diff --git a/.env.dev b/.env.dev index 63b879d7c6..cf77db28c1 100644 --- a/.env.dev +++ b/.env.dev @@ -15,9 +15,6 @@ TRUSTED_SERVER__SYNTHETIC__COUNTER_STORE=counter_store TRUSTED_SERVER__SYNTHETIC__OPID_STORE=opid_store # [proxy] -# Disable TLS certificate verification for local dev with self-signed certs -# TRUSTED_SERVER__PROXY__CERTIFICATE_CHECK=false -# # Restrict first-party proxy redirect targets to an allowlist (JSON array or indexed form). # Leave unset in local dev; configure in production to prevent SSRF via redirect chains # initiated by signed first-party proxy URLs. diff --git a/.github/actions/setup-integration-test-env/action.yml b/.github/actions/setup-integration-test-env/action.yml index a25f53ce66..38ebc9f1bf 100644 --- a/.github/actions/setup-integration-test-env/action.yml +++ b/.github/actions/setup-integration-test-env/action.yml @@ -91,7 +91,6 @@ runs: TRUSTED_SERVER__EC__PASSPHRASE: integration-test-ec-secret-padded-32 TRUSTED_SERVER__EC__PARTNERS: >- [{"name":"Integration Test Partner","source_domain":"inttest.example.com","bidstream_enabled":true,"api_token":"integration-test-token-alpha-32-bytes-ok"},{"name":"Integration Test Partner 2","source_domain":"inttest2.example.com","bidstream_enabled":true,"api_token":"integration-test-token-bravo-32-bytes-ok"}] - TRUSTED_SERVER__PROXY__CERTIFICATE_CHECK: "false" run: cargo build --package trusted-server-adapter-fastly --release --target wasm32-wasip1 - name: Build Axum native binary @@ -101,7 +100,6 @@ runs: TRUSTED_SERVER__PUBLISHER__ORIGIN_URL: http://127.0.0.1:${{ inputs.origin-port }} TRUSTED_SERVER__PUBLISHER__PROXY_SECRET: integration-test-proxy-secret TRUSTED_SERVER__EC__PASSPHRASE: integration-test-ec-secret-padded-32 - TRUSTED_SERVER__PROXY__CERTIFICATE_CHECK: "false" run: cargo build -p trusted-server-adapter-axum - name: Build WordPress test container @@ -132,5 +130,4 @@ runs: TRUSTED_SERVER__PUBLISHER__ORIGIN_URL: http://127.0.0.1:${{ inputs.origin-port }} TRUSTED_SERVER__PUBLISHER__PROXY_SECRET: integration-test-proxy-secret TRUSTED_SERVER__EC__PASSPHRASE: integration-test-ec-secret-padded-32 - TRUSTED_SERVER__PROXY__CERTIFICATE_CHECK: "false" run: bash crates/trusted-server-adapter-cloudflare/build.sh diff --git a/.github/workflows/format.yml b/.github/workflows/format.yml index 299768ffda..eb6566a3e6 100644 --- a/.github/workflows/format.yml +++ b/.github/workflows/format.yml @@ -63,6 +63,20 @@ jobs: - name: Verify jobs pin Node before compiling Rust run: node scripts/check-ci-node-pin.mjs + fastly-response-diff: + name: Fastly response diff script + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + + - name: Use Node.js + uses: actions/setup-node@v4 + with: + node-version-file: .tool-versions + + - name: Test the response comparison + run: node --test scripts/fastly-response-diff.test.mjs + format-rust: name: cargo fmt runs-on: ubuntu-latest diff --git a/.github/workflows/integration-tests.yml b/.github/workflows/integration-tests.yml index ab3e4ed991..fe2a824044 100644 --- a/.github/workflows/integration-tests.yml +++ b/.github/workflows/integration-tests.yml @@ -105,6 +105,9 @@ jobs: -- --include-ignored --skip test_wordpress_fastly --skip test_nextjs_fastly --skip trace_browser_workflow --skip trace_runtime_boundary + --skip multi_provider_auction_runs_spin --skip spin_runtime_ + --skip navigation_bids_include_180_and_320_ms_and_exclude_600_ms_spin + --skip publisher_head_and_conditional_304_return_real_status_spin --test-threads=1 env: WASM_BINARY_PATH: ${{ env.WASM_ARTIFACT_PATH }} @@ -229,6 +232,7 @@ jobs: run: | cargo test --manifest-path crates/trusted-server-integration-tests/Cargo.toml --target x86_64-unknown-linux-gnu --test integration trace_runtime_boundary -- --ignored --test-threads=1 --nocapture cargo test --manifest-path crates/trusted-server-integration-tests/Cargo.toml --target x86_64-unknown-linux-gnu --test integration trace_browser_workflow -- --ignored --test-threads=1 --nocapture + cargo test --manifest-path crates/trusted-server-integration-tests/Cargo.toml --target x86_64-unknown-linux-gnu --test integration -- --ignored --test-threads=1 --nocapture multi_provider_auction_runs_spin spin_runtime_ navigation_bids_include_180_and_320_ms_and_exclude_600_ms_spin publisher_head_and_conditional_304_return_real_status_spin env: WASM_BINARY_PATH: ${{ env.WASM_ARTIFACT_PATH }} AXUM_BINARY_PATH: ${{ env.AXUM_ARTIFACT_PATH }} diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 4b0a26c648..1a339ac5f0 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -279,7 +279,6 @@ jobs: TRUSTED_SERVER__PUBLISHER__ORIGIN_URL: http://127.0.0.1:8080 TRUSTED_SERVER__PUBLISHER__PROXY_SECRET: integration-test-proxy-secret TRUSTED_SERVER__EC__PASSPHRASE: integration-test-ec-secret-padded-32 - TRUSTED_SERVER__PROXY__CERTIFICATE_CHECK: "false" run: cargo build --package trusted-server-adapter-spin --target wasm32-wasip1 --features spin --release - name: Run Spin adapter tests (native host) diff --git a/AGENTS.md b/AGENTS.md index bf31f54858..d450f7888a 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -378,6 +378,7 @@ Every PR must pass: 14. Fastly absolute-deadline network fixtures under Viceroy (the harness builds the adapter test binary itself): `node crates/trusted-server-adapter-fastly/tests/deadline-origin.mjs "$(command -v viceroy)"` 15. Worker auction-deadline runtime fixtures (`worker-auction-deadlines.yml`): build and lint the runtime fixture, then run it under the local runtime. For Cloudflare, `cargo build --locked -p trusted-server-adapter-cloudflare --example deadline-runtime --target wasm32-unknown-unknown --features cloudflare` and `node scripts/test-worker-auction-deadlines.mjs cloudflare --runtime "$(command -v wrangler)"` (requires wrangler and the `Cargo.lock` wasm-bindgen CLI). For Spin, `cargo build --locked -p trusted-server-adapter-spin --example deadline-runtime --target wasm32-wasip1 --features spin` and `node scripts/test-worker-auction-deadlines.mjs spin --runtime "$(command -v spin)"`. CI lints each fixture with the same `cargo clippy` arguments plus `-- -D warnings` 16. Release producer checks: `node --test scripts/release/phase1.test.mjs && shellcheck scripts/release/phase1-build.sh` +17. Fastly response diff script: `node --test scripts/fastly-response-diff.test.mjs` with the `.tool-versions` Node CI also runs these HTML EOF checks. Run them locally only when changing EOF finalization, publisher or HTML-processor code: diff --git a/CHANGELOG.md b/CHANGELOG.md index dc8a24e872..41ad75e14c 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -13,6 +13,19 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Changed +- **Breaking:** EdgeZero is pinned to `14a6ca8bb5a5cbe055a1f725bdab8f42569a3bf0`, the head of EdgeZero pull request 275, in place of `v0.0.11` (#1506). Every adapter now serves requests through EdgeZero's request lifecycle and sends outbound HTTP through EdgeZero's `HttpClient`. Trusted Server's own outbound layer is removed: `PlatformHttpClient`, `PlatformBackend`, backend naming, the adapter HTTP clients, the redirect loop, the capability methods, and Trusted Server's Fastly response delivery. `RuntimeServices` drops `http_client` and `backend` and requires `outbound_client` and `clock`, and the config and secret store write methods are async. Code that implemented or called the removed traits must move to EdgeZero's client; see the [integration guide](docs/guide/integration-guide.md#proxy-implementation-rules). +- **Breaking:** Outbound requests have constant timeouts: a 1 s connect timeout, then a total timeout that is the configured value where a setting names one, 2 s for Tinybird, and otherwise 15 s for a buffered response or 60 s for a streamed one (#1506). The 15 s first-byte and between-bytes timers are gone, so a stalled streamed origin fails at 60 s, and a healthy streamed transfer longer than 60 s is cut. A redirect chain has a 60 s deadline, and a response body must finish within 60 s after egress starts. +- **Breaking:** Fastly runs on EdgeZero's canonical lifecycle (#1506). Dynamic backend names become `ez_b31v1_*`, so update dashboards keyed on backend names. Provider timers round up to a power of two of milliseconds, and a backend name collision fails the request with the route's 502. Framework errors such as 404, 405, 414, and 431 render as EdgeZero JSON with redacted text, and a 405 carries `Allow`. HEAD, 204, 205, and 304 responses skip the SDK's bulk framing, so their `Content-Length` and `Transfer-Encoding` can differ, and a streamed relay keeps the origin's `Content-Length`. The template-cache build digest changes, so the first deploy misses shared template-cache entries once. Each request that reaches EdgeZero logs one info line with its backend counts on the `trusted_server::outbound_observation` target. +- **Breaking:** App startup failures go through EdgeZero's app assembly, and a failed assembly is not retained, so the next request retries (#1506). Fastly and Cloudflare answer with EdgeZero's redacted JSON error at the failure's status, normally 500, and the Fastly response no longer carries `x-ts-version`; Fastly still answers `/health` first. Spin answers every route with 503, `/health` included. The Axum dev server exits with status 1 before it listens, in place of answering every route with 500. +- **Breaking:** Inbound requests follow EdgeZero's ingress rules on every adapter (#1506). A request target over 8192 bytes returns 414, more than 100 header fields or 64 KiB of headers returns 431, and ambiguous request framing returns 400. A body still unread 30 s after the request arrived returns 408. A publisher upload over 16 MiB returns 413; on Fastly the origin can see the head and up to 16 MiB of the aborted upload. A forwarded `GET` or `HEAD` that carries body bytes returns 400, and so does a publisher request whose target has a fragment, a backslash, or userinfo, or whose `Connection` header is malformed. +- **Breaking:** Outbound header rules are EdgeZero's on every adapter (#1506). Hop-by-hop fields, and the fields `Connection` names, no longer reach upstreams or come back from them, and a malformed upstream `Connection` returns the route's 502. An upstream request that sets no `Accept-Encoding` carries `identity`, and the request URI is normalized as a WHATWG URL. A request or response header whose value is not UTF-8 is dropped whole, with a warning that names it. That includes a raw-byte `Cookie` on the publisher forward, which drops every cookie to the origin, consent cookies included, and a raw-byte origin `Set-Cookie`. +- Redirects follow EdgeZero's rules (#1506). A hop to another origin drops `Authorization` and `Cookie`, a `Location` with userinfo fails with the route's 502, a UTF-8 `Location` with non-ASCII characters is percent-encoded and followed, and each hop logs at debug level without its query. +- **Breaking:** Every adapter sends an auction's bidder requests together as one EdgeZero request batch, after all of them are built and signed (#1506). Cloudflare and Spin now load and fan out multi-provider auctions instead of failing startup (#1484). An abandoned auction, or a page response that ends early, cancels its in-flight bidder and mediator requests on the guest side; Fastly host cancellation is best effort. A provider's reported response time now runs from the batch start to its bid's arrival. On Fastly, a bid first observed after the cutoff is dropped, and the 10 MiB cap across retained partial bid bodies is gone; each bidder response keeps its 1 MiB cap. Fastly startup still rejects a plan that could need more than 160 provider backends. +- **Breaking:** Fastly queues auction telemetry during the request and sends it to Tinybird after the response is delivered, then runs EC pull sync (#1506). Both run only after the application's response body was committed, and each pull-sync group has a 15 s cutoff. +- **Breaking:** `ts config validate` and `ts config push` reject `[response_headers]` entries named `Trailer`, `Content-Length`, `Connection`, `Keep-Alive`, `TE`, `Transfer-Encoding`, `Upgrade`, `Proxy-Connection`, `Proxy-Authenticate`, or `Proxy-Authorization` (#1506). A live config that predates the rule keeps loading, and each request skips such an entry with one warning. A skipped `Connection` entry no longer removes the fields it names. +- **Breaking:** `ts config validate` and `ts config push` reject `creative_opportunities.origin_readthrough_enabled = true`, because origin readthrough is unavailable; a live config keeps loading and treats it as `false` with a warning (#1506). They also reject a `publisher.origin_url` with userinfo, which would answer every publisher request with 400. +- Spin logs a warning at settings load when `publisher.origin_host_header_override` is set, because EdgeZero's Spin client cannot send a different `Host` (#1506). Fastly, Axum, and Cloudflare send the override as the origin `Host`. +- `ts build`, `ts serve`, and `ts deploy` find the manifest through `EDGEZERO_MANIFEST`, then the nearest `edgezero.toml` in the working directory or a parent, then the nearest one under the workspace root, in place of only `./edgezero.toml` (#1506). - **Breaking:** Known `[integrations.*]` tables, `[tester_cookie]`, and `[tinybird]` now reject unknown fields, including in disabled integration blocks. Integration parse errors name the offending key. `ts config validate` and `ts config push` also reject unknown integration IDs and list the supported IDs; runtime startup warns and ignores unknown IDs for compatibility with stored configs. Before deploying the new binary, use the new CLI to run `ts config validate` on the current config, correct the reported fields and IDs, then push the cleaned config while the old binary is still running. On Fastly, wait for the config-store propagation window to elapse before activating the new binary; a successful push does not mean every point of presence (POP) has the new config. Stored unknown fields can otherwise prevent startup. Intentionally open payload maps remain supported. Lockr uses `app_id`; remove unsupported `organization_id` and `project_id` fields. - The attested Fastly wasm reports `x-ts-version` as the full 40-character source commit; deploy-pipeline builds keep the tag, branch or short-commit value described below. - **Breaking:** `ts --version` now prints `ts package= source= target= variant=default` instead of `ts `. `source` is the full commit in attested release binaries, and otherwise the development ref, short commit or `unknown`. Update scripts that parse the old output. @@ -22,17 +35,16 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 measured from dispatch, including delayed headers and trickling bodies on all adapters. Complete available responses survive a slow publisher origin; unfinished providers time out and mediation is skipped after budget exhaustion. - Provider bids that are already complete at the final nonblocking snapshot remain - eligible even after `tmax`; mediation starts only with remaining budget and - rejects responses collected after its cutoff. - Fastly abandons unfinished requests; Axum, Cloudflare, and Spin cancel owned work. + Bids that arrive before the cutoff of the auction's EdgeZero request batch + remain eligible; mediation starts only with remaining budget. Dropping the + batch cancels unfinished requests on every adapter. - Explicit TS Console activation with `?ts_console=1` now sets a 30-minute diagnostics cookie instead of a browser-session cookie, independently of the mobile trace flag. Ordinary requests do not refresh it; end and explicitly enable diagnostics again to adopt the lifetime for an existing session. - Publisher documents with diagnostics active now use private, no-store responses and strip conditional and range request headers before fetching the origin. An unexpected origin `304` becomes a private `502` because it cannot supply an instrumented document, including when mobile tracing is disabled. - **Breaking:** `x-ts-version` now reports the deployed git version — the release tag, else the branch, else the first 6 characters of the commit — compiled in from the build-time `TRUSTED_SERVER__GIT_VERSION` (set by the deploy pipeline) or local git, and is sent by every adapter, including on the Fastly `GET /health` probe. The Fastly service version it previously carried is now `x-ts-fastly-version`; update dashboards, monitors, and scripts that read `x-ts-version` as the Fastly version number. Builds with neither the override nor git omit the header. - **Breaking:** `ts audit ad-templates generate` now previews the managed-field diff on stderr and asks before writing a changed config. Runs whose stdin or stderr is not a terminal fail without writing unless `--yes` is passed, so add `--yes` to scripts and CI jobs that apply generated slots. A declined prompt exits non-zero, `--yes` cannot be combined with `--dry-run`, and a candidate whose managed fields are unchanged is no longer rewritten. - **Breaking:** `ts prebid bundle` is now `ts prebid client`, alongside the new `ts prebid server` namespace. Update scripts and runbooks to use `ts prebid client` with the same arguments. The old `bundle` spelling is no longer accepted and has no compatibility alias. - The S2S `/_ts/api/v1/batch-sync` endpoint now validates the full batch and calls the CAS-protected update path once per distinct normalized EC ID. The last valid UID wins within a group, and infrastructure failures reject the failing and each unprocessed group, so accepted and `kv_unavailable` input indexes may interleave. -- **Breaking:** Auction providers and bidder routes now use the configuration-first `[auction.providers.]` and `[auction.bidders.]` maps. The removed `[auction].providers = [...]` list and removed server fields under `[integrations.prebid]` and `[integrations.aps]` are rejected even when those integrations are disabled, and `ts config push` rejects the old shape before publication. Move PBS `server_url` to provider `endpoint`, server timeout to provider `timeout_ms`, request controls and bidder-parameter overrides to the `prebid-server` `profile_config`, notification suppression to `notifications`, and each former server bidder to an `[auction.bidders.]` route. Move APS endpoint, timeout, account, inventory, debug, and creative controls to an `aps` provider and its `profile_config`. Browser Prebid settings remain under `[integrations.prebid]`; values such as timeout and debug that previously affected both browser and server behavior must now be configured for each owner. Provider endpoints must be absolute HTTPS URLs. Only bidder codes present in `[auction.bidders]` are folded into Trusted Server requests; unlisted publisher bids remain native browser demand. Provider response names now use the configured provider ID, such as `pbs-main`, instead of the legacy literal `prebid`; audit consumers that match `AuctionResponse.provider`. This schema has no mixed-version-safe deployment order: old binaries reject the maps and new binaries reject the retired fields, so activate the new binary and config blob together. Rollbacks must restore an old-schema blob together with the old binary. +- **Breaking:** Auction providers and bidder routes now use the configuration-first `[auction.providers.]` and `[auction.bidders.]` maps. The removed `[auction].providers = [...]` list and removed server fields under `[integrations.prebid]` and `[integrations.aps]` are rejected even when those integrations are disabled, and `ts config push` rejects the old shape before publication. Move PBS `server_url` to provider `endpoint`, server timeout to provider `timeout_ms`, request controls and bidder-parameter overrides to the `prebid-server` `profile_config`, notification suppression to `notifications`, and each former server bidder to an `[auction.bidders.]` route. Move APS endpoint, timeout, account, inventory, debug, and creative controls to an `aps` provider and its `profile_config`. Browser Prebid settings remain under `[integrations.prebid]`; values such as timeout and debug that previously affected both browser and server behavior must now be configured for each owner. Provider endpoints must be absolute HTTPS URLs, or plain HTTP to `127.0.0.1` or `[::1]` for local test bidders. Only bidder codes present in `[auction.bidders]` are folded into Trusted Server requests; unlisted publisher bids remain native browser demand. Provider response names now use the configured provider ID, such as `pbs-main`, instead of the legacy literal `prebid`; audit consumers that match `AuctionResponse.provider`. This schema has no mixed-version-safe deployment order: old binaries reject the maps and new binaries reject the retired fields, so activate the new binary and config blob together. Rollbacks must restore an old-schema blob together with the old binary. - **Breaking** — Admin Basic-auth coverage now includes `GET /_ts/admin/ec`, `GET /_ts/admin/ec/{id}`, and `GET /_ts/admin/eids`. Existing configurations whose `[[handlers]]` patterns protect only the key-management endpoints now fail startup; broaden coverage before deploying, preferably with a namespace-boundary pattern such as `^/_ts/admin(?:/|$)`. Coverage of the dynamic `/_ts/admin/ec/{id}` route is no longer inferred from ID-shaped samples: the router accepts any segment after `/_ts/admin/ec/` and Basic Auth runs on the raw path before routing, so patterns anchored to the EC ID grammar (for example `^/_ts/admin/ec/[a-f0-9]{64}[.][A-Za-z0-9]{6}$`) are rejected in favor of a prefix-level matcher. Placeholder and well-known weak handler passwords (`changeme`, `password`, `admin`, `replace-with-…`) now fail startup on every handler rather than only on handlers inferred to cover an admin endpoint, because first-match-wins handler selection lets a narrow handler shadow the admin namespace. - Prebid Server provider endpoints now normalize origin-only legacy `server_url` values to `/openrtb2/auction`. Query parameters are preserved, the canonical path loses a trailing slash, and configured non-root custom paths remain exact. - Publisher HTML uses the browser-only `Cache-Control: private, max-age=60` policy for successful GET document responses and their `304 Not Modified` revalidations when server-side ad templates are structurally inactive, while preserving origin `private`/`no-store` policies and request-scoped bot, prefetch, or consent-denied responses. The `private` directive prevents shared caches that use `Cache-Control` from storing the document. Cookie-bearing responses using the generated inactive policy are finalized as `private, max-age=0`; CDN-specific cache headers remain unchanged and continue to control supporting CDNs independently. Set `[creative_opportunities].enabled = false` to disable publisher HTML and SPA template delivery without disabling direct `POST /auction` callers; an absent configuration, an unmatched slot, or a disabled auction also make the stack structurally inactive. An explicit `enabled = false` is not compatible with older binaries: restore the default, re-push and finalize the config before rolling back. @@ -48,6 +60,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Security +- **Breaking:** `proxy.certificate_check = false` is no longer supported; origin TLS is always verified on every adapter (#1506). `ts config validate` and `ts config push` reject it. A live config that still sets it keeps loading, logs a warning, and ignores the setting, so an origin with an invalid certificate fails with the route's 502. - Prebid Server receives only allowlisted consent cookies in `both` and `cookies_only`, and no Cookie header in `openrtb_only` (#763). Cookie selection matches consent extraction, including last-duplicate selection and GPP section whitespace normalization. If an extracted consent cookie cannot be forwarded safely, `cookies_only` retains only the affected signal in the OpenRTB body; GPP and its section IDs fall back together. Publisher session and identity cookies are excluded in every mode. **Breaking** for deployments that relied on other cookies reaching PBS: Prebid Server's auction endpoint reads its `uids` user-sync cookie and configured host cookie from the request, so when that cookie domain covers the publisher site, those buyer UIDs no longer arrive through Trusted Server. - Didomi SDK requests now derive `X-Geo-Country`, `X-Geo-Region`, and `CloudFront-Viewer-Country` only from trusted platform geo (#1151). Caller-supplied `FastlyGeo-CountryCode` and `FastlyGeo-Region` headers previously let callers choose the geography sent to Didomi. **Breaking** for deployments that injected those headers: configure CDN-fronted client IP instead. Without a complete trusted country/region pair (currently always on Cloudflare, Axum, and Spin), the geo header group is omitted. Noncanonical loaders and SDK POSTs bypass the outbound cache and return `no-store, private`; other SDK GETs retain Didomi's cache policy. Purge cached SDK entries before rollout. Geo-less loaders risk Didomi throttling and disablement; production Fastly deployments should validate geo coverage and enable `geo_query_parameters = true`. Other adapters currently have no compliant loader-caching mode through this proxy; use direct SDK delivery instead. Audit-generated configurations leave detected Didomi disabled for manual review. - `ts prebid client` now keeps the config file's mode bits when it writes `external_bundle_sha256` and `external_bundle_sri`; previously a `0600` config was reset to the umask default (usually `0644`). Configs already rewritten by an earlier version keep the widened mode, so restrict them again (for example `chmod 600 trusted-server.toml`). The rewrite is fsynced and leaves no temporary copy behind on failure. TOML parse errors from `ts prebid client` and `ts audit generate` now name the file, line and column without quoting the offending config line (#1202). @@ -77,6 +90,8 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Added +- The Axum dev server reads its listener limits from `EDGEZERO__ADAPTER__INGRESS__MAX_CONNECTIONS` (default 256), `EDGEZERO__ADAPTER__INGRESS__MAX_RAW_HEAD_BYTES` (65536), `EDGEZERO__ADAPTER__INGRESS__MAX_HEADER_COUNT` (100), and `EDGEZERO__ADAPTER__INGRESS__HEAD_READ_TIMEOUT_MS` (10000) (#1506). An invalid value stops startup. +- Auction provider endpoints accept plain HTTP to `127.0.0.1` or `[::1]`, for local test bidders. Other `http` endpoints are still rejected (#1506). - Add opt-in mobile ad-render tracing through `[integrations.gpt_diagnostics].trace_page_enabled` (default `false`, requiring GPT diagnostics). The same-tab `/_ts/trace` viewer presents bounded, redacted browser-local request, auction and GPT observations with copy, download, share and independent local/server cleanup controls. See the GPT diagnostics guide for deployment and rollout checks. - Add the opt-in `[trusted_forwarder]` section with required `auth_header` and secret-store `shared_secret` reference fields for authenticated public-origin forwarding. - `[rewrite] include_domains` limits creative asset rewriting to listed hosts. When non-empty, only asset URLs on matching hosts are proxied through `/first-party/proxy`, so only those hosts receive the EC ID on newly rewritten or signed asset URLs. The list is not enforced when a proxy URL is fetched, so previously issued proxy URLs can still forward the EC ID to hosts later removed from it. Other assets keep their original URL, and `/first-party/sign` returns `502` for them so the creative runtime loads them directly. Click-through links are not affected, and `exclude_domains` still wins. Entries must be host names, optionally prefixed with `*.`, dotted-quad IPv4 addresses, or bracketed IPv6 addresses; entries with a scheme, port, path, spaces or non-ASCII characters are rejected at load. Upgrading: deploy the binary first, then push a config that sets `include_domains`. Rolling back: remove `include_domains`, push the resulting config, then roll back the binary; older binaries reject the key because `[rewrite]` denies unknown fields. diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index f52d2c4bde..d63713c757 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -148,11 +148,10 @@ link it in the description, and mark it ready for review again. ## Fastly dependency updates -Update the workspace `fastly` and `fastly-sys` dependencies together. The -Fastly deadline collector uses raw asynchronous I/O readiness calls from -`fastly-sys` with handles created by `fastly`, so both must use a compatible -SDK ABI. Check the resolved versions in `Cargo.lock` and run the Fastly -adapter tests, including the real deadline network fixture, after an SDK update. +After a workspace `fastly` update, check that `Cargo.lock` resolves one +`fastly` version for Trusted Server and `edgezero-adapter-fastly`, then run +the Fastly adapter tests, including the real deadline network fixture +(`crates/trusted-server-adapter-fastly/tests/deadline-origin.mjs`). ## Verify diff --git a/Cargo.lock b/Cargo.lock index 70316712ac..542d235694 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -206,9 +206,9 @@ dependencies = [ [[package]] name = "async-compression" -version = "0.4.42" +version = "0.4.43" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e79b3f8a79cccc2898f31920fc69f304859b3bd567490f75ebf51ae1c792a9ac" +checksum = "3976abdc8fe7d1133d43d304afd42abdf5bc3e1319d263d223bde07b5efc4be8" dependencies = [ "compression-codecs", "compression-core", @@ -338,7 +338,7 @@ dependencies = [ "serde_urlencoded", "sync_wrapper", "tokio", - "tower 0.5.3", + "tower", "tower-layer", "tower-service", "tracing", @@ -528,9 +528,9 @@ dependencies = [ [[package]] name = "brotli-decompressor" -version = "5.0.3" +version = "5.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3a32acac15fe1967bc3986b2a6347dffc965602354ea6f450ad07e8bfd253583" +checksum = "5962523e1b92ce1b5e793d9169b9943eece10d39f62550bc04bb605d75b94924" dependencies = [ "alloc-no-stdlib", "alloc-stdlib", @@ -576,9 +576,9 @@ checksum = "1fd0f2584146f6f2ef48085050886acf353beff7305ebd1ae69500e27c67f64b" [[package]] name = "bytes" -version = "1.12.0" +version = "1.12.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8ae3f5d315924270530207e2a68396c3cc547f6dca3fbdca317cfb1a51edb593" +checksum = "fc652a48c352aef3ea3aed32080501cf3ef6ed5da78602a020c991775b0aff04" dependencies = [ "serde", ] @@ -1530,9 +1530,10 @@ dependencies = [ [[package]] name = "edgezero-adapter" version = "0.1.0" -source = "git+https://github.com/IABTechLab/edgezero?tag=v0.0.11#87456c708ffe9edf86b3a0f8b06dfd15d60b07b5" +source = "git+https://github.com/IABTechLab/edgezero?rev=14a6ca8bb5a5cbe055a1f725bdab8f42569a3bf0#14a6ca8bb5a5cbe055a1f725bdab8f42569a3bf0" dependencies = [ "base16ct 1.0.0", + "edgezero-core", "serde", "serde_json", "sha2 0.11.1", @@ -1543,9 +1544,10 @@ dependencies = [ [[package]] name = "edgezero-adapter-axum" version = "0.1.0" -source = "git+https://github.com/IABTechLab/edgezero?tag=v0.0.11#87456c708ffe9edf86b3a0f8b06dfd15d60b07b5" +source = "git+https://github.com/IABTechLab/edgezero?rev=14a6ca8bb5a5cbe055a1f725bdab8f42569a3bf0#14a6ca8bb5a5cbe055a1f725bdab8f42569a3bf0" dependencies = [ "anyhow", + "async-stream", "async-trait", "axum", "bytes", @@ -1555,15 +1557,20 @@ dependencies = [ "futures", "futures-util", "http", + "http-body", + "hyper", + "hyper-util", + "libc", "log", "redb", "reqwest 0.13.4", + "serde", "serde_json", "simple_logger", "thiserror 2.0.18", "tokio", "toml", - "tower 0.5.3", + "tower", "tracing", "walkdir", ] @@ -1571,9 +1578,10 @@ dependencies = [ [[package]] name = "edgezero-adapter-cloudflare" version = "0.1.0" -source = "git+https://github.com/IABTechLab/edgezero?tag=v0.0.11#87456c708ffe9edf86b3a0f8b06dfd15d60b07b5" +source = "git+https://github.com/IABTechLab/edgezero?rev=14a6ca8bb5a5cbe055a1f725bdab8f42569a3bf0#14a6ca8bb5a5cbe055a1f725bdab8f42569a3bf0" dependencies = [ "anyhow", + "async-stream", "async-trait", "brotli", "bytes", @@ -1588,13 +1596,15 @@ dependencies = [ "tempfile", "toml_edit 0.25.12+spec-1.1.0", "walkdir", + "wasm-streams", + "web-sys", "worker", ] [[package]] name = "edgezero-adapter-fastly" version = "0.1.0" -source = "git+https://github.com/IABTechLab/edgezero?tag=v0.0.11#87456c708ffe9edf86b3a0f8b06dfd15d60b07b5" +source = "git+https://github.com/IABTechLab/edgezero?rev=14a6ca8bb5a5cbe055a1f725bdab8f42569a3bf0#14a6ca8bb5a5cbe055a1f725bdab8f42569a3bf0" dependencies = [ "anyhow", "async-stream", @@ -1607,6 +1617,8 @@ dependencies = [ "edgezero-adapter", "edgezero-core", "fastly", + "fastly-shared", + "fastly-sys", "fern", "flate2", "futures", @@ -1624,16 +1636,15 @@ dependencies = [ [[package]] name = "edgezero-adapter-spin" version = "0.1.0" -source = "git+https://github.com/IABTechLab/edgezero?tag=v0.0.11#87456c708ffe9edf86b3a0f8b06dfd15d60b07b5" +source = "git+https://github.com/IABTechLab/edgezero?rev=14a6ca8bb5a5cbe055a1f725bdab8f42569a3bf0#14a6ca8bb5a5cbe055a1f725bdab8f42569a3bf0" dependencies = [ "anyhow", + "async-stream", "async-trait", - "brotli", "bytes", "ctor", "edgezero-adapter", "edgezero-core", - "flate2", "futures", "futures-util", "log", @@ -1651,7 +1662,7 @@ dependencies = [ [[package]] name = "edgezero-cli" version = "0.1.0" -source = "git+https://github.com/IABTechLab/edgezero?tag=v0.0.11#87456c708ffe9edf86b3a0f8b06dfd15d60b07b5" +source = "git+https://github.com/IABTechLab/edgezero?rev=14a6ca8bb5a5cbe055a1f725bdab8f42569a3bf0#14a6ca8bb5a5cbe055a1f725bdab8f42569a3bf0" dependencies = [ "chrono", "clap", @@ -1676,31 +1687,43 @@ dependencies = [ [[package]] name = "edgezero-core" version = "0.1.0" -source = "git+https://github.com/IABTechLab/edgezero?tag=v0.0.11#87456c708ffe9edf86b3a0f8b06dfd15d60b07b5" +source = "git+https://github.com/IABTechLab/edgezero?rev=14a6ca8bb5a5cbe055a1f725bdab8f42569a3bf0#14a6ca8bb5a5cbe055a1f725bdab8f42569a3bf0" dependencies = [ + "adler2", + "alloc-no-stdlib", + "alloc-stdlib", "anyhow", "async-compression", "async-stream", "async-trait", "base16ct 1.0.0", + "brotli", + "brotli-decompressor", "bytes", + "compression-codecs", + "compression-core", + "crc32fast", "edgezero-macros", + "flate2", "futures", "futures-util", "http", "http-body", "log", "matchit 0.9.2", + "miniz_oxide", "ryu", "serde", "serde_json", "serde_path_to_error", "serde_urlencoded", "sha2 0.11.1", + "simd-adler32", "thiserror 2.0.18", "toml", "tower-service", "tracing", + "url", "validator", "web-time", ] @@ -1708,14 +1731,14 @@ dependencies = [ [[package]] name = "edgezero-macros" version = "0.1.0" -source = "git+https://github.com/IABTechLab/edgezero?tag=v0.0.11#87456c708ffe9edf86b3a0f8b06dfd15d60b07b5" +source = "git+https://github.com/IABTechLab/edgezero?rev=14a6ca8bb5a5cbe055a1f725bdab8f42569a3bf0#14a6ca8bb5a5cbe055a1f725bdab8f42569a3bf0" dependencies = [ "log", "proc-macro2", "quote", "serde", "serde_json", - "syn 3.0.4", + "syn 3.0.9", "toml", "validator", ] @@ -3311,9 +3334,9 @@ dependencies = [ [[package]] name = "hyper" -version = "1.10.1" +version = "1.12.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "55281c53a1894c864990125767da440a4e630446785086f52523b20033b74498" +checksum = "2c3e324da4c95177d6291d4c8730197c0d1822f8a9766814a4a44fa5ab797c9c" dependencies = [ "atomic-waker", "bytes", @@ -3425,7 +3448,7 @@ dependencies = [ "num-derive", "num-iter", "num-traits", - "prettyplease", + "prettyplease 0.2.37", "proc-macro2", "quote", "strum_macros", @@ -4714,6 +4737,16 @@ dependencies = [ "syn 2.0.118", ] +[[package]] +name = "prettyplease" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2bfe0f4c752e450fc2faf62654f1c134747922825d5b04ca717b8874f41a40c0" +dependencies = [ + "proc-macro2", + "syn 3.0.9", +] + [[package]] name = "primeorder" version = "0.13.6" @@ -4795,7 +4828,7 @@ dependencies = [ "multimap", "once_cell", "petgraph", - "prettyplease", + "prettyplease 0.2.37", "prost 0.13.5", "prost-types 0.13.5", "regex", @@ -5140,7 +5173,7 @@ dependencies = [ "sync_wrapper", "tokio", "tokio-rustls", - "tower 0.5.3", + "tower", "tower-http", "tower-service", "url", @@ -5180,7 +5213,7 @@ dependencies = [ "sync_wrapper", "tokio", "tokio-rustls", - "tower 0.5.3", + "tower", "tower-http", "tower-service", "url", @@ -5897,8 +5930,7 @@ checksum = "6980e8d7511241f8acf4aebddbb1ff938df5eebe98691418c4468d0b72a96a67" [[package]] name = "spin-macro" version = "7.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ac8a641da857d4e7cada31a853c40180eeb10485a0f2f0282d0a7e5a2a4105d4" +source = "git+https://github.com/stackpop/spin-rust-sdk.git?rev=6c59ed5b4fcb377072763aee0b27a3abdd8edb72#6c59ed5b4fcb377072763aee0b27a3abdd8edb72" dependencies = [ "proc-macro2", "quote", @@ -5908,8 +5940,7 @@ dependencies = [ [[package]] name = "spin-sdk" version = "7.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2b7a3d9a6805ccdb4d404e91ccf7198ac5e486b6199dc9387aeebaa368272112" +source = "git+https://github.com/stackpop/spin-rust-sdk.git?rev=6c59ed5b4fcb377072763aee0b27a3abdd8edb72#6c59ed5b4fcb377072763aee0b27a3abdd8edb72" dependencies = [ "anyhow", "bytes", @@ -6049,9 +6080,9 @@ dependencies = [ [[package]] name = "syn" -version = "3.0.4" +version = "3.0.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e6275cddf4610d1775e6d1fe9469b2e77d0f39fd98fb7450901b821e0c53649f" +checksum = "d78c8dee4c7bf0e14673097256fed6142ce9d3b85a408189d07482442145823b" dependencies = [ "proc-macro2", "quote", @@ -6420,7 +6451,7 @@ dependencies = [ "sync_wrapper", "tokio", "tokio-stream", - "tower 0.5.3", + "tower", "tower-layer", "tower-service", "tracing", @@ -6437,21 +6468,6 @@ dependencies = [ "tonic", ] -[[package]] -name = "tower" -version = "0.4.13" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b8fa9be0de6cf49e536ce1851f987bd21a43b771b09473c3549a6c853db37c1c" -dependencies = [ - "futures-core", - "futures-util", - "pin-project", - "pin-project-lite", - "tower-layer", - "tower-service", - "tracing", -] - [[package]] name = "tower" version = "0.5.3" @@ -6488,7 +6504,7 @@ dependencies = [ "pin-project-lite", "tokio", "tokio-util", - "tower 0.5.3", + "tower", "tower-layer", "tower-service", "url", @@ -6543,7 +6559,6 @@ name = "trusted-server-adapter-axum" version = "0.1.0" dependencies = [ "async-trait", - "axum", "base64", "edgezero-adapter-axum", "edgezero-core", @@ -6554,9 +6569,7 @@ dependencies = [ "simple_logger", "temp-env", "tokio", - "tower 0.4.13", "trusted-server-core", - "web-time", ] [[package]] @@ -6571,7 +6584,6 @@ dependencies = [ "edgezero-core", "error-stack", "futures", - "js-sys", "log", "serde_json", "tokio", @@ -6595,7 +6607,6 @@ dependencies = [ "error-stack", "esi", "fastly", - "fastly-sys", "fern", "futures", "log", @@ -6604,9 +6615,7 @@ dependencies = [ "serde_json", "toml", "trusted-server-core", - "url", "urlencoding", - "web-time", ] [[package]] @@ -6616,14 +6625,11 @@ dependencies = [ "anyhow", "async-trait", "base64", - "brotli", "bytes", "edgezero-adapter-spin", "edgezero-core", "error-stack", - "flate2", "futures", - "http-body-util", "log", "spin-sdk", "tokio", @@ -6747,15 +6753,18 @@ dependencies = [ "bytes", "derive_more", "edgezero-adapter-axum", + "edgezero-adapter-cloudflare", + "edgezero-adapter-spin", "edgezero-core", "env_logger", "error-stack", "futures", "http", - "http-body-util", "libc", "log", + "rcgen", "reqwest 0.12.28", + "rustls", "scraper", "serde_json", "temp-env", @@ -6763,14 +6772,12 @@ dependencies = [ "testcontainers", "tokio", "toml", - "tower 0.4.13", "trusted-server-adapter-axum", "trusted-server-adapter-cloudflare", "trusted-server-adapter-spin", "trusted-server-core", "trusted-server-js", "url", - "urlencoding", ] [[package]] @@ -7055,9 +7062,9 @@ dependencies = [ [[package]] name = "want" -version = "0.3.1" +version = "0.3.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bfa7760aed19e106de2c7c0b581b509f2f25d3dacaf737cb82ac61bc6d760b0e" +checksum = "ec4cdd0dd910afe868b7ef477227d8d538b46b3075031afee8a9f2acb0a2ed0b" dependencies = [ "try-lock", ] @@ -7079,15 +7086,15 @@ dependencies = [ [[package]] name = "wasip3" -version = "0.7.1+wasi-0.3.0" +version = "0.10.0+wasi-0.3.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "75e9ed91ee5a995b6fe16c89d4511a8c12eb76967be20b57557618e02e781bcc" +checksum = "d1657e9f48c63d6c3f143f70d5c6e2be7d463371a7af1ed939a5b83d3ca080d0" dependencies = [ "bytes", "http", "http-body", "thiserror 2.0.18", - "wit-bindgen 0.57.1", + "wit-bindgen 0.63.0", ] [[package]] @@ -7147,9 +7154,9 @@ dependencies = [ [[package]] name = "wasm-encoder" -version = "0.247.0" +version = "0.262.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "30b6733b8b91d010a6ac5b0fb237dc46a19650bc4c67db66857e2e787d437204" +checksum = "4495a6f85dd94c411c749303efcbc5defade0b0cbdbc0541495241ab917a9b5a" dependencies = [ "leb128fmt", "wasmparser", @@ -7157,9 +7164,9 @@ dependencies = [ [[package]] name = "wasm-metadata" -version = "0.247.0" +version = "0.262.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "665fe59e56cc9b419ca6fcca56673e3421d1a5011e3b65caf6b726fd9e041d10" +checksum = "c86ff6668190d9fb70372b306b4f9fe8930f0e2f57ce72e4922bf2f117900949" dependencies = [ "anyhow", "indexmap 2.14.0", @@ -7182,9 +7189,9 @@ dependencies = [ [[package]] name = "wasmparser" -version = "0.247.0" +version = "0.262.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8e6fb4c2bee46c5ea4d40f8cdb5c131725cd976718ec56f1c8e82fbde5fa2a80" +checksum = "974d643b125b272607e2fff8012ea14a8328acd7621e4bafa7afb32500052b1b" dependencies = [ "bitflags 2.13.0", "hashbrown 0.17.1", @@ -7541,6 +7548,15 @@ name = "wit-bindgen" version = "0.57.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1ebf944e87a7c253233ad6766e082e3cd714b5d03812acc24c318f549614536e" +dependencies = [ + "bitflags 2.13.0", +] + +[[package]] +name = "wit-bindgen" +version = "0.63.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b85a987ab61d365382d25c7d47e26edd48941b3bf6e3fbf76eff918bc71dab06" dependencies = [ "bitflags 2.13.0", "futures", @@ -7549,9 +7565,9 @@ dependencies = [ [[package]] name = "wit-bindgen-core" -version = "0.57.1" +version = "0.63.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "02dee27a2dc20d1008016c742ec9fc6ea498492994ba3750be7454cbc97ff04c" +checksum = "775e122e390603abfacd548027016cd0113a6e2bf2f8ce6bdd3658b013456ea8" dependencies = [ "anyhow", "heck", @@ -7560,15 +7576,15 @@ dependencies = [ [[package]] name = "wit-bindgen-rust" -version = "0.57.1" +version = "0.63.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b5007dae772945b7a5003d69d90a3a4a78929d41f19d004e980c4259a6af4484" +checksum = "84e26e13b9d972026778d2ffb7985736de116026fdc8d9464f966e12df2eb240" dependencies = [ "anyhow", "heck", "indexmap 2.14.0", - "prettyplease", - "syn 2.0.118", + "prettyplease 0.3.0", + "syn 3.0.9", "wasm-metadata", "wit-bindgen-core", "wit-component", @@ -7576,24 +7592,24 @@ dependencies = [ [[package]] name = "wit-bindgen-rust-macro" -version = "0.57.1" +version = "0.63.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "af9237d678e3513ad24e96fe98beacdc0db6405284ba2a2400418cf0d42caa89" +checksum = "16a008a0d6b4840677326beb4edbeb210a2f1b7d0183fb7cc33f466b90fd7fb6" dependencies = [ "anyhow", - "prettyplease", + "prettyplease 0.3.0", "proc-macro2", "quote", - "syn 2.0.118", + "syn 3.0.9", "wit-bindgen-core", "wit-bindgen-rust", ] [[package]] name = "wit-component" -version = "0.247.0" +version = "0.262.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9d567162a6b9843080e5e0053f696623ff694bae8ae017c9ec536d1873bbe3d8" +checksum = "078def49fdb49cab405c3006e05793b3fdaf7069126ba6b9c97d01ec8c98fe4c" dependencies = [ "anyhow", "bitflags 2.13.0", @@ -7610,9 +7626,9 @@ dependencies = [ [[package]] name = "wit-parser" -version = "0.247.0" +version = "0.262.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8ffe4064318cdf3c08cb99343b44c039fcefe61ccdf58aa9975285f13d74d1fc" +checksum = "00f0dc051466ffe46787721fca3d088536f158ae400be044f1b1c386ba9397ad" dependencies = [ "anyhow", "hashbrown 0.17.1", @@ -7623,7 +7639,7 @@ dependencies = [ "serde", "serde_derive", "serde_json", - "unicode-xid", + "unicode-ident", "wasmparser", ] diff --git a/Cargo.toml b/Cargo.toml index 139cf4f610..7447eb4073 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -56,17 +56,16 @@ cssparser = "0.36" derive_more = { version = "2.0", features = ["display", "error"] } directories = "5" ed25519-dalek = { version = "2.2", features = ["rand_core"] } -edgezero-adapter-axum = { git = "https://github.com/IABTechLab/edgezero", tag = "v0.0.11", default-features = false } -edgezero-adapter-cloudflare = { git = "https://github.com/IABTechLab/edgezero", tag = "v0.0.11", default-features = false } -edgezero-adapter-fastly = { git = "https://github.com/IABTechLab/edgezero", tag = "v0.0.11", default-features = false } -edgezero-adapter-spin = { git = "https://github.com/IABTechLab/edgezero", tag = "v0.0.11", default-features = false } -edgezero-cli = { git = "https://github.com/IABTechLab/edgezero", tag = "v0.0.11" } -edgezero-core = { git = "https://github.com/IABTechLab/edgezero", tag = "v0.0.11", default-features = false } +edgezero-adapter-axum = { git = "https://github.com/IABTechLab/edgezero", rev = "14a6ca8bb5a5cbe055a1f725bdab8f42569a3bf0", default-features = false } +edgezero-adapter-cloudflare = { git = "https://github.com/IABTechLab/edgezero", rev = "14a6ca8bb5a5cbe055a1f725bdab8f42569a3bf0", default-features = false } +edgezero-adapter-fastly = { git = "https://github.com/IABTechLab/edgezero", rev = "14a6ca8bb5a5cbe055a1f725bdab8f42569a3bf0", default-features = false } +edgezero-adapter-spin = { git = "https://github.com/IABTechLab/edgezero", rev = "14a6ca8bb5a5cbe055a1f725bdab8f42569a3bf0", default-features = false } +edgezero-cli = { git = "https://github.com/IABTechLab/edgezero", rev = "14a6ca8bb5a5cbe055a1f725bdab8f42569a3bf0" } +edgezero-core = { git = "https://github.com/IABTechLab/edgezero", rev = "14a6ca8bb5a5cbe055a1f725bdab8f42569a3bf0", default-features = false } env_logger = "0.11" error-stack = "0.6" esi = "0.7.2" fastly = "0.13" -fastly-sys = "0.13.1" fern = "0.7.1" flate2 = "1.1" futures = "0.3" @@ -85,7 +84,6 @@ hyper = "1" hyper-util = "0.1" iab_gpp = "0.1" jose-jwk = "0.1.2" -js-sys = "0.3" libc = "0.2" log = "0.4.29" log-fastly = "0.13" @@ -108,7 +106,7 @@ serde_yaml_ng = "0.10" sha2 = "0.11" similar = "2.7" simple_logger = "5" -spin-sdk = { version = "~7.0", default-features = false, features = ["http", "key-value", "variables"] } +spin-sdk = { version = "=7.0.0", git = "https://github.com/stackpop/spin-rust-sdk.git", rev = "6c59ed5b4fcb377072763aee0b27a3abdd8edb72", default-features = false, features = ["http", "key-value", "variables"] } subtle = "2.6" temp-env = "0.3.6" tempfile = "3.24" @@ -118,7 +116,6 @@ tokio = { version = "1.49", features = ["sync", "macros", "io-util", "rt", "time tokio-rustls = "0.26" toml = "1.1" toml_edit = "0.23.10" -tower = "0.4" tracing = "0.1" trusted-server-core = { path = "crates/trusted-server-core" } trusted-server-js = { path = "crates/trusted-server-js" } diff --git a/TESTING.md b/TESTING.md index 7565ca7dde..58deb2e0e8 100644 --- a/TESTING.md +++ b/TESTING.md @@ -9,10 +9,11 @@ Fastly development server: fastly compute serve ``` -Provider endpoints must use HTTPS. Fastly and Viceroy also need a backend that -matches the provider host and TLS settings. For a deterministic local bidder, -use `scripts/template-cache-local-test.sh`, which creates a temporary CA and -registers the matching backend. +Provider endpoints must use HTTPS, except plain HTTP to `127.0.0.1` or `[::1]`, +which local test bidders use. EdgeZero registers each provider's Fastly backend +at request time, so Viceroy needs no static backend for it. For a deterministic +local bidder, use `scripts/template-cache-local-test.sh`, which serves a stub +bidder over plain HTTP on `127.0.0.1`. ## Example configuration @@ -97,18 +98,17 @@ Building plan-backed auction orchestrator Auction orchestrator built with 2 bidder providers ``` -A launched request logs the configured provider ID, predicted backend, and -budget. Collection logs the pending and immediate response counts: +All bidder requests leave together as one batch. Collection logs how many are +in flight and the remaining auction budget, then the winner count: ```text -Dispatching bid request to 'pbs-main' (backend: ..., budget: ...ms) -Dispatching bid request to 'aps-main' (backend: ..., budget: ...ms) -Dispatched 2 SSP request(s) with 0 immediate response(s) (timeout: ...ms) +Collecting 2 in-flight SSP responses (...ms of the auction budget remaining) +Selected ... winning bids ``` -Exact backend names and budgets depend on the adapter and remaining auction -deadline. Provider failures are isolated and appear in response metadata under -the configured provider ID. +The remaining budget depends on the adapter and how long the requests took to +build. Provider failures are isolated and appear in response metadata under the +configured provider ID. ## Disabled auction @@ -145,9 +145,9 @@ cd crates/trusted-server-js/lib npx vitest run ``` -The template-cache harness exercises plan compilation, HTTPS backend naming, -provider dispatch, local highest-bid winner selection, and both ESI and inline -delivery modes: +The template-cache harness exercises plan compilation, a plain HTTP loopback +bidder, provider dispatch, local highest-bid winner selection, and both ESI and +inline delivery modes: ```bash ./scripts/template-cache-local-test.sh esi diff --git a/clippy.toml b/clippy.toml index bb3a6138d4..4f350d1270 100644 --- a/clippy.toml +++ b/clippy.toml @@ -14,3 +14,11 @@ allow-unwrap-in-tests = true # Keep current Trusted Server thresholds while strict lint cleanup is underway. cognitive-complexity-threshold = 30 too-many-lines-threshold = 200 + +# `Body::into_bytes` returns `None` for a streamed body, so a caller that +# treats `None` as empty silently drops the body. Production code reads bodies +# through the bounded readers instead. Unit tests may buffer freely: crate +# roots allow the lint under `cfg(test)`. +disallowed-methods = [ + { path = "edgezero_core::body::Body::into_bytes", reason = "returns None for a stream; read inbound bodies with `trusted_server_core::http_util::read_request_body` and outbound responses with `OutboundResponse::into_bytes_bounded`" }, +] diff --git a/crates/trusted-server-adapter-axum/Cargo.toml b/crates/trusted-server-adapter-axum/Cargo.toml index af707aa24e..9bf06bd8da 100644 --- a/crates/trusted-server-adapter-axum/Cargo.toml +++ b/crates/trusted-server-adapter-axum/Cargo.toml @@ -25,16 +25,14 @@ edgezero-core = { workspace = true } error-stack = { workspace = true } futures = { workspace = true } log = { workspace = true } -reqwest = { workspace = true } simple_logger = { workspace = true } tokio = { workspace = true, features = ["rt-multi-thread", "macros", "sync", "time"] } trusted-server-core = { workspace = true } -web-time = { workspace = true } [dev-dependencies] trusted-server-core = { workspace = true, features = ["test-utils"] } -axum = { workspace = true } +edgezero-adapter-axum = { workspace = true, features = ["axum", "test-utils"] } base64 = { workspace = true } +reqwest = { workspace = true } temp-env = { workspace = true } tokio = { workspace = true, features = ["rt-multi-thread", "macros"] } -tower = { workspace = true, features = ["util"] } diff --git a/crates/trusted-server-adapter-axum/README.md b/crates/trusted-server-adapter-axum/README.md index 3e4863e140..6c13723e94 100644 --- a/crates/trusted-server-adapter-axum/README.md +++ b/crates/trusted-server-adapter-axum/README.md @@ -4,12 +4,18 @@ Native development adapter for Trusted Server. It runs the shared application through Axum on the host, without an edge simulator, and is not a production deployment target. -The crate owns Axum route registration, middleware, outbound HTTP through -`reqwest`, and environment-backed platform services. Its route surface follows -the shared core contract, but request-time KV operations are unavailable and a -startup error makes every route, including `/health`, return 500. Application -configuration reaches the process through the documented environment bridge; -the local EdgeZero config-store file is not read directly. +The crate owns Axum route registration, middleware, and environment-backed +platform services. Outbound HTTP goes through EdgeZero's Axum client. Its +route surface follows the shared core contract, but request-time KV operations +are unavailable. A startup error, such as a missing config or secret, stops the +process with a non-zero exit before it listens. Application configuration +reaches the process through the documented environment bridge; the local +EdgeZero config-store file is not read directly. + +The listener's ingress limits come from `EDGEZERO__ADAPTER__INGRESS__*` +environment variables, and an invalid value also stops startup. The +[Axum development guide](../../docs/guide/axum-dev.md#ingress-limits) lists +them. Build and test from the repository root: diff --git a/crates/trusted-server-adapter-axum/src/app.rs b/crates/trusted-server-adapter-axum/src/app.rs index 0b1858e208..e64ab9d234 100644 --- a/crates/trusted-server-adapter-axum/src/app.rs +++ b/crates/trusted-server-adapter-axum/src/app.rs @@ -2,13 +2,14 @@ use core::future::Future; use std::sync::Arc; use edgezero_adapter_axum::context::AxumRequestContext; -use edgezero_core::app::Hooks; +use edgezero_core::app::{App, Hooks}; use edgezero_core::context::RequestContext; use edgezero_core::error::EdgeError; use edgezero_core::http::{ - HandlerFuture, HeaderValue, Method, Request, Response, StatusCode, header, + HandlerFuture, HeaderValue, Method, Request, RequestParts, Response, StatusCode, header, }; -use edgezero_core::router::RouterService; +use edgezero_core::outbound::ResponseDecoding; +use edgezero_core::router::{RouterBuilder, RouterService}; use error_stack::Report; use trusted_server_core::auction::endpoints::handle_auction; use trusted_server_core::auction::{ @@ -20,15 +21,17 @@ use trusted_server_core::ec::admin::{ admin_ec_lookup_not_supported, deny_admin_diagnostic_fallback, handle_admin_eids_lookup, }; use trusted_server_core::ec::registry::PartnerRegistry; -use trusted_server_core::error::{IntoHttpResponse as _, TrustedServerError}; +use trusted_server_core::edge_app::install_edge_policies; +use trusted_server_core::error::{TrustedServerError, http_error, startup_error}; use trusted_server_core::integrations::{IntegrationRegistry, ProxyDispatchInput}; use trusted_server_core::proxy::{ handle_first_party_click, handle_first_party_proxy, handle_first_party_proxy_rebuild, handle_first_party_proxy_sign, }; use trusted_server_core::publisher::{ - AuctionDispatch, PAGE_BIDS_LEGACY_PATH, PAGE_BIDS_PATH, buffer_publisher_response_async, - handle_page_bids, handle_publisher_request, handle_tsjs_dynamic, page_bids_preflight_denied, + AuctionDispatch, PAGE_BIDS_LEGACY_PATH, PAGE_BIDS_PATH, PublisherOriginOptions, + buffer_publisher_response_async, handle_page_bids, handle_publisher_request, + handle_tsjs_dynamic, page_bids_preflight_denied, publisher_input_cap, publisher_owner_error, }; use trusted_server_core::request_signing::{ handle_trusted_server_discovery, handle_verify_signature, @@ -38,7 +41,7 @@ use trusted_server_core::settings_data::{ default_config_key, default_config_store_name, get_settings_from_config_store, }; -use trusted_server_core::platform::{ClientInfo, RuntimeServices}; +use trusted_server_core::platform::{ClientInfo, HttpClient, RuntimeServices}; use trusted_server_core::trace::{TraceMetadata, TracePreDispatchHook}; use crate::middleware::{AuthMiddleware, FinalizeResponseMiddleware, SanitizeRequestMiddleware}; @@ -92,7 +95,6 @@ fn build_state_with_services( services: Option, ) -> Result, Report> { let plan = Arc::new(compile_auction_plan(&settings)?); - plan.validate_for_target(trusted_server_core::platform::AuctionTargetId::Axum)?; let orchestrator = build_orchestrator_with_plan(Arc::clone(&plan), &settings)?; let registry = IntegrationRegistry::with_plan(&settings, plan)?; @@ -105,32 +107,13 @@ fn build_state_with_services( } impl AppState { - fn services_for_request(&self, ctx: &RequestContext) -> RuntimeServices { + fn services_for_request(&self, ctx: &RequestContext) -> Result { self.services .clone() - .unwrap_or_else(|| build_runtime_services(ctx)) + .map_or_else(|| build_runtime_services(ctx), Ok) } } -// --------------------------------------------------------------------------- -// Error helper -// --------------------------------------------------------------------------- - -/// Convert a [`Report`] into an HTTP [`Response`]. -pub(crate) fn http_error(report: &Report) -> Response { - let root_error = report.current_context(); - log::error!("Error occurred: {:?}", report); - - let body = edgezero_core::body::Body::from(format!("{}\n", root_error.user_message())); - let mut response = Response::new(body); - *response.status_mut() = root_error.status_code(); - response.headers_mut().insert( - header::CONTENT_TYPE, - HeaderValue::from_static("text/plain; charset=utf-8"), - ); - response -} - /// Builds the local `404 Not Found` returned for legacy `/admin/keys/*` /// aliases on the Axum dev server. /// @@ -161,8 +144,8 @@ where F: FnOnce(Arc, RuntimeServices, Request) -> Fut, Fut: Future>>, { - let services = state.services_for_request(&ctx); - let mut req = ctx.into_request(); + let services = state.services_for_request(&ctx)?; + let mut req = ctx.into_request()?; if let Err(error) = trusted_server_core::integrations::gpt_diagnostics::prepare_request( &state.settings, &mut req, @@ -253,27 +236,36 @@ async fn dispatch_fallback( slots: state.settings.creative_opportunity_slots(), registry: None, }; - let publisher_response = handle_publisher_request( + // Axum has no runtime memory limit of its own, so the publisher setting + // alone bounds the origin body this adapter collects before egress. + let input_cap = publisher_input_cap(&state.settings, u64::MAX); + let (publisher_response, owner) = handle_publisher_request( &state.settings, services, None, &mut ec_context, auction, req, - EdgeCacheHeader::SMaxageFallback, + PublisherOriginOptions { + edge_header: EdgeCacheHeader::SMaxageFallback, + input_cap: Some(input_cap), + response_decoding: ResponseDecoding::Original, + }, ) .await?; // Async finalize so the dispatched auction is collected and its bids are // injected before `` (the sync buffer path would drop them). - buffer_publisher_response_async( + let response = buffer_publisher_response_async( publisher_response, &method, &state.settings, &state.registry, &state.orchestrator, services, + Some(input_cap), ) - .await + .await?; + owner.attach(response).map_err(publisher_owner_error) } fn fallback_handler( @@ -459,7 +451,7 @@ fn named_route_handler( handle_trusted_server_discovery(&state.settings, &services, req) } NamedRouteHandler::VerifySignature => { - handle_verify_signature(&state.settings, &services, req) + handle_verify_signature(&state.settings, &services, req).await } NamedRouteHandler::CachePurgeNotSupported => { // The Axum dev server has no template cache to purge. 501 rather @@ -564,7 +556,7 @@ fn named_route_handler( } // --------------------------------------------------------------------------- -// Startup error fallback +// Publisher fallback methods // --------------------------------------------------------------------------- fn publisher_fallback_methods() -> [Method; 7] { @@ -579,34 +571,6 @@ fn publisher_fallback_methods() -> [Method; 7] { ] } -/// Returns a [`RouterService`] that responds to every route with the startup error. -fn startup_error_router(e: &Report) -> RouterService { - let message = Arc::new(format!("{}\n", e.current_context().user_message())); - let status = e.current_context().status_code(); - - let make_handler = |msg: Arc| { - move |_ctx: RequestContext| { - let body = edgezero_core::body::Body::from((*msg).clone()); - let mut resp = Response::new(body); - *resp.status_mut() = status; - resp.headers_mut().insert( - header::CONTENT_TYPE, - HeaderValue::from_static("text/plain; charset=utf-8"), - ); - async move { Ok::(resp) } - } - }; - - let mut router = RouterService::builder().middleware(FinalizeResponseMiddleware::new( - Arc::new(Settings::default()), - )); - for method in publisher_fallback_methods() { - router = router.route("/", method.clone(), make_handler(Arc::clone(&message))); - router = router.route("/{*rest}", method, make_handler(Arc::clone(&message))); - } - router.build() -} - // --------------------------------------------------------------------------- // TrustedServerApp // --------------------------------------------------------------------------- @@ -615,20 +579,21 @@ fn startup_error_router(e: &Report) -> RouterService { pub struct TrustedServerApp; impl Hooks for TrustedServerApp { + fn configure(app: &mut App) -> Result<(), EdgeError> { + install_edge_policies(app) + } + fn name() -> &'static str { "TrustedServer" } - fn routes() -> RouterService { - let state = match build_state() { - Ok(s) => s, - Err(ref e) => { - log::error!("failed to build application state: {:?}", e); - return startup_error_router(e); - } - }; + fn routes() -> Result { + let state = build_state().map_err(|report| { + log::error!("{report:?}"); + startup_error(&report) + })?; - build_router(&state) + Ok(router_builder(&state).build()) } } @@ -647,7 +612,34 @@ impl TrustedServerApp { settings: Settings, ) -> Result> { let state = build_state_with_settings(settings)?; - Ok(build_router(&state)) + Ok(router_builder(&state).build()) + } + + /// Build the full application router from explicit settings, with `client` + /// in router state. + /// + /// Testing seam: `EdgeZero` hands a client in router state to every + /// request in place of the adapter's own, so an app served from this + /// router on `EdgeZero`'s Axum test server starts no real transport. + /// Production installs no client in router state. + /// + /// # Errors + /// + /// Returns an error when the auction orchestrator or the integration + /// registry fail to initialise. + /// + /// # Examples + /// + /// ```ignore + /// let client = HttpClient::with_client(ScriptedHttpClient::forbidding()); + /// let router = TrustedServerApp::routes_with_settings_and_outbound_client(settings, client)?; + /// ``` + pub fn routes_with_settings_and_outbound_client( + settings: Settings, + client: HttpClient, + ) -> Result> { + let state = build_state_with_settings(settings)?; + Ok(router_builder(&state).with_state(client).build()) } /// Build the full router with explicit settings and runtime services. @@ -655,6 +647,8 @@ impl TrustedServerApp { /// Each request receives a clone of the supplied services, allowing callers /// to exercise production routes with deterministic platform dependencies. /// The supplied client metadata applies to every request to this router. + /// The services' outbound client also goes into router state, so an app + /// served from this router starts no real `EdgeZero` transport. /// /// # Errors /// @@ -670,12 +664,14 @@ impl TrustedServerApp { settings: Settings, services: RuntimeServices, ) -> Result> { + let client = services.outbound_client().clone(); let state = build_state_with_services(settings, Some(services))?; - Ok(build_router(&state)) + Ok(router_builder(&state).with_state(client).build()) } } -fn build_router(state: &Arc) -> RouterService { +/// Registers every route, middleware, and hook for `state` on a router builder. +fn router_builder(state: &Arc) -> RouterBuilder { let fallback = fallback_handler(Arc::clone(state)); let trace_state = Arc::clone(state); @@ -722,10 +718,10 @@ fn build_router(state: &Arc) -> RouterService { router = router.route("/{*rest}", method, fallback.clone()); } - router.build() + router } -fn trace_metadata(state: &AppState, request: &Request) -> TraceMetadata { +fn trace_metadata(state: &AppState, request: &RequestParts) -> TraceMetadata { if let Some(services) = &state.services { let client_info = services.client_info().clone(); let geo = client_info.client_ip.and_then(|client_ip| { @@ -738,7 +734,9 @@ fn trace_metadata(state: &AppState, request: &Request) -> TraceMetadata { } TraceMetadata { client_info: ClientInfo { - client_ip: AxumRequestContext::get(request) + client_ip: request + .extensions + .get::() .and_then(|context| context.remote_addr) .map(|peer| peer.ip()), ..ClientInfo::default() diff --git a/crates/trusted-server-adapter-axum/src/lib.rs b/crates/trusted-server-adapter-axum/src/lib.rs index 2f15e566d2..4a8fb58ba6 100644 --- a/crates/trusted-server-adapter-axum/src/lib.rs +++ b/crates/trusted-server-adapter-axum/src/lib.rs @@ -4,9 +4,12 @@ //! the Fastly Compute adapter (via Viceroy). All routes and middleware mirror //! the Fastly adapter; store and geo primitives fall back to env vars and no-ops. +// Unit tests may buffer bodies with `Body::into_bytes`; production code must not. +#![cfg_attr(test, allow(clippy::disallowed_methods))] + /// Application routing and handler registration for the Axum dev server. pub mod app; /// Request middleware (auth, response finalisation). pub mod middleware; -/// Platform-trait implementations backed by env vars and `reqwest`. +/// Platform-trait implementations backed by env vars. pub mod platform; diff --git a/crates/trusted-server-adapter-axum/src/main.rs b/crates/trusted-server-adapter-axum/src/main.rs index 66d592e0b7..327fef24b4 100644 --- a/crates/trusted-server-adapter-axum/src/main.rs +++ b/crates/trusted-server-adapter-axum/src/main.rs @@ -1,5 +1,8 @@ +use edgezero_adapter_axum::AXUM_PLATFORM; use edgezero_adapter_axum::dev_server::{AxumDevServer, AxumDevServerConfig}; -use edgezero_core::app::Hooks as _; +use edgezero_adapter_axum::ingress_config::AxumIngressConfig; +use edgezero_core::app::App; +use edgezero_core::env_config::EnvConfig; use log::LevelFilter; use trusted_server_adapter_axum::app::TrustedServerApp; @@ -26,20 +29,43 @@ fn main() { eprintln!("warning: logger init failed: {e}"); } + // `EDGEZERO__ADAPTER__INGRESS__*` sets the listener limits. An invalid + // value stops startup instead of falling back to a default. + let ingress = match AxumIngressConfig::from_env(&EnvConfig::from_env()) { + Ok(ingress) => ingress, + Err(error) => { + log::error!("trusted-server-adapter-axum failed to start: {error}"); + std::process::exit(1); + } + }; + let config = match port_from_env() { // When PORT is set, bind to a specific address so integration tests // can allocate a fresh OS port each run and avoid TIME_WAIT flakiness. Some(port) => AxumDevServerConfig { addr: std::net::SocketAddr::from(([127, 0, 0, 1], port)), enable_ctrl_c: true, + ingress, }, // Normal development path: read bind address from axum.toml. - None => AxumDevServerConfig::default(), + None => AxumDevServerConfig { + ingress, + ..AxumDevServerConfig::default() + }, + }; + + // A settings failure stops startup before the listener binds. `routes` + // has already logged the full report. + let app = match App::build::(AXUM_PLATFORM) { + Ok(app) => app, + Err(error) => { + log::error!("trusted-server-adapter-axum failed to start: {error}"); + std::process::exit(1); + } }; log::info!("Listening on http://{}", config.addr); - let router = TrustedServerApp::routes(); - if let Err(err) = AxumDevServer::with_config(router, config).run() { + if let Err(err) = AxumDevServer::with_config(app, config).run() { log::error!("trusted-server-adapter-axum failed: {err}"); std::process::exit(1); } diff --git a/crates/trusted-server-adapter-axum/src/middleware.rs b/crates/trusted-server-adapter-axum/src/middleware.rs index 893cdc12db..8e51666125 100644 --- a/crates/trusted-server-adapter-axum/src/middleware.rs +++ b/crates/trusted-server-adapter-axum/src/middleware.rs @@ -7,6 +7,7 @@ use edgezero_core::http::{HeaderValue, Response}; use edgezero_core::middleware::{Middleware, Next}; use trusted_server_core::auth::enforce_basic_auth; use trusted_server_core::constants::HEADER_X_GEO_INFO_AVAILABLE; +use trusted_server_core::error::http_error; use trusted_server_core::http_util::sanitize_trusted_client_ip_headers; use trusted_server_core::settings::Settings; @@ -39,7 +40,7 @@ impl SanitizeRequestMiddleware { impl Middleware for SanitizeRequestMiddleware { async fn handle(&self, mut ctx: RequestContext, next: Next<'_>) -> Result { sanitize_trusted_client_ip_headers( - ctx.request_mut(), + ctx.parts_mut(), self.settings.trusted_client_ip.as_ref(), ); next.run(ctx).await @@ -106,12 +107,12 @@ impl AuthMiddleware { #[async_trait(?Send)] impl Middleware for AuthMiddleware { async fn handle(&self, mut ctx: RequestContext, next: Next<'_>) -> Result { - match enforce_basic_auth(&self.settings, ctx.request_mut()) { + match enforce_basic_auth(&self.settings, ctx.parts_mut()) { Ok(Some(response)) => return Ok(response), Ok(None) => {} Err(report) => { log::error!("auth check failed: {:?}", report); - return Ok(crate::app::http_error(&report)); + return Ok(http_error(&report)); } } @@ -138,6 +139,10 @@ pub(crate) fn apply_finalize_headers(settings: &Settings, response: &mut Respons trusted_server_core::version_header::apply_git_version_header(response); + // The finalize middleware runs once per request, so each skipped operator + // entry warns once. + trusted_server_core::response_privacy::warn_skipped_operator_response_headers(settings); + // Cookie-bearing responses stay private to shared caches and operator // headers cannot re-enable caching for uncacheable per-user payloads. trusted_server_core::response_privacy::apply_response_headers_with_cache_privacy( @@ -164,6 +169,7 @@ mod tests { use futures::executor::block_on; use trusted_server_core::redacted::Redacted; use trusted_server_core::settings::TrustedClientIpConfig; + use trusted_server_core::test_support::logging::WarningCapture; fn empty_response() -> Response { response_builder() @@ -208,6 +214,29 @@ mod tests { s } + #[test] + fn warns_once_about_a_skipped_operator_entry() { + let settings = settings_with_response_headers(vec![("connection", "keep-alive")]); + let mut response = empty_response(); + let capture = WarningCapture::start(); + + apply_finalize_headers(&settings, &mut response); + + let skipped = capture + .take() + .into_iter() + .filter(|warning| warning.message.contains("`connection`")) + .count(); + assert_eq!( + skipped, 1, + "should warn once about the skipped operator entry" + ); + assert!( + !response.headers().contains_key("connection"), + "should skip the operator entry" + ); + } + #[test] fn sets_geo_unavailable_header() { let settings = settings_with_response_headers(vec![]); @@ -275,8 +304,8 @@ mod tests { let handler_observed = Arc::clone(&handler_observed); async move { *handler_observed.lock().expect("should lock observation") = Some(( - ctx.request().headers().contains_key("x-reader-ip"), - ctx.request().headers().contains_key("x-reader-ip-auth"), + ctx.headers().contains_key("x-reader-ip"), + ctx.headers().contains_key("x-reader-ip-auth"), )); Ok::(empty_response()) } diff --git a/crates/trusted-server-adapter-axum/src/platform.rs b/crates/trusted-server-adapter-axum/src/platform.rs index 229bb902f7..67ee7da5ca 100644 --- a/crates/trusted-server-adapter-axum/src/platform.rs +++ b/crates/trusted-server-adapter-axum/src/platform.rs @@ -1,18 +1,14 @@ -use std::future::Future; use std::net::IpAddr; -use std::pin::Pin; use std::sync::Arc; -use std::task::{Context, Poll}; -use std::time::Duration; use async_trait::async_trait; -use edgezero_core::http::{HeaderMap, HeaderName, HeaderValue, header}; -use error_stack::{Report, ResultExt as _}; +use edgezero_adapter_axum::context::AxumRequestContext; +use edgezero_core::error::EdgeError; +use error_stack::Report; +use trusted_server_core::outbound::request_outbound_client; use trusted_server_core::platform::{ - BackendNamingPolicy, ClientInfo, GeoInfo, PlatformBackend, PlatformBackendSpec, - PlatformConfigStore, PlatformError, PlatformGeo, PlatformHttpClient, PlatformHttpRequest, - PlatformPendingRequest, PlatformResponse, PlatformSecretStore, PlatformSelectResult, - RuntimeServices, StoreId, StoreName, + ClientInfo, GeoInfo, PlatformConfigStore, PlatformError, PlatformGeo, PlatformKvStore, + PlatformSecretStore, RuntimeServices, StoreId, StoreName, }; // --------------------------------------------------------------------------- @@ -54,6 +50,7 @@ fn secret_env_var(store_name: &str, key: &str) -> String { /// Write operations are unsupported in local development. pub struct AxumPlatformConfigStore; +#[async_trait(?Send)] impl PlatformConfigStore for AxumPlatformConfigStore { fn get(&self, store_name: &StoreName, key: &str) -> Result> { let var_name = config_env_var(store_name.as_ref(), key); @@ -64,7 +61,7 @@ impl PlatformConfigStore for AxumPlatformConfigStore { }) } - fn put( + async fn put( &self, store_id: &StoreId, key: &str, @@ -80,7 +77,7 @@ impl PlatformConfigStore for AxumPlatformConfigStore { .attach("config store writes are not supported on the Axum dev server")) } - fn delete(&self, store_id: &StoreId, key: &str) -> Result<(), Report> { + async fn delete(&self, store_id: &StoreId, key: &str) -> Result<(), Report> { log::warn!( "AxumPlatformConfigStore: delete from store '{}' key '{}' ignored \ (config store deletes are not supported on the Axum dev server)", @@ -102,6 +99,7 @@ impl PlatformConfigStore for AxumPlatformConfigStore { /// Write operations are unsupported in local development. pub struct AxumPlatformSecretStore; +#[async_trait(?Send)] impl PlatformSecretStore for AxumPlatformSecretStore { fn get_bytes( &self, @@ -118,7 +116,7 @@ impl PlatformSecretStore for AxumPlatformSecretStore { }) } - fn create( + async fn create( &self, store_id: &StoreId, name: &str, @@ -134,7 +132,7 @@ impl PlatformSecretStore for AxumPlatformSecretStore { .attach("secret store writes are not supported on the Axum dev server")) } - fn delete(&self, store_id: &StoreId, name: &str) -> Result<(), Report> { + async fn delete(&self, store_id: &StoreId, name: &str) -> Result<(), Report> { log::warn!( "AxumPlatformSecretStore: delete '{}' from store '{}' ignored \ (secret store deletes are not supported on the Axum dev server)", @@ -146,33 +144,6 @@ impl PlatformSecretStore for AxumPlatformSecretStore { } } -// --------------------------------------------------------------------------- -// PlatformBackend -// --------------------------------------------------------------------------- - -/// No-op backend for the Axum dev server. -/// -/// Returns a deterministic name; `ensure` is a no-op returning the same name. -/// The Axum HTTP client sends directly to URIs and ignores backend names. -pub struct AxumPlatformBackend; - -impl PlatformBackend for AxumPlatformBackend { - fn naming_policy(&self) -> BackendNamingPolicy { - BackendNamingPolicy::Axum - } - - fn predict_name(&self, spec: &PlatformBackendSpec) -> Result> { - self.naming_policy() - .predict(spec) - .map(|prediction| prediction.name) - .change_context(PlatformError::Backend) - } - - fn ensure(&self, spec: &PlatformBackendSpec) -> Result> { - self.predict_name(spec) - } -} - // --------------------------------------------------------------------------- // PlatformGeo // --------------------------------------------------------------------------- @@ -186,385 +157,6 @@ impl PlatformGeo for AxumPlatformGeo { } } -// --------------------------------------------------------------------------- -// PlatformHttpClient -// --------------------------------------------------------------------------- - -type SpawnedRequestResult = Result<(u16, Vec<(String, Vec)>, Vec), Report>; - -fn sanitized_response_headers(headers: &HeaderMap) -> Vec<(String, Vec)> { - let connection_tokens = connection_header_tokens(headers); - - headers - .iter() - .filter(|(name, _)| !is_hop_by_hop_response_header(name, &connection_tokens)) - .map(|(name, value)| (name.to_string(), value.as_bytes().to_vec())) - .collect() -} - -fn connection_header_tokens(headers: &HeaderMap) -> Vec { - headers - .get_all(header::CONNECTION) - .iter() - .filter_map(header_value_to_str) - .flat_map(|value| value.split(',')) - .map(str::trim) - .filter(|token| !token.is_empty()) - .filter_map(|token| HeaderName::from_bytes(token.as_bytes()).ok()) - .collect() -} - -fn header_value_to_str(value: &HeaderValue) -> Option<&str> { - value.to_str().ok() -} - -fn is_hop_by_hop_response_header(name: &HeaderName, connection_tokens: &[HeaderName]) -> bool { - name == header::CONNECTION - || name == header::PROXY_AUTHENTICATE - || name == header::PROXY_AUTHORIZATION - || name == header::TE - || name == header::TRAILER - || name == header::TRANSFER_ENCODING - || name == header::UPGRADE - || name.as_str().eq_ignore_ascii_case("keep-alive") - || connection_tokens.iter().any(|token| token == name) -} - -/// Buffered response parts from a spawned outbound request. -/// -/// Stored inside [`PlatformPendingRequest`] so that [`AxumPlatformHttpClient::select`] -/// can poll multiple in-flight handles concurrently via -/// [`futures::future::select_all`]. -struct AxumPendingHandle { - backend_name: String, - handle: tokio::task::JoinHandle, -} - -impl Drop for AxumPendingHandle { - fn drop(&mut self) { - // Abort instead of detaching: when the orchestrator hits the auction - // deadline and drops the remaining pending requests, the abandoned - // bidder tasks would otherwise keep running for up to the 30s - // transport timeout. - self.handle.abort(); - } -} - -/// Resolves to the backend name together with the task result so that -/// [`futures::future::select_all`] callers never have to reconstruct which -/// backend a completion belongs to by position. `select_all` removes the -/// ready future with `swap_remove` and makes no ordering guarantee for the -/// remaining futures, so positional bookkeeping would mislabel them. -impl Future for AxumPendingHandle { - type Output = (String, Result); - - fn poll(mut self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll { - match Pin::new(&mut self.handle).poll(cx) { - Poll::Ready(result) => { - let backend_name = std::mem::take(&mut self.backend_name); - Poll::Ready((backend_name, result)) - } - Poll::Pending => Poll::Pending, - } - } -} - -/// reqwest-backed HTTP client for the Axum dev server. -/// -/// `send_async` buffers any `Body::Stream` in the calling context, then spawns -/// a `tokio` task for each outbound request so that multiple `send_async` calls -/// run concurrently. `select` uses [`futures::future::select_all`] to wait for -/// the first completing handle, preserving fan-out semantics. -pub struct AxumPlatformHttpClient { - client: reqwest::Client, -} - -impl AxumPlatformHttpClient { - /// Create a new client with sensible dev-server timeouts. - /// - /// # Panics - /// - /// Panics if the underlying `reqwest::Client` cannot be built (should not - /// happen with the default TLS configuration on any supported platform). - #[must_use] - pub fn new() -> Self { - Self { - client: reqwest::Client::builder() - .connect_timeout(Duration::from_secs(5)) - .timeout(Duration::from_secs(30)) - // Disable automatic redirects: core proxy code enforces redirect - // limits and allowed_domains checks itself. Without this, reqwest - // would follow Location headers internally and bypass those checks. - .redirect(reqwest::redirect::Policy::none()) - .build() - .expect("should build reqwest client"), - } - } - - /// Drain `body` to a `Vec`. - /// - /// For `Body::Stream` this awaits every chunk in the current async context - /// (where `LocalBoxStream` is valid) before the bytes are moved into a - /// `tokio::spawn` task that requires `Send`. - async fn buffer_body( - body: edgezero_core::body::Body, - ) -> Result, Report> { - match body { - edgezero_core::body::Body::Once(bytes) => Ok(bytes.to_vec()), - edgezero_core::body::Body::Stream(mut stream) => { - log::debug!("buffering Body::Stream into Vec for outbound request"); - use futures::StreamExt as _; - let mut buf = Vec::new(); - while let Some(chunk) = stream.next().await { - let bytes = chunk.map_err(|e| { - Report::new(PlatformError::HttpClient) - .attach(format!("failed to buffer outbound streaming body: {e}")) - })?; - buf.extend_from_slice(&bytes); - } - Ok(buf) - } - } - } - - async fn execute( - &self, - request: PlatformHttpRequest, - ) -> Result> { - let uri = request.request.uri().to_string(); - let method = reqwest::Method::from_bytes(request.request.method().as_str().as_bytes()) - .change_context(PlatformError::HttpClient)?; - - let mut builder = self.client.request(method, &uri); - for (name, value) in request.request.headers() { - builder = builder.header(name.as_str(), value.as_bytes()); - } - - let (_, body) = request.request.into_parts(); - let body_bytes = Self::buffer_body(body).await?; - if !body_bytes.is_empty() { - builder = builder.body(body_bytes); - } - - let resp = builder - .send() - .await - .change_context(PlatformError::HttpClient) - .attach(format!("outbound request to {uri} failed"))?; - - let status = resp.status().as_u16(); - let mut edge_builder = edgezero_core::http::response_builder().status(status); - for (name, value) in sanitized_response_headers(resp.headers()) { - edge_builder = edge_builder.header(name.as_str(), value.as_slice()); - } - let resp_bytes = resp - .bytes() - .await - .change_context(PlatformError::HttpClient)?; - // Upstream responses are buffered whole with no cap — acceptable for - // the dev server, but log the size so a large or hostile upstream is - // visible instead of silently growing the heap. - log::debug!( - "buffered {} upstream response bytes from {uri}", - resp_bytes.len() - ); - let edge_resp = edge_builder - .body(edgezero_core::body::Body::from(resp_bytes.to_vec())) - .change_context(PlatformError::HttpClient)?; - - Ok(PlatformResponse::new(edge_resp).with_backend_name(request.backend_name)) - } -} - -impl Default for AxumPlatformHttpClient { - fn default() -> Self { - Self::new() - } -} - -#[async_trait(?Send)] -impl PlatformHttpClient for AxumPlatformHttpClient { - async fn send( - &self, - request: PlatformHttpRequest, - ) -> Result> { - self.execute(request).await - } - - async fn send_async( - &self, - request: PlatformHttpRequest, - ) -> Result> { - let backend_name = request.backend_name.clone(); - - // Extract all Send-compatible parts before spawning. - let uri = request.request.uri().to_string(); - let method_bytes = request.request.method().as_str().as_bytes().to_vec(); - let headers: Vec<(String, Vec)> = request - .request - .headers() - .iter() - .map(|(n, v)| (n.to_string(), v.as_bytes().to_vec())) - .collect(); - - // Buffer any LocalBoxStream body here in the ?Send context before spawn. - let (_, body) = request.request.into_parts(); - let body_bytes = Self::buffer_body(body).await?; - - let client = self.client.clone(); - let handle = tokio::spawn(async move { - let method = reqwest::Method::from_bytes(&method_bytes) - .map_err(|e| Report::new(PlatformError::HttpClient).attach(e.to_string()))?; - let mut builder = client.request(method, &uri); - for (name, value) in &headers { - builder = builder.header(name.as_str(), value.as_slice()); - } - if !body_bytes.is_empty() { - builder = builder.body(body_bytes); - } - let resp = builder.send().await.map_err(|e| { - Report::new(PlatformError::HttpClient) - .attach(format!("outbound request to {uri} failed: {e}")) - })?; - let status = resp.status().as_u16(); - let resp_headers = sanitized_response_headers(resp.headers()); - let body = resp - .bytes() - .await - .map_err(|e| Report::new(PlatformError::HttpClient).attach(e.to_string()))? - .to_vec(); - // Same unbounded-buffering note as the synchronous path: log the - // size so large upstream responses are visible in dev. - log::debug!("buffered {} upstream response bytes from {uri}", body.len()); - Ok::<_, Report>((status, resp_headers, body)) - }); - - let pending = AxumPendingHandle { - backend_name: backend_name.clone(), - handle, - }; - Ok(PlatformPendingRequest::new(pending).with_backend_name(backend_name)) - } - - async fn select_until( - &self, - pending_requests: Vec, - deadline: web_time::Instant, - ) -> Result, Report> { - let mut handles = recover_pending_handles(pending_requests)?; - if handles.is_empty() { - return Err(Report::new(PlatformError::HttpClient) - .attach("select_until called with no pending requests")); - } - let remaining = deadline.saturating_duration_since(web_time::Instant::now()); - if !remaining.is_zero() { - // Borrow the handles so timer expiry cannot drop completed siblings - // before the final nonblocking snapshot inspects them. - let selected = - tokio::time::timeout(remaining, futures::future::select_all(handles.iter_mut())) - .await; - if let Ok(((backend_name, result), index, _)) = selected { - drop(handles.swap_remove(index)); - return Ok(Some(completed_selection(backend_name, result, handles))); - } - } - // Snapshot once: only completed tasks survive into subsequent calls. - // An unfinished task is aborted on drop and cannot become a late bid. - handles.retain(|pending| pending.handle.is_finished()); - if handles.is_empty() { - return Ok(None); - } - let ((backend_name, result), _, remaining_handles) = - futures::future::select_all(handles).await; - Ok(Some(completed_selection( - backend_name, - result, - remaining_handles, - ))) - } - - async fn select( - &self, - pending_requests: Vec, - ) -> Result> { - if pending_requests.is_empty() { - return Err(Report::new(PlatformError::HttpClient) - .attach("select called with an empty pending_requests list")); - } - - let handles = recover_pending_handles(pending_requests)?; - - // Each AxumPendingHandle resolves to (backend_name, result), so the - // remaining handles keep their own backend names — no positional - // reconstruction (select_all does not preserve the order of the - // remaining futures). - let ((backend_name, result), _ready_idx, remaining_handles) = - futures::future::select_all(handles).await; - - Ok(completed_selection(backend_name, result, remaining_handles)) - } -} - -fn completed_selection( - backend_name: String, - result: Result, - remaining_handles: Vec, -) -> PlatformSelectResult { - let remaining: Vec = remaining_handles - .into_iter() - .map(|handle| { - let backend_name = handle.backend_name.clone(); - PlatformPendingRequest::new(handle).with_backend_name(backend_name) - }) - .collect(); - - // Map join panics and per-request errors into ready: Err(...) so that the - // auction orchestrator can log the failure and continue with remaining providers - // rather than treating one bad provider as a fatal select() failure. - let ready = result - .map_err(|e| { - Report::new(PlatformError::HttpClient) - .attach(format!("auction request task panicked: {e}")) - }) - .and_then(|inner| inner) - .and_then(|(status, headers, body)| { - let mut builder = edgezero_core::http::response_builder().status(status); - for (name, value) in &headers { - builder = builder.header(name.as_str(), value.as_slice()); - } - builder - .body(edgezero_core::body::Body::from(body)) - .change_context(PlatformError::HttpClient) - }) - .map(|edge_resp| PlatformResponse::new(edge_resp).with_backend_name(backend_name.clone())); - - // Attribute the failure to its backend so the orchestrator can remove - // the provider and record a BidStatus::Error, matching the Fastly - // adapter. Without this, a failed provider silently vanishes through - // the orchestrator's "backend not identified" branch. - let failed_backend_name = ready.as_ref().err().map(|_| backend_name); - - PlatformSelectResult { - ready, - remaining, - failed_backend_name, - } -} - -fn recover_pending_handles( - pending_requests: Vec, -) -> Result, Report> { - pending_requests - .into_iter() - .map(|pending| { - pending.downcast::().map_err(|_| { - Report::new(PlatformError::HttpClient) - .attach("unexpected pending type in Axum selection") - }) - }) - .collect() -} - // --------------------------------------------------------------------------- // build_runtime_services // --------------------------------------------------------------------------- @@ -576,20 +168,28 @@ fn recover_pending_handles( /// The generic runtime KV slot uses /// [`trusted_server_core::platform::UnavailableKvStore`]. A `warn` log is /// emitted once per process. -pub fn build_runtime_services(ctx: &edgezero_core::context::RequestContext) -> RuntimeServices { +/// +/// The outbound client is the one the request context carries, with the proxy +/// marker turned off, and the clock is the context's monotonic clock. +/// +/// # Errors +/// +/// Returns an internal [`EdgeError`] when the request carries no outbound +/// client. +pub fn build_runtime_services( + ctx: &edgezero_core::context::RequestContext, +) -> Result { static KV_WARNED: std::sync::OnceLock<()> = std::sync::OnceLock::new(); KV_WARNED.get_or_init(|| { log::warn!("Axum dev server: generic runtime KV is unavailable (UnavailableKvStore)."); }); - let client_ip = edgezero_adapter_axum::context::AxumRequestContext::get(ctx.request()) + let client_ip = ctx + .extensions() + .get::() .and_then(|c| c.remote_addr) .map(|addr| addr.ip()); - use trusted_server_core::platform::{ - PlatformBackend, PlatformConfigStore, PlatformGeo, PlatformKvStore, PlatformSecretStore, - }; - // Stateless shims are promoted to process-wide statics so callers clone // an existing Arc instead of allocating a new one per request. static CONFIG_STORE: std::sync::OnceLock> = @@ -597,13 +197,14 @@ pub fn build_runtime_services(ctx: &edgezero_core::context::RequestContext) -> R static SECRET_STORE: std::sync::OnceLock> = std::sync::OnceLock::new(); static KV_STORE: std::sync::OnceLock> = std::sync::OnceLock::new(); - static BACKEND: std::sync::OnceLock> = std::sync::OnceLock::new(); static GEO: std::sync::OnceLock> = std::sync::OnceLock::new(); static TIMER: std::sync::OnceLock> = std::sync::OnceLock::new(); - RuntimeServices::builder() + let outbound_client = request_outbound_client(ctx)?; + + Ok(RuntimeServices::builder() .timer(Arc::clone(TIMER.get_or_init(|| Arc::new(AxumTimer)))) .config_store(Arc::clone(CONFIG_STORE.get_or_init(|| { Arc::new(AxumPlatformConfigStore) as Arc @@ -614,14 +215,8 @@ pub fn build_runtime_services(ctx: &edgezero_core::context::RequestContext) -> R .kv_store(Arc::clone(KV_STORE.get_or_init(|| { Arc::new(trusted_server_core::platform::UnavailableKvStore) as Arc }))) - .backend(Arc::clone(BACKEND.get_or_init(|| { - Arc::new(AxumPlatformBackend) as Arc - }))) - // Keep the HTTP client request-scoped in the dev adapter. Sharing a pooled - // client across requests previously regressed the Next.js server-action → - // API-route integration flow by reusing a poisoned connection after a - // truncated POST. Revisit pooling if profiling shows allocation cost. - .http_client(Arc::new(AxumPlatformHttpClient::new())) + .outbound_client(outbound_client) + .clock(ctx.monotonic_clock()) .geo(Arc::clone(GEO.get_or_init(|| { Arc::new(AxumPlatformGeo) as Arc }))) @@ -631,7 +226,7 @@ pub fn build_runtime_services(ctx: &edgezero_core::context::RequestContext) -> R tls_cipher: None, ..ClientInfo::default() }) - .build() + .build()) } struct AxumTimer; @@ -651,18 +246,63 @@ impl trusted_server_core::platform::PlatformTimer for AxumTimer { #[cfg(test)] mod tests { use super::*; - use edgezero_core::body::Body as EdgeBody; - use std::time::Duration; - use tokio::io::{AsyncReadExt as _, AsyncWriteExt as _}; + use trusted_server_core::platform::HttpClient; + use trusted_server_core::test_support::scripted_client::ScriptedHttpClient; - #[test] - fn html_finalization_timer_is_shared_without_an_executor() { - let request = edgezero_core::http::request_builder() + /// Builds a context whose request carries `client`, as `EdgeZero`'s Axum + /// request conversion installs its default client. + fn context_with_client(client: Option) -> edgezero_core::context::RequestContext { + let mut request = edgezero_core::http::request_builder() .body(edgezero_core::body::Body::empty()) .expect("should build a request"); - let context = edgezero_core::context::RequestContext::new(request, Default::default()); - let services = build_runtime_services(&context); - let next_services = build_runtime_services(&context); + if let Some(client) = client { + request.extensions_mut().insert(client); + } + edgezero_core::context::RequestContext::new(request, Default::default()) + } + + #[test] + fn runtime_services_carry_the_context_client_with_the_proxy_marker_off() { + let client = HttpClient::with_client(ScriptedHttpClient::forbidding()); + assert!( + client.proxy_marker_enabled(), + "should start from a client whose marker is on" + ); + let context = context_with_client(Some(client)); + + let services = build_runtime_services(&context).expect("should build runtime services"); + + assert!( + !services.outbound_client().proxy_marker_enabled(), + "should turn the proxy marker off on the context client" + ); + } + + #[test] + fn runtime_services_reject_a_request_without_an_outbound_client() { + let context = context_with_client(None); + + let error = build_runtime_services(&context) + .expect_err("should reject a request with no outbound client"); + + assert_eq!( + error.status(), + edgezero_core::http::StatusCode::INTERNAL_SERVER_ERROR, + "should answer a missing client with an internal error" + ); + assert_eq!( + error.to_string(), + "internal error: request has no outbound client", + "should name the missing client" + ); + } + + #[test] + fn html_finalization_timer_is_shared_without_an_executor() { + let context = context_with_client(Some(ScriptedHttpClient::forbidding().http_client())); + let services = build_runtime_services(&context).expect("should build runtime services"); + let next_services = + build_runtime_services(&context).expect("should build runtime services again"); assert!( std::ptr::eq( services @@ -676,16 +316,6 @@ mod tests { ); } - #[test] - fn auction_http_capabilities_are_explicit() { - let client = AxumPlatformHttpClient::new(); - let capabilities = trusted_server_core::platform::AuctionTargetId::Axum - .descriptor() - .capabilities(); - assert!(client.supports_concurrent_fanout()); - assert!(capabilities.supports_concurrent_provider_fanout()); - } - #[test] fn config_env_var_normalizes_store_and_key() { assert_eq!( @@ -735,47 +365,6 @@ mod tests { ); } - #[test] - fn backend_predict_name_returns_deterministic_string() { - let backend = AxumPlatformBackend; - let spec = PlatformBackendSpec { - scheme: "https".to_string(), - host: "example.com".to_string(), - port: None, - certificate_check: true, - first_byte_timeout: Duration::from_secs(15), - between_bytes_timeout: Duration::from_secs(15), - host_header_override: None, - discriminator: None, - }; - let name1 = backend.predict_name(&spec).expect("should return a name"); - let name2 = backend - .predict_name(&spec) - .expect("should return same name"); - assert!(!name1.is_empty(), "should return a non-empty name"); - assert_eq!(name1, name2, "should be deterministic"); - } - - #[test] - fn backend_ensure_returns_same_name_as_predict() { - let backend = AxumPlatformBackend; - let spec = PlatformBackendSpec { - scheme: "https".to_string(), - host: "example.com".to_string(), - port: None, - certificate_check: true, - first_byte_timeout: Duration::from_secs(15), - between_bytes_timeout: Duration::from_secs(15), - host_header_override: None, - discriminator: None, - }; - assert_eq!( - backend.predict_name(&spec).expect("should return name"), - backend.ensure(&spec).expect("should return name"), - "ensure should equal predict_name" - ); - } - #[test] fn geo_always_returns_none() { let geo = AxumPlatformGeo; @@ -786,426 +375,4 @@ mod tests { .expect("should not error"); assert!(with_ip.is_none(), "should return None for any IP"); } - - #[tokio::test(flavor = "multi_thread", worker_threads = 2)] - async fn http_client_surfaces_redirect_without_following() { - let url = serve_raw_response( - b"HTTP/1.1 302 Found\r\nLocation: https://redirect.example/next\r\nContent-Length: 0\r\n\r\n", - ) - .await; - let request = edgezero_core::http::request_builder() - .uri(url) - .body(EdgeBody::empty()) - .expect("should build outbound request"); - - let response = AxumPlatformHttpClient::new() - .send(PlatformHttpRequest::new(request, "test_backend")) - .await - .expect("should surface redirect") - .response; - - assert_eq!(response.status().as_u16(), 302); - assert_eq!( - response - .headers() - .get(edgezero_core::http::header::LOCATION) - .and_then(|value| value.to_str().ok()), - Some("https://redirect.example/next") - ); - } - - #[tokio::test(flavor = "multi_thread", worker_threads = 2)] - async fn http_client_strips_hop_by_hop_response_headers() { - let url = serve_raw_response( - b"HTTP/1.1 200 OK\r\n\ - Transfer-Encoding: chunked\r\n\ - Connection: keep-alive, x-remove-me\r\n\ - Keep-Alive: timeout=5\r\n\ - X-Remove-Me: listed-by-connection\r\n\ - X-Preserve-Me: application-header\r\n\ - \r\n\ - 2\r\n\ - ok\r\n\ - 0\r\n\ - \r\n", - ) - .await; - - let request = edgezero_core::http::request_builder() - .uri(url) - .body(EdgeBody::empty()) - .expect("should build outbound request"); - - let response = AxumPlatformHttpClient::new() - .send(PlatformHttpRequest::new(request, "test_backend")) - .await - .expect("should proxy raw response") - .response; - - assert!( - response.headers().get(header::TRANSFER_ENCODING).is_none(), - "should strip transfer-encoding" - ); - assert!( - response.headers().get(header::CONNECTION).is_none(), - "should strip connection" - ); - assert!( - response.headers().get("keep-alive").is_none(), - "should strip keep-alive" - ); - assert!( - response.headers().get("x-remove-me").is_none(), - "should strip headers named by connection" - ); - assert_eq!( - response - .headers() - .get("x-preserve-me") - .and_then(|value| value.to_str().ok()), - Some("application-header"), - "should preserve end-to-end headers" - ); - assert_eq!( - response - .into_body() - .into_bytes() - .unwrap_or_default() - .as_ref(), - b"ok", - "should preserve decoded response body" - ); - } - - #[tokio::test(flavor = "multi_thread", worker_threads = 2)] - async fn select_attributes_failed_backend_name() { - // Bind and immediately drop a listener so the port is closed — the - // request fails with connection refused. - let listener = tokio::net::TcpListener::bind("127.0.0.1:0") - .await - .expect("should bind probe listener"); - let addr = listener.local_addr().expect("should read local address"); - drop(listener); - - let request = edgezero_core::http::request_builder() - .uri(format!("http://{addr}/")) - .body(EdgeBody::empty()) - .expect("should build outbound request"); - - let client = AxumPlatformHttpClient::new(); - let pending = client - .send_async(PlatformHttpRequest::new(request, "failing_backend")) - .await - .expect("should spawn async request"); - - let result = client - .select(vec![pending]) - .await - .expect("select should surface the failure via ready, not a fatal error"); - - assert!( - result.ready.is_err(), - "request to a closed port should fail" - ); - assert_eq!( - result.failed_backend_name.as_deref(), - Some("failing_backend"), - "failed provider must be attributed to its backend so the orchestrator can record BidStatus::Error" - ); - assert!( - result.remaining.is_empty(), - "no remaining requests expected" - ); - } - - async fn deadline_pending( - client: &AxumPlatformHttpClient, - url: String, - backend: &str, - ) -> PlatformPendingRequest { - let request = edgezero_core::http::request_builder() - .uri(url) - .body(EdgeBody::empty()) - .expect("should build deadline request"); - client - .send_async(PlatformHttpRequest::new(request, backend)) - .await - .expect("should dispatch deadline request") - } - - async fn serve_slow_response(trickle_body: bool) -> (String, tokio::task::JoinHandle<()>) { - let listener = tokio::net::TcpListener::bind("127.0.0.1:0") - .await - .expect("should bind deadline server"); - let address = listener - .local_addr() - .expect("should read deadline server address"); - let task = tokio::spawn(async move { - let (mut stream, _) = listener - .accept() - .await - .expect("should accept deadline request"); - let mut request = [0; 1024]; - let bytes_read = stream - .read(&mut request) - .await - .expect("should read deadline request"); - assert!(bytes_read > 0, "should receive a deadline request"); - if trickle_body { - if stream - .write_all(b"HTTP/1.1 200 OK\r\nContent-Length: 100\r\n\r\n") - .await - .is_err() - { - return; - } - for _ in 0..100 { - if stream.write_all(b"x").await.is_err() { - return; - } - tokio::time::sleep(Duration::from_millis(50)).await; - } - } else { - tokio::time::sleep(Duration::from_secs(5)).await; - let _ = stream - .write_all(b"HTTP/1.1 200 OK\r\nContent-Length: 2\r\n\r\nok") - .await; - } - }); - (format!("http://{address}/"), task) - } - - #[tokio::test] - async fn select_until_bounds_delayed_headers_and_trickling_bodies() { - let client = AxumPlatformHttpClient::new(); - for trickle_body in [false, true] { - let (url, server) = serve_slow_response(trickle_body).await; - let pending = deadline_pending(&client, url, "slow").await; - let start = web_time::Instant::now(); - let result = tokio::time::timeout( - Duration::from_secs(2), - client.select_until(vec![pending], start + Duration::from_millis(200)), - ) - .await - .expect("should finish before the slow response") - .expect("should enforce the deadline"); - server.abort(); - assert!( - result.is_none(), - "should time out before the entire response completes" - ); - assert!( - start.elapsed() < Duration::from_secs(2), - "should bound the complete response wait" - ); - } - } - - #[tokio::test] - async fn select_until_preserves_fast_success_before_slow_body_expires() { - let client = AxumPlatformHttpClient::new(); - let (slow_url, server) = serve_slow_response(true).await; - let fast_url = serve_raw_response(b"HTTP/1.1 200 OK\r\nContent-Length: 2\r\n\r\nok").await; - let slow = deadline_pending(&client, slow_url, "slow").await; - let fast = deadline_pending(&client, fast_url, "fast").await; - let deadline = web_time::Instant::now() + Duration::from_millis(500); - let result = client - .select_until(vec![slow, fast], deadline) - .await - .expect("should collect fast response") - .expect("should complete before the deadline"); - assert_eq!( - result - .ready - .expect("should succeed") - .backend_name - .as_deref(), - Some("fast"), - "should collect the fast complete body first" - ); - assert_eq!( - result.remaining.len(), - 1, - "should preserve the slow body handle" - ); - let result = client - .select_until(result.remaining, deadline) - .await - .expect("should bound slow response"); - server.abort(); - assert!(result.is_none(), "should expire the remaining slow body"); - } - - #[tokio::test] - async fn select_until_expired_deadline_preserves_buffered_response() { - let client = AxumPlatformHttpClient::new(); - let handle = tokio::spawn(async { Ok((200, Vec::new(), b"ok".to_vec())) }); - while !handle.is_finished() { - tokio::task::yield_now().await; - } - let pending = PlatformPendingRequest::new(AxumPendingHandle { - backend_name: "ready".to_owned(), - handle, - }); - let result = client - .select_until(vec![pending], web_time::Instant::now()) - .await - .expect("should recognize expired deadline"); - assert!( - result.is_some(), - "should preserve a complete buffered response without waiting after cutoff" - ); - } - - #[tokio::test] - async fn expired_snapshot_preserves_finished_siblings_and_aborts_unfinished_task() { - let mut tasks = Vec::new(); - for name in ["ready-a", "ready-b"] { - tasks.push(AxumPendingHandle { - backend_name: name.to_owned(), - handle: tokio::spawn(async { Ok((200, Vec::new(), b"ok".to_vec())) }), - }); - } - while tasks.iter().any(|pending| !pending.handle.is_finished()) { - tokio::task::yield_now().await; - } - let (started_sender, started_receiver) = tokio::sync::oneshot::channel(); - let (drop_sender, drop_receiver) = tokio::sync::oneshot::channel::<()>(); - tasks.push(AxumPendingHandle { - backend_name: "unfinished".to_owned(), - handle: tokio::spawn(async move { - let _drop_signal = drop_sender; - started_sender - .send(()) - .expect("should signal unfinished task start"); - futures::future::pending::().await - }), - }); - started_receiver - .await - .expect("should start unfinished task"); - let pending = tasks - .into_iter() - .map(|task| { - let name = task.backend_name.clone(); - PlatformPendingRequest::new(task).with_backend_name(name) - }) - .collect(); - let client = AxumPlatformHttpClient::new(); - let deadline = web_time::Instant::now(); - let first = client - .select_until(pending, deadline) - .await - .expect("should snapshot ready responses") - .expect("should retain first response"); - assert!( - first.ready.is_ok(), - "should preserve first completed result" - ); - assert_eq!( - first.remaining.len(), - 1, - "should retain only the other completed response" - ); - let second = client - .select_until(first.remaining, deadline) - .await - .expect("should drain the completed snapshot") - .expect("should retain second response"); - assert!( - second.ready.is_ok(), - "should preserve the second completed result" - ); - assert!( - second.remaining.is_empty(), - "should leave no unfinished handles" - ); - assert!( - tokio::time::timeout(Duration::from_secs(1), drop_receiver) - .await - .expect("should promptly abort unfinished work") - .is_err(), - "should release unfinished task resources" - ); - } - - #[tokio::test] - async fn select_until_aborts_task_on_expiration() { - let (started_sender, started_receiver) = tokio::sync::oneshot::channel(); - let (dropped_sender, dropped_receiver) = tokio::sync::oneshot::channel::<()>(); - let handle = tokio::spawn(async move { - let _drop_signal = dropped_sender; - started_sender.send(()).expect("should signal task start"); - futures::future::pending::().await - }); - started_receiver - .await - .expect("should start task before selection"); - let pending = PlatformPendingRequest::new(AxumPendingHandle { - backend_name: "slow".to_owned(), - handle, - }); - let result = AxumPlatformHttpClient::new() - .select_until( - vec![pending], - web_time::Instant::now() + Duration::from_millis(50), - ) - .await - .expect("should expire task"); - assert!(result.is_none(), "should report deadline expiration"); - assert!( - tokio::time::timeout(Duration::from_secs(2), dropped_receiver) - .await - .expect("should promptly drop aborted task state") - .is_err(), - "should drop task resources on expiration" - ); - } - - #[tokio::test] - async fn select_until_preserves_failed_backend_attribution() { - let listener = tokio::net::TcpListener::bind("127.0.0.1:0") - .await - .expect("should bind probe"); - let address = listener.local_addr().expect("should read probe address"); - drop(listener); - let client = AxumPlatformHttpClient::new(); - let pending = deadline_pending(&client, format!("http://{address}/"), "failed").await; - let result = client - .select_until( - vec![pending], - web_time::Instant::now() + Duration::from_secs(2), - ) - .await - .expect("should collect transport failure") - .expect("should fail before deadline"); - assert!(result.ready.is_err(), "should preserve transport failure"); - assert_eq!( - result.failed_backend_name.as_deref(), - Some("failed"), - "should attribute failure to its backend" - ); - } - - async fn serve_raw_response(response: &'static [u8]) -> String { - let listener = tokio::net::TcpListener::bind("127.0.0.1:0") - .await - .expect("should bind raw HTTP test server"); - let addr = listener.local_addr().expect("should read local address"); - - tokio::spawn(async move { - let (mut stream, _) = listener.accept().await.expect("should accept request"); - let mut request = [0; 1024]; - let _ = stream - .read(&mut request) - .await - .expect("should read request"); - stream - .write_all(response) - .await - .expect("should write response"); - }); - - format!("http://{addr}/") - } } diff --git a/crates/trusted-server-adapter-axum/tests/routes.rs b/crates/trusted-server-adapter-axum/tests/routes.rs index b785734968..918171085a 100644 --- a/crates/trusted-server-adapter-axum/tests/routes.rs +++ b/crates/trusted-server-adapter-axum/tests/routes.rs @@ -1,27 +1,44 @@ //! Integration tests for the Axum dev server. //! -//! Uses `EdgeZeroAxumService` directly (no live TCP server) so tests remain fast -//! and self-contained. Each test builds the full `TrustedServerApp` router and -//! drives it through the Tower `Service` interface. - -use axum::body::Body as AxumBody; -use axum::http::Request; -use edgezero_adapter_axum::service::EdgeZeroAxumService; -use std::sync::Arc; -use tower::{Service as _, ServiceExt as _}; +//! Each route test assembles the full `TrustedServerApp` from explicit test +//! settings, serves it through `EdgeZero`'s `AxumTestServer` on a loopback port, +//! and sends real HTTP/1 requests over TCP. The production listener therefore +//! applies its ingress and egress handling to every request. The startup test +//! runs the compiled `trusted-server-axum` binary. + +use std::io::{Read as _, Write as _}; +use std::net::{SocketAddr, TcpStream}; +use std::process::{Child, Command, Stdio}; +use std::time::{Duration, Instant}; + +use edgezero_adapter_axum::AXUM_PLATFORM; +use edgezero_adapter_axum::dev_server::{AxumDevServer, AxumDevServerConfig}; +use edgezero_adapter_axum::test_utils::AxumTestServer; +use edgezero_core::app::App; +use edgezero_core::router::RouterService; use trusted_server_adapter_axum::app::TrustedServerApp; +use trusted_server_core::platform::HttpClient; +use trusted_server_core::settings::Settings; use trusted_server_core::test_support::nextjs_auction; +use trusted_server_core::test_support::scripted_client::{ + ScriptedHttpClient, ScriptedReply, TestClock, +}; const LEGACY_ADMIN_DENY_METHODS: &[&str] = &["GET", "POST", "HEAD", "OPTIONS", "PUT", "PATCH", "DELETE"]; -/// Build the full application router from explicit test settings. +/// Environment variable from which the Axum binary reads its app config. +const AXUM_CONFIG_ENV: &str = "TRUSTED_SERVER_CONFIG_TRUSTED_SERVER_CONFIG_TRUSTED_SERVER_CONFIG"; + +/// How long the startup test waits for the binary to exit. +const STARTUP_EXIT_DEADLINE: Duration = Duration::from_secs(10); + +/// Settings the route tests build the application from. /// /// The settings baked into the binary contain placeholder secrets that -/// `get_settings()` rejects by design, which would turn every route into a -/// startup error page (and its route table into the fallback-only set). -fn test_settings() -> trusted_server_core::settings::Settings { - trusted_server_core::settings::Settings::from_toml( +/// `get_settings()` rejects by design, which would fail application startup. +fn test_settings() -> Settings { + Settings::from_toml( r#" [[handlers]] path = "^/_ts/admin" @@ -41,15 +58,11 @@ fn test_settings() -> trusted_server_core::settings::Settings { .expect("should parse route test settings") } -fn test_router() -> edgezero_core::router::RouterService { +fn test_router() -> RouterService { TrustedServerApp::routes_with_settings(test_settings()) .expect("should build router from test settings") } -fn make_service() -> EdgeZeroAxumService { - EdgeZeroAxumService::new(test_router()) -} - fn registered_routes() -> Vec<(String, String)> { test_router() .routes() @@ -66,6 +79,175 @@ fn assert_route_registered(method: &str, path: &str) { ); } +/// A `TrustedServerApp` served over loopback TCP, with an HTTP client for it. +/// +/// The app has no `EdgeZero` stores attached. The Axum adapter reads its +/// config and secrets from environment-backed platform stores, and these tests +/// inject settings directly, so no route needs one. +struct TestServer { + server: AxumTestServer, + client: reqwest::Client, + /// The script in router state, when the test supplied one. + outbound: Option, +} + +impl TestServer { + /// Serves the app built from `settings`, with a script that fails every + /// outbound request in router state, so no real `EdgeZero` transport starts. + async fn start(settings: Settings) -> Self { + Self::with_origin(settings, ScriptedHttpClient::forbidding()).await + } + + /// Serves the app built from `settings`, with `outbound` in router state. + async fn with_origin(settings: Settings, outbound: ScriptedHttpClient) -> Self { + let router = TrustedServerApp::routes_with_settings_and_outbound_client( + settings, + HttpClient::with_client(outbound.clone()), + ) + .expect("should build router from test settings"); + Self { + outbound: Some(outbound), + ..Self::serve(router).await + } + } + + /// Assembles `router` as the Trusted Server app and serves it on loopback port 0. + async fn serve(router: RouterService) -> Self { + let app = App::from_router::(router, AXUM_PLATFORM) + .expect("should assemble the Trusted Server app"); + let config = AxumDevServerConfig { + addr: SocketAddr::from(([127, 0, 0, 1], 0)), + enable_ctrl_c: false, + ..AxumDevServerConfig::default() + }; + let server = AxumTestServer::start(AxumDevServer::with_config(app, config)) + .await + .expect("should start the Axum test server"); + // No proxy, redirect following, or transparent decompression: the + // tests assert on exactly what the listener sends. + let client = reqwest::Client::builder() + .no_proxy() + .no_gzip() + .redirect(reqwest::redirect::Policy::none()) + .build() + .expect("should build the test HTTP client"); + Self { + server, + client, + outbound: None, + } + } + + /// Fails the test if the app sent any outbound request through its router-state script. + fn assert_no_outbound_requests(&self) { + self.outbound + .as_ref() + .expect("should serve with a router-state script") + .assert_no_requests(); + } + + /// Serves the router built from the default test settings. + async fn with_test_settings() -> Self { + Self::start(test_settings()).await + } + + /// Starts a request for `path` on the served app. + fn request(&self, method: &str, path: &str) -> reqwest::RequestBuilder { + let method = + reqwest::Method::from_bytes(method.as_bytes()).expect("should parse the test method"); + self.client + .request(method, format!("http://{}{path}", self.server.address())) + } +} + +/// Sends `request` and returns the response, with a bound on the wait. +async fn send(request: reqwest::RequestBuilder) -> reqwest::Response { + tokio::time::timeout(Duration::from_secs(10), request.send()) + .await + .expect("should answer promptly") + .expect("should respond") +} + +/// Returns a loopback port that is free at the time of the call. +fn free_loopback_port() -> u16 { + std::net::TcpListener::bind("127.0.0.1:0") + .expect("should reserve a loopback port") + .local_addr() + .expect("should read the reserved port") + .port() +} + +/// Waits up to `deadline` for `child` to exit, killing it if it does not. +/// +/// Returns `None` when the child was still running at the deadline. +fn wait_for_exit(child: &mut Child, deadline: Duration) -> Option { + let started = Instant::now(); + loop { + if let Some(status) = child.try_wait().expect("should poll the Axum binary") { + return Some(status); + } + if started.elapsed() >= deadline { + child.kill().expect("should stop the Axum binary"); + child.wait().expect("should reap the Axum binary"); + return None; + } + std::thread::sleep(Duration::from_millis(20)); + } +} + +/// Reads everything left in a child's output pipe. +fn read_pipe(pipe: Option) -> String { + let mut output = String::new(); + if let Some(mut pipe) = pipe { + pipe.read_to_string(&mut output) + .expect("should read the Axum binary output"); + } + output +} + +#[test] +fn axum_main_exits_non_zero_when_settings_fail() { + // A clean environment keeps developer store-name overrides out of the + // test. The config value is not a Trusted Server app-config envelope. + let mut child = Command::new(env!("CARGO_BIN_EXE_trusted-server-axum")) + .env_clear() + .env(AXUM_CONFIG_ENV, "not-a-trusted-server-config-envelope") + .env("PORT", free_loopback_port().to_string()) + .env("RUST_LOG", "info") + .stdin(Stdio::null()) + .stdout(Stdio::piped()) + .stderr(Stdio::piped()) + .spawn() + .expect("should spawn the Axum binary"); + + let status = wait_for_exit(&mut child, STARTUP_EXIT_DEADLINE); + let output = format!( + "{}{}", + read_pipe(child.stdout.take()), + read_pipe(child.stderr.take()) + ); + + let status = status.unwrap_or_else(|| { + panic!("should exit instead of serving after a settings failure; output:\n{output}") + }); + assert!( + !status.success(), + "should exit non-zero after a settings failure, got {status}; output:\n{output}" + ); + assert!( + output.contains("failed to parse Trusted Server app-config blob envelope"), + "should log the settings failure report; output:\n{output}" + ); + assert!( + output.contains("trusted-server-adapter-axum failed to start"), + "should log the startup failure before exiting; output:\n{output}" + ); + assert!( + !output.contains("Listening on"), + "should not reach the listener; output:\n{output}" + ); +} + #[tokio::test(flavor = "multi_thread", worker_threads = 2)] async fn aps_profile_serves_renderer_through_adapter_fallback() { let mut settings = test_settings(); @@ -83,23 +265,11 @@ async fn aps_profile_serves_renderer_through_adapter_fallback() { .expect("should parse APS profile config"), }, ); - let router = TrustedServerApp::routes_with_settings(settings) - .expect("should build router with APS profile"); - let mut service = EdgeZeroAxumService::new(router); - let request = Request::builder() - .method("GET") - .uri("/integrations/aps/renderer") - .body(AxumBody::empty()) - .expect("should build APS renderer request"); - - let response = service - .ready() - .await - .expect("should be ready") - .call(request) - .await - .expect("should serve APS renderer"); + let server = TestServer::start(settings).await; + + let response = send(server.request("GET", "/integrations/aps/renderer")).await; assert_eq!(response.status().as_u16(), 200); + server.assert_no_outbound_requests(); } /// Verify that every expected explicit route is registered in the route table. @@ -174,100 +344,61 @@ fn legacy_admin_aliases_are_registered_to_local_deny() { async fn discovery_endpoint_is_routed() { // Verifies the route exists — 5xx from missing signing keys is acceptable; // 404 is not (that would mean the route was not registered). - let mut svc = make_service(); + let server = TestServer::with_test_settings().await; - let req = Request::builder() - .method("GET") - .uri("/.well-known/trusted-server.json") - .body(AxumBody::empty()) - .expect("should build request"); - - let resp = svc - .ready() - .await - .expect("should be ready") - .call(req) - .await - .expect("should respond"); + let resp = send(server.request("GET", "/.well-known/trusted-server.json")).await; assert_ne!( resp.status().as_u16(), 404, "discovery endpoint must be routed" ); + server.assert_no_outbound_requests(); } #[tokio::test(flavor = "multi_thread", worker_threads = 2)] async fn verify_signature_endpoint_is_routed() { - let mut svc = make_service(); - - let req = Request::builder() - .method("POST") - .uri("/verify-signature") - .header("content-type", "application/json") - .body(AxumBody::from("{}")) - .expect("should build request"); + let server = TestServer::with_test_settings().await; - let resp = svc - .ready() - .await - .expect("should be ready") - .call(req) - .await - .expect("should respond"); + let resp = send( + server + .request("POST", "/verify-signature") + .header("content-type", "application/json") + .body("{}"), + ) + .await; assert_ne!( resp.status().as_u16(), 404, "verify-signature must be routed" ); + server.assert_no_outbound_requests(); } #[tokio::test(flavor = "multi_thread", worker_threads = 2)] async fn tsjs_route_prefix_is_handled_not_5xx() { - let mut svc = make_service(); + let server = TestServer::with_test_settings().await; // /static/tsjs= is a GET /{*rest} catch-all path. The handler returns 404 // for an unknown hash, which is correct application behaviour (not a routing 404). // This test verifies the handler is reached (no 5xx/panic) and that routing works. - let req = Request::builder() - .method("GET") - .uri("/static/tsjs=0000000000000000") - .body(AxumBody::empty()) - .expect("should build request"); - - let resp = svc - .ready() - .await - .expect("should be ready") - .call(req) - .await - .expect("should respond"); + let resp = send(server.request("GET", "/static/tsjs=0000000000000000")).await; let status = resp.status().as_u16(); assert!( status < 500, "tsjs catch-all handler must not return 5xx: got {status}" ); + server.assert_no_outbound_requests(); } #[tokio::test(flavor = "multi_thread", worker_threads = 2)] async fn tsjs_route_matching_hash_uses_s_maxage_fallback() { - let mut svc = make_service(); + let server = TestServer::with_test_settings().await; let src = trusted_server_core::tsjs::tsjs_script_src(&["creative"]); - let req = Request::builder() - .method("GET") - .uri(src) - .body(AxumBody::empty()) - .expect("should build request"); - - let resp = svc - .ready() - .await - .expect("should be ready") - .call(req) - .await - .expect("should respond"); + + let resp = send(server.request("GET", &src)).await; assert_eq!( resp.status().as_u16(), @@ -285,6 +416,7 @@ async fn tsjs_route_matching_hash_uses_s_maxage_fallback() { resp.headers().get("surrogate-control").is_none(), "s-maxage fallback must not emit Fastly Surrogate-Control" ); + server.assert_no_outbound_requests(); } // --------------------------------------------------------------------------- @@ -293,22 +425,15 @@ async fn tsjs_route_matching_hash_uses_s_maxage_fallback() { #[tokio::test(flavor = "multi_thread", worker_threads = 2)] async fn finalize_middleware_sets_geo_unavailable_header() { - let mut svc = make_service(); + let server = TestServer::with_test_settings().await; - let req = Request::builder() - .method("POST") - .uri("/verify-signature") - .header("content-type", "application/json") - .body(AxumBody::from("{}")) - .expect("should build request"); - - let resp = svc - .ready() - .await - .expect("should be ready") - .call(req) - .await - .expect("should respond"); + let resp = send( + server + .request("POST", "/verify-signature") + .header("content-type", "application/json") + .body("{}"), + ) + .await; assert_eq!( resp.headers() @@ -317,6 +442,7 @@ async fn finalize_middleware_sets_geo_unavailable_header() { Some("false"), "finalize middleware should set X-Geo-Info-Available: false on every response" ); + server.assert_no_outbound_requests(); } // --------------------------------------------------------------------------- @@ -325,25 +451,20 @@ async fn finalize_middleware_sets_geo_unavailable_header() { #[tokio::test(flavor = "multi_thread", worker_threads = 2)] async fn admin_route_without_credentials_returns_401() { - let mut svc = make_service(); - let req = Request::builder() - .method("POST") - .uri("/_ts/admin/keys/rotate") - .header("content-type", "application/json") - .body(AxumBody::from("{}")) - .expect("should build request"); - let resp = svc - .ready() - .await - .expect("should be ready") - .call(req) - .await - .expect("should respond"); + let server = TestServer::with_test_settings().await; + let resp = send( + server + .request("POST", "/_ts/admin/keys/rotate") + .header("content-type", "application/json") + .body("{}"), + ) + .await; assert_eq!( resp.status().as_u16(), 401, "admin route must return 401 without credentials" ); + server.assert_no_outbound_requests(); } #[tokio::test(flavor = "multi_thread", worker_threads = 2)] @@ -356,20 +477,13 @@ async fn authenticated_admin_ec_routes_return_501() { "/_ts/admin/ec".to_owned(), format!("/_ts/admin/ec/{sample_ec_id}"), ] { - let mut svc = make_service(); - let req = Request::builder() - .method("GET") - .uri(&path) - .header("authorization", "Basic YWRtaW46YWRtaW4tcGFzcw==") - .body(AxumBody::empty()) - .expect("should build request"); - let resp = svc - .ready() - .await - .expect("should be ready") - .call(req) - .await - .expect("should respond"); + let server = TestServer::with_test_settings().await; + let resp = send( + server + .request("GET", &path) + .header("authorization", "Basic YWRtaW46YWRtaW4tcGFzcw=="), + ) + .await; assert_eq!( resp.status().as_u16(), 501, @@ -392,50 +506,34 @@ async fn authenticated_admin_ec_routes_return_501() { #[tokio::test(flavor = "multi_thread", worker_threads = 2)] async fn admin_ec_route_without_credentials_returns_401() { - let mut svc = make_service(); - let req = Request::builder() - .method("GET") - .uri("/_ts/admin/ec") - .body(AxumBody::empty()) - .expect("should build unauthenticated admin EC request"); - let resp = svc - .ready() - .await - .expect("should be ready") - .call(req) - .await - .expect("should respond"); + let server = TestServer::with_test_settings().await; + let resp = send(server.request("GET", "/_ts/admin/ec")).await; assert_eq!(resp.status().as_u16(), 401); assert!( resp.headers().contains_key("www-authenticate"), "admin EC 401 should include the Basic authentication challenge" ); + server.assert_no_outbound_requests(); } #[tokio::test(flavor = "multi_thread", worker_threads = 2)] async fn authenticated_admin_eids_route_returns_200() { // The EIDs echo is pure request inspection (no KV), so the dev server // serves the real handler. - let mut svc = make_service(); - let req = Request::builder() - .method("GET") - .uri("/_ts/admin/eids") - .header("authorization", "Basic YWRtaW46YWRtaW4tcGFzcw==") - .body(AxumBody::empty()) - .expect("should build request"); - let resp = svc - .ready() - .await - .expect("should be ready") - .call(req) - .await - .expect("should respond"); + let server = TestServer::with_test_settings().await; + let resp = send( + server + .request("GET", "/_ts/admin/eids") + .header("authorization", "Basic YWRtaW46YWRtaW4tcGFzcw=="), + ) + .await; assert_eq!( resp.status().as_u16(), 200, "/_ts/admin/eids should serve the real EIDs echo handler" ); + server.assert_no_outbound_requests(); } #[tokio::test(flavor = "multi_thread", worker_threads = 2)] @@ -446,22 +544,17 @@ async fn authenticated_admin_diagnostic_fallback_is_denied_locally() { format!("/_ts/admin/ec/{ec_id}"), "/_ts/admin/eids".to_owned(), ]; + let server = TestServer::with_test_settings().await; for path in valid_paths { for method in ["POST", "HEAD", "OPTIONS", "PUT", "PATCH", "DELETE"] { - let request = Request::builder() - .method(method) - .uri(&path) - .header("authorization", "Basic YWRtaW46YWRtaW4tcGFzcw==") - .body(AxumBody::from("sensitive-admin-body")) - .expect("should build authenticated admin request"); - let response = make_service() - .ready() - .await - .expect("should be ready") - .call(request) - .await - .expect("should respond"); + let response = send( + server + .request(method, &path) + .header("authorization", "Basic YWRtaW46YWRtaW4tcGFzcw==") + .body("sensitive-admin-body"), + ) + .await; assert_eq!(response.status().as_u16(), 405); assert_eq!( @@ -507,19 +600,13 @@ async fn authenticated_admin_diagnostic_fallback_is_denied_locally() { "/_ts%252Fadmin/ec".to_owned(), ] { for method in ["GET", "POST"] { - let request = Request::builder() - .method(method) - .uri(&path) - .header("authorization", "Basic YWRtaW46YWRtaW4tcGFzcw==") - .body(AxumBody::from("sensitive-admin-body")) - .expect("should build malformed admin request"); - let response = make_service() - .ready() - .await - .expect("should be ready") - .call(request) - .await - .expect("should respond"); + let response = send( + server + .request(method, &path) + .header("authorization", "Basic YWRtaW46YWRtaW4tcGFzcw==") + .body("sensitive-admin-body"), + ) + .await; assert_eq!(response.status().as_u16(), 404); assert_eq!( @@ -531,6 +618,7 @@ async fn authenticated_admin_diagnostic_fallback_is_denied_locally() { ); } } + server.assert_no_outbound_requests(); } #[tokio::test(flavor = "multi_thread", worker_threads = 2)] @@ -542,23 +630,17 @@ async fn legacy_admin_aliases_denied_locally_not_proxied_to_publisher() { // origin (which would leak the admin credentials and key body). A // publisher-fallback proxy without a backend would surface as a 5xx, so 404 // proves the local deny ran. + let server = TestServer::with_test_settings().await; for path in ["/admin/keys/rotate", "/admin/keys/deactivate"] { for method in LEGACY_ADMIN_DENY_METHODS { - let mut svc = make_service(); - let req = Request::builder() - .method(*method) - .uri(path) - .header("authorization", "Basic YWRtaW46YWRtaW4tcGFzcw==") - .header("content-type", "application/json") - .body(AxumBody::from("{\"key_id\":\"leak-me\"}")) - .expect("should build authorized legacy-alias request"); - let resp = svc - .ready() - .await - .expect("should be ready") - .call(req) - .await - .expect("should respond"); + let resp = send( + server + .request(method, path) + .header("authorization", "Basic YWRtaW46YWRtaW4tcGFzcw==") + .header("content-type", "application/json") + .body("{\"key_id\":\"leak-me\"}"), + ) + .await; assert_eq!( resp.status().as_u16(), 404, @@ -566,24 +648,19 @@ async fn legacy_admin_aliases_denied_locally_not_proxied_to_publisher() { ); } } + server.assert_no_outbound_requests(); } #[tokio::test(flavor = "multi_thread", worker_threads = 2)] async fn admin_route_without_credentials_includes_www_authenticate_header() { - let mut svc = make_service(); - let req = Request::builder() - .method("POST") - .uri("/_ts/admin/keys/rotate") - .header("content-type", "application/json") - .body(AxumBody::from("{}")) - .expect("should build request"); - let resp = svc - .ready() - .await - .expect("should be ready") - .call(req) - .await - .expect("should respond"); + let server = TestServer::with_test_settings().await; + let resp = send( + server + .request("POST", "/_ts/admin/keys/rotate") + .header("content-type", "application/json") + .body("{}"), + ) + .await; assert_eq!( resp.status().as_u16(), 401, @@ -603,103 +680,78 @@ async fn admin_route_without_credentials_includes_www_authenticate_header() { www_auth.starts_with("Basic realm="), "WWW-Authenticate must be Basic scheme, got: {www_auth}" ); + server.assert_no_outbound_requests(); } #[tokio::test(flavor = "multi_thread", worker_threads = 2)] async fn admin_route_with_wrong_credentials_returns_401() { use base64::Engine as _; let creds = base64::engine::general_purpose::STANDARD.encode("admin:wrong-password"); - let mut svc = make_service(); - let req = Request::builder() - .method("POST") - .uri("/_ts/admin/keys/rotate") - .header("content-type", "application/json") - .header("authorization", format!("Basic {creds}")) - .body(AxumBody::from("{}")) - .expect("should build request"); - let resp = svc - .ready() - .await - .expect("should be ready") - .call(req) - .await - .expect("should respond"); + let server = TestServer::with_test_settings().await; + let resp = send( + server + .request("POST", "/_ts/admin/keys/rotate") + .header("content-type", "application/json") + .header("authorization", format!("Basic {creds}")) + .body("{}"), + ) + .await; assert_eq!( resp.status().as_u16(), 401, "admin route must reject wrong credentials with 401" ); + server.assert_no_outbound_requests(); } #[tokio::test(flavor = "multi_thread", worker_threads = 2)] async fn discovery_endpoint_does_not_require_auth() { - let mut svc = make_service(); - let req = Request::builder() - .method("GET") - .uri("/.well-known/trusted-server.json") - .body(AxumBody::empty()) - .expect("should build request"); - let resp = svc - .ready() - .await - .expect("should be ready") - .call(req) - .await - .expect("should respond"); + let server = TestServer::with_test_settings().await; + let resp = send(server.request("GET", "/.well-known/trusted-server.json")).await; assert_ne!( resp.status().as_u16(), 401, "/.well-known/trusted-server.json must not require auth" ); + server.assert_no_outbound_requests(); } #[tokio::test(flavor = "multi_thread", worker_threads = 2)] async fn auction_endpoint_does_not_require_auth() { - let mut svc = make_service(); - let req = Request::builder() - .method("POST") - .uri("/auction") - .header("content-type", "application/json") - .body(AxumBody::from(r#"{"adUnits":[]}"#)) - .expect("should build request"); - let resp = svc - .ready() - .await - .expect("should be ready") - .call(req) - .await - .expect("should respond"); + let server = TestServer::with_test_settings().await; + let resp = send( + server + .request("POST", "/auction") + .header("content-type", "application/json") + .body(r#"{"adUnits":[]}"#), + ) + .await; assert_ne!( resp.status().as_u16(), 401, "/auction must not apply admin basic-auth gate" ); + server.assert_no_outbound_requests(); } #[tokio::test(flavor = "multi_thread", worker_threads = 2)] async fn admin_route_returns_non_404_non_5xx() { - let mut svc = make_service(); - - let req = Request::builder() - .method("POST") - .uri("/_ts/admin/keys/rotate") - .header("content-type", "application/json") - .body(AxumBody::from("{}")) - .expect("should build request"); + let server = TestServer::with_test_settings().await; - let resp = svc - .ready() - .await - .expect("should be ready") - .call(req) - .await - .expect("should respond"); + let resp = send( + server + .request("POST", "/_ts/admin/keys/rotate") + .header("content-type", "application/json") + .body("{}"), + ) + .await; let status = resp.status().as_u16(); assert_ne!(status, 404, "admin route must be routed"); // The auth gate short-circuits with 401 before the handler; only an // unhandled 500 indicates a panic or missing handler. assert_ne!(status, 500, "admin route must not panic: got {status}"); + server.assert_no_outbound_requests(); } // --------------------------------------------------------------------------- @@ -710,48 +762,38 @@ async fn admin_route_returns_non_404_non_5xx() { // generic `admin_route_without_credentials_returns_401` above). #[tokio::test(flavor = "multi_thread", worker_threads = 2)] async fn admin_rotate_key_auth_fail_returns_401() { - let mut svc = make_service(); - let req = Request::builder() - .method("POST") - .uri("/_ts/admin/keys/rotate") - .header("content-type", "application/json") - .body(AxumBody::from(r#"{"keyId":"test-key"}"#)) - .expect("should build request"); - let resp = svc - .ready() - .await - .expect("should be ready") - .call(req) - .await - .expect("should respond"); + let server = TestServer::with_test_settings().await; + let resp = send( + server + .request("POST", "/_ts/admin/keys/rotate") + .header("content-type", "application/json") + .body(r#"{"keyId":"test-key"}"#), + ) + .await; assert_eq!( resp.status().as_u16(), 401, "admin/keys/rotate without credentials must return 401" ); + server.assert_no_outbound_requests(); } #[tokio::test(flavor = "multi_thread", worker_threads = 2)] async fn admin_deactivate_key_auth_fail_returns_401() { - let mut svc = make_service(); - let req = Request::builder() - .method("POST") - .uri("/_ts/admin/keys/deactivate") - .header("content-type", "application/json") - .body(AxumBody::from(r#"{"keyId":"test-key"}"#)) - .expect("should build request"); - let resp = svc - .ready() - .await - .expect("should be ready") - .call(req) - .await - .expect("should respond"); + let server = TestServer::with_test_settings().await; + let resp = send( + server + .request("POST", "/_ts/admin/keys/deactivate") + .header("content-type", "application/json") + .body(r#"{"keyId":"test-key"}"#), + ) + .await; assert_eq!( resp.status().as_u16(), 401, "admin/keys/deactivate without credentials must return 401" ); + server.assert_no_outbound_requests(); } // --------------------------------------------------------------------------- @@ -760,114 +802,125 @@ async fn admin_deactivate_key_auth_fail_returns_401() { #[tokio::test(flavor = "multi_thread", worker_threads = 2)] async fn first_party_proxy_is_routed() { - let mut svc = make_service(); - let req = Request::builder() - .method("GET") - .uri("/first-party/proxy") - .body(AxumBody::empty()) - .expect("should build request"); - let resp = svc - .ready() - .await - .expect("should be ready") - .call(req) - .await - .expect("should respond"); + let server = TestServer::with_test_settings().await; + let resp = send(server.request("GET", "/first-party/proxy")).await; assert_ne!( resp.status().as_u16(), 404, "/first-party/proxy must be routed" ); + server.assert_no_outbound_requests(); } #[tokio::test(flavor = "multi_thread", worker_threads = 2)] async fn first_party_click_is_routed() { - let mut svc = make_service(); - let req = Request::builder() - .method("GET") - .uri("/first-party/click") - .body(AxumBody::empty()) - .expect("should build request"); - let resp = svc - .ready() - .await - .expect("should be ready") - .call(req) - .await - .expect("should respond"); + let server = TestServer::with_test_settings().await; + let resp = send(server.request("GET", "/first-party/click")).await; assert_ne!( resp.status().as_u16(), 404, "/first-party/click must be routed" ); + server.assert_no_outbound_requests(); } #[tokio::test(flavor = "multi_thread", worker_threads = 2)] async fn first_party_sign_get_is_routed() { - let mut svc = make_service(); - let req = Request::builder() - .method("GET") - .uri("/first-party/sign") - .body(AxumBody::empty()) - .expect("should build request"); - let resp = svc - .ready() - .await - .expect("should be ready") - .call(req) - .await - .expect("should respond"); + let server = TestServer::with_test_settings().await; + let resp = send(server.request("GET", "/first-party/sign")).await; assert_ne!( resp.status().as_u16(), 404, "GET /first-party/sign must be routed" ); + server.assert_no_outbound_requests(); } #[tokio::test(flavor = "multi_thread", worker_threads = 2)] async fn first_party_sign_post_is_routed() { - let mut svc = make_service(); - let req = Request::builder() - .method("POST") - .uri("/first-party/sign") - .header("content-type", "application/json") - .body(AxumBody::from("{}")) - .expect("should build request"); - let resp = svc - .ready() - .await - .expect("should be ready") - .call(req) - .await - .expect("should respond"); + let server = TestServer::with_test_settings().await; + let resp = send( + server + .request("POST", "/first-party/sign") + .header("content-type", "application/json") + .body("{}"), + ) + .await; assert_ne!( resp.status().as_u16(), 404, "POST /first-party/sign must be routed" ); + server.assert_no_outbound_requests(); +} + +/// The listener hands every POST body to the handler as a stream, so a signed +/// URL in the answer proves the bounded reader collected it end to end. +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn first_party_sign_post_signs_the_url_in_its_body() { + let server = TestServer::with_test_settings().await; + let resp = send( + server + .request("POST", "/first-party/sign") + .header("content-type", "application/json") + .body(r#"{"url":"https://cdn.example.com/pixel.gif?a=1"}"#), + ) + .await; + + let status = resp.status().as_u16(); + let body = resp.text().await.expect("should read the sign response"); + assert_eq!(status, 200, "POST /first-party/sign should sign: {body}"); + assert!( + body.contains(r#""base":"https://cdn.example.com/pixel.gif""#), + "should sign the URL from the request body: {body}" + ); + assert!( + body.contains(r#""href":"https://test-publisher.example.com/first-party/proxy?tsurl="#), + "should return the first-party proxy URL on the default public origin: {body}" + ); + server.assert_no_outbound_requests(); +} + +/// A JSON auction body reaches the handler instead of being read as empty. +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn auction_post_parses_its_json_body() { + let server = TestServer::with_test_settings().await; + let resp = send( + server + .request("POST", "/auction") + .header("content-type", "application/json") + .body( + r#"{"adUnits":[{"code":"slot-1","mediaTypes":{"banner":{"sizes":[[300,250]]}}}]}"#, + ), + ) + .await; + + let status = resp.status().as_u16(); + let body = resp.text().await.expect("should read the auction response"); + assert_eq!(status, 200, "POST /auction should parse its body: {body}"); + assert!( + body.contains(r#""id":""#), + "should return an OpenRTB response with an auction id: {body}" + ); + server.assert_no_outbound_requests(); } #[tokio::test(flavor = "multi_thread", worker_threads = 2)] async fn first_party_proxy_rebuild_is_routed() { - let mut svc = make_service(); - let req = Request::builder() - .method("POST") - .uri("/first-party/proxy-rebuild") - .header("content-type", "application/json") - .body(AxumBody::from("{}")) - .expect("should build request"); - let resp = svc - .ready() - .await - .expect("should be ready") - .call(req) - .await - .expect("should respond"); + let server = TestServer::with_test_settings().await; + let resp = send( + server + .request("POST", "/first-party/proxy-rebuild") + .header("content-type", "application/json") + .body("{}"), + ) + .await; assert_ne!( resp.status().as_u16(), 404, "/first-party/proxy-rebuild must be routed" ); + server.assert_no_outbound_requests(); } /// Regression test: a Next.js navigation with a pending auction must buffer to @@ -880,28 +933,25 @@ async fn first_party_proxy_rebuild_is_routed() { /// Fastly adapter alone and not by this test. #[tokio::test(flavor = "multi_thread", worker_threads = 2)] async fn nextjs_auction_output_holds_until_the_structural_body_close() { - let client = Arc::new(nextjs_auction::NextJsAuctionOrigin::default()); + let client = nextjs_auction::NextJsAuctionOrigin::default(); let router = TrustedServerApp::routes_with_settings_and_services( nextjs_auction::settings(), - nextjs_auction::services(Arc::clone(&client)), + nextjs_auction::services(&client), ) .expect("should build router with fixture services"); + let server = TestServer::serve(router).await; - let request = edgezero_core::http::request_builder() - .method("GET") - .uri("https://test-publisher.example.com/article") - .header("host", "test-publisher.example.com") - .header("accept", "text/html") - .body(edgezero_core::body::Body::empty()) - .expect("should build publisher navigation"); - let response = router - .oneshot(request) - .await - .expect("should serve publisher navigation"); + let response = send( + server + .request("GET", "/article") + .header("host", "test-publisher.example.com") + .header("accept", "text/html"), + ) + .await; assert_eq!(response.status(), 200, "should serve fixture HTML"); let body = response - .into_body() - .into_bytes() + .bytes() + .await .expect("should buffer adapter output"); let html = String::from_utf8(body.to_vec()).expect("should emit UTF-8 HTML"); @@ -942,46 +992,36 @@ async fn nextjs_gtm_rejects_malformed_request_hosts() { // A hostile `Host` is rejected. Unauthenticated forwarded fields are // stripped before routing, so their hostile values never reach rewriting. for header in ["host", "x-forwarded-host", "forwarded"] { - let origin = Arc::new(nextjs_auction::NextJsAuctionOrigin::with_html_response( + let origin = nextjs_auction::NextJsAuctionOrigin::with_html_response( &nextjs_auction::script_composition_html(), false, true, - )); + ); let router = TrustedServerApp::routes_with_settings_and_services( nextjs_auction::script_composition_settings(), - nextjs_auction::services(origin), + nextjs_auction::services(&origin), ) .expect("should build script composition router"); - let mut service = EdgeZeroAxumService::new(router); + let server = TestServer::serve(router).await; let hostile_host = "example.com"; let value = if header == "forwarded" { format!("host=\"{hostile_host}\"") } else { hostile_host.to_owned() }; - let mut request = Request::builder() - .uri("/script-composition") - .header("host", nextjs_auction::PUBLISHER_HOST) - .body(AxumBody::empty()) - .expect("should build hostile host request"); - request.headers_mut().insert( - header, - value.parse().expect("should encode hostile host header"), - ); - let response = service - .ready() - .await - .expect("should be ready") - .call(request) - .await - .expect("should respond to hostile host request"); + let mut request = server.request("GET", "/script-composition"); + if header != "host" { + request = request.header("host", nextjs_auction::PUBLISHER_HOST); + } + let response = send(request.header(header, value)).await; let expected_status = if header == "host" { 400 } else { 200 }; assert_eq!( response.status().as_u16(), expected_status, "should reject malformed Host and ignore unauthenticated {header}" ); - let body = axum::body::to_bytes(response.into_body(), usize::MAX) + let body = response + .bytes() .await .expect("should consume hostile host response"); assert!( @@ -1000,35 +1040,30 @@ async fn nextjs_gtm_accepts_local_request_hosts_and_ports() { "my_service:8080", "_service_.example.com:8443", ] { - let origin = Arc::new(nextjs_auction::NextJsAuctionOrigin::with_html_response( + let origin = nextjs_auction::NextJsAuctionOrigin::with_html_response( &nextjs_auction::script_composition_html(), false, true, - )); + ); let router = TrustedServerApp::routes_with_settings_and_services( nextjs_auction::script_composition_settings(), - nextjs_auction::services(origin), + nextjs_auction::services(&origin), ) .expect("should build script composition router"); - let mut service = EdgeZeroAxumService::new(router); - let request = Request::builder() - .uri("/script-composition") - .header("host", host) - .body(AxumBody::empty()) - .expect("should build local host request"); - let response = service - .ready() - .await - .expect("should be ready") - .call(request) - .await - .expect("should serve local host request"); + let server = TestServer::serve(router).await; + let response = send( + server + .request("GET", "/script-composition") + .header("host", host), + ) + .await; assert_eq!( response.status().as_u16(), 200, "should accept host: {host}" ); - let body = axum::body::to_bytes(response.into_body(), usize::MAX) + let body = response + .bytes() .await .expect("should consume local host response"); let html = String::from_utf8(body.to_vec()).expect("should emit UTF-8 HTML"); @@ -1050,31 +1085,24 @@ async fn nextjs_gtm_accepts_local_request_hosts_and_ports() { #[tokio::test(flavor = "multi_thread", worker_threads = 2)] async fn nextjs_gtm_html_response_is_complete() { for gzip in [false, true] { - let origin = Arc::new(nextjs_auction::NextJsAuctionOrigin::with_html_response( + let origin = nextjs_auction::NextJsAuctionOrigin::with_html_response( &nextjs_auction::script_composition_html(), gzip, false, - )); + ); let router = TrustedServerApp::routes_with_settings_and_services( nextjs_auction::script_composition_settings(), - nextjs_auction::services(origin), + nextjs_auction::services(&origin), ) .expect("should build mixed integration router"); - let mut service = EdgeZeroAxumService::new(router); - let request = Request::builder() - .method("GET") - .uri("/article") - .header("host", nextjs_auction::PUBLISHER_HOST) - .header("accept", "text/html") - .body(AxumBody::empty()) - .expect("should build navigation"); - let response = service - .ready() - .await - .expect("should be ready") - .call(request) - .await - .expect("should serve HTML"); + let server = TestServer::serve(router).await; + let response = send( + server + .request("GET", "/article") + .header("host", nextjs_auction::PUBLISHER_HOST) + .header("accept", "text/html"), + ) + .await; assert_eq!( response.status().as_u16(), 200, @@ -1088,7 +1116,8 @@ async fn nextjs_gtm_html_response_is_complete() { gzip.then_some("gzip"), "should retain the requested response compression" ); - let body = axum::body::to_bytes(response.into_body(), usize::MAX) + let body = response + .bytes() .await .expect("should consume entire adapter body"); nextjs_auction::assert_script_composition_response(&body, gzip); @@ -1097,20 +1126,9 @@ async fn nextjs_gtm_html_response_is_complete() { #[tokio::test(flavor = "multi_thread", worker_threads = 2)] async fn health_reports_git_version() { - let mut service = make_service(); - let request = Request::builder() - .method("GET") - .uri("/health") - .body(AxumBody::empty()) - .expect("should build health request"); - - let response = service - .ready() - .await - .expect("should be ready") - .call(request) - .await - .expect("should serve health"); + let server = TestServer::with_test_settings().await; + + let response = send(server.request("GET", "/health")).await; assert_eq!( response.status().as_u16(), @@ -1125,4 +1143,202 @@ async fn health_reports_git_version() { trusted_server_core::constants::TS_GIT_VERSION, "should report the compiled-in git version on /health" ); + server.assert_no_outbound_requests(); +} + +// --------------------------------------------------------------------------- +// Operator response headers that egress framing owns +// --------------------------------------------------------------------------- + +/// The route test settings with `headers` as the operator's `response_headers`. +fn settings_with_response_headers(headers: &[(&str, &str)]) -> Settings { + let mut settings = test_settings(); + settings.response_headers = headers + .iter() + .map(|(name, value)| ((*name).to_owned(), (*value).to_owned())) + .collect(); + settings +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn runtime_skips_malformed_connection_entry_and_serves_real_status() { + // A live config that predates validation can hold any of these. `x example` + // is not a field name, so egress would refuse it with a framing 500. + for value in ["keep-alive,", "", "x example"] { + let server = + TestServer::start(settings_with_response_headers(&[("connection", value)])).await; + + let response = send(server.request("GET", "/health")).await; + + assert_eq!( + response.status().as_u16(), + 200, + "should skip the operator `connection: {value}` entry and serve the real status" + ); + server.assert_no_outbound_requests(); + } +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn runtime_skip_of_connection_x_example_serves_x_example() { + let server = TestServer::start(settings_with_response_headers(&[ + ("connection", "x-example"), + ("x-example", "example-value"), + ])) + .await; + + let response = send(server.request("GET", "/health")).await; + + assert_eq!( + response.status().as_u16(), + 200, + "should serve the real status" + ); + // Skipping the `connection` entry also drops its nomination, so egress no + // longer removes the field it named. + assert_eq!( + response + .headers() + .get("x-example") + .and_then(|value| value.to_str().ok()), + Some("example-value"), + "should serve x-example once the connection entry naming it is skipped" + ); + server.assert_no_outbound_requests(); +} + +// --------------------------------------------------------------------------- +// Synthesized headers on EdgeZero's canonical responses +// --------------------------------------------------------------------------- + +/// Sends `request` as raw bytes to `address` and returns the whole response. +/// +/// For request targets the HTTP client cannot express, such as asterisk-form. +/// The request must ask the server to close the connection. +fn send_raw(address: SocketAddr, request: &'static str) -> String { + let mut stream = TcpStream::connect(address).expect("should connect to the Axum test server"); + stream + .set_read_timeout(Some(Duration::from_secs(10))) + .expect("should set a read timeout"); + stream + .write_all(request.as_bytes()) + .expect("should send the raw request"); + let mut response = Vec::new(); + stream + .read_to_end(&mut response) + .expect("should read the raw response"); + String::from_utf8(response).expect("should answer with a UTF-8 response head") +} + +/// Returns the value of header `name` from a raw HTTP/1 response. +fn raw_header<'response>(response: &'response str, name: &str) -> Option<&'response str> { + response + .split("\r\n\r\n") + .next() + .into_iter() + .flat_map(|head| head.split("\r\n").skip(1)) + .filter_map(|line| line.split_once(':')) + .find(|(field, _)| field.trim().eq_ignore_ascii_case(name)) + .map(|(_, value)| value.trim()) +} + +/// Returns the git version compiled into this build. +/// +/// # Panics +/// +/// Panics when the build has none, so a version assertion cannot pass on an +/// absent header. +fn compiled_git_version() -> &'static str { + trusted_server_core::constants::TS_GIT_VERSION + .expect("should compile a git version into the test build") +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn canonical_not_found_carries_x_ts_version() { + let server = TestServer::with_test_settings().await; + let address = server.server.address(); + + // The publisher fallback claims every origin-form path, so only an + // asterisk-form target has no route and gets EdgeZero's canonical 404. + let response = tokio::task::spawn_blocking(move || { + send_raw( + address, + "OPTIONS * HTTP/1.1\r\nHost: 127.0.0.1\r\nConnection: close\r\n\r\n", + ) + }) + .await + .expect("should finish the raw request"); + + assert!( + response.starts_with("HTTP/1.1 404 "), + "should answer an unrouted target with the canonical 404; response:\n{response}" + ); + assert_eq!( + raw_header(&response, "x-ts-version"), + Some(compiled_git_version()), + "should add the compiled-in git version to the canonical 404; response:\n{response}" + ); + server.assert_no_outbound_requests(); +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn canonical_method_not_allowed_carries_allow_and_x_ts_version() { + let server = TestServer::with_test_settings().await; + + let response = send(server.request("TRACE", "/auction")).await; + + assert_eq!( + response.status().as_u16(), + 405, + "should answer an unrouted method on a known path with the canonical 405" + ); + assert_eq!( + response + .headers() + .get("allow") + .and_then(|value| value.to_str().ok()), + Some("DELETE, GET, HEAD, OPTIONS, PATCH, POST, PUT"), + "should list the path's methods in sorted order" + ); + assert_eq!( + response + .headers() + .get("x-ts-version") + .and_then(|value| value.to_str().ok()), + Some(compiled_git_version()), + "should add the compiled-in git version to the canonical 405" + ); + server.assert_no_outbound_requests(); +} + +/// Origin of [`test_settings`]. +const ORIGIN: &str = "https://origin.test-publisher.example.com"; + +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn raw_body_at_cap_passes_and_one_over_fails_502_axum() { + // Axum's cap is `max_buffered_body_bytes` alone, 16 MiB by default. + let cap = test_settings().publisher.max_buffered_body_bytes; + let origin = ScriptedHttpClient::new(&TestClock::new()); + for (path, len) in [("/at-cap.bin", cap), ("/over-cap.bin", cap + 1)] { + origin.respond_to( + edgezero_core::http::Method::GET, + &format!("{ORIGIN}{path}"), + ScriptedReply::status(edgezero_core::http::StatusCode::OK) + .header("content-type", "application/octet-stream") + .body(vec![b'x'; len]), + ); + } + let server = TestServer::with_origin(test_settings(), origin).await; + + let at_cap = send(server.request("GET", "/at-cap.bin")).await; + assert_eq!(at_cap.status(), 200, "should serve a raw body at the cap"); + let body = at_cap.bytes().await.expect("should read the at-cap body"); + assert_eq!(body.len(), cap, "should serve every byte at the cap"); + + let over_cap = send(server.request("GET", "/over-cap.bin")).await; + assert_eq!( + over_cap.status(), + 502, + "should fail a raw body one byte over the cap with 502 before egress" + ); } diff --git a/crates/trusted-server-adapter-cloudflare/Cargo.toml b/crates/trusted-server-adapter-cloudflare/Cargo.toml index 36a9041641..fe579a2760 100644 --- a/crates/trusted-server-adapter-cloudflare/Cargo.toml +++ b/crates/trusted-server-adapter-cloudflare/Cargo.toml @@ -31,14 +31,12 @@ edgezero-core = { workspace = true } error-stack = { workspace = true } futures = { workspace = true } log = { workspace = true } -web-time = { workspace = true } serde_json = { workspace = true } trusted-server-core = { workspace = true } worker = { workspace = true, optional = true } [target.'cfg(target_arch = "wasm32")'.dependencies] edgezero-adapter-cloudflare = { workspace = true, features = ["cloudflare"] } -js-sys = { workspace = true } worker = { workspace = true } [dev-dependencies] @@ -46,6 +44,7 @@ trusted-server-core = { workspace = true, features = ["test-utils"] } base64 = { workspace = true } edgezero-core = { workspace = true } toml = { workspace = true } +web-time = { workspace = true } [target.'cfg(not(target_arch = "wasm32"))'.dev-dependencies] tokio = { workspace = true, features = ["rt-multi-thread", "macros"] } diff --git a/crates/trusted-server-adapter-cloudflare/README.md b/crates/trusted-server-adapter-cloudflare/README.md index 208827e689..acd9b58089 100644 --- a/crates/trusted-server-adapter-cloudflare/README.md +++ b/crates/trusted-server-adapter-cloudflare/README.md @@ -5,11 +5,13 @@ Cloudflare Workers adapter for Trusted Server. Production code targets adapter tests and deliberately exclude the Workers entry point. The crate translates Worker requests and responses, registers the shared -router, and implements Cloudflare-specific HTTP and secret access. The current +router, and implements Cloudflare-specific secret access. The current runtime does not open the KV config value written by `ts config push`; startup instead consumes the nested `TRUSTED_SERVER_CONFIG` variable bridge. It also -has no `/health` route or request-time KV registry and permits one enabled -auction provider. +has no `/health` route or request-time KV registry. Multi-provider auctions fan +out as one EdgeZero request batch. Workers rewrites the inbound +`Accept-Encoding`, so the publisher origin request decodes the origin body and +serves identity bytes. Build and test from the repository root: diff --git a/crates/trusted-server-adapter-cloudflare/src/app.rs b/crates/trusted-server-adapter-cloudflare/src/app.rs index 668b025ccb..9521222848 100644 --- a/crates/trusted-server-adapter-cloudflare/src/app.rs +++ b/crates/trusted-server-adapter-cloudflare/src/app.rs @@ -2,11 +2,14 @@ use core::future::Future; use core::pin::Pin; use std::sync::Arc; -use edgezero_core::app::Hooks; +use edgezero_core::app::{App, Hooks}; use edgezero_core::context::RequestContext; use edgezero_core::error::EdgeError; -use edgezero_core::http::{HeaderValue, Method, Request, Response, StatusCode, header}; -use edgezero_core::router::RouterService; +use edgezero_core::http::{ + HeaderValue, Method, Request, RequestParts, Response, StatusCode, header, +}; +use edgezero_core::outbound::ResponseDecoding; +use edgezero_core::router::{RouterBuilder, RouterService}; use error_stack::Report; use trusted_server_core::auction::endpoints::handle_auction; use trusted_server_core::auction::{ @@ -23,19 +26,20 @@ use trusted_server_core::ec::admin::{ deny_admin_diagnostic_fallback, handle_admin_eids_lookup, }; use trusted_server_core::ec::registry::PartnerRegistry; -use trusted_server_core::error::{IntoHttpResponse as _, TrustedServerError}; +use trusted_server_core::edge_app::install_edge_policies; +use trusted_server_core::error::{TrustedServerError, http_error, startup_error}; use trusted_server_core::integrations::{IntegrationRegistry, ProxyDispatchInput}; #[cfg(target_arch = "wasm32")] use trusted_server_core::platform::StoreName; -use trusted_server_core::platform::{ClientInfo, GeoInfo, RuntimeServices}; +use trusted_server_core::platform::{ClientInfo, GeoInfo, HttpClient, RuntimeServices}; use trusted_server_core::proxy::{ handle_first_party_click, handle_first_party_proxy, handle_first_party_proxy_rebuild, handle_first_party_proxy_sign, }; use trusted_server_core::publisher::{ - AuctionDispatch, PAGE_BIDS_LEGACY_PATH, PAGE_BIDS_PATH, PublisherResponse, - buffer_publisher_response_async, handle_page_bids, handle_publisher_request, - handle_tsjs_dynamic, page_bids_preflight_denied, + AuctionDispatch, PAGE_BIDS_LEGACY_PATH, PAGE_BIDS_PATH, PublisherOriginOptions, + PublisherResponse, buffer_publisher_response_async, handle_page_bids, handle_publisher_request, + handle_tsjs_dynamic, page_bids_preflight_denied, publisher_input_cap, publisher_owner_error, }; use trusted_server_core::request_signing::{ handle_trusted_server_discovery, handle_verify_signature, @@ -205,7 +209,6 @@ fn build_state_with_services( services: Option, ) -> Result, Report> { let plan = Arc::new(compile_auction_plan(&settings)?); - plan.validate_for_target(trusted_server_core::platform::AuctionTargetId::Cloudflare)?; let orchestrator = build_orchestrator_with_plan(Arc::clone(&plan), &settings)?; let registry = IntegrationRegistry::with_plan(&settings, plan)?; @@ -218,10 +221,10 @@ fn build_state_with_services( } impl AppState { - fn services_for_request(&self, ctx: &RequestContext) -> RuntimeServices { + fn services_for_request(&self, ctx: &RequestContext) -> Result { self.services .clone() - .unwrap_or_else(|| build_runtime_services(ctx)) + .map_or_else(|| build_runtime_services(ctx), Ok) } } @@ -276,8 +279,8 @@ where let s = Arc::clone(&state); let f = f.clone(); Box::pin(async move { - let services = s.services_for_request(&ctx); - let mut req = ctx.into_request(); + let services = s.services_for_request(&ctx)?; + let mut req = ctx.into_request()?; if let Err(error) = trusted_server_core::integrations::gpt_diagnostics::prepare_request( &s.settings, &mut req, @@ -293,12 +296,19 @@ where // Publisher response helper // --------------------------------------------------------------------------- +/// Largest publisher origin body the Cloudflare adapter collects before egress. +/// +/// It keeps the 10 MiB limit the adapter's own client applied before the +/// publisher origin moved to `EdgeZero`'s client. +const PUBLISHER_ORIGIN_INPUT_BYTES: u64 = 10 * 1024 * 1024; + /// Collapse a [`PublisherResponse`] into a plain [`Response`]. /// /// Delegates to the shared [`buffer_publisher_response_async`], which collects -/// the dispatched server-side auction and enforces -/// `settings.publisher.max_buffered_body_bytes`, then removes any -/// `Transfer-Encoding` header since the buffered body is no longer chunked. +/// the dispatched server-side auction, collects raw origin bodies under +/// `input_cap`, and enforces `settings.publisher.max_buffered_body_bytes` on +/// processed output, then removes any `Transfer-Encoding` header since the +/// buffered body is no longer chunked. async fn resolve_publisher_response( publisher_response: PublisherResponse, method: &Method, @@ -306,6 +316,7 @@ async fn resolve_publisher_response( registry: &IntegrationRegistry, orchestrator: &AuctionOrchestrator, services: &RuntimeServices, + input_cap: u64, ) -> Result> { let mut response = buffer_publisher_response_async( publisher_response, @@ -314,31 +325,13 @@ async fn resolve_publisher_response( registry, orchestrator, services, + Some(input_cap), ) .await?; response.headers_mut().remove(header::TRANSFER_ENCODING); Ok(response) } -// --------------------------------------------------------------------------- -// Error helper -// --------------------------------------------------------------------------- - -/// Convert a [`Report`] into an HTTP [`Response`]. -pub(crate) fn http_error(report: &Report) -> Response { - let root_error = report.current_context(); - log::error!("Error occurred: {:?}", report); - - let body = edgezero_core::body::Body::from(format!("{}\n", root_error.user_message())); - let mut response = Response::new(body); - *response.status_mut() = root_error.status_code(); - response.headers_mut().insert( - header::CONTENT_TYPE, - HeaderValue::from_static("text/plain; charset=utf-8"), - ); - response -} - fn admin_key_management_not_supported() -> Response { let body = edgezero_core::body::Body::from( "Admin key management is not supported on Cloudflare Workers.\n\ @@ -389,7 +382,7 @@ fn legacy_admin_alias_denied() -> Response { } // --------------------------------------------------------------------------- -// Startup error fallback +// Publisher fallback methods // --------------------------------------------------------------------------- /// HTTP methods the publisher fallback proxies, mirroring the Axum/Fastly @@ -407,34 +400,6 @@ fn publisher_fallback_methods() -> [Method; 7] { ] } -/// Returns a [`RouterService`] that responds to every route with the startup error. -fn startup_error_router(e: &Report) -> RouterService { - let message = Arc::new(format!("{}\n", e.current_context().user_message())); - let status = e.current_context().status_code(); - - let make = move |msg: Arc| { - move |_ctx: RequestContext| { - let body = edgezero_core::body::Body::from((*msg).clone()); - let mut resp = Response::new(body); - *resp.status_mut() = status; - resp.headers_mut().insert( - header::CONTENT_TYPE, - HeaderValue::from_static("text/plain; charset=utf-8"), - ); - async move { Ok::(resp) } - } - }; - - let mut router = RouterService::builder().middleware(FinalizeResponseMiddleware::new( - Arc::new(Settings::default()), - )); - for method in publisher_fallback_methods() { - router = router.route("/", method.clone(), make(Arc::clone(&message))); - router = router.route("/{*rest}", method, make(Arc::clone(&message))); - } - router.build() -} - // --------------------------------------------------------------------------- // TrustedServerApp // --------------------------------------------------------------------------- @@ -443,20 +408,21 @@ fn startup_error_router(e: &Report) -> RouterService { pub struct TrustedServerApp; impl Hooks for TrustedServerApp { + fn configure(app: &mut App) -> Result<(), EdgeError> { + install_edge_policies(app) + } + fn name() -> &'static str { "TrustedServer" } - fn routes() -> RouterService { - let state = match build_state() { - Ok(s) => s, - Err(ref e) => { - log::error!("failed to build application state: {:?}", e); - return startup_error_router(e); - } - }; + fn routes() -> Result { + let state = build_state().map_err(|report| { + log::error!("{report:?}"); + startup_error(&report) + })?; - build_router(&state) + Ok(router_builder(&state).build()) } } @@ -475,7 +441,34 @@ impl TrustedServerApp { settings: Settings, ) -> Result> { let state = build_state_with_settings(settings)?; - Ok(build_router(&state)) + Ok(router_builder(&state).build()) + } + + /// Build the full application router from explicit settings, with `client` + /// in router state. + /// + /// Testing seam: `EdgeZero` hands a client in router state to every + /// request in place of the adapter's own, so an app served from this + /// router on `EdgeZero`'s Axum test server starts no real transport. + /// Production installs no client in router state. + /// + /// # Errors + /// + /// Returns an error when the auction orchestrator or the integration + /// registry fail to initialise. + /// + /// # Examples + /// + /// ```ignore + /// let client = HttpClient::with_client(ScriptedHttpClient::forbidding()); + /// let router = TrustedServerApp::routes_with_settings_and_outbound_client(settings, client)?; + /// ``` + pub fn routes_with_settings_and_outbound_client( + settings: Settings, + client: HttpClient, + ) -> Result> { + let state = build_state_with_settings(settings)?; + Ok(router_builder(&state).with_state(client).build()) } /// Build the full router with explicit settings and runtime services. @@ -483,6 +476,8 @@ impl TrustedServerApp { /// Each request receives a clone of the supplied services, allowing callers /// to exercise production routes with deterministic platform dependencies. /// The supplied client metadata applies to every request to this router. + /// The services' outbound client also goes into router state, so an app + /// served from this router starts no real `EdgeZero` transport. /// /// # Errors /// @@ -498,12 +493,14 @@ impl TrustedServerApp { settings: Settings, services: RuntimeServices, ) -> Result> { + let client = services.outbound_client().clone(); let state = build_state_with_services(settings, Some(services))?; - Ok(build_router(&state)) + Ok(router_builder(&state).with_state(client).build()) } } -fn build_router(state: &Arc) -> RouterService { +/// Registers every route, middleware, and hook for `state` on a router builder. +fn router_builder(state: &Arc) -> RouterBuilder { { let state = Arc::clone(state); @@ -512,8 +509,8 @@ fn build_router(state: &Arc) -> RouterService { state: Arc, ctx: RequestContext, ) -> Result { - let services = state.services_for_request(&ctx); - let mut req = ctx.into_request(); + let services = state.services_for_request(&ctx)?; + let mut req = ctx.into_request()?; if let Some(response) = deny_admin_diagnostic_fallback(&req) { return Ok(response); } @@ -561,6 +558,7 @@ fn build_router(state: &Arc) -> RouterService { slots: state.settings.creative_opportunity_slots(), registry: None, }; + let input_cap = publisher_input_cap(&state.settings, PUBLISHER_ORIGIN_INPUT_BYTES); match handle_publisher_request( &state.settings, &services, @@ -568,21 +566,30 @@ fn build_router(state: &Arc) -> RouterService { &mut ec_context, auction, req, - EdgeCacheHeader::CloudflareCdnCacheControl, + PublisherOriginOptions { + edge_header: EdgeCacheHeader::CloudflareCdnCacheControl, + input_cap: Some(input_cap), + // Workers rewrites the inbound `Accept-Encoding` (to `br, gzip` + // whether the client sent none or `identity`), so the client's + // real preference is unknown here and the origin answers in a + // coding the client may not accept. Decoding the origin body + // serves identity bytes whatever the client sent. + response_decoding: ResponseDecoding::Decode, + }, ) .await { - Ok(pr) => { - resolve_publisher_response( - pr, - &method, - &state.settings, - &state.registry, - &state.orchestrator, - &services, - ) - .await - } + Ok((pr, owner)) => resolve_publisher_response( + pr, + &method, + &state.settings, + &state.registry, + &state.orchestrator, + &services, + input_cap, + ) + .await + .and_then(|response| owner.attach(response).map_err(publisher_owner_error)), Err(e) => Err(e), } }; @@ -620,7 +627,7 @@ fn build_router(state: &Arc) -> RouterService { .post( "/verify-signature", make_handler(Arc::clone(&state), |s, services, req| async move { - handle_verify_signature(&s.settings, &services, req) + handle_verify_signature(&s.settings, &services, req).await }), ) // Canonical admin key routes. These match `Settings::ADMIN_ENDPOINTS` @@ -777,11 +784,11 @@ fn build_router(state: &Arc) -> RouterService { router = router.route("/{*rest}", method, fallback.clone()); } - router.build() + router } } -fn trace_metadata(state: &AppState, request: &Request) -> TraceMetadata { +fn trace_metadata(state: &AppState, request: &RequestParts) -> TraceMetadata { if let Some(services) = &state.services { let client_info = services.client_info().clone(); let geo = client_info.client_ip.and_then(|client_ip| { @@ -793,12 +800,12 @@ fn trace_metadata(state: &AppState, request: &Request) -> TraceMetadata { return TraceMetadata { client_info, geo }; } let client_ip = request - .headers() + .headers .get("cf-connecting-ip") .and_then(|value| value.to_str().ok()) .and_then(|value| value.parse().ok()); let geo = request - .headers() + .headers .get("cf-ipcountry") .and_then(|value| value.to_str().ok()) .filter(|country| *country != "XX") @@ -931,7 +938,7 @@ mod tests { } #[test] - fn startup_rejects_multi_provider_auction_plan() { + fn cloudflare_and_spin_load_multi_provider_plan() { let mut settings = Settings::from_toml( r#" [[handlers]] @@ -970,13 +977,13 @@ mod tests { }) .collect(); - let error = match build_state_with_settings(settings) { - Ok(_) => panic!("Cloudflare startup should reject multi-provider fanout"), - Err(error) => error, - }; - assert!( - format!("{error:?}").contains("concurrent provider fanout"), - "should identify unsupported fanout: {error:?}" + let state = build_state_with_settings(settings) + .expect("should load a multi-provider auction plan on Cloudflare"); + + assert_eq!( + state.orchestrator.provider_count(), + 2, + "should fan out to both configured providers" ); } diff --git a/crates/trusted-server-adapter-cloudflare/src/lib.rs b/crates/trusted-server-adapter-cloudflare/src/lib.rs index 80c24e8f1f..a1fe4a5772 100644 --- a/crates/trusted-server-adapter-cloudflare/src/lib.rs +++ b/crates/trusted-server-adapter-cloudflare/src/lib.rs @@ -1,3 +1,6 @@ +// Unit tests may buffer bodies with `Body::into_bytes`; production code must not. +#![cfg_attr(test, allow(clippy::disallowed_methods))] + // The `cloudflare` feature activates the `worker` crate which requires // wasm-bindgen and only compiles for `wasm32-unknown-unknown`. Enabling it on // a native target produces cryptic linker errors — catch it early instead. diff --git a/crates/trusted-server-adapter-cloudflare/src/middleware.rs b/crates/trusted-server-adapter-cloudflare/src/middleware.rs index af19241163..aa5a0bb985 100644 --- a/crates/trusted-server-adapter-cloudflare/src/middleware.rs +++ b/crates/trusted-server-adapter-cloudflare/src/middleware.rs @@ -7,6 +7,7 @@ use edgezero_core::http::{HeaderValue, Response}; use edgezero_core::middleware::{Middleware, Next}; use trusted_server_core::auth::enforce_basic_auth; use trusted_server_core::constants::HEADER_X_GEO_INFO_AVAILABLE; +use trusted_server_core::error::http_error; use trusted_server_core::http_util::sanitize_trusted_client_ip_headers; use trusted_server_core::settings::Settings; @@ -39,7 +40,7 @@ impl SanitizeRequestMiddleware { impl Middleware for SanitizeRequestMiddleware { async fn handle(&self, mut ctx: RequestContext, next: Next<'_>) -> Result { sanitize_trusted_client_ip_headers( - ctx.request_mut(), + ctx.parts_mut(), self.settings.trusted_client_ip.as_ref(), ); next.run(ctx).await @@ -75,7 +76,6 @@ impl FinalizeResponseMiddleware { impl Middleware for FinalizeResponseMiddleware { async fn handle(&self, ctx: RequestContext, next: Next<'_>) -> Result { let geo_available = ctx - .request() .headers() .get("cf-ipcountry") .and_then(|v| v.to_str().ok()) @@ -113,12 +113,12 @@ impl AuthMiddleware { #[async_trait(?Send)] impl Middleware for AuthMiddleware { async fn handle(&self, mut ctx: RequestContext, next: Next<'_>) -> Result { - match enforce_basic_auth(&self.settings, ctx.request_mut()) { + match enforce_basic_auth(&self.settings, ctx.parts_mut()) { Ok(Some(response)) => return Ok(response), Ok(None) => {} Err(report) => { log::error!("auth check failed: {:?}", report); - return Ok(crate::app::http_error(&report)); + return Ok(http_error(&report)); } } @@ -148,6 +148,10 @@ pub(crate) fn apply_finalize_headers( trusted_server_core::version_header::apply_git_version_header(response); + // The finalize middleware runs once per request, so each skipped operator + // entry warns once. + trusted_server_core::response_privacy::warn_skipped_operator_response_headers(settings); + // Cloudflare is a real shared cache: cookie-bearing responses must stay // private and operator headers must not re-enable caching for uncacheable // per-user payloads. @@ -175,6 +179,7 @@ mod tests { use futures::executor::block_on; use trusted_server_core::redacted::Redacted; use trusted_server_core::settings::TrustedClientIpConfig; + use trusted_server_core::test_support::logging::WarningCapture; fn empty_response() -> Response { response_builder() @@ -222,6 +227,29 @@ mod tests { s } + #[test] + fn warns_once_about_a_skipped_operator_entry() { + let settings = settings_with_response_headers(vec![("connection", "keep-alive")]); + let mut response = empty_response(); + let capture = WarningCapture::start(); + + apply_finalize_headers(&settings, false, &mut response); + + let skipped = capture + .take() + .into_iter() + .filter(|warning| warning.message.contains("`connection`")) + .count(); + assert_eq!( + skipped, 1, + "should warn once about the skipped operator entry" + ); + assert!( + !response.headers().contains_key("connection"), + "should skip the operator entry" + ); + } + #[test] fn sets_geo_available_false_when_no_country_header() { let settings = settings_with_response_headers(vec![]); @@ -306,8 +334,8 @@ mod tests { let handler_observed = Arc::clone(&handler_observed); async move { *handler_observed.lock().expect("should lock observation") = Some(( - ctx.request().headers().contains_key("x-reader-ip"), - ctx.request().headers().contains_key("x-reader-ip-auth"), + ctx.headers().contains_key("x-reader-ip"), + ctx.headers().contains_key("x-reader-ip-auth"), )); Ok::(empty_response()) } diff --git a/crates/trusted-server-adapter-cloudflare/src/platform.rs b/crates/trusted-server-adapter-cloudflare/src/platform.rs index 40cfc7b3a1..adec308f5d 100644 --- a/crates/trusted-server-adapter-cloudflare/src/platform.rs +++ b/crates/trusted-server-adapter-cloudflare/src/platform.rs @@ -6,78 +6,53 @@ use std::time::Duration; use bytes::Bytes; use edgezero_core::config_store::ConfigStoreHandle; +use edgezero_core::error::EdgeError; use edgezero_core::key_value_store::{KvHandle, KvPage, KvStore}; -use error_stack::{Report, ResultExt as _}; +use error_stack::Report; +use trusted_server_core::outbound::request_outbound_client; use trusted_server_core::platform::{ - BackendNamingPolicy, ClientInfo, GeoInfo, KvError, PlatformBackend, PlatformBackendSpec, - PlatformConfigStore, PlatformError, PlatformGeo, PlatformHttpClient, PlatformKvStore, + ClientInfo, GeoInfo, KvError, PlatformConfigStore, PlatformError, PlatformGeo, PlatformKvStore, PlatformSecretStore, RuntimeServices, StoreId, StoreName, UnavailableKvStore, }; -#[cfg(not(target_arch = "wasm32"))] -use trusted_server_core::platform::UnavailableHttpClient; - -#[cfg(target_arch = "wasm32")] -use trusted_server_core::platform::{ - PlatformHttpRequest, PlatformPendingRequest, PlatformPendingResponse, PlatformResponse, - PlatformSelectResult, -}; - // --------------------------------------------------------------------------- // Noop stubs — used when a handle is absent (native CI, missing binding) // --------------------------------------------------------------------------- struct NoopConfigStore; +#[async_trait::async_trait(?Send)] impl PlatformConfigStore for NoopConfigStore { fn get(&self, _: &StoreName, _: &str) -> Result> { Err(Report::new(PlatformError::ConfigStore).attach("config store not available")) } - fn put(&self, _: &StoreId, _: &str, _: &str) -> Result<(), Report> { + async fn put(&self, _: &StoreId, _: &str, _: &str) -> Result<(), Report> { Err(Report::new(PlatformError::ConfigStore).attach("config store not available")) } - fn delete(&self, _: &StoreId, _: &str) -> Result<(), Report> { + async fn delete(&self, _: &StoreId, _: &str) -> Result<(), Report> { Err(Report::new(PlatformError::ConfigStore).attach("config store not available")) } } struct NoopSecretStore; +#[async_trait::async_trait(?Send)] impl PlatformSecretStore for NoopSecretStore { fn get_bytes(&self, _: &StoreName, _: &str) -> Result, Report> { Err(Report::new(PlatformError::SecretStore).attach("secret store not available")) } - fn create(&self, _: &StoreId, _: &str, _: &str) -> Result<(), Report> { + async fn create(&self, _: &StoreId, _: &str, _: &str) -> Result<(), Report> { Err(Report::new(PlatformError::SecretStore).attach("secret store not available")) } - fn delete(&self, _: &StoreId, _: &str) -> Result<(), Report> { + async fn delete(&self, _: &StoreId, _: &str) -> Result<(), Report> { Err(Report::new(PlatformError::SecretStore).attach("secret store not available")) } } -struct NoopBackend; - -impl PlatformBackend for NoopBackend { - fn naming_policy(&self) -> BackendNamingPolicy { - BackendNamingPolicy::Cloudflare - } - - fn predict_name(&self, spec: &PlatformBackendSpec) -> Result> { - self.naming_policy() - .predict(spec) - .map(|prediction| prediction.name) - .change_context(PlatformError::Backend) - } - - fn ensure(&self, spec: &PlatformBackendSpec) -> Result> { - self.predict_name(spec) - } -} - // --------------------------------------------------------------------------- // edgezero handle adapters — no #[cfg] needed; platform-specific store // construction is handled by edgezero's run_app before we receive the ctx. @@ -94,6 +69,7 @@ impl PlatformBackend for NoopBackend { /// callers cannot route to a different store by passing a different name. struct ConfigStoreHandleAdapter(ConfigStoreHandle); +#[async_trait::async_trait(?Send)] impl PlatformConfigStore for ConfigStoreHandleAdapter { fn get(&self, _store_name: &StoreName, key: &str) -> Result> { futures::executor::block_on(self.0.get(key)) @@ -101,16 +77,17 @@ impl PlatformConfigStore for ConfigStoreHandleAdapter { Report::new(PlatformError::ConfigStore) .attach(format!("config store lookup failed: {e}")) })? + .map(|value| String::from(value.as_str())) .ok_or_else(|| { Report::new(PlatformError::ConfigStore).attach(format!("key not found: {key}")) }) } - fn put(&self, _: &StoreId, _: &str, _: &str) -> Result<(), Report> { + async fn put(&self, _: &StoreId, _: &str, _: &str) -> Result<(), Report> { Err(Report::new(PlatformError::ConfigStore).attach("config store writes are not supported")) } - fn delete(&self, _: &StoreId, _: &str) -> Result<(), Report> { + async fn delete(&self, _: &StoreId, _: &str) -> Result<(), Report> { Err(Report::new(PlatformError::ConfigStore).attach("config store writes are not supported")) } } @@ -161,406 +138,6 @@ impl KvStore for KvHandleAdapter { } } -// --------------------------------------------------------------------------- -// CloudflareHttpClient — WASM target only -// --------------------------------------------------------------------------- - -/// [`worker::Fetch`]-backed HTTP client for the Cloudflare Workers runtime. -/// -/// # Multi-provider auction limitation -/// -/// Auction configuration supports one provider at a time, so -/// [`PlatformHttpClient::supports_concurrent_fanout`] reports `false`. -/// Requests start in a runtime task at dispatch, overlapping publisher-origin work. -/// Deadline selection bounds waiting for fetch and body collection and aborts -/// the Workers fetch when its pending operation is dropped. -#[cfg(target_arch = "wasm32")] -pub struct CloudflareHttpClient; - -/// Maximum buffered upstream response body, mirroring the Fastly adapter's cap. -/// -/// Cloudflare Workers isolates have a bounded memory budget (~128 MB), so a -/// large or hostile upstream could OOM the isolate. `execute` buffers the whole -/// body via `resp.bytes()`, so it caps both the declared Content-Length and the -/// materialized byte count at this limit. -#[cfg(target_arch = "wasm32")] -const MAX_PLATFORM_RESPONSE_BODY_BYTES: usize = 10 * 1024 * 1024; // 10 MiB - -/// Collect the lowercased tokens listed in the upstream `Connection:` header(s). -/// -/// Every header named here is hop-by-hop (RFC 7230 §6.1) and must not be -/// forwarded downstream. -#[cfg(target_arch = "wasm32")] -fn response_connection_tokens(resp: &worker::Response) -> Vec { - resp.headers() - .entries() - .filter(|(name, _)| name.eq_ignore_ascii_case("connection")) - .flat_map(|(_, value)| { - value - .split(',') - .map(|token| token.trim().to_ascii_lowercase()) - .filter(|token| !token.is_empty()) - .collect::>() - }) - .collect() -} - -/// Returns `true` for hop-by-hop response headers that must not be forwarded -/// downstream (RFC 7230 §6.1), including any header named in the upstream -/// `Connection:` token list. Mirrors the Axum adapter's -/// `is_hop_by_hop_response_header`. `transfer-encoding` is stripped separately -/// at the call site alongside the other auto-decoded framing headers. -#[cfg(target_arch = "wasm32")] -fn is_hop_by_hop_response_header(name: &str, connection_tokens: &[String]) -> bool { - const HOP_BY_HOP: [&str; 6] = [ - "connection", - "keep-alive", - "proxy-authenticate", - "proxy-authorization", - "te", - "upgrade", - ]; - let lower = name.to_ascii_lowercase(); - HOP_BY_HOP.iter().any(|header| *header == lower) || connection_tokens.contains(&lower) -} - -/// Cache policy for the outbound Workers `fetch` derived from -/// [`PlatformHttpRequest::cache_intent`]. -/// -/// Workers subrequests are eligible for Cloudflare's cache by default, so an -/// ad-stack navigation could otherwise be satisfied from cache (or revalidated -/// into a bodyless 304) instead of receiving a complete origin body. Mapping -/// the bypass flag to the runtime's `no-store` mode keeps the core contract's -/// cache-bypass guarantee intact on this adapter. -/// -/// Extracted as a free function over a target-independent enum so the mapping -/// is testable on native targets, where the `#[cfg(target_arch = "wasm32")]` -/// `execute` impl and its `worker` dependency are excluded from the test -/// binary. -#[cfg(any(target_arch = "wasm32", test))] -#[derive(Debug, Clone, Copy, Eq, PartialEq)] -enum OutboundCacheMode { - /// Leave the Workers runtime's default cache behavior in place. - RuntimeDefault, - /// Maps to `worker::CacheMode::NoStore`. - NoStore, -} - -#[cfg(any(target_arch = "wasm32", test))] -fn outbound_cache_mode( - intent: &trusted_server_core::platform::PlatformCacheIntent, -) -> OutboundCacheMode { - // Workers has no surrogate-key concept, so `Shared` falls in with `Default`: let the - // runtime apply its own behavior rather than pretending to honor a key it cannot use. - if intent.is_bypass() { - OutboundCacheMode::NoStore - } else { - OutboundCacheMode::RuntimeDefault - } -} - -#[cfg(target_arch = "wasm32")] -fn outbound_request_init(method: worker::Method, headers: worker::Headers) -> worker::RequestInit { - let mut init = worker::RequestInit::new(); - init.with_method(method) - .with_headers(headers) - .with_redirect(worker::RequestRedirect::Manual); - init -} - -#[cfg(target_arch = "wasm32")] -struct AbortOnDrop(Option); - -#[cfg(target_arch = "wasm32")] -impl Drop for AbortOnDrop { - fn drop(&mut self) { - if let Some(controller) = self.0.take() { - controller.abort(); - } - } -} - -#[cfg(target_arch = "wasm32")] -impl CloudflareHttpClient { - async fn execute( - request: PlatformHttpRequest, - ) -> Result> { - use worker::{CacheMode, Fetch, Headers, Method, Request}; - - // The Cloudflare fetch path cannot honor Fastly-style Image Optimizer - // metadata, and it always buffers the response body (see below). The - // `PlatformHttpRequest` contract requires adapters that cannot honor - // these to reject rather than silently drop the behavior, so surface a - // typed error instead of returning an untransformed or fully-buffered - // response. These fields are only set by asset routes, which are not - // routed to the Cloudflare adapter today; the guard keeps the contract - // honest if that changes. - if request.image_optimizer.is_some() { - return Err(Report::new(PlatformError::HttpClient) - .attach("Image Optimizer is not supported on the Cloudflare Workers runtime")); - } - if request.stream_response { - return Err(Report::new(PlatformError::HttpClient).attach( - "streaming response bodies are not supported on the Cloudflare Workers runtime", - )); - } - - let cache_mode = outbound_cache_mode(&request.cache_intent); - - let uri = request.request.uri().to_string(); - // http::Method always stores uppercase; worker 0.7 implements From only. - let method = Method::from(request.request.method().to_string()); - - let headers = Headers::new(); - for (name, value) in request.request.headers() { - let value_str = std::str::from_utf8(value.as_bytes()) - .change_context(PlatformError::HttpClient) - .attach_with(|| { - format!("non-UTF-8 bytes in outbound header `{name}` — value dropped") - })?; - // `append` rather than `set`: a request carrying the same header - // name more than once must forward every value, matching the - // response path which appends via `edge_builder.header(...)`. - headers - .append(name.as_str(), value_str) - .change_context(PlatformError::HttpClient)?; - } - - let (_, body) = request.request.into_parts(); - let body_bytes = match body { - edgezero_core::body::Body::Once(bytes) => bytes.to_vec(), - edgezero_core::body::Body::Stream(_) => { - return Err(Report::new(PlatformError::HttpClient) - .attach("streaming request bodies are not supported on Cloudflare Workers")); - } - }; - - // Force manual redirect handling: the Workers runtime otherwise defaults - // to `RequestRedirect::Follow` and transparently chases 3xx responses to - // any host inside `Fetch::send()`. Core's `proxy_with_redirects` does its - // own per-hop redirect handling and validates each next hop against - // `allowed_domains`; auto-following here would bypass that allowlist - // (SSRF). `Manual` surfaces the 3xx + Location back to core unfollowed, - // matching the Axum adapter's `redirect::Policy::none()`. - let mut init = outbound_request_init(method, headers); - // Setting the `cache` field requires the `cache_option_enabled` - // compatibility flag, which is only on by default from compatibility - // date 2024-11-11. `wrangler.toml`/`wrangler.ci.toml` pin an earlier - // date and set the flag explicitly; without it the Workers runtime - // throws here rather than failing at deploy time. - match cache_mode { - OutboundCacheMode::NoStore => { - init.with_cache(CacheMode::NoStore); - } - OutboundCacheMode::RuntimeDefault => {} - } - if !body_bytes.is_empty() { - let uint8 = js_sys::Uint8Array::from(body_bytes.as_slice()); - init.with_body(Some(uint8.into())); - } - - let worker_req = - Request::new_with_init(&uri, &init).change_context(PlatformError::HttpClient)?; - - let abort = AbortOnDrop(Some(worker::AbortController::default())); - let signal = abort - .0 - .as_ref() - .expect("should retain abort controller") - .signal(); - let mut resp = Fetch::Request(worker_req) - .send_with_signal(&signal) - .await - .change_context(PlatformError::HttpClient) - .attach_with(|| format!("outbound request to {uri} failed"))?; - - let status = resp.status_code(); - - // Pre-flight: reject oversized responses before copying bytes into the - // isolate heap. Content-Length is advisory (and absent on chunked - // responses), so the post-buffer check below is the real guard; this - // just rejects honestly-declared large bodies cheaply. Mirrors the - // Fastly adapter's two-stage cap. - if let Some(claimed_len) = resp - .headers() - .get("content-length") - .ok() - .flatten() - .and_then(|v| v.trim().parse::().ok()) - && claimed_len > MAX_PLATFORM_RESPONSE_BODY_BYTES - { - return Err(Report::new(PlatformError::HttpClient).attach(format!( - "origin Content-Length {claimed_len} exceeds \ - {MAX_PLATFORM_RESPONSE_BODY_BYTES}-byte response body limit" - ))); - } - - let connection_tokens = response_connection_tokens(&resp); - let mut edge_builder = edgezero_core::http::response_builder().status(status); - for (name, value) in resp.headers().entries() { - // The Workers runtime auto-decompresses gzip/br/deflate and handles - // chunked transfer — strip these headers so the proxy layer does not - // attempt a second decompression pass on the already-decoded body. - // Content-Length is stripped too: the origin value describes the - // compressed payload, not the decoded bytes returned by `bytes()`, - // and forwarding it stale would truncate pass-through responses. - // The accurate length is set from the decoded body below. - if name.eq_ignore_ascii_case("content-encoding") - || name.eq_ignore_ascii_case("transfer-encoding") - || name.eq_ignore_ascii_case("content-length") - { - continue; - } - // Strip hop-by-hop headers (and any header named in the upstream - // `Connection:` token list) so they are not forwarded downstream, - // matching the Axum adapter's `is_hop_by_hop_response_header`. - if is_hop_by_hop_response_header(&name, &connection_tokens) { - continue; - } - edge_builder = edge_builder.header(name.as_str(), value.as_bytes()); - } - let body_bytes = resp - .bytes() - .await - .change_context(PlatformError::HttpClient)?; - - // Belt-and-suspenders: catches chunked responses with no Content-Length. - if body_bytes.len() > MAX_PLATFORM_RESPONSE_BODY_BYTES { - return Err(Report::new(PlatformError::HttpClient).attach(format!( - "origin response body {} bytes exceeds \ - {MAX_PLATFORM_RESPONSE_BODY_BYTES}-byte limit", - body_bytes.len() - ))); - } - edge_builder = edge_builder.header( - edgezero_core::http::header::CONTENT_LENGTH, - body_bytes.len(), - ); - let edge_resp = edge_builder - .body(edgezero_core::body::Body::from(body_bytes)) - .change_context(PlatformError::HttpClient)?; - - Ok(PlatformResponse::new(edge_resp).with_backend_name(request.backend_name)) - } -} - -#[cfg(target_arch = "wasm32")] -#[async_trait::async_trait(?Send)] -impl PlatformHttpClient for CloudflareHttpClient { - async fn send( - &self, - request: PlatformHttpRequest, - ) -> Result> { - Self::execute(request).await - } - - fn supports_concurrent_fanout(&self) -> bool { - // Requests start concurrently at dispatch, but `select` and - // `select_until` collect only one response, so multi-provider fan-out - // stays unsupported. - false - } - - async fn send_async( - &self, - request: PlatformHttpRequest, - ) -> Result> { - let backend_name = request.backend_name.clone(); - let pending = PlatformPendingResponse::start( - Self::execute(request), - worker::wasm_bindgen_futures::spawn_local, - ); - Ok(PlatformPendingRequest::new(pending).with_backend_name(backend_name)) - } - - async fn select( - &self, - mut pending_requests: Vec, - ) -> Result> { - if pending_requests.is_empty() { - return Err(Report::new(PlatformError::HttpClient) - .attach("select called with an empty pending_requests list")); - } - - reject_multi_provider_fanout(pending_requests.len())?; - - let ready_platform = pending_requests.remove(0); - let mut pending = ready_platform - .downcast::>>>() - .map_err(|_| { - Report::new(PlatformError::HttpClient) - .attach("unexpected inner type in CloudflareHttpClient::select") - })?; - - let ready = pending.wait().await.unwrap_or_else(|error| { - Err(Report::new(PlatformError::HttpClient) - .attach(format!("dispatched response task stopped: {error}"))) - }); - Ok(PlatformSelectResult { - ready, - remaining: pending_requests, - failed_backend_name: None, - }) - } - - async fn select_until( - &self, - mut pending_requests: Vec, - deadline: web_time::Instant, - ) -> Result, Report> { - if pending_requests.is_empty() { - return Err(Report::new(PlatformError::HttpClient) - .attach("select_until called with an empty pending_requests list")); - } - if pending_requests.len() > 1 { - return Err(Report::new(PlatformError::HttpClient) - .attach("multi-provider fan-out is not supported on this adapter")); - } - let platform_pending = pending_requests.remove(0); - let backend_name = platform_pending.backend_name().map(str::to_owned); - let mut pending = platform_pending - .downcast::>>>() - .map_err(|_| { - Report::new(PlatformError::HttpClient) - .attach("unexpected pending request type in deadline selection") - })?; - let result = pending - .complete_before_deadline( - async { - // Workers timers use whole signed milliseconds. Recheck the - // absolute deadline after each bounded timer interval. - loop { - let remaining = - deadline.saturating_duration_since(web_time::Instant::now()); - if remaining.is_zero() { - break; - } - worker::Delay::from( - remaining - .max(Duration::from_millis(1)) - .min(Duration::from_millis(i32::MAX as u64)), - ) - .await; - } - }, - deadline, - web_time::Instant::now, - ) - .await; - Ok(result.map(|result| { - let ready = result.unwrap_or_else(|error| { - Err(Report::new(PlatformError::HttpClient) - .attach(format!("dispatched response task stopped: {error}"))) - }); - PlatformSelectResult { - failed_backend_name: ready.as_ref().err().and(backend_name), - ready, - remaining: pending_requests, - } - })) - } -} - // --------------------------------------------------------------------------- // CloudflareSecretStoreAdapter — WASM target only // @@ -578,6 +155,7 @@ pub(crate) struct CloudflareSecretStoreAdapter { } #[cfg(target_arch = "wasm32")] +#[async_trait::async_trait(?Send)] impl PlatformSecretStore for CloudflareSecretStoreAdapter { fn get_bytes( &self, @@ -595,12 +173,12 @@ impl PlatformSecretStore for CloudflareSecretStoreAdapter { } } - fn create(&self, _: &StoreId, _: &str, _: &str) -> Result<(), Report> { + async fn create(&self, _: &StoreId, _: &str, _: &str) -> Result<(), Report> { Err(Report::new(PlatformError::SecretStore) .attach("secret store writes are not supported on Cloudflare Workers")) } - fn delete(&self, _: &StoreId, _: &str) -> Result<(), Report> { + async fn delete(&self, _: &StoreId, _: &str) -> Result<(), Report> { Err(Report::new(PlatformError::SecretStore) .attach("secret store writes are not supported on Cloudflare Workers")) } @@ -624,13 +202,19 @@ impl PlatformSecretStore for CloudflareSecretStoreAdapter { /// Geo information is read from Cloudflare's injected request headers /// (`cf-ipcountry`, etc.) which are present on all plans; headers absent on /// the native host target simply produce empty/zero defaults. -pub fn build_runtime_services(ctx: &edgezero_core::context::RequestContext) -> RuntimeServices { +/// +/// The outbound client is the one the request context carries, with the proxy +/// marker turned off, and the clock is the context's monotonic clock. +/// +/// # Errors +/// +/// Returns an internal [`EdgeError`] when the request carries no outbound +/// client. +pub fn build_runtime_services( + ctx: &edgezero_core::context::RequestContext, +) -> Result { let client_ip = extract_client_ip(ctx); - - #[cfg(target_arch = "wasm32")] - let http_client: Arc = Arc::new(CloudflareHttpClient); - #[cfg(not(target_arch = "wasm32"))] - let http_client: Arc = Arc::new(UnavailableHttpClient); + let outbound_client = request_outbound_client(ctx)?; // Config: use the ConfigStoreHandle injected by run_app — no #[cfg] needed. let config_store: Arc = ctx @@ -646,14 +230,15 @@ pub fn build_runtime_services(ctx: &edgezero_core::context::RequestContext) -> R // Secrets: still requires wasm32-specific env.secret() (async/sync mismatch). #[cfg(target_arch = "wasm32")] - let secret_store: Arc = - edgezero_adapter_cloudflare::context::CloudflareRequestContext::get(ctx.request()) - .map(|cf_ctx| { - Arc::new(CloudflareSecretStoreAdapter { - env: cf_ctx.env().clone(), - }) as Arc - }) - .unwrap_or_else(|| Arc::new(NoopSecretStore)); + let secret_store: Arc = ctx + .extensions() + .get::() + .map(|cf_ctx| { + Arc::new(CloudflareSecretStoreAdapter { + env: cf_ctx.env().clone(), + }) as Arc + }) + .unwrap_or_else(|| Arc::new(NoopSecretStore)); #[cfg(not(target_arch = "wasm32"))] let secret_store: Arc = Arc::new(NoopSecretStore); @@ -665,8 +250,8 @@ pub fn build_runtime_services(ctx: &edgezero_core::context::RequestContext) -> R .config_store(config_store) .secret_store(secret_store) .kv_store(kv_store) - .backend(Arc::new(NoopBackend)) - .http_client(http_client) + .outbound_client(outbound_client) + .clock(ctx.monotonic_clock()) .geo(Arc::new(geo)) .client_info(ClientInfo { client_ip, @@ -679,7 +264,7 @@ pub fn build_runtime_services(ctx: &edgezero_core::context::RequestContext) -> R std::sync::OnceLock::new(); #[cfg(target_arch = "wasm32")] let builder = builder.timer(Arc::clone(TIMER.get_or_init(|| Arc::new(CloudflareTimer)))); - builder.build() + Ok(builder.build()) } // --------------------------------------------------------------------------- @@ -719,7 +304,7 @@ impl PlatformGeo for CloudflareGeo { } fn build_geo(ctx: &edgezero_core::context::RequestContext) -> CloudflareGeo { - let headers = ctx.request().headers(); + let headers = ctx.headers(); let country = headers .get("cf-ipcountry") .and_then(|v| v.to_str().ok()) @@ -756,39 +341,12 @@ fn build_geo(ctx: &edgezero_core::context::RequestContext) -> CloudflareGeo { } fn extract_client_ip(ctx: &edgezero_core::context::RequestContext) -> Option { - ctx.request() - .headers() + ctx.headers() .get("cf-connecting-ip") .and_then(|v| v.to_str().ok()) .and_then(|s| s.parse().ok()) } -/// Reject multi-provider auction fan-out at the Cloudflare adapter level. -/// -/// Pending worker requests start in a runtime task at dispatch. Collection -/// still selects one response future at a time, so the adapter retains its -/// single-provider contract. -/// -/// The primary guard lives in the auction orchestrator, which checks -/// `supports_concurrent_fanout()` and rejects multi-provider configurations -/// before any request launches. This `select`-time rejection is -/// defense-in-depth for callers that bypass that check. -/// -/// Extracted as a free function so the critical control-flow is testable on -/// native targets where the `#[cfg(target_arch = "wasm32")]` `select` impl -/// is excluded from the test binary. -#[cfg(any(target_arch = "wasm32", test))] -fn reject_multi_provider_fanout(len: usize) -> Result<(), Report> { - if len >= 2 { - return Err(Report::new(PlatformError::HttpClient).attach(format!( - "CloudflareHttpClient: multi-provider fan-out is not supported \ - ({len} providers submitted). Configure a single auction provider \ - or use the Fastly adapter for parallel DSP fan-out." - ))); - } - Ok(()) -} - #[cfg(target_arch = "wasm32")] struct CloudflareTimer; @@ -908,22 +466,10 @@ mod tests { ); } - #[cfg(target_arch = "wasm32")] - #[test] - fn outbound_request_creation_sets_manual_redirect_mode() { - let init = outbound_request_init(worker::Method::Get, worker::Headers::new()); - assert!(matches!(init.redirect, worker::RequestRedirect::Manual)); - } - - #[test] - fn auction_http_capabilities_are_explicit() { - let capabilities = trusted_server_core::platform::AuctionTargetId::Cloudflare - .descriptor() - .capabilities(); - assert!(!capabilities.supports_concurrent_provider_fanout()); - } use edgezero_core::http::{HeaderValue, request_builder}; use edgezero_core::params::PathParams; + use trusted_server_core::platform::HttpClient; + use trusted_server_core::test_support::scripted_client::ScriptedHttpClient; fn make_ctx_with_header(name: &str, value: &str) -> RequestContext { let req = request_builder() @@ -947,6 +493,54 @@ mod tests { RequestContext::new(req, PathParams::default()) } + /// Builds a context whose request carries `client`, as `EdgeZero`'s + /// Cloudflare request conversion installs its default client. + fn make_ctx_with_outbound_client(client: HttpClient) -> RequestContext { + let mut req = request_builder() + .method("GET") + .uri("https://example.com/") + .body(edgezero_core::body::Body::empty()) + .expect("should build test request"); + req.extensions_mut().insert(client); + RequestContext::new(req, PathParams::default()) + } + + #[test] + fn build_runtime_services_carries_the_context_client_with_the_proxy_marker_off() { + let client = HttpClient::with_client(ScriptedHttpClient::forbidding()); + assert!( + client.proxy_marker_enabled(), + "should start from a client whose marker is on" + ); + let ctx = make_ctx_with_outbound_client(client); + + let services = build_runtime_services(&ctx).expect("should build runtime services"); + + assert!( + !services.outbound_client().proxy_marker_enabled(), + "should turn the proxy marker off on the context client" + ); + } + + #[test] + fn build_runtime_services_rejects_a_request_without_an_outbound_client() { + let ctx = make_ctx_without_header(); + + let error = build_runtime_services(&ctx) + .expect_err("should reject a request with no outbound client"); + + assert_eq!( + error.status(), + edgezero_core::http::StatusCode::INTERNAL_SERVER_ERROR, + "should answer a missing client with an internal error" + ); + assert_eq!( + error.to_string(), + "internal error: request has no outbound client", + "should name the missing client" + ); + } + #[test] fn extract_client_ip_parses_cf_connecting_ip() { let ctx = make_ctx_with_header("cf-connecting-ip", "203.0.113.42"); @@ -1032,64 +626,4 @@ mod tests { "should populate longitude" ); } - - // --------------------------------------------------------------------------- - // reject_multi_provider_fanout tests - // --------------------------------------------------------------------------- - - #[test] - fn reject_multi_provider_fanout_passes_empty() { - assert!( - reject_multi_provider_fanout(0).is_ok(), - "len=0 should pass (empty list caught separately in select)" - ); - } - - #[test] - fn reject_multi_provider_fanout_passes_single_provider() { - assert!( - reject_multi_provider_fanout(1).is_ok(), - "single provider should be allowed" - ); - } - - #[test] - fn reject_multi_provider_fanout_rejects_two_providers() { - assert!( - reject_multi_provider_fanout(2).is_err(), - "len=2 should be rejected" - ); - } - - #[test] - fn reject_multi_provider_fanout_rejects_five_providers() { - let err = reject_multi_provider_fanout(5).expect_err("should reject five providers"); - let msg = format!("{err:?}"); - assert!( - msg.contains("5"), - "error message should include provider count" - ); - } - - // ----------------------------------------------------------------------- - // outbound_cache_mode tests - // ----------------------------------------------------------------------- - - #[test] - fn outbound_cache_mode_maps_bypass_to_no_store() { - assert_eq!( - outbound_cache_mode(&trusted_server_core::platform::PlatformCacheIntent::Bypass), - OutboundCacheMode::NoStore, - "a bypass intent should force the Workers `no-store` cache mode" - ); - } - - #[test] - fn outbound_cache_mode_leaves_default_when_not_bypassing() { - assert_eq!( - outbound_cache_mode(&trusted_server_core::platform::PlatformCacheIntent::Default), - OutboundCacheMode::RuntimeDefault, - "a default intent should keep the runtime default cache behavior" - ); - } } diff --git a/crates/trusted-server-adapter-cloudflare/tests/routes.rs b/crates/trusted-server-adapter-cloudflare/tests/routes.rs index f385bf5295..4a53d7ed92 100644 --- a/crates/trusted-server-adapter-cloudflare/tests/routes.rs +++ b/crates/trusted-server-adapter-cloudflare/tests/routes.rs @@ -4,11 +4,25 @@ //! `TrustedServerApp::routes()` builds without panicking. Does not exercise //! the platform layer or outbound network calls. -use edgezero_core::app::Hooks as _; -use edgezero_core::http::{Request, Response, request_builder}; +// Test crates may buffer bodies with `Body::into_bytes`. +#![allow(clippy::disallowed_methods)] + +use edgezero_adapter_cloudflare::CLOUDFLARE_PLATFORM; +use edgezero_core::app::{App, Hooks as _}; +use edgezero_core::body::Body; +use edgezero_core::http::{Method, Request, Response, StatusCode, header, request_builder}; +use edgezero_core::ingress::{IngressDispatchOutcome, IngressFraming, IngressHeadAccounting}; +use edgezero_core::outbound::{ResponseDecoding, ResponseMode}; +use edgezero_core::response_egress_framing::ResponseEgressFramingMode; use edgezero_core::router::RouterService; use trusted_server_adapter_cloudflare::app::TrustedServerApp; +use trusted_server_core::constants::TS_GIT_VERSION; +use trusted_server_core::platform::HttpClient; use trusted_server_core::settings::Settings; +use trusted_server_core::test_support::scripted_client::{ + ScriptedHttpClient, ScriptedReply, TestClock, +}; +use trusted_server_core::test_support::{gzip, insert_outbound_client}; const LEGACY_ADMIN_DENY_METHODS: &[&str] = &["GET", "POST", "HEAD", "OPTIONS", "PUT", "PATCH", "DELETE"]; @@ -54,7 +68,13 @@ fn registered_routes() -> Vec<(String, String)> { .collect() } -async fn route(router: RouterService, req: Request) -> Response { +async fn route(router: RouterService, mut req: Request) -> Response { + // `oneshot` skips `EdgeZero`'s request conversion, which installs the + // adapter's default client, so install one that fails every request. + insert_outbound_client( + &mut req, + HttpClient::with_client(ScriptedHttpClient::forbidding()), + ); router.oneshot(req).await.expect("should route request") } @@ -67,9 +87,8 @@ fn assert_route_registered(method: &str, path: &str) { } /// Build a router from explicit test settings so routes resolve to their real -/// handlers instead of the `startup_error_router` fallback. The settings baked -/// into the binary carry placeholder secrets that `get_settings()` rejects, -/// which would otherwise turn every route into a startup error page. +/// handlers. The settings baked into the binary carry placeholder secrets that +/// `get_settings()` rejects, which would otherwise fail startup. fn make_router() -> RouterService { let settings = trusted_server_core::settings::Settings::from_toml( r#" @@ -95,10 +114,117 @@ fn make_router() -> RouterService { } #[test] -fn routes_build_without_panic() { - // build_state() may fail (no real settings in CI) — startup_error_router - // is the fallback. Either way, routes() must not panic. - let _router = TrustedServerApp::routes(); +fn routes_report_missing_worker_config_as_internal_error() { + // Host targets have no Worker config binding, so startup fails and + // `routes()` returns the error EdgeZero answers with. + let Err(error) = TrustedServerApp::routes() else { + panic!("should fail startup without a Worker config binding"); + }; + + assert_eq!( + error.status(), + StatusCode::INTERNAL_SERVER_ERROR, + "should map the configuration error to a 500" + ); +} + +// --------------------------------------------------------------------------- +// Synthesized headers on EdgeZero's canonical responses +// --------------------------------------------------------------------------- + +/// Dispatches one request through the app `run_app` assembles, so the +/// policies `TrustedServerApp::configure` installs apply, and returns the +/// response head as egress would write it. +async fn dispatch_through_app(method: Method, target: &str) -> Response { + let app = App::from_router::(test_router(), CLOUDFLARE_PLATFORM) + .expect("should assemble the app with its edge policies"); + let request = request_builder() + .method(method) + .uri(target) + .body(Body::empty()) + .expect("should build request"); + let outcome = app + .dispatch_ingress( + request, + app.monotonic_now(), + IngressHeadAccounting::HostManaged, + IngressFraming::HostManaged, + ) + .await + .expect("should admit the request"); + let envelope = match outcome { + IngressDispatchOutcome::Response(envelope) => Some(envelope), + _ => None, + } + .expect("should answer with a response"); + let (prepared, _policy, mut attempt, clock) = envelope + .begin(ResponseEgressFramingMode::ExplicitBufferedLength) + .map_err(|_failure| "response egress failed before conversion") + .expect("should begin response egress"); + assert!(attempt.begin_writing(), "should start writing the response"); + assert!( + attempt.complete(clock.now()), + "should complete the response" + ); + prepared.into_response() +} + +/// Returns the git version compiled into this build. +/// +/// # Panics +/// +/// Panics when the build has none, so a version assertion cannot pass on an +/// absent header. +fn compiled_git_version() -> &'static str { + TS_GIT_VERSION.expect("should compile a git version into the test build") +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn canonical_not_found_carries_x_ts_version() { + // The publisher fallback claims every origin-form path, so only an + // asterisk-form target has no route and gets EdgeZero's canonical 404. + let response = dispatch_through_app(Method::OPTIONS, "*").await; + + assert_eq!( + response.status(), + StatusCode::NOT_FOUND, + "should answer an unrouted target with the canonical 404" + ); + assert_eq!( + response + .headers() + .get("x-ts-version") + .and_then(|value| value.to_str().ok()), + Some(compiled_git_version()), + "should add the compiled-in git version to the canonical 404" + ); +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn canonical_method_not_allowed_carries_allow_and_x_ts_version() { + let response = dispatch_through_app(Method::TRACE, "/auction").await; + + assert_eq!( + response.status(), + StatusCode::METHOD_NOT_ALLOWED, + "should answer an unrouted method on a known path with the canonical 405" + ); + assert_eq!( + response + .headers() + .get(header::ALLOW) + .and_then(|value| value.to_str().ok()), + Some("DELETE, GET, HEAD, OPTIONS, PATCH, POST, PUT"), + "should list the path's methods in sorted order" + ); + assert_eq!( + response + .headers() + .get("x-ts-version") + .and_then(|value| value.to_str().ok()), + Some(compiled_git_version()), + "should add the compiled-in git version to the canonical 405" + ); } // --------------------------------------------------------------------------- @@ -686,14 +812,12 @@ async fn tsjs_route_prefix_is_handled_not_5xx() { /// Fastly adapter alone and not by this test. #[tokio::test(flavor = "multi_thread", worker_threads = 2)] async fn nextjs_auction_output_holds_until_the_structural_body_close() { - use std::sync::Arc; - use trusted_server_core::test_support::nextjs_auction; - let client = Arc::new(nextjs_auction::NextJsAuctionOrigin::default()); + let client = nextjs_auction::NextJsAuctionOrigin::default(); let router = TrustedServerApp::routes_with_settings_and_services( nextjs_auction::settings(), - nextjs_auction::services(Arc::clone(&client)), + nextjs_auction::services(&client), ) .expect("should build router with fixture services"); @@ -746,3 +870,224 @@ async fn nextjs_auction_output_holds_until_the_structural_body_close() { "should not leak generated placeholders: {html}" ); } + +// --------------------------------------------------------------------------- +// Publisher origin through EdgeZero's client +// --------------------------------------------------------------------------- + +/// Origin of the route test settings. +const ORIGIN: &str = "https://origin.test-publisher.example.com"; + +/// Cloudflare's origin input cap with the default 16 MiB `max_buffered_body_bytes`. +const CLOUDFLARE_INPUT_CAP: usize = 10 * 1024 * 1024; + +/// Sends `req` through `router` with `origin` as the request's outbound client. +async fn route_to_origin( + router: RouterService, + mut req: Request, + origin: &ScriptedHttpClient, +) -> Response { + insert_outbound_client(&mut req, HttpClient::with_client(origin.clone())); + router.oneshot(req).await.expect("should route request") +} + +/// A publisher GET for `path` on the test publisher host. +fn publisher_get(path: &str) -> Request { + request_builder() + .method(Method::GET) + .uri(format!("https://test-publisher.example.com{path}")) + .header(header::HOST, "test-publisher.example.com") + .body(Body::empty()) + .expect("should build publisher request") +} + +/// An identity HTML document exactly `len` bytes long. +fn html_document(len: usize) -> Vec { + let mut document = b"".to_vec(); + document.resize(len - b"".len(), b'a'); + document.extend_from_slice(b""); + document +} + +/// The coding the scripted origin serves a body in. +#[derive(Clone, Copy, Debug)] +enum OriginCoding { + Identity, + Gzip, +} + +/// A 200 origin reply carrying `body` with `content_type`, coded as `coding`. +fn origin_reply(content_type: &str, body: &[u8], coding: OriginCoding) -> ScriptedReply { + let reply = ScriptedReply::status(StatusCode::OK).header("content-type", content_type); + match coding { + OriginCoding::Identity => reply.body(body.to_vec()), + OriginCoding::Gzip => reply.header("content-encoding", "gzip").body(gzip(body)), + } +} + +/// Asserts that an origin body that decodes to [`CLOUDFLARE_INPUT_CAP`] bytes +/// is served as identity and one byte more fails with 502 before egress. +async fn assert_cloudflare_input_cap( + content_type: &str, + body: fn(usize) -> Vec, + coding: OriginCoding, +) { + let origin = ScriptedHttpClient::new(&TestClock::new()); + for (path, len) in [ + ("/at-cap", CLOUDFLARE_INPUT_CAP), + ("/over-cap", CLOUDFLARE_INPUT_CAP + 1), + ] { + origin.respond_to( + Method::GET, + &format!("{ORIGIN}{path}"), + origin_reply(content_type, &body(len), coding), + ); + } + + let at_cap = route_to_origin(test_router(), publisher_get("/at-cap"), &origin).await; + assert_eq!( + ( + at_cap.status(), + at_cap.headers().get(header::CONTENT_ENCODING) + ), + (StatusCode::OK, None), + "should serve a {coding:?} {content_type} body at the cap as identity" + ); + let served = at_cap + .into_body() + .into_bytes() + .expect("should buffer the at-cap body"); + assert!( + served.len() >= CLOUDFLARE_INPUT_CAP, + "should serve the whole {coding:?} {content_type} body at the cap, got {} bytes", + served.len() + ); + + let over_cap = route_to_origin(test_router(), publisher_get("/over-cap"), &origin).await; + assert_eq!( + over_cap.status(), + StatusCode::BAD_GATEWAY, + "should fail a {coding:?} {content_type} body one byte over the cap with 502" + ); +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn raw_body_at_cap_passes_and_one_over_fails_502_cloudflare() { + assert_cloudflare_input_cap( + "application/octet-stream", + |len| vec![b'x'; len], + OriginCoding::Identity, + ) + .await; +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn identity_html_at_cap_passes_and_one_over_fails_502_cloudflare() { + assert_cloudflare_input_cap("text/html", html_document, OriginCoding::Identity).await; +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn gzip_raw_body_decoded_at_cap_passes_and_one_over_fails_502_cloudflare() { + assert_cloudflare_input_cap( + "application/octet-stream", + |len| vec![b'x'; len], + OriginCoding::Gzip, + ) + .await; +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn gzip_html_decoded_at_cap_passes_and_one_over_fails_502_cloudflare() { + assert_cloudflare_input_cap("text/html", html_document, OriginCoding::Gzip).await; +} + +/// Serves `path` from a gzip origin to a client that sends no `Accept-Encoding`. +async fn fetch_gzip_origin(path: &str, content_type: &str, body: &[u8]) -> (Response, Vec) { + let origin = ScriptedHttpClient::new(&TestClock::new()); + origin.respond_to( + Method::GET, + &format!("{ORIGIN}{path}"), + origin_reply(content_type, body, OriginCoding::Gzip), + ); + let request = publisher_get(path); + assert!( + !request.headers().contains_key(header::ACCEPT_ENCODING), + "should send no Accept-Encoding" + ); + let mut response = route_to_origin(test_router(), request, &origin).await; + let served = std::mem::replace(response.body_mut(), Body::empty()) + .into_bytes() + .expect("should buffer the served body"); + (response, served.to_vec()) +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn gzip_origin_html_reaches_a_client_without_accept_encoding_as_identity_cloudflare() { + let page = b"Example

example

"; + + let (response, served) = + fetch_gzip_origin("/gzip-page", "text/html; charset=utf-8", page).await; + + assert_eq!( + ( + response.status(), + response.headers().get(header::CONTENT_ENCODING) + ), + (StatusCode::OK, None), + "should serve the processed page as identity" + ); + let html = String::from_utf8(served).expect("should serve UTF-8 HTML"); + assert!( + html.contains("/static/tsjs=") && html.contains("

example

"), + "should serve the decoded page with the TSJS tag: {html}" + ); +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn gzip_origin_raw_body_reaches_a_client_without_accept_encoding_as_identity_cloudflare() { + let bytes = b"example raw origin bytes"; + + let (response, served) = + fetch_gzip_origin("/asset.bin", "application/octet-stream", bytes).await; + + assert_eq!( + ( + response.status(), + response.headers().get(header::CONTENT_ENCODING), + served.as_slice() + ), + (StatusCode::OK, None, bytes.as_slice()), + "should relay the raw body decoded, as identity" + ); +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn publisher_origin_streams_its_decoded_response_under_the_input_cap_cloudflare() { + let origin = ScriptedHttpClient::new(&TestClock::new()); + origin.respond_to( + Method::GET, + &format!("{ORIGIN}/page"), + origin_reply("text/html", &html_document(64), OriginCoding::Gzip), + ); + + let response = route_to_origin(test_router(), publisher_get("/page"), &origin).await; + + assert_eq!(response.status(), StatusCode::OK, "should serve the page"); + let requests = origin.requests(); + let [forwarded] = requests.as_slice() else { + panic!("should send one origin request, got {requests:?}"); + }; + assert_eq!( + ( + forwarded.response_mode, + forwarded.response_decoding, + forwarded.max_encoded_response_bytes, + ), + ( + ResponseMode::Streamed, + ResponseDecoding::Decode, + Some(u64::try_from(CLOUDFLARE_INPUT_CAP).expect("should fit the cap in u64")), + ), + "should stream the decoded origin response and cap its encoded bytes" + ); +} diff --git a/crates/trusted-server-adapter-cloudflare/tests/runtime/deadline.rs b/crates/trusted-server-adapter-cloudflare/tests/runtime/deadline.rs index d3875d76b1..449a7a2f68 100644 --- a/crates/trusted-server-adapter-cloudflare/tests/runtime/deadline.rs +++ b/crates/trusted-server-adapter-cloudflare/tests/runtime/deadline.rs @@ -1,14 +1,29 @@ -//! Local runtime fixture exercising the production HTTP client implementation. +//! Local runtime fixture exercising `EdgeZero`'s Cloudflare HTTP client. +//! +//! `scripts/test-worker-auction-deadlines.mjs` calls it with a delayed +//! upstream. It starts one bidder-shaped request in a batch, sleeps for the +//! simulated publisher work, then collects the batch at the auction cutoff. #[cfg(target_arch = "wasm32")] -#[allow(dead_code)] -#[path = "../../src/platform.rs"] -mod platform; +use std::time::Duration; #[cfg(target_arch = "wasm32")] -use std::time::Duration; +use edgezero_adapter_cloudflare::outbound::CloudflareOutboundClient; +#[cfg(target_arch = "wasm32")] +use edgezero_core::error::EdgeError; +#[cfg(target_arch = "wasm32")] +use edgezero_core::http::Method; #[cfg(target_arch = "wasm32")] -use trusted_server_core::platform::{PlatformHttpClient as _, PlatformHttpRequest}; +use edgezero_core::outbound::{ + HttpClient, OutboundBatchTermination, OutboundCachePolicy, OutboundCollectedSlot, + ResponseDecoding, +}; +#[cfg(target_arch = "wasm32")] +use edgezero_core::time::Deadline; +#[cfg(target_arch = "wasm32")] +use trusted_server_core::outbound::{ + BUFFERED_TIMEOUT, DEFAULT_BUFFERED_RESPONSE_BYTES, buffered, outbound_request, +}; #[cfg(target_arch = "wasm32")] #[worker::event(fetch)] @@ -25,9 +40,7 @@ async fn main( } #[cfg(target_arch = "wasm32")] -async fn exercise( - uri: &str, -) -> Result> { +async fn exercise(uri: &str) -> Result { let parameter = |name: &str| { uri.split_once('?').and_then(|(_, query)| { query @@ -46,30 +59,41 @@ async fn exercise( .expect("should supply origin delay") .parse() .expect("should parse origin delay"); - let client = platform::CloudflareHttpClient; - let request = edgezero_core::http::request_builder() - .uri(upstream) - .body(edgezero_core::body::Body::empty()) - .expect("should construct fixture request"); + let client = HttpClient::with_client(CloudflareOutboundClient::new()); + let request = buffered( + outbound_request(Method::GET, upstream)?, + BUFFERED_TIMEOUT, + DEFAULT_BUFFERED_RESPONSE_BYTES, + ResponseDecoding::Decode, + ) + .cache_policy(OutboundCachePolicy::Bypass); let started = web_time::Instant::now(); - let pending = client - .send_async(PlatformHttpRequest::new(request, "fixture")) - .await?; + let batch = client.start_batch_until( + vec![request], + Deadline::after(Duration::from_millis(budget_ms)), + ); worker::Delay::from(Duration::from_millis(origin_ms)).await; - let collected = client - .select_until(vec![pending], started + Duration::from_millis(budget_ms)) - .await?; - let status = match collected { - Some(result) => { - let response = result.ready?; + let (slots, termination) = batch + .collect() + .await + .map_err(|failure| failure.into_parts().0)? + .into_parts(); + let status = match (slots.into_iter().next(), termination) { + (Some(OutboundCollectedSlot::Resolved(result)), _) => { + let response = result.outcome?; assert_eq!( - response.response.body().as_bytes(), + response.body().as_bytes(), Some(b"okay".as_slice()), "should retain the complete nonempty upstream body" ); "complete" } - None => "timeout", + (Some(OutboundCollectedSlot::Unresolved), OutboundBatchTermination::Cutoff) => "timeout", + (slot, termination) => { + return Err(EdgeError::internal(std::io::Error::other(format!( + "unexpected batch outcome {slot:?} with {termination:?}" + )))); + } }; Ok(format!("{status}:{}", started.elapsed().as_millis())) } diff --git a/crates/trusted-server-adapter-cloudflare/wrangler.ci.toml b/crates/trusted-server-adapter-cloudflare/wrangler.ci.toml index 9992db7123..5018ed7979 100644 --- a/crates/trusted-server-adapter-cloudflare/wrangler.ci.toml +++ b/crates/trusted-server-adapter-cloudflare/wrangler.ci.toml @@ -3,7 +3,8 @@ main = "build/index.js" compatibility_date = "2024-09-23" # Keep in sync with wrangler.toml. `cache_option_enabled` is required for the # outbound `CacheMode::NoStore` cache bypass under this compatibility date. -compatibility_flags = ["nodejs_compat", "cache_option_enabled"] +# `enable_weak_ref` and `enable_request_signal` are required by EdgeZero. +compatibility_flags = ["nodejs_compat", "cache_option_enabled", "enable_weak_ref", "enable_request_signal"] # No [build] section — bundle is pre-built in CI; wrangler dev must not rebuild. [[kv_namespaces]] diff --git a/crates/trusted-server-adapter-cloudflare/wrangler.toml b/crates/trusted-server-adapter-cloudflare/wrangler.toml index f3b5ef6aa3..9c9d090e8e 100644 --- a/crates/trusted-server-adapter-cloudflare/wrangler.toml +++ b/crates/trusted-server-adapter-cloudflare/wrangler.toml @@ -9,7 +9,9 @@ compatibility_date = "2024-09-23" # the runtime throws when a subrequest sets the `cache` field on `RequestInit`. # The outbound HTTP client sets `CacheMode::NoStore` for cache-bypass requests # (auction-eligible publisher navigations), so this is a hard requirement. -compatibility_flags = ["nodejs_compat", "cache_option_enabled"] +# EdgeZero's bounded config and KV reads need `enable_weak_ref`, and its client +# disconnect observation needs `enable_request_signal`. +compatibility_flags = ["nodejs_compat", "cache_option_enabled", "enable_weak_ref", "enable_request_signal"] [build] command = "bash build.sh" diff --git a/crates/trusted-server-adapter-fastly/Cargo.toml b/crates/trusted-server-adapter-fastly/Cargo.toml index 6c7121d0d6..57cb492f29 100644 --- a/crates/trusted-server-adapter-fastly/Cargo.toml +++ b/crates/trusted-server-adapter-fastly/Cargo.toml @@ -28,8 +28,6 @@ edgezero-core = { workspace = true } error-stack = { workspace = true } esi = { workspace = true } fastly = { workspace = true } -# The SDK keeps timed body readiness private; auction deadlines use the ABI. -fastly-sys = { workspace = true } fern = { workspace = true } futures = { workspace = true } log = { workspace = true } @@ -37,12 +35,11 @@ log-fastly = { workspace = true } serde = { workspace = true } serde_json = { workspace = true } trusted-server-core = { workspace = true } -url = { workspace = true } urlencoding = { workspace = true } -web-time = { workspace = true } [dev-dependencies] bytes = { workspace = true } +edgezero-adapter-fastly = { workspace = true, features = ["fastly", "test-utils"] } edgezero-core = { workspace = true, features = ["test-utils"] } toml = { workspace = true } trusted-server-core = { workspace = true, features = ["test-utils"] } diff --git a/crates/trusted-server-adapter-fastly/README.md b/crates/trusted-server-adapter-fastly/README.md index 24558148ae..52700dd7ac 100644 --- a/crates/trusted-server-adapter-fastly/README.md +++ b/crates/trusted-server-adapter-fastly/README.md @@ -3,12 +3,12 @@ Production Fastly Compute adapter for Trusted Server, targeting `wasm32-wasip1`. -This crate owns the Fastly entry point, dynamic backend construction, edge -cache behavior, config and secret-store access, EC KV operations, rate limits, -streaming template assembly, and Fastly-only Tinybird auction emission. Its -`/health` response is served before application construction, so liveness does -not prove that configuration loaded. Platform-neutral routing, settings, -auctions, integrations, and rewrites remain in `trusted-server-core`. +This crate owns the Fastly entry point, edge cache behavior, config and +secret-store access, EC KV operations, rate limits, streaming template +assembly, and Fastly-only Tinybird auction emission. Its `/health` response is +served before application construction, so liveness does not prove that +configuration loaded. Platform-neutral routing, settings, auctions, +integrations, and rewrites remain in `trusted-server-core`. Build and test from the repository root: diff --git a/crates/trusted-server-adapter-fastly/src/app.rs b/crates/trusted-server-adapter-fastly/src/app.rs index 69675e2669..1f370ef99a 100644 --- a/crates/trusted-server-adapter-fastly/src/app.rs +++ b/crates/trusted-server-adapter-fastly/src/app.rs @@ -1,13 +1,12 @@ //! Full `EdgeZero` application wiring for Trusted Server. //! //! Registers all routes for Trusted Server into a -//! [`RouterService`]. On successful startup, attaches [`FinalizeResponseMiddleware`] -//! (outermost) and [`AuthMiddleware`] (inner). When startup fails, -//! [`startup_error_router`] returns a bare router without middleware. -//! Builds the [`AppState`] once per Wasm instance. +//! [`RouterService`], with [`FinalizeResponseMiddleware`] (outermost) and +//! [`AuthMiddleware`] (inner). [`assemble_app`] builds the [`AppState`] and +//! the [`App`] once per sandbox through [`App::from_router`]. //! //! `EdgeZero`'s current Fastly request context exposes client IP but not TLS -//! protocol or cipher metadata. `edgezero_main` injects a trusted `fastly-ssl` +//! protocol or cipher metadata. The entry point injects a trusted `fastly-ssl` //! header after stripping client-spoofable headers, so [`detect_request_scheme`] //! in `http_util` can still derive the correct scheme for HTTPS traffic. //! It also captures the full [`ClientInfo`] (TLS, JA4, H2 fingerprint, server @@ -43,8 +42,8 @@ //! //! > **Note:** Methods not in the list above (e.g. `TRACE`, `CONNECT`, WebDAV verbs) return a //! > router-level 405. Legacy routing proxied *every* method through to the publisher origin. -//! > This is a known intentional restriction of the EdgeZero router; the entry-point -//! > `apply_finalize_headers` call in `main.rs` still adds TS headers to those 405 responses. +//! > This is a known intentional restriction of the EdgeZero router; the entry point's +//! > `finalize` step still adds TS headers to those 405 responses. //! //! # EC identity lifecycle //! @@ -60,8 +59,8 @@ //! [`EcContext`], [`KvIdentityGraph`], and [`PartnerRegistry`] inputs as //! legacy; the publisher fallback generates EC IDs for browser navigations. //! - Handlers attach an [`EcFinalizeState`] to the response via extensions; -//! `edgezero_main` pops it and runs `ec_finalize_response` plus the -//! pull-sync hook on the converted fastly response before sending. +//! the entry point's `finalize` step pops it and runs `ec_finalize_response`, +//! and post-delivery work runs pull sync once the response is delivered. //! //! ## Intentional deviations from legacy //! @@ -79,33 +78,35 @@ //! //! # Startup error handling //! -//! When [`build_state`] fails, [`startup_error_router`] returns a minimal router -//! that responds to all routes with the startup error. This router does **not** -//! attach middleware. Startup-error responses may still receive entry-point -//! finalization (geo and TS headers) when settings can be reloaded via -//! [`load_settings_from_config_store`]; if settings loading itself fails, they -//! are returned without geo or TS headers. +//! [`assemble_app`] and [`Hooks::routes`] return a settings or state failure +//! as the [`EdgeError`] from [`startup_error`], and [`App::from_router`] +//! returns a `configure` failure. The entry point answers either with +//! `EdgeZero`'s assembly-error response, which carries no TS headers, and +//! retains nothing, so the next request retries. +use std::fmt; use std::sync::Arc; use crate::rate_limiter::{FastlyRateLimiter, RATE_COUNTER_NAME}; +use edgezero_adapter_fastly::FASTLY_PLATFORM; use edgezero_adapter_fastly::context::FastlyRequestContext; use edgezero_core::app::{App, Hooks, StoreMetadata, StoresMetadata}; use edgezero_core::context::RequestContext; use edgezero_core::error::EdgeError; use edgezero_core::http::{ - HandlerFuture, HeaderValue, Method, Request, Response, StatusCode, header, + HandlerFuture, HeaderValue, Method, Request, RequestParts, Response, StatusCode, header, }; +use edgezero_core::outbound::ResponseDecoding; use edgezero_core::router::RouterService; use error_stack::Report; -use trusted_server_core::auction::AuctionTelemetrySink; use trusted_server_core::auction::endpoints::handle_auction; +use trusted_server_core::auction::plan::validate_fastly_backend_bound; use trusted_server_core::auction::{ AuctionOrchestrator, build_orchestrator_with_plan, compile_auction_plan, }; use trusted_server_core::cache_policy::EdgeCacheHeader; use trusted_server_core::config_payload::DEFAULT_SECRET_STORE_ID; -use trusted_server_core::constants::{COOKIE_SHAREDID, COOKIE_TS_EIDS}; +use trusted_server_core::constants::{COOKIE_SHAREDID, COOKIE_TS_EIDS, ENV_FASTLY_SERVICE_VERSION}; use trusted_server_core::ec::admin::{ deny_admin_diagnostic_fallback, handle_admin_ec_lookup, handle_admin_eids_lookup, }; @@ -116,23 +117,25 @@ use trusted_server_core::ec::identify::{cors_preflight_identify, handle_identify use trusted_server_core::ec::kv::KvIdentityGraph; use trusted_server_core::ec::registry::PartnerRegistry; use trusted_server_core::ec::{EcContext, EidSyncSource}; -use trusted_server_core::error::{IntoHttpResponse as _, TrustedServerError}; +use trusted_server_core::edge_app::install_edge_policies; +use trusted_server_core::error::{TrustedServerError, http_error, startup_error}; use trusted_server_core::http_util::is_navigation_request; use trusted_server_core::integrations::{ IntegrationRegistry, ProxyDispatchInput, RequestFilterEffects, RequestFilterRegistryInput, RequestFilterRegistryOutcome, }; +use trusted_server_core::outbound::request_outbound_client; use trusted_server_core::platform::{ - ClientInfo, GeoInfo, PlatformGeo as _, PlatformKvStore, RuntimeServices, StoreName, + ClientInfo, GeoInfo, PlatformKvStore, RuntimeServices, StoreName, }; use trusted_server_core::proxy::{ AssetProxyCachePolicy, handle_asset_proxy_request, handle_first_party_click, handle_first_party_proxy, handle_first_party_proxy_rebuild, handle_first_party_proxy_sign, }; use trusted_server_core::publisher::{ - AuctionDispatch, PAGE_BIDS_LEGACY_PATH, PAGE_BIDS_PATH, handle_page_bids, - handle_publisher_request, handle_tsjs_dynamic, page_bids_preflight_denied, - publisher_response_into_streaming_response, + AuctionDispatch, PAGE_BIDS_LEGACY_PATH, PAGE_BIDS_PATH, PublisherOriginOptions, + handle_page_bids, handle_publisher_request, handle_tsjs_dynamic, page_bids_preflight_denied, + publisher_owner_error, publisher_response_into_streaming_response, }; use trusted_server_core::request_signing::{ handle_deactivate_key, handle_rotate_key, handle_trusted_server_discovery, @@ -143,11 +146,12 @@ use trusted_server_core::settings_data::{DEFAULT_CONFIG_STORE_ID, get_settings_f use trusted_server_core::tester_cookie::{handle_clear_tester, handle_set_tester}; use trusted_server_core::trace::{TraceMetadata, TracePreDispatchHook}; -use crate::middleware::{AuthMiddleware, FinalizeResponseMiddleware}; +use crate::middleware::{AuthMiddleware, FinalizeResponseMiddleware, version_synthesized_headers}; use crate::platform::{ - FastlyPlatformBackend, FastlyPlatformConfigStore, FastlyPlatformGeo, FastlyPlatformHttpClient, - FastlyPlatformSecretStore, UnavailableKvStore, + FastlyPlatformConfigStore, FastlyPlatformSecretStore, RequestGeo, UnavailableKvStore, }; +use crate::sandbox::RetainedApp; +use crate::tinybird::{QueueingTelemetrySink, TelemetryQueue}; // --------------------------------------------------------------------------- // AppState @@ -189,27 +193,14 @@ pub(crate) struct AppState { pub(crate) orchestrator: Arc, pub(crate) registry: Arc, pub(crate) default_kv_store: Arc, - pub(crate) auction_telemetry_sink: Arc, -} - -/// Build the application state, loading settings and constructing all per-application components. -/// -/// # Errors -/// -/// Returns an error when settings, the auction orchestrator, or the integration -/// registry fail to initialise. -pub(crate) fn build_state( - stores: &RuntimeStoreConfig, -) -> Result, Report> { - build_state_from_settings(load_settings_from_config_store(stores)?) } pub(crate) fn load_settings_from_config_store( stores: &RuntimeStoreConfig, ) -> Result> { get_settings_from_config_store( - &FastlyPlatformConfigStore, - &FastlyPlatformSecretStore, + &FastlyPlatformConfigStore::read_only(), + &FastlyPlatformSecretStore::read_only(), &stores.config_store_name, &stores.config_key, &stores.secret_store_name, @@ -219,14 +210,11 @@ pub(crate) fn load_settings_from_config_store( pub(crate) fn build_state_from_settings( settings: Settings, ) -> Result, Report> { - warn_if_certificate_check_disabled(&settings); - let plan = Arc::new(compile_auction_plan(&settings)?); - plan.validate_for_target(trusted_server_core::platform::AuctionTargetId::Fastly)?; + validate_fastly_backend_bound(&plan)?; let orchestrator = build_orchestrator_with_plan(Arc::clone(&plan), &settings)?; let registry = IntegrationRegistry::with_plan(&settings, plan)?; - let auction_telemetry_sink = crate::tinybird::auction_sink_from_settings(&settings); let default_kv_store = Arc::new(UnavailableKvStore) as Arc; Ok(Arc::new(AppState { @@ -234,61 +222,96 @@ pub(crate) fn build_state_from_settings( orchestrator: Arc::new(orchestrator), registry: Arc::new(registry), default_kv_store, - auction_telemetry_sink, })) } -fn warn_if_certificate_check_disabled(settings: &Settings) { - if !settings.proxy.certificate_check { - log::warn!( - "INSECURE: proxy.certificate_check is disabled; HTTPS origin certificate verification is disabled" - ); - } -} - // --------------------------------------------------------------------------- // Per-request RuntimeServices // --------------------------------------------------------------------------- /// Construct per-request [`RuntimeServices`] from the `EdgeZero` request context. /// -/// Prefers the full [`ClientInfo`] captured by `edgezero_main` from the original +/// Prefers the full [`ClientInfo`] captured by the entry point from the original /// `FastlyRequest` (TLS protocol/cipher, JA4, H2 fingerprint, and server /// hostname/region) and stored in the request extensions — the metadata /// integration bot protection (e.g. `DataDome`) serializes, which the /// reconstructed `EdgeZero` request cannot expose. Falls back to the client IP /// from the dispatch-inserted [`FastlyRequestContext`] when the extension is /// absent (e.g. tests that dispatch without the entry point). Scheme detection -/// continues to rely on the trusted `fastly-ssl` header injected by -/// `edgezero_main` after sanitization. -fn build_per_request_services(state: &AppState, ctx: &RequestContext) -> RuntimeServices { +/// continues to rely on the trusted `fastly-ssl` header injected by the entry +/// point after sanitization. +/// +/// The outbound client is the one `EdgeZero`'s Fastly request conversion +/// installs in the request context, with the proxy marker turned off. The +/// config and secret stores make their management API writes through it. The +/// clock is the context's monotonic clock. Geo answers from the request's +/// [`RequestGeo`], the lookup the entry point ran once. +/// +/// With Tinybird enabled, the auction telemetry sink queues NDJSON on the +/// request's [`TelemetryQueue`], which the entry point inserts and sends after +/// delivery. Otherwise the sink is the no-op default. +/// +/// # Errors +/// +/// Returns an internal [`EdgeError`] when the request carries no outbound +/// client, no [`RequestGeo`], or no telemetry queue while Tinybird is enabled. +fn build_per_request_services( + state: &AppState, + ctx: &RequestContext, +) -> Result { // No cooperative timer is installed: Fastly host calls block. EOF consumers // must bound their own transport calls using the remaining finish budget. let client_info = ctx - .request() .extensions() .get::() .cloned() .unwrap_or_else(|| ClientInfo { - client_ip: FastlyRequestContext::get(ctx.request()).and_then(|c| c.client_ip), + client_ip: ctx + .extensions() + .get::() + .and_then(|c| c.client_ip), ..ClientInfo::default() }); + let outbound_client = request_outbound_client(ctx)?; + let request_geo = ctx + .extensions() + .get::() + .cloned() + .ok_or_else(|| EdgeError::internal(std::io::Error::other("request has no geo")))?; - RuntimeServices::builder() - .config_store(Arc::new(FastlyPlatformConfigStore)) - .secret_store(Arc::new(FastlyPlatformSecretStore)) + let mut services = RuntimeServices::builder(); + if state.settings.tinybird.enabled { + let queue = ctx + .extensions() + .get::() + .cloned() + .ok_or_else(|| { + EdgeError::internal(std::io::Error::other("request has no telemetry queue")) + })?; + services = services.auction_telemetry_sink(Arc::new(QueueingTelemetrySink::new( + &state.settings.tinybird, + queue, + ))); + } + + Ok(services + .config_store(Arc::new(FastlyPlatformConfigStore::with_outbound_client( + outbound_client.clone(), + ))) + .secret_store(Arc::new(FastlyPlatformSecretStore::with_outbound_client( + outbound_client.clone(), + ))) .kv_store(Arc::clone(&state.default_kv_store)) // Constructed unconditionally, but only read when the assembly mode is a // shared-template one — which defaults to Inline, so this is inert until an // operator opts in. .template_cache(Arc::new(crate::template_cache::FastlyTemplateCache::new())) .template_assembler(Arc::new(crate::esi_assembly::FastlyTemplateAssembler)) - .backend(Arc::new(FastlyPlatformBackend)) - .http_client(Arc::new(FastlyPlatformHttpClient)) - .geo(Arc::new(FastlyPlatformGeo)) - .auction_telemetry_sink(Arc::clone(&state.auction_telemetry_sink)) + .outbound_client(outbound_client) + .clock(ctx.monotonic_clock()) + .geo(Arc::new(request_geo)) .client_info(client_info) - .build() + .build()) } fn publisher_fallback_methods() -> [Method; 7] { @@ -314,24 +337,21 @@ fn uses_dynamic_tsjs_fallback(method: &Method, path: &str) -> bool { /// EC state threaded from route handlers to the `main.rs` entry point via /// response extensions. /// -/// `edgezero_main` pops this from the response after dispatch and runs -/// [`trusted_server_core::ec::finalize::ec_finalize_response`] plus the -/// pull-sync hook on the converted fastly response — the same EC response -/// lifecycle the legacy path drives through `RouteResult`. +/// The entry point's `finalize` step pops this from the routed response and +/// runs [`trusted_server_core::ec::finalize::ec_finalize_response`], and +/// post-delivery work runs pull sync from it, the same EC response lifecycle +/// the legacy path drives through `RouteResult`. #[derive(Clone)] pub(crate) struct EcFinalizeState { pub(crate) ec_context: EcContext, /// Whether EC finalization may write to the KV identity graph. /// `KvIdentityGraph` wraps a non-`Sync` `dyn EcKvStore` and cannot ride - /// in response extensions, so `edgezero_main` rebuilds the graph from + /// in response extensions, so the entry point rebuilds the graph from /// settings when this is set. pub(crate) use_finalize_kv: bool, pub(crate) eids_cookie: Option, pub(crate) sharedid_cookie: Option, pub(crate) is_real_browser: bool, - /// Per-request services carried to the entry point so the pull-sync - /// dispatcher can reuse the same platform HTTP client. - pub(crate) services: RuntimeServices, } /// Per-request EC identity state built before dispatch, mirroring the @@ -345,7 +365,6 @@ struct EcRequestState { eids_cookie: Option, sharedid_cookie: Option, is_real_browser: bool, - services: RuntimeServices, /// Geo lookup result reused by the pre-route request-filter step so it does /// not repeat the lookup the legacy path also shares between EC setup and /// filtering. @@ -365,7 +384,6 @@ impl EcRequestState { eids_cookie: self.eids_cookie, sharedid_cookie: self.sharedid_cookie, is_real_browser: self.is_real_browser, - services: self.services, } } } @@ -388,7 +406,7 @@ fn derive_request_device_signals(req: &Request) -> DeviceSignals { /// Returns the device signals for an `EdgeZero` request. /// -/// `edgezero_main` derives [`DeviceSignals`] from the original `FastlyRequest` +/// The entry point derives [`DeviceSignals`] from the original `FastlyRequest` /// — the only place Fastly's TLS JA4 and HTTP/2 fingerprint accessors return /// real values — and stores them in the request extensions. Reading them back /// here preserves the bot gate's browser classification, which the `EdgeZero` @@ -467,7 +485,6 @@ fn build_ec_request_state( eids_cookie, sharedid_cookie, is_real_browser, - services: services.clone(), geo_info, setup_error, } @@ -528,8 +545,9 @@ async fn run_pre_route_filters( } /// Attaches the EC finalize state and any non-empty request-filter response -/// effects to a dispatched response via extensions, so `edgezero_main` can run -/// EC finalization and apply the filter effects after it (legacy ordering). +/// effects to a dispatched response via extensions, so the entry point's +/// `finalize` step can run EC finalization and apply the filter effects after it +/// (legacy ordering). fn attach_dispatch_extensions( mut response: Response, ec: EcRequestState, @@ -547,15 +565,15 @@ async fn execute_named( ctx: RequestContext, handler: NamedRouteHandler, ) -> Result { - let services = build_per_request_services(&state, &ctx); - let mut req = ctx.into_request(); + let services = build_per_request_services(&state, &ctx)?; + let mut req = ctx.into_request()?; // S2S batch sync uses Bearer auth (not EC cookies), so it skips EC // context creation entirely — mirroring the dedicated early arm in the // legacy route_request. Batch-sync also skips request filters, matching // legacy, which returns before the filter step for this route. if matches!(handler, NamedRouteHandler::BatchSync) { - return Ok(run_batch_sync(&state, &services, req)); + return Ok(run_batch_sync(&state, req).await); } // An operator cache purge is not a reader request: running the EC lifecycle would @@ -640,10 +658,12 @@ async fn run_named_route( handle_trusted_server_discovery(&state.settings, services, req) } NamedRouteHandler::VerifySignature => { - handle_verify_signature(&state.settings, services, req) + handle_verify_signature(&state.settings, services, req).await + } + NamedRouteHandler::RotateKey => handle_rotate_key(&state.settings, services, req).await, + NamedRouteHandler::DeactivateKey => { + handle_deactivate_key(&state.settings, services, req).await } - NamedRouteHandler::RotateKey => handle_rotate_key(&state.settings, services, req), - NamedRouteHandler::DeactivateKey => handle_deactivate_key(&state.settings, services, req), NamedRouteHandler::AdminEcLookup | NamedRouteHandler::AdminEidsLookup => { unreachable!("admin diagnostics should be handled before EC setup") } @@ -738,17 +758,19 @@ async fn run_named_route( /// Handles `POST /_ts/api/v1/batch-sync`, mirroring the legacy arm: identity /// graph + partner registry + rate limiter, with a default EC context for /// response finalization. -fn run_batch_sync(state: &AppState, services: &RuntimeServices, req: Request) -> Response { +async fn run_batch_sync(state: &AppState, req: Request) -> Response { let device_signals = device_signals_for(&req); let is_real_browser = device_signals.looks_like_browser(); let eids_cookie = crate::extract_cookie_value(&req, COOKIE_TS_EIDS); let sharedid_cookie = crate::extract_cookie_value(&req, COOKIE_SHAREDID); - let result = crate::require_identity_graph(&state.settings).and_then(|kv| { + let result = async { + let kv = crate::require_identity_graph(&state.settings)?; let partner_registry = PartnerRegistry::from_config(&state.settings.ec.partners)?; let limiter = FastlyRateLimiter::new(RATE_COUNTER_NAME); - handle_batch_sync(&kv, &partner_registry, &limiter, req) - }); + handle_batch_sync(&kv, &partner_registry, &limiter, req).await + } + .await; let mut response = result.unwrap_or_else(|e| http_error(&e)); // Legacy parity: batch-sync responses still pass through @@ -759,7 +781,6 @@ fn run_batch_sync(state: &AppState, services: &RuntimeServices, req: Request) -> eids_cookie, sharedid_cookie, is_real_browser, - services: services.clone(), }); response } @@ -768,8 +789,8 @@ async fn execute_fallback( state: Arc, ctx: RequestContext, ) -> Result { - let services = build_per_request_services(&state, &ctx); - let req = ctx.into_request(); + let services = build_per_request_services(&state, &ctx)?; + let req = ctx.into_request()?; Ok(dispatch_fallback(&state, &services, req).await) } @@ -882,17 +903,24 @@ async fn dispatch_fallback( &mut ec.ec_context, auction, req, - EdgeCacheHeader::SurrogateControl, + PublisherOriginOptions { + edge_header: EdgeCacheHeader::SurrogateControl, + input_cap: None, + response_decoding: ResponseDecoding::Original, + }, ) .await { - Ok(pub_response) => { + Ok((pub_response, owner)) => { // Origin start succeeded on the sole publisher-page // path. Authorize orphan recovery only for real-browser // document navigations so named routes, integration // proxies, and filter short circuits cannot rotate an // identity. ec.ec_context.set_recovery_eligible(is_publisher_navigation); + // The owner keeps the auction batch and the origin body + // alive until the client body ends: `EdgeZero`'s + // delivery takes it from the body and holds it to the end. publisher_response_into_streaming_response( pub_response, &method, @@ -902,6 +930,7 @@ async fn dispatch_fallback( services.clone(), ) .await + .and_then(|response| owner.attach(response).map_err(publisher_owner_error)) } Err(e) => Err(e), } @@ -931,16 +960,15 @@ fn asset_response_carries_body(method: &Method, status: StatusCode) -> bool { /// `route_request` asset branch. /// /// Proxies the request to the configured asset origin and threads the -/// [`AssetProxyCachePolicy`] out via response extensions so `edgezero_main` -/// can reapply protected cache directives after finalization. EC finalization -/// is intentionally skipped: no [`EcFinalizeState`] is attached, matching the -/// legacy `should_finalize_ec = false` behavior for asset responses. +/// [`AssetProxyCachePolicy`] out via response extensions so the entry point's +/// `finalize` step can reapply protected cache directives after finalization. +/// EC finalization is intentionally skipped: no [`EcFinalizeState`] is +/// attached, matching the legacy `should_finalize_ec = false` behavior for +/// asset responses. /// /// Like legacy `route_request`, asset bodies are streamed straight to the client -/// with no cap: the origin stream is attached to the response and `edgezero_main` -/// commits the headers, then pipes the body chunk-by-chunk via -/// [`fastly::Response::stream_to_client`] and -/// [`stream_asset_body`](trusted_server_core::proxy::stream_asset_body). The +/// with no cap: the origin stream is attached to the response, and `EdgeZero`'s +/// response delivery commits the headers and pumps the body chunk by chunk. The /// origin stream is left untouched here, so large images never materialize in /// the Wasm heap. async fn dispatch_asset_fallback( @@ -959,7 +987,7 @@ async fn dispatch_asset_fallback( let cache_policy = asset_response.cache_policy(); let (mut response, stream_body) = asset_response.into_response_and_body(); - // Attach the origin stream so `edgezero_main` streams it to the + // Attach the origin stream so delivery streams it to the // client. HEAD and bodiless statuses (204, 304) advertise the origin // Content-Length but carry no body, so their stream is dropped to // preserve that header and avoid a length/body mismatch. @@ -996,29 +1024,6 @@ fn attach_request_filter_effects(response: &mut Response, effects: &RequestFilte } } -// --------------------------------------------------------------------------- -// Error helper -// --------------------------------------------------------------------------- - -/// Convert a [`Report`] into an HTTP [`Response`], -/// mirroring [`crate::http_error_response`] exactly. -/// -/// The near-identical function in `main.rs` is intentional: the legacy path -/// uses fastly HTTP types while this path uses `edgezero_core` types. -pub(crate) fn http_error(report: &Report) -> Response { - let root_error = report.current_context(); - log::error!("Error occurred: {:?}", report); - - let body = edgezero_core::body::Body::from(format!("{}\n", root_error.user_message())); - let mut response = Response::new(body); - *response.status_mut() = root_error.status_code(); - response.headers_mut().insert( - header::CONTENT_TYPE, - HeaderValue::from_static("text/plain; charset=utf-8"), - ); - response -} - /// Builds the local `404 Not Found` returned for legacy `/admin/keys/*` /// aliases on the `EdgeZero` path. /// @@ -1036,39 +1041,6 @@ fn legacy_admin_alias_denied() -> Response { response } -// --------------------------------------------------------------------------- -// Startup error fallback -// --------------------------------------------------------------------------- - -/// Returns a [`RouterService`] that responds to every registered route with the startup error. -/// -/// Called when [`build_state`] fails so that request handling degrades to a -/// structured HTTP error response rather than an unrecoverable panic. -fn startup_error_router(e: &Report) -> RouterService { - let message = Arc::new(format!("{}\n", e.current_context().user_message())); - let status = e.current_context().status_code(); - - let make = move |msg: Arc| { - move |_ctx: RequestContext| { - let body = edgezero_core::body::Body::from((*msg).clone()); - let mut resp = Response::new(body); - *resp.status_mut() = status; - resp.headers_mut().insert( - header::CONTENT_TYPE, - HeaderValue::from_static("text/plain; charset=utf-8"), - ); - async move { Ok::(resp) } - } - }; - - let mut router = RouterService::builder(); - for method in publisher_fallback_methods() { - router = router.route("/", method.clone(), make(Arc::clone(&message))); - router = router.route("/{*rest}", method, make(Arc::clone(&message))); - } - router.build() -} - // --------------------------------------------------------------------------- // Route registration // --------------------------------------------------------------------------- @@ -1276,38 +1248,57 @@ fn fallback_route_handler( /// `EdgeZero` [`Hooks`] implementation for the Trusted Server application. pub struct TrustedServerApp; -impl TrustedServerApp { - pub(crate) fn build_app_with_state( - stores: &RuntimeStoreConfig, - ) -> (App, Option>) { - let (router, state) = Self::router_with_state(stores); - let mut app = App::with_name(router, Self::name()); - Self::configure(&mut app); - (app, state) - } - - fn router_with_state(stores: &RuntimeStoreConfig) -> (RouterService, Option>) { - let state = match build_state(stores) { - Ok(state) => state, - Err(ref e) => { - log::error!("failed to build application state: {:?}", e); - return (startup_error_router(e), None); - } - }; +/// Assembles the application from loaded `settings`. +/// +/// Builds the [`AppState`] and its routes, then [`App::from_router`], which +/// applies the name, [`Hooks::configure`], and the platform check exactly once. +/// The entry point retains the result in its sandbox only on success. +/// `log_context` ends the settings or state failure log, so it matches the +/// request that triggered assembly. +/// +/// # Errors +/// +/// Returns the [`EdgeError`] from [`startup_error`] when settings or state +/// fail, or the `configure` error, either of which the entry point answers +/// with `EdgeZero`'s assembly-error response. +pub(crate) fn assemble_app( + settings: Result>, + log_context: &dyn fmt::Display, +) -> Result { + let state = startup_state(settings, log_context)?; + let app = App::from_router::( + TrustedServerApp::routes_for_state(&state), + FASTLY_PLATFORM, + )?; + Ok(RetainedApp { app, state }) +} - (Self::routes_for_state(&state), Some(state)) - } +/// Builds the [`AppState`] from loaded `settings`. +/// +/// # Errors +/// +/// Returns the [`EdgeError`] from [`startup_error`] after logging the full +/// report, followed by `log_context`. +fn startup_state( + settings: Result>, + log_context: &dyn fmt::Display, +) -> Result, EdgeError> { + settings + .and_then(build_state_from_settings) + .map_err(|report| { + log::error!("failed to build application state: {report:?}{log_context}"); + startup_error(&report) + }) +} +impl TrustedServerApp { fn routes_for_state(state: &Arc) -> RouterService { let mut router = RouterService::builder() .pre_dispatch_hook_arc(Arc::new(TracePreDispatchHook::new( Arc::clone(&state.settings), Arc::new(trace_metadata), ))) - .middleware(FinalizeResponseMiddleware::new( - Arc::clone(&state.settings), - Arc::new(FastlyPlatformGeo), - )) + .middleware(FinalizeResponseMiddleware::new(Arc::clone(&state.settings))) .middleware(AuthMiddleware::new(Arc::clone(&state.settings))); let fallback_handler = fallback_route_handler(Arc::clone(state)); @@ -1343,34 +1334,51 @@ impl TrustedServerApp { } } -fn trace_metadata(request: &Request) -> TraceMetadata { +fn trace_metadata(request: &RequestParts) -> TraceMetadata { let client_info = request - .extensions() + .extensions .get::() .cloned() .unwrap_or_else(|| ClientInfo { - client_ip: FastlyRequestContext::get(request).and_then(|context| context.client_ip), + client_ip: request + .extensions + .get::() + .and_then(|context| context.client_ip), ..ClientInfo::default() }); - let geo = client_info.client_ip.and_then(|client_ip| { - FastlyPlatformGeo - .lookup(Some(client_ip)) - .unwrap_or_else(|_| { - log::warn!("trace_geo_unavailable"); - None - }) - }); + let geo = request + .extensions + .get::() + .and_then(RequestGeo::geo) + .cloned(); TraceMetadata { client_info, geo } } impl Hooks for TrustedServerApp { + /// Installs the edge policies with the version headers as the synthesized set. + /// + /// The set is [`version_synthesized_headers`] for `FASTLY_SERVICE_VERSION`, + /// read as the finalize middleware reads it. The entry point replaces the + /// set for each request with the finalize headers that do not depend on + /// the response. + fn configure(app: &mut App) -> Result<(), EdgeError> { + install_edge_policies(app)?; + app.set_synthesized_response_headers(version_synthesized_headers( + std::env::var(ENV_FASTLY_SERVICE_VERSION).ok().as_deref(), + )) + .map_err(EdgeError::internal) + } + fn name() -> &'static str { "TrustedServer" } - fn routes() -> RouterService { - let stores = RuntimeStoreConfig::logical(); - Self::router_with_state(&stores).0 + fn routes() -> Result { + let state = startup_state( + load_settings_from_config_store(&RuntimeStoreConfig::logical()), + &"", + )?; + Ok(Self::routes_for_state(&state)) } fn stores() -> StoresMetadata { @@ -1399,11 +1407,12 @@ mod tests { use std::time::Duration; use super::{ - AppState, AuctionDispatch, EcContext, EdgeCacheHeader, EidSyncSource, HandlerFuture, - NAMED_ROUTES, NamedRouteHandler, PAGE_BIDS_LEGACY_PATH, PAGE_BIDS_PATH, RuntimeStoreConfig, - TrustedServerApp, build_orchestrator_with_plan, build_per_request_services, - build_state_from_settings, compile_auction_plan, handle_publisher_request, - publisher_response_into_streaming_response, startup_error_router, + App, AppState, AuctionDispatch, EcContext, EdgeCacheHeader, EidSyncSource, FASTLY_PLATFORM, + HandlerFuture, NAMED_ROUTES, NamedRouteHandler, PAGE_BIDS_LEGACY_PATH, PAGE_BIDS_PATH, + PublisherOriginOptions, RuntimeStoreConfig, TrustedServerApp, assemble_app, + build_orchestrator_with_plan, build_per_request_services, build_state_from_settings, + compile_auction_plan, handle_publisher_request, load_settings_from_config_store, + publisher_owner_error, publisher_response_into_streaming_response, }; use base64::Engine as _; use bytes::Bytes; @@ -1414,6 +1423,7 @@ mod tests { HeaderValue, Method, Request, Response, StatusCode, header, request_builder, }; use edgezero_core::key_value_store::NoopKvStore; + use edgezero_core::outbound::ResponseDecoding; use edgezero_core::params::PathParams; use edgezero_core::router::RouterService; use std::net::{IpAddr, Ipv4Addr}; @@ -1421,7 +1431,8 @@ mod tests { use error_stack::Report; use futures::StreamExt as _; use futures::executor::block_on; - use trusted_server_core::constants::HEADER_X_GEO_INFO_AVAILABLE; + use trusted_server_core::auction::AuctionEventBatch; + use trusted_server_core::constants::{ENV_FASTLY_SERVICE_VERSION, HEADER_X_GEO_INFO_AVAILABLE}; use trusted_server_core::ec::device::DeviceSignals; use trusted_server_core::error::TrustedServerError; use trusted_server_core::integrations::{ @@ -1429,14 +1440,19 @@ mod tests { RequestFilterEffects, RequestFilterInput, }; use trusted_server_core::platform::{ - ClientInfo, PlatformBackend, PlatformBackendSpec, PlatformError, PlatformHttpClient, - PlatformHttpRequest, PlatformKvStore, PlatformPendingRequest, PlatformResponse, - PlatformSelectResult, PlatformTemplateCache, PlatformTemplateCacheReservation, - RuntimeServices, TemplateCacheError, TemplateCacheKey, TemplateCacheLookup, - TemplateCacheMiss, TemplateCacheReservation, TemplateEntry, TemplateMetadata, + ClientInfo, GeoInfo, HttpClient, PlatformKvStore, PlatformTemplateCache, + PlatformTemplateCacheReservation, RuntimeServices, TemplateCacheError, TemplateCacheKey, + TemplateCacheLookup, TemplateCacheMiss, TemplateCacheReservation, TemplateEntry, + TemplateMetadata, }; use trusted_server_core::settings::Settings; - use trusted_server_core::test_support::nextjs_auction; + use trusted_server_core::test_support::scripted_client::{ + ScriptedHttpClient, ScriptedReply, TestClock, + }; + use trusted_server_core::test_support::{insert_outbound_client, nextjs_auction}; + + use crate::platform::RequestGeo; + use crate::tinybird::TelemetryQueue; #[test] fn hooks_store_metadata_matches_edgezero_manifest() { @@ -1505,7 +1521,22 @@ mod tests { .expect("should build request") } - fn route(router: &RouterService, request: edgezero_core::http::Request) -> Response { + fn route(router: &RouterService, mut request: edgezero_core::http::Request) -> Response { + // `oneshot` skips `EdgeZero`'s request conversion, which installs the + // adapter's default client, so install one that fails every request. + // It also skips the entry point's `prepare`, so add the geo `prepare` + // would hold for the request's client IP, without host geo data. + insert_outbound_client( + &mut request, + HttpClient::with_client(ScriptedHttpClient::forbidding()), + ); + let client_ip = request + .extensions() + .get::() + .and_then(|info| info.client_ip); + request + .extensions_mut() + .insert(RequestGeo::new(client_ip, None)); block_on(router.oneshot(request)).expect("should route request") } @@ -1560,15 +1591,194 @@ mod tests { TrustedServerApp::routes_for_state(&state) } + /// Wraps `request` in a context that carries `client`, as `EdgeZero`'s Fastly + /// request conversion installs its default client, and an empty + /// [`RequestGeo`], as the entry point inserts one. + fn context_with_client( + mut request: edgezero_core::http::Request, + client: HttpClient, + ) -> RequestContext { + request.extensions_mut().insert(client); + request.extensions_mut().insert(RequestGeo::default()); + RequestContext::new(request, PathParams::default()) + } + #[test] - fn per_request_services_register_the_fastly_template_assembler() { + fn per_request_services_carry_the_context_client_with_the_proxy_marker_off() { + let state = build_state_from_settings(test_settings()).expect("should build test state"); + let client = HttpClient::with_client(ScriptedHttpClient::forbidding()); + assert!( + client.proxy_marker_enabled(), + "should start from a client whose marker is on" + ); + let context = context_with_client(empty_request(Method::GET, "/article"), client); + + let services = build_per_request_services(&state, &context) + .expect("should build per-request services"); + + assert!( + !services.outbound_client().proxy_marker_enabled(), + "should turn the proxy marker off on the context client" + ); + } + + #[test] + fn per_request_services_reject_a_request_without_an_outbound_client() { let state = build_state_from_settings(test_settings()).expect("should build test state"); let context = RequestContext::new( empty_request(Method::GET, "/article"), PathParams::default(), ); - let services = build_per_request_services(&state, &context); + let error = build_per_request_services(&state, &context) + .expect_err("should reject a request with no outbound client"); + + assert_eq!( + error.status(), + StatusCode::INTERNAL_SERVER_ERROR, + "should answer a missing client with an internal error" + ); + assert_eq!( + error.to_string(), + "internal error: request has no outbound client", + "should name the missing client" + ); + } + + #[test] + fn per_request_services_reject_a_request_without_geo() { + let state = build_state_from_settings(test_settings()).expect("should build test state"); + let mut request = empty_request(Method::GET, "/article"); + request + .extensions_mut() + .insert(ScriptedHttpClient::forbidding().http_client()); + let context = RequestContext::new(request, PathParams::default()); + + let error = build_per_request_services(&state, &context) + .expect_err("should reject a request with no geo"); + + assert_eq!( + error.to_string(), + "internal error: request has no geo", + "should name the missing geo" + ); + } + + #[test] + fn per_request_services_answer_geo_from_the_entry_point_lookup() { + let state = build_state_from_settings(test_settings()).expect("should build test state"); + let reader_ip = IpAddr::V4(std::net::Ipv4Addr::new(198, 51, 100, 7)); + let held = GeoInfo { + city: "Springfield".to_owned(), + country: "US".to_owned(), + continent: "NorthAmerica".to_owned(), + latitude: 39.78, + longitude: -89.65, + metro_code: 648, + region: Some("IL".to_owned()), + asn: None, + }; + let mut request = empty_request(Method::GET, "/article"); + request.extensions_mut().insert(ClientInfo { + client_ip: Some(reader_ip), + ..ClientInfo::default() + }); + let mut context = + context_with_client(request, ScriptedHttpClient::forbidding().http_client()); + context + .extensions_mut() + .insert(RequestGeo::new(Some(reader_ip), Some(held))); + + let services = build_per_request_services(&state, &context) + .expect("should build per-request services"); + + assert_eq!( + services + .geo() + .lookup(services.client_info().client_ip) + .expect("should answer geo") + .map(|geo| geo.city), + Some("Springfield".to_owned()), + "should answer the client IP from the entry point's lookup" + ); + } + + #[test] + fn per_request_services_queue_telemetry_when_tinybird_is_enabled() { + let mut settings = test_settings(); + settings.tinybird = crate::tinybird::tests::enabled_config(); + let state = build_state_from_settings(settings).expect("should build test state"); + let queue = TelemetryQueue::default(); + let mut request = empty_request(Method::GET, "/article"); + request.extensions_mut().insert(queue.clone()); + let context = context_with_client(request, ScriptedHttpClient::forbidding().http_client()); + + let services = build_per_request_services(&state, &context) + .expect("should build per-request services"); + block_on(services.auction_telemetry_sink().emit_auction_events( + &services, + AuctionEventBatch::new(vec![crate::tinybird::tests::test_row()]), + )) + .expect("should queue the batch"); + + assert!( + services.auction_telemetry_sink().is_enabled(), + "should enable the telemetry sink" + ); + assert_eq!( + queue.take().len(), + 1, + "should queue the batch on the request's queue instead of sending it" + ); + } + + #[test] + fn per_request_services_reject_tinybird_without_a_telemetry_queue() { + let mut settings = test_settings(); + settings.tinybird = crate::tinybird::tests::enabled_config(); + let state = build_state_from_settings(settings).expect("should build test state"); + let context = context_with_client( + empty_request(Method::GET, "/article"), + ScriptedHttpClient::forbidding().http_client(), + ); + + let error = build_per_request_services(&state, &context) + .expect_err("should reject a request with no telemetry queue"); + + assert_eq!( + error.to_string(), + "internal error: request has no telemetry queue", + "should name the missing queue" + ); + } + + #[test] + fn per_request_services_send_no_telemetry_when_tinybird_is_disabled() { + let state = build_state_from_settings(test_settings()).expect("should build test state"); + let context = context_with_client( + empty_request(Method::GET, "/article"), + ScriptedHttpClient::forbidding().http_client(), + ); + + let services = build_per_request_services(&state, &context) + .expect("should build per-request services without a queue"); + + assert!( + !services.auction_telemetry_sink().is_enabled(), + "should keep the no-op sink" + ); + } + + #[test] + fn per_request_services_register_the_fastly_template_assembler() { + let state = build_state_from_settings(test_settings()).expect("should build test state"); + let context = context_with_client( + empty_request(Method::GET, "/article"), + ScriptedHttpClient::forbidding().http_client(), + ); + + let services = build_per_request_services(&state, &context) + .expect("should build per-request services"); let template = format!( "article{}", trusted_server_core::publisher::AD_ASSEMBLY_SEAM @@ -1608,9 +1818,6 @@ mod tests { let default_kv_store = Arc::new(crate::platform::UnavailableKvStore) as Arc; let state = Arc::new(super::AppState { - auction_telemetry_sink: Arc::new( - trusted_server_core::auction::NoopAuctionTelemetrySink, - ), settings: Arc::new(settings), orchestrator: Arc::new(orchestrator), registry: Arc::new(registry), @@ -1716,24 +1923,6 @@ mod tests { } } - struct TraceCountingTelemetry(Arc); - - #[async_trait::async_trait(?Send)] - impl trusted_server_core::auction::AuctionTelemetrySink for TraceCountingTelemetry { - fn is_enabled(&self) -> bool { - self.0.fetch_add(1, Ordering::SeqCst); - true - } - async fn emit_auction_events( - &self, - _services: &RuntimeServices, - _batch: trusted_server_core::auction::AuctionEventBatch, - ) -> Result<(), Report> { - self.0.fetch_add(1, Ordering::SeqCst); - Ok(()) - } - } - #[test] fn trace_dispatch_bypasses_fastly_identity_filters_and_telemetry() { let mut settings = test_settings(); @@ -1744,14 +1933,14 @@ mod tests { &serde_json::json!({"enabled":true,"trace_page_enabled":true}), ) .expect("should configure trace"); + settings.tinybird = crate::tinybird::tests::enabled_config(); let mut state = build_state_from_settings(settings).expect("should build trace state"); let kv = Arc::new(AtomicUsize::new(0)); - let telemetry = Arc::new(AtomicUsize::new(0)); + let telemetry = TelemetryQueue::default(); let filter = Arc::new(Mutex::new(None)); let mutable = Arc::get_mut(&mut state) .expect("should retain unique test state before router construction"); mutable.default_kv_store = Arc::new(TraceCountingKv(Arc::clone(&kv))); - mutable.auction_telemetry_sink = Arc::new(TraceCountingTelemetry(Arc::clone(&telemetry))); mutable.registry = Arc::new(IntegrationRegistry::from_request_filters(vec![Arc::new( ClientInfoCapturingFilter(Arc::clone(&filter)), )])); @@ -1763,7 +1952,9 @@ mod tests { (Method::POST, "/_ts/trace/enable", StatusCode::FORBIDDEN), (Method::PATCH, "/_ts/trace", StatusCode::METHOD_NOT_ALLOWED), ] { - let response = route(&router, empty_request(method, path)); + let mut request = empty_request(method, path); + request.extensions_mut().insert(telemetry.clone()); + let response = route(&router, request); assert_eq!( response.status(), status, @@ -1789,10 +1980,9 @@ mod tests { 0, "should never touch identity KV for trace" ); - assert_eq!( - telemetry.load(Ordering::SeqCst), - 0, - "should never inspect or emit auction telemetry for trace" + assert!( + telemetry.take().is_empty(), + "should never queue auction telemetry for trace" ); assert!( filter @@ -1822,6 +2012,45 @@ mod tests { } } + /// [`test_settings`] with `count` standard providers in place of its own, + /// each on its own port, so each is a distinct Fastly backend tuple. + fn settings_with_provider_tuples(count: u16) -> Settings { + let mut settings = test_settings(); + settings.auction.providers = (0..count) + .map(|index| { + ( + format!("provider-{index}") + .parse() + .expect("should parse provider ID"), + trusted_server_core::auction::ProviderConfig { + protocol: "openrtb-2.6".to_string(), + profile: "standard".to_string(), + endpoint: format!("https://bid.example:{}/openrtb", 9000 + index), + timeout_ms: Some(1000), + routing: trusted_server_core::auction::RoutingMode::AllEligible, + notifications: trusted_server_core::auction::NotificationConfig::default(), + profile_config: serde_json::json!({}), + }, + ) + }) + .collect(); + settings + } + + #[test] + fn state_builder_enforces_the_fastly_backend_bound() { + build_state_from_settings(settings_with_provider_tuples(160)) + .expect("should start with 160 distinct provider backend tuples"); + + let Err(error) = build_state_from_settings(settings_with_provider_tuples(161)) else { + panic!("should refuse to start with 161 distinct provider backend tuples"); + }; + assert!( + error.to_string().contains("161") && error.to_string().contains("160"), + "should name the tuple count and the bound: {error}" + ); + } + #[test] fn startup_registers_aps_renderer_route() { let mut settings = test_settings(); @@ -1851,40 +2080,191 @@ mod tests { } #[test] - fn startup_error_router_handles_head_and_options() { - let report = Report::new(TrustedServerError::BadRequest { - message: "startup failed".to_string(), - }); - let router = startup_error_router(&report); + fn hooks_routes_report_a_missing_config_blob_as_the_startup_error() { + // The test runtime links an empty `trusted_server_config` store, so the + // app config blob is missing. + let Err(error) = TrustedServerApp::routes() else { + panic!("should fail route building without an app config blob"); + }; + + assert_eq!( + error.status(), + StatusCode::INTERNAL_SERVER_ERROR, + "should keep today's configuration-failure status" + ); + } - let head_response = route(&router, empty_request(Method::HEAD, "/")); - let options_response = route(&router, empty_request(Method::OPTIONS, "/any")); + /// Completes `envelope` and returns the response the client would receive. + fn delivered_response(envelope: edgezero_core::ResponseEgressEnvelope) -> Response { + let Ok((prepared, _policy, mut attempt, clock)) = envelope.begin( + edgezero_core::response_egress_framing::ResponseEgressFramingMode::HostAutomatic, + ) else { + panic!("should begin response egress"); + }; + assert!(attempt.begin_writing(), "should start writing"); + assert!(attempt.complete(clock.now()), "should complete the body"); + prepared.into_response() + } + #[test] + fn missing_config_blob_gets_500_then_recovers_in_same_sandbox() { + let mut sandbox = crate::sandbox::Sandbox::default(); + + // Request 1: the linked store has no app config blob. + let error = sandbox + .initialize(|| { + assemble_app( + load_settings_from_config_store(&RuntimeStoreConfig::logical()), + &"", + ) + }) + .expect_err("should fail assembly without an app config blob"); + assert!( + sandbox.state().is_none(), + "should retain nothing after a failed assembly" + ); + let first = crate::sandbox::SandboxCounters::capture(&sandbox, 1, "request-1"); + let clock = edgezero_core::time::MonotonicClock::default(); + let response = delivered_response(App::assembly_error_egress( + error, + Method::GET, + clock.now(), + clock, + )); assert_eq!( - head_response.status(), - StatusCode::BAD_REQUEST, - "HEAD should use the degraded startup-error response" + response.status(), + StatusCode::INTERNAL_SERVER_ERROR, + "should answer with today's configuration-failure status" + ); + assert!( + response + .headers() + .get(trusted_server_core::constants::HEADER_X_TS_VERSION) + .is_none(), + "should carry no configured headers on an assembly error" + ); + let body = block_on(response.into_body().into_bytes_bounded(4096)) + .expect("should read the assembly error body"); + let json: serde_json::Value = + serde_json::from_slice(&body).expect("should answer with EdgeZero's JSON error"); + assert!( + json["error"].is_object(), + "should render EdgeZero's error body: {json}" + ); + + // Request 2: the blob is restored. + sandbox + .initialize(|| assemble_app(Ok(test_settings()), &"")) + .expect("should assemble once the blob is restored"); + let second = crate::sandbox::SandboxCounters::capture(&sandbox, 2, "request-2"); + + assert!( + sandbox.state().is_some(), + "should retain the recovered application" ); assert_eq!( - options_response.status(), - StatusCode::BAD_REQUEST, - "OPTIONS should use the degraded startup-error response" + second.builds, + first.builds + 1, + "should count one more build in the same sandbox" ); assert_eq!( - head_response - .headers() - .get(header::CONTENT_TYPE) - .and_then(|value| value.to_str().ok()), - Some("text/plain; charset=utf-8"), - "startup errors should stay plain-text for HEAD requests" + second.log_context().to_string().split(' ').nth(1), + first.log_context().to_string().split(' ').nth(1), + "should keep serving from the same sandbox instance" + ); + } + + #[test] + fn assembly_runs_configure_once_with_the_edge_policies() { + let retained = assemble_app(Ok(test_settings()), &"").expect("should assemble the app"); + let headers = edgezero_core::http::HeaderMap::new(); + let request_start = edgezero_core::time::MonotonicInstant::now(); + let egress_started_at = request_start + .checked_add(Duration::from_secs(5)) + .expect("should represent an egress start after the request start"); + let head = edgezero_core::response_egress::ResponseEgressHead::new( + StatusCode::OK, + edgezero_core::http::Version::HTTP_11, + &headers, + None, + request_start, + None, ); + + let policy = (retained.app.response_egress_policy())(&head, egress_started_at); + assert_eq!( - options_response - .headers() - .get(header::CONTENT_TYPE) - .and_then(|value| value.to_str().ok()), - Some("text/plain; charset=utf-8"), - "startup errors should stay plain-text for OPTIONS requests" + retained.app.platform(), + FASTLY_PLATFORM, + "should assemble under Fastly's platform metadata" + ); + assert_eq!( + retained.app.name(), + TrustedServerApp::name(), + "should name the app after its hooks" + ); + assert_eq!( + policy.write_deadline.instant(), + egress_started_at + .checked_add(trusted_server_core::edge_app::RESPONSE_WRITE_DEADLINE) + .expect("should represent the expected deadline"), + "should install configure's write deadline instead of EdgeZero's default" + ); + } + + /// Sets `FASTLY_SERVICE_VERSION` for one test and restores it on drop. + struct ServiceVersionGuard(Option); + + impl ServiceVersionGuard { + fn set(value: &str) -> Self { + let previous = std::env::var(ENV_FASTLY_SERVICE_VERSION).ok(); + // SAFETY: the wasm32-wasip1 test binary runs its tests one at a + // time on one thread, so nothing reads the environment meanwhile. + unsafe { std::env::set_var(ENV_FASTLY_SERVICE_VERSION, value) }; + Self(previous) + } + } + + impl Drop for ServiceVersionGuard { + fn drop(&mut self) { + // SAFETY: as in `set`, the test binary is single-threaded. + unsafe { + match &self.0 { + Some(previous) => std::env::set_var(ENV_FASTLY_SERVICE_VERSION, previous), + None => std::env::remove_var(ENV_FASTLY_SERVICE_VERSION), + } + } + } + } + + #[test] + fn configure_failure_fails_assembly_with_500_and_retries_next_request() { + let mut sandbox = crate::sandbox::Sandbox::default(); + + // A service version past the synthesized set's 4096-byte bound is the + // smallest configure failure a deployment can reach. + let guard = ServiceVersionGuard::set(&"9".repeat(4097)); + let error = sandbox + .initialize(|| assemble_app(Ok(test_settings()), &"")) + .expect_err("should fail assembly when configure rejects the synthesized set"); + drop(guard); + + assert_eq!( + error.status(), + StatusCode::INTERNAL_SERVER_ERROR, + "should answer a configure failure with a 500" + ); + assert!( + sandbox.state().is_none(), + "should retain nothing after configure fails" + ); + sandbox + .initialize(|| assemble_app(Ok(test_settings()), &"")) + .expect("should assemble on the next request once configure succeeds"); + assert_eq!( + sandbox.initialization_attempts(), + 2, + "should retry assembly on the next request" ); } @@ -1943,8 +2323,9 @@ mod tests { // FinalizeResponseMiddleware for header injection. // // This is safe to run without Viceroy: enforce_basic_auth is pure Rust - // (reads settings + request headers only) and FastlyPlatformGeo.lookup(None) - // short-circuits without calling any Fastly ABI. + // (reads settings + request headers only) and the finalize middleware + // reads the geo-less RequestGeo that `route` inserts, so nothing calls a + // Fastly ABI. let router = test_router(); let req = empty_request(Method::POST, "/_ts/admin/keys/rotate"); @@ -2490,7 +2871,8 @@ mod tests { fn dispatch_fallback_attaches_ec_finalize_state() { // The publisher fallback must thread EC finalize state to the entry // point via response extensions — even on error responses — so that - // edgezero_main can run ec_finalize_response and pull sync. + // its `finalize` step can run ec_finalize_response and post-delivery + // work can run pull sync. let router = test_router(); let response = route(&router, empty_request(Method::GET, "/some-page")); @@ -2594,7 +2976,7 @@ mod tests { #[test] fn browser_device_signals_from_extension_reach_ec_finalize_state() { - // Regression guard for the EdgeZero JA4/H2 signal loss: `edgezero_main` + // Regression guard for the EdgeZero JA4/H2 signal loss: the entry point // derives DeviceSignals from the original FastlyRequest (the only place // get_tls_ja4/get_client_h2_fingerprint return real values) and stores // them in the request extensions. A browser-shaped signal must survive @@ -2643,7 +3025,7 @@ mod tests { #[test] fn entry_point_client_info_reaches_request_filters() { // Regression guard for the EdgeZero bot-protection metadata loss: - // `edgezero_main` captures the full ClientInfo (TLS protocol/cipher, JA4, + // the entry point captures the full ClientInfo (TLS protocol/cipher, JA4, // H2 fingerprint, server hostname/region) from the original FastlyRequest // and stores it in the request extensions. It must survive dispatch so // integration request filters (e.g. DataDome) serialize the same signals @@ -2943,74 +3325,26 @@ mod tests { ); } - struct FixedBackend; - - impl PlatformBackend for FixedBackend { - fn naming_policy(&self) -> trusted_server_core::platform::BackendNamingPolicy { - trusted_server_core::platform::BackendNamingPolicy::Fastly - } - - fn predict_name( - &self, - spec: &PlatformBackendSpec, - ) -> Result> { - Ok(format!("{}-{}", spec.scheme, spec.host)) - } - - fn ensure(&self, spec: &PlatformBackendSpec) -> Result> { - self.predict_name(spec) - } - } - - struct StreamingHttpClient; - - #[async_trait::async_trait(?Send)] - impl PlatformHttpClient for StreamingHttpClient { - fn supports_streaming_responses(&self) -> bool { - true - } - - async fn send( - &self, - request: PlatformHttpRequest, - ) -> Result> { - assert!( - request.stream_response, - "streaming Fastly routes should request an origin response stream" - ); - let response = edgezero_core::http::response_builder() - .status(StatusCode::OK) - .header(header::CONTENT_TYPE, "text/html") - .body(Body::stream(futures::stream::iter(vec![ - Bytes::from_static(b"streamed-asset"), - ]))) - .map_err(|_| Report::new(PlatformError::HttpClient))?; - Ok(PlatformResponse::new(response)) - } - - async fn send_async( - &self, - _request: PlatformHttpRequest, - ) -> Result> { - Err(Report::new(PlatformError::Unsupported)) - } - - async fn select( - &self, - _pending_requests: Vec, - ) -> Result> { - Err(Report::new(PlatformError::Unsupported)) - } - } - - fn streaming_runtime_services() -> RuntimeServices { + /// Streaming runtime services whose `EdgeZero` outbound requests go to `upstream`. + fn runtime_services_with_upstream( + upstream: &ScriptedHttpClient, + clock: &TestClock, + ) -> RuntimeServices { RuntimeServices::builder() - .config_store(Arc::new(crate::platform::FastlyPlatformConfigStore)) - .secret_store(Arc::new(crate::platform::FastlyPlatformSecretStore)) + .config_store(Arc::new( + crate::platform::FastlyPlatformConfigStore::with_outbound_client( + upstream.http_client(), + ), + )) + .secret_store(Arc::new( + crate::platform::FastlyPlatformSecretStore::with_outbound_client( + upstream.http_client(), + ), + )) .kv_store(Arc::new(NoopKvStore) as Arc) - .backend(Arc::new(FixedBackend)) - .http_client(Arc::new(StreamingHttpClient)) - .geo(Arc::new(crate::platform::FastlyPlatformGeo)) + .outbound_client(upstream.http_client()) + .clock(clock.monotonic_clock()) + .geo(Arc::new(RequestGeo::default())) .client_info(ClientInfo::default()) .build() } @@ -3116,44 +3450,6 @@ mod tests { } } - #[derive(Default)] - struct DispatchOriginClient { - calls: AtomicUsize, - } - - #[async_trait::async_trait(?Send)] - impl PlatformHttpClient for DispatchOriginClient { - async fn send( - &self, - _request: PlatformHttpRequest, - ) -> Result> { - self.calls.fetch_add(1, Ordering::Relaxed); - let response = edgezero_core::http::response_builder() - .status(StatusCode::OK) - .header(header::CONTENT_TYPE, "text/html; charset=utf-8") - .header(header::CACHE_CONTROL, "public, max-age=300") - .body(Body::from( - b"origin".as_ref(), - )) - .map_err(|_| Report::new(PlatformError::HttpClient))?; - Ok(PlatformResponse::new(response)) - } - - async fn send_async( - &self, - _request: PlatformHttpRequest, - ) -> Result> { - Err(Report::new(PlatformError::Unsupported)) - } - - async fn select( - &self, - _pending_requests: Vec, - ) -> Result> { - Err(Report::new(PlatformError::Unsupported)) - } - } - #[test] fn dispatch_edge_authenticated_esi_request_stores_then_hits_template() { let settings = Arc::new( @@ -3195,16 +3491,36 @@ mod tests { .expect("should parse dispatch cache settings"), ); let cache = Arc::new(DispatchTemplateCache::default()); - let origin = Arc::new(DispatchOriginClient::default()); + let clock = TestClock::new(); + let origin = ScriptedHttpClient::new(&clock); + // Two replies: a warm dispatch that wrongly fetched the origin would take the second. + for _ in 0..2 { + origin.respond_to( + Method::GET, + &format!("{}/secure/article", settings.publisher.origin_url), + ScriptedReply::status(StatusCode::OK) + .header("content-type", "text/html; charset=utf-8") + .header("cache-control", "public, max-age=300") + .body("origin"), + ); + } let services = RuntimeServices::builder() - .config_store(Arc::new(crate::platform::FastlyPlatformConfigStore)) - .secret_store(Arc::new(crate::platform::FastlyPlatformSecretStore)) + .config_store(Arc::new( + crate::platform::FastlyPlatformConfigStore::with_outbound_client( + origin.http_client(), + ), + )) + .secret_store(Arc::new( + crate::platform::FastlyPlatformSecretStore::with_outbound_client( + origin.http_client(), + ), + )) .kv_store(Arc::new(NoopKvStore) as Arc) .template_cache(Arc::clone(&cache) as Arc) .template_assembler(Arc::new(crate::esi_assembly::FastlyTemplateAssembler)) - .backend(Arc::new(FixedBackend)) - .http_client(Arc::clone(&origin) as Arc) - .geo(Arc::new(crate::platform::FastlyPlatformGeo)) + .outbound_client(origin.http_client()) + .clock(clock.monotonic_clock()) + .geo(Arc::new(RequestGeo::default())) .client_info(ClientInfo::default()) .build(); let plan = Arc::new(compile_auction_plan(&settings).expect("should compile auction plan")); @@ -3228,7 +3544,7 @@ mod tests { let registry = Arc::clone(®istry); let orchestrator = Arc::clone(&orchestrator); Box::pin(async move { - let request = ctx.into_request(); + let request = ctx.into_request()?; let method = request.method().clone(); let mut ec_context = match EcContext::read_from_request(&settings, &request, &services) { @@ -3247,13 +3563,18 @@ mod tests { registry: None, }, request, - EdgeCacheHeader::SurrogateControl, + PublisherOriginOptions { + edge_header: EdgeCacheHeader::SurrogateControl, + input_cap: None, + response_decoding: ResponseDecoding::Original, + }, ) .await { Ok(response) => response, Err(report) => return Ok(super::http_error(&report)), }; + let (response, owner) = response; match publisher_response_into_streaming_response( response, &method, @@ -3263,6 +3584,7 @@ mod tests { services, ) .await + .and_then(|response| owner.attach(response).map_err(publisher_owner_error)) { Ok(response) => Ok(response), Err(report) => Ok(super::http_error(&report)), @@ -3308,7 +3630,7 @@ mod tests { block_on(warm.into_body().into_bytes_bounded(1024 * 1024)) .expect("should drain warm response"); assert_eq!( - origin.calls.load(Ordering::Relaxed), + origin.requests().len(), 1, "the warm dispatch must not fetch the publisher origin" ); @@ -3317,10 +3639,10 @@ mod tests { #[test] fn dispatch_asset_fallback_streams_origin_body_without_buffering() { // Regression guard for the EdgeZero asset streaming cutover: a successful - // asset proxy must hand `edgezero_main` a streaming body (`Body::Stream`) - // rather than draining the origin into a buffered `Body::Once`. Injecting a - // streaming HTTP client lets us assert the body type the entry point will - // pipe straight to the client via `stream_to_client`. + // asset proxy must hand the entry point a streaming body (`Body::Stream`) + // rather than draining the origin into a buffered `Body::Once`. A scripted + // upstream with a chunked body lets us assert the body type the entry point + // hands to `EdgeZero`'s delivery to pump straight to the client. let settings = Settings::from_toml( r#" [[handlers]] @@ -3352,7 +3674,16 @@ mod tests { .expect("should parse asset-route settings"); let state = build_state_from_settings(settings).expect("should build state"); - let services = streaming_runtime_services(); + let clock = TestClock::new(); + let upstream = ScriptedHttpClient::new(&clock); + upstream.respond_to( + Method::GET, + "https://assets.example.com/.images/logo.png", + ScriptedReply::status(StatusCode::OK) + .header("content-type", "image/png") + .chunks(vec![Bytes::from_static(b"streamed-asset")]), + ); + let services = runtime_services_with_upstream(&upstream, &clock); let req = empty_request(Method::GET, "/.images/logo.png"); let asset_route = state .settings @@ -3374,7 +3705,7 @@ mod tests { "should preserve the streaming origin status" ); assert!( - matches!(response.body(), Body::Stream(_)), + response.body().is_stream(), "EdgeZero asset dispatch must stream the origin body, not buffer it" ); } @@ -3384,12 +3715,12 @@ mod tests { for gzip in [false, true] { let state = build_state_from_settings(nextjs_auction::script_composition_settings()) .expect("should build mixed state"); - let origin = Arc::new(nextjs_auction::NextJsAuctionOrigin::with_html_response( + let origin = nextjs_auction::NextJsAuctionOrigin::with_html_response( &nextjs_auction::script_composition_html(), gzip, true, - )); - let services = nextjs_auction::services(origin); + ); + let services = nextjs_auction::services(&origin); let request = request_builder() .method(Method::GET) .uri(format!( @@ -3418,7 +3749,7 @@ mod tests { gzip.then_some("gzip"), "should retain the requested response compression" ); - let Body::Stream(mut stream) = response.into_body() else { + let Ok(mut stream) = response.into_body().into_stream() else { panic!("should retain lazy Fastly body"); }; let mut bytes = Vec::new(); @@ -3434,14 +3765,26 @@ mod tests { #[test] fn dispatch_fallback_streams_publisher_body_without_buffering() { // Regression guard for the publisher streaming cutover (#849): a - // successful publisher origin fetch must hand `edgezero_main` a lazy + // successful publisher origin fetch must hand the entry point a lazy // streaming body (`Body::Stream`) so headers commit at origin first // byte, rather than draining the processed page into a buffered // `Body::Once`. Core tests cover the rewrite pipeline itself; this // guards the adapter wiring that could silently re-buffer. let settings = test_settings(); let state = build_state_from_settings(settings).expect("should build state"); - let services = streaming_runtime_services(); + let clock = TestClock::new(); + let origin = ScriptedHttpClient::new(&clock); + origin.respond_to( + Method::GET, + &format!("{}/article", state.settings.publisher.origin_url), + ScriptedReply::status(StatusCode::OK) + .header("content-type", "text/html") + .chunks(vec![ + Bytes::from_static(b""), + Bytes::from_static(b"streamed"), + ]), + ); + let services = runtime_services_with_upstream(&origin, &clock); let req = empty_request(Method::GET, "/article"); let response = block_on(super::dispatch_fallback(&state, &services, req)); @@ -3452,7 +3795,7 @@ mod tests { "publisher proxy should succeed against the streaming origin stub" ); assert!( - matches!(response.body(), Body::Stream(_)), + response.body().is_stream(), "EdgeZero publisher dispatch must attach the lazy streaming body, not buffer it" ); assert!( diff --git a/crates/trusted-server-adapter-fastly/src/backend.rs b/crates/trusted-server-adapter-fastly/src/backend.rs deleted file mode 100644 index 0151cbb165..0000000000 --- a/crates/trusted-server-adapter-fastly/src/backend.rs +++ /dev/null @@ -1,757 +0,0 @@ -use std::time::Duration; - -use error_stack::{Report, ResultExt as _}; -use fastly::backend::Backend; -use url::Url; - -use trusted_server_core::error::TrustedServerError; -use trusted_server_core::platform::{BackendNamingPolicy, PlatformBackendSpec, PredictedBackend}; - -#[cfg(test)] -const MAX_BACKEND_NAME_LEN: usize = 255; -#[cfg(test)] -const SPEC_DIGEST_HEX_LEN: usize = 32; - -/// Returns the default port for the given scheme (443 for HTTPS, 80 for HTTP). -#[inline] -fn default_port_for_scheme(scheme: &str) -> u16 { - if scheme.eq_ignore_ascii_case("https") { - 443 - } else { - 80 - } -} - -#[derive(Debug, Clone, Eq, PartialEq)] -struct NormalizedBackendHost { - identity: String, - authority: String, - is_ip_literal: bool, -} - -/// Normalize URL-derived and direct hosts for transport and TLS use. -/// -/// `url::Url::host_str()` preserves brackets around IPv6 literals. Fastly's -/// backend target and HTTP authority require those brackets, while certificate -/// identity matching requires the bare address and SNI must not be sent for IP -/// literals. -#[inline] -fn normalize_backend_host(host: &str) -> NormalizedBackendHost { - let unbracketed = host - .strip_prefix('[') - .and_then(|value| value.strip_suffix(']')) - .unwrap_or(host); - match unbracketed.parse::() { - Ok(std::net::IpAddr::V6(_)) => NormalizedBackendHost { - identity: unbracketed.to_owned(), - authority: format!("[{unbracketed}]"), - is_ip_literal: true, - }, - Ok(std::net::IpAddr::V4(_)) => NormalizedBackendHost { - identity: unbracketed.to_owned(), - authority: unbracketed.to_owned(), - is_ip_literal: true, - }, - Err(_) => NormalizedBackendHost { - identity: host.to_owned(), - authority: host.to_owned(), - is_ip_literal: false, - }, - } -} - -/// Compute the Host header value for a backend request. -/// -/// For standard ports (443 for HTTPS, 80 for HTTP), returns just the hostname. -/// For non-standard ports, returns "hostname:port" to ensure backends that -/// generate URLs based on the Host header include the port. -/// -/// This fixes the issue where backends behind reverse proxies (like Caddy) -/// would generate URLs without the port when the Host header didn't include it. -#[inline] -fn compute_host_header(scheme: &str, host: &str, port: u16) -> String { - let host = normalize_backend_host(host).authority; - if port == default_port_for_scheme(scheme) { - host - } else { - format!("{host}:{port}") - } -} - -/// Default first-byte timeout for backends (15 seconds). -pub(crate) const DEFAULT_FIRST_BYTE_TIMEOUT: Duration = Duration::from_secs(15); -/// Default timeout between response body bytes for backends (10 seconds). -pub(crate) const DEFAULT_BETWEEN_BYTES_TIMEOUT: Duration = Duration::from_secs(10); - -/// Configuration for creating a dynamic Fastly backend. -/// -/// Uses the builder pattern so that new options can be added without changing -/// existing call sites — fields carry sensible defaults. -pub struct BackendConfig<'a> { - scheme: &'a str, - host: &'a str, - port: Option, - certificate_check: bool, - first_byte_timeout: Duration, - between_bytes_timeout: Duration, - host_header_override: Option<&'a str>, - discriminator: Option<&'a str>, -} - -impl<'a> BackendConfig<'a> { - /// Create a new configuration with required fields and safe defaults. - /// - /// `certificate_check` defaults to `true`. - /// `first_byte_timeout` defaults to 15 seconds. - #[must_use] - pub fn new(scheme: &'a str, host: &'a str) -> Self { - Self { - scheme, - host, - port: None, - certificate_check: true, - first_byte_timeout: DEFAULT_FIRST_BYTE_TIMEOUT, - between_bytes_timeout: DEFAULT_BETWEEN_BYTES_TIMEOUT, - host_header_override: None, - discriminator: None, - } - } - - /// Set the port for the backend. When `None`, the default port for the - /// scheme is used (443 for HTTPS, 80 for HTTP). - #[must_use] - pub fn port(mut self, port: Option) -> Self { - self.port = port; - self - } - - /// Control TLS certificate verification. Defaults to `true`. - #[must_use] - pub fn certificate_check(mut self, check: bool) -> Self { - self.certificate_check = check; - self - } - - /// Set the maximum time to wait for the first byte of the response. - /// - /// Defaults to 15 seconds. For latency-sensitive paths like auction - /// requests, callers should set a tighter timeout derived from the - /// auction deadline. - #[must_use] - pub fn first_byte_timeout(mut self, timeout: Duration) -> Self { - self.first_byte_timeout = timeout; - self - } - - /// Set the maximum time to wait between response body bytes. - /// - /// Defaults to 10 seconds. Auction backends should set this to the same - /// remaining budget as the first-byte timeout so slow-drip bodies cannot - /// hold the auction past its deadline. - #[must_use] - pub fn between_bytes_timeout(mut self, timeout: Duration) -> Self { - self.between_bytes_timeout = timeout; - self - } - - /// Set the outbound Host header sent to the backend origin. - #[must_use] - pub fn host_header_override(mut self, host: Option<&'a str>) -> Self { - self.host_header_override = host; - self - } - - /// Set an optional stable discriminator folded into the backend name. - /// - /// Two callers targeting the same origin with the same transport timeout - /// otherwise share a backend name. Auction response correlation keys on the - /// backend name, so a shared name would let one provider's response be - /// parsed as another's. A per-provider discriminator keeps the names - /// distinct while staying stable across requests. - #[must_use] - pub fn discriminator(mut self, discriminator: Option<&'a str>) -> Self { - self.discriminator = discriminator; - self - } - - fn platform_spec(&self) -> PlatformBackendSpec { - PlatformBackendSpec { - scheme: self.scheme.to_owned(), - host: normalize_backend_host(self.host).identity, - port: self.port, - host_header_override: self.host_header_override.map(str::to_owned), - certificate_check: self.certificate_check, - first_byte_timeout: self.first_byte_timeout, - between_bytes_timeout: self.between_bytes_timeout, - discriminator: self.discriminator.map(str::to_owned), - } - } - - /// Compute the deterministic backend name and resolved port without - /// registering anything. - fn predict_backend(&self) -> Result> { - BackendNamingPolicy::Fastly - .predict(&self.platform_spec()) - .change_context(TrustedServerError::Proxy { - message: "backend name prediction failed".to_owned(), - }) - } - - /// Return the deterministic backend name without registering anything. - /// - /// Convenience wrapper over `Self::predict_backend` that discards the - /// resolved port, used by [`crate::platform::PlatformBackend`] - /// implementations that only need the name for correlation. - /// - /// # Errors - /// - /// Returns an error if the host is empty. - pub fn predict_name(self) -> Result> { - self.predict_backend().map(|prediction| prediction.name) - } - - /// Ensure a dynamic backend exists for this configuration and return its name. - /// - /// The name is a collision-resistant function of the complete backend spec - /// (see `Self::predict_backend`), so different specs — for example, different - /// timeout values — always produce different backend registrations and a - /// tight deadline cannot be silently widened by an earlier registration. - /// - /// # Errors - /// - /// Returns an error if the host is empty or if backend creation fails - /// (except for `NameInUse` which reuses the existing backend). - pub fn ensure(self) -> Result> { - let prediction = self.predict_backend()?; - let backend_name = prediction.name; - let target_port = prediction.port; - let host = normalize_backend_host(self.host); - - let host_with_port = format!("{}:{target_port}", host.authority); - - let host_header = self.host_header_override.map_or_else( - || compute_host_header(self.scheme, &host.identity, target_port), - str::to_owned, - ); - - // Target base is host[:port]; SSL is enabled only for https scheme - let mut builder = Backend::builder(&backend_name, &host_with_port) - .override_host(&host_header) - .connect_timeout(Duration::from_secs(1)) - .first_byte_timeout(self.first_byte_timeout) - .between_bytes_timeout(self.between_bytes_timeout); - if self.scheme.eq_ignore_ascii_case("https") { - builder = builder.enable_ssl(); - if !host.is_ip_literal { - builder = builder.sni_hostname(&host.identity); - } - if self.certificate_check { - builder = builder.check_certificate(&host.identity); - } else { - log::warn!("INSECURE: certificate check disabled for backend: {backend_name}"); - } - log::info!("enable ssl for backend: {backend_name}"); - } - - match builder.finish() { - Ok(_) => { - log::info!("created dynamic backend: {backend_name} -> {host_with_port}"); - Ok(backend_name) - } - Err(e) => { - let msg = e.to_string(); - if msg.contains("NameInUse") || msg.contains("already in use") { - log::info!("reusing existing dynamic backend: {backend_name}"); - Ok(backend_name) - } else { - Err(Report::new(TrustedServerError::Proxy { - message: format!( - "dynamic backend creation failed ({backend_name} -> {host_with_port}): {msg}" - ), - })) - } - } - } - } - - /// Parse an origin URL into its (scheme, host, port) components. - /// - /// Centralises URL parsing so that [`from_url`](Self::from_url) and - /// [`from_url_with_first_byte_timeout`](Self::from_url_with_first_byte_timeout) - /// share one code-path. - fn parse_origin( - origin_url: &str, - ) -> Result<(String, String, Option), Report> { - let parsed_url = Url::parse(origin_url).change_context(TrustedServerError::Proxy { - message: format!("Invalid origin_url: {origin_url}"), - })?; - - let scheme = parsed_url.scheme().to_owned(); - let host = parsed_url - .host_str() - .ok_or_else(|| { - Report::new(TrustedServerError::Proxy { - message: "Missing host in origin_url".to_owned(), - }) - })? - .to_owned(); - let port = parsed_url.port(); - - Ok((scheme, host, port)) - } - - /// Parse an origin URL and ensure a dynamic backend exists for it. - /// - /// This is a convenience constructor that parses the URL, extracts scheme, - /// host, and port, then calls [`ensure`](Self::ensure) with the default - /// 15 s first-byte timeout. - /// - /// # Errors - /// - /// Returns an error if the URL cannot be parsed or lacks a host, or if - /// backend creation fails. - pub fn from_url( - origin_url: &str, - certificate_check: bool, - ) -> Result> { - Self::from_url_with_first_byte_timeout( - origin_url, - certificate_check, - DEFAULT_FIRST_BYTE_TIMEOUT, - ) - } - - /// Parse an origin URL and ensure a dynamic backend with a custom - /// first-byte timeout. - /// - /// For latency-sensitive paths (e.g. auction bid requests) callers should - /// pass the remaining auction budget so that individual requests don't hang - /// longer than the overall deadline allows. - /// - /// # Errors - /// - /// Returns an error if the URL cannot be parsed or lacks a host, or if - /// backend creation fails. - pub fn from_url_with_first_byte_timeout( - origin_url: &str, - certificate_check: bool, - first_byte_timeout: Duration, - ) -> Result> { - Self::from_url_with_first_byte_timeout_and_host_header_override( - origin_url, - certificate_check, - first_byte_timeout, - None, - ) - } - - fn from_url_with_first_byte_timeout_and_host_header_override( - origin_url: &str, - certificate_check: bool, - first_byte_timeout: Duration, - host_header_override: Option<&str>, - ) -> Result> { - let (scheme, host, port) = Self::parse_origin(origin_url)?; - - BackendConfig::new(&scheme, &host) - .port(port) - .certificate_check(certificate_check) - .first_byte_timeout(first_byte_timeout) - .host_header_override(host_header_override) - .ensure() - } -} - -#[cfg(test)] -mod tests { - use trusted_server_core::platform::BackendNamingError; - - use super::{ - BackendConfig, MAX_BACKEND_NAME_LEN, SPEC_DIGEST_HEX_LEN, compute_host_header, - normalize_backend_host, - }; - - /// Assert a computed name is `backend__` and stays within - /// Fastly's length limit. The digest is what makes the name injective, so - /// checking its presence and width guards the collision-safety property. - fn assert_backend_name_shape(name: &str, expected_body: &str) { - let prefix = format!("backend_{expected_body}_"); - assert!( - name.starts_with(&prefix), - "name should start with the readable body `{prefix}`, got {name}" - ); - let digest = &name[prefix.len()..]; - assert_eq!( - digest.len(), - SPEC_DIGEST_HEX_LEN, - "digest suffix should be {SPEC_DIGEST_HEX_LEN} hex chars, got {digest}" - ); - assert!( - digest.bytes().all(|byte| byte.is_ascii_hexdigit()), - "digest suffix should be hex, got {digest}" - ); - assert!( - name.len() <= MAX_BACKEND_NAME_LEN, - "name should stay within the {MAX_BACKEND_NAME_LEN}-char limit, got {}", - name.len() - ); - } - - // Tests for compute_host_header - the fix for port preservation in Host header - #[test] - fn ipv6_hosts_are_bracketed_only_for_authority_values() { - let bare = normalize_backend_host("2001:db8::1"); - let bracketed = normalize_backend_host("[2001:db8::1]"); - assert_eq!(bare, bracketed); - assert_eq!(bare.identity, "2001:db8::1"); - assert_eq!(bare.authority, "[2001:db8::1]"); - assert!(bare.is_ip_literal, "IPv6 must not be sent as TLS SNI"); - assert_eq!( - normalize_backend_host("cdn.example.com"), - super::NormalizedBackendHost { - identity: "cdn.example.com".to_string(), - authority: "cdn.example.com".to_string(), - is_ip_literal: false, - } - ); - assert_eq!( - compute_host_header("https", "[2001:db8::1]", 443), - "[2001:db8::1]" - ); - assert_eq!( - compute_host_header("https", "[2001:db8::1]", 8443), - "[2001:db8::1]:8443" - ); - } - - #[test] - fn url_derived_ipv6_host_uses_bare_tls_identity_without_sni() { - let (scheme, url_host, port) = - BackendConfig::parse_origin("https://[2001:db8::7]:8443/openrtb") - .expect("should parse IPv6 provider URL"); - assert_eq!(scheme, "https"); - assert_eq!(url_host, "[2001:db8::7]"); - assert_eq!(port, Some(8443)); - - let normalized = normalize_backend_host(&url_host); - assert_eq!(normalized.identity, "2001:db8::7"); - assert_eq!(normalized.authority, "[2001:db8::7]"); - assert!( - normalized.is_ip_literal, - "IP literals must omit TLS SNI while retaining a bare certificate identity" - ); - - let from_url_name = BackendConfig::new(&scheme, &url_host) - .port(port) - .predict_name() - .expect("should predict URL-derived IPv6 backend name"); - let from_bare_name = BackendConfig::new(&scheme, "2001:db8::7") - .port(port) - .predict_name() - .expect("should predict bare IPv6 backend name"); - assert_eq!( - from_url_name, from_bare_name, - "URL and direct IPv6 paths must preserve backend naming parity" - ); - } - - #[test] - fn host_header_includes_port_for_non_standard_https() { - assert_eq!( - compute_host_header("https", "cdn.example.com", 9443), - "cdn.example.com:9443", - "should include non-standard HTTPS port 9443 in Host header" - ); - assert_eq!( - compute_host_header("https", "cdn.example.com", 8443), - "cdn.example.com:8443", - "should include non-standard HTTPS port 8443 in Host header" - ); - } - - #[test] - fn host_header_excludes_port_for_standard_https() { - assert_eq!( - compute_host_header("https", "cdn.example.com", 443), - "cdn.example.com", - "should omit standard HTTPS port 443 from Host header" - ); - } - - #[test] - fn host_header_includes_port_for_non_standard_http() { - assert_eq!( - compute_host_header("http", "cdn.example.com", 8080), - "cdn.example.com:8080", - "should include non-standard HTTP port 8080 in Host header" - ); - } - - #[test] - fn host_header_excludes_port_for_standard_http() { - assert_eq!( - compute_host_header("http", "cdn.example.com", 80), - "cdn.example.com", - "should omit standard HTTP port 80 from Host header" - ); - } - - #[test] - fn returns_name_for_https_with_cert_check() { - let name = BackendConfig::new("https", "origin.example.com") - .ensure() - .expect("should create backend for valid HTTPS origin"); - assert_backend_name_shape(&name, "https_origin_example_com_443_fb15000_bb10000"); - } - - #[test] - fn returns_name_for_https_without_cert_check() { - let name = BackendConfig::new("https", "origin.example.com") - .certificate_check(false) - .ensure() - .expect("should create backend with cert check disabled"); - assert_backend_name_shape(&name, "https_origin_example_com_443_nocert_fb15000_bb10000"); - } - - #[test] - fn returns_name_for_http_with_port_and_sanitizes() { - let name = BackendConfig::new("http", "api.test-site.org") - .port(Some(8080)) - .ensure() - .expect("should create backend for HTTP origin with explicit port"); - assert_backend_name_shape(&name, "http_api_test-site_org_8080_fb15000_bb10000"); - } - - #[test] - fn returns_name_for_http_without_port_defaults_to_80() { - let name = BackendConfig::new("http", "example.org") - .ensure() - .expect("should create backend defaulting to port 80 for HTTP"); - assert_backend_name_shape(&name, "http_example_org_80_fb15000_bb10000"); - } - - #[test] - fn error_on_host_with_control_characters() { - let err = BackendConfig::new("https", "evil.com\nINFO fake log entry") - .predict_name() - .expect_err("should reject host containing newline"); - assert!( - err.contains::(), - "should preserve the backend naming error report context" - ); - } - - #[test] - fn error_on_missing_host() { - let err = BackendConfig::new("https", "") - .ensure() - .expect_err("should reject empty host"); - assert!( - err.contains::(), - "should preserve the original backend naming error report context" - ); - } - - #[test] - fn second_call_reuses_existing_backend() { - let first = BackendConfig::new("https", "reuse.example.com") - .ensure() - .expect("should create backend on first call"); - let second = BackendConfig::new("https", "reuse.example.com") - .ensure() - .expect("should reuse backend on second call"); - assert_eq!( - first, second, - "should return same backend name on repeat call" - ); - } - - #[test] - fn host_header_overrides_produce_different_names() { - let name_a = BackendConfig::new("https", "origin.example.com") - .host_header_override(Some("www.example.com")) - .predict_name() - .expect("should compute name with host header override"); - let name_b = BackendConfig::new("https", "origin.example.com") - .host_header_override(Some("m.example.com")) - .predict_name() - .expect("should compute name with different host header override"); - - assert_ne!( - name_a, name_b, - "backends with different host header overrides should have different names" - ); - assert_backend_name_shape( - &name_a, - "https_origin_example_com_443_oh_www_example_com_fb15000_bb10000", - ); - assert_backend_name_shape( - &name_b, - "https_origin_example_com_443_oh_m_example_com_fb15000_bb10000", - ); - } - - #[test] - fn host_header_override_rejects_control_characters() { - let err = BackendConfig::new("https", "origin.example.com") - .host_header_override(Some("www\n.example.com")) - .predict_name() - .expect_err("should reject host header override containing newline"); - - assert!( - err.contains::(), - "should preserve the backend naming error report context" - ); - } - - #[test] - fn host_header_override_rejects_invalid_values() { - for host_header_override in [ - "https://www.example.com", - "www.example.com/path", - "www.example.com:", - "example..com", - "-", - ] { - let err = BackendConfig::new("https", "origin.example.com") - .host_header_override(Some(host_header_override)) - .predict_name() - .expect_err("should reject invalid host header override"); - - assert!( - err.contains::(), - "should preserve the backend naming error report context for {host_header_override:?}" - ); - } - } - - #[test] - fn different_timeouts_produce_different_names() { - use std::time::Duration; - - let name_a = BackendConfig::new("https", "origin.example.com") - .first_byte_timeout(Duration::from_secs(2)) - .predict_name() - .expect("should compute name with 2000ms timeout"); - let name_b = BackendConfig::new("https", "origin.example.com") - .first_byte_timeout(Duration::from_millis(500)) - .predict_name() - .expect("should compute name with 500ms timeout"); - assert_ne!( - name_a, name_b, - "backends with different timeouts should have different names" - ); - assert!( - name_a.contains("_fb2000_bb10000_"), - "name should include first-byte and between-bytes timeout in the readable body" - ); - assert!( - name_b.contains("_fb500_bb10000_"), - "name should include first-byte and between-bytes timeout in the readable body" - ); - } - - #[test] - fn different_between_bytes_timeouts_produce_different_names() { - use std::time::Duration; - - let name_a = BackendConfig::new("https", "origin.example.com") - .between_bytes_timeout(Duration::from_secs(2)) - .predict_name() - .expect("should compute name with 2000ms between-bytes timeout"); - let name_b = BackendConfig::new("https", "origin.example.com") - .between_bytes_timeout(Duration::from_millis(500)) - .predict_name() - .expect("should compute name with 500ms between-bytes timeout"); - - assert_ne!( - name_a, name_b, - "backends with different between-bytes timeouts should have different names" - ); - assert!( - name_a.contains("_fb15000_bb2000_"), - "name should include first-byte and between-bytes timeout in the readable body" - ); - assert!( - name_b.contains("_fb15000_bb500_"), - "name should include first-byte and between-bytes timeout in the readable body" - ); - } - - #[test] - fn discriminators_that_sanitize_alike_produce_distinct_names() { - // `provider.a` and `provider_a` both sanitize to the same readable slug - // (`.` maps to `_`). Before the spec digest they collided to one backend - // name, so the second registration silently reused the first — routing - // one provider's auction traffic through another's backend. The digest - // over the raw spec must keep them distinct. - let dotted = BackendConfig::new("https", "gateway.example.com") - .discriminator(Some("provider.a")) - .predict_name() - .expect("should predict name for dotted discriminator"); - let underscored = BackendConfig::new("https", "gateway.example.com") - .discriminator(Some("provider_a")) - .predict_name() - .expect("should predict name for underscored discriminator"); - assert_ne!( - dotted, underscored, - "discriminators differing only by a sanitized character must not collide" - ); - } - - #[test] - fn host_overrides_that_sanitize_alike_produce_distinct_names() { - // `host.example.com:8443` (host+port) and `host.example.com.8443` (DNS - // label) are both valid overrides that sanitize to the same readable - // slug (`:` and `.` both map to `_`). The digest over the raw value must - // keep the two backends — with different Host routing — distinct. - let with_port = BackendConfig::new("https", "origin.example.com") - .host_header_override(Some("host.example.com:8443")) - .predict_name() - .expect("should predict name for host:port override"); - let with_label = BackendConfig::new("https", "origin.example.com") - .host_header_override(Some("host.example.com.8443")) - .predict_name() - .expect("should predict name for dotted-label override"); - assert_ne!( - with_port, with_label, - "host overrides differing only by a sanitized character must not collide" - ); - } - - #[test] - fn long_host_and_discriminator_stay_within_the_length_limit() { - // A syntactically valid maximum-length DNS host plus a discriminator - // previously pushed the name past Fastly's 255-char limit, so - // `predict_name` succeeded while `ensure` failed at registration. The - // bounded prefix + fixed-width digest must keep prediction, and the name - // it predicts, within the limit. - let label = "a".repeat(63); - let long_host = format!("{label}.{label}.{label}.{label}.example.com"); - assert!( - long_host.len() > 200, - "should exercise a host longer than the readable-prefix bound" - ); - let name = BackendConfig::new("https", &long_host) - .discriminator(Some("prebid")) - .predict_name() - .expect("should predict a bounded name for a long host and discriminator"); - assert!( - name.len() <= MAX_BACKEND_NAME_LEN, - "name should stay within the {MAX_BACKEND_NAME_LEN}-char limit, got {}", - name.len() - ); - - // Two long hosts sharing the truncated prefix must still resolve to - // different backends via the digest. - let other_host = format!("{label}.{label}.{label}.{label}.example.net"); - let other = BackendConfig::new("https", &other_host) - .discriminator(Some("prebid")) - .predict_name() - .expect("should predict a bounded name for the sibling host"); - assert_ne!( - name, other, - "hosts sharing a truncated prefix must stay distinct via the digest" - ); - } -} diff --git a/crates/trusted-server-adapter-fastly/src/compat.rs b/crates/trusted-server-adapter-fastly/src/compat.rs index 4b679faecf..cd2cc8a984 100644 --- a/crates/trusted-server-adapter-fastly/src/compat.rs +++ b/crates/trusted-server-adapter-fastly/src/compat.rs @@ -1,9 +1,7 @@ -//! Compatibility bridge between `fastly` SDK types and `http` crate types. +//! Native request work the Fastly entry point runs before `EdgeZero` converts a request. use std::net::IpAddr; -use edgezero_core::body::Body as EdgeBody; -use edgezero_core::http::Response as HttpResponse; use edgezero_core::http::{HeaderMap, HeaderName, HeaderValue}; use trusted_server_core::forwarder::ForwarderPreparation; use trusted_server_core::http_util::SPOOFABLE_FORWARDED_HEADERS; @@ -12,52 +10,6 @@ use trusted_server_core::settings::TrustedClientIpConfig; use crate::platform::resolve_client_ip; -/// Convert an [`HttpResponse`] into a `fastly::Response`. -pub(crate) fn to_fastly_response(resp: HttpResponse) -> fastly::Response { - let (parts, body) = resp.into_parts(); - let mut fastly_resp = fastly::Response::from_status(parts.status.as_u16()); - for (name, value) in &parts.headers { - fastly_resp.append_header(name.as_str(), value.as_bytes()); - } - - match body { - EdgeBody::Once(bytes) => { - if !bytes.is_empty() { - fastly_resp.set_body(bytes.to_vec()); - } - } - EdgeBody::Stream(_) => { - // Streaming bodies cannot cross the compat boundary. Both audited call sites - // (legacy_main buffered arm and edgezero_main after EdgeZero collapses bodies - // to Once) only pass Once bodies — a Stream here is a caller error. - // The assert is suppressed in test builds where the behavior-documentation - // test deliberately exercises this path. - #[cfg(not(test))] - debug_assert!( - false, - "to_fastly_response: streaming body will be silently dropped; \ - use to_fastly_response_skeleton + stream_to_client for streaming responses" - ); - log::warn!("streaming body in compat::to_fastly_response; body will be empty"); - } - } - - fastly_resp -} - -/// Convert an [`HttpResponse`] into a `fastly::Response` without a body. -/// -/// Use this when the caller will stream the body separately through -/// [`fastly::Response::stream_to_client`]. -pub(crate) fn to_fastly_response_skeleton(resp: HttpResponse) -> fastly::Response { - let (parts, _body) = resp.into_parts(); - let mut fastly_resp = fastly::Response::from_status(parts.status.as_u16()); - for (name, value) in &parts.headers { - fastly_resp.append_header(name.as_str(), value.as_bytes()); - } - fastly_resp -} - /// Sanitize forwarded headers on a `fastly::Request`. /// /// Strips configured trust headers and headers that clients can spoof before @@ -134,6 +86,7 @@ pub(crate) fn capture_fastly_forwarder( #[cfg(test)] mod tests { use super::*; + use edgezero_core::body::Body as EdgeBody; use trusted_server_core::forwarder::prepare_trusted_forwarder; use trusted_server_core::http_util::RequestInfo; use trusted_server_core::platform::ClientInfo; @@ -165,13 +118,15 @@ mod tests { "should remove native forwarding credential" ); sanitize_fastly_forwarded_headers(&mut native, None); - let mut core = edgezero_core::http::request_builder() + let (mut parts, body) = edgezero_core::http::request_builder() .uri("http://upstream.example.com/path") .header("host", "upstream.example.com") .body(EdgeBody::empty()) - .expect("should construct converted request"); - core.extensions_mut().insert(prepared); - prepare_trusted_forwarder(&mut core, &settings); + .expect("should construct converted request") + .into_parts(); + parts.extensions.insert(prepared); + prepare_trusted_forwarder(&mut parts, &settings); + let core = edgezero_core::http::Request::from_parts(parts, body); let info = RequestInfo::from_request(&core, &ClientInfo::default()) .expect("should accept converted request metadata"); assert_eq!( @@ -306,29 +261,4 @@ mod tests { "should strip the configured auth header after resolving" ); } - - #[test] - fn to_fastly_response_with_streaming_body_produces_empty_body() { - use edgezero_core::http::StatusCode; - - let stream = futures::stream::empty::(); - let stream_body = EdgeBody::stream(stream); - - let http_resp = edgezero_core::http::response_builder() - .status(StatusCode::OK) - .body(stream_body) - .expect("should build response"); - - let mut fastly_resp = to_fastly_response(http_resp); - - assert_eq!( - fastly_resp.get_status().as_u16(), - 200, - "should preserve status" - ); - assert!( - fastly_resp.take_body_bytes().is_empty(), - "should produce empty body for streaming response" - ); - } } diff --git a/crates/trusted-server-adapter-fastly/src/deadline_network.rs b/crates/trusted-server-adapter-fastly/src/deadline_network.rs new file mode 100644 index 0000000000..a5077e099a --- /dev/null +++ b/crates/trusted-server-adapter-fastly/src/deadline_network.rs @@ -0,0 +1,234 @@ +//! Auction deadline cases for `EdgeZero`'s Fastly client against a slow origin. +//! +//! `tests/deadline-origin.mjs` starts a local origin, writes its port into the +//! inline `deadline_fixture` config store, and runs the ignored `network_` +//! cases under Viceroy. Each case drives a batch from +//! [`HttpClient::start_batch_until`](edgezero_core::outbound::HttpClient::start_batch_until) +//! with no request lifecycle, so the batch gets a caller-owned scope and every +//! wait reaches the host select. + +mod tests { + use std::time::{Duration, Instant}; + + use edgezero_adapter_fastly::outbound::FastlyOutboundClient; + use edgezero_core::http::{Method, StatusCode}; + use edgezero_core::outbound::{ + HttpClient, OutboundBatchNext, OutboundBatchTermination, OutboundCachePolicy, + OutboundCollectedSlot, OutboundRequest, ResponseDecoding, + }; + use edgezero_core::time::Deadline; + use futures::executor::block_on; + use trusted_server_core::outbound::{ + BUFFERED_TIMEOUT, DEFAULT_BUFFERED_RESPONSE_BYTES, buffered, outbound_request, + }; + + fn client() -> HttpClient { + HttpClient::with_client(FastlyOutboundClient::new()) + } + + /// Builds a bidder-shaped request to `path` on the harness origin. + /// + /// The request is buffered (10 MiB cap) with cache bypass, like a bid + /// request. + fn fixture_request(method: Method, path: &str) -> OutboundRequest { + let port = fastly::ConfigStore::open("deadline_fixture") + .get("port") + .expect("should read the origin port the harness wrote"); + let target = outbound_request(method, &format!("http://127.0.0.1:{port}/{path}")) + .expect("should build the fixture request"); + buffered( + target, + BUFFERED_TIMEOUT, + DEFAULT_BUFFERED_RESPONSE_BYTES, + ResponseDecoding::Decode, + ) + .cache_policy(OutboundCachePolicy::Bypass) + } + + #[test] + #[ignore = "requires the local origin started by tests/deadline-origin.mjs"] + fn network_stalled_headers_end_at_cutoff() { + let request = fixture_request(Method::GET, "stalled"); + let started = Instant::now(); + let batch = + client().start_batch_until(vec![request], Deadline::after(Duration::from_millis(80))); + let (slots, termination) = block_on(batch.collect()) + .expect("should collect the stalled batch") + .into_parts(); + + assert_eq!( + termination, + OutboundBatchTermination::Cutoff, + "should end at the cutoff while headers stay pending" + ); + assert!( + matches!(slots.as_slice(), [OutboundCollectedSlot::Unresolved]), + "should leave the stalled slot unresolved, got {slots:?}" + ); + assert!( + started.elapsed() < Duration::from_millis(500), + "should bound the header wait at the cutoff" + ); + } + + #[test] + #[ignore = "requires the local origin started by tests/deadline-origin.mjs"] + fn network_trickling_body_ends_at_cutoff() { + let request = fixture_request(Method::GET, "trickle"); + let started = Instant::now(); + let batch = + client().start_batch_until(vec![request], Deadline::after(Duration::from_millis(80))); + let (slots, termination) = block_on(batch.collect()) + .expect("should collect the trickling batch") + .into_parts(); + + assert_eq!( + termination, + OutboundBatchTermination::Cutoff, + "should end at the cutoff before the trickling body completes" + ); + assert!( + matches!(slots.as_slice(), [OutboundCollectedSlot::Unresolved]), + "should leave the trickling slot unresolved, got {slots:?}" + ); + assert!( + started.elapsed() < Duration::from_millis(500), + "should bound body reads at the cutoff" + ); + } + + #[test] + #[ignore = "requires the local origin started by tests/deadline-origin.mjs"] + fn network_head_skips_declared_length() { + let request = fixture_request(Method::HEAD, "head"); + let batch = + client().start_batch_until(vec![request], Deadline::after(Duration::from_secs(1))); + let (slots, termination) = block_on(batch.collect()) + .expect("should collect the HEAD batch") + .into_parts(); + + assert_eq!( + termination, + OutboundBatchTermination::Completed, + "should complete before the cutoff" + ); + let [OutboundCollectedSlot::Resolved(head)] = slots.as_slice() else { + panic!("should resolve the HEAD slot, got {slots:?}"); + }; + let response = head + .outcome + .as_ref() + .expect("should accept a HEAD whose declared length exceeds the response cap"); + assert_eq!( + response.status(), + StatusCode::OK, + "should keep the HEAD status" + ); + } + + #[test] + #[ignore = "requires the local origin started by tests/deadline-origin.mjs"] + fn network_fast_sibling_completes() { + let requests = vec![ + fixture_request(Method::GET, "trickle"), + fixture_request(Method::GET, "fast"), + ]; + let batch = + client().start_batch_until(requests, Deadline::after(Duration::from_millis(200))); + let (slots, termination) = block_on(batch.collect()) + .expect("should collect the mixed batch") + .into_parts(); + + assert_eq!( + termination, + OutboundBatchTermination::Cutoff, + "should end at the cutoff while the trickling body is incomplete" + ); + let [ + OutboundCollectedSlot::Unresolved, + OutboundCollectedSlot::Resolved(fast), + ] = slots.as_slice() + else { + panic!("should resolve only the fast sibling, got {slots:?}"); + }; + let response = fast + .outcome + .as_ref() + .expect("should collect the fast sibling"); + assert_eq!( + response.body().as_bytes(), + Some(b"fast".as_slice()), + "should not let a trickling sibling starve the fast reply" + ); + } + + #[test] + #[ignore = "requires the local origin started by tests/deadline-origin.mjs"] + fn network_expired_snapshot_drops_reply_landed_during_sleep() { + let requests = vec![ + fixture_request(Method::GET, "fast"), + fixture_request(Method::GET, "stalled"), + ]; + let mut batch = + client().start_batch_until(requests, Deadline::after(Duration::from_millis(40))); + // The fast reply lands about 30 ms in, while the guest sleeps past the + // cutoff without driving the batch. + std::thread::sleep(Duration::from_millis(100)); + let woke = Instant::now(); + let next = block_on(batch.next()).expect("should observe the batch end"); + + assert!( + matches!( + next, + OutboundBatchNext::Finished(OutboundBatchTermination::Cutoff) + ), + "should drop the reply that landed during the sleep, got {next:?}" + ); + assert!( + woke.elapsed() < Duration::from_millis(100), + "should end at the expired cutoff without a host wait" + ); + } + + #[test] + #[ignore = "requires the local origin started by tests/deadline-origin.mjs"] + fn network_reply_observed_during_wait_before_cutoff_is_kept() { + let requests = vec![ + fixture_request(Method::GET, "stalled"), + fixture_request(Method::GET, "fast"), + ]; + let started = Instant::now(); + let mut batch = + client().start_batch_until(requests, Deadline::after(Duration::from_millis(300))); + let first = block_on(batch.next()).expect("should observe the fast reply"); + let observed = started.elapsed(); + let last = block_on(batch.next()).expect("should observe the batch end"); + + let OutboundBatchNext::Item(item) = first else { + panic!("should keep the reply observed before the cutoff, got {first:?}"); + }; + let (index, result) = item.into_parts(); + assert_eq!(index, 1, "should deliver the fast slot"); + let response = result.outcome.expect("should answer the fast slot"); + assert_eq!( + response.body().as_bytes(), + Some(b"fast".as_slice()), + "should keep the fast reply body" + ); + assert!( + observed < Duration::from_millis(300), + "should deliver the reply before the cutoff, took {observed:?}" + ); + assert!( + matches!( + last, + OutboundBatchNext::Finished(OutboundBatchTermination::Cutoff) + ), + "should end at the cutoff while the stalled sibling waits, got {last:?}" + ); + assert!( + started.elapsed() < Duration::from_millis(600), + "should bound the wait for the stalled sibling at the cutoff" + ); + } +} diff --git a/crates/trusted-server-adapter-fastly/src/delivery.rs b/crates/trusted-server-adapter-fastly/src/delivery.rs new file mode 100644 index 0000000000..d5a524bad6 --- /dev/null +++ b/crates/trusted-server-adapter-fastly/src/delivery.rs @@ -0,0 +1,546 @@ +//! What the Fastly entry point does once `EdgeZero` has delivered a request. +//! +//! `EdgeZero` owns the response from the routed handler to the client and +//! reports how its delivery ended as a [`FastlyRequestDelivery`]. Every +//! delivery is handled here the same way, including the one +//! `send_assembly_error` returns. Each outcome selects exactly one +//! [`DeliveryAction`]: a native 500 when nothing reached the client, +//! post-delivery work after a committed application body, or nothing more. +//! Every failure log carries the sandbox counters, so it can be matched to +//! the response that carried them. + +use edgezero_adapter_fastly::delivery::{ + DeliveryError, FastlyNoAttemptReason, FastlyRequestDelivery, FastlyResponseCommitState, +}; +use edgezero_core::response_egress::ResponseEgressBodyKind; +use fastly::Response as FastlyResponse; +use fastly::http::StatusCode; + +use crate::sandbox::SandboxCounters; + +/// What the entry point still has to do for one delivered request. +#[derive(Debug, PartialEq, Eq)] +pub(crate) enum DeliveryAction { + /// Nothing reached the client, so it gets a native 500. + SendNativeError, + /// The application's body was committed, so post-delivery work runs with `S`. + RunPostDelivery(S), + /// The request is finished. No second response and no post-delivery work. + Done, +} + +/// Handles one delivery: logs its outcome, then sends a native 500 or runs `post`. +/// +/// `post` runs only after a committed application body, with the routed +/// finalize state, even when the body failed after commitment. +pub(crate) fn handle_delivery( + delivery: FastlyRequestDelivery, + counters: &SandboxCounters, + post: impl FnOnce(S), +) { + match delivery_action(delivery, counters) { + DeliveryAction::SendNativeError => { + FastlyResponse::from_status(StatusCode::INTERNAL_SERVER_ERROR) + .with_body_text_plain("Internal Server Error") + .send_to_client(); + } + DeliveryAction::RunPostDelivery(state) => post(state), + DeliveryAction::Done => {} + } +} + +/// Logs `delivery`'s outcome and selects its action, one arm per outcome-table row. +pub(crate) fn delivery_action( + delivery: FastlyRequestDelivery, + counters: &SandboxCounters, +) -> DeliveryAction { + let context = counters.log_context(); + let may_send_native_response = delivery.may_send_native_response(); + match delivery { + FastlyRequestDelivery::NoAttempt(failure) => { + match failure.reason() { + FastlyNoAttemptReason::RequestHeadConversion(cause) => log::error!( + "request head conversion failed ({cause:?}); sending a native 500{context}" + ), + FastlyNoAttemptReason::RequiredStore(reason) => log::error!( + "required store failed to open ({reason:?}), although Trusted Server declares none; sending a native 500{context}" + ), + reason => log::error!( + "request was not attempted: {}; sending a native 500{context}", + reason.message() + ), + } + DeliveryAction::SendNativeError + } + FastlyRequestDelivery::Routed { state, delivery } + if delivery.commitment() == FastlyResponseCommitState::Committed + && delivery.report().body_kind == ResponseEgressBodyKind::Application => + { + if let Some(error) = delivery.error() { + log::warn!( + "application body failed after commitment: {}{context}", + error.message() + ); + } + DeliveryAction::RunPostDelivery(state) + } + FastlyRequestDelivery::Routed { delivery, .. } + | FastlyRequestDelivery::Detached(delivery) + if may_send_native_response => + { + log::error!( + "response was not committed ({}); sending a native 500{context}", + delivery.error().map_or("no error", DeliveryError::message) + ); + DeliveryAction::SendNativeError + } + FastlyRequestDelivery::Routed { delivery, .. } => { + log::error!( + "routed response ended {:?} with a {:?} body ({}); sending nothing more{context}", + delivery.commitment(), + delivery.report().body_kind, + delivery.error().map_or("no error", DeliveryError::message) + ); + DeliveryAction::Done + } + FastlyRequestDelivery::Detached(delivery) => { + if delivery.error() == Some(DeliveryError::Source) { + log::warn!( + "detached response body source failed: {:?}{context}", + delivery.report() + ); + } else { + log::debug!( + "delivered detached response: {:?}{context}", + delivery.report() + ); + } + DeliveryAction::Done + } + FastlyRequestDelivery::Aborted => { + log::error!( + "ingress admission aborted the request, although Trusted Server installs no abort policy{context}" + ); + DeliveryAction::Done + } + _ => { + log::error!("unrecognized Fastly request delivery{context}"); + DeliveryAction::Done + } + } +} + +#[cfg(test)] +pub(crate) mod tests { + use edgezero_adapter_fastly::FASTLY_PLATFORM; + use edgezero_adapter_fastly::delivery::testing::{ + Dispatch, deliver_with_hooks, response_delivery, + }; + use edgezero_adapter_fastly::delivery::{ + DeliveryError, FastlyNoAttemptReason, FastlyRequestHeadFailure, FastlyResponseCommitState, + FastlyResponseDelivery, + }; + use edgezero_core::app::App; + use edgezero_core::body::Body; + use edgezero_core::context::RequestContext; + use edgezero_core::error::EdgeError; + use edgezero_core::http::{Method, Response, request_builder}; + use edgezero_core::ingress::{ + IngressBeginOutcome, IngressDispatchOutcome, IngressFraming, IngressHeadAccounting, + IngressHeadParts, + }; + use edgezero_core::response_egress::{ + ResponseEgressEnvelope, ResponseEgressFallbackDisposition, ResponseEgressOutcome, + }; + use edgezero_core::response_egress_framing::ResponseEgressFramingMode; + use edgezero_core::router::RouterService; + use edgezero_core::time::MonotonicClock; + use futures::executor::block_on; + + use super::*; + + /// Finalize state the routed rows carry through to post-delivery work. + const ROUTED_STATE: u8 = 7; + + pub(crate) fn counters() -> SandboxCounters { + SandboxCounters { + ordinal: 3, + builds: 1, + request_id: "request-example".to_owned(), + } + } + + fn app() -> App { + let handler = |_ctx: RequestContext| async { + Ok::<_, EdgeError>(Response::new(Body::from("example page"))) + }; + App::with_name_and_platform( + RouterService::builder() + .route("/", Method::GET, handler) + .build(), + "delivery-test", + FASTLY_PLATFORM, + ) + } + + fn get(path: &str) -> edgezero_core::http::Request { + request_builder() + .method(Method::GET) + .uri(format!("https://publisher.example.com{path}")) + .body(Body::empty()) + .expect("should build the example request") + } + + /// A routed envelope from `App::dispatch_admitted`. + fn routed(app: &App) -> ResponseEgressEnvelope { + let request = get("/"); + let head = IngressHeadParts::from_request( + &request, + IngressHeadAccounting::HostManaged, + IngressFraming::HostManaged, + ); + let Ok(IngressBeginOutcome::Admitted(prepared)) = + app.begin_ingress(head, app.monotonic_now()) + else { + panic!("should admit the example request"); + }; + block_on(app.dispatch_admitted(prepared, request)) + } + + /// A detached head-limit envelope from `App::detached_ingress_error_egress`. + fn detached(app: &App) -> ResponseEgressEnvelope { + let IngressDispatchOutcome::Response(envelope) = app.detached_ingress_error_egress( + EdgeError::bad_request("example request head over its limit"), + Method::GET, + app.monotonic_now(), + ) else { + panic!("should answer a detached head-limit error"); + }; + *envelope + } + + /// A detached admitted-source-error envelope from `App::admitted_error_egress`. + fn admitted_source_error(app: &App) -> ResponseEgressEnvelope { + let request = get("/"); + let head = IngressHeadParts::from_request( + &request, + IngressHeadAccounting::HostManaged, + IngressFraming::HostManaged, + ); + let Ok(IngressBeginOutcome::Admitted(prepared)) = + app.begin_ingress(head, app.monotonic_now()) + else { + panic!("should admit the example request"); + }; + app.admitted_error_egress( + prepared, + EdgeError::internal(std::io::Error::other("example body source failed")), + ) + } + + /// Writes `envelope`'s application body to completion. + pub(crate) fn committed(envelope: ResponseEgressEnvelope) -> FastlyResponseDelivery { + let Ok((_prepared, _policy, mut attempt, clock)) = + envelope.begin(ResponseEgressFramingMode::HostAutomatic) + else { + panic!("should begin response egress"); + }; + assert!(attempt.begin_writing(), "should start writing"); + assert!(attempt.complete(clock.now()), "should complete the body"); + response_delivery( + FastlyResponseCommitState::Committed, + attempt.into_terminal_report(), + Ok(()), + ) + } + + /// Commits `envelope`'s application head, then fails its body with `error`. + pub(crate) fn committed_then_failed( + envelope: ResponseEgressEnvelope, + error: DeliveryError, + ) -> FastlyResponseDelivery { + let Ok((_prepared, _policy, mut attempt, clock)) = + envelope.begin(ResponseEgressFramingMode::HostAutomatic) + else { + panic!("should begin response egress"); + }; + assert!(attempt.begin_writing(), "should start writing"); + assert!( + attempt.terminate(error.outcome(), clock.now()), + "should fail the body" + ); + response_delivery( + FastlyResponseCommitState::Committed, + attempt.into_terminal_report(), + Err(error), + ) + } + + /// Replaces `envelope`'s body with the adapter fallback and delivers that. + pub(crate) fn committed_fallback(envelope: ResponseEgressEnvelope) -> FastlyResponseDelivery { + let Ok((_prepared, _policy, mut attempt, clock)) = + envelope.begin(ResponseEgressFramingMode::HostAutomatic) + else { + panic!("should begin response egress"); + }; + assert!( + attempt.begin_fallback(ResponseEgressOutcome::ConversionError), + "should select the fallback" + ); + assert!(attempt.begin_writing(), "should start writing the fallback"); + assert!( + attempt.finish_fallback(ResponseEgressFallbackDisposition::Completed, clock.now()), + "should finish the fallback" + ); + response_delivery( + FastlyResponseCommitState::Committed, + attempt.into_terminal_report(), + Ok(()), + ) + } + + /// Fails `envelope` before its head reaches the host. + pub(crate) fn not_committed(envelope: ResponseEgressEnvelope) -> FastlyResponseDelivery { + let Ok((_prepared, _policy, attempt, _clock)) = + envelope.begin(ResponseEgressFramingMode::HostAutomatic) + else { + panic!("should begin response egress"); + }; + response_delivery( + FastlyResponseCommitState::NotCommitted, + attempt.into_terminal_report(), + Err(DeliveryError::Conversion), + ) + } + + /// Hands `envelope`'s head to a host that rejects the send. + pub(crate) fn unknown_commit(envelope: ResponseEgressEnvelope) -> FastlyResponseDelivery { + let Ok((_prepared, _policy, mut attempt, _clock)) = + envelope.begin(ResponseEgressFramingMode::HostAutomatic) + else { + panic!("should begin response egress"); + }; + assert!(attempt.begin_writing(), "should start writing"); + response_delivery( + FastlyResponseCommitState::Unknown, + attempt.into_terminal_report(), + Err(DeliveryError::Transport), + ) + } + + pub(crate) fn routed_delivery( + deliver: impl FnOnce(ResponseEgressEnvelope) -> FastlyResponseDelivery, + ) -> FastlyRequestDelivery { + deliver_with_hooks(Dispatch::Routed(routed(&app())), |_| ROUTED_STATE, deliver) + } + + fn detached_delivery( + envelope: ResponseEgressEnvelope, + deliver: impl FnOnce(ResponseEgressEnvelope) -> FastlyResponseDelivery, + ) -> FastlyRequestDelivery { + deliver_with_hooks( + Dispatch::Detached(envelope), + |_| panic!("should never finalize a detached response"), + deliver, + ) + } + + fn no_attempt(reason: FastlyNoAttemptReason) -> FastlyRequestDelivery { + deliver_with_hooks( + Dispatch::NoAttempt(reason), + |_| panic!("should never finalize an unattempted request"), + |_| panic!("should never deliver an unattempted request"), + ) + } + + #[test] + fn delivery_no_attempt_head_conversion_sends_native_500() { + let delivery = no_attempt(FastlyNoAttemptReason::RequestHeadConversion( + FastlyRequestHeadFailure::InvalidUri, + )); + + assert_eq!( + delivery_action(delivery, &counters()), + DeliveryAction::SendNativeError, + "should answer an unconvertible request head with a native 500" + ); + } + + #[test] + fn delivery_no_attempt_required_store_sends_native_500() { + let delivery = no_attempt(FastlyNoAttemptReason::RequiredStore( + edgezero_core::key_value_store::KvStoreOpenFailure::MissingBinding, + )); + + assert_eq!( + delivery_action(delivery, &counters()), + DeliveryAction::SendNativeError, + "should answer the unreachable required-store failure with a native 500" + ); + } + + #[test] + fn delivery_routed_committed_application_body_runs_post_delivery() { + let delivery = routed_delivery(committed); + + assert_eq!( + delivery_action(delivery, &counters()), + DeliveryAction::RunPostDelivery(ROUTED_STATE), + "should run post-delivery work with the finalize state" + ); + } + + #[test] + fn delivery_routed_committed_application_body_with_error_runs_post_delivery() { + for error in [DeliveryError::Source, DeliveryError::Transport] { + let delivery = routed_delivery(|envelope| committed_then_failed(envelope, error)); + + assert_eq!( + delivery_action(delivery, &counters()), + DeliveryAction::RunPostDelivery(ROUTED_STATE), + "should still run post-delivery work after a {error:?} failure once committed" + ); + } + } + + #[test] + fn delivery_routed_not_committed_sends_native_500() { + let delivery = routed_delivery(not_committed); + + assert_eq!( + delivery_action(delivery, &counters()), + DeliveryAction::SendNativeError, + "should replace an uncommitted routed response with a native 500" + ); + } + + #[test] + fn delivery_detached_not_committed_sends_native_500() { + let delivery = detached_delivery(detached(&app()), not_committed); + + assert_eq!( + delivery_action(delivery, &counters()), + DeliveryAction::SendNativeError, + "should replace an uncommitted detached response with a native 500" + ); + } + + #[test] + fn delivery_routed_unknown_commit_sends_nothing_more() { + let delivery = routed_delivery(unknown_commit); + + assert_eq!( + delivery_action(delivery, &counters()), + DeliveryAction::Done, + "should neither resend nor run post-delivery work when commitment is unknown" + ); + } + + #[test] + fn delivery_routed_committed_fallback_body_sends_nothing_more() { + let delivery = routed_delivery(committed_fallback); + + assert_eq!( + delivery_action(delivery, &counters()), + DeliveryAction::Done, + "should run no post-delivery work for a delivered fallback" + ); + } + + #[test] + fn delivery_detached_committed_sends_nothing_more() { + for envelope in [detached(&app()), admitted_source_error(&app())] { + let delivery = detached_delivery(envelope, committed); + + assert_eq!( + delivery_action(delivery, &counters()), + DeliveryAction::Done, + "should only log a delivered detached response" + ); + } + } + + #[test] + fn delivery_detached_source_error_sends_nothing_more() { + let delivery = detached_delivery(detached(&app()), |envelope| { + committed_then_failed(envelope, DeliveryError::Source) + }); + + assert_eq!( + delivery_action(delivery, &counters()), + DeliveryAction::Done, + "should only log a detached response whose body source failed" + ); + } + + #[test] + fn delivery_aborted_sends_nothing_more() { + let delivery = deliver_with_hooks::( + Dispatch::Aborted, + |_| panic!("should never finalize an aborted request"), + |_| panic!("should never deliver an aborted request"), + ); + + assert_eq!( + delivery_action(delivery, &counters()), + DeliveryAction::Done, + "should never answer a request admission aborted" + ); + } + + #[test] + fn assembly_error_delivery_is_handled_like_any_other() { + let clock = MonotonicClock::default(); + let assembly = || { + App::assembly_error_egress( + EdgeError::service_unavailable("example settings unavailable"), + Method::GET, + clock.now(), + clock.clone(), + ) + }; + + let delivered = deliver_with_hooks::<(), _, _>( + Dispatch::Detached(assembly()), + |_| panic!("should never finalize an assembly error"), + committed, + ); + let not_delivered = deliver_with_hooks::<(), _, _>( + Dispatch::Detached(assembly()), + |_| panic!("should never finalize an assembly error"), + not_committed, + ); + + assert_eq!( + delivery_action(delivered, &counters()), + DeliveryAction::Done, + "should only log a delivered assembly error" + ); + assert_eq!( + delivery_action(not_delivered, &counters()), + DeliveryAction::SendNativeError, + "should replace an uncommitted assembly error with a native 500" + ); + } + + #[test] + fn handle_delivery_runs_post_only_after_a_committed_application_body() { + let mut ran = Vec::new(); + + handle_delivery(routed_delivery(committed), &counters(), |state| { + ran.push(state); + }); + handle_delivery(routed_delivery(committed_fallback), &counters(), |state| { + ran.push(state); + }); + handle_delivery(routed_delivery(unknown_commit), &counters(), |state| { + ran.push(state); + }); + + assert_eq!( + ran, + vec![ROUTED_STATE], + "should run post-delivery work once, for the committed application body" + ); + } +} diff --git a/crates/trusted-server-adapter-fastly/src/main.rs b/crates/trusted-server-adapter-fastly/src/main.rs index 3cc13fdfa8..64a3728b1e 100644 --- a/crates/trusted-server-adapter-fastly/src/main.rs +++ b/crates/trusted-server-adapter-fastly/src/main.rs @@ -1,12 +1,16 @@ -use std::sync::Arc; +// Unit tests may buffer bodies with `Body::into_bytes`; production code must not. +#![cfg_attr(test, allow(clippy::disallowed_methods))] -use edgezero_adapter_fastly::config_store::FastlyConfigStore as EdgeZeroFastlyConfigStore; -use edgezero_adapter_fastly::request::into_core_request; -use edgezero_core::body::Body as EdgeBody; -use edgezero_core::config_store::ConfigStoreHandle; +use std::convert::Infallible; + +use edgezero_adapter_fastly::context::FastlyOutboundObservation; +use edgezero_adapter_fastly::request::{ + send_assembly_error, send_request_with_registries_and_hooks, +}; +use edgezero_core::app::Hooks as _; use edgezero_core::error::EdgeError; -use edgezero_core::http::{Request as HttpRequest, Response as HttpResponse}; -use edgezero_core::response::IntoResponse; +use edgezero_core::http::{Extensions, Request as HttpRequest, Response as HttpResponse}; +use edgezero_core::time::MonotonicClock; use error_stack::Report; use fastly::http::Method as FastlyMethod; use fastly::{Request as FastlyRequest, Response as FastlyResponse}; @@ -16,59 +20,55 @@ use trusted_server_core::constants::HEADER_X_TS_VERSION; use trusted_server_core::ec::device::DeviceSignals; use trusted_server_core::ec::finalize::ec_finalize_response; use trusted_server_core::ec::kv::KvIdentityGraph; -use trusted_server_core::ec::pull_sync::{ - PullSyncContext, build_pull_sync_context, dispatch_pull_sync, -}; use trusted_server_core::ec::registry::PartnerRegistry; use trusted_server_core::error::TrustedServerError; +use trusted_server_core::geo::GeoInfo; use trusted_server_core::integrations::RequestFilterEffects; -use trusted_server_core::platform::PlatformGeo as _; -use trusted_server_core::proxy::{AssetProxyCachePolicy, stream_asset_body}; +use trusted_server_core::proxy::AssetProxyCachePolicy; use trusted_server_core::response_privacy::TerminalPrivateResponse; use trusted_server_core::settings::Settings; use trusted_server_core::trace::{TraceTerminalResponse, is_trace_path}; -use trusted_server_core::version_header::{apply_git_version_header, git_version_header_value}; +use trusted_server_core::version_header::git_version_header_value; mod app; -mod backend; mod compat; +#[cfg(test)] +mod deadline_network; +mod delivery; mod ec_kv; mod esi_assembly; mod logging; mod management_api; mod middleware; mod platform; +mod post_delivery; mod rate_limiter; mod sandbox; mod template_cache; mod tinybird; use crate::app::{ - AppState, EcFinalizeState, RuntimeStoreConfig, TrustedServerApp, + EcFinalizeState, RuntimeStoreConfig, TrustedServerApp, assemble_app, load_settings_from_config_store, }; +use crate::delivery::handle_delivery; use crate::ec_kv::FastlyEcKvStore; -use crate::middleware::{HEADER_X_TS_FINALIZED, apply_finalize_headers, resolve_geo_for_response}; -use crate::platform::{FastlyPlatformGeo, client_info_from_request, resolve_client_ip}; -use crate::rate_limiter::{FastlyRateLimiter, RATE_COUNTER_NAME}; -use crate::sandbox::{RetainedApp, Sandbox, SandboxCounters, ServeMode, StartupDiagnostics}; +use crate::middleware::{ + HEADER_X_TS_FINALIZED, apply_finalize_headers, per_request_synthesized_headers, + resolve_geo_for_response, +}; +use crate::platform::{RequestGeo, client_info_from_request, resolve_client_ip}; +use crate::post_delivery::{FinalizeState, run_post_delivery}; +use crate::sandbox::{ + LogContext, RetainedApp, Sandbox, SandboxCounters, ServeMode, StartupDiagnostics, instance_id, + metrics_enabled, +}; +use crate::tinybird::TelemetryQueue; // Only the reuse path builds a serving loop, so the retirement snapshots have // no consumer in the default build. #[cfg(feature = "reusable-sandbox")] use crate::sandbox::RetirementCounters; -/// Opens the Fastly Config Store used by the `EdgeZero` dispatcher. -/// -/// # Errors -/// -/// Returns [`fastly::Error`] if the config store cannot be opened. -fn open_trusted_server_config_store(store_name: &str) -> Result { - let store = EdgeZeroFastlyConfigStore::try_open(store_name).map_err(|e| { - fastly::Error::msg(format!("failed to open config store `{store_name}`: {e}")) - })?; - Ok(ConfigStoreHandle::new(Arc::new(store))) -} - fn health_response(req: &FastlyRequest) -> Option { if req.get_method() == FastlyMethod::GET && req.get_path() == "/health" { let mut response = FastlyResponse::from_status(200).with_body_text_plain("ok"); @@ -85,9 +85,9 @@ fn health_response(req: &FastlyRequest) -> Option { /// Entry point for the Fastly Compute program. /// /// Uses an undecorated `main()` with `FastlyRequest::from_client()` instead of -/// `#[fastly::main]` so the `EdgeZero` streaming publisher path can call -/// [`fastly::Response::stream_to_client`] explicitly. It owns the sandbox -/// lifecycle and delegates each request to [`handle_request`]. +/// `#[fastly::main]` so the entry point owns the sandbox lifecycle. Each +/// request goes to [`handle_request`], which sends it through `EdgeZero`'s +/// canonical Fastly lifecycle. /// /// Without the `reusable-sandbox` feature this takes one request and returns, /// which is the original behaviour. With the feature, the bounds still have to @@ -106,11 +106,15 @@ fn main() { match mode { ServeMode::Single => { // `run_custom` completes the callback's SDK result exactly once. - // The callback sends its own response and returns `()`, so there - // is no error for the SDK to turn into a second response. + // The callback always returns `Ok`: `EdgeZero` or the callback has + // already answered, so there is no error for the SDK to turn into a + // second response. let Ok(()) = edgezero_adapter_fastly::lifecycle::run_custom( FastlyRequest::from_client(), - |request, sandbox: &mut Sandbox| handle_request(request, sandbox, &mut startup), + |request, sandbox: &mut Sandbox| { + handle_request(request, sandbox, &mut startup); + Ok::<(), Infallible>(()) + }, ); } ServeMode::Reuse(limits) => serve_loop(limits, startup), @@ -139,6 +143,9 @@ fn serve_loop(limits: crate::sandbox::SandboxLimits, mut startup: StartupDiagnos counters.observe(sandbox, |sandbox| { handle_request(request, sandbox, &mut startup); }); + // An `Err` would end serving with no response, so the callback + // reports none, whatever happened to this request's delivery. + Ok::<(), Infallible>(()) }, ); @@ -155,17 +162,22 @@ fn serve_loop(_limits: crate::sandbox::SandboxLimits, _startup: StartupDiagnosti unreachable!("serve_mode never selects reuse without the reusable-sandbox feature") } -/// Handles one request end to end, sending its own response. +/// Handles one request end to end. /// -/// Returns `()` rather than a response so the streaming publisher path can -/// call [`fastly::Response::stream_to_client`] itself. The SDK's -/// `HandlerResult` impl for `()` treats that as already sent. +/// Native short-circuits answer first. Every other request is assembled into +/// the sandbox's retained app and sent through `EdgeZero`, which owns the +/// response; [`handle_delivery`] then decides whether anything is left to do, +/// and [`log_outbound_observation`] logs the request's backend registrations. /// -/// Every non-panicking path through this function must send exactly once. In a -/// reused sandbox the SDK refuses to wait for the next request until the -/// current one is complete, so a missed send stalls the loop rather than +/// Every non-panicking path through this function must answer exactly once. +/// In a reused sandbox the SDK refuses to wait for the next request until the +/// current one is complete, so a missed answer stalls the loop rather than /// merely dropping one response. fn handle_request(req: FastlyRequest, sandbox: &mut Sandbox, startup: &mut StartupDiagnostics) { + // Taken at entry, so the assembly-error response and every deadline share + // one clock domain with `EdgeZero`'s. + let clock = MonotonicClock::default(); + let request_start = clock.now(); // The framework counts the callback before invoking it, including early // returns, so this is already this request's 1-based ordinal. let ordinal = sandbox.requests(); @@ -203,145 +215,13 @@ fn handle_request(req: FastlyRequest, sandbox: &mut Sandbox, startup: &mut Start .map(str::to_owned) .unwrap_or_else(|| format!("{}-{ordinal}", instance_id())); - edgezero_main(req, sandbox, ordinal, &request_id, trace_path); -} - -/// Builds the counters snapshot response. -/// -/// A snapshot only. It reports the sandbox that served *this* probe, which is -/// not necessarily the sandbox that served any preceding workload request, so -/// reuse is established from the counters attached to workload responses -/// rather than from polling this. -/// -/// `ordinal` is this probe's own 1-based position in the sandbox, which is -/// also the number of callbacks the sandbox has served including this one. -/// The lifetime count is therefore not reported separately. -#[cfg(feature = "reusable-sandbox")] -fn sandbox_metrics_response(sandbox: &Sandbox, ordinal: u64) -> FastlyResponse { - let body = serde_json::json!({ - "instance": instance_id(), - "ordinal": ordinal, - "builds": sandbox.initialization_attempts(), - }); - - FastlyResponse::from_status(fastly::http::StatusCode::OK) - .with_header("cache-control", "private, no-store") - .with_body_json(&body) - .unwrap_or_else(|e| { - log::error!("failed to serialize sandbox metrics: {e}"); - FastlyResponse::from_status(fastly::http::StatusCode::INTERNAL_SERVER_ERROR) - }) -} - -/// Attaches sandbox counters only to private, no-store workload responses. -/// -/// Called before headers are committed, which on the streaming path means -/// before `stream_to_client`. The counters therefore describe the request up -/// to commitment and cannot report its eventual outcome; a failure after -/// commitment is recorded in logs instead and reconciled during analysis. -fn attach_sandbox_counters(response: &mut HttpResponse, counters: &SandboxCounters) { - // Per-request measurements must never be replayed from a cache. Requiring - // no-store also excludes browser-cacheable and field-qualified privacy. - if !cache_control_headers_have_directive(response.headers(), "private") - || !cache_control_headers_have_directive(response.headers(), "no-store") - { - return; - } - - let headers = response.headers_mut(); - for (name, value) in [ - (sandbox::HEADER_SANDBOX_INSTANCE, instance_id()), - ( - sandbox::HEADER_SANDBOX_ORDINAL, - counters.ordinal.to_string(), - ), - (sandbox::HEADER_SANDBOX_BUILDS, counters.builds.to_string()), - ( - sandbox::HEADER_SANDBOX_REQUEST_ID, - counters.request_id.clone(), - ), - (sandbox::HEADER_SANDBOX_VCPU_MS, vcpu_ms()), - (sandbox::HEADER_SANDBOX_HEAP_MIB, heap_mib()), - ] { - match edgezero_core::http::HeaderValue::from_str(&value) { - Ok(value) => { - headers.insert(name, value); - } - Err(e) => log::warn!("sandbox counter `{name}` is not a valid header value: {e}"), - } - } -} - -/// Resolves how this sandbox will serve requests. -/// -/// Without the `reusable-sandbox` feature this is unconditionally -/// [`ServeMode::Single`] and reads nothing, so the default build does no -/// startup work the original entry point did not do. -/// Returns the mode alongside diagnostics that must wait for the logger. -/// -/// This runs before any logger exists, so the reasons reuse was declined are -/// carried out rather than logged here, where they would be discarded. -#[cfg(feature = "reusable-sandbox")] -fn serve_mode() -> (ServeMode, Vec) { - let (raw, diagnostics) = crate::sandbox::read_raw_limits(); - (crate::sandbox::resolve_mode(raw), diagnostics) -} - -#[cfg(not(feature = "reusable-sandbox"))] -fn serve_mode() -> (ServeMode, Vec) { - (ServeMode::Single, Vec::new()) -} - -/// Guest-instance identifier used to attribute requests to a sandbox. -/// -/// `FASTLY_TRACE_ID` describes the sandbox, which is exactly what is wanted -/// here and exactly why it must not be used as a request id. An absent value -/// is reported rather than synthesized, so a measurement run cannot silently -/// claim reuse it never observed. -fn instance_id() -> String { - // The SDK documents this as the per-sandbox identifier; on wasm32-wasip1 - // it resolves to `FASTLY_TRACE_ID`, which is why that value must never be - // used as a request id. - let id = fastly::compute_runtime::sandbox_id(); - if id.is_empty() { - return sandbox::INSTANCE_ID_UNAVAILABLE.to_owned(); - } - id.to_owned() -} - -/// Cumulative guest vCPU milliseconds, or a marker when unsupported. -fn vcpu_ms() -> String { - fastly::compute_runtime::elapsed_vcpu_ms().map_or_else( - |_| sandbox::COUNTER_UNSUPPORTED.to_owned(), - |v| v.to_string(), - ) -} - -/// Guest heap snapshot in MiB, or a marker when unsupported. -fn heap_mib() -> String { - fastly::compute_runtime::heap_memory_snapshot_mib().map_or_else( - |_| sandbox::COUNTER_UNSUPPORTED.to_owned(), - |v| v.to_string(), - ) -} - -/// Handles a request through the `EdgeZero` router path. -fn edgezero_main( - mut req: FastlyRequest, - sandbox: &mut Sandbox, - ordinal: u64, - request_id: &str, - trace_path: bool, -) { - let runtime_stores = RuntimeStoreConfig::logical(); - // Short-circuit the sandbox counters probe before app construction. It must // not build the application: polling it would otherwise increment the very // build counter it reports. #[cfg(feature = "reusable-sandbox")] if req.get_method() == FastlyMethod::GET && req.get_path() == sandbox::SANDBOX_METRICS_PATH { - match load_settings_from_config_store(&runtime_stores) { - Ok(settings) if sandbox::metrics_enabled(&settings) => { + match load_settings_from_config_store(&RuntimeStoreConfig::logical()) { + Ok(settings) if metrics_enabled(&settings) => { sandbox_metrics_response(sandbox, ordinal).send_to_client(); } Ok(_) => { @@ -361,7 +241,7 @@ fn edgezero_main( // because TLS/JA4 accessors are only available on FastlyRequest before // conversion to edgezero types. if !trace_path && req.get_method() == FastlyMethod::GET && req.get_path() == "/_ts/debug/ja4" { - match load_settings_from_config_store(&runtime_stores) { + match load_settings_from_config_store(&RuntimeStoreConfig::logical()) { Ok(settings) if settings.debug.ja4_endpoint_enabled => { build_ja4_debug_response(&req).send_to_client(); } @@ -378,61 +258,82 @@ fn edgezero_main( return; } - let config_store = - match open_trusted_server_config_store(runtime_stores.config_store_name.as_ref()) { - Ok(cs) => cs, - Err(e) => { - log::error!("failed to open config store: {e}"); - FastlyResponse::from_status(fastly::http::StatusCode::INTERNAL_SERVER_ERROR) - .with_body_text_plain("Internal Server Error") - .send_to_client(); - return; - } - }; - - // Build lazily, once per sandbox. Reached only past the health, JA4, and + // Assemble lazily, once per sandbox. Reached only past the health, JA4, and // counters short-circuits, so none of those pays for construction. - // - // `initialize` builds only when the sandbox is empty, retains only - // success, and returns the error unchanged. A failed build hands back its - // error router as the error payload: that serves this request and is then - // dropped, so a transient config-store failure cannot pin the sandbox into - // permanent error mode, and the next callback retries construction. - let failed_build = sandbox - .initialize(|| { - let (app, state) = TrustedServerApp::build_app_with_state(&runtime_stores); - match state { - Some(state) => Ok(RetainedApp { app, state }), - None => Err(app), - } - }) - .err(); - - let (app, app_state): (&edgezero_core::app::App, Option>) = - match (failed_build.as_ref(), sandbox.state()) { - (Some(app), _) => (app, None), - (None, Some(retained)) => (&retained.app, Some(Arc::clone(&retained.state))), - (None, None) => { - log::error!("no application available after initialization"); - FastlyResponse::from_status(fastly::http::StatusCode::INTERNAL_SERVER_ERROR) - .with_body_text_plain("Internal Server Error") - .send_to_client(); - return; - } - }; + // `initialize` retains only success and returns the error unchanged, so a + // transient config-store failure answers this request and the next + // callback retries assembly. + let assembled = sandbox.initialize(|| { + assemble_app( + load_settings_from_config_store(&RuntimeStoreConfig::logical()), + &LogContext::new(ordinal, &request_id), + ) + }); + let counters = SandboxCounters::capture(sandbox, ordinal, &request_id); + match assembled.and_then(|()| retained_app(sandbox)) { + Ok(retained) => serve(req, retained, &counters, trace_path), + Err(error) => handle_delivery( + send_assembly_error(req, error, request_start, clock), + &counters, + |()| {}, + ), + } + log_outbound_observation(); +} - let settings_snapshot = app_state.as_ref().map(|state| Arc::clone(&state.settings)); - let counters = - SandboxCounters::capture(sandbox, ordinal, request_id, settings_snapshot.as_deref()); - let trusted_client_ip = settings_snapshot - .as_deref() - .and_then(|settings| settings.trusted_client_ip.as_ref()); +/// Logs this callback's backend registration counts at info. +/// +/// Runs after delivery and post-delivery work, so pull-sync and telemetry +/// registrations count. Hosted Fastly logs at info, so every request that +/// reached `EdgeZero` leaves one line. +fn log_outbound_observation() { + if let Some(observation) = FastlyOutboundObservation::current() { + let stats = observation.snapshot(); + log::info!( + target: "trusted_server::outbound_observation", + "configurations={} registrations={} total_registration_us={} max_registration_us={}", + stats.configuration_count(), + stats.registration_count(), + stats.total_registration_duration().as_micros(), + stats.max_registration_duration().as_micros() + ); + } +} + +/// The app a successful `initialize` retained. +/// +/// # Errors +/// +/// Returns an internal [`EdgeError`] if the sandbox retained nothing, which +/// the entry point answers like any other assembly error. +fn retained_app(sandbox: &Sandbox) -> Result<&RetainedApp, EdgeError> { + sandbox.state().ok_or_else(|| { + EdgeError::internal(std::io::Error::other( + "no application was retained after a successful build", + )) + }) +} + +/// Runs the native request work, sends the request through `EdgeZero`, and handles its delivery. +/// +/// The native work reads what only the SDK request exposes, before `EdgeZero` +/// converts it: the trusted forwarder, the resolved client IP, TLS metadata, +/// and device signals. `prepare` moves them into the request extensions, with +/// the telemetry queue and this request's synthesized headers. `finalize` +/// takes over the post-routing steps. Post-delivery work runs only after a +/// committed application body. +fn serve( + mut req: FastlyRequest, + retained: &RetainedApp, + counters: &SandboxCounters, + trace_path: bool, +) { + let settings = retained.state.settings.as_ref(); + let trusted_client_ip = settings.trusted_client_ip.as_ref(); // Preserve the public origin before native sanitation removes its source // fields. The frozen decision carries no authentication material. - let forwarder = settings_snapshot - .as_deref() - .map(|settings| compat::capture_fastly_forwarder(&mut req, settings)); + let forwarder = compat::capture_fastly_forwarder(&mut req, settings); // Resolve the trusted client IP, then strip client-spoofable forwarded // headers before dispatch. One call keeps resolution ahead of the @@ -471,169 +372,219 @@ fn edgezero_main( } } - // Capture metadata from the original FastlyRequest before conversion. These - // accessors only return real values on the client request, so store them in - // request extensions for build_per_request_services and EC bot classification. + // These accessors only return real values on the client request, so they + // are read here and carried in the request extensions for + // build_per_request_services and EC bot classification. let client_info = client_info_from_request(&req, resolved_client_ip); - let client_ip = client_info.client_ip; let device_signals = (!trace_path).then(|| derive_device_signals(&req)); - - // Dispatch directly through the EdgeZero router without an intermediate - // fastly::Response conversion. That preserves duplicate header values such - // as multiple Set-Cookie headers. - let mut response = match into_core_request(req) { - Ok(mut core_req) => { - if let Some(forwarder) = forwarder { - core_req.extensions_mut().insert(forwarder); - } - core_req.extensions_mut().insert(config_store); - if let Some(device_signals) = device_signals { - core_req.extensions_mut().insert(device_signals); - } - core_req.extensions_mut().insert(client_info); - match futures::executor::block_on(app.router().oneshot(core_req)) { - Ok(response) => response, - Err(error) => edge_error_response(error), - } - } - Err(e) => { - log::error!("EdgeZero request conversion failed: {e}"); - FastlyResponse::from_status(fastly::http::StatusCode::INTERNAL_SERVER_ERROR) - .with_body_text_plain("Internal Server Error") - .send_to_client(); - return; + // The request's one geo lookup. It serves the synthesized headers and + // `finalize` here, and, from the request extensions, the finalize + // middleware, the trace metadata and the per-request services. + let request_geo = RequestGeo::lookup_once(client_info.client_ip); + let queue = TelemetryQueue::default(); + + let prepare = |_req: &mut FastlyRequest, extensions: &mut Extensions| { + extensions.insert(forwarder); + if let Some(device_signals) = device_signals { + extensions.insert(device_signals); } + extensions.insert(client_info); + extensions.insert(queue.clone()); + extensions.insert(per_request_synthesized_headers(settings, request_geo.geo())); + extensions.insert(request_geo.clone()); + }; + let inputs = FinalizeInputs { + settings, + geo: request_geo.geo(), + counters, }; + let delivery = send_request_with_registries_and_hooks( + &retained.app, + TrustedServerApp::stores(), + req, + prepare, + |response: &mut HttpResponse| finalize_response(response, &inputs), + ); + handle_delivery(delivery, counters, |state| { + run_post_delivery(state, &queue, settings, &counters.log_context()); + }); +} +/// What the `finalize` step reads besides the routed response. +struct FinalizeInputs<'a> { + settings: &'a Settings, + /// The geo lookup the entry point ran once for this request. + geo: Option<&'a GeoInfo>, + counters: &'a SandboxCounters, +} + +/// Takes over the post-routing steps for a routed response, before egress policy and framing. +/// +/// `EdgeZero` runs it for every routed response, handler errors and its +/// canonical 404 and 405 included. Detached responses never reach it. A trace +/// response is returned untouched. Otherwise it applies the finalize headers +/// unless the middleware already did, the asset cache policy, EC +/// finalization, the terminal privacy effects, and, with metrics enabled, the +/// sandbox counters. +/// +/// Returns the pull-sync inputs post-delivery work needs, or `None` when the +/// response carries no EC state or its partner registry cannot be built. +fn finalize_response( + response: &mut HttpResponse, + inputs: &FinalizeInputs<'_>, +) -> Option { + // Trace responses keep their exact hardened headers; no finalizer touches them. if response .extensions() .get::() .is_some() { - // Trace responses keep their exact hardened headers; no sandbox counters. - send_response_to_client(response, None); - return; + return None; } - // Pop response extensions before the Fastly conversion, which drops them. let ec_state = response.extensions_mut().remove::(); let asset_cache_policy = response.extensions_mut().remove::(); let request_filter_effects = response.extensions_mut().remove::(); - if !take_finalize_sentinel(&mut response) { - if let Some(settings) = settings_snapshot.as_deref() { - apply_entry_point_finalize_headers(settings, &mut response, client_ip); - } else { - match load_settings_from_config_store(&runtime_stores) { - Ok(settings) => { - apply_entry_point_finalize_headers(&settings, &mut response, client_ip); - } - Err(e) => { - log::warn!("entry-point finalize skipped: failed to reload settings: {e:?}"); - // The compiled-in version needs no settings, so a deploy that cannot - // load its config still reports which build is serving. - apply_git_version_header(&mut response); - } - } - } + if !take_finalize_sentinel(response) { + let geo = resolve_geo_for_response(response, inputs.geo); + apply_finalize_headers(inputs.settings, geo, response); } if let Some(policy) = asset_cache_policy { - policy.apply_after_route_finalization(&mut response, EdgeCacheHeader::SurrogateControl); + policy.apply_after_route_finalization(response, EdgeCacheHeader::SurrogateControl); } - if let Some(mut ec_state) = ec_state { - if let Some(settings) = settings_snapshot.as_deref() { - match apply_edgezero_ec_finalize(settings, &mut ec_state, &mut response) { - Ok(partner_registry) => { - send_edgezero_response( - response, - request_filter_effects.as_ref(), - counters.as_ref(), - ); - run_edgezero_pull_sync_after_send(settings, &partner_registry, &ec_state); - return; - } - Err(e) => { - log::error!( - "EdgeZero EC finalize skipped: failed to build partner registry: {e:?}" - ); - } - } - } else { - match load_settings_from_config_store(&runtime_stores) { - Ok(settings) => { - match apply_edgezero_ec_finalize(&settings, &mut ec_state, &mut response) { - Ok(partner_registry) => { - send_edgezero_response( - response, - request_filter_effects.as_ref(), - counters.as_ref(), - ); - run_edgezero_pull_sync_after_send( - &settings, - &partner_registry, - &ec_state, - ); - return; - } - Err(e) => { - log::error!( - "EdgeZero EC finalize skipped: failed to build partner registry: {e:?}" - ); - } - } - } - Err(e) => { - log::warn!("EdgeZero EC finalize skipped: failed to reload settings: {e:?}"); - } + let state = ec_state.and_then(|mut ec_state| { + match apply_edgezero_ec_finalize(inputs.settings, &mut ec_state, response) { + Ok(partner_registry) => Some(FinalizeState { + partner_registry, + ec_state, + }), + Err(e) => { + log::error!( + "EC finalize skipped: failed to build partner registry: {e:?}{}", + inputs.counters.log_context() + ); + None } } + }); + + apply_terminal_response_effects(response, request_filter_effects.as_ref()); + if metrics_enabled(inputs.settings) { + attach_sandbox_counters(response, inputs.counters); } + state +} + +/// Builds the counters snapshot response. +/// +/// A snapshot only. It reports the sandbox that served *this* probe, which is +/// not necessarily the sandbox that served any preceding workload request, so +/// reuse is established from the counters attached to workload responses +/// rather than from polling this. +/// +/// `ordinal` is this probe's own 1-based position in the sandbox, which is +/// also the number of callbacks the sandbox has served including this one. +/// The lifetime count is therefore not reported separately. +#[cfg(feature = "reusable-sandbox")] +fn sandbox_metrics_response(sandbox: &Sandbox, ordinal: u64) -> FastlyResponse { + let body = serde_json::json!({ + "instance": instance_id(), + "ordinal": ordinal, + "builds": sandbox.initialization_attempts(), + }); - send_edgezero_response(response, request_filter_effects.as_ref(), counters.as_ref()); + FastlyResponse::from_status(fastly::http::StatusCode::OK) + .with_header("cache-control", "private, no-store") + .with_body_json(&body) + .unwrap_or_else(|e| { + log::error!("failed to serialize sandbox metrics: {e}"); + FastlyResponse::from_status(fastly::http::StatusCode::INTERNAL_SERVER_ERROR) + }) } -fn edge_error_response(error: EdgeError) -> HttpResponse { - log::error!("EdgeZero router returned error: {error:?}"); - match error.into_response() { - Ok(response) => response, - Err(error) => { - log::error!("failed to convert EdgeZero error into response: {error:?}"); - edgezero_core::http::response_builder() - .status(edgezero_core::http::StatusCode::INTERNAL_SERVER_ERROR) - .body(EdgeBody::from("Internal Server Error")) - .expect("should build EdgeZero error response") +/// Attaches sandbox counters only to private, no-store workload responses. +/// +/// Called from `finalize`, before headers are committed. The counters +/// therefore describe the request up to commitment and cannot report its +/// eventual outcome; a failure after commitment is recorded in logs instead, +/// with the same counters, and reconciled during analysis. +fn attach_sandbox_counters(response: &mut HttpResponse, counters: &SandboxCounters) { + // Per-request measurements must never be replayed from a cache. Requiring + // no-store also excludes browser-cacheable and field-qualified privacy. + if !cache_control_headers_have_directive(response.headers(), "private") + || !cache_control_headers_have_directive(response.headers(), "no-store") + { + return; + } + + let headers = response.headers_mut(); + for (name, value) in [ + (sandbox::HEADER_SANDBOX_INSTANCE, instance_id()), + ( + sandbox::HEADER_SANDBOX_ORDINAL, + counters.ordinal.to_string(), + ), + (sandbox::HEADER_SANDBOX_BUILDS, counters.builds.to_string()), + ( + sandbox::HEADER_SANDBOX_REQUEST_ID, + counters.request_id.clone(), + ), + (sandbox::HEADER_SANDBOX_VCPU_MS, vcpu_ms()), + (sandbox::HEADER_SANDBOX_HEAP_MIB, heap_mib()), + ] { + match edgezero_core::http::HeaderValue::from_str(&value) { + Ok(value) => { + headers.insert(name, value); + } + Err(e) => log::warn!("sandbox counter `{name}` is not a valid header value: {e}"), } } } +/// Resolves how this sandbox will serve requests. +/// +/// Without the `reusable-sandbox` feature this is unconditionally +/// [`ServeMode::Single`] and reads nothing, so the default build does no +/// startup work the original entry point did not do. +/// Returns the mode alongside diagnostics that must wait for the logger. +/// +/// This runs before any logger exists, so the reasons reuse was declined are +/// carried out rather than logged here, where they would be discarded. +#[cfg(feature = "reusable-sandbox")] +fn serve_mode() -> (ServeMode, Vec) { + let (raw, diagnostics) = crate::sandbox::read_raw_limits(); + (crate::sandbox::resolve_mode(raw), diagnostics) +} + +#[cfg(not(feature = "reusable-sandbox"))] +fn serve_mode() -> (ServeMode, Vec) { + (ServeMode::Single, Vec::new()) +} + +/// Cumulative guest vCPU milliseconds, or a marker when unsupported. +fn vcpu_ms() -> String { + fastly::compute_runtime::elapsed_vcpu_ms().map_or_else( + |_| sandbox::COUNTER_UNSUPPORTED.to_owned(), + |v| v.to_string(), + ) +} + +/// Guest heap snapshot in MiB, or a marker when unsupported. +fn heap_mib() -> String { + fastly::compute_runtime::heap_memory_snapshot_mib().map_or_else( + |_| sandbox::COUNTER_UNSUPPORTED.to_owned(), + |v| v.to_string(), + ) +} + fn take_finalize_sentinel(response: &mut HttpResponse) -> bool { response .headers_mut() .remove(HEADER_X_TS_FINALIZED) - .is_some() -} - -fn apply_entry_point_finalize_headers( - settings: &Settings, - response: &mut HttpResponse, - client_ip: Option, -) { - if response - .extensions() - .get::() - .is_some() - { - return; - } - let geo_info = resolve_geo_for_response(response, client_ip, |client_ip| { - FastlyPlatformGeo.lookup(client_ip).unwrap_or_else(|e| { - log::warn!("entry-point geo lookup failed: {e}"); - None - }) - }); - apply_finalize_headers(settings, geo_info.as_ref(), response); + .is_some() } fn apply_edgezero_ec_finalize( @@ -659,131 +610,11 @@ fn apply_edgezero_ec_finalize( Ok(partner_registry) } -fn run_edgezero_pull_sync_after_send( - settings: &Settings, - partner_registry: &PartnerRegistry, - ec_state: &EcFinalizeState, -) { - if !ec_state.is_real_browser { - return; - } - - let prepared_context = build_pull_sync_context(&ec_state.ec_context, partner_registry); - let Some((context, kv)) = - prepare_pull_sync_after_send(prepared_context, || require_identity_graph(settings)) - else { - return; - }; - - let limiter = FastlyRateLimiter::new(RATE_COUNTER_NAME); - dispatch_pull_sync( - settings, - &kv, - partner_registry, - &limiter, - &context, - &ec_state.services, - ); -} - -fn prepare_pull_sync_after_send( - context: Option, - graph_factory: F, -) -> Option<(PullSyncContext, KvIdentityGraph)> -where - F: FnOnce() -> Result>, -{ - let context = context?; - let kv = match graph_factory() { - Ok(kv) => kv, - Err(err) => { - log::debug!("Pull sync: identity graph unavailable, skipping: {err:?}"); - return None; - } - }; - Some((context, kv)) -} - -/// Sends a finalized `EdgeZero` response to the client. -/// -/// Streaming `EdgeZero` bodies commit headers first, then pipe chunks to Fastly's -/// client stream so large asset and publisher-origin responses do not -/// materialize in the Wasm heap. -fn send_edgezero_response( - mut response: HttpResponse, - request_filter_effects: Option<&RequestFilterEffects>, - counters: Option<&SandboxCounters>, -) { - apply_terminal_response_effects(&mut response, request_filter_effects); - - send_response_to_client(response, counters); -} - -/// Sends a response, attaching sandbox counters first when supplied. -fn send_response_to_client(mut response: HttpResponse, counters: Option<&SandboxCounters>) { - // Captured before the body is consumed so post-commitment failures can be - // matched back to the response that carried these counters. - let counter_context = counters.map_or_else(String::new, |counters| { - format!( - " [instance={} ordinal={} request={}]", - instance_id(), - counters.ordinal, - counters.request_id - ) - }); - - // Before headers commit, including before `stream_to_client` below. - if let Some(counters) = counters.as_ref() { - attach_sandbox_counters(&mut response, counters); - } - - let (parts, body) = response.into_parts(); - - match body { - EdgeBody::Stream(_) => { - let skeleton = compat::to_fastly_response_skeleton(HttpResponse::from_parts( - parts, - EdgeBody::empty(), - )); - let mut streaming_body = skeleton.stream_to_client(); - match futures::executor::block_on(stream_asset_body(body, &mut streaming_body)) { - Ok(()) => { - if let Err(e) = streaming_body.finish() { - // Also post-commitment: same attribution as the - // streaming failure below. - log::error!( - "failed to finish EdgeZero streaming body{counter_context}: {e}" - ); - } - } - Err(e) => { - // After commitment: log and stop. Returning an error here - // would let the SDK attempt a second response. Counters - // already went out with the headers, so the failure is - // tagged with the same identity for reconciliation. - log::error!("EdgeZero streaming failed{counter_context}: {e:?}"); - drop(streaming_body); - } - } - } - once => { - compat::to_fastly_response(HttpResponse::from_parts(parts, once)).send_to_client(); - } - } -} - /// Apply every late response mutation, then restore privacy invariants before headers commit. fn apply_terminal_response_effects( response: &mut HttpResponse, request_filter_effects: Option<&RequestFilterEffects>, ) { - if response - .extensions() - .get::() - .is_some() - { - return; - } let must_remain_private = response .extensions() .get::() @@ -901,16 +732,28 @@ pub(crate) fn derive_device_signals(req: &FastlyRequest) -> DeviceSignals { #[cfg(test)] mod tests { use super::*; + use std::sync::atomic::{AtomicBool, Ordering}; + use std::sync::{Arc, Mutex}; + use base64::Engine as _; + use edgezero_adapter_fastly::request::FastlyService; + use edgezero_core::app::App; use edgezero_core::body::Body as EdgeBody; + use edgezero_core::context::RequestContext; use edgezero_core::http::response_builder; use edgezero_core::http::{HeaderValue, Method, StatusCode}; use edgezero_core::ingress::{ CapturedTarget, HeaderFidelity, InboundOrigin, OriginSource, RequestIngress, TargetUnavailable, }; + use edgezero_core::pre_dispatch::PreDispatchRequest; + use edgezero_core::router::{PreDispatchHook, RouterService}; use fastly::mime; use trusted_server_core::integrations::HeaderMutation; + use trusted_server_core::test_support::logging::WarningCapture; + use trusted_server_core::trace::TracePreDispatchHook; + + use crate::delivery::tests::{committed, committed_then_failed}; fn test_settings() -> Settings { Settings::from_toml( @@ -938,8 +781,8 @@ mod tests { .expect("should parse test settings") } - #[test] - fn trace_dispatch_terminal_asset_bypasses_hostile_native_finalizers() { + /// [`test_settings`] with the trace page on, behind fictional credentials. + fn trace_settings() -> Settings { let mut settings = test_settings(); settings .integrations @@ -950,7 +793,37 @@ mod tests { .expect("should configure trace"); settings.handlers.insert(0,serde_json::from_value(serde_json::json!({"path":"^/_ts/trace","username":"example-user","password":"example-password"})) .expect("should configure trace protection")); - let mut request = edgezero_core::http::request_builder() + // Counters would land on a private, no-store response, so a trace + // response that kept its headers proves `finalize` never reached it. + settings.debug.sandbox_metrics_enabled = true; + settings + } + + fn counters() -> SandboxCounters { + SandboxCounters { + ordinal: 2, + builds: 1, + request_id: "request-example".to_owned(), + } + } + + /// Request filter effects that would break a trace response's contract. + fn hostile_effects() -> RequestFilterEffects { + RequestFilterEffects { + request_headers: Vec::new(), + response_headers: vec![ + HeaderMutation::set("content-security-policy", "default-src *"), + HeaderMutation::set("content-type", "text/plain"), + HeaderMutation::append("set-cookie", "example=operator"), + HeaderMutation::set("cache-control", "public, max-age=3600"), + ], + } + } + + #[test] + fn trace_response_passes_untouched() { + let settings = trace_settings(); + let (mut request, _body) = edgezero_core::http::request_builder() .uri("https://publisher.example.com/_ts/trace/assets/v1.js") .header( "authorization", @@ -961,67 +834,78 @@ mod tests { ), ) .body(EdgeBody::empty()) - .expect("should build protected asset request"); + .expect("should build protected asset request") + .into_parts(); let trusted_server_core::trace::TracePreflight::Ready(dispatch) = trusted_server_core::trace::preflight(&settings, &mut request) else { panic!("should accept protected fixed asset"); }; let mut response = dispatch.fixed_asset(b"example immutable asset"); + response.extensions_mut().insert(hostile_effects()); let original = response.headers().clone(); - let effects = RequestFilterEffects { - request_headers: Vec::new(), - response_headers: vec![ - HeaderMutation::set("content-security-policy", "default-src *"), - HeaderMutation::set("content-type", "text/plain"), - HeaderMutation::append("set-cookie", "example=operator"), - HeaderMutation::set("cache-control", "public, max-age=3600"), - ], - }; - apply_entry_point_finalize_headers(&settings, &mut response, None); - apply_terminal_response_effects(&mut response, Some(&effects)); + let counters = counters(); + + let state = finalize_response( + &mut response, + &FinalizeInputs { + settings: &settings, + geo: None, + counters: &counters, + }, + ); + + assert!(state.is_none(), "should hand no pull-sync inputs on"); assert_eq!( response.headers(), &original, - "should bypass every ordinary trace finalizer and preserve protected digest" - ); - let native = compat::to_fastly_response(response); - assert_eq!( - native.get_header("etag").map(HeaderValue::as_bytes), - original.get("etag").map(HeaderValue::as_bytes), - "should preserve strong ETag through native conversion" + "should bypass every ordinary finalizer and preserve the protected digest" ); assert!( - native.get_header("set-cookie").is_none(), + response.headers().get("set-cookie").is_none(), "should never acquire an operator cookie" ); } + /// Drives `request` through `hook` on a router whose routes record a continuation. + /// + /// # Panics + /// + /// Panics when the hook continues to a route instead of answering. + fn dispatch_through_hook( + hook: &Arc, + request: HttpRequest, + ) -> HttpResponse { + let continued = Arc::new(AtomicBool::new(false)); + let route_continued = Arc::clone(&continued); + let route = move |_ctx: RequestContext| { + route_continued.store(true, Ordering::SeqCst); + async { Ok::<_, EdgeError>(HttpResponse::new(EdgeBody::empty())) } + }; + let method = request.method().clone(); + let hook = Arc::clone(hook); + let router = RouterService::builder() + .pre_dispatch_hook_arc(hook) + .route("/", method.clone(), route.clone()) + .route("/{*rest}", method, route) + .build(); + let response = futures::executor::block_on(router.oneshot(request)) + .expect("should build local trace response"); + assert!( + !continued.load(Ordering::SeqCst), + "should intercept reserved request" + ); + response + } + #[test] - fn trace_dispatch_terminal_actions_errors_and_head_preserve_native_contract() { - let mut settings = test_settings(); - settings - .integrations - .insert_config( - "gpt_diagnostics", - &serde_json::json!({"enabled":true,"trace_page_enabled":true}), - ) - .expect("should configure trace"); - settings.handlers.insert(0,serde_json::from_value(serde_json::json!({"path":"^/_ts/trace","username":"example-user","password":"example-password"})) - .expect("should configure trace protection")); - let hook = trusted_server_core::trace::TracePreDispatchHook::new( + fn trace_terminal_actions_errors_and_head_pass_finalize_untouched() { + let settings = trace_settings(); + let hook = Arc::new(TracePreDispatchHook::new( Arc::new(settings.clone()), Arc::new(|_| panic!("should never query metadata for actions, errors or HEAD")), - ); - let effects = RequestFilterEffects { - request_headers: Vec::new(), - response_headers: vec![ - HeaderMutation::set("content-security-policy", "default-src *"), - HeaderMutation::set("content-type", "text/plain"), - HeaderMutation::append("set-cookie", "example=operator"), - HeaderMutation::set("cache-control", "public, max-age=3600"), - ], - }; + )); + let counters = counters(); for (method, path, authenticated, expected, action) in [ ( Method::GET, @@ -1103,71 +987,75 @@ mod tests { HeaderFidelity::default(), )); } - let mut response = futures::executor::block_on( - edgezero_core::router::PreDispatchHook::handle(&hook, &mut request), - ) - .expect("should build local trace response") - .expect("should intercept reserved request"); + let mut response = dispatch_through_hook(&hook, request); assert_eq!( response.status(), expected, "should preserve action and preflight status" ); + response.extensions_mut().insert(hostile_effects()); let original = response.headers().clone(); - apply_entry_point_finalize_headers(&settings, &mut response, None); - apply_terminal_response_effects(&mut response, Some(&effects)); + + let state = finalize_response( + &mut response, + &FinalizeInputs { + settings: &settings, + geo: None, + counters: &counters, + }, + ); + + assert!(state.is_none(), "should hand no pull-sync inputs on"); assert_eq!( response.headers(), &original, "should bypass hostile ordinary finalizers for every terminal response" ); - let mut native = compat::to_fastly_response(response); - for (name, value) in &original { - assert_eq!( - native.get_header(name.as_str()).map(HeaderValue::as_bytes), - Some(value.as_bytes()), - "should preserve complete trace headers through native conversion" - ); - } assert_eq!( - native - .get_headers() - .filter(|(name, _)| name.as_str() == "set-cookie") - .count(), + response.headers().get_all("set-cookie").iter().count(), usize::from(action.is_some()), "should emit only one deliberate action cookie and none on reads or errors" ); if head { - assert!( - native.take_body_bytes().is_empty(), - "should preserve bodyless trace HEAD through native conversion" - ); + let body = futures::executor::block_on(response.into_body().into_bytes_bounded(1)) + .expect("should read the trace HEAD body"); + assert!(body.is_empty(), "should keep the trace HEAD bodiless"); } } } - #[test] - fn pull_sync_noop_states_skip_post_send_graph_factory() { - let calls = std::cell::Cell::new(0); - let result = prepare_pull_sync_after_send(None, || { - calls.set(calls.get() + 1); - Err(Report::new(TrustedServerError::KvStore { - store_name: "unexpected".to_owned(), - message: "graph factory should not run".to_owned(), - })) - }); - assert!( - result.is_none(), - "a skipped pull-sync plan should return none" - ); - assert_eq!(calls.get(), 0, "should not invoke the graph factory"); + /// Records the path every request reaches the pre-dispatch hook with. + struct PathRecorder(Arc>>); + + #[async_trait::async_trait(?Send)] + impl PreDispatchHook for PathRecorder { + async fn handle( + &self, + request: &mut PreDispatchRequest, + ) -> Result, EdgeError> { + self.0 + .lock() + .expect("should record the hook path") + .push(request.uri().path().to_owned()); + Ok(Some(HttpResponse::new(EdgeBody::empty()))) + } } #[test] - fn trace_path_classification_survives_core_request_conversion() { - // `main` skips client-IP sanitization when the native path is a trace - // path; the router hook later classifies the converted URI. Both must - // agree, or unsanitized forwarded headers could reach ordinary dispatch. + fn trace_path_classification_survives_canonical_request_conversion() { + // The entry point skips client-IP sanitization when the native path is a + // trace path; the trace hook later classifies the path `EdgeZero`'s + // canonical conversion hands it. Both must agree, or unsanitized + // forwarded headers could reach ordinary dispatch. + let paths = Arc::new(Mutex::new(Vec::new())); + let app = App::from_router::( + RouterService::builder() + .pre_dispatch_hook_arc(Arc::new(PathRecorder(Arc::clone(&paths)))) + .build(), + edgezero_adapter_fastly::FASTLY_PLATFORM, + ) + .expect("should assemble the recording app"); + let mut hooked = Vec::new(); for url in [ "https://example.com/_ts/trace", "https://example.com/_ts/trace/", @@ -1187,12 +1075,150 @@ mod tests { ] { let req = FastlyRequest::get(url); let native = is_trace_path(req.get_path()); - let core = into_core_request(req).expect("should convert the native request"); + let _delivery = FastlyService::new(&app).run_request_with_hooks_for_test( + req, + |_req, _extensions| {}, + |_| (), + committed, + ); + + // Admission may refuse an odd target before the hook runs, which + // keeps it out of ordinary dispatch too. + let reached = paths.lock().expect("should read the hook paths").pop(); + if let Some(path) = reached { + assert_eq!( + is_trace_path(&path), + native, + "should classify {url} identically before and after conversion" + ); + hooked.push(url); + } + } + for url in [ + "https://example.com/_ts/trace", + "https://example.com/_ts/trace/enable?source=example", + "https://example.com/auction", + "https://example.com/", + ] { + assert!( + hooked.contains(&url), + "should reach the pre-dispatch hook with {url}" + ); + } + } + + #[test] + fn reused_sandbox_keeps_serving_after_delivery_failure() { + let mut sandbox = Sandbox::default(); + let settings = test_settings(); + let mut statuses = Vec::new(); + let mut post_delivery = 0; + + // Two requests through one sandbox, each the way `handle_request` + // runs them. The first fails after commitment, which must neither + // end serving nor drop the retained app. + for request in 0..2 { + sandbox + .initialize(|| assemble_app(Ok(test_settings()), &"")) + .expect("should assemble or reuse the app"); + let retained = retained_app(&sandbox).expect("should retain the app"); + let counters = SandboxCounters::capture(&sandbox, request + 1, "request-example"); + let inputs = FinalizeInputs { + settings: &settings, + geo: None, + counters: &counters, + }; + + let delivery = FastlyService::new(&retained.app).run_request_with_hooks_for_test( + FastlyRequest::get("https://publisher.example.com/static/tsjs=tsjs-unified.min.js"), + |_req, extensions| { + extensions.insert(RequestGeo::default()); + }, + |response: &mut HttpResponse| { + statuses.push(response.status()); + finalize_response(response, &inputs) + }, + |envelope| { + if request == 0 { + committed_then_failed( + envelope, + edgezero_adapter_fastly::delivery::DeliveryError::Transport, + ) + } else { + committed(envelope) + } + }, + ); + handle_delivery(delivery, &counters, |_state| post_delivery += 1); + } + + assert_eq!( + statuses, + [StatusCode::OK, StatusCode::OK], + "should route both requests through the retained app" + ); + assert_eq!( + sandbox.initialization_attempts(), + 1, + "should serve the second request from the first build" + ); + assert_eq!( + post_delivery, 2, + "should run post-delivery work after each committed application body, the one that failed after commitment included" + ); + } + + #[test] + fn skipped_operator_header_warns_once_per_request() { + let mut settings = test_settings(); + settings + .response_headers + .insert("connection".to_owned(), "x example".to_owned()); + let mut sandbox = Sandbox::default(); + sandbox + .initialize(|| assemble_app(Ok(settings.clone()), &"")) + .expect("should assemble the app"); + let retained = retained_app(&sandbox).expect("should retain the app"); + let counters = SandboxCounters::capture(&sandbox, 1, "request-example"); + let inputs = FinalizeInputs { + settings: &settings, + geo: None, + counters: &counters, + }; + + // A routed response, which the finalize middleware finalizes, and a + // canonical 405, which only the entry point's finalize step reaches. + for (label, request) in [ + ( + "routed", + FastlyRequest::get("https://publisher.example.com/static/tsjs=tsjs-unified.min.js"), + ), + ( + "canonical 405", + FastlyRequest::new(FastlyMethod::TRACE, "https://publisher.example.com/"), + ), + ] { + let capture = WarningCapture::start(); + let delivery = FastlyService::new(&retained.app).run_request_with_hooks_for_test( + request, + |_req, extensions| { + extensions.insert(per_request_synthesized_headers(&settings, None)); + extensions.insert(RequestGeo::default()); + }, + |response: &mut HttpResponse| finalize_response(response, &inputs), + committed, + ); + handle_delivery(delivery, &counters, |_state| {}); + + let skipped = capture + .take() + .into_iter() + .filter(|warning| warning.message.contains("`connection`")) + .count(); assert_eq!( - is_trace_path(core.uri().path()), - native, - "should classify {url} identically before and after conversion" + skipped, 1, + "should warn once about the skipped operator entry for the {label} request" ); } } @@ -1259,13 +1285,193 @@ mod tests { ); } + fn sample_geo() -> GeoInfo { + GeoInfo { + city: "San Francisco".to_owned(), + country: "US".to_owned(), + continent: "NorthAmerica".to_owned(), + latitude: 37.7749, + longitude: -122.4194, + metro_code: 807, + region: Some("CA".to_owned()), + asn: None, + } + } + #[test] - fn sandbox_counters_never_appear_on_cacheable_responses() { - let counters = SandboxCounters { - ordinal: 2, - builds: 1, - request_id: "request-example".to_owned(), + fn finalize_sets_ec_cookies_geo_headers_cache_policy_and_counters() { + let mut settings = test_settings(); + settings.debug.sandbox_metrics_enabled = true; + settings.response_headers = + std::iter::once(("x-operator".to_owned(), "kept".to_owned())).collect(); + let geo = sample_geo(); + let counters = counters(); + let inputs = FinalizeInputs { + settings: &settings, + geo: Some(&geo), + counters: &counters, + }; + + // A returning reader withdraws consent with GPC in a US state. + let ec_id = format!("{}.ABC123", "a".repeat(64)); + let request = edgezero_core::http::request_builder() + .uri("https://www.example.com/article") + .header("cookie", format!("ts-ec={ec_id}")) + .header("sec-gpc", "1") + .body(EdgeBody::empty()) + .expect("should build the reader request"); + let ec_context = trusted_server_core::ec::EcContext::read_from_request_with_geo( + &settings, + &request, + &crate::tinybird::tests::test_services(), + Some(&geo), + ) + .expect("should read the EC context"); + let mut page = response_builder() + .header("cache-control", "public, max-age=600") + .body(EdgeBody::empty()) + .expect("should build the page response"); + page.extensions_mut().insert(EcFinalizeState { + ec_context, + use_finalize_kv: false, + eids_cookie: None, + sharedid_cookie: None, + is_real_browser: true, + }); + + let state = finalize_response(&mut page, &inputs); + + assert!(state.is_some(), "should hand the EC state on for pull sync"); + let page_header = |name: &str| { + page.headers() + .get(name) + .and_then(|value| value.to_str().ok()) + .map(str::to_owned) + }; + let cookie = page_header("set-cookie").expect("should write the EC cookie"); + assert!( + cookie.starts_with("ts-ec=;") && cookie.contains("Max-Age=0"), + "should expire the withdrawn EC cookie, got {cookie}" + ); + assert_eq!( + page_header("x-geo-country").as_deref(), + Some("US"), + "should apply the request's geo lookup" + ); + assert_eq!( + page_header("x-operator").as_deref(), + Some("kept"), + "should apply the operator headers" + ); + assert_eq!( + page_header("cache-control").as_deref(), + Some("private, max-age=0"), + "should keep a cookie-bearing response out of shared caches" + ); + + // An asset response keeps its protected cache policy and, being + // private and no-store, carries the sandbox counters. + let mut asset = response_builder() + .header("cache-control", "public, max-age=600") + .body(EdgeBody::empty()) + .expect("should build the asset response"); + asset + .extensions_mut() + .insert(AssetProxyCachePolicy::NoStorePrivate); + + assert!( + finalize_response(&mut asset, &inputs).is_none(), + "should hand no pull-sync inputs on without EC state" + ); + let asset_header = |name: &str| { + asset + .headers() + .get(name) + .and_then(|value| value.to_str().ok()) }; + assert_eq!( + asset_header("cache-control"), + Some("no-store, private"), + "should apply the asset cache policy after the finalize headers" + ); + assert_eq!( + asset_header("x-ts-version"), + trusted_server_core::constants::TS_GIT_VERSION, + "should apply the version header" + ); + assert_eq!( + asset_header(sandbox::HEADER_SANDBOX_REQUEST_ID), + Some("request-example"), + "should attach the sandbox counters to the private response" + ); + assert_eq!( + asset_header(sandbox::HEADER_SANDBOX_ORDINAL), + Some("2"), + "should report the request ordinal" + ); + } + + #[test] + fn finalize_keeps_counters_off_without_metrics() { + let settings = test_settings(); + let counters = counters(); + let mut response = response_builder() + .header("cache-control", "private, no-store") + .body(EdgeBody::empty()) + .expect("should build response"); + + finalize_response( + &mut response, + &FinalizeInputs { + settings: &settings, + geo: None, + counters: &counters, + }, + ); + + assert!( + response + .headers() + .get(sandbox::HEADER_SANDBOX_REQUEST_ID) + .is_none(), + "counters must stay off responses in the default configuration" + ); + } + + #[test] + fn finalize_skips_headers_the_middleware_already_set() { + let settings = test_settings(); + let counters = counters(); + let mut response = response_builder() + .header(HEADER_X_TS_FINALIZED, "1") + .body(EdgeBody::empty()) + .expect("should build response"); + + finalize_response( + &mut response, + &FinalizeInputs { + settings: &settings, + geo: None, + counters: &counters, + }, + ); + + assert!( + response.headers().get(HEADER_X_TS_FINALIZED).is_none(), + "should strip the sentinel" + ); + assert!( + response + .headers() + .get(trusted_server_core::constants::HEADER_X_GEO_INFO_AVAILABLE) + .is_none(), + "should not finalize a middleware-finalized response twice" + ); + } + + #[test] + fn sandbox_counters_never_appear_on_cacheable_responses() { + let counters = counters(); for policy in [ None, Some("public, s-maxage=3600"), @@ -1295,11 +1501,7 @@ mod tests { #[test] fn sandbox_counters_follow_final_private_no_store_policy() { - let counters = SandboxCounters { - ordinal: 2, - builds: 1, - request_id: "request-example".to_owned(), - }; + let counters = counters(); let mut response = response_builder() .header("cache-control", "public, s-maxage=3600") .body(EdgeBody::empty()) @@ -1325,7 +1527,6 @@ mod tests { "should retain terminal privacy" ); } - #[test] fn late_filter_effects_cannot_make_an_assembled_response_public() { let mut response = response_builder() @@ -1508,19 +1709,30 @@ mod tests { } #[test] - #[allow(clippy::panic)] - fn entry_point_finalize_skips_geo_lookup_for_401() { + fn entry_point_finalize_withholds_geo_from_401() { let settings = test_settings(); + let request_geo = GeoInfo { + city: "Springfield".to_owned(), + country: "US".to_owned(), + continent: "NorthAmerica".to_owned(), + latitude: 39.78, + longitude: -89.65, + metro_code: 648, + region: Some("IL".to_owned()), + asn: None, + }; let mut response = response_builder() .status(edgezero_core::http::StatusCode::UNAUTHORIZED) .body(EdgeBody::empty()) .expect("should build response"); - let geo_info = resolve_geo_for_response(&response, None, |_| { - panic!("should skip entry-point geo lookup for 401 responses"); - }); - apply_finalize_headers(&settings, geo_info.as_ref(), &mut response); + let geo = resolve_geo_for_response(&response, Some(&request_geo)); + apply_finalize_headers(&settings, geo, &mut response); + assert!( + geo.is_none(), + "should withhold the request's geo from a 401" + ); assert_eq!( response .headers() diff --git a/crates/trusted-server-adapter-fastly/src/management_api.rs b/crates/trusted-server-adapter-fastly/src/management_api.rs index a8469bcca7..9a035a7910 100644 --- a/crates/trusted-server-adapter-fastly/src/management_api.rs +++ b/crates/trusted-server-adapter-fastly/src/management_api.rs @@ -5,32 +5,43 @@ //! Used by [`super::platform::FastlyPlatformConfigStore`] and //! [`super::platform::FastlyPlatformSecretStore`] to back store write methods. //! +//! Each call goes through the request's `EdgeZero` [`HttpClient`] as a +//! buffered request with the one second connect timeout, the +//! [`BUFFERED_TIMEOUT`] total timeout, a 1 MiB response cap, and a decoded +//! response body. +//! //! # Credentials //! //! The Fastly API token is read from the `api-keys` secret store under the //! `api_key` entry. The token must have config-store write and secret-store -//! write permissions only — no service-level admin or purge permissions. +//! write permissions only, with no service-level admin or purge permissions. //! //! # Security //! //! Credential values are never logged. Log messages include store IDs and //! operation names only. -use std::io::Read; - use base64::{Engine as _, engine::general_purpose}; -use error_stack::{Report, ResultExt}; -use fastly::http::{Method, StatusCode}; -use fastly::{Request, Response}; -use trusted_server_core::platform::{PlatformError, PlatformSecretStore, StoreName}; +use bytes::Bytes; +use edgezero_core::http::{Method, StatusCode, header}; +use edgezero_core::outbound::ResponseDecoding; +use error_stack::{Report, ResultExt as _}; +use trusted_server_core::error::TrustedServerError; +use trusted_server_core::outbound::{ + BUFFERED_TIMEOUT, OutboundRequestSource, buffered, outbound_error, outbound_request, +}; +use trusted_server_core::platform::{ + HttpClient, PlatformError, PlatformSecretStore as _, StoreName, +}; -use crate::backend::BackendConfig; use crate::platform::FastlyPlatformSecretStore; const FASTLY_API_HOST: &str = "https://api.fastly.com"; const API_KEYS_STORE: &str = "api-keys"; const API_KEY_ENTRY: &str = "api_key"; const ERROR_BODY_LIMIT: usize = 200; +/// Cap on a management API response body. +const MAX_RESPONSE_BYTES: u64 = 1024 * 1024; fn encode_path_segment(value: &str) -> String { urlencoding::encode(value).into_owned() @@ -76,31 +87,21 @@ fn truncate_error_body(body: &str) -> String { body.trim().chars().take(ERROR_BODY_LIMIT).collect() } -fn check_response( - response: &mut Response, - error_kind: fn() -> PlatformError, - operation: &str, - entity_description: &str, - store_id: &str, -) -> Result<(), Report> { - if response.get_status().is_success() { - return Ok(()); - } +/// A management API request body with the media type its endpoint takes. +enum ManagementBody { + /// A URL-encoded form, which the config item update takes. + Form(String), + /// A JSON document, which the secret upsert takes. + Json(String), +} - let mut body = String::new(); - response - .get_body_mut() - .read_to_string(&mut body) - .change_context(error_kind())?; - - Err(Report::new(error_kind()).attach(format!( - "{} failed with HTTP {} - {} for {} in store '{}'", - operation, - response.get_status(), - truncate_error_body(&body), - entity_description, - store_id - ))) +impl ManagementBody { + fn into_parts(self) -> (&'static str, Bytes) { + match self { + Self::Form(body) => ("application/x-www-form-urlencoded", Bytes::from(body)), + Self::Json(body) => ("application/json", Bytes::from(body)), + } + } } /// HTTP client for Fastly management API write operations. @@ -111,7 +112,6 @@ fn check_response( pub(crate) struct FastlyManagementApiClient { api_key: String, base_url: &'static str, - backend_name: String, } impl FastlyManagementApiClient { @@ -119,15 +119,9 @@ impl FastlyManagementApiClient { /// /// # Errors /// - /// Returns [`PlatformError::Backend`] if the management API backend cannot - /// be registered, or [`PlatformError::SecretStore`] if the API key cannot - /// be read. + /// Returns [`PlatformError::SecretStore`] if the API key cannot be read. pub(crate) fn new() -> Result> { - let backend_name = BackendConfig::from_url(FASTLY_API_HOST, true) - .change_context(PlatformError::Backend) - .attach("failed to register Fastly management API backend")?; - - let api_key = FastlyPlatformSecretStore + let api_key = FastlyPlatformSecretStore::read_only() .get_string(&StoreName::from(API_KEYS_STORE), API_KEY_ENTRY) .change_context(PlatformError::SecretStore) .attach("failed to read Fastly API key from secret store")?; @@ -137,81 +131,90 @@ impl FastlyManagementApiClient { Ok(Self { api_key, base_url: FASTLY_API_HOST, - backend_name, }) } /// Construct a client from explicit components for tests. #[cfg(test)] - pub(crate) fn with_components( - api_key: String, - base_url: &'static str, - backend_name: String, - ) -> Self { - Self { - api_key, - base_url, - backend_name, - } + pub(crate) fn with_components(api_key: String, base_url: &'static str) -> Self { + Self { api_key, base_url } } - fn make_request( + /// Sends one management API request through `client` and returns its body. + /// + /// A DELETE that the API answers 404 succeeds, because the entry is already + /// gone, so retries after a partial failure converge. + async fn make_request( &self, + client: &HttpClient, method: Method, path: &str, - body: Option, - content_type: &str, - error_kind: fn() -> PlatformError, - ) -> Result> { - let url = format!("{}{}", self.base_url, path); - - let mut request = Request::new(method, &url); - - request = request - .with_header("Fastly-Key", &self.api_key) - .with_header("Accept", "application/json"); - - if let Some(body_content) = body { + body: Option, + ) -> Result> { + let failed = || TrustedServerError::Configuration { + message: format!("Fastly management API {method} {path} failed"), + }; + let mut request = outbound_request(method.clone(), &format!("{}{path}", self.base_url)) + .and_then(|request| request.header("Fastly-Key", &self.api_key)) + .and_then(|request| request.header(header::ACCEPT, "application/json")) + .map_err(|error| outbound_error(error, failed(), OutboundRequestSource::Internal))?; + if let Some(body) = body { + let (content_type, bytes) = body.into_parts(); request = request - .with_header("Content-Type", content_type) - .with_body(body_content); + .header(header::CONTENT_TYPE, content_type) + .map_err(|error| outbound_error(error, failed(), OutboundRequestSource::Internal))? + .body(bytes); } - request.send(&self.backend_name).map_err(|e| { - Report::new(error_kind()).attach(format!("management API request failed: {}", e)) - }) + let response = client + .send(buffered( + request, + BUFFERED_TIMEOUT, + MAX_RESPONSE_BYTES, + ResponseDecoding::Decode, + )) + .await + .map_err(|error| outbound_error(error, failed(), OutboundRequestSource::Internal))?; + let status = response.status(); + let body = response + .into_bytes_bounded(MAX_RESPONSE_BYTES) + .await + .map_err(|error| outbound_error(error, failed(), OutboundRequestSource::Internal))?; + + if status.is_success() || (method == Method::DELETE && status == StatusCode::NOT_FOUND) { + return Ok(body); + } + Err(Report::new(failed()).attach(format!( + "management API answered HTTP {status} - {}", + truncate_error_body(&String::from_utf8_lossy(&body)) + ))) } /// Update or create a config store item. /// /// # Errors /// - /// Returns an error if the API request fails or returns a non-OK status. - pub(crate) fn update_config_item( + /// Returns [`PlatformError::ConfigStore`] if the API request fails or + /// returns a non-2xx status. + pub(crate) async fn update_config_item( &self, + client: &HttpClient, store_id: &str, key: &str, value: &str, ) -> Result<(), Report> { let path = build_config_item_path(store_id, key); - let payload = build_config_item_payload(value); - - let mut response = self.make_request( - Method::PUT, - &path, - Some(payload), - "application/x-www-form-urlencoded", - || PlatformError::ConfigStore, - )?; - - let entity_description = format!("key '{}'", key); - check_response( - &mut response, - || PlatformError::ConfigStore, - "config item update", - &entity_description, - store_id, - )?; + let body = ManagementBody::Form(build_config_item_payload(value)); + + self.make_request(client, Method::PUT, &path, Some(body)) + .await + .map_err(|report| { + report + .change_context(PlatformError::ConfigStore) + .attach(format!( + "config item update failed for key '{key}' in store '{store_id}'" + )) + })?; log::debug!( "FastlyManagementApiClient: updated config key '{}' in store '{}'", @@ -228,32 +231,26 @@ impl FastlyManagementApiClient { /// /// # Errors /// - /// Returns an error if the API request fails or returns an unexpected non-2xx status. - pub(crate) fn delete_config_item( + /// Returns [`PlatformError::ConfigStore`] if the API request fails or + /// returns an unexpected non-2xx status. + pub(crate) async fn delete_config_item( &self, + client: &HttpClient, store_id: &str, key: &str, ) -> Result<(), Report> { let path = build_config_item_path(store_id, key); - let mut response = - self.make_request(Method::DELETE, &path, None, "application/json", || { - PlatformError::ConfigStore + self.make_request(client, Method::DELETE, &path, None) + .await + .map_err(|report| { + report + .change_context(PlatformError::ConfigStore) + .attach(format!( + "config item delete failed for key '{key}' in store '{store_id}'" + )) })?; - if response.get_status() == StatusCode::NOT_FOUND { - return Ok(()); - } - - let entity_description = format!("key '{}'", key); - check_response( - &mut response, - || PlatformError::ConfigStore, - "config item delete", - &entity_description, - store_id, - )?; - log::debug!( "FastlyManagementApiClient: deleted config key '{}' from store '{}'", key, @@ -266,32 +263,27 @@ impl FastlyManagementApiClient { /// /// # Errors /// - /// Returns an error if the API request fails or returns a non-OK status. - pub(crate) fn create_secret( + /// Returns [`PlatformError::SecretStore`] if the API request fails or + /// returns a non-2xx status. + pub(crate) async fn create_secret( &self, + client: &HttpClient, store_id: &str, secret_name: &str, secret_value: &str, ) -> Result<(), Report> { let path = build_secret_collection_path(store_id); - let payload = build_secret_payload(secret_name, secret_value); - - let mut response = self.make_request( - Method::POST, - &path, - Some(payload), - "application/json", - || PlatformError::SecretStore, - )?; - - let entity_description = format!("name '{}'", secret_name); - check_response( - &mut response, - || PlatformError::SecretStore, - "secret upsert", - &entity_description, - store_id, - )?; + let body = ManagementBody::Json(build_secret_payload(secret_name, secret_value)); + + self.make_request(client, Method::POST, &path, Some(body)) + .await + .map_err(|report| { + report + .change_context(PlatformError::SecretStore) + .attach(format!( + "secret upsert failed for name '{secret_name}' in store '{store_id}'" + )) + })?; log::debug!( "FastlyManagementApiClient: upserted secret '{}' in store '{}'", @@ -308,32 +300,26 @@ impl FastlyManagementApiClient { /// /// # Errors /// - /// Returns an error if the API request fails or returns an unexpected non-2xx status. - pub(crate) fn delete_secret( + /// Returns [`PlatformError::SecretStore`] if the API request fails or + /// returns an unexpected non-2xx status. + pub(crate) async fn delete_secret( &self, + client: &HttpClient, store_id: &str, secret_name: &str, ) -> Result<(), Report> { let path = build_secret_path(store_id, secret_name); - let mut response = - self.make_request(Method::DELETE, &path, None, "application/json", || { - PlatformError::SecretStore + self.make_request(client, Method::DELETE, &path, None) + .await + .map_err(|report| { + report + .change_context(PlatformError::SecretStore) + .attach(format!( + "secret delete failed for name '{secret_name}' in store '{store_id}'" + )) })?; - if response.get_status() == StatusCode::NOT_FOUND { - return Ok(()); - } - - let entity_description = format!("name '{}'", secret_name); - check_response( - &mut response, - || PlatformError::SecretStore, - "secret delete", - &entity_description, - store_id, - )?; - log::debug!( "FastlyManagementApiClient: deleted secret '{}' from store '{}'", secret_name, @@ -345,7 +331,40 @@ impl FastlyManagementApiClient { #[cfg(test)] mod tests { + use std::sync::Arc; + + use edgezero_core::error::EdgeError; + use edgezero_core::key_value_store::NoopKvStore; + use edgezero_core::outbound::{OutboundCachePolicy, ResponseMode}; + use futures::executor::block_on; + use trusted_server_core::platform::{ + ClientInfo, PlatformConfigStore, PlatformSecretStore, RuntimeServices, StoreId, + }; + use trusted_server_core::request_signing::{KeyRotationManager, KeyRotationResult}; + use trusted_server_core::test_support::scripted_client::{ + ExpectedTransport, RecordedRequest, ScriptedHttpClient, ScriptedReply, TestClock, + }; + use super::*; + use crate::platform::RequestGeo; + + const BASE_URL: &str = "https://management.example.com"; + const API_KEY: &str = "example-api-key"; + + fn test_management_client() -> FastlyManagementApiClient { + FastlyManagementApiClient::with_components(API_KEY.to_owned(), BASE_URL) + } + + fn management_transport() -> ExpectedTransport { + ExpectedTransport { + timeout: BUFFERED_TIMEOUT, + response_mode: ResponseMode::Buffered { + max_bytes: 1024 * 1024, + }, + response_decoding: ResponseDecoding::Decode, + cache_policy: OutboundCachePolicy::PlatformDefault, + } + } #[test] fn build_config_item_payload_url_encodes_reserved_characters() { @@ -396,21 +415,395 @@ mod tests { assert_eq!(truncated, "a".repeat(200), "should trim before truncating"); } + /// Config and secret stores whose writes go through the management client + /// over a scripted outbound client. + /// + /// Reads fail, as they do for a store with no entries, so a rotation starts + /// with no current or active kid. + struct ScriptedManagementStores { + management: FastlyManagementApiClient, + client: HttpClient, + } + + #[async_trait::async_trait(?Send)] + impl PlatformConfigStore for ScriptedManagementStores { + fn get( + &self, + _store_name: &StoreName, + _key: &str, + ) -> Result> { + Err(Report::new(PlatformError::ConfigStore)) + } + + async fn put( + &self, + store_id: &StoreId, + key: &str, + value: &str, + ) -> Result<(), Report> { + self.management + .update_config_item(&self.client, store_id.as_ref(), key, value) + .await + } + + async fn delete(&self, store_id: &StoreId, key: &str) -> Result<(), Report> { + self.management + .delete_config_item(&self.client, store_id.as_ref(), key) + .await + } + } + + #[async_trait::async_trait(?Send)] + impl PlatformSecretStore for ScriptedManagementStores { + fn get_bytes( + &self, + _store_name: &StoreName, + _key: &str, + ) -> Result, Report> { + Err(Report::new(PlatformError::SecretStore)) + } + + async fn create( + &self, + store_id: &StoreId, + name: &str, + value: &str, + ) -> Result<(), Report> { + self.management + .create_secret(&self.client, store_id.as_ref(), name, value) + .await + } + + async fn delete( + &self, + store_id: &StoreId, + name: &str, + ) -> Result<(), Report> { + self.management + .delete_secret(&self.client, store_id.as_ref(), name) + .await + } + } + + /// Runtime services whose config and secret store writes go to `upstream`. + fn rotation_services(upstream: &ScriptedHttpClient, clock: &TestClock) -> RuntimeServices { + let stores = Arc::new(ScriptedManagementStores { + management: test_management_client(), + client: upstream.http_client(), + }); + RuntimeServices::builder() + .config_store(Arc::clone(&stores) as Arc) + .secret_store(stores) + .kv_store(Arc::new(NoopKvStore)) + .outbound_client(upstream.http_client()) + .clock(clock.monotonic_clock()) + .geo(Arc::new(RequestGeo::default())) + .client_info(ClientInfo::default()) + .build() + } + + fn rotate_example_kid( + services: &RuntimeServices, + ) -> Result> { + block_on( + KeyRotationManager::new("config-store-id", "secret-store-id") + .rotate_key(services, Some("example-kid".to_owned())), + ) + } + + fn secrets_uri() -> String { + format!("{BASE_URL}/resources/stores/secret/secret-store-id/secrets") + } + + fn item_uri(key: &str) -> String { + format!("{BASE_URL}/resources/stores/config/config-store-id/item/{key}") + } + + fn methods_and_uris(requests: &[RecordedRequest]) -> Vec<(Method, String)> { + requests + .iter() + .map(|request| (request.method.clone(), request.uri.to_string())) + .collect() + } + + #[test] + fn rotation_sends_expected_method_headers_and_body() { + let clock = TestClock::new(); + let upstream = ScriptedHttpClient::new(&clock); + upstream.respond_to( + Method::POST, + &secrets_uri(), + ScriptedReply::status(StatusCode::OK), + ); + for key in ["example-kid", "active-kids", "current-kid"] { + upstream.respond_to( + Method::PUT, + &item_uri(key), + ScriptedReply::status(StatusCode::OK), + ); + } + + let rotation = rotate_example_kid(&rotation_services(&upstream, &clock)) + .expect("should rotate through the management API"); + + assert_eq!(rotation.new_kid, "example-kid", "should use the given kid"); + assert!( + rotation.previous_kid.is_none(), + "should find no previous kid in an empty store" + ); + assert_eq!( + rotation.active_kids, + vec!["example-kid".to_owned()], + "should activate only the new kid" + ); + let requests = upstream.requests(); + assert_eq!( + methods_and_uris(&requests), + vec![ + (Method::POST, secrets_uri()), + (Method::PUT, item_uri("example-kid")), + (Method::PUT, item_uri("active-kids")), + (Method::PUT, item_uri("current-kid")), + ], + "should write the private key, the JWK, active-kids, then current-kid" + ); + for request in &requests { + request.assert_transport(management_transport()); + } + + let upsert_request = &requests[0]; + upsert_request.assert_headers(&[ + ("fastly-key", API_KEY), + ("accept", "application/json"), + ("content-type", "application/json"), + ]); + let upsert: serde_json::Value = serde_json::from_slice(&upsert_request.body) + .expect("should send the secret upsert as JSON"); + assert_eq!( + (&upsert["name"], &upsert["method"]), + ( + &serde_json::json!("example-kid"), + &serde_json::json!("create_or_recreate") + ), + "should upsert the secret under the new kid" + ); + let stored_key = upsert["secret"] + .as_str() + .and_then(|secret| general_purpose::STANDARD.decode(secret).ok()) + .and_then(|bytes| String::from_utf8(bytes).ok()) + .expect("should send the secret as base64 of a text value"); + let signing_key = general_purpose::STANDARD + .decode(stored_key) + .expect("should store the signing key as base64 text"); + assert_eq!( + signing_key.len(), + 32, + "should store a 32-byte Ed25519 signing key" + ); + + let jwk = serde_json::to_string(&rotation.jwk).expect("should serialize the rotated JWK"); + let item_bodies = [ + format!("item_value={}", urlencoding::encode(&jwk)), + "item_value=example-kid".to_owned(), + "item_value=example-kid".to_owned(), + ]; + for (request, body) in requests[1..].iter().zip(item_bodies) { + request.assert_headers(&[ + ("fastly-key", API_KEY), + ("accept", "application/json"), + ("content-type", "application/x-www-form-urlencoded"), + ]); + assert_eq!( + request.body.as_ref(), + body.as_bytes(), + "should send {} {} its config item value as a URL-encoded form", + request.method, + request.uri + ); + } + } + + #[test] + fn rotation_rolls_back_jwk_and_private_key_when_active_kids_write_fails() { + let clock = TestClock::new(); + let upstream = ScriptedHttpClient::new(&clock); + let secret_uri = format!("{}/example-kid", secrets_uri()); + upstream + .respond_to( + Method::POST, + &secrets_uri(), + ScriptedReply::status(StatusCode::OK), + ) + .respond_to( + Method::PUT, + &item_uri("example-kid"), + ScriptedReply::status(StatusCode::OK), + ) + .respond_to( + Method::PUT, + &item_uri("active-kids"), + ScriptedReply::status(StatusCode::INTERNAL_SERVER_ERROR), + ) + .respond_to( + Method::DELETE, + &item_uri("example-kid"), + ScriptedReply::status(StatusCode::OK), + ) + .respond_to( + Method::DELETE, + &secret_uri, + ScriptedReply::status(StatusCode::OK), + ); + + let error = rotate_example_kid(&rotation_services(&upstream, &clock)) + .expect_err("should fail when the active-kids write fails"); + + assert!( + matches!( + error.current_context(), + TrustedServerError::Configuration { message } if message == "failed to update active-kids" + ), + "should report the failed active-kids write" + ); + let requests = upstream.requests(); + assert_eq!( + methods_and_uris(&requests), + vec![ + (Method::POST, secrets_uri()), + (Method::PUT, item_uri("example-kid")), + (Method::PUT, item_uri("active-kids")), + (Method::DELETE, item_uri("example-kid")), + (Method::DELETE, secret_uri), + ], + "should roll back the JWK, then the private key, and never flip current-kid" + ); + for rollback in &requests[3..] { + rollback.assert_transport(management_transport()); + rollback.assert_headers(&[("fastly-key", API_KEY), ("accept", "application/json")]); + assert!( + rollback.body.is_empty(), + "should send a rollback DELETE with no body" + ); + } + } + + #[test] + fn non_2xx_management_response_truncates_body_to_200_chars() { + let clock = TestClock::new(); + let upstream = ScriptedHttpClient::new(&clock); + let uri = format!("{BASE_URL}/resources/stores/config/config-store-id/item/current-kid"); + upstream.respond_to( + Method::PUT, + &uri, + ScriptedReply::status(StatusCode::INTERNAL_SERVER_ERROR) + .body(format!(" {}example-tail ", "x".repeat(200))), + ); + + let error = block_on(test_management_client().update_config_item( + &upstream.http_client(), + "config-store-id", + "current-kid", + "example-kid", + )) + .expect_err("should fail when the management API answers 500"); + + assert!( + matches!(error.current_context(), PlatformError::ConfigStore), + "should classify the failed write as a config-store error" + ); + let answer = error + .frames() + .filter_map(|frame| frame.downcast_ref::()) + .find(|text| text.starts_with("management API answered")) + .expect("should attach the management API answer"); + assert_eq!( + answer, + &format!( + "management API answered HTTP 500 Internal Server Error - {}", + "x".repeat(200) + ), + "should keep the trimmed body truncated to 200 chars" + ); + } + + #[test] + fn delete_treats_missing_entry_as_deleted() { + let clock = TestClock::new(); + let upstream = ScriptedHttpClient::new(&clock); + let item_uri = + format!("{BASE_URL}/resources/stores/config/config-store-id/item/example-kid"); + let secret_uri = + format!("{BASE_URL}/resources/stores/secret/secret-store-id/secrets/example-kid"); + upstream + .respond_to( + Method::DELETE, + &item_uri, + ScriptedReply::status(StatusCode::NOT_FOUND), + ) + .respond_to( + Method::DELETE, + &secret_uri, + ScriptedReply::status(StatusCode::NOT_FOUND), + ) + .respond_to( + Method::PUT, + &item_uri, + ScriptedReply::status(StatusCode::NOT_FOUND), + ); + let client = upstream.http_client(); + let management = test_management_client(); + + block_on(management.delete_config_item(&client, "config-store-id", "example-kid")) + .expect("should treat a missing config item as deleted"); + block_on(management.delete_secret(&client, "secret-store-id", "example-kid")) + .expect("should treat a missing secret as deleted"); + let error = block_on(management.update_config_item( + &client, + "config-store-id", + "example-kid", + "example-value", + )) + .expect_err("should fail a PUT that the management API answers 404"); + + assert!( + matches!(error.current_context(), PlatformError::ConfigStore), + "should classify the failed write as a config-store error" + ); + let requests = upstream.requests(); + assert_eq!(requests.len(), 3, "should send each management call once"); + for delete in &requests[..2] { + assert_eq!(delete.method, Method::DELETE, "should send a DELETE"); + delete.assert_transport(management_transport()); + delete.assert_headers(&[("fastly-key", API_KEY), ("accept", "application/json")]); + assert!(delete.body.is_empty(), "should send a DELETE with no body"); + } + } + #[test] fn create_secret_uses_secret_store_error_for_transport_failures() { - let client = FastlyManagementApiClient::with_components( - "test-api-key".to_string(), - FASTLY_API_HOST, - "missing-management-backend".to_string(), + let clock = TestClock::new(); + let upstream = ScriptedHttpClient::new(&clock); + upstream.respond_to( + Method::POST, + &format!("{BASE_URL}/resources/stores/secret/store-id/secrets"), + ScriptedReply::error(EdgeError::bad_gateway("example connection failure")), ); - let err = client - .create_secret("store-id", "secret-name", "secret-value") - .expect_err("should fail when the management API backend is unavailable"); + let err = block_on(test_management_client().create_secret( + &upstream.http_client(), + "store-id", + "secret-name", + "secret-value", + )) + .expect_err("should fail when the management API is unreachable"); assert!( matches!(err.current_context(), &PlatformError::SecretStore), "should classify secret transport failures as secret-store errors" ); + assert!( + err.downcast_ref::().is_some(), + "should keep the transport error on the report" + ); } } diff --git a/crates/trusted-server-adapter-fastly/src/middleware.rs b/crates/trusted-server-adapter-fastly/src/middleware.rs index 9bbc25d7ba..b9565cef40 100644 --- a/crates/trusted-server-adapter-fastly/src/middleware.rs +++ b/crates/trusted-server-adapter-fastly/src/middleware.rs @@ -2,7 +2,7 @@ //! //! Provides two middleware types used by the `EdgeZero` entry point: //! -//! - [`FinalizeResponseMiddleware`] — geo lookup and standard TS header injection +//! - [`FinalizeResponseMiddleware`]: standard TS header injection, with the request's geo //! - [`AuthMiddleware`] — basic-auth enforcement via [`enforce_basic_auth`] //! //! Registration order in [`crate::app`]: `FinalizeResponseMiddleware` outermost, @@ -12,34 +12,44 @@ use std::sync::Arc; use async_trait::async_trait; -use edgezero_adapter_fastly::context::FastlyRequestContext; +use edgezero_core::body::Body; use edgezero_core::context::RequestContext; use edgezero_core::error::EdgeError; -use edgezero_core::http::{HeaderValue, Response, StatusCode}; +use edgezero_core::http::{HeaderMap, HeaderName, HeaderValue, Response, StatusCode}; use edgezero_core::middleware::{Middleware, Next}; use edgezero_core::response::IntoResponse; -use std::net::IpAddr; +use edgezero_core::response_headers::{ + SynthesizedResponseHeaders, SynthesizedResponseHeadersError, +}; use trusted_server_core::auth::enforce_basic_auth; use trusted_server_core::constants::{ ENV_FASTLY_IS_STAGING, ENV_FASTLY_SERVICE_VERSION, HEADER_X_GEO_INFO_AVAILABLE, HEADER_X_TS_ENV, HEADER_X_TS_FASTLY_VERSION, }; +use trusted_server_core::edge_app::synthesized_response_headers; +use trusted_server_core::error::http_error; use trusted_server_core::geo::GeoInfo; -use trusted_server_core::platform::{ClientInfo, PlatformGeo}; +use trusted_server_core::response_privacy::warn_skipped_operator_response_headers; use trusted_server_core::settings::Settings; use trusted_server_core::version_header::apply_git_version_header; +use crate::platform::RequestGeo; + pub(crate) const HEADER_X_TS_FINALIZED: &str = "x-ts-finalized"; // --------------------------------------------------------------------------- // FinalizeResponseMiddleware // --------------------------------------------------------------------------- -/// Outermost middleware: performs geo lookup and injects all standard TS response headers. +/// Outermost middleware: injects all standard TS response headers. /// /// Registered first in the middleware chain so that it wraps all inner middleware /// (including [`AuthMiddleware`]) and the handler. This guarantees every registered-route -/// response — including auth-rejected ones — carries a consistent set of headers. +/// response, including auth-rejected ones, carries a consistent set of headers. +/// +/// Geo comes from the request's [`RequestGeo`], the lookup the entry point ran +/// once for the resolved client IP. A request without one carries +/// `X-Geo-Info-Available: false`. /// /// Router-level 405/404 responses for unregistered HTTP methods (e.g. TRACE) bypass the /// middleware chain. Those are covered by a second call to [`apply_finalize_headers`] at @@ -57,23 +67,19 @@ pub(crate) const HEADER_X_TS_FINALIZED: &str = "x-ts-finalized"; /// 4. Operator-configured `settings.response_headers` (can override any managed header) pub struct FinalizeResponseMiddleware { settings: Arc, - geo: Arc, } impl FinalizeResponseMiddleware { - /// Creates a new [`FinalizeResponseMiddleware`] with the given settings and geo lookup service. - pub fn new(settings: Arc, geo: Arc) -> Self { - Self { settings, geo } + /// Creates a new [`FinalizeResponseMiddleware`] with the given settings. + pub fn new(settings: Arc) -> Self { + Self { settings } } } #[async_trait(?Send)] impl Middleware for FinalizeResponseMiddleware { async fn handle(&self, ctx: RequestContext, next: Next<'_>) -> Result { - let client_ip = ctx.request().extensions().get::().map_or_else( - || FastlyRequestContext::get(ctx.request()).and_then(|c| c.client_ip), - |info| info.client_ip, - ); + let request_geo = ctx.extensions().get::().cloned(); let mut response = match next.run(ctx).await { Ok(r) => r, @@ -83,14 +89,9 @@ impl Middleware for FinalizeResponseMiddleware { } }; - let geo_info = resolve_geo_for_response(&response, client_ip, |ip| { - self.geo.lookup(ip).unwrap_or_else(|e| { - log::warn!("geo lookup failed: {e}"); - None - }) - }); - - apply_finalize_headers(&self.settings, geo_info.as_ref(), &mut response); + let geo = + resolve_geo_for_response(&response, request_geo.as_ref().and_then(RequestGeo::geo)); + apply_finalize_headers(&self.settings, geo, &mut response); response .headers_mut() .insert(HEADER_X_TS_FINALIZED, HeaderValue::from_static("1")); @@ -109,12 +110,12 @@ impl Middleware for FinalizeResponseMiddleware { /// challenge response (bubbles through [`FinalizeResponseMiddleware`] for header injection). /// - `Ok(None)` → no auth required or credentials accepted; continue the chain. /// - `Err(report)` → internal error; log and convert to an HTTP response via -/// [`crate::app::http_error`] using the error's documented status code. +/// [`http_error`] using the error's documented status code. /// /// # Errors /// /// When [`enforce_basic_auth`] returns an error report, converts it to an HTTP -/// response via [`crate::app::http_error`] (preserving the error's status code) +/// response via [`http_error`] (preserving the error's status code) /// so that [`FinalizeResponseMiddleware`] can still inject standard TS headers /// before the response reaches the client. pub struct AuthMiddleware { @@ -131,12 +132,12 @@ impl AuthMiddleware { #[async_trait(?Send)] impl Middleware for AuthMiddleware { async fn handle(&self, mut ctx: RequestContext, next: Next<'_>) -> Result { - match enforce_basic_auth(&self.settings, ctx.request_mut()) { + match enforce_basic_auth(&self.settings, ctx.parts_mut()) { Ok(Some(response)) => return Ok(response), Ok(None) => {} Err(report) => { log::error!("auth check failed: {:?}", report); - return Ok(crate::app::http_error(&report)); + return Ok(http_error(&report)); } } @@ -148,14 +149,13 @@ impl Middleware for AuthMiddleware { // Shared geo resolution helper // --------------------------------------------------------------------------- -/// Resolves geo for a response, skipping the lookup for 401 responses. +/// The geo a response's headers report: none for a 401, the request's `geo` otherwise. /// -/// Returns `None` for authentication rejections (401) without calling `lookup_geo` -/// to avoid unnecessary work and exposing geo data to unauthenticated callers. -/// All other responses call `lookup_geo` and return its result. +/// Authentication rejections (401) carry no geo, so geo data never reaches an +/// unauthenticated caller. /// -/// Used by both [`FinalizeResponseMiddleware`] and the entry-point finalization -/// in `main.rs` so the 401-skip rule is defined in one place. +/// Used by both [`FinalizeResponseMiddleware`] and the entry point's `finalize` +/// step in `main.rs` so the 401 rule is defined in one place. /// /// # Parity note /// @@ -165,18 +165,14 @@ impl Middleware for AuthMiddleware { /// is intentionally more conservative: geo data is not sent to any /// unauthenticated caller regardless of whether the 401 originated from this /// server or the upstream origin. -pub(crate) fn resolve_geo_for_response( +pub(crate) fn resolve_geo_for_response<'geo>( response: &Response, - client_ip: Option, - lookup_geo: F, -) -> Option -where - F: FnOnce(Option) -> Option, -{ + geo: Option<&'geo GeoInfo>, +) -> Option<&'geo GeoInfo> { if response.status() == StatusCode::UNAUTHORIZED { None } else { - lookup_geo(client_ip) + geo } } @@ -199,6 +195,9 @@ where /// 5. `settings.response_headers` — operator-configured overrides, except the /// cache-controlling headers are skipped on uncacheable (`private`/`no-store`) /// responses so operators cannot re-enable shared caching for per-user payloads +/// +/// Operator entries that are never sent are skipped without a log here. +/// [`per_request_synthesized_headers`] warns about them once per request. pub(crate) fn apply_finalize_headers( settings: &Settings, geo_info: Option<&GeoInfo>, @@ -247,35 +246,119 @@ pub(crate) use trusted_server_core::response_privacy::{ }; // --------------------------------------------------------------------------- -// Tests +// Synthesized response headers // --------------------------------------------------------------------------- -/// Sets `X-TS-Fastly-Version` to `service_version`, or removes an inherited -/// value when the version is unset or invalid, so it is never reported as ours. -fn apply_fastly_version_header(service_version: Option<&str>, response: &mut Response) { - match service_version.map(HeaderValue::from_str) { - Some(Ok(value)) => { - response - .headers_mut() - .insert(HEADER_X_TS_FASTLY_VERSION, value); +/// The app-level synthesized set: `x-ts-version` and the service version. +/// +/// `x-ts-fastly-version` is added only when `service_version` is a portable +/// synthesized value. A value `EdgeZero` would reject, such as one with a +/// leading or trailing space, is left out with a warning, so the base set +/// still installs. +pub(crate) fn version_synthesized_headers(service_version: Option<&str>) -> HeaderMap { + let mut fields = synthesized_response_headers(); + if let Some(value) = fastly_version_header_value(service_version) + && is_synthesizable(&HEADER_X_TS_FASTLY_VERSION, &value) + { + fields.insert(HEADER_X_TS_FASTLY_VERSION, value); + } + fields +} + +/// This request's synthesized set: the finalize headers that do not depend on the response. +/// +/// `EdgeZero` adds it, where a field is absent, to the responses it generates +/// for the request, such as head-limit errors, framing rejections, and +/// fallbacks. It holds what [`apply_finalize_headers`] sets on a response +/// without headers: `x-ts-version`, `x-ts-fastly-version`, `x-ts-env` on +/// staging, the geo headers or `x-geo-info-available: false`, and the +/// operator's `response_headers`. A field `EdgeZero` forbids, or whose value is +/// not portable, such as a geo city with accents, is left out with a warning. +/// When the rest still exceeds the set's bounds, only +/// [`version_synthesized_headers`] remain, and the bound is logged. +/// +/// The entry point builds this set once for every request, so it also warns +/// about each operator entry the finalize headers skip. [`apply_finalize_headers`] +/// skips them silently, so a routed response does not warn a second time. +pub(crate) fn per_request_synthesized_headers( + settings: &Settings, + geo: Option<&GeoInfo>, +) -> SynthesizedResponseHeaders { + warn_skipped_operator_response_headers(settings); + let mut finalized = Response::new(Body::empty()); + apply_finalize_headers(settings, geo, &mut finalized); + let mut fields = HeaderMap::new(); + for (name, value) in finalized.headers() { + if is_synthesizable(name, value) { + fields.append(name.clone(), value.clone()); } - Some(Err(_)) => { - log::warn!("Skipping invalid FASTLY_SERVICE_VERSION response header value"); - response.headers_mut().remove(HEADER_X_TS_FASTLY_VERSION); + } + + SynthesizedResponseHeaders::new(fields).unwrap_or_else(|error| { + log::warn!("Keeping only the version headers in the synthesized response headers: {error}"); + let service_version = std::env::var(ENV_FASTLY_SERVICE_VERSION).ok(); + SynthesizedResponseHeaders::new(version_synthesized_headers(service_version.as_deref())) + .unwrap_or_else(|error| { + log::warn!("Synthesizing no response headers: {error}"); + SynthesizedResponseHeaders::default() + }) + }) +} + +/// Whether `name: value` may join a synthesized set, warning when it may not. +/// +/// A single field never exceeds the set's field count, and its byte bound is +/// checked with the whole set, so only a forbidden name or an unportable value +/// leaves it out here. +fn is_synthesizable(name: &HeaderName, value: &HeaderValue) -> bool { + let mut field = HeaderMap::new(); + field.insert(name.clone(), value.clone()); + match SynthesizedResponseHeaders::new(field) { + Ok(_) | Err(SynthesizedResponseHeadersError::TooManyBytes) => true, + Err(error) => { + log::warn!("Leaving `{name}` out of the synthesized response headers: {error}"); + false } - None => { - response.headers_mut().remove(HEADER_X_TS_FASTLY_VERSION); + } +} + +/// Converts `service_version` to the `X-TS-Fastly-Version` value. +/// +/// `None` when the version is unset, or when it is not a valid header value, +/// which is logged. +fn fastly_version_header_value(service_version: Option<&str>) -> Option { + match HeaderValue::from_str(service_version?) { + Ok(value) => Some(value), + Err(_) => { + log::warn!("Skipping invalid FASTLY_SERVICE_VERSION response header value"); + None } } } +/// Sets `X-TS-Fastly-Version` to `service_version`, or removes an inherited +/// value when the version is unset or invalid, so it is never reported as ours. +fn apply_fastly_version_header(service_version: Option<&str>, response: &mut Response) { + if let Some(value) = fastly_version_header_value(service_version) { + response + .headers_mut() + .insert(HEADER_X_TS_FASTLY_VERSION, value); + } else { + response.headers_mut().remove(HEADER_X_TS_FASTLY_VERSION); + } +} + +// --------------------------------------------------------------------------- +// Tests +// --------------------------------------------------------------------------- + #[cfg(test)] mod tests { use super::*; use std::collections::HashMap; use std::net::IpAddr; - use std::sync::{Arc, Mutex}; + use std::sync::Arc; use edgezero_core::body::Body; use edgezero_core::context::RequestContext; @@ -283,9 +366,7 @@ mod tests { use edgezero_core::http::{HeaderName, Method, StatusCode, request_builder, response_builder}; use edgezero_core::middleware::Next; use edgezero_core::params::PathParams; - use error_stack::Report; use futures::executor::block_on; - use trusted_server_core::platform::{ClientInfo, PlatformError, PlatformGeo}; use trusted_server_core::response_privacy::apply_inactive_ad_stack_browser_cache_policy; fn empty_response() -> Response { @@ -303,31 +384,6 @@ mod tests { RequestContext::new(req, PathParams::new(HashMap::new())) } - struct FixedGeo(Option); - - impl PlatformGeo for FixedGeo { - fn lookup(&self, _: Option) -> Result, Report> { - Ok(self.0.clone()) - } - } - - struct RecordingGeo { - lookups: Arc>>>, - } - - impl PlatformGeo for RecordingGeo { - fn lookup( - &self, - client_ip: Option, - ) -> Result, Report> { - self.lookups - .lock() - .expect("should lock recorded geo lookups") - .push(client_ip); - Ok(None) - } - } - fn test_settings() -> Settings { Settings::from_toml( r#" @@ -579,114 +635,42 @@ mod tests { // --------------------------------------------------------------------------- #[test] - fn finalize_handle_uses_client_info_ip_for_geo_lookup() { + fn finalize_handle_reports_the_request_geo() { let reader_ip = IpAddr::V4(std::net::Ipv4Addr::new(198, 51, 100, 7)); - let peer_ip = IpAddr::V4(std::net::Ipv4Addr::new(203, 0, 113, 9)); let settings = settings_with_response_headers(vec![]); - let lookups = Arc::new(Mutex::new(Vec::new())); - let middleware = FinalizeResponseMiddleware::new( - Arc::new(settings), - Arc::new(RecordingGeo { - lookups: Arc::clone(&lookups), - }), - ); + let middleware = FinalizeResponseMiddleware::new(Arc::new(settings)); let mut ctx = empty_ctx(); - ctx.request_mut().extensions_mut().insert(ClientInfo { - client_ip: Some(reader_ip), - ..ClientInfo::default() - }); - FastlyRequestContext::insert( - ctx.request_mut(), - FastlyRequestContext { - client_ip: Some(peer_ip), - }, - ); + ctx.extensions_mut().insert(RequestGeo::new( + Some(reader_ip), + Some(sample_geo("Springfield")), + )); let handler = Arc::new( |_ctx: RequestContext| async move { Ok::(empty_response()) }, ); - block_on(middleware.handle(ctx, Next::new(&[], &*handler))).expect("should succeed"); + let response = + block_on(middleware.handle(ctx, Next::new(&[], &*handler))).expect("should succeed"); - assert_eq!( - *lookups.lock().expect("should lock recorded geo lookups"), - vec![Some(reader_ip)], - "should look up geo using the resolved ClientInfo IP" - ); - } - - #[test] - fn finalize_handle_preserves_none_from_present_client_info() { - let peer_ip = IpAddr::V4(std::net::Ipv4Addr::new(203, 0, 113, 9)); - let settings = settings_with_response_headers(vec![]); - let lookups = Arc::new(Mutex::new(Vec::new())); - let middleware = FinalizeResponseMiddleware::new( - Arc::new(settings), - Arc::new(RecordingGeo { - lookups: Arc::clone(&lookups), - }), - ); - let mut ctx = empty_ctx(); - ctx.request_mut() - .extensions_mut() - .insert(ClientInfo::default()); - FastlyRequestContext::insert( - ctx.request_mut(), - FastlyRequestContext { - client_ip: Some(peer_ip), - }, - ); - let handler = - Arc::new( - |_ctx: RequestContext| async move { Ok::(empty_response()) }, - ); - - block_on(middleware.handle(ctx, Next::new(&[], &*handler))).expect("should succeed"); - - assert_eq!( - *lookups.lock().expect("should lock recorded geo lookups"), - vec![None], - "should preserve no IP from the authoritative ClientInfo" - ); - } - - #[test] - fn finalize_handle_uses_fastly_context_ip_when_client_info_is_absent() { - let peer_ip = IpAddr::V4(std::net::Ipv4Addr::new(203, 0, 113, 9)); - let settings = settings_with_response_headers(vec![]); - let lookups = Arc::new(Mutex::new(Vec::new())); - let middleware = FinalizeResponseMiddleware::new( - Arc::new(settings), - Arc::new(RecordingGeo { - lookups: Arc::clone(&lookups), - }), - ); - let mut ctx = empty_ctx(); - FastlyRequestContext::insert( - ctx.request_mut(), - FastlyRequestContext { - client_ip: Some(peer_ip), - }, - ); - let handler = - Arc::new( - |_ctx: RequestContext| async move { Ok::(empty_response()) }, + for (name, value) in [ + ("x-geo-city", "Springfield"), + ("x-geo-info-available", "true"), + ] { + assert_eq!( + response + .headers() + .get(name) + .and_then(|value| value.to_str().ok()), + Some(value), + "should report {name} from the entry point's geo" ); - - block_on(middleware.handle(ctx, Next::new(&[], &*handler))).expect("should succeed"); - - assert_eq!( - *lookups.lock().expect("should lock recorded geo lookups"), - vec![Some(peer_ip)], - "should fall back to the Fastly request context IP" - ); + } } #[test] fn finalize_handle_injects_geo_unavailable_on_ok_response() { let settings = settings_with_response_headers(vec![]); - let middleware = - FinalizeResponseMiddleware::new(Arc::new(settings), Arc::new(FixedGeo(None))); + let middleware = FinalizeResponseMiddleware::new(Arc::new(settings)); let handler = Arc::new( |_ctx: RequestContext| async move { Ok::(empty_response()) }, @@ -708,8 +692,7 @@ mod tests { #[test] fn finalize_handle_marks_response_as_finalized() { let settings = settings_with_response_headers(vec![]); - let middleware = - FinalizeResponseMiddleware::new(Arc::new(settings), Arc::new(FixedGeo(None))); + let middleware = FinalizeResponseMiddleware::new(Arc::new(settings)); let handler = Arc::new( |_ctx: RequestContext| async move { Ok::(empty_response()) }, @@ -731,8 +714,7 @@ mod tests { #[test] fn finalize_handle_absorbs_handler_error_and_injects_headers() { let settings = settings_with_response_headers(vec![]); - let middleware = - FinalizeResponseMiddleware::new(Arc::new(settings), Arc::new(FixedGeo(None))); + let middleware = FinalizeResponseMiddleware::new(Arc::new(settings)); let handler = Arc::new(|_ctx: RequestContext| async move { Err::(EdgeError::service_unavailable("test error")) }); @@ -751,25 +733,22 @@ mod tests { } #[test] - #[allow(clippy::panic)] - fn finalize_handle_skips_geo_lookup_for_401() { - struct PanicGeo; - impl PlatformGeo for PanicGeo { - fn lookup(&self, _: Option) -> Result, Report> { - panic!("should not call geo for 401 responses") - } - } - + fn finalize_handle_withholds_geo_for_401() { let settings = settings_with_response_headers(vec![]); - let middleware = FinalizeResponseMiddleware::new(Arc::new(settings), Arc::new(PanicGeo)); + let middleware = FinalizeResponseMiddleware::new(Arc::new(settings)); + let mut ctx = empty_ctx(); + ctx.extensions_mut().insert(RequestGeo::new( + Some(IpAddr::V4(std::net::Ipv4Addr::new(198, 51, 100, 7))), + Some(sample_geo("Springfield")), + )); let handler = Arc::new(|_ctx: RequestContext| async move { let mut resp = empty_response(); *resp.status_mut() = StatusCode::UNAUTHORIZED; Ok::(resp) }); - let response = block_on(middleware.handle(empty_ctx(), Next::new(&[], &*handler))) - .expect("should succeed without calling geo"); + let response = + block_on(middleware.handle(ctx, Next::new(&[], &*handler))).expect("should succeed"); assert_eq!( response.status(), @@ -782,7 +761,11 @@ mod tests { .get("x-geo-info-available") .and_then(|v| v.to_str().ok()), Some("false"), - "should set geo-unavailable header without calling geo for 401" + "should withhold the request geo from a 401" + ); + assert!( + response.headers().get("x-geo-city").is_none(), + "should leave out the geo city on a 401" ); } @@ -797,8 +780,7 @@ mod tests { // collapsed them because fastly::Response uses set_header (last-wins). // This test verifies the EdgeZero middleware chain is header-transparent. let settings = settings_with_response_headers(vec![]); - let middleware = - FinalizeResponseMiddleware::new(Arc::new(settings), Arc::new(FixedGeo(None))); + let middleware = FinalizeResponseMiddleware::new(Arc::new(settings)); let handler = Arc::new(|_ctx: RequestContext| async move { let resp = response_builder() .header("set-cookie", "session=abc; Path=/; HttpOnly") @@ -878,4 +860,171 @@ mod tests { ); } } + + // --------------------------------------------------------------------------- + // Synthesized response header tests + // --------------------------------------------------------------------------- + + /// The fields `set` adds to a response without headers. + fn synthesized_fields(set: &SynthesizedResponseHeaders) -> HeaderMap { + let mut fields = HeaderMap::new(); + set.apply(&mut fields); + fields + } + + fn sample_geo(city: &str) -> GeoInfo { + GeoInfo { + city: city.to_owned(), + country: "US".to_owned(), + continent: "NorthAmerica".to_owned(), + latitude: 37.7749, + longitude: -122.4194, + metro_code: 807, + region: Some("CA".to_owned()), + asn: None, + } + } + + #[test] + fn version_synthesized_headers_skip_a_service_version_with_surrounding_space() { + for service_version in [" 42", "42 ", "\t42"] { + let fields = version_synthesized_headers(Some(service_version)); + + assert!( + fields.get(HEADER_X_TS_FASTLY_VERSION).is_none(), + "should leave out the unportable service version {service_version:?}" + ); + assert_eq!( + fields.get(trusted_server_core::constants::HEADER_X_TS_VERSION), + synthesized_response_headers() + .get(trusted_server_core::constants::HEADER_X_TS_VERSION), + "should keep the base set for {service_version:?}" + ); + SynthesizedResponseHeaders::new(fields).expect("should keep a set EdgeZero installs"); + } + } + + #[test] + fn version_synthesized_headers_add_a_portable_service_version() { + let fields = version_synthesized_headers(Some("42")); + + assert_eq!( + fields + .get(HEADER_X_TS_FASTLY_VERSION) + .and_then(|value| value.to_str().ok()), + Some("42"), + "should add the service version to the base set" + ); + assert!( + version_synthesized_headers(None) + .get(HEADER_X_TS_FASTLY_VERSION) + .is_none(), + "should add nothing when the service version is unset" + ); + } + + #[test] + fn per_request_synthesized_headers_carry_geo_version_and_operator_headers() { + let settings = settings_with_response_headers(vec![("x-operator", "kept")]); + + let fields = synthesized_fields(&per_request_synthesized_headers( + &settings, + Some(&sample_geo("San Francisco")), + )); + + for (name, value) in [ + ("x-geo-city", "San Francisco"), + ("x-geo-country", "US"), + ("x-geo-info-available", "true"), + ("x-operator", "kept"), + ] { + assert_eq!( + fields.get(name).and_then(|value| value.to_str().ok()), + Some(value), + "should synthesize {name}" + ); + } + assert_eq!( + fields.get(trusted_server_core::constants::HEADER_X_TS_VERSION), + synthesized_response_headers().get(trusted_server_core::constants::HEADER_X_TS_VERSION), + "should synthesize the git version" + ); + } + + #[test] + fn per_request_synthesized_headers_report_unavailable_geo() { + let fields = synthesized_fields(&per_request_synthesized_headers( + &settings_with_response_headers(vec![]), + None, + )); + + assert_eq!( + fields + .get(HEADER_X_GEO_INFO_AVAILABLE) + .and_then(|value| value.to_str().ok()), + Some("false"), + "should synthesize x-geo-info-available: false without geo" + ); + } + + #[test] + fn per_request_synthesized_headers_leave_out_unportable_and_forbidden_values() { + let settings = settings_with_response_headers(vec![ + ("content-type", "text/plain"), + ("x-operator", " padded"), + ("x-kept", "kept"), + ]); + + let fields = synthesized_fields(&per_request_synthesized_headers( + &settings, + Some(&sample_geo("Zürich")), + )); + + assert!( + fields.get("x-geo-city").is_none(), + "should leave out a geo city with accents" + ); + assert_eq!( + fields + .get("x-geo-country") + .and_then(|value| value.to_str().ok()), + Some("US"), + "should keep the other geo headers" + ); + assert!( + fields.get("content-type").is_none(), + "should leave out an operator field EdgeZero forbids" + ); + assert!( + fields.get("x-operator").is_none(), + "should leave out an operator value with a leading space" + ); + assert_eq!( + fields.get("x-kept").and_then(|value| value.to_str().ok()), + Some("kept"), + "should keep the portable operator fields" + ); + } + + #[test] + fn per_request_synthesized_headers_fall_back_to_version_headers_over_bounds() { + let operator: Vec<(String, String)> = (0..40) + .map(|index| (format!("x-operator-{index}"), "value".to_owned())) + .collect(); + let settings = settings_with_response_headers( + operator + .iter() + .map(|(name, value)| (name.as_str(), value.as_str())) + .collect(), + ); + + let fields = synthesized_fields(&per_request_synthesized_headers(&settings, None)); + + let service_version = std::env::var(ENV_FASTLY_SERVICE_VERSION).ok(); + assert_eq!( + fields, + version_synthesized_headers(service_version.as_deref()), + "should keep only the version headers when the set exceeds 32 fields" + ); + } } diff --git a/crates/trusted-server-adapter-fastly/src/platform.rs b/crates/trusted-server-adapter-fastly/src/platform.rs index ea2193d090..2c9b724c06 100644 --- a/crates/trusted-server-adapter-fastly/src/platform.rs +++ b/crates/trusted-server-adapter-fastly/src/platform.rs @@ -1,59 +1,60 @@ //! Fastly-backed implementations of the platform traits defined in //! `trusted-server-core::platform`. -mod deadline; - -use bytes::Bytes; -use error_stack::{Report, ResultExt}; +use error_stack::Report; use fastly::geo::{Geo, geo_lookup}; use fastly::{ConfigStore, Request, SecretStore}; -use std::io::Read as _; use std::net::IpAddr; -use crate::backend::BackendConfig; +use crate::management_api::FastlyManagementApiClient; pub(crate) use trusted_server_core::platform::UnavailableKvStore; use trusted_server_core::platform::{ - BackendNamingPolicy, ClientInfo, GeoInfo, PlatformBackend, PlatformBackendSpec, - PlatformCacheIntent, PlatformConfigStore, PlatformError, PlatformGeo, PlatformHttpClient, - PlatformHttpRequest, PlatformImageOptimizerCrop, PlatformImageOptimizerCropMode, - PlatformImageOptimizerOptions, PlatformImageOptimizerParams, PlatformImageOptimizerRegion, - PlatformPendingRequest, PlatformResponse, PlatformSecretStore, PlatformSelectResult, StoreId, - StoreName, + ClientInfo, GeoInfo, HttpClient, PlatformConfigStore, PlatformError, PlatformGeo, + PlatformSecretStore, StoreId, StoreName, }; use trusted_server_core::settings::TrustedClientIpConfig; -// Cache-pass requests always have a direct SDK pending handle. Keep this proof -// with the pending request until the deadline collector consumes it. -struct CacheBypassPending(fastly::http::request::PendingRequest); - -fn unwrap_pending( - pending: PlatformPendingRequest, -) -> Result { - match pending.downcast::() { - Ok(pending) => Ok(pending.0), - Err(pending) => pending.downcast(), - } -} - // --------------------------------------------------------------------------- // FastlyPlatformConfigStore // --------------------------------------------------------------------------- /// Fastly [`ConfigStore`]-backed implementation of [`PlatformConfigStore`]. /// -/// Stateless — the store name is supplied per call, matching the trait -/// signature. This replaces the store-name-at-construction pattern of -/// the legacy `FastlyConfigStore` (removed). +/// The store name is supplied per call, matching the trait signature. /// -/// # Write cost +/// # Writes /// -/// `put` and `delete` each perform a synchronous outbound HTTPS request to the -/// Fastly management API (`api.fastly.com`). Callers that issue many writes in -/// one request pay one round-trip per call. The `"api-keys"` secret store is -/// opened per call to read the management token; the Fastly Compute SDK caches -/// the open handle so that cost is negligible. -pub struct FastlyPlatformConfigStore; +/// `put` and `delete` each send one buffered HTTPS request to the Fastly +/// management API (`api.fastly.com`) through the request's outbound +/// [`HttpClient`]. Callers that issue many writes in one request pay one +/// round trip per call. The `"api-keys"` secret store is opened per call to +/// read the management token; the Fastly Compute SDK caches the open handle so +/// that cost is negligible. A store built with +/// [`FastlyPlatformConfigStore::read_only`], as settings load uses outside a +/// request, has no client and rejects writes. +pub struct FastlyPlatformConfigStore { + outbound_client: Option, +} + +impl FastlyPlatformConfigStore { + /// Builds a store whose writes go through `client`, the request's outbound client. + #[must_use] + pub fn with_outbound_client(client: HttpClient) -> Self { + Self { + outbound_client: Some(client), + } + } + /// Builds a store that only reads, for settings load outside a request. + #[must_use] + pub fn read_only() -> Self { + Self { + outbound_client: None, + } + } +} + +#[async_trait::async_trait(?Send)] impl PlatformConfigStore for FastlyPlatformConfigStore { fn get(&self, store_name: &StoreName, key: &str) -> Result> { let name = store_name.as_ref(); @@ -74,14 +75,23 @@ impl PlatformConfigStore for FastlyPlatformConfigStore { }) } - fn put(&self, store_id: &StoreId, key: &str, value: &str) -> Result<(), Report> { - let client = crate::management_api::FastlyManagementApiClient::new()?; - client.update_config_item(store_id.as_ref(), key, value) + async fn put( + &self, + store_id: &StoreId, + key: &str, + value: &str, + ) -> Result<(), Report> { + let client = write_client(self.outbound_client.as_ref(), PlatformError::ConfigStore)?; + FastlyManagementApiClient::new()? + .update_config_item(client, store_id.as_ref(), key, value) + .await } - fn delete(&self, store_id: &StoreId, key: &str) -> Result<(), Report> { - let client = crate::management_api::FastlyManagementApiClient::new()?; - client.delete_config_item(store_id.as_ref(), key) + async fn delete(&self, store_id: &StoreId, key: &str) -> Result<(), Report> { + let client = write_client(self.outbound_client.as_ref(), PlatformError::ConfigStore)?; + FastlyManagementApiClient::new()? + .delete_config_item(client, store_id.as_ref(), key) + .await } } @@ -91,17 +101,37 @@ impl PlatformConfigStore for FastlyPlatformConfigStore { /// Fastly [`SecretStore`]-backed implementation of [`PlatformSecretStore`]. /// -/// Stateless — the store name is supplied per call. This replaces the -/// store-name-at-construction pattern of the legacy `FastlySecretStore` -/// (removed). +/// The store name is supplied per call. /// -/// # Write cost +/// # Writes /// /// `create` and `delete` have the same per-call -/// [`crate::management_api::FastlyManagementApiClient`] cost described on -/// [`FastlyPlatformConfigStore`]. -pub struct FastlyPlatformSecretStore; +/// [`crate::management_api::FastlyManagementApiClient`] cost, and the same +/// read-only rule, described on [`FastlyPlatformConfigStore`]. +pub struct FastlyPlatformSecretStore { + outbound_client: Option, +} + +impl FastlyPlatformSecretStore { + /// Builds a store whose writes go through `client`, the request's outbound client. + #[must_use] + pub fn with_outbound_client(client: HttpClient) -> Self { + Self { + outbound_client: Some(client), + } + } + + /// Builds a store that only reads, as settings load and the management + /// token read use. + #[must_use] + pub fn read_only() -> Self { + Self { + outbound_client: None, + } + } +} +#[async_trait::async_trait(?Send)] impl PlatformSecretStore for FastlyPlatformSecretStore { fn get_bytes( &self, @@ -135,575 +165,46 @@ impl PlatformSecretStore for FastlyPlatformSecretStore { }) } - fn create( + async fn create( &self, store_id: &StoreId, name: &str, value: &str, ) -> Result<(), Report> { - let client = crate::management_api::FastlyManagementApiClient::new()?; - client.create_secret(store_id.as_ref(), name, value) + let client = write_client(self.outbound_client.as_ref(), PlatformError::SecretStore)?; + FastlyManagementApiClient::new()? + .create_secret(client, store_id.as_ref(), name, value) + .await } - fn delete(&self, store_id: &StoreId, name: &str) -> Result<(), Report> { - let client = crate::management_api::FastlyManagementApiClient::new()?; - client.delete_secret(store_id.as_ref(), name) + async fn delete(&self, store_id: &StoreId, name: &str) -> Result<(), Report> { + let client = write_client(self.outbound_client.as_ref(), PlatformError::SecretStore)?; + FastlyManagementApiClient::new()? + .delete_secret(client, store_id.as_ref(), name) + .await } } -// --------------------------------------------------------------------------- -// FastlyPlatformBackend -// --------------------------------------------------------------------------- - -/// Fastly dynamic-backend implementation of [`PlatformBackend`]. -/// -/// Delegates name computation and registration to [`BackendConfig`], preserving -/// the existing deterministic naming scheme (scheme + host + port + cert + -/// timeout → unique name). -pub struct FastlyPlatformBackend; - -#[cfg(test)] -const TRANSPORT_TIMEOUT_QUANTUM_MS: u32 = 250; -#[cfg(test)] -const SUB_QUANTUM_LADDER_MS: [u32; 4] = [200, 150, 100, 50]; - -fn backend_config_from_spec(spec: &PlatformBackendSpec) -> BackendConfig<'_> { - BackendConfig::new(&spec.scheme, &spec.host) - .port(spec.port) - .host_header_override(spec.host_header_override.as_deref()) - .certificate_check(spec.certificate_check) - .first_byte_timeout(spec.first_byte_timeout) - .between_bytes_timeout(spec.between_bytes_timeout) - .discriminator(spec.discriminator.as_deref()) -} - -impl PlatformBackend for FastlyPlatformBackend { - fn naming_policy(&self) -> BackendNamingPolicy { - BackendNamingPolicy::Fastly - } - - fn predict_name(&self, spec: &PlatformBackendSpec) -> Result> { - // Use the same host normalization as registration. In particular, - // URL-derived IPv6 hosts arrive bracketed, but both forms must predict - // the backend that `ensure` actually registers. - backend_config_from_spec(spec) - .predict_name() - .change_context(PlatformError::Backend) - } - - fn ensure(&self, spec: &PlatformBackendSpec) -> Result> { - backend_config_from_spec(spec) - .ensure() - .change_context(PlatformError::Backend) - } -} - -fn buffered_response_to_platform( - response: &fastly::Response, - backend_name: String, - bytes: Vec, -) -> Result> { - let mut builder = edgezero_core::http::response_builder().status(response.get_status()); - for (name, value) in response.get_headers() { - builder = builder.header(name.as_str(), value.as_bytes()); - } - let response = builder - .body(edgezero_core::body::Body::from(bytes)) - .change_context(PlatformError::HttpClient)?; - Ok(PlatformResponse::new(response).with_backend_name(backend_name)) -} - -// --------------------------------------------------------------------------- -// FastlyPlatformHttpClient — helpers -// --------------------------------------------------------------------------- - -fn fastly_image_optimizer_region( - region: &str, -) -> Result> { - use fastly::image_optimizer::ImageOptimizerRegion; - - match PlatformImageOptimizerRegion::parse(region) { - Some(PlatformImageOptimizerRegion::UsEast) => Ok(ImageOptimizerRegion::UsEast), - Some(PlatformImageOptimizerRegion::UsCentral) => Ok(ImageOptimizerRegion::UsCentral), - Some(PlatformImageOptimizerRegion::UsWest) => Ok(ImageOptimizerRegion::UsWest), - Some(PlatformImageOptimizerRegion::EuCentral) => Ok(ImageOptimizerRegion::EuCentral), - Some(PlatformImageOptimizerRegion::EuWest) => Ok(ImageOptimizerRegion::EuWest), - Some(PlatformImageOptimizerRegion::Asia) => Ok(ImageOptimizerRegion::Asia), - Some(PlatformImageOptimizerRegion::Australia) => Ok(ImageOptimizerRegion::Australia), - None => Err(Report::new(PlatformError::HttpClient) - .attach(format!("unsupported Image Optimizer region: {region}"))), - } -} - -fn fastly_image_optimizer_format( - format: &str, -) -> Result> { - use fastly::image_optimizer::Format; - - match format.trim().to_ascii_lowercase().as_str() { - "auto" => Ok(Format::Auto), - "avif" => Ok(Format::AVIF), - "gif" => Ok(Format::GIF), - "jpeg" | "jpg" => Ok(Format::JPEG), - "jxl" | "jpegxl" => Ok(Format::JPEGXL), - "mp4" => Ok(Format::MP4), - "png" => Ok(Format::PNG), - "webp" => Ok(Format::WebP), - other => Err(Report::new(PlatformError::HttpClient) - .attach(format!("unsupported Image Optimizer format: {other}"))), - } -} - -fn fastly_resize_filter( - resize_filter: &str, -) -> Result> { - use fastly::image_optimizer::ResizeAlgorithm; - - match resize_filter.trim().to_ascii_lowercase().as_str() { - "nearest" => Ok(ResizeAlgorithm::Nearest), - "bilinear" => Ok(ResizeAlgorithm::Bilinear), - "bicubic" => Ok(ResizeAlgorithm::Bicubic), - "lanczos2" => Ok(ResizeAlgorithm::Lanczos2), - "lanczos3" => Ok(ResizeAlgorithm::Lanczos3), - other => Err(Report::new(PlatformError::HttpClient).attach(format!( - "unsupported Image Optimizer resize filter: {other}" - ))), - } -} - -fn fastly_crop(crop: &PlatformImageOptimizerCrop) -> fastly::image_optimizer::Crop { - use fastly::image_optimizer::{Area, Crop, CropMode, PointOrOffset, Position}; - - let position = match (crop.offset_x, crop.offset_y) { - (Some(x), Some(y)) => Some(Position { - x: Some(PointOrOffset::Offset(x)), - y: Some(PointOrOffset::Offset(y)), - }), - _ => None, - }; - let mode = crop - .mode - .map(|PlatformImageOptimizerCropMode::Smart| CropMode::Smart); - - Crop { - size: Area::AspectRatio((crop.width, crop.height)), - position, - mode, - } -} - -fn apply_fastly_image_optimizer_params( - target: &mut fastly::image_optimizer::ImageOptimizerOptions, - params: PlatformImageOptimizerParams, -) -> Result<(), Report> { - use fastly::image_optimizer::PixelsOrPercentage; - - if let Some(format) = params.format { - target.format = Some(fastly_image_optimizer_format(&format)?); - } - if let Some(quality) = params.quality { - target.quality = Some(quality); - } - if let Some(resize_filter) = params.resize_filter { - target.resize_filter = Some(fastly_resize_filter(&resize_filter)?); - } - if let Some(width) = params.width { - target.width = Some(PixelsOrPercentage::Pixels(width)); - } - if let Some(height) = params.height { - target.height = Some(PixelsOrPercentage::Pixels(height)); - } - if let Some(crop) = params.crop { - target.crop = Some(fastly_crop(&crop)); - } - - Ok(()) -} - -fn apply_fastly_image_optimizer( - req: &mut fastly::Request, - options: PlatformImageOptimizerOptions, -) -> Result<(), Report> { - let region = fastly_image_optimizer_region(&options.region)?; - let mut fastly_options = fastly::image_optimizer::ImageOptimizerOptions::from_region(region); - fastly_options.preserve_query_string_on_origin_request = - Some(options.preserve_query_string_on_origin_request); - apply_fastly_image_optimizer_params(&mut fastly_options, options.params)?; - req.set_image_optimizer(fastly_options); - Ok(()) -} - -/// Convert a platform-neutral [`edgezero_core::http::Request`] to a [`fastly::Request`]. -/// -/// Only buffered `Body::Once` bodies are supported on this path. +/// Returns the outbound client a store write goes through. /// /// # Errors /// -/// Returns [`PlatformError::HttpClient`] when the request body is streaming. -fn edge_request_to_fastly( - request: edgezero_core::http::Request, -) -> Result> { - let (parts, body) = request.into_parts(); - let mut fastly_req = fastly::Request::new(parts.method, parts.uri.to_string()); - for (name, value) in parts.headers.iter() { - // `fastly::Request::new` derives a Host header from the request URI, so - // appending the edge request's own Host would leave a duplicate. Replace - // it instead to keep the in-memory request well-formed. The Host actually - // sent on the wire is still governed by the backend's `override_host` - // (see `BackendConfig::ensure`), which forces the value regardless. - if name == edgezero_core::http::header::HOST { - fastly_req.set_header(name.as_str(), value.as_bytes()); - } else { - fastly_req.append_header(name.as_str(), value.as_bytes()); - } - } - match body { - edgezero_core::body::Body::Once(bytes) => { - if !bytes.is_empty() { - fastly_req.set_body(bytes.to_vec()); - } - } - edgezero_core::body::Body::Stream(_) => { - return Err(Report::new(PlatformError::HttpClient) - .attach("streaming request body is not supported by Fastly request conversion")); - } - } - Ok(fastly_req) -} - -/// Maximum origin response body size copied into WASM heap. -/// -/// `take_body_bytes()` copies the full origin response into a single -/// allocation. This cap prevents oversized origin responses from exhausting -/// the WASM address space. The Content-Length pre-check avoids the copy -/// entirely for responses that declare their size. -const MAX_PLATFORM_RESPONSE_BODY_BYTES: usize = 10 * 1024 * 1024; // 10 MiB - -fn fastly_body_to_edge_stream(body: fastly::Body) -> edgezero_core::body::Body { - let stream = futures::stream::unfold(Some(body), |state| async move { - let mut body = state?; - let mut chunk = vec![0; 8192]; - match body.read(&mut chunk) { - Ok(0) => None, - Ok(bytes_read) => { - chunk.truncate(bytes_read); - Some((Ok(Bytes::from(chunk)), Some(body))) - } - Err(err) => Some((Err(err), None)), - } - }); - - edgezero_core::body::Body::from_stream(stream) -} - -/// Return whether a response is allowed to carry a body. -/// -/// `HEAD` responses and informational, 204, 205, and 304 statuses carry no -/// content. In particular, HEAD and 304 may legitimately retain a -/// `Content-Length` for the corresponding GET representation. -fn response_carries_body(request_is_head: bool, status: fastly::http::StatusCode) -> bool { - !request_is_head - && !status.is_informational() - && status != fastly::http::StatusCode::NO_CONTENT - && status != fastly::http::StatusCode::RESET_CONTENT - && status != fastly::http::StatusCode::NOT_MODIFIED -} - -/// Convert a [`fastly::Response`] to a [`PlatformResponse`] with the given backend name. -fn fastly_response_to_platform( - mut resp: fastly::Response, - backend_name: impl Into, - stream_response: bool, - request_is_head: bool, -) -> Result> { - let status = resp.get_status(); - let response_body_expected = response_carries_body(request_is_head, status); - - // Pre-flight: reject oversized responses before copying bytes into WASM heap. - // Content-Length is advisory but covers most origin responses; chunked - // responses without it fall through to the post-materialization check below. - if response_body_expected - && !stream_response - && let Some(claimed_len) = resp - .get_header("content-length") - .and_then(|v| v.to_str().ok()) - .and_then(|s| s.trim().parse::().ok()) - && claimed_len > MAX_PLATFORM_RESPONSE_BODY_BYTES - { - return Err(Report::new(PlatformError::HttpClient).attach(format!( - "origin Content-Length {claimed_len} exceeds \ - {MAX_PLATFORM_RESPONSE_BODY_BYTES}-byte response body limit" - ))); - } - - let mut builder = edgezero_core::http::response_builder().status(status); - for (name, value) in resp.get_headers() { - builder = builder.header(name.as_str(), value.as_bytes()); - } - let body = if !response_body_expected { - edgezero_core::body::Body::empty() - } else if stream_response { - fastly_body_to_edge_stream(resp.take_body()) - } else { - let body_bytes = resp.take_body_bytes(); - - // Belt-and-suspenders: catches chunked responses without Content-Length. - if body_bytes.len() > MAX_PLATFORM_RESPONSE_BODY_BYTES { - return Err(Report::new(PlatformError::HttpClient).attach(format!( - "origin response body {} bytes exceeds \ - {MAX_PLATFORM_RESPONSE_BODY_BYTES}-byte limit", - body_bytes.len() - ))); - } - - edgezero_core::body::Body::from(body_bytes) - }; - let edge_response = builder - .body(body) - .change_context(PlatformError::HttpClient)?; - Ok(PlatformResponse::new(edge_response).with_backend_name(backend_name)) -} - -// --------------------------------------------------------------------------- -// FastlyPlatformHttpClient -// --------------------------------------------------------------------------- - -/// Apply the caller's cache intent to a Fastly request. -/// -/// The two branches are mutually exclusive by construction, which is the point of -/// [`PlatformCacheIntent`]: `set_surrogate_key` "overrides any previous -/// `Request::set_pass` call" (`fastly-0.12.1/src/http/request.rs:2462`), so calling both -/// would silently cancel the bypass. -/// -/// Readthrough is enabled by *omitting* `set_pass`, never by adding a TTL. `set_ttl` -/// carries the same override note and additionally overrides the origin's own -/// `Cache-Control`, including `private` and `no-store` — it would turn the hazard this -/// gate exists to avoid into an API. -fn apply_fastly_cache_intent(request: &mut fastly::Request, intent: &PlatformCacheIntent) { - match intent { - PlatformCacheIntent::Bypass => request.set_pass(true), - PlatformCacheIntent::Shared { surrogate_key } => { - match fastly::http::HeaderValue::from_str(surrogate_key) { - Ok(value) => request.set_surrogate_key(value), - Err(error) => { - // Fail closed. Caching without the key would store an object no purge - // can reach, which is worse than not caching it: the whole rollback - // story for readthrough is "purge the key". - log::error!( - "Surrogate key {surrogate_key:?} is not a valid header value \ - ({error}); bypassing the cache rather than storing an \ - unpurgeable object" - ); - request.set_pass(true); - } - } - } - PlatformCacheIntent::Default => {} - } -} - -/// Fastly implementation of [`PlatformHttpClient`]. -/// -/// - [`send`](PlatformHttpClient::send) converts the platform request to a -/// `fastly::Request`, applies Image Optimizer metadata when present, calls -/// `.send()`, and wraps the response. Asset requests can ask to preserve the -/// response body as a stream instead of buffering it into a single `Vec`. -/// - [`send_async`](PlatformHttpClient::send_async) converts the request and -/// calls `.send_async()`. It rejects Image Optimizer metadata because Fastly's -/// async request path does not expose the IO attachment used by asset routes. -/// - [`select`](PlatformHttpClient::select) downcasts each -/// [`PlatformPendingRequest`] back to `fastly::PendingRequest` and calls -/// `fastly::http::request::select()`. -/// -/// Fastly's Compute HTTP API sends one request to the named backend and returns -/// the origin response; it has no client-side redirect-follow mode. Consequently -/// each trait call below performs exactly one underlying `.send()` or -/// `.send_async()`, and an original 3xx remains visible to core. The host test -/// environment cannot register a real Fastly backend, so the common -/// `StubHttpClient` driver test records the one-send 3xx behavior while adapter -/// tests cover request conversion and the single-send boundary. -/// -/// Deadline collection requires [`PlatformCacheIntent::Bypass`], which all -/// auction requests use. This guarantees a direct SDK pending handle that can -/// join body handles in one timed host wait. Shared-cache requests retain the -/// SDK caching state machine through the unbounded send/select/wait APIs. -pub struct FastlyPlatformHttpClient; - -#[async_trait::async_trait(?Send)] -impl PlatformHttpClient for FastlyPlatformHttpClient { - fn supports_streaming_responses(&self) -> bool { - true - } - - fn supports_pending_streaming_responses(&self) -> bool { - true - } - - async fn send( - &self, - request: PlatformHttpRequest, - ) -> Result> { - let backend_name = request.backend_name.clone(); - let image_optimizer = request.image_optimizer; - let stream_response = request.stream_response; - let cache_intent = request.cache_intent.clone(); - let request_is_head = request.request.method() == edgezero_core::http::Method::HEAD; - let mut fastly_req = edge_request_to_fastly(request.request)?; - if let Some(options) = image_optimizer { - apply_fastly_image_optimizer(&mut fastly_req, options)?; - } - apply_fastly_cache_intent(&mut fastly_req, &cache_intent); - let fastly_resp = fastly_req - .send(&backend_name) - .change_context(PlatformError::HttpClient)?; - fastly_response_to_platform(fastly_resp, backend_name, stream_response, request_is_head) - } - - async fn send_async( - &self, - request: PlatformHttpRequest, - ) -> Result> { - let backend_name = request.backend_name.clone(); - if request.image_optimizer.is_some() { - return Err(Report::new(PlatformError::HttpClient) - .attach("Image Optimizer is not supported with Fastly send_async")); - } - let stream_response = request.stream_response; - let request_method = request.request.method().clone(); - let cache_intent = request.cache_intent.clone(); - let mut fastly_req = edge_request_to_fastly(request.request)?; - apply_fastly_cache_intent(&mut fastly_req, &cache_intent); - let pending = fastly_req - .send_async(&backend_name) - .change_context(PlatformError::HttpClient)?; - let pending = if cache_intent.is_bypass() { - PlatformPendingRequest::new(CacheBypassPending(pending)) - } else { - PlatformPendingRequest::new(pending) - }; - Ok(pending - .with_backend_name(backend_name) - .with_response_handling(stream_response, request_method)) - } - - async fn select( - &self, - pending_requests: Vec, - ) -> Result> { - use fastly::http::request::{PendingRequest, select}; - - if pending_requests.is_empty() { - return Err(Report::new(PlatformError::HttpClient) - .attach("select called with an empty pending_requests list")); - } - - if pending_requests - .iter() - .any(PlatformPendingRequest::stream_response) - { - return Err(Report::new(PlatformError::HttpClient) - .attach("stream-marked pending request requires direct wait")); - } - - let mut fastly_pending: Vec = Vec::with_capacity(pending_requests.len()); - - for platform_req in pending_requests { - let inner = unwrap_pending(platform_req).map_err(|platform_req| { - let backend_name = platform_req.backend_name().unwrap_or(""); - Report::new(PlatformError::HttpClient).attach(format!( - "PlatformPendingRequest inner type is not fastly::PendingRequest for backend '{backend_name}'" - )) - })?; - fastly_pending.push(inner); - } - - let (result, remaining_fastly) = select(fastly_pending); - - // Fastly's select() does not preserve input order for remaining requests, - // so positional backend-name re-association is unreliable. Backend names - // are re-derived from get_backend_name() when each remaining request completes. - let remaining: Vec = remaining_fastly - .into_iter() - .map(PlatformPendingRequest::new) - .collect(); - - let (ready, failed_backend_name) = match result { - Ok(fastly_resp) => { - let Some(backend_name) = fastly_resp.get_backend_name().map(str::to_string) else { - return Err(Report::new(PlatformError::HttpClient) - .attach("select: response has no backend name; correlation impossible")); - }; - let Some(backend_request) = fastly_resp.get_backend_request() else { - return Err(Report::new(PlatformError::HttpClient).attach( - "select: response has no originating request; body semantics unknown", - )); - }; - let request_is_head = backend_request.get_method() == fastly::http::Method::HEAD; - ( - fastly_response_to_platform(fastly_resp, backend_name, false, request_is_head), - None, - ) - } - Err(e) => { - let failed_name = e.backend_name().to_string(); - ( - Err(Report::new(PlatformError::HttpClient).attach(format!( - "fastly select error for backend '{failed_name}': {e}" - ))), - Some(failed_name), - ) - } - }; - - Ok(PlatformSelectResult { - ready, - remaining, - failed_backend_name, - }) - } - - async fn select_until( - &self, - pending_requests: Vec, - deadline: web_time::Instant, - ) -> Result, Report> { - deadline::select_until(pending_requests, deadline) - } - - async fn wait( - &self, - pending: PlatformPendingRequest, - ) -> Result> { - let backend_hint = pending.backend_name().map(str::to_owned); - let stream_response = pending.stream_response(); - let request_is_head = pending.request_method() == Some(&edgezero_core::http::Method::HEAD); - let pending = unwrap_pending(pending).map_err(|pending| { - let backend_name = pending.backend_name().unwrap_or(""); - Report::new(PlatformError::HttpClient).attach(format!( - "PlatformPendingRequest inner type is not fastly::PendingRequest for backend '{backend_name}'" - )) - })?; - let response = pending.wait().change_context(PlatformError::HttpClient)?; - let backend_name = response - .get_backend_name() - .map(str::to_owned) - .or(backend_hint) - .ok_or_else(|| { - Report::new(PlatformError::HttpClient) - .attach("wait: response has no backend name; correlation impossible") - })?; - fastly_response_to_platform(response, backend_name, stream_response, request_is_head) - } +/// Returns `error` when the store was built read-only and so has no client. +fn write_client( + client: Option<&HttpClient>, + error: PlatformError, +) -> Result<&HttpClient, Report> { + client.ok_or_else(|| { + Report::new(error) + .attach("a read-only store cannot write; build it with the request's outbound client") + }) } // --------------------------------------------------------------------------- -// FastlyPlatformGeo +// RequestGeo // --------------------------------------------------------------------------- /// Convert a Fastly [`Geo`] value into a platform-neutral [`GeoInfo`]. -/// -/// Shared by `FastlyPlatformGeo::lookup` in `trusted-server-adapter-fastly` so -/// that field mapping is never duplicated. fn geo_from_fastly(geo: &Geo) -> GeoInfo { GeoInfo { city: geo.city().to_string(), @@ -717,14 +218,51 @@ fn geo_from_fastly(geo: &Geo) -> GeoInfo { } } -/// Fastly geo-lookup implementation of [`PlatformGeo`]. -pub struct FastlyPlatformGeo; +/// The geo result the entry point looks up once for a request's resolved client IP. +/// +/// `prepare` inserts it into the request extensions. The finalize middleware, +/// the trace metadata, and the per-request services read it, so a request makes +/// one host geo lookup however many steps use geo. +#[derive(Debug, Clone, Default)] +pub(crate) struct RequestGeo { + client_ip: Option, + geo: Option, +} -impl PlatformGeo for FastlyPlatformGeo { +impl RequestGeo { + /// Holds `geo` as the lookup result for `client_ip`. + #[cfg(test)] + pub(crate) fn new(client_ip: Option, geo: Option) -> Self { + Self { client_ip, geo } + } + + /// Looks up `client_ip` once with Fastly's geo database. + pub(crate) fn lookup_once(client_ip: Option) -> Self { + Self { + client_ip, + geo: client_ip + .and_then(geo_lookup) + .map(|geo| geo_from_fastly(&geo)), + } + } + + /// The request's geo, if the lookup found any. + pub(crate) fn geo(&self) -> Option<&GeoInfo> { + self.geo.as_ref() + } +} + +impl PlatformGeo for RequestGeo { + /// Answers the request's own IP from the held result, without a host lookup. + /// + /// Any other IP gets `Ok(None)`: there is no data for an IP this request + /// did not resolve. fn lookup(&self, client_ip: Option) -> Result, Report> { - Ok(client_ip - .and_then(geo_lookup) - .map(|geo| geo_from_fastly(&geo))) + if client_ip == self.client_ip { + Ok(self.geo.clone()) + } else { + Ok(None) + } } } @@ -805,12 +343,7 @@ pub fn client_info_from_request(req: &Request, client_ip: Option) -> Cli #[cfg(test)] mod tests { - use std::io; - use std::time::Duration; - use super::*; - use edgezero_core::body::Body; - use edgezero_core::http::request_builder; use fastly::http::HeaderValue; use trusted_server_core::redacted::Redacted; use trusted_server_core::settings::TrustedClientIpConfig; @@ -835,6 +368,83 @@ mod tests { req } + #[test] + fn request_geo_answers_only_the_ip_it_resolved() { + let resolved_ip = IpAddr::V4(std::net::Ipv4Addr::new(198, 51, 100, 7)); + // Viceroy's default geo data answers loopback, so a restored host + // lookup for this IP would return data and fail the test. + let other_ip = IpAddr::V4(std::net::Ipv4Addr::LOCALHOST); + let request_geo = RequestGeo::new( + Some(resolved_ip), + Some(GeoInfo { + city: "Springfield".to_owned(), + country: "US".to_owned(), + continent: "NorthAmerica".to_owned(), + latitude: 39.78, + longitude: -89.65, + metro_code: 648, + region: Some("IL".to_owned()), + asn: None, + }), + ); + + let held = request_geo + .lookup(Some(resolved_ip)) + .expect("should answer the resolved IP"); + let other = request_geo + .lookup(Some(other_ip)) + .expect("should answer another IP"); + + assert_eq!( + held.map(|geo| geo.city), + Some("Springfield".to_owned()), + "should return the held geo for the resolved IP" + ); + assert!( + other.is_none(), + "should have no data for an IP this request did not resolve" + ); + } + + #[test] + fn read_only_stores_reject_writes() { + let config = FastlyPlatformConfigStore::read_only(); + let secrets = FastlyPlatformSecretStore::read_only(); + let store_id = StoreId::from("example-store-id"); + + let errors = futures::executor::block_on(async { + [ + ( + config.put(&store_id, "example-key", "example-value").await, + PlatformError::ConfigStore, + ), + ( + config.delete(&store_id, "example-key").await, + PlatformError::ConfigStore, + ), + ( + secrets + .create(&store_id, "example-name", "example-value") + .await, + PlatformError::SecretStore, + ), + ( + secrets.delete(&store_id, "example-name").await, + PlatformError::SecretStore, + ), + ] + }); + + for (result, expected) in errors { + let error = result.expect_err("should reject a write without an outbound client"); + assert_eq!( + error.current_context().to_string(), + expected.to_string(), + "should report the write against its own store kind" + ); + } + } + #[test] fn resolve_client_ip_uses_peer_when_config_is_absent() { let req = authenticated_request("198.51.100.7"); @@ -989,783 +599,4 @@ mod tests { "should preserve the supplied client IP" ); } - - #[test] - fn edge_request_to_fastly_replaces_url_derived_host_header() { - let request = request_builder() - .method("GET") - .uri("https://origin.example.com/") - .header(edgezero_core::http::header::HOST, "www.example.com") - .body(Body::empty()) - .expect("should build request"); - - let fastly_req = edge_request_to_fastly(request).expect("should convert request"); - - assert_eq!( - fastly_req.get_header_str(fastly::http::header::HOST), - Some("www.example.com"), - "should replace the URL-derived Host instead of appending a duplicate" - ); - } - - #[test] - fn edge_request_to_fastly_preserves_query_encoding_order_and_duplicates() { - let expected_query = "space=a+b&plus=%2B"e=%27&empty=&x=1&x=2&country=US®ion=CA"; - let request = request_builder() - .method("GET") - .uri(format!( - "https://sdk.example.com/key/loader.js?{expected_query}" - )) - .body(Body::empty()) - .expect("should build request with encoded query"); - - let fastly_req = edge_request_to_fastly(request).expect("should convert request"); - - assert_eq!( - fastly_req.get_url().query(), - Some(expected_query), - "should preserve query encoding and order across Fastly conversion" - ); - assert_eq!( - fastly_req - .get_url() - .query_pairs() - .filter(|(name, _)| name.eq_ignore_ascii_case("country")) - .count(), - 1, - "should retain exactly one country pair" - ); - assert_eq!( - fastly_req - .get_url() - .query_pairs() - .filter(|(name, _)| name.eq_ignore_ascii_case("region")) - .count(), - 1, - "should retain exactly one region pair" - ); - } - - // --- FastlyPlatformBackend::predict_name -------------------------------- - - #[test] - fn predict_name_produces_same_name_as_backend_config() { - let backend = FastlyPlatformBackend; - let spec = PlatformBackendSpec { - scheme: "https".to_string(), - host: "origin.example.com".to_string(), - port: None, - host_header_override: None, - certificate_check: true, - first_byte_timeout: Duration::from_secs(15), - between_bytes_timeout: Duration::from_secs(15), - discriminator: None, - }; - - let name = backend - .predict_name(&spec) - .expect("should compute backend name for valid spec"); - - assert!( - name.starts_with("backend_https_origin_example_com_443_fb15000_bb15000_"), - "should match BackendConfig naming convention, got {name}" - ); - } - - #[test] - fn predict_name_includes_host_header_override_suffix() { - let backend = FastlyPlatformBackend; - let spec = PlatformBackendSpec { - scheme: "https".to_string(), - host: "origin.example.com".to_string(), - port: None, - host_header_override: Some("www.example.com".to_string()), - certificate_check: true, - first_byte_timeout: Duration::from_secs(15), - between_bytes_timeout: Duration::from_secs(15), - discriminator: None, - }; - - let name = backend - .predict_name(&spec) - .expect("should compute backend name for host header override"); - - assert!( - name.starts_with( - "backend_https_origin_example_com_443_oh_www_example_com_fb15000_bb15000_" - ), - "should match BackendConfig naming convention with host header override, got {name}" - ); - } - - #[test] - fn predict_name_includes_nocert_suffix_when_cert_check_disabled() { - let backend = FastlyPlatformBackend; - let spec = PlatformBackendSpec { - scheme: "https".to_string(), - host: "origin.example.com".to_string(), - port: None, - host_header_override: None, - certificate_check: false, - first_byte_timeout: Duration::from_secs(15), - between_bytes_timeout: Duration::from_secs(15), - discriminator: None, - }; - - let name = backend - .predict_name(&spec) - .expect("should compute name with cert check disabled"); - - assert!( - name.contains("nocert"), - "should include nocert suffix when certificate_check is false" - ); - } - - #[test] - fn predict_name_returns_error_for_empty_host() { - let backend = FastlyPlatformBackend; - let spec = PlatformBackendSpec { - scheme: "https".to_string(), - host: String::new(), - port: None, - host_header_override: None, - certificate_check: true, - first_byte_timeout: Duration::from_secs(15), - between_bytes_timeout: Duration::from_secs(15), - discriminator: None, - }; - - let result = backend.predict_name(&spec); - - assert!(result.is_err(), "should return an error for empty host"); - } - - #[test] - fn predict_name_encodes_custom_timeout() { - let backend = FastlyPlatformBackend; - let spec = PlatformBackendSpec { - scheme: "https".to_string(), - host: "origin.example.com".to_string(), - port: None, - host_header_override: None, - certificate_check: true, - first_byte_timeout: Duration::from_millis(2000), - between_bytes_timeout: Duration::from_millis(2000), - discriminator: None, - }; - - let name = backend - .predict_name(&spec) - .expect("should compute name with custom timeout"); - - assert!( - name.contains("_fb2000_bb2000_"), - "should encode 2000ms first-byte and between-bytes timeouts in name, got {name}" - ); - } - - #[test] - fn predict_name_matches_ensured_backend_name() { - // The auction orchestrator maps responses back to providers by the - // predicted backend name, so predict_name and ensure must return the - // identical string for the same spec — a divergence would make - // responses land in the "unknown backend" branch and drop bids - // silently. - let backend = FastlyPlatformBackend; - let spec = PlatformBackendSpec { - scheme: "https".to_string(), - host: "consistency.example.com".to_string(), - port: None, - host_header_override: None, - certificate_check: true, - first_byte_timeout: Duration::from_millis(750), - between_bytes_timeout: Duration::from_millis(750), - discriminator: None, - }; - - let predicted = backend - .predict_name(&spec) - .expect("should predict backend name"); - let ensured = backend - .ensure(&spec) - .expect("should register backend for valid spec"); - - assert_eq!( - predicted, ensured, - "predicted backend name should match the registered backend name" - ); - } - - #[test] - fn bracketed_ipv6_predict_name_matches_bare_and_ensured_backend_name() { - let backend = FastlyPlatformBackend; - let bracketed = PlatformBackendSpec { - scheme: "https".to_string(), - host: "[2001:db8::9]".to_string(), - port: Some(8443), - host_header_override: None, - certificate_check: true, - first_byte_timeout: Duration::from_millis(750), - between_bytes_timeout: Duration::from_millis(750), - discriminator: Some("ipv6-provider".to_string()), - }; - let mut bare = bracketed.clone(); - bare.host = "2001:db8::9".to_string(); - - let predicted = backend - .predict_name(&bracketed) - .expect("should predict bracketed IPv6 backend name"); - let bare_predicted = backend - .predict_name(&bare) - .expect("should predict bare IPv6 backend name"); - let ensured = backend - .ensure(&bracketed) - .expect("should register bracketed IPv6 backend"); - - assert_eq!(predicted, bare_predicted); - assert_eq!(predicted, ensured); - } - - // --- FastlyPlatformHttpClient ------------------------------------------- - - #[test] - fn auction_http_capabilities_are_explicit() { - let client = FastlyPlatformHttpClient; - let capabilities = trusted_server_core::platform::AuctionTargetId::Fastly - .descriptor() - .capabilities(); - assert!(client.supports_concurrent_fanout()); - assert!(capabilities.supports_concurrent_provider_fanout()); - } - - #[test] - fn response_conversion_preserves_original_redirect_at_single_send_boundary() { - let mut response = fastly::Response::from_status(fastly::http::StatusCode::FOUND); - response.set_header("location", "https://redirect.example/next"); - - let platform = fastly_response_to_platform(response, "origin", false, false) - .expect("should convert redirect response"); - - assert_eq!(platform.response.status().as_u16(), 302); - assert_eq!( - platform - .response - .headers() - .get("location") - .and_then(|value| value.to_str().ok()), - Some("https://redirect.example/next") - ); - } - - #[test] - fn apply_fastly_cache_intent_sets_pass_for_bypass() { - let mut request = fastly::Request::get("https://example.com/"); - apply_fastly_cache_intent(&mut request, &PlatformCacheIntent::Bypass); - assert!( - format!("{request:?}").contains("cache_override: Pass"), - "bypass should select Fastly pass mode" - ); - } - - #[test] - fn apply_fastly_cache_intent_leaves_default_alone() { - let mut request = fastly::Request::get("https://example.com/"); - apply_fastly_cache_intent(&mut request, &PlatformCacheIntent::Default); - assert!( - format!("{request:?}").contains("cache_override: None"), - "the default intent should preserve Fastly read-through caching" - ); - } - - #[test] - fn apply_fastly_cache_intent_never_passes_on_the_shared_branch() { - // Readthrough is enabled by *omitting* set_pass. If this branch ever set pass as - // well, the surrogate key would reverse it — and the resulting behavior would - // depend on call order rather than on what the code says. - let mut request = fastly::Request::get("https://example.com/"); - apply_fastly_cache_intent( - &mut request, - &PlatformCacheIntent::Shared { - surrogate_key: "ts-origin".to_owned(), - }, - ); - let rendered = format!("{request:?}"); - assert!( - !rendered.contains("cache_override: Pass"), - "the shared branch must not bypass, got: {rendered}" - ); - assert!( - rendered.contains("ts-origin"), - "the shared branch must attach the surrogate key, got: {rendered}" - ); - } - - #[test] - fn a_surrogate_key_that_cannot_be_a_header_value_falls_back_to_bypass() { - // Fail closed: caching without the key would store an object no purge can reach, - // and "purge the key" is the entire rollback story for readthrough. - let mut request = fastly::Request::get("https://example.com/"); - apply_fastly_cache_intent( - &mut request, - &PlatformCacheIntent::Shared { - surrogate_key: "bad\nkey".to_owned(), - }, - ); - assert!( - format!("{request:?}").contains("cache_override: Pass"), - "an unusable key must bypass rather than store something unpurgeable" - ); - } - - #[test] - fn fastly_response_to_platform_allows_oversized_bodiless_content_length() { - let oversized_content_length = (MAX_PLATFORM_RESPONSE_BODY_BYTES + 1).to_string(); - for (request_is_head, status) in [ - (true, fastly::http::StatusCode::OK), - (false, fastly::http::StatusCode::CONTINUE), - (false, fastly::http::StatusCode::NO_CONTENT), - (false, fastly::http::StatusCode::RESET_CONTENT), - (false, fastly::http::StatusCode::NOT_MODIFIED), - ] { - let mut fastly_response = fastly::Response::from_status(status); - fastly_response.set_header( - fastly::http::header::CONTENT_LENGTH, - oversized_content_length.as_str(), - ); - - let platform_response = - fastly_response_to_platform(fastly_response, "origin", false, request_is_head) - .expect("should allow oversized metadata for a bodiless response"); - - assert_eq!( - platform_response - .response - .headers() - .get(edgezero_core::http::header::CONTENT_LENGTH) - .and_then(|value| value.to_str().ok()), - Some(oversized_content_length.as_str()), - "should preserve the origin Content-Length" - ); - let body = platform_response - .response - .into_body() - .into_bytes() - .expect("should return a buffered body"); - assert!(body.is_empty(), "should return an empty bodiless response"); - } - } - - #[test] - fn fastly_response_to_platform_rejects_oversized_buffered_get_content_length() { - let mut fastly_response = fastly::Response::from_status(200); - fastly_response.set_header( - fastly::http::header::CONTENT_LENGTH, - (MAX_PLATFORM_RESPONSE_BODY_BYTES + 1).to_string(), - ); - - let error = fastly_response_to_platform(fastly_response, "origin", false, false) - .expect_err("should reject oversized buffered GET metadata"); - let expected_error = - format!("exceeds {MAX_PLATFORM_RESPONSE_BODY_BYTES}-byte response body limit"); - - assert!( - format!("{error:?}").contains(&expected_error), - "should retain the buffered response size limit: {error:?}" - ); - } - - #[test] - fn response_carries_body_rejects_all_informational_statuses() { - let status = fastly::http::StatusCode::from_u16(199) - .expect("should construct an informational status code"); - - assert!( - !response_carries_body(false, status), - "should reject every informational response body" - ); - assert!( - !response_carries_body(true, fastly::http::StatusCode::OK), - "should reject HEAD response bodies" - ); - assert!( - response_carries_body(false, fastly::http::StatusCode::OK), - "should retain ordinary GET response bodies" - ); - } - - #[test] - fn fastly_platform_http_client_send_returns_error_for_unregistered_backend() { - let client = FastlyPlatformHttpClient; - let request = request_builder() - .method("GET") - .uri("https://example.com/") - .body(Body::empty()) - .expect("should build test request"); - let err = futures::executor::block_on( - client.send(PlatformHttpRequest::new(request, "nonexistent-backend")), - ) - .expect_err("should return error for unregistered backend"); - - assert!( - matches!(err.current_context(), &PlatformError::HttpClient), - "should be HttpClient error, got: {:?}", - err.current_context() - ); - } - - #[test] - fn fastly_platform_http_client_send_async_returns_error_for_unregistered_backend() { - let client = FastlyPlatformHttpClient; - let request = request_builder() - .method("GET") - .uri("https://example.com/") - .body(Body::empty()) - .expect("should build test request"); - let err = futures::executor::block_on( - client.send_async(PlatformHttpRequest::new(request, "nonexistent-backend")), - ) - .expect_err("should return error for unregistered backend"); - - assert!( - matches!(err.current_context(), &PlatformError::HttpClient), - "should be HttpClient error, got: {:?}", - err.current_context() - ); - } - - #[test] - fn fastly_platform_http_client_select_returns_error_for_empty_list() { - let client = FastlyPlatformHttpClient; - let err = futures::executor::block_on(client.select(vec![])) - .expect_err("should return error for empty pending list"); - - assert!( - matches!(err.current_context(), &PlatformError::HttpClient), - "should be HttpClient error, got: {:?}", - err.current_context() - ); - } - - #[test] - fn fastly_platform_http_client_select_returns_error_for_wrong_inner_type() { - let client = FastlyPlatformHttpClient; - // Wrap a non-PendingRequest type to trigger the downcast failure. - let wrong = PlatformPendingRequest::new(42u32).with_backend_name("origin-a"); - let err = futures::executor::block_on(client.select(vec![wrong])) - .expect_err("should return error for wrong inner type"); - - assert!( - matches!(err.current_context(), &PlatformError::HttpClient), - "should be HttpClient error, got: {:?}", - err.current_context() - ); - assert!( - format!("{err:?}").contains("origin-a"), - "should include backend name in error report: {err:?}" - ); - } - - #[test] - fn fastly_platform_http_client_rejects_stream_marked_pending_from_select() { - let client = FastlyPlatformHttpClient; - let pending = PlatformPendingRequest::new(42_u32) - .with_backend_name("origin-a") - .with_response_handling(true, edgezero_core::http::Method::GET); - let err = futures::executor::block_on(client.select(vec![pending])) - .expect_err("should reject stream-marked pending handles from select"); - - assert!( - format!("{err:?}").contains("stream-marked pending request"), - "should explain that streaming pendings require direct wait: {err:?}" - ); - } - - #[test] - fn fastly_platform_http_client_send_returns_error_for_streaming_body() { - let client = FastlyPlatformHttpClient; - let request = request_builder() - .method("POST") - .uri("https://example.com/") - .body(Body::from_stream(futures::stream::empty::< - Result<_, io::Error>, - >())) - .expect("should build streaming test request"); - - let err = futures::executor::block_on( - client.send(PlatformHttpRequest::new(request, "nonexistent-backend")), - ) - .expect_err("should reject streaming request bodies before sending"); - - assert!( - matches!(err.current_context(), &PlatformError::HttpClient), - "should be HttpClient error, got: {:?}", - err.current_context() - ); - assert!( - format!("{err:?}").contains("streaming request body"), - "should describe the unsupported streaming body: {err:?}" - ); - } - - #[test] - fn fastly_platform_http_client_send_async_rejects_image_optimizer_metadata() { - let client = FastlyPlatformHttpClient; - let request = request_builder() - .method("GET") - .uri("https://example.com/image.jpg") - .body(Body::empty()) - .expect("should build test request"); - let platform_request = PlatformHttpRequest::new(request, "nonexistent-backend") - .with_image_optimizer(PlatformImageOptimizerOptions::new( - "us_east", - PlatformImageOptimizerParams::default(), - )); - - let err = futures::executor::block_on(client.send_async(platform_request)) - .expect_err("should reject async Image Optimizer requests"); - - assert!( - format!("{err:?}").contains("Image Optimizer"), - "should explain unsupported async IO path: {err:?}" - ); - } - - #[test] - fn fastly_platform_http_client_supports_pending_streaming_responses() { - let client = FastlyPlatformHttpClient; - assert!( - client.supports_pending_streaming_responses(), - "should advertise direct pending-response streaming" - ); - - let request = request_builder() - .method("GET") - .uri("https://example.com/image.jpg") - .body(Body::empty()) - .expect("should build test request"); - let platform_request = - PlatformHttpRequest::new(request, "nonexistent-backend").with_stream_response(); - - let err = futures::executor::block_on(client.send_async(platform_request)) - .expect_err("should fail only because the backend is unregistered"); - - assert!( - !format!("{err:?}").contains("streaming responses are not supported"), - "should accept streaming on the async path before backend dispatch: {err:?}" - ); - } - - #[test] - fn fastly_platform_http_client_send_async_returns_error_for_streaming_body() { - let client = FastlyPlatformHttpClient; - let request = request_builder() - .method("POST") - .uri("https://example.com/") - .body(Body::from_stream(futures::stream::empty::< - Result<_, io::Error>, - >())) - .expect("should build streaming test request"); - - let err = futures::executor::block_on( - client.send_async(PlatformHttpRequest::new(request, "nonexistent-backend")), - ) - .expect_err("should reject streaming request bodies before launching async send"); - - assert!( - matches!(err.current_context(), &PlatformError::HttpClient), - "should be HttpClient error, got: {:?}", - err.current_context() - ); - assert!( - format!("{err:?}").contains("streaming request body"), - "should describe the unsupported streaming body: {err:?}" - ); - } - - // --- FastlyPlatformBackend::canonicalize_transport_timeout_ms ----------- - - #[test] - fn canonicalize_prefers_configured_timeout_when_budget_allows() { - let backend = FastlyPlatformBackend; - assert_eq!( - backend.canonicalize_transport_timeout_ms(2000, 1000), - 1000, - "should use the configured timeout verbatim when the budget allows" - ); - assert_eq!( - backend.canonicalize_transport_timeout_ms(2000, 100), - 100, - "should preserve a sub-quantum configured constant — it is name-stable on its own" - ); - } - - #[test] - fn canonicalize_floors_budget_bound_value_to_quantum() { - let backend = FastlyPlatformBackend; - assert_eq!( - backend.canonicalize_transport_timeout_ms(999, 2000), - 750, - "should floor a 999ms budget to the 750ms quantum bucket" - ); - assert_eq!( - backend.canonicalize_transport_timeout_ms(300, 2000), - 250, - "should floor a tight budget down to one quantum" - ); - assert_eq!( - backend.canonicalize_transport_timeout_ms(250, 2000), - 250, - "should keep an exact quantum multiple" - ); - } - - #[test] - fn canonicalize_snaps_sub_quantum_budget_to_bounded_ladder() { - let backend = FastlyPlatformBackend; - // Exact wall-clock values in 1..250 must NOT pass through — that is the - // unbounded-cardinality regression this ladder closes. - assert_eq!( - backend.canonicalize_transport_timeout_ms(249, 2000), - 200, - "should snap a sub-quantum budget down to the greatest ladder rung, not pass 249 through" - ); - assert_eq!(backend.canonicalize_transport_timeout_ms(200, 2000), 200); - assert_eq!(backend.canonicalize_transport_timeout_ms(150, 2000), 150); - assert_eq!(backend.canonicalize_transport_timeout_ms(100, 2000), 100); - assert_eq!(backend.canonicalize_transport_timeout_ms(50, 2000), 50); - assert_eq!( - backend.canonicalize_transport_timeout_ms(49, 2000), - 0, - "a budget below the smallest rung rounds to zero (launch skipped)" - ); - assert_eq!( - backend.canonicalize_transport_timeout_ms(0, 1000), - 0, - "an exhausted budget canonicalizes to zero" - ); - assert_eq!( - backend.canonicalize_transport_timeout_ms(100, 0), - 0, - "a zero configured timeout canonicalizes to zero" - ); - } - - #[test] - fn canonicalize_budget_derived_names_stay_within_a_safe_cardinality() { - // Enumerate every reachable remaining budget for a normal 2000ms - // ceiling and confirm the number of distinct backend-name-bearing - // transport values an origin can mint stays far below Fastly's - // per-service dynamic backend limit (documented default 200). - let backend = FastlyPlatformBackend; - let configured = 2000; - let mut distinct = std::collections::BTreeSet::new(); - for remaining in 0..=configured { - let value = backend.canonicalize_transport_timeout_ms(remaining, configured); - if value > 0 { - distinct.insert(value); - } - // No arbitrary clock-derived value may leak: every canonical value - // is either a quantum multiple or one of the bounded ladder rungs. - assert!( - value == 0 - || value % TRANSPORT_TIMEOUT_QUANTUM_MS == 0 - || SUB_QUANTUM_LADDER_MS.contains(&value), - "canonical value {value}ms (from remaining {remaining}ms) is neither a quantum \ - multiple nor a ladder rung" - ); - // The mediator hold bound relies on canonicalization never - // extending a transport cap past the wall-clock budget. - assert!( - value <= remaining, - "canonical value {value}ms must not extend past the remaining {remaining}ms budget" - ); - } - assert!( - distinct.len() <= 16, - "budget-derived transport values should stay well under the dynamic backend limit, \ - got {} distinct values: {distinct:?}", - distinct.len() - ); - } - - #[test] - fn canonicalize_budget_derived_names_stay_bounded_for_large_ceiling() { - // A large configured ceiling (e.g. a 60s mediator budget) must not let - // the budget-derived buckets grow with the ceiling. Without the coarse - // ladder a 60,000ms ceiling would mint ~240 distinct 250ms buckets and - // blow past Fastly's documented per-service dynamic backend limit (200). - let backend = FastlyPlatformBackend; - let configured = 60_000; - let mut distinct = std::collections::BTreeSet::new(); - for remaining in 0..=configured { - let value = backend.canonicalize_transport_timeout_ms(remaining, configured); - if value > 0 { - distinct.insert(value); - } - assert!( - value == 0 - || value % TRANSPORT_TIMEOUT_QUANTUM_MS == 0 - || SUB_QUANTUM_LADDER_MS.contains(&value), - "canonical value {value}ms (from remaining {remaining}ms) is neither a quantum \ - multiple nor a ladder rung" - ); - // The mediator hold bound relies on canonicalization never - // extending a transport cap past the wall-clock budget. - assert!( - value <= remaining, - "canonical value {value}ms must not extend past the remaining {remaining}ms budget" - ); - } - // A budget above the top coarse rung must clamp to it, not open a new - // bucket per 250ms step. - assert_eq!( - backend.canonicalize_transport_timeout_ms(120_000, 240_000), - 60_000, - "a budget above the top coarse rung clamps to it" - ); - assert!( - distinct.len() <= 20, - "large-ceiling budget-derived values must stay bounded, got {} distinct values: {distinct:?}", - distinct.len() - ); - } - - // --- FastlyPlatformBackend::predict_name discriminator ------------------ - - #[test] - fn predict_name_includes_provider_discriminator() { - let backend = FastlyPlatformBackend; - let base = PlatformBackendSpec { - scheme: "https".to_string(), - host: "gateway.example.com".to_string(), - port: None, - host_header_override: None, - certificate_check: true, - first_byte_timeout: Duration::from_millis(750), - between_bytes_timeout: Duration::from_millis(750), - discriminator: Some("prebid".to_string()), - }; - let prebid_name = backend - .predict_name(&base) - .expect("should predict name with discriminator"); - assert!( - prebid_name.contains("_p_prebid"), - "should fold the provider discriminator into the name, got {prebid_name}" - ); - - // Same origin + same transport timeout, different provider → distinct - // backend names, so auction response correlation cannot cross them. - let aps = PlatformBackendSpec { - discriminator: Some("aps".to_string()), - ..base.clone() - }; - let aps_name = backend - .predict_name(&aps) - .expect("should predict name for the second provider"); - assert_ne!( - prebid_name, aps_name, - "two providers on one origin must not share a backend name" - ); - } } diff --git a/crates/trusted-server-adapter-fastly/src/platform/deadline.rs b/crates/trusted-server-adapter-fastly/src/platform/deadline.rs deleted file mode 100644 index 6a8842d180..0000000000 --- a/crates/trusted-server-adapter-fastly/src/platform/deadline.rs +++ /dev/null @@ -1,947 +0,0 @@ -//! Deadline-bounded collection of Fastly auction responses. - -use error_stack::{Report, ResultExt as _}; -use fastly::handle::{BodyHandle, PendingRequestHandle, PollHandleResult}; -use std::io::Read as _; -use trusted_server_core::platform::{ - PlatformError, PlatformPendingRequest, PlatformResponse, PlatformSelectResult, -}; -use web_time::Instant; - -use super::{ - CacheBypassPending, MAX_PLATFORM_RESPONSE_BODY_BYTES, buffered_response_to_platform, - fastly_response_to_platform, response_carries_body, -}; - -// Bound payload retained across all partial bodies, as well as each response. -const MAX_TOTAL_BUFFERED_BODY_BYTES: usize = MAX_PLATFORM_RESPONSE_BODY_BYTES; - -// Keep partially buffered responses alongside requests still awaiting headers so -// an early header with a slow body cannot hold up a completed sibling response. -struct DeadlinePending { - backend_name: String, - request_method: Option, - state: Option, -} - -enum PendingState { - Headers(Option), - Ready(Result>), - Body { - response: Box, - body: BodyHandle, - bytes: Vec, - }, -} - -impl DeadlinePending { - fn from_platform(pending: PlatformPendingRequest) -> Result> { - if pending.stream_response() { - return Err(Report::new(PlatformError::HttpClient) - .attach("stream-marked pending request requires direct wait")); - } - let request_method = pending.request_method().cloned(); - let pending = match pending.downcast::() { - Ok(mut pending) => { - pending.request_method = request_method.or(pending.request_method); - return Ok(pending); - } - Err(pending) => pending, - }; - let backend_name = pending.backend_name().unwrap_or("").to_owned(); - // The marker is constructed only after set_pass(true), which selects - // HostCaching in fastly 0.13.1's Request::get_caching_mode. Its direct - // handle conversion therefore cannot encounter a guest caching state. - let pending = pending.downcast::().map_err(|_| { - Report::new(PlatformError::Unsupported).attach(format!( - "deadline collection requires a cache-bypass request for backend '{backend_name}'" - )) - })?; - let pending = PendingRequestHandle::from(pending.0); - Ok(Self { - backend_name, - request_method, - state: Some(PendingState::Headers(Some(pending))), - }) - } - - fn into_platform(self) -> PlatformPendingRequest { - let backend_name = self.backend_name.clone(); - let request_method = self.request_method.clone(); - let pending = PlatformPendingRequest::new(self).with_backend_name(backend_name); - match request_method { - Some(method) => pending.with_response_handling(false, method), - None => pending, - } - } - - fn request_is_head(&self, response: &fastly::Response) -> Result> { - if let Some(method) = &self.request_method { - return Ok(method == edgezero_core::http::Method::HEAD); - } - response - .get_backend_request() - .map(|request| request.get_method() == fastly::http::Method::HEAD) - .ok_or_else(|| { - Report::new(PlatformError::HttpClient).attach(format!( - "deadline response for backend '{}' has no originating request or method hint", - self.backend_name - )) - }) - } - - fn buffered_bytes(&self) -> usize { - match &self.state { - Some(PendingState::Body { bytes, .. }) => bytes.len(), - Some(PendingState::Ready(Ok(response))) => { - response.response.body().as_bytes().map_or(0, <[u8]>::len) - } - _ => 0, - } - } - - fn poll(&mut self) -> Result, Report> { - let state = self - .state - .take() - .expect("should retain pending collection state"); - let state = match state { - PendingState::Ready(result) => return result.map(Some), - PendingState::Headers(mut slot) => { - match slot.take().expect("should retain a pending request").poll() { - PollHandleResult::Pending(pending) => { - slot = Some(pending); - self.state = Some(PendingState::Headers(slot)); - return Ok(None); - } - PollHandleResult::Done(Err(error)) => { - return Err( - Report::new(PlatformError::HttpClient).attach(error.to_string()) - ); - } - PollHandleResult::Done(Ok((response_handle, body_handle))) => { - let mut response = - fastly::Response::from_handles(response_handle, body_handle); - let request_is_head = self.request_is_head(&response)?; - if !response_carries_body(request_is_head, response.get_status()) { - return fastly_response_to_platform( - response, - self.backend_name.clone(), - false, - request_is_head, - ) - .map(Some); - } - if response - .get_header("content-length") - .and_then(|value| value.to_str().ok()) - .and_then(|value| value.trim().parse::().ok()) - .is_some_and(|length| length > MAX_PLATFORM_RESPONSE_BODY_BYTES as u64) - { - return Err(Report::new(PlatformError::HttpClient) - .attach("origin Content-Length exceeds response body limit")); - } - let body = response.take_body().into_handle(); - PendingState::Body { - response: Box::new(response), - body, - bytes: Vec::new(), - } - } - } - } - state => state, - }; - let PendingState::Body { - response, - mut body, - mut bytes, - } = state - else { - unreachable!("headers are polled before buffering bodies"); - }; - if !body_is_ready(&body)? { - self.state = Some(PendingState::Body { - response, - body, - bytes, - }); - return Ok(None); - } - let mut chunk = [0_u8; 8192]; - let read = body - .read(&mut chunk) - .change_context(PlatformError::HttpClient)?; - if read == 0 { - return buffered_response_to_platform(&response, self.backend_name.clone(), bytes) - .map(Some); - } - if bytes.len() + read > MAX_PLATFORM_RESPONSE_BODY_BYTES { - return Err(Report::new(PlatformError::HttpClient) - .attach("origin response body exceeds response body limit")); - } - if bytes.len() + read > bytes.capacity() { - let capacity = bytes - .capacity() - .max(chunk.len()) - .saturating_mul(2) - .min(MAX_PLATFORM_RESPONSE_BODY_BYTES); - bytes.reserve_exact(capacity - bytes.len()); - } - bytes.extend_from_slice(&chunk[..read]); - self.state = Some(PendingState::Body { - response, - body, - bytes, - }); - Ok(None) - } -} - -fn body_is_ready(body: &BodyHandle) -> Result> { - let mut ready = 0; - // SAFETY: the live owned body retains the handle while this hostcall borrows it. - unsafe { fastly_sys::fastly_async_io::is_ready(body.as_u32(), &mut ready) } - .result() - .map_err(|status| { - Report::new(PlatformError::HttpClient) - .attach(format!("body readiness hostcall failed: {status:?}")) - })?; - Ok(ready != 0) -} - -/// Collect one fully buffered response without waiting past the absolute deadline. -/// -/// Dropping response bodies closes their host handles. Fastly has no pending -/// request cancellation ABI: unfinished header requests are abandoned and -/// never waited on past the auction deadline. If collection starts after the -/// deadline, one nonblocking snapshot preserves already available responses. -/// -/// Partial bodies share a 10 MiB payload budget. If a chunk exceeds that -/// capacity, the largest partial response fails with an attributed HTTP client -/// error; its buffered bytes are released and smaller siblings remain collectible. -/// Completed response payload plus retained partial bodies fits the same budget. -/// -/// # Errors -/// -/// Returns an error for an empty request list, invalid aggregate buffer state, -/// or a failure of the timed wait hostcall. Individual provider errors, including -/// unsupported pending handles, are attributed in the returned collection result. -pub(super) fn select_until( - pending_requests: Vec, - deadline: Instant, -) -> Result, Report> { - if pending_requests.is_empty() { - return Err(Report::new(PlatformError::HttpClient) - .attach("select_until called with an empty pending_requests list")); - } - let mut pending = pending_requests - .into_iter() - .map(|pending| { - let backend_name = pending.backend_name().unwrap_or("").to_owned(); - let request_method = pending.request_method().cloned(); - match DeadlinePending::from_platform(pending) { - Ok(pending) => pending, - Err(error) => DeadlinePending { - backend_name, - request_method, - state: Some(PendingState::Ready(Err(error))), - }, - } - }) - .collect::>(); - let mut buffered_bytes: usize = pending.iter().map(DeadlinePending::buffered_bytes).sum(); - if buffered_bytes > MAX_TOTAL_BUFFERED_BODY_BYTES { - return Err(Report::new(PlatformError::HttpClient) - .attach("resumed auction responses exceed aggregate buffer capacity")); - } - let expired_at_entry = Instant::now() >= deadline; - if expired_at_entry { - // Take one nonblocking snapshot. Fully buffered results are cached; - // responses that would require a wait are abandoned, never retried. - for index in 0..pending.len() { - loop { - let previous = pending[index].buffered_bytes(); - let result = pending[index].poll(); - let progressed = - matches!(result, Ok(None)) && pending[index].buffered_bytes() > previous; - match result { - Ok(None) if progressed => {} - Ok(None) => { - pending[index].state = None; - } - result => { - pending[index].state = Some(PendingState::Ready(match result { - Ok(Some(response)) => Ok(response), - Err(error) => Err(error), - Ok(None) => unreachable!("pending results handled above"), - })); - } - } - buffered_bytes = buffered_bytes - previous + pending[index].buffered_bytes(); - enforce_aggregate_budget(&mut pending, &mut buffered_bytes); - if !progressed || !matches!(pending[index].state, Some(PendingState::Body { .. })) { - break; - } - } - } - pending.retain(|item| item.state.is_some()); - } - loop { - for index in 0..pending.len() { - if !expired_at_entry && Instant::now() >= deadline { - // A timed host wait may wake with a ready event at the cutoff. - // Finish with the same nonblocking snapshot as expired entry. - return select_until( - pending - .into_iter() - .map(DeadlinePending::into_platform) - .collect(), - deadline, - ); - } - let previous_bytes = pending[index].buffered_bytes(); - let result = pending[index].poll(); - if let Ok(Some(response)) = result { - pending[index].state = Some(PendingState::Ready(Ok(response))); - } else if let Err(error) = result { - pending[index].state = Some(PendingState::Ready(Err(error))); - } - buffered_bytes = buffered_bytes - previous_bytes + pending[index].buffered_bytes(); - enforce_aggregate_budget(&mut pending, &mut buffered_bytes); - // Capacity eviction may have failed an earlier large sibling. - let ready_index = pending - .iter() - .position(|item| matches!(item.state, Some(PendingState::Ready(_)))); - let Some(ready_index) = ready_index else { - continue; - }; - let mut completed = pending.swap_remove(ready_index); - let Some(PendingState::Ready(ready)) = completed.state.take() else { - unreachable!("should select a ready result"); - }; - let failed_backend_name = ready.as_ref().err().map(|_| completed.backend_name); - return Ok(Some(PlatformSelectResult { - ready, - remaining: pending - .into_iter() - .map(DeadlinePending::into_platform) - .collect(), - failed_backend_name, - })); - } - if expired_at_entry || pending.is_empty() { - return Ok(None); - } - wait_for_progress(&pending, deadline)?; - } -} - -fn enforce_aggregate_budget(pending: &mut [DeadlinePending], buffered_bytes: &mut usize) { - if *buffered_bytes <= MAX_TOTAL_BUFFERED_BODY_BYTES { - return; - } - // The overflow is at most one read chunk. Evict the largest partial body - // first, preserving completed responses and healthy smaller siblings. - let largest = pending - .iter() - .enumerate() - .filter(|(_, item)| matches!(item.state, Some(PendingState::Body { .. }))) - .max_by_key(|(_, item)| item.buffered_bytes()) - .or_else(|| { - pending - .iter() - .enumerate() - .max_by_key(|(_, item)| item.buffered_bytes()) - }) - .map(|(index, _)| index) - .expect("should retain the response exceeding buffer capacity"); - *buffered_bytes -= pending[largest].buffered_bytes(); - pending[largest].state = Some(PendingState::Ready(Err(Report::new( - PlatformError::HttpClient, - ) - .attach("largest partial response exceeds aggregate auction buffer capacity")))); -} - -fn wait_for_progress( - pending: &[DeadlinePending], - deadline: Instant, -) -> Result<(), Report> { - let Some(remaining) = deadline.checked_duration_since(Instant::now()) else { - return Ok(()); - }; - if remaining.is_zero() { - return Ok(()); - } - let mut bodies = Vec::new(); - for pending in pending { - match &pending.state { - Some(PendingState::Headers(Some(handle))) => bodies.push(handle.as_u32()), - Some(PendingState::Body { body, .. }) => { - // SAFETY: these handles remain owned by pending throughout select. - bodies.push(unsafe { body.as_u32() }); - } - _ => {} - } - } - if bodies.is_empty() { - return Ok(()); - } - let timeout_ms = host_select_timeout_ms(remaining); - let mut ready_index = 0; - // SAFETY: bodies is a live slice, and ready_index is writable. A positive - // timeout is essential: the host interprets zero as an unbounded wait. - unsafe { - fastly_sys::fastly_async_io::select( - bodies.as_ptr(), - bodies.len(), - timeout_ms, - &mut ready_index, - ) - } - .result() - .map_err(|status| { - Report::new(PlatformError::HttpClient) - .attach(format!("timed select hostcall failed: {status:?}")) - })?; - Ok(()) -} - -/// Convert a remaining budget into the timed-select hostcall argument. -/// -/// The host treats a zero timeout as unbounded, so sub-millisecond budgets -/// round up to one millisecond instead. -fn host_select_timeout_ms(remaining: std::time::Duration) -> u32 { - u32::try_from(remaining.as_millis()) - .unwrap_or(u32::MAX) - .max(1) -} - -#[cfg(test)] -mod tests { - use std::time::Duration; - - use super::super::{FastlyPlatformHttpClient, MAX_PLATFORM_RESPONSE_BODY_BYTES}; - use super::{DeadlinePending, PendingState, host_select_timeout_ms}; - use edgezero_core::body::Body; - use trusted_server_core::platform::{ - PlatformHttpClient, PlatformHttpRequest, PlatformPendingRequest, - }; - use web_time::Instant; - - #[test] - fn sub_millisecond_budget_never_requests_an_unbounded_host_wait() { - assert_eq!( - host_select_timeout_ms(Duration::from_micros(500)), - 1, - "should round a sub-millisecond budget up instead of passing 0" - ); - assert_eq!( - host_select_timeout_ms(Duration::from_millis(25)), - 25, - "should pass whole-millisecond budgets through unchanged" - ); - assert_eq!( - host_select_timeout_ms(Duration::from_secs(u64::MAX)), - u32::MAX, - "should saturate budgets beyond the hostcall range" - ); - } - - #[test] - fn expired_deadline_attributes_an_invalid_pending_handle() { - let result = futures::executor::block_on(FastlyPlatformHttpClient.select_until( - vec![PlatformPendingRequest::new(42_u32).with_backend_name("example")], - Instant::now(), - )) - .expect("should isolate invalid handles even in the nonblocking snapshot") - .expect("should report the invalid pending handle"); - assert_eq!( - result.failed_backend_name.as_deref(), - Some("example"), - "should attribute the invalid pending handle" - ); - let error = result - .ready - .expect_err("should reject the invalid pending handle"); - assert!( - matches!( - error.current_context(), - trusted_server_core::platform::PlatformError::Unsupported - ), - "should reject unsupported deadline handles without panicking" - ); - } - - #[test] - fn deadline_collection_rejects_malformed_handles_with_backend_context() { - let result = futures::executor::block_on(FastlyPlatformHttpClient.select_until( - vec![PlatformPendingRequest::new(42_u32).with_backend_name("example")], - Instant::now() + Duration::from_secs(1), - )); - let result = result - .expect("should isolate malformed pending handles") - .expect("should return the malformed provider's failure"); - assert_eq!( - result.failed_backend_name.as_deref(), - Some("example"), - "should attribute the malformed handle" - ); - let error = result - .ready - .expect_err("should reject a malformed pending handle"); - assert!( - format!("{error:?}").contains("example"), - "should identify the malformed handle's backend" - ); - } - - fn buffered_pending(bytes: Vec, backend_name: &str) -> PlatformPendingRequest { - let mut response = fastly::Response::from_body(bytes); - let body = response.take_body().into_handle(); - DeadlinePending { - backend_name: backend_name.to_owned(), - request_method: Some(edgezero_core::http::Method::GET), - state: Some(PendingState::Body { - response: Box::new(response), - body, - bytes: Vec::new(), - }), - } - .into_platform() - } - - #[test] - fn unsupported_provider_does_not_discard_a_healthy_sibling() { - let deadline = Instant::now() + Duration::from_secs(1); - let failed = PlatformPendingRequest::new(42_u32).with_backend_name("unsupported"); - let healthy = buffered_pending(b"healthy".to_vec(), "healthy"); - let result = futures::executor::block_on( - FastlyPlatformHttpClient.select_until(vec![failed, healthy], deadline), - ) - .expect("should isolate a provider with an unsupported pending handle") - .expect("should report the failed provider"); - assert_eq!( - result.failed_backend_name.as_deref(), - Some("unsupported"), - "should attribute the conversion error to its provider" - ); - assert!( - result.ready.is_err(), - "should report the unsupported handle" - ); - let result = futures::executor::block_on( - FastlyPlatformHttpClient.select_until(result.remaining, deadline), - ) - .expect("should continue collecting the healthy sibling") - .expect("should complete the healthy sibling"); - let response = result.ready.expect("should retain the healthy response"); - assert_eq!( - response.backend_name.as_deref(), - Some("healthy"), - "should preserve the unaffected provider's bid" - ); - } - - #[test] - fn expired_deadline_preserves_an_already_ready_body() { - let pending = buffered_pending(b"ready".to_vec(), "example"); - let result = futures::executor::block_on( - FastlyPlatformHttpClient.select_until(vec![pending], Instant::now()), - ) - .expect("should snapshot an expired ready response") - .expect("should preserve an already complete response without waiting"); - let response = result - .ready - .expect("should retain a successful buffered response"); - assert_eq!( - response.backend_name.as_deref(), - Some("example"), - "should retain the ready response's backend" - ); - assert_eq!( - response.response.body().as_bytes(), - Some(b"ready".as_slice()), - "should retain the complete ready payload" - ); - } - - #[test] - fn remaining_handle_retains_the_original_request_method_hint() { - let pending = buffered_pending(b"body".to_vec(), "example") - .with_response_handling(false, edgezero_core::http::Method::HEAD); - let resumed = DeadlinePending::from_platform(pending) - .expect("should resume a deadline pending request"); - let remaining = resumed.into_platform(); - assert_eq!( - remaining.request_method(), - Some(&edgezero_core::http::Method::HEAD), - "should retain the HEAD method through remaining-handle reconstruction" - ); - let resumed = - DeadlinePending::from_platform(remaining).expect("should resume the remaining handle"); - let response = fastly::Response::from_body("advisory body"); - assert!( - response.get_backend_request().is_none(), - "should exercise a response without SDK originating-request metadata" - ); - assert!( - resumed - .request_is_head(&response) - .expect("should use the preserved method hint"), - "should retain HEAD body semantics after resuming without response origin metadata" - ); - } - - #[test] - fn missing_request_method_and_response_origin_returns_typed_error() { - let pending = buffered_pending(b"body".to_vec(), "example"); - let mut pending = DeadlinePending::from_platform(pending) - .expect("should inspect a buffered pending request"); - pending.request_method = None; - let response = fastly::Response::new(); - let error = pending - .request_is_head(&response) - .expect_err("should reject a response whose request method is unknown"); - assert!( - matches!( - error.current_context(), - trusted_server_core::platform::PlatformError::HttpClient - ), - "should return a typed HTTP client error" - ); - assert!( - format!("{error:?}").contains("example"), - "should identify the backend whose originating method is missing" - ); - } - - #[test] - fn completed_body_moves_into_platform_without_a_payload_copy() { - let bytes = vec![b'x'; 32768]; - let pointer = bytes.as_ptr(); - let response = super::buffered_response_to_platform( - &fastly::Response::new(), - "example".to_owned(), - bytes, - ) - .expect("should move a buffered response into the platform"); - assert_eq!( - response - .response - .body() - .as_bytes() - .expect("should retain a buffered body") - .as_ptr(), - pointer, - "should retain the original allocation without a Fastly host-body roundtrip" - ); - } - - #[test] - fn body_collection_is_fair_and_preserves_partial_siblings() { - let large = vec![b'x'; 32768]; - let pending = vec![ - buffered_pending(large.clone(), "large"), - buffered_pending(b"fast".to_vec(), "fast"), - ]; - let deadline = Instant::now() + Duration::from_secs(10); - let result = - futures::executor::block_on(FastlyPlatformHttpClient.select_until(pending, deadline)) - .expect("should collect one response") - .expect("should complete before deadline"); - let response = result - .ready - .expect("should collect the short response first"); - assert_eq!( - response.backend_name.as_deref(), - Some("fast"), - "should give siblings a fair opportunity to complete" - ); - assert_eq!( - result.remaining.len(), - 1, - "should retain the partial response" - ); - let result = futures::executor::block_on( - FastlyPlatformHttpClient.select_until(result.remaining, deadline), - ) - .expect("should resume body collection") - .expect("should finish the remaining body"); - let response = result.ready.expect("should finish the partial body"); - let (_, body) = response.response.into_parts(); - let bytes = body - .into_bytes() - .expect("should read the complete buffered body"); - assert_eq!( - bytes.as_ref(), - large.as_slice(), - "should preserve all bytes across collector calls" - ); - } - - #[test] - fn body_at_the_exact_limit_remains_eligible() { - let pending = buffered_pending(vec![b'x'; MAX_PLATFORM_RESPONSE_BODY_BYTES], "exact"); - let result = futures::executor::block_on( - FastlyPlatformHttpClient - .select_until(vec![pending], Instant::now() + Duration::from_secs(10)), - ) - .expect("should collect an exactly sized body") - .expect("should complete before the deadline"); - let response = result - .ready - .expect("should accept the configured body-size boundary"); - assert_eq!( - response - .response - .body() - .as_bytes() - .expect("should retain buffered bytes") - .len(), - MAX_PLATFORM_RESPONSE_BODY_BYTES, - "should accept the exact limit without evicting its EOF probe" - ); - } - - #[test] - fn incremental_body_limit_reports_the_failed_backend() { - let pending = buffered_pending(vec![b'x'; MAX_PLATFORM_RESPONSE_BODY_BYTES + 1], "large"); - let result = futures::executor::block_on( - FastlyPlatformHttpClient - .select_until(vec![pending], Instant::now() + Duration::from_secs(30)), - ) - .expect("should return a provider failure") - .expect("should reject the body before deadline"); - assert!( - result.ready.is_err(), - "should enforce the incremental body cap" - ); - assert_eq!( - result.failed_backend_name.as_deref(), - Some("large"), - "should attribute body conversion failures" - ); - } - - #[test] - fn aggregate_body_pressure_evicts_largest_partial_provider() { - let mut large = - DeadlinePending::from_platform(buffered_pending(vec![b'x'; 16384], "large")) - .expect("should construct a large partial response"); - if let Some(PendingState::Body { bytes, .. }) = &mut large.state { - *bytes = vec![b'x'; MAX_PLATFORM_RESPONSE_BODY_BYTES - 1]; - } - let pending = vec![ - buffered_pending(b"healthy".to_vec(), "healthy"), - large.into_platform(), - ]; - let deadline = Instant::now() + Duration::from_secs(10); - let result = - futures::executor::block_on(FastlyPlatformHttpClient.select_until(pending, deadline)) - .expect("should return a capacity result") - .expect("should reject excess growth"); - assert!( - result.ready.is_err(), - "should reject excess aggregate buffering" - ); - assert_eq!( - result.failed_backend_name.as_deref(), - Some("large"), - "should attribute capacity pressure to the largest partial response" - ); - let result = futures::executor::block_on( - FastlyPlatformHttpClient.select_until(result.remaining, deadline), - ) - .expect("should retain the healthy provider") - .expect("should complete the healthy provider"); - assert_eq!( - result - .ready - .expect("should preserve the healthy bid") - .backend_name - .as_deref(), - Some("healthy"), - "should avoid blaming the healthy provider for another provider's large body" - ); - } - - #[test] - fn expired_snapshot_retains_only_ready_results_across_calls() { - let pending = vec![ - buffered_pending(b"one".to_vec(), "one"), - buffered_pending(b"two".to_vec(), "two"), - ]; - let deadline = Instant::now(); - let result = - futures::executor::block_on(FastlyPlatformHttpClient.select_until(pending, deadline)) - .expect("should snapshot ready results") - .expect("should return first ready result"); - assert!( - result.ready.is_ok(), - "should preserve a complete ready response" - ); - let result = futures::executor::block_on( - FastlyPlatformHttpClient.select_until(result.remaining, deadline), - ) - .expect("should use cached ready results") - .expect("should return second ready result"); - assert!( - result.ready.is_ok(), - "should preserve the second complete response" - ); - } - - fn network_pending(path: &str, method: &str) -> PlatformPendingRequest { - let request = edgezero_core::http::request_builder() - .method(method) - .uri(format!("http://example.com/{path}")) - .body(Body::empty()) - .expect("should build fixture request"); - futures::executor::block_on( - FastlyPlatformHttpClient.send_async( - PlatformHttpRequest::new(request, "deadline_origin").with_cache_bypass(), - ), - ) - .expect("should launch fixture request") - } - - #[test] - #[ignore = "requires the local origin started by tests/deadline-origin.mjs"] - fn network_slow_origin_preserves_ready_bid_without_waiting_for_stalled_sibling() { - let fast = network_pending("fast", "GET"); - let stalled = network_pending("stalled", "GET"); - let deadline = Instant::now() + Duration::from_millis(40); - std::thread::sleep(Duration::from_millis(100)); - let started = Instant::now(); - let result = futures::executor::block_on( - FastlyPlatformHttpClient.select_until(vec![stalled, fast], deadline), - ) - .expect("should snapshot already ready bids") - .expect("should retain the complete fast response"); - assert!( - result.ready.is_ok(), - "should preserve a bid available after a slow publisher origin" - ); - assert!( - result.remaining.is_empty(), - "should abandon the stalled sibling in the nonblocking snapshot" - ); - assert!( - started.elapsed() < Duration::from_millis(100), - "should never wait for the stalled sibling after expiration" - ); - } - - #[test] - #[ignore = "requires the local origin started by tests/deadline-origin.mjs"] - fn network_stalled_headers_stop_at_deadline() { - let pending = network_pending("stalled", "GET"); - let started = Instant::now(); - let result = futures::executor::block_on( - FastlyPlatformHttpClient - .select_until(vec![pending], started + Duration::from_millis(80)), - ) - .expect("should stop waiting for stalled headers"); - assert!( - result.is_none(), - "should expire while headers remain pending" - ); - assert!( - started.elapsed() < Duration::from_millis(500), - "should bound the synchronous header wait" - ); - } - - #[test] - #[ignore = "requires the local origin started by tests/deadline-origin.mjs"] - fn network_trickling_body_stops_at_deadline() { - let pending = network_pending("trickle", "GET"); - let started = Instant::now(); - let result = futures::executor::block_on( - FastlyPlatformHttpClient - .select_until(vec![pending], started + Duration::from_millis(80)), - ) - .expect("should stop buffering a trickling body"); - assert!( - result.is_none(), - "should expire before the trickling body completes" - ); - assert!( - started.elapsed() < Duration::from_millis(500), - "should bound body reads at the auction deadline" - ); - } - - #[test] - #[ignore = "requires the local origin started by tests/deadline-origin.mjs"] - fn network_fast_sibling_completes_while_another_body_trickles() { - let slow = network_pending("trickle", "GET"); - let fast = network_pending("fast", "GET"); - let request = edgezero_core::http::request_builder() - .uri("http://example.com/fast") - .body(Body::empty()) - .expect("should build the request missing cache bypass"); - let unsupported = futures::executor::block_on( - FastlyPlatformHttpClient - .send_async(PlatformHttpRequest::new(request, "deadline_origin")), - ) - .expect("should dispatch the request missing cache bypass"); - let started = Instant::now(); - let deadline = started + Duration::from_millis(200); - let result = futures::executor::block_on( - FastlyPlatformHttpClient.select_until(vec![unsupported, slow, fast], deadline), - ) - .expect("should isolate a request missing cache bypass") - .expect("should return the unsupported provider's failure"); - assert_eq!( - result.failed_backend_name.as_deref(), - Some("deadline_origin"), - "should attribute missing cache bypass to that request" - ); - let error = result - .ready - .expect_err("should reject the unsupported handle"); - assert!( - matches!( - error.current_context(), - trusted_server_core::platform::PlatformError::Unsupported - ), - "should reject SDK guest-caching handles before direct handle conversion" - ); - let result = futures::executor::block_on( - FastlyPlatformHttpClient.select_until(result.remaining, deadline), - ) - .expect("should collect the completed sibling") - .expect("should finish before deadline"); - let response = result.ready.expect("should collect the fast body"); - let (_, body) = response.response.into_parts(); - let bytes = body.into_bytes().expect("should read the fast body"); - assert_eq!( - bytes.as_ref(), - b"fast", - "should avoid starvation by a trickling sibling" - ); - assert_eq!( - result.remaining.len(), - 1, - "should retain the incomplete sibling" - ); - } - - #[test] - #[ignore = "requires the local origin started by tests/deadline-origin.mjs"] - fn network_head_skips_the_body_and_its_declared_length_limit() { - let pending = network_pending("head", "HEAD"); - let result = futures::executor::block_on( - FastlyPlatformHttpClient - .select_until(vec![pending], Instant::now() + Duration::from_secs(1)), - ) - .expect("should collect HEAD response") - .expect("should finish before deadline"); - assert!( - result.ready.is_ok(), - "should accept HEAD with an advisory large content-length" - ); - } -} diff --git a/crates/trusted-server-adapter-fastly/src/post_delivery.rs b/crates/trusted-server-adapter-fastly/src/post_delivery.rs new file mode 100644 index 0000000000..63d5324bda --- /dev/null +++ b/crates/trusted-server-adapter-fastly/src/post_delivery.rs @@ -0,0 +1,460 @@ +//! Work the Fastly entry point runs after a response is delivered. +//! +//! [`run_post_delivery`] sends the request's queued auction telemetry, then +//! runs EC pull sync. Telemetry goes first because its requests are short, +//! while a pull-sync group can take its whole cutoff. Both run only after a +//! committed application body (see [`crate::delivery`]). +//! +//! Both use a fresh `EdgeZero` client and clock, because the request's client +//! went away with the request. They run under +//! [`futures::executor::block_on`] inside the serving callback, after +//! `send_request_with_registries_and_hooks` returned. That is the only safe +//! place: dispatch itself runs under `block_on`, and nesting there panics. +//! `EdgeZero` releases callback-owned batch drivers when the callback returns, +//! so both finish before it does. The client already has its response; each +//! extends only the sandbox's lifetime. + +use std::fmt; + +use edgezero_adapter_fastly::outbound::FastlyOutboundClient; +use edgezero_core::outbound::HttpClient; +use edgezero_core::time::MonotonicClock; +use error_stack::Report; +use trusted_server_core::ec::kv::KvIdentityGraph; +use trusted_server_core::ec::pull_sync::{ + PullSyncContext, build_pull_sync_context, dispatch_pull_sync, +}; +use trusted_server_core::ec::registry::PartnerRegistry; +use trusted_server_core::error::TrustedServerError; +use trusted_server_core::settings::Settings; + +use crate::app::EcFinalizeState; +use crate::rate_limiter::{FastlyRateLimiter, RATE_COUNTER_NAME}; +use crate::require_identity_graph; +use crate::tinybird::{TelemetryQueue, send_queued_batches}; + +/// Pull-sync inputs a routed response's finalize step hands to post-delivery work. +pub(crate) struct FinalizeState { + pub(crate) partner_registry: PartnerRegistry, + pub(crate) ec_state: EcFinalizeState, +} + +/// Sends the queued telemetry, then runs pull sync for `state`, on a fresh client and clock. +/// +/// `log_context` ends each failure log, so it matches the delivered response. +pub(crate) fn run_post_delivery( + state: Option, + queue: &TelemetryQueue, + settings: &Settings, + log_context: &dyn fmt::Display, +) { + let client = HttpClient::with_client(FastlyOutboundClient::new()).with_proxy_marker(false); + let clock = MonotonicClock::default(); + futures::executor::block_on(post_delivery_work( + state, + queue, + settings, + &client, + &clock, + log_context, + )); +} + +/// The post-delivery work itself, on `client` and `clock`: telemetry, then pull sync. +async fn post_delivery_work( + state: Option, + queue: &TelemetryQueue, + settings: &Settings, + client: &HttpClient, + clock: &MonotonicClock, + log_context: &dyn fmt::Display, +) { + send_queued_batches(client, &settings.tinybird, queue, log_context).await; + if let Some(state) = state { + pull_sync(state, settings, client, clock, log_context).await; + } +} + +/// Runs pull sync for a real browser whose EC state still has partners to fill. +async fn pull_sync( + state: FinalizeState, + settings: &Settings, + client: &HttpClient, + clock: &MonotonicClock, + log_context: &dyn fmt::Display, +) { + let FinalizeState { + partner_registry, + ec_state, + } = state; + if !ec_state.is_real_browser { + return; + } + + let prepared_context = build_pull_sync_context(&ec_state.ec_context, &partner_registry); + let Some((context, kv)) = prepare_pull_sync_after_send( + prepared_context, + || require_identity_graph(settings), + log_context, + ) else { + return; + }; + + let limiter = FastlyRateLimiter::new(RATE_COUNTER_NAME); + dispatch_pull_sync( + settings, + &kv, + &partner_registry, + &limiter, + &context, + client, + clock, + ) + .await; +} + +/// Opens the identity graph only when there is a pull-sync context to run. +fn prepare_pull_sync_after_send( + context: Option, + graph_factory: F, + log_context: &dyn fmt::Display, +) -> Option<(PullSyncContext, KvIdentityGraph)> +where + F: FnOnce() -> Result>, +{ + let context = context?; + let kv = match graph_factory() { + Ok(kv) => kv, + Err(err) => { + log::debug!("Pull sync: identity graph unavailable, skipping: {err:?}{log_context}"); + return None; + } + }; + Some((context, kv)) +} + +#[cfg(test)] +mod tests { + use edgezero_adapter_fastly::delivery::DeliveryError; + use edgezero_core::http::{Method, StatusCode}; + use trusted_server_core::auction::telemetry::{AuctionEventBatch, AuctionTelemetrySink as _}; + use trusted_server_core::ec::EcContext; + use trusted_server_core::geo::GeoInfo; + use trusted_server_core::redacted::Redacted; + use trusted_server_core::settings::EcPartner; + use trusted_server_core::test_support::scripted_client::{ + ScriptedHttpClient, ScriptedReply, TestClock, + }; + + use super::*; + use crate::delivery::tests::{ + committed, committed_fallback, committed_then_failed, counters, not_committed, + routed_delivery, unknown_commit, + }; + use crate::delivery::{DeliveryAction, delivery_action, handle_delivery}; + use crate::tinybird::QueueingTelemetrySink; + use crate::tinybird::tests::{ + EVENTS_URI, assert_events_request, enabled_config, test_row, test_services, + }; + + fn settings_with_tinybird() -> Settings { + Settings { + tinybird: enabled_config(), + ..Settings::default() + } + } + + /// A queue holding the NDJSON the queueing sink wrote for one auction, and that body. + fn queue_with_one_auction() -> (TelemetryQueue, String) { + let queue = TelemetryQueue::default(); + let sink = QueueingTelemetrySink::new(&enabled_config(), queue.clone()); + let batch = AuctionEventBatch::new(vec![test_row()]); + let body = batch + .to_ndjson(enabled_config().max_body_bytes) + .expect("should serialize the auction's telemetry"); + futures::executor::block_on(sink.emit_auction_events(&test_services(), batch)) + .expect("should queue the auction's telemetry"); + (queue, body) + } + + #[test] + fn telemetry_rows_queue_during_request_and_send_after_delivery() { + let clock = TestClock::new(); + let client = ScriptedHttpClient::new(&clock); + client.respond_to( + Method::POST, + EVENTS_URI, + ScriptedReply::status(StatusCode::ACCEPTED), + ); + let settings = settings_with_tinybird(); + let (queue, body) = queue_with_one_auction(); + + assert!( + client.requests().is_empty(), + "should send nothing while the request is still being served" + ); + handle_delivery(routed_delivery(committed), &counters(), |_state| { + clock.drive(post_delivery_work( + None, + &queue, + &settings, + &client.http_client(), + &clock.monotonic_clock(), + &"", + )); + }); + + let requests = client.requests(); + let [request] = requests.as_slice() else { + panic!("should send the queued auction once after delivery, got {requests:?}"); + }; + assert_events_request(request, &body); + } + + #[test] + fn no_post_delivery_when_not_committed_or_fallback_body() { + let client = ScriptedHttpClient::forbidding(); + let clock = TestClock::new(); + let settings = settings_with_tinybird(); + let (queue, _body) = queue_with_one_auction(); + let run = |_state| { + clock.drive(post_delivery_work( + None, + &queue, + &settings, + &client.http_client(), + &clock.monotonic_clock(), + &"", + )); + }; + + handle_delivery(routed_delivery(committed_fallback), &counters(), run); + handle_delivery(routed_delivery(unknown_commit), &counters(), run); + assert_eq!( + delivery_action(routed_delivery(not_committed), &counters()), + DeliveryAction::SendNativeError, + "an uncommitted delivery gets a native 500 and no post-delivery work" + ); + + client.assert_no_requests(); + assert_eq!(queue.take().len(), 1, "should leave the telemetry unsent"); + } + + #[test] + fn post_delivery_without_finalize_state_still_sends_telemetry() { + let clock = TestClock::new(); + let client = ScriptedHttpClient::new(&clock); + client.respond_to( + Method::POST, + EVENTS_URI, + ScriptedReply::status(StatusCode::ACCEPTED), + ); + let (queue, body) = queue_with_one_auction(); + + clock.drive(post_delivery_work( + None, + &queue, + &settings_with_tinybird(), + &client.http_client(), + &clock.monotonic_clock(), + &"", + )); + + let requests = client.requests(); + let [request] = requests.as_slice() else { + panic!("should send the queued auction once, got {requests:?}"); + }; + assert_events_request(request, &body); + assert!(queue.take().is_empty(), "should drain the queue"); + } + + #[test] + fn pull_sync_noop_states_skip_post_send_graph_factory() { + let calls = std::cell::Cell::new(0); + let result = prepare_pull_sync_after_send( + None, + || { + calls.set(calls.get() + 1); + Err(Report::new(TrustedServerError::KvStore { + store_name: "unexpected".to_owned(), + message: "graph factory should not run".to_owned(), + })) + }, + &"", + ); + assert!( + result.is_none(), + "a skipped pull-sync plan should return none" + ); + assert_eq!(calls.get(), 0, "should not invoke the graph factory"); + } + + #[test] + fn post_delivery_with_nothing_queued_makes_no_request() { + let client = ScriptedHttpClient::forbidding(); + let clock = TestClock::new(); + + clock.drive(post_delivery_work( + None, + &TelemetryQueue::default(), + &settings_with_tinybird(), + &client.http_client(), + &clock.monotonic_clock(), + &"", + )); + + client.assert_no_requests(); + } + + // ----------------------------------------------------------------------- + // Pull sync after delivery, against the local identity store + // ----------------------------------------------------------------------- + + /// EC ID `fastly.toml` seeds, consented, in the local `ec_identity_store`. + fn seeded_ec_id() -> String { + format!("{}.test01", "a".repeat(64)) + } + + /// Pull-sync URL the example partner receives for [`seeded_ec_id`]. + fn partner_pull_url() -> String { + format!("https://sync.example.com/pull?ec_id={}", seeded_ec_id()) + } + + /// Settings with Tinybird, the local identity store, and one pull-sync partner. + fn pull_sync_settings() -> Settings { + let mut settings = settings_with_tinybird(); + settings.ec.ec_store = Some("ec_identity_store".to_owned()); + settings.ec.partners = vec![EcPartner { + name: "Example partner".to_owned(), + source_domain: "partner.example.com".to_owned(), + openrtb_atype: EcPartner::default_openrtb_atype(), + bidstream_enabled: true, + api_token: Some(Redacted::new( + "example-partner-api-token-32-bytes-ok".to_owned(), + )), + batch_rate_limit: EcPartner::default_batch_rate_limit(), + pull_sync_enabled: true, + pull_sync_url: Some("https://sync.example.com/pull".to_owned()), + pull_sync_allowed_domains: vec!["sync.example.com".to_owned()], + pull_sync_ttl_sec: 3600, + pull_sync_rate_limit: 100, + ts_pull_token: Some(Redacted::new("example-pull-token".to_owned())), + }]; + settings + } + + /// Finalize state for a returning browser whose EC row the local store holds. + fn returning_reader_state(settings: &Settings) -> FinalizeState { + let request = edgezero_core::http::request_builder() + .uri("https://test-publisher.example.com/article") + .header("cookie", format!("ts-ec={}", seeded_ec_id())) + .body(edgezero_core::body::Body::empty()) + .expect("should build the reader request"); + let geo = GeoInfo { + city: "Tokyo".to_owned(), + country: "JP".to_owned(), + continent: "Asia".to_owned(), + latitude: 35.6762, + longitude: 139.6503, + metro_code: 0, + region: None, + asn: None, + }; + let mut ec_context = + EcContext::read_from_request_with_geo(settings, &request, &test_services(), Some(&geo)) + .expect("should read the reader's EC context"); + let kv = require_identity_graph(settings).expect("should open the local identity store"); + ec_context.set_kv_snapshot(kv.load_snapshot(&seeded_ec_id())); + FinalizeState { + partner_registry: PartnerRegistry::from_config(&settings.ec.partners) + .expect("should build the partner registry"), + ec_state: EcFinalizeState { + ec_context, + use_finalize_kv: false, + eids_cookie: None, + sharedid_cookie: None, + is_real_browser: true, + }, + } + } + + #[test] + fn pull_sync_reaches_partner_after_origin_aborts_mid_body() { + let clock = TestClock::new(); + let client = ScriptedHttpClient::new(&clock); + client + .respond_to( + Method::POST, + EVENTS_URI, + ScriptedReply::status(StatusCode::ACCEPTED), + ) + .respond_to( + Method::GET, + &partner_pull_url(), + ScriptedReply::status(StatusCode::OK).body(r#"{"uid":"example-partner-uid"}"#), + ); + let settings = pull_sync_settings(); + let (queue, _body) = queue_with_one_auction(); + let state = returning_reader_state(&settings); + + // The origin aborted after the page head was committed. + handle_delivery( + routed_delivery(|envelope| committed_then_failed(envelope, DeliveryError::Source)), + &counters(), + |_routed| { + clock.drive(post_delivery_work( + Some(state), + &queue, + &settings, + &client.http_client(), + &clock.monotonic_clock(), + &"", + )); + }, + ); + + let requested: Vec<(Method, String)> = client + .requests() + .iter() + .map(|request| (request.method.clone(), request.uri.to_string())) + .collect(); + assert_eq!( + requested, + vec![ + (Method::POST, EVENTS_URI.to_owned()), + (Method::GET, partner_pull_url()), + ], + "should send the telemetry first, then reach the pull-sync partner" + ); + } + + #[test] + fn framing_fallback_delivery_makes_no_pull_sync_call() { + let client = ScriptedHttpClient::forbidding(); + let clock = TestClock::new(); + let settings = pull_sync_settings(); + let state = returning_reader_state(&settings); + let mut state = Some(state); + + handle_delivery( + routed_delivery(committed_fallback), + &counters(), + |_routed| { + clock.drive(post_delivery_work( + state.take(), + &TelemetryQueue::default(), + &settings, + &client.http_client(), + &clock.monotonic_clock(), + &"", + )); + }, + ); + + client.assert_no_requests(); + assert!( + state.is_some(), + "should never run post-delivery work for a delivered framing fallback" + ); + } +} diff --git a/crates/trusted-server-adapter-fastly/src/sandbox.rs b/crates/trusted-server-adapter-fastly/src/sandbox.rs index db825427a1..ca9ea94327 100644 --- a/crates/trusted-server-adapter-fastly/src/sandbox.rs +++ b/crates/trusted-server-adapter-fastly/src/sandbox.rs @@ -62,11 +62,12 @@ pub(crate) const INSTANCE_ID_UNAVAILABLE: &str = "unavailable"; /// names the application-specific payload it retains. pub(crate) type Sandbox = edgezero_adapter_fastly::lifecycle::Sandbox; -/// The application state a retained sandbox carries. +/// The application a retained sandbox carries: the assembled app and its state. /// -/// Only ever reachable after a successful build: a failed build returns the -/// error router as the `initialize` error instead, so it is served for the -/// current request and dropped rather than retained. +/// Only ever reachable after a successful assembly. A failed assembly returns +/// its error from `initialize` instead, which the entry point answers with +/// `EdgeZero`'s assembly-error response for the current request only, so the +/// next request retries. pub(crate) struct RetainedApp { pub(crate) app: App, pub(crate) state: Arc, @@ -146,10 +147,12 @@ impl RetirementCounters { } } -/// Counter values captured for one response. +/// Counter values captured for one request. /// /// An owned snapshot rather than a borrow of [`Sandbox`], so attaching counters -/// never competes with the mutable borrow the request path holds. +/// never competes with the mutable borrow the request path holds. Every +/// request captures them, because every delivery failure log carries them. +/// They reach a response only when [`metrics_enabled`] says so. #[derive(Debug, Clone, PartialEq, Eq)] pub(crate) struct SandboxCounters { pub(crate) ordinal: u64, @@ -158,26 +161,64 @@ pub(crate) struct SandboxCounters { } impl SandboxCounters { - /// Captures the current counters, or `None` when metrics are disabled. + /// Captures the current counters. + pub(crate) fn capture(sandbox: &Sandbox, ordinal: u64, request_id: &str) -> Self { + Self { + ordinal, + builds: sandbox.initialization_attempts(), + request_id: request_id.to_owned(), + } + } + + /// The ` [instance=.. ordinal=.. request=..]` suffix of a delivery failure log. /// - /// Returning `None` is what keeps the counters off every response in the - /// default configuration, including the correlation id. - pub(crate) fn capture( - sandbox: &Sandbox, - ordinal: u64, - request_id: &str, - settings: Option<&Settings>, - ) -> Option { - settings - .filter(|settings| metrics_enabled(settings)) - .map(|_| Self { - ordinal, - builds: sandbox.initialization_attempts(), - request_id: request_id.to_owned(), - }) + /// It matches the counters a committed response carries, so a failure + /// after commitment can be reconciled with that response. + pub(crate) fn log_context(&self) -> LogContext<'_> { + LogContext::new(self.ordinal, &self.request_id) + } +} + +/// The ` [instance=.. ordinal=.. request=..]` suffix of a failure log. +/// +/// It is formatted only when a log writes it, so a request that logs no +/// failure pays nothing for it. The entry point builds one before the +/// counters are captured, so the assembly failure log matches +/// [`SandboxCounters::log_context`]. +#[derive(Debug, Clone, Copy, derive_more::Display)] +#[display(" [instance={} ordinal={ordinal} request={request_id}]", instance_id())] +pub(crate) struct LogContext<'a> { + ordinal: u64, + request_id: &'a str, +} + +impl<'a> LogContext<'a> { + /// The suffix for request `ordinal`, identified by `request_id`. + pub(crate) fn new(ordinal: u64, request_id: &'a str) -> Self { + Self { + ordinal, + request_id, + } } } +/// Guest-instance identifier used to attribute requests to a sandbox. +/// +/// `FASTLY_TRACE_ID` describes the sandbox, which is exactly what is wanted +/// here and exactly why it must not be used as a request id. An absent value +/// is reported rather than synthesized, so a measurement run cannot silently +/// claim reuse it never observed. +pub(crate) fn instance_id() -> String { + // The SDK documents this as the per-sandbox identifier; on wasm32-wasip1 + // it resolves to `FASTLY_TRACE_ID`, which is why that value must never be + // used as a request id. + let id = fastly::compute_runtime::sandbox_id(); + if id.is_empty() { + return INSTANCE_ID_UNAVAILABLE.to_owned(); + } + id.to_owned() +} + /// Whether the counters endpoint and response counters are enabled. pub(crate) fn metrics_enabled(settings: &Settings) -> bool { settings.debug.sandbox_metrics_enabled @@ -773,14 +814,14 @@ mod tests { let builds = Cell::new(0_u32); // Request 1: the build fails. The error payload stands in for the - // error router `build_app_with_state` returns when state is `None`. + // assembly error the entry point answers the request with. let failed = sandbox.initialize(|| { builds.set(builds.get() + 1); - Err::("error router") + Err::("assembly error") }); assert_eq!( failed.err(), - Some("error router"), + Some("assembly error"), "a failed build must be handed back for this request only" ); assert!( @@ -843,7 +884,7 @@ mod tests { counters.observe(&mut sandbox, |sandbox| { sandbox .initialize(|| Ok::<_, &str>(test_retained())) - .expect("the build should succeed"); + .expect("should build the app"); }); assert_eq!( counters.attempts(), @@ -861,10 +902,10 @@ mod tests { // in `initialization_attempts()` until the sandbox is dropped. What a // stale snapshot loses is the report, not the count. counters.observe(&mut sandbox, |sandbox| { - let failed = sandbox.initialize(|| Err::("error router")); + let failed = sandbox.initialize(|| Err::("assembly error")); assert_eq!( failed.err(), - Some("error router"), + Some("assembly error"), "the failed build should be handed back" ); }); @@ -907,32 +948,42 @@ mod tests { } #[test] - fn counters_are_captured_only_when_metrics_are_enabled() { - let sandbox = Sandbox::default(); - let mut settings = Settings::default(); + fn counters_capture_the_ordinal_builds_and_request_id() { + let mut sandbox = Sandbox::default(); + sandbox + .initialize(|| Ok::<_, &str>(test_retained())) + .expect("should build the app"); + + let counters = SandboxCounters::capture(&sandbox, 7, "req-7"); assert_eq!( - SandboxCounters::capture(&sandbox, 1, "req-1", None), - None, - "no settings should mean no counters" + counters, + SandboxCounters { + ordinal: 7, + builds: 1, + request_id: "req-7".to_owned(), + }, + "should capture the current counters whatever the metrics setting" ); - - settings.debug.sandbox_metrics_enabled = false; assert_eq!( - SandboxCounters::capture(&sandbox, 1, "req-1", Some(&settings)), - None, - "counters must stay off in the default configuration" + counters.log_context().to_string(), + format!(" [instance={} ordinal=7 request=req-7]", instance_id()), + "should name the instance, ordinal, and request in failure logs" + ); + } + + #[test] + fn metrics_follow_the_debug_setting() { + let mut settings = Settings::default(); + assert!( + !metrics_enabled(&settings), + "counters must stay off responses in the default configuration" ); settings.debug.sandbox_metrics_enabled = true; - assert_eq!( - SandboxCounters::capture(&sandbox, 7, "req-7", Some(&settings)), - Some(SandboxCounters { - ordinal: 7, - builds: 0, - request_id: "req-7".to_owned(), - }), - "enabling the flag should capture the current counters" + assert!( + metrics_enabled(&settings), + "enabling the flag should put counters on responses" ); } diff --git a/crates/trusted-server-adapter-fastly/src/tinybird.rs b/crates/trusted-server-adapter-fastly/src/tinybird.rs index 504dbd9fba..6565531b8a 100644 --- a/crates/trusted-server-adapter-fastly/src/tinybird.rs +++ b/crates/trusted-server-adapter-fastly/src/tinybird.rs @@ -1,76 +1,70 @@ -//! Tinybird direct-ingest telemetry sink for the Fastly adapter. - -use std::sync::Arc; +//! Tinybird direct-ingest telemetry for the Fastly adapter. +//! +//! During a request, [`QueueingTelemetrySink`] serializes each auction +//! telemetry batch to NDJSON and queues the body on the request's +//! [`TelemetryQueue`]. After the response is delivered, +//! [`send_queued_batches`] sends each queued body as one Events API request. +//! `EdgeZero` cancels an outbound request whose pending handle is dropped, so a +//! request cannot be left to finish on its own while the response streams. + +use std::fmt; +use std::sync::{Arc, Mutex, PoisonError}; use std::time::Duration; -use edgezero_core::body::Body; -use edgezero_core::http::{HeaderValue, Method, header, request_builder}; -use error_stack::{Report, ResultExt as _}; -use trusted_server_core::auction::telemetry::{ - AuctionEventBatch, AuctionTelemetrySink, NoopAuctionTelemetrySink, -}; +use edgezero_core::error::EdgeError; +use edgezero_core::http::{Method, header}; +use edgezero_core::outbound::{HttpClient, OutboundCachePolicy, ResponseDecoding}; +use error_stack::Report; +use trusted_server_core::auction::telemetry::{AuctionEventBatch, AuctionTelemetrySink}; use trusted_server_core::error::TrustedServerError; -use trusted_server_core::platform::{PlatformBackendSpec, PlatformHttpRequest, RuntimeServices}; -use trusted_server_core::redacted::Redacted; -use trusted_server_core::settings::{Settings, TinybirdSettings}; +use trusted_server_core::outbound::{ + DEFAULT_BUFFERED_RESPONSE_BYTES, OutboundRequestSource, buffered, outbound_error, + outbound_request, +}; +use trusted_server_core::platform::RuntimeServices; +use trusted_server_core::settings::TinybirdSettings; const TINYBIRD_EVENTS_PATH: &str = "/v0/events"; const TINYBIRD_NDJSON_CONTENT_TYPE: &str = "application/x-ndjson"; -const TINYBIRD_FIRST_BYTE_TIMEOUT: Duration = Duration::from_secs(2); -const TINYBIRD_BETWEEN_BYTES_TIMEOUT: Duration = Duration::from_secs(2); +/// Total timeout for one Events API request. +const TINYBIRD_TIMEOUT: Duration = Duration::from_secs(2); const TINYBIRD_MAX_ROWS_PER_AUCTION_BATCH: usize = 512; -/// Build the configured auction telemetry sink. -#[must_use] -pub(crate) fn auction_sink_from_settings(settings: &Settings) -> Arc { - if settings.tinybird.enabled { - Arc::new(FastlyTinybirdAuctionTelemetrySink::new( - settings.tinybird.clone(), - )) - } else { - Arc::new(NoopAuctionTelemetrySink) +/// NDJSON bodies one request queued for sending after its response is delivered. +/// +/// The Fastly entry point creates one per request and `prepare` inserts a +/// clone into the request extensions, where the per-request services find it. +/// Clones share one queue: [`QueueingTelemetrySink`] pushes onto it during +/// the request, and [`send_queued_batches`] takes the bodies after delivery. +#[derive(Debug, Clone, Default)] +pub(crate) struct TelemetryQueue(Arc>>); + +impl TelemetryQueue { + fn push(&self, body: String) { + self.0 + .lock() + .unwrap_or_else(PoisonError::into_inner) + .push(body); } -} -#[derive(Debug, Clone)] -struct FastlyTinybirdAuctionTelemetrySink { - enabled: bool, - target: TinybirdEventsTarget, + /// Takes every queued body, leaving the queue empty. + pub(crate) fn take(&self) -> Vec { + std::mem::take(&mut *self.0.lock().unwrap_or_else(PoisonError::into_inner)) + } } -#[derive(Debug, Clone)] -struct TinybirdEventsTarget { - api_host: String, - dataset: String, - append_token: Redacted, - uri: String, - backend_spec: PlatformBackendSpec, +/// Auction telemetry sink that queues each batch as NDJSON instead of sending it. +pub(crate) struct QueueingTelemetrySink { max_body_bytes: usize, + queue: TelemetryQueue, } -impl TinybirdEventsTarget { - fn from_config(config: TinybirdSettings) -> Self { - let uri = tinybird_events_uri(&config.api_host, &config.auction_dataset); - let backend_spec = tinybird_backend_spec(&config.api_host); +impl QueueingTelemetrySink { + /// Creates a sink that queues on `queue`, bounded by `config`'s body limit. + pub(crate) fn new(config: &TinybirdSettings, queue: TelemetryQueue) -> Self { Self { - api_host: config.api_host, - dataset: config.auction_dataset, - append_token: config - .auction_token_secret - .expect("should contain a resolved Tinybird auction token when enabled"), - uri, - backend_spec, max_body_bytes: config.max_body_bytes, - } - } -} - -impl FastlyTinybirdAuctionTelemetrySink { - fn new(config: TinybirdSettings) -> Self { - let enabled = config.enabled; - Self { - enabled, - target: TinybirdEventsTarget::from_config(config), + queue, } } @@ -86,115 +80,118 @@ impl FastlyTinybirdAuctionTelemetrySink { } Ok(()) } - - fn serialize_batch( - &self, - batch: &AuctionEventBatch, - ) -> Result> { - batch.to_ndjson(self.target.max_body_bytes) - } - - fn ensure_backend( - &self, - services: &RuntimeServices, - ) -> Result> { - services - .backend() - .ensure(&self.target.backend_spec) - .change_context(TrustedServerError::Proxy { - message: "Tinybird backend registration failed".to_owned(), - }) - } - - fn authorization_header(token: &str) -> Result> { - HeaderValue::from_str(&format!("Bearer {token}")).change_context( - TrustedServerError::InvalidHeaderValue { - message: "invalid Tinybird authorization header".to_owned(), - }, - ) - } - - fn build_events_request( - &self, - body: String, - auth_header: HeaderValue, - ) -> Result> { - request_builder() - .method(Method::POST) - .uri(self.target.uri.as_str()) - .header(header::AUTHORIZATION, auth_header) - .header(header::CONTENT_TYPE, TINYBIRD_NDJSON_CONTENT_TYPE) - .body(Body::from(body)) - .change_context(TrustedServerError::Proxy { - message: "failed to build Tinybird Events API request".to_owned(), - }) - } - - async fn send_fire_and_forget( - services: &RuntimeServices, - request: edgezero_core::http::Request, - backend_name: String, - ) -> Result<(), Report> { - let pending = services - .http_client() - .send_async(PlatformHttpRequest::new(request, backend_name)) - .await - .change_context(TrustedServerError::Proxy { - message: "failed to start Tinybird Events API request".to_owned(), - })?; - drop(pending); - Ok(()) - } } #[async_trait::async_trait(?Send)] -impl AuctionTelemetrySink for FastlyTinybirdAuctionTelemetrySink { - fn is_enabled(&self) -> bool { - self.enabled - } - +impl AuctionTelemetrySink for QueueingTelemetrySink { async fn emit_auction_events( &self, - services: &RuntimeServices, + _services: &RuntimeServices, batch: AuctionEventBatch, ) -> Result<(), Report> { - if !self.enabled || batch.is_empty() { + if batch.is_empty() { return Ok(()); } Self::validate_batch(&batch)?; - let body = self.serialize_batch(&batch)?; - let body_len = body.len(); - let auth_header = Self::authorization_header(self.target.append_token.expose())?; - let backend_name = self.ensure_backend(services)?; - let request = self.build_events_request(body, auth_header)?; - + let body = batch.to_ndjson(self.max_body_bytes)?; log::info!( - "sending auction telemetry to Tinybird dataset={} rows={} bytes={} host={} backend={}", - self.target.dataset, + "queueing auction telemetry for Tinybird rows={} bytes={}", batch.row_count(), - body_len, - self.target.api_host, - backend_name + body.len() ); - - Self::send_fire_and_forget(services, request, backend_name).await + self.queue.push(body); + Ok(()) } } -fn tinybird_backend_spec(api_host: &str) -> PlatformBackendSpec { - PlatformBackendSpec { - scheme: "https".to_owned(), - host: api_host.to_owned(), - port: None, - host_header_override: None, - certificate_check: true, - first_byte_timeout: TINYBIRD_FIRST_BYTE_TIMEOUT, - between_bytes_timeout: TINYBIRD_BETWEEN_BYTES_TIMEOUT, - discriminator: None, +/// Sends every body queued on `queue` to `config`'s Events API, one request each. +/// +/// Best effort: a failed request is logged, followed by `log_context`, and the +/// rest are still sent. Each request carries the 2 s Tinybird timeout and +/// bypasses the cache. +pub(crate) async fn send_queued_batches( + client: &HttpClient, + config: &TinybirdSettings, + queue: &TelemetryQueue, + log_context: &dyn fmt::Display, +) { + let bodies = queue.take(); + if bodies.is_empty() { + return; + } + let Some(token) = config.auction_token_secret.as_ref() else { + log::error!( + "dropping {} queued auction telemetry batch(es): no Tinybird auction token is resolved{log_context}", + bodies.len() + ); + return; + }; + let uri = tinybird_events_uri(&config.api_host, &config.auction_dataset); + + for body in bodies { + let bytes = body.len(); + let request = outbound_request(Method::POST, &uri) + .and_then(|request| { + request.header(header::AUTHORIZATION, format!("Bearer {}", token.expose())) + }) + .and_then(|request| request.header(header::CONTENT_TYPE, TINYBIRD_NDJSON_CONTENT_TYPE)); + let request = match request { + Ok(request) => buffered( + request.body(body), + TINYBIRD_TIMEOUT, + DEFAULT_BUFFERED_RESPONSE_BYTES, + ResponseDecoding::Original, + ) + .cache_policy(OutboundCachePolicy::Bypass), + Err(error) => { + log_send_failure( + error, + "the Events API request could not be built".to_owned(), + log_context, + ); + continue; + } + }; + + log::info!( + "sending auction telemetry to Tinybird dataset={} bytes={bytes} host={}", + config.auction_dataset, + config.api_host + ); + match client.send(request).await { + Ok(response) if response.status().is_success() => {} + Ok(response) => log::warn!( + "Tinybird Events API answered {} for dataset {}{log_context}", + response.status(), + config.auction_dataset + ), + Err(error) => log_send_failure( + error, + format!( + "the Events API request for dataset {} failed", + config.auction_dataset + ), + log_context, + ), + } } } +/// Logs one failed Tinybird send at warn, through [`outbound_error`], followed by `log_context`. +/// +/// `outbound_error` logs the failure's status, kind, budget source, and provider +/// detail, never the error's message, which can hold provider text or a URL. +/// This then logs the report with the `failed` step. +fn log_send_failure(error: EdgeError, failed: String, log_context: &dyn fmt::Display) { + let report = outbound_error( + error, + TrustedServerError::Proxy { message: failed }, + OutboundRequestSource::Internal, + ); + log::warn!("Tinybird telemetry send failed: {report:?}{log_context}"); +} + fn tinybird_events_uri(api_host: &str, dataset: &str) -> String { format!( "https://{api_host}{TINYBIRD_EVENTS_PATH}?name={}", @@ -203,177 +200,30 @@ fn tinybird_events_uri(api_host: &str, dataset: &str) -> String { } #[cfg(test)] -mod tests { - use std::collections::HashMap; - use std::sync::{Arc, Mutex}; - - use error_stack::Report; - use trusted_server_core::auction::telemetry::{AuctionEventBatch, AuctionEventRow}; - use trusted_server_core::platform::{ - ClientInfo, PlatformBackend, PlatformConfigStore, PlatformError, PlatformGeo, - PlatformHttpClient, PlatformPendingRequest, PlatformResponse, PlatformSecretStore, - PlatformSelectResult, RuntimeServices, StoreId, StoreName, +pub(crate) mod tests { + use edgezero_core::http::StatusCode; + use edgezero_core::outbound::ResponseMode; + use trusted_server_core::auction::telemetry::AuctionEventRow; + use trusted_server_core::platform::ClientInfo; + use trusted_server_core::redacted::Redacted; + use trusted_server_core::test_support::logging::WarningCapture; + use trusted_server_core::test_support::scripted_client::{ + ExpectedTransport, ScriptedHttpClient, ScriptedReply, TestClock, }; use super::*; + use crate::platform::{ + FastlyPlatformConfigStore, FastlyPlatformSecretStore, RequestGeo, UnavailableKvStore, + }; const TEST_USER_AGENT: &str = "FictionalBrowser/123.4 (FictionalOS 10.2; FictionalDevice) ExampleRenderer/567.8"; - struct NoopConfigStore; - - impl PlatformConfigStore for NoopConfigStore { - fn get( - &self, - _store_name: &StoreName, - _key: &str, - ) -> Result> { - Err(Report::new(PlatformError::Unsupported)) - } - - fn put( - &self, - _store_id: &StoreId, - _key: &str, - _value: &str, - ) -> Result<(), Report> { - Err(Report::new(PlatformError::Unsupported)) - } - - fn delete(&self, _store_id: &StoreId, _key: &str) -> Result<(), Report> { - Err(Report::new(PlatformError::Unsupported)) - } - } - - struct MapSecretStore(HashMap>); - - impl PlatformSecretStore for MapSecretStore { - fn get_bytes( - &self, - _store_name: &StoreName, - key: &str, - ) -> Result, Report> { - self.0 - .get(key) - .cloned() - .ok_or_else(|| Report::new(PlatformError::SecretStore)) - } - - fn create( - &self, - _store_id: &StoreId, - _name: &str, - _value: &str, - ) -> Result<(), Report> { - Err(Report::new(PlatformError::Unsupported)) - } - - fn delete(&self, _store_id: &StoreId, _name: &str) -> Result<(), Report> { - Err(Report::new(PlatformError::Unsupported)) - } - } - - #[derive(Default)] - struct RecordingBackend { - specs: Mutex>, - } - - impl PlatformBackend for RecordingBackend { - fn naming_policy(&self) -> trusted_server_core::platform::BackendNamingPolicy { - trusted_server_core::platform::BackendNamingPolicy::Fastly - } - - fn predict_name( - &self, - _spec: &PlatformBackendSpec, - ) -> Result> { - Ok("tinybird-backend".to_owned()) - } - - fn ensure(&self, spec: &PlatformBackendSpec) -> Result> { - self.specs - .lock() - .expect("should lock backend specs") - .push(spec.clone()); - Ok("tinybird-backend".to_owned()) - } - } - - #[derive(Debug)] - struct RecordedRequest { - backend_name: String, - method: String, - uri: String, - headers: Vec<(String, String)>, - body: Vec, - } - - #[derive(Default)] - struct RecordingHttpClient { - requests: Mutex>, - select_calls: Mutex, - } - - #[async_trait::async_trait(?Send)] - impl PlatformHttpClient for RecordingHttpClient { - async fn send( - &self, - _request: PlatformHttpRequest, - ) -> Result> { - Err(Report::new(PlatformError::Unsupported)) - } - - async fn send_async( - &self, - request: PlatformHttpRequest, - ) -> Result> { - let backend_name = request.backend_name; - let (parts, body) = request.request.into_parts(); - let headers = parts - .headers - .iter() - .filter_map(|(name, value)| { - value - .to_str() - .ok() - .map(|value| (name.as_str().to_owned(), value.to_owned())) - }) - .collect(); - let recorded = RecordedRequest { - backend_name, - method: parts.method.to_string(), - uri: parts.uri.to_string(), - headers, - body: body.into_bytes().unwrap_or_default().to_vec(), - }; - self.requests - .lock() - .expect("should lock recorded requests") - .push(recorded); - Ok(PlatformPendingRequest::new(()).with_backend_name("tinybird-backend")) - } - - async fn select( - &self, - _pending_requests: Vec, - ) -> Result> { - *self.select_calls.lock().expect("should lock select calls") += 1; - Err(Report::new(PlatformError::Unsupported)) - } - } - - struct NoopGeo; + /// Events API URI of [`enabled_config`]. + pub(crate) const EVENTS_URI: &str = + "https://api.tinybird.example.com/v0/events?name=auction_events_raw"; - impl PlatformGeo for NoopGeo { - fn lookup( - &self, - _client_ip: Option, - ) -> Result, Report> { - Ok(None) - } - } - - fn test_row() -> AuctionEventRow { + pub(crate) fn test_row() -> AuctionEventRow { AuctionEventRow { event_ts: "2026-06-23 00:00:00.000".to_owned(), event_kind: "summary".to_owned(), @@ -412,26 +262,10 @@ mod tests { } } - fn services( - backend: Arc, - http_client: Arc, - secrets: HashMap>, - ) -> RuntimeServices { - RuntimeServices::builder() - .config_store(Arc::new(NoopConfigStore)) - .secret_store(Arc::new(MapSecretStore(secrets))) - .kv_store(Arc::new(edgezero_core::key_value_store::NoopKvStore)) - .backend(backend) - .http_client(http_client) - .geo(Arc::new(NoopGeo)) - .client_info(ClientInfo::default()) - .build() - } - - fn enabled_config() -> TinybirdSettings { + pub(crate) fn enabled_config() -> TinybirdSettings { TinybirdSettings { enabled: true, - api_host: "api.us-east.aws.tinybird.co".to_owned(), + api_host: "api.tinybird.example.com".to_owned(), secret_store: None, auction_dataset: "auction_events_raw".to_owned(), auction_token_secret: Some(Redacted::new("append-token".to_owned())), @@ -443,186 +277,262 @@ mod tests { } } - #[test] - fn events_uri_targets_dataset_on_region_host() { + pub(crate) fn test_services() -> RuntimeServices { + RuntimeServices::builder() + .config_store(Arc::new(FastlyPlatformConfigStore::read_only())) + .secret_store(Arc::new(FastlyPlatformSecretStore::read_only())) + .kv_store(Arc::new(UnavailableKvStore)) + .outbound_client(ScriptedHttpClient::forbidding().http_client()) + .clock(TestClock::new().monotonic_clock()) + .geo(Arc::new(RequestGeo::default())) + .client_info(ClientInfo::default()) + .build() + } + + fn queue_of(bodies: &[&str]) -> TelemetryQueue { + let queue = TelemetryQueue::default(); + for body in bodies { + queue.push((*body).to_owned()); + } + queue + } + + /// Asserts `request` is one Events API POST of `body` with today's headers. + pub(crate) fn assert_events_request( + request: &trusted_server_core::test_support::scripted_client::RecordedRequest, + body: &str, + ) { + assert_eq!(request.method, Method::POST, "should POST to Tinybird"); + assert_eq!( + request.uri.to_string(), + EVENTS_URI, + "should target the dataset on the region host" + ); + request.assert_headers(&[ + (header::AUTHORIZATION.as_str(), "Bearer append-token"), + (header::CONTENT_TYPE.as_str(), TINYBIRD_NDJSON_CONTENT_TYPE), + ]); assert_eq!( - tinybird_events_uri("api.us-east.aws.tinybird.co", "auction_events_raw"), - "https://api.us-east.aws.tinybird.co/v0/events?name=auction_events_raw" + request.body.as_ref(), + body.as_bytes(), + "should send the queued NDJSON body" ); + request.assert_transport(ExpectedTransport { + timeout: TINYBIRD_TIMEOUT, + response_mode: ResponseMode::Buffered { + max_bytes: DEFAULT_BUFFERED_RESPONSE_BYTES, + }, + response_decoding: ResponseDecoding::Original, + cache_policy: OutboundCachePolicy::Bypass, + }); } #[test] - fn events_uri_urlencodes_dataset_name() { + fn events_uri_targets_dataset_on_region_host() { assert_eq!( - tinybird_events_uri("api.us-east.aws.tinybird.co", "auction events/raw"), - "https://api.us-east.aws.tinybird.co/v0/events?name=auction%20events%2Fraw" + tinybird_events_uri("api.tinybird.example.com", "auction_events_raw"), + EVENTS_URI ); } #[test] - fn backend_spec_uses_matching_tls_host() { - let spec = tinybird_backend_spec("api.us-east.aws.tinybird.co"); - assert_eq!(spec.scheme, "https"); - assert_eq!(spec.host, "api.us-east.aws.tinybird.co"); - assert_eq!(spec.host_header_override, None); - assert!(spec.certificate_check, "should verify Tinybird TLS cert"); + fn events_uri_urlencodes_dataset_name() { + assert_eq!( + tinybird_events_uri("api.tinybird.example.com", "auction events/raw"), + "https://api.tinybird.example.com/v0/events?name=auction%20events%2Fraw" + ); } #[test] - fn sink_posts_ndjson_with_resolved_token_and_does_not_wait() { - let backend = Arc::new(RecordingBackend::default()); - let http_client = Arc::new(RecordingHttpClient::default()); - let services = services( - Arc::clone(&backend), - Arc::clone(&http_client), - HashMap::new(), - ); - let sink = FastlyTinybirdAuctionTelemetrySink::new(enabled_config()); + fn sink_queues_ndjson_with_the_full_user_agent_and_sends_nothing() { + let queue = TelemetryQueue::default(); + let sink = QueueingTelemetrySink::new(&enabled_config(), queue.clone()); futures::executor::block_on( - sink.emit_auction_events(&services, AuctionEventBatch::new(vec![test_row()])), + sink.emit_auction_events(&test_services(), AuctionEventBatch::new(vec![test_row()])), ) - .expect("should start Tinybird request"); - - let specs = backend.specs.lock().expect("should lock specs"); - assert_eq!(specs.len(), 1, "should ensure one backend"); - assert_eq!(specs[0].host, "api.us-east.aws.tinybird.co"); - drop(specs); + .expect("should queue the batch"); - let requests = http_client - .requests - .lock() - .expect("should lock recorded requests"); - assert_eq!(requests.len(), 1, "should start one async POST"); - assert_eq!(requests[0].backend_name, "tinybird-backend"); - assert_eq!(requests[0].method, Method::POST.to_string()); - assert_eq!( - requests[0].uri, - "https://api.us-east.aws.tinybird.co/v0/events?name=auction_events_raw" - ); - assert_eq!( - header_value(&requests[0].headers, header::CONTENT_TYPE.as_str()), - Some(TINYBIRD_NDJSON_CONTENT_TYPE) - ); - assert_eq!( - header_value(&requests[0].headers, header::AUTHORIZATION.as_str()), - Some("Bearer append-token") - ); - let body = std::str::from_utf8(&requests[0].body).expect("should record utf8 ndjson body"); - assert!(body.ends_with('\n'), "should send newline-delimited JSON"); + let bodies = queue.take(); + let [body] = bodies.as_slice() else { + panic!("should queue one body, got {bodies:?}"); + }; + assert!(body.ends_with('\n'), "should queue newline-delimited JSON"); assert!( body.contains(TEST_USER_AGENT), - "should send the complete user agent to Tinybird" - ); - assert_eq!( - *http_client - .select_calls - .lock() - .expect("should lock select calls"), - 0, - "should not wait for the Tinybird response" + "should keep the complete user agent for Tinybird" ); } #[test] - fn disabled_sink_does_not_dispatch() { - let backend = Arc::new(RecordingBackend::default()); - let http_client = Arc::new(RecordingHttpClient::default()); - let services = services( - Arc::clone(&backend), - Arc::clone(&http_client), - HashMap::new(), - ); - let mut config = enabled_config(); - config.enabled = false; - let sink = FastlyTinybirdAuctionTelemetrySink::new(config); + fn empty_batch_queues_nothing() { + let queue = TelemetryQueue::default(); + let sink = QueueingTelemetrySink::new(&enabled_config(), queue.clone()); futures::executor::block_on( - sink.emit_auction_events(&services, AuctionEventBatch::new(vec![test_row()])), + sink.emit_auction_events(&test_services(), AuctionEventBatch::new(Vec::new())), ) - .expect("should ignore disabled sink"); + .expect("should ignore the empty batch"); - assert!( - backend.specs.lock().expect("should lock specs").is_empty(), - "should not ensure backend when disabled" - ); - assert!( - http_client - .requests - .lock() - .expect("should lock recorded requests") - .is_empty(), - "should not send when disabled" - ); + assert!(queue.take().is_empty(), "should queue no empty batch"); } #[test] - fn empty_batch_does_not_dispatch() { - let backend = Arc::new(RecordingBackend::default()); - let http_client = Arc::new(RecordingHttpClient::default()); - let services = services( - Arc::clone(&backend), - Arc::clone(&http_client), - HashMap::new(), - ); - let sink = FastlyTinybirdAuctionTelemetrySink::new(enabled_config()); + fn sink_rejects_a_row_count_oversize_before_queueing() { + let queue = TelemetryQueue::default(); + let sink = QueueingTelemetrySink::new(&enabled_config(), queue.clone()); + let rows = vec![test_row(); TINYBIRD_MAX_ROWS_PER_AUCTION_BATCH + 1]; - futures::executor::block_on( - sink.emit_auction_events(&services, AuctionEventBatch::new(Vec::new())), - ) - .expect("should ignore empty batch"); + let result = futures::executor::block_on( + sink.emit_auction_events(&test_services(), AuctionEventBatch::new(rows)), + ); assert!( - backend.specs.lock().expect("should lock specs").is_empty(), - "should not ensure backend for empty batches" + result.is_err(), + "best-effort caller will suppress this error" ); assert!( - http_client - .requests - .lock() - .expect("should lock recorded requests") - .is_empty(), - "should not send empty batches" + queue.take().is_empty(), + "should not queue an oversized row batch" ); } #[test] - fn sink_drops_row_count_oversize_before_sending() { - let backend = Arc::new(RecordingBackend::default()); - let http_client = Arc::new(RecordingHttpClient::default()); - let services = services( - backend, - Arc::clone(&http_client), - HashMap::from([( - "tinybird_auction_append_token".to_owned(), - b"append-token".to_vec(), - )]), - ); - let sink = FastlyTinybirdAuctionTelemetrySink::new(enabled_config()); - let rows = vec![test_row(); TINYBIRD_MAX_ROWS_PER_AUCTION_BATCH + 1]; + fn queued_batches_post_once_each_with_the_token_timeout_and_bypass() { + let clock = TestClock::new(); + let client = ScriptedHttpClient::new(&clock); + for _ in 0..2 { + client.respond_to( + Method::POST, + EVENTS_URI, + ScriptedReply::status(StatusCode::ACCEPTED), + ); + } + let queue = queue_of(&["{\"row\":1}\n", "{\"row\":2}\n"]); + + clock.drive(send_queued_batches( + &client.http_client(), + &enabled_config(), + &queue, + &"", + )); + + let requests = client.requests(); + let [first, second] = requests.as_slice() else { + panic!("should send one request per queued body, got {requests:?}"); + }; + assert_events_request(first, "{\"row\":1}\n"); + assert_events_request(second, "{\"row\":2}\n"); + assert!(queue.take().is_empty(), "should drain the queue"); + } - let result = futures::executor::block_on( - sink.emit_auction_events(&services, AuctionEventBatch::new(rows)), + #[test] + fn a_failed_batch_does_not_stop_the_next_one() { + let clock = TestClock::new(); + let client = ScriptedHttpClient::new(&clock); + client + .respond_to( + Method::POST, + EVENTS_URI, + ScriptedReply::error(edgezero_core::error::EdgeError::bad_gateway( + "scripted Tinybird failure", + )), + ) + .respond_to( + Method::POST, + EVENTS_URI, + ScriptedReply::status(StatusCode::ACCEPTED), + ); + let queue = queue_of(&["{\"row\":1}\n", "{\"row\":2}\n"]); + + clock.drive(send_queued_batches( + &client.http_client(), + &enabled_config(), + &queue, + &"", + )); + + assert_eq!( + client.requests().len(), + 2, + "should still send the second body after the first fails" ); + } - assert!( - result.is_err(), - "best-effort caller will suppress this error" + #[test] + fn send_and_build_failures_log_through_outbound_error() { + let clock = TestClock::new(); + let client = ScriptedHttpClient::new(&clock); + client.respond_to( + Method::POST, + EVENTS_URI, + ScriptedReply::error(EdgeError::bad_gateway("tinybird provider text")), ); + let mut unbuildable = enabled_config(); + unbuildable.api_host = "api tinybird.example.com".to_owned(); + let capture = WarningCapture::start(); + + clock.drive(send_queued_batches( + &client.http_client(), + &enabled_config(), + &queue_of(&["{\"row\":1}\n"]), + &" [request=example]", + )); + clock.drive(send_queued_batches( + &client.http_client(), + &unbuildable, + &queue_of(&["{\"row\":2}\n"]), + &" [request=example]", + )); + + let warnings = capture.take(); + let typed = warnings + .iter() + .filter(|warning| warning.message.starts_with("Outbound request failed with")) + .count(); + assert_eq!( + typed, 2, + "should log the send failure and the build failure through outbound_error" + ); + for step in [ + "the Events API request for dataset auction_events_raw failed", + "the Events API request could not be built", + ] { + assert!( + warnings.iter().any(|warning| { + warning + .message + .starts_with("Tinybird telemetry send failed:") + && warning.message.contains(step) + && warning.message.ends_with(" [request=example]") + }), + "should log the report for `{step}` with the log context" + ); + } assert!( - http_client - .requests - .lock() - .expect("should lock recorded requests") - .is_empty(), - "should not send oversized row batches" + warnings + .iter() + .all(|warning| !warning.message.contains("tinybird provider text")), + "should keep the provider text out of the logs" + ); + assert_eq!( + client.requests().len(), + 1, + "should send only the request that could be built" ); } - fn header_value<'a>(headers: &'a [(String, String)], name: &str) -> Option<&'a str> { - headers - .iter() - .find(|(header_name, _)| header_name.eq_ignore_ascii_case(name)) - .map(|(_, value)| value.as_str()) + #[test] + fn an_empty_queue_sends_nothing() { + let client = ScriptedHttpClient::forbidding(); + + futures::executor::block_on(send_queued_batches( + &client.http_client(), + &enabled_config(), + &TelemetryQueue::default(), + &"", + )); + + client.assert_no_requests(); } } diff --git a/crates/trusted-server-adapter-fastly/tests/deadline-origin.mjs b/crates/trusted-server-adapter-fastly/tests/deadline-origin.mjs index c985097677..a8d1407c47 100644 --- a/crates/trusted-server-adapter-fastly/tests/deadline-origin.mjs +++ b/crates/trusted-server-adapter-fastly/tests/deadline-origin.mjs @@ -107,8 +107,10 @@ async function main() { const config = join(directory, 'fastly.toml') await writeFile( config, - '[local_server.backends.deadline_origin]\n' + - `url = "http://127.0.0.1:${server.address().port}"\n` + '[local_server.config_stores.deadline_fixture]\n' + + 'format = "inline-toml"\n' + + '[local_server.config_stores.deadline_fixture.contents]\n' + + `port = "${server.address().port}"\n` ) const environment = { ...process.env } if (existsSync('/etc/ssl/cert.pem')) @@ -120,7 +122,7 @@ async function main() { '-C', config, binary, - 'platform::deadline::tests::network_', + 'deadline_network::tests::network_', '--ignored', '--nocapture', ], @@ -147,8 +149,8 @@ async function main() { assert.equal(code, 0, 'should run the network tests successfully') assert.match( output, - /^test result: ok\. 5 passed; 0 failed; 0 ignored; 0 measured;/m, - 'should execute all five network tests, including when the filter changes' + /^test result: ok\. 6 passed; 0 failed; 0 ignored; 0 measured;/m, + 'should execute all six network tests, including when the filter changes' ) return code ?? 1 } finally { diff --git a/crates/trusted-server-adapter-spin/Cargo.toml b/crates/trusted-server-adapter-spin/Cargo.toml index 47cf7b589c..f7b14891ff 100644 --- a/crates/trusted-server-adapter-spin/Cargo.toml +++ b/crates/trusted-server-adapter-spin/Cargo.toml @@ -24,16 +24,12 @@ spin = ["edgezero-adapter-spin/spin"] [dependencies] anyhow = { workspace = true } async-trait = { workspace = true } -brotli = { workspace = true } bytes = { workspace = true } edgezero-adapter-spin = { workspace = true } edgezero-core = { workspace = true } error-stack = { workspace = true } -flate2 = { workspace = true } futures = { workspace = true } -http-body-util = { workspace = true } log = { workspace = true } -web-time = { workspace = true } trusted-server-core = { workspace = true } [target.'cfg(target_arch = "wasm32")'.dependencies] @@ -43,6 +39,7 @@ spin-sdk = { workspace = true } trusted-server-core = { workspace = true, features = ["test-utils"] } base64 = { workspace = true } edgezero-core = { workspace = true } +web-time = { workspace = true } [target.'cfg(not(target_arch = "wasm32"))'.dev-dependencies] tokio = { workspace = true, features = ["rt-multi-thread", "macros"] } diff --git a/crates/trusted-server-adapter-spin/README.md b/crates/trusted-server-adapter-spin/README.md index fe33c90557..4aebea1cc6 100644 --- a/crates/trusted-server-adapter-spin/README.md +++ b/crates/trusted-server-adapter-spin/README.md @@ -5,9 +5,11 @@ Experimental Fermyon Spin adapter for Trusted Server. The component targets platform tests. The crate maps Spin HTTP, variables, and the `default` key-value store into the -shared runtime. Startup failures install a restricted 503 router that keeps -`/health` live. The adapter permits one enabled auction provider. Request-time -KV services beyond startup config loading remain unavailable. Its entry point +shared runtime. A startup failure answers every route with 503, `/health` +included. Multi-provider auctions fan out as one EdgeZero request batch. +Request-time KV services beyond startup config loading remain unavailable. +EdgeZero's Spin client cannot override the outbound `Host`, so the adapter +ignores `publisher.origin_host_header_override` with a warning. Its entry point uses `anyhow::Result` only because the EdgeZero Spin FFI requires that type. Build and test from the repository root: diff --git a/crates/trusted-server-adapter-spin/src/app.rs b/crates/trusted-server-adapter-spin/src/app.rs index 9edce69fa3..5447396e67 100644 --- a/crates/trusted-server-adapter-spin/src/app.rs +++ b/crates/trusted-server-adapter-spin/src/app.rs @@ -4,14 +4,17 @@ use std::sync::Arc; #[cfg(all(feature = "spin", target_arch = "wasm32"))] use edgezero_adapter_spin::config_store::SpinConfigStore; use edgezero_adapter_spin::context::SpinRequestContext; -use edgezero_core::app::Hooks; +use edgezero_core::app::{App, Hooks}; #[cfg(all(feature = "spin", target_arch = "wasm32"))] use edgezero_core::config_store::ConfigStoreHandle; use edgezero_core::context::RequestContext; use edgezero_core::error::EdgeError; -use edgezero_core::http::{HeaderValue, Method, Request, Response, StatusCode, header}; +use edgezero_core::http::{ + HeaderValue, Method, Request, RequestParts, Response, StatusCode, header, +}; use edgezero_core::ingress::{InboundOrigin, OriginSource}; -use edgezero_core::router::RouterService; +use edgezero_core::outbound::ResponseDecoding; +use edgezero_core::router::{RouterBuilder, RouterService}; use error_stack::Report; use trusted_server_core::auction::endpoints::handle_auction; use trusted_server_core::auction::{ @@ -26,10 +29,11 @@ use trusted_server_core::ec::admin::{ deny_admin_diagnostic_fallback, handle_admin_eids_lookup, }; use trusted_server_core::ec::registry::PartnerRegistry; -use trusted_server_core::error::{IntoHttpResponse as _, TrustedServerError}; +use trusted_server_core::edge_app::install_edge_policies; +use trusted_server_core::error::{TrustedServerError, http_error}; use trusted_server_core::http_util::{RuntimeRequestOrigin, sanitize_forwarded_headers}; use trusted_server_core::integrations::{IntegrationRegistry, ProxyDispatchInput}; -use trusted_server_core::platform::{ClientInfo, RuntimeServices}; +use trusted_server_core::platform::{ClientInfo, HttpClient, RuntimeServices}; #[cfg(all(feature = "spin", target_arch = "wasm32"))] use trusted_server_core::platform::{PlatformConfigStore, StoreName}; use trusted_server_core::proxy::{ @@ -37,9 +41,9 @@ use trusted_server_core::proxy::{ handle_first_party_proxy_sign, }; use trusted_server_core::publisher::{ - AuctionDispatch, PAGE_BIDS_LEGACY_PATH, PAGE_BIDS_PATH, PublisherResponse, - buffer_publisher_response_async, handle_page_bids, handle_publisher_request, - handle_tsjs_dynamic, page_bids_preflight_denied, + AuctionDispatch, PAGE_BIDS_LEGACY_PATH, PAGE_BIDS_PATH, PublisherOriginOptions, + PublisherResponse, buffer_publisher_response_async, handle_page_bids, handle_publisher_request, + handle_tsjs_dynamic, page_bids_preflight_denied, publisher_input_cap, publisher_owner_error, }; use trusted_server_core::request_signing::{ handle_trusted_server_discovery, handle_verify_signature, @@ -87,14 +91,9 @@ fn build_state() -> Result, Report> { fn load_startup_settings() -> Result> { let config_store_name = StoreName::from(SPIN_DEFAULT_CONFIG_STORE); let config_key = default_config_key(); - let config_store = - futures::executor::block_on(SpinConfigStore::open(config_store_name.as_ref().to_owned())) - .map_err(|error| { - Report::new(TrustedServerError::Configuration { - message: "failed to open Spin Trusted Server config store".to_string(), - }) - .attach(error.to_string()) - })?; + // `for_label` defers open failures to the first read, which the + // read error branch below reports. + let config_store = SpinConfigStore::for_label(config_store_name.as_ref().to_owned()); let config_handle = ConfigStoreHandle::new(Arc::new(config_store)); let config_adapter = ConfigStoreHandleAdapter(config_handle); let raw_envelope = config_adapter @@ -130,11 +129,14 @@ fn build_state_with_settings( } fn build_state_with_services( - settings: Settings, + mut settings: Settings, services: Option, ) -> Result, Report> { + // Spin cannot send a host other than the origin URI authority. + settings + .publisher + .drop_origin_host_header_override_for_spin(); let plan = Arc::new(compile_auction_plan(&settings)?); - plan.validate_for_target(trusted_server_core::platform::AuctionTargetId::Spin)?; let orchestrator = build_orchestrator_with_plan(Arc::clone(&plan), &settings)?; let registry = IntegrationRegistry::with_plan(&settings, plan)?; @@ -147,10 +149,10 @@ fn build_state_with_services( } impl AppState { - fn services_for_request(&self, ctx: &RequestContext) -> RuntimeServices { + fn services_for_request(&self, ctx: &RequestContext) -> Result { self.services .clone() - .unwrap_or_else(|| build_runtime_services(ctx)) + .map_or_else(|| build_runtime_services(ctx), Ok) } } @@ -158,12 +160,19 @@ impl AppState { // Publisher response helper // --------------------------------------------------------------------------- +/// Largest publisher origin body the Spin adapter collects before egress. +/// +/// It keeps the 8 MiB limit the adapter's own client applied before the +/// publisher origin moved to `EdgeZero`'s client. +const PUBLISHER_ORIGIN_INPUT_BYTES: u64 = 8 * 1024 * 1024; + /// Collapse a [`PublisherResponse`] into a plain [`Response`]. /// /// Delegates to the shared [`buffer_publisher_response_async`], which collects -/// the dispatched server-side auction and enforces -/// `settings.publisher.max_buffered_body_bytes` so a large processable origin -/// response fails safely instead of exhausting the Wasm heap. +/// the dispatched server-side auction, collects raw origin bodies under +/// `input_cap`, and enforces `settings.publisher.max_buffered_body_bytes` on +/// processed output, so a large origin response fails safely instead of +/// exhausting the Wasm heap. async fn resolve_publisher_response( publisher_response: PublisherResponse, method: &Method, @@ -171,6 +180,7 @@ async fn resolve_publisher_response( registry: &IntegrationRegistry, orchestrator: &AuctionOrchestrator, services: &RuntimeServices, + input_cap: u64, ) -> Result> { buffer_publisher_response_async( publisher_response, @@ -179,6 +189,7 @@ async fn resolve_publisher_response( registry, orchestrator, services, + Some(input_cap), ) .await } @@ -293,8 +304,8 @@ fn parse_client_addr(raw: &str) -> Option { // which rejects a relative path. Rebuilding an absolute URI from the trusted // scheme+host lets those handlers validate the signed target instead of failing // with "Invalid URL". -pub(crate) fn normalize_spin_request(req: &mut Request) { - sanitize_forwarded_headers(req); +pub(crate) fn normalize_spin_request(req: &mut RequestParts) { + sanitize_forwarded_headers(&mut req.headers); // Strip any client-supplied `x-forwarded-for`. Spin exposes no trusted // upstream forwarded-for chain — the only trusted client IP comes from the @@ -302,7 +313,7 @@ pub(crate) fn normalize_spin_request(req: &mut Request) { // inbound `x-forwarded-for` to origins and Prebid copies it into its outbound // headers, so leaving it would let a Spin client spoof the IP seen by // downstream services while `device.ip` uses the trusted runtime address. - req.headers_mut().remove("x-forwarded-for"); + req.headers.remove("x-forwarded-for"); // Re-derive the trusted client IP from the *last* `spin-client-addr` header // and overwrite `SpinRequestContext`, which `build_runtime_services` reads @@ -313,26 +324,23 @@ pub(crate) fn normalize_spin_request(req: &mut Request) { // hashing, `/auction` `device.ip`, and integration `X-Forwarded-For` consume. // Selecting the last occurrence pins the value to the trusted runtime one. let trusted_client_addr = req - .headers() + .headers .get_all("spin-client-addr") .iter() .next_back() .and_then(|v| v.to_str().ok()) .and_then(parse_client_addr); let trusted_full_url = req - .headers() + .headers .get_all("spin-full-url") .iter() .next_back() .and_then(|v| v.to_str().ok()) .map(str::to_owned); - SpinRequestContext::insert( - req, - SpinRequestContext { - client_addr: trusted_client_addr, - full_url: trusted_full_url.clone(), - }, - ); + req.extensions.insert(SpinRequestContext { + client_addr: trusted_client_addr, + full_url: trusted_full_url.clone(), + }); // Spin's WASI HTTP bridge copies synthetic `spin-client-addr` and // `spin-full-url` headers onto the core request. They are runtime metadata, @@ -340,7 +348,7 @@ pub(crate) fn normalize_spin_request(req: &mut Request) { // values are captured into `SpinRequestContext` above; remove every copy here // so the shared publisher/integration handlers neither forward them to // publisher origins nor mistake them for client-controlled input. - req.headers_mut().remove("spin-client-addr"); + req.headers.remove("spin-client-addr"); // Spin's WASI HTTP bridge appends its synthetic `spin-full-url` *after* the // original client headers, so when duplicate names are present the trusted @@ -350,7 +358,7 @@ pub(crate) fn normalize_spin_request(req: &mut Request) { // by publisher HTML rewriting, integration URL rewriting, and request // signing. Select the last occurrence as the trusted authority, then strip // every copy so none reach publisher origins or the shared handlers. - req.headers_mut().remove("spin-full-url"); + req.headers.remove("spin-full-url"); let Some((scheme, host)) = trusted_full_url .as_deref() .and_then(scheme_host_from_spin_url) @@ -360,8 +368,7 @@ pub(crate) fn normalize_spin_request(req: &mut Request) { let Ok(origin) = InboundOrigin::parse(&scheme, &host, OriginSource::RuntimeUri) else { return; }; - req.extensions_mut() - .insert(RuntimeRequestOrigin::new(origin)); + req.extensions.insert(RuntimeRequestOrigin::new(origin)); // Always set Host from the trusted spin-full-url rather than preserving any // incoming value. Spin's WASI HTTP bridge does not normally surface the @@ -373,18 +380,18 @@ pub(crate) fn normalize_spin_request(req: &mut Request) { // signing context) read one host while handlers parsing req.uri() see another. // Overriding from the single trusted authority keeps both consistent. if let Ok(hval) = HeaderValue::from_str(&host) { - req.headers_mut().insert(header::HOST, hval); + req.headers.insert(header::HOST, hval); } // Promote the path-only URI to an absolute one so the shared first-party // proxy/click/sign handlers can parse `req.uri()` as a full URL. let path_and_query = req - .uri() + .uri .path_and_query() .map(|pq| pq.as_str().to_owned()) .unwrap_or_else(|| "/".to_string()); if let Ok(uri) = format!("{scheme}://{host}{path_and_query}").parse() { - *req.uri_mut() = uri; + req.uri = uri; } } @@ -395,10 +402,9 @@ pub(crate) fn normalize_spin_request(req: &mut Request) { /// Builds the `GET /health` liveness response (`200 ok`, `text/plain`). /// /// Mirrors the Fastly entry point and Axum adapter so deployments reusing -/// Trusted Server health probes see identical behaviour on Spin. Served from -/// both the healthy router and the startup-error fallback so the probe answers -/// even before (or when) application state is usable, leaving Spin's -/// platform-provided `/.well-known/spin/health` untouched. +/// Trusted Server health probes see identical behaviour on Spin. Served only +/// once application state is built, so a startup failure answers 503 here too. +/// Spin's platform-provided `/.well-known/spin/health` is left untouched. fn health_response() -> Response { let mut resp = Response::new(edgezero_core::body::Body::from("ok")); *resp.status_mut() = StatusCode::OK; @@ -463,25 +469,6 @@ fn admin_ec_lookup_not_supported() -> Response { core_admin_ec_lookup_not_supported() } -// --------------------------------------------------------------------------- -// Error helper -// --------------------------------------------------------------------------- - -/// Convert a [`Report`] into an HTTP [`Response`]. -pub(crate) fn http_error(report: &Report) -> Response { - let root_error = report.current_context(); - log::error!("Error occurred: {:?}", report); - - let body = edgezero_core::body::Body::from(format!("{}\n", root_error.user_message())); - let mut response = Response::new(body); - *response.status_mut() = root_error.status_code(); - response.headers_mut().insert( - header::CONTENT_TYPE, - HeaderValue::from_static("text/plain; charset=utf-8"), - ); - response -} - /// Builds the local `404 Not Found` returned for legacy `/admin/keys/*` /// aliases on the Spin adapter. /// @@ -499,46 +486,6 @@ fn legacy_admin_alias_denied() -> Response { response } -// --------------------------------------------------------------------------- -// Startup error fallback -// --------------------------------------------------------------------------- - -/// Returns a [`RouterService`] that responds to every route with a generic -/// 503 Service Unavailable. The startup error is logged but not echoed in the -/// response body so that deployment state is not leaked to anonymous callers. -fn startup_error_router(e: &Report) -> RouterService { - log::error!("startup failed, serving error fallback: {:?}", e); - - let handler = |_ctx: RequestContext| { - let body = edgezero_core::body::Body::from("Service Unavailable\n"); - let mut resp = Response::new(body); - *resp.status_mut() = StatusCode::SERVICE_UNAVAILABLE; - resp.headers_mut().insert( - header::CONTENT_TYPE, - HeaderValue::from_static("text/plain; charset=utf-8"), - ); - async move { Ok::(resp) } - }; - - // Cover the full publisher fallback method set (GET, POST, HEAD, OPTIONS, - // PUT, PATCH, DELETE) so degraded behaviour stays consistent with the - // healthy router: every method on `/` and `/{*rest}` returns the generic - // 503 instead of a router-level 405 for HEAD/OPTIONS/PATCH. - let mut builder = RouterService::builder().middleware(FinalizeResponseMiddleware::new( - Arc::new(Settings::default()), - )); - // Keep the liveness probe answering 200 even while state construction is - // failing, matching the Fastly/Axum health behaviour. - builder = builder.get("/health", |_ctx: RequestContext| async { - Ok::(health_response()) - }); - for method in publisher_fallback_methods() { - builder = builder.route("/", method.clone(), handler); - builder = builder.route("/{*rest}", method, handler); - } - builder.build() -} - // --------------------------------------------------------------------------- // TrustedServerApp // --------------------------------------------------------------------------- @@ -547,20 +494,23 @@ fn startup_error_router(e: &Report) -> RouterService { pub struct TrustedServerApp; impl Hooks for TrustedServerApp { + fn configure(app: &mut App) -> Result<(), EdgeError> { + install_edge_policies(app) + } + fn name() -> &'static str { "TrustedServer" } - fn routes() -> RouterService { - let state = match build_state() { - Ok(s) => s, - Err(ref e) => { - log::error!("failed to build application state: {:?}", e); - return startup_error_router(e); - } - }; + fn routes() -> Result { + // Every startup failure answers 503 with a generic message, so the + // logged report never reaches an anonymous caller. + let state = build_state().map_err(|report| { + log::error!("{report:?}"); + EdgeError::service_unavailable("Service Unavailable") + })?; - build_router(&state) + Ok(router_builder(&state).build()) } } @@ -579,7 +529,34 @@ impl TrustedServerApp { settings: Settings, ) -> Result> { let state = build_state_with_settings(settings)?; - Ok(build_router(&state)) + Ok(router_builder(&state).build()) + } + + /// Build the full application router from explicit settings, with `client` + /// in router state. + /// + /// Testing seam: `EdgeZero` hands a client in router state to every + /// request in place of the adapter's own, so an app served from this + /// router on `EdgeZero`'s Axum test server starts no real transport. + /// Production installs no client in router state. + /// + /// # Errors + /// + /// Returns an error when the auction orchestrator or the integration + /// registry fail to initialise. + /// + /// # Examples + /// + /// ```ignore + /// let client = HttpClient::with_client(ScriptedHttpClient::forbidding()); + /// let router = TrustedServerApp::routes_with_settings_and_outbound_client(settings, client)?; + /// ``` + pub fn routes_with_settings_and_outbound_client( + settings: Settings, + client: HttpClient, + ) -> Result> { + let state = build_state_with_settings(settings)?; + Ok(router_builder(&state).with_state(client).build()) } /// Build the full router with explicit settings and runtime services. @@ -587,6 +564,8 @@ impl TrustedServerApp { /// Each request receives a clone of the supplied services, allowing callers /// to exercise production routes with deterministic platform dependencies. /// The supplied client metadata applies to every request to this router. + /// The services' outbound client also goes into router state, so an app + /// served from this router starts no real `EdgeZero` transport. /// /// # Errors /// @@ -602,12 +581,14 @@ impl TrustedServerApp { settings: Settings, services: RuntimeServices, ) -> Result> { + let client = services.outbound_client().clone(); let state = build_state_with_services(settings, Some(services))?; - Ok(build_router(&state)) + Ok(router_builder(&state).with_state(client).build()) } } -fn build_router(state: &Arc) -> RouterService { +/// Registers every route, middleware, and hook for `state` on a router builder. +fn router_builder(state: &Arc) -> RouterBuilder { { let state = Arc::clone(state); @@ -616,10 +597,12 @@ fn build_router(state: &Arc) -> RouterService { let discovery_handler = move |ctx: RequestContext| { let s = Arc::clone(&s); async move { - let services = s.services_for_request(&ctx); - let req = ctx.into_request(); - Ok(handle_trusted_server_discovery(&s.settings, &services, req) - .unwrap_or_else(|e| http_error(&e))) + let services = s.services_for_request(&ctx)?; + let req = ctx.into_request()?; + Ok::( + handle_trusted_server_discovery(&s.settings, &services, req) + .unwrap_or_else(|e| http_error(&e)), + ) } }; @@ -628,10 +611,13 @@ fn build_router(state: &Arc) -> RouterService { let verify_handler = move |ctx: RequestContext| { let s = Arc::clone(&s); async move { - let services = s.services_for_request(&ctx); - let req = ctx.into_request(); - Ok(handle_verify_signature(&s.settings, &services, req) - .unwrap_or_else(|e| http_error(&e))) + let services = s.services_for_request(&ctx)?; + let req = ctx.into_request()?; + Ok::( + handle_verify_signature(&s.settings, &services, req) + .await + .unwrap_or_else(|e| http_error(&e)), + ) } }; @@ -652,7 +638,7 @@ fn build_router(state: &Arc) -> RouterService { let admin_eids_handler = move |ctx: RequestContext| { let s = Arc::clone(&s); async move { - let req = ctx.into_request(); + let req = ctx.into_request()?; let result = PartnerRegistry::from_config(&s.settings.ec.partners) .and_then(|registry| handle_admin_eids_lookup(®istry, &req)); Ok::(result.unwrap_or_else(|e| http_error(&e))) @@ -664,13 +650,13 @@ fn build_router(state: &Arc) -> RouterService { let auction_handler = move |ctx: RequestContext| { let s = Arc::clone(&s); async move { - let services = s.services_for_request(&ctx); + let services = s.services_for_request(&ctx)?; // Request normalization (forwarded-header stripping, trusted // Host/scheme/client-IP derivation) is applied centrally by // `NormalizeMiddleware` before this handler runs, so the signed // OpenRTB metadata that auction signing derives from // `RequestInfo::from_request` uses the trusted runtime authority. - let mut req = ctx.into_request(); + let mut req = ctx.into_request()?; if let Err(error) = trusted_server_core::integrations::gpt_diagnostics::prepare_request( &s.settings, @@ -683,17 +669,19 @@ fn build_router(state: &Arc) -> RouterService { // the caller's jurisdiction — `EcContext::default()` fails it // closed for consented users. let mut ec_context = build_ec_context(&s.settings, &services, &req); - Ok(handle_auction( - &s.settings, - &s.orchestrator, - None, - None, - &mut ec_context, - &services, - req, + Ok::( + handle_auction( + &s.settings, + &s.orchestrator, + None, + None, + &mut ec_context, + &services, + req, + ) + .await + .unwrap_or_else(|e| http_error(&e)), ) - .await - .unwrap_or_else(|e| http_error(&e))) } }; @@ -702,8 +690,8 @@ fn build_router(state: &Arc) -> RouterService { let page_bids_handler = move |ctx: RequestContext| { let s = Arc::clone(&s); async move { - let services = s.services_for_request(&ctx); - let mut req = ctx.into_request(); + let services = s.services_for_request(&ctx)?; + let mut req = ctx.into_request()?; if let Err(error) = trusted_server_core::integrations::gpt_diagnostics::prepare_request( &s.settings, @@ -719,7 +707,7 @@ fn build_router(state: &Arc) -> RouterService { slots: s.settings.creative_opportunity_slots(), registry: None, }; - Ok( + Ok::( handle_page_bids(&s.settings, &services, None, auction, &mut ec_context, req) .await .unwrap_or_else(|e| http_error(&e)), @@ -738,11 +726,13 @@ fn build_router(state: &Arc) -> RouterService { let fp_proxy_handler = move |ctx: RequestContext| { let s = Arc::clone(&s); async move { - let services = s.services_for_request(&ctx); - let req = ctx.into_request(); - Ok(handle_first_party_proxy(&s.settings, &services, req) - .await - .unwrap_or_else(|e| http_error(&e))) + let services = s.services_for_request(&ctx)?; + let req = ctx.into_request()?; + Ok::( + handle_first_party_proxy(&s.settings, &services, req) + .await + .unwrap_or_else(|e| http_error(&e)), + ) } }; @@ -751,11 +741,13 @@ fn build_router(state: &Arc) -> RouterService { let fp_click_handler = move |ctx: RequestContext| { let s = Arc::clone(&s); async move { - let services = s.services_for_request(&ctx); - let req = ctx.into_request(); - Ok(handle_first_party_click(&s.settings, &services, req) - .await - .unwrap_or_else(|e| http_error(&e))) + let services = s.services_for_request(&ctx)?; + let req = ctx.into_request()?; + Ok::( + handle_first_party_click(&s.settings, &services, req) + .await + .unwrap_or_else(|e| http_error(&e)), + ) } }; @@ -764,11 +756,13 @@ fn build_router(state: &Arc) -> RouterService { let fp_sign_handler = move |ctx: RequestContext| { let s = Arc::clone(&s); async move { - let services = s.services_for_request(&ctx); - let req = ctx.into_request(); - Ok(handle_first_party_proxy_sign(&s.settings, &services, req) - .await - .unwrap_or_else(|e| http_error(&e))) + let services = s.services_for_request(&ctx)?; + let req = ctx.into_request()?; + Ok::( + handle_first_party_proxy_sign(&s.settings, &services, req) + .await + .unwrap_or_else(|e| http_error(&e)), + ) } }; let fp_sign_post_handler = fp_sign_handler.clone(); @@ -781,9 +775,9 @@ fn build_router(state: &Arc) -> RouterService { let fp_rebuild_handler = move |ctx: RequestContext| { let s = Arc::clone(&s); async move { - let services = s.services_for_request(&ctx); - let req = ctx.into_request(); - Ok( + let services = s.services_for_request(&ctx)?; + let req = ctx.into_request()?; + Ok::( handle_first_party_proxy_rebuild(&s.settings, &services, req) .await .unwrap_or_else(|e| http_error(&e)), @@ -797,8 +791,8 @@ fn build_router(state: &Arc) -> RouterService { state: Arc, ctx: RequestContext, ) -> Result { - let services = state.services_for_request(&ctx); - let mut req = ctx.into_request(); + let services = state.services_for_request(&ctx)?; + let mut req = ctx.into_request()?; if let Some(response) = deny_admin_diagnostic_fallback(&req) { return Ok(response); } @@ -843,6 +837,7 @@ fn build_router(state: &Arc) -> RouterService { slots: state.settings.creative_opportunity_slots(), registry: None, }; + let input_cap = publisher_input_cap(&state.settings, PUBLISHER_ORIGIN_INPUT_BYTES); match handle_publisher_request( &state.settings, &services, @@ -850,21 +845,25 @@ fn build_router(state: &Arc) -> RouterService { &mut ec_context, auction, req, - EdgeCacheHeader::SMaxageFallback, + PublisherOriginOptions { + edge_header: EdgeCacheHeader::SMaxageFallback, + input_cap: Some(input_cap), + response_decoding: ResponseDecoding::Original, + }, ) .await { - Ok(pr) => { - resolve_publisher_response( - pr, - &method, - &state.settings, - &state.registry, - &state.orchestrator, - &services, - ) - .await - } + Ok((pr, owner)) => resolve_publisher_response( + pr, + &method, + &state.settings, + &state.registry, + &state.orchestrator, + &services, + input_cap, + ) + .await + .and_then(|response| owner.attach(response).map_err(publisher_owner_error)), Err(e) => Err(e), } }; @@ -977,11 +976,11 @@ fn build_router(state: &Arc) -> RouterService { builder = builder.route("/{*rest}", method, fallback.clone()); } - builder.build() + builder } } -fn trace_metadata(state: &AppState, request: &Request) -> TraceMetadata { +fn trace_metadata(state: &AppState, request: &RequestParts) -> TraceMetadata { if let Some(services) = &state.services { let client_info = services.client_info().clone(); let geo = client_info.client_ip.and_then(|client_ip| { @@ -993,7 +992,7 @@ fn trace_metadata(state: &AppState, request: &Request) -> TraceMetadata { return TraceMetadata { client_info, geo }; } let client_ip = request - .headers() + .headers .get_all("spin-client-addr") .iter() .next_back() @@ -1012,6 +1011,14 @@ fn trace_metadata(state: &AppState, request: &Request) -> TraceMetadata { mod tests { use super::*; + // Runs the production normalizer on the head of a whole test request. + fn normalize_request(req: &mut Request) { + let request = std::mem::replace(req, Request::new(edgezero_core::body::Body::empty())); + let (mut parts, body) = request.into_parts(); + normalize_spin_request(&mut parts); + *req = Request::from_parts(parts, body); + } + fn multi_provider_settings() -> Settings { let mut settings = Settings::from_toml( r#" @@ -1095,14 +1102,14 @@ mod tests { } #[test] - fn startup_rejects_multi_provider_auction_plan() { - let error = match build_state_with_settings(multi_provider_settings()) { - Ok(_) => panic!("Spin startup should reject multi-provider fanout"), - Err(error) => error, - }; - assert!( - format!("{error:?}").contains("concurrent provider fanout"), - "should identify unsupported fanout: {error:?}" + fn cloudflare_and_spin_load_multi_provider_plan() { + let state = build_state_with_settings(multi_provider_settings()) + .expect("should load a multi-provider auction plan on Spin"); + + assert_eq!( + state.orchestrator.provider_count(), + 2, + "should fan out to both configured providers" ); } @@ -1169,7 +1176,7 @@ mod tests { ("spin-full-url", value), ]); - normalize_spin_request(&mut req); + normalize_request(&mut req); assert!( req.extensions().get::().is_none(), @@ -1210,7 +1217,7 @@ mod tests { ]); *req.method_mut() = Method::POST; *req.uri_mut() = "/_ts/trace/enable".parse().expect("should set action path"); - normalize_spin_request(&mut req); + normalize_request(&mut req); let response = RouterService::oneshot(&router, req) .await @@ -1246,11 +1253,13 @@ mod tests { edgezero_core::ingress::HeaderFidelity::default(), )); - normalize_spin_request(&mut req); + normalize_request(&mut req); + let (mut parts, body) = req.into_parts(); trusted_server_core::forwarder::prepare_trusted_forwarder( - &mut req, + &mut parts, &multi_provider_settings(), ); + let req = Request::from_parts(parts, body); let info = trusted_server_core::http_util::RequestInfo::from_request( &req, @@ -1291,7 +1300,7 @@ mod tests { ("forwarded", "host=evil.example;proto=http"), ]); - normalize_spin_request(&mut req); + normalize_request(&mut req); // Spoofable host overrides are stripped, leaving only the trusted Host. assert!( @@ -1343,7 +1352,7 @@ mod tests { ("spin-full-url", "https://www.publisher.example/cars/"), ]); - normalize_spin_request(&mut req); + normalize_request(&mut req); assert_eq!( req.headers() @@ -1392,7 +1401,7 @@ mod tests { }, ); - normalize_spin_request(&mut req); + normalize_request(&mut req); // All `spin-full-url` copies are stripped after normalization. assert!( @@ -1449,7 +1458,7 @@ mod tests { }, ); - normalize_spin_request(&mut req); + normalize_request(&mut req); assert_eq!( SpinRequestContext::get(&req).and_then(|c| c.client_addr), @@ -1474,7 +1483,7 @@ mod tests { ("x-forwarded-for", "198.51.100.99, 10.0.0.1"), ]); - normalize_spin_request(&mut req); + normalize_request(&mut req); assert!( req.headers().get("x-forwarded-for").is_none(), @@ -1496,74 +1505,21 @@ mod tests { ); } - #[tokio::test(flavor = "multi_thread", worker_threads = 2)] - async fn startup_error_router_serves_503_for_all_fallback_methods() { - // The degraded router must answer every publisher-fallback method - // (including HEAD/OPTIONS/PATCH) on both "/" and nested paths with the - // generic 503, never a router-level 405, so startup-failure behaviour - // stays consistent with the healthy router. - let report = Report::new(TrustedServerError::BadRequest { - message: "startup failure".to_string(), - }); - let router = startup_error_router(&report); - - for method in ["GET", "POST", "HEAD", "OPTIONS", "PUT", "PATCH", "DELETE"] { - for path in ["/", "/some/nested/page"] { - let req = edgezero_core::http::request_builder() - .method(method) - .uri(path) - .body(edgezero_core::body::Body::empty()) - .expect("should build request"); - let status = router - .oneshot(req) - .await - .expect("should route startup-error request") - .status() - .as_u16(); - assert_eq!( - status, 503, - "{method} {path} must return 503 from the startup fallback, got {status}" - ); - } - } - } - - #[tokio::test(flavor = "multi_thread", worker_threads = 2)] - async fn startup_error_router_answers_health_with_200() { - // The liveness probe must keep returning 200 even while application state - // construction is failing, matching the Fastly/Axum health behaviour. - let report = Report::new(TrustedServerError::BadRequest { - message: "startup failure".to_string(), - }); - let router = startup_error_router(&report); - - let req = edgezero_core::http::request_builder() - .method("GET") - .uri("/health") - .body(edgezero_core::body::Body::empty()) - .expect("should build request"); - let resp = router - .oneshot(req) - .await - .expect("should route startup-error health request"); + #[test] + fn routes_answer_startup_failure_with_generic_service_unavailable() { + let Err(error) = TrustedServerApp::routes() else { + panic!("should fail startup without the production config store"); + }; assert_eq!( - resp.status().as_u16(), - 200, - "GET /health must return 200 from the startup fallback" - ); - assert_eq!( - resp.headers() - .get("x-ts-version") - .and_then(|v| v.to_str().ok()), - trusted_server_core::constants::TS_GIT_VERSION, - "startup-fallback /health should report the compiled-in git version" + error.status(), + StatusCode::SERVICE_UNAVAILABLE, + "should answer 503 for every startup failure" ); - let body = resp.into_body().into_bytes().unwrap_or_default(); assert_eq!( - &body[..], - b"ok", - "startup-fallback health body should be `ok`" + error.message(), + "Service Unavailable", + "should not echo the startup report to callers" ); } } diff --git a/crates/trusted-server-adapter-spin/src/lib.rs b/crates/trusted-server-adapter-spin/src/lib.rs index adbf61c9d0..fd85ca8987 100644 --- a/crates/trusted-server-adapter-spin/src/lib.rs +++ b/crates/trusted-server-adapter-spin/src/lib.rs @@ -1,18 +1,21 @@ //! Fermyon Spin adapter for Trusted Server. +// Unit tests may buffer bodies with `Body::into_bytes`; production code must not. +#![cfg_attr(test, allow(clippy::disallowed_methods))] + pub mod app; pub mod middleware; pub mod platform; #[cfg(all(feature = "spin", target_arch = "wasm32"))] -use spin_sdk::http::{IntoResponse, Request}; +use spin_sdk::http::Request; #[cfg(all(feature = "spin", target_arch = "wasm32"))] use spin_sdk::http_service; #[cfg(all(feature = "spin", target_arch = "wasm32"))] #[http_service] // FORCED: edgezero_adapter_spin::run_app returns anyhow::Result — EdgeZero SDK constraint, not a project choice. -async fn handle(req: Request) -> anyhow::Result { +async fn handle(req: Request) -> anyhow::Result { #[cfg(feature = "test-utils")] install_fixture_logger(); edgezero_adapter_spin::run_app::(req).await diff --git a/crates/trusted-server-adapter-spin/src/middleware.rs b/crates/trusted-server-adapter-spin/src/middleware.rs index 4dc3cbee47..734973970e 100644 --- a/crates/trusted-server-adapter-spin/src/middleware.rs +++ b/crates/trusted-server-adapter-spin/src/middleware.rs @@ -7,6 +7,7 @@ use edgezero_core::http::{HeaderValue, Response}; use edgezero_core::middleware::{Middleware, Next}; use trusted_server_core::auth::enforce_basic_auth; use trusted_server_core::constants::HEADER_X_GEO_INFO_AVAILABLE; +use trusted_server_core::error::http_error; use trusted_server_core::http_util::sanitize_trusted_client_ip_headers; use trusted_server_core::settings::Settings; @@ -39,7 +40,7 @@ impl SanitizeRequestMiddleware { impl Middleware for SanitizeRequestMiddleware { async fn handle(&self, mut ctx: RequestContext, next: Next<'_>) -> Result { sanitize_trusted_client_ip_headers( - ctx.request_mut(), + ctx.parts_mut(), self.settings.trusted_client_ip.as_ref(), ); next.run(ctx).await @@ -106,12 +107,12 @@ impl AuthMiddleware { #[async_trait(?Send)] impl Middleware for AuthMiddleware { async fn handle(&self, mut ctx: RequestContext, next: Next<'_>) -> Result { - match enforce_basic_auth(&self.settings, ctx.request_mut()) { + match enforce_basic_auth(&self.settings, ctx.parts_mut()) { Ok(Some(response)) => return Ok(response), Ok(None) => {} Err(report) => { log::error!("auth check failed: {:?}", report); - return Ok(crate::app::http_error(&report)); + return Ok(http_error(&report)); } } @@ -149,7 +150,7 @@ impl NormalizeMiddleware { #[async_trait(?Send)] impl Middleware for NormalizeMiddleware { async fn handle(&self, mut ctx: RequestContext, next: Next<'_>) -> Result { - crate::app::normalize_spin_request(ctx.request_mut()); + crate::app::normalize_spin_request(ctx.parts_mut()); next.run(ctx).await } } @@ -176,6 +177,10 @@ pub(crate) fn apply_finalize_headers( trusted_server_core::version_header::apply_git_version_header(response); + // The finalize middleware runs once per request, so each skipped operator + // entry warns once. + trusted_server_core::response_privacy::warn_skipped_operator_response_headers(settings); + // Cookie-bearing responses stay private to shared caches and operator // headers cannot re-enable caching for uncacheable per-user payloads. trusted_server_core::response_privacy::apply_response_headers_with_cache_privacy( @@ -202,6 +207,7 @@ mod tests { use futures::executor::block_on; use trusted_server_core::redacted::Redacted; use trusted_server_core::settings::TrustedClientIpConfig; + use trusted_server_core::test_support::logging::WarningCapture; fn empty_response() -> Response { response_builder() @@ -249,6 +255,29 @@ mod tests { s } + #[test] + fn warns_once_about_a_skipped_operator_entry() { + let settings = settings_with_response_headers(vec![("connection", "keep-alive")]); + let mut response = empty_response(); + let capture = WarningCapture::start(); + + apply_finalize_headers(&settings, false, &mut response); + + let skipped = capture + .take() + .into_iter() + .filter(|warning| warning.message.contains("`connection`")) + .count(); + assert_eq!( + skipped, 1, + "should warn once about the skipped operator entry" + ); + assert!( + !response.headers().contains_key("connection"), + "should skip the operator entry" + ); + } + #[test] fn sets_geo_available_false_when_spin_has_no_geo() { let settings = settings_with_response_headers(vec![]); @@ -333,8 +362,8 @@ mod tests { let handler_observed = Arc::clone(&handler_observed); async move { *handler_observed.lock().expect("should lock observation") = Some(( - ctx.request().headers().contains_key("x-reader-ip"), - ctx.request().headers().contains_key("x-reader-ip-auth"), + ctx.headers().contains_key("x-reader-ip"), + ctx.headers().contains_key("x-reader-ip-auth"), )); Ok::(empty_response()) } diff --git a/crates/trusted-server-adapter-spin/src/platform.rs b/crates/trusted-server-adapter-spin/src/platform.rs index 53b6386373..032cc4d414 100644 --- a/crates/trusted-server-adapter-spin/src/platform.rs +++ b/crates/trusted-server-adapter-spin/src/platform.rs @@ -4,67 +4,15 @@ use std::time::Duration; use bytes::Bytes; use edgezero_core::config_store::ConfigStoreHandle; +use edgezero_core::error::EdgeError; use edgezero_core::key_value_store::{KvHandle, KvPage, KvStore}; -use error_stack::{Report, ResultExt as _}; -#[cfg(any(test, all(feature = "spin", target_arch = "wasm32")))] -use http_body_util::BodyExt; +use error_stack::Report; +use trusted_server_core::outbound::request_outbound_client; use trusted_server_core::platform::{ - BackendNamingPolicy, ClientInfo, GeoInfo, KvError, PlatformBackend, PlatformBackendSpec, - PlatformConfigStore, PlatformError, PlatformGeo, PlatformHttpClient, PlatformKvStore, + ClientInfo, GeoInfo, KvError, PlatformConfigStore, PlatformError, PlatformGeo, PlatformKvStore, PlatformSecretStore, RuntimeServices, StoreId, StoreName, UnavailableKvStore, }; -#[cfg(not(all(feature = "spin", target_arch = "wasm32")))] -use trusted_server_core::platform::UnavailableHttpClient; - -#[cfg(any(test, all(feature = "spin", target_arch = "wasm32")))] -use std::io::Read as _; -#[cfg(any(test, all(feature = "spin", target_arch = "wasm32")))] -use trusted_server_core::platform::PlatformHttpRequest; -#[cfg(all(feature = "spin", target_arch = "wasm32"))] -use trusted_server_core::platform::{ - PlatformPendingRequest, PlatformPendingResponse, PlatformResponse, PlatformSelectResult, -}; - -// Limit both streamed bytes and decompressed bytes to 8 MiB to preserve -// headroom in the Spin WASM component heap. The streaming limit rejects an -// oversized frame before retaining it in the collected response body. -#[cfg(any(test, all(feature = "spin", target_arch = "wasm32")))] -const MAX_DECOMPRESSED_SIZE: usize = 8 * 1024 * 1024; - -#[cfg(any(test, all(feature = "spin", target_arch = "wasm32")))] -type HeaderPairs = Vec<(String, Vec)>; -#[cfg(any(test, all(feature = "spin", target_arch = "wasm32")))] -type BufferedResponseParts = (HeaderPairs, Vec); - -/// Collect an outbound response body, rejecting it once it exceeds -/// [`MAX_DECOMPRESSED_SIZE`] streamed bytes. -/// -/// The limit applies frame by frame, so an oversized body fails before it is -/// fully retained in the component heap. -/// -/// # Errors -/// -/// Returns `PlatformError::HttpClient` when the body exceeds the limit or a -/// frame cannot be read. -#[cfg(any(test, all(feature = "spin", target_arch = "wasm32")))] -async fn collect_limited_body(body: B) -> Result, Report> -where - B: BodyExt, - B::Data: bytes::Buf, - B::Error: Into>, -{ - let collected = http_body_util::Limited::new(body, MAX_DECOMPRESSED_SIZE) - .collect() - .await - .map_err(|error| { - Report::new(PlatformError::HttpClient).attach(format!( - "failed to read Spin outbound response body: {error}" - )) - })?; - Ok(collected.to_bytes().to_vec()) -} - #[cfg(any(test, all(feature = "spin", target_arch = "wasm32")))] const SPIN_VARIABLE_HEX: &[u8; 16] = b"0123456789abcdef"; @@ -74,16 +22,17 @@ const SPIN_VARIABLE_HEX: &[u8; 16] = b"0123456789abcdef"; struct NoopConfigStore; +#[async_trait::async_trait(?Send)] impl PlatformConfigStore for NoopConfigStore { fn get(&self, _: &StoreName, _: &str) -> Result> { Err(Report::new(PlatformError::ConfigStore).attach("config store not available")) } - fn put(&self, _: &StoreId, _: &str, _: &str) -> Result<(), Report> { + async fn put(&self, _: &StoreId, _: &str, _: &str) -> Result<(), Report> { Err(Report::new(PlatformError::ConfigStore).attach("config store writes are not supported")) } - fn delete(&self, _: &StoreId, _: &str) -> Result<(), Report> { + async fn delete(&self, _: &StoreId, _: &str) -> Result<(), Report> { Err(Report::new(PlatformError::ConfigStore).attach("config store writes are not supported")) } } @@ -92,39 +41,21 @@ impl PlatformConfigStore for NoopConfigStore { struct NoopSecretStore; #[cfg(not(all(feature = "spin", target_arch = "wasm32")))] +#[async_trait::async_trait(?Send)] impl PlatformSecretStore for NoopSecretStore { fn get_bytes(&self, _: &StoreName, _: &str) -> Result, Report> { Err(Report::new(PlatformError::SecretStore).attach("secret store not available")) } - fn create(&self, _: &StoreId, _: &str, _: &str) -> Result<(), Report> { + async fn create(&self, _: &StoreId, _: &str, _: &str) -> Result<(), Report> { Err(Report::new(PlatformError::SecretStore).attach("secret store not available")) } - fn delete(&self, _: &StoreId, _: &str) -> Result<(), Report> { + async fn delete(&self, _: &StoreId, _: &str) -> Result<(), Report> { Err(Report::new(PlatformError::SecretStore).attach("secret store not available")) } } -struct NoopBackend; - -impl PlatformBackend for NoopBackend { - fn naming_policy(&self) -> BackendNamingPolicy { - BackendNamingPolicy::Spin - } - - fn predict_name(&self, spec: &PlatformBackendSpec) -> Result> { - self.naming_policy() - .predict(spec) - .map(|prediction| prediction.name) - .change_context(PlatformError::Backend) - } - - fn ensure(&self, spec: &PlatformBackendSpec) -> Result> { - self.predict_name(spec) - } -} - // --------------------------------------------------------------------------- // edgezero handle adapters — injected by edgezero_adapter_spin::run_app // --------------------------------------------------------------------------- @@ -136,6 +67,7 @@ impl PlatformBackend for NoopBackend { /// typed errors. pub(crate) struct ConfigStoreHandleAdapter(pub(crate) ConfigStoreHandle); +#[async_trait::async_trait(?Send)] impl PlatformConfigStore for ConfigStoreHandleAdapter { fn get(&self, _store_name: &StoreName, key: &str) -> Result> { futures::executor::block_on(self.0.get(key)) @@ -144,18 +76,19 @@ impl PlatformConfigStore for ConfigStoreHandleAdapter { "config store lookup failed for key `{key}`: {error}" )) })? + .map(|value| String::from(value.as_str())) .ok_or_else(|| { Report::new(PlatformError::ConfigStore) .attach(format!("key `{key}` not found in Spin config store")) }) } - fn put(&self, _: &StoreId, _: &str, _: &str) -> Result<(), Report> { + async fn put(&self, _: &StoreId, _: &str, _: &str) -> Result<(), Report> { Err(Report::new(PlatformError::ConfigStore) .attach("config store writes are not supported on Spin")) } - fn delete(&self, _: &StoreId, _: &str) -> Result<(), Report> { + async fn delete(&self, _: &StoreId, _: &str) -> Result<(), Report> { Err(Report::new(PlatformError::ConfigStore) .attach("config store writes are not supported on Spin")) } @@ -262,414 +195,6 @@ impl KvStore for KvHandleAdapter { } } -// --------------------------------------------------------------------------- -// Response policy helpers -// --------------------------------------------------------------------------- - -#[cfg(any(test, all(feature = "spin", target_arch = "wasm32")))] -fn sanitize_response_headers(headers: HeaderPairs) -> HeaderPairs { - let connection_tokens = connection_header_tokens(&headers); - - headers - .into_iter() - .filter(|(name, _)| !is_hop_by_hop_response_header(name, &connection_tokens)) - .collect() -} - -#[cfg(any(test, all(feature = "spin", target_arch = "wasm32")))] -fn connection_header_tokens(headers: &HeaderPairs) -> Vec { - headers - .iter() - .filter(|(name, _)| name.eq_ignore_ascii_case("connection")) - .filter_map(|(_, value)| std::str::from_utf8(value).ok()) - .flat_map(|value| value.split(',')) - .map(str::trim) - .filter(|token| !token.is_empty()) - .map(str::to_ascii_lowercase) - .collect() -} - -#[cfg(any(test, all(feature = "spin", target_arch = "wasm32")))] -fn is_hop_by_hop_response_header(name: &str, connection_tokens: &[String]) -> bool { - let lowercase_name = name.to_ascii_lowercase(); - matches!( - lowercase_name.as_str(), - "connection" - | "proxy-authenticate" - | "proxy-authorization" - | "te" - | "trailer" - | "transfer-encoding" - | "upgrade" - | "keep-alive" - ) || connection_tokens - .iter() - .any(|token| token == &lowercase_name) -} - -#[cfg(any(test, all(feature = "spin", target_arch = "wasm32")))] -fn content_encoding(headers: &HeaderPairs) -> Option { - headers - .iter() - .find(|(name, _)| name.eq_ignore_ascii_case("content-encoding")) - .and_then(|(_, value)| std::str::from_utf8(value).ok()) - .map(str::trim) - .map(str::to_ascii_lowercase) -} - -#[cfg(any(test, all(feature = "spin", target_arch = "wasm32")))] -fn reject_unsupported_request_contracts( - request: &PlatformHttpRequest, -) -> Result<(), Report> { - if request.image_optimizer.is_some() { - return Err(Report::new(PlatformError::HttpClient) - .attach("Spin outbound HTTP does not support Image Optimizer metadata")); - } - if request.stream_response { - return Err(Report::new(PlatformError::HttpClient) - .attach("Spin outbound HTTP does not support streaming responses")); - } - Ok(()) -} - -#[cfg(any(test, all(feature = "spin", target_arch = "wasm32")))] -fn response_must_not_have_body(method: &edgezero_core::http::Method, status: u16) -> bool { - method == edgezero_core::http::Method::HEAD - || (100..200).contains(&status) - || status == 204 - || status == 304 -} - -#[cfg(any(test, all(feature = "spin", target_arch = "wasm32")))] -fn apply_spin_response_policy( - method: &edgezero_core::http::Method, - status: u16, - headers: HeaderPairs, - body: Vec, -) -> Result> { - let mut headers = sanitize_response_headers(headers); - - // Bound the buffered body on every path, not just decompressed ones. An - // identity response or an unsupported encoding (e.g. zstd) is returned as-is, - // so without this check a large origin/ad-server/proxy response would be - // forwarded after being fully buffered instead of failing with a typed error. - // For gzip/br this also rejects an overlarge compressed body before - // decompression; `decompress_body` separately bounds the expanded size. - enforce_response_size(body.len())?; - - if response_must_not_have_body(method, status) { - return Ok((headers, body)); - } - - let Some(encoding) = content_encoding(&headers) else { - return Ok((headers, body)); - }; - - if !matches!(encoding.as_str(), "gzip" | "br") { - return Ok((headers, body)); - } - - let body = decompress_body(&body, &encoding, MAX_DECOMPRESSED_SIZE)?; - headers.retain(|(name, _)| { - !name.eq_ignore_ascii_case("content-encoding") - && !name.eq_ignore_ascii_case("content-length") - }); - Ok((headers, body)) -} - -#[cfg(any(test, all(feature = "spin", target_arch = "wasm32")))] -fn enforce_response_size(len: usize) -> Result<(), Report> { - if len > MAX_DECOMPRESSED_SIZE { - return Err(Report::new(PlatformError::HttpClient).attach(format!( - "response body exceeded maximum size of {MAX_DECOMPRESSED_SIZE} bytes" - ))); - } - Ok(()) -} - -#[cfg(any(test, all(feature = "spin", target_arch = "wasm32")))] -fn decompress_body( - body: &[u8], - encoding: &str, - max_size: usize, -) -> Result, Report> { - match encoding { - "gzip" => { - let decoder = flate2::read::GzDecoder::new(body); - read_decompressed(decoder, encoding, max_size) - } - "br" => { - let decoder = brotli::Decompressor::new(body, 8192); - read_decompressed(decoder, encoding, max_size) - } - _ => Ok(body.to_vec()), - } -} - -#[cfg(any(test, all(feature = "spin", target_arch = "wasm32")))] -fn read_decompressed( - reader: R, - encoding: &str, - max_size: usize, -) -> Result, Report> -where - R: std::io::Read, -{ - let mut decoded = Vec::new(); - reader - .take(max_size as u64 + 1) - .read_to_end(&mut decoded) - .map_err(|e| { - Report::new(PlatformError::HttpClient) - .attach(format!("{encoding} decompression failed: {e}")) - })?; - if decoded.len() > max_size { - return Err(Report::new(PlatformError::HttpClient).attach(format!( - "{encoding} decompression exceeded maximum size of {max_size} bytes" - ))); - } - Ok(decoded) -} - -// --------------------------------------------------------------------------- -// SpinPlatformHttpClient — WASM target + spin feature only -// --------------------------------------------------------------------------- - -/// Returns `true` for headers forbidden on WASI HTTP outbound requests. -/// -/// WASI HTTP (wasi:http@0.2) forbids `host` on outgoing requests — the -/// authority is conveyed separately via `OutgoingRequest::set_authority`. -/// Hop-by-hop headers (`connection`, `keep-alive`, `transfer-encoding`, -/// `upgrade`, `proxy-connection`) are HTTP/1.1 transport-layer concerns -/// that must not appear in WASI HTTP requests. -// Compiled for tests on native targets so unit tests can exercise the filter -// without requiring wasm32; the only production call site is inside the -// `#[cfg(all(feature = "spin", target_arch = "wasm32"))]` block below. -#[cfg(any(test, all(feature = "spin", target_arch = "wasm32")))] -fn is_wasi_forbidden_outbound_header(name: &str) -> bool { - name.eq_ignore_ascii_case("host") - || name.eq_ignore_ascii_case("connection") - || name.eq_ignore_ascii_case("keep-alive") - || name.eq_ignore_ascii_case("transfer-encoding") - || name.eq_ignore_ascii_case("upgrade") - || name.eq_ignore_ascii_case("proxy-connection") -} - -/// `spin_sdk::http::send`-backed HTTP client for the Spin runtime. -/// -/// Auction configuration supports one provider at a time, so -/// [`PlatformHttpClient::supports_concurrent_fanout`] reports `false`. -/// Requests start in a runtime task at dispatch, overlapping publisher-origin work. -/// Deadline selection bounds waiting for the full send and response body collection. -/// Dropping the pending operation cancels its WASI asynchronous operations. -/// -/// Spin's WASI HTTP API sends one request and returns the original response; it -/// exposes no redirect-follow policy. Each trait call therefore reaches exactly -/// one `spin_sdk::http::send` boundary and returns an original 3xx to core. Host -/// tests cannot instantiate Spin's WASI transport, so the common -/// `StubHttpClient` driver records the one-send 3xx behavior while adapter tests -/// cover request/response policy around that single boundary. -/// -/// Backend phase timeouts remain unsupported. Callers requiring an absolute -/// cutoff use [`PlatformHttpClient::select_until`]. -#[cfg(all(feature = "spin", target_arch = "wasm32"))] -pub struct SpinPlatformHttpClient; - -#[cfg(all(feature = "spin", target_arch = "wasm32"))] -impl SpinPlatformHttpClient { - async fn execute( - request: PlatformHttpRequest, - ) -> Result> { - reject_unsupported_request_contracts(&request)?; - - let method = request.request.method().clone(); - let uri = request.request.uri().to_string(); - - let mut builder = spin_sdk::http::Request::builder() - .method(into_spin_method(&method)) - .uri(uri.clone()); - - for (name, value) in request.request.headers() { - // WASI HTTP forbids these headers on outbound requests: - // `host` is conveyed via set_authority; hop-by-hop headers are HTTP/1.1 only. - if is_wasi_forbidden_outbound_header(name.as_str()) { - continue; - } - match value.to_str() { - Ok(value) => { - builder = builder.header(name.as_str(), value); - } - Err(_) => { - log::warn!( - "dropping non-UTF-8 outbound request header (Spin WASI limitation): {}", - name - ); - } - } - } - - let (_, body) = request.request.into_parts(); - let body_bytes = match body { - edgezero_core::body::Body::Once(bytes) => bytes.to_vec(), - edgezero_core::body::Body::Stream(_) => { - // TODO: streaming request bodies unsupported; large proxy POBs (e.g. - // /first-party/proxy-rebuild) will fail until Spin WASI HTTP buffering - // is added to this client. - return Err(Report::new(PlatformError::HttpClient) - .attach("streaming request bodies are not supported on Spin outbound HTTP")); - } - }; - let spin_request = builder - .body(spin_sdk::http::FullBody::new(Bytes::from(body_bytes))) - .map_err(|error| { - Report::new(PlatformError::HttpClient) - .attach(format!("failed to build Spin outbound request: {error}")) - })?; - - let spin_response: spin_sdk::http::Response = - spin_sdk::http::send(spin_request).await.map_err(|e| { - Report::new(PlatformError::HttpClient) - .attach(format!("outbound request to {uri} failed: {e}")) - })?; - - let status = spin_response.status().as_u16(); - let headers: HeaderPairs = spin_response - .headers() - .iter() - .map(|(name, value)| (name.to_string(), value.as_bytes().to_vec())) - .collect(); - let body = collect_limited_body(spin_response.into_body()).await?; - - let (headers, body) = apply_spin_response_policy(&method, status, headers, body)?; - let mut edge_builder = edgezero_core::http::response_builder().status(status); - for (name, value) in headers { - edge_builder = edge_builder.header(name.as_str(), value.as_slice()); - } - let edge_resp = edge_builder - .body(edgezero_core::body::Body::from(body)) - .change_context(PlatformError::HttpClient)?; - - Ok(PlatformResponse::new(edge_resp).with_backend_name(request.backend_name)) - } -} - -#[cfg(all(feature = "spin", target_arch = "wasm32"))] -#[async_trait::async_trait(?Send)] -impl PlatformHttpClient for SpinPlatformHttpClient { - fn supports_concurrent_fanout(&self) -> bool { - // Requests start concurrently at dispatch, but `select` and - // `select_until` collect only one response, so multi-provider fan-out - // stays unsupported. - false - } - - async fn send( - &self, - request: PlatformHttpRequest, - ) -> Result> { - Self::execute(request).await - } - - async fn send_async( - &self, - request: PlatformHttpRequest, - ) -> Result> { - let backend_name = request.backend_name.clone(); - let pending = - PlatformPendingResponse::start(Self::execute(request), spin_sdk::wasip3::spawn); - Ok(PlatformPendingRequest::new(pending).with_backend_name(backend_name)) - } - - async fn select( - &self, - mut pending_requests: Vec, - ) -> Result> { - if pending_requests.is_empty() { - return Err(Report::new(PlatformError::HttpClient) - .attach("select called with an empty pending_requests list")); - } - - if pending_requests.len() >= 2 { - return Err(Report::new(PlatformError::HttpClient).attach(format!( - "SpinPlatformHttpClient: multi-provider fan-out is not supported \ - ({} providers submitted). Configure a single auction provider \ - or use the Fastly adapter for parallel DSP fan-out.", - pending_requests.len() - ))); - } - - let ready_platform = pending_requests.remove(0); - let mut pending = ready_platform - .downcast::>>>() - .map_err(|_| { - Report::new(PlatformError::HttpClient) - .attach("unexpected inner type in SpinPlatformHttpClient::select") - })?; - - let ready = pending.wait().await.unwrap_or_else(|error| { - Err(Report::new(PlatformError::HttpClient) - .attach(format!("dispatched response task stopped: {error}"))) - }); - Ok(PlatformSelectResult { - ready, - remaining: pending_requests, - failed_backend_name: None, - }) - } - - async fn select_until( - &self, - mut pending_requests: Vec, - deadline: web_time::Instant, - ) -> Result, Report> { - if pending_requests.is_empty() { - return Err(Report::new(PlatformError::HttpClient) - .attach("select_until called with an empty pending_requests list")); - } - if pending_requests.len() > 1 { - return Err(Report::new(PlatformError::HttpClient) - .attach("multi-provider fan-out is not supported on this adapter")); - } - let platform_pending = pending_requests.remove(0); - let backend_name = platform_pending.backend_name().map(str::to_owned); - let mut pending = platform_pending - .downcast::>>>() - .map_err(|_| { - Report::new(PlatformError::HttpClient) - .attach("unexpected pending request type in deadline selection") - })?; - let result = pending - .complete_before_deadline( - async { - let remaining = deadline.saturating_duration_since(web_time::Instant::now()); - if !remaining.is_zero() { - spin_sdk::time::sleep(remaining).await; - } - }, - deadline, - web_time::Instant::now, - ) - .await; - Ok(result.map(|result| { - let ready = result.unwrap_or_else(|error| { - Err(Report::new(PlatformError::HttpClient) - .attach(format!("dispatched response task stopped: {error}"))) - }); - PlatformSelectResult { - failed_backend_name: ready.as_ref().err().and(backend_name), - ready, - remaining: pending_requests, - } - })) - } -} - -#[cfg(all(feature = "spin", target_arch = "wasm32"))] -fn into_spin_method(method: &edgezero_core::http::Method) -> spin_sdk::http::Method { - spin_sdk::http::Method::from_bytes(method.as_str().as_bytes()) - .expect("should convert valid HTTP method") -} - // --------------------------------------------------------------------------- // SpinSecretStoreAdapter — WASM target + spin feature only // --------------------------------------------------------------------------- @@ -691,6 +216,7 @@ fn into_spin_method(method: &edgezero_core::http::Method) -> spin_sdk::http::Met pub(crate) struct SpinSecretStoreAdapter; #[cfg(all(feature = "spin", target_arch = "wasm32"))] +#[async_trait::async_trait(?Send)] impl PlatformSecretStore for SpinSecretStoreAdapter { fn get_bytes( &self, @@ -706,12 +232,12 @@ impl PlatformSecretStore for SpinSecretStoreAdapter { } } - fn create(&self, _: &StoreId, _: &str, _: &str) -> Result<(), Report> { + async fn create(&self, _: &StoreId, _: &str, _: &str) -> Result<(), Report> { Err(Report::new(PlatformError::SecretStore) .attach("secret store writes are not supported on Spin")) } - fn delete(&self, _: &StoreId, _: &str) -> Result<(), Report> { + async fn delete(&self, _: &StoreId, _: &str) -> Result<(), Report> { Err(Report::new(PlatformError::SecretStore) .attach("secret store deletes are not supported on Spin")) } @@ -726,14 +252,19 @@ impl PlatformSecretStore for SpinSecretStoreAdapter { /// Config and KV are sourced from the `EdgeZero` handles that `run_app` injects /// before routing. Secrets are read synchronously from Spin component /// variables because Trusted Server's platform secret trait is sync. -#[must_use] -pub fn build_runtime_services(ctx: &edgezero_core::context::RequestContext) -> RuntimeServices { +/// +/// The outbound client is the one the request context carries, with the proxy +/// marker turned off, and the clock is the context's monotonic clock. +/// +/// # Errors +/// +/// Returns an internal [`EdgeError`] when the request carries no outbound +/// client. +pub fn build_runtime_services( + ctx: &edgezero_core::context::RequestContext, +) -> Result { let client_ip = extract_client_ip(ctx); - - #[cfg(all(feature = "spin", target_arch = "wasm32"))] - let http_client: Arc = Arc::new(SpinPlatformHttpClient); - #[cfg(not(all(feature = "spin", target_arch = "wasm32")))] - let http_client: Arc = Arc::new(UnavailableHttpClient); + let outbound_client = request_outbound_client(ctx)?; let config_store: Arc = ctx .config_store_default() @@ -754,8 +285,8 @@ pub fn build_runtime_services(ctx: &edgezero_core::context::RequestContext) -> R .config_store(config_store) .secret_store(secret_store) .kv_store(kv_store) - .backend(Arc::new(NoopBackend)) - .http_client(http_client) + .outbound_client(outbound_client) + .clock(ctx.monotonic_clock()) .geo(Arc::new(NullGeo)) .client_info(ClientInfo { client_ip, @@ -773,7 +304,7 @@ pub fn build_runtime_services(ctx: &edgezero_core::context::RequestContext) -> R std::sync::OnceLock::new(); #[cfg(all(feature = "spin", target_arch = "wasm32"))] let builder = builder.timer(Arc::clone(TIMER.get_or_init(|| Arc::new(SpinTimer)))); - builder.build() + Ok(builder.build()) } // --------------------------------------------------------------------------- @@ -802,7 +333,8 @@ impl PlatformGeo for NullGeo { /// Cloudflare adapters receive. Deployments fronting Spin with such a hop must /// account for this. fn extract_client_ip(ctx: &edgezero_core::context::RequestContext) -> Option { - edgezero_adapter_spin::context::SpinRequestContext::get(ctx.request()) + ctx.extensions() + .get::() .and_then(|c| c.client_addr) } @@ -823,27 +355,42 @@ mod tests { use super::*; use edgezero_core::body::Body; - use edgezero_core::config_store::{ConfigStore, ConfigStoreError}; - use trusted_server_core::platform::AuctionTargetId; - - #[test] - fn auction_http_capabilities_are_explicit() { - let capabilities = AuctionTargetId::Spin.descriptor().capabilities(); - assert!(!capabilities.supports_concurrent_provider_fanout()); - } + use edgezero_core::config_store::{ + BoundedStoreRead, ConfigStore, ConfigStoreError, ConfigValue, finish_bounded_config_read, + }; use edgezero_core::context::RequestContext; use edgezero_core::http::request_builder; use edgezero_core::params::PathParams; - use flate2::Compression; - use flate2::write::GzEncoder; - use std::io::Write as _; + use edgezero_core::time::{Deadline, MonotonicClock}; + use trusted_server_core::platform::HttpClient; + use trusted_server_core::test_support::scripted_client::ScriptedHttpClient; struct InMemoryConfigStore(std::collections::BTreeMap); #[async_trait::async_trait(?Send)] impl ConfigStore for InMemoryConfigStore { - async fn get(&self, key: &str) -> Result, ConfigStoreError> { - Ok(self.0.get(key).cloned()) + async fn get(&self, key: &str) -> Result, ConfigStoreError> { + Ok(self + .0 + .get(key) + .map(|value| ConfigValue::from(value.as_str()))) + } + + async fn get_bounded( + &self, + key: &str, + clock: &MonotonicClock, + deadline: Deadline, + max_backend_bytes: u64, + max_value_bytes: u64, + ) -> Result, ConfigStoreError> { + finish_bounded_config_read( + self.get(key).await, + clock, + deadline, + max_backend_bytes, + max_value_bytes, + ) } } @@ -856,64 +403,6 @@ mod tests { RequestContext::new(req, PathParams::default()) } - fn gzip_bytes(input: &[u8]) -> Vec { - let mut encoder = GzEncoder::new(Vec::new(), Compression::default()); - encoder - .write_all(input) - .expect("should write gzip test payload"); - encoder.finish().expect("should finish gzip test payload") - } - - fn brotli_bytes(input: &[u8]) -> Vec { - let mut compressed = Vec::new(); - { - let mut compressor = brotli::CompressorWriter::new(&mut compressed, 4096, 5, 21); - compressor - .write_all(input) - .expect("should write brotli test payload"); - } - compressed - } - - fn platform_request() -> PlatformHttpRequest { - let req = request_builder() - .method("GET") - .uri("https://origin.example/asset.png") - .body(Body::empty()) - .expect("should build platform HTTP request"); - PlatformHttpRequest::new(req, "origin") - } - - fn apply_get_response_policy( - headers: HeaderPairs, - body: Vec, - ) -> Result> { - apply_spin_response_policy(&edgezero_core::http::Method::GET, 200, headers, body) - } - - #[test] - fn response_policy_preserves_original_redirect_at_single_send_boundary() { - let (headers, body) = apply_spin_response_policy( - &edgezero_core::http::Method::GET, - 302, - vec![( - "location".to_string(), - b"https://redirect.example/next".to_vec(), - )], - Vec::new(), - ) - .expect("should preserve redirect response"); - - assert_eq!( - headers, - vec![( - "location".to_string(), - b"https://redirect.example/next".to_vec(), - )] - ); - assert!(body.is_empty()); - } - #[test] fn extract_client_ip_reads_spin_request_context() { let mut req = request_builder() @@ -1073,313 +562,82 @@ mod tests { ); } - #[tokio::test(flavor = "multi_thread", worker_threads = 2)] - async fn build_runtime_services_uses_noop_native_stores_without_handles() { - let ctx = make_ctx_without_spin_context(); - let services = build_runtime_services(&ctx); - - assert!( - services.client_info().client_ip.is_none(), - "should not set client IP without Spin request context" - ); - assert!( - services - .geo() - .lookup(None) - .expect("should not fail") - .is_none(), - "should use null geo" - ); - assert!( - services - .config_store() - .get(&StoreName::from("config"), "missing") - .is_err(), - "should return typed config error without injected handle" - ); - assert!( - services.kv_store().get_bytes("missing").await.is_err(), - "should return typed KV error without injected handle" - ); + /// Builds a context without a Spin request context whose request carries + /// `client`, as `EdgeZero`'s Spin request conversion installs its default + /// client. + fn make_ctx_with_outbound_client(client: HttpClient) -> RequestContext { + let mut req = request_builder() + .method("GET") + .uri("https://example.com/") + .body(Body::empty()) + .expect("should build request"); + req.extensions_mut().insert(client); + RequestContext::new(req, PathParams::default()) } #[test] - fn unsupported_request_contracts_reject_image_optimizer_metadata() { - use trusted_server_core::platform::{ - PlatformImageOptimizerOptions, PlatformImageOptimizerParams, - }; - - let request = platform_request().with_image_optimizer(PlatformImageOptimizerOptions::new( - "us_west", - PlatformImageOptimizerParams::default(), - )); - + fn build_runtime_services_carries_the_context_client_with_the_proxy_marker_off() { + let client = HttpClient::with_client(ScriptedHttpClient::forbidding()); assert!( - reject_unsupported_request_contracts(&request).is_err(), - "Spin must reject Image Optimizer metadata instead of silently dropping it" + client.proxy_marker_enabled(), + "should start from a client whose marker is on" ); - } - - #[test] - fn unsupported_request_contracts_reject_stream_response() { - let request = platform_request().with_stream_response(); + let ctx = make_ctx_with_outbound_client(client); - assert!( - reject_unsupported_request_contracts(&request).is_err(), - "Spin must reject streaming-response requests instead of buffering silently" - ); - } + let services = build_runtime_services(&ctx).expect("should build runtime services"); - #[test] - fn response_policy_strips_transfer_encoding() { - let (headers, body) = apply_get_response_policy( - vec![ - ("transfer-encoding".to_string(), b"chunked".to_vec()), - ("x-preserve".to_string(), b"yes".to_vec()), - ], - b"ok".to_vec(), - ) - .expect("should apply response policy"); - - assert_eq!(body, b"ok", "should preserve body"); - assert!( - headers.iter().all(|(name, _)| name != "transfer-encoding"), - "should strip transfer-encoding" - ); assert!( - headers.iter().any(|(name, _)| name == "x-preserve"), - "should preserve ordinary headers" + !services.outbound_client().proxy_marker_enabled(), + "should turn the proxy marker off on the context client" ); } #[test] - fn response_policy_strips_headers_named_by_connection() { - let (headers, _) = apply_get_response_policy( - vec![ - ("connection".to_string(), b"keep-alive, x-remove".to_vec()), - ("x-remove".to_string(), b"drop".to_vec()), - ("x-keep".to_string(), b"keep".to_vec()), - ], - b"ok".to_vec(), - ) - .expect("should apply response policy"); + fn build_runtime_services_rejects_a_request_without_an_outbound_client() { + let ctx = make_ctx_without_spin_context(); - assert!( - headers.iter().all(|(name, _)| name != "connection"), - "should strip connection" - ); - assert!( - headers.iter().all(|(name, _)| name != "x-remove"), - "should strip headers named by connection" - ); - assert!( - headers.iter().any(|(name, _)| name == "x-keep"), - "should preserve ordinary response headers" - ); - } + let error = build_runtime_services(&ctx) + .expect_err("should reject a request with no outbound client"); - #[test] - fn response_policy_decodes_gzip_and_strips_encoding_headers() { - let (headers, body) = apply_get_response_policy( - vec![ - ("content-encoding".to_string(), b"gzip".to_vec()), - ("content-length".to_string(), b"999".to_vec()), - ("content-type".to_string(), b"text/plain".to_vec()), - ], - gzip_bytes(b"hello gzip"), - ) - .expect("should decode gzip body"); - - assert_eq!(body, b"hello gzip", "should decode gzip payload"); - assert!( - headers.iter().all(|(name, _)| { - !name.eq_ignore_ascii_case("content-encoding") - && !name.eq_ignore_ascii_case("content-length") - }), - "should strip content-encoding and content-length after decode" - ); - assert!( - headers - .iter() - .any(|(name, value)| name == "content-type" && value == b"text/plain"), - "should preserve ordinary headers" + assert_eq!( + error.status(), + edgezero_core::http::StatusCode::INTERNAL_SERVER_ERROR, + "should answer a missing client with an internal error" ); - } - - #[test] - fn response_policy_decodes_brotli_and_strips_encoding_headers() { - let (headers, body) = apply_get_response_policy( - vec![ - ("content-encoding".to_string(), b"br".to_vec()), - ("content-length".to_string(), b"999".to_vec()), - ], - brotli_bytes(b"hello brotli"), - ) - .expect("should decode brotli body"); - - assert_eq!(body, b"hello brotli", "should decode brotli payload"); - assert!( - headers.iter().all(|(name, _)| { - !name.eq_ignore_ascii_case("content-encoding") - && !name.eq_ignore_ascii_case("content-length") - }), - "should strip content-encoding and content-length after decode" + assert_eq!( + error.to_string(), + "internal error: request has no outbound client", + "should name the missing client" ); } - #[test] - fn response_policy_preserves_unsupported_encoding() { - let (headers, body) = apply_get_response_policy( - vec![("content-encoding".to_string(), b"zstd".to_vec())], - b"raw body".to_vec(), - ) - .expect("should preserve unsupported encoding"); - - assert_eq!(body, b"raw body", "should preserve raw body bytes"); - assert!( - headers.iter().any(|(name, value)| { - name.eq_ignore_ascii_case("content-encoding") && value == b"zstd" - }), - "should preserve unsupported encoding header" - ); - } + #[tokio::test(flavor = "multi_thread", worker_threads = 2)] + async fn build_runtime_services_uses_noop_native_stores_without_handles() { + let ctx = make_ctx_with_outbound_client(ScriptedHttpClient::forbidding().http_client()); + let services = build_runtime_services(&ctx).expect("should build runtime services"); - #[test] - fn response_policy_skips_decoding_for_head_responses() { - let (headers, body) = apply_spin_response_policy( - &edgezero_core::http::Method::HEAD, - 200, - vec![ - ("content-encoding".to_string(), b"gzip".to_vec()), - ("content-length".to_string(), b"999".to_vec()), - ("transfer-encoding".to_string(), b"chunked".to_vec()), - ], - Vec::new(), - ) - .expect("should apply HEAD response policy without decoding"); - - assert!(body.is_empty(), "should preserve the empty HEAD body"); - assert!( - headers.iter().any(|(name, value)| { - name.eq_ignore_ascii_case("content-encoding") && value.as_slice() == b"gzip" - }), - "should preserve content-encoding on HEAD responses" - ); - assert!( - headers.iter().any(|(name, value)| { - name.eq_ignore_ascii_case("content-length") && value.as_slice() == b"999" - }), - "should preserve content-length on HEAD responses" - ); assert!( - headers - .iter() - .all(|(name, _)| !name.eq_ignore_ascii_case("transfer-encoding")), - "should still strip hop-by-hop headers on HEAD responses" - ); - } - - #[test] - fn response_policy_skips_decoding_for_no_body_statuses() { - for status in [100, 101, 204, 304] { - let (headers, body) = apply_spin_response_policy( - &edgezero_core::http::Method::GET, - status, - vec![ - ("content-encoding".to_string(), b"gzip".to_vec()), - ("content-length".to_string(), b"999".to_vec()), - ("connection".to_string(), b"close".to_vec()), - ], - Vec::new(), - ) - .expect("should apply no-body response policy without decoding"); - - assert!( - body.is_empty(), - "status {status} should preserve the empty body" - ); - assert!( - headers.iter().any(|(name, value)| { - name.eq_ignore_ascii_case("content-encoding") && value.as_slice() == b"gzip" - }), - "status {status} should preserve content-encoding" - ); - assert!( - headers.iter().any(|(name, value)| { - name.eq_ignore_ascii_case("content-length") && value.as_slice() == b"999" - }), - "status {status} should preserve content-length" - ); - assert!( - headers - .iter() - .all(|(name, _)| !name.eq_ignore_ascii_case("connection")), - "status {status} should still strip hop-by-hop headers" - ); - } - } - - #[test] - fn response_policy_reports_gzip_decode_failure() { - let result = apply_get_response_policy( - vec![("content-encoding".to_string(), b"gzip".to_vec())], - b"not gzip".to_vec(), + services.client_info().client_ip.is_none(), + "should not set client IP without Spin request context" ); - - assert!(result.is_err(), "should reject invalid gzip payload"); - } - - #[test] - fn response_policy_rejects_oversized_identity_body() { - let oversized = vec![0u8; MAX_DECOMPRESSED_SIZE + 1]; - let result = apply_get_response_policy(Vec::new(), oversized); - assert!( - result.is_err(), - "should reject an identity response larger than the size ceiling" - ); - } - - #[test] - fn response_policy_rejects_oversized_unsupported_encoding_body() { - let oversized = vec![0u8; MAX_DECOMPRESSED_SIZE + 1]; - let result = apply_get_response_policy( - vec![("content-encoding".to_string(), b"zstd".to_vec())], - oversized, + services + .geo() + .lookup(None) + .expect("should not fail") + .is_none(), + "should use null geo" ); - assert!( - result.is_err(), - "should reject an unsupported-encoding response larger than the size ceiling" - ); - } - - #[test] - fn response_policy_allows_body_at_size_ceiling() { - let at_limit = vec![0u8; MAX_DECOMPRESSED_SIZE]; - let (_, body) = apply_get_response_policy(Vec::new(), at_limit) - .expect("should accept an identity body exactly at the ceiling"); - - assert_eq!( - body.len(), - MAX_DECOMPRESSED_SIZE, - "should pass through a body at the ceiling unchanged" + services + .config_store() + .get(&StoreName::from("config"), "missing") + .is_err(), + "should return typed config error without injected handle" ); - } - - #[test] - fn decompression_limit_is_enforced() { - let compressed = gzip_bytes(b"expanded body"); - let result = decompress_body(&compressed, "gzip", 4); - assert!( - result.is_err(), - "should reject decoded bodies larger than the configured limit" - ); - assert_eq!( - MAX_DECOMPRESSED_SIZE, - 8 * 1024 * 1024, - "production limit must stay within Spin WASM heap budget" + services.kv_store().get_bytes("missing").await.is_err(), + "should return typed KV error without injected handle" ); } @@ -1402,67 +660,4 @@ mod tests { "should reject uppercase-leading key at the encoder boundary" ); } - - #[test] - fn wasi_forbidden_headers_are_identified() { - for header in &[ - "host", - "Host", - "HOST", - "connection", - "keep-alive", - "transfer-encoding", - "upgrade", - "proxy-connection", - ] { - assert!( - is_wasi_forbidden_outbound_header(header), - "{header} should be forbidden in WASI HTTP outbound requests" - ); - } - for header in &[ - "user-agent", - "accept", - "content-type", - "x-custom", - "authorization", - ] { - assert!( - !is_wasi_forbidden_outbound_header(header), - "{header} should be allowed in WASI HTTP outbound requests" - ); - } - } -} - -#[cfg(test)] -mod deadline_tests { - use futures::executor::block_on; - - #[test] - fn spin_body_collection_rejects_oversized_stream() { - let body = http_body_util::Full::new(bytes::Bytes::from(vec![ - 0; - super::MAX_DECOMPRESSED_SIZE - + 1 - ])); - let result = block_on(super::collect_limited_body(body)); - assert!( - result.is_err(), - "should reject a response exceeding the streamed body cap" - ); - } - - #[test] - fn spin_body_collection_accepts_body_at_limit() { - let body = - http_body_util::Full::new(bytes::Bytes::from(vec![0; super::MAX_DECOMPRESSED_SIZE])); - let result = block_on(super::collect_limited_body(body)) - .expect("should collect a body at the limit"); - assert_eq!( - result.len(), - super::MAX_DECOMPRESSED_SIZE, - "should retain the complete permitted body" - ); - } } diff --git a/crates/trusted-server-adapter-spin/tests/routes.rs b/crates/trusted-server-adapter-spin/tests/routes.rs index 6fa45aa6fa..19adb598a5 100644 --- a/crates/trusted-server-adapter-spin/tests/routes.rs +++ b/crates/trusted-server-adapter-spin/tests/routes.rs @@ -6,11 +6,25 @@ //! that depends on external stores assert routing only; deterministic auth and //! method gates assert exact status codes. -use edgezero_core::app::Hooks as _; -use edgezero_core::http::{Request, Response, request_builder}; +// Test crates may buffer bodies with `Body::into_bytes`. +#![allow(clippy::disallowed_methods)] + +use edgezero_adapter_spin::SPIN_PLATFORM; +use edgezero_core::app::{App, Hooks as _}; +use edgezero_core::body::Body; +use edgezero_core::http::{Method, Request, Response, StatusCode, header, request_builder}; +use edgezero_core::ingress::{IngressDispatchOutcome, IngressFraming, IngressHeadAccounting}; +use edgezero_core::response_egress_framing::ResponseEgressFramingMode; use edgezero_core::router::RouterService; use trusted_server_adapter_spin::app::TrustedServerApp; +use trusted_server_core::constants::TS_GIT_VERSION; +use trusted_server_core::platform::HttpClient; use trusted_server_core::settings::Settings; +use trusted_server_core::test_support::insert_outbound_client; +use trusted_server_core::test_support::logging::WarningCapture; +use trusted_server_core::test_support::scripted_client::{ + ScriptedHttpClient, ScriptedReply, TestClock, +}; /// Build the full application router from explicit test settings. /// @@ -21,7 +35,13 @@ use trusted_server_core::settings::Settings; /// `Settings::ADMIN_ENDPOINTS` and the default config, so the canonical /// `/_ts/admin/keys/*` routes are auth-gated exactly as in production. fn test_router() -> RouterService { - let settings = Settings::from_toml( + TrustedServerApp::routes_with_settings(test_settings()) + .expect("should build router from test settings") +} + +/// The settings [`test_router`] builds its router from. +fn test_settings() -> Settings { + Settings::from_toml( r#" [[handlers]] path = "^/_ts/admin" @@ -38,23 +58,125 @@ fn test_router() -> RouterService { passphrase = "test-secret-key-32-bytes-minimum" "#, ) - .expect("should parse route test settings"); - - TrustedServerApp::routes_with_settings(settings) - .expect("should build router from test settings") + .expect("should parse route test settings") } -async fn route(router: RouterService, req: Request) -> Response { +async fn route(router: RouterService, mut req: Request) -> Response { + // `oneshot` skips `EdgeZero`'s request conversion, which installs the + // adapter's default client, so install one that fails every request. + insert_outbound_client( + &mut req, + HttpClient::with_client(ScriptedHttpClient::forbidding()), + ); router.oneshot(req).await.expect("should route request") } #[test] fn routes_build_without_panic() { - // build_state() may fail (no real settings in CI) — startup_error_router - // is the fallback. Either way, routes() must not panic. + // Host targets have no production config store, so `routes()` returns + // the startup error instead of panicking. let _router = TrustedServerApp::routes(); } +// --------------------------------------------------------------------------- +// Synthesized headers on EdgeZero's canonical responses +// --------------------------------------------------------------------------- + +/// Dispatches one request through the app `run_app` assembles, so the +/// policies `TrustedServerApp::configure` installs apply, and returns the +/// response head as egress would write it. +async fn dispatch_through_app(method: Method, target: &str) -> Response { + let app = App::from_router::(test_router(), SPIN_PLATFORM) + .expect("should assemble the app with its edge policies"); + let request = request_builder() + .method(method) + .uri(target) + .body(Body::empty()) + .expect("should build request"); + let outcome = app + .dispatch_ingress( + request, + app.monotonic_now(), + IngressHeadAccounting::HostManaged, + IngressFraming::HostManaged, + ) + .await + .expect("should admit the request"); + let envelope = match outcome { + IngressDispatchOutcome::Response(envelope) => Some(envelope), + _ => None, + } + .expect("should answer with a response"); + let (prepared, _policy, mut attempt, clock) = envelope + .begin(ResponseEgressFramingMode::ExplicitBufferedLength) + .map_err(|_failure| "response egress failed before conversion") + .expect("should begin response egress"); + assert!(attempt.begin_writing(), "should start writing the response"); + assert!( + attempt.complete(clock.now()), + "should complete the response" + ); + prepared.into_response() +} + +/// Returns the git version compiled into this build. +/// +/// # Panics +/// +/// Panics when the build has none, so a version assertion cannot pass on an +/// absent header. +fn compiled_git_version() -> &'static str { + TS_GIT_VERSION.expect("should compile a git version into the test build") +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn canonical_not_found_carries_x_ts_version() { + // The publisher fallback claims every origin-form path, so only an + // asterisk-form target has no route and gets EdgeZero's canonical 404. + let response = dispatch_through_app(Method::OPTIONS, "*").await; + + assert_eq!( + response.status(), + StatusCode::NOT_FOUND, + "should answer an unrouted target with the canonical 404" + ); + assert_eq!( + response + .headers() + .get("x-ts-version") + .and_then(|value| value.to_str().ok()), + Some(compiled_git_version()), + "should add the compiled-in git version to the canonical 404" + ); +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn canonical_method_not_allowed_carries_allow_and_x_ts_version() { + let response = dispatch_through_app(Method::TRACE, "/auction").await; + + assert_eq!( + response.status(), + StatusCode::METHOD_NOT_ALLOWED, + "should answer an unrouted method on a known path with the canonical 405" + ); + assert_eq!( + response + .headers() + .get(header::ALLOW) + .and_then(|value| value.to_str().ok()), + Some("DELETE, GET, HEAD, OPTIONS, PATCH, POST, PUT"), + "should list the path's methods in sorted order" + ); + assert_eq!( + response + .headers() + .get("x-ts-version") + .and_then(|value| value.to_str().ok()), + Some(compiled_git_version()), + "should add the compiled-in git version to the canonical 405" + ); +} + #[test] fn edgezero_manifest_loads_and_resolves_spin_stores() { let loader = edgezero_core::manifest::ManifestLoader::load_from_str(include_str!( @@ -988,14 +1110,12 @@ async fn admin_deactivate_key_auth_fail_returns_401() { /// Fastly adapter alone and not by this test. #[tokio::test(flavor = "multi_thread", worker_threads = 2)] async fn nextjs_auction_output_holds_until_the_structural_body_close() { - use std::sync::Arc; - use trusted_server_core::test_support::nextjs_auction; - let client = Arc::new(nextjs_auction::NextJsAuctionOrigin::default()); + let client = nextjs_auction::NextJsAuctionOrigin::default(); let router = TrustedServerApp::routes_with_settings_and_services( nextjs_auction::settings(), - nextjs_auction::services(Arc::clone(&client)), + nextjs_auction::services(&client), ) .expect("should build router with fixture services"); @@ -1048,3 +1168,133 @@ async fn nextjs_auction_output_holds_until_the_structural_body_close() { "should not leak generated placeholders: {html}" ); } + +// --------------------------------------------------------------------------- +// Publisher origin through EdgeZero's client +// --------------------------------------------------------------------------- + +/// Origin of [`test_settings`]. +const ORIGIN: &str = "https://origin.test-publisher.example.com"; + +/// Spin's origin input cap with the default 16 MiB `max_buffered_body_bytes`. +const SPIN_INPUT_CAP: usize = 8 * 1024 * 1024; + +/// Sends `req` through `router` with `origin` as the request's outbound client. +async fn route_to_origin( + router: RouterService, + mut req: Request, + origin: &ScriptedHttpClient, +) -> Response { + insert_outbound_client(&mut req, HttpClient::with_client(origin.clone())); + router.oneshot(req).await.expect("should route request") +} + +/// A publisher GET for `path` on the test publisher host. +fn publisher_get(path: &str) -> Request { + request_builder() + .method(Method::GET) + .uri(format!("https://test-publisher.example.com{path}")) + .header(header::HOST, "test-publisher.example.com") + .body(Body::empty()) + .expect("should build publisher request") +} + +/// An identity HTML document exactly `len` bytes long. +fn html_document(len: usize) -> Vec { + let mut document = b"".to_vec(); + document.resize(len - b"".len(), b'a'); + document.extend_from_slice(b""); + document +} + +/// Asserts that an origin body of [`SPIN_INPUT_CAP`] bytes is served and one +/// byte more fails with 502 before egress. +async fn assert_spin_input_cap(content_type: &str, body: fn(usize) -> Vec) { + let origin = ScriptedHttpClient::new(&TestClock::new()); + for (path, len) in [ + ("/at-cap", SPIN_INPUT_CAP), + ("/over-cap", SPIN_INPUT_CAP + 1), + ] { + origin.respond_to( + Method::GET, + &format!("{ORIGIN}{path}"), + ScriptedReply::status(StatusCode::OK) + .header("content-type", content_type) + .body(body(len)), + ); + } + + let at_cap = route_to_origin(test_router(), publisher_get("/at-cap"), &origin).await; + assert_eq!( + at_cap.status(), + StatusCode::OK, + "should serve a {content_type} body at the cap" + ); + let served = at_cap + .into_body() + .into_bytes() + .expect("should buffer the at-cap body"); + assert!( + served.len() >= SPIN_INPUT_CAP, + "should serve the whole {content_type} body at the cap, got {} bytes", + served.len() + ); + + let over_cap = route_to_origin(test_router(), publisher_get("/over-cap"), &origin).await; + assert_eq!( + over_cap.status(), + StatusCode::BAD_GATEWAY, + "should fail a {content_type} body one byte over the cap with 502" + ); +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn raw_body_at_cap_passes_and_one_over_fails_502_spin() { + assert_spin_input_cap("application/octet-stream", |len| vec![b'x'; len]).await; +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn identity_html_at_cap_passes_and_one_over_fails_502_spin() { + assert_spin_input_cap("text/html", html_document).await; +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn spin_ignores_host_override_with_warning() { + let mut settings = test_settings(); + settings.publisher.origin_host_header_override = + Some("www.test-publisher.example.com".to_owned()); + let capture = WarningCapture::start(); + let router = TrustedServerApp::routes_with_settings(settings) + .expect("should keep serving a config that sets the override"); + assert!( + capture.take().iter().any(|warning| { + warning.message.contains("origin_host_header_override") + && warning.message.contains("ignored on Spin") + }), + "should warn at settings load that Spin ignores the override" + ); + let origin = ScriptedHttpClient::new(&TestClock::new()); + origin.respond_to( + Method::GET, + &format!("{ORIGIN}/page"), + ScriptedReply::status(StatusCode::OK).header("content-type", "application/octet-stream"), + ); + + let response = route_to_origin(router, publisher_get("/page"), &origin).await; + + assert_eq!( + response.status(), + StatusCode::OK, + "should serve from the origin" + ); + let requests = origin.requests(); + assert_eq!(requests.len(), 1, "should send one origin request"); + assert_eq!( + ( + requests[0].host_authority.as_str(), + requests[0].has_host_authority_override + ), + ("origin.test-publisher.example.com", false), + "should send the URI authority with no override" + ); +} diff --git a/crates/trusted-server-adapter-spin/tests/runtime/deadline.rs b/crates/trusted-server-adapter-spin/tests/runtime/deadline.rs index 95cd0013da..54b09bfe7c 100644 --- a/crates/trusted-server-adapter-spin/tests/runtime/deadline.rs +++ b/crates/trusted-server-adapter-spin/tests/runtime/deadline.rs @@ -1,14 +1,31 @@ -//! Local runtime fixture exercising the production HTTP client implementation. +//! Local runtime fixture exercising `EdgeZero`'s Spin HTTP client. +//! +//! `scripts/test-worker-auction-deadlines.mjs` calls it with a delayed +//! upstream. It starts one bidder-shaped request in a batch, sleeps for the +//! simulated publisher work, then collects the batch at the auction cutoff. #[cfg(all(feature = "spin", target_arch = "wasm32"))] -#[allow(dead_code)] -#[path = "../../src/platform.rs"] -mod platform; +use std::time::Duration; #[cfg(all(feature = "spin", target_arch = "wasm32"))] -use std::time::Duration; +use edgezero_adapter_spin::outbound::SpinOutboundClient; +#[cfg(all(feature = "spin", target_arch = "wasm32"))] +use edgezero_core::error::EdgeError; +#[cfg(all(feature = "spin", target_arch = "wasm32"))] +use edgezero_core::http::Method; +#[cfg(all(feature = "spin", target_arch = "wasm32"))] +use edgezero_core::outbound::{ + HttpClient, OutboundBatchTermination, OutboundCachePolicy, OutboundCollectedSlot, + ResponseDecoding, +}; +#[cfg(all(feature = "spin", target_arch = "wasm32"))] +use edgezero_core::time::Deadline; #[cfg(all(feature = "spin", target_arch = "wasm32"))] -use trusted_server_core::platform::{PlatformHttpClient as _, PlatformHttpRequest}; +use trusted_server_core::outbound::{BUFFERED_TIMEOUT, buffered, outbound_request}; + +/// Response cap the script's oversized upstream exceeds by one byte. +#[cfg(all(feature = "spin", target_arch = "wasm32"))] +const RESPONSE_CAP_BYTES: u64 = 8 * 1024 * 1024; #[cfg(all(feature = "spin", target_arch = "wasm32"))] #[spin_sdk::http_service] @@ -20,9 +37,7 @@ async fn handle(request: spin_sdk::http::Request) -> Result Result> { +async fn exercise(uri: &str) -> Result { let parameter = |name: &str| { uri.split_once('?').and_then(|(_, query)| { query @@ -41,40 +56,44 @@ async fn exercise( .expect("should supply origin delay") .parse() .expect("should parse origin delay"); - let client = platform::SpinPlatformHttpClient; - let request = edgezero_core::http::request_builder() - .uri(upstream) - .body(edgezero_core::body::Body::empty()) - .expect("should construct fixture request"); + let client = HttpClient::with_client(SpinOutboundClient::new()); + let request = buffered( + outbound_request(Method::GET, upstream)?, + BUFFERED_TIMEOUT, + RESPONSE_CAP_BYTES, + ResponseDecoding::Decode, + ) + .cache_policy(OutboundCachePolicy::Bypass); let started = web_time::Instant::now(); - let pending = client - .send_async(PlatformHttpRequest::new(request, "fixture")) - .await?; + let batch = client.start_batch_until( + vec![request], + Deadline::after(Duration::from_millis(budget_ms)), + ); spin_sdk::time::sleep(Duration::from_millis(origin_ms)).await; - let collected = client - .select_until(vec![pending], started + Duration::from_millis(budget_ms)) - .await?; - let status = match collected { - Some(result) => match result.ready { + let (slots, termination) = batch + .collect() + .await + .map_err(|failure| failure.into_parts().0)? + .into_parts(); + let status = match (slots.into_iter().next(), termination) { + (Some(OutboundCollectedSlot::Resolved(result)), _) => match result.outcome { Ok(response) => { assert_eq!( - response.response.body().as_bytes(), + response.body().as_bytes(), Some(b"okay".as_slice()), "should retain the complete nonempty upstream body" ); "complete" } - Err(error) if format!("{error:?}").contains("length limit exceeded") => { - assert_eq!( - result.failed_backend_name.as_deref(), - Some("fixture"), - "should attribute oversized response to its provider" - ); - "oversized" - } + Err(EdgeError::ResponseTooLarge { .. }) => "oversized", Err(error) => return Err(error), }, - None => "timeout", + (Some(OutboundCollectedSlot::Unresolved), OutboundBatchTermination::Cutoff) => "timeout", + (slot, termination) => { + return Err(EdgeError::internal(std::io::Error::other(format!( + "unexpected batch outcome {slot:?} with {termination:?}" + )))); + } }; Ok(format!("{status}:{}", started.elapsed().as_millis())) } diff --git a/crates/trusted-server-cli/Cargo.toml b/crates/trusted-server-cli/Cargo.toml index 4d6af83fd5..3fb4303960 100644 --- a/crates/trusted-server-cli/Cargo.toml +++ b/crates/trusted-server-cli/Cargo.toml @@ -112,7 +112,7 @@ x509-parser = { workspace = true } [target.'cfg(not(target_arch = "wasm32"))'.dev-dependencies] assert_cmd = { workspace = true } -edgezero-adapter-axum = { workspace = true, features = ["axum"] } +edgezero-adapter-axum = { workspace = true, features = ["axum", "test-utils"] } flate2 = { workspace = true } predicates = { workspace = true } temp-env = { workspace = true } diff --git a/crates/trusted-server-cli/tests/proxy_trusted_server.rs b/crates/trusted-server-cli/tests/proxy_trusted_server.rs index a9dcccc15d..13d25a826e 100644 --- a/crates/trusted-server-cli/tests/proxy_trusted_server.rs +++ b/crates/trusted-server-cli/tests/proxy_trusted_server.rs @@ -1,21 +1,24 @@ //! Exercises the proxy against the production Axum listener and trace router. //! -//! A child test process owns the real HTTP server. A TLS relay additionally -//! exercises the proxy's encrypted upstream transport without inventing ingress -//! metadata or replacing the server's trace authorization with an echo handler. +//! `EdgeZero`'s `AxumTestServer` serves the real Trusted Server app over +//! loopback TCP. A TLS relay additionally exercises the proxy's encrypted +//! upstream transport without inventing ingress metadata or replacing the +//! server's trace authorization with an echo handler. #![cfg(any(target_os = "macos", target_os = "linux"))] use std::convert::Infallible; use std::io::Cursor; use std::net::SocketAddr; -use std::process::{Child, Command, Stdio}; use std::sync::{Arc, Mutex}; use std::time::Duration; use bytes::Bytes; use clap::Parser as _; +use edgezero_adapter_axum::AXUM_PLATFORM; use edgezero_adapter_axum::dev_server::{AxumDevServer, AxumDevServerConfig}; +use edgezero_adapter_axum::test_utils::AxumTestServer; +use edgezero_core::app::App; use http::{HeaderMap, HeaderValue, Method, Request, Response, StatusCode, header}; use http_body_util::{BodyExt as _, Full}; use hyper::client::conn::http1::SendRequest; @@ -36,24 +39,14 @@ mod support; const PUBLIC_HOST: &str = "www.publisher.example.com"; const PUBLIC_AUTHORITY: &str = "www.publisher.example.com:8443"; const PUBLIC_ORIGIN: &str = "https://www.publisher.example.com:8443"; -const FIXTURE_ADDR: &str = "TS_TEST_AXUM_ADDR"; -const FIXTURE_ORIGIN: &str = "TS_TEST_AXUM_ORIGIN"; -const FIXTURE_TRUST: &str = "TS_TEST_AXUM_TRUST"; -/// Run only as a fixture child; the parent always kills and reaps this process. -#[test] -#[ignore = "server fixture invoked only by proxy integration tests"] -fn trusted_server_fixture() { - // Direct ignored-test runs have no parent process to serve. - let Ok(addr) = std::env::var(FIXTURE_ADDR) else { - return; - }; +/// Serves the actual Trusted Server app on a loopback port. +/// +/// The app reads no `EdgeZero` store: its settings are injected here, so no +/// store is attached. +async fn serve_trusted_server(origin: &str, trust: bool) -> AxumTestServer { + // The adapter's outbound client expects a process-level crypto provider. let _ = rustls::crypto::aws_lc_rs::default_provider().install_default(); - let addr: SocketAddr = addr - .parse() - .expect("should parse the fixture listener address"); - let origin = std::env::var(FIXTURE_ORIGIN).expect("should supply the publisher origin"); - let trust = std::env::var(FIXTURE_TRUST).expect("should supply the forwarder policy"); let mut value = json!({ "handlers": [{ "path": "^/_ts/admin", @@ -72,7 +65,7 @@ fn trusted_server_fixture() { "gpt_diagnostics": {"enabled": true, "trace_page_enabled": true} } }); - if trust == "enabled" { + if trust { value["trusted_forwarder"] = json!({ "auth_header": "x-ts-forwarder-auth", "shared_secret": support::FORWARDER_TOKEN @@ -81,20 +74,24 @@ fn trusted_server_fixture() { let settings = Settings::from_json_value(value).expect("should load fixture settings"); let router = TrustedServerApp::routes_with_settings(settings) .expect("should build the actual Trusted Server router"); - AxumDevServer::with_config( - router, + let app = App::from_router::(router, AXUM_PLATFORM) + .expect("should assemble the actual Trusted Server app"); + AxumTestServer::start(AxumDevServer::with_config( + app, AxumDevServerConfig { - addr, + addr: SocketAddr::from(([127, 0, 0, 1], 0)), enable_ctrl_c: false, + ..AxumDevServerConfig::default() }, - ) - .run() - .expect("should run the actual Axum HTTP listener"); + )) + .await + .expect("should start the actual Axum HTTP listener") } struct Fixture { addr: SocketAddr, - child: Child, + // Held for its lifetime: dropping it shuts the listener down. + _server: AxumTestServer, origin_headers: Arc>>, tasks: Vec>, } @@ -104,8 +101,6 @@ impl Drop for Fixture { for task in &self.tasks { task.abort(); } - let _ = self.child.kill(); - let _ = self.child.wait(); } } @@ -147,49 +142,14 @@ async fn start_fixture(trust: bool) -> Fixture { }); } }); - let reserved = std::net::TcpListener::bind("127.0.0.1:0") - .expect("should reserve the Axum fixture address"); - let addr = reserved.local_addr().expect("should read the Axum address"); - drop(reserved); - let child = Command::new(std::env::current_exe().expect("should locate the test executable")) - .args([ - "--ignored", - "--exact", - "trusted_server_fixture", - "--nocapture", - ]) - .env(FIXTURE_ADDR, addr.to_string()) - .env(FIXTURE_ORIGIN, origin_url) - .env(FIXTURE_TRUST, if trust { "enabled" } else { "disabled" }) - .stdout(Stdio::null()) - .stderr(Stdio::inherit()) - .spawn() - .expect("should spawn the real Axum fixture"); - let mut fixture = Fixture { - addr, - child, + // `start` returns once the listener is bound, so no readiness poll is needed. + let server = serve_trusted_server(&origin_url, trust).await; + Fixture { + addr: server.address(), + _server: server, origin_headers: headers, tasks: vec![origin_task], - }; - tokio::time::timeout(Duration::from_secs(10), async { - loop { - if TcpStream::connect(addr).await.is_ok() { - return; - } - assert!( - fixture - .child - .try_wait() - .expect("should inspect the child") - .is_none(), - "should keep the Axum fixture running until ready" - ); - tokio::time::sleep(Duration::from_millis(10)).await; - } - }) - .await - .expect("should start the actual Axum listener promptly"); - fixture + } } async fn add_tls_relay(fixture: &mut Fixture) -> SocketAddr { diff --git a/crates/trusted-server-core/README.md b/crates/trusted-server-core/README.md index 1d8c54ea93..ad3b792456 100644 --- a/crates/trusted-server-core/README.md +++ b/crates/trusted-server-core/README.md @@ -8,8 +8,12 @@ on an edge SDK, Tokio runtime, filesystem, socket, or host-only process API. - `settings` and `settings_data` define typed application configuration, validation, secret references, and runtime normalization. -- `platform` defines the HTTP, backend, store, geo, client-info, and telemetry - service boundary adapters implement. +- `platform` defines the store, geo, client-info, template, and telemetry + service boundary adapters implement, and carries each request's EdgeZero + `HttpClient` and `MonotonicClock`. +- `outbound` sets the connect timeout, constant timeouts, and response caps + every outbound request carries, and maps client failures to Trusted Server + errors. - `publisher`, `router`, `handlers`, and `response` dispatch publisher and administrative requests through platform-neutral request/response types. - `auction` builds plans, invokes providers and mediators, selects bids, and @@ -30,10 +34,11 @@ on an edge SDK, Tokio runtime, filesystem, socket, or host-only process API. - `tsjs` selects embedded browser modules supplied by `trusted-server-js`. - `openrtb` connects auction logic to the checked OpenRTB data model. -Adapter crates own runtime startup, SDK conversion, concrete storage, outbound -transport, and target-specific limitations. Adding an integration should use -the narrowest registration hook and the core-neutral `RuntimeServices` -boundary; see the [Integration Guide](../../docs/guide/integration-guide.md). +Adapter crates own runtime startup, SDK conversion, concrete storage, and +target-specific limitations. Outbound HTTP goes through EdgeZero's client. +Adding an integration should use the narrowest registration hook and the +core-neutral `RuntimeServices` boundary; see the +[Integration Guide](../../docs/guide/integration-guide.md). ## Build and test diff --git a/crates/trusted-server-core/src/asset_image_optimizer.rs b/crates/trusted-server-core/src/asset_image_optimizer.rs index bbc2393fb4..0482811832 100644 --- a/crates/trusted-server-core/src/asset_image_optimizer.rs +++ b/crates/trusted-server-core/src/asset_image_optimizer.rs @@ -1,32 +1,108 @@ -//! Platform-neutral Image Optimizer profile-table handling for asset routes. +//! Image Optimizer profile-table handling for asset routes. //! //! Asset routes accept small, publisher-defined query controls such as -//! `profile`, `ar`, `x`, and `y`. This module converts those controls into a -//! closed transformation set before the request reaches a platform adapter. It -//! does not pass arbitrary client query parameters through as Image Optimizer -//! options. +//! `profile`, `ar`, `x`, and `y`. This module resolves those controls into a +//! closed transformation set, an [`ImageOptimizerProfile`], and maps it to the +//! `EdgeZero` options the final asset request carries. It does not pass +//! arbitrary client query parameters through as Image Optimizer options. //! //! The profile table supports shared base parameters, per-profile overrides, //! optional aspect-ratio overrides, crop offset bucketing, and a debug bypass //! parameter that disables image optimization for a single request. -use std::borrow::Cow; -use std::collections::HashSet; - +use edgezero_core::error::EdgeError; +use edgezero_core::outbound::{ + OutboundCropFocus, OutboundImageFormat, OutboundImageOptimizerOptions, + OutboundImageOptimizerRegion, OutboundResizeFilter, +}; use error_stack::Report; use url::form_urlencoded; use crate::error::TrustedServerError; -use crate::platform::{ - PlatformImageOptimizerCrop, PlatformImageOptimizerCropMode, PlatformImageOptimizerOptions, - PlatformImageOptimizerParams, -}; use crate::settings::{ ImageOptimizerCropOffsetsConfig, ImageOptimizerProfileSet, MissingCropOffsetMode, OriginQueryPolicy, ProxyAssetRoute, Settings, UnknownProfilePolicy, }; -/// Build Image Optimizer metadata for a route and request query. +/// An Image Optimizer profile resolved for one asset request. +/// +/// It holds the selected profile's parameters over the profile set's base +/// parameters, with the request's aspect ratio and crop offsets applied, plus +/// the route's region and origin query policy. [`image_optimizer_options`] maps +/// it to the options the asset request carries. +#[derive(Debug, Clone, PartialEq, Eq)] +pub(crate) struct ImageOptimizerProfile { + region: String, + preserve_origin_query: bool, + params: ImageOptimizerParams, +} + +/// The transformation parameters a profile table can set. +#[derive(Debug, Default, Clone, Copy, PartialEq, Eq)] +struct ImageOptimizerParams { + format: Option, + quality: Option, + resize_filter: Option, + width: Option, + height: Option, + crop: Option, +} + +impl ImageOptimizerParams { + /// Merges `other` into these parameters, with `other` taking precedence. + fn merge_from(&mut self, other: Self) { + if other.format.is_some() { + self.format = other.format; + } + if other.quality.is_some() { + self.quality = other.quality; + } + if other.resize_filter.is_some() { + self.resize_filter = other.resize_filter; + } + if other.width.is_some() { + self.width = other.width; + } + if other.height.is_some() { + self.height = other.height; + } + if other.crop.is_some() { + self.crop = other.crop; + } + } +} + +/// An aspect-ratio crop and where it focuses. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +struct ImageOptimizerCrop { + width: u32, + height: u32, + focus: ImageOptimizerCropFocus, +} + +impl ImageOptimizerCrop { + /// A bare aspect-ratio crop around the default focus. + fn aspect_ratio(width: u32, height: u32) -> Self { + Self { + width, + height, + focus: ImageOptimizerCropFocus::Default, + } + } +} + +/// Where a crop focuses. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +enum ImageOptimizerCropFocus { + /// Image Optimizer's default focus, for a bare aspect ratio. + Default, + /// Image Optimizer's smart crop. + Smart, + /// Crop offsets, as percentages of the image. + Offset { x: u32, y: u32 }, +} + +/// Resolves the Image Optimizer profile for a route and request query. /// /// The incoming query is read only as profile-table input. The asset proxy /// applies the route origin-query policy separately before signing or sending @@ -40,11 +116,11 @@ use crate::settings::{ /// Returns a proxy/configuration error if the configured profile set is missing, /// a profile parameter cannot be parsed, or the request references an unknown /// profile in `reject` mode. -pub(crate) fn options_for_asset_request( +pub(crate) fn profile_for_asset_request( settings: &Settings, route: &ProxyAssetRoute, query: &str, -) -> Result, Report> { +) -> Result, Report> { let Some(route_config) = &route.image_optimizer else { return Ok(None); }; @@ -88,12 +164,95 @@ pub(crate) fn options_for_asset_request( apply_aspect_ratio_override(profile_set, selected_profile, query, &mut params); apply_crop_offsets(profile_set, query, &mut params); - Ok(Some( - PlatformImageOptimizerOptions::new(route_config.region.clone(), params) - .with_preserve_query_string_on_origin_request( - route.origin_query_policy() == OriginQueryPolicy::Preserve, - ), - )) + Ok(Some(ImageOptimizerProfile { + region: route_config.region.clone(), + preserve_origin_query: route.origin_query_policy() == OriginQueryPolicy::Preserve, + params, + })) +} + +/// Maps a resolved profile to the `EdgeZero` Image Optimizer options its asset +/// request carries. +/// +/// Trusted Server's region aliases, such as `us-east-1`, map to `EdgeZero`'s +/// seven regions through [`image_optimizer_region`]. The route's origin query +/// policy maps to `preserve_origin_query`. +/// +/// # Errors +/// +/// Returns [`EdgeError::BadRequest`] for an unknown region, or for a quality, +/// dimension, or crop value that Image Optimizer does not accept. Settings +/// validation rejects all of them, so the asset proxy serves the image +/// unoptimized if this ever fails. +pub(crate) fn image_optimizer_options( + profile: &ImageOptimizerProfile, +) -> Result { + let region = image_optimizer_region(&profile.region) + .ok_or_else(|| EdgeError::bad_request("unsupported image optimizer region"))?; + let params = &profile.params; + let mut options = OutboundImageOptimizerOptions::new() + .region(region) + .preserve_origin_query(profile.preserve_origin_query); + if let Some(format) = params.format { + options = options.format(format); + } + if let Some(quality) = params.quality { + options = options.quality(u8::try_from(quality).map_err(|_| { + EdgeError::bad_request("image optimizer quality must be between 0 and 100") + })?)?; + } + if let Some(resize_filter) = params.resize_filter { + options = options.resize_filter(resize_filter); + } + if let Some(width) = params.width { + options = options.width(width)?; + } + if let Some(height) = params.height { + options = options.height(height)?; + } + if let Some(crop) = params.crop { + let focus = match crop.focus { + ImageOptimizerCropFocus::Default => OutboundCropFocus::Default, + ImageOptimizerCropFocus::Smart => OutboundCropFocus::Smart, + ImageOptimizerCropFocus::Offset { x, y } => OutboundCropFocus::Percent { + x: crop_percentage(x)?, + y: crop_percentage(y)?, + }, + }; + options = options.crop(crop.width, crop.height, focus)?; + } + + Ok(options) +} + +/// Maps a configured Image Optimizer region, or one of its aliases, to an +/// `EdgeZero` region. +/// +/// Matching ignores surrounding space, case, and `-` versus `_`, so `us-east-1` +/// and `US_EAST` both name [`OutboundImageOptimizerRegion::UsEast`]. Settings +/// validation accepts exactly the regions this maps. +pub(crate) fn image_optimizer_region(region: &str) -> Option { + match region + .trim() + .to_ascii_lowercase() + .replace('-', "_") + .as_str() + { + "us_east" | "us_east_1" => Some(OutboundImageOptimizerRegion::UsEast), + "us_central" | "us_central_1" => Some(OutboundImageOptimizerRegion::UsCentral), + "us_west" | "us_west_1" | "us_west_2" => Some(OutboundImageOptimizerRegion::UsWest), + "eu_central" | "eu_central_1" => Some(OutboundImageOptimizerRegion::EuCentral), + "eu_west" | "eu_west_1" => Some(OutboundImageOptimizerRegion::EuWest), + "asia" => Some(OutboundImageOptimizerRegion::Asia), + "australia" => Some(OutboundImageOptimizerRegion::Australia), + _ => None, + } +} + +fn crop_percentage(offset: u32) -> Result { + u8::try_from(offset).map_err(|_| { + EdgeError::bad_request("image optimizer crop percentages must be between 0 and 100") + }) } fn select_profile<'a>( @@ -126,7 +285,7 @@ fn apply_aspect_ratio_override( profile_set: &ImageOptimizerProfileSet, selected_profile: &str, query: &str, - params: &mut PlatformImageOptimizerParams, + params: &mut ImageOptimizerParams, ) { let Some(aspect_config) = &profile_set.aspect_ratios else { return; @@ -153,13 +312,13 @@ fn apply_aspect_ratio_override( return; }; - params.crop = Some(PlatformImageOptimizerCrop::aspect_ratio(width, height)); + params.crop = Some(ImageOptimizerCrop::aspect_ratio(width, height)); } fn apply_crop_offsets( profile_set: &ImageOptimizerProfileSet, query: &str, - params: &mut PlatformImageOptimizerParams, + params: &mut ImageOptimizerParams, ) { let Some(offset_config) = &profile_set.crop_offsets else { return; @@ -170,7 +329,7 @@ fn apply_crop_offsets( let Some(crop) = &mut params.crop else { return; }; - if !crop.is_bare_aspect_ratio() { + if crop.focus != ImageOptimizerCropFocus::Default { return; } @@ -178,13 +337,15 @@ fn apply_crop_offsets( let y = query_param_value(query, &offset_config.y_param); if x.is_none() && y.is_none() { if offset_config.when_missing == MissingCropOffsetMode::Smart { - crop.mode = Some(PlatformImageOptimizerCropMode::Smart); + crop.focus = ImageOptimizerCropFocus::Smart; } return; } - crop.offset_x = Some(normalize_offset(x.as_deref(), offset_config)); - crop.offset_y = Some(normalize_offset(y.as_deref(), offset_config)); + crop.focus = ImageOptimizerCropFocus::Offset { + x: normalize_offset(x.as_deref(), offset_config), + y: normalize_offset(y.as_deref(), offset_config), + }; } fn normalize_offset(value: Option<&str>, config: &ImageOptimizerCropOffsetsConfig) -> u32 { @@ -213,10 +374,8 @@ fn normalize_offset(value: Option<&str>, config: &ImageOptimizerCropOffsetsConfi config.buckets.last().copied().unwrap_or(first) } -fn parse_param_string( - params: &str, -) -> Result> { - let mut parsed = PlatformImageOptimizerParams::default(); +fn parse_param_string(params: &str) -> Result> { + let mut parsed = ImageOptimizerParams::default(); if params.trim().is_empty() { return Ok(parsed); } @@ -244,22 +403,29 @@ fn parse_param_string( Ok(parsed) } -fn parse_format(value: &str) -> Result> { - let normalized = value.trim().to_ascii_lowercase(); - match normalized.as_str() { - "auto" | "avif" | "gif" | "jpeg" | "jpg" | "jxl" | "jpegxl" | "mp4" | "png" | "webp" => { - Ok(normalized) - } +fn parse_format(value: &str) -> Result> { + match value.trim().to_ascii_lowercase().as_str() { + "auto" => Ok(OutboundImageFormat::Auto), + "avif" => Ok(OutboundImageFormat::Avif), + "gif" => Ok(OutboundImageFormat::Gif), + "jpeg" | "jpg" => Ok(OutboundImageFormat::Jpeg), + "jxl" | "jpegxl" => Ok(OutboundImageFormat::JpegXl), + "mp4" => Ok(OutboundImageFormat::Mp4), + "png" => Ok(OutboundImageFormat::Png), + "webp" => Ok(OutboundImageFormat::WebP), _ => Err(Report::new(TrustedServerError::Configuration { message: format!("unsupported image optimizer format `{value}`"), })), } } -fn parse_resize_filter(value: &str) -> Result> { - let normalized = value.trim().to_ascii_lowercase(); - match normalized.as_str() { - "nearest" | "bilinear" | "bicubic" | "lanczos2" | "lanczos3" => Ok(normalized), +fn parse_resize_filter(value: &str) -> Result> { + match value.trim().to_ascii_lowercase().as_str() { + "nearest" => Ok(OutboundResizeFilter::Nearest), + "bilinear" => Ok(OutboundResizeFilter::Bilinear), + "bicubic" => Ok(OutboundResizeFilter::Bicubic), + "lanczos2" => Ok(OutboundResizeFilter::Lanczos2), + "lanczos3" => Ok(OutboundResizeFilter::Lanczos3), _ => Err(Report::new(TrustedServerError::Configuration { message: format!("unsupported image optimizer resize-filter `{value}`"), })), @@ -299,27 +465,25 @@ fn parse_bounded_u32( Ok(parsed) } -fn parse_crop(value: &str) -> Result> { +fn parse_crop(value: &str) -> Result> { let mut parts = value.split(','); let ratio = parts.next().unwrap_or_default(); let (width, height) = parse_crop_ratio(ratio)?; - let mut crop = PlatformImageOptimizerCrop::aspect_ratio(width, height); - let mut seen_suffixes = HashSet::new(); + let mut smart = false; + let mut offset_x = None; + let mut offset_y = None; for suffix in parts { if suffix == "smart" { - crop.mode = Some(PlatformImageOptimizerCropMode::Smart); - seen_suffixes.insert(Cow::Borrowed("smart")); + smart = true; continue; } if let Some(offset) = suffix.strip_prefix("offset-x") { - crop.offset_x = Some(parse_bounded_u32("crop offset-x", offset, 0, 100)?); - seen_suffixes.insert(Cow::Borrowed("offset-x")); + offset_x = Some(parse_bounded_u32("crop offset-x", offset, 0, 100)?); continue; } if let Some(offset) = suffix.strip_prefix("offset-y") { - crop.offset_y = Some(parse_bounded_u32("crop offset-y", offset, 0, 100)?); - seen_suffixes.insert(Cow::Borrowed("offset-y")); + offset_y = Some(parse_bounded_u32("crop offset-y", offset, 0, 100)?); continue; } return Err(Report::new(TrustedServerError::Configuration { @@ -327,22 +491,29 @@ fn parse_crop(value: &str) -> Result ImageOptimizerCropFocus::Default, + (true, None, None) => ImageOptimizerCropFocus::Smart, + (false, Some(x), Some(y)) => ImageOptimizerCropFocus::Offset { x, y }, + (true, _, _) => { + return Err(Report::new(TrustedServerError::Configuration { + message: "image optimizer crop cannot combine smart mode with explicit offsets" + .to_string(), + })); + } + (false, _, _) => { + return Err(Report::new(TrustedServerError::Configuration { + message: "image optimizer crop offsets must include both offset-x and offset-y" + .to_string(), + })); + } + }; - Ok(crop) + Ok(ImageOptimizerCrop { + width, + height, + focus, + }) } fn parse_crop_ratio(value: &str) -> Result<(u32, u32), Report> { @@ -401,8 +572,29 @@ mod tests { } } + fn medium_params(query: &str) -> ImageOptimizerParams { + let set = profile_set(); + let selected = select_profile(&set, Some("medium")).expect("should select profile"); + let mut params = parse_param_string(&set.base_params).expect("should parse base params"); + params.merge_from( + parse_param_string(set.profiles.get(selected).expect("should get profile")) + .expect("should parse profile params"), + ); + apply_aspect_ratio_override(&set, selected, query, &mut params); + apply_crop_offsets(&set, query, &mut params); + params + } + + fn us_east_profile(params: ImageOptimizerParams) -> ImageOptimizerProfile { + ImageOptimizerProfile { + region: "us_east".to_string(), + preserve_origin_query: false, + params, + } + } + #[test] - fn options_for_asset_request_returns_error_when_selected_profile_is_missing() { + fn profile_for_asset_request_returns_error_when_selected_profile_is_missing() { let mut settings = crate::test_support::tests::create_test_settings(); let mut profile_set = profile_set(); profile_set.profiles.remove("default"); @@ -420,7 +612,7 @@ mod tests { origin_query: None, }); - let err = options_for_asset_request(&settings, &route, "") + let err = profile_for_asset_request(&settings, &route, "") .expect_err("should return a configuration error for broken profile sets"); assert!( @@ -431,39 +623,201 @@ mod tests { #[test] fn profile_conversion_adds_aspect_ratio_and_smart_crop() { - let set = profile_set(); - let selected = select_profile(&set, Some("medium")).expect("should select profile"); - let mut params = parse_param_string(&set.base_params).expect("should parse base params"); - params.merge_from( - parse_param_string(set.profiles.get(selected).expect("should get profile")) - .expect("should parse profile params"), + let params = medium_params("profile=medium&ar=1-1"); + + assert_eq!( + params, + ImageOptimizerParams { + format: Some(OutboundImageFormat::Auto), + quality: Some(70), + resize_filter: Some(OutboundResizeFilter::Bicubic), + width: Some(828), + height: None, + crop: Some(ImageOptimizerCrop { + width: 1, + height: 1, + focus: ImageOptimizerCropFocus::Smart, + }), + }, + "should merge the profile over the base and add a smart aspect-ratio crop" ); - apply_aspect_ratio_override(&set, selected, "profile=medium&ar=1-1", &mut params); - apply_crop_offsets(&set, "profile=medium&ar=1-1", &mut params); - - assert_eq!(params.quality, Some(70)); - assert_eq!(params.resize_filter.as_deref(), Some("bicubic")); - assert_eq!(params.format.as_deref(), Some("auto")); - assert_eq!(params.width, Some(828)); - let crop = params.crop.expect("should add crop"); - assert_eq!((crop.width, crop.height), (1, 1)); - assert_eq!(crop.mode, Some(PlatformImageOptimizerCropMode::Smart)); } #[test] fn profile_conversion_buckets_offsets() { - let set = profile_set(); - let selected = select_profile(&set, Some("medium")).expect("should select profile"); - let mut params = parse_param_string("width=828").expect("should parse params"); - apply_aspect_ratio_override( - &set, - selected, - "profile=medium&ar=16-9&x=79&y=bad", - &mut params, + let params = medium_params("profile=medium&ar=16-9&x=79&y=bad"); + + assert_eq!( + params.crop, + Some(ImageOptimizerCrop { + width: 16, + height: 9, + focus: ImageOptimizerCropFocus::Offset { x: 70, y: 50 }, + }), + "should bucket x and fall back to the default y" ); - apply_crop_offsets(&set, "profile=medium&ar=16-9&x=79&y=bad", &mut params); + } + + #[test] + fn image_optimizer_options_maps_the_profile_to_edgezero_options() { + let profile = ImageOptimizerProfile { + preserve_origin_query: true, + ..us_east_profile(medium_params("profile=medium&ar=16-9&x=79&y=bad")) + }; + + let options = image_optimizer_options(&profile).expect("should map a validated profile"); + + assert_eq!( + options, + OutboundImageOptimizerOptions::new() + .region(OutboundImageOptimizerRegion::UsEast) + .preserve_origin_query(true) + .format(OutboundImageFormat::Auto) + .quality(70) + .expect("should accept quality 70") + .resize_filter(OutboundResizeFilter::Bicubic) + .width(828) + .expect("should accept width 828") + .crop(16, 9, OutboundCropFocus::Percent { x: 70, y: 50 }) + .expect("should accept the bucketed crop"), + "should carry every profile parameter, the region, and the query policy" + ); + } + + #[test] + fn image_optimizer_options_maps_each_crop_focus() { + for (focus, expected) in [ + (ImageOptimizerCropFocus::Default, OutboundCropFocus::Default), + (ImageOptimizerCropFocus::Smart, OutboundCropFocus::Smart), + ( + ImageOptimizerCropFocus::Offset { x: 0, y: 100 }, + OutboundCropFocus::Percent { x: 0, y: 100 }, + ), + ] { + let profile = us_east_profile(ImageOptimizerParams { + crop: Some(ImageOptimizerCrop { + width: 4, + height: 3, + focus, + }), + ..ImageOptimizerParams::default() + }); + + let options = image_optimizer_options(&profile).expect("should map the crop"); + + assert_eq!( + options.crop_setting(), + Some((4, 3, expected)), + "should map {focus:?} to {expected:?}" + ); + } + } + + #[test] + fn image_optimizer_options_rejects_values_validation_rules_out() { + let crop = |x| ImageOptimizerCrop { + width: 1, + height: 1, + focus: ImageOptimizerCropFocus::Offset { x, y: 50 }, + }; + let profiles = [ + ImageOptimizerProfile { + region: "mars".to_string(), + ..us_east_profile(ImageOptimizerParams::default()) + }, + us_east_profile(ImageOptimizerParams { + quality: Some(101), + ..ImageOptimizerParams::default() + }), + us_east_profile(ImageOptimizerParams { + quality: Some(300), + ..ImageOptimizerParams::default() + }), + us_east_profile(ImageOptimizerParams { + width: Some(0), + ..ImageOptimizerParams::default() + }), + us_east_profile(ImageOptimizerParams { + crop: Some(crop(150)), + ..ImageOptimizerParams::default() + }), + us_east_profile(ImageOptimizerParams { + crop: Some(crop(300)), + ..ImageOptimizerParams::default() + }), + ]; + + for profile in profiles { + let error = image_optimizer_options(&profile) + .expect_err("should reject a profile that validation rules out"); + + assert_eq!( + error.status(), + http::StatusCode::BAD_REQUEST, + "should reject {profile:?} as a bad request" + ); + } + } - let crop = params.crop.expect("should add crop"); - assert_eq!((crop.offset_x, crop.offset_y), (Some(70), Some(50))); + #[test] + fn image_optimizer_region_maps_aliases_to_the_seven_regions() { + for (aliases, expected) in [ + ( + &["us_east", "us-east-1", "US_EAST"][..], + OutboundImageOptimizerRegion::UsEast, + ), + ( + &["us_central", "us-central-1"][..], + OutboundImageOptimizerRegion::UsCentral, + ), + ( + &["us_west", "us-west-1", "us-west-2"][..], + OutboundImageOptimizerRegion::UsWest, + ), + ( + &["eu_central", "eu-central-1"][..], + OutboundImageOptimizerRegion::EuCentral, + ), + ( + &["eu_west", "eu-west-1"][..], + OutboundImageOptimizerRegion::EuWest, + ), + (&[" asia "][..], OutboundImageOptimizerRegion::Asia), + (&["Australia"][..], OutboundImageOptimizerRegion::Australia), + ] { + for alias in aliases { + assert_eq!( + image_optimizer_region(alias), + Some(expected), + "should map `{alias}` to {expected:?}" + ); + } + } + for unknown in ["", "mars", "us_east_2", "eu_north_1"] { + assert_eq!( + image_optimizer_region(unknown), + None, + "should not map `{unknown}`" + ); + } + } + + #[test] + fn parse_crop_rejects_smart_with_offsets_and_lone_offsets() { + for (value, message) in [ + ( + "1:1,smart,offset-x10,offset-y10", + "cannot combine smart mode", + ), + ("1:1,offset-x10", "must include both offset-x and offset-y"), + ("1:1,offset-y10", "must include both offset-x and offset-y"), + ] { + let error = parse_crop(value).expect_err("should reject the crop"); + + assert!( + format!("{error:?}").contains(message), + "should reject `{value}` with `{message}`: {error:?}" + ); + } } } diff --git a/crates/trusted-server-core/src/auction/README.md b/crates/trusted-server-core/src/auction/README.md index 74ba790bc3..c8454817f3 100644 --- a/crates/trusted-server-core/src/auction/README.md +++ b/crates/trusted-server-core/src/auction/README.md @@ -1,8 +1,8 @@ # Auction orchestration The auction module compiles operator configuration into one immutable plan, -routes browser demand to providers, runs provider requests concurrently where -the adapter permits it, and returns normalized OpenRTB bids. +routes browser demand to providers, sends provider requests concurrently as +one EdgeZero request batch, and returns normalized OpenRTB bids. The maintained operator guide is [`docs/guide/auction-orchestration.md`](../../../../docs/guide/auction-orchestration.md). @@ -51,12 +51,13 @@ startup and operator validation. It validates: - mediator and request-signing references; - bounded configuration values. -Adapters then call `AuctionPlan::validate_for_target` for backend naming, -fan-out support, and target resource limits. +The Fastly state builder then calls `plan::validate_fastly_backend_bound`, +which rejects an enabled plan that could need more than 160 Fastly backend +configurations. The other adapters add no plan check. A plan-backed orchestrator contains generic OpenRTB providers compiled from the -plan. `AuctionOrchestrator::register_provider` and the old concrete Prebid -provider remain test-only parity code. They are not extension APIs. +plan. The `cfg(test)` `AuctionProvider` impls on the concrete Prebid and APS +providers remain test-only parity code. They are not extension APIs. ## Routing @@ -81,10 +82,13 @@ bidders remain browser demand. `standard`, `prebid-server`, and `aps` profiles. Profiles receive routed and privacy-approved facts, not the raw inbound request. -The orchestrator launches all eligible providers before collecting responses. -It uses adapter `PlatformHttpClient` handles and predicted backend names for -correlation. Provider launch, transport, HTTP, parse, and admission failures are -provider-local when another provider can continue. +The orchestrator builds and signs every routed provider's request, then sends +them together with `HttpClient::start_batch_until`, whose cutoff is the auction +deadline. Batch results are positional, so each result maps back to its +provider. A bid that has not arrived by the cutoff becomes a provider timeout, +and dropping the batch cancels the requests still in flight. Provider launch, +transport, HTTP, parse, and admission failures are provider-local when another +provider can continue. When no mediator is configured, the orchestrator selects the highest decoded CPM per slot and applies floors locally. When a mediator is configured, it sends @@ -157,14 +161,15 @@ routing = "all_eligible" profile_config = { account_id = "example-account" } ``` -Provider endpoints must be absolute HTTPS URLs. Replace all example values -before enabling an auction. +Provider endpoints must be absolute HTTPS URLs. Plain HTTP is accepted only +for `127.0.0.1` and `[::1]`, which local test bidders use. Replace all example +values before enabling an auction. ## Code map - `mod.rs` compiles plans and builds the shared orchestrator. - `endpoints.rs` handles `POST /auction` and converts the browser request. -- `plan.rs` owns plan validation and target capability checks. +- `plan.rs` owns plan validation and the Fastly backend bound. - `profile.rs` owns typed OpenRTB profile configuration. - `routing.rs` assigns slots and bidder params to providers. - `openrtb.rs` builds shared requests and parses standard responses. @@ -178,7 +183,9 @@ before enabling an auction. Use `compile_auction_plan` in tests, then construct the orchestrator and integration registry from the same `Arc`. Profile tests should cover typed configuration, exact request output, response admission, routing, -provider-local failures, and target validation. +provider-local failures, and the Fastly backend bound. Send bidder requests +through the scripted client in `test_support::scripted_client`, which records +each request and answers from a script. Run target-matched aliases rather than bare workspace tests: diff --git a/crates/trusted-server-core/src/auction/endpoints.rs b/crates/trusted-server-core/src/auction/endpoints.rs index 086515fa20..c1e4e03fc1 100644 --- a/crates/trusted-server-core/src/auction/endpoints.rs +++ b/crates/trusted-server-core/src/auction/endpoints.rs @@ -20,6 +20,7 @@ use crate::ec::kv_types::MAX_UID_LENGTH; use crate::ec::prebid_eids::parse_prebid_eids_cookie; use crate::ec::registry::PartnerRegistry; use crate::error::TrustedServerError; +use crate::http_util::read_request_body; use crate::openrtb::{Eid, Uid}; use crate::platform::RuntimeServices; use crate::settings::Settings; @@ -46,6 +47,23 @@ const MAX_CLIENT_EID_SOURCE_BYTES: usize = 255; /// arbitrary WASM linear memory. const MAX_AUCTION_BODY_SIZE: usize = 256 * 1024; +/// Builds the `text/plain` 413 answered for an auction body over [`MAX_AUCTION_BODY_SIZE`]. +/// +/// # Errors +/// +/// Returns [`TrustedServerError::Auction`] when the response cannot be built. +fn auction_body_too_large_response() -> Result, Report> { + Response::builder() + .status(StatusCode::PAYLOAD_TOO_LARGE) + .header(header::CONTENT_TYPE, "text/plain") + .body(EdgeBody::from(format!( + "Request body exceeds {MAX_AUCTION_BODY_SIZE} byte limit" + ))) + .change_context(TrustedServerError::Auction { + message: "Auction request body exceeded maximum size".to_string(), + }) +} + /// Handle auction request from `POST /auction`. /// /// Accepts a JSON body matching [`AdRequest`][`super::formats::AdRequest`]. @@ -115,6 +133,8 @@ const MAX_AUCTION_BODY_SIZE: usize = 256 * 1024; /// # Errors /// /// Returns an error if: +/// - The request body cannot be read (see [`read_request_body`]); a body over +/// 256 KiB is answered with a `text/plain` 413 instead /// - The request body cannot be parsed /// - The auction request conversion fails (e.g., invalid ad units) /// - The auction execution fails @@ -128,40 +148,23 @@ pub async fn handle_auction( services: &RuntimeServices, req: Request, ) -> Result, Report> { - // Reject oversized bodies before core buffers/parses them. The Content-Length - // pre-check stops well-behaved clients early; the post-read check defends - // against clients that lie about (or omit) the header. - let content_length_exceeded = req - .headers() - .get(header::CONTENT_LENGTH) - .and_then(|v| v.to_str().ok()) - .and_then(|v| v.parse::().ok()) - .is_some_and(|length| length > MAX_AUCTION_BODY_SIZE); - if content_length_exceeded { - return Response::builder() - .status(StatusCode::PAYLOAD_TOO_LARGE) - .header(header::CONTENT_TYPE, "text/plain") - .body(EdgeBody::from(format!( - "Request body exceeds {MAX_AUCTION_BODY_SIZE} byte limit" - ))) - .change_context(TrustedServerError::Auction { - message: "Auction request body exceeded maximum size".to_string(), - }); - } - + // The bounded read rejects a declared overflow before polling the body and + // stops a streamed body at the first chunk past the cap, so a client that + // omits or understates `Content-Length` is still held to it. let (parts, body) = req.into_parts(); - let body_bytes = body.into_bytes().unwrap_or_default(); - if body_bytes.len() > MAX_AUCTION_BODY_SIZE { - return Response::builder() - .status(StatusCode::PAYLOAD_TOO_LARGE) - .header(header::CONTENT_TYPE, "text/plain") - .body(EdgeBody::from(format!( - "Request body exceeds {MAX_AUCTION_BODY_SIZE} byte limit" - ))) - .change_context(TrustedServerError::Auction { - message: "Auction request body exceeded maximum size".to_string(), - }); - } + let body_bytes = match read_request_body(&parts.headers, body, MAX_AUCTION_BODY_SIZE).await { + Ok(bytes) => bytes, + Err(report) + if matches!( + report.current_context(), + TrustedServerError::RequestTooLarge { .. } + ) => + { + log::warn!("Rejecting auction request: {report:?}"); + return auction_body_too_large_response(); + } + Err(report) => return Err(report), + }; let body: AdRequest = serde_json::from_slice(&body_bytes).change_context(TrustedServerError::BadRequest { message: "Failed to parse auction request body".to_string(), @@ -392,7 +395,6 @@ pub async fn handle_auction( settings, request: &http_req, timeout_ms: settings.auction.timeout_ms, - transport_timeout_ms: settings.auction.timeout_ms, provider_responses: None, services, }; @@ -695,24 +697,31 @@ pub(crate) fn merge_auction_eids( mod tests { use super::*; use crate::auction::config::AuctionConfig; - use crate::auction::provider::{AuctionProvider, ProviderRequestOutcome}; + use crate::auction::telemetry::{AuctionEventBatch, AuctionTelemetrySink}; - use crate::auction::types::{AuctionRequest, AuctionResponse}; + use crate::consent::jurisdiction::Jurisdiction; use crate::consent::types::ConsentContext; use crate::error::IntoHttpResponse as _; + use crate::integrations::adserver_mock::{AdServerMockConfig, AdServerMockProvider}; use crate::openrtb::Uid; + use crate::platform::ClientInfo; use crate::platform::test_support::{ - NoopBackend, NoopConfigStore, NoopGeo, NoopHttpClient, NoopSecretStore, StubBackend, - StubHttpClient, noop_services, + NoopConfigStore, NoopGeo, NoopSecretStore, build_services_with_outbound_client, + noop_services, + }; + use crate::test_support::body_read::body_read_failure_cases; + use crate::test_support::logging::WarningCapture; + use crate::test_support::scripted_client::{ScriptedHttpClient, ScriptedReply, TestClock}; + use crate::test_support::tests::{ + crate_test_settings_str, create_test_settings, run_pre_dispatch_hook, }; - use crate::platform::{ClientInfo, PlatformHttpClient, PlatformHttpRequest, PlatformResponse}; - use crate::test_support::tests::{crate_test_settings_str, create_test_settings}; use crate::trace::TracePreDispatchHook; use base64::Engine as _; use base64::engine::general_purpose::STANDARD as BASE64; - use edgezero_core::router::PreDispatchHook as _; + use edgezero_core::error::{BadGatewayReason, EdgeError}; + use edgezero_core::{FastlyFailureCause, ProviderFailureDetail, ProviderFailurePhase}; use serde_json::json; use std::sync::{Arc, Mutex}; @@ -742,14 +751,24 @@ mod tests { .config_store(Arc::new(NoopConfigStore)) .secret_store(Arc::new(NoopSecretStore)) .kv_store(Arc::new(edgezero_core::key_value_store::NoopKvStore)) - .backend(Arc::new(NoopBackend)) - .http_client(Arc::new(NoopHttpClient)) + .outbound_client(ScriptedHttpClient::forbidding().http_client()) + .clock(TestClock::new().monotonic_clock()) .geo(Arc::new(NoopGeo)) .auction_telemetry_sink(telemetry_sink) .client_info(ClientInfo::default()) .build() } + /// Endpoint of the `eid-capture` provider that `legacy_provider_map` configures. + const EID_CAPTURE_ENDPOINT: &str = "https://eid-capture.example/openrtb2/auction"; + + /// Builds the plan-backed orchestrator for test settings with `auction` in place. + fn orchestrator_with(auction: AuctionConfig) -> AuctionOrchestrator { + let mut settings = create_test_settings(); + settings.auction = auction; + crate::auction::build_orchestrator(&settings).expect("should build the orchestrator") + } + fn make_ec_context(jurisdiction: Jurisdiction, ec_value: Option<&str>) -> EcContext { EcContext::new_for_test( ec_value.map(str::to_owned), @@ -787,16 +806,13 @@ mod tests { // read: before the snapshot was shared, finalization saw `NotRead` and // paid for a second lookup. let settings = create_test_settings(); - let mut orchestrator = AuctionOrchestrator::new(AuctionConfig { + let orchestrator = orchestrator_with(AuctionConfig { enabled: true, - providers: AuctionConfig::legacy_provider_map(&["eid_capturing_provider"]), + providers: AuctionConfig::legacy_provider_map(&["eid-capture"]), timeout_ms: 2000, mediator: None, ..Default::default() }); - orchestrator.register_provider(Arc::new(EidCapturingProvider { - had_eids: Arc::new(std::sync::Mutex::new(None)), - })); let registry = PartnerRegistry::from_config(&[counting_test_partner("sharedid.org")]) .expect("should build partner registry"); @@ -885,16 +901,13 @@ mod tests { // hand its parsed body EIDs to `ec_context` so finalization ingests // them regardless of what the cookie carried. let settings = create_test_settings(); - let mut orchestrator = AuctionOrchestrator::new(AuctionConfig { + let orchestrator = orchestrator_with(AuctionConfig { enabled: true, - providers: AuctionConfig::legacy_provider_map(&["eid_capturing_provider"]), + providers: AuctionConfig::legacy_provider_map(&["eid-capture"]), timeout_ms: 2000, mediator: None, ..Default::default() }); - orchestrator.register_provider(Arc::new(EidCapturingProvider { - had_eids: Arc::new(std::sync::Mutex::new(None)), - })); let registry = PartnerRegistry::from_config(&[counting_test_partner("id5-sync.com")]) .expect("should build partner registry"); @@ -969,99 +982,6 @@ mod tests { ); } - /// Provider that fails the test if it is ever contacted. Used to prove the - /// `/auction` consent gate short-circuits before any outbound bid request. - struct PanicOnBidProvider; - - #[async_trait::async_trait(?Send)] - impl AuctionProvider for PanicOnBidProvider { - fn provider_name(&self) -> &str { - "panic_provider" - } - - async fn request_bids( - &self, - _request: &AuctionRequest, - _context: &AuctionContext<'_>, - ) -> Result> { - panic!("provider must not be contacted when the consent gate denies the auction"); - } - - async fn parse_response( - &self, - _response: PlatformResponse, - _response_time_ms: u64, - ) -> Result> { - panic!("provider must not parse a response when the auction is gated off"); - } - - fn timeout_ms(&self) -> u32 { - 100 - } - - fn backend_name(&self, _services: &RuntimeServices, _timeout_ms: u32) -> Option { - Some("panic-backend".to_string()) - } - } - - /// Provider used to prove that direct `/auction` remains available when - /// publisher server-side ad templates are disabled. - struct TemplateSwitchProbeProvider { - calls: Arc>, - } - - #[async_trait::async_trait(?Send)] - impl AuctionProvider for TemplateSwitchProbeProvider { - fn provider_name(&self) -> &'static str { - "template-switch-probe" - } - - async fn request_bids( - &self, - _request: &AuctionRequest, - context: &AuctionContext<'_>, - ) -> Result> { - *self.calls.lock().expect("should lock provider call count") += 1; - let request = Request::builder() - .method("POST") - .uri("https://bidder.example/auction") - .body(EdgeBody::empty()) - .expect("should build probe provider request"); - context - .services - .http_client() - .send_async( - PlatformHttpRequest::new(request, "template-switch-probe-backend") - .with_cache_bypass(), - ) - .await - .change_context(TrustedServerError::Auction { - message: "probe provider launch failed".to_string(), - }) - .map(ProviderRequestOutcome::pending) - } - - async fn parse_response( - &self, - _response: PlatformResponse, - _response_time_ms: u64, - ) -> Result> { - Ok(AuctionResponse::success( - self.provider_name(), - Vec::new(), - 0, - )) - } - - fn timeout_ms(&self) -> u32 { - 100 - } - - fn backend_name(&self, _services: &RuntimeServices, _timeout_ms: u32) -> Option { - Some("template-switch-probe-backend".to_string()) - } - } - #[tokio::test] async fn direct_auction_remains_available_when_templates_are_disabled() { let settings_toml = format!( @@ -1070,23 +990,16 @@ mod tests { ); let settings = Settings::from_toml(&settings_toml) .expect("should parse settings with disabled templates"); - let calls = Arc::new(Mutex::new(0)); - let mut orchestrator = AuctionOrchestrator::new(settings.auction.clone()); - orchestrator.register_provider(Arc::new(TemplateSwitchProbeProvider { - calls: Arc::clone(&calls), - })); - - let stub = Arc::new(StubHttpClient::new()); - stub.push_response(200, b"probe response".to_vec()); - let services = RuntimeServices::builder() - .config_store(Arc::new(NoopConfigStore)) - .secret_store(Arc::new(NoopSecretStore)) - .kv_store(Arc::new(edgezero_core::key_value_store::NoopKvStore)) - .backend(Arc::new(NoopBackend)) - .http_client(Arc::clone(&stub) as Arc) - .geo(Arc::new(NoopGeo)) - .client_info(ClientInfo::default()) - .build(); + let orchestrator = + crate::auction::build_orchestrator(&settings).expect("should build the orchestrator"); + let clock = TestClock::new(); + let bidders = ScriptedHttpClient::new(&clock); + bidders.respond_to( + http::Method::POST, + "https://bidder.example/auction", + ScriptedReply::status(StatusCode::NO_CONTENT), + ); + let services = build_services_with_outbound_client(&bidders, &clock); let mut ec_context = make_ec_context(Jurisdiction::NonRegulated, None); let body = json!({ "adUnits": [{ @@ -1115,27 +1028,119 @@ mod tests { .expect("direct auction should remain available"); assert_eq!( - *calls.lock().expect("should lock provider call count"), + bidders.requests().len(), 1, "disabling publisher templates must not disable direct /auction" ); assert_eq!(response.status(), StatusCode::OK); } + #[tokio::test] + async fn all_planned_launch_failures_return_bad_gateway_and_execution_failed_telemetry() { + // The plan accepts a backslash in the endpoint query, but `EdgeZero` + // builds no request for it. The other endpoint gets the error Fastly + // reports when it cannot register the bidder backend. + let unbuildable = r"https://failing.example/auction?x=\"; + let unregistered = "https://failing.example/auction"; + for endpoint in [unbuildable, unregistered] { + let settings_toml = format!( + "{}\n[auction]\nenabled = true\n\n[auction.providers.launch-fail]\nprotocol = \"openrtb-2.6\"\nprofile = \"standard\"\nendpoint = '{endpoint}'\nrouting = \"all_eligible\"\n", + crate_test_settings_str() + ); + let settings = + Settings::from_toml(&settings_toml).expect("should parse launch-failure settings"); + let orchestrator = crate::auction::build_orchestrator(&settings) + .expect("should build the orchestrator"); + let telemetry_sink = Arc::new(RecordingTelemetrySink::default()); + let clock = TestClock::new(); + let bidders = ScriptedHttpClient::new(&clock); + bidders.respond_to( + http::Method::POST, + unregistered, + ScriptedReply::error(EdgeError::bad_gateway_with_detail( + "Fastly failed to register a dynamic backend", + BadGatewayReason::Unspecified, + ProviderFailureDetail::Fastly { + phase: ProviderFailurePhase::BackendRegistration, + cause: FastlyFailureCause::BackendHostError { status: 1 }, + }, + )), + ); + let services = RuntimeServices { + outbound_client: bidders.http_client(), + clock: clock.monotonic_clock(), + ..services_with_telemetry(Arc::clone(&telemetry_sink)) + }; + let mut ec_context = make_ec_context(Jurisdiction::NonRegulated, None); + let body = json!({ + "adUnits": [{ + "code": "div-gpt-ad-1", + "mediaTypes": { "banner": { "sizes": [[300, 250]] } } + }] + }); + let request = Request::builder() + .method("POST") + .uri("https://test-publisher.example/auction") + .body(EdgeBody::from( + serde_json::to_vec(&body).expect("should serialize launch-failure body"), + )) + .expect("should build launch-failure request"); + + let error = handle_auction( + &settings, + &orchestrator, + None, + None, + &mut ec_context, + &services, + request, + ) + .await + .expect_err("should fail the auction endpoint when every planned launch fails"); + + assert_eq!( + error.current_context().status_code(), + StatusCode::BAD_GATEWAY, + "{endpoint} should answer 502" + ); + let batches = telemetry_sink + .batches + .lock() + .expect("should lock telemetry batches"); + assert_eq!( + batches.len(), + 1, + "{endpoint} should emit one telemetry batch" + ); + let rows = batches[0].rows(); + assert_eq!( + rows.len(), + 1, + "{endpoint} should emit one execution-failure summary" + ); + assert_eq!(rows[0].event_kind, "summary"); + assert_eq!(rows[0].terminal_status.as_deref(), Some("execution_failed")); + assert_eq!(rows[0].terminal_reason.as_deref(), Some("execution_failed")); + } + } + #[tokio::test] async fn disabled_auction_endpoint_emits_skipped_telemetry_without_provider_work() { let settings = create_test_settings(); let config = AuctionConfig { enabled: false, - providers: AuctionConfig::legacy_provider_map(&["panic_provider"]), + providers: AuctionConfig::legacy_provider_map(&["never-contacted"]), timeout_ms: 2000, mediator: None, ..Default::default() }; - let mut orchestrator = AuctionOrchestrator::new(config); - orchestrator.register_provider(Arc::new(PanicOnBidProvider)); + let orchestrator = orchestrator_with(config); let telemetry_sink = Arc::new(RecordingTelemetrySink::default()); - let services = services_with_telemetry(Arc::clone(&telemetry_sink)); + let bidders = ScriptedHttpClient::forbidding(); + let services = RuntimeServices { + outbound_client: bidders.http_client(), + ..services_with_telemetry(Arc::clone(&telemetry_sink)) + }; let mut ec_context = make_ec_context(Jurisdiction::NonRegulated, None); let body = json!({ "adUnits": [{ @@ -1182,6 +1187,7 @@ mod tests { Some("auction_disabled"), "should identify the disabled auction policy" ); + bidders.assert_no_requests(); } #[tokio::test] @@ -1212,20 +1218,21 @@ mod tests { ..Default::default() }); let orchestrator = AuctionOrchestrator::from_plan(plan, Some(Arc::new(mediator))); - let http = Arc::new(StubHttpClient::new()); - http.push_response(204, Vec::new()); - http.push_response(204, Vec::new()); - let services = RuntimeServices::builder() - .config_store(Arc::new(NoopConfigStore)) - .secret_store(Arc::new(NoopSecretStore)) - .kv_store(Arc::new(edgezero_core::key_value_store::NoopKvStore)) - .backend(Arc::new(StubBackend)) - .http_client(Arc::clone(&http) as Arc<_>) - .geo(Arc::new(NoopGeo)) - .client_info(ClientInfo::default()) - .build(); + let clock = TestClock::new(); + let http = ScriptedHttpClient::new(&clock); + for endpoint in [ + "https://bidder.example.com/auction", + "https://mediator.example.com/auction", + ] { + http.respond_to( + http::Method::POST, + endpoint, + ScriptedReply::status(StatusCode::NO_CONTENT), + ); + } + let services = build_services_with_outbound_client(&http, &clock); let body = json!({"adUnits":[{"code":"slot","mediaTypes":{"banner":{"sizes":[[300,250]]}},"ext":{"trusted_server":{"trace_slot_ref":SLOT_REF}}}]}); - let mut request = Request::builder() + let request = Request::builder() .method("POST") .uri("https://publisher.example.com/auction") .header(header::COOKIE, "__Host-ts-console=1") @@ -1233,11 +1240,13 @@ mod tests { serde_json::to_vec(&body).expect("should encode source token"), )) .expect("should build request"); - TracePreDispatchHook::new( - Arc::clone(&settings), - Arc::new(|_| panic!("should not load API setup metadata")), + let request = run_pre_dispatch_hook( + TracePreDispatchHook::new( + Arc::clone(&settings), + Arc::new(|_| panic!("should not load API setup metadata")), + ), + request, ) - .handle(&mut request) .await .expect("should freeze trace gate"); let mut ec_context = make_ec_context(Jurisdiction::NonRegulated, None); @@ -1273,14 +1282,15 @@ mod tests { evidence["slots"][0]["candidate"], "no_candidate", "should observe no delivered candidate" ); - let bodies = http.recorded_request_bodies(); + let sent = http.requests(); assert_eq!( - bodies.len(), + sent.len(), 2, "should actually serialize bidder and mediator requests" ); - for bytes in bodies { - let body = String::from_utf8(bytes).expect("should serialize UTF-8 outbound JSON"); + for request in sent { + let body = String::from_utf8(request.body.to_vec()) + .expect("should serialize UTF-8 outbound JSON"); for private in [ SLOT_REF, auction_token, @@ -1310,7 +1320,7 @@ mod tests { ) .expect("should enable trace feature"); let settings = Arc::new(settings); - let orchestrator = AuctionOrchestrator::new(AuctionConfig { + let orchestrator = orchestrator_with(AuctionConfig { enabled: false, ..Default::default() }); @@ -1335,11 +1345,13 @@ mod tests { "__Host-ts-console=1".parse().expect("should build cookie"), ); } - TracePreDispatchHook::new( - Arc::clone(&settings), - Arc::new(|_| panic!("should not load API setup metadata")), + let request = run_pre_dispatch_hook( + TracePreDispatchHook::new( + Arc::clone(&settings), + Arc::new(|_| panic!("should not load API setup metadata")), + ), + request, ) - .handle(&mut request) .await .expect("should freeze cookie gate"); let response = handle_auction( @@ -1433,15 +1445,18 @@ mod tests { let config = AuctionConfig { enabled: false, - providers: AuctionConfig::legacy_provider_map(&["panic_provider"]), + providers: AuctionConfig::legacy_provider_map(&["never-contacted"]), timeout_ms: 2000, mediator: None, ..Default::default() }; - let mut orchestrator = AuctionOrchestrator::new(config); - orchestrator.register_provider(Arc::new(PanicOnBidProvider)); + let orchestrator = orchestrator_with(config); let telemetry_sink = Arc::new(RecordingTelemetrySink::default()); - let services = services_with_telemetry(Arc::clone(&telemetry_sink)); + let bidders = ScriptedHttpClient::forbidding(); + let services = RuntimeServices { + outbound_client: bidders.http_client(), + ..services_with_telemetry(Arc::clone(&telemetry_sink)) + }; let mut ec_context = make_ec_context(Jurisdiction::NonRegulated, None); let body = json!({ "adUnits": [{ @@ -1449,7 +1464,7 @@ mod tests { "mediaTypes": { "banner": { "sizes": [[300, 250]] } } }] }); - let mut request = Request::builder() + let request = Request::builder() .method("POST") .header("cookie", "__Host-ts-console=1") .uri("https://test-publisher.example/auction") @@ -1462,13 +1477,9 @@ mod tests { Arc::clone(&settings), Arc::new(|_| panic!("should not load setup metadata on API capture")), ); - assert!( - hook.handle(&mut request) - .await - .expect("should freeze API cookies") - .is_none(), - "should retain ordinary API routing" - ); + let request = run_pre_dispatch_hook(hook, request) + .await + .expect("should retain ordinary API routing"); let response = handle_auction( &settings, @@ -1515,64 +1526,7 @@ mod tests { response["ext"]["trusted_server"]["trace_auction"], facts, "should deliver the same checked terminal facts" ); - } - - #[tokio::test] - async fn all_planned_launch_failures_return_bad_gateway_and_execution_failed_telemetry() { - let settings_toml = format!( - "{}\n[auction]\nenabled = true\n\n[auction.providers.launch-fail]\nprotocol = \"openrtb-2.6\"\nprofile = \"standard\"\nendpoint = \"https://bidder.example/auction\"\nrouting = \"all_eligible\"\n", - crate_test_settings_str() - ); - let settings = - Settings::from_toml(&settings_toml).expect("should parse launch-failure settings"); - let plan = Arc::new( - crate::auction::compile_auction_plan(&settings) - .expect("should compile launch-failure plan"), - ); - let orchestrator = AuctionOrchestrator::from_plan(plan, None); - let telemetry_sink = Arc::new(RecordingTelemetrySink::default()); - let services = services_with_telemetry(Arc::clone(&telemetry_sink)); - let mut ec_context = make_ec_context(Jurisdiction::NonRegulated, None); - let body = json!({ - "adUnits": [{ - "code": "div-gpt-ad-1", - "mediaTypes": { "banner": { "sizes": [[300, 250]] } } - }] - }); - let request = Request::builder() - .method("POST") - .uri("https://test-publisher.example/auction") - .body(EdgeBody::from( - serde_json::to_vec(&body).expect("should serialize launch-failure body"), - )) - .expect("should build launch-failure request"); - - let error = handle_auction( - &settings, - &orchestrator, - None, - None, - &mut ec_context, - &services, - request, - ) - .await - .expect_err("all planned launch failures should fail the auction endpoint"); - - assert_eq!( - error.current_context().status_code(), - StatusCode::BAD_GATEWAY - ); - let batches = telemetry_sink - .batches - .lock() - .expect("should lock telemetry batches"); - assert_eq!(batches.len(), 1, "should emit one telemetry batch"); - let rows = batches[0].rows(); - assert_eq!(rows.len(), 1, "should emit one execution-failure summary"); - assert_eq!(rows[0].event_kind, "summary"); - assert_eq!(rows[0].terminal_status.as_deref(), Some("execution_failed")); - assert_eq!(rows[0].terminal_reason.as_deref(), Some("execution_failed")); + bidders.assert_no_requests(); } #[tokio::test] @@ -1584,15 +1538,18 @@ mod tests { let settings = create_test_settings(); let config = AuctionConfig { enabled: true, - providers: AuctionConfig::legacy_provider_map(&["panic_provider"]), + providers: AuctionConfig::legacy_provider_map(&["never-contacted"]), timeout_ms: 2000, mediator: None, ..Default::default() }; - let mut orchestrator = AuctionOrchestrator::new(config); - orchestrator.register_provider(Arc::new(PanicOnBidProvider)); + let orchestrator = orchestrator_with(config); let telemetry_sink = Arc::new(RecordingTelemetrySink::default()); - let services = services_with_telemetry(Arc::clone(&telemetry_sink)); + let bidders = ScriptedHttpClient::forbidding(); + let services = RuntimeServices { + outbound_client: bidders.http_client(), + ..services_with_telemetry(Arc::clone(&telemetry_sink)) + }; let ec_id = format!("{}.ABC123", "a".repeat(64)); let mut ec_context = make_ec_context(Jurisdiction::Unknown, Some(&ec_id)); @@ -1658,47 +1615,7 @@ mod tests { !ndjson.contains(&ec_id), "telemetry must not serialize EC identifiers" ); - } - - /// Provider that records whether the auction request it received carried - /// EIDs, then fails its launch so no real transport handle is needed. - struct EidCapturingProvider { - had_eids: Arc>>, - } - - #[async_trait::async_trait(?Send)] - impl AuctionProvider for EidCapturingProvider { - fn provider_name(&self) -> &str { - "eid_capturing_provider" - } - - async fn request_bids( - &self, - request: &AuctionRequest, - _context: &AuctionContext<'_>, - ) -> Result> { - *self.had_eids.lock().expect("should lock captured eids") = - Some(request.user.eids.is_some()); - Err(Report::new(TrustedServerError::Auction { - message: "capture only".to_string(), - })) - } - - async fn parse_response( - &self, - _response: PlatformResponse, - _response_time_ms: u64, - ) -> Result> { - panic!("parse_response must not run when the launch fails"); - } - - fn timeout_ms(&self) -> u32 { - 100 - } - - fn backend_name(&self, _services: &RuntimeServices, _timeout_ms: u32) -> Option { - Some("capture-backend".to_string()) - } + bidders.assert_no_requests(); } #[tokio::test] @@ -1712,17 +1629,20 @@ mod tests { let settings = create_test_settings(); let config = AuctionConfig { enabled: true, - providers: AuctionConfig::legacy_provider_map(&["eid_capturing_provider"]), + providers: AuctionConfig::legacy_provider_map(&["eid-capture"]), timeout_ms: 2000, mediator: None, ..Default::default() }; - let mut orchestrator = AuctionOrchestrator::new(config); - let had_eids = Arc::new(std::sync::Mutex::new(None)); - orchestrator.register_provider(Arc::new(EidCapturingProvider { - had_eids: Arc::clone(&had_eids), - })); - let services = noop_services(); + let orchestrator = orchestrator_with(config); + let clock = TestClock::new(); + let bidders = ScriptedHttpClient::new(&clock); + bidders.respond_to( + http::Method::POST, + EID_CAPTURE_ENDPOINT, + ScriptedReply::status(StatusCode::NO_CONTENT), + ); + let services = build_services_with_outbound_client(&bidders, &clock); // US-state jurisdiction with an explicit GPC opt-out: auction allowed, // EC identity denied. @@ -1767,8 +1687,7 @@ mod tests { )) .expect("should build auction request"); - // The capturing provider fails its launch, so the auction errors overall; - // the assertion is on the EIDs observed by the provider, not the result. + // The assertion is on the EIDs the bidder received, not the auction result. let _ = handle_auction( &settings, &orchestrator, @@ -1780,10 +1699,17 @@ mod tests { ) .await; + let sent = bidders.requests(); assert_eq!( - *had_eids.lock().expect("should lock captured eids"), - Some(false), - "outgoing auction request must carry no EIDs when EC identity is denied" + sent.len(), + 1, + "should send the auction request to the bidder" + ); + let body: serde_json::Value = + serde_json::from_slice(&sent[0].body).expect("should send a JSON bid request"); + assert!( + body["user"]["ext"]["eids"].is_null(), + "outgoing auction request must carry no EIDs when EC identity is denied: {body}" ); } @@ -2199,50 +2125,108 @@ mod tests { }); } + /// Returns an orchestrator with the auction disabled, so a parsed body gets a 200 no-bid. + fn disabled_orchestrator() -> AuctionOrchestrator { + orchestrator_with(AuctionConfig { + enabled: false, + ..Default::default() + }) + } + #[test] - fn auction_rejects_streaming_body_instead_of_treating_as_empty() { + fn auction_reads_a_streamed_body() { futures::executor::block_on(async { - use bytes::Bytes; - use edgezero_core::body::Body as EdgeBody; - use http::{Method, Request as HttpRequest}; - - use crate::auction::build_orchestrator; - use crate::consent::ConsentContext; - use crate::ec::EcContext; - use crate::error::TrustedServerError; - use crate::platform::test_support::noop_services; - use crate::test_support::tests::create_test_settings; - let settings = create_test_settings(); - let orchestrator = build_orchestrator(&settings).expect("should build orchestrator"); + let orchestrator = disabled_orchestrator(); let services = noop_services(); let mut ec_context = EcContext::new_for_test(None, ConsentContext::default()); - let stream = futures::stream::iter([Bytes::from_static(br#"{}"#)]); - let req = HttpRequest::builder() - .method(Method::POST) - .uri("https://test.com/auction") - .body(EdgeBody::stream(stream)) - .expect("should build request"); + let body = serde_json::to_vec(&json!({ + "adUnits": [{ + "code": "div-gpt-ad-1", + "mediaTypes": { "banner": { "sizes": [[300, 250]] } } + }] + })) + .expect("should serialize the auction body"); + let (head, tail) = body.split_at(body.len() / 2); + let chunks = [ + bytes::Bytes::copy_from_slice(head), + bytes::Bytes::copy_from_slice(tail), + ]; + let request = Request::builder() + .method("POST") + .uri("https://test-publisher.example/auction") + .body(EdgeBody::stream(futures::stream::iter(chunks))) + .expect("should build the streamed auction request"); - let result = handle_auction( + let response = handle_auction( &settings, &orchestrator, None, None, &mut ec_context, &services, - req, + request, ) - .await; + .await + .expect("should parse a streamed auction body"); - let err = match result { - Ok(_) => panic!("streaming body should be rejected"), - Err(err) => err, - }; - assert!( - matches!(err.current_context(), TrustedServerError::BadRequest { .. }), - "streaming request body should fail as bad request" + assert_eq!( + response.status(), + StatusCode::OK, + "should answer a streamed body as it answers a buffered one" ); }); } + + #[test] + fn auction_body_read_failures_map_to_statuses() { + futures::executor::block_on(async { + let settings = create_test_settings(); + let orchestrator = disabled_orchestrator(); + let services = noop_services(); + + for case in body_read_failure_cases(MAX_AUCTION_BODY_SIZE) { + let mut ec_context = EcContext::new_for_test(None, ConsentContext::default()); + let (request, expectation) = case.into_request( + Request::builder() + .method("POST") + .uri("https://test-publisher.example/auction"), + ); + + let capture = WarningCapture::start(); + let outcome = handle_auction( + &settings, + &orchestrator, + None, + None, + &mut ec_context, + &services, + request, + ) + .await; + + expectation.assert_outcome("/auction", &outcome); + if expectation.expected() == StatusCode::PAYLOAD_TOO_LARGE { + let levels: Vec = + capture.take().iter().map(|record| record.level).collect(); + assert_eq!( + levels, + [log::Level::Warn], + "should log the {} case once, at warn", + expectation.name() + ); + let response = outcome.expect("should answer an overflow with a response"); + assert_eq!( + response + .headers() + .get(header::CONTENT_TYPE) + .and_then(|value| value.to_str().ok()), + Some("text/plain"), + "should keep the text/plain 413 body for the {} case", + expectation.name() + ); + } + } + }); + } } diff --git a/crates/trusted-server-core/src/auction/mod.rs b/crates/trusted-server-core/src/auction/mod.rs index 4323039284..6495bf1d6f 100644 --- a/crates/trusted-server-core/src/auction/mod.rs +++ b/crates/trusted-server-core/src/auction/mod.rs @@ -186,29 +186,6 @@ mod plan_sharing_tests { .expect("should build with enabled exact mediator registration"); } - #[test] - fn cloudflare_and_spin_reject_multi_provider_plans_before_runtime_construction() { - let mut settings = create_test_settings(); - settings.auction.enabled = true; - settings.auction.providers = - AuctionConfig::legacy_provider_map(&["provider-a", "provider-b"]); - let plan = compile_auction_plan(&settings).expect("should compile target-independent plan"); - - for target in [ - crate::platform::AuctionTargetId::Cloudflare, - crate::platform::AuctionTargetId::Spin, - ] { - let error = plan - .validate_for_target(target) - .expect_err("should reject unsupported multi-provider fanout"); - assert!( - error - .to_string() - .contains("does not support concurrent provider fanout") - ); - } - } - #[test] fn aps_profile_registers_renderer_without_browser_aps_config() { let mut settings = create_test_settings(); diff --git a/crates/trusted-server-core/src/auction/openrtb.rs b/crates/trusted-server-core/src/auction/openrtb.rs index 12c885d88e..43c1ce2658 100644 --- a/crates/trusted-server-core/src/auction/openrtb.rs +++ b/crates/trusted-server-core/src/auction/openrtb.rs @@ -995,7 +995,5 @@ mod routing_metadata_tests { } } -#[cfg(test)] -mod test_executor; #[cfg(test)] mod tests; diff --git a/crates/trusted-server-core/src/auction/openrtb/test_executor.rs b/crates/trusted-server-core/src/auction/openrtb/test_executor.rs deleted file mode 100644 index d322102e62..0000000000 --- a/crates/trusted-server-core/src/auction/openrtb/test_executor.rs +++ /dev/null @@ -1,107 +0,0 @@ -//! Fictional standard-profile executor compiled only for automated tests. - -use edgezero_core::body::Body as EdgeBody; -use error_stack::{Report, ResultExt as _}; -use http::{Method, Request, StatusCode, header}; -use serde_json::{Value, json}; - -use super::{apply_notification_policy, extract_standard_response, unused_bidder_params_count}; -use crate::auction::plan::ProviderPlan; -use crate::auction::routing::ProviderAuctionInput; -use crate::auction::types::AuctionResponse; -use crate::error::TrustedServerError; -use crate::platform::{PlatformBackend, PlatformHttpClient, PlatformHttpRequest}; - -const MAX_STANDARD_RESPONSE_BYTES: usize = 1024 * 1024; - -/// Execute one fictional standard-profile request through a supplied test client. -/// -/// The HTTP client receives exactly one request. Redirect statuses are -/// classified as the original provider error and never followed. -pub(super) async fn execute_standard_fixture( - provider: &ProviderPlan, - input: &ProviderAuctionInput, - request: &trusted_server_openrtb::BidRequest, - backend: &dyn PlatformBackend, - http_client: &dyn PlatformHttpClient, -) -> Result> { - let spec = provider.backend_spec(); - let predicted_name = - backend - .predict_name(&spec) - .change_context(TrustedServerError::Auction { - message: "Failed to predict fictional standard backend".to_string(), - })?; - let backend_name = backend - .ensure(&spec) - .change_context(TrustedServerError::Auction { - message: "Failed to ensure fictional standard backend".to_string(), - })?; - if backend_name != predicted_name { - return Err(Report::new(TrustedServerError::Auction { - message: "Fictional standard backend ensure did not match prediction".to_string(), - })); - } - let body = serde_json::to_vec(request).change_context(TrustedServerError::Auction { - message: "Failed to serialize fictional standard request".to_string(), - })?; - let outbound = Request::builder() - .method(Method::POST) - .uri(provider.endpoint.as_str()) - .header(header::CONTENT_TYPE, "application/json") - .header(header::ACCEPT, "application/json") - .body(EdgeBody::from(body)) - .change_context(TrustedServerError::Auction { - message: "Failed to build fictional standard request".to_string(), - })?; - let response = http_client - .send(PlatformHttpRequest::new(outbound, backend_name)) - .await - .change_context(TrustedServerError::Auction { - message: "Fictional standard transport failed".to_string(), - })? - .response; - let status = response.status(); - if status == StatusCode::NO_CONTENT { - return Ok( - AuctionResponse::no_bid(provider.id.as_str(), 0).with_metadata( - "routing", - json!({"unused_bidder_params_count": unused_bidder_params_count(&provider.profile, input)}), - ), - ); - } - if !status.is_success() { - return Ok(AuctionResponse::error(provider.id.as_str(), 0) - .with_metadata("error_type", json!("http_status")) - .with_metadata("http_status", json!(status.as_u16())) - .with_metadata( - "routing", - json!({"unused_bidder_params_count": unused_bidder_params_count(&provider.profile, input)}), - )); - } - let body = response - .into_body() - .into_bytes_bounded(MAX_STANDARD_RESPONSE_BYTES) - .await - .change_context(TrustedServerError::Auction { - message: "Fictional standard response exceeded its limit".to_string(), - })?; - let value: Value = match serde_json::from_slice(&body) { - Ok(value) => value, - Err(_) => { - return Ok(AuctionResponse::error(provider.id.as_str(), 0) - .with_metadata("error_type", json!("parse_response")) - .with_metadata( - "routing", - json!({"unused_bidder_params_count": unused_bidder_params_count(&provider.profile, input)}), - )); - } - }; - let mut parsed = extract_standard_response(provider.id.as_str(), input, &value, 0); - apply_notification_policy(&mut parsed.bids, &provider.notifications); - parsed.metadata.insert( - "routing".to_string(), - json!({"unused_bidder_params_count": unused_bidder_params_count(&provider.profile, input)}), - ); - Ok(parsed) -} diff --git a/crates/trusted-server-core/src/auction/openrtb/tests.rs b/crates/trusted-server-core/src/auction/openrtb/tests.rs index f47efa475e..63c54f7669 100644 --- a/crates/trusted-server-core/src/auction/openrtb/tests.rs +++ b/crates/trusted-server-core/src/auction/openrtb/tests.rs @@ -1,30 +1,29 @@ -use std::collections::{BTreeMap, BTreeSet, HashMap, HashSet}; +use std::collections::{BTreeMap, BTreeSet, HashMap}; use std::str::FromStr as _; -use std::sync::Arc; use base64::Engine as _; use edgezero_core::body::Body as EdgeBody; -use http::{Request, header}; +use http::{Method, Request, StatusCode, header}; use serde_json::{Value, json}; -use super::test_executor::execute_standard_fixture; use super::*; use crate::auction::plan::{ AuctionPlan, AuctionPlanConfig, BidderId, BidderRouteConfig, NotificationConfig, ProviderConfig, ProviderId, RoutingMode, }; use crate::auction::provider::{GenericOpenRtbProvider, ProviderRequestOutcome}; +use crate::auction::routing::RoutedAuction; use crate::auction::routing::route_auction; use crate::auction::test_support::canonical_parity_auction_request; +use crate::auction::types::AuctionResponse; use crate::auction::types::{AdFormat, AdSlot, BidStatus, MediaType}; use crate::consent::jurisdiction::Jurisdiction; use crate::consent::{ConsentContext, ConsentSource}; -use crate::platform::PlatformHttpClient; use crate::platform::test_support::{ - HashMapConfigStore, HashMapSecretStore, NoopHttpClient, StubBackend, StubHttpClient, - build_services_with_backend_and_http_client, build_services_with_config_secret_and_http_client, + HashMapConfigStore, HashMapSecretStore, build_services_with_config_and_secret, }; use crate::request_signing::RequestSigner; +use crate::test_support::scripted_client::{ScriptedHttpClient, ScriptedReply, TestClock}; fn config(profile: &str, profile_config: Value) -> AuctionPlanConfig { config_with_endpoint( @@ -163,10 +162,9 @@ fn deterministic_signer() -> RequestSigner { .encode([7_u8; 32]) .into_bytes(), ); - let services = build_services_with_config_secret_and_http_client( + let services = build_services_with_config_and_secret( HashMapConfigStore::new(config_data), HashMapSecretStore::new(secret_data), - Arc::new(NoopHttpClient), ); RequestSigner::from_services(&services).expect("should load deterministic signer") } @@ -1387,105 +1385,116 @@ fn notification_suppression_matrix_uses_only_exact_valid_returned_seat() { ); } +/// Builds, sends, and parses one request through the real provider and a scripted client. +/// +/// The client answers the provider's endpoint with `reply`, and the parsed response comes +/// back with the client so the test can read what was sent. +async fn execute_through_provider( + plan: &AuctionPlan, + routed: &RoutedAuction, + reply: ScriptedReply, +) -> (AuctionResponse, ScriptedHttpClient) { + let provider_plan = plan.providers()[0].clone(); + let provider = GenericOpenRtbProvider::new(provider_plan.clone()); + let client = ScriptedHttpClient::new(&TestClock::new()); + client.respond_to(Method::POST, provider_plan.endpoint.as_str(), reply); + let ProviderRequestOutcome::Ready { + request, + parse_state, + } = provider + .request_bids_routed(&routed.inputs()[0], routed, 321, None) + .expect("should build one provider request") + else { + panic!("should build an unsent provider request"); + }; + let response = client + .http_client() + .send(request) + .await + .expect("should answer the provider request"); + let parsed = provider + .parse_response_with_state(response, 0, parse_state.as_deref()) + .await + .expect("should parse the provider response"); + (parsed, client) +} + #[test] -fn fictional_standard_executor_covers_bid_no_bid_malformed_unused_and_redirect() { +fn standard_provider_covers_bid_no_bid_malformed_unused_and_redirect() { futures::executor::block_on(async { - let (plan, routed, request) = standard_fixture(); - let provider = &plan.providers()[0]; - let client = Arc::new(StubHttpClient::new()); - client.push_response( - 200, - serde_json::to_vec(&json!({"id":"mismatch","seatbid":[{"seat":"fictional-seat","bid":[ - {"id":"good","impid":"fictional-slot","price":1.25,"adm":"
fictional
","w":300,"h":250}, - {"id":"bad","impid":"fictional-slot","price":null,"adm":"bad","w":300,"h":250} - ]}]})).expect("should serialize response"), - ); - let response = execute_standard_fixture( - provider, - &routed.inputs()[0], - &request, - &StubBackend, - client.as_ref(), + let (plan, routed, _request) = standard_fixture(); + let endpoint = plan.providers()[0].endpoint.as_str().to_owned(); + let (response, client) = execute_through_provider( + &plan, + &routed, + ScriptedReply::status(StatusCode::OK).body( + serde_json::to_vec(&json!({"id":"mismatch","seatbid":[{"seat":"fictional-seat","bid":[ + {"id":"good","impid":"fictional-slot","price":1.25,"adm":"
fictional
","w":300,"h":250}, + {"id":"bad","impid":"fictional-slot","price":null,"adm":"bad","w":300,"h":250} + ]}]})).expect("should serialize response"), + ), ) - .await - .expect("should execute ordinary fixture"); + .await; assert_eq!(response.status, BidStatus::Success); assert_eq!(response.bids.len(), 1, "should isolate malformed sibling"); assert_eq!( response.metadata["routing"]["unused_bidder_params_count"], 1 ); + let sent = client.requests(); assert_eq!( - client.recorded_request_uris(), - vec![provider.endpoint.as_str()] + sent.iter() + .map(|request| request.uri.to_string()) + .collect::>(), + vec![endpoint.clone()] ); - let headers = &client.recorded_request_headers()[0]; assert!( - !headers.iter().any(|(name, _)| name == "authorization"), - "fixture must add no authentication" + !sent[0].headers.contains_key(header::AUTHORIZATION), + "should add no authentication" ); - let body: Value = serde_json::from_slice(&client.recorded_request_bodies()[0]) - .expect("should parse recorded body"); + let body: Value = + serde_json::from_slice(&sent[0].body).expect("should parse recorded body"); assert_eq!(body["ext"]["fixture"], true, "should send static extension"); - let no_bid = Arc::new(StubHttpClient::new()); - no_bid.push_response(204, Vec::new()); - let response = execute_standard_fixture( - provider, - &routed.inputs()[0], - &request, - &StubBackend, - no_bid.as_ref(), + let (response, _client) = execute_through_provider( + &plan, + &routed, + ScriptedReply::status(StatusCode::NO_CONTENT), ) - .await - .expect("should execute no-bid fixture"); + .await; assert_eq!(response.status, BidStatus::NoBid); - let malformed = Arc::new(StubHttpClient::new()); - malformed.push_response(200, b"not-json".to_vec()); - let response = execute_standard_fixture( - provider, - &routed.inputs()[0], - &request, - &StubBackend, - malformed.as_ref(), + let (response, _client) = execute_through_provider( + &plan, + &routed, + ScriptedReply::status(StatusCode::OK).body("not-json"), ) - .await - .expect("should classify malformed response"); + .await; assert_eq!(response.status, BidStatus::Error); assert_eq!(response.metadata["error_type"], "parse_response"); - let redirect = Arc::new(StubHttpClient::new()); - redirect.push_response_with_headers( - 302, - Vec::new(), - vec![("location", "https://redirect.example.test/openrtb")], - ); - let response = execute_standard_fixture( - provider, - &routed.inputs()[0], - &request, - &StubBackend, - redirect.as_ref(), + let (response, redirect) = execute_through_provider( + &plan, + &routed, + ScriptedReply::status(StatusCode::FOUND) + .header("location", "https://redirect.example.test/openrtb"), ) - .await - .expect("should classify redirect"); + .await; assert_eq!(response.status, BidStatus::Error); assert_eq!(response.metadata["error_type"], "http_status"); assert_eq!(response.metadata["http_status"], 302); + let sent = redirect.requests(); assert_eq!( - redirect.recorded_request_uris(), - vec![provider.endpoint.as_str()], + sent.iter() + .map(|request| request.uri.to_string()) + .collect::>(), + vec![endpoint], "a 3xx Location must not trigger a second HTTP request" ); assert_eq!( - redirect.recorded_backend_names(), - vec!["stub-backend"], - "the common driver must perform exactly one underlying send for a 3xx" + sent[0].host_authority, "exchange.example.test", + "should send the request to the provider's own authority" ); - let spec = provider.backend_spec(); - assert_eq!(spec.host, "exchange.example.test"); - assert_eq!(spec.discriminator.as_deref(), Some("fictional-provider")); }); } @@ -1510,47 +1519,27 @@ fn prebid_endpoint_normalization_reaches_generic_execution_and_preserves_custom_ .body(EdgeBody::empty()) .expect("should build inbound request"); let routed = route_auction(canonical_parity_auction_request(), &inbound, &plan, None); - let provider_plan = plan.providers()[0].clone(); - let provider = GenericOpenRtbProvider::new(provider_plan.clone()); - let client = Arc::new(StubHttpClient::new()); - client.push_response(204, Vec::new()); - let services = build_services_with_backend_and_http_client( - Arc::new(StubBackend), - Arc::clone(&client) as Arc, - ); - let mut reserved_backend_names = HashSet::new(); - let outcome = provider - .request_bids_routed( - &routed.inputs()[0], - &routed, - 321, - 321, - None, - &services, - &mut reserved_backend_names, - ) - .await - .expect("should launch one Prebid Server request"); - let ProviderRequestOutcome::Pending { request, .. } = outcome else { - panic!("should launch a pending Prebid Server request"); - }; - let selected = services - .http_client() - .select(vec![request]) - .await - .expect("should select one Prebid Server response"); - let response = selected - .ready - .expect("should receive the Prebid Server response"); - - assert_eq!(response.response.status(), http::StatusCode::NO_CONTENT); - assert_eq!(client.recorded_backend_names(), vec!["stub-backend"]); - assert_eq!(client.recorded_request_methods(), vec!["POST"]); - assert_eq!(client.recorded_request_uris(), vec![expected_endpoint]); - assert_eq!(provider_plan.backend_spec().host, "pbs.example"); - let body: Value = serde_json::from_slice(&client.recorded_request_bodies()[0]) - .expect("should parse recorded Prebid Server body"); + let (response, client) = execute_through_provider( + &plan, + &routed, + ScriptedReply::status(StatusCode::NO_CONTENT), + ) + .await; + + let sent = client.requests(); + assert_eq!(sent.len(), 1, "should send exactly one request"); + assert_eq!( + response.provider, "fictional-provider", + "should parse the provider response" + ); + assert_eq!( + (sent[0].method.as_str(), sent[0].uri.to_string()), + ("POST", expected_endpoint.to_owned()) + ); + assert_eq!(sent[0].host_authority, "pbs.example"); + let body: Value = + serde_json::from_slice(&sent[0].body).expect("should parse recorded body"); assert!( !body .as_object() diff --git a/crates/trusted-server-core/src/auction/orchestrator.rs b/crates/trusted-server-core/src/auction/orchestrator.rs index fb229faa5e..82ff701cdf 100644 --- a/crates/trusted-server-core/src/auction/orchestrator.rs +++ b/crates/trusted-server-core/src/auction/orchestrator.rs @@ -1,25 +1,25 @@ //! Auction orchestrator for managing multi-provider auctions. -use edgezero_core::body::Body as EdgeBody; -use error_stack::{Report, ResultExt}; -use http::Request; -use std::collections::{HashMap, HashSet, hash_map::Entry}; +use edgezero_core::ProviderFailurePhase; +use edgezero_core::error::EdgeError; +use edgezero_core::outbound::{ + OutboundBatch, OutboundBatchCheck, OutboundBatchNext, OutboundResponse, OutboundSlotResult, +}; +use edgezero_core::time::{Deadline, MonotonicClock, MonotonicInstant}; +use error_stack::Report; +use std::collections::HashMap; use std::sync::Arc; use std::time::Duration; -use web_time::Instant; use crate::error::TrustedServerError; -use crate::platform::{PlatformPendingRequest, RuntimeServices}; +use crate::outbound::{OutboundRequestSource, outbound_error}; +use crate::platform::RuntimeServices; -#[cfg(test)] -use super::config::AuctionConfig; use super::openrtb::unused_bidder_params_count; use super::plan::AuctionPlan; use super::provider::{ AuctionProvider, GenericOpenRtbProvider, ProviderParseState, ProviderRequestOutcome, }; -#[cfg(test)] -use super::routing::RoutedAuction; use super::routing::route_auction; use super::telemetry::AbandonedProviderCall; use super::types::{AuctionContext, AuctionRequest, AuctionResponse, Bid, BidStatus}; @@ -32,19 +32,22 @@ use crate::trace::{ /// In-flight auction requests dispatched to SSP backends. /// /// Created by [`AuctionOrchestrator::dispatch_auction`] and consumed by -/// [`AuctionOrchestrator::collect_dispatched_auction`]. Carrying this handle -/// across the publisher origin wait lets provider operations progress while -/// the page loads. Collection retains available responses and never restarts -/// the original auction budget. +/// [`AuctionOrchestrator::collect_dispatched_auction`]. The bidder requests +/// travel as one `EdgeZero` batch whose cutoff is the auction deadline, so they +/// progress while the page loads. Collection keeps the bids that arrived in +/// time and never restarts the auction budget. Dropping the token cancels +/// every bidder request still in flight. pub struct DispatchedAuction { - pending_requests: Vec, - backend_to_provider: HashMap, - planned_backend_to_provider: HashMap, + /// The bidder batch, absent when no bidder request was built. + batch: Option, + /// The bidder behind each batch slot, at the slot's index, until the slot + /// is settled. + slot_providers: Vec>, completed_responses: Vec, - auction_start: Instant, - timeout_ms: u32, + clock: MonotonicClock, + auction_start: MonotonicInstant, + auction_deadline: Deadline, floor_prices: HashMap, - provider_request_context: Box>, /// Carried so the mediator call in collect can pass it as the auction request. request: AuctionRequest, planned_unused_bidder_params: HashMap, @@ -53,12 +56,11 @@ pub struct DispatchedAuction { trace: Option, } -struct ProviderLaunchState { - provider_name: String, - started_at: Instant, - provider: Arc, - effective_timeout_ms: u32, +/// One bidder request's provider and the state its response needs. +struct BidderSlot { + provider: Arc, parse_state: Option, + trace_observation: Option, } /// Outcome of attempting to dispatch split-phase auction provider requests. @@ -91,6 +93,8 @@ pub enum DispatchAuctionOutcome { impl DispatchedAuction { /// Consume the dispatch token without collecting provider responses. + /// + /// Cancels every bidder request still in flight, on the guest side. #[must_use] pub fn abandon( self, @@ -106,20 +110,15 @@ impl DispatchedAuction { Some(TraceAuctionTerminalReason::Unknown), ); } - let elapsed_ms = self.auction_start.elapsed().as_millis() as u64; - let abandoned = self - .backend_to_provider - .into_values() - .map(|state| (state.provider_name, state.started_at)) - .chain( - self.planned_backend_to_provider - .into_values() - .map(|state| (state.provider.provider_name().to_string(), state.started_at)), - ) - .map(|(provider_name, started_at)| { + let elapsed_ms = elapsed_ms(&self.clock, self.auction_start); + let undelivered = self.batch.map(OutboundBatch::cancel).unwrap_or_default(); + let abandoned = undelivered + .into_iter() + .filter_map(|index| self.slot_providers.get(index).and_then(Option::as_ref)) + .map(|slot| { AbandonedProviderCall::bidder( - provider_name, - Some(u32::try_from(started_at.elapsed().as_millis()).unwrap_or(u32::MAX)), + slot.provider.provider_name(), + Some(u32::try_from(elapsed_ms).unwrap_or(u32::MAX)), ) }) .collect(); @@ -138,16 +137,20 @@ impl DispatchedAuction { #[cfg(test)] impl DispatchedAuction { + /// A dispatched auction with no bidder in flight, for publisher tests. + /// + /// Its deadline is `timeout_ms` after now on a fresh test clock. pub(crate) fn empty_for_test(request: AuctionRequest, timeout_ms: u32) -> Self { + let clock = crate::test_support::scripted_client::TestClock::new().monotonic_clock(); + let auction_start = clock.now(); Self { - pending_requests: Vec::new(), - backend_to_provider: HashMap::new(), - planned_backend_to_provider: HashMap::new(), + batch: None, + slot_providers: Vec::new(), completed_responses: Vec::new(), - auction_start: Instant::now(), - timeout_ms, + auction_deadline: clock.deadline_after(Duration::from_millis(u64::from(timeout_ms))), + clock, + auction_start, floor_prices: HashMap::new(), - provider_request_context: Box::new(Request::new(EdgeBody::empty())), request, planned_unused_bidder_params: HashMap::new(), planned_unroutable_bidder_count: 0, @@ -156,6 +159,7 @@ impl DispatchedAuction { } } + /// Attaches `trace` to this test auction. pub(crate) fn with_trace_for_test(mut self, trace: TraceAuctionCarry) -> Self { self.trace = Some(trace); self @@ -219,9 +223,9 @@ fn provider_launch_failed_response(provider_name: &str, response_time_ms: u64) - .with_metadata("message", serde_json::json!("Provider launch failed")) } -// Transport failures carry a static message: the underlying select() error is a -// `Report` that may reference upstream-controlled content, so it -// is logged server-side rather than surfaced in the public /auction response. +// Transport failures carry a static message: the underlying client error may +// reference upstream-controlled content, so it is logged server-side rather +// than surfaced in the public /auction response. fn provider_transport_failed_response( provider_name: &str, response_time_ms: u64, @@ -244,19 +248,32 @@ fn provider_skipped_response(provider_name: &str) -> AuctionResponse { ) } -/// Compute the remaining time budget from a deadline. +/// Whether `error` is a failed backend registration, which no request byte +/// follows. /// -/// Returns the number of milliseconds left before `timeout_ms` is exceeded, -/// measured from `start`. Returns `0` when the deadline has already passed. -#[inline] -fn remaining_budget_ms(start: Instant, timeout_ms: u32) -> u32 { - let elapsed = u32::try_from(start.elapsed().as_millis()).unwrap_or(u32::MAX); - timeout_ms.saturating_sub(elapsed) +/// A registration that runs out of budget stays a timeout. +fn failed_backend_registration(error: &EdgeError) -> bool { + !matches!(error, EdgeError::GatewayTimeout { .. }) + && error + .provider_failure_detail() + .is_some_and(|detail| detail.phase() == ProviderFailurePhase::BackendRegistration) +} + +/// Whole milliseconds in `duration`, saturating. +fn duration_ms(duration: Duration) -> u64 { + u64::try_from(duration.as_millis()).unwrap_or(u64::MAX) } -/// Returns the original absolute cutoff without rounding transport budgets. -fn collection_deadline(start: Instant, timeout_ms: u32) -> Instant { - start + Duration::from_millis(u64::from(timeout_ms)) +/// Whole milliseconds `clock` has moved since `since`. +fn elapsed_ms(clock: &MonotonicClock, since: MonotonicInstant) -> u64 { + duration_ms(clock.now().saturating_duration_since(since)) +} + +/// Whole milliseconds left before `deadline` on `clock`, or zero once it passed. +fn remaining_ms(clock: &MonotonicClock, deadline: Deadline) -> u32 { + deadline.remaining_at(clock.now()).map_or(0, |remaining| { + u32::try_from(remaining.as_millis()).unwrap_or(u32::MAX) + }) } fn routing_metadata(unroutable_bidder_count: u32) -> HashMap { @@ -298,582 +315,34 @@ fn materialize_planned_response( response } -fn snapshot_context_request(request: &Request) -> Request { - let mut snapshot = Request::new(EdgeBody::empty()); - *snapshot.method_mut() = request.method().clone(); - *snapshot.uri_mut() = request.uri().clone(); - *snapshot.version_mut() = request.version(); - *snapshot.headers_mut() = request.headers().clone(); - if let Some(prepared) = request - .extensions() - .get::() - { - snapshot.extensions_mut().insert(prepared.clone()); - } - if let Some(origin) = request - .extensions() - .get::() - { - snapshot.extensions_mut().insert(origin.clone()); - } - if let Some(ingress) = request - .extensions() - .get::() - { - snapshot.extensions_mut().insert(ingress.clone()); - } - snapshot +/// Logs one failed bidder or mediator request without its provider text. +/// +/// A timeout is an expected auction outcome, so it logs at debug. Every other +/// failure goes through [`outbound_error`], which logs its typed detail. +fn log_request_failure(role: &str, provider_name: &str, error: EdgeError) { + if matches!(error, EdgeError::GatewayTimeout { .. }) { + log::debug!("{role} '{provider_name}' exhausted its response budget"); + return; + } + let report = outbound_error( + error, + TrustedServerError::Auction { + message: format!("{role} {provider_name} request failed"), + }, + OutboundRequestSource::Internal, + ); + log::warn!("{role} '{provider_name}' request failed: {report:?}"); } /// Manages auction execution across multiple providers. pub struct AuctionOrchestrator { enabled: bool, - plan_backed: bool, plan: Arc, planned_providers: Vec>, mediator: Option>, - #[cfg(test)] - config: AuctionConfig, - #[cfg(test)] - providers: HashMap>, -} - -/// Test harness for the live plan-backed orchestrator semantics. -#[cfg(test)] -pub(crate) struct AuctionOrchestratorHarness { - plan: Arc, - providers: Vec>, - mediator: Option>, -} - -struct PlannedLaunchState { - provider: Arc, - started_at: Instant, - parse_state: Option, - trace_observation: Option, -} - -#[cfg(test)] -#[allow( - dead_code, - reason = "test harness exercises plan-backed runtime behavior" -)] -impl AuctionOrchestratorHarness { - pub(crate) fn new( - plan: impl Into>, - mediator: Option>, - ) -> Self { - let plan = plan.into(); - let providers = plan - .providers() - .iter() - .cloned() - .map(GenericOpenRtbProvider::new) - .map(Arc::new) - .collect(); - Self { - plan, - providers, - mediator, - } - } - - pub(crate) fn provider_count(&self) -> usize { - self.providers.len() - } - - pub(crate) fn mediator(&self) -> Option<&Arc> { - self.mediator.as_ref() - } - - /// Route and execute config-first bidder providers in deterministic order. - pub(crate) async fn run_auction( - &self, - request: &AuctionRequest, - context: &AuctionContext<'_>, - ) -> Result> { - // Admission, including signer-store reads and routing, consumes the same - // request-local deadline as provider transport and response collection. - let auction_start = Instant::now(); - let routed = route_auction( - request.clone(), - context.request, - &self.plan, - context.services.client_info().client_ip, - ); - if context.timeout_ms == 0 { - return self - .run_routed(request, &routed, context, None, auction_start) - .await; - } - if self.providers.len() > 1 && !context.services.http_client().supports_concurrent_fanout() - { - return Err(Report::new(TrustedServerError::Auction { - message: format!( - "{} auction providers configured, but this platform's HTTP client does not support concurrent fanout", - self.providers.len() - ), - })); - } - - // Signing admission deliberately precedes every backend call. - let signer = self - .plan - .signing_enabled() - .then(|| RequestSigner::from_services(context.services)) - .transpose()?; - self.run_routed(request, &routed, context, signer.as_ref(), auction_start) - .await - } - - async fn run_routed( - &self, - original_request: &AuctionRequest, - routed: &RoutedAuction, - context: &AuctionContext<'_>, - signer: Option<&RequestSigner>, - auction_start: Instant, - ) -> Result> { - let mut responses = routed - .skipped_no_eligible_provider_ids() - .iter() - .map(|id| provider_skipped_response(id.as_str())) - .collect::>(); - let planned_unused_bidder_params = routed - .inputs() - .iter() - .map(|input| { - ( - input.provider_id().as_str().to_string(), - unused_bidder_params_count( - &self - .plan - .provider(input.provider_id()) - .expect("should find routed provider in compiled plan") - .profile, - input, - ), - ) - }) - .collect::>(); - let mut pending = Vec::new(); - let mut launches = HashMap::new(); - let mut reserved_backend_names = HashSet::new(); - - for input in routed.inputs() { - let Some(provider) = self - .providers - .iter() - .find(|provider| provider.provider_name() == input.provider_id().as_str()) - .cloned() - else { - responses.push(provider_launch_failed_response( - input.provider_id().as_str(), - 0, - )); - continue; - }; - let remaining_ms = remaining_budget_ms(auction_start, context.timeout_ms); - let logical_budget_ms = remaining_ms.min(provider.timeout_ms()); - if logical_budget_ms == 0 { - log::debug!( - "Planned provider '{}' has no remaining auction budget before launch", - provider.provider_name() - ); - responses.push(provider_timeout_response(provider.provider_name(), 0)); - continue; - } - let transport_timeout_ms = context - .services - .backend() - .canonicalize_transport_timeout_ms(logical_budget_ms, provider.timeout_ms()); - if transport_timeout_ms == 0 { - log::debug!( - "Planned provider '{}' has no remaining transport budget before launch", - provider.provider_name() - ); - responses.push(provider_timeout_response(provider.provider_name(), 0)); - continue; - } - let started_at = Instant::now(); - match provider - .request_bids_routed( - input, - routed, - logical_budget_ms, - transport_timeout_ms, - signer, - context.services, - &mut reserved_backend_names, - ) - .await - { - Ok(ProviderRequestOutcome::Pending { - request: launched, - parse_state, - }) => { - let Some(backend_name) = launched.backend_name().map(str::to_string) else { - log::warn!( - "Planned provider '{}' pending request had no backend name", - provider.provider_name() - ); - responses.push(provider_launch_failed_response( - provider.provider_name(), - started_at.elapsed().as_millis() as u64, - )); - continue; - }; - match launches.entry(backend_name) { - Entry::Vacant(entry) => { - entry.insert(PlannedLaunchState { - provider, - started_at, - parse_state, - trace_observation: None, - }); - pending.push(launched); - } - Entry::Occupied(entry) => { - log::warn!( - "Planned provider '{}' pending backend '{}' already belongs to another provider", - provider.provider_name(), - entry.key(), - ); - responses.push(provider_launch_failed_response( - provider.provider_name(), - started_at.elapsed().as_millis() as u64, - )); - } - } - } - Ok(ProviderRequestOutcome::Immediate(response)) => responses.push(response), - Err(error) => { - log::warn!( - "Planned provider '{}' failed to launch: {:?}", - provider.provider_name(), - error - ); - responses.push(provider_launch_failed_response( - provider.provider_name(), - started_at.elapsed().as_millis() as u64, - )); - } - } - } - - while !pending.is_empty() { - let select_result = match context - .services - .http_client() - .select_until( - pending, - collection_deadline(auction_start, context.timeout_ms), - ) - .await - { - Ok(Some(result)) => result, - Ok(None) => break, - Err(error) => { - log::warn!("Planned provider select failed: {:?}", error); - let mut transport_failures = launches - .drain() - .map(|(_, state)| { - provider_transport_failed_response( - state.provider.provider_name(), - state.started_at.elapsed().as_millis() as u64, - ) - }) - .collect::>(); - transport_failures.sort_by(|left, right| left.provider.cmp(&right.provider)); - responses.extend(transport_failures); - break; - } - }; - pending = select_result.remaining; - match select_result.ready { - Ok(platform_response) => { - let backend_name = platform_response - .backend_name - .as_deref() - .unwrap_or_default() - .to_string(); - if let Some(state) = launches.remove(&backend_name) { - let elapsed_ms = state.started_at.elapsed().as_millis() as u64; - match state - .provider - .parse_response_with_state( - platform_response, - elapsed_ms, - state.parse_state.as_deref(), - ) - .await - { - Ok(response) => responses.push(response), - Err(error) => responses.push(provider_error_response( - state.provider.provider_name(), - elapsed_ms, - ERROR_TYPE_PARSE_RESPONSE, - &error, - )), - } - } - } - Err(error) => { - if let Some(backend_name) = select_result.failed_backend_name - && let Some(state) = launches.remove(&backend_name) - { - let elapsed_ms = state.started_at.elapsed().as_millis() as u64; - log::warn!( - "Planned provider '{}' transport failed: {:?}", - state.provider.provider_name(), - error - ); - responses.push(provider_transport_failed_response( - state.provider.provider_name(), - elapsed_ms, - )); - } - } - } - } - for state in launches.into_values() { - log::debug!( - "Planned provider '{}' exhausted its auction collection budget", - state.provider.provider_name() - ); - responses.push(provider_timeout_response( - state.provider.provider_name(), - state.started_at.elapsed().as_millis() as u64, - )); - } - - for response in &mut responses { - if let Some(&unused_bidder_params_count) = - planned_unused_bidder_params.get(response.provider.as_str()) - { - *response = - materialize_planned_response(response.clone(), unused_bidder_params_count); - } - } - - let provider_order = self - .plan - .providers() - .iter() - .enumerate() - .map(|(index, provider)| (provider.id.as_str(), index)) - .collect::>(); - responses.sort_by_key(|response| { - provider_order - .get(response.provider.as_str()) - .copied() - .unwrap_or(usize::MAX) - }); - - let floor_prices = original_request - .slots - .iter() - .filter_map(|slot| slot.floor_price.map(|floor| (slot.id.clone(), floor))) - .collect::>(); - let helper = AuctionOrchestrator::new(AuctionConfig::default()); - let local_winners = || helper.select_winning_bids(&responses, &floor_prices); - let (mediator_response, winning_bids) = if let Some(mediator) = &self.mediator { - let remaining_ms = remaining_budget_ms(auction_start, context.timeout_ms); - let logical_budget_ms = remaining_ms.min(mediator.timeout_ms()); - if logical_budget_ms == 0 { - log::warn!( - "Auction deadline exhausted before planned mediator; using local ranking" - ); - (None, local_winners()) - } else { - let transport_timeout_ms = context - .services - .backend() - .canonicalize_transport_timeout_ms(logical_budget_ms, mediator.timeout_ms()); - if transport_timeout_ms == 0 { - log::warn!( - "Planned mediator transport budget canonicalized to zero; using local ranking" - ); - let winning_bids = local_winners(); - return Ok(OrchestrationResult { - provider_responses: responses, - mediator_response: None, - winning_bids, - total_time_ms: auction_start.elapsed().as_millis() as u64, - metadata: routing_metadata(routed.diagnostics().unroutable_bidder_count()), - }); - } - let mediator_context = AuctionContext { - settings: context.settings, - request: context.request, - timeout_ms: logical_budget_ms, - transport_timeout_ms, - provider_responses: Some(&responses), - services: context.services, - }; - let mediator_start = Instant::now(); - let mediator_deadline = collection_deadline(auction_start, context.timeout_ms) - .min(collection_deadline(mediator_start, logical_budget_ms)); - let mediated = match mediator - .request_bids(original_request, &mediator_context) - .await - { - Ok(_) if Instant::now() >= mediator_deadline => { - log::debug!( - "Mediator '{}' exhausted its response budget during launch; using local ranking", - mediator.provider_name() - ); - None - } - Ok(ProviderRequestOutcome::Immediate(response)) => Some(response), - Ok(ProviderRequestOutcome::Pending { - request: pending, - parse_state, - }) => match context - .services - .http_client() - .wait_until(pending, mediator_deadline) - .await - { - Ok(platform_response) => { - let response_time_ms = mediator_start.elapsed().as_millis() as u64; - if Instant::now() >= mediator_deadline { - log::debug!( - "Planned mediator '{}' completed after its response deadline; using local ranking ({}ms)", - mediator.provider_name(), - response_time_ms - ); - None - } else { - mediator - .parse_response_with_context_and_state( - platform_response, - response_time_ms, - original_request, - &mediator_context, - parse_state.as_deref(), - ) - .await - .map_err(|error| { - log::warn!( - "Planned mediator '{}' response processing failed; using local ranking: {:?}", - mediator.provider_name(), - error - ); - }) - .ok() - } - } - Err(error) => { - if matches!( - error.current_context(), - crate::platform::PlatformError::DeadlineExceeded - ) { - log::debug!( - "Planned mediator '{}' exhausted its response budget; using local ranking", - mediator.provider_name() - ); - } else { - log::warn!( - "Planned mediator '{}' request failed: {:?}", - mediator.provider_name(), - error - ); - } - None - } - }, - Err(error) => { - log::warn!( - "Planned mediator '{}' failed to launch: {:?}", - mediator.provider_name(), - error - ); - None - } - }; - // An error status is not a mediation decision, however the provider - // reports it, so it falls back to local ranking like other failures. - if mediated - .as_ref() - .is_some_and(|response| response.status == BidStatus::Error) - { - log::warn!( - "Planned mediator '{}' reported an error response; using local ranking", - mediator.provider_name() - ); - } - if let Some(mediated) = - mediated.filter(|response| response.status != BidStatus::Error) - { - let winners = mediated - .bids - .iter() - .filter_map(|bid| { - if bid.price.is_none() { - log::warn!( - "Planned mediator returned a bid without a decoded price" - ); - None - } else { - Some((bid.slot_id.clone(), bid.clone())) - } - }) - .collect(); - ( - Some(mediated), - helper.apply_floor_prices(winners, &floor_prices), - ) - } else { - (None, local_winners()) - } - } - } else { - (None, local_winners()) - }; - let unroutable_bidder_count = routed.diagnostics().unroutable_bidder_count(); - // This logs only a bounded routing count, never request data or secrets. - log::info!( - "Auction routing diagnostics: unroutable_bidder_count={}", - unroutable_bidder_count - ); - Ok(OrchestrationResult { - provider_responses: responses, - mediator_response, - winning_bids, - total_time_ms: auction_start.elapsed().as_millis() as u64, - metadata: routing_metadata(unroutable_bidder_count), - }) - } } impl AuctionOrchestrator { - /// Create a legacy orchestrator for parity tests. - #[cfg(test)] - #[must_use] - pub(crate) fn new(config: AuctionConfig) -> Self { - let plan = Arc::new( - AuctionPlan::compile(super::plan::AuctionPlanConfig { - timeout_ms: config.timeout_ms, - providers: std::collections::BTreeMap::new(), - bidders: std::collections::BTreeMap::new(), - mediator: None, - request_signing: None, - }) - .expect("should compile empty legacy test plan") - .with_enabled(config.enabled), - ); - Self { - enabled: config.enabled, - plan_backed: false, - config, - plan, - planned_providers: Vec::new(), - mediator: None, - providers: HashMap::new(), - } - } - /// Create the live orchestrator from one shared compiled auction plan. #[must_use] pub fn from_plan(plan: Arc, mediator: Option>) -> Self { @@ -886,14 +355,9 @@ impl AuctionOrchestrator { .collect(); Self { enabled: plan.enabled(), - plan_backed: true, plan, planned_providers, mediator, - #[cfg(test)] - config: AuctionConfig::default(), - #[cfg(test)] - providers: HashMap::new(), } } @@ -903,14 +367,6 @@ impl AuctionOrchestrator { Arc::ptr_eq(&self.plan, plan) } - /// Register an auction provider in the legacy parity harness. - #[cfg(test)] - pub(crate) fn register_provider(&mut self, provider: Arc) { - let name = provider.provider_name().to_string(); - log::info!("Registering auction provider: {}", name); - self.providers.insert(name, provider); - } - /// Get the number of registered providers. #[must_use] pub fn provider_count(&self) -> usize { @@ -982,605 +438,8 @@ impl AuctionOrchestrator { } return Ok(OrchestrationResult::no_bid()); } - #[cfg(not(test))] - return self.run_planned_auction(request, context).await; - #[cfg(test)] - if self.plan_backed { - return self.run_planned_auction(request, context).await; - } - #[cfg(test)] - let start_time = Instant::now(); - - // Auto-detect strategy based on mediator configuration. - #[cfg(test)] - let (strategy_name, result) = if self.config.has_mediator() { - ( - "parallel_mediation", - self.run_parallel_mediation(request, context).await?, - ) - } else { - ( - "parallel_only", - self.run_parallel_only(request, context).await?, - ) - }; - - #[cfg(test)] - log::info!( - "Running auction with strategy: {} (auto-detected from mediator config)", - strategy_name - ); - - #[cfg(test)] - Ok(OrchestrationResult { - total_time_ms: start_time.elapsed().as_millis() as u64, - ..result - }) - } - - /// Run auction with parallel bidding + mediation. - #[cfg(test)] - /// - /// Flow: - /// 1. Run all bidders in parallel - /// 2. Collect bids from all bidders - /// 3. Send combined bids to mediator for final decision - async fn run_parallel_mediation( - &self, - request: &AuctionRequest, - context: &AuctionContext<'_>, - ) -> Result> { - let mediation_start = Instant::now(); - let provider_responses = self.run_providers_parallel(request, context).await?; - - let floor_prices = self.floor_prices_by_slot(request); - let (mediator_response, winning_bids) = if let Some(mediator_name) = &self.config.mediator { - let mediator = self.get_provider(mediator_name)?; - - log::info!( - "Sending {} provider responses to mediator: {}", - provider_responses.len(), - mediator.provider_name() - ); - - // Give the mediator only the remaining time from the auction - // deadline, not the full timeout — the bidding phase already - // consumed part of it. Canonicalize the transport timeout so the - // backend name remains stable across equivalent budget values. - let remaining_ms = remaining_budget_ms(mediation_start, context.timeout_ms); - let logical_budget_ms = remaining_ms.min(mediator.timeout_ms()); - let mediator_timeout = context - .services - .backend() - .canonicalize_transport_timeout_ms(logical_budget_ms, mediator.timeout_ms()); - - if mediator_timeout == 0 { - log::warn!("Auction timeout exhausted during bidding phase; skipping mediator"); - let winning = self.select_winning_bids(&provider_responses, &floor_prices); - return Ok(OrchestrationResult { - provider_responses, - mediator_response: None, - winning_bids: winning, - total_time_ms: 0, - metadata: HashMap::new(), - }); - } - - let mediator_context = AuctionContext { - settings: context.settings, - request: context.request, - timeout_ms: logical_budget_ms, - transport_timeout_ms: mediator_timeout, - provider_responses: Some(&provider_responses), - services: context.services, - }; - - let start_time = Instant::now(); - let mediator_deadline = collection_deadline(mediation_start, context.timeout_ms) - .min(collection_deadline(start_time, logical_budget_ms)); - let mediator_outcome = mediator - .request_bids(request, &mediator_context) - .await - .change_context(TrustedServerError::Auction { - message: format!("Mediator {} failed to launch", mediator.provider_name()), - })?; - if Instant::now() >= mediator_deadline { - log::debug!( - "Mediator '{}' exhausted its response budget during launch; using local ranking", - mediator.provider_name() - ); - let winning_bids = self.select_winning_bids(&provider_responses, &floor_prices); - return Ok(OrchestrationResult { - provider_responses, - mediator_response: None, - winning_bids, - total_time_ms: mediation_start.elapsed().as_millis() as u64, - metadata: HashMap::new(), - }); - } - let mediator_resp = match mediator_outcome { - ProviderRequestOutcome::Immediate(response) => response, - ProviderRequestOutcome::Pending { - request: pending, - parse_state, - } => { - let platform_resp = mediator_context - .services - .http_client() - .wait_until(pending, mediator_deadline) - .await; - if platform_resp.as_ref().is_err_and(|error| { - matches!( - error.current_context(), - crate::platform::PlatformError::DeadlineExceeded - ) - }) { - let winning_bids = - self.select_winning_bids(&provider_responses, &floor_prices); - return Ok(OrchestrationResult { - provider_responses, - mediator_response: None, - winning_bids, - total_time_ms: mediation_start.elapsed().as_millis() as u64, - metadata: HashMap::new(), - }); - } - let platform_resp = - platform_resp.change_context(TrustedServerError::Auction { - message: format!( - "Mediator {} request failed", - mediator.provider_name() - ), - })?; - let response_time_ms = start_time.elapsed().as_millis() as u64; - if Instant::now() >= mediator_deadline { - log::debug!( - "Mediator '{}' completed after its response deadline; using local ranking ({}ms)", - mediator.provider_name(), - response_time_ms - ); - let winning = self.select_winning_bids(&provider_responses, &floor_prices); - return Ok(OrchestrationResult { - provider_responses, - mediator_response: None, - winning_bids: winning, - total_time_ms: mediation_start.elapsed().as_millis() as u64, - metadata: HashMap::new(), - }); - } - - mediator - .parse_response_with_context_and_state( - platform_resp, - response_time_ms, - request, - &mediator_context, - parse_state.as_deref(), - ) - .await - .change_context(TrustedServerError::Auction { - message: format!("Mediator {} parse failed", mediator.provider_name()), - })? - } - }; - - // Extract only mediator bids with comparable numeric prices. - let winning = mediator_resp - .bids - .iter() - .filter_map(|bid| { - if bid.price.is_none() { - log::warn!( - "Mediator '{}' returned bid for slot '{}' without a price - skipping", - mediator.provider_name(), - bid.slot_id - ); - None - } else { - Some((bid.slot_id.clone(), bid.clone())) - } - }) - .collect(); - - ( - Some(mediator_resp), - self.apply_floor_prices(winning, &floor_prices), - ) - } else { - // No mediator - select best bid per slot from bidder responses - let winning = self.select_winning_bids(&provider_responses, &floor_prices); - (None, winning) - }; - - Ok(OrchestrationResult { - provider_responses, - mediator_response, - winning_bids, - total_time_ms: 0, // Will be set by caller - metadata: HashMap::new(), - }) - } - - /// Run auction with only parallel bidding (no mediation). - #[cfg(test)] - async fn run_parallel_only( - &self, - request: &AuctionRequest, - context: &AuctionContext<'_>, - ) -> Result> { - let provider_responses = self.run_providers_parallel(request, context).await?; - let floor_prices = self.floor_prices_by_slot(request); - let winning_bids = self.select_winning_bids(&provider_responses, &floor_prices); - - Ok(OrchestrationResult { - provider_responses, - mediator_response: None, - winning_bids, - total_time_ms: 0, - metadata: HashMap::new(), - }) - } - - /// Run all providers in parallel and collect responses. - #[cfg(test)] - /// - /// Uses `PlatformHttpClient::select()` to process responses as they - /// become ready, rather than waiting for each response sequentially. - async fn run_providers_parallel( - &self, - request: &AuctionRequest, - context: &AuctionContext<'_>, - ) -> Result, Report> { - let provider_names = self - .config - .providers - .keys() - .map(super::plan::ProviderId::as_str) - .collect::>(); - - if provider_names.is_empty() { - return Err(Report::new(TrustedServerError::Auction { - message: "No providers configured".to_string(), - })); - } - - // Enforce each adapter's supported provider count before dispatch. - // Worker adapters retain their single-provider collection contract. - if provider_names.len() > 1 && !context.services.http_client().supports_concurrent_fanout() - { - return Err(Report::new(TrustedServerError::Auction { - message: format!( - "{} auction providers configured, but this platform's HTTP \ - client executes requests sequentially — configure a single \ - provider, or use an adapter with concurrent fan-out support", - provider_names.len(), - ), - })); - } - - log::info!( - "Running {} providers in parallel using select", - provider_names.len() - ); - - // Track auction start time for deadline enforcement - let auction_start = Instant::now(); - - // Phase 1: Launch all requests concurrently and build mapping - // Maps backend_name to provider state retained for response parsing. - let mut backend_to_provider: HashMap = HashMap::new(); - let mut pending_requests: Vec = Vec::new(); - let mut responses = Vec::new(); - let mut immediate_response_count = 0usize; - - for provider_name in &provider_names { - let provider = match self.providers.get(*provider_name) { - Some(p) => p, - None => { - // This logs a configured provider identifier, not request data or secrets. - log::warn!("Provider '{}' not registered, skipping", provider_name); - continue; - } - }; - - if !provider.is_enabled() { - log::debug!( - "Provider '{}' is disabled, skipping", - provider.provider_name() - ); - continue; - } - - // Give each provider only the remaining time from the auction - // deadline so that backend transport timeouts do not extend past - // the overall budget. Canonicalizing keeps backend names stable. - let remaining_ms = remaining_budget_ms(auction_start, context.timeout_ms); - let effective_timeout = context - .services - .backend() - .canonicalize_transport_timeout_ms(remaining_ms, provider.timeout_ms()); - - // The deadline gate intentionally precedes `request_bids`: zero - // budget skips every provider, including one that might respond immediately. - if effective_timeout == 0 { - log::warn!("Auction timeout exhausted before launching provider request; skipping"); - continue; - } - - // Immediate providers have no backend name and must remain eligible - // to return a synchronous result. Pending providers are still - // guarded before dispatch when their name can be predicted. - let predicted_backend_name = provider.backend_name(context.services, effective_timeout); - if let Some(backend_name) = predicted_backend_name.as_ref() - && backend_to_provider.contains_key(backend_name) - { - log::warn!( - "Provider '{}' predicted backend name '{}' already belongs to another provider; skipping launch", - provider.provider_name(), - backend_name, - ); - responses.push(provider_launch_failed_response(provider.provider_name(), 0)); - continue; - } - - let provider_context = AuctionContext { - settings: context.settings, - request: context.request, - timeout_ms: effective_timeout, - transport_timeout_ms: effective_timeout, - provider_responses: context.provider_responses, - services: context.services, - }; - - // This logs a configured provider identifier and timeout, not request data or secrets. - log::info!( - "Launching bid request to '{}' with a {}ms budget", - provider.provider_name(), - effective_timeout - ); - - let start_time = Instant::now(); - match provider.request_bids(request, &provider_context).await { - Ok(ProviderRequestOutcome::Pending { - request: pending, - parse_state, - }) => { - let request_backend_name = pending.backend_name().map(str::to_string).or_else(|| { - if let Some(backend_name) = predicted_backend_name.as_ref() { - log::warn!( - "Provider '{}' pending request returned no backend name; using predicted name '{}'", - provider.provider_name(), - backend_name, - ); - } - predicted_backend_name.clone() - }); - let Some(request_backend_name) = request_backend_name else { - log::warn!( - "Provider '{}' pending request has no backend name; response cannot be correlated", - provider.provider_name() - ); - responses.push(provider_launch_failed_response( - provider.provider_name(), - start_time.elapsed().as_millis() as u64, - )); - continue; - }; - if backend_to_provider.contains_key(&request_backend_name) { - log::warn!( - "Provider '{}' resolved backend name '{}' already belongs to another provider; skipping launch", - provider.provider_name(), - request_backend_name, - ); - responses.push(provider_launch_failed_response( - provider.provider_name(), - start_time.elapsed().as_millis() as u64, - )); - continue; - } - backend_to_provider.insert( - request_backend_name, - ProviderLaunchState { - provider_name: provider.provider_name().to_string(), - started_at: start_time, - provider: Arc::clone(provider), - effective_timeout_ms: effective_timeout, - parse_state, - }, - ); - pending_requests.push(pending); - log::debug!( - "Request to '{}' launched successfully", - provider.provider_name() - ); - } - Ok(ProviderRequestOutcome::Immediate(response)) => { - immediate_response_count += 1; - log::debug!( - "Provider '{}' completed without an upstream request", - provider.provider_name() - ); - responses.push(response); - } - Err(e) => { - let response_time_ms = start_time.elapsed().as_millis() as u64; - log::warn!( - "Provider '{}' failed to launch request: {:?}", - provider.provider_name(), - e - ); - responses.push(provider_launch_failed_response( - provider.provider_name(), - response_time_ms, - )); - } - } - } - - if pending_requests.is_empty() { - // An immediate response (for example, an APS-only Prebid no-bid) is - // a completed provider outcome. Launch failures alone remain a - // terminal auction error rather than being converted to a 200 no-bid. - if immediate_response_count > 0 { - return Ok(responses); - } - return Err(Report::new(TrustedServerError::Auction { - message: format!( - "All {} configured provider(s) skipped or failed to launch", - provider_names.len() - ), - })); - } - log::info!( - "Launched {} concurrent provider request(s); waiting for responses", - pending_requests.len() - ); - - // Collection bounds header waits and body buffering by the original cutoff. - let mut remaining = pending_requests; - - while !remaining.is_empty() { - let platform_result = match context - .services - .http_client() - .select_until( - remaining, - collection_deadline(auction_start, context.timeout_ms), - ) - .await - { - Ok(Some(r)) => r, - Ok(None) => break, - Err(e) => { - log::warn!("select() failed: {:?}", e); - let mut failures = backend_to_provider - .drain() - .map(|(_, state)| { - provider_transport_failed_response( - &state.provider_name, - state.started_at.elapsed().as_millis() as u64, - ) - }) - .collect::>(); - failures.sort_by(|left, right| left.provider.cmp(&right.provider)); - responses.extend(failures); - break; - } - }; - let crate::platform::PlatformSelectResult { - ready, - remaining: new_remaining, - failed_backend_name, - } = platform_result; - remaining = new_remaining; - - match ready { - Ok(response) => { - // Identify the provider from the backend name - let backend_name = response - .backend_name - .as_deref() - .unwrap_or_default() - .to_string(); - - if let Some(state) = backend_to_provider.remove(&backend_name) { - let response_time_ms = state.started_at.elapsed().as_millis() as u64; - let provider_context = AuctionContext { - settings: context.settings, - request: context.request, - timeout_ms: state.effective_timeout_ms, - transport_timeout_ms: state.effective_timeout_ms, - provider_responses: context.provider_responses, - services: context.services, - }; - - match state - .provider - .parse_response_with_context_and_state( - response, - response_time_ms, - request, - &provider_context, - state.parse_state.as_deref(), - ) - .await - { - Ok(auction_response) => { - log::info!( - "Provider '{}' returned {} bids (status: {:?}, time: {}ms)", - auction_response.provider, - auction_response.bids.len(), - auction_response.status, - auction_response.response_time_ms - ); - responses.push(auction_response); - } - Err(e) => { - // This warning reports provider parse failures only; no secret values are logged. - log::warn!( - "Provider '{}' failed to parse response: {:?}", - state.provider_name, - e - ); - responses.push(provider_error_response( - &state.provider_name, - response_time_ms, - ERROR_TYPE_PARSE_RESPONSE, - &e, - )); - } - } - } else { - log::warn!( - "Received response from unknown backend '{}', ignoring", - backend_name - ); - } - } - Err(e) => { - if let Some(ref backend_name) = failed_backend_name { - if let Some(state) = backend_to_provider.remove(backend_name) { - let response_time_ms = state.started_at.elapsed().as_millis() as u64; - log::warn!( - "Provider '{}' request failed: {:?}", - state.provider_name, - e - ); - responses.push(provider_transport_failed_response( - &state.provider_name, - response_time_ms, - )); - } else { - log::warn!( - "A provider request failed (backend '{}' not tracked): {:?}", - backend_name, - e - ); - } - } else { - log::warn!( - "A provider request failed (backend not identified): {:?}", - e - ); - } - } - } - } - - for state in backend_to_provider.into_values() { - let response_time_ms = state.started_at.elapsed().as_millis() as u64; - log::warn!( - "Provider '{}' timed out before auction collection completed", - state.provider_name - ); - responses.push(provider_timeout_response( - &state.provider_name, - response_time_ms, - )); - } - - Ok(responses) - } + self.run_planned_auction(request, context).await + } /// Select the best decoded-price bid for each slot from all responses. /// @@ -1683,24 +542,6 @@ impl AuctionOrchestrator { .collect() } - /// Get a provider by name. - #[cfg(test)] - fn get_provider( - &self, - name: &str, - ) -> Result<&Arc, Report> { - self.providers.get(name).ok_or_else(|| { - log::warn!( - "Provider '{}' configured but not registered. Available providers: {:?}", - name, - self.providers.keys().collect::>() - ); - Report::new(TrustedServerError::Auction { - message: format!("Provider '{}' not registered", name), - }) - }) - } - async fn dispatch_planned_auction( &self, request: &AuctionRequest, @@ -1715,17 +556,11 @@ impl AuctionOrchestrator { if self.planned_providers.is_empty() { return DispatchAuctionOutcome::NotStarted; } - if self.planned_providers.len() > 1 - && !context.services.http_client().supports_concurrent_fanout() - { - log::warn!( - "{} planned auction providers configured on a runtime without concurrent fanout", - self.planned_providers.len() - ); - return DispatchAuctionOutcome::NotStarted; - } - let auction_start = Instant::now(); + let clock = context.services.clock().clone(); + let auction_start = clock.now(); + let auction_deadline = + clock.deadline_after(Duration::from_millis(u64::from(context.timeout_ms))); let routed = route_auction( request.clone(), context.request, @@ -1797,7 +632,7 @@ impl AuctionOrchestrator { provider_responses, fatal_admission_error: Some(error), metadata: routing_metadata(planned_unroutable_bidder_count), - elapsed_ms: auction_start.elapsed().as_millis() as u64, + elapsed_ms: elapsed_ms(&clock, auction_start), }; } }; @@ -1812,10 +647,8 @@ impl AuctionOrchestrator { .enumerate() .map(|(index, provider)| (provider.id.as_str().to_string(), index)) .collect::>(); - let mut pending_requests = Vec::new(); - let mut planned_backend_to_provider = HashMap::new(); - let mut reserved_backend_names = HashSet::new(); - let mut immediate_response_count = 0usize; + let mut bidder_requests = Vec::new(); + let mut slot_providers = Vec::new(); let mut launch_failure_count = 0usize; for input in routed.inputs() { @@ -1833,7 +666,7 @@ impl AuctionOrchestrator { continue; }; let logical_budget_ms = - remaining_budget_ms(auction_start, context.timeout_ms).min(provider.timeout_ms()); + remaining_ms(&clock, auction_deadline).min(provider.timeout_ms()); if logical_budget_ms == 0 { log::debug!( "Planned provider '{}' has no remaining auction budget before launch", @@ -1842,76 +675,26 @@ impl AuctionOrchestrator { completed_responses.push(provider_timeout_response(provider.provider_name(), 0)); continue; } - let transport_timeout_ms = context - .services - .backend() - .canonicalize_transport_timeout_ms(logical_budget_ms, provider.timeout_ms()); - if transport_timeout_ms == 0 { - log::debug!( - "Planned provider '{}' has no remaining transport budget before launch", - provider.provider_name() - ); - completed_responses.push(provider_timeout_response(provider.provider_name(), 0)); - continue; - } - let started_at = Instant::now(); + let built_at = clock.now(); let trace_observation = trace .as_ref() .map(|trace| trace.launch_provider(TraceProviderRole::Bidder)); - match provider - .request_bids_routed( - input, - &routed, - logical_budget_ms, - transport_timeout_ms, - signer.as_ref(), - context.services, - &mut reserved_backend_names, - ) - .await - { - Ok(ProviderRequestOutcome::Pending { - request: pending, + match provider.request_bids_routed(input, &routed, logical_budget_ms, signer.as_ref()) { + Ok(ProviderRequestOutcome::Ready { + request: bidder_request, parse_state, }) => { - let Some(backend_name) = pending.backend_name().map(str::to_string) else { - if let (Some(trace), Some(observation)) = (&trace, trace_observation) { - trace.observe_failure(observation); - } - launch_failure_count += 1; - completed_responses.push(provider_launch_failed_response( - provider.provider_name(), - started_at.elapsed().as_millis() as u64, - )); - continue; - }; - match planned_backend_to_provider.entry(backend_name.clone()) { - Entry::Vacant(entry) => { - entry.insert(PlannedLaunchState { - provider, - started_at, - parse_state, - trace_observation, - }); - pending_requests.push(pending.with_backend_name(backend_name)); - } - Entry::Occupied(_) => { - if let (Some(trace), Some(observation)) = (&trace, trace_observation) { - trace.observe_failure(observation); - } - launch_failure_count += 1; - completed_responses.push(provider_launch_failed_response( - provider.provider_name(), - started_at.elapsed().as_millis() as u64, - )) - } - } + bidder_requests.push(bidder_request); + slot_providers.push(Some(BidderSlot { + provider, + parse_state, + trace_observation, + })); } Ok(ProviderRequestOutcome::Immediate(response)) => { if let (Some(trace), Some(observation)) = (&trace, trace_observation) { trace.observe_response(observation, &response); } - immediate_response_count += 1; completed_responses.push(response); } Err(error) => { @@ -1925,54 +708,68 @@ impl AuctionOrchestrator { launch_failure_count += 1; completed_responses.push(provider_launch_failed_response( provider.provider_name(), - started_at.elapsed().as_millis() as u64, + elapsed_ms(&clock, built_at), )); } } } - if pending_requests.is_empty() && immediate_response_count == 0 { - // A pre-launch timeout is a completed provider outcome. Only a set - // made entirely of launch failures crosses the fatal boundary. - if launch_failure_count > 0 && launch_failure_count == routed.inputs().len() { - for response in &mut completed_responses { - if let Some(&count) = - planned_unused_bidder_params.get(response.provider.as_str()) - { - *response = materialize_planned_response(response.clone(), count); - } + if routed.inputs().is_empty() && completed_responses.is_empty() { + return DispatchAuctionOutcome::NotStarted; + } + + // Every bidder request leaves together, after all are built and + // signed. An auction whose providers were all filtered out starts no + // batch, so it completes without waiting for the deadline. + let batch = if bidder_requests.is_empty() { + None + } else { + let mut batch = context + .services + .outbound_client() + .start_batch_until(bidder_requests, auction_deadline); + // Fastly registers every bidder backend while it starts the batch, + // so a registration failure is already recorded here. + launch_failure_count += settle_recorded_slots( + &mut batch, + &mut slot_providers, + &mut completed_responses, + trace.as_ref(), + ) + .await; + Some(batch) + }; + + // A pre-launch timeout is a completed provider outcome. Only a set + // made entirely of launch failures crosses the fatal boundary. + if launch_failure_count > 0 && launch_failure_count == routed.inputs().len() { + for response in &mut completed_responses { + if let Some(&count) = planned_unused_bidder_params.get(response.provider.as_str()) { + *response = materialize_planned_response(response.clone(), count); } - completed_responses.sort_by_key(|response| { - planned_provider_order - .get(response.provider.as_str()) - .copied() - .unwrap_or(usize::MAX) - }); - return DispatchAuctionOutcome::DispatchFailed { - request: request.clone(), - provider_responses: completed_responses, - fatal_admission_error: None, - metadata: routing_metadata(planned_unroutable_bidder_count), - elapsed_ms: auction_start.elapsed().as_millis() as u64, - }; - } - if routed.inputs().is_empty() && completed_responses.is_empty() { - return DispatchAuctionOutcome::NotStarted; } + completed_responses.sort_by_key(|response| { + planned_provider_order + .get(response.provider.as_str()) + .copied() + .unwrap_or(usize::MAX) + }); + return DispatchAuctionOutcome::DispatchFailed { + request: request.clone(), + provider_responses: completed_responses, + fatal_admission_error: None, + metadata: routing_metadata(planned_unroutable_bidder_count), + elapsed_ms: elapsed_ms(&clock, auction_start), + }; } - DispatchAuctionOutcome::Dispatched(DispatchedAuction { - pending_requests, - backend_to_provider: HashMap::new(), - planned_backend_to_provider, + batch, + slot_providers, completed_responses, + clock, auction_start, - timeout_ms: context.timeout_ms, + auction_deadline, floor_prices: self.floor_prices_by_slot(request), - // Planned providers carry their typed parse state, so collection - // does not need to retain the inbound client request. Keep the - // explicit request boundary empty across the origin wait. - provider_request_context: Box::new(Request::new(EdgeBody::empty())), request: request.clone(), planned_unused_bidder_params, planned_unroutable_bidder_count, @@ -1981,12 +778,12 @@ impl AuctionOrchestrator { }) } - /// Dispatch SSP bid requests without blocking WASM. + /// Dispatch SSP bid requests without waiting for their responses. /// - /// Calls each enabled provider's [`AuctionProvider::request_bids`] (which - /// internally calls Fastly's `send_async`), then returns immediately with a - /// [`DispatchedAuction`] token. The Fastly host begins the SSP round-trips - /// while WASM continues to `pending_origin.wait()`. + /// Builds every routed provider request and starts them together as one + /// `EdgeZero` batch whose cutoff is the auction deadline, then returns a + /// [`DispatchedAuction`] token. The bids progress while the caller waits + /// on the publisher origin. /// /// Returns [`DispatchAuctionOutcome::NotStarted`] when no providers are configured or /// all providers are disabled / over budget. Returns @@ -1998,8 +795,9 @@ impl AuctionOrchestrator { request: &AuctionRequest, context: &AuctionContext<'_>, ) -> DispatchAuctionOutcome { + let trace = context.request.extensions().get::(); if !self.enabled { - if let Some(trace) = context.request.extensions().get::() { + if let Some(trace) = trace { trace.finish( TraceAuctionTerminalStatus::Skipped, Some(TraceAuctionTerminalReason::PolicySkipped), @@ -2007,263 +805,33 @@ impl AuctionOrchestrator { } return DispatchAuctionOutcome::NotStarted; } - if !cfg!(test) || self.plan_backed { - let trace = context.request.extensions().get::(); - let mut guard = trace.map(TraceAuctionCarry::cancellation_guard); - let outcome = self.dispatch_planned_auction(request, context).await; - match &outcome { - DispatchAuctionOutcome::Dispatched(_) => { - if let Some(guard) = &mut guard { - guard.disarm(); - } - } - DispatchAuctionOutcome::DispatchFailed { .. } => { - if let Some(trace) = trace { - trace.finish_dispatch_failed(false); - } - } - DispatchAuctionOutcome::NotStarted => { - if let Some(trace) = trace { - trace.finish_dispatch_failed(true); - } + let mut guard = trace.map(TraceAuctionCarry::cancellation_guard); + let outcome = self.dispatch_planned_auction(request, context).await; + match &outcome { + DispatchAuctionOutcome::Dispatched(_) => { + if let Some(guard) = &mut guard { + guard.disarm(); } } - return outcome; - } - #[cfg(test)] - let provider_names = self - .config - .providers - .keys() - .map(super::plan::ProviderId::as_str) - .collect::>(); - #[cfg(test)] - if provider_names.is_empty() { - return DispatchAuctionOutcome::NotStarted; - } - - // Mirror the adapter capability guard before dispatch. Worker adapters - // start requests eagerly but retain single-provider collection support. - #[cfg(test)] - if provider_names.len() > 1 && !context.services.http_client().supports_concurrent_fanout() - { - log::warn!( - "{} auction providers configured, but this platform's HTTP client \ - executes requests sequentially — skipping initial-page auction \ - dispatch; configure a single provider, or use an adapter with \ - concurrent fan-out support", - provider_names.len(), - ); - return DispatchAuctionOutcome::NotStarted; - } - - let auction_start = Instant::now(); - #[cfg(test)] - let mut backend_to_provider: HashMap = HashMap::new(); - #[cfg(not(test))] - let backend_to_provider: HashMap = HashMap::new(); - #[cfg(test)] - let mut pending_requests: Vec = Vec::new(); - #[cfg(not(test))] - let pending_requests: Vec = Vec::new(); - #[cfg(test)] - let mut completed_responses: Vec = Vec::new(); - #[cfg(not(test))] - let completed_responses: Vec = Vec::new(); - #[cfg(test)] - let mut immediate_response_count = 0usize; - #[cfg(not(test))] - let immediate_response_count = 0usize; - - #[cfg(test)] - for provider_name in &provider_names { - let provider = match self.providers.get(*provider_name) { - Some(p) => p, - None => { - // The provider name is a static config identifier (e.g. "prebid"), not a secret. - log::warn!("Provider '{}' not registered, skipping", provider_name); - continue; + DispatchAuctionOutcome::DispatchFailed { .. } => { + if let Some(trace) = trace { + trace.finish_dispatch_failed(false); } - }; - - if !provider.is_enabled() { - log::debug!( - "Provider '{}' is disabled, skipping", - provider.provider_name() - ); - continue; - } - - let remaining_ms = remaining_budget_ms(auction_start, context.timeout_ms); - let effective_timeout = context - .services - .backend() - .canonicalize_transport_timeout_ms(remaining_ms, provider.timeout_ms()); - - // Match the synchronous path's strict deadline semantics: do not - // invoke even an immediate provider after the budget reaches zero. - if effective_timeout == 0 { - // This logs a configured provider identifier and timeout, not request data or secrets. - log::warn!( - "Auction timeout ({}ms) exhausted before launching '{}' — skipping", - context.timeout_ms, - provider.provider_name() - ); - continue; - } - - // Do not require a backend name before dispatch: an immediate - // provider intentionally has none. Guard predicted names when - // available; pending requests without either name fail below. - let predicted_backend_name = provider.backend_name(context.services, effective_timeout); - if let Some(backend_name) = predicted_backend_name.as_ref() - && backend_to_provider.contains_key(backend_name) - { - log::warn!( - "Provider '{}' predicted backend name '{}' already belongs to another provider; skipping dispatch", - provider.provider_name(), - backend_name, - ); - completed_responses - .push(provider_launch_failed_response(provider.provider_name(), 0)); - continue; } - - let provider_context = AuctionContext { - settings: context.settings, - request: context.request, - timeout_ms: effective_timeout, - transport_timeout_ms: effective_timeout, - provider_responses: context.provider_responses, - services: context.services, - }; - - let start_time = Instant::now(); - match provider.request_bids(request, &provider_context).await { - Ok(ProviderRequestOutcome::Pending { - request: pending, - parse_state, - }) => { - let backend_name = pending.backend_name().map(str::to_string).or_else(|| { - if let Some(backend_name) = predicted_backend_name.as_ref() { - log::warn!( - "Provider '{}' pending request returned no backend name; using predicted name '{}'", - provider.provider_name(), - backend_name, - ); - } - predicted_backend_name.clone() - }); - let Some(backend_name) = backend_name else { - log::warn!( - "Provider '{}' pending request has no backend name; response cannot be correlated", - provider.provider_name() - ); - completed_responses.push(provider_launch_failed_response( - provider.provider_name(), - start_time.elapsed().as_millis() as u64, - )); - continue; - }; - if backend_to_provider.contains_key(&backend_name) { - log::warn!( - "Provider '{}' resolved backend name '{}' already belongs to another provider; skipping dispatch", - provider.provider_name(), - backend_name, - ); - completed_responses.push(provider_launch_failed_response( - provider.provider_name(), - start_time.elapsed().as_millis() as u64, - )); - continue; - } - // This logs configured provider and backend identifiers plus a timeout, not request data or secrets. - log::info!( - "Dispatching bid request to '{}' (backend: {}, budget: {}ms)", - provider.provider_name(), - backend_name, - effective_timeout - ); - backend_to_provider.insert( - backend_name.clone(), - ProviderLaunchState { - provider_name: provider.provider_name().to_string(), - started_at: start_time, - provider: Arc::clone(provider), - effective_timeout_ms: effective_timeout, - parse_state, - }, - ); - pending_requests.push(pending.with_backend_name(backend_name)); - } - Ok(ProviderRequestOutcome::Immediate(response)) => { - immediate_response_count += 1; - completed_responses.push(response); - } - Err(e) => { - let response_time_ms = start_time.elapsed().as_millis() as u64; - log::warn!( - "Provider '{}' failed to dispatch request: {:?}", - provider.provider_name(), - e - ); - completed_responses.push(provider_launch_failed_response( - provider.provider_name(), - response_time_ms, - )); + DispatchAuctionOutcome::NotStarted => { + if let Some(trace) = trace { + trace.finish_dispatch_failed(true); } } } - - if pending_requests.is_empty() && immediate_response_count == 0 { - return if completed_responses.is_empty() { - DispatchAuctionOutcome::NotStarted - } else { - DispatchAuctionOutcome::DispatchFailed { - request: request.clone(), - provider_responses: completed_responses, - fatal_admission_error: None, - metadata: HashMap::new(), - elapsed_ms: auction_start.elapsed().as_millis() as u64, - } - }; - } - - // This logs bounded request counts and a timeout, not request data or secrets. - log::info!( - "Dispatched {} SSP request(s) with {} immediate response(s) (timeout: {}ms)", - pending_requests.len(), - immediate_response_count, - context.timeout_ms - ); - - DispatchAuctionOutcome::Dispatched(DispatchedAuction { - pending_requests, - backend_to_provider, - planned_backend_to_provider: HashMap::new(), - completed_responses, - auction_start, - timeout_ms: context.timeout_ms, - floor_prices: self.floor_prices_by_slot(request), - provider_request_context: Box::new(snapshot_context_request(context.request)), - request: request.clone(), - planned_unused_bidder_params: HashMap::new(), - planned_unroutable_bidder_count: 0, - planned_provider_order: HashMap::new(), - trace: context - .request - .extensions() - .get::() - .cloned(), - }) + outcome } /// Collect bid responses from a previously-dispatched auction. /// - /// Runs the select-loop phase (equivalent to Phase 2 of - /// `run_providers_parallel`) and, if the orchestrator has a mediator - /// configured, forwards collected bids to it. The overall auction deadline - /// is enforced from `dispatched.auction_start`. + /// Observes the bidder batch until it finishes, then forwards the + /// collected bids to the mediator, if one is configured. A bid that has + /// not arrived by the auction deadline becomes a provider timeout. /// /// On any error or partial failure the method returns the best available /// result rather than propagating — the caller should still inject the @@ -2275,14 +843,13 @@ impl AuctionOrchestrator { context: &AuctionContext<'_>, ) -> OrchestrationResult { let DispatchedAuction { - pending_requests, - mut backend_to_provider, - mut planned_backend_to_provider, + batch, + slot_providers, completed_responses, + clock, auction_start, - timeout_ms, + auction_deadline, floor_prices, - provider_request_context, request, planned_unused_bidder_params, planned_unroutable_bidder_count, @@ -2292,456 +859,116 @@ impl AuctionOrchestrator { let _trace_guard = trace.as_ref().map(TraceAuctionCarry::cancellation_guard); log::info!( - "Collecting {} in-flight SSP responses (timeout: {}ms remaining: {}ms)", - pending_requests.len(), - timeout_ms, - remaining_budget_ms(auction_start, timeout_ms), - ); - - let mut responses: Vec = completed_responses; - let mut remaining = pending_requests; - - while !remaining.is_empty() { - let select_result = match services - .http_client() - .select_until(remaining, collection_deadline(auction_start, timeout_ms)) - .await - .change_context(TrustedServerError::Auction { - message: "HTTP select failed".to_string(), - }) { - Ok(Some(r)) => r, - Ok(None) => break, - Err(e) => { - if let Some(trace) = &trace { - trace.observe_collection_failure(); - } - log::warn!("select() failed during auction collection: {:?}", e); - // An outer select failure means the platform could not poll - // any outstanding handle. Attribute every tracked launch as - // a transport failure rather than later relabeling it as a - // timeout. Drain through a sorted buffer because HashMap - // iteration order is intentionally nondeterministic. - let mut transport_failures = backend_to_provider - .drain() - .map(|(_, state)| { - let response_time_ms = state.started_at.elapsed().as_millis() as u64; - provider_transport_failed_response( - &state.provider_name, - response_time_ms, - ) - }) - .chain(planned_backend_to_provider.drain().map(|(_, state)| { - if let (Some(trace), Some(observation)) = - (&trace, state.trace_observation) - { - trace.observe_failure(observation); - } - let response_time_ms = state.started_at.elapsed().as_millis() as u64; - provider_transport_failed_response( - state.provider.provider_name(), - response_time_ms, - ) - })) - .collect::>(); - transport_failures.sort_by(|left, right| left.provider.cmp(&right.provider)); - responses.extend(transport_failures); - break; - } - }; - // Destructure so transport failures can be attributed to a provider - // via `failed_backend_name`, mirroring run_providers_parallel. - let crate::platform::PlatformSelectResult { - ready, - remaining: new_remaining, - failed_backend_name, - } = select_result; - remaining = new_remaining; - - match ready { - Ok(platform_response) => { - let backend_name = platform_response.backend_name.clone().unwrap_or_default(); - if let Some(state) = backend_to_provider.remove(&backend_name) { - let response_time_ms = state.started_at.elapsed().as_millis() as u64; - let provider_context = AuctionContext { - settings: context.settings, - request: &provider_request_context, - timeout_ms: state.effective_timeout_ms, - transport_timeout_ms: state.effective_timeout_ms, - provider_responses: context.provider_responses, - services: context.services, + "Collecting {} in-flight SSP responses ({}ms of the auction budget remaining)", + slot_providers.iter().flatten().count(), + remaining_ms(&clock, auction_deadline), + ); + + let mut responses = completed_responses; + let mut unresolved = slot_providers; + if let Some(mut batch) = batch { + let failed = loop { + match batch.next().await { + Ok(OutboundBatchNext::Item(item)) => { + let (index, result) = item.into_parts(); + let Some(slot) = unresolved.get_mut(index).and_then(Option::take) else { + log::warn!("Auction batch reported unknown slot {index}, ignoring"); + continue; }; - match state - .provider - .parse_response_with_context_and_state( - platform_response, - response_time_ms, - &request, - &provider_context, - state.parse_state.as_deref(), - ) - .await - { - Ok(auction_response) => responses.push(auction_response), - Err(error) => responses.push(provider_error_response( - &state.provider_name, - response_time_ms, - ERROR_TYPE_PARSE_RESPONSE, - &error, - )), - } - } else if let Some(state) = planned_backend_to_provider.remove(&backend_name) { - let response_time_ms = state.started_at.elapsed().as_millis() as u64; - match state - .provider - .parse_response_with_state( - platform_response, - response_time_ms, - state.parse_state.as_deref(), - ) - .await - { - Ok(response) => { - if let (Some(trace), Some(observation)) = - (&trace, state.trace_observation) - { - trace.observe_response(observation, &response); - } - responses.push(response); - } - Err(error) => { - if let (Some(trace), Some(observation)) = - (&trace, state.trace_observation) - { - trace.observe_failure(observation); - } - responses.push(provider_error_response( - state.provider.provider_name(), - response_time_ms, - ERROR_TYPE_PARSE_RESPONSE, - &error, - )); - } - } - } else { - log::warn!( - "Received response from unknown backend '{}', ignoring", - backend_name - ); + responses.push(settle_bidder_slot(slot, result, trace.as_ref()).await); } - } - Err(e) => { - // Mirror the parallel path: attribute the transport failure to - // the provider behind `failed_backend_name` so it appears in - // provider_details instead of vanishing. - if let Some(ref backend_name) = failed_backend_name { - if let Some(state) = backend_to_provider.remove(backend_name) { - let response_time_ms = state.started_at.elapsed().as_millis() as u64; - log::warn!( - "Provider '{}' request failed: {:?}", - state.provider_name, - e - ); - responses.push(provider_transport_failed_response( - &state.provider_name, - response_time_ms, - )); - } else if let Some(state) = planned_backend_to_provider.remove(backend_name) - { - if let (Some(trace), Some(observation)) = - (&trace, state.trace_observation) - { - trace.observe_failure(observation); - } - let response_time_ms = state.started_at.elapsed().as_millis() as u64; - log::warn!( - "Planned provider '{}' request failed: {:?}", - state.provider.provider_name(), - e - ); - responses.push(provider_transport_failed_response( - state.provider.provider_name(), - response_time_ms, - )); - } else { - log::warn!( - "A provider request failed (backend '{}' not tracked): {:?}", - backend_name, - e - ); + Ok(OutboundBatchNext::Finished(_termination)) => break false, + Err(error) => { + if let Some(trace) = &trace { + trace.observe_collection_failure(); } - } else { - log::warn!( - "A provider request failed during collection (backend not identified): {:?}", - e + let report = outbound_error( + error, + TrustedServerError::Auction { + message: "Auction bidder batch failed".to_string(), + }, + OutboundRequestSource::Internal, ); + log::warn!("Auction bidder batch failed: {report:?}"); + break true; } } + }; + // The batch never resolves these slots: a cutoff leaves a late bid + // unresolved, and a failed batch leaves every slot it had not + // reported. + let unresolved_ms = duration_ms( + clock + .now() + .min(auction_deadline.instant()) + .saturating_duration_since(auction_start), + ); + for slot in unresolved.into_iter().flatten() { + if let (Some(trace), Some(observation)) = (&trace, slot.trace_observation) { + trace.observe_failure(observation); + } + let provider_name = slot.provider.provider_name(); + responses.push(if failed { + provider_transport_failed_response(provider_name, unresolved_ms) + } else { + log::debug!( + "Planned provider '{provider_name}' exhausted its dispatched auction collection budget" + ); + provider_timeout_response(provider_name, unresolved_ms) + }); } } - - for state in backend_to_provider.values() { - let response_time_ms = state.started_at.elapsed().as_millis() as u64; - log::warn!( - "Provider '{}' timed out before dispatched auction collection completed", - state.provider_name - ); - responses.push(provider_timeout_response( - &state.provider_name, - response_time_ms, - )); - } - backend_to_provider.clear(); - for state in planned_backend_to_provider.into_values() { - log::debug!( - "Planned provider '{}' exhausted its dispatched auction collection budget", - state.provider.provider_name() - ); - if let (Some(trace), Some(observation)) = (&trace, state.trace_observation) { - trace.observe_failure(observation); - } - responses.push(provider_timeout_response( - state.provider.provider_name(), - state.started_at.elapsed().as_millis() as u64, - )); - } for response in &mut responses { if let Some(&count) = planned_unused_bidder_params.get(response.provider.as_str()) { *response = materialize_planned_response(response.clone(), count); } } - if !planned_provider_order.is_empty() { - responses.sort_by_key(|response| { - planned_provider_order - .get(response.provider.as_str()) - .copied() - .unwrap_or(usize::MAX) - }); - } - - #[cfg(not(test))] - let mediator = self.mediator.as_ref(); - #[cfg(test)] - let mediator = self.mediator.as_ref().or_else(|| { - self.config - .mediator - .as_ref() - .and_then(|name| self.providers.get(name)) + responses.sort_by_key(|response| { + planned_provider_order + .get(response.provider.as_str()) + .copied() + .unwrap_or(usize::MAX) }); - let (mediator_response, winning_bids) = if let Some(mediator) = mediator { - { - // Cap the mediator at whichever is tighter: its own configured - // timeout or the remaining auction budget (A_deadline). Backend - // first-byte and between-bytes timeouts bound normal collection, but - // they are transport timers rather than absolute wall-clock limits: - // connection setup and byte-trickling can still consume more of the - // auction budget. Recomputing the remaining budget here prevents the - // mediator from extending that bounded response hold. - let remaining = remaining_budget_ms(auction_start, timeout_ms); - let logical_budget_ms = remaining.min(mediator.timeout_ms()); - if logical_budget_ms == 0 { - if let Some(trace) = &trace { - trace.finish(TraceAuctionTerminalStatus::Completed, None); - } - log::warn!( - "A_deadline exhausted before mediator '{}' — returning {} SSP bids without mediation", - mediator.provider_name(), - responses.len(), - ); - let winning = self.select_winning_bids(&responses, &floor_prices); - return OrchestrationResult { - provider_responses: responses, - mediator_response: None, - winning_bids: winning, - total_time_ms: auction_start.elapsed().as_millis() as u64, - metadata: routing_metadata(planned_unroutable_bidder_count), - }; - } - let transport_timeout_ms = services - .backend() - .canonicalize_transport_timeout_ms(logical_budget_ms, mediator.timeout_ms()); - if transport_timeout_ms == 0 { - if let Some(trace) = &trace { - trace.finish(TraceAuctionTerminalStatus::Completed, None); - } - log::warn!( - "Mediator '{}' transport budget canonicalized to zero — returning {} SSP bids without mediation", - mediator.provider_name(), - responses.len(), - ); - let winning = self.select_winning_bids(&responses, &floor_prices); - return OrchestrationResult { - provider_responses: responses, - mediator_response: None, - winning_bids: winning, - total_time_ms: auction_start.elapsed().as_millis() as u64, - metadata: routing_metadata(planned_unroutable_bidder_count), - }; - } - let mediator_start = Instant::now(); - // This logs a configured mediator identifier and timeout values, not request data or secrets. - log::info!( - "Running mediator '{}' with {}ms logical budget and {}ms transport timeout (A_deadline remaining: {}ms, configured: {}ms)", - mediator.provider_name(), - logical_budget_ms, - transport_timeout_ms, - remaining, - mediator.timeout_ms(), - ); - // The mediator runs on the collect path. See the doc-comment on - // `AuctionContext::request`: the real client request was already - // consumed by `send_async` during dispatch, so we substitute a - // canonical placeholder URL. Any future mediator that needs real - // client headers must snapshot them at dispatch time onto - // `DispatchedAuction` rather than reading `context.request` here. - let placeholder = http::Request::builder() - .uri(crate::auction::types::MEDIATOR_PLACEHOLDER_URL) - .body(edgezero_core::body::Body::empty()) - .unwrap_or_else(|_| http::Request::new(edgezero_core::body::Body::empty())); - let mediator_context = AuctionContext { - settings: context.settings, - request: &placeholder, - timeout_ms: logical_budget_ms, - transport_timeout_ms, - provider_responses: Some(&responses), - services: context.services, - }; - let mediator_deadline = collection_deadline(auction_start, timeout_ms) - .min(collection_deadline(mediator_start, logical_budget_ms)); - let mut trace_observation = trace - .as_ref() - .map(|trace| trace.launch_provider(TraceProviderRole::Mediator)); - let mediator_response = match mediator - .request_bids(&request, &mediator_context) - .await - { - Ok(_) if Instant::now() >= mediator_deadline => { - log::debug!( - "Mediator '{}' exhausted its response budget during launch; using local ranking", - mediator.provider_name() - ); - None - } - Ok(ProviderRequestOutcome::Immediate(response)) => { - if let (Some(trace), Some(observation)) = (&trace, trace_observation.take()) - { - trace.observe_response(observation, &response); - } - Some(response) - } - Ok(ProviderRequestOutcome::Pending { - request: pending, - parse_state, - }) => match services - .http_client() - .wait_until(pending, mediator_deadline) - .await - { - Ok(platform_resp) => { - let response_time_ms = mediator_start.elapsed().as_millis() as u64; - if Instant::now() >= mediator_deadline { - log::debug!( - "Mediator '{}' completed after its response deadline; using local ranking ({}ms)", - mediator.provider_name(), - response_time_ms - ); - None - } else { - match mediator - .parse_response_with_context_and_state( - platform_resp, - response_time_ms, - &request, - &mediator_context, - parse_state.as_deref(), - ) - .await - { - Ok(response) => { - if let (Some(trace), Some(observation)) = - (&trace, trace_observation.take()) - { - trace.observe_response(observation, &response); - } - Some(response) - } - Err(error) => { - log::warn!( - "Mediator '{}' response processing failed; using local ranking: {:?}", - mediator.provider_name(), - error - ); - None - } - } - } - } - Err(error) => { - if matches!( - error.current_context(), - crate::platform::PlatformError::DeadlineExceeded - ) { - log::debug!( - "Mediator '{}' exhausted its response budget; using local ranking", - mediator.provider_name() - ); - } else { - log::warn!("Mediator request failed: {:?}", error); - } + + let mediated = match &self.mediator { + Some(mediator) => { + self.run_mediator( + mediator.as_ref(), + &responses, + &MediatorRun { + request: &request, + clock: &clock, + auction_deadline, + trace: trace.as_ref(), + services, + context, + }, + ) + .await + } + None => None, + }; + let winning_bids = match &mediated { + Some(mediator_response) => { + let winning = mediator_response + .bids + .iter() + .filter_map(|bid| { + if bid.price.is_none() { + log::warn!( + "Mediator '{}' returned bid for slot '{}' without decoded price - skipping", + mediator_response.provider, + bid.slot_id + ); None + } else { + Some((bid.slot_id.clone(), bid.clone())) } - }, - Err(error) => { - log::warn!( - "Mediator '{}' failed to dispatch: {:?}", - mediator.provider_name(), - error - ); - None - } - }; - - if let (Some(trace), Some(observation)) = (&trace, trace_observation.take()) { - trace.observe_failure(observation); - } - - // An error status is not a mediation decision, however the provider - // reports it, so it falls back to local ranking like other failures. - if mediator_response - .as_ref() - .is_some_and(|response| response.status == BidStatus::Error) - { - log::warn!( - "Mediator '{}' reported an error response; using local ranking", - mediator.provider_name() - ); - } - if let Some(mediator_response) = - mediator_response.filter(|response| response.status != BidStatus::Error) - { - let winning = mediator_response - .bids - .iter() - .filter_map(|bid| { - if bid.price.is_none() { - log::warn!( - "Mediator '{}' returned bid for slot '{}' without decoded price - skipping", - mediator.provider_name(), - bid.slot_id - ); - None - } else { - Some((bid.slot_id.clone(), bid.clone())) - } - }) - .collect(); - let winning = self.apply_floor_prices(winning, &floor_prices); - (Some(mediator_response), winning) - } else { - (None, self.select_winning_bids(&responses, &floor_prices)) - } + }) + .collect(); + self.apply_floor_prices(winning, &floor_prices) } - } else { - (None, self.select_winning_bids(&responses, &floor_prices)) + None => self.select_winning_bids(&responses, &floor_prices), }; if let Some(trace) = &trace { @@ -2749,13 +976,120 @@ impl AuctionOrchestrator { } OrchestrationResult { provider_responses: responses, - mediator_response, + mediator_response: mediated, winning_bids, - total_time_ms: auction_start.elapsed().as_millis() as u64, + total_time_ms: elapsed_ms(&clock, auction_start), metadata: routing_metadata(planned_unroutable_bidder_count), } } + /// Sends the mediator after the bidder batch, bounded by the auction deadline. + /// + /// Returns `None`, so the caller ranks bids locally, when the deadline has + /// passed, or the mediator fails, times out, returns a response it cannot + /// process (such as a non-success HTTP status), or reports an error status. + async fn run_mediator( + &self, + mediator: &dyn AuctionProvider, + responses: &[AuctionResponse], + run: &MediatorRun<'_>, + ) -> Option { + let remaining = remaining_ms(run.clock, run.auction_deadline); + let logical_budget_ms = remaining.min(mediator.timeout_ms()); + if logical_budget_ms == 0 { + log::warn!( + "A_deadline exhausted before mediator '{}' — returning {} SSP bids without mediation", + mediator.provider_name(), + responses.len(), + ); + return None; + } + // This logs a configured mediator identifier and timeout values, not request data or secrets. + log::info!( + "Running mediator '{}' with {}ms logical budget (A_deadline remaining: {}ms, configured: {}ms)", + mediator.provider_name(), + logical_budget_ms, + remaining, + mediator.timeout_ms(), + ); + // The mediator runs on the collect path. See the doc-comment on + // `AuctionContext::request`: the real client request was already + // consumed during dispatch, so we substitute a canonical placeholder + // URL. Any future mediator that needs real client headers must + // snapshot them at dispatch time onto `DispatchedAuction` rather than + // reading `context.request` here. + let placeholder = http::Request::builder() + .uri(crate::auction::types::MEDIATOR_PLACEHOLDER_URL) + .body(edgezero_core::body::Body::empty()) + .unwrap_or_else(|_| http::Request::new(edgezero_core::body::Body::empty())); + let mediator_context = AuctionContext { + settings: run.context.settings, + request: &placeholder, + timeout_ms: logical_budget_ms, + provider_responses: Some(responses), + services: run.context.services, + }; + let mut trace_observation = run + .trace + .map(|trace| trace.launch_provider(TraceProviderRole::Mediator)); + let mediator_start = run.clock.now(); + let mediated = match mediator.request_bids(run.request, &mediator_context).await { + Ok(ProviderRequestOutcome::Immediate(response)) => Some(response), + Ok(ProviderRequestOutcome::Ready { + request: mediator_request, + parse_state, + }) => match run + .services + .outbound_client() + .send(mediator_request.deadline(run.auction_deadline)) + .await + { + Ok(response) => { + parse_mediator_response( + mediator, + response, + elapsed_ms(run.clock, mediator_start), + run.request, + &mediator_context, + parse_state.as_deref(), + ) + .await + } + Err(error) => { + log_request_failure("Mediator", mediator.provider_name(), error); + None + } + }, + Err(error) => { + log::warn!( + "Mediator '{}' failed to dispatch: {:?}", + mediator.provider_name(), + error + ); + None + } + }; + if let (Some(trace), Some(observation)) = (run.trace, trace_observation.take()) { + match &mediated { + Some(response) => trace.observe_response(observation, response), + None => trace.observe_failure(observation), + } + } + // An error status is not a mediation decision, however the provider + // reports it, so it falls back to local ranking like other failures. + if mediated + .as_ref() + .is_some_and(|response| response.status == BidStatus::Error) + { + log::warn!( + "Mediator '{}' reported an error response; using local ranking", + mediator.provider_name() + ); + return None; + } + mediated + } + /// Check if orchestrator is enabled. #[must_use] pub fn is_enabled(&self) -> bool { @@ -2763,11 +1097,151 @@ impl AuctionOrchestrator { } } -/// Result of an orchestrated auction. -#[derive(Debug, Clone)] -pub struct OrchestrationResult { - /// All responses from providers - pub provider_responses: Vec, +/// What the mediator call needs from one collection. +struct MediatorRun<'a> { + request: &'a AuctionRequest, + clock: &'a MonotonicClock, + auction_deadline: Deadline, + trace: Option<&'a TraceAuctionCarry>, + services: &'a RuntimeServices, + context: &'a AuctionContext<'a>, +} + +/// Turns one resolved bidder slot into the provider's auction response. +/// +/// The slot's `elapsed` runs from the batch start to the bid's arrival, so a +/// collection delayed after a timely arrival reports the same response time. +async fn settle_bidder_slot( + slot: BidderSlot, + result: OutboundSlotResult, + trace: Option<&TraceAuctionCarry>, +) -> AuctionResponse { + let OutboundSlotResult { + elapsed, outcome, .. + } = result; + let response_time_ms = duration_ms(elapsed); + let provider_name = slot.provider.provider_name(); + let observation = slot.trace_observation; + let (response, succeeded) = match outcome { + Ok(response) => match slot + .provider + .parse_response_with_state(response, response_time_ms, slot.parse_state.as_deref()) + .await + { + Ok(parsed) => (parsed, true), + Err(error) => ( + provider_error_response( + provider_name, + response_time_ms, + ERROR_TYPE_PARSE_RESPONSE, + &error, + ), + false, + ), + }, + Err(error) => { + let timed_out = matches!(error, EdgeError::GatewayTimeout { .. }); + let launch_failed = failed_backend_registration(&error); + log_request_failure("Planned provider", provider_name, error); + let response = if timed_out { + provider_timeout_response(provider_name, response_time_ms) + } else if launch_failed { + provider_launch_failed_response(provider_name, response_time_ms) + } else { + provider_transport_failed_response(provider_name, response_time_ms) + }; + (response, false) + } + }; + if let (Some(trace), Some(observation)) = (trace, observation) { + if succeeded { + trace.observe_response(observation, &response); + } else { + trace.observe_failure(observation); + } + } + response +} + +/// Settles the bidder slots `batch` has already recorded, without waiting. +/// +/// Returns how many of them failed backend registration. A failed batch fails +/// every later observation too, so collection marks the slots it leaves. +async fn settle_recorded_slots( + batch: &mut OutboundBatch, + slots: &mut [Option], + responses: &mut Vec, + trace: Option<&TraceAuctionCarry>, +) -> usize { + let mut registration_failures = 0; + loop { + match batch.check() { + Ok(OutboundBatchCheck::Item(item)) => { + let (index, result) = item.into_parts(); + let Some(slot) = slots.get_mut(index).and_then(Option::take) else { + log::warn!("Auction batch reported unknown slot {index}, ignoring"); + continue; + }; + if result + .outcome + .as_ref() + .is_err_and(failed_backend_registration) + { + registration_failures += 1; + } + responses.push(settle_bidder_slot(slot, result, trace).await); + } + Ok(OutboundBatchCheck::Pending | OutboundBatchCheck::Finished(_)) => { + return registration_failures; + } + Err(error) => { + let report = outbound_error( + error, + TrustedServerError::Auction { + message: "Auction bidder batch failed".to_string(), + }, + OutboundRequestSource::Internal, + ); + log::warn!("Auction bidder batch failed as it started: {report:?}"); + return registration_failures; + } + } + } +} + +/// Parses a mediator response, logging and dropping a processing failure. +async fn parse_mediator_response( + mediator: &dyn AuctionProvider, + response: OutboundResponse, + response_time_ms: u64, + request: &AuctionRequest, + context: &AuctionContext<'_>, + parse_state: Option<&(dyn core::any::Any + Send + Sync)>, +) -> Option { + mediator + .parse_response_with_context_and_state( + response, + response_time_ms, + request, + context, + parse_state, + ) + .await + .map_err(|error| { + log::warn!( + "Mediator '{}' response processing failed; using local ranking: {:?}", + mediator.provider_name(), + error + ); + }) + .ok() +} + +/// Result of an orchestrated auction. +#[derive(Debug, Clone)] +pub struct OrchestrationResult { + /// All responses from providers + pub provider_responses: Vec, /// Final response from mediator (if used) pub mediator_response: Option, /// Winning bids per slot @@ -2818,9 +1292,11 @@ mod tests { use std::time::Duration; use base64::Engine as _; - use web_time::Instant; + use edgezero_core::error::{BadGatewayReason, BudgetSource, EdgeError}; + use edgezero_core::outbound::OutboundResponse; + use edgezero_core::{FastlyFailureCause, ProviderFailureDetail, ProviderFailurePhase}; + use http::{Method, StatusCode}; - use crate::auction::config::AuctionConfig; use crate::auction::orchestrator::DispatchAuctionOutcome; use crate::auction::plan::{ AuctionPlan, AuctionPlanConfig, NotificationConfig, ProviderConfig, ProviderId, RoutingMode, @@ -2832,7 +1308,6 @@ mod tests { use crate::auction::telemetry::{ AuctionObservationContext, AuctionSource, AuctionTerminalOutcome, build_auction_events, }; - use crate::auction::test_support::create_test_auction_context; use crate::auction::types::{ AdFormat, AdSlot, ApsRendererV1, ApsTagType, AuctionContext, AuctionRequest, AuctionResponse, Bid, BidRenderer, BidStatus, MediaType, PublisherInfo, UserInfo, @@ -2841,27 +1316,140 @@ mod tests { use crate::ec::EcContext; use crate::error::TrustedServerError; use crate::integrations::adserver_mock::{AdServerMockConfig, AdServerMockProvider}; - use crate::platform::test_support::{ - StubHttpClient, build_services_with_backend_and_http_client, - build_services_with_http_client, noop_services, - }; + use crate::platform::test_support::build_services_with_outbound_client; use crate::platform::{ - BackendNamingPolicy, PlatformBackend, PlatformBackendSpec, PlatformConfigStore, - PlatformError, PlatformHttpClient, PlatformHttpRequest, PlatformPendingRequest, - PlatformResponse, PlatformSecretStore, PlatformSelectResult, RuntimeServices, StoreId, + PlatformConfigStore, PlatformError, PlatformSecretStore, RuntimeServices, StoreId, StoreName, }; + use crate::test_support::scripted_client::{ + ScriptedHttpClient, ScriptedReply, TestClock, outbound_response, + }; use crate::test_support::tests::crate_test_settings_str; - use error_stack::{Report, ResultExt}; - use std::collections::{BTreeMap, BTreeSet, HashMap, HashSet}; + use error_stack::Report; + use std::collections::{BTreeMap, HashMap}; + use std::sync::Arc; use std::sync::atomic::{AtomicUsize, Ordering}; - use std::sync::{Arc, Mutex}; use super::{ - AuctionOrchestrator, AuctionOrchestratorHarness, DispatchedAuction, - ERROR_TYPE_LAUNCH_FAILED, ERROR_TYPE_TIMEOUT, ERROR_TYPE_TRANSPORT, OrchestrationResult, + AuctionOrchestrator, DispatchedAuction, ERROR_TYPE_LAUNCH_FAILED, ERROR_TYPE_TIMEOUT, + ERROR_TYPE_TRANSPORT, OrchestrationResult, }; + /// Endpoint every [`planned_config`] provider shares. + const PLANNED_ENDPOINT: &str = "https://example.test/openrtb"; + + /// Endpoint every [`planned_aps_instances_config`] provider shares. + const APS_ENDPOINT: &str = "https://aps.example/e/pb/bid"; + + /// Builds the live orchestrator over `plan` with an optional mediator. + fn planned_orchestrator( + plan: impl Into>, + mediator: Option>, + ) -> AuctionOrchestrator { + AuctionOrchestrator::from_plan(plan.into(), mediator) + } + + /// Builds an orchestrator over an empty plan, for the ranking helpers. + fn empty_orchestrator(enabled: bool) -> AuctionOrchestrator { + planned_orchestrator( + AuctionPlan::compile(AuctionPlanConfig { + timeout_ms: 777, + ..AuctionPlanConfig::default() + }) + .expect("should compile an empty plan") + .with_enabled(enabled), + None, + ) + } + + /// A scripted reply with `status` and `body`. + fn reply(status: u16, body: Vec) -> ScriptedReply { + ScriptedReply::status(StatusCode::from_u16(status).expect("should use a valid status")) + .body(body) + } + + /// Queues one reply for the next request to `endpoint`. + fn respond(client: &ScriptedHttpClient, endpoint: &str, status: u16, body: Vec) { + client.respond_to(Method::POST, endpoint, reply(status, body)); + } + + /// Queues one reply for the next request to [`PLANNED_ENDPOINT`]. + fn respond_planned(client: &ScriptedHttpClient, status: u16, body: Vec) { + respond(client, PLANNED_ENDPOINT, status, body); + } + + /// The URI of every request sent so far, in order. + fn recorded_uris(client: &ScriptedHttpClient) -> Vec { + client + .requests() + .iter() + .map(|request| request.uri.to_string()) + .collect() + } + + /// The headers of every request sent so far, as lowercase name and value pairs. + fn recorded_headers(client: &ScriptedHttpClient) -> Vec> { + client + .requests() + .iter() + .map(|request| { + request + .headers + .iter() + .map(|(name, value)| { + ( + name.as_str().to_owned(), + value + .to_str() + .expect("should send UTF-8 header values") + .to_owned(), + ) + }) + .collect() + }) + .collect() + } + + /// Queues one reply with response `headers` for the next request to `endpoint`. + fn respond_with_headers( + client: &ScriptedHttpClient, + endpoint: &str, + status: u16, + body: Vec, + headers: &[(&str, &str)], + ) { + let reply = headers + .iter() + .fold(reply(status, body), |reply, (name, value)| { + reply.header(name, value) + }); + client.respond_to(Method::POST, endpoint, reply); + } + + /// The headers of every request sent so far, with raw value bytes. + fn recorded_header_bytes(client: &ScriptedHttpClient) -> Vec)>> { + client + .requests() + .iter() + .map(|request| { + request + .headers + .iter() + .map(|(name, value)| (name.as_str().to_owned(), value.as_bytes().to_vec())) + .collect() + }) + .collect() + } + + /// The body of every request sent so far. + fn recorded_bodies(client: &ScriptedHttpClient) -> Vec> { + client + .requests() + .iter() + .map(|request| request.body.to_vec()) + .collect() + } + fn planned_config(providers: &[(&str, RoutingMode)], signing: bool) -> AuctionPlanConfig { AuctionPlanConfig { timeout_ms: 777, @@ -2873,7 +1461,7 @@ mod tests { ProviderConfig { protocol: "openrtb-2.6".to_string(), profile: "standard".to_string(), - endpoint: "https://example.test/openrtb".to_string(), + endpoint: PLANNED_ENDPOINT.to_string(), timeout_ms: Some(1_000), routing: *routing, notifications: Default::default(), @@ -2892,6 +1480,293 @@ mod tests { } } + /// Endpoint of the bidder that the launch-failure tests make fail. + const FAILING_BIDDER: &str = "https://failing.example.test/openrtb"; + + /// [`FAILING_BIDDER`] with a backslash in its query. The plan accepts it, + /// but `EdgeZero` builds no outbound request for a target with a backslash. + const UNBUILDABLE_BIDDER: &str = r"https://failing.example.test/openrtb?x=\"; + + /// Endpoint of the bidder whose backend registration runs out of budget. + const TIMED_OUT_BIDDER: &str = "https://timed-out.example.test/openrtb"; + + /// How a launch-failure test makes its bidder fail to launch. + #[derive(Clone, Copy, Debug)] + enum LaunchFailure { + /// Trusted Server cannot build the request, so it never joins the batch. + Construction, + /// Fastly cannot register the bidder backend as the batch starts. + Registration, + } + + impl LaunchFailure { + const ALL: [Self; 2] = [Self::Construction, Self::Registration]; + + /// The endpoint a provider failing this way is configured with. + fn endpoint(self) -> &'static str { + match self { + Self::Construction => UNBUILDABLE_BIDDER, + Self::Registration => FAILING_BIDDER, + } + } + + /// Scripts the failure for `auctions` auctions on `bidders`. + fn script(self, bidders: &ScriptedHttpClient, auctions: usize) { + if matches!(self, Self::Registration) { + for _ in 0..auctions { + bidders.respond_to( + Method::POST, + FAILING_BIDDER, + ScriptedReply::error(backend_registration_error()), + ); + } + } + } + + /// How many bid requests reach `EdgeZero` in one auction. + fn requests_per_auction(self) -> usize { + match self { + Self::Construction => 0, + Self::Registration => 1, + } + } + } + + /// The error Fastly reports when it cannot register a bidder's backend. + fn backend_registration_error() -> EdgeError { + EdgeError::bad_gateway_with_detail( + "Fastly failed to register a dynamic backend", + BadGatewayReason::Unspecified, + ProviderFailureDetail::Fastly { + phase: ProviderFailurePhase::BackendRegistration, + cause: FastlyFailureCause::BackendHostError { status: 1 }, + }, + ) + } + + /// The error Fastly reports when a backend registration overruns the + /// request budget. It carries the registration phase, but it is a timeout. + fn backend_registration_timeout() -> EdgeError { + EdgeError::gateway_timeout_with_detail( + "outbound request deadline expired", + BudgetSource::PerCallTimeout, + ProviderFailureDetail::Fastly { + phase: ProviderFailurePhase::BackendRegistration, + cause: FastlyFailureCause::BackendHostError { status: 1 }, + }, + ) + } + + /// Compiles one standard provider per `(id, endpoint)`. + fn launch_plan(providers: &[(&str, &str)]) -> Arc { + let mut config = planned_config( + &providers + .iter() + .map(|(id, _)| (*id, RoutingMode::AllEligible)) + .collect::>(), + false, + ); + for (id, endpoint) in providers { + config + .providers + .get_mut(&ProviderId::from_str(id).expect("should parse fictional provider ID")) + .expect("should configure the provider") + .endpoint = (*endpoint).to_string(); + } + Arc::new(AuctionPlan::compile(config).expect("should compile the launch-failure plan")) + } + + fn launch_context<'a>( + settings: &'a crate::settings::Settings, + inbound: &'a http::Request, + services: &'a RuntimeServices, + ) -> AuctionContext<'a> { + AuctionContext { + settings, + request: inbound, + timeout_ms: 777, + provider_responses: None, + services, + } + } + + /// Each provider with the `error_type` its response carries, if any. + fn error_types(result: &OrchestrationResult) -> Vec<(&str, Option)> { + result + .provider_responses + .iter() + .map(|response| { + ( + response.provider.as_str(), + response.metadata.get("error_type").cloned(), + ) + }) + .collect() + } + + /// Bidders that fail to launch, through construction or backend registration. + mod launch_failure_tests { + use super::*; + + #[tokio::test] + async fn all_planned_launch_failures_error_direct_and_surface_split_failure() { + for failure in LaunchFailure::ALL { + let clock = TestClock::new(); + let bidders = ScriptedHttpClient::new(&clock); + failure.script(&bidders, 2); + let services = build_services_with_outbound_client(&bidders, &clock); + let orchestrator = AuctionOrchestrator::from_plan( + launch_plan(&[("launch-fail", failure.endpoint())]), + None, + ); + let settings = create_test_settings(); + let inbound = http::Request::new(edgezero_core::body::Body::empty()); + let context = launch_context(&settings, &inbound, &services); + let request = planned_request(); + + let error = orchestrator + .run_auction(&request, &context) + .await + .expect_err("should fail direct execution when every planned launch fails"); + assert!( + error + .to_string() + .contains("All eligible planned provider requests failed to launch"), + "{failure:?} should distinguish launch failure from an ordinary no-bid: {error:?}" + ); + + let DispatchAuctionOutcome::DispatchFailed { + provider_responses, + fatal_admission_error, + .. + } = orchestrator.dispatch_auction(&request, &context).await + else { + panic!("{failure:?} should surface a split dispatch failure"); + }; + assert!( + fatal_admission_error.is_none(), + "{failure:?} should carry no fatal admission error" + ); + assert_eq!( + provider_responses + .iter() + .map(|response| ( + response.provider.as_str(), + response.metadata["error_type"].clone() + )) + .collect::>(), + vec![("launch-fail", serde_json::json!(ERROR_TYPE_LAUNCH_FAILED))], + "{failure:?} should keep one launch failure for the failed provider" + ); + assert_eq!( + bidders.requests().len(), + 2 * failure.requests_per_auction(), + "{failure:?} should send no bid request beyond the failed registrations" + ); + } + } + + #[tokio::test] + async fn launch_failure_plus_prelaunch_timeout_remains_a_completed_auction() { + let settings = create_test_settings(); + let inbound = http::Request::new(edgezero_core::body::Body::empty()); + for failure in LaunchFailure::ALL { + let clock = TestClock::new(); + let bidders = ScriptedHttpClient::new(&clock); + failure.script(&bidders, 1); + bidders.respond_to( + Method::POST, + TIMED_OUT_BIDDER, + ScriptedReply::error(backend_registration_timeout()), + ); + let services = build_services_with_outbound_client(&bidders, &clock); + let orchestrator = AuctionOrchestrator::from_plan( + launch_plan(&[ + ("launch-fail", failure.endpoint()), + ("timeout", TIMED_OUT_BIDDER), + ]), + None, + ); + let context = launch_context(&settings, &inbound, &services); + + let result = orchestrator + .run_auction(&planned_request(), &context) + .await + .expect("should treat a registration timeout as a completed provider outcome"); + + assert_eq!( + error_types(&result), + vec![ + ( + "launch-fail", + Some(serde_json::json!(ERROR_TYPE_LAUNCH_FAILED)) + ), + ("timeout", Some(serde_json::json!(ERROR_TYPE_TIMEOUT))), + ], + "{failure:?} should keep the launch failure beside the timeout" + ); + } + + let clock = TestClock::new(); + let bidders = ScriptedHttpClient::new(&clock); + bidders.respond_to( + Method::POST, + TIMED_OUT_BIDDER, + ScriptedReply::error(backend_registration_timeout()), + ); + let services = build_services_with_outbound_client(&bidders, &clock); + let orchestrator = + AuctionOrchestrator::from_plan(launch_plan(&[("timeout", TIMED_OUT_BIDDER)]), None); + let context = launch_context(&settings, &inbound, &services); + + let result = orchestrator + .run_auction(&planned_request(), &context) + .await + .expect("should complete an auction whose only bidder timed out registering"); + + assert_eq!( + error_types(&result), + vec![("timeout", Some(serde_json::json!(ERROR_TYPE_TIMEOUT)))], + "should report the lone registration timeout as a timeout, not a launch failure" + ); + } + + #[tokio::test] + async fn provider_launch_failures_error_when_no_requests_launch() { + let clock = TestClock::new(); + let bidders = ScriptedHttpClient::new(&clock); + LaunchFailure::Registration.script(&bidders, 1); + let services = build_services_with_outbound_client(&bidders, &clock); + let orchestrator = AuctionOrchestrator::from_plan( + launch_plan(&[ + ("unbuildable", UNBUILDABLE_BIDDER), + ("unregistered", FAILING_BIDDER), + ]), + None, + ); + let settings = create_test_settings(); + let inbound = http::Request::new(edgezero_core::body::Body::empty()); + let context = launch_context(&settings, &inbound, &services); + + let error = orchestrator + .run_auction(&planned_request(), &context) + .await + .expect_err("should fail when every provider launch fails"); + + assert!( + error + .to_string() + .contains("All eligible planned provider requests failed to launch"), + "should explain that no provider request launched: {error:?}" + ); + assert_eq!( + recorded_uris(&bidders), + vec![FAILING_BIDDER.to_string()], + "should send only the request whose registration failed" + ); + } + } + fn planned_prebid_config( providers: &[(&str, serde_json::Value, NotificationConfig)], ) -> AuctionPlanConfig { @@ -2941,7 +1816,7 @@ mod tests { ProviderConfig { protocol: "openrtb-2.6".to_string(), profile: "aps".to_string(), - endpoint: "https://aps.example/e/pb/bid".to_string(), + endpoint: APS_ENDPOINT.to_string(), timeout_ms: Some(1_000), routing: RoutingMode::AllEligible, notifications: notifications.clone(), @@ -2993,3609 +1868,1924 @@ mod tests { request } - mod trace_auction_live_tests { + /// Bidders and the mediator through `EdgeZero` batches on the scripted client. + mod batch_tests { use super::*; - use crate::trace::{TraceAuctionCarry, TraceAuctionSource}; - use edgezero_core::body::Body as EdgeBody; - use http::Request; - - fn captured_request(request: &AuctionRequest) -> (Request, TraceAuctionCarry) { - let carry = TraceAuctionCarry::capture_if_enabled( - true, - TraceAuctionSource::AuctionApi, - &request.slots, + use crate::auction::telemetry::{ + AuctionObservationContext, AuctionSource, AuctionTerminalOutcome, build_auction_events, + }; + use crate::platform::test_support::{ + build_services_with_outbound_client, noop_services_with_client_ip, + }; + use crate::test_support::scripted_client::{ + ExpectedTransport, RecordedRequest, ReplyRelease, ScriptedReply, + }; + use edgezero_core::error::{BudgetSource, EdgeError}; + use edgezero_core::outbound::{ + OutboundCachePolicy, OutboundSendMode, ResponseDecoding, ResponseMode, + }; + use http::{Method, StatusCode}; + use std::net::{IpAddr, Ipv4Addr}; + + const AUCTION_TIMEOUT: Duration = Duration::from_millis(500); + const FAST_BIDDER: &str = "https://fast.example.test/openrtb"; + const SLOW_BIDDER: &str = "https://slow.example.test/openrtb"; + const MEDIATOR_ENDPOINT: &str = "https://mediator.example.test/mediate"; + const BIDDER_RESPONSE_BYTES: u64 = 1024 * 1024; + + /// Compiles standard providers from `(id, endpoint, timeout)` triples. + fn batch_plan(providers: &[(&str, &str, Duration)]) -> Arc { + Arc::new( + AuctionPlan::compile(AuctionPlanConfig { + timeout_ms: duration_ms(AUCTION_TIMEOUT), + providers: providers + .iter() + .map(|(id, endpoint, timeout)| { + ( + ProviderId::from_str(id) + .expect("should parse fictional provider ID"), + ProviderConfig { + protocol: "openrtb-2.6".to_string(), + profile: "standard".to_string(), + endpoint: (*endpoint).to_string(), + timeout_ms: Some(duration_ms(*timeout)), + routing: RoutingMode::AllEligible, + notifications: NotificationConfig::default(), + profile_config: serde_json::json!({}), + }, + ) + }) + .collect(), + bidders: BTreeMap::new(), + mediator: None, + request_signing: None, + }) + .expect("should compile the batch plan"), ) - .expect("should capture the enabled request"); - let mut inbound = Request::new(EdgeBody::empty()); - inbound.extensions_mut().insert(carry.clone()); - (inbound, carry) } - fn evidence(carry: &TraceAuctionCarry) -> serde_json::Value { - serde_json::to_value( - carry - .transport() - .expect("should terminalize the live auction"), - ) - .expect("should serialize redacted facts")["evidence"] - .clone() + fn duration_ms(duration: Duration) -> u32 { + u32::try_from(duration.as_millis()).expect("should fit the fixture duration in u32") } - #[tokio::test] - async fn trace_auction_live_empty_plan_distinguishes_sync_success_from_split_failure() { - let plan = Arc::new( - AuctionPlan::compile(planned_config(&[], false)) - .expect("should compile an empty plan"), - ); - let orchestrator = AuctionOrchestrator::from_plan(plan, None); - let http = Arc::new(StubHttpClient::new()); - let services = build_services_with_http_client(Arc::clone(&http) as Arc<_>); - let settings = create_test_settings(); - let request = planned_request(); - for split in [false, true] { - let (inbound, carry) = captured_request(&request); - let token = carry.token(); - let context = AuctionContext { - settings: &settings, - request: &inbound, - timeout_ms: 777, - transport_timeout_ms: 777, - provider_responses: None, - services: &services, - }; - if split { - assert!( - matches!( - orchestrator.dispatch_auction(&request, &context).await, - DispatchAuctionOutcome::NotStarted - ), - "should preserve the ordinary split outcome" - ); - } else { - let result = orchestrator - .run_auction(&request, &context) - .await - .expect("should preserve successful empty-plan execution"); - assert!( - result.winning_bids.is_empty(), - "should preserve an ordinary zero-bid result" - ); + /// A standard `OpenRTB` reply with one bid for the fixture slot. + fn bid_reply(price: f64) -> ScriptedReply { + ScriptedReply::status(StatusCode::OK) + .header("content-type", "application/json") + .body( + serde_json::to_vec(&serde_json::json!({ + "seatbid": [{"seat": "fictional-seat", "bid": [{ + "id": "fictional-bid", + "impid": "fictional-slot", + "price": price, + "adm": "
fictional creative
", + "w": 300, + "h": 250 + }]}] + })) + .expect("should serialize the bid reply"), + ) + } + + fn batch_context<'a>( + settings: &'a crate::settings::Settings, + inbound: &'a http::Request, + services: &'a RuntimeServices, + timeout: Duration, + ) -> AuctionContext<'a> { + AuctionContext { + settings, + request: inbound, + timeout_ms: duration_ms(timeout), + provider_responses: None, + services, + } + } + + fn inbound_request() -> http::Request { + http::Request::builder() + .uri("https://publisher.example/article") + .body(edgezero_core::body::Body::empty()) + .expect("should build the inbound request") + } + + fn dispatched(outcome: DispatchAuctionOutcome) -> DispatchedAuction { + match outcome { + DispatchAuctionOutcome::Dispatched(dispatched) => dispatched, + DispatchAuctionOutcome::NotStarted => panic!("should dispatch, not skip"), + DispatchAuctionOutcome::DispatchFailed { .. } => { + panic!("should dispatch, not fail") } - let value = evidence(&carry); - assert_eq!( - value["diagnostic_auction_id"], - token.as_str(), - "should preserve the pre-dispatch token" - ); - assert_eq!( - value["terminal_status"], - if split { - "dispatch_failed" - } else { - "completed" - }, - "should map the observed path semantics" - ); - assert_eq!( - value["provider_calls"], - serde_json::json!([]), - "should not fabricate a launched provider" - ); } } - #[tokio::test] - async fn trace_auction_live_launch_failure_survives_the_ordinary_error() { - let plan = Arc::new( - AuctionPlan::compile(planned_config( - &[("launch-fail", RoutingMode::AllEligible)], - false, - )) - .expect("should compile the launch-failure plan"), - ); - let orchestrator = AuctionOrchestrator::from_plan(plan, None); - let backend = Arc::new(NamingBackend::new(BackendNamingPolicy::Axum)); - backend.fail_ensure_for("launch-fail"); - let http = Arc::new(StubHttpClient::new()); - let services = build_services_with_backend_and_http_client( - Arc::clone(&backend) as Arc<_>, - Arc::clone(&http) as Arc<_>, + fn response_for<'a>( + result: &'a OrchestrationResult, + provider: &str, + ) -> &'a AuctionResponse { + result + .provider_responses + .iter() + .find(|response| response.provider == provider) + .unwrap_or_else(|| panic!("should report provider {provider}")) + } + + fn request_to<'a>(requests: &'a [RecordedRequest], uri: &str) -> &'a RecordedRequest { + requests + .iter() + .find(|request| request.uri == uri) + .unwrap_or_else(|| panic!("should send a request to {uri}")) + } + + #[test] + fn bidders_leave_together_in_one_batch() { + let clock = TestClock::new(); + let client = ScriptedHttpClient::new(&clock); + client + .respond_to( + Method::POST, + FAST_BIDDER, + bid_reply(1.5).after(Duration::from_millis(100)), + ) + .respond_to( + Method::POST, + SLOW_BIDDER, + bid_reply(2.5).after(Duration::from_millis(200)), + ); + let services = build_services_with_outbound_client(&client, &clock); + let orchestrator = AuctionOrchestrator::from_plan( + batch_plan(&[ + ("fast", FAST_BIDDER, Duration::from_secs(1)), + ("slow", SLOW_BIDDER, Duration::from_secs(1)), + ]), + None, ); let settings = create_test_settings(); - let request = planned_request(); - let (inbound, carry) = captured_request(&request); - let context = AuctionContext { - settings: &settings, - request: &inbound, - timeout_ms: 777, - transport_timeout_ms: 777, - provider_responses: None, - services: &services, - }; - let _error = orchestrator - .run_auction(&request, &context) - .await - .expect_err("should preserve the ordinary launch failure"); - let value = evidence(&carry); + let inbound = inbound_request(); + let context = batch_context(&settings, &inbound, &services, AUCTION_TIMEOUT); + let started = clock.monotonic_clock().now(); + + let dispatched = dispatched( + clock.drive(orchestrator.dispatch_auction(&planned_request(), &context)), + ); + let sent_at_dispatch = client.requests(); + let result = clock + .drive(orchestrator.collect_dispatched_auction(dispatched, &services, &context)); + assert_eq!( - value["terminal_status"], "dispatch_failed", - "should preserve the actual dispatch failure" + sent_at_dispatch.len(), + 2, + "should send both bidder requests before any reply arrives" ); + let cutoff = started + .checked_add(AUCTION_TIMEOUT) + .expect("should compute the auction cutoff"); + for request in &sent_at_dispatch { + assert_eq!( + ( + request.mode, + request.budget.cause, + request.budget.deadline.instant() + ), + (OutboundSendMode::Batch, BudgetSource::BatchCutoff, cutoff), + "should send {} in the auction batch, capped by its cutoff", + request.uri + ); + } assert_eq!( - value["terminal_reason"], "provider_execution_failed", - "should use the directly observed launch failure" + result.provider_responses.len(), + 2, + "should report both bidders" ); assert_eq!( - value["provider_calls"][0]["status"], "error", - "should retain the failed actual invocation" + result + .get_winning_bid("fictional-slot") + .and_then(|bid| bid.price), + Some(2.5), + "should rank both bidders' bids" + ); + } + + #[test] + fn bid_after_cutoff_becomes_provider_timeout() { + let clock = TestClock::new(); + let client = ScriptedHttpClient::new(&clock); + client + .respond_to( + Method::POST, + FAST_BIDDER, + bid_reply(1.5).after(Duration::from_millis(100)), + ) + .respond_to( + Method::POST, + SLOW_BIDDER, + bid_reply(9.0).after(Duration::from_millis(600)), + ); + let services = build_services_with_outbound_client(&client, &clock); + let orchestrator = AuctionOrchestrator::from_plan( + batch_plan(&[ + ("fast", FAST_BIDDER, Duration::from_secs(1)), + ("slow", SLOW_BIDDER, Duration::from_secs(1)), + ]), + None, ); - assert!( - !value.to_string().contains("launch-fail"), - "should exclude provider identities from public facts" + let settings = create_test_settings(); + let inbound = inbound_request(); + let context = batch_context(&settings, &inbound, &services, AUCTION_TIMEOUT); + + let result = clock + .drive(orchestrator.run_auction(&planned_request(), &context)) + .expect("should complete the auction at the cutoff"); + + let slow = response_for(&result, "slow"); + assert_eq!( + (slow.status.clone(), slow.metadata.get("error_type")), + ( + BidStatus::Error, + Some(&serde_json::json!(ERROR_TYPE_TIMEOUT)) + ), + "should report the bid after the cutoff as a provider timeout" ); - assert!( - http.recorded_backend_names().is_empty(), - "should preserve ordinary network behavior" + assert_eq!( + slow.response_time_ms, 500, + "should time the late bidder to the cutoff" ); - } - - #[tokio::test] - async fn trace_auction_live_collection_failure_keeps_launch_order_through_local_fallback() { - let http = Arc::new(OuterSelectErrorHttpClient::new()); - http.push_response(204, Vec::new()); - http.push_response(204, Vec::new()); - let services = build_services_with_http_client(Arc::clone(&http) as Arc<_>); - let plan = Arc::new( - AuctionPlan::compile(planned_config( - &[ - ("provider-b", RoutingMode::AllEligible), - ("provider-a", RoutingMode::AllEligible), - ], - false, - )) - .expect("should compile the ordered plan"), + assert_eq!( + result + .get_winning_bid("fictional-slot") + .and_then(|bid| bid.price), + Some(1.5), + "should keep only the bid that answered before the cutoff" + ); + } + + #[test] + fn mediator_runs_after_batch_with_auction_deadline() { + let clock = TestClock::new(); + let client = ScriptedHttpClient::new(&clock); + client + .respond_to( + Method::POST, + FAST_BIDDER, + bid_reply(1.5).after(Duration::from_millis(100)), + ) + .respond_to( + Method::POST, + MEDIATOR_ENDPOINT, + ScriptedReply::status(StatusCode::OK) + .body( + serde_json::to_vec(&serde_json::json!({ + "seatbid": [{"seat": "fast", "bid": [{ + "id": "mediated-bid", + "impid": "fictional-slot", + "price": 1.75, + "adm": "
mediated
", + "w": 300, + "h": 250, + "crid": "fictional-seat-creative" + }]}] + })) + .expect("should serialize the mediator reply"), + ) + .after(Duration::from_millis(50)), + ); + let services = build_services_with_outbound_client(&client, &clock); + let mediator = AdServerMockProvider::new(AdServerMockConfig { + enabled: true, + endpoint: MEDIATOR_ENDPOINT.to_string(), + timeout_ms: 300, + price_floor: None, + context_query_params: BTreeMap::new(), + }); + let orchestrator = AuctionOrchestrator::from_plan( + batch_plan(&[("fast", FAST_BIDDER, Duration::from_secs(1))]), + Some(Arc::new(mediator)), ); - let orchestrator = AuctionOrchestrator::from_plan(plan, None); - let request = planned_request(); let settings = create_test_settings(); - let (inbound, carry) = captured_request(&request); - let context = AuctionContext { - settings: &settings, - request: &inbound, - timeout_ms: 777, - transport_timeout_ms: 777, - provider_responses: None, - services: &services, - }; - let result = orchestrator - .run_auction(&request, &context) - .await - .expect("should preserve ordinary local fallback"); - assert!( - result.winning_bids.is_empty(), - "should preserve ordinary ranking" + let inbound = inbound_request(); + let context = batch_context(&settings, &inbound, &services, AUCTION_TIMEOUT); + let auction_deadline = clock.monotonic_clock().deadline_after(AUCTION_TIMEOUT); + + let dispatched = dispatched( + clock.drive(orchestrator.dispatch_auction(&planned_request(), &context)), ); - let value = evidence(&carry); + let sent_before_collection = client.requests().len(); + let result = clock + .drive(orchestrator.collect_dispatched_auction(dispatched, &services, &context)); + assert_eq!( - value["terminal_status"], "execution_failed", - "should not infer successful collection from an ordinary Ok result" + sent_before_collection, 1, + "should not send the mediator request while bidders are in flight" ); + let requests = client.requests(); assert_eq!( - value["terminal_reason"], "collection_failed", - "should record the failure at its source" + requests + .iter() + .map(|request| request.uri.to_string()) + .collect::>(), + vec![FAST_BIDDER.to_string(), MEDIATOR_ENDPOINT.to_string()], + "should send the mediator after the bidder batch" ); + let mediator_request = &requests[1]; assert_eq!( - value["provider_calls"][0]["provider_number"], 1, - "should preserve first launch order" + ( + mediator_request.mode, + mediator_request.timeout, + mediator_request.deadline.map(|deadline| deadline.instant()) + ), + ( + OutboundSendMode::Send, + Some(Duration::from_millis(300)), + Some(auction_deadline.instant()) + ), + "should send the mediator with its own timeout and the auction deadline" ); assert_eq!( - value["provider_calls"][1]["provider_number"], 2, - "should preserve second launch order" + ( + mediator_request.connect_timeout, + mediator_request.cache_policy, + mediator_request.response_decoding, + mediator_request.response_mode + ), + ( + Some(Duration::from_secs(1)), + OutboundCachePolicy::Bypass, + ResponseDecoding::Decode, + ResponseMode::Buffered { + max_bytes: 2 * 1024 * 1024 + } + ), + "should buffer the mediator response under 2 MiB, decoded, uncached" ); + let mediated = result + .mediator_response + .as_ref() + .expect("should keep the mediator response"); assert_eq!( - value["provider_calls"][0]["status"], "error", - "should retain transport failure facts" + mediated.bids.first().and_then(|bid| bid.price), + Some(1.75), + "should parse the mediated bid" ); } - #[tokio::test] - async fn trace_auction_live_mediator_no_bid_is_an_actual_ordered_call() { - let http = Arc::new(StubHttpClient::new()); - http.push_response(204, Vec::new()); - let services = build_services_with_http_client(Arc::clone(&http) as Arc<_>); - let plan = Arc::new( - AuctionPlan::compile(planned_config( - &[("bidder", RoutingMode::AllEligible)], - false, - )) - .expect("should compile bidder plan"), + #[test] + fn bidder_timeout_is_configured_value_not_remaining_budget() { + let clock = TestClock::new(); + let client = ScriptedHttpClient::new(&clock); + client.respond_to(Method::POST, FAST_BIDDER, bid_reply(1.5)); + let services = build_services_with_outbound_client(&client, &clock); + let orchestrator = AuctionOrchestrator::from_plan( + batch_plan(&[("fast", FAST_BIDDER, Duration::from_millis(300))]), + None, ); - let orchestrator = - AuctionOrchestrator::from_plan(plan, Some(Arc::new(ImmediateNoBidProvider))); - let request = planned_request(); let settings = create_test_settings(); - let (inbound, carry) = captured_request(&request); - let context = AuctionContext { - settings: &settings, - request: &inbound, - timeout_ms: 777, - transport_timeout_ms: 777, - provider_responses: None, - services: &services, - }; - orchestrator - .run_auction(&request, &context) - .await - .expect("should retain ordinary successful zero-bid execution"); - let value = evidence(&carry); + let inbound = inbound_request(); + let auction_budget = Duration::from_millis(200); + let context = batch_context(&settings, &inbound, &services, auction_budget); + let cutoff = clock.monotonic_clock().deadline_after(auction_budget); + + clock + .drive(orchestrator.run_auction(&planned_request(), &context)) + .expect("should complete the auction"); + + let requests = client.requests(); + let bidder = request_to(&requests, FAST_BIDDER); + bidder.assert_transport(ExpectedTransport { + timeout: Duration::from_millis(300), + response_mode: ResponseMode::Buffered { + max_bytes: BIDDER_RESPONSE_BYTES, + }, + response_decoding: ResponseDecoding::Decode, + cache_policy: OutboundCachePolicy::Bypass, + }); assert_eq!( - value["terminal_status"], "completed", - "should not label ordinary zero bids a failure" + (bidder.budget.cause, bidder.budget.deadline.instant()), + (BudgetSource::BatchCutoff, cutoff.instant()), + "should let the batch cutoff, not the timeout, bound the shorter auction budget" ); + let body: serde_json::Value = + serde_json::from_slice(&bidder.body).expect("should send a JSON bid request"); assert_eq!( - value["provider_calls"][0]["role"], "bidder", - "should retain the bidder launch" + body["tmax"], + serde_json::json!(200), + "should keep the remaining auction budget in the payload's tmax" ); - assert_eq!( - value["provider_calls"][1]["role"], "mediator", - "should retain the real mediator invocation" + } + + #[test] + fn two_auctions_with_different_budgets_record_identical_request_options() { + let clock = TestClock::new(); + let client = ScriptedHttpClient::new(&clock); + client + .respond_to(Method::POST, FAST_BIDDER, bid_reply(1.5)) + .respond_to(Method::POST, FAST_BIDDER, bid_reply(1.5)); + let services = build_services_with_outbound_client(&client, &clock); + let orchestrator = AuctionOrchestrator::from_plan( + batch_plan(&[("fast", FAST_BIDDER, Duration::from_secs(1))]), + None, ); + let settings = create_test_settings(); + let inbound = inbound_request(); + + for budget in [Duration::from_millis(450), Duration::from_millis(250)] { + let context = batch_context(&settings, &inbound, &services, budget); + clock + .drive(orchestrator.run_auction(&planned_request(), &context)) + .expect("should complete the auction"); + } + + let requests = client.requests(); + assert_eq!(requests.len(), 2, "should send one request per auction"); + let options = |request: &RecordedRequest| { + ( + request.method.clone(), + request.uri.clone(), + request.timeout, + request.connect_timeout, + request.deadline.map(|deadline| deadline.instant()), + request.cache_policy, + request.response_decoding, + request.response_mode, + request.host_authority.clone(), + request.has_host_authority_override, + ) + }; assert_eq!( - value["provider_calls"][1]["status"], "no_bid", - "should retain mediator zero-bid status" + options(&requests[0]), + options(&requests[1]), + "should send the same backend-identity options whatever the remaining budget" + ); + assert_ne!( + requests[0].budget.deadline.instant(), + requests[1].budget.deadline.instant(), + "should still cap each auction at its own cutoff" ); } - struct NeverFinishesMediator; - #[async_trait::async_trait(?Send)] - impl AuctionProvider for NeverFinishesMediator { - fn provider_name(&self) -> &str { - "pending-mediator" - } - async fn request_bids( - &self, - _request: &AuctionRequest, - _context: &AuctionContext<'_>, - ) -> Result> { - std::future::pending().await - } - async fn parse_response( - &self, - _response: PlatformResponse, - _response_time_ms: u64, - ) -> Result> { - panic!("should never parse an uncompleted mediator invocation"); - } - fn timeout_ms(&self) -> u32 { - 777 - } - } - - #[tokio::test] - async fn trace_auction_live_cancellation_preserves_pending_mediator_after_bidder_collection() - { - let http = Arc::new(StubHttpClient::new()); - http.push_response(204, Vec::new()); - let services = build_services_with_http_client(Arc::clone(&http) as Arc<_>); - let plan = Arc::new( - AuctionPlan::compile(planned_config( - &[("bidder", RoutingMode::AllEligible)], - false, - )) - .expect("should compile the bidder plan"), + #[test] + fn abandoned_auction_cancels_pending_slots() { + let clock = TestClock::new(); + let client = ScriptedHttpClient::new(&clock); + let (held, release): (ScriptedReply, ReplyRelease) = bid_reply(1.5).held(); + client.respond_to(Method::POST, FAST_BIDDER, held); + let services = build_services_with_outbound_client(&client, &clock); + let orchestrator = AuctionOrchestrator::from_plan( + batch_plan(&[("fast", FAST_BIDDER, Duration::from_secs(1))]), + None, ); - let orchestrator = - AuctionOrchestrator::from_plan(plan, Some(Arc::new(NeverFinishesMediator))); - let request = planned_request(); let settings = create_test_settings(); - let (inbound, carry) = captured_request(&request); - let context = AuctionContext { - settings: &settings, - request: &inbound, - timeout_ms: 777, - transport_timeout_ms: 777, - provider_responses: None, - services: &services, - }; - let mut pending = Box::pin(orchestrator.run_auction(&request, &context)); - let mut task = std::task::Context::from_waker(std::task::Waker::noop()); + let inbound = inbound_request(); + let context = batch_context(&settings, &inbound, &services, AUCTION_TIMEOUT); + + let dispatched = dispatched( + clock.drive(orchestrator.dispatch_auction(&planned_request(), &context)), + ); + let awaited_in_flight = release.is_awaited(); + let (_request, completed, abandoned, _elapsed_ms) = dispatched.abandon(); + assert!( - std::future::Future::poll(pending.as_mut(), &mut task).is_pending(), - "should stop at the actual mediator await" + awaited_in_flight, + "should have the bidder request in flight before abandoning" ); assert!( - carry.transport().is_none(), - "should not invent a terminal result while work is pending" + completed.is_empty(), + "should complete no provider before abandoning" ); - - drop(pending); - - let facts = evidence(&carry); assert_eq!( - facts["terminal_status"], "abandoned", - "should finish synchronously when collection is cancelled" + abandoned + .iter() + .map(|call| call.provider.as_str()) + .collect::>(), + vec!["fast"], + "should report the in-flight bidder as abandoned" ); - assert_eq!( - facts["provider_calls"][0]["status"], "no_bid", - "should retain the collected bidder result" + assert!( + !release.is_awaited(), + "should cancel the bidder request, so nothing awaits its held reply" ); - assert_eq!( - facts["provider_calls"][1]["role"], "mediator", - "should retain the actual pending mediator launch" + } + + #[test] + fn delayed_collection_keeps_slot_elapsed_and_response_time() { + let clock = TestClock::new(); + let client = ScriptedHttpClient::new(&clock); + client.respond_to( + Method::POST, + FAST_BIDDER, + bid_reply(1.5).after(Duration::from_millis(100)), ); + let services = build_services_with_outbound_client(&client, &clock); + let orchestrator = AuctionOrchestrator::from_plan( + batch_plan(&[("fast", FAST_BIDDER, Duration::from_secs(1))]), + None, + ); + let settings = create_test_settings(); + let inbound = inbound_request(); + let context = batch_context(&settings, &inbound, &services, AUCTION_TIMEOUT); + let request = planned_request(); + + let dispatched = + dispatched(clock.drive(orchestrator.dispatch_auction(&request, &context))); + // The bid arrives at 100 ms, and a slow origin delays collection to 400 ms. + clock.advance(Duration::from_millis(400)); + let result = clock + .drive(orchestrator.collect_dispatched_auction(dispatched, &services, &context)); + assert_eq!( - facts["provider_calls"][1]["status"], "abandoned", - "should close only still-pending observations" + response_for(&result, "fast").response_time_ms, + 100, + "should time the bid to its arrival, not its delayed collection" + ); + let rows = build_auction_events( + AuctionObservationContext::for_test( + AuctionSource::InitialNavigation, + "/article", + 1, + ), + AuctionTerminalOutcome::Completed { + request: &request, + result: &result, + delivered_winner_slots: None, + }, + ); + let provider_times = rows + .rows() + .iter() + .filter(|row| { + row.event_kind == "provider_call" && row.provider.as_deref() == Some("fast") + }) + .map(|row| row.provider_response_time_ms) + .collect::>(); + assert!( + !provider_times.is_empty() && provider_times.iter().all(|time| *time == Some(100)), + "should report the arrival time in telemetry, got {provider_times:?}" ); } - #[tokio::test] - async fn trace_auction_live_mediator_failure_preserves_successful_local_fallback() { - let http = Arc::new(StubHttpClient::new()); - http.push_response(204, Vec::new()); - let services = build_services_with_http_client(Arc::clone(&http) as Arc<_>); - let plan = Arc::new( - AuctionPlan::compile(planned_config( - &[("bidder", RoutingMode::AllEligible)], - false, - )) - .expect("should compile the bidder plan"), + #[test] + fn empty_auction_returns_without_waiting_for_deadline() { + let clock = TestClock::new(); + let client = ScriptedHttpClient::new(&clock); + let services = build_services_with_outbound_client(&client, &clock); + let orchestrator = AuctionOrchestrator::from_plan( + batch_plan(&[("fast", FAST_BIDDER, Duration::from_secs(1))]), + None, ); - let orchestrator = - AuctionOrchestrator::from_plan(plan, Some(Arc::new(LaunchFailingProvider))); - let request = planned_request(); let settings = create_test_settings(); - let (inbound, carry) = captured_request(&request); - let context = AuctionContext { - settings: &settings, - request: &inbound, - timeout_ms: 777, - transport_timeout_ms: 777, - provider_responses: None, - services: &services, + let inbound = inbound_request(); + let context = batch_context(&settings, &inbound, &services, AUCTION_TIMEOUT); + // A video-only slot leaves no provider an eligible banner slot. + let mut request = planned_request(); + request.slots[0].formats[0].media_type = MediaType::Video; + let started = clock.monotonic_clock().now(); + + let direct = clock + .drive(orchestrator.run_auction(&request, &context)) + .expect("should complete the empty auction"); + let split = clock.drive(async { + let dispatched = + dispatched(orchestrator.dispatch_auction(&request, &context).await); + orchestrator + .collect_dispatched_auction(dispatched, &services, &context) + .await + }); + + for result in [&direct, &split] { + assert!( + result.winning_bids.is_empty(), + "should return no winning bids" + ); + assert_eq!( + response_for(result, "fast").metadata["routing"]["skipped_no_eligible_slots"], + serde_json::json!(true), + "should report the filtered provider as skipped" + ); + } + assert_eq!( + clock.monotonic_clock().now(), + started, + "should not wait for the auction deadline" + ); + client.assert_no_requests(); + } + + #[test] + fn bidder_requests_carry_consent_and_attested_client_ip_headers() { + let clock = TestClock::new(); + let client = ScriptedHttpClient::new(&clock); + let endpoint = "https://pbs.example.test/openrtb"; + client.respond_to( + Method::POST, + endpoint, + ScriptedReply::status(StatusCode::NO_CONTENT), + ); + let attested = IpAddr::V4(Ipv4Addr::new(192, 0, 2, 44)); + let services = RuntimeServices { + outbound_client: client.http_client(), + clock: clock.monotonic_clock(), + ..noop_services_with_client_ip(attested) }; + let mut config = planned_prebid_config(&[( + "pbs", + serde_json::json!({"consent_forwarding": "both"}), + NotificationConfig::default(), + )]); + config + .providers + .values_mut() + .for_each(|provider| provider.endpoint = endpoint.to_string()); + let orchestrator = AuctionOrchestrator::from_plan( + Arc::new(AuctionPlan::compile(config).expect("should compile the PBS plan")), + None, + ); + let settings = create_test_settings(); + let inbound = http::Request::builder() + .uri("https://publisher.example/article") + .header( + http::header::COOKIE, + "session=private; euconsent-v2=fictional-tcf", + ) + .header(http::header::USER_AGENT, "Fictional Browser/7") + .header("x-forwarded-for", "203.0.113.250") + .body(edgezero_core::body::Body::empty()) + .expect("should build the inbound request"); + let context = batch_context(&settings, &inbound, &services, AUCTION_TIMEOUT); - let result = orchestrator - .run_auction(&request, &context) - .await - .expect("should preserve successful ordinary local fallback"); + clock + .drive(orchestrator.run_auction(&planned_prebid_request(), &context)) + .expect("should complete the auction"); - assert!( - result.mediator_response.is_none(), - "should preserve the ordinary discarded mediator response" - ); - let facts = evidence(&carry); + let requests = client.requests(); + let headers = &request_to(&requests, endpoint).headers; assert_eq!( - facts["terminal_status"], "completed", - "should not infer whole-auction failure from provider error" + ( + headers.get(http::header::COOKIE), + headers.get("x-forwarded-for"), + headers.get(http::header::USER_AGENT) + ), + ( + Some(&http::HeaderValue::from_static( + "euconsent-v2=fictional-tcf" + )), + Some(&http::HeaderValue::from_static("192.0.2.44")), + Some(&http::HeaderValue::from_static("Fictional Browser/7")) + ), + "should send only the consent cookie and the attested client IP, never the inbound XFF" ); - assert!( - facts.get("terminal_reason").is_none(), - "should omit a reason for completed execution" + } + + fn mediator_with_timeout(timeout: Duration) -> AdServerMockProvider { + AdServerMockProvider::new(AdServerMockConfig { + enabled: true, + endpoint: MEDIATOR_ENDPOINT.to_string(), + timeout_ms: duration_ms(timeout), + price_floor: None, + context_query_params: BTreeMap::new(), + }) + } + + #[test] + fn mediator_ends_at_its_own_timeout_or_the_auction_deadline_and_ranks_locally() { + // The mediator's own timeout ends first in the first case and the + // auction deadline in the second. + for (mediator_timeout, bid_delay, cause, ends_at) in [ + ( + Duration::from_millis(100), + Duration::from_millis(50), + BudgetSource::PerCallTimeout, + Duration::from_millis(150), + ), + ( + Duration::from_secs(1), + Duration::from_millis(400), + BudgetSource::RequestDeadline, + AUCTION_TIMEOUT, + ), + ] { + let clock = TestClock::new(); + let client = ScriptedHttpClient::new(&clock); + let (mediator_reply, _never_released) = + ScriptedReply::status(StatusCode::OK).held(); + client + .respond_to(Method::POST, FAST_BIDDER, bid_reply(1.5).after(bid_delay)) + .respond_to(Method::POST, MEDIATOR_ENDPOINT, mediator_reply); + let services = build_services_with_outbound_client(&client, &clock); + let mediator = mediator_with_timeout(mediator_timeout); + let orchestrator = AuctionOrchestrator::from_plan( + batch_plan(&[("fast", FAST_BIDDER, Duration::from_secs(1))]), + Some(Arc::new(mediator)), + ); + let settings = create_test_settings(); + let inbound = inbound_request(); + let context = batch_context(&settings, &inbound, &services, AUCTION_TIMEOUT); + + let result = clock + .drive(orchestrator.run_auction(&planned_request(), &context)) + .expect("should complete without the mediator"); + + assert!( + result.mediator_response.is_none(), + "should drop the mediator that never answered" + ); + assert_eq!( + result + .get_winning_bid("fictional-slot") + .and_then(|bid| bid.price), + Some(1.5), + "should rank the bidder's bid locally" + ); + assert_eq!( + ( + request_to(&client.requests(), MEDIATOR_ENDPOINT) + .budget + .cause, + result.total_time_ms + ), + (cause, duration_ms(ends_at).into()), + "should end the mediator at the earlier of its timeout and the auction deadline" + ); + } + } + + #[test] + fn deadline_spent_by_bidders_skips_the_mediator() { + let clock = TestClock::new(); + let client = ScriptedHttpClient::new(&clock); + client.respond_to( + Method::POST, + FAST_BIDDER, + bid_reply(1.5).after(Duration::from_millis(600)), ); - assert_eq!( - facts["provider_calls"][1]["status"], "error", - "should preserve the failed mediator invocation before fallback" + let services = build_services_with_outbound_client(&client, &clock); + let mediator = mediator_with_timeout(Duration::from_millis(300)); + let orchestrator = AuctionOrchestrator::from_plan( + batch_plan(&[("fast", FAST_BIDDER, Duration::from_secs(1))]), + Some(Arc::new(mediator)), ); - } + let settings = create_test_settings(); + let inbound = inbound_request(); + let context = batch_context(&settings, &inbound, &services, AUCTION_TIMEOUT); - #[tokio::test] - async fn trace_auction_live_zero_budget_has_no_synthetic_provider_launches() { - let http = Arc::new(StubHttpClient::new()); - let services = build_services_with_http_client(Arc::clone(&http) as Arc<_>); - let plan = Arc::new( - AuctionPlan::compile(planned_config( - &[("bidder", RoutingMode::AllEligible)], - false, - )) - .expect("should compile the bidder plan"), + let result = clock + .drive(orchestrator.run_auction(&planned_request(), &context)) + .expect("should complete at the auction deadline"); + + assert!( + result.mediator_response.is_none() && result.winning_bids.is_empty(), + "should return the late bidder's timeout without mediation" + ); + assert_eq!( + client + .requests() + .iter() + .map(|request| request.uri.to_string()) + .collect::>(), + vec![FAST_BIDDER.to_string()], + "should not send the mediator once the auction deadline passed" + ); + } + + #[test] + fn batch_failure_marks_unresolved_slots_as_provider_errors() { + let clock = TestClock::new(); + let client = ScriptedHttpClient::new(&clock); + client + .respond_to( + Method::POST, + FAST_BIDDER, + bid_reply(1.5).after(Duration::from_millis(100)), + ) + .respond_to( + Method::POST, + SLOW_BIDDER, + ScriptedReply::failing_batch(EdgeError::internal(std::io::Error::other( + "fictional batch driver failure", + ))) + .after(Duration::from_millis(200)), + ); + let services = build_services_with_outbound_client(&client, &clock); + let orchestrator = AuctionOrchestrator::from_plan( + batch_plan(&[ + ("fast", FAST_BIDDER, Duration::from_secs(1)), + ("slow", SLOW_BIDDER, Duration::from_secs(1)), + ]), + None, ); - let orchestrator = - AuctionOrchestrator::from_plan(plan, Some(Arc::new(NeverFinishesMediator))); - let request = planned_request(); let settings = create_test_settings(); - let (inbound, carry) = captured_request(&request); - let context = AuctionContext { - settings: &settings, - request: &inbound, - timeout_ms: 0, - transport_timeout_ms: 0, - provider_responses: None, - services: &services, - }; + let inbound = inbound_request(); + let context = batch_context(&settings, &inbound, &services, AUCTION_TIMEOUT); - orchestrator - .run_auction(&request, &context) - .await - .expect("should preserve the ordinary zero-budget result"); + let result = clock + .drive(orchestrator.run_auction(&planned_request(), &context)) + .expect("should complete the auction after a batch failure"); assert_eq!( - evidence(&carry)["provider_calls"], - serde_json::json!([]), - "should exclude synthetic timeout responses from launched work" + response_for(&result, "fast").bids.len(), + 1, + "should keep the bid observed before the batch failed" ); - assert!( - http.recorded_backend_names().is_empty(), - "should not add network work for trace" + let slow = response_for(&result, "slow"); + assert_eq!( + (slow.status.clone(), slow.metadata.get("error_type")), + ( + BidStatus::Error, + Some(&serde_json::json!(ERROR_TYPE_TRANSPORT)) + ), + "should report the slot the failed batch left unresolved as a provider error" ); } } - #[tokio::test] - async fn disabled_from_plan_is_a_no_work_kill_switch_for_sync_and_split_paths() { - let plan = Arc::new( - AuctionPlan::compile(planned_config( - &[("provider-a", RoutingMode::AllEligible)], - false, - )) - .expect("should compile plan") - .with_enabled(false), - ); - let orchestrator = AuctionOrchestrator::from_plan(plan, None); - let http = Arc::new(StubHttpClient::new()); - let services = build_services_with_http_client(Arc::clone(&http) as Arc<_>); - let settings = create_test_settings(); - let inbound = http::Request::new(edgezero_core::body::Body::empty()); - let context = AuctionContext { - settings: &settings, - request: &inbound, - timeout_ms: 777, - transport_timeout_ms: 777, - provider_responses: None, - services: &services, - }; - let request = planned_request(); - - let result = orchestrator - .run_auction(&request, &context) - .await - .expect("disabled auction should complete as no-bid"); - assert!(result.provider_responses.is_empty()); - assert!(result.winning_bids.is_empty()); - assert!(matches!( - orchestrator.dispatch_auction(&request, &context).await, - DispatchAuctionOutcome::NotStarted - )); - assert!(http.recorded_backend_names().is_empty()); - } + mod trace_auction_live_tests { + use super::*; + use crate::trace::{TraceAuctionCarry, TraceAuctionSource}; + use edgezero_core::body::Body as EdgeBody; + use http::Request; - #[tokio::test] - async fn enabled_empty_plan_is_successful_no_bid_without_dispatch() { - let plan = Arc::new( - AuctionPlan::compile(planned_config(&[], false)).expect("should compile empty plan"), - ); - let orchestrator = AuctionOrchestrator::from_plan(plan, None); - let http = Arc::new(StubHttpClient::new()); - let services = build_services_with_http_client(Arc::clone(&http) as Arc<_>); - let settings = create_test_settings(); - let inbound = http::Request::new(edgezero_core::body::Body::empty()); - let context = AuctionContext { - settings: &settings, - request: &inbound, - timeout_ms: 777, - transport_timeout_ms: 777, - provider_responses: None, - services: &services, - }; - let request = planned_request(); - - let result = orchestrator - .run_auction(&request, &context) - .await - .expect("empty auction should complete as no-bid"); - assert!(result.provider_responses.is_empty()); - assert!(result.winning_bids.is_empty()); - assert!(matches!( - orchestrator.dispatch_auction(&request, &context).await, - DispatchAuctionOutcome::NotStarted - )); - assert!(http.recorded_backend_names().is_empty()); - } - - #[tokio::test] - async fn all_planned_launch_failures_error_direct_and_surface_split_failure() { - let plan = Arc::new( - AuctionPlan::compile(planned_config( - &[("launch-fail", RoutingMode::AllEligible)], - false, - )) - .expect("should compile launch-failure plan"), - ); - let orchestrator = AuctionOrchestrator::from_plan(plan, None); - let backend = Arc::new(NamingBackend::new(BackendNamingPolicy::Axum)); - backend.fail_ensure_for("launch-fail"); - let http = Arc::new(StubHttpClient::new()); - let services = build_services_with_backend_and_http_client( - Arc::clone(&backend) as Arc<_>, - Arc::clone(&http) as Arc<_>, - ); - let settings = create_test_settings(); - let inbound = http::Request::new(edgezero_core::body::Body::empty()); - let context = AuctionContext { - settings: &settings, - request: &inbound, - timeout_ms: 777, - transport_timeout_ms: 777, - provider_responses: None, - services: &services, - }; - let request = planned_request(); + fn captured_request(request: &AuctionRequest) -> (Request, TraceAuctionCarry) { + let carry = TraceAuctionCarry::capture_if_enabled( + true, + TraceAuctionSource::AuctionApi, + &request.slots, + ) + .expect("should capture the enabled request"); + let mut inbound = Request::new(EdgeBody::empty()); + inbound.extensions_mut().insert(carry.clone()); + (inbound, carry) + } - let error = orchestrator - .run_auction(&request, &context) - .await - .expect_err("all planned launch failures should fail direct execution"); - assert!( - error - .to_string() - .contains("All eligible planned provider requests failed to launch"), - "should distinguish launch failure from an ordinary no-bid: {error:?}" - ); + fn evidence(carry: &TraceAuctionCarry) -> serde_json::Value { + serde_json::to_value( + carry + .transport() + .expect("should terminalize the live auction"), + ) + .expect("should serialize redacted facts")["evidence"] + .clone() + } - let DispatchAuctionOutcome::DispatchFailed { - provider_responses, - fatal_admission_error, - .. - } = orchestrator.dispatch_auction(&request, &context).await - else { - panic!("all planned launch failures should surface a split dispatch failure"); - }; - assert!( - fatal_admission_error.is_none(), - "should carry no fatal admission error" - ); - assert_eq!( - provider_responses.len(), - 1, - "should keep one launch-failure response" - ); - assert_eq!( - provider_responses[0].provider, "launch-fail", - "should attribute the response to the failed provider" - ); - assert_eq!( - provider_responses[0].metadata["error_type"], - ERROR_TYPE_LAUNCH_FAILED - ); - assert!(http.recorded_backend_names().is_empty()); - } + #[tokio::test] + async fn trace_auction_live_launch_failure_survives_the_ordinary_error() { + for failure in LaunchFailure::ALL { + let clock = TestClock::new(); + let bidders = ScriptedHttpClient::new(&clock); + failure.script(&bidders, 1); + let services = build_services_with_outbound_client(&bidders, &clock); + let orchestrator = AuctionOrchestrator::from_plan( + launch_plan(&[("launch-fail", failure.endpoint())]), + None, + ); + let settings = create_test_settings(); + let request = planned_request(); + let (inbound, carry) = captured_request(&request); + let context = launch_context(&settings, &inbound, &services); - #[tokio::test] - async fn launch_failure_plus_prelaunch_timeout_remains_a_completed_auction() { - let plan = Arc::new( - AuctionPlan::compile(planned_config( - &[ - ("launch-fail", RoutingMode::AllEligible), - ("timeout", RoutingMode::AllEligible), - ], - false, - )) - .expect("should compile mixed-outcome plan"), - ); - let orchestrator = AuctionOrchestrator::from_plan(plan, None); - let backend = Arc::new(FailThenZeroCanonicalBackend::new()); - let http = Arc::new(StubHttpClient::new()); - let services = build_services_with_backend_and_http_client( - Arc::clone(&backend) as Arc<_>, - Arc::clone(&http) as Arc<_>, - ); - let settings = create_test_settings(); - let inbound = http::Request::new(edgezero_core::body::Body::empty()); - let context = AuctionContext { - settings: &settings, - request: &inbound, - timeout_ms: 777, - transport_timeout_ms: 777, - provider_responses: None, - services: &services, - }; + let _error = orchestrator + .run_auction(&request, &context) + .await + .expect_err("should preserve the ordinary launch failure"); - let result = orchestrator - .run_auction(&planned_request(), &context) - .await - .expect("should treat a pre-launch timeout as a completed provider outcome"); + let value = evidence(&carry); + assert_eq!( + value["terminal_status"], "dispatch_failed", + "{failure:?} should preserve the actual dispatch failure" + ); + assert_eq!( + value["terminal_reason"], "provider_execution_failed", + "{failure:?} should use the directly observed launch failure" + ); + assert_eq!( + value["provider_calls"][0]["status"], "error", + "{failure:?} should retain the failed actual invocation" + ); + assert!( + !value.to_string().contains("launch-fail"), + "{failure:?} should exclude provider identities from public facts" + ); + assert_eq!( + bidders.requests().len(), + failure.requests_per_auction(), + "{failure:?} should send no bid request beyond the failed registration" + ); + } + } - assert_eq!( - result.provider_responses.len(), - 2, - "should retain both provider outcomes" - ); - assert!( - result.provider_responses.iter().any(|response| { - response.provider == "launch-fail" - && response.metadata["error_type"] == ERROR_TYPE_LAUNCH_FAILED - }), - "should retain the launch-failure outcome" - ); - assert!( - result.provider_responses.iter().any(|response| { - response.provider == "timeout" - && response.metadata["error_type"] == ERROR_TYPE_TIMEOUT - }), - "should retain the pre-launch timeout outcome" - ); - assert_eq!( - backend.ensured.load(Ordering::Relaxed), - 1, - "should attempt only the provider with a transport budget" - ); - assert!( - http.recorded_backend_names().is_empty(), - "should start no provider network requests" - ); - } + #[tokio::test] + async fn trace_auction_live_empty_plan_distinguishes_sync_success_from_split_failure() { + let plan = Arc::new( + AuctionPlan::compile(planned_config(&[], false)) + .expect("should compile an empty plan"), + ); + let orchestrator = AuctionOrchestrator::from_plan(plan, None); + let clock = TestClock::new(); + let http = ScriptedHttpClient::new(&clock); + let services = build_services_with_outbound_client(&http, &clock); + let settings = create_test_settings(); + let request = planned_request(); + for split in [false, true] { + let (inbound, carry) = captured_request(&request); + let token = carry.token(); + let context = AuctionContext { + settings: &settings, + request: &inbound, + timeout_ms: 777, + provider_responses: None, + services: &services, + }; + if split { + assert!( + matches!( + orchestrator.dispatch_auction(&request, &context).await, + DispatchAuctionOutcome::NotStarted + ), + "should preserve the ordinary split outcome" + ); + } else { + let result = orchestrator + .run_auction(&request, &context) + .await + .expect("should preserve successful empty-plan execution"); + assert!( + result.winning_bids.is_empty(), + "should preserve an ordinary zero-bid result" + ); + } + let value = evidence(&carry); + assert_eq!( + value["diagnostic_auction_id"], + token.as_str(), + "should preserve the pre-dispatch token" + ); + assert_eq!( + value["terminal_status"], + if split { + "dispatch_failed" + } else { + "completed" + }, + "should map the observed path semantics" + ); + assert_eq!( + value["provider_calls"], + serde_json::json!([]), + "should not fabricate a launched provider" + ); + } + } - #[tokio::test] - async fn all_skipped_from_plan_completes_with_routing_metadata_in_sync_and_split_paths() { - for split in [false, true] { + #[tokio::test] + async fn trace_auction_live_collection_failure_keeps_launch_order_through_local_fallback() { + let clock = TestClock::new(); + let http = ScriptedHttpClient::new(&clock); + // The first slot's arrival fails the whole batch driver. + http.respond_to( + Method::POST, + PLANNED_ENDPOINT, + ScriptedReply::failing_batch(edgezero_core::error::EdgeError::internal( + std::io::Error::other("fictional batch driver failure"), + )), + ); + respond_planned(&http, 204, Vec::new()); + let services = build_services_with_outbound_client(&http, &clock); let plan = Arc::new( - AuctionPlan::compile(planned_config(&[("skipped", RoutingMode::Explicit)], false)) - .expect("should compile all-skipped plan"), + AuctionPlan::compile(planned_config( + &[ + ("provider-b", RoutingMode::AllEligible), + ("provider-a", RoutingMode::AllEligible), + ], + false, + )) + .expect("should compile the ordered plan"), ); let orchestrator = AuctionOrchestrator::from_plan(plan, None); - let http = Arc::new(StubHttpClient::new()); - let services = build_services_with_http_client(Arc::clone(&http) as Arc<_>); + let request = planned_request(); let settings = create_test_settings(); - let inbound = http::Request::new(edgezero_core::body::Body::empty()); + let (inbound, carry) = captured_request(&request); let context = AuctionContext { settings: &settings, request: &inbound, timeout_ms: 777, - transport_timeout_ms: 777, provider_responses: None, services: &services, }; - let request = planned_request(); - - let result = if split { - let DispatchAuctionOutcome::Dispatched(dispatched) = - orchestrator.dispatch_auction(&request, &context).await - else { - panic!("all-skipped auction should produce a completed dispatch token"); - }; - orchestrator - .collect_dispatched_auction(dispatched, &services, &context) - .await - } else { - orchestrator - .run_auction(&request, &context) - .await - .expect("all-skipped auction should complete") - }; - - assert_eq!(result.provider_responses.len(), 1); + let result = orchestrator + .run_auction(&request, &context) + .await + .expect("should preserve ordinary local fallback"); + assert!( + result.winning_bids.is_empty(), + "should preserve ordinary ranking" + ); + let value = evidence(&carry); assert_eq!( - result.provider_responses[0].metadata["routing"]["skipped_no_eligible_slots"], - true + value["terminal_status"], "execution_failed", + "should not infer successful collection from an ordinary Ok result" ); - assert!(result.metadata.contains_key("routing")); - assert!(http.recorded_backend_names().is_empty()); - } - } - - struct NamingBackend { - policy: BackendNamingPolicy, - predicted: AtomicUsize, - ensured: AtomicUsize, - specs: Mutex>, - fail_ensure_for: Mutex>, - } - - impl NamingBackend { - fn new(policy: BackendNamingPolicy) -> Self { - Self { - policy, - predicted: AtomicUsize::new(0), - ensured: AtomicUsize::new(0), - specs: Mutex::new(Vec::new()), - fail_ensure_for: Mutex::new(HashSet::new()), - } - } - - fn fail_ensure_for(&self, provider_id: &str) { - self.fail_ensure_for - .lock() - .expect("should lock failing provider IDs") - .insert(provider_id.to_string()); - } - - fn name(&self, spec: &PlatformBackendSpec) -> Result> { - self.policy - .predict(spec) - .map(|prediction| prediction.name) - .change_context(PlatformError::Backend) - } - } - - impl PlatformBackend for NamingBackend { - fn naming_policy(&self) -> BackendNamingPolicy { - self.policy - } - - fn predict_name( - &self, - spec: &PlatformBackendSpec, - ) -> Result> { - self.predicted.fetch_add(1, Ordering::Relaxed); - self.name(spec) - } - - fn ensure(&self, spec: &PlatformBackendSpec) -> Result> { - self.ensured.fetch_add(1, Ordering::Relaxed); - if spec.discriminator.as_deref().is_some_and(|provider_id| { - self.fail_ensure_for - .lock() - .expect("should lock failing provider IDs") - .contains(provider_id) - }) { - return Err(Report::new(PlatformError::Backend)); - } - self.specs - .lock() - .expect("should lock planned backend specs") - .push(spec.clone()); - self.name(spec) - } - } - - struct ZeroCanonicalBackend { - predicted: AtomicUsize, - ensured: AtomicUsize, - } - - impl ZeroCanonicalBackend { - fn new() -> Self { - Self { - predicted: AtomicUsize::new(0), - ensured: AtomicUsize::new(0), - } - } - } - - impl PlatformBackend for ZeroCanonicalBackend { - fn naming_policy(&self) -> BackendNamingPolicy { - BackendNamingPolicy::Fastly - } - - fn predict_name( - &self, - _spec: &PlatformBackendSpec, - ) -> Result> { - self.predicted.fetch_add(1, Ordering::Relaxed); - Ok("zero-canonical-backend".to_string()) - } - - fn ensure(&self, _spec: &PlatformBackendSpec) -> Result> { - self.ensured.fetch_add(1, Ordering::Relaxed); - Ok("zero-canonical-backend".to_string()) - } - - fn canonicalize_transport_timeout_ms( - &self, - _remaining_ms: u32, - _configured_ms: u32, - ) -> u32 { - 0 - } - } - - struct FailThenZeroCanonicalBackend { - canonicalizations: AtomicUsize, - ensured: AtomicUsize, - } - - impl FailThenZeroCanonicalBackend { - fn new() -> Self { - Self { - canonicalizations: AtomicUsize::new(0), - ensured: AtomicUsize::new(0), - } - } - } - - impl PlatformBackend for FailThenZeroCanonicalBackend { - fn naming_policy(&self) -> BackendNamingPolicy { - BackendNamingPolicy::Axum - } - - fn predict_name( - &self, - spec: &PlatformBackendSpec, - ) -> Result> { - BackendNamingPolicy::Axum - .predict(spec) - .map(|prediction| prediction.name) - .change_context(PlatformError::Backend) - } - - fn ensure(&self, _spec: &PlatformBackendSpec) -> Result> { - self.ensured.fetch_add(1, Ordering::Relaxed); - Err(Report::new(PlatformError::Backend)) - } - - fn canonicalize_transport_timeout_ms(&self, remaining_ms: u32, configured_ms: u32) -> u32 { - if self.canonicalizations.fetch_add(1, Ordering::Relaxed) == 0 { - remaining_ms.min(configured_ms) - } else { - 0 - } - } - } - - struct CollidingBackend; - - impl PlatformBackend for CollidingBackend { - fn naming_policy(&self) -> BackendNamingPolicy { - BackendNamingPolicy::Axum - } - - fn predict_name( - &self, - _spec: &PlatformBackendSpec, - ) -> Result> { - Ok("colliding-backend".to_string()) - } - - fn ensure(&self, _spec: &PlatformBackendSpec) -> Result> { - Ok("colliding-backend".to_string()) - } - } - - struct FailingCountingConfigStore { - reads: AtomicUsize, - } - - impl PlatformConfigStore for FailingCountingConfigStore { - fn get( - &self, - _store_name: &StoreName, - _key: &str, - ) -> Result> { - self.reads.fetch_add(1, Ordering::Relaxed); - Err(Report::new(PlatformError::ConfigStore)) - } - - fn put( - &self, - _store_id: &StoreId, - _key: &str, - _value: &str, - ) -> Result<(), Report> { - Err(Report::new(PlatformError::Unsupported)) - } - - fn delete(&self, _store_id: &StoreId, _key: &str) -> Result<(), Report> { - Err(Report::new(PlatformError::Unsupported)) - } - } - - struct CountingConfigStore { - reads: AtomicUsize, - current_kid: String, - delay: Duration, - } - - impl PlatformConfigStore for CountingConfigStore { - fn get(&self, _store_name: &StoreName, key: &str) -> Result> { - self.reads.fetch_add(1, Ordering::Relaxed); - if !self.delay.is_zero() { - std::thread::sleep(self.delay); - } - (key == "current-kid") - .then(|| self.current_kid.clone()) - .ok_or_else(|| Report::new(PlatformError::ConfigStore)) - } - - fn put( - &self, - _store_id: &StoreId, - _key: &str, - _value: &str, - ) -> Result<(), Report> { - Err(Report::new(PlatformError::Unsupported)) - } - - fn delete(&self, _store_id: &StoreId, _key: &str) -> Result<(), Report> { - Err(Report::new(PlatformError::Unsupported)) - } - } - - struct CountingSecretStore { - reads: AtomicUsize, - key: Vec, - } - - impl PlatformSecretStore for CountingSecretStore { - fn get_bytes( - &self, - _store_name: &StoreName, - _key: &str, - ) -> Result, Report> { - self.reads.fetch_add(1, Ordering::Relaxed); - Ok(self.key.clone()) - } - - fn create( - &self, - _store_id: &StoreId, - _name: &str, - _value: &str, - ) -> Result<(), Report> { - Err(Report::new(PlatformError::Unsupported)) - } - - fn delete(&self, _store_id: &StoreId, _name: &str) -> Result<(), Report> { - Err(Report::new(PlatformError::Unsupported)) - } - } - - struct OuterSelectErrorHttpClient { - inner: StubHttpClient, - selected_pending: AtomicUsize, - } - - impl OuterSelectErrorHttpClient { - fn new() -> Self { - Self { - inner: StubHttpClient::new(), - selected_pending: AtomicUsize::new(0), - } - } - - fn push_response(&self, status: u16, body: Vec) { - self.inner.push_response(status, body); - } - } - - #[async_trait::async_trait(?Send)] - impl PlatformHttpClient for OuterSelectErrorHttpClient { - async fn send( - &self, - request: PlatformHttpRequest, - ) -> Result> { - self.inner.send(request).await - } - - async fn send_async( - &self, - request: PlatformHttpRequest, - ) -> Result> { - self.inner.send_async(request).await - } - - async fn select_until( - &self, - pending_requests: Vec, - _deadline: Instant, - ) -> Result, Report> { - self.select(pending_requests).await.map(Some) - } - - async fn select( - &self, - pending_requests: Vec, - ) -> Result> { - self.selected_pending - .store(pending_requests.len(), Ordering::Relaxed); - Err(Report::new(PlatformError::HttpClient)) - } - } - - struct LateReadyMediatorHttpClient { - inner: Arc, - waits: Arc, - } - - #[async_trait::async_trait(?Send)] - impl PlatformHttpClient for LateReadyMediatorHttpClient { - async fn send( - &self, - request: PlatformHttpRequest, - ) -> Result> { - self.inner.send(request).await - } - - async fn send_async( - &self, - request: PlatformHttpRequest, - ) -> Result> { - self.inner.send_async(request).await - } - - async fn select( - &self, - pending: Vec, - ) -> Result> { - self.inner.select(pending).await - } - - async fn select_until( - &self, - pending: Vec, - deadline: Instant, - ) -> Result, Report> { - self.inner.select_until(pending, deadline).await - } - - async fn wait_until( - &self, - pending: PlatformPendingRequest, - _deadline: Instant, - ) -> Result> { - self.waits.fetch_add(1, Ordering::Relaxed); - // Model an executor waking after the mediator cutoff with a complete - // result returned by the adapter's final nonblocking snapshot. - std::thread::sleep(Duration::from_millis(40)); - self.inner.wait(pending).await - } - } - - struct UnusedSecretStore; - - impl PlatformSecretStore for UnusedSecretStore { - fn get_bytes( - &self, - _store_name: &StoreName, - _key: &str, - ) -> Result, Report> { - panic!("signing key should not be read after current-kid failure") - } - - fn create( - &self, - _store_id: &StoreId, - _name: &str, - _value: &str, - ) -> Result<(), Report> { - Err(Report::new(PlatformError::Unsupported)) - } - - fn delete(&self, _store_id: &StoreId, _name: &str) -> Result<(), Report> { - Err(Report::new(PlatformError::Unsupported)) - } - } - - // --------------------------------------------------------------------------- - // Minimal test double for AuctionProvider - // --------------------------------------------------------------------------- - - struct StubAuctionProvider { - name: &'static str, - backend: &'static str, - } - - #[async_trait::async_trait(?Send)] - impl AuctionProvider for StubAuctionProvider { - fn provider_name(&self) -> &str { - self.name - } - - async fn request_bids( - &self, - _request: &AuctionRequest, - context: &AuctionContext<'_>, - ) -> Result> { - let req = PlatformHttpRequest::new( - http::Request::builder() - .method("POST") - .uri("https://example.com/bid") - .body(edgezero_core::body::Body::empty()) - .expect("should build stub bid request"), - self.backend, - ); - context - .services - .http_client() - .send_async(req.with_cache_bypass()) - .await - .change_context(TrustedServerError::Auction { - message: "stub launch failed".to_string(), - }) - .map(ProviderRequestOutcome::pending) - } - - async fn parse_response( - &self, - _response: PlatformResponse, - response_time_ms: u64, - ) -> Result> { - Ok(AuctionResponse::success( - self.name, - vec![], - response_time_ms, - )) - } - - async fn parse_response_with_context( - &self, - response: PlatformResponse, - response_time_ms: u64, - _request: &AuctionRequest, - context: &AuctionContext<'_>, - ) -> Result> { - let referer = context - .request - .headers() - .get(http::header::REFERER) - .and_then(|value| value.to_str().ok()); - Ok(self - .parse_response(response, response_time_ms) - .await? - .with_metadata("context_referer", serde_json::json!(referer)) - .with_metadata("context_timeout_ms", serde_json::json!(context.timeout_ms))) - } - - fn timeout_ms(&self) -> u32 { - 125 - } - - fn backend_name(&self, _services: &RuntimeServices, _timeout_ms: u32) -> Option { - Some(self.backend.to_string()) - } - } - - struct DeadlineBidProvider { - name: &'static str, - backend: &'static str, - } - - #[async_trait::async_trait(?Send)] - impl AuctionProvider for DeadlineBidProvider { - fn provider_name(&self) -> &str { - self.name - } - - async fn request_bids( - &self, - _request: &AuctionRequest, - context: &AuctionContext<'_>, - ) -> Result> { - let request = PlatformHttpRequest::new( - http::Request::builder() - .method("POST") - .uri("https://example.com/bid") - .body(edgezero_core::body::Body::empty()) - .expect("should build deadline test request"), - self.backend, - ); - context - .services - .http_client() - .send_async(request.with_cache_bypass()) - .await - .change_context(TrustedServerError::Auction { - message: "deadline test provider launch failed".to_string(), - }) - .map(ProviderRequestOutcome::pending) - } - - async fn parse_response( - &self, - _response: PlatformResponse, - response_time_ms: u64, - ) -> Result> { - Ok(AuctionResponse::success( - self.name, - vec![auction_bid(self.name, 3.0)], - response_time_ms, - )) - } - - fn timeout_ms(&self) -> u32 { - 1_000 - } - - fn backend_name(&self, _services: &RuntimeServices, _timeout_ms: u32) -> Option { - Some(self.backend.to_string()) - } - } - - type RecordedMediatorBudgets = Arc>>; - - struct DeadlineRecordingMediator { - launches: Arc, - budgets: Option, - } - - struct PendingDeadlineMediator(u32); - - #[async_trait::async_trait(?Send)] - impl AuctionProvider for PendingDeadlineMediator { - fn provider_name(&self) -> &str { - "pending-deadline-mediator" - } - - async fn request_bids( - &self, - _request: &AuctionRequest, - context: &AuctionContext<'_>, - ) -> Result> { - let request = PlatformHttpRequest::new( - http::Request::builder() - .method("POST") - .uri("https://example.com/mediate") - .body(edgezero_core::body::Body::empty()) - .expect("should build pending mediator request"), - "pending-mediator-backend", - ); - context - .services - .http_client() - .send_async(request.with_cache_bypass()) - .await - .change_context(TrustedServerError::Auction { - message: "pending mediator launch failed".to_string(), - }) - .map(ProviderRequestOutcome::pending) - } - - async fn parse_response( - &self, - _response: PlatformResponse, - response_time_ms: u64, - ) -> Result> { - Ok(AuctionResponse::success( - self.provider_name(), - vec![auction_bid("mediated", 9.0)], - response_time_ms, - )) - } - - fn timeout_ms(&self) -> u32 { - self.0 - } - - fn backend_name(&self, _services: &RuntimeServices, _timeout_ms: u32) -> Option { - Some("pending-mediator-backend".to_string()) - } - } - - struct SlowLaunchMediator { - immediate: bool, - slow_launch: bool, - parses: Arc, - } - - #[async_trait::async_trait(?Send)] - impl AuctionProvider for SlowLaunchMediator { - fn provider_name(&self) -> &str { - "pending-deadline-mediator" - } - - async fn request_bids( - &self, - request: &AuctionRequest, - context: &AuctionContext<'_>, - ) -> Result> { - let outcome = if self.immediate { - ProviderRequestOutcome::Immediate(AuctionResponse::success( - self.provider_name(), - vec![auction_bid("mediated", 9.0)], - 0, - )) - } else { - PendingDeadlineMediator(20) - .request_bids(request, context) - .await? - }; - // Simulate launch work that outlives the mediator's own budget while - // a complete transport result is already buffered. - if self.slow_launch { - std::thread::sleep(Duration::from_millis(40)); - } - Ok(outcome) - } - - async fn parse_response( - &self, - response: PlatformResponse, - response_time_ms: u64, - ) -> Result> { - self.parses.fetch_add(1, Ordering::Relaxed); - PendingDeadlineMediator(20) - .parse_response(response, response_time_ms) - .await - } - - fn timeout_ms(&self) -> u32 { - 20 - } - - fn backend_name(&self, _services: &RuntimeServices, _timeout_ms: u32) -> Option { - Some("pending-mediator-backend".to_string()) - } - } - - #[async_trait::async_trait(?Send)] - impl AuctionProvider for DeadlineRecordingMediator { - fn provider_name(&self) -> &str { - "deadline-mediator" - } - - async fn request_bids( - &self, - _request: &AuctionRequest, - context: &AuctionContext<'_>, - ) -> Result> { - self.launches.fetch_add(1, Ordering::Relaxed); - if let Some(budgets) = &self.budgets { - budgets - .lock() - .expect("should lock mediator budgets") - .push((context.timeout_ms, context.transport_timeout_ms)); - } - Ok(ProviderRequestOutcome::Immediate(AuctionResponse::no_bid( - self.provider_name(), - 0, - ))) - } - - async fn parse_response( - &self, - _response: PlatformResponse, - _response_time_ms: u64, - ) -> Result> { - panic!("immediate mediator response should not be parsed"); - } - - fn timeout_ms(&self) -> u32 { - 1_000 - } - } - - struct RecordingTimeoutProvider { - name: &'static str, - backend: &'static str, - configured_timeout_ms: u32, - predicted: Arc>>, - requested: Arc>>, - } - - #[async_trait::async_trait(?Send)] - impl AuctionProvider for RecordingTimeoutProvider { - fn provider_name(&self) -> &str { - self.name - } - - async fn request_bids( - &self, - _request: &AuctionRequest, - context: &AuctionContext<'_>, - ) -> Result> { - self.requested - .lock() - .expect("should lock requested timeouts") - .push(context.transport_timeout_ms); - let request = PlatformHttpRequest::new( - http::Request::builder() - .method("POST") - .uri("https://example.com/bid") - .body(edgezero_core::body::Body::empty()) - .expect("should build recording request"), - self.backend, - ); - context - .services - .http_client() - .send_async(request.with_cache_bypass()) - .await - .change_context(TrustedServerError::Auction { - message: "recording launch failed".to_string(), - }) - .map(ProviderRequestOutcome::pending) - } - - async fn parse_response( - &self, - _response: PlatformResponse, - response_time_ms: u64, - ) -> Result> { - Ok(AuctionResponse::success( - self.name, - vec![], - response_time_ms, - )) - } - - fn timeout_ms(&self) -> u32 { - self.configured_timeout_ms - } - - fn backend_name(&self, _services: &RuntimeServices, timeout_ms: u32) -> Option { - self.predicted - .lock() - .expect("should lock predicted timeouts") - .push(timeout_ms); - Some(self.backend.to_string()) - } - } - - struct DivergentBackendProvider { - name: &'static str, - predicted: &'static str, - resolved: &'static str, - } - - #[async_trait::async_trait(?Send)] - impl AuctionProvider for DivergentBackendProvider { - fn provider_name(&self) -> &str { - self.name - } - - async fn request_bids( - &self, - _request: &AuctionRequest, - context: &AuctionContext<'_>, - ) -> Result> { - let request = PlatformHttpRequest::new( - http::Request::builder() - .method("POST") - .uri("https://example.com/bid") - .body(edgezero_core::body::Body::empty()) - .expect("should build divergent request"), - self.resolved, - ); - context - .services - .http_client() - .send_async(request.with_cache_bypass()) - .await - .change_context(TrustedServerError::Auction { - message: "divergent launch failed".to_string(), - }) - .map(ProviderRequestOutcome::pending) - } - - async fn parse_response( - &self, - _response: PlatformResponse, - response_time_ms: u64, - ) -> Result> { - Ok(AuctionResponse::success( - self.name, - vec![], - response_time_ms, - )) - } - - fn timeout_ms(&self) -> u32 { - 2000 - } - - fn backend_name(&self, _services: &RuntimeServices, _timeout_ms: u32) -> Option { - Some(self.predicted.to_string()) - } - } - - struct CanonicalTimeoutBackend { - canonical_ms: u32, - calls: Arc>>, - } - - impl PlatformBackend for CanonicalTimeoutBackend { - fn naming_policy(&self) -> crate::platform::BackendNamingPolicy { - crate::platform::BackendNamingPolicy::Axum - } - - fn predict_name( - &self, - _spec: &PlatformBackendSpec, - ) -> Result> { - Ok("stub-backend".to_string()) - } - - fn ensure(&self, _spec: &PlatformBackendSpec) -> Result> { - Ok("stub-backend".to_string()) - } - - fn canonicalize_transport_timeout_ms(&self, remaining_ms: u32, configured_ms: u32) -> u32 { - self.calls - .lock() - .expect("should lock canonicalization calls") - .push((remaining_ms, configured_ms)); - self.canonical_ms - } - } - - fn recording_provider( - name: &'static str, - backend: &'static str, - configured_timeout_ms: u32, - predicted: &Arc>>, - requested: &Arc>>, - ) -> RecordingTimeoutProvider { - RecordingTimeoutProvider { - name, - backend, - configured_timeout_ms, - predicted: Arc::clone(predicted), - requested: Arc::clone(requested), - } - } - - /// Mediator whose context-aware parse restores `nurl`/`ad_id` (mirroring - /// `adserver_mock`), while its context-free parse does not. Lets a test prove - /// the synchronous mediation path calls `parse_response_with_context`. - struct CacheRestoringMediator; - - fn auction_bid(bidder: &str, price: f64) -> Bid { - let renderer = (bidder == "aps").then(|| { - BidRenderer::Aps(ApsRendererV1 { - version: 1, - account_id: "example-account".to_string(), - bid_id: "aps-selected-bid".to_string(), - creative_id: None, - tag_type: ApsTagType::Iframe, - creative_url: "https://creative.example/render".to_string(), - aax_response: "fictional-base64".to_string(), - width: 300, - height: 250, - }) - }); - Bid { - slot_id: "slot-1".to_string(), - price: Some(price), - currency: "USD".to_string(), - creative: renderer - .is_none() - .then(|| "
ordinary
".to_string()), - adomain: None, - bidder: bidder.to_string(), - returned_seat: None, - width: 300, - height: 250, - nurl: None, - burl: None, - bid_id: (bidder == "aps").then(|| "aps-selected-bid".to_string()), - ad_id: None, - creative_id: None, - renderer, - cache_id: None, - cache_host: None, - cache_path: None, - metadata: HashMap::new(), - } - } - - fn mediated_bid(nurl: Option) -> Bid { - Bid { - slot_id: "header-banner".to_string(), - price: Some(2.5), - currency: "USD".to_string(), - creative: Some("
ad
".to_string()), - adomain: None, - bidder: "mediator".to_string(), - returned_seat: None, - width: 728, - height: 90, - nurl: nurl.clone(), - burl: nurl, - bid_id: None, - ad_id: Some("creative-123".to_string()), - creative_id: None, - renderer: None, - cache_id: Some("cache-abc".to_string()), - cache_host: None, - cache_path: None, - metadata: HashMap::new(), - } - } - - #[async_trait::async_trait(?Send)] - impl AuctionProvider for CacheRestoringMediator { - fn provider_name(&self) -> &str { - "mediator" - } - - async fn request_bids( - &self, - _request: &AuctionRequest, - context: &AuctionContext<'_>, - ) -> Result> { - let req = PlatformHttpRequest::new( - http::Request::builder() - .method("POST") - .uri("https://example.com/mediate") - .body(edgezero_core::body::Body::empty()) - .expect("should build mediator request"), - "mediator-backend", - ); - context - .services - .http_client() - .send_async(req.with_cache_bypass()) - .await - .change_context(TrustedServerError::Auction { - message: "mediator launch failed".to_string(), - }) - .map(ProviderRequestOutcome::pending) - } - - async fn parse_response( - &self, - _response: PlatformResponse, - response_time_ms: u64, - ) -> Result> { - // Context-free path: cannot restore SSP-only render/accounting fields. - Ok(AuctionResponse::success( - "mediator", - vec![mediated_bid(None)], - response_time_ms, - )) - } - - async fn parse_response_with_context( - &self, - _response: PlatformResponse, - response_time_ms: u64, - _request: &AuctionRequest, - _context: &AuctionContext<'_>, - ) -> Result> { - // Context-aware path: restores nurl/ad_id from the collected SSP bids. - Ok(AuctionResponse::success( - "mediator", - vec![mediated_bid(Some("https://nurl.example/win".to_string()))], - response_time_ms, - )) - } - - fn timeout_ms(&self) -> u32 { - 2000 - } - - fn backend_name(&self, _services: &RuntimeServices, _timeout_ms: u32) -> Option { - Some("mediator-backend".to_string()) - } - } - - struct ImmediateMediator; - - #[async_trait::async_trait(?Send)] - impl AuctionProvider for ImmediateMediator { - fn provider_name(&self) -> &str { - "immediate-mediator" - } - - async fn request_bids( - &self, - _request: &AuctionRequest, - _context: &AuctionContext<'_>, - ) -> Result> { - Ok(ProviderRequestOutcome::Immediate(AuctionResponse::success( - self.provider_name(), - vec![mediated_bid(Some( - "https://nurl.example/immediate".to_string(), - ))], - 0, - ))) - } - - async fn parse_response( - &self, - _response: PlatformResponse, - _response_time_ms: u64, - ) -> Result> { - panic!("immediate mediator response should not be parsed"); - } - - fn timeout_ms(&self) -> u32 { - 2000 - } - } - - #[tokio::test] - async fn mediated_bid_preserves_restored_fields_through_run_auction() { - // run_parallel_mediation must parse the mediator response via - // parse_response_with_context so cache/nurl fields restored from SSP - // responses survive the synchronous mediation path (POST /auction, - // /_ts/page-bids), matching the dispatched collect path. - let stub = Arc::new(StubHttpClient::new()); - stub.push_response(200, b"{}".to_vec()); // bidder send_async - stub.push_response(200, b"{}".to_vec()); // mediator send_async - let services = build_services_with_http_client(stub); - // SAFETY: `Box::leak` creates a `'static` reference for test use only. - let services: &'static RuntimeServices = Box::leak(Box::new(services)); - - let config = AuctionConfig { - enabled: true, - providers: AuctionConfig::legacy_provider_map(&["bidder"]), - mediator: Some("mediator".to_string()), - timeout_ms: 2000, - ..Default::default() - }; - let mut orchestrator = AuctionOrchestrator::new(config); - orchestrator.register_provider(Arc::new(StubAuctionProvider { - name: "bidder", - backend: "bidder-backend", - })); - orchestrator.register_provider(Arc::new(CacheRestoringMediator)); - - let request = create_test_auction_request(); - let settings = create_test_settings(); - let req = http::Request::builder() - .method(http::Method::GET) - .uri("https://example.com/test") - .body(edgezero_core::body::Body::empty()) - .expect("should build request"); - let context = AuctionContext { - settings: &settings, - request: &req, - timeout_ms: 2000, - transport_timeout_ms: 2000, - provider_responses: None, - services, - }; - - let result = orchestrator - .run_auction(&request, &context) - .await - .expect("mediated auction should complete"); - - let bid = result - .winning_bids - .get("header-banner") - .expect("mediator should produce a winning bid for the slot"); - assert_eq!( - bid.nurl.as_deref(), - Some("https://nurl.example/win"), - "synchronous mediation must restore nurl via parse_response_with_context" - ); - assert_eq!( - bid.ad_id.as_deref(), - Some("creative-123"), - "mediated bid must keep its restored ad_id" - ); - } - - #[tokio::test] - async fn immediate_mediator_completes_in_sync_and_split_paths() { - for split in [false, true] { - let stub = Arc::new(StubHttpClient::new()); - stub.push_response(200, b"{}".to_vec()); - let services = build_services_with_http_client(stub); - let config = AuctionConfig { - enabled: true, - providers: AuctionConfig::legacy_provider_map(&["bidder"]), - mediator: Some("immediate-mediator".to_string()), - timeout_ms: 2000, - ..Default::default() - }; - let mut orchestrator = AuctionOrchestrator::new(config); - orchestrator.register_provider(Arc::new(StubAuctionProvider { - name: "bidder", - backend: "bidder-backend", - })); - orchestrator.register_provider(Arc::new(ImmediateMediator)); - let request = create_test_auction_request(); - let settings = create_test_settings(); - let downstream = http::Request::new(edgezero_core::body::Body::empty()); - let context = AuctionContext { - settings: &settings, - request: &downstream, - timeout_ms: 2000, - transport_timeout_ms: 2000, - provider_responses: None, - services: &services, - }; - - let result = if split { - let DispatchAuctionOutcome::Dispatched(dispatched) = - orchestrator.dispatch_auction(&request, &context).await - else { - panic!("bidder request should dispatch"); - }; - orchestrator - .collect_dispatched_auction(dispatched, &services, &context) - .await - } else { - orchestrator - .run_auction(&request, &context) - .await - .expect("auction with immediate mediator should complete") - }; - - assert_eq!( - result - .mediator_response - .as_ref() - .map(|response| response.provider.as_str()), - Some("immediate-mediator") - ); - assert_eq!( - result - .winning_bids - .get("header-banner") - .and_then(|bid| bid.nurl.as_deref()), - Some("https://nurl.example/immediate") - ); - } - } - - async fn collect_deadline_test_result(split: bool) -> OrchestrationResult { - let stub = Arc::new(StubHttpClient::new()); - stub.push_response(200, b"{}".to_vec()); - stub.push_response(200, b"{}".to_vec()); - stub.push_select_delay(Duration::from_millis(200)); - let services = build_services_with_http_client(Arc::clone(&stub) as Arc<_>); - let config = AuctionConfig { - enabled: true, - providers: AuctionConfig::legacy_provider_map(&["late-one", "late-two"]), - timeout_ms: 10, - ..Default::default() - }; - let mut orchestrator = AuctionOrchestrator::new(config); - orchestrator.register_provider(Arc::new(DeadlineBidProvider { - name: "late-one", - backend: "late-one-backend", - })); - orchestrator.register_provider(Arc::new(DeadlineBidProvider { - name: "late-two", - backend: "late-two-backend", - })); - let request = create_test_auction_request(); - let settings = create_test_settings(); - let downstream = http::Request::new(edgezero_core::body::Body::empty()); - let context = AuctionContext { - settings: &settings, - request: &downstream, - timeout_ms: 10, - transport_timeout_ms: 10, - provider_responses: None, - services: &services, - }; - - let result = if split { - let DispatchAuctionOutcome::Dispatched(dispatched) = - orchestrator.dispatch_auction(&request, &context).await - else { - panic!("deadline test providers should dispatch"); - }; - orchestrator - .collect_dispatched_auction(dispatched, &services, &context) - .await - } else { - orchestrator - .run_auction(&request, &context) - .await - .expect("deadline test auction should complete") - }; - assert_eq!( - stub.recorded_select_count(), - 0, - "should skip the delayed unbounded collection path" - ); - result - } - - #[tokio::test] - async fn current_adapter_deadline_stops_collection_in_both_paths() { - for split in [false, true] { - let result = collect_deadline_test_result(split).await; - assert_eq!( - result.provider_responses.len(), - 2, - "should account for every provider" - ); - assert!( - result.provider_responses.iter().all(|response| { - response.status == BidStatus::Error - && response.metadata["error_type"] == ERROR_TYPE_TIMEOUT - }), - "should time out outstanding providers on every adapter" - ); - assert!( - result.winning_bids.is_empty(), - "should not accept late bids" - ); - } - } - - async fn pending_mediator_deadline_test_result(split: bool) -> OrchestrationResult { - let stub = Arc::new(StubHttpClient::new()); - stub.push_response(200, b"{}".to_vec()); - stub.push_response(200, b"{}".to_vec()); - stub.push_select_delay(Duration::ZERO); - stub.push_wait_delay(Duration::from_millis(200)); - let services = build_services_with_http_client(Arc::clone(&stub) as Arc<_>); - let config = AuctionConfig { - enabled: true, - providers: AuctionConfig::legacy_provider_map(&["local"]), - mediator: Some("pending-deadline-mediator".to_string()), - timeout_ms: 20, - ..Default::default() - }; - let mut orchestrator = AuctionOrchestrator::new(config); - orchestrator.register_provider(Arc::new(DeadlineBidProvider { - name: "local", - backend: "local-backend", - })); - orchestrator.register_provider(Arc::new(PendingDeadlineMediator(1_000))); - let request = create_test_auction_request(); - let settings = create_test_settings(); - let downstream = http::Request::new(edgezero_core::body::Body::empty()); - let context = AuctionContext { - settings: &settings, - request: &downstream, - timeout_ms: 20, - transport_timeout_ms: 20, - provider_responses: None, - services: &services, - }; - - let result = if split { - let DispatchAuctionOutcome::Dispatched(dispatched) = - orchestrator.dispatch_auction(&request, &context).await - else { - panic!("deadline test provider should dispatch"); - }; - orchestrator - .collect_dispatched_auction(dispatched, &services, &context) - .await - } else { - orchestrator - .run_auction(&request, &context) - .await - .expect("deadline test auction should complete") - }; - assert_eq!( - stub.recorded_wait_count(), - 0, - "should skip the delayed unbounded collection path" - ); - result - } - - #[tokio::test] - async fn pending_mediator_is_bounded_in_sync_and_split_paths() { - for split in [false, true] { - let result = pending_mediator_deadline_test_result(split).await; - assert!( - result.mediator_response.is_none(), - "should abandon a late mediator" + assert_eq!( + value["terminal_reason"], "collection_failed", + "should record the failure at its source" ); assert_eq!( - result.winning_bids["slot-1"].bidder, "local", - "should retain provider winner" + value["provider_calls"][0]["provider_number"], 1, + "should preserve first launch order" + ); + assert_eq!( + value["provider_calls"][1]["provider_number"], 2, + "should preserve second launch order" + ); + assert_eq!( + value["provider_calls"][0]["status"], "error", + "should retain transport failure facts" ); } - } - - #[tokio::test] - async fn split_deadline_skips_mediator_and_discards_late_provider() { - let stub = Arc::new(StubHttpClient::new()); - stub.push_response(200, b"{}".to_vec()); - stub.push_select_delay(Duration::from_millis(50)); - let services = build_services_with_http_client(Arc::clone(&stub) as Arc<_>); - let launches = Arc::new(AtomicUsize::new(0)); - let config = AuctionConfig { - enabled: true, - providers: AuctionConfig::legacy_provider_map(&["late-one"]), - mediator: Some("deadline-mediator".to_string()), - timeout_ms: 10, - ..Default::default() - }; - let mut orchestrator = AuctionOrchestrator::new(config); - orchestrator.register_provider(Arc::new(DeadlineBidProvider { - name: "late-one", - backend: "late-one-backend", - })); - orchestrator.register_provider(Arc::new(DeadlineRecordingMediator { - launches: Arc::clone(&launches), - budgets: None, - })); - let request = create_test_auction_request(); - let settings = create_test_settings(); - let downstream = http::Request::new(edgezero_core::body::Body::empty()); - let context = AuctionContext { - settings: &settings, - request: &downstream, - timeout_ms: 10, - transport_timeout_ms: 10, - provider_responses: None, - services: &services, - }; - let DispatchAuctionOutcome::Dispatched(dispatched) = - orchestrator.dispatch_auction(&request, &context).await - else { - panic!("deadline test provider should dispatch"); - }; - let result = orchestrator - .collect_dispatched_auction(dispatched, &services, &context) - .await; - - assert_eq!(launches.load(Ordering::Relaxed), 0); - assert!(result.mediator_response.is_none()); - assert!( - result.winning_bids.is_empty(), - "should not retain a late provider bid" - ); - } - - #[tokio::test] - async fn synchronous_deadline_skips_mediator_and_discards_late_provider() { - let stub = Arc::new(StubHttpClient::new()); - stub.push_response(200, b"{}".to_vec()); - stub.push_select_delay(Duration::from_millis(50)); - let services = build_services_with_http_client(Arc::clone(&stub) as Arc<_>); - let launches = Arc::new(AtomicUsize::new(0)); - let config = AuctionConfig { - enabled: true, - providers: AuctionConfig::legacy_provider_map(&["late-one"]), - mediator: Some("deadline-mediator".to_string()), - timeout_ms: 10, - ..Default::default() - }; - let mut orchestrator = AuctionOrchestrator::new(config); - orchestrator.register_provider(Arc::new(DeadlineBidProvider { - name: "late-one", - backend: "late-one-backend", - })); - orchestrator.register_provider(Arc::new(DeadlineRecordingMediator { - launches: Arc::clone(&launches), - budgets: None, - })); - let request = create_test_auction_request(); - let settings = create_test_settings(); - let downstream = http::Request::new(edgezero_core::body::Body::empty()); - let context = AuctionContext { - settings: &settings, - request: &downstream, - timeout_ms: 10, - transport_timeout_ms: 10, - provider_responses: None, - services: &services, - }; - let result = orchestrator - .run_auction(&request, &context) - .await - .expect("synchronous deadline test should complete"); - - assert_eq!(launches.load(Ordering::Relaxed), 0); - assert!(result.mediator_response.is_none()); - assert!( - result.winning_bids.is_empty(), - "should not retain a late provider bid" - ); - } - - fn create_test_auction_request() -> AuctionRequest { - AuctionRequest { - id: "test-auction-123".to_string(), - slots: vec![ - AdSlot { - id: "header-banner".to_string(), - formats: vec![AdFormat { - media_type: MediaType::Banner, - width: 728, - height: 90, - }], - floor_price: Some(1.50), - targeting: HashMap::new(), - bidders: HashMap::new(), - }, - AdSlot { - id: "sidebar".to_string(), - formats: vec![AdFormat { - media_type: MediaType::Banner, - width: 300, - height: 250, - }], - floor_price: Some(1.00), - targeting: HashMap::new(), - bidders: HashMap::new(), - }, - ], - publisher: PublisherInfo { - domain: "test.com".to_string(), - page_url: Some("https://test.com/article".to_string()), - }, - user: UserInfo { - id: Some("user-123".to_string()), - consent: None, - eids: None, - }, - device: None, - site: None, - context: HashMap::new(), - } - } - - fn create_test_settings() -> crate::settings::Settings { - let settings_str = crate_test_settings_str(); - crate::settings::Settings::from_toml(&settings_str).expect("should parse test settings") - } - - struct ImmediateNoBidProvider; - - #[async_trait::async_trait(?Send)] - impl AuctionProvider for ImmediateNoBidProvider { - fn provider_name(&self) -> &str { - "immediate" - } - - async fn request_bids( - &self, - _request: &AuctionRequest, - _context: &AuctionContext<'_>, - ) -> Result> { - Ok(ProviderRequestOutcome::Immediate(AuctionResponse::no_bid( - "immediate", - 0, - ))) - } - - async fn parse_response( - &self, - _response: PlatformResponse, - _response_time_ms: u64, - ) -> Result> { - panic!("immediate response should not be parsed"); - } - - fn timeout_ms(&self) -> u32 { - 2000 - } - } - - struct LaunchFailingProvider; - - #[async_trait::async_trait(?Send)] - impl AuctionProvider for LaunchFailingProvider { - fn provider_name(&self) -> &str { - "launch-failing" - } - - async fn request_bids( - &self, - _request: &AuctionRequest, - _context: &AuctionContext<'_>, - ) -> Result> { - Err(Report::new(TrustedServerError::Auction { - message: "launch failed in test provider".to_string(), - })) - } - - async fn parse_response( - &self, - _response: PlatformResponse, - _response_time_ms: u64, - ) -> Result> { - Err(Report::new(TrustedServerError::Auction { - message: "launch-failing provider should not parse responses".to_string(), - })) - } - - fn timeout_ms(&self) -> u32 { - 2000 - } - - fn backend_name(&self, _services: &RuntimeServices, _timeout_ms: u32) -> Option { - Some("launch-failing-backend".to_string()) - } - } - - fn immediate_test_context<'a>( - settings: &'a crate::settings::Settings, - request: &'a http::Request, - services: &'a RuntimeServices, - ) -> AuctionContext<'a> { - AuctionContext { - settings, - request, - timeout_ms: 2000, - transport_timeout_ms: 2000, - provider_responses: None, - services, - } - } - - #[tokio::test] - async fn synchronous_auction_accepts_an_all_immediate_no_bid_result() { - let config = AuctionConfig { - enabled: true, - providers: AuctionConfig::legacy_provider_map(&["immediate"]), - timeout_ms: 2000, - ..Default::default() - }; - let mut orchestrator = AuctionOrchestrator::new(config); - orchestrator.register_provider(Arc::new(ImmediateNoBidProvider)); - let settings = create_test_settings(); - let services = noop_services(); - let downstream = http::Request::new(edgezero_core::body::Body::empty()); - let context = immediate_test_context(&settings, &downstream, &services); - - let result = orchestrator - .run_auction(&create_test_auction_request(), &context) - .await - .expect("all-immediate auction should complete"); - - assert_eq!(result.provider_responses.len(), 1); - assert_eq!(result.provider_responses[0].status, BidStatus::NoBid); - assert!(result.winning_bids.is_empty()); - } - - #[tokio::test] - async fn split_auction_accepts_an_all_immediate_no_bid_result() { - let config = AuctionConfig { - enabled: true, - providers: AuctionConfig::legacy_provider_map(&["immediate"]), - timeout_ms: 2000, - ..Default::default() - }; - let mut orchestrator = AuctionOrchestrator::new(config); - orchestrator.register_provider(Arc::new(ImmediateNoBidProvider)); - let settings = create_test_settings(); - let services = noop_services(); - let downstream = http::Request::new(edgezero_core::body::Body::empty()); - let context = immediate_test_context(&settings, &downstream, &services); - - let DispatchAuctionOutcome::Dispatched(dispatched) = orchestrator - .dispatch_auction(&create_test_auction_request(), &context) - .await - else { - panic!("enabled immediate provider should dispatch"); - }; - let result = orchestrator - .collect_dispatched_auction(dispatched, &services, &context) - .await; - - assert_eq!(result.provider_responses.len(), 1); - assert_eq!(result.provider_responses[0].status, BidStatus::NoBid); - assert!(result.winning_bids.is_empty()); - } - #[tokio::test] - async fn immediate_and_pending_providers_complete_in_sync_and_split_paths() { - for split in [false, true] { - let config = AuctionConfig { - enabled: true, - providers: AuctionConfig::legacy_provider_map(&["immediate", "pending"]), - timeout_ms: 2000, - ..Default::default() - }; - let mut orchestrator = AuctionOrchestrator::new(config); - orchestrator.register_provider(Arc::new(ImmediateNoBidProvider)); - orchestrator.register_provider(Arc::new(StubAuctionProvider { - name: "pending", - backend: "pending-backend", - })); - let stub = Arc::new(StubHttpClient::new()); - stub.push_response(200, b"{}".to_vec()); - let services = build_services_with_http_client(stub); + #[tokio::test] + async fn trace_auction_live_mediator_no_bid_is_an_actual_ordered_call() { + let clock = TestClock::new(); + let http = ScriptedHttpClient::new(&clock); + respond_planned(&http, 204, Vec::new()); + let services = build_services_with_outbound_client(&http, &clock); + let plan = Arc::new( + AuctionPlan::compile(planned_config( + &[("bidder", RoutingMode::AllEligible)], + false, + )) + .expect("should compile bidder plan"), + ); + let orchestrator = + AuctionOrchestrator::from_plan(plan, Some(Arc::new(ImmediateNoBidProvider))); + let request = planned_request(); let settings = create_test_settings(); - let downstream = http::Request::new(edgezero_core::body::Body::empty()); - let context = immediate_test_context(&settings, &downstream, &services); - let request = create_test_auction_request(); - - let result = if split { - let DispatchAuctionOutcome::Dispatched(dispatched) = - orchestrator.dispatch_auction(&request, &context).await - else { - panic!("mixed immediate/pending auction should dispatch"); - }; - orchestrator - .collect_dispatched_auction(dispatched, &services, &context) - .await - } else { - orchestrator - .run_auction(&request, &context) - .await - .expect("mixed immediate/pending auction should complete") + let (inbound, carry) = captured_request(&request); + let context = AuctionContext { + settings: &settings, + request: &inbound, + timeout_ms: 777, + provider_responses: None, + services: &services, }; + orchestrator + .run_auction(&request, &context) + .await + .expect("should retain ordinary successful zero-bid execution"); + let value = evidence(&carry); + assert_eq!( + value["terminal_status"], "completed", + "should not label ordinary zero bids a failure" + ); + assert_eq!( + value["provider_calls"][0]["role"], "bidder", + "should retain the bidder launch" + ); + assert_eq!( + value["provider_calls"][1]["role"], "mediator", + "should retain the real mediator invocation" + ); + assert_eq!( + value["provider_calls"][1]["status"], "no_bid", + "should retain mediator zero-bid status" + ); + } + struct NeverFinishesMediator; - assert_eq!(result.provider_responses.len(), 2); - assert!(result.provider_responses.iter().any(|response| { - response.provider == "immediate" && response.status == BidStatus::NoBid - })); - assert!(result.provider_responses.iter().any(|response| { - response.provider == "pending" && response.status == BidStatus::Success - })); - } - } - - #[test] - fn runtime_origin_snapshot_preserves_https_fallback() { - let mut request = http::Request::builder() - .uri("/article") - .header(http::header::HOST, "client-supplied.example.com") - .body(edgezero_core::body::Body::empty()) - .expect("should construct normalized runtime request"); - request - .extensions_mut() - .insert(crate::http_util::RuntimeRequestOrigin::new( - edgezero_core::ingress::InboundOrigin::parse( - "https", - "publisher.example.com:8443", - edgezero_core::ingress::OriginSource::RuntimeUri, - ) - .expect("should validate runtime origin"), - )); - - let snapshot = super::snapshot_context_request(&request); - let info = crate::http_util::RequestInfo::from_request( - &snapshot, - &crate::platform::ClientInfo::default(), - ) - .expect("should accept runtime snapshot metadata"); - - assert_eq!( - info.scheme, "https", - "should preserve runtime HTTPS scheme in auction snapshot" - ); - assert_eq!( - info.host, "publisher.example.com:8443", - "should preserve runtime authority in auction snapshot" - ); - } - - #[test] - fn public_origin_snapshot_preserves_accepted_and_frozen_rejected_decisions() { - let mut settings = crate::test_support::tests::create_test_settings(); - settings.publisher.domain = "publisher.example.com".to_owned(); - settings.trusted_forwarder = Some(crate::settings::TrustedForwarderConfig { - auth_header: crate::forwarder::FORWARDER_AUTH_HEADER.to_owned(), - shared_secret: crate::redacted::Redacted::new( - "fictional-forwarder-secret-0123456789".to_owned(), - ), - }); - for accepted in [true, false] { - let mut request = http::Request::builder() - .uri("http://internal.example.com/article?x=1") - .header(http::header::HOST, "internal.example.com") - .header( - crate::forwarder::FORWARDER_AUTH_HEADER, - "fictional-forwarder-secret-0123456789", - ) - .header("x-forwarded-host", "publisher.example.com:8443") - .header("x-forwarded-proto", "https") - .body(edgezero_core::body::Body::empty()) - .expect("should construct request"); - if !accepted { - request - .headers_mut() - .remove(crate::forwarder::FORWARDER_AUTH_HEADER); + #[async_trait::async_trait(?Send)] + impl AuctionProvider for NeverFinishesMediator { + fn provider_name(&self) -> &str { + "pending-mediator" } - let ingress = edgezero_core::ingress::RequestIngress::new( - edgezero_core::ingress::CapturedTarget::Unavailable( - edgezero_core::ingress::TargetUnavailable::NotExposed, - ), - Some( - edgezero_core::ingress::InboundOrigin::parse( - "http", - "internal.example.com", - edgezero_core::ingress::OriginSource::RuntimeUri, - ) - .expect("should construct transport origin"), - ), - edgezero_core::ingress::HeaderFidelity::default(), + async fn request_bids( + &self, + _request: &AuctionRequest, + _context: &AuctionContext<'_>, + ) -> Result> { + std::future::pending().await + } + async fn parse_response( + &self, + _response: OutboundResponse, + _response_time_ms: u64, + ) -> Result> { + panic!("should never parse an uncompleted mediator invocation"); + } + fn timeout_ms(&self) -> u32 { + 777 + } + } + + #[tokio::test] + async fn trace_auction_live_cancellation_preserves_pending_mediator_after_bidder_collection() + { + let clock = TestClock::new(); + let http = ScriptedHttpClient::new(&clock); + respond_planned(&http, 204, Vec::new()); + let services = build_services_with_outbound_client(&http, &clock); + let plan = Arc::new( + AuctionPlan::compile(planned_config( + &[("bidder", RoutingMode::AllEligible)], + false, + )) + .expect("should compile the bidder plan"), ); - request.extensions_mut().insert(ingress); - crate::forwarder::prepare_trusted_forwarder(&mut request, &settings); - let mut snapshot = super::snapshot_context_request(&request); + let orchestrator = + AuctionOrchestrator::from_plan(plan, Some(Arc::new(NeverFinishesMediator))); + let request = planned_request(); + let settings = create_test_settings(); + let (inbound, carry) = captured_request(&request); + let context = AuctionContext { + settings: &settings, + request: &inbound, + timeout_ms: 777, + provider_responses: None, + services: &services, + }; + let mut pending = Box::pin(orchestrator.run_auction(&request, &context)); + let mut task = std::task::Context::from_waker(std::task::Waker::noop()); assert!( - snapshot - .extensions() - .get::() - .is_some(), - "should preserve frozen forwarding decision" - ); - let retained = snapshot - .extensions() - .get::() - .expect("should preserve immutable ingress"); - let original = request - .extensions() - .get::() - .expect("should retain ingress on source request"); - assert_eq!( - retained.origin().map(|origin| ( - origin.scheme(), - origin.authority(), - origin.source() - )), - original.origin().map(|origin| ( - origin.scheme(), - origin.authority(), - origin.source() - )), - "should preserve actual transport origin" + std::future::Future::poll(pending.as_mut(), &mut task).is_pending(), + "should stop at the actual mediator await" ); assert!( - matches!( - retained.target(), - edgezero_core::ingress::CapturedTarget::Unavailable( - edgezero_core::ingress::TargetUnavailable::NotExposed - ) - ), - "should preserve unavailable target evidence" + carry.transport().is_none(), + "should not invent a terminal result while work is pending" ); + + drop(pending); + + let facts = evidence(&carry); assert_eq!( - retained.header_fidelity(&http::header::COOKIE), - original.header_fidelity(&http::header::COOKIE), - "should preserve cookie fidelity" - ); - snapshot.headers_mut().insert( - crate::forwarder::FORWARDER_AUTH_HEADER, - http::HeaderValue::from_static("fictional-forwarder-secret-0123456789"), + facts["terminal_status"], "abandoned", + "should finish synchronously when collection is cancelled" ); - snapshot.headers_mut().insert( - "x-forwarded-host", - http::HeaderValue::from_static("publisher.example.com:8443"), + assert_eq!( + facts["provider_calls"][0]["status"], "no_bid", + "should retain the collected bidder result" ); - snapshot - .headers_mut() - .insert("x-forwarded-proto", http::HeaderValue::from_static("https")); - crate::forwarder::prepare_trusted_forwarder(&mut snapshot, &settings); - let info = crate::http_util::RequestInfo::from_request( - &snapshot, - &crate::platform::ClientInfo::default(), - ) - .expect("should accept frozen snapshot metadata"); assert_eq!( - info.host, - if accepted { - "publisher.example.com:8443" - } else { - "internal.example.com" - }, - "should retain initial trust decision" + facts["provider_calls"][1]["role"], "mediator", + "should retain the actual pending mediator launch" ); assert_eq!( - info.scheme, - if accepted { "https" } else { "http" }, - "should retain initial public or ingress scheme" + facts["provider_calls"][1]["status"], "abandoned", + "should close only still-pending observations" ); } - } - - #[test] - fn provider_error_response_includes_diagnostic_metadata() { - let error = Report::new(TrustedServerError::Auction { - message: "parse failed".to_string(), - }) - .attach("internal/source.rs:12:34"); - - let response = - super::provider_error_response("prebid", 37, super::ERROR_TYPE_PARSE_RESPONSE, &error); - - assert_eq!( - response.status, - BidStatus::Error, - "should mark diagnostic provider responses as errors" - ); - assert_eq!( - response.metadata["error_type"], - serde_json::json!("parse_response"), - "should include the provider error classification" - ); - - let message = response.metadata["message"] - .as_str() - .expect("should include provider error message"); - assert!( - message.contains("parse failed"), - "should include user-safe diagnostic detail" - ); - assert!( - !message.contains("internal/source.rs"), - "should not include attached internal details" - ); - } - - #[test] - fn launch_failed_response_has_safe_static_message() { - let response = super::provider_launch_failed_response("prebid", 58); - assert_eq!( - response.status, - BidStatus::Error, - "should mark launch failures as errors" - ); - assert_eq!( - response.metadata["error_type"], - serde_json::json!("launch_failed"), - "should include launch_failed classification" - ); - assert_eq!( - response.metadata["message"], - serde_json::json!("Provider launch failed"), - "should use a safe, stable public launch failure message" - ); - } + #[tokio::test] + async fn trace_auction_live_mediator_failure_preserves_successful_local_fallback() { + let clock = TestClock::new(); + let http = ScriptedHttpClient::new(&clock); + respond_planned(&http, 204, Vec::new()); + let services = build_services_with_outbound_client(&http, &clock); + let plan = Arc::new( + AuctionPlan::compile(planned_config( + &[("bidder", RoutingMode::AllEligible)], + false, + )) + .expect("should compile the bidder plan"), + ); + let orchestrator = + AuctionOrchestrator::from_plan(plan, Some(Arc::new(LaunchFailingProvider))); + let request = planned_request(); + let settings = create_test_settings(); + let (inbound, carry) = captured_request(&request); + let context = AuctionContext { + settings: &settings, + request: &inbound, + timeout_ms: 777, + provider_responses: None, + services: &services, + }; - #[test] - fn transport_failed_response_has_safe_static_message() { - let response = super::provider_transport_failed_response("prebid", 64); + let result = orchestrator + .run_auction(&request, &context) + .await + .expect("should preserve successful ordinary local fallback"); - assert_eq!( - response.status, - BidStatus::Error, - "should mark transport failures as errors" - ); - assert_eq!( - response.metadata["error_type"], - serde_json::json!("transport"), - "should classify transport failures consistently with other failure modes" - ); - assert_eq!( - response.metadata["message"], - serde_json::json!("Provider request failed"), - "should use a safe, stable public transport failure message" - ); - } + assert!( + result.mediator_response.is_none(), + "should preserve the ordinary discarded mediator response" + ); + let facts = evidence(&carry); + assert_eq!( + facts["terminal_status"], "completed", + "should not infer whole-auction failure from provider error" + ); + assert!( + facts.get("terminal_reason").is_none(), + "should omit a reason for completed execution" + ); + assert_eq!( + facts["provider_calls"][1]["status"], "error", + "should preserve the failed mediator invocation before fallback" + ); + } - #[test] - fn provider_error_message_truncates_user_safe_context() { - let long_message = "x".repeat(super::PROVIDER_ERROR_MESSAGE_CHARS + 100); - let error = Report::new(TrustedServerError::Auction { - message: long_message, - }); + #[tokio::test] + async fn trace_auction_live_zero_budget_has_no_synthetic_provider_launches() { + let clock = TestClock::new(); + let http = ScriptedHttpClient::new(&clock); + let services = build_services_with_outbound_client(&http, &clock); + let plan = Arc::new( + AuctionPlan::compile(planned_config( + &[("bidder", RoutingMode::AllEligible)], + false, + )) + .expect("should compile the bidder plan"), + ); + let orchestrator = + AuctionOrchestrator::from_plan(plan, Some(Arc::new(NeverFinishesMediator))); + let request = planned_request(); + let settings = create_test_settings(); + let (inbound, carry) = captured_request(&request); + let context = AuctionContext { + settings: &settings, + request: &inbound, + timeout_ms: 0, + provider_responses: None, + services: &services, + }; - let message = super::provider_error_message(&error); + orchestrator + .run_auction(&request, &context) + .await + .expect("should preserve the ordinary zero-budget result"); - assert_eq!( - message.chars().count(), - super::PROVIDER_ERROR_MESSAGE_CHARS, - "should cap provider error messages" - ); - assert!( - message.starts_with("Auction error: "), - "should preserve the current context display text" - ); + assert_eq!( + evidence(&carry)["provider_calls"], + serde_json::json!([]), + "should exclude synthetic timeout responses from launched work" + ); + assert!( + recorded_uris(&http).is_empty(), + "should not add network work for trace" + ); + } } - #[test] - fn filters_winning_bids_below_floor() { - let orchestrator = AuctionOrchestrator::new(AuctionConfig::default()); - let mut floor_prices = HashMap::new(); - floor_prices.insert("slot-1".to_string(), 1.00); - floor_prices.insert("slot-2".to_string(), 2.00); - - // Arrange winning bids with one below floor. - let mut winning_bids = HashMap::new(); - winning_bids.insert( - "slot-1".to_string(), - Bid { - slot_id: "slot-1".to_string(), - price: Some(0.50), - currency: "USD".to_string(), - creative: Some("
Ad
".to_string()), - adomain: None, - bidder: "test-bidder".to_string(), - returned_seat: None, - width: 300, - height: 250, - nurl: None, - burl: None, - bid_id: None, - ad_id: None, - creative_id: None, - renderer: None, - cache_id: None, - cache_host: None, - cache_path: None, - metadata: HashMap::new(), - }, - ); - winning_bids.insert( - "slot-2".to_string(), - Bid { - slot_id: "slot-2".to_string(), - price: Some(2.00), - currency: "USD".to_string(), - creative: Some("
Ad
".to_string()), - adomain: None, - bidder: "test-bidder".to_string(), - returned_seat: None, - width: 300, - height: 250, - nurl: None, - burl: None, - bid_id: None, - ad_id: None, - creative_id: None, - renderer: None, - cache_id: None, - cache_host: None, - cache_path: None, - metadata: HashMap::new(), - }, + #[tokio::test] + async fn disabled_from_plan_is_a_no_work_kill_switch_for_sync_and_split_paths() { + let plan = Arc::new( + AuctionPlan::compile(planned_config( + &[("provider-a", RoutingMode::AllEligible)], + false, + )) + .expect("should compile plan") + .with_enabled(false), ); + let orchestrator = AuctionOrchestrator::from_plan(plan, None); + let clock = TestClock::new(); + let http = ScriptedHttpClient::new(&clock); + let services = build_services_with_outbound_client(&http, &clock); + let settings = create_test_settings(); + let inbound = http::Request::new(edgezero_core::body::Body::empty()); + let context = AuctionContext { + settings: &settings, + request: &inbound, + timeout_ms: 777, + provider_responses: None, + services: &services, + }; + let request = planned_request(); - // Apply floor pricing and validate the results. - let filtered = orchestrator.apply_floor_prices(winning_bids, &floor_prices); + let result = orchestrator + .run_auction(&request, &context) + .await + .expect("disabled auction should complete as no-bid"); + assert!(result.provider_responses.is_empty()); + assert!(result.winning_bids.is_empty()); + assert!(matches!( + orchestrator.dispatch_auction(&request, &context).await, + DispatchAuctionOutcome::NotStarted + )); + assert!(recorded_uris(&http).is_empty()); + } - assert_eq!( - filtered.len(), - 1, - "Filtered bids should keep only those meeting floor price" - ); - assert!( - filtered.contains_key("slot-2"), - "Filtered bids should include slot-2 winner" + #[tokio::test] + async fn enabled_empty_plan_is_successful_no_bid_without_dispatch() { + let plan = Arc::new( + AuctionPlan::compile(planned_config(&[], false)).expect("should compile empty plan"), ); + let orchestrator = AuctionOrchestrator::from_plan(plan, None); + let clock = TestClock::new(); + let http = ScriptedHttpClient::new(&clock); + let services = build_services_with_outbound_client(&http, &clock); + let settings = create_test_settings(); + let inbound = http::Request::new(edgezero_core::body::Body::empty()); + let context = AuctionContext { + settings: &settings, + request: &inbound, + timeout_ms: 777, + provider_responses: None, + services: &services, + }; + let request = planned_request(); + + let result = orchestrator + .run_auction(&request, &context) + .await + .expect("empty auction should complete as no-bid"); + assert!(result.provider_responses.is_empty()); + assert!(result.winning_bids.is_empty()); + assert!(matches!( + orchestrator.dispatch_auction(&request, &context).await, + DispatchAuctionOutcome::NotStarted + )); + assert!(recorded_uris(&http).is_empty()); } - // Timeout paths still need a controllable delayed pending response: - // - Deadline check in select() loop (drops remaining requests) - // - Mediator skip when remaining_ms == 0 (bidding exhausts budget) - // - Provider skip when effective_timeout == 0 (budget exhausted before launch) - // - Provider context receives reduced timeout_ms per remaining budget - // - // Follow-up: extend StubHttpClient with delayed responses so these paths can - // be asserted deterministically without real backends or wall-clock races. + #[tokio::test] + async fn all_skipped_from_plan_completes_with_routing_metadata_in_sync_and_split_paths() { + for split in [false, true] { + let plan = Arc::new( + AuctionPlan::compile(planned_config(&[("skipped", RoutingMode::Explicit)], false)) + .expect("should compile all-skipped plan"), + ); + let orchestrator = AuctionOrchestrator::from_plan(plan, None); + let clock = TestClock::new(); + let http = ScriptedHttpClient::new(&clock); + let services = build_services_with_outbound_client(&http, &clock); + let settings = create_test_settings(); + let inbound = http::Request::new(edgezero_core::body::Body::empty()); + let context = AuctionContext { + settings: &settings, + request: &inbound, + timeout_ms: 777, + provider_responses: None, + services: &services, + }; + let request = planned_request(); - #[test] - fn test_no_providers_configured() { - futures::executor::block_on(async { - let config = AuctionConfig { - enabled: true, - sanitize_creatives: true, - rewrite_creatives: true, - rewrite_clicks: None, - providers: AuctionConfig::legacy_provider_map(&[]), - bidders: Default::default(), - mediator: None, - timeout_ms: 2000, - creative_store: "creative_store".to_string(), - allowed_context_keys: BTreeSet::from(["permutive_segments".to_string()]), + let result = if split { + let DispatchAuctionOutcome::Dispatched(dispatched) = + orchestrator.dispatch_auction(&request, &context).await + else { + panic!("all-skipped auction should produce a completed dispatch token"); + }; + orchestrator + .collect_dispatched_auction(dispatched, &services, &context) + .await + } else { + orchestrator + .run_auction(&request, &context) + .await + .expect("all-skipped auction should complete") }; - let orchestrator = AuctionOrchestrator::new(config); + assert_eq!(result.provider_responses.len(), 1); + assert_eq!( + result.provider_responses[0].metadata["routing"]["skipped_no_eligible_slots"], + true + ); + assert!(result.metadata.contains_key("routing")); + assert!(recorded_uris(&http).is_empty()); + } + } - let request = create_test_auction_request(); - let settings = create_test_settings(); - let req = http::Request::builder() - .method(http::Method::GET) - .uri("https://test.com/test") - .body(edgezero_core::body::Body::empty()) - .expect("should build request"); - let context = create_test_auction_context(&settings, &req, 2000); + struct FailingCountingConfigStore { + reads: AtomicUsize, + } - let result = orchestrator.run_auction(&request, &context).await; + #[async_trait::async_trait(?Send)] + impl PlatformConfigStore for FailingCountingConfigStore { + fn get( + &self, + _store_name: &StoreName, + _key: &str, + ) -> Result> { + self.reads.fetch_add(1, Ordering::Relaxed); + Err(Report::new(PlatformError::ConfigStore)) + } - assert!(result.is_err()); - let err = result.unwrap_err(); - assert!(format!("{}", err).contains("No providers configured")); - }); + async fn put( + &self, + _store_id: &StoreId, + _key: &str, + _value: &str, + ) -> Result<(), Report> { + Err(Report::new(PlatformError::Unsupported)) + } + + async fn delete( + &self, + _store_id: &StoreId, + _key: &str, + ) -> Result<(), Report> { + Err(Report::new(PlatformError::Unsupported)) + } } - #[test] - fn provider_launch_failures_error_when_no_requests_launch() { - futures::executor::block_on(async { - let config = AuctionConfig { - enabled: true, - providers: AuctionConfig::legacy_provider_map(&["launch-failing"]), - timeout_ms: 2000, - ..Default::default() - }; - let mut orchestrator = AuctionOrchestrator::new(config); - orchestrator.register_provider(Arc::new(LaunchFailingProvider)); + struct CountingConfigStore { + reads: AtomicUsize, + current_kid: String, + /// Time each read spends, on the given test clock. + admission_delay: Option<(TestClock, Duration)>, + } - let request = create_test_auction_request(); - let settings = create_test_settings(); - let req = http::Request::builder() - .method(http::Method::GET) - .uri("https://test.com/test") - .body(edgezero_core::body::Body::empty()) - .expect("should build request"); - let context = create_test_auction_context(&settings, &req, 2000); + #[async_trait::async_trait(?Send)] + impl PlatformConfigStore for CountingConfigStore { + fn get(&self, _store_name: &StoreName, key: &str) -> Result> { + self.reads.fetch_add(1, Ordering::Relaxed); + if let Some((clock, delay)) = &self.admission_delay { + clock.advance(*delay); + } + (key == "current-kid") + .then(|| self.current_kid.clone()) + .ok_or_else(|| Report::new(PlatformError::ConfigStore)) + } - let error = orchestrator - .run_auction(&request, &context) - .await - .expect_err("should fail when every provider launch fails"); + async fn put( + &self, + _store_id: &StoreId, + _key: &str, + _value: &str, + ) -> Result<(), Report> { + Err(Report::new(PlatformError::Unsupported)) + } - assert!( - error - .to_string() - .contains("All 1 configured provider(s) skipped or failed to launch"), - "should explain that no configured provider request launched" - ); - }); + async fn delete( + &self, + _store_id: &StoreId, + _key: &str, + ) -> Result<(), Report> { + Err(Report::new(PlatformError::Unsupported)) + } } - #[tokio::test] - async fn duplicate_backend_name_fails_second_provider_attributably_in_both_paths() { - for split in [false, true] { - let config = AuctionConfig { - enabled: true, - providers: AuctionConfig::legacy_provider_map(&["provider-a", "provider-b"]), - timeout_ms: 2000, - ..Default::default() - }; - let mut orchestrator = AuctionOrchestrator::new(config); - orchestrator.register_provider(Arc::new(StubAuctionProvider { - name: "provider-a", - backend: "shared-backend", - })); - orchestrator.register_provider(Arc::new(StubAuctionProvider { - name: "provider-b", - backend: "shared-backend", - })); - let stub = Arc::new(StubHttpClient::new()); - stub.push_response(200, b"{}".to_vec()); - let services = build_services_with_http_client(stub); - let settings = create_test_settings(); - let downstream = http::Request::new(edgezero_core::body::Body::empty()); - let context = immediate_test_context(&settings, &downstream, &services); - let request = create_test_auction_request(); + struct CountingSecretStore { + reads: AtomicUsize, + key: Vec, + } + + #[async_trait::async_trait(?Send)] + impl PlatformSecretStore for CountingSecretStore { + fn get_bytes( + &self, + _store_name: &StoreName, + _key: &str, + ) -> Result, Report> { + self.reads.fetch_add(1, Ordering::Relaxed); + Ok(self.key.clone()) + } - let result = if split { - let DispatchAuctionOutcome::Dispatched(dispatched) = - orchestrator.dispatch_auction(&request, &context).await - else { - panic!("should dispatch the first provider"); - }; - orchestrator - .collect_dispatched_auction(dispatched, &services, &context) - .await - } else { - orchestrator - .run_auction(&request, &context) - .await - .expect("should complete auction despite the collision") - }; + async fn create( + &self, + _store_id: &StoreId, + _name: &str, + _value: &str, + ) -> Result<(), Report> { + Err(Report::new(PlatformError::Unsupported)) + } - assert!(result.provider_responses.iter().any(|response| { - response.provider == "provider-a" && response.status == BidStatus::Success - })); - assert!(result.provider_responses.iter().any(|response| { - response.provider == "provider-b" && response.status == BidStatus::Error - })); + async fn delete( + &self, + _store_id: &StoreId, + _name: &str, + ) -> Result<(), Report> { + Err(Report::new(PlatformError::Unsupported)) } } - #[test] - fn test_orchestrator_is_enabled() { - let config = AuctionConfig { - enabled: true, - ..Default::default() - }; - let orchestrator = AuctionOrchestrator::new(config); - assert!(orchestrator.is_enabled()); + struct UnusedSecretStore; - let config = AuctionConfig { - enabled: false, - ..Default::default() - }; - let orchestrator = AuctionOrchestrator::new(config); - assert!(!orchestrator.is_enabled()); - } + #[async_trait::async_trait(?Send)] + impl PlatformSecretStore for UnusedSecretStore { + fn get_bytes( + &self, + _store_name: &StoreName, + _key: &str, + ) -> Result, Report> { + panic!("signing key should not be read after current-kid failure") + } - #[test] - fn remaining_budget_returns_full_timeout_immediately() { - let start = Instant::now(); - let result = super::remaining_budget_ms(start, 2000); - // Should be very close to 2000 (allow a few ms for test execution) - assert!( - result >= 1990, - "should return ~full timeout immediately, got {result}" - ); - } + async fn create( + &self, + _store_id: &StoreId, + _name: &str, + _value: &str, + ) -> Result<(), Report> { + Err(Report::new(PlatformError::Unsupported)) + } - #[test] - fn remaining_budget_saturates_at_zero() { - // Create an instant in the past by sleeping briefly with a tiny timeout - let start = Instant::now(); - // Use a timeout of 0 — elapsed will always exceed it - let result = super::remaining_budget_ms(start, 0); - assert_eq!(result, 0, "should return 0 when timeout is 0"); + async fn delete( + &self, + _store_id: &StoreId, + _name: &str, + ) -> Result<(), Report> { + Err(Report::new(PlatformError::Unsupported)) + } } - #[test] - fn remaining_budget_decreases_over_time() { - let start = Instant::now(); - std::thread::sleep(Duration::from_millis(50)); - let result = super::remaining_budget_ms(start, 2000); - assert!( - result < 2000, - "should be less than full timeout after sleeping" - ); - assert!( - result > 1900, - "should still have most of the budget, got {result}" - ); - } + // --------------------------------------------------------------------------- + // Minimal test double for AuctionProvider + // --------------------------------------------------------------------------- - #[test] - fn parallel_launch_applies_canonical_timeout_to_name_and_request() { - futures::executor::block_on(async { - let stub = Arc::new(StubHttpClient::new()); - stub.push_response(200, b"{}".to_vec()); - let calls = Arc::new(Mutex::new(Vec::new())); - let services = build_services_with_backend_and_http_client( - Arc::new(CanonicalTimeoutBackend { - canonical_ms: 750, - calls: Arc::clone(&calls), - }), - stub, - ); - let predicted = Arc::new(Mutex::new(Vec::new())); - let requested = Arc::new(Mutex::new(Vec::new())); - let mut orchestrator = AuctionOrchestrator::new(AuctionConfig { - enabled: true, - providers: AuctionConfig::legacy_provider_map(&["bidder"]), - timeout_ms: 2000, - ..Default::default() - }); - orchestrator.register_provider(Arc::new(recording_provider( - "bidder", - "bidder-backend", - 1000, - &predicted, - &requested, - ))); - let settings = create_test_settings(); - let downstream = http::Request::new(edgezero_core::body::Body::empty()); - let context = immediate_test_context(&settings, &downstream, &services); + /// Mediator whose context-aware parse restores `nurl`/`ad_id` (mirroring + /// `adserver_mock`), while its context-free parse does not. Lets a test prove + /// the synchronous mediation path calls `parse_response_with_context`. + struct CacheRestoringMediator; - orchestrator - .run_auction(&create_test_auction_request(), &context) - .await - .expect("should complete auction"); - - assert_eq!(*predicted.lock().expect("should lock predicted"), vec![750]); - assert_eq!(*requested.lock().expect("should lock requested"), vec![750]); - let calls = calls.lock().expect("should lock calls"); - assert_eq!(calls.len(), 1); - assert_eq!(calls[0].1, 1000); - assert!(calls[0].0 > 0 && calls[0].0 <= 2000); + fn auction_bid(bidder: &str, price: f64) -> Bid { + let renderer = (bidder == "aps").then(|| { + BidRenderer::Aps(ApsRendererV1 { + version: 1, + account_id: "example-account".to_string(), + bid_id: "aps-selected-bid".to_string(), + creative_id: None, + tag_type: ApsTagType::Iframe, + creative_url: "https://creative.example/render".to_string(), + aax_response: "fictional-base64".to_string(), + width: 300, + height: 250, + }) }); + Bid { + slot_id: "slot-1".to_string(), + price: Some(price), + currency: "USD".to_string(), + creative: renderer + .is_none() + .then(|| "
ordinary
".to_string()), + adomain: None, + bidder: bidder.to_string(), + returned_seat: None, + width: 300, + height: 250, + nurl: None, + burl: None, + bid_id: (bidder == "aps").then(|| "aps-selected-bid".to_string()), + ad_id: None, + creative_id: None, + renderer, + cache_id: None, + cache_host: None, + cache_path: None, + metadata: HashMap::new(), + } } - #[test] - fn zero_canonical_timeout_skips_parallel_launch() { - futures::executor::block_on(async { - let calls = Arc::new(Mutex::new(Vec::new())); - let services = build_services_with_backend_and_http_client( - Arc::new(CanonicalTimeoutBackend { - canonical_ms: 0, - calls, - }), - Arc::new(StubHttpClient::new()), - ); - let predicted = Arc::new(Mutex::new(Vec::new())); - let requested = Arc::new(Mutex::new(Vec::new())); - let mut orchestrator = AuctionOrchestrator::new(AuctionConfig { - enabled: true, - providers: AuctionConfig::legacy_provider_map(&["bidder"]), - timeout_ms: 2000, - ..Default::default() - }); - orchestrator.register_provider(Arc::new(recording_provider( - "bidder", - "bidder-backend", - 1000, - &predicted, - &requested, - ))); - let settings = create_test_settings(); - let downstream = http::Request::new(edgezero_core::body::Body::empty()); - let context = immediate_test_context(&settings, &downstream, &services); + fn mediated_bid(nurl: Option) -> Bid { + Bid { + slot_id: "header-banner".to_string(), + price: Some(2.5), + currency: "USD".to_string(), + creative: Some("
ad
".to_string()), + adomain: None, + bidder: "mediator".to_string(), + returned_seat: None, + width: 728, + height: 90, + nurl: nurl.clone(), + burl: nurl, + bid_id: None, + ad_id: Some("creative-123".to_string()), + creative_id: None, + renderer: None, + cache_id: Some("cache-abc".to_string()), + cache_host: None, + cache_path: None, + metadata: HashMap::new(), + } + } - let result = orchestrator - .run_auction(&create_test_auction_request(), &context) - .await; + /// Endpoint the [`CacheRestoringMediator`] sends its mediation request to. + const CACHE_RESTORING_MEDIATOR: &str = "https://mediator.example.com/mediate"; - assert!(result.is_err(), "zero budget should skip every provider"); - assert!(predicted.lock().expect("should lock predicted").is_empty()); - assert!(requested.lock().expect("should lock requested").is_empty()); - }); - } + #[async_trait::async_trait(?Send)] + impl AuctionProvider for CacheRestoringMediator { + fn provider_name(&self) -> &str { + "mediator" + } - #[test] - fn synchronous_mediation_applies_canonical_timeout_to_mediator() { - futures::executor::block_on(async { - let stub = Arc::new(StubHttpClient::new()); - stub.push_response(200, b"{}".to_vec()); - stub.push_response(200, b"{}".to_vec()); - let calls = Arc::new(Mutex::new(Vec::new())); - let services = build_services_with_backend_and_http_client( - Arc::new(CanonicalTimeoutBackend { - canonical_ms: 500, - calls, - }), - stub, - ); - let predicted = Arc::new(Mutex::new(Vec::new())); - let requested = Arc::new(Mutex::new(Vec::new())); - let mut orchestrator = AuctionOrchestrator::new(AuctionConfig { - enabled: true, - providers: AuctionConfig::legacy_provider_map(&["bidder"]), - mediator: Some("mediator".to_string()), - timeout_ms: 2000, - ..Default::default() - }); - orchestrator.register_provider(Arc::new(StubAuctionProvider { - name: "bidder", - backend: "bidder-backend", - })); - orchestrator.register_provider(Arc::new(recording_provider( + async fn request_bids( + &self, + _request: &AuctionRequest, + _context: &AuctionContext<'_>, + ) -> Result> { + Ok(ProviderRequestOutcome::ready( + edgezero_core::outbound::OutboundRequest::post(CACHE_RESTORING_MEDIATOR) + .expect("should build the mediator request"), + )) + } + + async fn parse_response( + &self, + _response: OutboundResponse, + response_time_ms: u64, + ) -> Result> { + // Context-free path: cannot restore SSP-only render/accounting fields. + Ok(AuctionResponse::success( "mediator", - "mediator-backend", - 2000, - &predicted, - &requested, - ))); - let settings = create_test_settings(); - let downstream = http::Request::new(edgezero_core::body::Body::empty()); - let context = immediate_test_context(&settings, &downstream, &services); + vec![mediated_bid(None)], + response_time_ms, + )) + } - orchestrator - .run_auction(&create_test_auction_request(), &context) - .await - .expect("should complete mediated auction"); + async fn parse_response_with_context( + &self, + _response: OutboundResponse, + response_time_ms: u64, + _request: &AuctionRequest, + _context: &AuctionContext<'_>, + ) -> Result> { + // Context-aware path: restores nurl/ad_id from the collected SSP bids. + Ok(AuctionResponse::success( + "mediator", + vec![mediated_bid(Some("https://nurl.example/win".to_string()))], + response_time_ms, + )) + } - assert!(predicted.lock().expect("should lock predicted").is_empty()); - assert_eq!(*requested.lock().expect("should lock requested"), vec![500]); - }); + fn timeout_ms(&self) -> u32 { + 2000 + } } - #[test] - fn dispatched_collect_applies_canonical_timeout_to_both_paths() { - futures::executor::block_on(async { - let stub = Arc::new(StubHttpClient::new()); - stub.push_response(200, b"{}".to_vec()); - stub.push_response(200, b"{}".to_vec()); - let calls = Arc::new(Mutex::new(Vec::new())); - let services = build_services_with_backend_and_http_client( - Arc::new(CanonicalTimeoutBackend { - canonical_ms: 500, - calls, - }), - stub, - ); - let bidder_predicted = Arc::new(Mutex::new(Vec::new())); - let bidder_requested = Arc::new(Mutex::new(Vec::new())); - let mediator_predicted = Arc::new(Mutex::new(Vec::new())); - let mediator_requested = Arc::new(Mutex::new(Vec::new())); - let mut orchestrator = AuctionOrchestrator::new(AuctionConfig { - enabled: true, - providers: AuctionConfig::legacy_provider_map(&["bidder"]), - mediator: Some("mediator".to_string()), - timeout_ms: 2000, - ..Default::default() - }); - orchestrator.register_provider(Arc::new(recording_provider( - "bidder", - "bidder-backend", - 2000, - &bidder_predicted, - &bidder_requested, - ))); - orchestrator.register_provider(Arc::new(recording_provider( - "mediator", - "mediator-backend", - 2000, - &mediator_predicted, - &mediator_requested, - ))); - let settings = create_test_settings(); - let downstream = http::Request::new(edgezero_core::body::Body::empty()); - let context = immediate_test_context(&settings, &downstream, &services); - let request = create_test_auction_request(); + struct ImmediateMediator; - let DispatchAuctionOutcome::Dispatched(dispatched) = - orchestrator.dispatch_auction(&request, &context).await - else { - panic!("should dispatch bidder request"); - }; - orchestrator - .collect_dispatched_auction(dispatched, &services, &context) - .await; + #[async_trait::async_trait(?Send)] + impl AuctionProvider for ImmediateMediator { + fn provider_name(&self) -> &str { + "immediate-mediator" + } - assert_eq!( - *bidder_predicted.lock().expect("should lock predicted"), - vec![500] - ); - assert_eq!( - *bidder_requested.lock().expect("should lock requested"), - vec![500] - ); - assert!( - mediator_predicted - .lock() - .expect("should lock predicted") - .is_empty() - ); - assert_eq!( - *mediator_requested.lock().expect("should lock requested"), - vec![500] - ); - }); - } + async fn request_bids( + &self, + _request: &AuctionRequest, + _context: &AuctionContext<'_>, + ) -> Result> { + Ok(ProviderRequestOutcome::Immediate(AuctionResponse::success( + self.provider_name(), + vec![mediated_bid(Some( + "https://nurl.example/immediate".to_string(), + ))], + 0, + ))) + } - #[test] - fn planned_collect_skips_mediator_with_zero_canonical_transport_budget() { - futures::executor::block_on(async { - let calls = Arc::new(Mutex::new(Vec::new())); - let services = build_services_with_backend_and_http_client( - Arc::new(CanonicalTimeoutBackend { - canonical_ms: 0, - calls: Arc::clone(&calls), - }), - Arc::new(StubHttpClient::new()), - ); - let launches = Arc::new(AtomicUsize::new(0)); - let budgets = Arc::new(Mutex::new(Vec::new())); - let plan = AuctionPlan::compile(AuctionPlanConfig { - timeout_ms: 49, - providers: BTreeMap::new(), - bidders: BTreeMap::new(), - mediator: Some("adserver_mock".to_string()), - request_signing: None, - }) - .expect("should compile mediator-only plan") - .with_enabled(true); - let orchestrator = AuctionOrchestrator::from_plan( - Arc::new(plan), - Some(Arc::new(DeadlineRecordingMediator { - launches: Arc::clone(&launches), - budgets: Some(Arc::clone(&budgets)), - })), - ); - let request = create_test_auction_request(); - let settings = create_test_settings(); - let downstream = http::Request::new(edgezero_core::body::Body::empty()); - let context = AuctionContext { - settings: &settings, - request: &downstream, - timeout_ms: 49, - transport_timeout_ms: 49, - provider_responses: None, - services: &services, - }; - let dispatched = DispatchedAuction::empty_for_test(request, 49); + async fn parse_response( + &self, + _response: OutboundResponse, + _response_time_ms: u64, + ) -> Result> { + panic!("immediate mediator response should not be parsed"); + } - let result = orchestrator - .collect_dispatched_auction(dispatched, &services, &context) - .await; + fn timeout_ms(&self) -> u32 { + 2000 + } + } - assert_eq!(launches.load(Ordering::Relaxed), 0); - assert!( - budgets - .lock() - .expect("should lock mediator budgets") - .is_empty() - ); - assert_eq!(calls.lock().expect("should lock calls").len(), 1); - assert!(result.mediator_response.is_none()); - }); + fn create_test_settings() -> crate::settings::Settings { + let settings_str = crate_test_settings_str(); + crate::settings::Settings::from_toml(&settings_str).expect("should parse test settings") } - #[test] - fn dispatched_resolved_backend_name_diverging_from_prediction_still_correlates() { - futures::executor::block_on(async { - let stub = Arc::new(StubHttpClient::new()); - stub.push_response(200, b"{}".to_vec()); - let services = build_services_with_http_client(stub); - let mut orchestrator = AuctionOrchestrator::new(AuctionConfig { - enabled: true, - providers: AuctionConfig::legacy_provider_map(&["provider-a"]), - timeout_ms: 2000, - ..Default::default() - }); - orchestrator.register_provider(Arc::new(DivergentBackendProvider { - name: "provider-a", - predicted: "predicted-backend", - resolved: "resolved-backend", - })); - let settings = create_test_settings(); - let downstream = http::Request::new(edgezero_core::body::Body::empty()); - let context = immediate_test_context(&settings, &downstream, &services); - let request = create_test_auction_request(); + struct ImmediateNoBidProvider; - let DispatchAuctionOutcome::Dispatched(dispatched) = - orchestrator.dispatch_auction(&request, &context).await - else { - panic!("should dispatch provider"); - }; - let result = orchestrator - .collect_dispatched_auction(dispatched, &services, &context) - .await; + #[async_trait::async_trait(?Send)] + impl AuctionProvider for ImmediateNoBidProvider { + fn provider_name(&self) -> &str { + "immediate" + } - assert!(result.provider_responses.iter().any(|response| { - response.provider == "provider-a" && response.status == BidStatus::Success - })); - }); - } + async fn request_bids( + &self, + _request: &AuctionRequest, + _context: &AuctionContext<'_>, + ) -> Result> { + Ok(ProviderRequestOutcome::Immediate(AuctionResponse::no_bid( + "immediate", + 0, + ))) + } - #[test] - fn dispatched_post_launch_resolved_name_collision_fails_second_provider_attributably() { - futures::executor::block_on(async { - let stub = Arc::new(StubHttpClient::new()); - stub.push_response(200, b"{}".to_vec()); - stub.push_response(200, b"{}".to_vec()); - let services = build_services_with_http_client(stub); - let mut orchestrator = AuctionOrchestrator::new(AuctionConfig { - enabled: true, - providers: AuctionConfig::legacy_provider_map(&["provider-a", "provider-b"]), - timeout_ms: 2000, - ..Default::default() - }); - orchestrator.register_provider(Arc::new(DivergentBackendProvider { - name: "provider-a", - predicted: "predicted-a", - resolved: "shared-resolved", - })); - orchestrator.register_provider(Arc::new(DivergentBackendProvider { - name: "provider-b", - predicted: "predicted-b", - resolved: "shared-resolved", - })); - let settings = create_test_settings(); - let downstream = http::Request::new(edgezero_core::body::Body::empty()); - let context = immediate_test_context(&settings, &downstream, &services); - let request = create_test_auction_request(); + async fn parse_response( + &self, + _response: OutboundResponse, + _response_time_ms: u64, + ) -> Result> { + panic!("immediate response should not be parsed"); + } - let DispatchAuctionOutcome::Dispatched(dispatched) = - orchestrator.dispatch_auction(&request, &context).await - else { - panic!("should dispatch first provider"); - }; - let result = orchestrator - .collect_dispatched_auction(dispatched, &services, &context) - .await; - - assert!(result.provider_responses.iter().any(|response| { - response.provider == "provider-a" && response.status == BidStatus::Success - })); - assert!(result.provider_responses.iter().any(|response| { - response.provider == "provider-b" && response.status == BidStatus::Error - })); - }); + fn timeout_ms(&self) -> u32 { + 2000 + } } - #[test] - fn select_error_is_attributed_to_correct_provider() { - futures::executor::block_on(async { - // Arrange: two stub providers backed by distinct backend names. - // The stub HTTP client injects a select() error for the first request - // that completes (backend-a). backend-b should still produce a success. - let stub = Arc::new(StubHttpClient::new()); - stub.push_response(200, b"{}".to_vec()); // consumed by send_async for backend-a - stub.push_response(200, b"{}".to_vec()); // consumed by send_async for backend-b - stub.push_select_error(); // first select() reports backend-a as failed - - let services = build_services_with_http_client(stub); - // SAFETY: `Box::leak` creates a `'static` reference for test use only. - // The leaked allocation is bounded to the test process lifetime. - let services: &'static RuntimeServices = Box::leak(Box::new(services)); - - let config = AuctionConfig { - enabled: true, - providers: AuctionConfig::legacy_provider_map(&["provider-a", "provider-b"]), - timeout_ms: 2000, - mediator: None, - ..Default::default() - }; - let mut orchestrator = AuctionOrchestrator::new(config); - orchestrator.register_provider(Arc::new(StubAuctionProvider { - name: "provider-a", - backend: "backend-a", - })); - orchestrator.register_provider(Arc::new(StubAuctionProvider { - name: "provider-b", - backend: "backend-b", - })); - - let request = create_test_auction_request(); - let settings = create_test_settings(); - let req = http::Request::builder() - .method(http::Method::GET) - .uri("https://example.com/test") - .body(edgezero_core::body::Body::empty()) - .expect("should build request"); - let context = AuctionContext { - settings: &settings, - request: &req, - timeout_ms: 2000, - transport_timeout_ms: 2000, - provider_responses: None, - services, - }; + struct LaunchFailingProvider; - // Act - let result = orchestrator - .run_auction(&request, &context) - .await - .expect("should complete auction even when one provider errors"); + #[async_trait::async_trait(?Send)] + impl AuctionProvider for LaunchFailingProvider { + fn provider_name(&self) -> &str { + "launch-failing" + } - // Assert: exactly two responses — one error, one success. - assert_eq!( - result.provider_responses.len(), - 2, - "should collect responses from both providers" - ); + async fn request_bids( + &self, + _request: &AuctionRequest, + _context: &AuctionContext<'_>, + ) -> Result> { + Err(Report::new(TrustedServerError::Auction { + message: "launch failed in test provider".to_string(), + })) + } - let provider_a = result - .provider_responses - .iter() - .find(|r| r.provider == "provider-a") - .expect("should have provider-a response"); - let provider_b = result - .provider_responses - .iter() - .find(|r| r.provider == "provider-b") - .expect("should have provider-b response"); + async fn parse_response( + &self, + _response: OutboundResponse, + _response_time_ms: u64, + ) -> Result> { + Err(Report::new(TrustedServerError::Auction { + message: "launch-failing provider should not parse responses".to_string(), + })) + } - assert_eq!( - provider_a.status, - BidStatus::Error, - "provider-a should be marked error — select() Err was attributed via failed_backend_name" - ); - assert_eq!( - provider_b.status, - BidStatus::Success, - "provider-b should succeed — error was correctly isolated to provider-a" - ); - }); + fn timeout_ms(&self) -> u32 { + 2000 + } } - #[tokio::test] - async fn harness_outer_select_error_materializes_all_launches_as_transport_failures() { - let http = Arc::new(OuterSelectErrorHttpClient::new()); - http.push_response(204, Vec::new()); - let backend = Arc::new(NamingBackend::new(BackendNamingPolicy::Axum)); - let services = build_services_with_backend_and_http_client( - Arc::clone(&backend) as Arc<_>, - Arc::clone(&http) as Arc<_>, - ); - let plan = Arc::new( - AuctionPlan::compile(planned_config( - &[("provider-a", RoutingMode::AllEligible)], - false, - )) - .expect("should compile planned auction"), - ); - let harness = AuctionOrchestratorHarness::new(plan, None); - let request = planned_request(); - let settings = create_test_settings(); - let inbound = http::Request::new(edgezero_core::body::Body::empty()); - let context = AuctionContext { - settings: &settings, - request: &inbound, - timeout_ms: 777, - transport_timeout_ms: 777, - provider_responses: None, - services: &services, - }; + #[test] + fn provider_error_response_includes_diagnostic_metadata() { + let error = Report::new(TrustedServerError::Auction { + message: "parse failed".to_string(), + }) + .attach("internal/source.rs:12:34"); - let result = harness - .run_auction(&request, &context) - .await - .expect("should collect a harness auction after select failure"); + let response = + super::provider_error_response("prebid", 37, super::ERROR_TYPE_PARSE_RESPONSE, &error); assert_eq!( - result.provider_responses.len(), - 1, - "should report one provider response" - ); - assert_eq!( - result.provider_responses[0].status, + response.status, BidStatus::Error, - "outer select failure should report an error" + "should mark diagnostic provider responses as errors" ); assert_eq!( - result.provider_responses[0].metadata["error_type"], ERROR_TYPE_TRANSPORT, - "harness should match production transport classification" + response.metadata["error_type"], + serde_json::json!("parse_response"), + "should include the provider error classification" ); - } - #[tokio::test] - async fn planned_dispatch_does_not_snapshot_inbound_headers() { - let http = Arc::new(StubHttpClient::new()); - http.push_response(204, Vec::new()); - let backend = Arc::new(NamingBackend::new(BackendNamingPolicy::Axum)); - let services = build_services_with_backend_and_http_client( - Arc::clone(&backend) as Arc<_>, - Arc::clone(&http) as Arc<_>, - ); - let plan = Arc::new( - AuctionPlan::compile(planned_config( - &[("provider-a", RoutingMode::AllEligible)], - false, - )) - .expect("should compile planned auction"), + let message = response.metadata["message"] + .as_str() + .expect("should include provider error message"); + assert!( + message.contains("parse failed"), + "should include user-safe diagnostic detail" ); - let orchestrator = AuctionOrchestrator::from_plan(plan, None); - let request = planned_request(); - let settings = create_test_settings(); - let inbound = http::Request::builder() - .header(http::header::AUTHORIZATION, "Bearer should-not-be-retained") - .header(http::header::COOKIE, "session=should-not-be-retained") - .body(edgezero_core::body::Body::empty()) - .expect("should build inbound request"); - let context = AuctionContext { - settings: &settings, - request: &inbound, - timeout_ms: 777, - transport_timeout_ms: 777, - provider_responses: None, - services: &services, - }; - - let DispatchAuctionOutcome::Dispatched(dispatched) = - orchestrator.dispatch_auction(&request, &context).await - else { - panic!("should dispatch planned provider"); - }; - assert!( - dispatched.provider_request_context.headers().is_empty(), - "planned dispatch should retain an empty request context" + !message.contains("internal/source.rs"), + "should not include attached internal details" ); } - #[tokio::test] - async fn outer_select_error_materializes_all_planned_launches_as_transport_failures() { - let http = Arc::new(OuterSelectErrorHttpClient::new()); - http.push_response(204, Vec::new()); - http.push_response(204, Vec::new()); - let backend = Arc::new(NamingBackend::new(BackendNamingPolicy::Axum)); - let services = build_services_with_backend_and_http_client( - Arc::clone(&backend) as Arc<_>, - Arc::clone(&http) as Arc<_>, - ); - let plan = Arc::new( - AuctionPlan::compile(planned_config( - &[ - ("provider-b", RoutingMode::AllEligible), - ("provider-a", RoutingMode::AllEligible), - ], - false, - )) - .expect("should compile planned auction"), - ); - let orchestrator = AuctionOrchestrator::from_plan(plan, None); - let request = planned_request(); - let settings = create_test_settings(); - let inbound = http::Request::new(edgezero_core::body::Body::empty()); - let context = AuctionContext { - settings: &settings, - request: &inbound, - timeout_ms: 777, - transport_timeout_ms: 777, - provider_responses: None, - services: &services, - }; - - let DispatchAuctionOutcome::Dispatched(dispatched) = - orchestrator.dispatch_auction(&request, &context).await - else { - panic!("should dispatch both planned providers"); - }; - tokio::time::sleep(Duration::from_millis(5)).await; - let result = orchestrator - .collect_dispatched_auction(dispatched, &services, &context) - .await; + #[test] + fn launch_failed_response_has_safe_static_message() { + let response = super::provider_launch_failed_response("prebid", 58); - assert_eq!(http.selected_pending.load(Ordering::Relaxed), 2); assert_eq!( - result - .provider_responses - .iter() - .map(|response| response.provider.as_str()) - .collect::>(), - vec!["provider-a", "provider-b"], - "outer select errors should retain deterministic plan order" + response.status, + BidStatus::Error, + "should mark launch failures as errors" + ); + assert_eq!( + response.metadata["error_type"], + serde_json::json!("launch_failed"), + "should include launch_failed classification" + ); + assert_eq!( + response.metadata["message"], + serde_json::json!("Provider launch failed"), + "should use a safe, stable public launch failure message" ); - for response in &result.provider_responses { - assert_eq!(response.status, BidStatus::Error); - assert_eq!(response.metadata["error_type"], "transport"); - assert!( - response.response_time_ms >= 5, - "transport failure should preserve launch elapsed time" - ); - } - } - - #[test] - fn dispatched_collection_reuses_provider_launch_context() { - futures::executor::block_on(async { - let stub = Arc::new(StubHttpClient::new()); - stub.push_response(200, b"{}".to_vec()); - let services = build_services_with_http_client(stub); - let config = AuctionConfig { - enabled: true, - providers: AuctionConfig::legacy_provider_map(&["provider-a"]), - timeout_ms: 750, - mediator: None, - ..Default::default() - }; - let mut orchestrator = AuctionOrchestrator::new(config); - orchestrator.register_provider(Arc::new(StubAuctionProvider { - name: "provider-a", - backend: "backend-a", - })); - let request = create_test_auction_request(); - let settings = create_test_settings(); - let downstream = http::Request::builder() - .uri("https://publisher.example/article") - .header(http::header::REFERER, "https://referrer.example/source") - .body(edgezero_core::body::Body::empty()) - .expect("should build downstream request"); - let dispatch_context = AuctionContext { - settings: &settings, - request: &downstream, - timeout_ms: 750, - transport_timeout_ms: 750, - provider_responses: None, - services: &services, - }; - let dispatched = match orchestrator - .dispatch_auction(&request, &dispatch_context) - .await - { - DispatchAuctionOutcome::Dispatched(dispatched) => dispatched, - _ => panic!("should dispatch provider request"), - }; - let placeholder = http::Request::builder() - .uri("https://placeholder.invalid/") - .body(edgezero_core::body::Body::empty()) - .expect("should build placeholder request"); - let collect_context = AuctionContext { - settings: &settings, - request: &placeholder, - timeout_ms: 750, - transport_timeout_ms: 750, - provider_responses: None, - services: &services, - }; - - let result = orchestrator - .collect_dispatched_auction(dispatched, &services, &collect_context) - .await; - - let response = result - .provider_responses - .first() - .expect("should collect provider response"); - assert_eq!( - response.metadata["context_referer"], "https://referrer.example/source", - "should parse with the downstream request used at launch" - ); - assert_eq!( - response.metadata["context_timeout_ms"], 125, - "should parse with the provider-capped launch timeout" - ); - }); } #[test] - fn rejects_multi_provider_fanout_before_launch_on_sequential_platform() { - futures::executor::block_on(async { - // Arrange: two configured providers on a platform whose HTTP - // client is configured without multi-provider collection support. - let stub = Arc::new(StubHttpClient::new()); - stub.set_concurrent_fanout(false); - let stub_for_assertion = Arc::clone(&stub); - - let services = build_services_with_http_client(stub); - // SAFETY: `Box::leak` creates a `'static` reference for test use only. - // The leaked allocation is bounded to the test process lifetime. - let services: &'static RuntimeServices = Box::leak(Box::new(services)); - - let config = AuctionConfig { - enabled: true, - providers: AuctionConfig::legacy_provider_map(&["provider-a", "provider-b"]), - timeout_ms: 2000, - mediator: None, - ..Default::default() - }; - let mut orchestrator = AuctionOrchestrator::new(config); - orchestrator.register_provider(Arc::new(StubAuctionProvider { - name: "provider-a", - backend: "backend-a", - })); - orchestrator.register_provider(Arc::new(StubAuctionProvider { - name: "provider-b", - backend: "backend-b", - })); - - let request = create_test_auction_request(); - let settings = create_test_settings(); - let req = http::Request::builder() - .method(http::Method::GET) - .uri("https://example.com/test") - .body(edgezero_core::body::Body::empty()) - .expect("should build request"); - let context = AuctionContext { - settings: &settings, - request: &req, - timeout_ms: 2000, - transport_timeout_ms: 2000, - provider_responses: None, - services, - }; + fn transport_failed_response_has_safe_static_message() { + let response = super::provider_transport_failed_response("prebid", 64); - // Act - let result = orchestrator.run_auction(&request, &context).await; + assert_eq!( + response.status, + BidStatus::Error, + "should mark transport failures as errors" + ); + assert_eq!( + response.metadata["error_type"], + serde_json::json!("transport"), + "should classify transport failures consistently with other failure modes" + ); + assert_eq!( + response.metadata["message"], + serde_json::json!("Provider request failed"), + "should use a safe, stable public transport failure message" + ); + } - // Assert: rejected before any provider request launches. - let err = result.expect_err("should reject multi-provider fan-out"); - assert!( - format!("{err}").contains("sequentially"), - "should explain the sequential-execution limitation" - ); - assert!( - stub_for_assertion.recorded_backend_names().is_empty(), - "should not launch any provider request before rejecting" - ); + #[test] + fn provider_error_message_truncates_user_safe_context() { + let long_message = "x".repeat(super::PROVIDER_ERROR_MESSAGE_CHARS + 100); + let error = Report::new(TrustedServerError::Auction { + message: long_message, }); + + let message = super::provider_error_message(&error); + + assert_eq!( + message.chars().count(), + super::PROVIDER_ERROR_MESSAGE_CHARS, + "should cap provider error messages" + ); + assert!( + message.starts_with("Auction error: "), + "should preserve the current context display text" + ); } #[test] - fn dispatch_auction_skips_multi_provider_fanout_on_sequential_platform() { - futures::executor::block_on(async { - // Arrange: two configured providers on a platform whose HTTP - // client is configured without multi-provider collection support. - // The initial-page dispatch path must apply the same guard as - // run_providers_parallel or the summed provider latency lands - // before the origin fetch. - let stub = Arc::new(StubHttpClient::new()); - stub.set_concurrent_fanout(false); - let stub_for_assertion = Arc::clone(&stub); - - let services = build_services_with_http_client(stub); - // SAFETY: `Box::leak` creates a `'static` reference for test use only. - // The leaked allocation is bounded to the test process lifetime. - let services: &'static RuntimeServices = Box::leak(Box::new(services)); - - let config = AuctionConfig { - enabled: true, - providers: AuctionConfig::legacy_provider_map(&["provider-a", "provider-b"]), - timeout_ms: 2000, - mediator: None, - ..Default::default() - }; - let mut orchestrator = AuctionOrchestrator::new(config); - orchestrator.register_provider(Arc::new(StubAuctionProvider { - name: "provider-a", - backend: "backend-a", - })); - orchestrator.register_provider(Arc::new(StubAuctionProvider { - name: "provider-b", - backend: "backend-b", - })); - - let request = create_test_auction_request(); - let settings = create_test_settings(); - let req = http::Request::builder() - .method(http::Method::GET) - .uri("https://example.com/test") - .body(edgezero_core::body::Body::empty()) - .expect("should build request"); - let context = AuctionContext { - settings: &settings, - request: &req, - timeout_ms: 2000, - transport_timeout_ms: 2000, - provider_responses: None, - services, - }; + fn filters_winning_bids_below_floor() { + let orchestrator = empty_orchestrator(false); + let mut floor_prices = HashMap::new(); + floor_prices.insert("slot-1".to_string(), 1.00); + floor_prices.insert("slot-2".to_string(), 2.00); + + // Arrange winning bids with one below floor. + let mut winning_bids = HashMap::new(); + winning_bids.insert( + "slot-1".to_string(), + Bid { + slot_id: "slot-1".to_string(), + price: Some(0.50), + currency: "USD".to_string(), + creative: Some("
Ad
".to_string()), + adomain: None, + bidder: "test-bidder".to_string(), + returned_seat: None, + width: 300, + height: 250, + nurl: None, + burl: None, + bid_id: None, + ad_id: None, + creative_id: None, + renderer: None, + cache_id: None, + cache_host: None, + cache_path: None, + metadata: HashMap::new(), + }, + ); + winning_bids.insert( + "slot-2".to_string(), + Bid { + slot_id: "slot-2".to_string(), + price: Some(2.00), + currency: "USD".to_string(), + creative: Some("
Ad
".to_string()), + adomain: None, + bidder: "test-bidder".to_string(), + returned_seat: None, + width: 300, + height: 250, + nurl: None, + burl: None, + bid_id: None, + ad_id: None, + creative_id: None, + renderer: None, + cache_id: None, + cache_host: None, + cache_path: None, + metadata: HashMap::new(), + }, + ); - // Act - let dispatched = orchestrator.dispatch_auction(&request, &context).await; + // Apply floor pricing and validate the results. + let filtered = orchestrator.apply_floor_prices(winning_bids, &floor_prices); - // Assert: no dispatch and no provider request launched. - assert!( - matches!(dispatched, DispatchAuctionOutcome::NotStarted), - "should skip initial-page dispatch on sequential platforms" - ); - assert!( - stub_for_assertion.recorded_backend_names().is_empty(), - "should not launch any provider request on a sequential platform" - ); - }); + assert_eq!( + filtered.len(), + 1, + "Filtered bids should keep only those meeting floor price" + ); + assert!( + filtered.contains_key("slot-2"), + "Filtered bids should include slot-2 winner" + ); + } + + #[test] + fn test_orchestrator_is_enabled() { + assert!(empty_orchestrator(true).is_enabled()); + assert!(!empty_orchestrator(false).is_enabled()); } #[tokio::test] async fn from_plan_standard_provider_runs_direct_and_split_with_correlation_and_metadata() { for split in [false, true] { - let http = Arc::new(StubHttpClient::new()); - http.push_response( + let clock = TestClock::new(); + let http = ScriptedHttpClient::new(&clock); + respond_planned( + &http, 200, serde_json::to_vec(&serde_json::json!({ "seatbid": [{"seat": "provider-seat", "bid": [{ @@ -6605,11 +3795,7 @@ mod tests { })) .expect("should serialize provider response"), ); - let backend = Arc::new(NamingBackend::new(BackendNamingPolicy::Axum)); - let services = build_services_with_backend_and_http_client( - Arc::clone(&backend) as Arc<_>, - Arc::clone(&http) as Arc<_>, - ); + let services = build_services_with_outbound_client(&http, &clock); let mut config = planned_config(&[("provider-a", RoutingMode::AllEligible)], false); config.bidders.insert( "routed-bidder" @@ -6638,7 +3824,6 @@ mod tests { settings: &settings, request: &inbound, timeout_ms: 777, - transport_timeout_ms: 777, provider_responses: None, services: &services, }; @@ -6649,332 +3834,41 @@ mod tests { else { panic!("standard provider should dispatch"); }; - orchestrator - .collect_dispatched_auction(dispatched, &services, &context) - .await - } else { - orchestrator - .run_auction(&request, &context) - .await - .expect("standard provider should run") - }; - - assert_eq!(http.recorded_backend_names().len(), 1); - assert_eq!(backend.ensured.load(Ordering::Relaxed), 1); - assert_eq!(result.provider_responses.len(), 1); - assert_eq!(result.provider_responses[0].provider, "provider-a"); - assert_eq!(result.provider_responses[0].status, BidStatus::Success); - assert_eq!( - result.provider_responses[0].metadata["routing"]["unused_bidder_params_count"], - 1 - ); - assert_eq!(result.metadata["routing"]["unroutable_bidder_count"], 1); - assert_eq!( - result.winning_bids["fictional-slot"].bid_id.as_deref(), - Some("provider-bid") - ); - let metadata = - serde_json::to_string(&result.metadata).expect("should serialize auction metadata"); - assert!(!metadata.contains("unknown-private-id") && !metadata.contains("secret")); - } - } - - #[tokio::test] - async fn planned_executor_invokes_immediate_mediator_and_applies_floor() { - let http = Arc::new(StubHttpClient::new()); - http.push_response( - 200, - serde_json::to_vec(&serde_json::json!({ - "seatbid": [{"seat": "provider-seat", "bid": [{ - "id": "provider", "impid": "fictional-slot", "price": 2.0, - "adm": "
provider
", "w": 300, "h": 250 - }]}] - })) - .expect("should serialize provider response"), - ); - let backend = Arc::new(NamingBackend::new(BackendNamingPolicy::Axum)); - let services = build_services_with_backend_and_http_client( - Arc::clone(&backend) as Arc<_>, - Arc::clone(&http) as Arc<_>, - ); - let plan = AuctionPlan::compile(planned_config( - &[("provider-a", RoutingMode::AllEligible)], - false, - )) - .expect("should compile planned auction"); - let orchestrator = AuctionOrchestratorHarness::new(plan, Some(Arc::new(ImmediateMediator))); - let request = planned_request(); - let settings = create_test_settings(); - let inbound = http::Request::new(edgezero_core::body::Body::empty()); - let context = AuctionContext { - settings: &settings, - request: &inbound, - timeout_ms: 777, - transport_timeout_ms: 777, - provider_responses: None, - services: &services, - }; - - let result = orchestrator - .run_auction(&request, &context) - .await - .expect("should execute planned mediation"); - - assert_eq!( - result - .mediator_response - .as_ref() - .map(|response| response.provider.as_str()), - Some("immediate-mediator") - ); - assert_eq!( - result.winning_bids["header-banner"].nurl.as_deref(), - Some("https://nurl.example/immediate") - ); - assert!( - !result.winning_bids.contains_key("fictional-slot"), - "mediator output owns final selection" - ); - } - - #[tokio::test] - async fn planned_collection_preserves_on_time_bid_and_times_out_pending_sibling() { - for split in [false, true] { - let http = Arc::new(StubHttpClient::new()); - http.push_response( - 200, - serde_json::to_vec(&serde_json::json!({ - "seatbid": [{"seat": "example", "bid": [{ - "id":"on-time", "impid":"fictional-slot", "price":2.0, - "adm":"
fictional
", "w":300, "h":250 - }]}] - })) - .expect("should serialize on-time bid"), - ); - http.push_response(204, Vec::new()); - http.push_select_delay(Duration::ZERO); - http.push_select_delay(Duration::from_millis(250)); - let services = build_services_with_backend_and_http_client( - Arc::new(NamingBackend::new(BackendNamingPolicy::Axum)), - Arc::clone(&http) as Arc<_>, - ); - let plan = AuctionPlan::compile(planned_config( - &[ - ("provider-a", RoutingMode::AllEligible), - ("provider-b", RoutingMode::AllEligible), - ], - false, - )) - .expect("should compile two providers"); - let orchestrator = AuctionOrchestrator::from_plan(Arc::new(plan), None); - let request = planned_request(); - let settings = create_test_settings(); - let inbound = http::Request::new(edgezero_core::body::Body::empty()); - let context = AuctionContext { - settings: &settings, - request: &inbound, - timeout_ms: 30, - transport_timeout_ms: 30, - provider_responses: None, - services: &services, - }; - let result = if split { - let DispatchAuctionOutcome::Dispatched(dispatched) = - orchestrator.dispatch_auction(&request, &context).await - else { - panic!("should dispatch providers"); - }; - orchestrator - .collect_dispatched_auction(dispatched, &services, &context) - .await - } else { - orchestrator - .run_auction(&request, &context) - .await - .expect("should collect partial results") - }; - assert!( - http.recorded_cache_intents() - .iter() - .all(crate::platform::PlatformCacheIntent::is_bypass), - "should explicitly bypass intermediary caches for personalized auctions" - ); - assert_eq!( - result.provider_responses.len(), - 2, - "should retain one result per provider" - ); - assert_eq!( - result.provider_responses[0].provider, "provider-a", - "should retain configured ordering" - ); - assert_eq!( - result.provider_responses[1].provider, "provider-b", - "should attribute timeout to outstanding sibling" - ); + orchestrator + .collect_dispatched_auction(dispatched, &services, &context) + .await + } else { + orchestrator + .run_auction(&request, &context) + .await + .expect("standard provider should run") + }; + + assert_eq!(recorded_uris(&http).len(), 1); + assert_eq!(result.provider_responses.len(), 1); + assert_eq!(result.provider_responses[0].provider, "provider-a"); + assert_eq!(result.provider_responses[0].status, BidStatus::Success); assert_eq!( - result.provider_responses[1].metadata["error_type"], ERROR_TYPE_TIMEOUT, - "should report deadline expiry as timeout" + result.provider_responses[0].metadata["routing"]["unused_bidder_params_count"], + 1 ); + assert_eq!(result.metadata["routing"]["unroutable_bidder_count"], 1); assert_eq!( result.winning_bids["fictional-slot"].bid_id.as_deref(), - Some("on-time"), - "should preserve the collected winner" - ); - assert!( - result.total_time_ms < 200, - "should not drain the delayed provider" + Some("provider-bid") ); + let metadata = + serde_json::to_string(&result.metadata).expect("should serialize auction metadata"); + assert!(!metadata.contains("unknown-private-id") && !metadata.contains("secret")); } } #[tokio::test] - async fn planned_split_collection_drains_ready_bid_after_slow_origin_without_mediation() { - let http = Arc::new(StubHttpClient::new()); - http.push_response( - 200, - serde_json::to_vec(&serde_json::json!({ - "seatbid": [{"seat": "example", "bid": [{ - "id": "on-time", "impid": "fictional-slot", "price": 2.0, - "adm": "
fictional
", "w": 300, "h": 250 - }]}] - })) - .expect("should serialize buffered bid"), - ); - http.push_response(204, Vec::new()); - http.push_select_delay(Duration::ZERO); - // Shorter than the 10 ms budget, so a collection clock restarted after - // the origin wait would collect this 204 instead of timing it out. - http.push_select_delay(Duration::from_millis(1)); - let services = build_services_with_backend_and_http_client( - Arc::new(NamingBackend::new(BackendNamingPolicy::Axum)), - Arc::clone(&http) as Arc<_>, - ); - let plan = AuctionPlan::compile(planned_config( - &[ - ("provider-a", RoutingMode::AllEligible), - ("provider-b", RoutingMode::AllEligible), - ], - false, - )) - .expect("should compile providers"); - let launches = Arc::new(AtomicUsize::new(0)); - let mediator = Arc::new(DeadlineRecordingMediator { - launches: Arc::clone(&launches), - budgets: None, - }); - let orchestrator = AuctionOrchestrator::from_plan(Arc::new(plan), Some(mediator)); - let request = planned_request(); - let settings = create_test_settings(); - let inbound = http::Request::new(edgezero_core::body::Body::empty()); - let context = AuctionContext { - settings: &settings, - request: &inbound, - timeout_ms: 10, - transport_timeout_ms: 10, - provider_responses: None, - services: &services, - }; - let DispatchAuctionOutcome::Dispatched(dispatched) = - orchestrator.dispatch_auction(&request, &context).await - else { - panic!("should dispatch providers"); - }; - std::thread::sleep(Duration::from_millis(25)); - let started = Instant::now(); - let result = orchestrator - .collect_dispatched_auction(dispatched, &services, &context) - .await; - assert_eq!( - result.provider_responses.len(), - 2, - "should retain one result for every provider" - ); - assert_eq!( - result - .winning_bids - .get("fictional-slot") - .and_then(|bid| bid.bid_id.as_deref()), - Some("on-time"), - "should retain the available bid after the origin wait" - ); - assert_eq!( - result.provider_responses[1].metadata["error_type"], ERROR_TYPE_TIMEOUT, - "should time out the unfinished sibling" - ); - assert!( - result - .provider_responses - .iter() - .all(|response| response.response_time_ms >= 10), - "should report elapsed dispatch-to-collection time" - ); - assert_eq!( - launches.load(Ordering::Relaxed), - 0, - "should skip mediation after budget exhaustion" - ); - assert!( - started.elapsed() < Duration::from_millis(100), - "should never wait again for an unfinished response" - ); - } - - #[tokio::test] - async fn planned_split_collection_uses_dispatch_deadline_after_slow_origin() { - let http = Arc::new(StubHttpClient::new()); - http.push_response(204, Vec::new()); - // Shorter than the 10 ms budget, so a collection clock restarted after - // the origin wait would collect this 204 instead of timing it out. - http.push_select_delay(Duration::from_millis(1)); - let services = build_services_with_backend_and_http_client( - Arc::new(NamingBackend::new(BackendNamingPolicy::Axum)), - http, - ); - let plan = AuctionPlan::compile(planned_config( - &[("provider-a", RoutingMode::AllEligible)], - false, - )) - .expect("should compile provider"); - let orchestrator = AuctionOrchestrator::from_plan(Arc::new(plan), None); - let request = planned_request(); - let settings = create_test_settings(); - let inbound = http::Request::new(edgezero_core::body::Body::empty()); - let context = AuctionContext { - settings: &settings, - request: &inbound, - timeout_ms: 10, - transport_timeout_ms: 10, - provider_responses: None, - services: &services, - }; - let DispatchAuctionOutcome::Dispatched(dispatched) = - orchestrator.dispatch_auction(&request, &context).await - else { - panic!("should dispatch provider"); - }; - std::thread::sleep(Duration::from_millis(25)); - let started = Instant::now(); - let result = orchestrator - .collect_dispatched_auction(dispatched, &services, &context) - .await; - assert_eq!( - result.provider_responses[0].metadata["error_type"], ERROR_TYPE_TIMEOUT, - "should not restart the auction clock at collection" - ); - assert!( - started.elapsed() < Duration::from_millis(100), - "should return without waiting again" - ); - } - - async fn planned_pending_mediator_deadline_result( - auction_timeout_ms: u32, - mediator_timeout_ms: u32, - policy: BackendNamingPolicy, - ) -> OrchestrationResult { - let http = Arc::new(StubHttpClient::new()); - http.push_response( + async fn planned_executor_invokes_immediate_mediator_and_applies_floor() { + let clock = TestClock::new(); + let http = ScriptedHttpClient::new(&clock); + respond_planned( + &http, 200, serde_json::to_vec(&serde_json::json!({ "seatbid": [{"seat": "provider-seat", "bid": [{ @@ -6984,32 +3878,20 @@ mod tests { })) .expect("should serialize provider response"), ); - http.push_response(200, b"{}".to_vec()); - http.push_select_delay(Duration::ZERO); - http.push_wait_delay(Duration::from_millis(200)); - let backend = Arc::new(NamingBackend::new(policy)); - let services = build_services_with_backend_and_http_client( - Arc::clone(&backend) as Arc<_>, - Arc::clone(&http) as Arc<_>, - ); + let services = build_services_with_outbound_client(&http, &clock); let plan = AuctionPlan::compile(planned_config( &[("provider-a", RoutingMode::AllEligible)], false, )) .expect("should compile planned auction"); - let orchestrator = AuctionOrchestrator::from_plan( - Arc::new(plan), - Some(Arc::new(PendingDeadlineMediator(mediator_timeout_ms))), - ); + let orchestrator = planned_orchestrator(plan, Some(Arc::new(ImmediateMediator))); let request = planned_request(); let settings = create_test_settings(); let inbound = http::Request::new(edgezero_core::body::Body::empty()); let context = AuctionContext { settings: &settings, request: &inbound, - timeout_ms: auction_timeout_ms, - transport_timeout_ms: policy - .canonicalize_transport_timeout_ms(auction_timeout_ms, 1_000), + timeout_ms: 777, provider_responses: None, services: &services, }; @@ -7017,206 +3899,64 @@ mod tests { let result = orchestrator .run_auction(&request, &context) .await - .expect("should execute planned pending mediator"); - assert_eq!( - http.recorded_wait_count(), - 0, - "should skip the delayed unbounded mediator wait" - ); - result - } + .expect("should execute planned mediation"); - #[tokio::test] - async fn planned_pending_mediator_observes_absolute_deadline() { - let result = - planned_pending_mediator_deadline_result(20, 1_000, BackendNamingPolicy::Axum).await; - assert!( - result.mediator_response.is_none(), - "should stop waiting for mediator" + assert_eq!( + result + .mediator_response + .as_ref() + .map(|response| response.provider.as_str()), + Some("immediate-mediator") ); assert_eq!( - result.winning_bids["fictional-slot"].bid_id.as_deref(), - Some("provider"), - "should preserve on-time provider bids" + result.winning_bids["header-banner"].nurl.as_deref(), + Some("https://nurl.example/immediate") ); - } - - #[tokio::test] - async fn planned_pending_mediator_observes_own_budget_before_auction_deadline() { - let result = - planned_pending_mediator_deadline_result(1_000, 20, BackendNamingPolicy::Fastly).await; assert!( - result.mediator_response.is_none(), - "should stop at the mediator budget before the auction deadline" - ); - assert_eq!( - result.winning_bids["fictional-slot"].bid_id.as_deref(), - Some("provider"), - "should preserve on-time provider bids" + !result.winning_bids.contains_key("fictional-slot"), + "mediator output owns final selection" ); } - #[tokio::test] - async fn mediator_own_deadline_rejects_ready_results_after_launch_or_wait() { - for planned in [false, true] { - for split in [false, true] { - for (immediate, slow_launch) in [(false, true), (true, true), (false, false)] { - let http = Arc::new(StubHttpClient::new()); - http.push_response( - 200, - serde_json::to_vec(&serde_json::json!({ - "seatbid": [{"seat": "provider-seat", "bid": [{ - "id": "provider", "impid": "fictional-slot", "price": 2.0, - "adm": "
provider
", "w": 300, "h": 250 - }]}] - })) - .expect("should serialize provider response"), - ); - if !immediate { - http.push_response(200, b"{}".to_vec()); - } - let waits = Arc::new(AtomicUsize::new(0)); - let client: Arc = if slow_launch { - http - } else { - Arc::new(LateReadyMediatorHttpClient { - inner: http, - waits: Arc::clone(&waits), - }) - }; - let services = build_services_with_backend_and_http_client( - Arc::new(NamingBackend::new(BackendNamingPolicy::Axum)), - client, - ); - let parses = Arc::new(AtomicUsize::new(0)); - let mediator = Arc::new(SlowLaunchMediator { - immediate, - slow_launch, - parses: Arc::clone(&parses), - }); - let orchestrator = if planned { - let plan = AuctionPlan::compile(planned_config( - &[("provider-a", RoutingMode::AllEligible)], - false, - )) - .expect("should compile planned auction"); - AuctionOrchestrator::from_plan(Arc::new(plan), Some(mediator)) - } else { - let mut orchestrator = AuctionOrchestrator::new(AuctionConfig { - enabled: true, - providers: AuctionConfig::legacy_provider_map(&["local"]), - mediator: Some("pending-deadline-mediator".to_string()), - timeout_ms: 1_000, - ..Default::default() - }); - orchestrator.register_provider(Arc::new(DeadlineBidProvider { - name: "local", - backend: "local-backend", - })); - orchestrator.register_provider(mediator); - orchestrator - }; - let request = if planned { - planned_request() - } else { - create_test_auction_request() - }; - let settings = create_test_settings(); - let inbound = http::Request::new(edgezero_core::body::Body::empty()); - let context = AuctionContext { - settings: &settings, - request: &inbound, - timeout_ms: 1_000, - transport_timeout_ms: 1_000, - provider_responses: None, - services: &services, - }; - let result = if split { - let DispatchAuctionOutcome::Dispatched(dispatched) = - orchestrator.dispatch_auction(&request, &context).await - else { - panic!("should dispatch provider"); - }; - orchestrator - .collect_dispatched_auction(dispatched, &services, &context) - .await - } else { - orchestrator - .run_auction(&request, &context) - .await - .expect("should complete auction") - }; - assert_eq!( - result.provider_responses.len(), - 1, - "should retain the on-time provider result" - ); - assert!( - result.mediator_response.is_none(), - "should reject expired mediator: planned={planned} split={split} immediate={immediate}" - ); - assert_eq!( - parses.load(Ordering::Relaxed), - 0, - "should reject expiry before parsing" - ); - assert_eq!( - waits.load(Ordering::Relaxed), - usize::from(!slow_launch), - "should exercise the intended launch or wait boundary" - ); - assert!( - result.total_time_ms >= 40, - "should report elapsed auction time after mediator expiry" - ); - let slot = if planned { "fictional-slot" } else { "slot-1" }; - assert_eq!( - result.winning_bids[slot].price, - Some(if planned { 2.0 } else { 3.0 }), - "should preserve the local winner" - ); - } - } - } - } + /// Endpoint of the [`AdServerMockProvider`] mediator in the fallback tests. + const MOCK_MEDIATOR_ENDPOINT: &str = "https://mediator.example/mediate"; async fn run_mock_mediator_response( - harness: bool, status: u16, body: &serde_json::Value, floor: f64, ) -> (AuctionRequest, OrchestrationResult) { let body = serde_json::to_vec(body).expect("should serialize mediator response"); - run_raw_mock_mediator_response(harness, status, body, floor).await + run_raw_mock_mediator_response(status, body, floor).await } async fn run_raw_mock_mediator_response( - harness: bool, status: u16, body: Vec, floor: f64, ) -> (AuctionRequest, OrchestrationResult) { let mediator = Arc::new(AdServerMockProvider::new(AdServerMockConfig { enabled: true, - endpoint: "https://mediator.example/mediate".to_string(), + endpoint: MOCK_MEDIATOR_ENDPOINT.to_string(), timeout_ms: 500, ..AdServerMockConfig::default() })); - run_mediated_provider_auction(harness, mediator, Some((status, body)), floor).await + run_mediated_provider_auction(mediator, Some((status, body)), floor).await } /// Runs a planned auction whose single provider returns bids at 1.5 and 2.0. /// - /// `mediator_http` is the stubbed mediator HTTP reply, or `None` for a + /// `mediator_http` is the scripted mediator HTTP reply, or `None` for a /// mediator that answers without HTTP. async fn run_mediated_provider_auction( - harness: bool, mediator: Arc, mediator_http: Option<(u16, Vec)>, floor: f64, ) -> (AuctionRequest, OrchestrationResult) { - let http = Arc::new(StubHttpClient::new()); - http.push_response( + let clock = TestClock::new(); + let client = ScriptedHttpClient::new(&clock); + respond_planned( + &client, 200, serde_json::to_vec(&serde_json::json!({ "seatbid": [{"seat": "provider-seat", "bid": [ @@ -7229,13 +3969,9 @@ mod tests { .expect("should serialize provider bids"), ); if let Some((status, body)) = mediator_http { - http.push_response(status, body); + respond(&client, MOCK_MEDIATOR_ENDPOINT, status, body); } - let backend = Arc::new(NamingBackend::new(BackendNamingPolicy::Axum)); - let services = build_services_with_backend_and_http_client( - Arc::clone(&backend) as Arc<_>, - Arc::clone(&http) as Arc<_>, - ); + let services = build_services_with_outbound_client(&client, &clock); let plan = AuctionPlan::compile(planned_config( &[("provider-a", RoutingMode::AllEligible)], false, @@ -7249,35 +3985,25 @@ mod tests { settings: &settings, request: &inbound, timeout_ms: 777, - transport_timeout_ms: 777, provider_responses: None, services: &services, }; - let result = if harness { - AuctionOrchestratorHarness::new(plan, Some(mediator)) - .run_auction(&request, &context) - .await - } else { - AuctionOrchestrator::from_plan(Arc::new(plan), Some(mediator)) - .run_auction(&request, &context) - .await - } - .expect("should complete auction with mock mediation"); + let result = planned_orchestrator(plan, Some(mediator)) + .run_auction(&request, &context) + .await + .expect("should complete auction with mock mediation"); (request, result) } - async fn assert_mediator_non_success_falls_back(harness: bool) { + #[tokio::test] + async fn mediator_non_success_status_falls_back_to_local_ranking() { for status in [302, 400, 503] { for floor in [1.0, 3.0] { - let (request, result) = run_mock_mediator_response( - harness, - status, - &serde_json::json!({"seatbid": []}), - floor, - ) - .await; + let (request, result) = + run_mock_mediator_response(status, &serde_json::json!({"seatbid": []}), floor) + .await; let expected_winner = if floor < 2.0 { Some("higher") } else { None }; assert_eq!( result @@ -7285,7 +4011,7 @@ mod tests { .get("fictional-slot") .and_then(|bid| bid.bid_id.as_deref()), expected_winner, - "should rank provider bids locally with floors after HTTP {status} (harness={harness})" + "should rank provider bids locally with floors after HTTP {status}" ); assert_eq!( result.provider_responses[0].bids.len(), @@ -7339,68 +4065,75 @@ mod tests { (Some(1.5), Some(0)), (Some(2.0), Some(u8::from(expected_winner.is_some()))), ], - "should mark only the selected provider bid as winning after HTTP {status} (harness={harness})" + "should mark only the selected provider bid as winning after HTTP {status}" ); } } } #[tokio::test] - async fn mediator_non_success_status_falls_back_to_local_ranking() { - assert_mediator_non_success_falls_back(false).await; - } - - #[tokio::test] - async fn planned_executor_mediator_non_success_status_falls_back_locally() { - assert_mediator_non_success_falls_back(true).await; + async fn mock_mediator_non_success_status_is_a_processing_failure() { + let warnings = crate::test_support::logging::WarningCapture::start(); + run_mock_mediator_response(503, &serde_json::json!({"seatbid": []}), 1.0).await; + let warnings = warnings.take(); + assert!( + warnings.iter().any(|warning| warning.message.starts_with( + "Mediator 'adserver_mock' response processing failed; using local ranking:" + ) && warning + .message + .contains("AdServer Mock returned non-success HTTP 503")), + "should fail mediator response processing on a non-success HTTP status" + ); + assert!( + !warnings + .iter() + .any(|warning| warning.message.contains("reported an error response")), + "should not report a non-success HTTP status as an error-status mediation response" + ); } #[tokio::test] async fn mock_mediator_success_and_no_bid_remain_authoritative() { - for harness in [false, true] { - for has_winner in [false, true] { - let body = if has_winner { - serde_json::json!({"seatbid": [{"seat": "provider-a", "bid": [{ - "id": "mediated", "impid": "fictional-slot", "price": 1.75, - "adm": "
mediated
", "w": 300, "h": 250 - }]}]}) - } else { - serde_json::json!({"seatbid": []}) - }; - let (_, result) = run_mock_mediator_response(harness, 200, &body, 1.0).await; - assert!( - result.mediator_response.is_some(), - "should preserve a valid mediation result, including no-bid" - ); - assert_eq!( - result - .winning_bids - .get("fictional-slot") - .and_then(|bid| bid.bid_id.as_deref()), - if has_winner { Some("mediated") } else { None }, - "should honor the mediator decision instead of the higher local bid" - ); - } + for has_winner in [false, true] { + let body = if has_winner { + serde_json::json!({"seatbid": [{"seat": "provider-a", "bid": [{ + "id": "mediated", "impid": "fictional-slot", "price": 1.75, + "adm": "
mediated
", "w": 300, "h": 250 + }]}]}) + } else { + serde_json::json!({"seatbid": []}) + }; + let (_, result) = run_mock_mediator_response(200, &body, 1.0).await; + assert!( + result.mediator_response.is_some(), + "should preserve a valid mediation result, including no-bid" + ); + assert_eq!( + result + .winning_bids + .get("fictional-slot") + .and_then(|bid| bid.bid_id.as_deref()), + if has_winner { Some("mediated") } else { None }, + "should honor the mediator decision instead of the higher local bid" + ); } } #[tokio::test] async fn mock_mediator_no_content_is_a_final_no_bid() { - for harness in [false, true] { - let (_, result) = run_raw_mock_mediator_response(harness, 204, Vec::new(), 1.0).await; - assert_eq!( - result - .mediator_response - .as_ref() - .map(|response| response.status.clone()), - Some(BidStatus::NoBid), - "should record a bodiless HTTP 204 as a mediation no-bid (harness={harness})" - ); - assert!( - result.winning_bids.is_empty(), - "should keep the mediator's no-bid instead of ranking provider bids (harness={harness})" - ); - } + let (_, result) = run_raw_mock_mediator_response(204, Vec::new(), 1.0).await; + assert_eq!( + result + .mediator_response + .as_ref() + .map(|response| response.status.clone()), + Some(BidStatus::NoBid), + "should record a bodiless HTTP 204 as a mediation no-bid" + ); + assert!( + result.winning_bids.is_empty(), + "should keep the mediator's no-bid instead of ranking provider bids" + ); } struct ErrorStatusMediator; @@ -7421,75 +4154,57 @@ mod tests { 3, ))) } - - async fn parse_response( - &self, - _response: PlatformResponse, - _response_time_ms: u64, - ) -> Result> { - panic!("error-status mediator response should not be parsed"); - } - - fn timeout_ms(&self) -> u32 { - 500 - } - } - - #[tokio::test] - async fn mediator_error_status_response_falls_back_to_local_ranking() { - for harness in [false, true] { - for floor in [1.0, 3.0] { - let (_, result) = run_mediated_provider_auction( - harness, - Arc::new(ErrorStatusMediator), - None, - floor, - ) - .await; - assert!( - result.mediator_response.is_none(), - "should treat an error-status mediator response as failed mediation (harness={harness})" - ); - assert_eq!( - result - .winning_bids - .get("fictional-slot") - .and_then(|bid| bid.bid_id.as_deref()), - if floor < 2.0 { Some("higher") } else { None }, - "should rank provider bids locally with floors (harness={harness}, floor={floor})" - ); - } + + async fn parse_response( + &self, + _response: OutboundResponse, + _response_time_ms: u64, + ) -> Result> { + panic!("error-status mediator response should not be parsed"); + } + + fn timeout_ms(&self) -> u32 { + 500 } } #[tokio::test] - async fn mediator_error_status_response_logs_local_ranking_fallback() { - for (harness, expected) in [ - ( - false, - "Mediator 'error-status-mediator' reported an error response; using local ranking", - ), - ( - true, - "Planned mediator 'error-status-mediator' reported an error response; using local ranking", - ), - ] { - let warnings = crate::test_support::logging::WarningCapture::start(); - run_mediated_provider_auction(harness, Arc::new(ErrorStatusMediator), None, 1.0).await; + async fn mediator_error_status_response_falls_back_to_local_ranking() { + for floor in [1.0, 3.0] { + let (_, result) = + run_mediated_provider_auction(Arc::new(ErrorStatusMediator), None, floor).await; assert!( - warnings - .take() - .iter() - .any(|warning| warning.message == expected), - "should warn when an error-status mediator response falls back to local ranking (harness={harness})" + result.mediator_response.is_none(), + "should treat an error-status mediator response as failed mediation" + ); + assert_eq!( + result + .winning_bids + .get("fictional-slot") + .and_then(|bid| bid.bid_id.as_deref()), + if floor < 2.0 { Some("higher") } else { None }, + "should rank provider bids locally with floors (floor={floor})" ); } } + #[tokio::test] + async fn mediator_error_status_response_logs_local_ranking_fallback() { + let warnings = crate::test_support::logging::WarningCapture::start(); + run_mediated_provider_auction(Arc::new(ErrorStatusMediator), None, 1.0).await; + assert!( + warnings.take().iter().any(|warning| warning.message + == "Mediator 'error-status-mediator' reported an error response; using local ranking"), + "should warn when an error-status mediator response falls back to local ranking" + ); + } + #[tokio::test] async fn planned_executor_mediator_transport_failure_falls_back_locally() { - let http = Arc::new(StubHttpClient::new()); - http.push_response( + let clock = TestClock::new(); + let http = ScriptedHttpClient::new(&clock); + respond_planned( + &http, 200, serde_json::to_vec(&serde_json::json!({ "seatbid": [{"seat": "provider-seat", "bid": [{ @@ -7499,21 +4214,20 @@ mod tests { })) .expect("should serialize provider response"), ); - http.push_response(200, b"{}".to_vec()); - http.push_select_success(); - http.push_wait_error(); - let backend = Arc::new(NamingBackend::new(BackendNamingPolicy::Axum)); - let services = build_services_with_backend_and_http_client( - Arc::clone(&backend) as Arc<_>, - Arc::clone(&http) as Arc<_>, + http.respond_to( + Method::POST, + CACHE_RESTORING_MEDIATOR, + ScriptedReply::error(edgezero_core::error::EdgeError::bad_gateway( + "fictional mediator transport failure", + )), ); + let services = build_services_with_outbound_client(&http, &clock); let plan = AuctionPlan::compile(planned_config( &[("provider-a", RoutingMode::AllEligible)], false, )) .expect("should compile planned auction"); - let orchestrator = - AuctionOrchestratorHarness::new(plan, Some(Arc::new(CacheRestoringMediator))); + let orchestrator = planned_orchestrator(plan, Some(Arc::new(CacheRestoringMediator))); let request = planned_request(); let settings = create_test_settings(); let inbound = http::Request::new(edgezero_core::body::Body::empty()); @@ -7521,7 +4235,6 @@ mod tests { settings: &settings, request: &inbound, timeout_ms: 777, - transport_timeout_ms: 777, provider_responses: None, services: &services, }; @@ -7541,11 +4254,18 @@ mod tests { #[tokio::test] async fn planned_prebid_stored_intent_filters_wire_demand_and_skips_empty_transports() { for inline_providers in 0..=2 { - let http = Arc::new(StubHttpClient::new()); + let clock = TestClock::new(); + let http = ScriptedHttpClient::new(&clock); // Providers launch in ID order: APS, then only PBS instances with usable demand. - http.push_response(204, Vec::new()); + respond(&http, APS_ENDPOINT, 204, Vec::new()); for index in 0..inline_providers { - http.push_response( + respond( + &http, + if index == 0 { + "https://pbs-a.example.test/openrtb" + } else { + "https://pbs-b.example.test/openrtb" + }, 200, serde_json::to_vec(&serde_json::json!({ "seatbid":[{"seat":"example-seat","bid":[ @@ -7562,11 +4282,7 @@ mod tests { .expect("should serialize PBS response"), ); } - let backend = Arc::new(NamingBackend::new(BackendNamingPolicy::Fastly)); - let services = build_services_with_backend_and_http_client( - Arc::clone(&backend) as Arc<_>, - Arc::clone(&http) as Arc<_>, - ); + let services = build_services_with_outbound_client(&http, &clock); let mut config = planned_prebid_config(&[ ( "pbs-a", @@ -7589,7 +4305,7 @@ mod tests { ); } let plan = AuctionPlan::compile(config).expect("should compile plan"); - let orchestrator = AuctionOrchestratorHarness::new(plan, None); + let orchestrator = planned_orchestrator(plan, None); let mut request = planned_request(); let template = request.slots[0].clone(); // Both PBS instances must evaluate candidates but omit them after overrides. @@ -7624,7 +4340,6 @@ mod tests { settings: &settings, request: &inbound, timeout_ms: 777, - transport_timeout_ms: 777, provider_responses: None, services: &services, }; @@ -7632,7 +4347,7 @@ mod tests { .run_auction(&request, &context) .await .expect("should run auction"); - let bodies = http.recorded_request_bodies(); + let bodies = recorded_bodies(&http); assert_eq!( bodies.len(), 1 + inline_providers, @@ -7708,8 +4423,9 @@ mod tests { #[tokio::test] async fn planned_prebid_instances_preserve_headers_metadata_suppression_and_identity() { - let http = Arc::new(StubHttpClient::new()); - http.push_response( + let clock = TestClock::new(); + let http = ScriptedHttpClient::new(&clock); + respond(&http, "https://pbs-a.example.test/openrtb", 200, serde_json::to_vec(&serde_json::json!({ "seatbid": [{"seat": "suppress-exact", "bid": [ @@ -7720,7 +4436,7 @@ mod tests { })) .expect("should serialize PBS response a"), ); - http.push_response( + respond(&http, "https://pbs-b.example.test/openrtb", 200, serde_json::to_vec(&serde_json::json!({ "seatbid": [{"seat": "keep-seat", "bid": [{ @@ -7729,11 +4445,7 @@ mod tests { })) .expect("should serialize PBS response b"), ); - let backend = Arc::new(NamingBackend::new(BackendNamingPolicy::Fastly)); - let services = build_services_with_backend_and_http_client( - Arc::clone(&backend) as Arc<_>, - Arc::clone(&http) as Arc<_>, - ); + let services = build_services_with_outbound_client(&http, &clock); let notifications = NotificationConfig { suppress_all: false, suppress_seats: vec!["suppress-exact".to_string()], @@ -7747,7 +4459,7 @@ mod tests { ("pbs-b", serde_json::json!({}), NotificationConfig::default()), ])) .expect("should compile planned PBS auction"); - let orchestrator = AuctionOrchestratorHarness::new(plan, None); + let orchestrator = planned_orchestrator(plan, None); let request = planned_prebid_request(); let settings = create_test_settings(); let inbound = http::Request::builder() @@ -7766,7 +4478,6 @@ mod tests { settings: &settings, request: &inbound, timeout_ms: 777, - transport_timeout_ms: 777, provider_responses: None, services: &services, }; @@ -7808,7 +4519,7 @@ mod tests { assert!(!second.metadata.contains_key("debug")); assert!(!second.metadata.contains_key("bidstatus")); - let headers = http.recorded_request_headers(); + let headers = recorded_headers(&http); assert_eq!(headers.len(), 2); for request_headers in &headers { assert!( @@ -7886,16 +4597,17 @@ mod tests { ] { let unreadable_cookie = cookie.as_ref().is_some_and(|value| value.to_str().is_err()); - let http = Arc::new(StubHttpClient::new()); - http.push_response(204, Vec::new()); - let services = build_services_with_http_client(Arc::clone(&http) as Arc<_>); + let clock = TestClock::new(); + let http = ScriptedHttpClient::new(&clock); + respond(&http, "https://pbs.example.test/openrtb", 204, Vec::new()); + let services = build_services_with_outbound_client(&http, &clock); let plan = AuctionPlan::compile(planned_prebid_config(&[( "pbs", serde_json::json!({"consent_forwarding":mode}), NotificationConfig::default(), )])) .expect("should compile PBS plan"); - let orchestrator = AuctionOrchestratorHarness::new(plan, None); + let orchestrator = planned_orchestrator(plan, None); let mut request = planned_prebid_request(); request.user.consent = Some(crate::consent::ConsentContext { source, @@ -7921,7 +4633,6 @@ mod tests { settings: &settings, request: &inbound, timeout_ms: 777, - transport_timeout_ms: 777, provider_responses: None, services: &services, }; @@ -7931,7 +4642,7 @@ mod tests { .await .expect("should run PBS auction"); - let headers = http.recorded_request_header_bytes(); + let headers = recorded_header_bytes(&http); assert_eq!(headers.len(), 1, "should dispatch one PBS request"); let cookies = headers[0] .iter() @@ -7950,7 +4661,7 @@ mod tests { .unwrap_or_default(), "should apply {mode} cookie policy" ); - let bodies = http.recorded_request_bodies(); + let bodies = recorded_bodies(&http); let body: serde_json::Value = serde_json::from_slice(&bodies[0]).expect("should parse PBS body"); let includes_body = @@ -8006,16 +4717,17 @@ mod tests { true, ), ] { - let http = Arc::new(StubHttpClient::new()); - http.push_response(204, Vec::new()); - let services = build_services_with_http_client(Arc::clone(&http) as Arc<_>); + let clock = TestClock::new(); + let http = ScriptedHttpClient::new(&clock); + respond(&http, "https://pbs.example.test/openrtb", 204, Vec::new()); + let services = build_services_with_outbound_client(&http, &clock); let plan = AuctionPlan::compile(planned_prebid_config(&[( "pbs", serde_json::json!({"consent_forwarding":mode}), NotificationConfig::default(), )])) .expect("should compile PBS plan"); - let orchestrator = AuctionOrchestratorHarness::new(plan, None); + let orchestrator = planned_orchestrator(plan, None); let inbound = http::Request::builder() .header(http::header::COOKIE, cookie) .body(edgezero_core::body::Body::empty()) @@ -8037,7 +4749,6 @@ mod tests { settings: &settings, request: &inbound, timeout_ms: 777, - transport_timeout_ms: 777, provider_responses: None, services: &services, }; @@ -8045,7 +4756,7 @@ mod tests { .run_auction(&request, &context) .await .expect("should dispatch PBS auction"); - let headers = http.recorded_request_headers(); + let headers = recorded_headers(&http); assert_eq!( headers[0] .iter() @@ -8054,9 +4765,8 @@ mod tests { (mode != "openrtb_only").then_some(forwarded), "should preserve only the selected consent values for {cookie:?} in {mode}" ); - let body: serde_json::Value = - serde_json::from_slice(&http.recorded_request_bodies()[0]) - .expect("should parse PBS body"); + let body: serde_json::Value = serde_json::from_slice(&recorded_bodies(&http)[0]) + .expect("should parse PBS body"); let includes_body = mode != "cookies_only"; assert_eq!( body.pointer("/regs/us_privacy"), @@ -8109,16 +4819,17 @@ mod tests { Some(r"bad\value"), ), ] { - let http = Arc::new(StubHttpClient::new()); - http.push_response(204, Vec::new()); - let services = build_services_with_http_client(Arc::clone(&http) as Arc<_>); + let clock = TestClock::new(); + let http = ScriptedHttpClient::new(&clock); + respond(&http, "https://pbs.example.test/openrtb", 204, Vec::new()); + let services = build_services_with_outbound_client(&http, &clock); let plan = AuctionPlan::compile(planned_prebid_config(&[( "pbs", serde_json::json!({"consent_forwarding":"cookies_only"}), NotificationConfig::default(), )])) .expect("should compile PBS plan"); - let orchestrator = AuctionOrchestratorHarness::new(plan, None); + let orchestrator = planned_orchestrator(plan, None); let inbound = http::Request::builder() .header(http::header::COOKIE, cookie) .body(edgezero_core::body::Body::empty()) @@ -8133,7 +4844,6 @@ mod tests { settings: &settings, request: &inbound, timeout_ms: 777, - transport_timeout_ms: 777, provider_responses: None, services: &services, }; @@ -8143,7 +4853,7 @@ mod tests { .await .expect("should dispatch PBS auction"); - let headers = http.recorded_request_headers(); + let headers = recorded_headers(&http); assert_eq!(headers.len(), 1, "should dispatch exactly one PBS request"); assert_eq!( headers[0] @@ -8153,7 +4863,7 @@ mod tests { Some(forwarded), "should keep unaffected safe consent cookies" ); - let bodies = http.recorded_request_bodies(); + let bodies = recorded_bodies(&http); let body: serde_json::Value = serde_json::from_slice(&bodies[0]).expect("should parse PBS body"); for path in ["/user/consent", "/user/ext/consent"] { @@ -8185,8 +4895,11 @@ mod tests { #[tokio::test] async fn planned_aps_mock_mediation_preserves_three_identities_and_renderer() { - let http = Arc::new(StubHttpClient::new()); - http.push_response( + let clock = TestClock::new(); + let http = ScriptedHttpClient::new(&clock); + respond( + &http, + APS_ENDPOINT, 200, serde_json::to_vec(&serde_json::json!({ "seatbid": [{"seat": "upstream-seat", "bid": [{ @@ -8197,7 +4910,9 @@ mod tests { })) .expect("should serialize APS response"), ); - http.push_response( + respond( + &http, + "https://mediator.example/mediate", 200, serde_json::to_vec(&serde_json::json!({ "seatbid": [{"seat": "aps-instance", "bid": [{ @@ -8207,11 +4922,7 @@ mod tests { })) .expect("should serialize mediator response"), ); - let backend = Arc::new(NamingBackend::new(BackendNamingPolicy::Axum)); - let services = build_services_with_backend_and_http_client( - Arc::clone(&backend) as Arc<_>, - Arc::clone(&http) as Arc<_>, - ); + let services = build_services_with_outbound_client(&http, &clock); let plan = AuctionPlan::compile(planned_aps_config()).expect("should compile planned APS auction"); let mediator = AdServerMockProvider::new(AdServerMockConfig { @@ -8220,7 +4931,7 @@ mod tests { timeout_ms: 500, ..AdServerMockConfig::default() }); - let orchestrator = AuctionOrchestratorHarness::new(plan, Some(Arc::new(mediator))); + let orchestrator = planned_orchestrator(plan, Some(Arc::new(mediator))); let request = planned_request(); let settings = create_test_settings(); let inbound = http::Request::new(edgezero_core::body::Body::empty()); @@ -8228,7 +4939,6 @@ mod tests { settings: &settings, request: &inbound, timeout_ms: 777, - transport_timeout_ms: 777, provider_responses: None, services: &services, }; @@ -8258,16 +4968,13 @@ mod tests { #[tokio::test] async fn planned_aps_transport_omits_accept_header() { - let http = Arc::new(StubHttpClient::new()); - http.push_response(400, Vec::new()); - let backend = Arc::new(NamingBackend::new(BackendNamingPolicy::Fastly)); - let services = build_services_with_backend_and_http_client( - Arc::clone(&backend) as Arc<_>, - Arc::clone(&http) as Arc<_>, - ); + let clock = TestClock::new(); + let http = ScriptedHttpClient::new(&clock); + respond(&http, APS_ENDPOINT, 400, Vec::new()); + let services = build_services_with_outbound_client(&http, &clock); let plan = AuctionPlan::compile(planned_aps_config()).expect("should compile planned APS auction"); - let orchestrator = AuctionOrchestratorHarness::new(plan, None); + let orchestrator = planned_orchestrator(plan, None); let request = planned_request(); let settings = create_test_settings(); let inbound = http::Request::builder() @@ -8278,7 +4985,6 @@ mod tests { settings: &settings, request: &inbound, timeout_ms: 777, - transport_timeout_ms: 777, provider_responses: None, services: &services, }; @@ -8288,7 +4994,7 @@ mod tests { .await .expect("should execute planned APS auction"); - let headers = http.recorded_request_headers(); + let headers = recorded_headers(&http); assert_eq!(headers.len(), 1); assert!( headers[0].iter().all(|(name, _)| name != "accept"), @@ -8298,8 +5004,11 @@ mod tests { #[tokio::test] async fn planned_aps_profile_normalizes_renderer_reduction_and_metadata() { - let http = Arc::new(StubHttpClient::new()); - http.push_response_with_headers( + let clock = TestClock::new(); + let http = ScriptedHttpClient::new(&clock); + respond_with_headers( + &http, + APS_ENDPOINT, 200, serde_json::to_vec(&serde_json::json!({ "cur": "USD", @@ -8337,16 +5046,12 @@ mod tests { ] })) .expect("should serialize APS profile response"), - vec![ + &[ ("content-type", "application/json"), ("authorization", "fictional-secret"), ], ); - let backend = Arc::new(NamingBackend::new(BackendNamingPolicy::Fastly)); - let services = build_services_with_backend_and_http_client( - Arc::clone(&backend) as Arc<_>, - Arc::clone(&http) as Arc<_>, - ); + let services = build_services_with_outbound_client(&http, &clock); let plan = AuctionPlan::compile(planned_aps_instances_config(&[( "aps-instance", serde_json::json!({"account_id": "example-account", "debug": true}), @@ -8356,7 +5061,7 @@ mod tests { }, )])) .expect("should compile planned APS profile"); - let orchestrator = AuctionOrchestratorHarness::new(plan, None); + let orchestrator = planned_orchestrator(plan, None); let request = planned_request(); let settings = create_test_settings(); let inbound = http::Request::new(edgezero_core::body::Body::empty()); @@ -8364,7 +5069,6 @@ mod tests { settings: &settings, request: &inbound, timeout_ms: 777, - transport_timeout_ms: 777, provider_responses: None, services: &services, }; @@ -8455,9 +5159,12 @@ mod tests { #[tokio::test] async fn two_planned_aps_instances_correlate_independently() { - let http = Arc::new(StubHttpClient::new()); + let clock = TestClock::new(); + let http = ScriptedHttpClient::new(&clock); for (seat, id, price) in [("seat-a", "bid-a", 1.0), ("seat-b", "bid-b", 2.0)] { - http.push_response( + respond( + &http, + APS_ENDPOINT, 200, serde_json::to_vec(&serde_json::json!({"seatbid":[{"seat":seat,"bid":[{ "id":id,"impid":"fictional-slot","price":price,"w":300,"h":250, @@ -8466,11 +5173,7 @@ mod tests { .expect("should serialize APS instance response"), ); } - let backend = Arc::new(NamingBackend::new(BackendNamingPolicy::Fastly)); - let services = build_services_with_backend_and_http_client( - Arc::clone(&backend) as Arc<_>, - Arc::clone(&http) as Arc<_>, - ); + let services = build_services_with_outbound_client(&http, &clock); let plan = AuctionPlan::compile(planned_aps_instances_config(&[ ( "aps-a", @@ -8484,7 +5187,7 @@ mod tests { ), ])) .expect("should compile two APS instances"); - let orchestrator = AuctionOrchestratorHarness::new(plan, None); + let orchestrator = planned_orchestrator(plan, None); let request = planned_request(); let settings = create_test_settings(); let inbound = http::Request::new(edgezero_core::body::Body::empty()); @@ -8492,7 +5195,6 @@ mod tests { settings: &settings, request: &inbound, timeout_ms: 777, - transport_timeout_ms: 777, provider_responses: None, services: &services, }; @@ -8513,15 +5215,12 @@ mod tests { result.provider_responses[1].bids[0].bid_id.as_deref(), Some("bid-b") ); - assert_eq!(http.recorded_request_bodies().len(), 2); + assert_eq!(recorded_bodies(&http).len(), 2); assert_eq!( result.winning_bids["fictional-slot"].bid_id.as_deref(), Some("bid-b"), "global ranking should remain orchestrator-owned" ); - let specs = backend.specs.lock().expect("should lock specs"); - assert_eq!(specs.len(), 2); - assert_ne!(specs[0].discriminator, specs[1].discriminator); } #[tokio::test] @@ -8541,7 +5240,7 @@ mod tests { (serde_json::json!("exact-seat"), Some("exact-seat")), ] { let state = provider.parse_state_for_test(routed.inputs()[0].clone()); - let response = PlatformResponse::new( + let response = outbound_response( edgezero_core::http::response_builder() .status(200) .body(edgezero_core::body::Body::from( @@ -8607,7 +5306,7 @@ mod tests { ]; for (status, body, expected, reason, error_type) in cases { let state = provider.parse_state_for_test(routed.inputs()[0].clone()); - let response = PlatformResponse::new( + let response = outbound_response( edgezero_core::http::response_builder() .status(status) .body(edgezero_core::body::Body::from(body)) @@ -8673,7 +5372,7 @@ mod tests { ); let provider = GenericOpenRtbProvider::new(plan.providers()[0].clone()); let state = provider.parse_state_for_test(routed.inputs()[0].clone()); - let response = PlatformResponse::new( + let response = outbound_response( edgezero_core::http::response_builder() .status(status) .body(edgezero_core::body::Body::from(body)) @@ -8722,7 +5421,7 @@ mod tests { ); let provider = GenericOpenRtbProvider::new(plan.providers()[0].clone()); let state = provider.parse_state_for_test(routed.inputs()[0].clone()); - let response = PlatformResponse::new( + let response = outbound_response( edgezero_core::http::response_builder() .status(200) .body(edgezero_core::body::Body::from( @@ -8785,7 +5484,7 @@ mod tests { ); let provider = GenericOpenRtbProvider::new(plan.providers()[0].clone()); let state = provider.parse_state_for_test(routed.inputs()[0].clone()); - let response = PlatformResponse::new( + let response = outbound_response( edgezero_core::http::response_builder() .status(200) .header("content-type", "application/json") @@ -8808,9 +5507,11 @@ mod tests { } #[tokio::test] - async fn planned_standard_instances_have_distinct_backends_and_independent_results() { - let http = Arc::new(StubHttpClient::new()); - http.push_response( + async fn planned_standard_instances_have_independent_results_and_configured_timeouts() { + let clock = TestClock::new(); + let http = ScriptedHttpClient::new(&clock); + respond_planned( + &http, 200, serde_json::to_vec(&serde_json::json!({ "seatbid": [{"seat": "fictional-seat-a", "bid": [{ @@ -8820,7 +5521,8 @@ mod tests { })) .expect("should serialize response a"), ); - http.push_response( + respond_planned( + &http, 200, serde_json::to_vec(&serde_json::json!({ "seatbid": [{"seat": "fictional-seat-b", "bid": [{ @@ -8830,11 +5532,7 @@ mod tests { })) .expect("should serialize response b"), ); - let backend = Arc::new(NamingBackend::new(BackendNamingPolicy::Fastly)); - let services = build_services_with_backend_and_http_client( - Arc::clone(&backend) as Arc<_>, - Arc::clone(&http) as Arc<_>, - ); + let services = build_services_with_outbound_client(&http, &clock); let plan = AuctionPlan::compile(planned_config( &[ ("provider-a", RoutingMode::AllEligible), @@ -8843,7 +5541,7 @@ mod tests { false, )) .expect("should compile planned auction"); - let orchestrator = AuctionOrchestratorHarness::new(plan, None); + let orchestrator = planned_orchestrator(plan, None); let request = planned_request(); let settings = create_test_settings(); let inbound = http::Request::builder() @@ -8854,7 +5552,6 @@ mod tests { settings: &settings, request: &inbound, timeout_ms: 777, - transport_timeout_ms: 777, provider_responses: None, services: &services, }; @@ -8865,7 +5562,6 @@ mod tests { .expect("should execute planned auction"); assert_eq!(orchestrator.provider_count(), 2); - assert!(orchestrator.mediator().is_none()); assert_eq!(result.provider_responses.len(), 2); assert_eq!(result.provider_responses[0].provider, "provider-a"); assert_eq!( @@ -8901,7 +5597,7 @@ mod tests { result.winning_bids["fictional-slot"].bid_id.as_deref(), Some("bid-b") ); - let request_headers = http.recorded_request_headers(); + let request_headers = recorded_headers(&http); assert_eq!(request_headers.len(), 2); for headers in request_headers { assert!( @@ -8911,10 +5607,7 @@ mod tests { "standard planned transport should retain its JSON Accept header" ); } - let backend_names = http.recorded_backend_names(); - assert_eq!(backend_names.len(), 2); - assert_ne!(backend_names[0], backend_names[1]); - let request_bodies = http.recorded_request_bodies(); + let request_bodies = recorded_bodies(&http); assert_eq!(request_bodies.len(), 2); for body in request_bodies { let value: serde_json::Value = @@ -8926,162 +5619,13 @@ mod tests { ); assert_eq!(value["imp"].as_array().map(Vec::len), Some(1)); } - let specs = backend.specs.lock().expect("should lock specs"); - assert_eq!(specs.len(), 2); - assert_eq!(specs[0].first_byte_timeout, Duration::from_millis(750)); - assert_eq!(specs[1].first_byte_timeout, Duration::from_millis(750)); - assert_ne!(specs[0].discriminator, specs[1].discriminator); - } - - #[tokio::test] - async fn planned_backend_collision_does_not_overwrite_first_launch_state() { - let http = Arc::new(StubHttpClient::new()); - http.push_response( - 200, - serde_json::to_vec(&serde_json::json!({ - "seatbid": [{"seat": "first", "bid": [{ - "id": "first-bid", "impid": "fictional-slot", "price": 2.0, - "adm": "
first
", "w": 300, "h": 250 - }]}] - })) - .expect("should serialize first response"), - ); - let backend = Arc::new(CollidingBackend); - let services = build_services_with_backend_and_http_client( - Arc::clone(&backend) as Arc<_>, - Arc::clone(&http) as Arc<_>, - ); - let plan = AuctionPlan::compile(planned_config( - &[ - ("provider-a", RoutingMode::AllEligible), - ("provider-b", RoutingMode::AllEligible), - ], - false, - )) - .expect("should compile planned auction"); - let orchestrator = AuctionOrchestratorHarness::new(plan, None); - let request = planned_request(); - let settings = create_test_settings(); - let inbound = http::Request::new(edgezero_core::body::Body::empty()); - let context = AuctionContext { - settings: &settings, - request: &inbound, - timeout_ms: 777, - transport_timeout_ms: 777, - provider_responses: None, - services: &services, - }; - - let result = orchestrator - .run_auction(&request, &context) - .await - .expect("should isolate backend collision"); - - assert_eq!(http.recorded_backend_names().len(), 1); - assert_eq!(result.provider_responses[0].provider, "provider-a"); - assert_eq!( - result.provider_responses[0].bids[0].bid_id.as_deref(), - Some("first-bid") - ); - assert_eq!(result.provider_responses[1].provider, "provider-b"); - assert_eq!( - result.provider_responses[1].metadata["error_type"], - "launch_failed" - ); - } - - #[tokio::test] - async fn planned_pending_backend_divergence_isolated_from_valid_provider() { - let http = Arc::new(StubHttpClient::new()); - http.push_response(204, Vec::new()); - http.push_response(204, Vec::new()); - http.push_pending_backend_name_override(Some("divergent-backend")); - let backend = Arc::new(NamingBackend::new(BackendNamingPolicy::Axum)); - let services = build_services_with_backend_and_http_client( - Arc::clone(&backend) as Arc<_>, - Arc::clone(&http) as Arc<_>, - ); - let plan = AuctionPlan::compile(planned_config( - &[ - ("provider-a", RoutingMode::AllEligible), - ("provider-b", RoutingMode::AllEligible), - ], - false, - )) - .expect("should compile planned auction"); - let orchestrator = AuctionOrchestratorHarness::new(plan, None); - let request = planned_request(); - let settings = create_test_settings(); - let inbound = http::Request::new(edgezero_core::body::Body::empty()); - let context = AuctionContext { - settings: &settings, - request: &inbound, - timeout_ms: 777, - transport_timeout_ms: 777, - provider_responses: None, - services: &services, - }; - - let result = orchestrator - .run_auction(&request, &context) - .await - .expect("should isolate divergent pending backend"); - - assert_eq!(result.provider_responses.len(), 2); - assert_eq!(result.provider_responses[0].provider, "provider-a"); - assert_eq!( - result.provider_responses[0].metadata["error_type"], - "launch_failed" - ); - assert_eq!(result.provider_responses[1].provider, "provider-b"); - assert_eq!(result.provider_responses[1].status, BidStatus::NoBid); - } - - #[tokio::test] - async fn planned_pending_backend_missing_isolated_from_valid_provider() { - let http = Arc::new(StubHttpClient::new()); - http.push_response(204, Vec::new()); - http.push_response(204, Vec::new()); - http.push_pending_backend_name_override(None); - let backend = Arc::new(NamingBackend::new(BackendNamingPolicy::Axum)); - let services = build_services_with_backend_and_http_client( - Arc::clone(&backend) as Arc<_>, - Arc::clone(&http) as Arc<_>, - ); - let plan = AuctionPlan::compile(planned_config( - &[ - ("provider-a", RoutingMode::AllEligible), - ("provider-b", RoutingMode::AllEligible), - ], - false, - )) - .expect("should compile planned auction"); - let orchestrator = AuctionOrchestratorHarness::new(plan, None); - let request = planned_request(); - let settings = create_test_settings(); - let inbound = http::Request::new(edgezero_core::body::Body::empty()); - let context = AuctionContext { - settings: &settings, - request: &inbound, - timeout_ms: 777, - transport_timeout_ms: 777, - provider_responses: None, - services: &services, - }; - - let result = orchestrator - .run_auction(&request, &context) - .await - .expect("should isolate missing pending backend"); - - assert_eq!(result.provider_responses.len(), 2); - assert_eq!(result.provider_responses[0].provider, "provider-a"); - assert_eq!( - result.provider_responses[0].metadata["error_type"], - "launch_failed" - ); - assert_eq!(result.provider_responses[1].provider, "provider-b"); - assert_eq!(result.provider_responses[1].status, BidStatus::NoBid); + for request in http.requests() { + assert_eq!( + request.timeout, + Some(Duration::from_secs(1)), + "should send each instance with its own configured timeout" + ); + } } #[tokio::test] @@ -9103,7 +5647,7 @@ mod tests { let provider_a = GenericOpenRtbProvider::new(plan.providers()[0].clone()); let provider_b = GenericOpenRtbProvider::new(plan.providers()[1].clone()); let parse_state = provider_a.parse_state_for_test(routed.inputs()[0].clone()); - let response = PlatformResponse::new( + let response = outbound_response( edgezero_core::http::response_builder() .status(204) .body(edgezero_core::body::Body::empty()) @@ -9145,7 +5689,7 @@ mod tests { let provider_a = GenericOpenRtbProvider::new(plan.providers()[0].clone()); let provider_b = GenericOpenRtbProvider::new(plan.providers()[1].clone()); let parse_state = provider_a.parse_state_for_test(routed.inputs()[0].clone()); - let response = PlatformResponse::new( + let response = outbound_response( edgezero_core::http::response_builder() .status(200) .body(edgezero_core::body::Body::from_bytes(b"{}".as_slice())) @@ -9165,13 +5709,10 @@ mod tests { #[tokio::test] async fn planned_skipped_provider_has_no_io_and_no_zero_impression_launch() { - let http = Arc::new(StubHttpClient::new()); - http.push_response(204, Vec::new()); - let backend = Arc::new(NamingBackend::new(BackendNamingPolicy::Axum)); - let services = build_services_with_backend_and_http_client( - Arc::clone(&backend) as Arc<_>, - Arc::clone(&http) as Arc<_>, - ); + let clock = TestClock::new(); + let http = ScriptedHttpClient::new(&clock); + respond_planned(&http, 204, Vec::new()); + let services = build_services_with_outbound_client(&http, &clock); let plan = AuctionPlan::compile(planned_config( &[ ("eligible", RoutingMode::AllEligible), @@ -9180,7 +5721,7 @@ mod tests { false, )) .expect("should compile planned auction"); - let orchestrator = AuctionOrchestratorHarness::new(plan, None); + let orchestrator = planned_orchestrator(plan, None); let request = planned_request(); let settings = create_test_settings(); let inbound = http::Request::new(edgezero_core::body::Body::empty()); @@ -9188,7 +5729,6 @@ mod tests { settings: &settings, request: &inbound, timeout_ms: 777, - transport_timeout_ms: 777, provider_responses: None, services: &services, }; @@ -9198,8 +5738,7 @@ mod tests { .await .expect("should execute eligible provider only"); - assert_eq!(http.recorded_backend_names().len(), 1); - assert_eq!(backend.ensured.load(Ordering::Relaxed), 1); + assert_eq!(recorded_uris(&http).len(), 1); let skipped = result .provider_responses .iter() @@ -9210,91 +5749,20 @@ mod tests { skipped.metadata["routing"]["skipped_no_eligible_slots"], true ); - let request_body = &http.recorded_request_bodies()[0]; + let request_body = &recorded_bodies(&http)[0]; let request_value: serde_json::Value = serde_json::from_slice(request_body).expect("should parse request body"); assert_eq!(request_value["imp"].as_array().map(Vec::len), Some(1)); } #[tokio::test] - async fn zero_canonical_timeout_skips_plan_backed_direct_and_split_launches() { - for split in [false, true] { - let http = Arc::new(StubHttpClient::new()); - let backend = Arc::new(ZeroCanonicalBackend::new()); - let services = build_services_with_backend_and_http_client( - Arc::clone(&backend) as Arc<_>, - Arc::clone(&http) as Arc<_>, - ); - let mut config = planned_config(&[("provider", RoutingMode::AllEligible)], false); - config.mediator = Some("adserver_mock".to_string()); - let plan = AuctionPlan::compile(config).expect("should compile planned auction"); - let mediator_predicted = Arc::new(Mutex::new(Vec::new())); - let mediator_requested = Arc::new(Mutex::new(Vec::new())); - let mediator = Arc::new(recording_provider( - "adserver_mock", - "mediator-backend", - 777, - &mediator_predicted, - &mediator_requested, - )); - let orchestrator = AuctionOrchestrator::from_plan(Arc::new(plan), Some(mediator)); - let request = planned_request(); - let settings = create_test_settings(); - let inbound = http::Request::new(edgezero_core::body::Body::empty()); - let context = AuctionContext { - settings: &settings, - request: &inbound, - timeout_ms: 777, - transport_timeout_ms: 777, - provider_responses: None, - services: &services, - }; - - let result = if split { - let DispatchAuctionOutcome::Dispatched(dispatched) = - orchestrator.dispatch_auction(&request, &context).await - else { - panic!("zero canonical timeout should materialize a split response"); - }; - orchestrator - .collect_dispatched_auction(dispatched, &services, &context) - .await - } else { - orchestrator - .run_auction(&request, &context) - .await - .expect("should materialize direct timeout response") - }; - - assert_eq!(result.provider_responses.len(), 1); - assert_eq!( - result.provider_responses[0].metadata["error_type"], - "timeout" - ); - assert_eq!(backend.predicted.load(Ordering::Relaxed), 0); - assert_eq!(backend.ensured.load(Ordering::Relaxed), 0); - assert!(http.recorded_backend_names().is_empty()); - assert!( - mediator_predicted - .lock() - .expect("should lock mediator predictions") - .is_empty() - ); - assert!( - mediator_requested - .lock() - .expect("should lock mediator requests") - .is_empty() - ); - } - } - - #[tokio::test] - async fn planned_launch_transport_parse_failures_are_isolated_from_valid_winner_and_floor() { - let http = Arc::new(StubHttpClient::new()); + async fn planned_transport_and_parse_failures_are_isolated_from_valid_winner_and_floor() { + let clock = TestClock::new(); + let http = ScriptedHttpClient::new(&clock); // BTreeMap plan order is alphabetical: below-floor, parse-fail, // transport-fail, valid-winner. Queue responses in that exact order. - http.push_response( + respond_planned( + &http, 200, serde_json::to_vec(&serde_json::json!({ "seatbid": [{"seat": "below-floor", "bid": [{ @@ -9307,9 +5775,16 @@ mod tests { })) .expect("should serialize below-floor response"), ); - http.push_response(200, b"not-json".to_vec()); - http.push_response(200, b"{}".to_vec()); - http.push_response( + respond_planned(&http, 200, b"not-json".to_vec()); + http.respond_to( + Method::POST, + PLANNED_ENDPOINT, + ScriptedReply::error(edgezero_core::error::EdgeError::bad_gateway( + "fictional transport failure", + )), + ); + respond_planned( + &http, 200, serde_json::to_vec(&serde_json::json!({ "seatbid": [{"seat": "winner", "bid": [{ @@ -9319,18 +5794,9 @@ mod tests { })) .expect("should serialize winner response"), ); - http.push_select_success(); - http.push_select_success(); - http.push_select_error(); - let backend = Arc::new(NamingBackend::new(BackendNamingPolicy::Axum)); - backend.fail_ensure_for("launch-fail"); - let services = build_services_with_backend_and_http_client( - Arc::clone(&backend) as Arc<_>, - Arc::clone(&http) as Arc<_>, - ); + let services = build_services_with_outbound_client(&http, &clock); let plan = AuctionPlan::compile(planned_config( &[ - ("launch-fail", RoutingMode::AllEligible), ("transport-fail", RoutingMode::AllEligible), ("parse-fail", RoutingMode::AllEligible), ("below-floor", RoutingMode::AllEligible), @@ -9339,7 +5805,7 @@ mod tests { false, )) .expect("should compile failure isolation plan"); - let orchestrator = AuctionOrchestratorHarness::new(plan, None); + let orchestrator = planned_orchestrator(plan, None); let mut request = planned_request(); let mut below_only_slot = request.slots[0].clone(); below_only_slot.id = "below-only-slot".to_string(); @@ -9350,7 +5816,6 @@ mod tests { settings: &settings, request: &inbound, timeout_ms: 777, - transport_timeout_ms: 777, provider_responses: None, services: &services, }; @@ -9365,16 +5830,6 @@ mod tests { .iter() .map(|response| (response.provider.as_str(), response)) .collect::>(); - assert_eq!( - by_provider - .get("launch-fail") - .unwrap_or_else(|| panic!( - "should include launch-fail response; got {:?}", - by_provider.keys().collect::>() - )) - .metadata["error_type"], - "launch_failed" - ); let below_floor = by_provider.get("below-floor").unwrap_or_else(|| { panic!( "should include below-floor response; got {:?}", @@ -9421,13 +5876,15 @@ mod tests { #[tokio::test] async fn planned_routing_count_survives_standard_and_aps_bounded_body_failures() { for (profile, provider_id) in [("standard", "standard"), ("aps", "aps-instance")] { - let http = Arc::new(StubHttpClient::new()); - http.push_response(200, vec![b'x'; 1024 * 1024 + 1]); - let backend = Arc::new(NamingBackend::new(BackendNamingPolicy::Axum)); - let services = build_services_with_backend_and_http_client( - Arc::clone(&backend) as Arc<_>, - Arc::clone(&http) as Arc<_>, - ); + let clock = TestClock::new(); + let http = ScriptedHttpClient::new(&clock); + let endpoint = if profile == "standard" { + PLANNED_ENDPOINT + } else { + APS_ENDPOINT + }; + respond(&http, endpoint, 200, vec![b'x'; 1024 * 1024 + 1]); + let services = build_services_with_outbound_client(&http, &clock); let mut config = if profile == "standard" { planned_config(&[(provider_id, RoutingMode::AllEligible)], false) } else { @@ -9444,7 +5901,7 @@ mod tests { }, ); let plan = AuctionPlan::compile(config).expect("should compile bounded-body plan"); - let orchestrator = AuctionOrchestratorHarness::new(plan, None); + let orchestrator = planned_orchestrator(plan, None); let mut request = planned_request(); request.slots[0].bidders.insert( "example-bidder".to_string(), @@ -9456,7 +5913,6 @@ mod tests { settings: &settings, request: &inbound, timeout_ms: 777, - transport_timeout_ms: 777, provider_responses: None, services: &services, }; @@ -9479,25 +5935,25 @@ mod tests { #[tokio::test] async fn planned_signer_admission_time_reduces_budget_and_total_time_includes_it() { + let clock = TestClock::new(); let config_store = Arc::new(CountingConfigStore { reads: AtomicUsize::new(0), current_kid: "test-kid".to_string(), - delay: Duration::from_millis(50), + admission_delay: Some((clock.clone(), Duration::from_millis(50))), }); let secret_store = Arc::new(CountingSecretStore { reads: AtomicUsize::new(0), key: base64::Engine::encode(&base64::engine::general_purpose::STANDARD, [7_u8; 32]) .into_bytes(), }); - let backend = Arc::new(NamingBackend::new(BackendNamingPolicy::Axum)); - let http = Arc::new(StubHttpClient::new()); - http.push_response(204, Vec::new()); + let http = ScriptedHttpClient::new(&clock); + respond_planned(&http, 204, Vec::new()); let services = RuntimeServices::builder() .config_store(Arc::clone(&config_store) as Arc<_>) .secret_store(Arc::clone(&secret_store) as Arc<_>) .kv_store(Arc::new(edgezero_core::key_value_store::NoopKvStore)) - .backend(Arc::clone(&backend) as Arc<_>) - .http_client(Arc::clone(&http) as Arc<_>) + .outbound_client(http.http_client()) + .clock(clock.monotonic_clock()) .geo(Arc::new(crate::platform::test_support::NoopGeo)) .auction_telemetry_sink(Arc::new( crate::auction::telemetry::NoopAuctionTelemetrySink, @@ -9509,7 +5965,7 @@ mod tests { true, )) .expect("should compile signed plan"); - let orchestrator = AuctionOrchestratorHarness::new(plan, None); + let orchestrator = planned_orchestrator(plan, None); let request = planned_request(); let settings = create_test_settings(); let inbound = http::Request::new(edgezero_core::body::Body::empty()); @@ -9517,7 +5973,6 @@ mod tests { settings: &settings, request: &inbound, timeout_ms: 200, - transport_timeout_ms: 200, provider_responses: None, services: &services, }; @@ -9527,15 +5982,15 @@ mod tests { .await .expect("should execute signed planned auction"); - let body: serde_json::Value = serde_json::from_slice(&http.recorded_request_bodies()[0]) + let body: serde_json::Value = serde_json::from_slice(&recorded_bodies(&http)[0]) .expect("should parse signed request"); let tmax = body["tmax"].as_u64().expect("should include tmax"); - assert!( - (100..=175).contains(&tmax), - "signer delay should reduce logical budget, got {tmax}" + assert_eq!( + tmax, 150, + "signer admission should reduce the logical budget" ); - assert!( - result.total_time_ms >= 50, + assert_eq!( + result.total_time_ms, 50, "total time should include signer admission" ); assert_eq!(config_store.reads.load(Ordering::Relaxed), 1); @@ -9547,23 +6002,23 @@ mod tests { let config_store = Arc::new(CountingConfigStore { reads: AtomicUsize::new(0), current_kid: "test-kid".to_string(), - delay: Duration::ZERO, + admission_delay: None, }); let secret_store = Arc::new(CountingSecretStore { reads: AtomicUsize::new(0), key: base64::Engine::encode(&base64::engine::general_purpose::STANDARD, [11_u8; 32]) .into_bytes(), }); - let backend = Arc::new(NamingBackend::new(BackendNamingPolicy::Axum)); - let http = Arc::new(StubHttpClient::new()); - http.push_response(204, Vec::new()); - http.push_response(204, Vec::new()); + let clock = TestClock::new(); + let http = ScriptedHttpClient::new(&clock); + respond_planned(&http, 204, Vec::new()); + respond_planned(&http, 204, Vec::new()); let services = RuntimeServices::builder() .config_store(Arc::clone(&config_store) as Arc<_>) .secret_store(Arc::clone(&secret_store) as Arc<_>) .kv_store(Arc::new(edgezero_core::key_value_store::NoopKvStore)) - .backend(Arc::clone(&backend) as Arc<_>) - .http_client(Arc::clone(&http) as Arc<_>) + .outbound_client(http.http_client()) + .clock(clock.monotonic_clock()) .geo(Arc::new(crate::platform::test_support::NoopGeo)) .auction_telemetry_sink(Arc::new( crate::auction::telemetry::NoopAuctionTelemetrySink, @@ -9578,7 +6033,7 @@ mod tests { true, )) .expect("should compile signed plan"); - let orchestrator = AuctionOrchestratorHarness::new(plan, None); + let orchestrator = planned_orchestrator(plan, None); let request = planned_request(); let settings = create_test_settings(); let inbound = http::Request::new(edgezero_core::body::Body::empty()); @@ -9586,7 +6041,6 @@ mod tests { settings: &settings, request: &inbound, timeout_ms: 777, - transport_timeout_ms: 777, provider_responses: None, services: &services, }; @@ -9599,8 +6053,8 @@ mod tests { assert_eq!(result.provider_responses.len(), 2); assert_eq!(config_store.reads.load(Ordering::Relaxed), 1); assert_eq!(secret_store.reads.load(Ordering::Relaxed), 1); - assert_eq!(http.recorded_backend_names().len(), 2); - for body in http.recorded_request_bodies() { + assert_eq!(recorded_uris(&http).len(), 2); + for body in recorded_bodies(&http) { let value: serde_json::Value = serde_json::from_slice(&body).expect("should parse signed provider request"); assert!( @@ -9616,14 +6070,14 @@ mod tests { let config_store = Arc::new(FailingCountingConfigStore { reads: AtomicUsize::new(0), }); - let backend = Arc::new(NamingBackend::new(BackendNamingPolicy::Axum)); - let http = Arc::new(StubHttpClient::new()); + let clock = TestClock::new(); + let http = ScriptedHttpClient::new(&clock); let services = RuntimeServices::builder() .config_store(Arc::clone(&config_store) as Arc<_>) .secret_store(Arc::new(UnusedSecretStore)) .kv_store(Arc::new(edgezero_core::key_value_store::NoopKvStore)) - .backend(Arc::clone(&backend) as Arc<_>) - .http_client(Arc::clone(&http) as Arc<_>) + .outbound_client(http.http_client()) + .clock(clock.monotonic_clock()) .geo(Arc::new(crate::platform::test_support::NoopGeo)) .auction_telemetry_sink(Arc::new( crate::auction::telemetry::NoopAuctionTelemetrySink, @@ -9654,7 +6108,6 @@ mod tests { settings: &settings, request: &inbound, timeout_ms: 777, - transport_timeout_ms: 777, provider_responses: None, services: &services, }; @@ -9693,9 +6146,7 @@ mod tests { } assert_eq!(config_store.reads.load(Ordering::Relaxed), 1); - assert_eq!(backend.predicted.load(Ordering::Relaxed), 0); - assert_eq!(backend.ensured.load(Ordering::Relaxed), 0); - assert!(http.recorded_backend_names().is_empty()); + assert!(recorded_uris(&http).is_empty()); } } @@ -9704,14 +6155,14 @@ mod tests { let config_store = Arc::new(FailingCountingConfigStore { reads: AtomicUsize::new(0), }); - let backend = Arc::new(NamingBackend::new(BackendNamingPolicy::Axum)); - let http = Arc::new(StubHttpClient::new()); + let clock = TestClock::new(); + let http = ScriptedHttpClient::new(&clock); let services = RuntimeServices::builder() .config_store(Arc::clone(&config_store) as Arc<_>) .secret_store(Arc::new(UnusedSecretStore)) .kv_store(Arc::new(edgezero_core::key_value_store::NoopKvStore)) - .backend(Arc::clone(&backend) as Arc<_>) - .http_client(Arc::clone(&http) as Arc<_>) + .outbound_client(http.http_client()) + .clock(clock.monotonic_clock()) .geo(Arc::new(crate::platform::test_support::NoopGeo)) .auction_telemetry_sink(Arc::new( crate::auction::telemetry::NoopAuctionTelemetrySink, @@ -9723,7 +6174,7 @@ mod tests { true, )) .expect("should compile signed plan"); - let orchestrator = AuctionOrchestratorHarness::new(plan, None); + let orchestrator = planned_orchestrator(plan, None); let request = planned_request(); let settings = create_test_settings(); let inbound = http::Request::new(edgezero_core::body::Body::empty()); @@ -9731,7 +6182,6 @@ mod tests { settings: &settings, request: &inbound, timeout_ms: 777, - transport_timeout_ms: 777, provider_responses: None, services: &services, }; @@ -9742,9 +6192,43 @@ mod tests { .expect_err("should fail signer admission"); assert_eq!(config_store.reads.load(Ordering::Relaxed), 1); - assert_eq!(backend.predicted.load(Ordering::Relaxed), 0); - assert_eq!(backend.ensured.load(Ordering::Relaxed), 0); - assert!(http.recorded_backend_names().is_empty()); + assert!(recorded_uris(&http).is_empty()); + } + + /// An immediate mediator that counts its launches. + struct CountingMediator { + launches: Arc, + } + + #[async_trait::async_trait(?Send)] + impl AuctionProvider for CountingMediator { + fn provider_name(&self) -> &str { + "counting-mediator" + } + + async fn request_bids( + &self, + _request: &AuctionRequest, + _context: &AuctionContext<'_>, + ) -> Result> { + self.launches.fetch_add(1, Ordering::Relaxed); + Ok(ProviderRequestOutcome::Immediate(AuctionResponse::no_bid( + "counting-mediator", + 0, + ))) + } + + async fn parse_response( + &self, + _response: OutboundResponse, + _response_time_ms: u64, + ) -> Result> { + panic!("should never parse an immediate mediator response"); + } + + fn timeout_ms(&self) -> u32 { + 777 + } } #[tokio::test] @@ -9752,21 +6236,21 @@ mod tests { let config_store = Arc::new(CountingConfigStore { reads: AtomicUsize::new(0), current_kid: "unused-kid".to_string(), - delay: Duration::ZERO, + admission_delay: None, }); let secret_store = Arc::new(CountingSecretStore { reads: AtomicUsize::new(0), key: base64::Engine::encode(&base64::engine::general_purpose::STANDARD, [13_u8; 32]) .into_bytes(), }); - let backend = Arc::new(NamingBackend::new(BackendNamingPolicy::Axum)); - let http = Arc::new(StubHttpClient::new()); + let clock = TestClock::new(); + let http = ScriptedHttpClient::new(&clock); let services = RuntimeServices::builder() .config_store(Arc::clone(&config_store) as Arc<_>) .secret_store(Arc::clone(&secret_store) as Arc<_>) .kv_store(Arc::new(edgezero_core::key_value_store::NoopKvStore)) - .backend(Arc::clone(&backend) as Arc<_>) - .http_client(Arc::clone(&http) as Arc<_>) + .outbound_client(http.http_client()) + .clock(clock.monotonic_clock()) .geo(Arc::new(crate::platform::test_support::NoopGeo)) .auction_telemetry_sink(Arc::new( crate::auction::telemetry::NoopAuctionTelemetrySink, @@ -9783,9 +6267,8 @@ mod tests { let mediator_launches = Arc::new(AtomicUsize::new(0)); let orchestrator = AuctionOrchestrator::from_plan( plan, - Some(Arc::new(DeadlineRecordingMediator { + Some(Arc::new(CountingMediator { launches: Arc::clone(&mediator_launches), - budgets: None, })), ); let request = planned_request(); @@ -9795,7 +6278,6 @@ mod tests { settings: &settings, request: &inbound, timeout_ms: 0, - transport_timeout_ms: 0, provider_responses: None, services: &services, }; @@ -9816,9 +6298,7 @@ mod tests { ); assert_eq!(config_store.reads.load(Ordering::Relaxed), 0); assert_eq!(secret_store.reads.load(Ordering::Relaxed), 0); - assert_eq!(backend.predicted.load(Ordering::Relaxed), 0); - assert_eq!(backend.ensured.load(Ordering::Relaxed), 0); - assert!(http.recorded_backend_names().is_empty()); + assert!(recorded_uris(&http).is_empty()); assert_eq!( mediator_launches.load(Ordering::Relaxed), 0, @@ -9826,46 +6306,6 @@ mod tests { ); } - #[tokio::test] - async fn planned_fanout_rejection_happens_before_backend_or_send() { - let http = Arc::new(StubHttpClient::new()); - http.set_concurrent_fanout(false); - let backend = Arc::new(NamingBackend::new(BackendNamingPolicy::Cloudflare)); - let services = build_services_with_backend_and_http_client( - Arc::clone(&backend) as Arc<_>, - Arc::clone(&http) as Arc<_>, - ); - let plan = AuctionPlan::compile(planned_config( - &[ - ("provider-a", RoutingMode::AllEligible), - ("provider-b", RoutingMode::AllEligible), - ], - false, - )) - .expect("should compile planned auction"); - let orchestrator = AuctionOrchestratorHarness::new(plan, None); - let request = planned_request(); - let settings = create_test_settings(); - let inbound = http::Request::new(edgezero_core::body::Body::empty()); - let context = AuctionContext { - settings: &settings, - request: &inbound, - timeout_ms: 777, - transport_timeout_ms: 777, - provider_responses: None, - services: &services, - }; - - let _error = orchestrator - .run_auction(&request, &context) - .await - .expect_err("should reject unsupported concurrent fanout"); - - assert_eq!(backend.predicted.load(Ordering::Relaxed), 0); - assert_eq!(backend.ensured.load(Ordering::Relaxed), 0); - assert!(http.recorded_backend_names().is_empty()); - } - #[test] fn routing_metadata_is_fixed_count_only_and_saturating() { let metadata = super::routing_metadata( @@ -9888,7 +6328,7 @@ mod tests { #[test] fn decoded_aps_bid_competes_directly_by_cpm() { - let orchestrator = AuctionOrchestrator::new(AuctionConfig::default()); + let orchestrator = empty_orchestrator(false); let floor_prices = HashMap::new(); let response = |provider: &str, bid: Bid| AuctionResponse::success(provider, vec![bid], 1); @@ -9923,7 +6363,7 @@ mod tests { #[test] fn test_apply_floor_prices_drops_bids_without_price() { // Price-less bids cannot be compared or delivered and remain fail-closed. - let orchestrator = AuctionOrchestrator::new(AuctionConfig::default()); + let orchestrator = empty_orchestrator(false); let mut floor_prices = HashMap::new(); floor_prices.insert("slot-1".to_string(), 1.00); @@ -9968,7 +6408,7 @@ mod tests { #[test] fn test_apply_floor_prices_drops_decoded_aps_bid_below_floor() { // APS supplies decoded price at the provider boundary, so normal floors apply. - let orchestrator = AuctionOrchestrator::new(AuctionConfig::default()); + let orchestrator = empty_orchestrator(false); let mut floor_prices = HashMap::new(); floor_prices.insert("atf".to_string(), 0.50); @@ -10008,7 +6448,7 @@ mod tests { #[test] fn test_apply_floor_prices_keeps_decoded_aps_bid_at_or_above_floor() { - let orchestrator = AuctionOrchestrator::new(AuctionConfig::default()); + let orchestrator = empty_orchestrator(false); let mut floor_prices = HashMap::new(); floor_prices.insert("atf".to_string(), 0.50); diff --git a/crates/trusted-server-core/src/auction/plan.rs b/crates/trusted-server-core/src/auction/plan.rs index df36fc09b7..56c7512ecb 100644 --- a/crates/trusted-server-core/src/auction/plan.rs +++ b/crates/trusted-server-core/src/auction/plan.rs @@ -1,17 +1,16 @@ //! Target-independent config-first auction plan compiler. use std::collections::{BTreeMap, BTreeSet}; +use std::net::{Ipv4Addr, Ipv6Addr}; use std::str::FromStr; -use std::time::Duration; -use error_stack::{Report, ResultExt as _}; +use error_stack::Report; use serde::{Deserialize, Serialize}; use serde_json::Value; -use url::Url; +use url::{Host, Url}; use super::profile::{CompiledOpenRtbProfile, ProfileTimeoutDefault, find_profile}; use crate::error::TrustedServerError; -use crate::platform::{AuctionTargetId, PlatformBackendSpec}; use crate::settings::RequestSigning; const MAX_ID_BYTES: usize = 128; @@ -19,6 +18,10 @@ const MAX_SUPPRESS_SEATS: usize = 128; const MAX_SUPPRESS_SEAT_BYTES: usize = 128; const MOCK_MEDIATOR_ID: &str = "adserver_mock"; const RESERVED_BROWSER_ENVELOPE_BIDDER_ID: &str = "trustedServer"; +/// Dynamic backends a Fastly service may register. +const FASTLY_DYNAMIC_BACKEND_LIMIT: usize = 200; +/// Fastly dynamic backends kept for the sites outside the auction. +const FASTLY_NON_AUCTION_BACKEND_RESERVE: usize = 40; /// Validated operator-defined provider identifier. #[derive(Debug, Clone, Eq, Hash, Ord, PartialEq, PartialOrd, derive_more::Display)] @@ -255,7 +258,6 @@ pub struct ProviderPlan { #[derive(Debug, Clone)] pub struct AuctionPlan { enabled: bool, - timeout_ms: u32, providers: Vec, bidder_routes: BTreeMap, signing_enabled: bool, @@ -361,7 +363,6 @@ impl AuctionPlan { } Ok(Self { enabled: true, - timeout_ms: config.timeout_ms, providers, bidder_routes, signing_enabled, @@ -370,106 +371,7 @@ impl AuctionPlan { } } -impl ProviderPlan { - /// Build the canonical backend specification with the configured timeout. - #[must_use] - pub(crate) fn backend_spec(&self) -> PlatformBackendSpec { - self.backend_spec_with_transport_timeout(self.timeout_ms) - } - - /// Build the canonical backend specification with request-local transport timers. - #[must_use] - pub(crate) fn backend_spec_with_transport_timeout( - &self, - transport_timeout_ms: u32, - ) -> PlatformBackendSpec { - let endpoint = self.endpoint.url(); - let timeout = Duration::from_millis(u64::from(transport_timeout_ms)); - PlatformBackendSpec { - scheme: endpoint.scheme().to_owned(), - host: endpoint - .host_str() - .expect("should retain validated provider endpoint host") - .to_owned(), - port: endpoint.port(), - host_header_override: None, - certificate_check: true, - first_byte_timeout: timeout, - between_bytes_timeout: timeout, - discriminator: Some(self.id.as_str().to_owned()), - } - } -} - impl AuctionPlan { - /// Validate adapter capabilities and backend-name correlation before I/O. - /// - /// Each provider is predicted from a canonical backend specification using - /// its exact configured provider timeout as both transport timers and its - /// provider ID as the stable discriminator. These transport timers do not - /// replace the auction-wide logical budget. - /// - /// # Errors - /// - /// Returns a configuration error when the target cannot fan out to every - /// configured provider, backend prediction fails, or two predicted names - /// collide. - pub fn validate_for_target( - &self, - target_id: AuctionTargetId, - ) -> Result<(), Report> { - if !self.enabled { - return Ok(()); - } - let target = target_id.descriptor(); - if self.providers.len() > 1 && !target.capabilities().supports_concurrent_provider_fanout() - { - return Err(configuration_error(format!( - "auction target `{}` does not support concurrent provider fanout; configured {} providers", - target_id.adapter_id(), - self.providers.len() - ))); - } - - let naming_policy = target.naming_policy(); - let backend_budget = naming_policy.auction_dynamic_backend_budget(); - let mut required_backend_names = 0_usize; - let mut predicted_names = BTreeMap::::new(); - for provider in &self.providers { - let reachable_timeout_ms = provider.timeout_ms.min(self.timeout_ms); - required_backend_names = required_backend_names - .saturating_add(naming_policy.transport_timeout_bucket_count(reachable_timeout_ms)); - if let Some(budget) = backend_budget - && required_backend_names > budget - { - return Err(configuration_error(format!( - "auction target `{}` requires up to {required_backend_names} dynamic provider backends, exceeding its auction budget of {budget}", - target_id.adapter_id(), - ))); - } - let spec = provider.backend_spec(); - let prediction = - naming_policy - .predict(&spec) - .change_context(TrustedServerError::Configuration { - message: format!( - "provider `{}` backend prediction failed for target `{}`", - provider.id, - target_id.adapter_id() - ), - })?; - if let Some(existing) = predicted_names.insert(prediction.name.clone(), &provider.id) { - return Err(configuration_error(format!( - "providers `{existing}` and `{}` predict the same backend name `{}` for target `{}`", - provider.id, - prediction.name, - target_id.adapter_id() - ))); - } - } - Ok(()) - } - /// Borrow compiled providers in deterministic provider-ID order. #[must_use] pub fn providers(&self) -> &[ProviderPlan] { @@ -522,6 +424,75 @@ impl AuctionPlan { } } +/// Rejects an enabled plan whose providers need more Fastly backends than the auction may use. +/// +/// Each distinct tuple of provider scheme, host, port, and configured timeout is at most one +/// `EdgeZero` backend configuration on Fastly, because bidders set no host override and every +/// request carries the same connect timeout. The count can only overstate: `EdgeZero` keys +/// identity on the rounded timer, so two timeouts that round alike share a configuration but +/// count twice here. The auction may use 160 of Fastly's 200 dynamic backends, which keeps 40 for +/// the other outbound sites. A disabled auction registers no provider backend and passes. +/// +/// # Errors +/// +/// Returns [`TrustedServerError::Configuration`] when an enabled plan has more than 160 distinct +/// provider tuples. +/// +/// # Examples +/// +/// ``` +/// use trusted_server_core::auction::plan::{ +/// AuctionPlan, AuctionPlanConfig, validate_fastly_backend_bound, +/// }; +/// +/// let plan = AuctionPlan::compile(AuctionPlanConfig { +/// timeout_ms: 1000, +/// ..AuctionPlanConfig::default() +/// }) +/// .expect("should compile an empty plan"); +/// validate_fastly_backend_bound(&plan).expect("should fit an empty plan"); +/// ``` +pub fn validate_fastly_backend_bound(plan: &AuctionPlan) -> Result<(), Report> { + if !plan.enabled { + return Ok(()); + } + let bound = FASTLY_DYNAMIC_BACKEND_LIMIT - FASTLY_NON_AUCTION_BACKEND_RESERVE; + let tuples = plan + .providers + .iter() + .map(|provider| { + let endpoint = provider.endpoint.url(); + ( + endpoint.scheme(), + endpoint.host_str(), + endpoint.port_or_known_default(), + provider.timeout_ms, + ) + }) + .collect::>(); + if tuples.len() > bound { + return Err(configuration_error(format!( + "auction providers need {} Fastly backend configurations, exceeding the bound of {bound} ({FASTLY_DYNAMIC_BACKEND_LIMIT} dynamic backends less {FASTLY_NON_AUCTION_BACKEND_RESERVE} kept for other sites)", + tuples.len() + ))); + } + Ok(()) +} + +/// Reports whether `url` is plain HTTP to the loopback IP literal `127.0.0.1` or `[::1]`. +/// +/// Hosted runtimes cannot reach loopback, so these endpoints serve only local test bidders. +/// Names and other addresses, such as `localhost` or `127.0.0.2`, do not qualify. The port is +/// optional. +fn is_loopback_http_endpoint(url: &Url) -> bool { + url.scheme() == "http" + && match url.host() { + Some(Host::Ipv4(address)) => address == Ipv4Addr::LOCALHOST, + Some(Host::Ipv6(address)) => address == Ipv6Addr::LOCALHOST, + Some(Host::Domain(_)) | None => false, + } +} + fn default_profile() -> String { "standard".to_string() } @@ -569,17 +540,17 @@ fn canonicalize_endpoint( ) -> Result> { let mut endpoint = Url::parse(value).map_err(|error| { configuration_error(format!( - "provider `{provider_id}` endpoint must be an absolute HTTPS URL: {error}" + "provider `{provider_id}` endpoint must be absolute HTTPS, or plain HTTP to 127.0.0.1 or [::1]: {error}" )) })?; - if endpoint.scheme() != "https" + if !(endpoint.scheme() == "https" || is_loopback_http_endpoint(&endpoint)) || endpoint.host_str().is_none() || !endpoint.username().is_empty() || endpoint.password().is_some() || endpoint.fragment().is_some() { return Err(configuration_error(format!( - "provider `{provider_id}` endpoint must be absolute HTTPS with a host and no credentials or fragment" + "provider `{provider_id}` endpoint must be absolute HTTPS, or plain HTTP to 127.0.0.1 or [::1], with a host and no credentials or fragment" ))); } if profile_id == "aps" @@ -689,199 +660,6 @@ mod tests { value } - #[test] - fn target_validation_accepts_fanout_and_rejects_unsupported_targets() { - let providers = BTreeMap::from([ - (id("provider-one"), provider("standard")), - (id("provider-two"), provider("standard")), - ]); - let plan = AuctionPlan::compile(config(providers)).expect("should compile plan"); - - assert!( - plan.validate_for_target(crate::platform::AuctionTargetId::Fastly) - .is_ok(), - "Fastly should accept provider fanout" - ); - assert!( - plan.validate_for_target(crate::platform::AuctionTargetId::Axum) - .is_ok(), - "Axum should accept provider fanout" - ); - for target in [ - crate::platform::AuctionTargetId::Cloudflare, - crate::platform::AuctionTargetId::Spin, - ] { - let error = plan - .validate_for_target(target) - .expect_err("should reject unsupported provider fanout"); - assert!( - error.to_string().contains("fanout"), - "should explain fanout rejection: {error:?}" - ); - } - } - - #[test] - fn fastly_target_validation_canonicalizes_url_derived_ipv6_prediction() { - let mut ipv6_provider = provider("standard"); - ipv6_provider.endpoint = "https://[2001:db8::5]:8443/openrtb".to_string(); - ipv6_provider.timeout_ms = Some(750); - let plan = AuctionPlan::compile(config(BTreeMap::from([( - id("ipv6-provider"), - ipv6_provider, - )]))) - .expect("should compile IPv6 provider plan"); - - plan.validate_for_target(crate::platform::AuctionTargetId::Fastly) - .expect("Fastly target validation should accept canonical IPv6 prediction"); - let bracketed_spec = plan.providers()[0].backend_spec(); - assert_eq!(bracketed_spec.host, "[2001:db8::5]"); - let mut bare_spec = bracketed_spec.clone(); - bare_spec.host = "2001:db8::5".to_string(); - let policy = crate::platform::BackendNamingPolicy::Fastly; - assert_eq!( - policy - .predict(&bracketed_spec) - .expect("should predict URL-derived bracketed IPv6 backend"), - policy - .predict(&bare_spec) - .expect("should predict runtime-normalized bare IPv6 backend"), - "startup target validation and Fastly runtime must hash the same backend spec" - ); - } - - #[test] - fn fastly_target_validation_reserves_dynamic_backends_outside_auction() { - let plan_with = |count: usize| { - let providers = (0..count) - .map(|index| { - let mut provider = provider("standard"); - provider.timeout_ms = Some(1000); - (id(&format!("provider-{index}")), provider) - }) - .collect(); - AuctionPlan::compile(config(providers)).expect("should compile provider plan") - }; - - plan_with(19) - .validate_for_target(crate::platform::AuctionTargetId::Fastly) - .expect("below-budget provider plan should validate"); - plan_with(20) - .validate_for_target(crate::platform::AuctionTargetId::Fastly) - .expect("at-budget provider plan should validate"); - let error = plan_with(21) - .validate_for_target(crate::platform::AuctionTargetId::Fastly) - .expect_err("over-budget provider plan should fail"); - assert!( - error - .to_string() - .contains("exceeding its auction budget of 160") - ); - } - - #[test] - fn fastly_backend_quota_uses_auction_timeout_as_reachable_bucket_ceiling() { - let providers = (0..21) - .map(|index| { - let mut provider = provider("standard"); - provider.timeout_ms = Some(1000); - (id(&format!("provider-{index}")), provider) - }) - .collect(); - let mut bounded = config(providers); - bounded.timeout_ms = 100; - let plan = AuctionPlan::compile(bounded).expect("should compile bounded provider plan"); - - assert!( - plan.providers() - .iter() - .all(|provider| provider.timeout_ms == 1000), - "quota validation must not rewrite configured provider timeouts" - ); - plan.validate_for_target(crate::platform::AuctionTargetId::Fastly) - .expect("21 providers reach only the 50ms and 100ms Fastly buckets"); - } - - #[test] - fn disabled_target_validation_skips_fanout_and_collision_checks() { - let providers = BTreeMap::from([ - (id("provider-one"), provider("standard")), - (id("provider-two"), provider("standard")), - ]); - let disabled = AuctionPlan::compile(config(providers)) - .expect("should compile plan") - .with_enabled(false); - - for target in [ - crate::platform::AuctionTargetId::Cloudflare, - crate::platform::AuctionTargetId::Spin, - ] { - disabled - .validate_for_target(target) - .expect("disabled dormant providers should skip target validation"); - } - - let provider = disabled.providers[0].clone(); - let disabled_collision = AuctionPlan { - enabled: false, - timeout_ms: disabled.timeout_ms, - providers: vec![provider.clone(), provider], - bidder_routes: BTreeMap::new(), - signing_enabled: false, - mediator: None, - }; - disabled_collision - .validate_for_target(crate::platform::AuctionTargetId::Axum) - .expect("disabled dormant providers should skip collision validation"); - } - - #[test] - fn target_validation_keeps_same_origin_timeout_profile_instances_distinct() { - let shared = ProviderConfig { - timeout_ms: Some(777), - ..provider("standard") - }; - let plan = AuctionPlan::compile(config(BTreeMap::from([ - (id("provider-one"), shared.clone()), - (id("provider-two"), shared), - ]))) - .expect("should compile same-origin provider instances"); - - for target in [ - crate::platform::AuctionTargetId::Fastly, - crate::platform::AuctionTargetId::Axum, - ] { - plan.validate_for_target(target) - .expect("provider ID discriminators should prevent predicted collisions"); - } - } - - #[test] - fn target_validation_rejects_predicted_name_collisions() { - let compiled = AuctionPlan::compile(config(BTreeMap::from([( - id("provider-a"), - provider("standard"), - )]))) - .expect("should compile plan"); - let provider = compiled.providers[0].clone(); - // The compiler prevents duplicate provider IDs. Construct the otherwise - // impossible duplicate internally to pin validation's defense-in-depth - // collision rejection independently of compiler invariants. - let collision_plan = AuctionPlan { - enabled: true, - timeout_ms: compiled.timeout_ms, - providers: vec![provider.clone(), provider], - bidder_routes: BTreeMap::new(), - signing_enabled: false, - mediator: None, - }; - - let error = collision_plan - .validate_for_target(crate::platform::AuctionTargetId::Axum) - .expect_err("should reject predicted backend collision"); - assert!(error.to_string().contains("same backend name")); - } - #[test] fn provider_id_enforces_exact_grammar_and_bounds() { for valid in ["a", "pbs-primary", &format!("a{}", "0".repeat(62))] { @@ -1123,6 +901,127 @@ mod tests { assert!(AuctionPlan::compile(config(BTreeMap::from([(id("aps"), aps)]))).is_err()); } + /// Compiles one standard provider per `(endpoint, timeout_ms)` pair. + fn plan_with_providers(endpoints: impl IntoIterator) -> AuctionPlan { + let providers = endpoints + .into_iter() + .enumerate() + .map(|(index, (endpoint, timeout_ms))| { + let mut raw_provider = provider("standard"); + raw_provider.endpoint = endpoint; + raw_provider.timeout_ms = Some(timeout_ms); + (id(&format!("provider-{index}")), raw_provider) + }) + .collect(); + AuctionPlan::compile(config(providers)).expect("should compile the provider plan") + } + + /// One endpoint per port, so every provider has its own backend tuple. + fn distinct_tuples(count: u16) -> impl Iterator { + (0..count).map(|index| { + ( + format!("https://bid.example:{}/openrtb", 9000 + index), + 1000, + ) + }) + } + + #[test] + fn fastly_backend_bound_accepts_160_and_rejects_161_tuples() { + validate_fastly_backend_bound(&plan_with_providers(distinct_tuples(160))) + .expect("should accept 160 distinct provider backend tuples"); + + let error = validate_fastly_backend_bound(&plan_with_providers(distinct_tuples(161))) + .expect_err("should reject 161 distinct provider backend tuples"); + assert!( + error.to_string().contains("161") && error.to_string().contains("160"), + "should name the tuple count and the bound: {error}" + ); + } + + #[test] + fn providers_sharing_a_tuple_count_once() { + // The extra provider differs from the first only in its path, which is + // not part of backend identity. + let shared = ("https://bid.example:9000/other-path".to_string(), 1000); + let plan = plan_with_providers(distinct_tuples(160).chain([shared])); + + assert_eq!(plan.providers().len(), 161, "should compile 161 providers"); + validate_fastly_backend_bound(&plan) + .expect("should count two providers that share a tuple once"); + + let other_timeout = ("https://bid.example:9000/openrtb".to_string(), 999); + let error = validate_fastly_backend_bound(&plan_with_providers( + distinct_tuples(160).chain([other_timeout]), + )) + .expect_err("should count a different configured timeout as its own tuple"); + assert!( + error.to_string().contains("161"), + "should count the timeout in the tuple: {error}" + ); + } + + #[test] + fn disabled_auction_skips_backend_bound() { + let disabled = plan_with_providers(distinct_tuples(161)).with_enabled(false); + + validate_fastly_backend_bound(&disabled) + .expect("should skip the bound while the auction is disabled"); + } + + fn compile_endpoint(endpoint: &str) -> Result> { + let mut raw_provider = provider("standard"); + raw_provider.endpoint = endpoint.to_string(); + AuctionPlan::compile(config(BTreeMap::from([(id("loopback"), raw_provider)]))) + } + + #[test] + fn loopback_http_endpoints_are_accepted() { + for endpoint in [ + "http://127.0.0.1:8080/bid", + "http://127.0.0.1/bid", + "http://[::1]:8080/bid", + "http://[::1]/bid", + ] { + let plan = compile_endpoint(endpoint) + .unwrap_or_else(|error| panic!("should accept {endpoint}: {error:?}")); + assert_eq!( + plan.providers()[0].endpoint.as_str(), + endpoint, + "should keep the loopback endpoint as configured" + ); + } + } + + #[test] + fn loopback_lookalike_endpoints_are_rejected() { + for endpoint in [ + "http://localhost:8080/bid", + "http://127.0.0.2:8080/bid", // allow-domain: 127.0.0.2 + "http://0.0.0.0:8080/bid", // allow-domain: 0.0.0.0 + "http://user@127.0.0.1:8080/bid", + "http://127.0.0.1:8080/bid#fragment", + ] { + assert!( + compile_endpoint(endpoint).is_err(), + "should reject plain HTTP to {endpoint}" + ); + } + } + + #[test] + fn unparsable_endpoint_error_states_the_loopback_rule() { + let Err(error) = compile_endpoint("not a URL") else { + panic!("should reject an unparsable endpoint"); + }; + assert!( + error + .to_string() + .contains("must be absolute HTTPS, or plain HTTP to 127.0.0.1 or [::1]"), + "should state the loopback-aware endpoint rule: {error}" + ); + } + #[test] fn compiler_normalizes_only_prebid_server_origin_and_canonical_paths() { for (configured, expected) in [ diff --git a/crates/trusted-server-core/src/auction/provider.rs b/crates/trusted-server-core/src/auction/provider.rs index d68ebde4d8..0aeea25243 100644 --- a/crates/trusted-server-core/src/auction/provider.rs +++ b/crates/trusted-server-core/src/auction/provider.rs @@ -2,20 +2,22 @@ use core::any::Any; use std::collections::HashSet; +use std::time::Duration; use async_trait::async_trait; -use edgezero_core::body::Body as EdgeBody; +use edgezero_core::outbound::{ + OutboundCachePolicy, OutboundRequest, OutboundResponse, ResponseDecoding, +}; use error_stack::{Report, ResultExt as _}; -use http::{Method, Request, StatusCode, header}; +use http::header::HeaderValue; +use http::{Method, StatusCode, header}; use serde_json::{Value, json}; use crate::integrations::aps::{ApsDebugRequest, parse_planned_aps_response}; use crate::integrations::prebid::{apply_prebid_transport_headers, parse_planned_prebid_response}; use crate::error::TrustedServerError; -use crate::platform::{ - PlatformHttpRequest, PlatformPendingRequest, PlatformResponse, RuntimeServices, -}; +use crate::outbound::{OutboundRequestSource, buffered, outbound_error, outbound_request}; use crate::request_signing::{RequestSigner, SigningParams}; use super::openrtb::{ @@ -27,7 +29,8 @@ use super::profile::CompiledOpenRtbProfile; use super::routing::{ProviderAuctionInput, RoutedAuction}; use super::types::{AuctionContext, AuctionRequest, AuctionResponse}; -const MAX_PLANNED_RESPONSE_BYTES: usize = 1024 * 1024; +/// Response cap for every bidder request. +const MAX_PLANNED_RESPONSE_BYTES: u64 = 1024 * 1024; fn attach_provider_routing_metadata( response: &mut AuctionResponse, @@ -44,11 +47,15 @@ fn attach_provider_routing_metadata( pub type ProviderParseState = Box; /// Result of asking a provider to start a bid request. +#[allow( + clippy::large_enum_variant, + reason = "the outcome moves once from the provider to the orchestrator, so boxing the request would only add an allocation" +)] pub enum ProviderRequestOutcome { - /// An upstream request is in flight and must be awaited by the orchestrator. - Pending { - /// Platform-specific pending request handle. - request: PlatformPendingRequest, + /// An upstream request built and unsent, which the orchestrator sends. + Ready { + /// The request, with its transport settings, ready for `EdgeZero`'s client. + request: OutboundRequest, /// Optional provider-local state consumed when the response is parsed. parse_state: Option, }, @@ -57,22 +64,19 @@ pub enum ProviderRequestOutcome { } impl ProviderRequestOutcome { - /// Wrap an ordinary pending provider request without parse state. + /// Wrap an unsent provider request without parse state. #[must_use] - pub fn pending(request: PlatformPendingRequest) -> Self { - Self::Pending { + pub fn ready(request: OutboundRequest) -> Self { + Self::Ready { request, parse_state: None, } } - /// Wrap a pending provider request with provider-local parse state. + /// Wrap an unsent provider request with provider-local parse state. #[must_use] - pub fn pending_with_state( - request: PlatformPendingRequest, - parse_state: ProviderParseState, - ) -> Self { - Self::Pending { + pub fn ready_with_state(request: OutboundRequest, parse_state: ProviderParseState) -> Self { + Self::Ready { request, parse_state: Some(parse_state), } @@ -88,20 +92,17 @@ pub trait AuctionProvider: Send + Sync { /// return their owned operator-defined [`super::plan::ProviderId`]. fn provider_name(&self) -> &str; - /// Submit a bid request to this provider. + /// Build a bid request for this provider, without sending it. /// - /// Implementations normally return a pending upstream request, but may return - /// an immediate response for a legitimate outcome that requires no HTTP call. - /// The orchestrator handles waiting for and parsing pending responses. - /// Outbound auction and mediator requests must use - /// [`PlatformHttpRequest::with_cache_bypass`]: they carry personalized data, - /// and Fastly requires this marker for deadline-bounded collection. + /// Implementations normally return [`ProviderRequestOutcome::Ready`], but may + /// return an immediate response for a legitimate outcome that requires no + /// HTTP call. The orchestrator sends the request and parses its response. + /// Auction and mediator requests set [`OutboundCachePolicy::Bypass`], + /// because they carry personalized data. /// /// # Errors /// - /// Returns an error if the request cannot be created or if the provider endpoint - /// cannot be reached (though usually network errors happen while the returned - /// [`PlatformPendingRequest`] is polled). + /// Returns an error if the request cannot be built. async fn request_bids( &self, request: &AuctionRequest, @@ -110,16 +111,14 @@ pub trait AuctionProvider: Send + Sync { /// Parse the response from the provider into an `AuctionResponse`. /// - /// Called by the orchestrator after the [`PlatformPendingRequest`] completes. - /// Declared async so implementations can safely drain streaming response bodies - /// without panicking on the `Body::Stream` variant. + /// Called by the orchestrator once the provider's request completes. /// /// # Errors /// /// Returns an error if the response cannot be parsed into a valid `AuctionResponse`. async fn parse_response( &self, - response: PlatformResponse, + response: OutboundResponse, response_time_ms: u64, ) -> Result>; @@ -136,7 +135,7 @@ pub trait AuctionProvider: Send + Sync { /// Returns an error if the response cannot be parsed into a valid [`AuctionResponse`]. async fn parse_response_with_context( &self, - response: PlatformResponse, + response: OutboundResponse, response_time_ms: u64, request: &AuctionRequest, context: &AuctionContext<'_>, @@ -155,7 +154,7 @@ pub trait AuctionProvider: Send + Sync { /// Returns an error if the response cannot be parsed into a valid [`AuctionResponse`]. async fn parse_response_with_context_and_state( &self, - response: PlatformResponse, + response: OutboundResponse, response_time_ms: u64, request: &AuctionRequest, context: &AuctionContext<'_>, @@ -179,17 +178,6 @@ pub trait AuctionProvider: Send + Sync { fn is_enabled(&self) -> bool { true } - - /// Return the backend name used by this provider for request routing. - /// - /// `timeout_ms` is the effective timeout that will be used when the backend - /// is registered in [`request_bids`](Self::request_bids). It must be - /// forwarded to [`crate::platform::PlatformBackend::predict_name`] through - /// `services` so the predicted name matches the actual platform backend - /// registration. - fn backend_name(&self, _services: &RuntimeServices, _timeout_ms: u32) -> Option { - None - } } /// One immutable config-first `OpenRTB` provider instance. @@ -257,28 +245,24 @@ impl GenericOpenRtbProvider { Box::new(state) } - /// Build, register, and start exactly one routed provider request. + /// Build exactly one routed provider request, unsent. /// /// The public [`AuctionProvider::request_bids`] seam cannot carry a /// [`ProviderAuctionInput`], the complete [`RoutedAuction`], or an /// auction-local [`RequestSigner`] without shared mutable provider state. - /// The plan-backed split dispatcher therefore supplies that explicit - /// execution context here, while this method reuses - /// [`ProviderRequestOutcome`] and [`ProviderParseState`] for the existing - /// request/parse token boundary. - #[allow( - clippy::too_many_arguments, - reason = "the internal driver keeps routed inputs, both budgets, signer, services, and collision state explicit" - )] - pub(crate) async fn request_bids_routed( + /// The plan-backed dispatcher therefore supplies that explicit execution + /// context here, while this method reuses [`ProviderRequestOutcome`] and + /// [`ProviderParseState`] for the request and parse token boundary. + /// + /// The request carries the configured provider timeout, never the + /// remaining auction budget, which only `logical_budget_ms` and the batch + /// cutoff reflect. Its response is buffered under 1 MiB and decoded. + pub(crate) fn request_bids_routed( &self, input: &ProviderAuctionInput, routed: &RoutedAuction, logical_budget_ms: u32, - transport_timeout_ms: u32, signer: Option<&RequestSigner>, - services: &RuntimeServices, - reserved_backend_names: &mut HashSet, ) -> Result> { let signing_params = SigningParams::new( input.common_request().id.clone(), @@ -304,101 +288,58 @@ impl GenericOpenRtbProvider { } }; - let spec = self - .plan - .backend_spec_with_transport_timeout(transport_timeout_ms); - let predicted_name = - services - .backend() - .predict_name(&spec) - .change_context(TrustedServerError::Auction { - message: format!( - "Provider {} backend prediction failed", - self.provider_name() - ), - })?; - let backend_name = - services - .backend() - .ensure(&spec) - .change_context(TrustedServerError::Auction { - message: format!( - "Provider {} backend registration failed", - self.provider_name() - ), - })?; - if backend_name != predicted_name { - return Err(Report::new(TrustedServerError::Auction { - message: format!( - "Provider {} backend registration did not match prediction", - self.provider_name() - ), - })); - } - if !reserved_backend_names.insert(backend_name.clone()) { - return Err(Report::new(TrustedServerError::Auction { - message: format!( - "Provider {} resolved an actual backend name already owned by another provider", - self.provider_name() - ), - })); - } - let body = serde_json::to_vec(&request).change_context(TrustedServerError::Auction { message: format!( "Provider {} request serialization failed", self.provider_name() ), })?; - let mut outbound = Request::builder() - .method(Method::POST) - .uri(self.plan.endpoint.as_str()) - .header(header::CONTENT_TYPE, "application/json"); - if matches!(&self.plan.profile, CompiledOpenRtbProfile::Standard(_)) { - outbound = outbound.header(header::ACCEPT, "application/json"); - } + let target = + outbound_request(Method::POST, self.plan.endpoint.as_str()).map_err(|error| { + outbound_error( + error, + TrustedServerError::Auction { + message: format!( + "Provider {} request construction failed", + self.provider_name() + ), + }, + OutboundRequestSource::Internal, + ) + })?; let aps_debug_body = matches!( &self.plan.profile, CompiledOpenRtbProfile::Aps(profile) if profile.debug ) .then(|| body.clone()); - let mut outbound = + let mut outbound = buffered( + target, + Duration::from_millis(u64::from(self.plan.timeout_ms)), + MAX_PLANNED_RESPONSE_BYTES, + ResponseDecoding::Decode, + ) + .cache_policy(OutboundCachePolicy::Bypass) + .body(body); + outbound.headers_mut().insert( + header::CONTENT_TYPE, + HeaderValue::from_static("application/json"), + ); + if matches!(&self.plan.profile, CompiledOpenRtbProfile::Standard(_)) { outbound - .body(EdgeBody::from(body)) - .change_context(TrustedServerError::Auction { - message: format!( - "Provider {} request construction failed", - self.provider_name() - ), - })?; + .headers_mut() + .insert(header::ACCEPT, HeaderValue::from_static("application/json")); + } let aps_debug_request = aps_debug_body .as_deref() .map(|body| ApsDebugRequest::capture(body, outbound.headers())); if let CompiledOpenRtbProfile::PrebidServer(profile) = &self.plan.profile { apply_prebid_transport_headers( routed.prebid_transport_headers(), - &mut outbound, + outbound.headers_mut(), profile.consent_forwarding, routed.attested_client_ip(), ); } - let pending = services - .http_client() - .send_async( - PlatformHttpRequest::new(outbound, backend_name.clone()).with_cache_bypass(), - ) - .await - .change_context(TrustedServerError::Auction { - message: format!("Provider {} request launch failed", self.provider_name()), - })?; - if pending.backend_name() != Some(backend_name.as_str()) { - return Err(Report::new(TrustedServerError::Auction { - message: format!( - "Provider {} pending request backend did not match registered backend", - self.provider_name() - ), - })); - } let parse_state = match &self.plan.profile { CompiledOpenRtbProfile::Standard(_) => GenericOpenRtbParseState::Standard { provider_id: self.provider_name().to_string(), @@ -428,8 +369,8 @@ impl GenericOpenRtbProvider { debug_request: aps_debug_request, }, }; - Ok(ProviderRequestOutcome::pending_with_state( - pending, + Ok(ProviderRequestOutcome::ready_with_state( + outbound, Box::new(parse_state), )) } @@ -437,7 +378,7 @@ impl GenericOpenRtbProvider { /// Parse a response using state created by this exact provider instance. pub(crate) async fn parse_response_with_state( &self, - response: PlatformResponse, + response: OutboundResponse, response_time_ms: u64, parse_state: Option<&(dyn Any + Send + Sync)>, ) -> Result> { @@ -548,7 +489,6 @@ impl GenericOpenRtbProvider { return Ok(parsed); } - let response = response.response; let status = response.status(); let GenericOpenRtbParseState::Standard { input, .. } = parse_state else { unreachable!("profile-specific states are handled before standard parsing"); @@ -573,11 +513,16 @@ impl GenericOpenRtbProvider { } let body = response - .into_body() .into_bytes_bounded(MAX_PLANNED_RESPONSE_BYTES) .await - .change_context(TrustedServerError::Auction { - message: format!("Provider {} response body failed", self.provider_name()), + .map_err(|error| { + outbound_error( + error, + TrustedServerError::Auction { + message: format!("Provider {} response body failed", self.provider_name()), + }, + OutboundRequestSource::Internal, + ) })?; let value: Value = match serde_json::from_slice(&body) { Ok(value) => value, diff --git a/crates/trusted-server-core/src/auction/telemetry.rs b/crates/trusted-server-core/src/auction/telemetry.rs index d718ee0765..c3d459146b 100644 --- a/crates/trusted-server-core/src/auction/telemetry.rs +++ b/crates/trusted-server-core/src/auction/telemetry.rs @@ -222,7 +222,11 @@ impl AuctionObservationContext { } #[cfg(test)] - fn for_test(auction_source: AuctionSource, page_path: &str, slot_count: u16) -> Self { + pub(crate) fn for_test( + auction_source: AuctionSource, + page_path: &str, + slot_count: u16, + ) -> Self { Self { auction_id: Uuid::new_v4(), auction_source, diff --git a/crates/trusted-server-core/src/auction/test_support.rs b/crates/trusted-server-core/src/auction/test_support.rs index a83f8899a8..fe3b436ad8 100644 --- a/crates/trusted-server-core/src/auction/test_support.rs +++ b/crates/trusted-server-core/src/auction/test_support.rs @@ -27,7 +27,6 @@ pub(crate) fn create_test_auction_context<'a>( settings, request, timeout_ms, - transport_timeout_ms: timeout_ms, provider_responses: None, services, } diff --git a/crates/trusted-server-core/src/auction/types.rs b/crates/trusted-server-core/src/auction/types.rs index 4069157068..1db3e8f782 100644 --- a/crates/trusted-server-core/src/auction/types.rs +++ b/crates/trusted-server-core/src/auction/types.rs @@ -131,8 +131,8 @@ pub struct SiteInfo { /// In the **collect path** ([`collect_dispatched_auction`][collect]) the /// mediator is invoked with a synthetic placeholder request /// (`https://placeholder.invalid/`), because the real client request has -/// already been consumed by `send_async` during dispatch and the host pipeline -/// can't lend it across the `.await`. **Mediators must not depend on reading +/// already been consumed by the publisher origin request during dispatch and +/// the host pipeline can't lend it across the `.await`. **Mediators must not depend on reading /// client state from `context.request`** — the placeholder has none of the /// real headers. If a future mediator needs that data, snapshot it into a new /// field on this struct at dispatch time and stash it on the @@ -147,13 +147,10 @@ pub struct AuctionContext<'a> { pub settings: &'a Settings, pub request: &'a Request, /// Exact logical provider budget used by auction policy and payloads. - pub timeout_ms: u32, - /// Canonical backend transport timeout used for provider registration. /// - /// This can be lower than `timeout_ms` on runtimes whose backend names - /// encode timers. Providers that register a backend should use this value - /// for transport timers while retaining `timeout_ms` for logical policy. - pub transport_timeout_ms: u32, + /// Requests never take their transport timeout from it. Each provider + /// sets its own configured timeout, and the auction deadline bounds it. + pub timeout_ms: u32, /// Provider responses from the bidding phase, used by mediators. /// This is `None` for regular bidders and `Some` when calling a mediator. pub provider_responses: Option<&'a [AuctionResponse]>, diff --git a/crates/trusted-server-core/src/auth.rs b/crates/trusted-server-core/src/auth.rs index e563fd6163..9cf5b85ea6 100644 --- a/crates/trusted-server-core/src/auth.rs +++ b/crates/trusted-server-core/src/auth.rs @@ -1,8 +1,9 @@ use base64::{Engine as _, engine::general_purpose::STANDARD}; use edgezero_core::body::Body as EdgeBody; +use edgezero_core::http::RequestParts; use error_stack::Report; use http::header; -use http::{Request, Response, StatusCode}; +use http::{HeaderMap, Request, Response, StatusCode}; use sha2::{Digest as _, Sha256}; use subtle::ConstantTimeEq as _; @@ -62,7 +63,7 @@ impl EdgeTerminatedAuthorization { /// /// # Request mutation /// -/// Takes `req` mutably because it owns [`EdgeTerminatedAuthorization`]. Any +/// Takes the request head mutably because it owns [`EdgeTerminatedAuthorization`]. Any /// inherited marker is cleared on entry, and a fresh one is inserted only on the /// success path, so the marker present after this call always describes this /// call's own decision. Nothing else about the request is touched — in @@ -82,15 +83,15 @@ impl EdgeTerminatedAuthorization { /// un-compilable path regex. pub fn enforce_basic_auth( settings: &Settings, - req: &mut Request, + req: &mut RequestParts, ) -> Result>, Report> { // Cleared before any early return so no inherited marker can survive a call // that did not itself validate a credential. Without this, a request marked // upstream and then routed to an unprotected path would keep an assertion // nothing checked. - req.extensions_mut().remove::(); + req.extensions.remove::(); - let path = req.uri().path(); + let path = req.uri.path(); let Some(handler) = settings.handler_for_path(path)? else { if Settings::is_admin_path(path) { return Err(Report::new(TrustedServerError::Configuration { @@ -100,7 +101,7 @@ pub fn enforce_basic_auth( return Ok(None); }; - let Some((username, password, authorization_digest)) = extract_credentials(req) else { + let Some((username, password, authorization_digest)) = extract_credentials(&req.headers) else { return Ok(Some(unauthorized_response())); }; @@ -119,14 +120,14 @@ pub fn enforce_basic_auth( if bool::from(username_match & password_match) { // Record that TS itself consumed this credential, so the shared template // cache can distinguish it from a credential meant for the origin. - req.extensions_mut() + req.extensions .insert(EdgeTerminatedAuthorization(authorization_digest)); Ok(None) } else { - if req.extensions().get::().is_some() { + if req.extensions.get::().is_some() { log::warn!("trace_auth_failed"); } else { - log::warn!("Basic auth failed for path: {}", req.uri().path()); + log::warn!("Basic auth failed for path: {}", req.uri.path()); } Ok(Some(unauthorized_response())) } @@ -142,11 +143,11 @@ pub fn enforce_basic_auth( /// operator credential was used. #[must_use] pub fn authenticated_username(req: &Request) -> Option { - extract_credentials(req).map(|(username, _password, _digest)| username) + extract_credentials(req.headers()).map(|(username, _password, _digest)| username) } -fn extract_credentials(req: &Request) -> Option<(String, String, [u8; 32])> { - let mut header_values = req.headers().get_all(header::AUTHORIZATION).iter(); +fn extract_credentials(headers: &HeaderMap) -> Option<(String, String, [u8; 32])> { + let mut header_values = headers.get_all(header::AUTHORIZATION).iter(); let header_value = header_values.next()?; if header_values.next().is_some() { return None; @@ -192,16 +193,18 @@ mod tests { use crate::test_support::tests::{crate_test_settings_str, create_test_settings}; - fn build_request(method: Method, uri: &str) -> Request { - Request::builder() + fn build_request(method: Method, uri: &str) -> RequestParts { + let (parts, _body) = Request::builder() .method(method) .uri(uri) .body(EdgeBody::empty()) .expect("should build request") + .into_parts(); + parts } - fn set_authorization(req: &mut Request, value: &str) { - req.headers_mut().insert( + fn set_authorization(req: &mut RequestParts, value: &str) { + req.headers.insert( header::AUTHORIZATION, HeaderValue::from_str(value).expect("should build authorization header"), ); @@ -244,24 +247,24 @@ mod tests { "valid credentials should be admitted" ); let marker = req - .extensions() + .extensions .get::() .expect("should mark a credential this edge consumed"); assert!( - marker.matches(req.headers()), + marker.matches(&req.headers), "the marker should match the unchanged validated authorization" ); - req.headers_mut().insert( + req.headers.insert( header::AUTHORIZATION, HeaderValue::from_static("Bearer publisher-origin-credential"), ); let marker = req - .extensions() + .extensions .get::() .expect("should retain the marker after an unrelated mutation"); assert!( - !marker.matches(req.headers()), + !marker.matches(&req.headers), "the marker must not match a replacement authorization value" ); } @@ -272,7 +275,7 @@ mod tests { let mut req = build_request(Method::GET, "https://example.com/secure"); let encoded = STANDARD.encode("user:pass"); set_authorization(&mut req, &format!("Basic {encoded}")); - req.headers_mut().append( + req.headers.append( header::AUTHORIZATION, HeaderValue::from_static("Bearer publisher-origin-credential"), ); @@ -282,7 +285,7 @@ mod tests { .expect("should challenge an ambiguous credential"); assert_eq!(response.status(), StatusCode::UNAUTHORIZED); assert!( - req.extensions() + req.extensions .get::() .is_none(), "a repeated authorization field must remain pass-through rather than being marked safe" @@ -297,7 +300,7 @@ mod tests { let settings = create_test_settings(); let mut req = build_request(Method::GET, "https://example.com/open"); set_authorization(&mut req, "Basic dXNlcjpwYXNz"); - req.extensions_mut() + req.extensions .insert(EdgeTerminatedAuthorization::for_test("Basic dXNlcjpwYXNz")); assert!( @@ -307,7 +310,7 @@ mod tests { "an unprotected path should not challenge" ); assert!( - req.extensions() + req.extensions .get::() .is_none(), "a marker no credential check produced must not survive this call" @@ -320,17 +323,16 @@ mod tests { let mut req = build_request(Method::GET, "https://example.com/secure"); let encoded = STANDARD.encode("user:wrong-pass"); set_authorization(&mut req, &format!("Basic {encoded}")); - req.extensions_mut() - .insert(EdgeTerminatedAuthorization::for_test( - "Basic dXNlcjp3cm9uZy1wYXNz", - )); + req.extensions.insert(EdgeTerminatedAuthorization::for_test( + "Basic dXNlcjp3cm9uZy1wYXNz", + )); let response = enforce_basic_auth(&settings, &mut req) .expect("should evaluate auth") .expect("should challenge"); assert_eq!(response.status(), StatusCode::UNAUTHORIZED); assert!( - req.extensions() + req.extensions .get::() .is_none(), "a failed check must strip an inherited marker rather than honour it" @@ -350,7 +352,7 @@ mod tests { "an unprotected path should not challenge" ); assert!( - req.extensions() + req.extensions .get::() .is_none(), "a credential no handler consumed is pass-through and must stay disqualifying" @@ -369,7 +371,7 @@ mod tests { .expect("should challenge"); assert_eq!(response.status(), StatusCode::UNAUTHORIZED); assert!( - req.extensions() + req.extensions .get::() .is_none(), "a failed credential must never be marked as terminated" diff --git a/crates/trusted-server-core/src/cache_purge.rs b/crates/trusted-server-core/src/cache_purge.rs index f77f43e4dd..c2b5871aaa 100644 --- a/crates/trusted-server-core/src/cache_purge.rs +++ b/crates/trusted-server-core/src/cache_purge.rs @@ -23,7 +23,7 @@ use http::{Method, Request, Response, StatusCode, header}; use serde::{Deserialize, Serialize}; use crate::error::TrustedServerError; -use crate::http_util::enforce_max_body_size; +use crate::http_util::read_request_body; use crate::platform::{RuntimeServices, reader_url_surrogate_key}; /// Purge bodies name a scope and, for a URL purge, one URL. Nothing here is unbounded. @@ -107,8 +107,9 @@ struct PurgeResponse { /// /// # Errors /// -/// Returns an error when the body cannot be read, exceeds [`PURGE_MAX_BODY_BYTES`], is not -/// valid JSON for a known scope, or when the platform's purge fails. +/// Returns an error when the body cannot be read in time, exceeds [`PURGE_MAX_BODY_BYTES`] (see +/// [`read_request_body`]), is not valid JSON for a known scope, or when the platform's purge +/// fails. pub async fn handle_cache_purge( services: &RuntimeServices, req: Request, @@ -132,14 +133,8 @@ pub async fn handle_cache_purge( )); } - let body = req.into_body(); - if body.is_stream() { - return Err(Report::new(TrustedServerError::BadRequest { - message: "cache-purge request body must be buffered, not streamed".into(), - })); - } - let bytes = body.into_bytes().unwrap_or_default(); - enforce_max_body_size(&bytes, PURGE_MAX_BODY_BYTES, "cache-purge")?; + let (parts, body) = req.into_parts(); + let bytes = read_request_body(&parts.headers, body, PURGE_MAX_BODY_BYTES).await?; let request = PurgeRequest::parse(&bytes).map_err(|message| TrustedServerError::BadRequest { @@ -239,6 +234,8 @@ fn json_response(status: StatusCode, body: &str) -> Response { #[cfg(test)] mod tests { use super::*; + use crate::platform::test_support::noop_services; + use crate::test_support::body_read::body_read_failure_cases; #[test] fn content_type_must_be_exactly_json() { @@ -370,4 +367,46 @@ mod tests { ); } } + /// Returns a JSON `POST` request builder for the purge route. + fn purge_builder() -> http::request::Builder { + Request::builder() + .method(Method::POST) + .uri("https://ts.example.com/_ts/admin/cache/purge") + .header(header::CONTENT_TYPE, "application/json") + } + + #[test] + fn cache_purge_body_read_failures_map_to_statuses() { + let services = noop_services(); + + for case in body_read_failure_cases(PURGE_MAX_BODY_BYTES) { + let (req, expectation) = case.into_request(purge_builder()); + + let outcome = futures::executor::block_on(handle_cache_purge(&services, req, None)); + + expectation.assert_outcome("cache purge", &outcome); + } + } + + #[test] + fn cache_purge_reads_a_streamed_body() { + let services = noop_services(); + let req = purge_builder() + .body(EdgeBody::stream(futures::stream::iter([ + bytes::Bytes::from_static(br#"{"scope":"#), + bytes::Bytes::from_static(br#""nope"}"#), + ]))) + .expect("should build the streamed purge request"); + + let error = futures::executor::block_on(handle_cache_purge(&services, req, None)) + .expect_err("should reject the unknown scope it read"); + + assert!( + matches!( + error.current_context(), + TrustedServerError::BadRequest { message } if message.contains("unknown scope") + ), + "should parse the streamed body rather than reject it unread, got {error:?}" + ); + } } diff --git a/crates/trusted-server-core/src/config.rs b/crates/trusted-server-core/src/config.rs index 50d5e9d8a9..4a66d3a180 100644 --- a/crates/trusted-server-core/src/config.rs +++ b/crates/trusted-server-core/src/config.rs @@ -33,6 +33,7 @@ use crate::integrations::{ sourcepoint::SourcepointConfig, testlight::TestlightConfig, }; +use crate::response_privacy::is_forbidden_operator_response_header; use crate::settings::{AssetOriginAuth, IntegrationConfig, IntegrationSettings, Settings}; const DEPLOY_VALIDATION_FIELD: &str = "trusted_server"; @@ -322,6 +323,10 @@ pub fn validate_settings_for_deploy(settings: &Settings) -> Result<(), Report Result<(), Report Result<(), Report> { + warn_ignored_certificate_check(settings); + settings.warn_ignored_origin_readthrough(); for id in unknown_integration_ids(settings) { log::warn!( "Unknown integration ID '{id}' is ignored; check the integration name or remove its configuration" @@ -359,6 +366,105 @@ pub fn validate_settings_for_runtime( Ok(()) } +/// Rejects `proxy.certificate_check = false`, because origin TLS is always verified. +/// +/// # Errors +/// +/// Returns [`TrustedServerError::Configuration`] when the setting is `false`. +fn reject_disabled_certificate_check( + settings: &Settings, +) -> Result<(), Report> { + if settings.proxy.certificate_check { + return Ok(()); + } + Err(Report::new(TrustedServerError::Configuration { + message: "proxy.certificate_check = false is not supported; origin TLS is always verified" + .to_owned(), + })) +} + +/// Rejects `creative_opportunities.origin_readthrough_enabled = true`, because origin +/// readthrough is unavailable. +/// +/// # Errors +/// +/// Returns [`TrustedServerError::Configuration`] when the setting is `true`. +fn reject_origin_readthrough(settings: &Settings) -> Result<(), Report> { + let enabled = settings + .creative_opportunities + .as_ref() + .is_some_and(|config| config.origin_readthrough_enabled == Some(true)); + if !enabled { + return Ok(()); + } + Err(Report::new(TrustedServerError::Configuration { + message: "creative_opportunities.origin_readthrough_enabled = true is not supported; origin readthrough is unavailable" + .to_owned(), + })) +} + +/// Rejects a `publisher.origin_url` that carries userinfo. +/// +/// The publisher forward refuses such a target, so every publisher request would +/// answer 400. The error never echoes the URL, which holds the credentials. +/// +/// # Errors +/// +/// Returns [`TrustedServerError::Configuration`] when the URL has a username or password. +fn reject_origin_url_userinfo(settings: &Settings) -> Result<(), Report> { + let has_userinfo = url::Url::parse(&settings.publisher.origin_url) + .is_ok_and(|url| !url.username().is_empty() || url.password().is_some()); + if !has_userinfo { + return Ok(()); + } + Err(Report::new(TrustedServerError::Configuration { + message: + "publisher.origin_url must not contain userinfo; put credentials in a header instead" + .to_owned(), + })) +} + +/// Rejects operator `response_headers` entries that Trusted Server never sends: +/// `Trailer`, `Content-Length`, and the hop-by-hop fields. +/// +/// Response egress owns message framing, so such an entry would fail the +/// response or silently change it. Runtime skips one with a warning instead. +/// +/// # Errors +/// +/// Returns [`TrustedServerError::Configuration`] naming every such entry. +fn reject_forbidden_operator_response_headers( + settings: &Settings, +) -> Result<(), Report> { + let mut names: Vec = settings + .response_headers + .keys() + .filter(|name| is_forbidden_operator_response_header(name)) + .map(|name| format!("`{name}`")) + .collect(); + if names.is_empty() { + return Ok(()); + } + names.sort_unstable(); + Err(Report::new(TrustedServerError::Configuration { + message: format!( + "response_headers must not set {}; framing and hop-by-hop fields belong to response egress", + names.join(", ") + ), + })) +} + +/// Warns that a live `proxy.certificate_check = false` is ignored. +/// +/// A config that predates the rejection keeps serving, and origin TLS stays verified. +fn warn_ignored_certificate_check(settings: &Settings) { + if !settings.proxy.certificate_check { + log::warn!( + "proxy.certificate_check = false is ignored; origin TLS is always verified. Remove the setting before the next config push" + ); + } +} + fn unknown_integration_ids(settings: &Settings) -> Vec<&str> { let mut ids = settings .integrations @@ -643,6 +749,7 @@ mod tests { use crate::auction_config_types::{NotificationConfig, ProviderConfig, RoutingMode}; use crate::redacted::Redacted; use crate::settings::{ProxyAssetRoute, S3SigV4AuthConfig, TrustedClientIpConfig}; + use crate::test_support::logging::WarningCapture; use crate::test_support::tests::crate_test_settings_str; use edgezero_core::app_config::AppConfigMeta; use edgezero_core::blob_envelope::BlobEnvelope; @@ -1616,6 +1723,177 @@ password = "production-admin-password-32-bytes" ); } + #[test] + fn validate_rejects_certificate_check_false() { + let mut settings = valid_settings(); + settings.proxy.certificate_check = false; + + let error = validate_settings_for_deploy(&settings) + .expect_err("should reject certificate_check = false at deploy"); + assert!( + format!("{error:?}").contains("proxy.certificate_check"), + "should name the rejected setting: {error:?}" + ); + let error = TrustedServerAppConfig::new(settings) + .expect_err("should reject certificate_check = false on validate and push"); + assert!( + error.to_string().contains("proxy.certificate_check"), + "should name the rejected setting on validate and push: {error:?}" + ); + } + + #[test] + fn runtime_ignores_certificate_check_false_with_warning() { + let mut settings = valid_settings(); + let capture = WarningCapture::start(); + validate_settings_for_runtime(&settings) + .expect("should accept the default certificate check at runtime"); + assert!( + !capture + .take() + .iter() + .any(|warning| warning.message.contains("proxy.certificate_check")), + "should not warn when certificate_check keeps its default" + ); + + settings.proxy.certificate_check = false; + validate_settings_for_runtime(&settings) + .expect("should keep serving a config that sets certificate_check = false"); + + let warnings = capture.take(); + assert!( + warnings.iter().any(|warning| { + warning.message.contains("proxy.certificate_check") + && warning.message.contains("ignored") + }), + "should warn that certificate_check = false is ignored" + ); + } + + #[test] + fn runtime_keeps_loading_operator_framing_headers() { + // Deploy validation rejects these entries, but a live config pushed + // before the rejection still has to load. The response-header helper + // skips them when it serves. + for (name, value) in [ + ("connection", "keep-alive,"), + ("connection", ""), + ("trailer", "expires"), + ] { + let mut settings = valid_settings(); + settings + .response_headers + .insert(name.to_owned(), value.to_owned()); + + validate_settings_for_runtime(&settings) + .expect("should keep loading a live config that predates the rejection"); + } + } + + #[test] + fn validate_rejects_origin_url_with_userinfo() { + for origin_url in [ + "https://example-user@origin.test-publisher.example.com", + "https://example-user:example-pass@origin.test-publisher.example.com", + ] { + let mut settings = valid_settings(); + settings.publisher.origin_url = origin_url.to_owned(); + + let error = validate_settings_for_deploy(&settings) + .expect_err("should reject an origin URL with userinfo at deploy"); + assert!( + format!("{error:?}").contains("publisher.origin_url"), + "should name the rejected setting: {error:?}" + ); + assert!( + !format!("{error:?}").contains("example-pass"), + "should never echo the credentials: {error:?}" + ); + let error = TrustedServerAppConfig::new(settings) + .expect_err("should reject an origin URL with userinfo on validate and push"); + assert!( + error.to_string().contains("publisher.origin_url"), + "should name the rejected setting on validate and push: {error:?}" + ); + } + } + + #[test] + fn validate_rejects_origin_readthrough_enabled() { + let mut settings = valid_settings(); + let readthrough = |enabled: bool| { + serde_json::from_value::( + serde_json::json!({ + "gam_network_id": "12345", + "origin_readthrough_enabled": enabled, + }), + ) + .expect("should build a creative opportunities config") + }; + settings.creative_opportunities = Some(readthrough(false)); + validate_settings_for_deploy(&settings) + .expect("should accept origin readthrough explicitly disabled"); + + settings.creative_opportunities = Some(readthrough(true)); + + let error = validate_settings_for_deploy(&settings) + .expect_err("should reject origin readthrough at deploy"); + assert!( + format!("{error:?}").contains("creative_opportunities.origin_readthrough_enabled"), + "should name the rejected setting: {error:?}" + ); + let error = TrustedServerAppConfig::new(settings) + .expect_err("should reject origin readthrough on validate and push"); + assert!( + error + .to_string() + .contains("creative_opportunities.origin_readthrough_enabled"), + "should name the rejected setting on validate and push: {error:?}" + ); + } + + #[test] + fn validate_rejects_operator_trailer_and_connection_entries() { + let mut settings = valid_settings(); + settings + .response_headers + .insert("x-example".to_owned(), "example-value".to_owned()); + validate_settings_for_deploy(&settings) + .expect("should accept an operator header that is not a framing field"); + + for name in [ + "Trailer", + "Content-Length", + "Connection", + "Keep-Alive", + "TE", + "Transfer-Encoding", + "Upgrade", + "Proxy-Connection", + "Proxy-Authenticate", + "Proxy-Authorization", + ] { + let mut settings = valid_settings(); + settings + .response_headers + .insert(name.to_owned(), "example-value".to_owned()); + + let quoted = format!("`{name}`"); + let error = validate_settings_for_deploy(&settings) + .expect_err("should reject an operator framing or hop-by-hop header at deploy"); + assert!( + format!("{error:?}").contains("ed), + "should name the rejected header {name}: {error:?}" + ); + let error = TrustedServerAppConfig::new(settings) + .expect_err("should reject an operator framing or hop-by-hop header on push"); + assert!( + error.to_string().contains("ed), + "should name the rejected header {name} on validate and push: {error:?}" + ); + } + } + #[test] fn deploy_validation_rejects_example_publisher_hosts() { let mut settings = valid_settings(); diff --git a/crates/trusted-server-core/src/config_payload.rs b/crates/trusted-server-core/src/config_payload.rs index e0615a7e7a..827439b080 100644 --- a/crates/trusted-server-core/src/config_payload.rs +++ b/crates/trusted-server-core/src/config_payload.rs @@ -156,6 +156,7 @@ mod tests { struct EchoSecretStore; + #[async_trait::async_trait(?Send)] impl PlatformSecretStore for EchoSecretStore { fn get_bytes( &self, @@ -170,7 +171,7 @@ mod tests { Ok(value.as_bytes().to_vec()) } - fn create( + async fn create( &self, _store_id: &StoreId, _name: &str, @@ -179,13 +180,18 @@ mod tests { Ok(()) } - fn delete(&self, _store_id: &StoreId, _name: &str) -> Result<(), Report> { + async fn delete( + &self, + _store_id: &StoreId, + _name: &str, + ) -> Result<(), Report> { Ok(()) } } struct UnifiedSecretStore; + #[async_trait::async_trait(?Send)] impl PlatformSecretStore for UnifiedSecretStore { fn get_bytes( &self, @@ -212,7 +218,7 @@ mod tests { Ok(value.as_bytes().to_vec()) } - fn create( + async fn create( &self, _store_id: &StoreId, _name: &str, @@ -221,7 +227,11 @@ mod tests { Ok(()) } - fn delete(&self, _store_id: &StoreId, _name: &str) -> Result<(), Report> { + async fn delete( + &self, + _store_id: &StoreId, + _name: &str, + ) -> Result<(), Report> { Ok(()) } } @@ -998,6 +1008,31 @@ mod tests { .expect("runtime should accept disabled browser bidder ownership overlap"); } + #[test] + fn live_blob_with_operator_framing_headers_still_loads() { + // Push rejects these entries, but a blob pushed before the rejection + // still has to load. The response-header helper skips them when it serves. + for (name, value) in [ + ("connection", "keep-alive,"), + ("connection", ""), + ("trailer", "expires"), + ] { + let mut settings = test_settings(); + settings + .response_headers + .insert(name.to_owned(), value.to_owned()); + + let loaded = load_settings(&envelope_json(&settings)) + .expect("should keep loading a live config that predates the rejection"); + + assert_eq!( + loaded.response_headers.get(name).map(String::as_str), + Some(value), + "should keep the {name} entry for the response-header helper to skip" + ); + } + } + #[test] fn tampered_blob_hash_is_rejected() { let mut envelope: BlobEnvelope = diff --git a/crates/trusted-server-core/src/creative_opportunities.rs b/crates/trusted-server-core/src/creative_opportunities.rs index 8a20c72965..af604f5053 100644 --- a/crates/trusted-server-core/src/creative_opportunities.rs +++ b/crates/trusted-server-core/src/creative_opportunities.rs @@ -360,39 +360,17 @@ pub struct CreativeOpportunitiesConfig { /// /// Named bypass cookies always disqualify. On the template cache path, an origin's /// `Vary: Cookie` still refuses storage regardless of this assertion or the key. - /// Readthrough performs no response-side check: when - /// [`Self::origin_readthrough_enabled`] is also `true`, otherwise eligible requests - /// with unlisted cookies rely on this assertion alone. Configured key-cookie - /// dimensions disable readthrough because its key does not include their values. - /// Verify the cookie axis with `ts origin probe-shareability` before enabling both - /// flags; see the configuration guide's template-cache section. + /// Verify the cookie axis with `ts origin probe-shareability` before enabling it; + /// see the configuration guide's template-cache section. #[serde(default, skip_serializing_if = "Option::is_none")] pub origin_is_cookie_independent: Option, - /// Whether this origin's responses may be held in the platform's shared readthrough + /// Retired opt-in for holding origin responses in the platform's shared readthrough /// cache. /// - /// Unset or `false` preserves the existing policy: ad-serving requests bypass the - /// cache, while other publisher requests retain the platform default. Setting `true` - /// uses request shareability instead: eligible ad-serving requests may use the cache, - /// and ineligible non-ad requests bypass it. - /// - /// **This flag is the whole opt-in.** Unlike - /// [`Self::origin_is_cookie_independent`], which only ever applies to cookie-bearing - /// requests, readthrough admits *cookieless* requests — first-time visitors, which is - /// exactly when an origin issues a session cookie. There is no response-side guard on - /// this path: the decision is made before the origin replies, and no post-response - /// hook is reachable on the Fastly adapter. Safety therefore rests on the origin's own - /// `Cache-Control` plus an operator's verification, so enabling it must be a - /// deliberate act rather than a consequence of deploying. - /// - /// Configured [`Self::template_cache_key_cookies`] disable readthrough because - /// its key does not include their values; template variants remain eligible. - /// Verify with `ts origin probe-shareability` before setting this. - /// - /// Setting this back to `false` restores the existing ad-stack bypass policy, not - /// a global cache bypass. Purge already-stored tagged objects with `ts cache purge` - /// (`--all` or the exact reader-facing `--page` URL). Objects stored by versions - /// without readthrough tags must still expire on the origin's TTL. + /// Origin readthrough is unavailable. Validation rejects `true`, and a live config + /// that still sets it serves as if it were `false`: ad-serving requests bypass the + /// platform cache and other publisher requests keep the platform default. Runtime + /// loading warns through [`crate::settings::Settings::warn_ignored_origin_readthrough`]. #[serde(default, skip_serializing_if = "Option::is_none")] pub origin_readthrough_enabled: Option, /// Slot templates. An empty vec or `enabled = false` disables template delivery. @@ -416,15 +394,6 @@ impl CreativeOpportunitiesConfig { self.origin_is_cookie_independent.unwrap_or(false) } - /// Whether the origin's responses may enter the platform's shared readthrough cache. - /// - /// Defaults to `false`: the conservative reading, and the one that preserves today's - /// shipped behavior for every deployment that does not ask for the change. - #[must_use] - pub fn origin_readthrough_enabled(&self) -> bool { - self.origin_readthrough_enabled.unwrap_or(false) - } - /// Exact cookie names included as bounded shared-template key dimensions. #[must_use] pub fn template_cache_key_cookies(&self) -> &[String] { diff --git a/crates/trusted-server-core/src/ec/batch_sync.rs b/crates/trusted-server-core/src/ec/batch_sync.rs index 4903589b29..b32b3c4855 100644 --- a/crates/trusted-server-core/src/ec/batch_sync.rs +++ b/crates/trusted-server-core/src/ec/batch_sync.rs @@ -21,6 +21,7 @@ use http::{Request, Response, StatusCode}; use serde::{Deserialize, Serialize}; use crate::error::TrustedServerError; +use crate::http_util::read_request_body; use super::auth::authenticate_bearer; use super::generation::{is_valid_ec_id, normalize_ec_id_for_kv}; @@ -37,6 +38,9 @@ const REASON_KV_UNAVAILABLE: &str = "kv_unavailable"; /// Maximum number of mappings allowed in a single batch request. const MAX_BATCH_SIZE: usize = 1000; +/// Maximum request body size accepted before mappings are parsed (2 MB). +const MAX_BODY_SIZE: usize = 2 * 1024 * 1024; + use super::kv_types::MAX_UID_LENGTH; trait BatchSyncWriter { @@ -108,19 +112,22 @@ struct MappingGroup { /// Handles `POST /_ts/api/v1/batch-sync`. /// +/// A body over the 2 MB cap is answered with a 413 `body_too_large` response. +/// /// # Errors /// -/// Returns [`TrustedServerError`] on serialization or KV store failures. -pub fn handle_batch_sync( +/// Returns [`TrustedServerError`] on serialization or KV store failures, and +/// the [`read_request_body`] error when the body read times out or fails. +pub async fn handle_batch_sync( kv: &KvIdentityGraph, registry: &PartnerRegistry, rate_limiter: &dyn RateLimiter, req: Request, ) -> Result, Report> { - handle_batch_sync_with_writer(kv, registry, rate_limiter, req) + handle_batch_sync_with_writer(kv, registry, rate_limiter, req).await } -fn handle_batch_sync_with_writer( +async fn handle_batch_sync_with_writer( writer: &dyn BatchSyncWriter, registry: &PartnerRegistry, rate_limiter: &dyn RateLimiter, @@ -141,21 +148,23 @@ fn handle_batch_sync_with_writer( } // 3. Parse body (with size limit to prevent OOM before validation) - const MAX_BODY_SIZE: usize = 2 * 1024 * 1024; // 2 MB - if content_length_exceeds_limit(&req, MAX_BODY_SIZE) { - return Ok(error_response( - StatusCode::PAYLOAD_TOO_LARGE, - "body_too_large", - )); - } - - let body_bytes = req.into_body().into_bytes().unwrap_or_default(); - if body_bytes.len() > MAX_BODY_SIZE { - return Ok(error_response( - StatusCode::PAYLOAD_TOO_LARGE, - "body_too_large", - )); - } + let (parts, body) = req.into_parts(); + let body_bytes = match read_request_body(&parts.headers, body, MAX_BODY_SIZE).await { + Ok(bytes) => bytes, + Err(report) + if matches!( + report.current_context(), + TrustedServerError::RequestTooLarge { .. } + ) => + { + log::warn!("Rejecting batch sync request: {report:?}"); + return Ok(error_response( + StatusCode::PAYLOAD_TOO_LARGE, + "body_too_large", + )); + } + Err(report) => return Err(report), + }; let body: BatchSyncRequest = serde_json::from_slice(&body_bytes).map_err(|e| { Report::new(TrustedServerError::BadRequest { message: format!("Invalid request body: {e}"), @@ -185,14 +194,6 @@ fn handle_batch_sync_with_writer( json_response(status, &response_body) } -fn content_length_exceeds_limit(req: &Request, max_body_size: usize) -> bool { - req.headers() - .get("content-length") - .and_then(|v| v.to_str().ok()) - .and_then(|value| value.parse::().ok()) - .is_some_and(|content_length| content_length > max_body_size) -} - /// Validates all mappings, then processes each normalized EC ID once. /// /// Successful and eligibility outcomes fan out to every valid input in a @@ -311,6 +312,8 @@ mod tests { use crate::error::TrustedServerError; use crate::redacted::Redacted; use crate::settings::EcPartner; + use crate::test_support::body_read::{body_read_failure_cases, streamed_over}; + use crate::test_support::logging::WarningCapture; // EC ID validation tests are in generation.rs (is_valid_ec_id). // Verify the import works here with a basic smoke test. @@ -437,43 +440,6 @@ mod tests { PartnerRegistry::from_config(&partners).expect("should build registry") } - #[test] - fn content_length_exceeds_limit_detects_oversized_header() { - let req = Request::builder() - .method("POST") - .uri("https://edge.example.com/_ts/api/v1/batch-sync") - .header("authorization", "Bearer test-token-32-bytes-minimum-value") - .header("content-length", "2097153") - .body(EdgeBody::from("{}")) - .expect("should build test request"); - - assert!( - content_length_exceeds_limit(&req, 2 * 1024 * 1024), - "should reject oversized content-length before reading body" - ); - } - - #[test] - fn content_length_exceeds_limit_ignores_missing_or_malformed_header() { - let missing = authorized_batch_request("{}"); - let malformed = Request::builder() - .method("POST") - .uri("https://edge.example.com/_ts/api/v1/batch-sync") - .header("authorization", "Bearer test-token-32-bytes-minimum-value") - .header("content-length", "not-a-number") - .body(EdgeBody::from("{}")) - .expect("should build test request"); - - assert!( - !content_length_exceeds_limit(&missing, 2 * 1024 * 1024), - "missing content-length should fall back to post-read size check" - ); - assert!( - !content_length_exceeds_limit(&malformed, 2 * 1024 * 1024), - "malformed content-length should fall back to post-read size check" - ); - } - #[test] fn handle_batch_sync_rejects_oversized_content_length_before_body_parse() { let writer = MockWriter::new(vec![]); @@ -489,8 +455,10 @@ mod tests { .body(EdgeBody::from("not-json")) .expect("should build test request"); - let response = handle_batch_sync_with_writer(&writer, ®istry, &limiter, req) - .expect("should return oversized response"); + let response = futures::executor::block_on(handle_batch_sync_with_writer( + &writer, ®istry, &limiter, req, + )) + .expect("should return oversized response"); assert_eq!( response.status(), @@ -515,8 +483,10 @@ mod tests { .body(EdgeBody::from(oversized_body)) .expect("should build test request"); - let response = handle_batch_sync_with_writer(&writer, ®istry, &limiter, req) - .expect("should return oversized response"); + let response = futures::executor::block_on(handle_batch_sync_with_writer( + &writer, ®istry, &limiter, req, + )) + .expect("should return oversized response"); assert_eq!( response.status(), @@ -525,6 +495,88 @@ mod tests { ); } + /// Returns an authorized batch-sync request builder with no body yet. + fn authorized_batch_builder() -> http::request::Builder { + Request::builder() + .method("POST") + .uri("https://edge.example.com/_ts/api/v1/batch-sync") + .header("authorization", "Bearer test-token-32-bytes-minimum-value") + } + + #[test] + fn batch_sync_body_read_failures_map_to_statuses() { + let registry = test_registry(); + let limiter = MockRateLimiter { + should_exceed: false, + }; + + for case in body_read_failure_cases(MAX_BODY_SIZE) { + let writer = MockWriter::new(vec![]); + let (req, expectation) = case.into_request(authorized_batch_builder()); + + let capture = WarningCapture::start(); + let outcome = futures::executor::block_on(handle_batch_sync_with_writer( + &writer, ®istry, &limiter, req, + )); + + expectation.assert_outcome("batch sync", &outcome); + assert!( + writer.calls().is_empty(), + "should not write any mapping for the {} case", + expectation.name() + ); + if expectation.expected() == StatusCode::PAYLOAD_TOO_LARGE { + let levels: Vec = + capture.take().iter().map(|record| record.level).collect(); + assert_eq!( + levels, + [log::Level::Warn], + "should log the {} case once, at warn", + expectation.name() + ); + let response = outcome.expect("should answer an overflow with a response"); + assert_eq!( + response_json(response), + serde_json::json!({ "error": "body_too_large" }), + "should keep the body_too_large error body for the {} case", + expectation.name() + ); + } + } + } + + #[test] + fn batch_sync_stream_past_declared_length_and_cap_returns_413() { + let writer = MockWriter::new(vec![]); + let registry = test_registry(); + let limiter = MockRateLimiter { + should_exceed: false, + }; + // The declared length is valid and small, so only the bounded read can + // stop a stream that runs past both it and the cap. + let req = authorized_batch_builder() + .header("content-length", "2") + .body(streamed_over(MAX_BODY_SIZE)) + .expect("should build test request"); + + let response = futures::executor::block_on(handle_batch_sync_with_writer( + &writer, ®istry, &limiter, req, + )) + .expect("should return oversized response"); + + assert_eq!( + response.status(), + StatusCode::PAYLOAD_TOO_LARGE, + "should reject a stream past its declared length and the cap" + ); + assert_eq!( + response_json(response), + serde_json::json!({ "error": "body_too_large" }), + "should keep the body_too_large error body" + ); + assert!(writer.calls().is_empty(), "should not write any mapping"); + } + #[test] fn process_mappings_returns_multistatus_errors_per_mapping() { let writer = MockWriter::new(vec![Ok(UpsertResult::Written)]); @@ -593,7 +645,8 @@ mod tests { .expect("should build test request"); let response = - handle_batch_sync(&kv, ®istry, &limiter, req).expect("should return response"); + futures::executor::block_on(handle_batch_sync(&kv, ®istry, &limiter, req)) + .expect("should return response"); assert_eq!( response.status(), StatusCode::UNAUTHORIZED, @@ -925,12 +978,12 @@ mod tests { r#"{{"mappings":[{{"ec_id":"{ec_id}","partner_uid":"one","timestamp":1}},{{"ec_id":"{ec_id}","partner_uid":"two","timestamp":2}}]}}"# ); - let response = handle_batch_sync_with_writer( + let response = futures::executor::block_on(handle_batch_sync_with_writer( &writer, ®istry, &limiter, authorized_batch_request(&body), - ) + )) .expect("should return success response"); assert_eq!( @@ -955,12 +1008,12 @@ mod tests { r#"{{"mappings":[{{"ec_id":"{ec_id}","partner_uid":"one","timestamp":1}},{{"ec_id":"{ec_id}","partner_uid":"two","timestamp":2}}]}}"# ); - let response = handle_batch_sync_with_writer( + let response = futures::executor::block_on(handle_batch_sync_with_writer( &writer, ®istry, &limiter, authorized_batch_request(&body), - ) + )) .expect("should return multi-status response"); assert_eq!( @@ -993,12 +1046,12 @@ mod tests { r#"{{"mappings":[{{"ec_id":"invalid","partner_uid":"one","timestamp":1}},{{"ec_id":"{valid_ec_id}","partner_uid":"","timestamp":2}}]}}"# ); - let response = handle_batch_sync_with_writer( + let response = futures::executor::block_on(handle_batch_sync_with_writer( &writer, ®istry, &limiter, authorized_batch_request(&body), - ) + )) .expect("should return validation response"); assert_eq!(response.status(), StatusCode::MULTI_STATUS); @@ -1038,12 +1091,12 @@ mod tests { r#"{{"mappings":[{{"ec_id":"{ec_id_a}","partner_uid":"a-first","timestamp":1}},{{"ec_id":"{ec_id_b}","partner_uid":"b","timestamp":2}},{{"ec_id":"{ec_id_a}","partner_uid":"a-last","timestamp":3}},{{"ec_id":"{ec_id_c}","partner_uid":"c","timestamp":4}}]}}"# ); - let response = handle_batch_sync_with_writer( + let response = futures::executor::block_on(handle_batch_sync_with_writer( &writer, ®istry, &limiter, authorized_batch_request(&body), - ) + )) .expect("should return infrastructure failure response"); assert_eq!(response.status(), StatusCode::MULTI_STATUS); diff --git a/crates/trusted-server-core/src/ec/pull_sync.rs b/crates/trusted-server-core/src/ec/pull_sync.rs index 191e6c6d24..77ae09d317 100644 --- a/crates/trusted-server-core/src/ec/pull_sync.rs +++ b/crates/trusted-server-core/src/ec/pull_sync.rs @@ -8,21 +8,27 @@ //! present in the EC identity entry, it is not periodically refreshed because //! the entry no longer stores per-partner sync timestamps. -use edgezero_core::body::Body as EdgeBody; +use std::time::Duration; + +use edgezero_core::error::EdgeError; +use edgezero_core::outbound::{ + HttpClient, OutboundCollectedSlot, OutboundRequest, OutboundResponse, ResponseDecoding, +}; +use edgezero_core::time::MonotonicClock; use http::{Method, StatusCode, header}; use serde::Deserialize; use url::Url; use crate::consent::allows_eid_persistence; -use crate::platform::{ - DEFAULT_FIRST_BYTE_TIMEOUT, PlatformBackendSpec, PlatformHttpRequest, PlatformPendingRequest, - PlatformResponse, RuntimeServices, +use crate::error::TrustedServerError; +use crate::outbound::{ + BUFFERED_TIMEOUT, OutboundRequestSource, buffered, outbound_error, outbound_request, }; use crate::settings::Settings; use super::generation::{ec_hash, is_valid_ec_id}; use super::kv::{KvIdentityGraph, PartnerIdUpdate}; -use super::kv_types::KvEntry; +use super::kv_types::{KvEntry, MAX_UID_LENGTH}; use super::rate_limiter::RateLimiter; use super::registry::{PartnerConfig, PartnerRegistry}; @@ -46,11 +52,17 @@ impl PullSyncContext { } } -struct InFlightPull { +/// One partner request waiting for its group to be sent. +struct QueuedPull { source_domain: String, - pending: PlatformPendingRequest, + request: OutboundRequest, } +/// How long a group of partner requests may run before it is cut off. +/// +/// Each request also carries it as its own timeout. +const PULL_SYNC_GROUP_CUTOFF: Duration = BUFFERED_TIMEOUT; + #[derive(Debug, Deserialize)] struct PullSyncResponse { uid: Option, @@ -97,6 +109,11 @@ pub fn build_pull_sync_context( /// /// This function is best-effort: all errors are logged and swallowed. /// +/// Partners are sent in groups of `ec.pull_sync_concurrency` through +/// [`HttpClient::send_all_until`]. Each group is cut off +/// [`PULL_SYNC_GROUP_CUTOFF`] after it starts, on `clock`, and a partner that +/// has not answered by then is skipped. +/// /// # Consent revalidation /// /// Pull sync runs *after* the page response has been flushed to the browser and @@ -108,18 +125,14 @@ pub fn build_pull_sync_context( /// snapshot-based eligibility filter, so a request with nothing to pull still /// costs no KV operation, and the fresh snapshot is reused for the write-back /// instead of the stale one. -/// -/// # Panics -/// -/// Panics if the HTTP request builder produces an invalid request, which -/// cannot happen with the hardcoded method and well-formed URI used here. -pub fn dispatch_pull_sync( +pub async fn dispatch_pull_sync( settings: &Settings, kv: &KvIdentityGraph, registry: &PartnerRegistry, rate_limiter: &dyn RateLimiter, context: &PullSyncContext, - services: &RuntimeServices, + client: &HttpClient, + clock: &MonotonicClock, ) { let now = current_timestamp(); let mut pull_partners = registry.pull_enabled_partners(); @@ -182,7 +195,7 @@ pub fn dispatch_pull_sync( } let max_concurrency = settings.ec.pull_sync_concurrency.max(1); - let mut in_flight: Vec = Vec::new(); + let mut group: Vec = Vec::new(); let mut updates = Vec::new(); for partner in pull_partners { @@ -225,63 +238,31 @@ pub fn dispatch_pull_sync( }; let request_url = build_pull_request_url(url, context.ec_id()); - let scheme = request_url.scheme().to_string(); - let host = request_url.host_str().unwrap_or_default().to_string(); - let port = request_url.port(); - - let backend_name = match services.backend().ensure(&PlatformBackendSpec { - scheme, - host, - port, - host_header_override: None, - certificate_check: settings.proxy.certificate_check, - first_byte_timeout: DEFAULT_FIRST_BYTE_TIMEOUT, - between_bytes_timeout: DEFAULT_FIRST_BYTE_TIMEOUT, - discriminator: None, - }) { - Ok(name) => name, + let request = match pull_request(&request_url, token.expose()) { + Ok(request) => request, Err(err) => { - log::warn!( - "Pull sync: failed to resolve backend for partner '{}': {err:?}", - partner.source_domain + log_pull_failure( + err, + format!( + "request for partner '{}' could not be built", + partner.source_domain + ), ); continue; } }; - let request = http::Request::builder() - .method(Method::GET) - .uri(request_url.as_str()) - .header("authorization", format!("Bearer {}", token.expose())) - .body(EdgeBody::empty()) - .expect("should build pull sync request"); - - let pending = match futures::executor::block_on( - services - .http_client() - .send_async(PlatformHttpRequest::new(request, backend_name)), - ) { - Ok(pending) => pending, - Err(err) => { - log::warn!( - "Pull sync: failed to dispatch partner '{}': {err:?}", - partner.source_domain - ); - continue; - } - }; - - in_flight.push(InFlightPull { + group.push(QueuedPull { source_domain: partner.source_domain.clone(), - pending, + request, }); - if in_flight.len() >= max_concurrency { - drain_pull_batch(&mut in_flight, services, &mut updates); + if group.len() >= max_concurrency { + send_pull_group(client, clock, &mut group, &mut updates).await; } } - drain_pull_batch(&mut in_flight, services, &mut updates); + send_pull_group(client, clock, &mut group, &mut updates).await; if !updates.is_empty() { // Write back from the revalidated snapshot, not the request one: its // generation is current, so the first CAS attempt is not spent losing a @@ -374,77 +355,113 @@ fn pull_rate_limit_key(source_domain: &str, ec_id: &str) -> String { format!("pull:{source_domain}:{}", ec_hash(ec_id)) } -fn drain_pull_batch( - in_flight: &mut Vec, - services: &RuntimeServices, +/// Builds one partner's pull-sync request. +/// +/// # Errors +/// +/// Returns the [`EdgeError`] that building the request or its +/// `Authorization` header raised. +fn pull_request(url: &Url, token: &str) -> Result { + let request = outbound_request(Method::GET, url.as_str())? + .header(header::AUTHORIZATION, format!("Bearer {token}"))?; + Ok(buffered( + request, + PULL_SYNC_GROUP_CUTOFF, + MAX_PULL_RESPONSE_BYTES, + ResponseDecoding::Decode, + )) +} + +/// Logs one failed pull-sync step at warn, through [`outbound_error`]. +/// +/// `outbound_error` logs the failure's status, kind, budget source, and +/// provider detail, never the error's message, which can hold provider text +/// or a URL. This then logs the report with the `failed` step. +fn log_pull_failure(error: EdgeError, failed: String) { + let report = outbound_error( + error, + TrustedServerError::EdgeCookie { message: failed }, + OutboundRequestSource::Internal, + ); + log::warn!("Pull sync failed: {report:?}"); +} + +/// Sends the queued `group` and collects the UIDs its partners answer with. +/// +/// The group ends [`PULL_SYNC_GROUP_CUTOFF`] after it starts. A partner that +/// has not answered by then is logged and skipped. +async fn send_pull_group( + client: &HttpClient, + clock: &MonotonicClock, + group: &mut Vec, updates: &mut Vec, ) { - for pending in in_flight.drain(..) { - let source_domain = pending.source_domain; - // All requests were dispatched up front via send_async, so waiting on - // each in turn does not change concurrency. - let response = - match futures::executor::block_on(services.http_client().wait(pending.pending)) { + if group.is_empty() { + return; + } + let (source_domains, requests): (Vec, Vec) = group + .drain(..) + .map(|queued| (queued.source_domain, queued.request)) + .unzip(); + let cutoff = clock.deadline_after(PULL_SYNC_GROUP_CUTOFF); + let slots = match client.send_all_until(requests, cutoff).await { + Ok(results) => results.into_parts().0, + Err(failure) => { + let (error, slots) = failure.into_parts(); + log_pull_failure(error, "partner group failed".to_owned()); + slots + } + }; + + for (source_domain, slot) in source_domains.into_iter().zip(slots) { + let response = match slot { + OutboundCollectedSlot::Resolved(result) => match result.outcome { Ok(response) => response, Err(err) => { - log::warn!( - "Pull sync: request failed for partner '{}': {err:?}", - source_domain - ); + log_pull_failure(err, format!("request for partner '{source_domain}' failed")); continue; } - }; - - let Some(uid) = extract_pull_uid(response, &source_domain) else { - continue; + }, + OutboundCollectedSlot::Unresolved => { + log::warn!( + "Pull sync: partner '{source_domain}' did not answer within the {}s group cutoff", + PULL_SYNC_GROUP_CUTOFF.as_secs() + ); + continue; + } + OutboundCollectedSlot::AlreadyDelivered => continue, }; - updates.push(PartnerIdUpdate::new(source_domain, uid)); + if let Some(uid) = extract_pull_uid(response, &source_domain).await { + updates.push(PartnerIdUpdate::new(source_domain, uid)); + } } } /// Maximum response body size accepted from pull sync partners (64 KiB). /// /// The expected response is `{"uid":""}`, so 64 KiB is generous. -/// This prevents a misbehaving partner from exhausting WASM memory. -const MAX_PULL_RESPONSE_BYTES: usize = 64 * 1024; - -fn response_content_length_exceeds_limit(response: &PlatformResponse, source_domain: &str) -> bool { - let Some(value) = response.response.headers().get(header::CONTENT_LENGTH) else { - return false; - }; - - let Some(value) = value.to_str().ok() else { - log::warn!( - "Pull sync: partner '{}' returned invalid Content-Length header, rejecting", - source_domain - ); - return true; - }; - - let Ok(length) = value.parse::() else { - log::warn!( - "Pull sync: partner '{}' returned malformed Content-Length header, rejecting", - source_domain - ); - return true; +/// Each request carries it as its response cap, so a misbehaving partner +/// cannot exhaust WASM memory. +const MAX_PULL_RESPONSE_BYTES: u64 = 64 * 1024; + +async fn extract_pull_uid(response: OutboundResponse, source_domain: &str) -> Option { + let status = response.status(); + let body = match response.into_bytes_bounded(MAX_PULL_RESPONSE_BYTES).await { + Ok(body) => body, + Err(err) => { + log_pull_failure( + err, + format!("response from partner '{source_domain}' could not be read"), + ); + return None; + } }; - - if length > MAX_PULL_RESPONSE_BYTES { - log::warn!( - "Pull sync: partner '{}' returned oversized Content-Length ({} bytes), rejecting", - source_domain, - length - ); - return true; - } - - false + pull_uid(status, &body, source_domain) } -fn extract_pull_uid(response: PlatformResponse, source_domain: &str) -> Option { - let status = response.response.status(); - +/// Reads the UID a partner answered with `status` and `body`. +fn pull_uid(status: StatusCode, body: &[u8], source_domain: &str) -> Option { if status == StatusCode::NOT_FOUND { log::debug!( "Pull sync: partner '{}' returned 404, treating as no-op", @@ -462,24 +479,7 @@ fn extract_pull_uid(response: PlatformResponse, source_domain: &str) -> Option MAX_PULL_RESPONSE_BYTES { - log::warn!( - "Pull sync: partner '{}' returned oversized response ({} bytes), rejecting", - source_domain, - body.len() - ); - return None; - } - let payload = match serde_json::from_slice::(&body) { + let payload = match serde_json::from_slice::(body) { Ok(payload) => payload, Err(err) => { log::warn!( @@ -490,8 +490,6 @@ fn extract_pull_uid(response: PlatformResponse, source_domain: &str) -> Option { @@ -518,34 +516,9 @@ fn extract_pull_uid(response: PlatformResponse, source_domain: &str) -> Option PlatformResponse { - PlatformResponse::new( - edgezero_core::http::response_builder() - .status(status) - .body(EdgeBody::from(body.to_vec())) - .expect("should build test response"), - ) - } - - fn make_response_with_content_length( - status: u16, - content_length: usize, - body: &[u8], - ) -> PlatformResponse { - PlatformResponse::new( - edgezero_core::http::response_builder() - .status(status) - .header(header::CONTENT_LENGTH, content_length.to_string()) - .body(EdgeBody::from(body.to_vec())) - .expect("should build test response"), - ) - } - fn pull_partner(ttl_sec: u64) -> PartnerConfig { PartnerConfig { name: "SSP X".to_owned(), @@ -784,36 +757,29 @@ mod tests { } #[test] - fn extract_pull_uid_treats_404_as_noop() { - let response = make_response(404, b""); - - let uid = extract_pull_uid(response, "ssp.example.com"); + fn pull_uid_treats_404_as_noop() { + let uid = pull_uid(StatusCode::NOT_FOUND, b"", "ssp.example.com"); assert!(uid.is_none(), "should treat 404 as no-op"); } #[test] - fn extract_pull_uid_treats_uid_null_as_noop() { - let response = make_response(200, b"{\"uid\":null}"); - - let uid = extract_pull_uid(response, "ssp.example.com"); + fn pull_uid_treats_uid_null_as_noop() { + let uid = pull_uid(StatusCode::OK, b"{\"uid\":null}", "ssp.example.com"); assert!(uid.is_none(), "should treat uid=null as no-op"); } #[test] - fn extract_pull_uid_rejects_oversized_uid() { + fn pull_uid_rejects_oversized_uid() { let long_uid = "x".repeat(513); let body = format!("{{\"uid\":\"{long_uid}\"}}"); - let response = make_response(200, body.as_bytes()); - let uid = extract_pull_uid(response, "ssp.example.com"); + let uid = pull_uid(StatusCode::OK, body.as_bytes(), "ssp.example.com"); assert!(uid.is_none(), "should reject uid exceeding 512 bytes"); } #[test] - fn extract_pull_uid_reads_uid_from_success_body() { - let response = make_response(200, b"{\"uid\":\"abc123\"}"); - - let uid = extract_pull_uid(response, "ssp.example.com"); + fn pull_uid_reads_uid_from_success_body() { + let uid = pull_uid(StatusCode::OK, b"{\"uid\":\"abc123\"}", "ssp.example.com"); assert_eq!( uid.as_deref(), Some("abc123"), @@ -821,42 +787,6 @@ mod tests { ); } - #[test] - fn extract_pull_uid_rejects_oversized_content_length_before_body_read() { - let response = make_response_with_content_length( - 200, - MAX_PULL_RESPONSE_BYTES + 1, - b"{\"uid\":\"abc123\"}", - ); - - let uid = extract_pull_uid(response, "ssp.example.com"); - assert!( - uid.is_none(), - "should reject oversized Content-Length before parsing body" - ); - } - - #[test] - fn extract_pull_uid_accepts_small_body_without_content_length() { - let response = make_response(200, b"{\"uid\":\"abc123\"}"); - - let uid = extract_pull_uid(response, "ssp.example.com"); - assert_eq!( - uid.as_deref(), - Some("abc123"), - "should accept small valid response without Content-Length" - ); - } - - #[test] - fn extract_pull_uid_rejects_body_larger_than_limit() { - let body = format!("{{\"uid\":\"{}\"}}", "x".repeat(MAX_PULL_RESPONSE_BYTES)); - let response = make_response(200, body.as_bytes()); - - let uid = extract_pull_uid(response, "ssp.example.com"); - assert!(uid.is_none(), "should reject body larger than limit"); - } - #[test] fn rotating_offset_distributes_partners_across_hours() { // Simulate 3 partners sorted by source domain: alpha, beta, gamma. @@ -894,12 +824,14 @@ mod tests { // ----------------------------------------------------------------------- use crate::ec::kv::TombstoneOutcome; - use crate::error::TrustedServerError; - use crate::platform::test_support::{StubHttpClient, build_services_with_http_client}; use crate::settings::EcPartner; + use crate::test_support::logging::WarningCapture; + use crate::test_support::scripted_client::{ + ExpectedTransport, ScriptedHttpClient, ScriptedReply, TestClock, + }; use crate::test_support::tests::create_test_settings; + use edgezero_core::outbound::{OutboundCachePolicy, ResponseMode}; use error_stack::Report; - use std::sync::Arc; struct AllowAllRateLimiter; @@ -936,6 +868,11 @@ mod tests { format!("{}.ABC123", "a".repeat(64)) } + /// The pull-sync URL `source_domain`'s partner receives for [`snapshot_ec_id`]. + fn pull_url(source_domain: &str) -> String { + format!("https://{source_domain}/sync?ec_id={}", snapshot_ec_id()) + } + fn seed_present_snapshot(graph: &KvIdentityGraph, ec_id: &str) -> EcKvSnapshot { let mut entry = KvEntry::tombstone(1000); entry.consent.ok = true; @@ -943,6 +880,26 @@ mod tests { graph.load_snapshot(ec_id) } + /// Runs pull sync to completion on `clock`, advancing it as the requests wait. + fn run_pull_sync( + settings: &Settings, + graph: &KvIdentityGraph, + registry: &PartnerRegistry, + context: &PullSyncContext, + client: &ScriptedHttpClient, + clock: &TestClock, + ) { + clock.drive(dispatch_pull_sync( + settings, + graph, + registry, + &AllowAllRateLimiter, + context, + &client.http_client(), + &clock.monotonic_clock(), + )); + } + #[test] fn dispatch_pull_sync_aggregates_batches_into_one_bulk_write() { let mut settings = create_test_settings(); @@ -958,29 +915,27 @@ mod tests { let ec_id = snapshot_ec_id(); let snapshot = seed_present_snapshot(&graph, &ec_id); - let stub = Arc::new(StubHttpClient::new()); - // One JSON response per partner, drained across two concurrency batches. - stub.push_response(200, br#"{"uid":"synced-uid"}"#.to_vec()); - stub.push_response(200, br#"{"uid":"synced-uid"}"#.to_vec()); - let services = build_services_with_http_client(stub.clone()); + let clock = TestClock::new(); + let client = ScriptedHttpClient::new(&clock); + // One JSON response per partner, collected across two concurrency batches. + for source_domain in ["alpha.example.com", "beta.example.com"] { + client.respond_to( + Method::GET, + &pull_url(source_domain), + ScriptedReply::status(StatusCode::OK).body(r#"{"uid":"synced-uid"}"#), + ); + } let context = PullSyncContext { ec_id: ec_id.clone(), snapshot, }; - dispatch_pull_sync( - &settings, - &graph, - ®istry, - &AllowAllRateLimiter, - &context, - &services, - ); + run_pull_sync(&settings, &graph, ®istry, &context, &client, &clock); let (entry, generation) = graph .get(&ec_id) .expect("should read store") - .expect("entry should exist"); + .expect("should keep the entry"); assert_eq!( entry.ids.get("alpha.example.com").map(|id| id.uid.as_str()), Some("synced-uid"), @@ -997,6 +952,222 @@ mod tests { ); } + #[test] + fn dispatch_pull_sync_sends_each_partner_its_bearer_token_and_transport() { + let mut settings = create_test_settings(); + settings.ec.pull_sync_concurrency = 4; + let registry = + PartnerRegistry::from_config(&[pull_enabled_ec_partner("alpha.example.com")]) + .expect("should build pull registry"); + let graph = KvIdentityGraph::in_memory("pull_store"); + let ec_id = snapshot_ec_id(); + let snapshot = seed_present_snapshot(&graph, &ec_id); + let clock = TestClock::new(); + let client = ScriptedHttpClient::new(&clock); + client.respond_to( + Method::GET, + &pull_url("alpha.example.com"), + ScriptedReply::status(StatusCode::OK).body(r#"{"uid":"synced-uid"}"#), + ); + + let context = PullSyncContext { ec_id, snapshot }; + run_pull_sync(&settings, &graph, ®istry, &context, &client, &clock); + + let requests = client.requests(); + let [request] = requests.as_slice() else { + panic!("should send one pull request, got {requests:?}"); + }; + assert_eq!(request.method, Method::GET, "should pull with GET"); + assert_eq!( + request.uri.to_string(), + pull_url("alpha.example.com"), + "should append the EC ID to the partner's pull URL" + ); + request.assert_headers(&[("authorization", "Bearer outbound-token")]); + request.assert_transport(ExpectedTransport { + timeout: PULL_SYNC_GROUP_CUTOFF, + response_mode: ResponseMode::Buffered { + max_bytes: MAX_PULL_RESPONSE_BYTES, + }, + response_decoding: ResponseDecoding::Decode, + cache_policy: OutboundCachePolicy::PlatformDefault, + }); + } + + #[test] + fn pull_sync_group_ends_at_fifteen_second_cutoff() { + let mut settings = create_test_settings(); + settings.ec.pull_sync_concurrency = 2; + let registry = PartnerRegistry::from_config(&[ + pull_enabled_ec_partner("alpha.example.com"), + pull_enabled_ec_partner("beta.example.com"), + ]) + .expect("should build pull registry"); + let graph = KvIdentityGraph::in_memory("pull_store"); + let ec_id = snapshot_ec_id(); + let snapshot = seed_present_snapshot(&graph, &ec_id); + let clock = TestClock::new(); + let started = clock.monotonic_clock().now(); + let client = ScriptedHttpClient::new(&clock); + client + .respond_to( + Method::GET, + &pull_url("alpha.example.com"), + ScriptedReply::status(StatusCode::OK) + .body(r#"{"uid":"fast-uid"}"#) + .after(Duration::from_millis(100)), + ) + .respond_to( + Method::GET, + &pull_url("beta.example.com"), + ScriptedReply::status(StatusCode::OK) + .body(r#"{"uid":"slow-uid"}"#) + .after(Duration::from_secs(20)), + ); + + let context = PullSyncContext { + ec_id: ec_id.clone(), + snapshot, + }; + run_pull_sync(&settings, &graph, ®istry, &context, &client, &clock); + + assert_eq!( + clock + .monotonic_clock() + .now() + .saturating_duration_since(started), + Duration::from_secs(15), + "should stop waiting for the group at its 15 s cutoff" + ); + let cutoff = started + .checked_add(Duration::from_secs(15)) + .expect("should represent the cutoff"); + for request in client.requests() { + assert_eq!( + request.budget.deadline.instant(), + cutoff, + "should bound {} by the group cutoff", + request.uri + ); + } + let (entry, _) = graph + .get(&ec_id) + .expect("should read store") + .expect("should keep the entry"); + assert_eq!( + entry.ids.get("alpha.example.com").map(|id| id.uid.as_str()), + Some("fast-uid"), + "should keep the partner that answered before the cutoff" + ); + assert!( + !entry.ids.contains_key("beta.example.com"), + "should drop the partner that answered after the cutoff" + ); + } + + #[test] + fn pull_sync_failures_log_through_outbound_error() { + let mut settings = create_test_settings(); + // One partner per group, so the failing batch ends only its own group. + settings.ec.pull_sync_concurrency = 1; + let mut unbuildable = pull_enabled_ec_partner("gamma.example.com"); + unbuildable.ts_pull_token = Some(Redacted::new("outbound\ntoken".to_owned())); + let registry = PartnerRegistry::from_config(&[ + pull_enabled_ec_partner("alpha.example.com"), + pull_enabled_ec_partner("beta.example.com"), + unbuildable, + ]) + .expect("should build pull registry"); + let graph = KvIdentityGraph::in_memory("pull_store"); + let ec_id = snapshot_ec_id(); + let snapshot = seed_present_snapshot(&graph, &ec_id); + let clock = TestClock::new(); + let client = ScriptedHttpClient::new(&clock); + client + .respond_to( + Method::GET, + &pull_url("alpha.example.com"), + ScriptedReply::failing_batch(EdgeError::bad_gateway("alpha provider text")), + ) + .respond_to( + Method::GET, + &pull_url("beta.example.com"), + ScriptedReply::error(EdgeError::bad_gateway("beta provider text")), + ); + let context = PullSyncContext { ec_id, snapshot }; + let capture = WarningCapture::start(); + + run_pull_sync(&settings, &graph, ®istry, &context, &client, &clock); + + let warnings = capture.take(); + let typed = warnings + .iter() + .filter(|warning| warning.message.starts_with("Outbound request failed with")) + .count(); + assert_eq!( + typed, 3, + "should log the group failure, the request failure, and the build failure through outbound_error" + ); + for step in [ + "partner group failed", + "request for partner 'beta.example.com' failed", + "request for partner 'gamma.example.com' could not be built", + ] { + assert!( + warnings.iter().any(|warning| { + warning.message.starts_with("Pull sync failed:") + && warning.message.contains(step) + }), + "should log the report for `{step}`" + ); + } + assert!( + warnings + .iter() + .all(|warning| !warning.message.contains("provider text")), + "should keep the provider text out of the logs" + ); + } + + #[test] + fn dispatch_pull_sync_drops_a_response_over_the_pull_cap() { + let mut settings = create_test_settings(); + settings.ec.pull_sync_concurrency = 4; + let registry = + PartnerRegistry::from_config(&[pull_enabled_ec_partner("alpha.example.com")]) + .expect("should build pull registry"); + let graph = KvIdentityGraph::in_memory("pull_store"); + let ec_id = snapshot_ec_id(); + let snapshot = seed_present_snapshot(&graph, &ec_id); + let clock = TestClock::new(); + let client = ScriptedHttpClient::new(&clock); + let oversized = format!( + "{{\"uid\":\"{}\"}}", + "x".repeat(usize::try_from(MAX_PULL_RESPONSE_BYTES).expect("should fit usize")) + ); + client.respond_to( + Method::GET, + &pull_url("alpha.example.com"), + ScriptedReply::status(StatusCode::OK).body(oversized), + ); + + let context = PullSyncContext { + ec_id: ec_id.clone(), + snapshot, + }; + run_pull_sync(&settings, &graph, ®istry, &context, &client, &clock); + + let (entry, generation) = graph + .get(&ec_id) + .expect("should read store") + .expect("should keep the entry"); + assert!( + entry.ids.is_empty(), + "should persist no UID from a response over the 64 KiB cap" + ); + assert_eq!(generation, 1, "should not write the row back"); + } + #[test] fn dispatch_pull_sync_calls_and_persists_only_missing_partner() { let mut settings = create_test_settings(); @@ -1015,32 +1186,34 @@ mod tests { ) .expect("should seed partial entry"); let snapshot = graph.load_snapshot(&ec_id); - let stub = Arc::new(StubHttpClient::new()); - stub.push_response(200, br#"{"uid":"new-beta"}"#.to_vec()); - let services = build_services_with_http_client(stub.clone()); + let clock = TestClock::new(); + let client = ScriptedHttpClient::new(&clock); + client.respond_to( + Method::GET, + &pull_url("beta.example.com"), + ScriptedReply::status(StatusCode::OK).body(r#"{"uid":"new-beta"}"#), + ); let context = PullSyncContext { ec_id: ec_id.clone(), snapshot, }; - dispatch_pull_sync( - &settings, - &graph, - ®istry, - &AllowAllRateLimiter, - &context, - &services, - ); + run_pull_sync(&settings, &graph, ®istry, &context, &client, &clock); + let requested: Vec = client + .requests() + .iter() + .map(|request| request.uri.to_string()) + .collect(); assert_eq!( - stub.recorded_backend_names().len(), - 1, + requested, + vec![pull_url("beta.example.com")], "should call only the missing partner" ); let (entry, generation) = graph .get(&ec_id) .expect("should read store") - .expect("entry should exist"); + .expect("should keep the entry"); assert_eq!( entry .ids @@ -1069,8 +1242,8 @@ mod tests { .expect("should build registry"); let graph = KvIdentityGraph::in_memory("pull_store"); let ec_id = snapshot_ec_id(); - let stub = Arc::new(StubHttpClient::new()); - let services = build_services_with_http_client(stub.clone()); + let client = ScriptedHttpClient::forbidding(); + let clock = TestClock::new(); for snapshot in [ EcKvSnapshot::NotRead, @@ -1085,20 +1258,10 @@ mod tests { ec_id: ec_id.clone(), snapshot, }; - dispatch_pull_sync( - &settings, - &graph, - ®istry, - &AllowAllRateLimiter, - &context, - &services, - ); + run_pull_sync(&settings, &graph, ®istry, &context, &client, &clock); } - assert!( - stub.recorded_backend_names().is_empty(), - "not-read, missing, and failed snapshots must not dispatch pull sync" - ); + client.assert_no_requests(); assert!( graph.get(&ec_id).expect("should read store").is_none(), "no snapshot state should create a missing root" @@ -1119,27 +1282,18 @@ mod tests { entry: Box::new(KvEntry::tombstone(1000)), generation: Some(1), }; - let stub = Arc::new(StubHttpClient::new()); - let services = build_services_with_http_client(stub.clone()); + let client = ScriptedHttpClient::forbidding(); + let clock = TestClock::new(); let context = PullSyncContext { ec_id: ec_id.clone(), snapshot, }; - dispatch_pull_sync( - &settings, - &graph, - ®istry, - &AllowAllRateLimiter, - &context, - &services, - ); + run_pull_sync(&settings, &graph, ®istry, &context, &client, &clock); - assert!( - stub.recorded_backend_names().is_empty(), - "a tombstone snapshot must not dispatch pull sync" - ); + client.assert_no_requests(); } + #[test] fn dispatch_pull_sync_skips_dispatch_when_ec_is_tombstoned_after_snapshot_capture() { // The request snapshot authorized pull sync, then a concurrent request @@ -1165,31 +1319,20 @@ mod tests { "should tombstone the row the snapshot still reports as present" ); - let stub = Arc::new(StubHttpClient::new()); - stub.push_response(200, br#"{"uid":"leaked-uid"}"#.to_vec()); - let services = build_services_with_http_client(stub.clone()); + let client = ScriptedHttpClient::forbidding(); + let clock = TestClock::new(); let context = PullSyncContext { ec_id: ec_id.clone(), snapshot, }; - dispatch_pull_sync( - &settings, - &graph, - ®istry, - &AllowAllRateLimiter, - &context, - &services, - ); + run_pull_sync(&settings, &graph, ®istry, &context, &client, &clock); - assert!( - stub.recorded_backend_names().is_empty(), - "a withdrawal that lands after snapshot capture must cancel post-send pull sync" - ); + client.assert_no_requests(); let (entry, _) = graph .get(&ec_id) .expect("should read store") - .expect("tombstone should remain"); + .expect("should keep the tombstone"); assert!( entry.ids.is_empty(), "no partner UID may be written back onto a tombstoned row" @@ -1226,25 +1369,15 @@ mod tests { generation: Some(1), }; - let stub = Arc::new(StubHttpClient::new()); - let services = build_services_with_http_client(stub.clone()); + let client = ScriptedHttpClient::forbidding(); + let clock = TestClock::new(); let context = PullSyncContext { ec_id: ec_id.clone(), snapshot, }; - dispatch_pull_sync( - &settings, - &graph, - ®istry, - &AllowAllRateLimiter, - &context, - &services, - ); + run_pull_sync(&settings, &graph, ®istry, &context, &client, &clock); - assert!( - stub.recorded_backend_names().is_empty(), - "a fully synced entry must not dispatch pull sync" - ); + client.assert_no_requests(); } } diff --git a/crates/trusted-server-core/src/edge_app.rs b/crates/trusted-server-core/src/edge_app.rs new file mode 100644 index 0000000000..20efd69f88 --- /dev/null +++ b/crates/trusted-server-core/src/edge_app.rs @@ -0,0 +1,128 @@ +//! App-level `EdgeZero` policies that every adapter installs from `Hooks::configure`. + +use std::time::Duration; + +use edgezero_core::app::App; +use edgezero_core::error::EdgeError; +use edgezero_core::http::HeaderMap; +use edgezero_core::response_egress::{ResponseEgressHead, ResponseEgressPolicy}; +use edgezero_core::time::{Deadline, MonotonicInstant}; + +use crate::constants::HEADER_X_TS_VERSION; +use crate::version_header::git_version_header_value; + +/// How long a response may take to reach the client once egress starts. +/// +/// `EdgeZero` defaults to 30 s, which would cut a streamed publisher page short +/// whatever its origin timeout. +pub const RESPONSE_WRITE_DEADLINE: Duration = Duration::from_secs(60); + +/// Installs the response write deadline and the app-level synthesized headers. +/// +/// The egress policy ends each response write [`RESPONSE_WRITE_DEADLINE`] after +/// egress starts. The synthesized set is [`synthesized_response_headers`]. +/// `EdgeZero` adds it, where a field is absent, to the responses it generates +/// itself, such as the canonical 404 and 405. Routed responses keep the headers +/// the finalize middleware sets. +/// +/// # Errors +/// +/// Returns [`EdgeError::Internal`] when `EdgeZero` rejects the synthesized set. +/// +/// # Examples +/// +/// ``` +/// use edgezero_core::app::App; +/// use edgezero_core::router::RouterService; +/// use trusted_server_core::edge_app::install_edge_policies; +/// +/// let mut app = App::new(RouterService::builder().build()); +/// install_edge_policies(&mut app).expect("should install the edge policies"); +/// ``` +pub fn install_edge_policies(app: &mut App) -> Result<(), EdgeError> { + app.set_response_egress_policy(response_write_policy); + app.set_synthesized_response_headers(synthesized_response_headers()) + .map_err(EdgeError::internal) +} + +/// Returns the app-level synthesized response headers. +/// +/// The set holds `x-ts-version` when a git version is compiled in, the value +/// the finalize middleware sets on routed responses. An adapter that adds its +/// own fields starts from this set and installs the result with +/// [`App::set_synthesized_response_headers`]. +/// +/// # Examples +/// +/// ``` +/// use trusted_server_core::constants::{HEADER_X_TS_VERSION, TS_GIT_VERSION}; +/// use trusted_server_core::edge_app::synthesized_response_headers; +/// +/// let headers = synthesized_response_headers(); +/// assert_eq!( +/// headers +/// .get(HEADER_X_TS_VERSION) +/// .and_then(|value| value.to_str().ok()), +/// TS_GIT_VERSION, +/// "should hold the compiled-in git version", +/// ); +/// ``` +#[must_use] +pub fn synthesized_response_headers() -> HeaderMap { + let mut headers = HeaderMap::new(); + if let Some(version) = git_version_header_value() { + headers.insert(HEADER_X_TS_VERSION, version); + } + headers +} + +fn response_write_policy( + _head: &ResponseEgressHead<'_>, + egress_started_at: MonotonicInstant, +) -> ResponseEgressPolicy { + // An unrepresentable deadline fails closed at the egress start, as + // EdgeZero's default policy does. + let deadline = egress_started_at + .checked_add(RESPONSE_WRITE_DEADLINE) + .unwrap_or(egress_started_at); + ResponseEgressPolicy { + write_deadline: Deadline::at_instant(deadline), + } +} + +#[cfg(test)] +mod tests { + use super::*; + + use edgezero_core::http::{StatusCode, Version}; + use edgezero_core::router::RouterService; + + #[test] + fn installed_egress_policy_sets_write_deadline_sixty_seconds_after_egress_start() { + let mut app = App::new(RouterService::builder().build()); + install_edge_policies(&mut app).expect("should install the edge policies"); + let headers = HeaderMap::new(); + let request_start = MonotonicInstant::now(); + let egress_started_at = request_start + .checked_add(Duration::from_secs(5)) + .expect("should represent an egress start after the request start"); + let head = ResponseEgressHead::new( + StatusCode::OK, + Version::HTTP_11, + &headers, + None, + request_start, + None, + ); + + let policy = (app.response_egress_policy())(&head, egress_started_at); + + assert_eq!( + policy.write_deadline.instant(), + egress_started_at + .checked_add(Duration::from_secs(60)) + .expect("should represent the expected deadline"), + "should end the response write 60 s after egress starts, not after the request starts" + ); + } +} diff --git a/crates/trusted-server-core/src/error.rs b/crates/trusted-server-core/src/error.rs index 69cfdd68a6..8db47d27be 100644 --- a/crates/trusted-server-core/src/error.rs +++ b/crates/trusted-server-core/src/error.rs @@ -6,6 +6,10 @@ use core::error::Error; use derive_more::Display; +use edgezero_core::body::Body; +use edgezero_core::error::EdgeError; +use edgezero_core::http::{HeaderValue, Response, header}; +use error_stack::Report; use http::StatusCode; /// The main error type for trusted server operations. @@ -49,6 +53,11 @@ pub enum TrustedServerError { #[display("Request payload too large: {message}")] RequestTooLarge { message: String }, + /// The request body was not read before the inbound read deadline, + /// resulting in a 408 Request Timeout. + #[display("Request timeout: {message}")] + RequestTimeout { message: String }, + /// HTTP header value creation failed. #[display("Invalid HTTP header value: {message}")] InvalidHeaderValue { message: String }, @@ -127,6 +136,7 @@ impl IntoHttpResponse for TrustedServerError { Self::Integration { .. } => StatusCode::BAD_GATEWAY, Self::Proxy { .. } => StatusCode::BAD_GATEWAY, Self::RequestTooLarge { .. } => StatusCode::PAYLOAD_TOO_LARGE, + Self::RequestTimeout { .. } => StatusCode::REQUEST_TIMEOUT, Self::Forbidden { .. } => StatusCode::FORBIDDEN, Self::AllowlistViolation { .. } => StatusCode::FORBIDDEN, Self::EdgeCookie { .. } => StatusCode::INTERNAL_SERVER_ERROR, @@ -142,13 +152,94 @@ impl IntoHttpResponse for TrustedServerError { // Consent strings may contain user data; return category only. Self::GdprConsent { .. } => "GDPR consent error".to_string(), Self::InvalidHeaderValue { .. } => "Invalid header value".to_string(), + // The client sent the body too slowly; the detail stays in the logs. + Self::RequestTimeout { .. } => "Request timeout".to_string(), // Server/integration errors (5xx/502/503) — generic message only. - // Full details are already logged via log::error! in to_error_response. + // Full details are already logged by http_error. _ => "An internal error occurred".to_string(), } } } +/// Maps a failed application startup to the [`EdgeError`] that `EdgeZero` answers with. +/// +/// The constructor follows the [`IntoHttpResponse::status_code`] of the report's +/// current context, the same error the old startup routers answered with: +/// +/// - 500 maps to [`EdgeError::internal`]. +/// - 503 maps to [`EdgeError::service_unavailable`]. +/// - 502 maps to [`EdgeError::bad_gateway`]. +/// - 400 maps to [`EdgeError::bad_request`]. +/// - Any other status maps to [`EdgeError::internal`]. +/// +/// The message is the error's [`IntoHttpResponse::user_message`], so no +/// internal detail reaches the client. This function does not log. Callers log +/// the full report at error level before returning the result. +/// +/// # Examples +/// +/// ``` +/// use edgezero_core::http::StatusCode; +/// use error_stack::Report; +/// use trusted_server_core::error::{TrustedServerError, startup_error}; +/// +/// let report = Report::new(TrustedServerError::KvStore { +/// store_name: "example-store".to_owned(), +/// message: "unavailable".to_owned(), +/// }); +/// assert_eq!(startup_error(&report).status(), StatusCode::SERVICE_UNAVAILABLE); +/// ``` +#[must_use] +pub fn startup_error(report: &Report) -> EdgeError { + let error = report.current_context(); + let message = error.user_message(); + match error.status_code() { + StatusCode::SERVICE_UNAVAILABLE => EdgeError::service_unavailable(message), + StatusCode::BAD_GATEWAY => EdgeError::bad_gateway(message), + StatusCode::BAD_REQUEST => EdgeError::bad_request(message), + _ => EdgeError::internal(std::io::Error::other(message)), + } +} + +/// Renders a failed request's [`Report`] as the plain-text error response. +/// +/// The response answers with the [`IntoHttpResponse::status_code`] of the +/// report's current context and its client-safe +/// [`IntoHttpResponse::user_message`]. The whole report is logged once: at +/// warn when the status is a client error, because the client caused it, and +/// at error otherwise. +/// +/// # Examples +/// +/// ``` +/// use edgezero_core::http::StatusCode; +/// use error_stack::Report; +/// use trusted_server_core::error::{TrustedServerError, http_error}; +/// +/// let report = Report::new(TrustedServerError::RequestTooLarge { +/// message: "example body over the limit".to_owned(), +/// }); +/// assert_eq!(http_error(&report).status(), StatusCode::PAYLOAD_TOO_LARGE); +/// ``` +#[must_use] +pub fn http_error(report: &Report) -> Response { + let error = report.current_context(); + let status = error.status_code(); + if status.is_client_error() { + log::warn!("Error occurred: {report:?}"); + } else { + log::error!("Error occurred: {report:?}"); + } + + let mut response = Response::new(Body::from(format!("{}\n", error.user_message()))); + *response.status_mut() = status; + response.headers_mut().insert( + header::CONTENT_TYPE, + HeaderValue::from_static("text/plain; charset=utf-8"), + ); + response +} + #[cfg(test)] mod tests { use super::*; @@ -198,6 +289,12 @@ mod tests { }, StatusCode::PAYLOAD_TOO_LARGE, ), + ( + TrustedServerError::RequestTimeout { + message: String::from("body read deadline passed"), + }, + StatusCode::REQUEST_TIMEOUT, + ), ( TrustedServerError::InvalidHeaderValue { message: String::from("non-ascii header"), @@ -284,6 +381,7 @@ mod tests { TrustedServerError::GdprConsent { .. } => StatusCode::BAD_REQUEST, TrustedServerError::InvalidUtf8 { .. } => StatusCode::INTERNAL_SERVER_ERROR, TrustedServerError::RequestTooLarge { .. } => StatusCode::PAYLOAD_TOO_LARGE, + TrustedServerError::RequestTimeout { .. } => StatusCode::REQUEST_TIMEOUT, TrustedServerError::InvalidHeaderValue { .. } => StatusCode::BAD_REQUEST, TrustedServerError::KvStore { .. } => StatusCode::SERVICE_UNAVAILABLE, TrustedServerError::Prebid { .. } => StatusCode::BAD_GATEWAY, @@ -398,6 +496,15 @@ mod tests { message: "non-ascii".into(), }; assert_eq!(error.user_message(), "Invalid header value"); + + let error = TrustedServerError::RequestTimeout { + message: "read deadline passed after 30s".into(), + }; + assert_eq!( + error.user_message(), + "Request timeout", + "should keep the timeout detail out of the client message" + ); } #[test] @@ -422,6 +529,7 @@ mod tests { | TrustedServerError::EdgeCookie { .. } | TrustedServerError::PartnerNotFound { .. } | TrustedServerError::RequestTooLarge { .. } + | TrustedServerError::RequestTimeout { .. } | TrustedServerError::InsecureDefault { .. } => (), }; @@ -536,6 +644,12 @@ mod tests { }, StatusCode::PAYLOAD_TOO_LARGE, ), + ( + TrustedServerError::RequestTimeout { + message: "body read deadline passed".to_string(), + }, + StatusCode::REQUEST_TIMEOUT, + ), ]; for (error, expected) in cases { @@ -546,4 +660,121 @@ mod tests { ); } } + + fn assert_startup_error( + error: TrustedServerError, + expected_kind: &str, + expected_status: StatusCode, + ) -> EdgeError { + let edge_error = startup_error(&Report::new(error)); + assert_eq!( + edge_error.kind(), + expected_kind, + "should use the {expected_kind} constructor" + ); + assert_eq!( + edge_error.status(), + expected_status, + "should keep the startup status" + ); + edge_error + } + + #[test] + fn startup_error_maps_internal_server_error_to_internal() { + let edge_error = assert_startup_error( + TrustedServerError::Configuration { + message: "fictional missing setting".to_string(), + }, + "internal", + StatusCode::INTERNAL_SERVER_ERROR, + ); + assert!( + !edge_error.message().contains("fictional missing setting"), + "should keep the configuration detail out of the client message" + ); + } + + #[test] + fn startup_error_maps_service_unavailable_to_service_unavailable() { + let edge_error = assert_startup_error( + TrustedServerError::KvStore { + store_name: "example-store".to_string(), + message: "fictional store outage".to_string(), + }, + "service_unavailable", + StatusCode::SERVICE_UNAVAILABLE, + ); + assert_eq!( + edge_error.message(), + "An internal error occurred", + "should use the generic user message" + ); + } + + #[test] + fn startup_error_maps_bad_gateway_to_bad_gateway() { + let edge_error = assert_startup_error( + TrustedServerError::Proxy { + message: "fictional upstream failure".to_string(), + }, + "bad_gateway", + StatusCode::BAD_GATEWAY, + ); + assert_eq!( + edge_error.message(), + "An internal error occurred", + "should use the generic user message" + ); + } + + #[test] + fn startup_error_maps_bad_request_to_bad_request() { + let edge_error = assert_startup_error( + TrustedServerError::BadRequest { + message: "fictional startup input".to_string(), + }, + "bad_request", + StatusCode::BAD_REQUEST, + ); + assert_eq!( + edge_error.message(), + "Bad request: fictional startup input", + "should keep the safe bad-request detail" + ); + } + + #[test] + fn startup_error_maps_other_statuses_to_internal() { + for error in [ + TrustedServerError::Forbidden { + message: "fictional denial".to_string(), + }, + TrustedServerError::PartnerNotFound { + partner_id: "example-partner".to_string(), + }, + TrustedServerError::RequestTooLarge { + message: "fictional size".to_string(), + }, + ] { + assert_startup_error(error, "internal", StatusCode::INTERNAL_SERVER_ERROR); + } + } + + #[test] + fn startup_error_follows_the_current_context() { + let report = Report::new(TrustedServerError::Configuration { + message: "fictional inner failure".to_string(), + }) + .change_context(TrustedServerError::KvStore { + store_name: "example-store".to_string(), + message: "fictional outer failure".to_string(), + }); + + assert_eq!( + startup_error(&report).status(), + StatusCode::SERVICE_UNAVAILABLE, + "should map the outermost context, as the startup routers did" + ); + } } diff --git a/crates/trusted-server-core/src/forwarder.rs b/crates/trusted-server-core/src/forwarder.rs index 94a8fc6343..120fe64b5c 100644 --- a/crates/trusted-server-core/src/forwarder.rs +++ b/crates/trusted-server-core/src/forwarder.rs @@ -1,7 +1,7 @@ //! Authenticated browser-facing origin metadata, separate from transport evidence. -use edgezero_core::body::Body; -use http::{HeaderMap, HeaderName, Request}; +use edgezero_core::http::{Extensions, RequestParts}; +use http::{HeaderMap, HeaderName}; use url::Url; use crate::settings::Settings; @@ -58,21 +58,21 @@ impl ForwarderPreparation { /// # Examples /// /// ```ignore -/// prepare_trusted_forwarder(&mut request, &settings); +/// prepare_trusted_forwarder(request.parts_mut(), &settings); /// ``` -pub fn prepare_trusted_forwarder(request: &mut Request, settings: &Settings) { - if request.extensions().get::().is_none() { - let prepared = ForwarderPreparation::capture(request.headers(), settings); - request.extensions_mut().insert(prepared); +pub fn prepare_trusted_forwarder(parts: &mut RequestParts, settings: &Settings) { + if parts.extensions.get::().is_none() { + let prepared = ForwarderPreparation::capture(&parts.headers, settings); + parts.extensions.insert(prepared); } - request.headers_mut().remove(FORWARDER_AUTH_HEADER); + parts.headers.remove(FORWARDER_AUTH_HEADER); if let Some(config) = &settings.trusted_forwarder { - request.headers_mut().remove(config.auth_header.as_str()); + parts.headers.remove(config.auth_header.as_str()); } // The decision is now typed metadata. Raw forwarding fields must not // influence later consumers or escape to publishers and vendors. for name in ["forwarded", "x-forwarded-host", "x-forwarded-proto"] { - request.headers_mut().remove(name); + parts.headers.remove(name); } } @@ -160,9 +160,8 @@ fn parse_public_origin(scheme: &str, authority: &str) -> Option { }) } -pub(crate) fn public_origin(request: &Request) -> Option<&PublicOrigin> { - request - .extensions() +pub(crate) fn public_origin(extensions: &Extensions) -> Option<&PublicOrigin> { + extensions .get::() .and_then(|prepared| prepared.origin.as_ref()) } @@ -170,15 +169,16 @@ pub(crate) fn public_origin(request: &Request) -> Option<&PublicOrigin> { #[cfg(test)] mod tests { use super::*; + use edgezero_core::body::Body; use edgezero_core::ingress::{ CapturedTarget, HeaderFidelity, InboundOrigin, OriginSource, RequestIngress, TargetUnavailable, }; - use http::{HeaderValue, header}; + use http::{HeaderValue, Request, header}; use crate::http_util::RequestInfo; use crate::platform::ClientInfo; - use crate::test_support::tests::create_test_settings; + use crate::test_support::tests::{create_test_settings, with_request_parts}; use crate::trace::inspect_cookies; const SECRET: &str = "fictional-forwarder-secret-0123456789"; @@ -210,7 +210,9 @@ mod tests { fn authenticated_origin_overrides_transport_without_mutating_it() { let mut request = request(); let original_uri = request.uri().clone(); - prepare_trusted_forwarder(&mut request, &settings()); + with_request_parts(&mut request, |parts| { + prepare_trusted_forwarder(parts, &settings()) + }); let info = RequestInfo::from_request(&request, &ClientInfo::default()) .expect("should accept request metadata"); @@ -333,9 +335,11 @@ mod tests { .headers_mut() .append(name, HeaderValue::from_static(value)); } - prepare_trusted_forwarder(&mut request, &settings()); + with_request_parts(&mut request, |parts| { + prepare_trusted_forwarder(parts, &settings()) + }); assert!( - public_origin(&request).is_none(), + public_origin(request.extensions()).is_none(), "should reject invalid {name}" ); assert!( @@ -359,8 +363,10 @@ mod tests { if !valid { request.headers_mut().remove(FORWARDER_AUTH_HEADER); } - prepare_trusted_forwarder(&mut request, &settings()); - let first = public_origin(&request).cloned(); + with_request_parts(&mut request, |parts| { + prepare_trusted_forwarder(parts, &settings()) + }); + let first = public_origin(request.extensions()).cloned(); request .headers_mut() .insert(FORWARDER_AUTH_HEADER, HeaderValue::from_static(SECRET)); @@ -371,9 +377,11 @@ mod tests { request .headers_mut() .insert("x-forwarded-proto", HeaderValue::from_static("http")); - prepare_trusted_forwarder(&mut request, &settings()); + with_request_parts(&mut request, |parts| { + prepare_trusted_forwarder(parts, &settings()) + }); assert_eq!( - public_origin(&request), + public_origin(request.extensions()), first.as_ref(), "should retain the first forwarding decision" ); @@ -387,9 +395,11 @@ mod tests { #[test] fn disabled_forwarding_ignores_and_strips_spoofed_metadata() { let mut request = request(); - prepare_trusted_forwarder(&mut request, &create_test_settings()); + with_request_parts(&mut request, |parts| { + prepare_trusted_forwarder(parts, &create_test_settings()); + }); assert!( - public_origin(&request).is_none(), + public_origin(request.extensions()).is_none(), "should require explicit configuration" ); assert!( @@ -417,10 +427,12 @@ mod tests { .headers_mut() .insert("x-example-forwarder-auth", HeaderValue::from_static(SECRET)); - prepare_trusted_forwarder(&mut request, &settings); + with_request_parts(&mut request, |parts| { + prepare_trusted_forwarder(parts, &settings) + }); assert!( - public_origin(&request).is_some(), + public_origin(request.extensions()).is_some(), "should authenticate the configured credential field" ); for name in ["x-example-forwarder-auth", FORWARDER_AUTH_HEADER] { @@ -460,7 +472,9 @@ mod tests { .insert(header::HOST, HeaderValue::from_static("other.example.com")); } - prepare_trusted_forwarder(&mut request, &settings()); + with_request_parts(&mut request, |parts| { + prepare_trusted_forwarder(parts, &settings()) + }); let retained = request .extensions() diff --git a/crates/trusted-server-core/src/html_processor/finish.rs b/crates/trusted-server-core/src/html_processor/finish.rs index 3e801deea9..36c03ecbb8 100644 --- a/crates/trusted-server-core/src/html_processor/finish.rs +++ b/crates/trusted-server-core/src/html_processor/finish.rs @@ -348,15 +348,15 @@ mod tests { use crate::html_processor::HtmlWithStreamingProcessors; use crate::platform::test_support::noop_services; #[cfg(not(target_arch = "wasm32"))] - use crate::platform::test_support::{ - NoopBackend, NoopConfigStore, NoopGeo, NoopHttpClient, NoopSecretStore, - }; + use crate::platform::test_support::{NoopConfigStore, NoopGeo, NoopSecretStore}; #[cfg(not(target_arch = "wasm32"))] use crate::platform::{PlatformTimer, RuntimeServices}; use crate::streaming_processor::StreamProcessor; use crate::streaming_processor::finish::{ StreamFinish, StreamFinishDeadline, StreamFinishError, StreamFinishInput, }; + #[cfg(not(target_arch = "wasm32"))] + use crate::test_support::scripted_client::{ScriptedHttpClient, TestClock}; struct Stage { name: &'static str, @@ -875,8 +875,8 @@ mod tests { .config_store(Arc::new(NoopConfigStore)) .secret_store(Arc::new(NoopSecretStore)) .kv_store(Arc::new(edgezero_core::key_value_store::NoopKvStore)) - .backend(Arc::new(NoopBackend)) - .http_client(Arc::new(NoopHttpClient)) + .outbound_client(ScriptedHttpClient::forbidding().http_client()) + .clock(TestClock::new().monotonic_clock()) .geo(Arc::new(NoopGeo)) .client_info(crate::platform::ClientInfo::default()) .timer(Arc::new(RecordingTimer(Arc::clone(&requested)))) diff --git a/crates/trusted-server-core/src/http_util.rs b/crates/trusted-server-core/src/http_util.rs index 4a6bf1edb4..e4d9e42195 100644 --- a/crates/trusted-server-core/src/http_util.rs +++ b/crates/trusted-server-core/src/http_util.rs @@ -1,9 +1,12 @@ use base64::{Engine as _, engine::general_purpose::URL_SAFE_NO_PAD}; +use bytes::Bytes; use chacha20poly1305::{XChaCha20Poly1305, XNonce, aead::Aead as _, aead::KeyInit as _}; -use edgezero_core::body::Body as EdgeBody; +use edgezero_core::body::{Body as EdgeBody, BodyPayload}; +use edgezero_core::error::EdgeError; +use edgezero_core::http::RequestParts; use edgezero_core::ingress::{InboundOrigin, RequestIngress}; use error_stack::Report; -use http::{Request, Response, StatusCode, header}; +use http::{HeaderMap, Request, Response, StatusCode, header}; use sha2::{Digest as _, Sha256}; use std::net::Ipv6Addr; use std::time::Duration; @@ -58,25 +61,163 @@ pub const SPOOFABLE_FORWARDED_HEADERS: &[&str] = &[ /// values before sanitation. Every adapter removes them so a shared configuration /// cannot expose an authentication secret to publisher or integration handling. pub fn sanitize_trusted_client_ip_headers( - req: &mut Request, + req: &mut RequestParts, config: Option<&TrustedClientIpConfig>, ) { if let Some(config) = config { - req.headers_mut().remove(config.ip_header.as_str()); - req.headers_mut().remove(config.auth_header.as_str()); + req.headers.remove(config.ip_header.as_str()); + req.headers.remove(config.auth_header.as_str()); } } +/// Split a body into buffered bytes or a stream without releasing a streamed body's scope. +/// +/// A streaming [`EdgeBody`] becomes a [`BodyPayload::Stream`] that keeps any +/// outbound scope owner and closes it at end of stream, on error, or on drop. +/// A buffered body becomes [`BodyPayload::Once`] and releases its owner, +/// because its bytes no longer depend on the scope. A streaming body is never +/// returned as empty bytes, unlike [`EdgeBody::into_bytes`]. +#[must_use] +pub(crate) fn into_body_payload(body: EdgeBody) -> BodyPayload { + match body.into_stream() { + Ok(stream) => BodyPayload::Stream(stream), + Err(buffered) => buffered.into_parts().0, + } +} + +/// Wraps a buffered body for a synchronous rewrite pipeline. +/// +/// Callers buffer the responses they rewrite, so a streamed body here is a +/// bug. It fails instead of reading as empty. `pipeline` names the caller's +/// pipeline in the error, such as `"proxy"` or `"publisher"`. +/// +/// # Errors +/// +/// Returns [`TrustedServerError::Proxy`] for a streamed body. +pub(crate) fn body_as_reader( + body: EdgeBody, + pipeline: &str, +) -> Result, Report> { + match into_body_payload(body) { + BodyPayload::Once(bytes) => Ok(std::io::Cursor::new(bytes)), + BodyPayload::Stream(_) => Err(Report::new(TrustedServerError::Proxy { + message: format!("streaming body cannot be processed by the sync {pipeline} pipeline"), + })), + } +} + +/// Read an inbound request body into memory, accepting at most `max` bytes. +/// +/// When `headers` carry exactly one `Content-Length` field whose value is a +/// decimal number above `max`, the body is rejected without being polled. +/// Otherwise the body is read with [`EdgeBody::into_bytes_bounded`], which +/// stops at the first chunk that would cross `max`, so a duplicate or +/// malformed `Content-Length` never decides the outcome on its own. The +/// read also ends with an error once the inbound read deadline passes. +/// +/// This function does not log. The report names the limit or carries the +/// read error, and [`http_error`](crate::error::http_error) logs it at the +/// level its status calls for. Body bytes never reach the report. +/// +/// # Errors +/// +/// - [`TrustedServerError::RequestTooLarge`] (413) when the declared length or +/// the bytes read exceed `max`. +/// - [`TrustedServerError::RequestTimeout`] (408) when the read passes the +/// inbound read deadline. +/// - [`TrustedServerError::BadRequest`] (400) for any other client error the +/// body stream reports. +/// - [`TrustedServerError::Proxy`] (502) when the host fails to read the body. +/// +/// # Examples +/// +/// ``` +/// use edgezero_core::body::Body; +/// use http::{HeaderMap, HeaderValue, header}; +/// use trusted_server_core::error::{IntoHttpResponse as _, TrustedServerError}; +/// use trusted_server_core::http_util::read_request_body; +/// +/// let bytes = futures::executor::block_on(read_request_body( +/// &HeaderMap::new(), +/// Body::from(r#"{"example":true}"#), +/// 64, +/// )) +/// .expect("should read a body under the cap"); +/// assert_eq!(bytes.as_ref(), br#"{"example":true}"#); +/// +/// let mut headers = HeaderMap::new(); +/// headers.insert(header::CONTENT_LENGTH, HeaderValue::from_static("65")); +/// let report = futures::executor::block_on(read_request_body(&headers, Body::empty(), 64)) +/// .expect_err("should reject a declared length over the cap"); +/// assert!(matches!( +/// report.current_context(), +/// TrustedServerError::RequestTooLarge { .. } +/// )); +/// assert_eq!(report.current_context().status_code().as_u16(), 413); +/// ``` +pub async fn read_request_body( + headers: &HeaderMap, + body: EdgeBody, + max: usize, +) -> Result> { + if let Some(declared) = single_declared_content_length(headers) + && declared > u64::try_from(max).unwrap_or(u64::MAX) + { + return Err(Report::new(TrustedServerError::RequestTooLarge { + message: format!("declared Content-Length {declared} exceeds the {max} byte limit"), + })); + } + + body.into_bytes_bounded(max) + .await + .map_err(|error| request_body_read_error(&error, max)) +} + +/// Returns the `Content-Length` when `headers` carry exactly one field whose +/// value is a decimal number that fits in a `u64`. +fn single_declared_content_length(headers: &HeaderMap) -> Option { + let mut values = headers.get_all(header::CONTENT_LENGTH).iter(); + let value = values.next()?; + if values.next().is_some() { + return None; + } + let value = value.to_str().ok()?; + // `u64::from_str` also accepts a leading `+`, which `Content-Length` does not. + if value.is_empty() || !value.bytes().all(|byte| byte.is_ascii_digit()) { + return None; + } + value.parse().ok() +} + +/// Maps a failed bounded body read onto the Trusted Server error for its status. +fn request_body_read_error(error: &EdgeError, max: usize) -> Report { + let status = error.status(); + let context = match status { + StatusCode::PAYLOAD_TOO_LARGE => TrustedServerError::RequestTooLarge { + message: format!("request body exceeds the {max} byte limit"), + }, + StatusCode::REQUEST_TIMEOUT => TrustedServerError::RequestTimeout { + message: "request body was not read before the read deadline".to_string(), + }, + status if status.is_client_error() => TrustedServerError::BadRequest { + message: "request body could not be read".to_string(), + }, + _ => TrustedServerError::Proxy { + message: "failed to read request body".to_string(), + }, + }; + Report::new(context).attach(format!("body read failed with {status}: {error}")) +} + /// Strip forwarded headers that clients can spoof. /// /// Call this at the edge entry point (before routing) to prevent /// `X-Forwarded-Host: evil.com` from hijacking all URL rewriting. /// See . -pub fn sanitize_forwarded_headers(req: &mut Request) { +pub fn sanitize_forwarded_headers(headers: &mut HeaderMap) { for header in SPOOFABLE_FORWARDED_HEADERS { - if req.headers().contains_key(*header) { + if headers.remove(*header).is_some() { log::debug!("Stripped spoofable header: {header}"); - req.headers_mut().remove(*header); } } } @@ -193,7 +334,7 @@ impl RequestInfo { req: &Request, client_info: &ClientInfo, ) -> Result> { - if let Some(origin) = public_origin(req) { + if let Some(origin) = public_origin(req.extensions()) { return Ok(Self { host: origin.authority.clone(), scheme: origin.scheme.clone(), @@ -486,35 +627,63 @@ pub fn compute_encrypted_sha256_token(settings: &Settings, full_url: &str) -> St URL_SAFE_NO_PAD.encode(digest) } -/// Return an error if `bytes` exceeds `limit`. -/// -/// # Errors -/// -/// Returns [`TrustedServerError::RequestTooLarge`] when `bytes.len() > limit`. -pub fn enforce_max_body_size( - bytes: &[u8], - limit: usize, - what: &str, -) -> Result<(), Report> { - if bytes.len() > limit { - return Err(Report::new(TrustedServerError::RequestTooLarge { - message: format!( - "{what} payload {} exceeds limit of {limit} bytes", - bytes.len() - ), - })); - } - Ok(()) -} - #[cfg(test)] mod tests { use super::*; + use crate::error::{IntoHttpResponse as _, http_error}; use crate::platform::ClientInfo; use crate::redacted::Redacted; use crate::settings::TrustedClientIpConfig; + use crate::test_support::body_read::body_read_failure_cases; + use crate::test_support::logging::WarningCapture; use http::{HeaderName, HeaderValue, Method}; + #[test] + fn into_body_payload_keeps_buffered_and_streamed_bodies_apart() { + let BodyPayload::Once(bytes) = into_body_payload(EdgeBody::from("example")) else { + panic!("should keep a buffered body buffered"); + }; + let streamed = EdgeBody::stream(futures::stream::iter([bytes::Bytes::from_static( + b"example", + )])); + + assert_eq!( + bytes.as_ref(), + b"example", + "should return the buffered bytes" + ); + assert!( + matches!(into_body_payload(streamed), BodyPayload::Stream(_)), + "should keep a streamed body streamed" + ); + } + + #[test] + fn body_as_reader_rejects_a_streamed_body() { + let buffered = body_as_reader(EdgeBody::from("example"), "proxy") + .expect("should read a buffered body") + .into_inner(); + let streamed = EdgeBody::stream(futures::stream::iter([bytes::Bytes::from_static( + b"example", + )])); + + let error = + body_as_reader(streamed, "publisher").expect_err("should reject a streamed body"); + + assert_eq!( + buffered.as_ref(), + b"example", + "should keep the buffered bytes" + ); + let TrustedServerError::Proxy { message } = error.current_context() else { + panic!("should fail with the proxy context instead of reading as empty, got {error:?}"); + }; + assert_eq!( + message, "streaming body cannot be processed by the sync publisher pipeline", + "should name the caller's pipeline" + ); + } + fn build_request(method: Method, uri: &str) -> Request { Request::builder() .method(method) @@ -648,7 +817,9 @@ mod tests { }), edgezero_core::ingress::HeaderFidelity::default(), )); - crate::forwarder::prepare_trusted_forwarder(&mut request, &settings); + crate::test_support::tests::with_request_parts(&mut request, |parts| { + crate::forwarder::prepare_trusted_forwarder(parts, &settings); + }); let info = RequestInfo::from_request(&request, &default_client_info()) .expect("should accept trusted origin metadata"); @@ -986,7 +1157,9 @@ mod tests { ); set_header(&mut req, "x-unrelated", "preserved"); - sanitize_trusted_client_ip_headers(&mut req, Some(&config)); + crate::test_support::tests::with_request_parts(&mut req, |parts| { + sanitize_trusted_client_ip_headers(parts, Some(&config)); + }); assert!( req.headers().get("x-reader-ip").is_none(), @@ -1014,7 +1187,7 @@ mod tests { set_header(&mut req, "x-forwarded-proto", "https"); set_header(&mut req, "fastly-ssl", "1"); - sanitize_forwarded_headers(&mut req); + sanitize_forwarded_headers(req.headers_mut()); assert!( req.headers().get("forwarded").is_none(), @@ -1050,7 +1223,7 @@ mod tests { set_header(&mut req, "x-forwarded-host", "evil.com"); set_header(&mut req, "x-forwarded-proto", "http"); - sanitize_forwarded_headers(&mut req); + sanitize_forwarded_headers(req.headers_mut()); let info = RequestInfo::from_request(&req, &default_client_info()) .expect("should accept sanitized Host header"); @@ -1322,4 +1495,230 @@ mod tests { "should match text/html case-insensitively in fallback" ); } + + /// Cap the `read_request_body` tests read against. + const READ_TEST_MAX: usize = 16; + + /// Returns headers that carry each of `values` as a `Content-Length` field. + fn content_length_headers(values: &[&str]) -> HeaderMap { + let mut headers = HeaderMap::new(); + for value in values { + headers.append( + header::CONTENT_LENGTH, + HeaderValue::from_str(value).expect("should build Content-Length value"), + ); + } + headers + } + + /// Returns a streamed body that yields `chunks` in order. + fn streamed_body(chunks: &[&'static [u8]]) -> EdgeBody { + let items: Vec> = chunks + .iter() + .map(|chunk| Ok(bytes::Bytes::from_static(chunk))) + .collect(); + EdgeBody::from_stream(futures::stream::iter(items)) + } + + /// Returns a streamed body whose first item is `error`. + fn failing_body(error: EdgeError) -> EdgeBody { + EdgeBody::from_stream(futures::stream::iter([Err::(error)])) + } + + fn read_body( + headers: &HeaderMap, + body: EdgeBody, + ) -> Result> { + futures::executor::block_on(read_request_body(headers, body, READ_TEST_MAX)) + } + + /// Asserts that `result` failed with the variant `is_expected` accepts and with `status`. + fn assert_read_error( + result: Result>, + is_expected: fn(&TrustedServerError) -> bool, + status: StatusCode, + ) { + let report = result.expect_err("should reject the body"); + let error = report.current_context(); + assert!( + is_expected(error), + "should map to the expected variant, got {error:?}" + ); + assert_eq!( + error.status_code(), + status, + "should answer {status} for {error:?}" + ); + } + + #[test] + fn declared_length_over_max_returns_413_without_polling() { + let polled = std::rc::Rc::new(std::cell::Cell::new(false)); + let poll_flag = std::rc::Rc::clone(&polled); + let body = EdgeBody::from_stream(futures::stream::poll_fn(move |_| { + poll_flag.set(true); + std::task::Poll::Ready(None::>) + })); + let headers = content_length_headers(&["17"]); + + let result = read_body(&headers, body); + + assert_read_error( + result, + |error| matches!(error, TrustedServerError::RequestTooLarge { .. }), + StatusCode::PAYLOAD_TOO_LARGE, + ); + assert!( + !polled.get(), + "should reject a declared overflow without polling the body" + ); + } + + #[test] + fn stream_crossing_max_returns_413() { + let body = streamed_body(&[b"0123456789", b"0123456789"]); + + let result = read_body(&HeaderMap::new(), body); + + assert_read_error( + result, + |error| matches!(error, TrustedServerError::RequestTooLarge { .. }), + StatusCode::PAYLOAD_TOO_LARGE, + ); + } + + #[test] + fn read_after_deadline_returns_408() { + let body = failing_body(EdgeError::request_timeout("expired")); + + let result = read_body(&HeaderMap::new(), body); + + assert_read_error( + result, + |error| matches!(error, TrustedServerError::RequestTimeout { .. }), + StatusCode::REQUEST_TIMEOUT, + ); + } + + #[test] + fn client_caused_stream_error_returns_400() { + let body = failing_body(EdgeError::bad_request("malformed chunk framing")); + + let result = read_body(&HeaderMap::new(), body); + + assert_read_error( + result, + |error| matches!(error, TrustedServerError::BadRequest { .. }), + StatusCode::BAD_REQUEST, + ); + } + + #[test] + fn host_read_error_returns_502() { + let body = failing_body(EdgeError::bad_gateway("host read failed")); + + let result = read_body(&HeaderMap::new(), body); + + assert_read_error( + result, + |error| matches!(error, TrustedServerError::Proxy { .. }), + StatusCode::BAD_GATEWAY, + ); + } + + #[test] + fn body_read_failure_logs_once_at_its_status_level() { + for case in body_read_failure_cases(READ_TEST_MAX) { + let (request, expectation) = + case.into_request(http::Request::builder().uri("https://example.com/")); + let (parts, body) = request.into_parts(); + let capture = WarningCapture::start(); + + let report = read_body(&parts.headers, body).expect_err("should reject the body"); + let response = http_error(&report); + + let records = capture.take(); + drop(capture); + let name = expectation.name(); + let expected = expectation.expected(); + assert_eq!( + response.status(), + expected, + "should answer the {name} case with {expected}" + ); + let levels: Vec = records.iter().map(|record| record.level).collect(); + let level = if expected.is_client_error() { + log::Level::Warn + } else { + log::Level::Error + }; + assert_eq!( + levels, + [level], + "should log the {name} case once, at {level}" + ); + } + } + + #[test] + fn duplicate_content_length_falls_through_to_bounded_read() { + // Both declarations exceed the cap, so a precheck that trusted either + // one would answer 413 instead of reading the body. + let headers = content_length_headers(&["999", "999"]); + + let bytes = read_body(&headers, streamed_body(&[b"hello"])) + .expect("should read a body under the cap despite duplicate declarations"); + + assert_eq!( + bytes.as_ref(), + b"hello", + "should return the body the bounded read collected" + ); + } + + #[test] + fn non_numeric_content_length_falls_through_to_bounded_read() { + // `+999` parses as a Rust integer but is not a valid `Content-Length`. + for declared in ["not-a-number", "+999", "1e9", ""] { + let headers = content_length_headers(&[declared]); + + let bytes = read_body(&headers, streamed_body(&[b"hello"])).unwrap_or_else(|error| { + panic!("should read the body when Content-Length is `{declared}`: {error:?}") + }); + + assert_eq!( + bytes.as_ref(), + b"hello", + "should return the body for Content-Length `{declared}`" + ); + } + } + + #[test] + fn body_under_max_returns_exact_bytes() { + let at_cap: &'static [u8] = b"0123456789abcdef"; + let declared = at_cap.len().to_string(); + let cases = [ + ("buffered", HeaderMap::new(), EdgeBody::from(at_cap)), + ( + "streamed", + content_length_headers(&[declared.as_str()]), + streamed_body(&[&at_cap[..7], &at_cap[7..]]), + ), + ("empty", HeaderMap::new(), EdgeBody::empty()), + ]; + + for (label, headers, body) in cases { + let expected: &[u8] = if label == "empty" { b"" } else { at_cap }; + + let bytes = read_body(&headers, body) + .unwrap_or_else(|error| panic!("should read the {label} body: {error:?}")); + + assert_eq!( + bytes.as_ref(), + expected, + "should return the exact {label} body bytes" + ); + } + } } diff --git a/crates/trusted-server-core/src/integrations/adserver_mock.rs b/crates/trusted-server-core/src/integrations/adserver_mock.rs index d511bcbb7a..5042f321b4 100644 --- a/crates/trusted-server-core/src/integrations/adserver_mock.rs +++ b/crates/trusted-server-core/src/integrations/adserver_mock.rs @@ -5,7 +5,7 @@ //! based on price (highest price wins). use async_trait::async_trait; -use edgezero_core::body::Body as EdgeBody; +use edgezero_core::outbound::{OutboundCachePolicy, OutboundResponse, ResponseDecoding}; use error_stack::{Report, ResultExt}; use http::{Method, StatusCode, header}; use serde::{Deserialize, Serialize}; @@ -25,13 +25,13 @@ use crate::auction::types::{ AuctionContext, AuctionRequest, AuctionResponse, Bid, BidStatus, MediaType, }; use crate::error::TrustedServerError; -use crate::integrations::{ - UPSTREAM_RTB_MAX_RESPONSE_BYTES, collect_response_bounded, - ensure_integration_backend_with_timeout, predict_integration_backend_name, -}; -use crate::platform::{PlatformHttpRequest, PlatformResponse, RuntimeServices}; +use crate::integrations::UPSTREAM_RTB_MAX_RESPONSE_BYTES; +use crate::outbound::{OutboundRequestSource, buffered, outbound_error, outbound_request}; use crate::settings::{IntegrationConfig, Settings}; +/// The mediator response cap, the RTB cap that Prebid and APS responses share. +const MEDIATOR_MAX_RESPONSE_BYTES: u64 = UPSTREAM_RTB_MAX_RESPONSE_BYTES as u64; + // ============================================================================ // Configuration // ============================================================================ @@ -382,13 +382,11 @@ impl AdServerMockProvider { /// its body cannot be read within the response limit, or it is not valid JSON. async fn parse_response_inner( &self, - response: PlatformResponse, + response: OutboundResponse, response_time_ms: u64, bid_index: &BidIndex, dimensions_by_slot: Option<&BidDimensionIndex>, ) -> Result> { - let response = response.response; - // OpenRTB signals an empty auction with a bodiless 204, and the bidder // providers already treat it that way. Keep it a final mediation no-bid. if response.status() == StatusCode::NO_CONTENT { @@ -404,16 +402,19 @@ impl AdServerMockProvider { })); } - // collect_response_bounded caps memory from misbehaving providers. - let body_bytes = collect_response_bounded( - response.into_body(), - UPSTREAM_RTB_MAX_RESPONSE_BYTES, - "adserver_mock", - ) - .await - .change_context(TrustedServerError::Auction { - message: "Failed to read AdServer Mock response body".to_string(), - })?; + // The request buffered the response under the same cap. + let body_bytes = response + .into_bytes_bounded(MEDIATOR_MAX_RESPONSE_BYTES) + .await + .map_err(|error| { + outbound_error( + error, + TrustedServerError::Auction { + message: "Failed to read AdServer Mock response body".to_string(), + }, + OutboundRequestSource::Internal, + ) + })?; let response_json: Json = serde_json::from_slice(&body_bytes).change_context(TrustedServerError::Auction { message: "Failed to parse mediation response".to_string(), @@ -475,69 +476,37 @@ impl AuctionProvider for AdServerMockProvider { serde_json::to_vec(&mediation_req).change_context(TrustedServerError::Auction { message: "Failed to serialize mediation request".to_string(), })?; - let mut req = http::Request::builder() - .method(Method::POST) - .uri(&endpoint_url) - .header(header::CONTENT_TYPE, "application/json") - .body(EdgeBody::from(mediation_body)) - .change_context(TrustedServerError::Auction { - message: "Failed to build mediation request".to_string(), - })?; - - // Set Host header with port to ensure mocktioneer generates correct iframe URLs - if let Ok(url) = url::Url::parse(&self.config.endpoint) - && let Some(host) = url.host_str() - { - let host_with_port = if let Some(port) = url.port() { - format!("{}:{}", host, port) - } else { - host.to_string() - }; - match header::HeaderValue::from_str(&host_with_port) { - Ok(value) => { - req.headers_mut().insert(header::HOST, value); - } - Err(e) => { - log::warn!( - "Failed to build Host header for '{}': {}", - host_with_port, - e - ); - } - } - } - - // Uses the auction-scoped canonical transport timeout rather than the - // 15 s fixed timeout in ensure_integration_backend, which is for proxy - // endpoints. The exact logical budget remains in context.timeout_ms. - let backend_name = ensure_integration_backend_with_timeout( - context.services, - &self.config.endpoint, - "adserver_mock", - Duration::from_millis(u64::from(context.transport_timeout_ms)), - ) - .change_context(TrustedServerError::Auction { - message: format!( - "Failed to resolve backend for mediation endpoint: {}", - self.config.endpoint - ), + let target = outbound_request(Method::POST, &endpoint_url).map_err(|error| { + outbound_error( + error, + TrustedServerError::Auction { + message: "Failed to build mediation request".to_string(), + }, + OutboundRequestSource::Internal, + ) })?; + // The mediator's own timeout bounds it, and the orchestrator adds the + // auction deadline. `EdgeZero` sends the endpoint's authority, port + // included, as the Host, which mocktioneer needs for its iframe URLs. + let mut mediation_request = buffered( + target, + Duration::from_millis(u64::from(self.config.timeout_ms)), + MEDIATOR_MAX_RESPONSE_BYTES, + ResponseDecoding::Decode, + ) + .cache_policy(OutboundCachePolicy::Bypass) + .body(mediation_body); + mediation_request.headers_mut().insert( + header::CONTENT_TYPE, + header::HeaderValue::from_static("application/json"), + ); - let pending = context - .services - .http_client() - .send_async(PlatformHttpRequest::new(req, backend_name).with_cache_bypass()) - .await - .change_context(TrustedServerError::Auction { - message: "Failed to send mediation request".to_string(), - })?; - - Ok(ProviderRequestOutcome::pending(pending)) + Ok(ProviderRequestOutcome::ready(mediation_request)) } async fn parse_response( &self, - response: PlatformResponse, + response: OutboundResponse, response_time_ms: u64, ) -> Result> { // No auction context available — nurl/burl/ad_id restoration from the @@ -551,7 +520,7 @@ impl AuctionProvider for AdServerMockProvider { async fn parse_response_with_context( &self, - response: PlatformResponse, + response: OutboundResponse, response_time_ms: u64, request: &AuctionRequest, context: &AuctionContext<'_>, @@ -581,26 +550,6 @@ impl AuctionProvider for AdServerMockProvider { fn is_enabled(&self) -> bool { self.config.enabled } - - fn backend_name( - &self, - services: &RuntimeServices, - transport_timeout_ms: u32, - ) -> Option { - predict_integration_backend_name( - services, - &self.config.endpoint, - "adserver_mock", - Duration::from_millis(u64::from(transport_timeout_ms)), - ) - .inspect_err(|e| { - log::error!( - "Failed to predict backend name for AdServer Mock endpoint '{}': {e:?}", - self.config.endpoint - ); - }) - .ok() - } } // ============================================================================ diff --git a/crates/trusted-server-core/src/integrations/aps.rs b/crates/trusted-server-core/src/integrations/aps.rs index 62f8c8f068..3abc7097e8 100644 --- a/crates/trusted-server-core/src/integrations/aps.rs +++ b/crates/trusted-server-core/src/integrations/aps.rs @@ -8,6 +8,9 @@ use std::time::Duration; use async_trait::async_trait; use base64::{Engine as _, engine::general_purpose::STANDARD as BASE64_STANDARD}; use edgezero_core::body::Body as EdgeBody; +use edgezero_core::outbound::OutboundResponse; +#[cfg(test)] +use edgezero_core::outbound::{OutboundCachePolicy, ResponseDecoding}; use error_stack::{Report, ResultExt}; use http::header::HeaderName; use http::{HeaderMap, Method, StatusCode, header}; @@ -36,10 +39,6 @@ use crate::integrations::{ IntegrationRegistration, UPSTREAM_RTB_MAX_RESPONSE_BYTES, collect_response_bounded, }; #[cfg(test)] -use crate::integrations::{ - ensure_integration_backend_with_timeout, predict_integration_backend_name, -}; -#[cfg(test)] use crate::openrtb::ToExt; #[cfg(test)] use crate::openrtb::{ @@ -47,8 +46,11 @@ use crate::openrtb::{ UserExt, to_openrtb_i32, }; #[cfg(test)] -use crate::platform::PlatformHttpRequest; -use crate::platform::{PlatformResponse, RuntimeServices}; +use crate::outbound::{ + DEFAULT_BUFFERED_RESPONSE_BYTES, OutboundRequestSource, buffered, outbound_error, + outbound_request, +}; +use crate::platform::RuntimeServices; use crate::settings::{IntegrationConfig, Settings}; const APS_INTEGRATION_ID: &str = "aps"; @@ -881,7 +883,7 @@ pub(crate) async fn parse_planned_aps_response( profile: &ApsProfilePlan, endpoint: &str, input: &ProviderAuctionInput, - response: PlatformResponse, + response: OutboundResponse, response_time_ms: u64, debug_request: Option, ) -> Result> { @@ -893,7 +895,6 @@ pub(crate) async fn parse_planned_aps_response( allow_script_creatives: profile.allow_script_creatives, publisher_domain: &input.common_request().publisher.domain, }; - let response = response.response; let status = response.status(); let response_headers = if policy.debug { aps_debug_headers(response.headers()) @@ -1561,13 +1562,12 @@ impl ApsAuctionProvider { async fn parse_response_inner( &self, - response: PlatformResponse, + response: OutboundResponse, response_time_ms: u64, request: Option<&AuctionRequest>, debug_request: Option, debug_enabled: bool, ) -> Result> { - let response = response.response; let status = response.status(); let response_headers = if debug_enabled { Self::debug_headers(response.headers()) @@ -1696,46 +1696,42 @@ impl AuctionProvider for ApsAuctionProvider { .config .debug .then(|| String::from_utf8_lossy(&body).into_owned()); - let outbound_request = http::Request::builder() - .method(Method::POST) - .uri(&self.config.endpoint) - .header(header::CONTENT_TYPE, "application/json") - .body(EdgeBody::from(body)) - .change_context(TrustedServerError::Auction { - message: "Failed to build APS request".to_string(), - })?; + let target = outbound_request(Method::POST, &self.config.endpoint).map_err(|error| { + outbound_error( + error, + TrustedServerError::Auction { + message: "Failed to build APS request".to_string(), + }, + OutboundRequestSource::Internal, + ) + })?; + let mut aps_request = buffered( + target, + Duration::from_millis(u64::from(context.timeout_ms)), + DEFAULT_BUFFERED_RESPONSE_BYTES, + ResponseDecoding::Decode, + ) + .cache_policy(OutboundCachePolicy::Bypass) + .body(body); + aps_request.headers_mut().insert( + header::CONTENT_TYPE, + header::HeaderValue::from_static("application/json"), + ); let debug_request = debug_body.map(|body| ApsDebugRequest { body, - headers: Self::debug_headers(outbound_request.headers()), + headers: Self::debug_headers(aps_request.headers()), }); - let backend = ensure_integration_backend_with_timeout( - context.services, - &self.config.endpoint, - APS_INTEGRATION_ID, - Duration::from_millis(u64::from(context.timeout_ms)), - ) - .change_context(TrustedServerError::Auction { - message: "Failed to resolve APS backend".to_string(), - })?; - let pending = context - .services - .http_client() - .send_async(PlatformHttpRequest::new(outbound_request, backend).with_cache_bypass()) - .await - .change_context(TrustedServerError::Auction { - message: "Failed to send APS request".to_string(), - })?; Ok(match debug_request { Some(debug_request) => { - ProviderRequestOutcome::pending_with_state(pending, Box::new(debug_request)) + ProviderRequestOutcome::ready_with_state(aps_request, Box::new(debug_request)) } - None => ProviderRequestOutcome::pending(pending), + None => ProviderRequestOutcome::ready(aps_request), }) } async fn parse_response( &self, - response: PlatformResponse, + response: OutboundResponse, response_time_ms: u64, ) -> Result> { self.parse_response_inner(response, response_time_ms, None, None, self.config.debug) @@ -1744,7 +1740,7 @@ impl AuctionProvider for ApsAuctionProvider { async fn parse_response_with_context( &self, - response: PlatformResponse, + response: OutboundResponse, response_time_ms: u64, request: &AuctionRequest, context: &AuctionContext<'_>, @@ -1762,7 +1758,7 @@ impl AuctionProvider for ApsAuctionProvider { async fn parse_response_with_context_and_state( &self, - response: PlatformResponse, + response: OutboundResponse, response_time_ms: u64, request: &AuctionRequest, context: &AuctionContext<'_>, @@ -1800,17 +1796,6 @@ impl AuctionProvider for ApsAuctionProvider { fn is_enabled(&self) -> bool { self.config.enabled } - - fn backend_name(&self, services: &RuntimeServices, timeout_ms: u32) -> Option { - predict_integration_backend_name( - services, - &self.config.endpoint, - APS_INTEGRATION_ID, - Duration::from_millis(u64::from(timeout_ms)), - ) - .inspect_err(|error| log::error!("Failed to predict APS backend name: {error:?}")) - .ok() - } } #[derive(Debug)] @@ -1986,8 +1971,9 @@ mod tests { AdFormat, AdSlot, AuctionContext, AuctionRequest, BidStatus, PublisherInfo, UserInfo, }; use crate::integrations::IntegrationDocumentState; - use crate::platform::test_support::{ - StubHttpClient, build_services_with_http_client, noop_services, + use crate::platform::test_support::{build_services_with_outbound_client, noop_services}; + use crate::test_support::scripted_client::{ + ScriptedHttpClient, ScriptedReply, TestClock, outbound_response, }; use crate::test_support::tests::create_test_settings; use serde_json::json; @@ -2056,7 +2042,7 @@ mod tests { fn parse_with_context( provider: &ApsAuctionProvider, - response: PlatformResponse, + response: OutboundResponse, ) -> AuctionResponse { let settings = create_test_settings(); let services = noop_services(); @@ -2068,7 +2054,6 @@ mod tests { settings: &settings, request: &downstream, timeout_ms: 321, - transport_timeout_ms: 321, provider_responses: None, services: &services, }; @@ -2290,7 +2275,6 @@ mod tests { settings: &settings, request: &downstream, timeout_ms: 321, - transport_timeout_ms: 321, provider_responses: None, services: &services, }; @@ -2334,7 +2318,6 @@ mod tests { settings: &settings, request: &downstream, timeout_ms: 321, - transport_timeout_ms: 321, provider_responses: None, services: &services, }; @@ -2432,7 +2415,6 @@ mod tests { settings: &settings, request: &downstream, timeout_ms: 321, - transport_timeout_ms: 321, provider_responses: None, services: &services, }; @@ -2556,7 +2538,7 @@ mod tests { "seatbid": [{"bid": [bid("fictional-debug-bid", 1.23, "iframe")]}] })) .expect("should serialize APS response fixture"); - let platform_response = PlatformResponse::new( + let platform_response = outbound_response( edgezero_core::http::response_builder() .status(StatusCode::OK) .header(header::CONTENT_TYPE, "application/json") @@ -2600,11 +2582,9 @@ mod tests { #[test] fn debug_request_metadata_matches_the_dispatched_request() { - let stub = Arc::new(StubHttpClient::new()); - stub.push_response(200, br#"{"seatbid":[]}"#.to_vec()); - let services = build_services_with_http_client( - Arc::clone(&stub) as Arc - ); + let clock = TestClock::new(); + let client = ScriptedHttpClient::new(&clock); + let services = build_services_with_outbound_client(&client, &clock); let settings = create_test_settings(); let downstream = http::Request::builder() .uri("https://publisher.example/auction") @@ -2614,27 +2594,32 @@ mod tests { settings: &settings, request: &downstream, timeout_ms: 321, - transport_timeout_ms: 321, provider_responses: None, services: &services, }; let mut provider_config = config(); provider_config.debug = true; + client.respond_to( + Method::POST, + &provider_config.endpoint, + ScriptedReply::status(StatusCode::OK).body(br#"{"seatbid":[]}"#.to_vec()), + ); let provider = ApsAuctionProvider::new(provider_config); let auction_request = request(); let outcome = futures::executor::block_on(provider.request_bids(&auction_request, &context)) .expect("should dispatch APS request"); - let ProviderRequestOutcome::Pending { - request: pending, + let ProviderRequestOutcome::Ready { + request: aps_request, parse_state, } = outcome else { - panic!("should return a pending APS request"); + panic!("should return an unsent APS request"); }; - let platform_response = futures::executor::block_on(services.http_client().wait(pending)) - .expect("should collect APS response"); + let platform_response = + futures::executor::block_on(services.outbound_client().send(aps_request)) + .expect("should collect APS response"); let response = futures::executor::block_on(provider.parse_response_with_context_and_state( platform_response, 12, @@ -2644,20 +2629,16 @@ mod tests { )) .expect("should parse APS response"); - let sent_body = stub - .recorded_request_bodies() + let sent = client + .requests() .into_iter() .next() - .expect("should capture outbound APS body"); - let sent_headers = stub - .recorded_request_headers() - .into_iter() - .next() - .expect("should capture outbound APS headers"); - let sent_content_type = sent_headers - .iter() - .find(|(name, _)| name.eq_ignore_ascii_case(header::CONTENT_TYPE.as_str())) - .map(|(_, value)| value.as_str()); + .expect("should capture the outbound APS request"); + let sent_body = sent.body.to_vec(); + let sent_content_type = sent + .headers + .get(header::CONTENT_TYPE) + .and_then(|value| value.to_str().ok()); let call = &response.metadata["debug"]["httpcalls"]["aps"][0]; assert_eq!( call["requestbody"], @@ -2674,7 +2655,7 @@ mod tests { #[test] fn disabled_debug_omits_httpcall_metadata() { let provider = ApsAuctionProvider::new(config()); - let platform_response = PlatformResponse::new( + let platform_response = outbound_response( edgezero_core::http::response_builder() .status(StatusCode::OK) .body(EdgeBody::from( @@ -2695,20 +2676,20 @@ mod tests { let mut provider_config = config(); provider_config.debug = true; let provider = ApsAuctionProvider::new(provider_config); - let malformed = PlatformResponse::new( + let malformed = outbound_response( edgezero_core::http::response_builder() .status(StatusCode::OK) .body(EdgeBody::from(b"{not-json".to_vec())) .expect("should build malformed APS response"), ); - let unavailable = PlatformResponse::new( + let unavailable = outbound_response( edgezero_core::http::response_builder() .status(StatusCode::SERVICE_UNAVAILABLE) .header("retry-after", "5") .body(EdgeBody::from(b"temporarily unavailable".to_vec())) .expect("should build unavailable APS response"), ); - let preview_limited = PlatformResponse::new( + let preview_limited = outbound_response( edgezero_core::http::response_builder() .status(StatusCode::BAD_GATEWAY) .body(EdgeBody::from(vec![ @@ -2717,7 +2698,7 @@ mod tests { ])) .expect("should build preview-limited APS response"), ); - let oversized = PlatformResponse::new( + let oversized = outbound_response( edgezero_core::http::response_builder() .status(StatusCode::BAD_GATEWAY) .body(EdgeBody::from(vec![ @@ -2777,7 +2758,7 @@ mod tests { let mut provider_config = config(); provider_config.debug = true; let provider = ApsAuctionProvider::new(provider_config); - let platform_response = PlatformResponse::new( + let platform_response = outbound_response( edgezero_core::http::response_builder() .status(StatusCode::NO_CONTENT) .body(EdgeBody::empty()) @@ -2795,7 +2776,7 @@ mod tests { #[test] fn malformed_json_is_a_safe_shape_error() { let provider = ApsAuctionProvider::new(config()); - let platform_response = PlatformResponse::new( + let platform_response = outbound_response( edgezero_core::http::response_builder() .status(StatusCode::OK) .body(EdgeBody::from(b"{not-json".to_vec())) @@ -2823,7 +2804,7 @@ mod tests { let mut provider_config = config(); provider_config.debug = true; let provider = ApsAuctionProvider::new(provider_config); - let platform_response = PlatformResponse::new( + let platform_response = outbound_response( edgezero_core::http::response_builder() .status(StatusCode::OK) .body(EdgeBody::from( @@ -2856,7 +2837,7 @@ mod tests { #[test] fn no_content_and_empty_responses_are_no_bids() { let provider = ApsAuctionProvider::new(config()); - let platform_response = PlatformResponse::new( + let platform_response = outbound_response( edgezero_core::http::response_builder() .status(StatusCode::NO_CONTENT) .body(EdgeBody::empty()) diff --git a/crates/trusted-server-core/src/integrations/datadome.rs b/crates/trusted-server-core/src/integrations/datadome.rs index 0935b4b1a3..1321dd5339 100644 --- a/crates/trusted-server-core/src/integrations/datadome.rs +++ b/crates/trusted-server-core/src/integrations/datadome.rs @@ -59,6 +59,7 @@ use std::sync::{Arc, LazyLock}; use async_trait::async_trait; use edgezero_core::body::Body as EdgeBody; +use edgezero_core::outbound::{OutboundRequest, ResponseDecoding}; use error_stack::{Report, ResultExt}; use http::header; use http::{Method, StatusCode}; @@ -70,14 +71,18 @@ use validator::Validate; use crate::constants::ENV_FASTLY_IS_STAGING; use crate::error::TrustedServerError; +use crate::http_util::read_request_body; use crate::integrations::{ AttributeRewriteAction, INTEGRATION_MAX_BODY_BYTES, IntegrationAttributeContext, IntegrationAttributeRewriter, IntegrationEndpoint, IntegrationHeadInjector, IntegrationHtmlContext, IntegrationProxy, IntegrationRegistration, IntegrationRequestFilter, RequestFilterDecision, RequestFilterInput, UPSTREAM_SDK_MAX_RESPONSE_BYTES, - collect_body_bounded, collect_response_bounded, ensure_integration_backend, }; -use crate::platform::{PlatformHttpRequest, RuntimeServices}; +use crate::outbound::{ + BUFFERED_TIMEOUT, DEFAULT_BUFFERED_RESPONSE_BYTES, OutboundRequestSource, buffered, + outbound_error, outbound_request, +}; +use crate::platform::RuntimeServices; use crate::redacted::Redacted; use crate::settings::{IntegrationConfig, Settings}; @@ -196,7 +201,7 @@ pub struct DataDomeConfig { #[validate(url)] pub protection_api_origin: String, - /// First-byte timeout for Protection API calls, in milliseconds. + /// Total timeout for each Protection API call, in milliseconds. #[serde(default = "default_timeout_ms")] #[validate(range(min = 1, max = 10000))] pub timeout_ms: u32, @@ -620,15 +625,6 @@ impl DataDomeIntegration { } } - /// Extract the host from a URL for use in the Host header. - fn extract_host(url: &str) -> &str { - url.trim_start_matches("https://") - .trim_start_matches("http://") - .split('/') - .next() - .unwrap_or("api-js.datadome.co") - } - /// Handle the /tags.js endpoint - fetch and rewrite the `DataDome` SDK. async fn handle_tags_js( &self, @@ -639,18 +635,16 @@ impl DataDomeIntegration { log::info!("[datadome] Fetching tags.js from {}", target_url); - let backend = Self::backend_name_for_url(services, &target_url) - .change_context(Self::error("Invalid SDK URL"))?; - - let sdk_host = Self::extract_host(&self.config.sdk_origin); - - let mut backend_req = http::Request::builder() - .method(Method::GET) - .uri(&target_url) - .header(header::HOST, sdk_host) - .header(header::ACCEPT, "application/javascript, */*") - .body(EdgeBody::empty()) - .change_context(Self::error("Failed to build DataDome SDK request"))?; + let fetch_failed = || Self::error("Failed to fetch tags.js from DataDome"); + let mut backend_req = OutboundRequest::get(&target_url) + .and_then(|request| request.header(header::ACCEPT, "application/javascript, */*")) + .map_err(|error| { + outbound_error( + error, + Self::error("Failed to build DataDome SDK request"), + OutboundRequestSource::Internal, + ) + })?; // Copy relevant headers from original request if let Some(ua) = req.headers().get(header::USER_AGENT) { @@ -660,32 +654,39 @@ impl DataDomeIntegration { } let backend_resp = services - .http_client() - .send(PlatformHttpRequest::new(backend_req, backend)) + .outbound_client() + .send(buffered( + backend_req, + BUFFERED_TIMEOUT, + UPSTREAM_SDK_MAX_RESPONSE_BYTES, + ResponseDecoding::Decode, + )) .await - .change_context(Self::error("Failed to fetch tags.js from DataDome"))?; + .map_err(|error| { + outbound_error(error, fetch_failed(), OutboundRequestSource::Internal) + })?; - if backend_resp.response.status() != StatusCode::OK { + if backend_resp.status() != StatusCode::OK { log::warn!( "[datadome] tags.js fetch returned status {}", - backend_resp.response.status() + backend_resp.status() ); - return Ok(backend_resp.response); + return backend_resp.into_response().map_err(|error| { + outbound_error(error, fetch_failed(), OutboundRequestSource::Internal) + }); } // Read and rewrite the script content let cors_header = backend_resp - .response .headers() .get(header::ACCESS_CONTROL_ALLOW_ORIGIN) .cloned(); - let body = collect_response_bounded( - backend_resp.response.into_body(), - UPSTREAM_SDK_MAX_RESPONSE_BYTES, - DATADOME_INTEGRATION_ID, - ) - .await - .change_context(Self::error("Failed to read DataDome SDK response body"))?; + let body = backend_resp + .into_bytes_bounded(UPSTREAM_SDK_MAX_RESPONSE_BYTES) + .await + .map_err(|error| { + outbound_error(error, fetch_failed(), OutboundRequestSource::Internal) + })?; let rewritten = self.rewrite_script_content(&String::from_utf8_lossy(&body)); // Build response with caching headers @@ -728,37 +729,34 @@ impl DataDomeIntegration { // Use api_origin (api-js.datadome.co) for signal collection requests let target_url = self.build_api_url(datadome_path, parts.uri.query()); - let api_host = Self::extract_host(&self.config.api_origin); log::info!( - "[datadome] Proxying signal request to {} (method: {}, host: {})", + "[datadome] Proxying signal request to {} (method: {})", target_url, parts.method, - api_host ); - let backend = Self::backend_name_for_url(services, &target_url) - .change_context(Self::error("Invalid API URL"))?; - let request_body = if parts.method == Method::POST || parts.method == Method::PUT { - let bytes = - collect_body_bounded(body, INTEGRATION_MAX_BODY_BYTES, DATADOME_INTEGRATION_ID) - .await?; + let bytes = read_request_body(&parts.headers, body, INTEGRATION_MAX_BODY_BYTES).await?; EdgeBody::from(bytes) } else { EdgeBody::empty() }; - let mut backend_req = http::Request::builder() - .method(parts.method.clone()) - .uri(&target_url) - .header(header::HOST, api_host) - .body(request_body) - .change_context(Self::error("Failed to build DataDome API request"))?; + let proxy_failed = || Self::error("Failed to proxy signal request to DataDome"); + let mut backend_req = outbound_request(parts.method.clone(), &target_url) + .map_err(|error| { + outbound_error( + error, + Self::error("Failed to build DataDome API request"), + OutboundRequestSource::Internal, + ) + })? + .body(request_body); // Copy relevant headers from the original client request. // CONTENT_LENGTH is intentionally omitted: the body is re-materialized - // via collect_body_bounded, so its length may differ from the original. + // via read_request_body, so its length may differ from the original. let headers_to_copy = [ header::USER_AGENT, header::ACCEPT, @@ -776,17 +774,26 @@ impl DataDomeIntegration { } let backend_resp = services - .http_client() - .send(PlatformHttpRequest::new(backend_req, backend)) + .outbound_client() + .send(buffered( + backend_req, + BUFFERED_TIMEOUT, + DEFAULT_BUFFERED_RESPONSE_BYTES, + ResponseDecoding::Original, + )) .await - .change_context(Self::error("Failed to proxy signal request to DataDome"))?; + .map_err(|error| { + outbound_error(error, proxy_failed(), OutboundRequestSource::Internal) + })?; log::info!( "[datadome] Signal request returned status {}", - backend_resp.response.status() + backend_resp.status() ); - Ok(backend_resp.response) + backend_resp + .into_response() + .map_err(|error| outbound_error(error, proxy_failed(), OutboundRequestSource::Internal)) } /// Extract the path portion after the `DataDome` domain from a URL. @@ -803,13 +810,6 @@ impl DataDomeIntegration { }) .unwrap_or("/tags.js") } - - fn backend_name_for_url( - services: &RuntimeServices, - target_url: &str, - ) -> Result> { - ensure_integration_backend(services, target_url, DATADOME_INTEGRATION_ID, None) - } } #[async_trait(?Send)] @@ -1010,10 +1010,16 @@ pub fn register( #[cfg(test)] mod tests { - use std::sync::Arc; + use edgezero_core::outbound::{OutboundCachePolicy, ResponseDecoding, ResponseMode}; + use futures::executor::block_on; use super::*; - use crate::platform::test_support::{StubHttpClient, build_services_with_http_client}; + use crate::outbound::{BUFFERED_TIMEOUT, DEFAULT_BUFFERED_RESPONSE_BYTES}; + use crate::platform::test_support::build_services_with_outbound_client; + use crate::test_support::body_read::body_read_failure_cases; + use crate::test_support::scripted_client::{ + ExpectedTransport, ScriptedHttpClient, ScriptedReply, TestClock, + }; use crate::test_support::tests::create_test_settings; fn test_config() -> DataDomeConfig { @@ -1511,22 +1517,6 @@ mod tests { ); } - #[test] - fn extract_host() { - assert_eq!( - DataDomeIntegration::extract_host("https://api-js.datadome.co"), - "api-js.datadome.co" - ); - assert_eq!( - DataDomeIntegration::extract_host("https://js.datadome.co/path"), - "js.datadome.co" - ); - assert_eq!( - DataDomeIntegration::extract_host("http://example.com:8080/path"), - "example.com:8080" - ); - } - #[test] fn extract_datadome_path() { assert_eq!( @@ -1633,33 +1623,158 @@ mod tests { } } + fn readback_config() -> DataDomeConfig { + DataDomeConfig { + sdk_origin: "https://sdk.example.com".to_string(), + api_origin: "https://api.example.com".to_string(), + ..test_config() + } + } + #[test] - fn datadome_proxy_uses_platform_http_client() { - let stub = Arc::new(StubHttpClient::new()); - stub.push_response(200, b"ok".to_vec()); - let services = build_services_with_http_client( - Arc::clone(&stub) as Arc + fn datadome_tags_js_request_readback() { + let clock = TestClock::new(); + let client = ScriptedHttpClient::new(&clock); + let target = "https://sdk.example.com/tags.js?key=example"; + client.respond_to( + Method::GET, + target, + ScriptedReply::status(StatusCode::OK) + .header("access-control-allow-origin", "*") + .body("var dd = 1;"), ); - let settings = create_test_settings(); - let integration = DataDomeIntegration::new(test_config()); + let services = build_services_with_outbound_client(&client, &clock); + let integration = DataDomeIntegration::new(readback_config()); let req = http::Request::builder() - .method(http::Method::GET) - .uri("https://publisher.example/integrations/datadome/js/check") + .method(Method::GET) + .uri("https://publisher.example.com/integrations/datadome/tags.js?key=example") + .header(header::USER_AGENT, "example-agent") + .header(header::ACCEPT_ENCODING, "gzip") + .header(header::AUTHORIZATION, "Basic dXNlcjpwYXNz") + .header(header::COOKIE, "session_id=secret") .body(EdgeBody::empty()) - .expect("should build request"); + .expect("should build tags.js request"); - let response = futures::executor::block_on(integration.handle(&settings, &services, req)) - .expect("should proxy request"); + let response = block_on(integration.handle(&create_test_settings(), &services, req)) + .expect("should serve tags.js"); assert_eq!( - response.status(), - http::StatusCode::OK, - "should return stubbed response" + response + .headers() + .get(header::ACCESS_CONTROL_ALLOW_ORIGIN) + .and_then(|value| value.to_str().ok()), + Some("*"), + "should copy the upstream CORS header" + ); + assert_eq!( + response + .into_body() + .into_bytes() + .expect("should buffer the SDK body") + .as_ref(), + b"var dd = 1;", + "should serve the fetched SDK bytes" + ); + let requests = client.requests(); + assert_eq!(requests.len(), 1, "should send one tags.js request"); + let tags = &requests[0]; + assert_eq!( + (&tags.method, tags.uri.to_string()), + (&Method::GET, target.to_owned()), + "should fetch tags.js with the client's query" + ); + tags.assert_headers(&[ + ("accept", "application/javascript, */*"), + ("user-agent", "example-agent"), + ]); + tags.assert_transport(ExpectedTransport { + timeout: BUFFERED_TIMEOUT, + response_mode: ResponseMode::Buffered { + max_bytes: 16 * 1024 * 1024, + }, + response_decoding: ResponseDecoding::Decode, + cache_policy: OutboundCachePolicy::PlatformDefault, + }); + } + + #[test] + fn datadome_js_api_request_readback() { + let clock = TestClock::new(); + let client = ScriptedHttpClient::new(&clock); + let target = "https://api.example.com/js/?ddv=1"; + client.respond_to( + Method::POST, + target, + ScriptedReply::status(StatusCode::OK).body("{}"), ); + let services = build_services_with_outbound_client(&client, &clock); + let integration = DataDomeIntegration::new(readback_config()); + let payload = b"jsData=example"; + let req = http::Request::builder() + .method(Method::POST) + .uri("https://publisher.example.com/integrations/datadome/js/?ddv=1") + .header(header::USER_AGENT, "example-agent") + .header(header::ACCEPT, "*/*") + .header(header::ACCEPT_LANGUAGE, "en") + .header(header::ACCEPT_ENCODING, "gzip") + .header(header::CONTENT_TYPE, "application/x-www-form-urlencoded") + .header(header::CONTENT_LENGTH, payload.len().to_string()) + .header(header::ORIGIN, "https://publisher.example.com") + .header(header::REFERER, "https://publisher.example.com/page") + .header(header::AUTHORIZATION, "Basic dXNlcjpwYXNz") + .header(header::COOKIE, "session_id=secret") + .body(EdgeBody::from(payload.to_vec())) + .expect("should build signal request"); + + let response = block_on(integration.handle(&create_test_settings(), &services, req)) + .expect("should proxy the signal request"); + + assert_eq!(response.status(), StatusCode::OK, "should relay the status"); + let requests = client.requests(); + assert_eq!(requests.len(), 1, "should send one signal request"); + let signal = &requests[0]; assert_eq!( - stub.recorded_backend_names(), - vec!["stub-backend".to_string()], - "should route outbound request through PlatformHttpClient" + (&signal.method, signal.uri.to_string(), signal.body.as_ref()), + (&Method::POST, target.to_owned(), payload.as_slice()), + "should forward the method, path, query, and body" ); + signal.assert_headers(&[ + ("user-agent", "example-agent"), + ("accept", "*/*"), + ("accept-language", "en"), + ("accept-encoding", "gzip"), + ("content-type", "application/x-www-form-urlencoded"), + ("origin", "https://publisher.example.com"), + ("referer", "https://publisher.example.com/page"), + ]); + signal.assert_transport(ExpectedTransport { + timeout: BUFFERED_TIMEOUT, + response_mode: ResponseMode::Buffered { + max_bytes: DEFAULT_BUFFERED_RESPONSE_BYTES, + }, + response_decoding: ResponseDecoding::Original, + cache_policy: OutboundCachePolicy::PlatformDefault, + }); + } + + #[test] + fn datadome_js_api_body_read_failures_map_to_statuses() { + let settings = create_test_settings(); + let integration = DataDomeIntegration::new(test_config()); + for case in body_read_failure_cases(INTEGRATION_MAX_BODY_BYTES) { + let clock = TestClock::new(); + let client = ScriptedHttpClient::forbidding(); + let services = build_services_with_outbound_client(&client, &clock); + let (req, expectation) = case.into_request( + http::Request::builder() + .method(http::Method::POST) + .uri("https://publisher.example/integrations/datadome/js/check"), + ); + + let outcome = block_on(integration.handle(&settings, &services, req)); + + expectation.assert_outcome("DataDome /js/", &outcome); + client.assert_no_requests(); + } } } diff --git a/crates/trusted-server-core/src/integrations/datadome/protection.rs b/crates/trusted-server-core/src/integrations/datadome/protection.rs index 1d153d087b..2eb2fab76d 100644 --- a/crates/trusted-server-core/src/integrations/datadome/protection.rs +++ b/crates/trusted-server-core/src/integrations/datadome/protection.rs @@ -1,19 +1,21 @@ use std::time::Duration; use edgezero_core::body::Body as EdgeBody; -use edgezero_core::http::{HeaderMap, HeaderName, request_builder}; -use error_stack::{Report, ResultExt}; +use edgezero_core::http::{HeaderMap, HeaderName, HeaderValue}; +use edgezero_core::outbound::{OutboundRequest, OutboundResponse, ResponseDecoding}; +use error_stack::Report; use http::{Method, Request, Response, StatusCode, header}; use sha2::{Digest as _, Sha256}; use subtle::ConstantTimeEq as _; -use url::Url; use crate::error::TrustedServerError; use crate::http_util::{RequestInfo, is_navigation_request}; use crate::integrations::{ HeaderMutation, RequestFilterDecision, RequestFilterEffects, RequestFilterInput, }; -use crate::platform::{PlatformBackendSpec, PlatformHttpRequest, RuntimeServices}; +use crate::outbound::{ + DEFAULT_BUFFERED_RESPONSE_BYTES, OutboundRequestSource, buffered, outbound_error, +}; use crate::redacted::Redacted; use super::DataDomeIntegration; @@ -79,51 +81,53 @@ impl DataDomeIntegration { &self, input: RequestFilterInput<'_>, ) -> Result { - let api_url = self.protection_validate_url(); - let backend_name = self - .ensure_protection_backend(input.services, &api_url) - .map_err(ProtectionRequestError::Setup)?; let server_side_key = self .server_side_key() .map_err(ProtectionRequestError::Setup)?; let payload = self.build_protection_payload(&input, server_side_key); - let encoded_body = form_encode(&payload.fields); - - let mut builder = request_builder() - .method(Method::POST.as_str()) - .uri(api_url.as_str()) - .header( - header::CONTENT_TYPE.as_str(), - "application/x-www-form-urlencoded", - ) - .header( - header::CONTENT_LENGTH.as_str(), - encoded_body.len().to_string(), - ); + let mut request = + OutboundRequest::post(self.protection_validate_url()).map_err(|error| { + ProtectionRequestError::Setup(outbound_error( + error, + Self::error("Failed to build DataDome Protection API request"), + OutboundRequestSource::Internal, + )) + })?; + request.headers_mut().insert( + header::CONTENT_TYPE, + HeaderValue::from_static("application/x-www-form-urlencoded"), + ); if payload.uses_header_client_id { - builder = builder.header(HEADER_DATADOME_X_SET_COOKIE, "true"); + request.headers_mut().insert( + HeaderName::from_static(HEADER_DATADOME_X_SET_COOKIE), + HeaderValue::from_static("true"), + ); } + let request = buffered( + request.body(EdgeBody::from(form_encode(&payload.fields))), + Duration::from_millis(u64::from(self.config.timeout_ms)), + DEFAULT_BUFFERED_RESPONSE_BYTES, + ResponseDecoding::Decode, + ); - let request = builder - .body(EdgeBody::from(encoded_body)) - .change_context(Self::error( - "Failed to build DataDome Protection API request", - )) - .map_err(ProtectionRequestError::Runtime)?; - - let platform_response = input + let response = input .services - .http_client() - .send(PlatformHttpRequest::new(request, backend_name)) + .outbound_client() + .send(request) .await - .change_context(Self::error("Failed to call DataDome Protection API")) - .map_err(ProtectionRequestError::Runtime)?; + .and_then(OutboundResponse::into_response) + .map_err(|error| { + ProtectionRequestError::Runtime(outbound_error( + error, + Self::error("Failed to call DataDome Protection API"), + OutboundRequestSource::Internal, + )) + })?; - let status = platform_response.response.status(); - let datadome_status = datadome_response_status(platform_response.response.headers()); - let decision = - self.classify_protection_response(platform_response.response, input.request.method()); + let status = response.status(); + let datadome_status = datadome_response_status(response.headers()); + let decision = self.classify_protection_response(response, input.request.method()); log_protection_result(&input, status, datadome_status, &decision); Ok(decision) @@ -219,32 +223,6 @@ impl DataDomeIntegration { ) } - fn ensure_protection_backend( - &self, - services: &RuntimeServices, - api_url: &str, - ) -> Result> { - let parsed = Url::parse(api_url) - .change_context(Self::error("Invalid DataDome Protection API URL"))?; - let host = parsed - .host_str() - .ok_or_else(|| Report::new(Self::error("Missing DataDome Protection API host")))?; - let spec = PlatformBackendSpec { - scheme: parsed.scheme().to_string(), - host: host.to_string(), - port: parsed.port(), - host_header_override: None, - certificate_check: true, - first_byte_timeout: Duration::from_millis(u64::from(self.config.timeout_ms)), - between_bytes_timeout: Duration::from_millis(u64::from(self.config.timeout_ms)), - discriminator: None, - }; - - services.backend().ensure(&spec).change_context(Self::error( - "Failed to register DataDome Protection API backend", - )) - } - fn server_side_key(&self) -> Result<&Redacted, Report> { self.config .server_side_key_secret_name @@ -263,7 +241,7 @@ impl DataDomeIntegration { ) -> ProtectionPayload { let req = &*input.request; let client_info = input.services.client_info(); - let public_info = crate::forwarder::public_origin(req) + let public_info = crate::forwarder::public_origin(req.extensions()) .and_then(|_| RequestInfo::from_request(req, client_info).ok()); let request_host = public_info .as_ref() @@ -458,17 +436,11 @@ impl DataDomeIntegration { } if matches!(status.as_u16(), 301 | 302 | 401 | 403 | 429) { + // The verdict request buffers its response, so the challenge body is in memory. let response_body = if request_method == Method::HEAD { EdgeBody::empty() } else { - if body.is_stream() { - log::warn!( - "[datadome] Protection API challenge body was streaming; failing open" - ); - return RequestFilterDecision::Continue(RequestFilterEffects::default()); - } - let body_bytes = body.into_bytes().unwrap_or_default(); - EdgeBody::from(body_bytes.as_ref().to_vec()) + body }; let challenge = Response::builder() .status(status) @@ -823,27 +795,113 @@ mod tests { DataDomeConfig, ProtectionExclusionRuleConfig, ProtectionMatcherConfig, ProtectionTestBypassConfig, }; - use crate::platform::GeoInfo; use crate::platform::test_support::{ - HashMapConfigStore, HashMapSecretStore, NoopConfigStore, NoopSecretStore, StubHttpClient, + HashMapConfigStore, NoopConfigStore, NoopSecretStore, build_services_with_config_and_secret, build_services_with_config_and_secret_and_client_ip, - build_services_with_secret_and_http_client, noop_services_with_client_ip, + build_services_with_outbound_client, noop_services_with_client_ip, }; + use crate::platform::{GeoInfo, RuntimeServices}; use crate::settings::Settings; + use crate::test_support::scripted_client::{ + ExpectedTransport, ScriptedHttpClient, ScriptedReply, TestClock, + }; + use edgezero_core::http::request_builder; + use edgezero_core::outbound::{OutboundCachePolicy, ResponseMode}; use super::*; static FASTLY_IS_STAGING_ENV_LOCK: Mutex<()> = Mutex::new(()); + /// Verdict endpoint of [`protection_config`]. + const PROTECTION_VALIDATE_URL: &str = "https://protection.example.com/validate-request"; + fn protection_config() -> DataDomeConfig { DataDomeConfig { enabled: true, enable_protection: true, server_side_key_secret_name: Some(Redacted::new("server-side-key".to_string())), + protection_api_origin: "https://protection.example.com".to_string(), ..DataDomeConfig::default() } } + /// Services whose Protection API answers one verdict request with an allow. + fn services_with_allowing_protection_api() -> (ScriptedHttpClient, RuntimeServices) { + let clock = TestClock::new(); + let client = ScriptedHttpClient::new(&clock); + client.respond_to( + Method::POST, + PROTECTION_VALIDATE_URL, + ScriptedReply::status(StatusCode::OK).header(HEADER_DATADOME_RESPONSE, "200"), + ); + let services = build_services_with_outbound_client(&client, &clock); + (client, services) + } + + /// Services that fail the test's assertions if the Protection API is called. + fn services_without_protection_api() -> (ScriptedHttpClient, RuntimeServices) { + let client = ScriptedHttpClient::forbidding(); + let services = build_services_with_outbound_client(&client, &TestClock::new()); + (client, services) + } + + #[test] + fn protection_api_request_readback() { + let (client, services) = services_with_allowing_protection_api(); + let config = DataDomeConfig { + timeout_ms: 750, + ..protection_config() + }; + let integration = + DataDomeIntegration::try_new(config).expect("should create DataDome integration"); + let settings = Settings::default(); + let mut request = request_for_filter(); + request.headers_mut().insert( + HEADER_DATADOME_CLIENT_ID, + edgezero_core::http::HeaderValue::from_static("client-123"), + ); + + let decision = futures::executor::block_on(integration.filter_protection_request( + RequestFilterInput { + settings: &settings, + services: &services, + request: &mut request, + geo_info: None, + is_integration_route: false, + }, + )); + + assert!( + matches!(decision, RequestFilterDecision::Continue(_)), + "an allow verdict should continue" + ); + let requests = client.requests(); + assert_eq!(requests.len(), 1, "should send one verdict request"); + let validate = &requests[0]; + assert_eq!( + (&validate.method, validate.uri.to_string()), + (&Method::POST, PROTECTION_VALIDATE_URL.to_owned()), + "should post to the validate endpoint" + ); + validate.assert_headers(&[ + ("content-type", "application/x-www-form-urlencoded"), + (HEADER_DATADOME_X_SET_COOKIE, "true"), + ]); + let body = std::str::from_utf8(&validate.body).expect("should send a UTF-8 form body"); + assert!( + body.starts_with("Key=server-side-key&") && body.contains("&ClientID=client-123&"), + "should send the server-side key and client ID in the form body: {body}" + ); + validate.assert_transport(ExpectedTransport { + timeout: Duration::from_millis(750), + response_mode: ResponseMode::Buffered { + max_bytes: DEFAULT_BUFFERED_RESPONSE_BYTES, + }, + response_decoding: ResponseDecoding::Decode, + cache_policy: OutboundCachePolicy::PlatformDefault, + }); + } + fn protection_integration() -> Arc { DataDomeIntegration::try_new(protection_config()).expect("should create integration") } @@ -960,7 +1018,9 @@ mod tests { .headers_mut() .remove(crate::forwarder::FORWARDER_AUTH_HEADER); } - crate::forwarder::prepare_trusted_forwarder(&mut request, &settings); + crate::test_support::tests::with_request_parts(&mut request, |parts| { + crate::forwarder::prepare_trusted_forwarder(parts, &settings); + }); let payload = integration.build_protection_payload( &RequestFilterInput { settings: &settings, @@ -1024,16 +1084,7 @@ mod tests { ..protection_config() }; let integration = DataDomeIntegration::try_new(config).expect("should create integration"); - let mut secrets = HashMap::new(); - secrets.insert( - "datadome_test_bypass".to_string(), - b"temporary-test-credential-32-bytes!".to_vec(), - ); - let http_client = Arc::new(StubHttpClient::new()); - let services = build_services_with_secret_and_http_client( - HashMapSecretStore::new(secrets), - http_client.clone(), - ); + let (client, services) = services_without_protection_api(); let settings = Settings::default(); let mut request = request_for_filter(); request.headers_mut().insert( @@ -1059,7 +1110,7 @@ mod tests { "the bypass credential must not reach the publisher origin" ); assert!( - http_client.recorded_backend_names().is_empty(), + client.requests().is_empty(), "a matching test credential must not call the Protection API" ); } @@ -1084,21 +1135,7 @@ mod tests { }; let integration = DataDomeIntegration::try_new(config).expect("should create integration"); - let mut secrets = HashMap::new(); - secrets.insert( - "datadome_server_side_key".to_string(), - b"server-side-key".to_vec(), - ); - let http_client = Arc::new(StubHttpClient::new()); - http_client.push_response_with_headers( - 200, - Vec::new(), - vec![(HEADER_DATADOME_RESPONSE, "200")], - ); - let services = build_services_with_secret_and_http_client( - HashMapSecretStore::new(secrets), - http_client.clone(), - ); + let (client, services) = services_with_allowing_protection_api(); let settings = Settings::default(); let mut request = request_for_filter(); request.headers_mut().insert( @@ -1124,7 +1161,7 @@ mod tests { "an inactive bypass must not suppress the DataDome client tag" ); assert_eq!( - http_client.recorded_backend_names().len(), + client.requests().len(), 1, "an inactive bypass must still call the Protection API" ); @@ -1146,25 +1183,7 @@ mod tests { ..protection_config() }; let integration = DataDomeIntegration::try_new(config).expect("should create integration"); - let mut secrets = HashMap::new(); - secrets.insert( - "datadome_server_side_key".to_string(), - b"server-side-key".to_vec(), - ); - secrets.insert( - "datadome_test_bypass".to_string(), - b"temporary-test-credential-32-bytes!".to_vec(), - ); - let http_client = Arc::new(StubHttpClient::new()); - http_client.push_response_with_headers( - 200, - Vec::new(), - vec![(HEADER_DATADOME_RESPONSE, "200")], - ); - let services = build_services_with_secret_and_http_client( - HashMapSecretStore::new(secrets), - http_client.clone(), - ); + let (client, services) = services_with_allowing_protection_api(); let settings = Settings::default(); let mut request = request_for_filter(); request.headers_mut().insert( @@ -1207,7 +1226,7 @@ mod tests { "the bypass must not suppress the DataDome client tag outside staging" ); assert_eq!( - http_client.recorded_backend_names().len(), + client.requests().len(), 1, "the bypass must still call the Protection API outside staging" ); @@ -1236,16 +1255,7 @@ mod tests { ..protection_config() }; let integration = DataDomeIntegration::try_new(config).expect("should create integration"); - let mut secrets = HashMap::new(); - secrets.insert( - "datadome_test_bypass".to_string(), - b"temporary-test-credential-32-bytes!".to_vec(), - ); - let http_client = Arc::new(StubHttpClient::new()); - let services = build_services_with_secret_and_http_client( - HashMapSecretStore::new(secrets), - http_client.clone(), - ); + let (client, services) = services_without_protection_api(); let settings = Settings::default(); let mut request = request_for_filter(); request.headers_mut().insert( @@ -1264,7 +1274,7 @@ mod tests { "a matching test credential should suppress the tag even on an excluded path" ); assert!( - http_client.recorded_backend_names().is_empty(), + client.requests().is_empty(), "a matching test credential must not call the Protection API" ); } @@ -1284,25 +1294,7 @@ mod tests { ..protection_config() }; let integration = DataDomeIntegration::try_new(config).expect("should create integration"); - let mut secrets = HashMap::new(); - secrets.insert( - "datadome_server_side_key".to_string(), - b"server-side-key".to_vec(), - ); - secrets.insert( - "datadome_test_bypass".to_string(), - b"temporary-test-credential-32-bytes!".to_vec(), - ); - let http_client = Arc::new(StubHttpClient::new()); - http_client.push_response_with_headers( - 200, - Vec::new(), - vec![(HEADER_DATADOME_RESPONSE, "200")], - ); - let services = build_services_with_secret_and_http_client( - HashMapSecretStore::new(secrets), - http_client.clone(), - ); + let (client, services) = services_with_allowing_protection_api(); let settings = Settings::default(); let mut request = request_for_filter(); request.headers_mut().insert( @@ -1328,7 +1320,7 @@ mod tests { "an invalid bypass credential must not reach the publisher origin" ); assert_eq!( - http_client.recorded_backend_names().len(), + client.requests().len(), 1, "a non-matching credential must still call the Protection API" ); @@ -1349,25 +1341,7 @@ mod tests { ..protection_config() }; let integration = DataDomeIntegration::try_new(config).expect("should create integration"); - let mut secrets = HashMap::new(); - secrets.insert( - "datadome_server_side_key".to_string(), - b"server-side-key".to_vec(), - ); - secrets.insert( - "datadome_test_bypass".to_string(), - b"temporary-test-credential-32-bytes!".to_vec(), - ); - let http_client = Arc::new(StubHttpClient::new()); - http_client.push_response_with_headers( - 200, - Vec::new(), - vec![(HEADER_DATADOME_RESPONSE, "200")], - ); - let services = build_services_with_secret_and_http_client( - HashMapSecretStore::new(secrets), - http_client.clone(), - ); + let (client, services) = services_with_allowing_protection_api(); let settings = Settings::default(); let mut request = request_for_filter(); for value in [ @@ -1391,7 +1365,7 @@ mod tests { "all duplicate bypass values should be stripped" ); assert!(!has_client_tag_suppression_marker(&request)); - assert_eq!(http_client.recorded_backend_names().len(), 1); + assert_eq!(client.requests().len(), 1); } #[test] @@ -1406,14 +1380,7 @@ mod tests { }; let integration = DataDomeIntegration::try_new(config).expect("should accept short bypass credential"); - let http_client = Arc::new(StubHttpClient::new()); - http_client.push_response_with_headers( - 200, - Vec::new(), - vec![(HEADER_DATADOME_RESPONSE, "200")], - ); - let services = - build_services_with_secret_and_http_client(NoopSecretStore, http_client.clone()); + let (client, services) = services_with_allowing_protection_api(); let settings = Settings::default(); let mut request = request_for_filter(); request.headers_mut().insert( @@ -1433,7 +1400,7 @@ mod tests { ); assert!(!has_client_tag_suppression_marker(&request)); assert_eq!( - http_client.recorded_backend_names().len(), + client.requests().len(), 1, "a short credential should not bypass the Protection API" ); diff --git a/crates/trusted-server-core/src/integrations/didomi.rs b/crates/trusted-server-core/src/integrations/didomi.rs index 7e9c65e74a..490e54a83b 100644 --- a/crates/trusted-server-core/src/integrations/didomi.rs +++ b/crates/trusted-server-core/src/integrations/didomi.rs @@ -2,6 +2,7 @@ use std::sync::Arc; use async_trait::async_trait; use edgezero_core::body::Body as EdgeBody; +use edgezero_core::outbound::{OutboundCachePolicy, OutboundResponse, ResponseDecoding}; use error_stack::{Report, ResultExt}; use http::Method; use http::header::{self, HeaderMap, HeaderValue}; @@ -10,12 +11,16 @@ use url::Url; use validator::{Validate, ValidationError}; use crate::error::TrustedServerError; +use crate::http_util::read_request_body; use crate::integrations::{ INTEGRATION_MAX_BODY_BYTES, IntegrationEndpoint, IntegrationHeadInjector, - IntegrationHtmlContext, IntegrationProxy, IntegrationRegistration, collect_body_bounded, - ensure_integration_backend, + IntegrationHtmlContext, IntegrationProxy, IntegrationRegistration, }; -use crate::platform::{GeoInfo, PlatformHttpRequest, RuntimeServices}; +use crate::outbound::{ + BUFFERED_TIMEOUT, DEFAULT_BUFFERED_RESPONSE_BYTES, OutboundRequestSource, buffered, + outbound_error, outbound_request, +}; +use crate::platform::{GeoInfo, RuntimeServices}; use crate::settings::{IntegrationConfig, Settings}; const DIDOMI_INTEGRATION_ID: &str = "didomi"; @@ -325,13 +330,6 @@ impl DidomiIntegration { ); } - fn backend_name_for_origin( - services: &RuntimeServices, - origin: &str, - ) -> Result> { - ensure_integration_backend(services, origin, DIDOMI_INTEGRATION_ID, None) - } - fn geo_failure_response(reason: &str) -> http::Response { log::warn!("Didomi loader geo unavailable: reason={reason}"); let mut response = http::Response::builder() @@ -461,23 +459,23 @@ impl IntegrationProxy for DidomiIntegration { let target_url = self .build_target_url(base_origin, consent_path, query) .change_context(Self::error("Failed to build Didomi target URL"))?; - let backend_name = Self::backend_name_for_origin(services, base_origin) - .change_context(Self::error("Failed to configure Didomi backend"))?; let request_body = if parts.method == Method::POST { - let bytes = - collect_body_bounded(body, INTEGRATION_MAX_BODY_BYTES, DIDOMI_INTEGRATION_ID) - .await?; + let bytes = read_request_body(&parts.headers, body, INTEGRATION_MAX_BODY_BYTES).await?; EdgeBody::from(bytes) } else { EdgeBody::empty() }; - let mut proxy_req = http::Request::builder() - .method(parts.method.clone()) - .uri(&target_url) - .body(request_body) - .change_context(Self::error("Failed to build Didomi proxy request"))?; + let mut proxy_req = outbound_request(parts.method.clone(), &target_url) + .map_err(|error| { + outbound_error( + error, + Self::error("Failed to build Didomi proxy request"), + OutboundRequestSource::Internal, + ) + })? + .body(request_body); self.copy_headers( &backend, services.client_info().client_ip, @@ -492,26 +490,39 @@ impl IntegrationProxy for DidomiIntegration { || (matches!(backend, DidomiBackend::Sdk) && parts.method == Method::GET && !consent_path.ends_with("/loader.js")); - let platform_request = PlatformHttpRequest::new(proxy_req, backend_name); - let platform_request = if shared_cacheable { - platform_request + let cache_policy = if shared_cacheable { + OutboundCachePolicy::PlatformDefault } else { - platform_request.with_cache_bypass() + OutboundCachePolicy::Bypass }; + let proxy_req = buffered( + proxy_req, + BUFFERED_TIMEOUT, + DEFAULT_BUFFERED_RESPONSE_BYTES, + ResponseDecoding::Original, + ) + .cache_policy(cache_policy); let mut response = services - .http_client() - .send(platform_request) + .outbound_client() + .send(proxy_req) .await - .change_context(Self::error("Didomi upstream request failed"))?; + .and_then(OutboundResponse::into_response) + .map_err(|error| { + outbound_error( + error, + Self::error("Didomi upstream request failed"), + OutboundRequestSource::Internal, + ) + })?; if matches!(backend, DidomiBackend::Sdk) { - Self::add_cors_headers(&mut response.response); + Self::add_cors_headers(&mut response); } if !shared_cacheable { - crate::response_privacy::enforce_terminal_private_cache_privacy(&mut response.response); + crate::response_privacy::enforce_terminal_private_cache_privacy(&mut response); } - Ok(response.response) + Ok(response) } } @@ -550,16 +561,19 @@ mod tests { use std::net::{IpAddr, Ipv4Addr}; use std::sync::Arc; + use edgezero_core::outbound::{OutboundCachePolicy, ResponseDecoding, ResponseMode}; + use http::{Method, StatusCode}; + use super::*; use crate::integrations::{IntegrationDocumentState, IntegrationRegistry}; - use crate::platform::PlatformCacheIntent; - use crate::platform::test_support::{ - NoopConfigStore, NoopSecretStore, StubBackend, StubHttpClient, - build_services_with_http_client, - }; + use crate::outbound::{BUFFERED_TIMEOUT, DEFAULT_BUFFERED_RESPONSE_BYTES}; + use crate::platform::test_support::build_services_with_outbound_client; use crate::platform::{ClientInfo, GeoInfo, PlatformError, PlatformGeo}; + use crate::test_support::body_read::body_read_failure_cases; + use crate::test_support::scripted_client::{ + ExpectedTransport, ScriptedHttpClient, ScriptedReply, TestClock, + }; use crate::test_support::tests::{crate_test_settings_str, create_test_settings}; - use http::Method; #[derive(Clone)] enum GeoResult { @@ -611,22 +625,170 @@ mod tests { } } - fn services_with_geo( - http_client: Arc, - geo_result: GeoResult, - ) -> RuntimeServices { - RuntimeServices::builder() - .config_store(Arc::new(NoopConfigStore)) - .secret_store(Arc::new(NoopSecretStore)) - .kv_store(Arc::new(edgezero_core::key_value_store::NoopKvStore)) - .backend(Arc::new(StubBackend)) - .http_client(http_client) - .geo(Arc::new(StubGeo(geo_result))) - .client_info(ClientInfo { + fn services_with_geo(client: &ScriptedHttpClient, geo_result: GeoResult) -> RuntimeServices { + RuntimeServices { + geo: Arc::new(StubGeo(geo_result)), + client_info: ClientInfo { client_ip: Some(IpAddr::V4(Ipv4Addr::new(203, 0, 113, 7))), ..ClientInfo::default() - }) - .build() + }, + ..build_services_with_outbound_client(client, &TestClock::new()) + } + } + + /// A Didomi upstream that answers one `method` request to `uri` with `reply`. + fn scripted_upstream(method: Method, uri: &str, reply: ScriptedReply) -> ScriptedHttpClient { + let client = ScriptedHttpClient::new(&TestClock::new()); + client.respond_to(method, uri, reply); + client + } + + /// A 200 reply with `body` and `headers`. + fn reply_with_headers(body: &'static str, headers: &[(&str, &str)]) -> ScriptedReply { + headers.iter().fold( + ScriptedReply::status(StatusCode::OK).body(body), + |reply, (name, value)| reply.header(name, value), + ) + } + + /// The URI of every request `client` received, in arrival order. + fn sent_uris(client: &ScriptedHttpClient) -> Vec { + client + .requests() + .iter() + .map(|request| request.uri.to_string()) + .collect() + } + + /// The cache policy of every request `client` received, in arrival order. + fn sent_cache_policies(client: &ScriptedHttpClient) -> Vec { + client + .requests() + .iter() + .map(|request| request.cache_policy) + .collect() + } + + /// The headers of the one request `client` received. + fn sent_headers(client: &ScriptedHttpClient) -> HeaderMap { + let requests = client.requests(); + assert_eq!(requests.len(), 1, "should send one Didomi request"); + requests[0].headers.clone() + } + + fn sent_header<'a>(headers: &'a HeaderMap, name: &str) -> Option<&'a str> { + headers.get(name).and_then(|value| value.to_str().ok()) + } + + #[test] + fn didomi_requests_readback() { + let sdk_target = "https://sdk.example.com/sdk/v1/core.js?v=1"; + let api_target = "https://api.example.com/api/events?x=1"; + let clock = TestClock::new(); + let client = ScriptedHttpClient::new(&clock); + client + .respond_to( + Method::GET, + sdk_target, + ScriptedReply::status(StatusCode::OK).body("sdk"), + ) + .respond_to( + Method::POST, + api_target, + ScriptedReply::status(StatusCode::OK).body("event"), + ); + let services = + services_with_geo(&client, GeoResult::Value(Some(geo_info("US", Some("CA"))))); + let settings = create_test_settings(); + let integration = DidomiIntegration::new(Arc::new(config(true))); + let browser_headers = [ + (header::ACCEPT, "*/*"), + (header::ACCEPT_LANGUAGE, "en"), + (header::ACCEPT_ENCODING, "gzip"), + (header::USER_AGENT, "example-agent"), + (header::REFERER, "https://publisher.example.com/page"), + (header::ORIGIN, "https://publisher.example.com"), + (header::AUTHORIZATION, "Basic dXNlcjpwYXNz"), + (header::COOKIE, "session_id=secret"), + ]; + let mut sdk_request = http::Request::builder() + .method(Method::GET) + .uri("https://publisher.example.com/integrations/didomi/consent/sdk/v1/core.js?v=1"); + let mut api_request = http::Request::builder() + .method(Method::POST) + .uri("https://publisher.example.com/integrations/didomi/consent/api/events?x=1") + .header(header::CONTENT_TYPE, "application/json") + .header(header::CONTENT_LENGTH, "2"); + for (name, value) in &browser_headers { + sdk_request = sdk_request.header(name, *value); + api_request = api_request.header(name, *value); + } + + for request in [ + sdk_request + .body(EdgeBody::empty()) + .expect("should build SDK request"), + api_request + .body(EdgeBody::from("{}")) + .expect("should build API request"), + ] { + let response = + futures::executor::block_on(integration.handle(&settings, &services, request)) + .expect("should proxy the Didomi request"); + assert_eq!(response.status(), StatusCode::OK, "should relay the status"); + } + + let requests = client.requests(); + assert_eq!(requests.len(), 2, "should send the SDK and API requests"); + let (sdk, api) = (&requests[0], &requests[1]); + assert_eq!( + (&sdk.method, sdk.uri.to_string()), + (&Method::GET, sdk_target.to_owned()), + "should fetch the SDK asset with its query" + ); + sdk.assert_headers(&[ + ("x-forwarded-for", "203.0.113.7"), + ("accept", "*/*"), + ("accept-language", "en"), + ("accept-encoding", "gzip"), + ("user-agent", "example-agent"), + ("referer", "https://publisher.example.com/page"), + ("origin", "https://publisher.example.com"), + ("x-geo-country", "US"), + ("x-geo-region", "CA"), + ("cloudfront-viewer-country", "US"), + ]); + sdk.assert_transport(ExpectedTransport { + timeout: BUFFERED_TIMEOUT, + response_mode: ResponseMode::Buffered { + max_bytes: DEFAULT_BUFFERED_RESPONSE_BYTES, + }, + response_decoding: ResponseDecoding::Original, + cache_policy: OutboundCachePolicy::PlatformDefault, + }); + assert_eq!( + (&api.method, api.uri.to_string(), api.body.as_ref()), + (&Method::POST, api_target.to_owned(), b"{}".as_slice()), + "should forward the API method, query, and body" + ); + api.assert_headers(&[ + ("x-forwarded-for", "203.0.113.7"), + ("accept", "*/*"), + ("accept-language", "en"), + ("accept-encoding", "gzip"), + ("content-type", "application/json"), + ("user-agent", "example-agent"), + ("referer", "https://publisher.example.com/page"), + ("origin", "https://publisher.example.com"), + ]); + api.assert_transport(ExpectedTransport { + timeout: BUFFERED_TIMEOUT, + response_mode: ResponseMode::Buffered { + max_bytes: DEFAULT_BUFFERED_RESPONSE_BYTES, + }, + response_decoding: ResponseDecoding::Original, + cache_policy: OutboundCachePolicy::Bypass, + }); } #[test] @@ -790,9 +952,9 @@ mod tests { #[test] fn enabled_geo_redirects_noncanonical_loader_without_upstream_call() { - let stub = Arc::new(StubHttpClient::new()); + let upstream = ScriptedHttpClient::forbidding(); let services = services_with_geo( - Arc::clone(&stub), + &upstream, GeoResult::Value(Some(geo_info("us", Some("ca")))), ); let settings = create_test_settings(); @@ -839,16 +1001,16 @@ mod tests { "should keep the redirect usable from cross-origin loader embeds" ); assert!( - stub.recorded_backend_names().is_empty(), + upstream.requests().is_empty(), "should not contact Didomi before canonical redirect" ); } #[test] fn enabled_geo_redirects_loader_under_custom_proxy_prefix() { - let stub = Arc::new(StubHttpClient::new()); + let upstream = ScriptedHttpClient::forbidding(); let services = services_with_geo( - Arc::clone(&stub), + &upstream, GeoResult::Value(Some(geo_info("us", Some("ca")))), ); let settings = create_test_settings(); @@ -876,17 +1038,20 @@ mod tests { "should preserve the custom proxy prefix in the canonical target" ); assert!( - stub.recorded_backend_names().is_empty(), + upstream.requests().is_empty(), "should not contact Didomi before the custom-prefix redirect" ); } #[test] fn enabled_geo_proxies_canonical_loader_with_authoritative_headers() { - let stub = Arc::new(StubHttpClient::new()); - stub.push_response(200, b"loader".to_vec()); + let upstream = scripted_upstream( + Method::GET, + "https://sdk.example.com/key/loader.js?target_type=notice&country=US®ion=CA", + ScriptedReply::status(StatusCode::OK).body("loader"), + ); let services = services_with_geo( - Arc::clone(&stub), + &upstream, GeoResult::Value(Some(geo_info("us", Some("us-ca")))), ); let settings = create_test_settings(); @@ -905,20 +1070,19 @@ mod tests { assert_eq!(response.status(), http::StatusCode::OK); assert_eq!( - stub.recorded_request_uris(), + sent_uris(&upstream), vec!["https://sdk.example.com/key/loader.js?target_type=notice&country=US®ion=CA"], "should send the canonical query to Didomi" ); - let headers = stub.recorded_request_headers(); + let headers = sent_headers(&upstream); for (name, expected) in [ ("x-geo-country", "US"), ("x-geo-region", "CA"), ("cloudfront-viewer-country", "US"), ] { - assert!( - headers[0] - .iter() - .any(|(actual_name, value)| actual_name == name && value == expected), + assert_eq!( + sent_header(&headers, name), + Some(expected), "should set {name} from authoritative geo" ); } @@ -926,10 +1090,13 @@ mod tests { #[test] fn disabled_geo_proxies_loader_without_geo_query() { - let stub = Arc::new(StubHttpClient::new()); - stub.push_response(200, b"loader".to_vec()); + let upstream = scripted_upstream( + Method::GET, + "https://sdk.example.com/key/loader.js?target_type=notice", + ScriptedReply::status(StatusCode::OK).body("loader"), + ); let services = services_with_geo( - Arc::clone(&stub), + &upstream, GeoResult::Value(Some(geo_info("US", Some("CA")))), ); let settings = create_test_settings(); @@ -946,7 +1113,7 @@ mod tests { assert_eq!(response.status(), http::StatusCode::OK); assert_eq!( - stub.recorded_request_uris(), + sent_uris(&upstream), vec!["https://sdk.example.com/key/loader.js?target_type=notice"], "should not add geo when the option is disabled" ); @@ -954,9 +1121,12 @@ mod tests { #[test] fn enabled_geo_leaves_unrelated_sdk_assets_unchanged() { - let stub = Arc::new(StubHttpClient::new()); - stub.push_response(200, b"sdk".to_vec()); - let services = services_with_geo(Arc::clone(&stub), GeoResult::Failure); + let upstream = scripted_upstream( + Method::GET, + "https://sdk.example.com/sdk/v1/core.js?v=1", + ScriptedReply::status(StatusCode::OK).body("sdk"), + ); + let services = services_with_geo(&upstream, GeoResult::Failure); let settings = create_test_settings(); let integration = DidomiIntegration::new(Arc::new(config_with_geo_query_parameters())); let request = http::Request::builder() @@ -971,7 +1141,7 @@ mod tests { assert_eq!(response.status(), http::StatusCode::OK); assert_eq!( - stub.recorded_request_uris(), + sent_uris(&upstream), vec!["https://sdk.example.com/sdk/v1/core.js?v=1"], "should not apply loader geo behavior to other SDK assets" ); @@ -989,10 +1159,13 @@ mod tests { fn noncanonical_sdk_uses_trusted_geo_instead_of_caller_headers() { for (enabled, method, path) in NONCANONICAL_SDK_CASES { // Arrange conflicting caller geography and a normalized platform pair. - let stub = Arc::new(StubHttpClient::new()); - stub.push_response(200, b"sdk".to_vec()); + let upstream = scripted_upstream( + method.clone(), + &format!("https://sdk.example.com{path}?v=1"), + ScriptedReply::status(StatusCode::OK).body("sdk"), + ); let services = services_with_geo( - Arc::clone(&stub), + &upstream, GeoResult::Value(Some(geo_info(" us ", Some(" us-ca ")))), ); let settings = create_test_settings(); @@ -1025,27 +1198,26 @@ mod tests { "should proxy without redirect" ); assert_eq!( - stub.recorded_request_uris(), + sent_uris(&upstream), vec![format!("https://sdk.example.com{path}?v=1")], "should preserve noncanonical SDK URL" ); - let headers = stub.recorded_request_headers(); + let headers = sent_headers(&upstream); for (name, expected) in [ ("x-geo-country", "US"), ("x-geo-region", "CA"), ("cloudfront-viewer-country", "US"), ] { - assert!( - headers[0] - .iter() - .any(|(actual, value)| actual == name && value == expected), + assert_eq!( + sent_header(&headers, name), + Some(expected), "should derive {name} only from trusted geo for {path}, geo option {enabled}" ); } assert!( - !headers[0] - .iter() - .any(|(name, _)| name.starts_with("fastlygeo-")), + !headers + .keys() + .any(|name| name.as_str().starts_with("fastlygeo-")), "should not forward raw caller geo headers" ); } @@ -1062,9 +1234,12 @@ mod tests { GeoResult::Value(Some(geo_info("XX", Some("CA")))), GeoResult::Value(Some(geo_info("US", Some("GB-LND")))), ] { - let stub = Arc::new(StubHttpClient::new()); - stub.push_response(200, b"sdk".to_vec()); - let services = services_with_geo(Arc::clone(&stub), geo_result); + let upstream = scripted_upstream( + method.clone(), + &format!("https://sdk.example.com{path}"), + ScriptedReply::status(StatusCode::OK).body("sdk"), + ); + let services = services_with_geo(&upstream, geo_result); let settings = create_test_settings(); let integration = DidomiIntegration::new(Arc::new(DidomiIntegrationConfig { geo_query_parameters: enabled, @@ -1092,7 +1267,7 @@ mod tests { http::StatusCode::OK, "should tolerate unavailable geo outside enabled GET loaders" ); - let headers = stub.recorded_request_headers(); + let headers = sent_headers(&upstream); for name in [ "x-geo-country", "x-geo-region", @@ -1101,7 +1276,7 @@ mod tests { "fastlygeo-region", ] { assert!( - !headers[0].iter().any(|(actual, _)| actual == name), + !headers.contains_key(name), "should omit {name} without a complete trusted pair for {path}, geo option {enabled}" ); } @@ -1126,23 +1301,25 @@ mod tests { GeoResult::Value(Some(geo_info("US", Some("CA")))), GeoResult::Value(None), ] { - let stub = Arc::new(StubHttpClient::new()); - stub.push_response_with_headers( - 200, - b"sdk".to_vec(), - vec![ - ("Cache-Control", "public, max-age=3600"), - ("Expires", "Wed, 21 Oct 2037 07:28:00 GMT"), - ("ETag", "\"sdk-response\""), - ("Last-Modified", "Wed, 21 Oct 2015 07:28:00 GMT"), - ("Age", "120"), - ("Surrogate-Control", "max-age=3600"), - ("CDN-Cache-Control", "max-age=3600"), - ("Cloudflare-CDN-Cache-Control", "max-age=3600"), - ("Fastly-Surrogate-Control", "max-age=3600"), - ], + let upstream = scripted_upstream( + method.clone(), + &format!("https://sdk.example.com{path}?country=GB®ion=LND"), + reply_with_headers( + "sdk", + &[ + ("Cache-Control", "public, max-age=3600"), + ("Expires", "Wed, 21 Oct 2037 07:28:00 GMT"), + ("ETag", "\"sdk-response\""), + ("Last-Modified", "Wed, 21 Oct 2015 07:28:00 GMT"), + ("Age", "120"), + ("Surrogate-Control", "max-age=3600"), + ("CDN-Cache-Control", "max-age=3600"), + ("Cloudflare-CDN-Cache-Control", "max-age=3600"), + ("Fastly-Surrogate-Control", "max-age=3600"), + ], + ), ); - let services = services_with_geo(Arc::clone(&stub), geo_result); + let services = services_with_geo(&upstream, geo_result); let settings = create_test_settings(); let integration = DidomiIntegration::new(Arc::new(DidomiIntegrationConfig { geo_query_parameters: enabled, @@ -1161,8 +1338,8 @@ mod tests { .expect("should proxy SDK request"); assert_eq!( - stub.recorded_cache_intents(), - vec![PlatformCacheIntent::Bypass], + sent_cache_policies(&upstream), + vec![OutboundCachePolicy::Bypass], "should bypass SDK cache without canonical geo identity for {path}, geo option {enabled}" ); assert_eq!( @@ -1203,12 +1380,10 @@ mod tests { "should remove SDK cache metadata {name}" ); } - let headers = stub.recorded_request_headers(); + let headers = sent_headers(&upstream); assert!( - !headers[0].iter().any(|(name, _)| matches!( - name.as_str(), - "if-none-match" | "if-modified-since" - )), + !headers.contains_key(header::IF_NONE_MATCH) + && !headers.contains_key(header::IF_MODIFIED_SINCE), "should not revalidate a previously cached geo response" ); } @@ -1223,8 +1398,7 @@ mod tests { GeoResult::Value(None), ] { for path in ["/sdk/v1/core.js", "/sdk/v1/ui.js", "/sdk/v1/iab-texts.js"] { - let stub = Arc::new(StubHttpClient::new()); - let cache_headers = vec![ + let cache_headers = [ ("Cache-Control", "public, max-age=31536000"), ("Expires", "Wed, 21 Oct 2037 07:28:00 GMT"), ("ETag", "\"sdk-chunk\""), @@ -1235,8 +1409,12 @@ mod tests { ("Cloudflare-CDN-Cache-Control", "max-age=31536000"), ("Fastly-Surrogate-Control", "max-age=31536000"), ]; - stub.push_response_with_headers(200, b"sdk".to_vec(), cache_headers.clone()); - let services = services_with_geo(Arc::clone(&stub), geo_result.clone()); + let upstream = scripted_upstream( + Method::GET, + &format!("https://sdk.example.com{path}?v=1"), + reply_with_headers("sdk", &cache_headers), + ); + let services = services_with_geo(&upstream, geo_result.clone()); let settings = create_test_settings(); let integration = DidomiIntegration::new(Arc::new(DidomiIntegrationConfig { geo_query_parameters: enabled, @@ -1256,8 +1434,8 @@ mod tests { .expect("should proxy SDK asset"); assert_eq!( - stub.recorded_cache_intents(), - vec![PlatformCacheIntent::Default], + sent_cache_policies(&upstream), + vec![OutboundCachePolicy::PlatformDefault], "should follow origin cache policy for {path}, geo option {enabled}" ); for (name, expected) in cache_headers { @@ -1293,10 +1471,13 @@ mod tests { #[test] fn api_requests_do_not_forward_geo_headers() { for enabled in [false, true] { - let stub = Arc::new(StubHttpClient::new()); - stub.push_response(200, b"event".to_vec()); + let upstream = scripted_upstream( + Method::GET, + "https://api.example.com/api/events", + ScriptedReply::status(StatusCode::OK).body("event"), + ); let services = services_with_geo( - Arc::clone(&stub), + &upstream, GeoResult::Value(Some(geo_info("US", Some("CA")))), ); let settings = create_test_settings(); @@ -1317,7 +1498,7 @@ mod tests { futures::executor::block_on(integration.handle(&settings, &services, request)) .expect("should proxy API request"); - let headers = stub.recorded_request_headers(); + let headers = sent_headers(&upstream); for name in [ "x-geo-country", "x-geo-region", @@ -1326,7 +1507,7 @@ mod tests { "fastlygeo-region", ] { assert!( - !headers[0].iter().any(|(actual, _)| actual == name), + !headers.contains_key(name), "should omit API geo header {name}" ); } @@ -1341,8 +1522,8 @@ mod tests { GeoResult::Value(Some(geo_info("XX", Some("CA")))), GeoResult::Failure, ] { - let stub = Arc::new(StubHttpClient::new()); - let services = services_with_geo(Arc::clone(&stub), geo_result); + let upstream = ScriptedHttpClient::forbidding(); + let services = services_with_geo(&upstream, geo_result); let settings = create_test_settings(); let integration = DidomiIntegration::new(Arc::new(config_with_geo_query_parameters())); let request = http::Request::builder() @@ -1377,7 +1558,7 @@ mod tests { "should keep geo failures usable from cross-origin loader embeds" ); assert!( - stub.recorded_backend_names().is_empty(), + upstream.requests().is_empty(), "should not contact Didomi on geo failure" ); } @@ -1385,21 +1566,23 @@ mod tests { #[test] fn api_requests_bypass_cache_and_strip_response_cache_metadata() { - let stub = Arc::new(StubHttpClient::new()); - stub.push_response_with_headers( - 200, - b"event".to_vec(), - vec![ - ("Cache-Control", "public, max-age=3600"), - ("Expires", "Wed, 21 Oct 2037 07:28:00 GMT"), - ("ETag", "\"api-response\""), - ("Last-Modified", "Wed, 21 Oct 2015 07:28:00 GMT"), - ("Age", "120"), - ("Surrogate-Control", "max-age=3600"), - ("CDN-Cache-Control", "max-age=3600"), - ], + let upstream = scripted_upstream( + Method::GET, + "https://api.example.com/api/events?x=1", + reply_with_headers( + "event", + &[ + ("Cache-Control", "public, max-age=3600"), + ("Expires", "Wed, 21 Oct 2037 07:28:00 GMT"), + ("ETag", "\"api-response\""), + ("Last-Modified", "Wed, 21 Oct 2015 07:28:00 GMT"), + ("Age", "120"), + ("Surrogate-Control", "max-age=3600"), + ("CDN-Cache-Control", "max-age=3600"), + ], + ), ); - let services = services_with_geo(Arc::clone(&stub), GeoResult::Value(None)); + let services = services_with_geo(&upstream, GeoResult::Value(None)); let settings = create_test_settings(); let integration = DidomiIntegration::new(Arc::new(config(true))); let request = http::Request::builder() @@ -1413,8 +1596,8 @@ mod tests { .expect("should proxy API request"); assert_eq!( - stub.recorded_cache_intents(), - vec![PlatformCacheIntent::Bypass], + sent_cache_policies(&upstream), + vec![OutboundCachePolicy::Bypass], "should bypass the platform cache for API requests" ); assert_eq!( @@ -1442,21 +1625,23 @@ mod tests { #[test] fn canonical_loader_responses_preserve_origin_cache_metadata() { - let stub = Arc::new(StubHttpClient::new()); - stub.push_response_with_headers( - 200, - b"sdk".to_vec(), - vec![ - ("Cache-Control", "public, max-age=3600"), - ("Expires", "Wed, 21 Oct 2037 07:28:00 GMT"), - ("ETag", "\"sdk-response\""), - ("Last-Modified", "Wed, 21 Oct 2015 07:28:00 GMT"), - ("Age", "120"), - ("Surrogate-Control", "max-age=3600"), - ], + let upstream = scripted_upstream( + Method::GET, + "https://sdk.example.com/key/loader.js?country=US®ion=CA", + reply_with_headers( + "sdk", + &[ + ("Cache-Control", "public, max-age=3600"), + ("Expires", "Wed, 21 Oct 2037 07:28:00 GMT"), + ("ETag", "\"sdk-response\""), + ("Last-Modified", "Wed, 21 Oct 2015 07:28:00 GMT"), + ("Age", "120"), + ("Surrogate-Control", "max-age=3600"), + ], + ), ); let services = services_with_geo( - Arc::clone(&stub), + &upstream, GeoResult::Value(Some(geo_info("US", Some("CA")))), ); let settings = create_test_settings(); @@ -1472,8 +1657,8 @@ mod tests { .expect("should proxy SDK request"); assert_eq!( - stub.recorded_cache_intents(), - vec![PlatformCacheIntent::Default], + sent_cache_policies(&upstream), + vec![OutboundCachePolicy::PlatformDefault], "should retain normal platform caching for canonical SDK loaders" ); for (name, expected) in [ @@ -1740,36 +1925,6 @@ mod tests { ); } - #[test] - fn didomi_proxy_uses_platform_http_client() { - let stub = Arc::new(StubHttpClient::new()); - stub.push_response(200, b"ok".to_vec()); - let services = build_services_with_http_client( - Arc::clone(&stub) as Arc - ); - let settings = create_test_settings(); - let integration = DidomiIntegration::new(Arc::new(config(true))); - let req = http::Request::builder() - .method(http::Method::GET) - .uri("https://publisher.example/integrations/didomi/consent/api/events") - .body(EdgeBody::empty()) - .expect("should build request"); - - let response = futures::executor::block_on(integration.handle(&settings, &services, req)) - .expect("should proxy request"); - - assert_eq!( - response.status(), - http::StatusCode::OK, - "should return stubbed response" - ); - assert_eq!( - stub.recorded_backend_names(), - vec!["stub-backend".to_string()], - "should route outbound request through PlatformHttpClient" - ); - } - #[test] fn head_injector_default_path() { let integration = DidomiIntegration::new(Arc::new(config(true))); @@ -1786,4 +1941,29 @@ mod tests { r#""# ); } + + #[test] + fn didomi_api_body_read_failures_map_to_statuses() { + let settings = create_test_settings(); + let integration = DidomiIntegration::new(Arc::new(config(true))); + for case in body_read_failure_cases(INTEGRATION_MAX_BODY_BYTES) { + let upstream = ScriptedHttpClient::forbidding(); + let services = services_with_geo(&upstream, GeoResult::Value(None)); + let (req, expectation) = case.into_request( + http::Request::builder() + .method(http::Method::POST) + .uri("https://publisher.example/integrations/didomi/consent/api/events"), + ); + + let outcome = + futures::executor::block_on(integration.handle(&settings, &services, req)); + + expectation.assert_outcome("Didomi API", &outcome); + assert!( + upstream.requests().is_empty(), + "should not call upstream for the {} case", + expectation.name() + ); + } + } } diff --git a/crates/trusted-server-core/src/integrations/google_tag_manager.rs b/crates/trusted-server-core/src/integrations/google_tag_manager.rs index 41850b2953..e2f327a493 100644 --- a/crates/trusted-server-core/src/integrations/google_tag_manager.rs +++ b/crates/trusted-server-core/src/integrations/google_tag_manager.rs @@ -15,20 +15,19 @@ use std::sync::{Arc, LazyLock, Mutex}; use async_trait::async_trait; -use edgezero_core::body::Body as EdgeBody; +use edgezero_core::body::{Body as EdgeBody, BodyPayload}; use error_stack::{Report, ResultExt}; -use futures::StreamExt as _; use http::{Method, Request, Response, StatusCode, header}; use regex::Regex; use serde::{Deserialize, Serialize}; use validator::{Validate, ValidationError}; -use crate::error::TrustedServerError; +use crate::error::{IntoHttpResponse as _, TrustedServerError}; +use crate::http_util::{into_body_payload, read_request_body}; use crate::integrations::{ AttributeRewriteAction, IntegrationAttributeContext, IntegrationAttributeRewriter, IntegrationEndpoint, IntegrationProxy, IntegrationRegistration, IntegrationScriptContext, - IntegrationScriptRewriter, ScriptRewriteAction, UPSTREAM_SDK_MAX_RESPONSE_BYTES, - collect_response_bounded, + IntegrationScriptRewriter, ScriptRewriteAction, }; use crate::platform::RuntimeServices; use crate::proxy::{FIRST_PARTY_PASSTHROUGH_STRIP_HEADERS, ProxyRequestConfig, proxy_request}; @@ -48,17 +47,6 @@ const GTM_SCRIPT_HOST: &str = "www.googletagmanager.com"; /// Alternate analytics host that appears in tag script bodies. const ANALYTICS_HOST: &str = "analytics.google.com"; -/// Error type for payload size validation -#[derive(Debug)] -enum PayloadSizeError { - TooLarge { - actual: usize, - max: usize, - }, - /// Transport error while reading a streaming body chunk. - StreamRead(String), -} - /// Regex pattern for validating GTM container IDs. /// Format: GTM-XXXXXX where X is alphanumeric. static GTM_CONTAINER_ID_PATTERN: LazyLock = LazyLock::new(|| { @@ -794,71 +782,24 @@ impl GoogleTagManagerIntegration { .change_context(Self::error("Failed to build GTM response")) } - /// Returns the status to answer with when a POST declares a body this - /// integration will not read, or `None` when the request may proceed. - /// - /// A missing `Content-Length` is not rejected here: the body is bounded - /// again while it is read, which keeps HTTP/2 and chunked intermediaries - /// working. - fn content_length_rejection(&self, req: &Request) -> Option { - if req.method() != Method::POST { - return None; - } - let declared = req - .headers() - .get(header::CONTENT_LENGTH) - .and_then(|value| value.to_str().ok())?; - match declared.parse::() { - Ok(content_length) if content_length > self.config.max_beacon_body_size => { - log::warn!( - "Rejecting POST beacon with Content-Length {} exceeding max {}", - content_length, - self.config.max_beacon_body_size - ); - Some(StatusCode::PAYLOAD_TOO_LARGE) - } - Ok(_) => None, - Err(_) => { - log::warn!("POST request with malformed Content-Length header"); - Some(StatusCode::BAD_REQUEST) - } - } - } - - /// Maps a body-size failure onto the status the client should see. - fn payload_size_status(error: &PayloadSizeError) -> StatusCode { - match error { - PayloadSizeError::TooLarge { actual, max } => { - log::warn!( - "Returning 413: actual body size {actual} exceeds max {max} (Content-Length mismatch)" - ); - StatusCode::PAYLOAD_TOO_LARGE - } - PayloadSizeError::StreamRead(error) => { - log::error!("Returning 502: failed to read GTM request body stream: {error}"); - StatusCode::BAD_GATEWAY - } - } - } - /// Rewrites a proxied script body so its internal URLs stay first-party. /// /// # Errors /// - /// Returns [`TrustedServerError`] when the body cannot be read within the - /// response cap or the response cannot be built. - async fn rewritten_script_response( + /// Returns [`TrustedServerError`] when the body was not buffered or the + /// response cannot be built. + fn rewritten_script_response( &self, response: Response, ) -> Result, Report> { log::debug!("Rewriting GTM/gtag script content"); let status = response.status(); - let body_bytes = collect_response_bounded( - response.into_body(), - UPSTREAM_SDK_MAX_RESPONSE_BYTES, - GTM_INTEGRATION_ID, - ) - .await?; + // `proxy_request` buffers the script under its response cap. + let BodyPayload::Once(body_bytes) = into_body_payload(response.into_body()) else { + return Err(Report::new(Self::error( + "GTM script response was not buffered", + ))); + }; let body_str = String::from_utf8_lossy(&body_bytes); let rewritten_body = Self::rewrite_gtm_urls(&body_str); @@ -903,14 +844,14 @@ impl GoogleTagManagerIntegration { /// /// # Errors /// - /// Returns [`PayloadSizeError`] when the request body exceeds + /// Returns the [`read_request_body`] error when a POST body exceeds /// `max_beacon_body_size` or cannot be read. async fn build_proxy_config<'a>( &'a self, path: &str, req: &mut Request, target_url: &'a str, - ) -> Result, PayloadSizeError> { + ) -> Result, Report> { // The upstream body is re-served from the publisher's origin, so the // fetch itself must not be downgradable, and a redirect must not be // able to carry it to a host the operator never configured. @@ -925,8 +866,8 @@ impl GoogleTagManagerIntegration { // Read body with size cap to prevent unbounded memory allocation. let body = std::mem::replace(req.body_mut(), EdgeBody::empty()); let body_bytes = - Self::collect_request_body_bounded(body, self.config.max_beacon_body_size).await?; - proxy_config.body = Some(body_bytes); + read_request_body(req.headers(), body, self.config.max_beacon_body_size).await?; + proxy_config.body = Some(body_bytes.to_vec()); } // Explicitly strip X-Forwarded-For to prevent client IP leakage to Google. @@ -945,53 +886,6 @@ impl GoogleTagManagerIntegration { Ok(proxy_config) } - - async fn collect_request_body_bounded( - body: EdgeBody, - max_size: usize, - ) -> Result, PayloadSizeError> { - match body { - EdgeBody::Once(bytes) => { - if bytes.len() > max_size { - log::warn!( - "POST body size {} exceeds max {} (rejected before proxy)", - bytes.len(), - max_size - ); - return Err(PayloadSizeError::TooLarge { - actual: bytes.len(), - max: max_size, - }); - } - Ok(bytes.to_vec()) - } - EdgeBody::Stream(mut stream) => { - let mut body_bytes = Vec::new(); - while let Some(chunk_result) = stream.next().await { - let chunk = chunk_result.map_err(|error| { - log::error!("Error reading request body stream: {}", error); - PayloadSizeError::StreamRead(error.to_string()) - })?; - - if body_bytes.len() + chunk.len() > max_size { - let total_size = body_bytes.len() + chunk.len(); - log::warn!( - "POST body size {} exceeds max {} (rejected during stream read)", - total_size, - max_size - ); - return Err(PayloadSizeError::TooLarge { - actual: total_size, - max: max_size, - }); - } - - body_bytes.extend_from_slice(&chunk); - } - Ok(body_bytes) - } - } - } } fn build( @@ -1059,10 +953,6 @@ impl IntegrationProxy for GoogleTagManagerIntegration { let path = req.uri().path().to_string(); log::debug!("Handling GTM request: {} {}", req.method(), path); - if let Some(status) = self.content_length_rejection(&req) { - return Self::empty_response(status); - } - let Some(target) = self.build_target_url(&req, &path) else { return Self::empty_response(StatusCode::NOT_FOUND); }; @@ -1076,7 +966,7 @@ impl IntegrationProxy for GoogleTagManagerIntegration { let proxy_config = match self.build_proxy_config(&path, &mut req, &target_url).await { Ok(config) => config, - Err(error) => return Self::empty_response(Self::payload_size_status(&error)), + Err(report) => return Self::empty_response(report.current_context().status_code()), }; let mut response = proxy_request(settings, req, proxy_config, services) @@ -1095,7 +985,7 @@ impl IntegrationProxy for GoogleTagManagerIntegration { log::warn!("GTM upstream returned status {}", response.status()); return Ok(response); } - return self.rewritten_script_response(response).await; + return self.rewritten_script_response(response); } Ok(response) @@ -1213,10 +1103,18 @@ mod tests { IntegrationDocumentState, IntegrationRegistry, IntegrationScriptContext, IntegrationScriptRewriter, ScriptRewriteAction, }; - use crate::platform::test_support::{StubHttpClient, build_services_with_http_client}; + use crate::outbound::{ + BUFFERED_TIMEOUT, DEFAULT_BUFFERED_RESPONSE_BYTES, REDIRECT_CHAIN_DEADLINE, + }; + use crate::platform::test_support::build_services_with_outbound_client; use crate::settings::Settings; use crate::streaming_processor::StreamProcessor as _; use crate::streaming_processor::{Compression, PipelineConfig, StreamingPipeline}; + use crate::test_support::body_read::body_read_failure_cases; + use crate::test_support::scripted_client::{ + ExpectedTransport, ScriptedHttpClient, ScriptedReply, TestClock, + }; + use edgezero_core::outbound::{OutboundCachePolicy, ResponseDecoding, ResponseMode}; #[test] fn container_id_validation_matches_gtm_pattern() { @@ -1837,31 +1735,112 @@ mod tests { }); } - /// Drives `handle` against a stubbed upstream that returns `headers`. + /// A scripted upstream that answers the URL `integration` proxies `req` to with `reply`. + /// + /// A request that resolves to no proxied target scripts nothing, so any request it sends + /// fails. + fn scripted_gtm_upstream( + integration: &GoogleTagManagerIntegration, + req: &http::Request, + reply: ScriptedReply, + ) -> (TestClock, ScriptedHttpClient, RuntimeServices) { + let clock = TestClock::new(); + let client = ScriptedHttpClient::new(&clock); + if let Some(GtmTarget::Proxy(target_url)) = + integration.build_target_url(req, req.uri().path()) + { + client.respond_to(req.method().clone(), &target_url, reply); + } + let services = build_services_with_outbound_client(&client, &clock); + (clock, client, services) + } + + /// Drives `handle` against a scripted upstream that returns `headers`. fn handle_with_upstream_headers( path_and_query: &str, status: u16, headers: Vec<(&'static str, &'static str)>, ) -> Response { - futures::executor::block_on(async { - let stub = Arc::new(StubHttpClient::new()); - stub.push_response_with_headers(status, b"upstream body".to_vec(), headers); - let services = build_services_with_http_client( - Arc::clone(&stub) as Arc - ); - let settings = create_test_settings(); - let integration = GoogleTagManagerIntegration::new(tag_config("GTM-CONFIGURED", &[])); - let req = build_http_request( - Method::GET, - &format!("https://edge.example.com{path_and_query}"), - EdgeBody::empty(), - ); + let settings = create_test_settings(); + let integration = GoogleTagManagerIntegration::new(tag_config("GTM-CONFIGURED", &[])); + let req = build_http_request( + Method::GET, + &format!("https://edge.example.com{path_and_query}"), + EdgeBody::empty(), + ); + let reply = headers.into_iter().fold( + ScriptedReply::status(StatusCode::from_u16(status).expect("should be a valid status")) + .body("upstream body"), + |reply, (name, value)| reply.header(name, value), + ); + let (_clock, _client, services) = scripted_gtm_upstream(&integration, &req, reply); + + futures::executor::block_on(integration.handle(&settings, &services, req)) + .expect("should handle the GTM request") + } + #[test] + fn gtm_script_request_readback() { + let settings = create_test_settings(); + let integration = GoogleTagManagerIntegration::new(tag_config("GTM-CONFIGURED", &[])); + let req = http::Request::builder() + .method(Method::GET) + .uri("https://edge.example.com/integrations/google_tag_manager/gtm.js") + .header(header::USER_AGENT, "fictional-agent/1.0") + .header(header::ACCEPT, "*/*") + .header(header::ACCEPT_LANGUAGE, "en-US") + .header(header::REFERER, "https://publisher.example.com/article") + .header(crate::constants::HEADER_X_FORWARDED_FOR, "198.51.100.10") + .header(header::ACCEPT_ENCODING, "gzip, br") + .header(header::COOKIE, "_ga=fictional") + .body(EdgeBody::empty()) + .expect("should build the GTM script request"); + let target_url = into_proxy_url( integration - .handle(&settings, &services, req) - .await - .expect("should handle the GTM request") - }) + .build_target_url(&req, req.uri().path()) + .expect("should resolve a gtm.js target"), + ); + let (clock, client, services) = scripted_gtm_upstream( + &integration, + &req, + ScriptedReply::status(StatusCode::OK) + .header("content-type", "application/javascript") + .body("var gtm = true;"), + ); + + let response = futures::executor::block_on(integration.handle(&settings, &services, req)) + .expect("should serve the GTM script"); + + assert_eq!(response.status(), StatusCode::OK, "should serve the script"); + let requests = client.requests(); + assert_eq!(requests.len(), 1, "should send one script request"); + let sent = &requests[0]; + assert_eq!( + (&sent.method, sent.uri.to_string()), + (&Method::GET, target_url), + "should fetch the configured container script" + ); + sent.assert_headers(&[ + ("user-agent", "fictional-agent/1.0"), + ("accept", "*/*"), + ("accept-language", "en-US"), + ("referer", "https://publisher.example.com/article"), + ("x-forwarded-for", ""), + ("accept-encoding", "identity"), + ]); + sent.assert_followed_transport( + ExpectedTransport { + timeout: BUFFERED_TIMEOUT, + response_mode: ResponseMode::Buffered { + max_bytes: DEFAULT_BUFFERED_RESPONSE_BYTES, + }, + response_decoding: ResponseDecoding::Original, + cache_policy: OutboundCachePolicy::PlatformDefault, + }, + clock + .monotonic_clock() + .deadline_after(REDIRECT_CHAIN_DEADLINE), + ); } #[test] @@ -1957,63 +1936,95 @@ mod tests { ); } + /// Builds the proxy config for `req` on its own path, as `handle` does. + async fn proxy_config_result( + integration: &GoogleTagManagerIntegration, + mut req: http::Request, + ) -> Result>, Report> { + let path = req.uri().path().to_string(); + let target_url = into_proxy_url( + integration + .build_target_url(&req, &path) + .expect("should resolve the target URL"), + ); + integration + .build_proxy_config(&path, &mut req, &target_url) + .await + .map(|config| config.body) + } + #[test] - fn content_length_rejection_matches_the_declared_body_size() { - let integration = GoogleTagManagerIntegration::new(tag_config("GTM-CONFIGURED", &[])); - let max = default_max_beacon_body_size(); + fn post_body_is_bounded_by_a_valid_declared_length_and_by_the_read() { + futures::executor::block_on(async { + let integration = GoogleTagManagerIntegration::new(tag_config("GTM-CONFIGURED", &[])); + let max = default_max_beacon_body_size(); + let payload = b"v=2&tid=G-TEST".to_vec(); + + // A malformed declaration is not trusted in either direction, so + // the bounded read decides, as it does with no declaration at all. + for (declared, expected) in [ + (None, None), + (Some(max.to_string()), None), + ( + Some((max + 1).to_string()), + Some(StatusCode::PAYLOAD_TOO_LARGE), + ), + (Some("not-a-number".to_string()), None), + ] { + let mut req = build_http_request( + Method::POST, + "https://edge.example.com/integrations/google_tag_manager/collect", + EdgeBody::from(payload.clone()), + ); + if let Some(declared) = &declared { + req.headers_mut().insert( + header::CONTENT_LENGTH, + http::HeaderValue::from_str(declared).expect("should build header"), + ); + } - let within = build_http_request( - Method::POST, - "https://edge.example.com/integrations/google_tag_manager/collect", - EdgeBody::empty(), - ); - assert_eq!( - integration.content_length_rejection(&within), - None, - "a POST without Content-Length proceeds and is bounded while read" - ); + match (proxy_config_result(&integration, req).await, expected) { + (Ok(body), None) => assert_eq!( + body.as_deref(), + Some(payload.as_slice()), + "Content-Length {declared:?} should forward the payload" + ), + (Err(report), Some(status)) => assert_eq!( + report.current_context().status_code(), + status, + "Content-Length {declared:?} should map to {status}" + ), + (Ok(_), Some(status)) => { + panic!("Content-Length {declared:?} should answer {status}") + } + (Err(report), None) => { + panic!("Content-Length {declared:?} should proceed: {report:?}") + } + } + } + }); + } - for (declared, expected) in [ - (max.to_string(), None), - ((max + 1).to_string(), Some(StatusCode::PAYLOAD_TOO_LARGE)), - ("not-a-number".to_string(), Some(StatusCode::BAD_REQUEST)), - ] { + #[test] + fn get_requests_are_not_subject_to_the_body_size_check() { + futures::executor::block_on(async { + let integration = GoogleTagManagerIntegration::new(tag_config("GTM-CONFIGURED", &[])); let mut req = build_http_request( - Method::POST, - "https://edge.example.com/integrations/google_tag_manager/collect", + Method::GET, + "https://edge.example.com/integrations/google_tag_manager/gtm.js", EdgeBody::empty(), ); req.headers_mut().insert( header::CONTENT_LENGTH, - http::HeaderValue::from_str(&declared).expect("should build header"), + http::HeaderValue::from_static("999999999"), ); - assert_eq!( - integration.content_length_rejection(&req), - expected, - "Content-Length `{declared}` should map to {expected:?}" - ); - } - } - - #[test] - fn get_requests_are_not_subject_to_the_body_size_check() { - let integration = GoogleTagManagerIntegration::new(tag_config("GTM-CONFIGURED", &[])); - let mut req = build_http_request( - Method::GET, - "https://edge.example.com/integrations/google_tag_manager/gtm.js", - EdgeBody::empty(), - ); - req.headers_mut().insert( - header::CONTENT_LENGTH, - http::HeaderValue::from_static("999999999"), - ); + let body = proxy_config_result(&integration, req) + .await + .expect("the beacon body cap applies to POST only"); - assert_eq!( - integration.content_length_rejection(&req), - None, - "the beacon body cap applies to POST only" - ); + assert_eq!(body, None, "should not read or forward a GET body"); + }); } #[test] @@ -2173,32 +2184,27 @@ mod tests { ); } - /// Drives `handle` with a stubbed upstream body and the given config. + /// Drives `handle` with a scripted upstream body and the given config. fn handle_with_upstream_body( config: GoogleTagManagerConfig, path_and_query: &str, status: u16, body: &[u8], ) -> Response { - futures::executor::block_on(async { - let stub = Arc::new(StubHttpClient::new()); - stub.push_response(status, body.to_vec()); - let services = build_services_with_http_client( - Arc::clone(&stub) as Arc - ); - let settings = create_test_settings(); - let integration = GoogleTagManagerIntegration::new(config); - let req = build_http_request( - Method::GET, - &format!("https://edge.example.com{path_and_query}"), - EdgeBody::empty(), - ); + let settings = create_test_settings(); + let integration = GoogleTagManagerIntegration::new(config); + let req = build_http_request( + Method::GET, + &format!("https://edge.example.com{path_and_query}"), + EdgeBody::empty(), + ); + let reply = + ScriptedReply::status(StatusCode::from_u16(status).expect("should be a valid status")) + .body(body.to_vec()); + let (_clock, _client, services) = scripted_gtm_upstream(&integration, &req, reply); - integration - .handle(&settings, &services, req) - .await - .expect("should handle the GTM request") - }) + futures::executor::block_on(integration.handle(&settings, &services, req)) + .expect("should handle the GTM request") } #[test] @@ -2254,15 +2260,10 @@ mod tests { "should carry the configured cache lifetime" ); - let body = futures::executor::block_on(async { - collect_response_bounded( - response.into_body(), - UPSTREAM_SDK_MAX_RESPONSE_BYTES, - GTM_INTEGRATION_ID, - ) - .await - .expect("should read the rewritten body") - }); + let body = response + .into_body() + .into_bytes() + .expect("should buffer the rewritten body"); assert_eq!( String::from_utf8_lossy(&body), r#"var beacon = "/integrations/google_tag_manager/g/collect";"#, @@ -2798,7 +2799,7 @@ j=d.createElement(s),dl=l!='dataLayer'?'&l='+l:'';j.async=true;j.src= let mut req = build_http_request( Method::POST, "https://edge.example.com/integrations/google_tag_manager/collect", - EdgeBody::from(oversized_payload.clone()), + EdgeBody::from(oversized_payload), ); let path = req.uri().path().to_string(); @@ -2813,14 +2814,21 @@ j=d.createElement(s),dl=l!='dataLayer'?'&l='+l:'';j.async=true;j.src= .build_proxy_config(&path, &mut req, &target_url) .await; - assert!(result.is_err(), "Oversized POST body should be rejected"); - - if let Err(PayloadSizeError::TooLarge { actual, max }) = result { - assert_eq!(actual, max_size + 1, "Should report actual size"); - assert_eq!(max, max_size, "Should report max size"); - } else { - panic!("Expected PayloadSizeError::TooLarge"); - } + let Err(report) = result else { + panic!("should reject the oversized POST body"); + }; + assert!( + matches!( + report.current_context(), + TrustedServerError::RequestTooLarge { .. } + ), + "should report the overflow as RequestTooLarge" + ); + assert_eq!( + report.current_context().status_code(), + StatusCode::PAYLOAD_TOO_LARGE, + "should answer 413 for the oversized POST body" + ); }); } @@ -2904,7 +2912,7 @@ j=d.createElement(s),dl=l!='dataLayer'?'&l='+l:'';j.async=true;j.src= let mut req = build_http_request( Method::POST, "https://edge.example.com/integrations/google_tag_manager/collect", - EdgeBody::from(oversized_payload.clone()), + EdgeBody::from(oversized_payload), ); // Set Content-Length to a small value (incorrect) req.headers_mut().insert( @@ -2920,23 +2928,26 @@ j=d.createElement(s),dl=l!='dataLayer'?'&l='+l:'';j.async=true;j.src= .expect("should resolve collect target URL"), ); - // build_proxy_config should detect the mismatch and return PayloadSizeError + // build_proxy_config should detect the mismatch and return RequestTooLarge let result = integration .build_proxy_config(&path, &mut req, &target_url) .await; + let Err(report) = result else { + panic!("should reject the oversized POST body"); + }; assert!( - result.is_err(), - "Should reject when actual body exceeds max despite low Content-Length" + matches!( + report.current_context(), + TrustedServerError::RequestTooLarge { .. } + ), + "should report the overflow as RequestTooLarge" + ); + assert_eq!( + report.current_context().status_code(), + StatusCode::PAYLOAD_TOO_LARGE, + "should answer 413 for the oversized POST body" ); - - // Verify it's a PayloadSizeError::TooLarge - if let Err(PayloadSizeError::TooLarge { actual, max }) = result { - assert_eq!(actual, oversized_payload.len(), "Should report actual size"); - assert_eq!(max, max_size, "Should report max size"); - } else { - panic!("Expected PayloadSizeError::TooLarge"); - } }); } @@ -2984,35 +2995,45 @@ j=d.createElement(s),dl=l!='dataLayer'?'&l='+l:'';j.async=true;j.src= } #[test] - fn test_handle_returns_400_for_invalid_content_length() { + fn test_handle_forwards_post_with_malformed_content_length() { futures::executor::block_on(async { - // Verify that handle() returns 400 Bad Request for malformed Content-Length - let config = tag_config("GTM-TEST1234", &[]); - let integration = GoogleTagManagerIntegration::new(config); - - // Create POST request with invalid Content-Length header + // A malformed Content-Length no longer answers 400 before the read: + // the bounded read governs, and a body under the cap is forwarded. + let integration = GoogleTagManagerIntegration::new(tag_config("GTM-TEST1234", &[])); let payload = b"v=2&tid=G-TEST&cid=123".to_vec(); - let mut req = http::Request::builder() - .method(Method::POST) - .uri("https://edge.example.com/integrations/google_tag_manager/collect") - .body(EdgeBody::from(payload)) - .expect("should build malformed request"); + let mut req = build_http_request( + Method::POST, + "https://edge.example.com/integrations/google_tag_manager/collect", + EdgeBody::from(payload.clone()), + ); req.headers_mut().insert( http::header::CONTENT_LENGTH, http::HeaderValue::from_static("not-a-number"), ); + let (_clock, upstream, services) = scripted_gtm_upstream( + &integration, + &req, + ScriptedReply::status(StatusCode::NO_CONTENT), + ); - let settings = make_settings(); let response = integration - .handle(&settings, &noop_services(), req) + .handle(&make_settings(), &services, req) .await - .expect("handle should not return error"); + .expect("should proxy the beacon"); - // Verify we get 400 Bad Request for malformed Content-Length assert_eq!( response.status(), - StatusCode::BAD_REQUEST, - "Should return 400 for malformed Content-Length" + StatusCode::NO_CONTENT, + "should return the upstream answer" + ); + assert_eq!( + upstream + .requests() + .iter() + .map(|request| request.body.to_vec()) + .collect::>(), + vec![payload], + "should forward the beacon body read under the cap" ); }); } @@ -4342,4 +4363,25 @@ container_id = "GTM-DEFAULT" ); } } + + #[test] + fn gtm_beacon_body_read_failures_map_to_statuses() { + let settings = make_settings(); + let integration = GoogleTagManagerIntegration::new(tag_config("GTM-TEST1234", &[])); + for case in body_read_failure_cases(default_max_beacon_body_size()) { + let upstream = ScriptedHttpClient::forbidding(); + let services = build_services_with_outbound_client(&upstream, &TestClock::new()); + let (req, expectation) = case.into_request( + http::Request::builder() + .method(Method::POST) + .uri("https://edge.example.com/integrations/google_tag_manager/g/collect"), + ); + + let outcome = + futures::executor::block_on(integration.handle(&settings, &services, req)); + + expectation.assert_outcome("GTM beacon", &outcome); + upstream.assert_no_requests(); + } + } } diff --git a/crates/trusted-server-core/src/integrations/gpt.rs b/crates/trusted-server-core/src/integrations/gpt.rs index 919c44ae96..5df321e36e 100644 --- a/crates/trusted-server-core/src/integrations/gpt.rs +++ b/crates/trusted-server-core/src/integrations/gpt.rs @@ -563,7 +563,13 @@ mod tests { use super::*; use crate::constants::HEADER_X_FORWARDED_FOR; use crate::integrations::IntegrationDocumentState; + use crate::outbound::{BUFFERED_TIMEOUT, DEFAULT_BUFFERED_RESPONSE_BYTES}; + use crate::platform::test_support::build_services_with_outbound_client; + use crate::test_support::scripted_client::{ + ExpectedTransport, ScriptedHttpClient, ScriptedReply, TestClock, + }; use crate::test_support::tests::create_test_settings; + use edgezero_core::outbound::{OutboundCachePolicy, ResponseDecoding, ResponseMode}; use http::Method; fn test_config() -> GptConfig { @@ -756,6 +762,73 @@ mod tests { // -- GPT proxy configuration -- + #[test] + fn gpt_script_request_readback() { + let clock = TestClock::new(); + let upstream = ScriptedHttpClient::new(&clock); + upstream.respond_to( + Method::GET, + "https://ads.example.com/gpt.js", + ScriptedReply::status(http::StatusCode::OK) + .header("content-type", "application/javascript") + .header("content-encoding", "gzip") + .body("gzip-bytes"), + ); + let services = build_services_with_outbound_client(&upstream, &clock); + let integration = GptIntegration::new(GptConfig { + script_url: "https://ads.example.com/gpt.js".to_owned(), + ..test_config() + }); + let req = http::Request::builder() + .method(Method::GET) + .uri("https://edge.example.com/integrations/gpt/script") + .header(HEADER_ACCEPT, "application/javascript") + .header(HEADER_ACCEPT_LANGUAGE, "en-US") + .header(HEADER_ACCEPT_ENCODING, "gzip") + .header(header::COOKIE, "session=fictional") + .header(HEADER_X_FORWARDED_FOR, "198.51.100.10") + .body(EdgeBody::empty()) + .expect("should build the GPT script request"); + + let response = futures::executor::block_on(integration.handle( + &create_test_settings(), + &services, + req, + )) + .expect("should serve the GPT script"); + + assert_eq!( + response + .headers() + .get(header::CONTENT_ENCODING) + .and_then(|value| value.to_str().ok()), + Some("gzip"), + "should relay the script in the coding the upstream sent" + ); + let requests = upstream.requests(); + assert_eq!(requests.len(), 1, "should send one script request"); + let sent = &requests[0]; + assert_eq!( + (&sent.method, sent.uri.to_string()), + (&Method::GET, "https://ads.example.com/gpt.js".to_owned()), + "should fetch the configured script URL" + ); + sent.assert_headers(&[ + ("accept", "application/javascript"), + ("accept-language", "en-US"), + ("accept-encoding", "gzip"), + ("user-agent", "TrustedServer/1.0"), + ]); + sent.assert_transport(ExpectedTransport { + timeout: BUFFERED_TIMEOUT, + response_mode: ResponseMode::Buffered { + max_bytes: DEFAULT_BUFFERED_RESPONSE_BYTES, + }, + response_decoding: ResponseDecoding::Original, + cache_policy: OutboundCachePolicy::PlatformDefault, + }); + } + #[test] fn build_proxy_config_uses_streaming_without_ec_forwarding_or_redirects() { let req = build_http_request( diff --git a/crates/trusted-server-core/src/integrations/gpt_diagnostics.rs b/crates/trusted-server-core/src/integrations/gpt_diagnostics.rs index a5f708b7ec..447e86dc90 100644 --- a/crates/trusted-server-core/src/integrations/gpt_diagnostics.rs +++ b/crates/trusted-server-core/src/integrations/gpt_diagnostics.rs @@ -411,6 +411,17 @@ fn console_cookie_state(request: &Request) -> ConsoleCookieState { } fn sanitize_console_cookie(request: &mut Request) { + // The rebuilt field keeps only UTF-8 values, so a non-UTF-8 field never reaches + // `EdgeZero`, which would drop it at dispatch with this warning. The value is + // never logged. + if request + .headers() + .get_all(header::COOKIE) + .iter() + .any(|value| value.to_str().is_err()) + { + log::warn!("dropping non-UTF-8 request header: cookie"); + } let retained = request .headers() .get_all(header::COOKIE) diff --git a/crates/trusted-server-core/src/integrations/js_asset_proxy.rs b/crates/trusted-server-core/src/integrations/js_asset_proxy.rs index 98571543c6..4e235f819d 100644 --- a/crates/trusted-server-core/src/integrations/js_asset_proxy.rs +++ b/crates/trusted-server-core/src/integrations/js_asset_proxy.rs @@ -576,9 +576,14 @@ mod tests { use crate::integrations::{ AttributeRewriteAction, IntegrationAttributeRewriter, IntegrationRegistry, }; - use crate::platform::test_support::{StubHttpClient, build_services_with_http_client}; + use crate::outbound::STREAMED_TIMEOUT; + use crate::platform::test_support::build_services_with_outbound_client; use crate::streaming_processor::{Compression, PipelineConfig, StreamingPipeline}; + use crate::test_support::scripted_client::{ + ExpectedTransport, ScriptedHttpClient, ScriptedReply, TestClock, + }; use crate::test_support::tests::create_test_settings; + use edgezero_core::outbound::{OutboundCachePolicy, ResponseDecoding, ResponseMode}; use http::header; use serde_json::json; @@ -1356,180 +1361,185 @@ mod tests { ); } + const VENDOR_ORIGIN: &str = "https://cdn.example.com/vendor.js"; + + fn vendor_integration() -> Arc { + JsAssetProxyIntegration::new(config_with_assets(vec![asset( + "/assets/vendor.js", + VENDOR_ORIGIN, + JsAssetProxyMode::Enabled, + )])) + } + + /// Proxies `/assets/vendor.js` against an upstream that answers with `reply`. + fn handle_vendor_request( + request: Request, + reply: Option, + ) -> (ScriptedHttpClient, Response) { + let clock = TestClock::new(); + let upstream = ScriptedHttpClient::new(&clock); + if let Some(reply) = reply { + upstream.respond_to(Method::GET, VENDOR_ORIGIN, reply); + } + let services = build_services_with_outbound_client(&upstream, &clock); + + let response = futures::executor::block_on(vendor_integration().handle( + &create_test_settings(), + &services, + request, + )) + .expect("should answer the JS asset request"); + + (upstream, response) + } + + fn vendor_request() -> Request { + build_http_request( + Method::GET, + "https://publisher.example.com/assets/vendor.js", + ) + } + #[test] - fn handle_forwards_conditional_headers_and_preserves_not_modified() { - futures::executor::block_on(async { - let settings = create_test_settings(); - let integration = JsAssetProxyIntegration::new(config_with_assets(vec![asset( - "/assets/vendor.js", - "https://cdn.example.com/vendor.js", - JsAssetProxyMode::Enabled, - )])); - let stub = Arc::new(StubHttpClient::new()); - stub.push_response_with_headers( - StatusCode::NOT_MODIFIED.as_u16(), - b"must not reach the client".to_vec(), - vec![(header::ETAG.as_str(), "\"current\"")], - ); - let services = build_services_with_http_client( - Arc::clone(&stub) as Arc - ); - let mut request = build_http_request( - Method::GET, - "https://publisher.example.com/assets/vendor.js", - ); - request.headers_mut().insert( - header::IF_NONE_MATCH, - http::HeaderValue::from_static("\"current\""), - ); - request.headers_mut().insert( - header::IF_MODIFIED_SINCE, - http::HeaderValue::from_static("Wed, 21 Oct 2015 07:28:00 GMT"), - ); + fn js_asset_proxy_request_readback() { + let request = Request::builder() + .method(Method::GET) + .uri("https://publisher.example.com/assets/vendor.js") + .header(header::ACCEPT, "*/*") + .header(header::ACCEPT_LANGUAGE, "en-US") + .header(header::ACCEPT_ENCODING, "gzip, br") + .header(header::IF_NONE_MATCH, "\"current\"") + .header(header::IF_MODIFIED_SINCE, "Wed, 21 Oct 2015 07:28:00 GMT") + .header(header::COOKIE, "session=fictional") + .header(HEADER_REFERER, "https://publisher.example.com/article") + .header(HEADER_X_FORWARDED_FOR, "198.51.100.10") + .body(EdgeBody::empty()) + .expect("should build the JS asset request"); - let response = integration - .handle(&settings, &services, request) - .await - .expect("should proxy conditional asset response"); - - assert_eq!(response.status(), StatusCode::NOT_MODIFIED); - assert_eq!( - response - .headers() - .get(header::ETAG) - .and_then(|value| value.to_str().ok()), - Some("\"current\"") - ); - let response_body = response - .into_body() - .into_bytes_bounded(1024) - .await - .expect("should collect not-modified JS asset body"); - assert!( - response_body.is_empty(), - "not-modified responses should not contain a body" - ); - let request_headers = stub.recorded_request_headers(); - assert_eq!(request_headers.len(), 1); - assert!(request_headers[0].iter().any(|(name, value)| { - name == header::IF_NONE_MATCH.as_str() && value == "\"current\"" - })); - assert!(request_headers[0].iter().any(|(name, value)| { - name == header::IF_MODIFIED_SINCE.as_str() - && value == "Wed, 21 Oct 2015 07:28:00 GMT" - })); + let (upstream, response) = handle_vendor_request( + request, + Some(ScriptedReply::status(StatusCode::OK).body("vendor();")), + ); + + assert_eq!(response.status(), StatusCode::OK, "should serve the asset"); + let requests = upstream.requests(); + assert_eq!(requests.len(), 1, "should send one asset request"); + let sent = &requests[0]; + assert_eq!( + (&sent.method, sent.uri.to_string()), + (&Method::GET, VENDOR_ORIGIN.to_owned()), + "should fetch the configured origin" + ); + sent.assert_headers(&[ + ("accept", "*/*"), + ("accept-language", "en-US"), + ("accept-encoding", "gzip, br"), + ("if-none-match", "\"current\""), + ("if-modified-since", "Wed, 21 Oct 2015 07:28:00 GMT"), + ("user-agent", "TrustedServer/1.0"), + ]); + sent.assert_transport(ExpectedTransport { + timeout: STREAMED_TIMEOUT, + response_mode: ResponseMode::Streamed, + response_decoding: ResponseDecoding::Original, + cache_policy: OutboundCachePolicy::PlatformDefault, }); } #[test] - fn handle_maps_upstream_outcomes_and_respects_streaming_capability() { - futures::executor::block_on(async { - let settings = create_test_settings(); - let integration = JsAssetProxyIntegration::new(config_with_assets(vec![asset( - "/assets/vendor.js", - "https://cdn.example.com/vendor.js", - JsAssetProxyMode::Enabled, - )])); + fn handle_forwards_conditional_headers_and_preserves_not_modified() { + let mut request = vendor_request(); + request.headers_mut().insert( + header::IF_NONE_MATCH, + http::HeaderValue::from_static("\"current\""), + ); + request.headers_mut().insert( + header::IF_MODIFIED_SINCE, + http::HeaderValue::from_static("Wed, 21 Oct 2015 07:28:00 GMT"), + ); - let buffered_stub = Arc::new(StubHttpClient::new()); - buffered_stub.set_streaming_responses_supported(false); - buffered_stub.push_response(200, b"ok".to_vec()); - let buffered_services = build_services_with_http_client( - Arc::clone(&buffered_stub) as Arc - ); - let success = integration - .handle( - &settings, - &buffered_services, - build_http_request( - Method::GET, - "https://publisher.example.com/assets/vendor.js", - ), - ) - .await - .expect("should proxy buffered asset response"); - assert_eq!(success.status(), StatusCode::OK); - assert_eq!(buffered_stub.recorded_stream_response_flags(), vec![false]); - - let streaming_stub = Arc::new(StubHttpClient::new()); - streaming_stub.set_streaming_responses_supported(true); - streaming_stub.push_streaming_response(200, b"streamed".to_vec()); - let streaming_services = build_services_with_http_client( - Arc::clone(&streaming_stub) as Arc - ); - let success = integration - .handle( - &settings, - &streaming_services, - build_http_request( - Method::GET, - "https://publisher.example.com/assets/vendor.js", - ), - ) - .await - .expect("should proxy streaming asset response"); - assert_eq!(success.status(), StatusCode::OK); - assert_eq!(streaming_stub.recorded_stream_response_flags(), vec![true]); - assert!( - matches!(success.body(), EdgeBody::Stream(_)), - "streaming-capable clients should preserve the origin response stream" - ); - let streamed_body = success - .into_body() - .into_bytes_bounded(1024) - .await + let (upstream, response) = handle_vendor_request( + request, + Some( + ScriptedReply::status(StatusCode::NOT_MODIFIED) + .header(header::ETAG.as_str(), "\"current\"") + .body("must not reach the client"), + ), + ); + + assert_eq!(response.status(), StatusCode::NOT_MODIFIED); + assert_eq!( + response + .headers() + .get(header::ETAG) + .and_then(|value| value.to_str().ok()), + Some("\"current\"") + ); + let response_body = + futures::executor::block_on(response.into_body().into_bytes_bounded(1024)) + .expect("should collect not-modified JS asset body"); + assert!( + response_body.is_empty(), + "not-modified responses should not contain a body" + ); + let requests = upstream.requests(); + assert_eq!(requests.len(), 1, "should send one conditional request"); + assert_eq!( + ( + requests[0].headers.get(header::IF_NONE_MATCH), + requests[0].headers.get(header::IF_MODIFIED_SINCE) + ), + ( + Some(&http::HeaderValue::from_static("\"current\"")), + Some(&http::HeaderValue::from_static( + "Wed, 21 Oct 2015 07:28:00 GMT" + )) + ), + "should forward both validators" + ); + } + + #[test] + fn handle_maps_upstream_outcomes() { + let (_upstream, success) = handle_vendor_request( + vendor_request(), + Some(ScriptedReply::status(StatusCode::OK).chunks(vec![ + bytes::Bytes::from_static(b"stream"), + bytes::Bytes::from_static(b"ed"), + ])), + ); + assert_eq!(success.status(), StatusCode::OK); + assert!( + success.body().is_stream(), + "should preserve the origin response stream" + ); + let streamed_body = + futures::executor::block_on(success.into_body().into_bytes_bounded(1024)) .expect("should collect streamed JS asset body"); - assert_eq!(streamed_body.as_ref(), b"streamed"); + assert_eq!(streamed_body.as_ref(), b"streamed"); - let unavailable_stub = Arc::new(StubHttpClient::new()); - let unavailable_services = build_services_with_http_client( - Arc::clone(&unavailable_stub) as Arc, - ); - let unavailable = integration - .handle( - &settings, - &unavailable_services, - build_http_request( - Method::GET, - "https://publisher.example.com/assets/vendor.js", - ), - ) - .await - .expect("should map unavailable origin response"); - assert_eq!(unavailable.status(), StatusCode::BAD_GATEWAY); - assert_eq!( - unavailable - .headers() - .get(HEADER_X_TS_ERROR) - .and_then(|value| value.to_str().ok()), - Some(ERROR_ORIGIN_UNREACHABLE) - ); + let (_upstream, unavailable) = handle_vendor_request(vendor_request(), None); + assert_eq!(unavailable.status(), StatusCode::BAD_GATEWAY); + assert_eq!( + unavailable + .headers() + .get(HEADER_X_TS_ERROR) + .and_then(|value| value.to_str().ok()), + Some(ERROR_ORIGIN_UNREACHABLE) + ); - let status_stub = Arc::new(StubHttpClient::new()); - status_stub.push_response(404, Vec::new()); - let status_services = build_services_with_http_client( - Arc::clone(&status_stub) as Arc - ); - let status = integration - .handle( - &settings, - &status_services, - build_http_request( - Method::GET, - "https://publisher.example.com/assets/vendor.js", - ), - ) - .await - .expect("should map non-success origin response"); - assert_eq!(status.status(), StatusCode::BAD_GATEWAY); - assert_eq!( - status - .headers() - .get(HEADER_X_TS_ERROR) - .and_then(|value| value.to_str().ok()), - Some(ERROR_ORIGIN_STATUS) - ); - }); + let (_upstream, status) = handle_vendor_request( + vendor_request(), + Some(ScriptedReply::status(StatusCode::NOT_FOUND)), + ); + assert_eq!(status.status(), StatusCode::BAD_GATEWAY); + assert_eq!( + status + .headers() + .get(HEADER_X_TS_ERROR) + .and_then(|value| value.to_str().ok()), + Some(ERROR_ORIGIN_STATUS) + ); } #[test] diff --git a/crates/trusted-server-core/src/integrations/lockr.rs b/crates/trusted-server-core/src/integrations/lockr.rs index e79c014518..56793409b0 100644 --- a/crates/trusted-server-core/src/integrations/lockr.rs +++ b/crates/trusted-server-core/src/integrations/lockr.rs @@ -11,6 +11,7 @@ use std::sync::Arc; use async_trait::async_trait; use edgezero_core::body::Body as EdgeBody; +use edgezero_core::outbound::ResponseDecoding; use error_stack::{Report, ResultExt}; use http::header::{self, HeaderMap, HeaderValue}; use http::{Method, StatusCode}; @@ -19,14 +20,18 @@ use validator::Validate; use crate::constants::INTERNAL_HEADERS; use crate::error::TrustedServerError; +use crate::http_util::read_request_body; use crate::integrations::sdk_url::parse_sdk_network_url; use crate::integrations::{ AttributeRewriteAction, INTEGRATION_MAX_BODY_BYTES, IntegrationAttributeContext, IntegrationAttributeRewriter, IntegrationEndpoint, IntegrationProxy, IntegrationRegistration, - UPSTREAM_SDK_MAX_RESPONSE_BYTES, collect_body_bounded, collect_response_bounded, - ensure_integration_backend, + UPSTREAM_SDK_MAX_RESPONSE_BYTES, }; -use crate::platform::{PlatformHttpRequest, RuntimeServices}; +use crate::outbound::{ + BUFFERED_TIMEOUT, DEFAULT_BUFFERED_RESPONSE_BYTES, OutboundRequestSource, buffered, + outbound_error, outbound_request, +}; +use crate::platform::RuntimeServices; use crate::settings::{IntegrationConfig, Settings}; const LOCKR_INTEGRATION_ID: &str = "lockr"; @@ -111,7 +116,7 @@ impl LockrIntegration { && path.ends_with(".js") } - /// Handle SDK serving — fetch from Lockr CDN and serve through first-party domain. + /// Handle SDK serving: fetch from Lockr CDN and serve through first-party domain. async fn handle_sdk_serving( &self, _settings: &Settings, @@ -122,28 +127,32 @@ impl LockrIntegration { // TODO: Check KV store cache first (future enhancement) - let lockr_req = http::Request::builder() - .method(Method::GET) - .uri(sdk_url) - .header(header::USER_AGENT, "TrustedServer/1.0") - .header(header::ACCEPT, "application/javascript, */*") - .body(EdgeBody::empty()) - .change_context(Self::error("Failed to build Lockr SDK request"))?; - - let backend_name = Self::backend_name_for_url(services, sdk_url) - .change_context(Self::error("Failed to determine backend for SDK fetch"))?; + let lockr_req = outbound_request(Method::GET, sdk_url) + .and_then(|request| request.header(header::USER_AGENT, "TrustedServer/1.0")) + .and_then(|request| request.header(header::ACCEPT, "application/javascript, */*")) + .map_err(|error| { + outbound_error( + error, + Self::error("Failed to build Lockr SDK request"), + OutboundRequestSource::Internal, + ) + })?; + let fetch_failed = || Self::error(format!("Failed to fetch Lockr SDK from {sdk_url}")); let lockr_response = services - .http_client() - .send(PlatformHttpRequest::new(lockr_req, backend_name)) + .outbound_client() + .send(buffered( + lockr_req, + BUFFERED_TIMEOUT, + UPSTREAM_SDK_MAX_RESPONSE_BYTES, + ResponseDecoding::Decode, + )) .await - .change_context(Self::error(format!( - "Failed to fetch Lockr SDK from {}", - sdk_url - )))? - .response; + .map_err(|error| { + outbound_error(error, fetch_failed(), OutboundRequestSource::Internal) + })?; - if !lockr_response.status().is_success() { + if !lockr_response.is_success() { log::error!( "Lockr SDK fetch failed with status {}", lockr_response.status() @@ -154,13 +163,12 @@ impl LockrIntegration { )))); } - let sdk_body = collect_response_bounded( - lockr_response.into_body(), - UPSTREAM_SDK_MAX_RESPONSE_BYTES, - LOCKR_INTEGRATION_ID, - ) - .await - .change_context(Self::error("Failed to read Lockr SDK response body"))?; + let sdk_body = lockr_response + .into_bytes_bounded(UPSTREAM_SDK_MAX_RESPONSE_BYTES) + .await + .map_err(|error| { + outbound_error(error, fetch_failed(), OutboundRequestSource::Internal) + })?; log::info!("Fetched Lockr SDK ({} bytes)", sdk_body.len()); // TODO: Cache in KV store (future enhancement) @@ -182,7 +190,7 @@ impl LockrIntegration { .change_context(Self::error("Failed to build Lockr SDK response")) } - /// Handle API proxy — forward requests to the configured Lockr API endpoint. + /// Handle API proxy: forward requests to the configured Lockr API endpoint. async fn handle_api_proxy( &self, _settings: &Settings, @@ -211,37 +219,42 @@ impl LockrIntegration { log::info!("Forwarding to Lockr API: {}", target_url); let request_body = if method == Method::POST { - let bytes = - collect_body_bounded(body, INTEGRATION_MAX_BODY_BYTES, LOCKR_INTEGRATION_ID) - .await?; + let bytes = read_request_body(&parts.headers, body, INTEGRATION_MAX_BODY_BYTES).await?; EdgeBody::from(bytes) } else { EdgeBody::empty() }; - let mut target_req = http::Request::builder() - .method(method.clone()) - .uri(&target_url) - .body(request_body) - .change_context(Self::error("Failed to build Lockr API proxy request"))?; + let forward_failed = || Self::error(format!("Failed to forward request to {target_url}")); + let mut target_req = outbound_request(method, &target_url) + .map_err(|error| { + outbound_error( + error, + Self::error("Failed to build Lockr API proxy request"), + OutboundRequestSource::Internal, + ) + })? + .body(request_body); self.copy_request_headers(&parts.headers, target_req.headers_mut())?; - let backend_name = Self::backend_name_for_url(services, &self.config.api_endpoint) - .change_context(Self::error("Failed to determine backend for API proxy"))?; - let response = services - .http_client() - .send(PlatformHttpRequest::new(target_req, backend_name)) + .outbound_client() + .send(buffered( + target_req, + BUFFERED_TIMEOUT, + DEFAULT_BUFFERED_RESPONSE_BYTES, + ResponseDecoding::Original, + )) .await - .change_context(Self::error(format!( - "Failed to forward request to {}", - target_url - )))? - .response; + .map_err(|error| { + outbound_error(error, forward_failed(), OutboundRequestSource::Internal) + })?; log::info!("Lockr API responded with status {}", response.status()); - Ok(response) + response.into_response().map_err(|error| { + outbound_error(error, forward_failed(), OutboundRequestSource::Internal) + }) } /// Copy relevant request headers for proxying. @@ -302,13 +315,6 @@ impl LockrIntegration { Ok(()) } - - fn backend_name_for_url( - services: &RuntimeServices, - target_url: &str, - ) -> Result> { - ensure_integration_backend(services, target_url, LOCKR_INTEGRATION_ID, None) - } } fn build(settings: &Settings) -> Result>, Report> { @@ -437,16 +443,20 @@ fn default_rewrite_sdk() -> bool { #[cfg(test)] mod tests { - use std::sync::Arc; - - use super::*; - use edgezero_core::http::Method as HttpMethod; + use edgezero_core::outbound::{OutboundCachePolicy, ResponseDecoding, ResponseMode}; + use futures::executor::block_on; use serde_json::json; + use super::*; use crate::integrations::sdk_url_test_support::{ assert_sdk_html_attribute_rewrite, assert_sdk_url_rewrite, sdk_url_cases, }; - use crate::platform::test_support::{StubHttpClient, build_services_with_http_client}; + use crate::outbound::{BUFFERED_TIMEOUT, DEFAULT_BUFFERED_RESPONSE_BYTES}; + use crate::platform::test_support::build_services_with_outbound_client; + use crate::test_support::body_read::body_read_failure_cases; + use crate::test_support::scripted_client::{ + ExpectedTransport, ScriptedHttpClient, ScriptedReply, TestClock, + }; use crate::test_support::tests::create_test_settings; fn test_config() -> LockrConfig { @@ -640,99 +650,181 @@ mod tests { ); } + fn readback_config() -> LockrConfig { + LockrConfig { + api_endpoint: "https://identity.example.com".to_string(), + sdk_url: "https://sdk.example.com/identity-lockr-trust-server.js".to_string(), + ..test_config() + } + } + #[test] - fn lockr_proxy_uses_platform_http_client() { - let stub = Arc::new(StubHttpClient::new()); - stub.push_response(200, b"ok".to_vec()); - let services = build_services_with_http_client( - Arc::clone(&stub) as Arc + fn lockr_sdk_url_with_fragment_is_fetched_without_it() { + let clock = TestClock::new(); + let client = ScriptedHttpClient::new(&clock); + client.respond_to( + Method::GET, + "https://sdk.example.com/identity-lockr-trust-server.js", + ScriptedReply::status(StatusCode::OK).body("window.lockr = {};"), ); - let settings = create_test_settings(); - let integration = LockrIntegration::new(test_config()); + let services = build_services_with_outbound_client(&client, &clock); + let integration = LockrIntegration::new(LockrConfig { + sdk_url: "https://sdk.example.com/identity-lockr-trust-server.js#v2".to_string(), + ..readback_config() + }); let req = http::Request::builder() - .method(HttpMethod::GET) - .uri("https://publisher.example/integrations/lockr/api/publisher/app/v1/identityLockr/settings") + .method(Method::GET) + .uri("https://publisher.example.com/integrations/lockr/sdk") .body(EdgeBody::empty()) - .expect("should build request"); + .expect("should build SDK request"); - let response = futures::executor::block_on(integration.handle(&settings, &services, req)) - .expect("should proxy request"); + let response = block_on(integration.handle(&create_test_settings(), &services, req)) + .expect("should serve an SDK whose configured URL carries a fragment"); + assert_eq!(response.status(), StatusCode::OK, "should serve the SDK"); assert_eq!( - response.status(), - http::StatusCode::OK, - "should return stubbed response" + client + .requests() + .iter() + .map(|request| request.uri.to_string()) + .collect::>(), + vec!["https://sdk.example.com/identity-lockr-trust-server.js"], + "should fetch the SDK URL without its fragment" ); + } + + #[test] + fn lockr_sdk_request_readback() { + let clock = TestClock::new(); + let client = ScriptedHttpClient::new(&clock); + client.respond_to( + Method::GET, + "https://sdk.example.com/identity-lockr-trust-server.js", + ScriptedReply::status(StatusCode::OK).body("window.lockr = {};"), + ); + let services = build_services_with_outbound_client(&client, &clock); + let integration = LockrIntegration::new(readback_config()); + let req = http::Request::builder() + .method(Method::GET) + .uri("https://publisher.example.com/integrations/lockr/sdk") + .header(header::ACCEPT_ENCODING, "gzip") + .header(header::AUTHORIZATION, "Basic dXNlcjpwYXNz") + .body(EdgeBody::empty()) + .expect("should build SDK request"); + + let response = block_on(integration.handle(&create_test_settings(), &services, req)) + .expect("should serve the Lockr SDK"); + + assert_eq!( + response + .into_body() + .into_bytes() + .expect("should buffer the SDK body") + .as_ref(), + b"window.lockr = {};", + "should serve the fetched SDK bytes" + ); + let requests = client.requests(); + assert_eq!(requests.len(), 1, "should send one SDK request"); + let sdk = &requests[0]; assert_eq!( - stub.recorded_backend_names().len(), - 1, - "should route one outbound request through PlatformHttpClient" + (&sdk.method, sdk.uri.to_string()), + ( + &Method::GET, + "https://sdk.example.com/identity-lockr-trust-server.js".to_owned() + ), + "should fetch the configured SDK URL" ); + sdk.assert_headers(&[ + ("user-agent", "TrustedServer/1.0"), + ("accept", "application/javascript, */*"), + ]); + sdk.assert_transport(ExpectedTransport { + timeout: BUFFERED_TIMEOUT, + response_mode: ResponseMode::Buffered { + max_bytes: 16 * 1024 * 1024, + }, + response_decoding: ResponseDecoding::Decode, + cache_policy: OutboundCachePolicy::PlatformDefault, + }); } #[test] - fn lockr_proxy_forwards_body_and_strips_publisher_credentials() { - // Regression guard for the upstream-rejection / credential-leak causes: - // 1. The POST body (and content-type) must be forwarded, otherwise the - // Lockr API returns `{"code":400,"message":"Invalid request"}`. - // 2. The publisher's `Authorization` header (e.g. site basic-auth) must - // NOT be forwarded — the Lockr API rejects it with the same 400, and - // forwarding it would leak the publisher credential to a third party. - // 3. The publisher's `Cookie` header (session/auth cookies the browser - // attaches to the first-party route) must NOT be forwarded either. - let stub = Arc::new(StubHttpClient::new()); - stub.push_response(200, br#"{"success":true,"data":{}}"#.to_vec()); - let services = build_services_with_http_client( - Arc::clone(&stub) as Arc + fn lockr_api_request_readback() { + // The POST body and content type must reach the Lockr API, which + // answers `{"code":400,"message":"Invalid request"}` without them. The + // publisher's `Authorization` and `Cookie` headers must not: the API + // rejects an unexpected `Authorization` with the same 400, and both + // carry publisher credentials that must not leak to a third party. + let clock = TestClock::new(); + let client = ScriptedHttpClient::new(&clock); + let target = + "https://identity.example.com/publisher/app/v2/identityLockr/settings?source=example"; + client.respond_to( + Method::POST, + target, + ScriptedReply::status(StatusCode::OK) + .header("content-type", "application/json") + .body(r#"{"success":true,"data":{}}"#), ); - let settings = create_test_settings(); - let integration = LockrIntegration::new(test_config()); - + let services = build_services_with_outbound_client(&client, &clock); + let integration = LockrIntegration::new(readback_config()); let payload = br#"{"appID":"test-app-id"}"#; let req = http::Request::builder() - .method(HttpMethod::POST) - .uri("https://publisher.example/integrations/lockr/api/publisher/app/v2/identityLockr/settings") + .method(Method::POST) + .uri("https://publisher.example.com/integrations/lockr/api/publisher/app/v2/identityLockr/settings?source=example") .header(header::CONTENT_TYPE, "application/json;charset=UTF-8") + .header(header::CONTENT_LENGTH, payload.len().to_string()) + .header(header::ACCEPT, "application/json") + .header(header::USER_AGENT, "example-agent") + .header(header::ACCEPT_LANGUAGE, "en") + .header(header::ACCEPT_ENCODING, "gzip") + .header(header::ORIGIN, "https://publisher.example.com") + .header("x-example-trace", "trace-1") + .header("x-ts-ec", "internal-ec") .header(header::AUTHORIZATION, "Basic dXNlcjpwYXNz") .header(header::COOKIE, "session_id=secret; euconsent-v2=tcf") .body(EdgeBody::from(payload.to_vec())) - .expect("should build request"); + .expect("should build API request"); - let response = futures::executor::block_on(integration.handle(&settings, &services, req)) - .expect("should proxy request"); - assert_eq!(response.status(), http::StatusCode::OK, "should return OK"); + let response = block_on(integration.handle(&create_test_settings(), &services, req)) + .expect("should proxy the Lockr API request"); - let bodies = stub.recorded_request_bodies(); + assert_eq!(response.status(), StatusCode::OK, "should relay the status"); assert_eq!( - bodies.len(), - 1, - "should forward exactly one upstream request" + response + .into_body() + .into_bytes() + .expect("should buffer the API body") + .as_ref(), + br#"{"success":true,"data":{}}"#, + "should relay the API body" ); + let requests = client.requests(); + assert_eq!(requests.len(), 1, "should send one API request"); + let api = &requests[0]; assert_eq!( - bodies[0], payload, - "should forward the POST body unchanged to the Lockr API" - ); - - let headers = stub.recorded_request_headers(); - assert!( - headers[0] - .iter() - .any(|(name, value)| name == "content-type" - && value == "application/json;charset=UTF-8"), - "should forward the content-type header to the Lockr API" - ); - assert!( - !headers[0] - .iter() - .any(|(name, _)| name.eq_ignore_ascii_case("authorization")), - "should NOT forward the publisher's Authorization header to the Lockr API" - ); - assert!( - !headers[0] - .iter() - .any(|(name, _)| name.eq_ignore_ascii_case("cookie")), - "should NOT forward the publisher's Cookie header to the Lockr API" + (&api.method, api.uri.to_string(), api.body.as_ref()), + (&Method::POST, target.to_owned(), payload.as_slice()), + "should forward the method, path, query, and body" ); + api.assert_headers(&[ + ("content-type", "application/json;charset=UTF-8"), + ("accept", "application/json"), + ("user-agent", "example-agent"), + ("accept-language", "en"), + ("accept-encoding", "gzip"), + ("origin", "https://publisher.example.com"), + ("x-example-trace", "trace-1"), + ]); + api.assert_transport(ExpectedTransport { + timeout: BUFFERED_TIMEOUT, + response_mode: ResponseMode::Buffered { + max_bytes: DEFAULT_BUFFERED_RESPONSE_BYTES, + }, + response_decoding: ResponseDecoding::Original, + cache_policy: OutboundCachePolicy::PlatformDefault, + }); } #[test] @@ -812,4 +904,25 @@ mod tests { "disabled invalid Lockr config should not register" ); } + + #[test] + fn lockr_api_body_read_failures_map_to_statuses() { + let settings = create_test_settings(); + let integration = LockrIntegration::new(test_config()); + for case in body_read_failure_cases(INTEGRATION_MAX_BODY_BYTES) { + let clock = TestClock::new(); + let client = ScriptedHttpClient::forbidding(); + let services = build_services_with_outbound_client(&client, &clock); + let (req, expectation) = case.into_request( + http::Request::builder() + .method(http::Method::POST) + .uri("https://publisher.example/integrations/lockr/api/publisher/app/v2/identityLockr/settings"), + ); + + let outcome = block_on(integration.handle(&settings, &services, req)); + + expectation.assert_outcome("Lockr API", &outcome); + client.assert_no_requests(); + } + } } diff --git a/crates/trusted-server-core/src/integrations/mod.rs b/crates/trusted-server-core/src/integrations/mod.rs index 953db44931..49f77262c3 100644 --- a/crates/trusted-server-core/src/integrations/mod.rs +++ b/crates/trusted-server-core/src/integrations/mod.rs @@ -1,14 +1,11 @@ //! Integration module registry and sample implementations. -use std::time::Duration; - -use edgezero_core::body::Body as EdgeBody; -use error_stack::{Report, ResultExt}; +use edgezero_core::body::{Body as EdgeBody, BodyPayload}; +use error_stack::Report; use futures::StreamExt as _; -use url::Url; use crate::error::TrustedServerError; -use crate::platform::{DEFAULT_FIRST_BYTE_TIMEOUT, PlatformBackendSpec, RuntimeServices}; +use crate::http_util::into_body_payload; use crate::settings::Settings; pub mod adserver_mock; @@ -42,195 +39,23 @@ pub use registry::{ RequestFilterRegistryInput, RequestFilterRegistryOutcome, ScriptRewriteAction, }; -/// Registers or retrieves a platform backend for the given URL. -/// -/// Parses `url`, builds a [`PlatformBackendSpec`] with TLS enabled and a -/// 15-second first-byte timeout, and delegates to -/// [`crate::platform::PlatformBackend::ensure`]. -/// -/// # Errors -/// -/// Returns an error when `url` cannot be parsed, is missing a host, or the -/// backend registration fails. -pub(crate) fn ensure_integration_backend( - services: &RuntimeServices, - url: &str, - integration: &'static str, - first_byte_timeout: Option, -) -> Result> { - services - .backend() - .ensure(&integration_backend_spec( - url, - integration, - true, - first_byte_timeout.unwrap_or(DEFAULT_FIRST_BYTE_TIMEOUT), - )?) - .change_context(TrustedServerError::Integration { - integration: integration.to_string(), - message: "Failed to register backend".to_string(), - }) -} - -/// Registers or retrieves a platform backend for the given URL with a custom -/// first-byte timeout. -/// -/// Parses `url`, builds a [`PlatformBackendSpec`] with TLS enabled and the -/// given `first_byte_timeout`, and delegates to -/// [`crate::platform::PlatformBackend::ensure`]. -/// -/// # Errors -/// -/// Returns an error when `url` cannot be parsed, is missing a host, or the -/// backend registration fails. -pub(crate) fn ensure_integration_backend_with_timeout( - services: &RuntimeServices, - url: &str, - integration: &'static str, - first_byte_timeout: Duration, -) -> Result> { - services - .backend() - .ensure(&integration_backend_spec( - url, - integration, - true, - first_byte_timeout, - )?) - .change_context(TrustedServerError::Integration { - integration: integration.to_string(), - message: "Failed to register backend".to_string(), - }) -} - -/// Compute the deterministic platform backend name for a URL without registering it. -/// -/// Parses `url`, builds a [`PlatformBackendSpec`], and delegates to -/// [`crate::platform::PlatformBackend::predict_name`]. -/// -/// # Errors -/// -/// Returns an error when the URL cannot be parsed, is missing a host, or the -/// platform backend cannot predict a name for the spec. -pub(crate) fn predict_integration_backend_name( - services: &RuntimeServices, - url: &str, - integration: &'static str, - first_byte_timeout: Duration, -) -> Result> { - services - .backend() - .predict_name(&integration_backend_spec( - url, - integration, - true, - first_byte_timeout, - )?) - .change_context(TrustedServerError::Integration { - integration: integration.to_string(), - message: "Failed to predict backend name".to_string(), - }) -} - -fn integration_backend_spec( - url: &str, - integration: &'static str, - certificate_check: bool, - first_byte_timeout: Duration, -) -> Result> { - let parsed = Url::parse(url).change_context(TrustedServerError::Integration { - integration: integration.to_string(), - message: format!("Invalid upstream URL: {url}"), - })?; - Ok(PlatformBackendSpec { - scheme: parsed.scheme().to_string(), - host: parsed - .host_str() - .ok_or_else(|| { - Report::new(TrustedServerError::Integration { - integration: integration.to_string(), - message: "Upstream URL missing host".to_string(), - }) - })? - .to_string(), - port: parsed.port(), - host_header_override: None, - certificate_check, - first_byte_timeout, - between_bytes_timeout: first_byte_timeout, - // Distinguish this integration's backend from any other provider that - // targets the same origin, so auction response correlation by backend - // name cannot cross providers. - discriminator: Some(integration.to_string()), - }) -} - /// Maximum body size accepted by integration proxy endpoints (256 KiB). pub(crate) const INTEGRATION_MAX_BODY_BYTES: usize = 256 * 1024; /// Maximum response body size from RTB providers (prebid, aps, mediator). pub(crate) const UPSTREAM_RTB_MAX_RESPONSE_BYTES: usize = 2 * 1024 * 1024; /// Maximum response body size from SDK/proxy integrations. -pub(crate) const UPSTREAM_SDK_MAX_RESPONSE_BYTES: usize = 16 * 1024 * 1024; - -/// Drains an [`EdgeBody`] into a byte vector, rejecting bodies larger than -/// `max_bytes` with [`TrustedServerError::RequestTooLarge`]. -/// -/// # Errors -/// -/// Returns an error when: -/// - The body exceeds `max_bytes`. -/// - A streaming body chunk cannot be read (mapped to an `Integration` error). -pub(crate) async fn collect_body_bounded( - body: EdgeBody, - max_bytes: usize, - integration: &'static str, -) -> Result, Report> { - match body { - EdgeBody::Once(bytes) => { - if bytes.len() > max_bytes { - return Err(Report::new(TrustedServerError::RequestTooLarge { - message: format!( - "{integration}: request body ({} bytes) exceeds the {max_bytes} byte limit", - bytes.len(), - ), - })); - } - Ok(bytes.to_vec()) - } - EdgeBody::Stream(mut stream) => { - let mut body_bytes = Vec::new(); - while let Some(chunk_result) = stream.next().await { - let chunk = chunk_result.map_err(|error| { - Report::new(TrustedServerError::Integration { - integration: integration.to_string(), - message: format!("Failed to read request body: {error}"), - }) - })?; - if body_bytes.len() + chunk.len() > max_bytes { - return Err(Report::new(TrustedServerError::RequestTooLarge { - message: format!( - "{integration}: request body exceeds the {max_bytes} byte limit", - ), - })); - } - // Size check runs after chunk is materialized — effective bound is - // ≤ max_bytes + one_chunk (Fastly H2/H3 chunks are ≤ 16 KiB in practice). - body_bytes.extend_from_slice(&chunk); - } - Ok(body_bytes) - } - } -} +pub(crate) const UPSTREAM_SDK_MAX_RESPONSE_BYTES: u64 = 16 * 1024 * 1024; /// Drains an upstream [`EdgeBody`] response into a byte vector, rejecting /// bodies larger than `max_bytes` with [`TrustedServerError::Integration`]. /// /// Use this for upstream (provider/integration) response bodies to bound -/// memory usage when a third-party server misbehaves. Unlike -/// [`collect_body_bounded`], oversized bodies are classified as -/// [`TrustedServerError::Integration`] (502 `BAD_GATEWAY`) rather than -/// [`TrustedServerError::RequestTooLarge`] (413). +/// memory usage when a third-party server misbehaves. An oversized body is +/// classified as [`TrustedServerError::Integration`] (502 `BAD_GATEWAY`). +/// Inbound request bodies use +/// [`read_request_body`](crate::http_util::read_request_body) instead, which +/// answers an overflow with [`TrustedServerError::RequestTooLarge`] (413). /// /// Note: the effective bound for streaming bodies is ≤ `max_bytes` + `one_chunk` /// because the size check runs after each chunk is materialized. Fastly @@ -246,8 +71,8 @@ pub(crate) async fn collect_response_bounded( max_bytes: usize, integration: &'static str, ) -> Result, Report> { - match body { - EdgeBody::Once(bytes) => { + match into_body_payload(body) { + BodyPayload::Once(bytes) => { if bytes.len() > max_bytes { return Err(Report::new(TrustedServerError::Integration { integration: integration.to_string(), @@ -259,7 +84,7 @@ pub(crate) async fn collect_response_bounded( } Ok(bytes.to_vec()) } - EdgeBody::Stream(mut stream) => { + BodyPayload::Stream(mut stream) => { let mut body_bytes = Vec::new(); while let Some(chunk_result) = stream.next().await { let chunk = chunk_result.map_err(|error| { diff --git a/crates/trusted-server-core/src/integrations/permutive.rs b/crates/trusted-server-core/src/integrations/permutive.rs index 93ec993b14..8d0ea20874 100644 --- a/crates/trusted-server-core/src/integrations/permutive.rs +++ b/crates/trusted-server-core/src/integrations/permutive.rs @@ -7,6 +7,7 @@ use std::sync::Arc; use async_trait::async_trait; use edgezero_core::body::Body as EdgeBody; +use edgezero_core::outbound::ResponseDecoding; use error_stack::{Report, ResultExt}; use http::header::{self, HeaderMap, HeaderValue}; use http::{Method, StatusCode}; @@ -15,14 +16,18 @@ use validator::Validate; use crate::constants::INTERNAL_HEADERS; use crate::error::TrustedServerError; +use crate::http_util::read_request_body; use crate::integrations::sdk_url::parse_sdk_network_url; use crate::integrations::{ AttributeRewriteAction, INTEGRATION_MAX_BODY_BYTES, IntegrationAttributeContext, IntegrationAttributeRewriter, IntegrationEndpoint, IntegrationProxy, IntegrationRegistration, - UPSTREAM_SDK_MAX_RESPONSE_BYTES, collect_body_bounded, collect_response_bounded, - ensure_integration_backend, + UPSTREAM_SDK_MAX_RESPONSE_BYTES, }; -use crate::platform::{PlatformHttpRequest, RuntimeServices}; +use crate::outbound::{ + BUFFERED_TIMEOUT, DEFAULT_BUFFERED_RESPONSE_BYTES, OutboundRequestSource, buffered, + outbound_error, outbound_request, +}; +use crate::platform::RuntimeServices; use crate::settings::{IntegrationConfig, Settings}; const PERMUTIVE_INTEGRATION_ID: &str = "permutive"; @@ -128,28 +133,32 @@ impl PermutiveIntegration { // TODO: Check KV store cache first (future enhancement) // Fetch SDK from Permutive CDN - let permutive_req = http::Request::builder() - .method(Method::GET) - .uri(&sdk_url) - .header(header::USER_AGENT, "TrustedServer/1.0") - .header(header::ACCEPT, "application/javascript, */*") - .body(EdgeBody::empty()) - .change_context(Self::error("Failed to build Permutive SDK request"))?; - - let backend_name = Self::backend_name_for_url(services, &sdk_url) - .change_context(Self::error("Failed to determine backend for SDK fetch"))?; + let permutive_req = outbound_request(Method::GET, &sdk_url) + .and_then(|request| request.header(header::USER_AGENT, "TrustedServer/1.0")) + .and_then(|request| request.header(header::ACCEPT, "application/javascript, */*")) + .map_err(|error| { + outbound_error( + error, + Self::error("Failed to build Permutive SDK request"), + OutboundRequestSource::Internal, + ) + })?; + let fetch_failed = || Self::error(format!("Failed to fetch Permutive SDK from {sdk_url}")); let permutive_response = services - .http_client() - .send(PlatformHttpRequest::new(permutive_req, backend_name)) + .outbound_client() + .send(buffered( + permutive_req, + BUFFERED_TIMEOUT, + UPSTREAM_SDK_MAX_RESPONSE_BYTES, + ResponseDecoding::Decode, + )) .await - .change_context(Self::error(format!( - "Failed to fetch Permutive SDK from {}", - sdk_url - )))? - .response; + .map_err(|error| { + outbound_error(error, fetch_failed(), OutboundRequestSource::Internal) + })?; - if !permutive_response.status().is_success() { + if !permutive_response.is_success() { log::error!( "Permutive SDK fetch failed with status: {}", permutive_response.status() @@ -160,13 +169,12 @@ impl PermutiveIntegration { )))); } - let sdk_body = collect_response_bounded( - permutive_response.into_body(), - UPSTREAM_SDK_MAX_RESPONSE_BYTES, - PERMUTIVE_INTEGRATION_ID, - ) - .await - .change_context(Self::error("Failed to read Permutive SDK response body"))?; + let sdk_body = permutive_response + .into_bytes_bounded(UPSTREAM_SDK_MAX_RESPONSE_BYTES) + .await + .map_err(|error| { + outbound_error(error, fetch_failed(), OutboundRequestSource::Internal) + })?; log::info!( "Successfully fetched Permutive SDK: {} bytes", sdk_body.len() @@ -223,38 +231,36 @@ impl PermutiveIntegration { log::info!("Forwarding {} to {}", route_name, target_url); let request_body = if method == Method::POST { - let bytes = - collect_body_bounded(body, INTEGRATION_MAX_BODY_BYTES, PERMUTIVE_INTEGRATION_ID) - .await?; + let bytes = read_request_body(&parts.headers, body, INTEGRATION_MAX_BODY_BYTES).await?; EdgeBody::from(bytes) } else { EdgeBody::empty() }; - let mut target_req = http::Request::builder() - .method(method) - .uri(&target_url) - .body(request_body) - .change_context(Self::error(format!( - "Failed to build {} proxy request", - route_name - )))?; + let forward_failed = || Self::error(format!("Failed to forward request to {target_url}")); + let mut target_req = outbound_request(method, &target_url) + .map_err(|error| { + outbound_error( + error, + Self::error(format!("Failed to build {} proxy request", route_name)), + OutboundRequestSource::Internal, + ) + })? + .body(request_body); self.copy_request_headers(&parts.headers, target_req.headers_mut()); - let backend_name = - Self::backend_name_for_url(services, upstream_base).change_context(Self::error( - format!("Failed to determine backend for {} proxy", route_name), - ))?; - let response = services - .http_client() - .send(PlatformHttpRequest::new(target_req, backend_name)) + .outbound_client() + .send(buffered( + target_req, + BUFFERED_TIMEOUT, + DEFAULT_BUFFERED_RESPONSE_BYTES, + ResponseDecoding::Original, + )) .await - .change_context(Self::error(format!( - "Failed to forward request to {}", - target_url - )))? - .response; + .map_err(|error| { + outbound_error(error, forward_failed(), OutboundRequestSource::Internal) + })?; log::info!( "{} responded with status: {}", @@ -262,7 +268,9 @@ impl PermutiveIntegration { response.status() ); - Ok(response) + response.into_response().map_err(|error| { + outbound_error(error, forward_failed(), OutboundRequestSource::Internal) + }) } /// Copy relevant request headers for proxying. @@ -293,13 +301,6 @@ impl PermutiveIntegration { } } } - - fn backend_name_for_url( - services: &RuntimeServices, - target_url: &str, - ) -> Result> { - ensure_integration_backend(services, target_url, PERMUTIVE_INTEGRATION_ID, None) - } } fn build( @@ -481,13 +482,19 @@ fn default_rewrite_sdk() -> bool { #[cfg(test)] mod tests { - use std::sync::Arc; + use edgezero_core::outbound::{OutboundCachePolicy, ResponseDecoding, ResponseMode}; + use futures::executor::block_on; use super::*; use crate::integrations::sdk_url_test_support::{ assert_sdk_html_attribute_rewrite, assert_sdk_url_rewrite, sdk_url_cases, }; - use crate::platform::test_support::{StubHttpClient, build_services_with_http_client}; + use crate::outbound::{BUFFERED_TIMEOUT, DEFAULT_BUFFERED_RESPONSE_BYTES}; + use crate::platform::test_support::build_services_with_outbound_client; + use crate::test_support::body_read::body_read_failure_cases; + use crate::test_support::scripted_client::{ + ExpectedTransport, ScriptedHttpClient, ScriptedReply, TestClock, + }; use crate::test_support::tests::create_test_settings; #[test] @@ -674,93 +681,160 @@ mod tests { ); } - #[test] - fn permutive_proxy_uses_platform_http_client() { - let stub = Arc::new(StubHttpClient::new()); - stub.push_response(200, b"ok".to_vec()); - let services = build_services_with_http_client( - Arc::clone(&stub) as Arc - ); - let settings = create_test_settings(); - let integration = PermutiveIntegration::new(PermutiveConfig { + fn readback_config() -> PermutiveConfig { + PermutiveConfig { enabled: true, organization_id: "myorg".to_string(), workspace_id: "workspace-123".to_string(), project_id: String::new(), - api_endpoint: default_api_endpoint(), - secure_signals_endpoint: default_secure_signals_endpoint(), + api_endpoint: "https://api.example.com".to_string(), + secure_signals_endpoint: "https://signals.example.com".to_string(), cache_ttl_seconds: 3600, rewrite_sdk: true, - }); + } + } + + #[test] + fn permutive_sdk_request_readback() { + let clock = TestClock::new(); + let client = ScriptedHttpClient::new(&clock); + let integration = PermutiveIntegration::new(readback_config()); + let sdk_url = integration.sdk_url(); + client.respond_to( + Method::GET, + &sdk_url, + ScriptedReply::status(StatusCode::OK).body("window.permutive = {};"), + ); + let services = build_services_with_outbound_client(&client, &clock); let req = http::Request::builder() - .method(http::Method::GET) - .uri("https://publisher.example/integrations/permutive/api/v2.0/events") + .method(Method::GET) + .uri("https://publisher.example.com/integrations/permutive/sdk") + .header(header::ACCEPT_ENCODING, "gzip") + .header(header::AUTHORIZATION, "Basic dXNlcjpwYXNz") .body(EdgeBody::empty()) - .expect("should build request"); + .expect("should build SDK request"); - let response = futures::executor::block_on(integration.handle(&settings, &services, req)) - .expect("should proxy request"); + let response = block_on(integration.handle(&create_test_settings(), &services, req)) + .expect("should serve the Permutive SDK"); assert_eq!( - response.status(), - http::StatusCode::OK, - "should return stubbed response" + response + .into_body() + .into_bytes() + .expect("should buffer the SDK body") + .as_ref(), + b"window.permutive = {};", + "should serve the fetched SDK bytes" ); + let requests = client.requests(); + assert_eq!(requests.len(), 1, "should send one SDK request"); + let sdk = &requests[0]; assert_eq!( - stub.recorded_backend_names(), - vec!["stub-backend".to_string()], - "should route outbound request through PlatformHttpClient" + (&sdk.method, sdk.uri.to_string()), + (&Method::GET, sdk_url), + "should fetch the workspace SDK URL" ); + sdk.assert_headers(&[ + ("user-agent", "TrustedServer/1.0"), + ("accept", "application/javascript, */*"), + ]); + sdk.assert_transport(ExpectedTransport { + timeout: BUFFERED_TIMEOUT, + response_mode: ResponseMode::Buffered { + max_bytes: 16 * 1024 * 1024, + }, + response_decoding: ResponseDecoding::Decode, + cache_policy: OutboundCachePolicy::PlatformDefault, + }); } #[test] - fn permutive_proxy_strips_authorization() { - // Security regression guard: the publisher's Authorization header must - // not be forwarded to the Permutive upstream (credential leak). - let stub = Arc::new(StubHttpClient::new()); - stub.push_response(200, b"ok".to_vec()); - let services = build_services_with_http_client( - Arc::clone(&stub) as Arc + fn permutive_proxy_request_readback() { + // The publisher's `Authorization` header must not reach Permutive: it + // carries the publisher's own credential and would leak it. + let clock = TestClock::new(); + let client = ScriptedHttpClient::new(&clock); + let target = "https://api.example.com/v2.0/events?k=example"; + client.respond_to( + Method::POST, + target, + ScriptedReply::status(StatusCode::ACCEPTED).body("accepted"), ); - let settings = create_test_settings(); - let integration = PermutiveIntegration::new(PermutiveConfig { - enabled: true, - organization_id: "myorg".to_string(), - workspace_id: "workspace-123".to_string(), - project_id: String::new(), - api_endpoint: default_api_endpoint(), - secure_signals_endpoint: default_secure_signals_endpoint(), - cache_ttl_seconds: 3600, - rewrite_sdk: true, - }); + let services = build_services_with_outbound_client(&client, &clock); + let integration = PermutiveIntegration::new(readback_config()); + let payload = br#"{"event":"Pageview"}"#; let req = http::Request::builder() - .method(http::Method::GET) - .uri("https://publisher.example/integrations/permutive/api/v2.0/events") - .header(header::AUTHORIZATION, "Basic dXNlcjpwYXNz") + .method(Method::POST) + .uri("https://publisher.example.com/integrations/permutive/api/v2.0/events?k=example") + .header(header::CONTENT_TYPE, "application/json") + .header(header::CONTENT_LENGTH, payload.len().to_string()) + .header(header::ACCEPT, "application/json") .header(header::USER_AGENT, "test-agent") - .body(EdgeBody::empty()) - .expect("should build request"); + .header(header::ACCEPT_LANGUAGE, "en") + .header(header::ACCEPT_ENCODING, "br") + .header("x-example-trace", "trace-1") + .header("x-ts-ec", "internal-ec") + .header(header::AUTHORIZATION, "Basic dXNlcjpwYXNz") + .header(header::COOKIE, "session_id=secret") + .body(EdgeBody::from(payload.to_vec())) + .expect("should build proxy request"); + + let response = block_on(integration.handle(&create_test_settings(), &services, req)) + .expect("should proxy the Permutive request"); - let response = futures::executor::block_on(integration.handle(&settings, &services, req)) - .expect("should proxy request"); assert_eq!( response.status(), - http::StatusCode::OK, - "should return stubbed response" + StatusCode::ACCEPTED, + "should relay the status" ); - - let headers = stub.recorded_request_headers(); - assert!( - !headers[0] - .iter() - .any(|(name, _)| name.eq_ignore_ascii_case("authorization")), - "should NOT forward the publisher's Authorization header to Permutive" - ); - assert!( - headers[0] - .iter() - .any(|(name, value)| name == "user-agent" && value == "test-agent"), - "should still forward required headers (user-agent)" + let requests = client.requests(); + assert_eq!(requests.len(), 1, "should send one proxy request"); + let proxied = &requests[0]; + assert_eq!( + ( + &proxied.method, + proxied.uri.to_string(), + proxied.body.as_ref() + ), + (&Method::POST, target.to_owned(), payload.as_slice()), + "should forward the method, path, query, and body" ); + proxied.assert_headers(&[ + ("content-type", "application/json"), + ("accept", "application/json"), + ("user-agent", "test-agent"), + ("accept-language", "en"), + ("accept-encoding", "br"), + ("x-example-trace", "trace-1"), + ]); + proxied.assert_transport(ExpectedTransport { + timeout: BUFFERED_TIMEOUT, + response_mode: ResponseMode::Buffered { + max_bytes: DEFAULT_BUFFERED_RESPONSE_BYTES, + }, + response_decoding: ResponseDecoding::Original, + cache_policy: OutboundCachePolicy::PlatformDefault, + }); + } + + #[test] + fn permutive_api_body_read_failures_map_to_statuses() { + let settings = create_test_settings(); + let integration = PermutiveIntegration::new(readback_config()); + for case in body_read_failure_cases(INTEGRATION_MAX_BODY_BYTES) { + let clock = TestClock::new(); + let client = ScriptedHttpClient::forbidding(); + let services = build_services_with_outbound_client(&client, &clock); + let (req, expectation) = case.into_request( + http::Request::builder() + .method(http::Method::POST) + .uri("https://publisher.example/integrations/permutive/api/v2.0/events"), + ); + + let outcome = block_on(integration.handle(&settings, &services, req)); + + expectation.assert_outcome("Permutive API", &outcome); + client.assert_no_requests(); + } } } diff --git a/crates/trusted-server-core/src/integrations/prebid.rs b/crates/trusted-server-core/src/integrations/prebid.rs index c13260704f..6c5981a6da 100644 --- a/crates/trusted-server-core/src/integrations/prebid.rs +++ b/crates/trusted-server-core/src/integrations/prebid.rs @@ -13,8 +13,11 @@ use base64::{ }, }; use edgezero_core::body::Body as EdgeBody; +use edgezero_core::outbound::OutboundResponse; +#[cfg(test)] +use edgezero_core::outbound::{OutboundCachePolicy, ResponseDecoding}; use error_stack::{Report, ResultExt}; -use http::header::HeaderValue; +use http::header::{HeaderMap, HeaderValue}; use http::{Method, StatusCode, header}; use regex::Regex; use serde::{Deserialize, Serialize}; @@ -48,18 +51,17 @@ use crate::integrations::{ IntegrationRegistration, UPSTREAM_RTB_MAX_RESPONSE_BYTES, collect_response_bounded, }; #[cfg(test)] -use crate::integrations::{ - ensure_integration_backend_with_timeout, predict_integration_backend_name, -}; -#[cfg(test)] use crate::openrtb::{ Banner, ConsentedProvidersSettings, Device, Format, Geo, Imp, ImpExt, ImpStoredRequest, OpenRtbRequest, PrebidExt, PrebidImpExt, Publisher, Regs, RegsExt, RequestExt, Site, ToExt, TrustedServerExt, User, UserExt, to_openrtb_i32, }; #[cfg(test)] -use crate::platform::PlatformHttpRequest; -use crate::platform::{PlatformResponse, RuntimeServices}; +use crate::outbound::{ + DEFAULT_BUFFERED_RESPONSE_BYTES, OutboundRequestSource, buffered, outbound_error, + outbound_request, +}; +use crate::platform::RuntimeServices; use crate::proxy::{ProxyRequestConfig, is_host_allowed, proxy_request}; #[cfg(test)] use crate::request_signing::{RequestSigner, SIGNING_VERSION, SigningParams}; @@ -2023,7 +2025,7 @@ fn non_empty_override_object( #[cfg(test)] fn copy_request_headers( from: &http::Request, - to: &mut http::Request, + to: &mut HeaderMap, consent_forwarding: ConsentForwardingMode, client_ip: Option, ) { @@ -2041,7 +2043,7 @@ fn copy_request_headers( /// Applies the common browser-header and consent-cookie policy for PBS. pub(crate) fn apply_prebid_transport_headers( from: &PrebidTransportHeaders, - to: &mut http::Request, + to: &mut HeaderMap, consent_forwarding: ConsentForwardingMode, client_ip: Option, ) { @@ -2065,7 +2067,7 @@ fn apply_prebid_header_values( user_agent: Option<&HeaderValue>, referer: Option<&HeaderValue>, accept_language: Option<&HeaderValue>, - to: &mut http::Request, + to: &mut HeaderMap, consent_forwarding: ConsentForwardingMode, client_ip: Option, ) { @@ -2075,24 +2077,23 @@ fn apply_prebid_header_values( (header::ACCEPT_LANGUAGE, accept_language), ] { if let Some(value) = value { - to.headers_mut().insert(name, value.clone()); + to.insert(name, value.clone()); } } if let Some(ip) = client_ip && let Ok(value) = HeaderValue::from_str(&ip.to_string()) { - to.headers_mut() - .insert(header::HeaderName::from_static("x-forwarded-for"), value); + to.insert(header::HeaderName::from_static("x-forwarded-for"), value); } // Rebuild the destination header so no stale or arbitrary cookies survive. - to.headers_mut().remove(header::COOKIE); + to.remove(header::COOKIE); if consent_forwarding == ConsentForwardingMode::OpenrtbOnly { return; } if let Some(value) = consent_cookies.header() { - to.headers_mut().insert(header::COOKIE, value.clone()); + to.insert(header::COOKIE, value.clone()); } } @@ -2281,11 +2282,10 @@ pub(crate) async fn parse_planned_prebid_response( provider_id: &str, profile: &PrebidProfilePlan, input: &ProviderAuctionInput, - response: PlatformResponse, + response: OutboundResponse, response_time_ms: u64, auction_id: &str, ) -> Result> { - let response = response.response; let status = response.status(); let content_type = response .headers() @@ -3219,11 +3219,10 @@ impl PrebidAuctionProvider { async fn parse_response_inner( &self, - response: PlatformResponse, + response: OutboundResponse, response_time_ms: u64, auction_id: Option<&str>, ) -> Result> { - let response = response.response; let status = response.status(); let content_type = response .headers() @@ -3589,60 +3588,47 @@ impl AuctionProvider for PrebidAuctionProvider { } } - // Create HTTP request - let mut pbs_req = http::Request::builder() - .method(http::Method::POST) - .uri(self.auction_endpoint_url()) - .body(EdgeBody::empty()) - .change_context(TrustedServerError::Prebid { - message: "Failed to build Prebid request".to_string(), - })?; + let pbs_body = serde_json::to_vec(&openrtb).change_context(TrustedServerError::Prebid { + message: "Failed to serialize Prebid request body".to_string(), + })?; + let target = outbound_request(http::Method::POST, &self.auction_endpoint_url()).map_err( + |error| { + outbound_error( + error, + TrustedServerError::Prebid { + message: "Failed to build Prebid request".to_string(), + }, + OutboundRequestSource::Internal, + ) + }, + )?; + // Uses context.timeout_ms (auction-scoped) rather than the 15 s fixed + // timeout proxy endpoints use. + let mut pbs_req = buffered( + target, + Duration::from_millis(u64::from(context.timeout_ms)), + DEFAULT_BUFFERED_RESPONSE_BYTES, + ResponseDecoding::Decode, + ) + .cache_policy(OutboundCachePolicy::Bypass) + .body(pbs_body); copy_request_headers( context.request, - &mut pbs_req, + pbs_req.headers_mut(), self.config.consent_forwarding, context.services.client_info().client_ip, ); - - let pbs_body = serde_json::to_vec(&openrtb).change_context(TrustedServerError::Prebid { - message: "Failed to serialize Prebid request body".to_string(), - })?; pbs_req.headers_mut().insert( header::CONTENT_TYPE, HeaderValue::from_static("application/json"), ); - *pbs_req.body_mut() = EdgeBody::from(pbs_body); - // Uses context.timeout_ms (auction-scoped) rather than the 15 s fixed - // timeout in ensure_integration_backend, which is for proxy endpoints. - // Send request asynchronously with auction-scoped timeout - let backend_name = ensure_integration_backend_with_timeout( - context.services, - &self.config.server_url, - "prebid", - Duration::from_millis(u64::from(context.timeout_ms)), - ) - .change_context(TrustedServerError::Auction { - message: format!( - "Failed to resolve backend for Prebid Server endpoint: {}", - self.config.server_url - ), - })?; - let pending = context - .services - .http_client() - .send_async(PlatformHttpRequest::new(pbs_req, backend_name).with_cache_bypass()) - .await - .change_context(TrustedServerError::Prebid { - message: "Failed to send async request to Prebid Server".to_string(), - })?; - - Ok(ProviderRequestOutcome::pending(pending)) + Ok(ProviderRequestOutcome::ready(pbs_req)) } async fn parse_response( &self, - response: PlatformResponse, + response: OutboundResponse, response_time_ms: u64, ) -> Result> { self.parse_response_inner(response, response_time_ms, None) @@ -3651,7 +3637,7 @@ impl AuctionProvider for PrebidAuctionProvider { async fn parse_response_with_context( &self, - response: PlatformResponse, + response: OutboundResponse, response_time_ms: u64, request: &AuctionRequest, _context: &AuctionContext<'_>, @@ -3671,22 +3657,6 @@ impl AuctionProvider for PrebidAuctionProvider { fn is_enabled(&self) -> bool { self.config.enabled } - - fn backend_name(&self, services: &RuntimeServices, timeout_ms: u32) -> Option { - predict_integration_backend_name( - services, - &self.config.server_url, - PREBID_INTEGRATION_ID, - Duration::from_millis(u64::from(timeout_ms)), - ) - .inspect_err(|e| { - log::error!( - "Failed to predict backend name for Prebid server URL '{}': {e:?}", - self.config.server_url - ); - }) - .ok() - } } // ============================================================================ @@ -3752,19 +3722,25 @@ mod tests { use crate::integrations::{ AttributeRewriteAction, IntegrationDocumentState, IntegrationRegistry, }; - use crate::platform::test_support::{ - HashMapConfigStore, HashMapSecretStore, NoopConfigStore, NoopGeo, NoopHttpClient, - NoopSecretStore, StubHttpClient, build_services_with_config_secret_and_http_client, - build_services_with_http_client, build_services_with_http_client_and_client_ip, + use crate::outbound::{ + BUFFERED_TIMEOUT, DEFAULT_BUFFERED_RESPONSE_BYTES, REDIRECT_CHAIN_DEADLINE, }; - use crate::platform::{ - ClientInfo, PlatformBackend, PlatformBackendSpec, PlatformError, RuntimeServices, + use crate::platform::test_support::build_services_with_outbound_client; + use crate::platform::test_support::{ + HashMapConfigStore, HashMapSecretStore, build_services_with_config_and_secret, }; + use crate::settings::Settings; use crate::streaming_processor::{Compression, PipelineConfig, StreamingPipeline}; + use crate::test_support::scripted_client::{ + ExpectedTransport, ScriptedHttpClient, ScriptedReply, TestClock, outbound_response, + }; use crate::test_support::tests::create_test_settings; use base64::engine::general_purpose::STANDARD as TEST_BASE64_STANDARD; use bytes::Bytes; + use edgezero_core::outbound::{ + OutboundCachePolicy, OutboundRequest, ResponseDecoding, ResponseMode, + }; use http::Method; use serde_json::json; use std::collections::{BTreeMap, HashMap}; @@ -3866,63 +3842,6 @@ mod tests { } } - struct PredictOnlyBackend; - - impl PlatformBackend for PredictOnlyBackend { - fn naming_policy(&self) -> crate::platform::BackendNamingPolicy { - crate::platform::BackendNamingPolicy::Axum - } - - fn predict_name( - &self, - spec: &PlatformBackendSpec, - ) -> Result> { - Ok(format!( - "predicted_{}_{}_{}_{}", - spec.scheme, - spec.host, - spec.first_byte_timeout.as_millis(), - spec.between_bytes_timeout.as_millis() - )) - } - - fn ensure(&self, _spec: &PlatformBackendSpec) -> Result> { - Ok("unused".to_string()) - } - } - - fn services_with_backend(backend: impl PlatformBackend + 'static) -> RuntimeServices { - RuntimeServices::builder() - .config_store(Arc::new(NoopConfigStore)) - .secret_store(Arc::new(NoopSecretStore)) - .kv_store(Arc::new(edgezero_core::key_value_store::NoopKvStore)) - .backend(Arc::new(backend)) - .http_client(Arc::new(NoopHttpClient)) - .geo(Arc::new(NoopGeo)) - .client_info(ClientInfo { - client_ip: None, - tls_protocol: None, - tls_cipher: None, - ..ClientInfo::default() - }) - .build() - } - - #[test] - fn prebid_backend_name_delegates_to_platform_backend_prediction() { - let provider = PrebidAuctionProvider::new(base_config()); - let services = services_with_backend(PredictOnlyBackend); - - let backend_name = provider - .backend_name(&services, 123) - .expect("should predict backend name through platform backend"); - - assert_eq!( - backend_name, "predicted_https_prebid.example_123_123", - "should cap both first-byte and between-bytes timeouts to the auction budget" - ); - } - fn test_sri(algorithm: &str, digest: &[u8]) -> String { format!("{algorithm}-{}", TEST_BASE64_STANDARD.encode(digest)) } @@ -3961,7 +3880,7 @@ mod tests { status: StatusCode, content_type: Option<&str>, body: impl Into>, - ) -> PlatformResponse { + ) -> OutboundResponse { prebid_platform_response_with_body(status, content_type, EdgeBody::from(body.into())) } @@ -3969,13 +3888,13 @@ mod tests { status: StatusCode, content_type: Option<&str>, body: EdgeBody, - ) -> PlatformResponse { + ) -> OutboundResponse { let mut builder = http::Response::builder().status(status); if let Some(content_type) = content_type { builder = builder.header(header::CONTENT_TYPE, content_type); } - PlatformResponse::new( + outbound_response( builder .body(body) .expect("should build Prebid platform response"), @@ -4020,12 +3939,8 @@ mod tests { } #[test] - fn prebid_provider_uses_platform_http_client_for_bid_request() { - let stub = Arc::new(StubHttpClient::new()); - stub.push_response(200, br#"{"seatbid":[]}"#.to_vec()); - let services = build_services_with_http_client( - Arc::clone(&stub) as Arc - ); + fn prebid_provider_returns_an_unsent_bid_request() { + let services = crate::platform::test_support::noop_services(); let settings = make_settings(); let provider = PrebidAuctionProvider::new(base_config()); let auction_request = create_test_auction_request(); @@ -4038,7 +3953,6 @@ mod tests { settings: &settings, request: &http_req, timeout_ms: 500, - transport_timeout_ms: 500, provider_responses: None, services: &services, }; @@ -4046,28 +3960,25 @@ mod tests { let outcome = futures::executor::block_on(provider.request_bids(&auction_request, &context)) .expect("should start request"); - let ProviderRequestOutcome::Pending { request, .. } = outcome else { - panic!("should return a pending Prebid request"); + let ProviderRequestOutcome::Ready { request, .. } = outcome else { + panic!("should return an unsent Prebid request"); }; - assert!( - request.backend_name().is_some(), - "should preserve backend correlation" + assert_eq!( + (request.method(), request.uri().to_string()), + (&http::Method::POST, provider.auction_endpoint_url()), + "should build one POST to the auction endpoint" ); assert_eq!( - stub.recorded_backend_names().len(), - 1, - "should launch one upstream request through PlatformHttpClient" + request.into_parts().cache_policy, + OutboundCachePolicy::Bypass, + "should bypass caches for personalized bid traffic" ); } #[test] fn request_bids_sets_trusted_server_request_host_to_publisher_domain_not_edge_host() { - let stub = Arc::new(StubHttpClient::new()); - stub.push_response(200, br#"{"seatbid":[]}"#.to_vec()); - let services = build_services_with_http_client( - Arc::clone(&stub) as Arc - ); + let services = crate::platform::test_support::noop_services(); let settings = make_settings(); let provider = PrebidAuctionProvider::new(base_config()); let auction_request = create_test_auction_request(); @@ -4085,18 +3996,13 @@ mod tests { settings: &settings, request: &http_req, timeout_ms: 500, - transport_timeout_ms: 500, provider_responses: None, services: &services, }; - futures::executor::block_on(provider.request_bids(&auction_request, &context)) - .expect("should start request"); - - let bodies = stub.recorded_request_bodies(); - assert_eq!(bodies.len(), 1, "should send one upstream PBS request"); - let sent: Json = - serde_json::from_slice(&bodies[0]).expect("should parse sent OpenRTB request body"); + let request = unsent_bid_request(&provider, &auction_request, &context); + let sent: Json = serde_json::from_slice(&request_body(request)) + .expect("should parse sent OpenRTB request body"); let request_host = sent["ext"]["trusted_server"]["request_host"] .as_str() .expect("should set ext.trusted_server.request_host"); @@ -4106,6 +4012,29 @@ mod tests { ); } + /// Runs the provider's `request_bids` and returns the request it built, unsent. + fn unsent_bid_request( + provider: &PrebidAuctionProvider, + request: &AuctionRequest, + context: &AuctionContext<'_>, + ) -> OutboundRequest { + let outcome = futures::executor::block_on(provider.request_bids(request, context)) + .expect("should build the PBS request"); + let ProviderRequestOutcome::Ready { request, .. } = outcome else { + panic!("should return an unsent PBS request"); + }; + request + } + + /// The buffered body of an unsent request. + fn request_body(request: OutboundRequest) -> Bytes { + request + .into_parts() + .body + .into_bytes() + .expect("should buffer the PBS request body") + } + fn create_test_auction_context<'a>( settings: &'a Settings, request: &'a http::Request, @@ -5305,7 +5234,7 @@ external_bundle_sri = "sha384-AAAA" ); } #[test] - fn external_bundle_handler_fetches_and_sanitizes_with_platform_client() { + fn external_bundle_handler_fetches_and_sanitizes_the_bundle() { futures::executor::block_on(async { let sha256 = "a".repeat(64); let mut config = base_config(); @@ -5314,20 +5243,19 @@ external_bundle_sri = "sha384-AAAA" let mut settings = make_settings(); settings.proxy.allowed_domains = vec!["assets.example".to_string()]; - let stub = Arc::new(StubHttpClient::new()); - stub.push_response_with_headers( - 200, - b"console.log('bundle');".to_vec(), - vec![ - (header::CONTENT_TYPE.as_str(), "text/html"), - (header::CACHE_CONTROL.as_str(), "private, max-age=0"), - (header::SET_COOKIE.as_str(), "bad=1; Path=/"), - ("x-upstream", "leak"), - ], - ); - let services = build_services_with_http_client( - Arc::clone(&stub) as Arc + let clock = TestClock::new(); + let upstream = ScriptedHttpClient::new(&clock); + upstream.respond_to( + Method::GET, + "https://assets.example/prebid/trusted-prebid.js", + ScriptedReply::status(StatusCode::OK) + .header(header::CONTENT_TYPE.as_str(), "text/html") + .header(header::CACHE_CONTROL.as_str(), "private, max-age=0") + .header(header::SET_COOKIE.as_str(), "bad=1; Path=/") + .header("x-upstream", "leak") + .body("console.log('bundle');"), ); + let services = build_services_with_outbound_client(&upstream, &clock); let req = http::Request::builder() .method(Method::GET) .uri(format!( @@ -5373,28 +5301,28 @@ external_bundle_sri = "sha384-AAAA" "should preserve bundle bytes" ); + let requests = upstream.requests(); + assert_eq!(requests.len(), 1, "should make one upstream request"); + let sent = &requests[0]; assert_eq!( - stub.recorded_request_uris(), - vec!["https://assets.example/prebid/trusted-prebid.js".to_string()], + sent.uri.to_string(), + "https://assets.example/prebid/trusted-prebid.js", "should fetch the configured external bundle URL without adding EC query params" ); - let recorded_headers = stub.recorded_request_headers(); - assert_eq!( - recorded_headers.len(), - 1, - "should make one upstream request" - ); - assert!( - !recorded_headers[0] - .iter() - .any(|(name, _)| name.eq_ignore_ascii_case(header::COOKIE.as_str())), - "external bundle fetch should not forward client cookies" - ); - assert!( - !recorded_headers[0] - .iter() - .any(|(name, _)| name.eq_ignore_ascii_case(header::ACCEPT.as_str())), - "external bundle fetch should not forward client headers" + // Neither the client's cookies nor its other headers reach the bundle host. + sent.assert_headers(&[]); + sent.assert_followed_transport( + ExpectedTransport { + timeout: BUFFERED_TIMEOUT, + response_mode: ResponseMode::Buffered { + max_bytes: DEFAULT_BUFFERED_RESPONSE_BYTES, + }, + response_decoding: ResponseDecoding::Original, + cache_policy: OutboundCachePolicy::PlatformDefault, + }, + clock + .monotonic_clock() + .deadline_after(REDIRECT_CHAIN_DEADLINE), ); }); } @@ -7590,7 +7518,6 @@ external_bundle_sri = "sha384-AAAA" settings: &settings, request: &http_req, timeout_ms: 1000, - transport_timeout_ms: 1000, provider_responses: None, services: &services, }; @@ -9581,7 +9508,7 @@ set = { networkId = 42 } ])), ), ] { - let expected_status = response.response.status().as_u16(); + let expected_status = response.status().as_u16(); let parsed = futures::executor::block_on(parse_planned_prebid_response( "pbs-instance", &profile, @@ -9918,7 +9845,7 @@ set = { networkId = 42 } apply_prebid_transport_headers( routed.prebid_transport_headers(), - &mut outbound, + outbound.headers_mut(), ConsentForwardingMode::OpenrtbOnly, routed.attested_client_ip(), ); @@ -10001,7 +9928,7 @@ set = { networkId = 42 } .headers_mut() .append(header::COOKIE, HeaderValue::from_static("another=secret")); - copy_request_headers(&inbound, &mut outbound, mode, None); + copy_request_headers(&inbound, outbound.headers_mut(), mode, None); let expected = if mode == ConsentForwardingMode::OpenrtbOnly { None @@ -10033,7 +9960,7 @@ set = { networkId = 42 } ); let mut outbound = http::Request::new(EdgeBody::empty()); - copy_request_headers(&inbound, &mut outbound, mode, None); + copy_request_headers(&inbound, outbound.headers_mut(), mode, None); assert!( !outbound.headers().contains_key(header::COOKIE), @@ -10151,7 +10078,7 @@ set = { networkId = 42 } copy_request_headers( &from, - &mut to, + to.headers_mut(), ConsentForwardingMode::Both, Some(std::net::IpAddr::from([203, 0, 113, 7])), ); @@ -10184,7 +10111,7 @@ set = { networkId = 42 } .body(EdgeBody::empty()) .expect("should build outbound request"); - copy_request_headers(&from, &mut to, ConsentForwardingMode::Both, None); + copy_request_headers(&from, to.headers_mut(), ConsentForwardingMode::Both, None); assert!( !to.headers().contains_key("x-forwarded-for"), @@ -10229,10 +10156,7 @@ set = { networkId = 42 } None, ), ] { - let stub = Arc::new(StubHttpClient::new()); - stub.push_response(204, Vec::new()); - let services = build_services_with_http_client_and_client_ip( - Arc::clone(&stub) as Arc, + let services = crate::platform::test_support::noop_services_with_client_ip( std::net::IpAddr::from([203, 0, 113, 9]), ); let settings = make_settings(); @@ -10256,17 +10180,17 @@ set = { networkId = 42 } settings: &settings, request: &inbound, timeout_ms: 321, - transport_timeout_ms: 321, provider_responses: None, services: &services, }; - futures::executor::block_on(provider.request_bids(&auction_request, &context)) - .expect("should dispatch PBS request"); - - let headers = stub.recorded_request_header_bytes(); - assert_eq!(headers.len(), 1, "should dispatch one request"); - let headers = &headers[0]; + let request = unsent_bid_request(&provider, &auction_request, &context); + let headers = request + .headers() + .iter() + .map(|(name, value)| (name.as_str().to_owned(), value.as_bytes().to_vec())) + .collect::>(); + let headers = &headers; assert_eq!( header_value(headers, "referer"), Some(b"https://referrer.example/story?fictional=1".as_slice()) @@ -10286,12 +10210,8 @@ set = { networkId = 42 } ); assert_eq!(header_value(headers, "cookie"), expected_cookie); - let body = stub - .recorded_request_bodies() - .into_iter() - .next() - .expect("should capture PBS request body"); - let body: Json = serde_json::from_slice(&body).expect("should parse PBS request body"); + let body: Json = serde_json::from_slice(&request_body(request)) + .expect("should parse PBS request body"); assert_eq!( body["site"]["ref"], "https://referrer.example/story?fictional=1", "should place the raw Referer in PBS site.ref as well as forwarding it" @@ -10318,7 +10238,6 @@ set = { networkId = 42 } settings: &settings, request: &inbound, timeout_ms: 321, - transport_timeout_ms: 321, provider_responses: None, services: &services, }; @@ -10351,10 +10270,9 @@ set = { networkId = 42 } .encode([7_u8; 32]) .into_bytes(), ); - let signing_services = build_services_with_config_secret_and_http_client( + let signing_services = build_services_with_config_and_secret( HashMapConfigStore::new(config_data), HashMapSecretStore::new(secret_data), - Arc::new(NoopHttpClient), ); let signer = RequestSigner::from_services(&signing_services) .expect("should load deterministic test signer"); diff --git a/crates/trusted-server-core/src/integrations/registry.rs b/crates/trusted-server-core/src/integrations/registry.rs index 15a6486fd8..2936fa8a1b 100644 --- a/crates/trusted-server-core/src/integrations/registry.rs +++ b/crates/trusted-server-core/src/integrations/registry.rs @@ -260,9 +260,10 @@ impl IntegrationEndpoint { /// /// `Send + Sync` bounds are required so trait objects can be stored in /// `Arc` and shared across the single-threaded WASM -/// request context. The `?Send` on the async methods is intentional — see the -/// `!Send` design rationale on [`crate::platform::PlatformPendingRequest`] for -/// the full explanation. On wasm32 these bounds are compatible because the runtime is +/// request context. The `?Send` on the async methods is intentional: +/// [`edgezero_core::body::Body`] wraps a `LocalBoxStream`, which is `!Send` for +/// wasm32 compatibility, so the futures that carry request bodies cannot be +/// `Send`. On wasm32 these bounds are compatible because the runtime is /// single-threaded. #[async_trait(?Send)] pub trait IntegrationProxy: Send + Sync { diff --git a/crates/trusted-server-core/src/integrations/sourcepoint.rs b/crates/trusted-server-core/src/integrations/sourcepoint.rs index d9b18a6f3d..f0ad3e05e0 100644 --- a/crates/trusted-server-core/src/integrations/sourcepoint.rs +++ b/crates/trusted-server-core/src/integrations/sourcepoint.rs @@ -24,8 +24,9 @@ use std::sync::{Arc, LazyLock}; use async_trait::async_trait; use edgezero_core::body::Body as EdgeBody; +use edgezero_core::outbound::OutboundResponse; use error_stack::{Report, ResultExt}; -use http::header::{self, HeaderValue}; +use http::header::{self, HeaderMap, HeaderValue}; use http::{Method, Request, Response, StatusCode}; use regex::Regex; use serde::Deserialize; @@ -33,13 +34,16 @@ use url::Url; use validator::{Validate, ValidationError}; use crate::error::TrustedServerError; +use crate::http_util::read_request_body; use crate::integrations::{ AttributeRewriteAction, INTEGRATION_MAX_BODY_BYTES, IntegrationAttributeContext, IntegrationAttributeRewriter, IntegrationEndpoint, IntegrationHeadInjector, - IntegrationHtmlContext, IntegrationProxy, IntegrationRegistration, collect_body_bounded, - collect_response_bounded, ensure_integration_backend, + IntegrationHtmlContext, IntegrationProxy, IntegrationRegistration, collect_response_bounded, }; -use crate::platform::{PlatformHttpRequest, RuntimeServices}; +use crate::outbound::{ + OutboundRequestSource, STREAMED_TIMEOUT, outbound_error, outbound_request, streamed, +}; +use crate::platform::RuntimeServices; use crate::settings::{IntegrationConfig, Settings}; const SOURCEPOINT_INTEGRATION_ID: &str = "sourcepoint"; @@ -368,12 +372,12 @@ impl SourcepointIntegration { &self, client_ip: Option, original_req: &Request, - proxy_req: &mut Request, + proxy_headers: &mut HeaderMap, ) -> bool { if let Some(client_ip) = client_ip && let Ok(val) = HeaderValue::from_str(&client_ip.to_string()) { - proxy_req.headers_mut().insert("x-forwarded-for", val); + proxy_headers.insert("x-forwarded-for", val); } // Accept-Encoding is deliberately omitted here and handled in the @@ -393,14 +397,14 @@ impl SourcepointIntegration { header::HeaderName::from_static("access-control-request-headers"), ] { if let Some(value) = original_req.headers().get(&header_name) { - proxy_req.headers_mut().insert(header_name, value.clone()); + proxy_headers.insert(header_name, value.clone()); } } if let Some(filtered_cookie_header) = self.filtered_sourcepoint_cookie_header(original_req) { if let Ok(val) = HeaderValue::from_str(&filtered_cookie_header) { - proxy_req.headers_mut().insert(header::COOKIE, val); + proxy_headers.insert(header::COOKIE, val); } return true; } @@ -836,26 +840,29 @@ impl IntegrationProxy for SourcepointIntegration { let (req_parts, req_body) = req.into_parts(); let request_body = if method == Method::POST { - let bytes = collect_body_bounded( - req_body, - INTEGRATION_MAX_BODY_BYTES, - SOURCEPOINT_INTEGRATION_ID, - ) - .await?; + let bytes = + read_request_body(&req_parts.headers, req_body, INTEGRATION_MAX_BODY_BYTES).await?; EdgeBody::from(bytes) } else { EdgeBody::empty() }; - let mut proxy_req = http::Request::builder() - .method(method.clone()) - .uri(&target_url) - .body(request_body) - .change_context(Self::error("Failed to build Sourcepoint proxy request"))?; + let mut proxy_req = outbound_request(method.clone(), &target_url) + .map_err(|error| { + outbound_error( + error, + Self::error("Failed to build Sourcepoint proxy request"), + OutboundRequestSource::Internal, + ) + })? + .body(request_body); let source_req = http::Request::from_parts(req_parts, EdgeBody::empty()); - let forwarded_cookies = - self.copy_headers(services.client_info.client_ip, &source_req, &mut proxy_req); + let forwarded_cookies = self.copy_headers( + services.client_info.client_ip, + &source_req, + proxy_req.headers_mut(), + ); // Request uncompressed content only for paths that are likely // JavaScript or HTML (the files we need to regex-rewrite). All other CDN @@ -883,30 +890,25 @@ impl IntegrationProxy for SourcepointIntegration { .insert(header::CONTENT_TYPE, content_type.clone()); } - let backend_name = ensure_integration_backend( - services, - &self.config.cdn_origin, - SOURCEPOINT_INTEGRATION_ID, - None, - )?; - let requested_identity_encoding = proxy_req .headers() .get(header::ACCEPT_ENCODING) .and_then(|value| value.to_str().ok()) .is_some_and(|encoding| encoding.trim().eq_ignore_ascii_case("identity")); - // Keep the body streaming where supported so the rewrite collector - // enforces its limit before the adapter buffers the entire response. - let mut platform_request = PlatformHttpRequest::new(proxy_req, backend_name); - if services.http_client().supports_streaming_responses() { - platform_request = platform_request.with_stream_response(); - } + // Stream the body, so the rewrite collector enforces its limit before + // anything buffers the entire response. let mut response = services - .http_client() - .send(platform_request) + .outbound_client() + .send(streamed(proxy_req, STREAMED_TIMEOUT)) .await - .change_context(Self::error("Sourcepoint upstream request failed"))? - .response; + .and_then(OutboundResponse::into_response) + .map_err(|error| { + outbound_error( + error, + Self::error("Sourcepoint upstream request failed"), + OutboundRequestSource::Internal, + ) + })?; log::info!( "Sourcepoint: upstream responded with status {}", @@ -1144,74 +1146,215 @@ impl IntegrationHeadInjector for SourcepointIntegration { #[cfg(test)] mod tests { + use std::sync::atomic::{AtomicUsize, Ordering}; + + use bytes::Bytes; + use edgezero_core::outbound::{OutboundCachePolicy, ResponseDecoding, ResponseMode}; + use serde_json::json; + use super::*; use crate::error::IntoHttpResponse as _; use crate::integrations::{IntegrationDocumentState, IntegrationRegistry}; - use crate::platform::test_support::{StubHttpClient, build_services_with_http_client}; - use crate::platform::{ - PlatformError, PlatformHttpClient, PlatformPendingRequest, PlatformResponse, - PlatformSelectResult, + use crate::outbound::STREAMED_TIMEOUT; + use crate::platform::test_support::build_services_with_outbound_client; + use crate::test_support::body_read::body_read_failure_cases; + use crate::test_support::scripted_client::{ + ExpectedTransport, RecordedRequest, ScriptedHttpClient, ScriptedReply, TestClock, }; use crate::test_support::tests::create_test_settings; - use serde_json::json; - use std::sync::atomic::{AtomicUsize, Ordering}; const TEST_CHUNK_SIZE: usize = 8192; - struct StreamingHttpClient { - stub: StubHttpClient, + /// A scripted Sourcepoint CDN that counts how much of each reply body is read. + struct Cdn { + client: ScriptedHttpClient, + clock: TestClock, reads: Arc, } - impl StreamingHttpClient { + impl Cdn { fn new() -> Self { + let clock = TestClock::new(); Self { - stub: StubHttpClient::new(), - reads: Arc::new(AtomicUsize::new(0)), + client: ScriptedHttpClient::new(&clock), + clock, + reads: Arc::default(), } } - } - #[async_trait(?Send)] - impl PlatformHttpClient for StreamingHttpClient { - fn supports_streaming_responses(&self) -> bool { - true + /// Answers one `method` request to the CDN `path` with `status`, `headers`, and + /// `body` delivered in [`TEST_CHUNK_SIZE`] chunks. + fn respond( + &self, + method: Method, + path: &str, + status: u16, + body: Vec, + headers: &[(&str, &str)], + ) { + let body = Bytes::from(body); + let chunks = (0..body.len()) + .step_by(TEST_CHUNK_SIZE) + .map(|offset| body.slice(offset..(offset + TEST_CHUNK_SIZE).min(body.len()))) + .collect(); + let status = StatusCode::from_u16(status).expect("should use a valid test status"); + let reply = headers.iter().fold( + ScriptedReply::status(status) + .chunks(chunks) + .counting_body_polls(&self.reads), + |reply, (name, value)| reply.header(name, value), + ); + self.client.respond_to( + method, + &format!("https://{SOURCEPOINT_CDN_HOST}{path}"), + reply, + ); } - async fn send( - &self, - request: PlatformHttpRequest, - ) -> Result> { - let mut response = self.stub.send(request).await?; - let body = std::mem::replace(response.response.body_mut(), EdgeBody::empty()); - let EdgeBody::Once(bytes) = body else { - panic!("should receive a buffered stub body"); - }; - let reads = Arc::clone(&self.reads); - let chunks = futures::stream::unfold((bytes, 0), move |(bytes, offset)| { - reads.fetch_add(1, Ordering::Relaxed); - let end = (offset + TEST_CHUNK_SIZE).min(bytes.len()); - futures::future::ready( - (offset < bytes.len()).then(|| (bytes.slice(offset..end), (bytes, end))), - ) - }); - *response.response.body_mut() = EdgeBody::stream(chunks); - Ok(response) + fn services(&self) -> RuntimeServices { + build_services_with_outbound_client(&self.client, &self.clock) } - async fn send_async( - &self, - request: PlatformHttpRequest, - ) -> Result> { - self.stub.send_async(request).await + /// Polls of every reply body so far, end-of-stream polls included. + fn reads(&self) -> usize { + self.reads.load(Ordering::SeqCst) } - async fn select( - &self, - pending_requests: Vec, - ) -> Result> { - self.stub.select(pending_requests).await + fn requests(&self) -> Vec { + self.client.requests() + } + } + + #[test] + fn sourcepoint_request_readback() { + let cdn = Cdn::new(); + cdn.respond( + Method::GET, + "/wrapper.js?v=1", + 200, + b"var x = 1;".to_vec(), + &[("content-type", "application/javascript")], + ); + cdn.respond( + Method::POST, + "/consent/tcfv2", + 200, + b"{}".to_vec(), + &[("content-type", "application/json")], + ); + let mut services = cdn.services(); + services.client_info.client_ip = + Some("203.0.113.10".parse().expect("should parse test IP")); + let settings = create_test_settings(); + let integration = SourcepointIntegration::new(Arc::new(config(true))); + let browser_headers = [ + (header::ACCEPT, "*/*"), + (header::ACCEPT_LANGUAGE, "en"), + (header::USER_AGENT, "example-agent"), + (header::REFERER, "https://publisher.example.com/page"), + (header::ORIGIN, "https://publisher.example.com"), + (header::COOKIE, "consentUUID=example; session=secret"), + (header::AUTHORIZATION, "Bearer publisher-token"), + ]; + let mut get = make_req( + Method::GET, + "https://publisher.example.com/integrations/sourcepoint/cdn/wrapper.js?v=1", + ); + set_req_header(&mut get, header::ACCEPT_ENCODING, "gzip, br"); + let mut post = http::Request::builder() + .method(Method::POST) + .uri("https://publisher.example.com/integrations/sourcepoint/cdn/consent/tcfv2") + .header(header::CONTENT_TYPE, "application/json") + .header(header::CONTENT_LENGTH, "7") + .body(EdgeBody::from(r#"{"a":1}"#)) + .expect("should build POST request"); + for request in [&mut get, &mut post] { + for (name, value) in &browser_headers { + set_req_header(request, name.clone(), value); + } } + + futures::executor::block_on(async { + for request in [get, post] { + let response = integration + .handle(&settings, &services, request) + .await + .expect("should proxy the Sourcepoint request"); + assert_eq!(response.status(), StatusCode::OK, "should relay the status"); + } + }); + + let requests = cdn.requests(); + assert_eq!(requests.len(), 2, "should send the GET and the POST"); + let shared = [ + ("x-forwarded-for", "203.0.113.10"), + ("accept", "*/*"), + ("accept-language", "en"), + ("user-agent", "example-agent"), + ("referer", "https://publisher.example.com/page"), + ("origin", "https://publisher.example.com"), + ("cookie", "consentUUID=example"), + ]; + let (get, post) = (&requests[0], &requests[1]); + assert_eq!( + (&get.method, get.uri.to_string()), + ( + &Method::GET, + format!("https://{SOURCEPOINT_CDN_HOST}/wrapper.js?v=1") + ), + "should fetch the CDN path and query" + ); + get.assert_headers(&[shared.as_slice(), &[("accept-encoding", "identity")]].concat()); + assert_eq!( + (&post.method, post.uri.to_string(), post.body.as_ref()), + ( + &Method::POST, + format!("https://{SOURCEPOINT_CDN_HOST}/consent/tcfv2"), + br#"{"a":1}"#.as_slice() + ), + "should forward the POST path and body" + ); + post.assert_headers(&[shared.as_slice(), &[("content-type", "application/json")]].concat()); + for request in [get, post] { + request.assert_transport(ExpectedTransport { + timeout: STREAMED_TIMEOUT, + response_mode: ResponseMode::Streamed, + response_decoding: ResponseDecoding::Original, + cache_policy: OutboundCachePolicy::PlatformDefault, + }); + } + } + + #[test] + fn sourcepoint_drops_non_utf8_request_header() { + let cdn = Cdn::new(); + cdn.respond( + Method::GET, + "/asset", + 200, + b"unchanged".to_vec(), + &[("content-type", "image/png")], + ); + let settings = create_test_settings(); + let integration = SourcepointIntegration::new(Arc::new(config(true))); + let mut request = make_req( + Method::GET, + "https://publisher.example.com/integrations/sourcepoint/cdn/asset", + ); + request.headers_mut().insert( + header::USER_AGENT, + HeaderValue::from_bytes(b"example-agent/\xff").expect("should build obs-text value"), + ); + set_req_header(&mut request, header::ACCEPT, "image/png"); + + let response = + futures::executor::block_on(integration.handle(&settings, &cdn.services(), request)) + .expect("should proxy despite a header value that is not UTF-8"); + + assert_eq!(response.status(), StatusCode::OK, "should relay the status"); + let requests = cdn.requests(); + assert_eq!(requests.len(), 1, "should send one CDN request"); + requests[0].assert_headers(&[("accept", "image/png")]); } #[test] @@ -1219,19 +1362,20 @@ mod tests { futures::executor::block_on(async { let settings = create_test_settings(); let integration = SourcepointIntegration::new(Arc::new(config(true))); - let client = Arc::new(StreamingHttpClient::new()); + let cdn = Cdn::new(); let input = format!(r#"var api="https://{SOURCEPOINT_CDN_HOST}/consent/tcfv2";"#); - client.stub.push_response_with_headers( + cdn.respond( + Method::GET, + "/wrapper.js", 200, input.into_bytes(), - vec![("content-type", "application/javascript")], + &[("content-type", "application/javascript")], ); - let services = build_services_with_http_client(client.clone()); let response = integration .handle( &settings, - &services, + &cdn.services(), make_req( Method::GET, "https://publisher.example.com/integrations/sourcepoint/cdn/wrapper.js", @@ -1251,8 +1395,8 @@ mod tests { "should rewrite a streamed CDN URL without Content-Length" ); assert_eq!( - client.stub.recorded_stream_response_flags(), - vec![true], + cdn.requests()[0].response_mode, + ResponseMode::Streamed, "should request streaming before collecting the upstream body" ); }); @@ -1276,7 +1420,7 @@ mod tests { vec!["gzip", "identity"], ] { for oversized in [false, true] { - let client = Arc::new(StreamingHttpClient::new()); + let cdn = Cdn::new(); // Valid UTF-8 with a rewrite target must still pass through // when the encoding header does not identify plain text. let mut bytes = @@ -1291,15 +1435,12 @@ mod tests { ("vary", "Accept-Encoding, Origin"), ]; headers.extend(encodings.iter().map(|value| ("content-encoding", *value))); - client - .stub - .push_response_with_headers(200, bytes.clone(), headers); - let services = build_services_with_http_client(client.clone()); + cdn.respond(Method::GET, "/asset", 200, bytes.clone(), &headers); let response = integration .handle( &settings, - &services, + &cdn.services(), make_req( Method::GET, "https://publisher.example.com/integrations/sourcepoint/cdn/asset", @@ -1309,11 +1450,11 @@ mod tests { .expect("should pass through encoded responses regardless of size"); assert!( - matches!(response.body(), EdgeBody::Stream(_)), + response.body().is_stream(), "should leave encoded response bodies streaming" ); assert_eq!( - client.reads.load(Ordering::Relaxed), + cdn.reads(), 0, "should not read encoded bodies for rewriting" ); @@ -1362,7 +1503,7 @@ mod tests { let integration = SourcepointIntegration::new(Arc::new(config(true))); for content_type in ["application/javascript", "text/html"] { for encoding in ["identity", " IdEnTiTy "] { - let client = Arc::new(StreamingHttpClient::new()); + let cdn = Cdn::new(); let (input, expected): (Vec, &[u8]) = if content_type == "text/html" { ( br#""#.to_vec(), @@ -1375,19 +1516,20 @@ mod tests { br#"var api="/integrations/sourcepoint/cdn/consent/tcfv2";"#, ) }; - client.stub.push_response_with_headers( + cdn.respond( + Method::GET, + "/asset", 200, input, - vec![ + &[ ("content-type", content_type), ("content-encoding", encoding), ], ); - let services = build_services_with_http_client(client.clone()); let response = integration .handle( &settings, - &services, + &cdn.services(), make_req( Method::GET, "https://publisher.example.com/integrations/sourcepoint/cdn/asset", @@ -1401,7 +1543,7 @@ mod tests { "should drop {encoding:?} Content-Encoding after rewriting" ); assert!( - client.reads.load(Ordering::Relaxed) > 0, + cdn.reads() > 0, "should collect {encoding:?} bodies for rewriting" ); assert_eq!( @@ -1419,18 +1561,19 @@ mod tests { futures::executor::block_on(async { let settings = create_test_settings(); let integration = SourcepointIntegration::new(Arc::new(config(true))); - let client = Arc::new(StreamingHttpClient::new()); - client.stub.push_response_with_headers( + let cdn = Cdn::new(); + cdn.respond( + Method::GET, + "/us_pm/index.html", 200, br#""#.to_vec(), - vec![("content-type", "text/html"), ("cache-control", "no-store")], + &[("content-type", "text/html"), ("cache-control", "no-store")], ); - let services = build_services_with_http_client(client.clone()); let response = integration .handle( &settings, - &services, + &cdn.services(), make_req(Method::GET, "https://publisher.example.com/integrations/sourcepoint/cdn/us_pm/index.html"), ) .await @@ -1458,21 +1601,22 @@ mod tests { for content_type in ["application/javascript", "text/html"] { for declared_length in [None, Some("1")] { for extra_bytes in [0, 1, TEST_CHUNK_SIZE * 2] { - let client = Arc::new(StreamingHttpClient::new()); + let cdn = Cdn::new(); let mut headers = vec![("content-type", content_type)]; if let Some(length) = declared_length { headers.push(("content-length", length)); } - client.stub.push_response_with_headers( + cdn.respond( + Method::GET, + "/asset", 200, vec![b' '; limit + extra_bytes], - headers, + &headers, ); - let services = build_services_with_http_client(client.clone()); let result = integration.handle( &settings, - &services, + &cdn.services(), make_req(Method::GET, "https://publisher.example.com/integrations/sourcepoint/cdn/asset"), ).await; @@ -1501,7 +1645,7 @@ mod tests { ); } assert_eq!( - client.reads.load(Ordering::Relaxed), + cdn.reads(), limit / TEST_CHUNK_SIZE + 1, "should stop at EOF or the first overflowing chunk without draining the stream" ); @@ -1517,23 +1661,24 @@ mod tests { let settings = create_test_settings(); let integration = SourcepointIntegration::new(Arc::new(config(true))); for content_type in ["application/javascript", "text/html"] { - let client = Arc::new(StreamingHttpClient::new()); + let cdn = Cdn::new(); let length = (MAX_REWRITE_BODY_SIZE + 1).to_string(); - client.stub.push_response_with_headers( + cdn.respond( + Method::GET, + "/asset", 200, vec![b' '; MAX_REWRITE_BODY_SIZE as usize + 1], - vec![ + &[ ("content-type", content_type), ("content-length", &length), ("cache-control", "no-store"), ], ); - let services = build_services_with_http_client(client.clone()); let response = integration .handle( &settings, - &services, + &cdn.services(), make_req( Method::GET, "https://publisher.example.com/integrations/sourcepoint/cdn/asset", @@ -1543,14 +1688,10 @@ mod tests { .expect("should pass through a declared oversized response"); assert!( - matches!(response.body(), EdgeBody::Stream(_)), + response.body().is_stream(), "should retain the original stream" ); - assert_eq!( - client.reads.load(Ordering::Relaxed), - 0, - "should not poll a declared oversized body" - ); + assert_eq!(cdn.reads(), 0, "should not poll a declared oversized body"); assert_eq!( get_header_str(&response, header::CONTENT_LENGTH), Some(length.as_str()), @@ -1583,42 +1724,42 @@ mod tests { let mut cfg = config(true); cfg.rewrite_sdk = rewrite_sdk; let integration = SourcepointIntegration::new(Arc::new(cfg)); - let client = Arc::new(StreamingHttpClient::new()); - client.stub.push_response_with_headers( + let cdn = Cdn::new(); + cdn.respond( + method.clone(), + "/asset", status, b"unchanged".to_vec(), - vec![ + &[ ("content-type", content_type), ("content-encoding", "identity"), ("cache-control", "no-store"), ], ); - let services = build_services_with_http_client(client.clone()); let expected_body: &[u8] = if method == Method::HEAD { b"" } else { b"unchanged" }; let mut request = make_req( - method, + method.clone(), "https://publisher.example.com/integrations/sourcepoint/cdn/asset", ); set_req_header(&mut request, header::ACCEPT_ENCODING, "gzip, br"); let response = integration - .handle(&settings, &services, request) + .handle(&settings, &cdn.services(), request) .await .expect("should pass through an ineligible response"); - assert!( - matches!(response.body(), EdgeBody::Stream(_)), - "should leave ineligible bodies streaming" - ); - assert_eq!( - client.reads.load(Ordering::Relaxed), - 0, - "should not poll an ineligible body" - ); + // EdgeZero drops a HEAD body before the response reaches the proxy. + if method != Method::HEAD { + assert!( + response.body().is_stream(), + "should leave ineligible bodies streaming" + ); + } + assert_eq!(cdn.reads(), 0, "should not poll an ineligible body"); assert_eq!( get_header_str(&response, header::CONTENT_ENCODING), Some("identity"), @@ -1639,10 +1780,12 @@ mod tests { expected_body, "should preserve pass-through bytes or omit them for HEAD" ); - assert!( - client.stub.recorded_request_headers()[0] - .iter() - .any(|(name, value)| name == "accept-encoding" && value == "gzip, br"), + assert_eq!( + cdn.requests()[0] + .headers + .get(header::ACCEPT_ENCODING) + .and_then(|value| value.to_str().ok()), + Some("gzip, br"), "should forward the client's encoding for non-script paths" ); } @@ -1650,12 +1793,12 @@ mod tests { } #[test] - fn handle_rewrites_on_buffered_adapters_with_or_without_content_length() { + fn handle_rewrite_drops_stale_length_encoding_and_vary_accept_encoding() { futures::executor::block_on(async { let settings = create_test_settings(); let integration = SourcepointIntegration::new(Arc::new(config(true))); for has_length in [false, true] { - let client = Arc::new(StubHttpClient::new()); + let cdn = Cdn::new(); let input = format!(r#"var api="https://{SOURCEPOINT_CDN_HOST}/consent/tcfv2";"#); let length = input.len().to_string(); let mut headers = vec![ @@ -1666,26 +1809,26 @@ mod tests { if has_length { headers.push(("content-length", &length)); } - client.push_response_with_headers(200, input.into_bytes(), headers); - let services = build_services_with_http_client(client.clone()); + cdn.respond( + Method::GET, + "/wrapper.js", + 200, + input.into_bytes(), + &headers, + ); let response = integration .handle( &settings, - &services, + &cdn.services(), make_req( Method::GET, "https://publisher.example.com/integrations/sourcepoint/cdn/wrapper.js", ), ) .await - .expect("should rewrite a buffered response"); + .expect("should rewrite the response"); - assert_eq!( - client.recorded_stream_response_flags(), - vec![false], - "should respect adapters without streaming support" - ); assert!( response.headers().get(header::CONTENT_LENGTH).is_none(), "should not forward a stale upstream length" @@ -1718,46 +1861,6 @@ mod tests { }); } - #[test] - fn handle_rejects_buffered_overflow_without_content_length() { - futures::executor::block_on(async { - let settings = create_test_settings(); - let integration = SourcepointIntegration::new(Arc::new(config(true))); - for content_type in ["application/javascript", "text/html"] { - let client = Arc::new(StubHttpClient::new()); - client.push_response_with_headers( - 200, - vec![b' '; MAX_REWRITE_BODY_SIZE as usize + 1], - vec![("content-type", content_type)], - ); - let services = build_services_with_http_client(client.clone()); - - let error = integration - .handle( - &settings, - &services, - make_req( - Method::GET, - "https://publisher.example.com/integrations/sourcepoint/cdn/asset", - ), - ) - .await - .expect_err("should reject a buffered body over the rewrite limit"); - - assert_eq!( - client.recorded_stream_response_flags(), - vec![false], - "should exercise the buffered adapter path" - ); - assert_eq!( - error.current_context().status_code(), - StatusCode::BAD_GATEWAY, - "should return 502 for buffered {content_type} overflow" - ); - } - }); - } - #[test] fn handle_javascript_cache_policy_follows_static_path_classification() { futures::executor::block_on(async { @@ -1789,7 +1892,7 @@ mod tests { ), ] { for has_length in [false, true] { - let client = Arc::new(StreamingHttpClient::new()); + let cdn = Cdn::new(); let input = format!(r#"var api="https://{SOURCEPOINT_CDN_HOST}/consent/tcfv2";"#); let length = input.len().to_string(); @@ -1803,10 +1906,8 @@ mod tests { if sets_cookie { headers.push(("set-cookie", "consentUUID=example; Path=/")); } - client - .stub - .push_response_with_headers(200, input.into_bytes(), headers); - let services = build_services_with_http_client(client.clone()); + cdn.respond(Method::GET, path, 200, input.into_bytes(), &headers); + let services = cdn.services(); let mut request = make_req( Method::GET, &format!("https://publisher.example.com{SOURCEPOINT_CDN_PREFIX}{path}"), @@ -1864,7 +1965,7 @@ mod tests { "private, no-store", ), ] { - let client = Arc::new(StreamingHttpClient::new()); + let cdn = Cdn::new(); let input = format!(r#"var api="https://{SOURCEPOINT_CDN_HOST}/consent/tcfv2";"#); let mut headers = vec![("content-type", "application/javascript")]; if let Some(cache) = upstream_cache { @@ -1873,10 +1974,14 @@ mod tests { if sets_cookie { headers.push(("set-cookie", "consentUUID=example; Path=/")); } - client - .stub - .push_response_with_headers(200, input.into_bytes(), headers); - let services = build_services_with_http_client(client.clone()); + cdn.respond( + Method::GET, + "/mms/v2/get_site_data?account_id=123", + 200, + input.into_bytes(), + &headers, + ); + let services = cdn.services(); let mut request = make_req( Method::GET, "https://publisher.example.com/integrations/sourcepoint/cdn/mms/v2/get_site_data?account_id=123", @@ -1892,9 +1997,11 @@ mod tests { .expect("should rewrite dynamic site data"); assert!( - client.stub.recorded_request_headers()[0] - .iter() - .any(|(name, value)| name == "accept-encoding" && value == "identity"), + cdn.requests()[0] + .headers + .get(header::ACCEPT_ENCODING) + .and_then(|value| value.to_str().ok()) + == Some("identity"), "should request uncompressed site data despite its extensionless path" ); assert_eq!( @@ -1916,18 +2023,20 @@ mod tests { futures::executor::block_on(async { let settings = create_test_settings(); let integration = SourcepointIntegration::new(Arc::new(config(true))); - let client = Arc::new(StreamingHttpClient::new()); + let cdn = Cdn::new(); let bytes = vec![b'a', 0xff, b'b']; - client.stub.push_response_with_headers( + cdn.respond( + Method::GET, + "/wrapper.js", 200, bytes.clone(), - vec![ + &[ ("content-type", "application/javascript"), ("content-encoding", "identity"), ("cache-control", "no-store"), ], ); - let services = build_services_with_http_client(client.clone()); + let services = cdn.services(); let response = integration .handle( @@ -1942,7 +2051,7 @@ mod tests { .expect("should retain non-UTF-8 content unchanged"); assert!( - client.reads.load(Ordering::Relaxed) > 0, + cdn.reads() > 0, "should exercise UTF-8 fallback after bounded collection" ); @@ -2656,10 +2765,10 @@ mod tests { } fn take_body_bytes(resp: Response) -> Vec { - match resp.into_body() { - EdgeBody::Once(b) => b.to_vec(), - EdgeBody::Stream(_) => vec![], - } + resp.into_body() + .into_bytes() + .expect("should buffer the response body") + .to_vec() } #[test] @@ -2670,7 +2779,7 @@ mod tests { let client_ip = "203.0.113.10".parse().expect("should parse test IP"); let forwarded_cookies = - integration.copy_headers(Some(client_ip), &original_req, &mut proxy_req); + integration.copy_headers(Some(client_ip), &original_req, proxy_req.headers_mut()); assert!( !forwarded_cookies, @@ -2696,7 +2805,7 @@ mod tests { ); let mut proxy_req = make_req(Method::OPTIONS, "https://cdn.privacy-mgmt.com/wrapper.js"); - integration.copy_headers(None, &original_req, &mut proxy_req); + integration.copy_headers(None, &original_req, proxy_req.headers_mut()); assert_eq!( get_req_header_str(&proxy_req, "access-control-request-method"), @@ -3061,4 +3170,25 @@ mod tests { "should rewrite relative redirect resolved against CDN base" ); } + + #[test] + fn sourcepoint_cdn_body_read_failures_map_to_statuses() { + let settings = create_test_settings(); + let integration = SourcepointIntegration::new(Arc::new(config(true))); + for case in body_read_failure_cases(INTEGRATION_MAX_BODY_BYTES) { + let client = ScriptedHttpClient::forbidding(); + let services = build_services_with_outbound_client(&client, &TestClock::new()); + let (req, expectation) = case.into_request( + http::Request::builder() + .method(http::Method::POST) + .uri("https://publisher.example.com/integrations/sourcepoint/cdn/asset"), + ); + + let outcome = + futures::executor::block_on(integration.handle(&settings, &services, req)); + + expectation.assert_outcome("Sourcepoint CDN", &outcome); + client.assert_no_requests(); + } + } } diff --git a/crates/trusted-server-core/src/integrations/testlight.rs b/crates/trusted-server-core/src/integrations/testlight.rs index b2b7f7f6c2..98b0b1b6a4 100644 --- a/crates/trusted-server-core/src/integrations/testlight.rs +++ b/crates/trusted-server-core/src/integrations/testlight.rs @@ -1,7 +1,7 @@ use std::sync::Arc; use async_trait::async_trait; -use edgezero_core::body::Body as EdgeBody; +use edgezero_core::body::{Body as EdgeBody, BodyPayload}; use error_stack::{Report, ResultExt}; use http::Response; use http::header::{self, HeaderValue}; @@ -11,10 +11,11 @@ use validator::Validate; use crate::edge_cookie::get_ec_id; use crate::error::TrustedServerError; +use crate::http_util::{into_body_payload, read_request_body}; use crate::integrations::{ AttributeRewriteAction, INTEGRATION_MAX_BODY_BYTES, IntegrationAttributeContext, IntegrationAttributeRewriter, IntegrationEndpoint, IntegrationProxy, IntegrationRegistration, - UPSTREAM_RTB_MAX_RESPONSE_BYTES, collect_body_bounded, collect_response_bounded, + UPSTREAM_RTB_MAX_RESPONSE_BYTES, }; use crate::platform::RuntimeServices; use crate::proxy::{ProxyRequestConfig, proxy_request}; @@ -181,8 +182,7 @@ impl IntegrationProxy for TestlightIntegration { ) -> Result, Report> { let (parts, body) = req.into_parts(); let payload_bytes = - collect_body_bounded(body, INTEGRATION_MAX_BODY_BYTES, TESTLIGHT_INTEGRATION_ID) - .await?; + read_request_body(&parts.headers, body, INTEGRATION_MAX_BODY_BYTES).await?; let req = http::Request::from_parts(parts, EdgeBody::empty()); // Read EC ID from the ts-ec cookie forwarded by the client. @@ -197,7 +197,11 @@ impl IntegrationProxy for TestlightIntegration { let payload_bytes = Self::rewrite_request_body(&payload_bytes, &ec_id)?; - let mut proxy_config = ProxyRequestConfig::new(&self.config.endpoint); + let mut proxy_config = ProxyRequestConfig::new(&self.config.endpoint) + .with_max_response_bytes( + u64::try_from(UPSTREAM_RTB_MAX_RESPONSE_BYTES) + .expect("should widen the RTB response cap to u64"), + ); proxy_config.forward_ec_id = false; proxy_config.body = Some(payload_bytes); proxy_config.stream_passthrough = true; @@ -211,13 +215,15 @@ impl IntegrationProxy for TestlightIntegration { .change_context(Self::error("Failed to contact upstream integration"))?; let (parts, body) = response.into_parts(); + // `proxy_request` buffers the answer under the Testlight response cap. + let BodyPayload::Once(response_body) = into_body_payload(body) else { + return Err(Report::new(Self::error( + "Testlight upstream response was not buffered", + ))); + }; + let response_body = response_body.to_vec(); + // Attempt to parse response into structured form for logging/future transforms. - let response_body = collect_response_bounded( - body, - UPSTREAM_RTB_MAX_RESPONSE_BYTES, - TESTLIGHT_INTEGRATION_ID, - ) - .await?; match serde_json::from_slice::(&response_body) { Ok(body) => { let response_body = serde_json::to_vec(&body) @@ -278,12 +284,17 @@ fn default_enabled() -> bool { #[cfg(test)] mod tests { use super::*; - use crate::platform::test_support::{StubHttpClient, build_services_with_http_client}; + use crate::outbound::{BUFFERED_TIMEOUT, REDIRECT_CHAIN_DEADLINE}; + use crate::platform::test_support::build_services_with_outbound_client; + use crate::test_support::body_read::body_read_failure_cases; + use crate::test_support::scripted_client::{ + ExpectedTransport, ScriptedHttpClient, ScriptedReply, TestClock, + }; use crate::test_support::tests::{VALID_SYNTHETIC_ID, create_test_settings}; use crate::tsjs; + use edgezero_core::outbound::{OutboundCachePolicy, ResponseDecoding, ResponseMode}; use http::Method; use serde_json::json; - use std::sync::Arc; #[test] fn build_requires_config() { @@ -421,13 +432,18 @@ mod tests { } #[test] - fn handle_uses_platform_http_client_with_http_request() { + fn testlight_request_readback() { futures::executor::block_on(async { - let stub = Arc::new(StubHttpClient::new()); - stub.push_response(200, br#"{"ok":true}"#.to_vec()); - let services = build_services_with_http_client( - Arc::clone(&stub) as Arc + let clock = TestClock::new(); + let upstream = ScriptedHttpClient::new(&clock); + upstream.respond_to( + Method::POST, + "https://example.com/openrtb", + ScriptedReply::status(http::StatusCode::OK) + .header("content-type", "application/json") + .body(r#"{"ok":true}"#), ); + let services = build_services_with_outbound_client(&upstream, &clock); let settings = create_test_settings(); let integration = TestlightIntegration::new(TestlightConfig { enabled: true, @@ -445,6 +461,10 @@ mod tests { crate::constants::HEADER_X_TS_EC.clone(), http::HeaderValue::from_static(VALID_SYNTHETIC_ID), ); + req.headers_mut().insert( + header::USER_AGENT, + http::HeaderValue::from_static("fictional-agent/1.0"), + ); let response = integration .handle(&settings, &services, req) @@ -454,12 +474,39 @@ mod tests { assert_eq!( response.status(), http::StatusCode::OK, - "should return stubbed upstream status" + "should return the upstream status" ); + let requests = upstream.requests(); + assert_eq!(requests.len(), 1, "should send one Testlight request"); + let sent = &requests[0]; + let sent_body: serde_json::Value = + serde_json::from_slice(&sent.body).expect("should send a JSON body"); assert_eq!( - stub.recorded_backend_names(), - vec!["stub-backend".to_string()], - "should route outbound request through PlatformHttpClient" + (&sent.method, sent.uri.to_string(), &sent_body["user"]["id"]), + ( + &Method::POST, + "https://example.com/openrtb".to_owned(), + &json!(VALID_SYNTHETIC_ID) + ), + "should POST the rewritten payload to the configured endpoint" + ); + sent.assert_headers(&[ + ("user-agent", "fictional-agent/1.0"), + ("accept-encoding", "gzip, deflate, br"), + ("content-type", "application/json"), + ]); + sent.assert_followed_transport( + ExpectedTransport { + timeout: BUFFERED_TIMEOUT, + response_mode: ResponseMode::Buffered { + max_bytes: 2 * 1024 * 1024, + }, + response_decoding: ResponseDecoding::Original, + cache_policy: OutboundCachePolicy::PlatformDefault, + }, + clock + .monotonic_clock() + .deadline_after(REDIRECT_CHAIN_DEADLINE), ); let response_json: serde_json::Value = serde_json::from_slice(&response.into_body().into_bytes().unwrap_or_default()) @@ -470,4 +517,32 @@ mod tests { ); }); } + + #[test] + fn testlight_auction_body_read_failures_map_to_statuses() { + let settings = create_test_settings(); + let integration = TestlightIntegration::new(TestlightConfig { + enabled: true, + endpoint: "https://example.com/openrtb".to_string(), + timeout_ms: 1000, + shim_src: tsjs::tsjs_unified_script_src(), + rewrite_scripts: true, + }); + for case in body_read_failure_cases(INTEGRATION_MAX_BODY_BYTES) { + let upstream = ScriptedHttpClient::forbidding(); + let services = build_services_with_outbound_client(&upstream, &TestClock::new()); + let (req, expectation) = case.into_request( + http::Request::builder() + .method(Method::POST) + .uri("https://edge.example.com/integrations/testlight/auction") + .header(crate::constants::HEADER_X_TS_EC.clone(), VALID_SYNTHETIC_ID), + ); + + let outcome = + futures::executor::block_on(integration.handle(&settings, &services, req)); + + expectation.assert_outcome("Testlight auction", &outcome); + upstream.assert_no_requests(); + } + } } diff --git a/crates/trusted-server-core/src/lib.rs b/crates/trusted-server-core/src/lib.rs index 5cdc1d1248..68d2d582a8 100644 --- a/crates/trusted-server-core/src/lib.rs +++ b/crates/trusted-server-core/src/lib.rs @@ -12,6 +12,7 @@ //! - [`consent`]: Consent signal extraction and logging //! - [`geo`]: Geographic location utilities and DMA code extraction //! - [`models`]: Data models for ad serving and callbacks +//! - [`outbound`]: Outbound request settings and failure mapping //! - [`integrations::prebid`]: Prebid integration and real-time bidding support //! - [`settings`]: Configuration management and validation //! - [`streaming_replacer`]: Streaming URL replacement for large responses @@ -19,6 +20,8 @@ //! - [`test_support`]: Testing utilities and mocks //! - [`tester_cookie`]: Optional tester-cookie endpoint helpers +// Unit tests may buffer bodies with `Body::into_bytes`; production code must not. +#![cfg_attr(test, allow(clippy::disallowed_methods))] #![cfg_attr( test, allow( @@ -47,6 +50,7 @@ pub mod cookies; pub mod creative; pub mod creative_opportunities; pub mod ec; +pub mod edge_app; pub(crate) mod edge_cookie; pub mod error; pub mod forwarder; @@ -58,6 +62,7 @@ pub mod http_util; pub mod integrations; pub mod models; pub mod openrtb; +pub mod outbound; pub mod platform; pub mod price_bucket; pub mod proxy; diff --git a/crates/trusted-server-core/src/migration_guards.rs b/crates/trusted-server-core/src/migration_guards.rs index 17b3d624f6..79f2ad6eff 100644 --- a/crates/trusted-server-core/src/migration_guards.rs +++ b/crates/trusted-server-core/src/migration_guards.rs @@ -172,11 +172,6 @@ fn checked_sources() -> &'static [(&'static str, &'static str)] { ("models.rs", include_str!("models.rs")), ("openrtb.rs", include_str!("openrtb.rs")), ("platform/error.rs", include_str!("platform/error.rs")), - ("platform/http.rs", include_str!("platform/http.rs")), - ( - "platform/image_optimizer.rs", - include_str!("platform/image_optimizer.rs"), - ), ("platform/kv.rs", include_str!("platform/kv.rs")), ("platform/mod.rs", include_str!("platform/mod.rs")), ( diff --git a/crates/trusted-server-core/src/outbound.rs b/crates/trusted-server-core/src/outbound.rs new file mode 100644 index 0000000000..8b7f5dc496 --- /dev/null +++ b/crates/trusted-server-core/src/outbound.rs @@ -0,0 +1,682 @@ +//! Outbound request settings and failure mapping for `EdgeZero`'s HTTP client. +//! +//! Every outbound request goes through [`buffered`] or [`streamed`]. Both set a +//! one second connect timeout and a constant total timeout, never one computed +//! from a remaining budget, so Fastly backend identity stays stable. Every +//! failure from the client, or from building the request, goes through +//! [`outbound_error`], which keeps the site's own error except for the three +//! statuses the client caused. + +use std::time::Duration; + +use edgezero_core::context::RequestContext; +use edgezero_core::error::{BudgetSource, EdgeError}; +use edgezero_core::error_detail::ProviderFailureDetail; +use edgezero_core::outbound::{HttpClient, OutboundRequest, ResponseDecoding}; +use error_stack::Report; +use http::{Method, StatusCode, Uri}; + +use crate::error::TrustedServerError; + +/// Connect timeout every outbound request carries. +pub const CONNECT_TIMEOUT: Duration = Duration::from_secs(1); + +/// Total timeout for a streamed request whose site names no timeout. +pub const STREAMED_TIMEOUT: Duration = Duration::from_secs(60); + +/// Total timeout for a buffered request whose site names no timeout. +pub const BUFFERED_TIMEOUT: Duration = Duration::from_secs(15); + +/// Deadline for a whole redirect chain, measured from its first request. +pub const REDIRECT_CHAIN_DEADLINE: Duration = Duration::from_secs(60); + +/// Response cap for a buffered site that has no Trusted Server cap of its own. +/// +/// It keeps the 10 MiB platform limit those sites had before `EdgeZero`'s +/// smaller default. +pub const DEFAULT_BUFFERED_RESPONSE_BYTES: u64 = 10 * 1024 * 1024; + +/// Cap on a publisher request body forwarded to the origin. +pub const PUBLISHER_UPLOAD_BYTES: u64 = 16 * 1024 * 1024; + +/// Returns the outbound client the adapter installed on the request, with the proxy marker off. +/// +/// # Errors +/// +/// Returns [`EdgeError::Internal`] when the request carries no outbound client. +/// +/// # Examples +/// +/// ``` +/// use edgezero_core::body::Body; +/// use edgezero_core::context::RequestContext; +/// use edgezero_core::http::request_builder; +/// use trusted_server_core::outbound::request_outbound_client; +/// +/// let request = request_builder() +/// .body(Body::empty()) +/// .expect("should build a request"); +/// let context = RequestContext::new(request, Default::default()); +/// let error = request_outbound_client(&context) +/// .err() +/// .expect("should reject a request without an outbound client"); +/// assert_eq!(error.to_string(), "internal error: request has no outbound client"); +/// ``` +pub fn request_outbound_client(ctx: &RequestContext) -> Result { + let client = ctx.http_client().ok_or_else(|| { + EdgeError::internal(std::io::Error::other("request has no outbound client")) + })?; + Ok(client.with_proxy_marker(false)) +} + +/// Builds a `method` request to `target`, a URL Trusted Server assembled. +/// +/// The target is parsed as an [`http::Uri`] first, so any method works, not +/// only the GET and POST that the raw-string constructors cover. +/// +/// # Errors +/// +/// Returns [`EdgeError::BadRequest`] when `target` is not an absolute HTTP or +/// HTTPS URI. +/// +/// # Examples +/// +/// ``` +/// use edgezero_core::http::Method; +/// use trusted_server_core::outbound::outbound_request; +/// +/// let request = outbound_request(Method::DELETE, "https://example.com/items/1") +/// .expect("should build a DELETE request"); +/// assert_eq!(request.method(), &Method::DELETE); +/// ``` +pub fn outbound_request(method: Method, target: &str) -> Result { + let uri = target + .parse::() + .map_err(|_| EdgeError::bad_request("outbound target is not a valid URI"))?; + OutboundRequest::new(method, uri) +} + +/// Applies the buffered-response settings every buffered site carries. +/// +/// Sets [`CONNECT_TIMEOUT`], the constant total `timeout`, the response cap +/// `max_response_bytes`, and the response `decoding`. +/// +/// # Examples +/// +/// ``` +/// use edgezero_core::outbound::{OutboundRequest, ResponseDecoding, ResponseMode}; +/// use trusted_server_core::outbound::{ +/// BUFFERED_TIMEOUT, CONNECT_TIMEOUT, DEFAULT_BUFFERED_RESPONSE_BYTES, buffered, +/// }; +/// +/// let request = OutboundRequest::get("https://example.com/config.json") +/// .expect("should build a GET request"); +/// let parts = buffered( +/// request, +/// BUFFERED_TIMEOUT, +/// DEFAULT_BUFFERED_RESPONSE_BYTES, +/// ResponseDecoding::Decode, +/// ) +/// .into_parts(); +/// assert_eq!(parts.connect_timeout, Some(CONNECT_TIMEOUT)); +/// assert_eq!(parts.timeout, Some(BUFFERED_TIMEOUT)); +/// assert_eq!( +/// parts.response_mode, +/// ResponseMode::Buffered { +/// max_bytes: DEFAULT_BUFFERED_RESPONSE_BYTES +/// } +/// ); +/// ``` +#[must_use] +pub fn buffered( + request: OutboundRequest, + timeout: Duration, + max_response_bytes: u64, + decoding: ResponseDecoding, +) -> OutboundRequest { + request + .connect_timeout(CONNECT_TIMEOUT) + .timeout(timeout) + .max_response_bytes(max_response_bytes) + .response_decoding(decoding) +} + +/// Applies the streamed-response settings every streamed site carries. +/// +/// Sets [`CONNECT_TIMEOUT`], the constant total `timeout`, a streamed +/// response, and [`ResponseDecoding::Original`], so the bytes reach the +/// caller in the coding the upstream sent. +/// +/// # Examples +/// +/// ``` +/// use edgezero_core::outbound::{OutboundRequest, ResponseDecoding, ResponseMode}; +/// use trusted_server_core::outbound::{CONNECT_TIMEOUT, STREAMED_TIMEOUT, streamed}; +/// +/// let request = OutboundRequest::get("https://example.com/article") +/// .expect("should build a GET request"); +/// let parts = streamed(request, STREAMED_TIMEOUT).into_parts(); +/// assert_eq!(parts.connect_timeout, Some(CONNECT_TIMEOUT)); +/// assert_eq!(parts.timeout, Some(STREAMED_TIMEOUT)); +/// assert_eq!(parts.response_mode, ResponseMode::Streamed); +/// assert_eq!(parts.response_decoding, ResponseDecoding::Original); +/// ``` +#[must_use] +pub fn streamed(request: OutboundRequest, timeout: Duration) -> OutboundRequest { + request + .connect_timeout(CONNECT_TIMEOUT) + .timeout(timeout) + .stream_response() + .response_decoding(ResponseDecoding::Original) +} + +/// Who shaped a failed outbound request, which decides what its 400 means. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum OutboundRequestSource { + /// The request forwards the client's own request, so a 400 is the client's. + ClientForward, + /// Trusted Server built the request, so a 400 is the site's own failure. + Internal, +} + +/// Budget input that selected the deadline a failed outbound request reached. +#[derive(Debug, Clone, Copy, PartialEq, Eq, derive_more::Display)] +#[display("outbound budget source: {_0:?}")] +struct OutboundBudgetSource(BudgetSource); + +/// Text-free provider evidence for a failed outbound request. +#[derive(Debug, Clone, Copy, PartialEq, Eq, derive_more::Display)] +#[display("outbound provider failure detail: {_0:?}")] +struct OutboundFailureDetail(ProviderFailureDetail); + +/// Maps an outbound client or request-construction failure onto a Trusted +/// Server error. +/// +/// The mapping follows [`EdgeError::status`]: +/// +/// - 413 becomes [`TrustedServerError::RequestTooLarge`]. +/// - 408 becomes [`TrustedServerError::RequestTimeout`]. +/// - 400 becomes [`TrustedServerError::BadRequest`] only for +/// [`OutboundRequestSource::ClientForward`]. +/// - Every other failure keeps `context`, the site's own error, which is a +/// 502 for every outbound site. +/// +/// The report carries `error` as an opaque attachment, the budget source of a +/// timeout, and the error's provider failure detail. The detail is logged at +/// warn with `{:?}`. Neither that log nor the report's `{:?}` rendering carries +/// the error's message, which can hold provider text or a URL. Callers that +/// need the error downcast the report to [`EdgeError`]. +/// +/// # Examples +/// +/// ``` +/// use edgezero_core::error::EdgeError; +/// use trusted_server_core::error::{IntoHttpResponse as _, TrustedServerError}; +/// use trusted_server_core::outbound::{OutboundRequestSource, outbound_error}; +/// +/// let site = TrustedServerError::Proxy { +/// message: "example origin request failed".to_owned(), +/// }; +/// let report = outbound_error( +/// EdgeError::bad_gateway("upstream connection failed"), +/// site, +/// OutboundRequestSource::Internal, +/// ); +/// assert_eq!(report.current_context().status_code().as_u16(), 502); +/// assert!(report.downcast_ref::().is_some()); +/// ``` +#[track_caller] +pub fn outbound_error( + error: EdgeError, + context: TrustedServerError, + source: OutboundRequestSource, +) -> Report { + let status = error.status(); + let budget_source = match &error { + EdgeError::GatewayTimeout { cause, .. } => Some(*cause), + _ => None, + }; + let detail = error.provider_failure_detail().copied(); + log::warn!( + "Outbound request failed with {status} ({kind}): budget source {budget_source:?}, provider detail {detail:?}", + kind = error.kind() + ); + + let mapped = match (status, source) { + (StatusCode::PAYLOAD_TOO_LARGE, _) => Some(TrustedServerError::RequestTooLarge { + message: "request body exceeds the outbound upload limit".to_owned(), + }), + (StatusCode::REQUEST_TIMEOUT, _) => Some(TrustedServerError::RequestTimeout { + message: "request body was not read before the read deadline".to_owned(), + }), + (StatusCode::BAD_REQUEST, OutboundRequestSource::ClientForward) => { + Some(TrustedServerError::BadRequest { + message: "request could not be forwarded".to_owned(), + }) + } + _ => None, + }; + + let report = match mapped { + // Keep the replaced site error readable, so the log still names the site. + Some(mapped) => Report::new(mapped).attach(format!("in place of: {context}")), + None => Report::new(context), + }; + let mut report = report.attach_opaque(error); + if let Some(budget_source) = budget_source { + report = report.attach(OutboundBudgetSource(budget_source)); + } + if let Some(detail) = detail { + report = report.attach(OutboundFailureDetail(detail)); + } + report +} + +#[cfg(test)] +mod tests { + use std::io; + use std::time::Duration; + + use edgezero_core::body::Body; + use edgezero_core::error::{BudgetSource, EdgeError, RequestBodyOverflowOrigin}; + use edgezero_core::error_detail::{ + FastlyFailureCause, ProviderFailureDetail, ProviderFailurePhase, + }; + use edgezero_core::outbound::{OutboundRequest, ResponseDecoding, ResponseMode}; + use http::StatusCode; + + use super::*; + use crate::error::IntoHttpResponse as _; + use crate::test_support::logging::WarningCapture; + use crate::test_support::scripted_client::ScriptedHttpClient; + + const BOTH_SOURCES: [OutboundRequestSource; 2] = [ + OutboundRequestSource::ClientForward, + OutboundRequestSource::Internal, + ]; + + fn site_error() -> TrustedServerError { + TrustedServerError::Proxy { + message: "example origin request failed".to_owned(), + } + } + + fn attached_edge_error(report: &Report) -> String { + report + .downcast_ref::() + .expect("should attach the EdgeError") + .to_string() + } + + fn example_request() -> OutboundRequest { + OutboundRequest::get("https://example.com/resource").expect("should build a GET request") + } + + #[test] + fn request_outbound_client_turns_the_proxy_marker_off() { + let client = HttpClient::with_client(ScriptedHttpClient::forbidding()); + assert!( + client.proxy_marker_enabled(), + "should start from a client whose marker is on" + ); + let mut request = edgezero_core::http::request_builder() + .body(Body::empty()) + .expect("should build a request"); + request.extensions_mut().insert(client); + let context = RequestContext::new(request, Default::default()); + + let client = request_outbound_client(&context).expect("should return the request's client"); + + assert!( + !client.proxy_marker_enabled(), + "should turn the proxy marker off on the request's client" + ); + } + + #[test] + fn outbound_error_maps_payload_too_large_to_413() { + for source in BOTH_SOURCES { + let report = outbound_error( + EdgeError::payload_too_large(RequestBodyOverflowOrigin::OutboundUpload), + site_error(), + source, + ); + + assert!( + matches!( + report.current_context(), + TrustedServerError::RequestTooLarge { .. } + ), + "should map a {source:?} upload overflow to RequestTooLarge" + ); + assert_eq!( + report.current_context().status_code(), + StatusCode::PAYLOAD_TOO_LARGE, + "should answer a {source:?} upload overflow with 413" + ); + assert_eq!( + attached_edge_error(&report), + "request body too large", + "should attach the {source:?} overflow EdgeError" + ); + } + } + + #[test] + fn outbound_error_maps_request_timeout_to_408() { + for source in BOTH_SOURCES { + let report = outbound_error( + EdgeError::request_timeout("inbound read deadline passed"), + site_error(), + source, + ); + + assert!( + matches!( + report.current_context(), + TrustedServerError::RequestTimeout { .. } + ), + "should map a {source:?} read deadline to RequestTimeout" + ); + assert_eq!( + report.current_context().status_code(), + StatusCode::REQUEST_TIMEOUT, + "should answer a {source:?} read deadline with 408" + ); + assert_eq!( + attached_edge_error(&report), + "inbound read deadline passed", + "should attach the {source:?} read deadline EdgeError" + ); + } + } + + #[test] + fn outbound_error_maps_bad_request_to_400_for_client_forward() { + let report = outbound_error( + EdgeError::bad_request("GET request carries a body"), + site_error(), + OutboundRequestSource::ClientForward, + ); + + assert!( + matches!( + report.current_context(), + TrustedServerError::BadRequest { .. } + ), + "should map a forwarded client's bad request to BadRequest" + ); + assert_eq!( + report.current_context().status_code(), + StatusCode::BAD_REQUEST, + "should answer a forwarded client's bad request with 400" + ); + assert_eq!( + attached_edge_error(&report), + "GET request carries a body", + "should attach the bad request EdgeError" + ); + } + + #[test] + fn outbound_error_maps_bad_request_to_site_502_for_internal() { + let report = outbound_error( + EdgeError::bad_request("request URI carries userinfo"), + site_error(), + OutboundRequestSource::Internal, + ); + + assert!( + matches!( + report.current_context(), + TrustedServerError::Proxy { message } if message == "example origin request failed" + ), + "should keep the site context for a request Trusted Server built" + ); + assert_eq!( + report.current_context().status_code(), + StatusCode::BAD_GATEWAY, + "should answer an internal bad request with the site's 502" + ); + assert_eq!( + attached_edge_error(&report), + "request URI carries userinfo", + "should attach the internal bad request EdgeError" + ); + } + + #[test] + fn outbound_error_keeps_site_502_for_gateway_timeout_bad_gateway_response_too_large_and_internal() + { + let failures: [fn() -> EdgeError; 4] = [ + || { + EdgeError::gateway_timeout_caused( + "upstream timed out", + BudgetSource::PerCallTimeout, + ) + }, + || EdgeError::bad_gateway("upstream connection failed"), + || EdgeError::response_too_large("upstream response exceeds its cap"), + || EdgeError::internal(io::Error::other("provider send failed")), + ]; + + for failure in failures { + for source in BOTH_SOURCES { + let expected_display = failure().to_string(); + let report = outbound_error(failure(), site_error(), source); + + assert!( + matches!( + report.current_context(), + TrustedServerError::Proxy { message } if message == "example origin request failed" + ), + "should keep the site context for {source:?} `{expected_display}`" + ); + assert_eq!( + report.current_context().status_code(), + StatusCode::BAD_GATEWAY, + "should answer {source:?} `{expected_display}` with the site's 502" + ); + assert_eq!( + attached_edge_error(&report), + expected_display, + "should attach the {source:?} EdgeError" + ); + } + } + } + + #[test] + fn outbound_error_attaches_edge_error_detail() { + let detail = ProviderFailureDetail::Fastly { + phase: ProviderFailurePhase::Send, + cause: FastlyFailureCause::ConnectionTimeout, + }; + let report = outbound_error( + EdgeError::gateway_timeout_with_detail( + "upstream timed out", + BudgetSource::ConnectTimeout, + detail, + ), + site_error(), + OutboundRequestSource::Internal, + ); + + assert_eq!( + report.current_context().status_code(), + StatusCode::BAD_GATEWAY, + "should answer a connect timeout with the site's 502" + ); + assert_eq!( + attached_edge_error(&report), + "upstream timed out", + "should attach the timeout EdgeError" + ); + assert_eq!( + report.downcast_ref::(), + Some(&OutboundBudgetSource(BudgetSource::ConnectTimeout)), + "should attach the budget source that selected the deadline" + ); + assert_eq!( + report.downcast_ref::(), + Some(&OutboundFailureDetail(detail)), + "should attach the provider failure detail" + ); + } + + #[test] + fn outbound_error_logs_provider_detail_without_provider_text() { + let detail = ProviderFailureDetail::Fastly { + phase: ProviderFailurePhase::Send, + cause: FastlyFailureCause::ConnectionRefused, + }; + let capture = WarningCapture::start(); + + let _report = outbound_error( + EdgeError::bad_gateway_with_detail( + "provider said no to https://example.com/secret-path", + edgezero_core::error::BadGatewayReason::Transport, + detail, + ), + site_error(), + OutboundRequestSource::Internal, + ); + + let warnings = capture.take(); + assert_eq!(warnings.len(), 1, "should log one warning per failure"); + let message = &warnings[0].message; + assert!( + message.contains(&format!("{detail:?}")), + "should log the provider detail with Debug, got `{message}`" + ); + assert!( + !message.contains("provider said no") && !message.contains("example.com"), + "should never log provider text or URLs, got `{message}`" + ); + } + + #[test] + fn outbound_error_keeps_edge_error_text_out_of_report_debug() { + let report = outbound_error( + EdgeError::internal(io::Error::other( + "provider failed for https://example.com/secret-path", + )), + site_error(), + OutboundRequestSource::Internal, + ); + + let rendered = format!("{report:?}"); + assert!( + !rendered.contains("provider failed") && !rendered.contains("example.com"), + "should keep the EdgeError text out of the logged report, got `{rendered}`" + ); + assert!( + report.downcast_ref::().is_some(), + "should still attach the EdgeError for downcasts" + ); + } + + #[test] + fn outbound_request_builds_any_method_and_rejects_invalid_targets() { + let request = outbound_request(Method::OPTIONS, "https://example.com/a/../b?x=1") + .expect("should build an OPTIONS request"); + assert_eq!( + (request.method(), request.uri().to_string()), + (&Method::OPTIONS, "https://example.com/b?x=1".to_owned()), + "should keep the method and canonicalize the target" + ); + + for target in ["not a url", "/relative/path", "ftp://example.com/file"] { + let error = outbound_request(Method::GET, target) + .expect_err("should reject a target that is not an absolute HTTP URI"); + assert_eq!( + error.status(), + StatusCode::BAD_REQUEST, + "should reject `{target}` as a bad request" + ); + } + } + + #[test] + fn buffered_sets_one_second_connect_timeout_and_constant_timeout() { + let site_timeout = Duration::from_millis(2_000); + + for (timeout, max_response_bytes, decoding) in [ + ( + BUFFERED_TIMEOUT, + DEFAULT_BUFFERED_RESPONSE_BYTES, + ResponseDecoding::Decode, + ), + (site_timeout, 64 * 1024, ResponseDecoding::Original), + ] { + let parts = + buffered(example_request(), timeout, max_response_bytes, decoding).into_parts(); + + assert_eq!( + parts.connect_timeout, + Some(Duration::from_secs(1)), + "should bound every buffered connect at one second" + ); + assert_eq!( + parts.timeout, + Some(timeout), + "should set the constant total timeout" + ); + assert!( + parts.deadline.is_none(), + "should never derive a deadline from a remaining budget" + ); + assert_eq!( + parts.response_mode, + ResponseMode::Buffered { + max_bytes: max_response_bytes + }, + "should buffer the response under the site cap" + ); + assert_eq!( + parts.response_decoding, decoding, + "should apply the site's response decoding" + ); + } + assert_eq!( + BUFFERED_TIMEOUT, + Duration::from_secs(15), + "should default buffered sites to 15 s" + ); + assert_eq!( + DEFAULT_BUFFERED_RESPONSE_BYTES, + 10 * 1024 * 1024, + "should keep today's 10 MiB platform cap for uncapped sites" + ); + } + + #[test] + fn streamed_sets_original_decoding_and_stream_response() { + let parts = streamed(example_request(), STREAMED_TIMEOUT).into_parts(); + + assert_eq!( + parts.connect_timeout, + Some(Duration::from_secs(1)), + "should bound every streamed connect at one second" + ); + assert_eq!( + parts.timeout, + Some(Duration::from_secs(60)), + "should default streamed sites to 60 s" + ); + assert!( + parts.deadline.is_none(), + "should never derive a deadline from a remaining budget" + ); + assert_eq!( + parts.response_mode, + ResponseMode::Streamed, + "should stream the response" + ); + assert_eq!( + parts.response_decoding, + ResponseDecoding::Original, + "should deliver streamed bytes in their original coding" + ); + } +} diff --git a/crates/trusted-server-core/src/platform/backend_naming.rs b/crates/trusted-server-core/src/platform/backend_naming.rs deleted file mode 100644 index 52487def53..0000000000 --- a/crates/trusted-server-core/src/platform/backend_naming.rs +++ /dev/null @@ -1,632 +0,0 @@ -//! Pure adapter backend naming and auction-target capability policies. -//! -//! These policies contain no platform SDK calls. Startup validation, CLI -//! validation, and runtime adapters can therefore predict the same backend -//! names before any backend registration occurs. - -use core::fmt::Write as _; - -use error_stack::Report; -use sha2::{Digest as _, Sha256}; - -use super::PlatformBackendSpec; -use crate::host_header::validate_host_header_override_value; - -const MAX_FASTLY_BACKEND_NAME_LEN: usize = 255; -const MAX_FASTLY_READABLE_PREFIX_LEN: usize = 200; -const FASTLY_SPEC_DIGEST_HEX_LEN: usize = 32; -const FASTLY_TRANSPORT_TIMEOUT_QUANTUM_MS: u32 = 250; -const FASTLY_TRANSPORT_TIMEOUT_QUANTUM_CEILING_MS: u32 = 2000; -const FASTLY_SUB_QUANTUM_LADDER_MS: [u32; 4] = [200, 150, 100, 50]; -const FASTLY_TRANSPORT_TIMEOUT_COARSE_LADDER_MS: [u32; 8] = - [2000, 3000, 5000, 10000, 20000, 30000, 45000, 60000]; -const FASTLY_DYNAMIC_BACKEND_LIMIT: usize = 200; -const FASTLY_NON_AUCTION_BACKEND_RESERVE: usize = 40; - -/// A pure backend-name and transport-timeout policy for one adapter. -/// -/// Cloudflare and Spin intentionally remain separate variants even though -/// their current no-registration name formats are identical. Keeping distinct -/// policies prevents a future adapter-specific change from silently affecting -/// the other target. -#[derive(Debug, Clone, Copy, Eq, PartialEq)] -pub enum BackendNamingPolicy { - /// Fastly dynamic backend naming and bounded timeout buckets. - Fastly, - /// Axum's environment-segment-compatible correlation name. - Axum, - /// Cloudflare's deterministic no-registration correlation name. - Cloudflare, - /// Spin's deterministic no-registration correlation name. - Spin, -} - -/// Pure backend prediction shared by validation and runtime registration. -#[derive(Debug, Clone, Eq, PartialEq)] -pub struct PredictedBackend { - /// Deterministic platform backend or correlation name. - pub name: String, - /// Explicit or scheme-derived target port. - pub port: u16, -} - -/// Detailed pure backend prediction error. -#[derive(Debug, Clone, Eq, PartialEq, derive_more::Display)] -pub enum BackendNamingError { - /// The backend host is empty. - #[display("missing host")] - MissingHost, - /// A field contains a control character. - #[display("{field} contains control characters")] - ControlCharacters { field: &'static str }, - /// An outbound Host override is invalid. - #[display("host header override {reason}")] - InvalidHostHeaderOverride { reason: &'static str }, - /// A generated Fastly backend name exceeded its documented limit. - #[display("backend name exceeds {limit}-char limit ({actual} chars)")] - NameTooLong { limit: usize, actual: usize }, -} - -impl core::error::Error for BackendNamingError {} - -impl BackendNamingPolicy { - /// Predict a backend name and resolved port without platform I/O. - /// - /// # Errors - /// - /// Returns a naming error when a Fastly backend specification contains an - /// invalid host, scheme, Host override, or cannot fit the platform limit. - pub fn predict( - self, - spec: &PlatformBackendSpec, - ) -> Result> { - match self { - Self::Fastly => predict_fastly(spec), - Self::Axum => Ok(predict_axum(spec)), - Self::Cloudflare => Ok(predict_cloudflare(spec)), - Self::Spin => Ok(predict_spin(spec)), - } - } - - /// Canonicalize a transport timer without changing the logical budget. - /// - /// The result configures adapter transport timers and backend-name - /// stability only. It is not an auction-wide deadline. Fastly bounds the - /// cardinality of budget-derived timers because timers are encoded in - /// dynamic backend names; other adapters preserve the exact bounded value. - #[must_use] - pub fn canonicalize_transport_timeout_ms(self, remaining_ms: u32, configured_ms: u32) -> u32 { - match self { - Self::Fastly => canonicalize_fastly_transport_timeout_ms(remaining_ms, configured_ms), - Self::Axum | Self::Cloudflare | Self::Spin => remaining_ms.min(configured_ms), - } - } - - /// Return the target-specific backend-name budget available to auction providers. - #[must_use] - pub(crate) fn auction_dynamic_backend_budget(self) -> Option { - matches!(self, Self::Fastly).then_some( - FASTLY_DYNAMIC_BACKEND_LIMIT.saturating_sub(FASTLY_NON_AUCTION_BACKEND_RESERVE), - ) - } - - /// Count every transport-timeout bucket one provider can reach. - #[must_use] - pub(crate) fn transport_timeout_bucket_count(self, configured_ms: u32) -> usize { - if !matches!(self, Self::Fastly) || configured_ms == 0 { - return 1; - } - let mut buckets = std::collections::BTreeSet::new(); - buckets.insert(configured_ms); - for remaining_ms in FASTLY_SUB_QUANTUM_LADDER_MS { - let timeout = self.canonicalize_transport_timeout_ms(remaining_ms, configured_ms); - if timeout > 0 { - buckets.insert(timeout); - } - } - for remaining_ms in (FASTLY_TRANSPORT_TIMEOUT_QUANTUM_MS - ..FASTLY_TRANSPORT_TIMEOUT_QUANTUM_CEILING_MS) - .step_by(FASTLY_TRANSPORT_TIMEOUT_QUANTUM_MS as usize) - { - let timeout = self.canonicalize_transport_timeout_ms(remaining_ms, configured_ms); - if timeout > 0 { - buckets.insert(timeout); - } - } - for remaining_ms in FASTLY_TRANSPORT_TIMEOUT_COARSE_LADDER_MS { - let timeout = self.canonicalize_transport_timeout_ms(remaining_ms, configured_ms); - if timeout > 0 { - buckets.insert(timeout); - } - } - buckets.len() - } -} - -/// Canonical adapter target identifier accepted by Trusted Server tooling. -#[derive(Debug, Clone, Copy, Eq, PartialEq)] -pub enum AuctionTargetId { - /// Fastly Compute. - Fastly, - /// Native Axum development server. - Axum, - /// Cloudflare Workers. - Cloudflare, - /// Fermyon Spin. - Spin, -} - -impl AuctionTargetId { - /// Map a canonical `EdgeZero` adapter registry ID to an auction target. - #[must_use] - pub fn from_adapter_id(adapter_id: &str) -> Option { - match adapter_id { - "fastly" => Some(Self::Fastly), - "axum" => Some(Self::Axum), - "cloudflare" => Some(Self::Cloudflare), - "spin" => Some(Self::Spin), - _ => None, - } - } - - /// Return the canonical `EdgeZero` adapter registry ID. - #[must_use] - pub const fn adapter_id(self) -> &'static str { - match self { - Self::Fastly => "fastly", - Self::Axum => "axum", - Self::Cloudflare => "cloudflare", - Self::Spin => "spin", - } - } - - /// Return the shared naming and capability descriptor for this target. - #[must_use] - pub const fn descriptor(self) -> AuctionTargetDescriptor { - match self { - Self::Fastly => AuctionTargetDescriptor { - id: self, - naming_policy: BackendNamingPolicy::Fastly, - capabilities: AuctionTargetCapabilities { - concurrent_provider_fanout: true, - }, - }, - Self::Axum => AuctionTargetDescriptor { - id: self, - naming_policy: BackendNamingPolicy::Axum, - capabilities: AuctionTargetCapabilities { - concurrent_provider_fanout: true, - }, - }, - Self::Cloudflare => AuctionTargetDescriptor { - id: self, - naming_policy: BackendNamingPolicy::Cloudflare, - capabilities: AuctionTargetCapabilities { - concurrent_provider_fanout: false, - }, - }, - Self::Spin => AuctionTargetDescriptor { - id: self, - naming_policy: BackendNamingPolicy::Spin, - capabilities: AuctionTargetCapabilities { - concurrent_provider_fanout: false, - }, - }, - } - } -} - -/// Adapter capabilities relevant to auction dispatch validation. -#[derive(Debug, Clone, Copy, Eq, PartialEq)] -pub struct AuctionTargetCapabilities { - concurrent_provider_fanout: bool, -} - -impl AuctionTargetCapabilities { - /// Return whether multiple provider requests can be in flight concurrently. - #[must_use] - pub const fn supports_concurrent_provider_fanout(self) -> bool { - self.concurrent_provider_fanout - } -} - -/// Shared target descriptor used by plan validation. -#[derive(Debug, Clone, Copy, Eq, PartialEq)] -pub struct AuctionTargetDescriptor { - id: AuctionTargetId, - naming_policy: BackendNamingPolicy, - capabilities: AuctionTargetCapabilities, -} - -impl AuctionTargetDescriptor { - /// Return the canonical target identity. - #[must_use] - pub const fn id(self) -> AuctionTargetId { - self.id - } - - /// Return the pure backend naming and transport timer policy. - #[must_use] - pub const fn naming_policy(self) -> BackendNamingPolicy { - self.naming_policy - } - - /// Return this target's auction dispatch capabilities. - #[must_use] - pub const fn capabilities(self) -> AuctionTargetCapabilities { - self.capabilities - } -} - -fn default_port(scheme: &str, https_case_insensitive: bool) -> u16 { - let https = if https_case_insensitive { - scheme.eq_ignore_ascii_case("https") - } else { - scheme == "https" - }; - if https { 443 } else { 80 } -} - -fn sanitize_fastly_component(value: &str) -> String { - value - .chars() - .map(|ch| { - if ch.is_ascii_alphanumeric() || matches!(ch, '-' | '_') { - ch - } else { - '_' - } - }) - .collect() -} - -fn canonical_fastly_host(host: &str) -> &str { - host.strip_prefix('[') - .and_then(|value| value.strip_suffix(']')) - .filter(|value| value.parse::().is_ok()) - .unwrap_or(host) -} - -fn fastly_canonical_spec(spec: &PlatformBackendSpec, target_port: u16) -> String { - fn push_field(buffer: &mut String, field: &str) { - buffer.push_str(&field.len().to_string()); - buffer.push(':'); - buffer.push_str(field); - } - - let mut buffer = String::new(); - push_field(&mut buffer, &spec.scheme); - push_field(&mut buffer, canonical_fastly_host(&spec.host)); - push_field(&mut buffer, &target_port.to_string()); - push_field(&mut buffer, if spec.certificate_check { "1" } else { "0" }); - match spec.host_header_override.as_deref() { - Some(value) => { - buffer.push('s'); - push_field(&mut buffer, value); - } - None => buffer.push('n'), - } - match spec.discriminator.as_deref() { - Some(value) => { - buffer.push('s'); - push_field(&mut buffer, value); - } - None => buffer.push('n'), - } - push_field( - &mut buffer, - &spec.first_byte_timeout.as_millis().to_string(), - ); - push_field( - &mut buffer, - &spec.between_bytes_timeout.as_millis().to_string(), - ); - buffer -} - -fn fastly_spec_digest(canonical: &str) -> String { - let mut hasher = Sha256::new(); - hasher.update(canonical.as_bytes()); - let digest = hasher.finalize(); - let mut hex = String::with_capacity(FASTLY_SPEC_DIGEST_HEX_LEN); - for byte in digest.iter().take(FASTLY_SPEC_DIGEST_HEX_LEN / 2) { - write!(hex, "{byte:02x}").expect("should write hex digit to string"); - } - hex -} - -fn predict_fastly( - spec: &PlatformBackendSpec, -) -> Result> { - if spec.host.is_empty() { - return Err(Report::new(BackendNamingError::MissingHost)); - } - if spec.host.chars().any(char::is_control) { - return Err(Report::new(BackendNamingError::ControlCharacters { - field: "host", - })); - } - if spec.scheme.chars().any(char::is_control) { - return Err(Report::new(BackendNamingError::ControlCharacters { - field: "scheme", - })); - } - if let Some(host_header_override) = spec.host_header_override.as_deref() { - validate_host_header_override_value(host_header_override).map_err(|reason| { - Report::new(BackendNamingError::InvalidHostHeaderOverride { reason }) - })?; - } - - let port = spec - .port - .unwrap_or_else(|| default_port(&spec.scheme, true)); - let name_base = format!( - "{}_{}_{}", - spec.scheme, - canonical_fastly_host(&spec.host), - port - ); - let host_override_suffix = spec - .host_header_override - .as_deref() - .map(|host| format!("_oh_{}", sanitize_fastly_component(host))) - .unwrap_or_default(); - let cert_suffix = if spec.certificate_check { - "" - } else { - "_nocert" - }; - let discriminator_suffix = spec - .discriminator - .as_deref() - .map(|value| format!("_p_{}", sanitize_fastly_component(value))) - .unwrap_or_default(); - let readable_full = format!( - "{}{}{}{}_fb{}_bb{}", - sanitize_fastly_component(&name_base), - host_override_suffix, - cert_suffix, - discriminator_suffix, - spec.first_byte_timeout.as_millis(), - spec.between_bytes_timeout.as_millis() - ); - let readable = readable_full - .chars() - .take(MAX_FASTLY_READABLE_PREFIX_LEN) - .collect::(); - let digest = fastly_spec_digest(&fastly_canonical_spec(spec, port)); - let name = format!("backend_{readable}_{digest}"); - if name.len() > MAX_FASTLY_BACKEND_NAME_LEN { - return Err(Report::new(BackendNamingError::NameTooLong { - limit: MAX_FASTLY_BACKEND_NAME_LEN, - actual: name.len(), - })); - } - - Ok(PredictedBackend { name, port }) -} - -fn normalize_axum_segment(value: &str) -> String { - value.to_uppercase().replace(['-', '.', ' '], "_") -} - -fn predict_axum(spec: &PlatformBackendSpec) -> PredictedBackend { - let port = spec - .port - .unwrap_or_else(|| default_port(&spec.scheme, false)); - let discriminator = spec - .discriminator - .as_deref() - .map(|value| format!("_p_{}", normalize_axum_segment(value))) - .unwrap_or_default(); - PredictedBackend { - name: format!( - "{}_{}_{}{}", - normalize_axum_segment(&spec.scheme), - normalize_axum_segment(&spec.host), - port, - discriminator - ), - port, - } -} - -fn predict_no_registration(spec: &PlatformBackendSpec) -> PredictedBackend { - let port = spec - .port - .unwrap_or_else(|| default_port(&spec.scheme, false)); - let cert_suffix = if spec.certificate_check { - "" - } else { - "_nocert" - }; - let discriminator = spec - .discriminator - .as_deref() - .map(|value| format!("_p_{value}")) - .unwrap_or_default(); - PredictedBackend { - name: format!( - "{}_{}_{}_{}ms{cert_suffix}{discriminator}", - spec.scheme, - spec.host, - port, - spec.first_byte_timeout.as_millis() - ), - port, - } -} - -fn predict_cloudflare(spec: &PlatformBackendSpec) -> PredictedBackend { - predict_no_registration(spec) -} - -fn predict_spin(spec: &PlatformBackendSpec) -> PredictedBackend { - predict_no_registration(spec) -} - -fn canonicalize_fastly_transport_timeout_ms(remaining_ms: u32, configured_ms: u32) -> u32 { - if remaining_ms >= configured_ms { - return configured_ms; - } - if remaining_ms >= FASTLY_TRANSPORT_TIMEOUT_QUANTUM_CEILING_MS { - return FASTLY_TRANSPORT_TIMEOUT_COARSE_LADDER_MS - .into_iter() - .rev() - .find(|&rung| rung <= remaining_ms) - .unwrap_or(FASTLY_TRANSPORT_TIMEOUT_QUANTUM_CEILING_MS); - } - let floored = - (remaining_ms / FASTLY_TRANSPORT_TIMEOUT_QUANTUM_MS) * FASTLY_TRANSPORT_TIMEOUT_QUANTUM_MS; - if floored > 0 { - return floored; - } - FASTLY_SUB_QUANTUM_LADDER_MS - .into_iter() - .find(|&rung| rung <= remaining_ms) - .unwrap_or(0) -} - -#[cfg(test)] -mod tests { - use std::time::Duration; - - use super::*; - - fn spec() -> PlatformBackendSpec { - PlatformBackendSpec { - scheme: "https".to_owned(), - host: "origin.example.com".to_owned(), - port: None, - host_header_override: None, - certificate_check: true, - first_byte_timeout: Duration::from_millis(1500), - between_bytes_timeout: Duration::from_millis(1500), - discriminator: Some("provider-one".to_owned()), - } - } - - #[test] - fn adapter_prediction_outputs_are_pinned() { - let spec = spec(); - assert_eq!( - BackendNamingPolicy::Fastly - .predict(&spec) - .expect("should predict Fastly backend"), - PredictedBackend { - name: "backend_https_origin_example_com_443_p_provider-one_fb1500_bb1500_51fb8dba14db39e759e2c1c5d204a6eb".to_owned(), - port: 443, - } - ); - assert_eq!( - BackendNamingPolicy::Axum - .predict(&spec) - .expect("should predict Axum backend"), - PredictedBackend { - name: "HTTPS_ORIGIN_EXAMPLE_COM_443_p_PROVIDER_ONE".to_owned(), - port: 443, - } - ); - let no_registration = PredictedBackend { - name: "https_origin.example.com_443_1500ms_p_provider-one".to_owned(), - port: 443, - }; - assert_eq!( - BackendNamingPolicy::Cloudflare - .predict(&spec) - .expect("should predict Cloudflare backend"), - no_registration - ); - assert_eq!( - BackendNamingPolicy::Spin - .predict(&spec) - .expect("should predict Spin backend"), - no_registration - ); - } - - #[test] - fn fastly_prediction_canonicalizes_only_bracketed_ipv6_hosts() { - let mut bare = spec(); - bare.host = "2001:db8::1".to_string(); - bare.port = Some(8443); - let mut bracketed = bare.clone(); - bracketed.host = "[2001:db8::1]".to_string(); - - let bare_prediction = BackendNamingPolicy::Fastly - .predict(&bare) - .expect("should predict bare IPv6 backend"); - let bracketed_prediction = BackendNamingPolicy::Fastly - .predict(&bracketed) - .expect("should predict bracketed IPv6 backend"); - assert_eq!(bare_prediction, bracketed_prediction); - assert_eq!( - bare_prediction - .name - .rsplit_once('_') - .map(|(_, digest)| digest), - bracketed_prediction - .name - .rsplit_once('_') - .map(|(_, digest)| digest), - "canonical forms must hash the identical Fastly specification" - ); - - assert_eq!( - canonical_fastly_host("origin.example.com"), - "origin.example.com" - ); - assert_eq!(canonical_fastly_host("192.0.2.1"), "192.0.2.1"); - assert_eq!(canonical_fastly_host("[not-ipv6]"), "[not-ipv6]"); - } - - #[test] - fn target_descriptors_pin_all_capabilities_and_policies() { - let cases = [ - (AuctionTargetId::Fastly, true, BackendNamingPolicy::Fastly), - (AuctionTargetId::Axum, true, BackendNamingPolicy::Axum), - ( - AuctionTargetId::Cloudflare, - false, - BackendNamingPolicy::Cloudflare, - ), - (AuctionTargetId::Spin, false, BackendNamingPolicy::Spin), - ]; - for (id, fanout, naming_policy) in cases { - let descriptor = id.descriptor(); - assert_eq!(descriptor.id(), id); - assert_eq!(descriptor.naming_policy(), naming_policy); - assert_eq!( - descriptor - .capabilities() - .supports_concurrent_provider_fanout(), - fanout, - "should declare fanout accurately for {}", - id.adapter_id() - ); - assert_eq!(AuctionTargetId::from_adapter_id(id.adapter_id()), Some(id)); - } - assert_eq!(AuctionTargetId::from_adapter_id("FASTLY"), None); - assert_eq!(AuctionTargetId::from_adapter_id("unknown"), None); - } - - #[test] - fn timeout_policies_preserve_logical_transport_distinction() { - assert_eq!( - BackendNamingPolicy::Fastly.canonicalize_transport_timeout_ms(999, 2000), - 750 - ); - assert_eq!( - BackendNamingPolicy::Fastly.canonicalize_transport_timeout_ms(2000, 100), - 100 - ); - for policy in [ - BackendNamingPolicy::Axum, - BackendNamingPolicy::Cloudflare, - BackendNamingPolicy::Spin, - ] { - assert_eq!(policy.canonicalize_transport_timeout_ms(999, 2000), 999); - assert_eq!(policy.canonicalize_transport_timeout_ms(2000, 100), 100); - } - } -} diff --git a/crates/trusted-server-core/src/platform/error.rs b/crates/trusted-server-core/src/platform/error.rs index 12d0c39c69..2fbad44739 100644 --- a/crates/trusted-server-core/src/platform/error.rs +++ b/crates/trusted-server-core/src/platform/error.rs @@ -14,15 +14,6 @@ pub enum PlatformError { /// Secret store access failed. #[display("secret store error")] SecretStore, - /// Backend registration or name computation failed. - #[display("backend error")] - Backend, - /// HTTP client request failed. - #[display("http client error")] - HttpClient, - /// The absolute HTTP collection deadline elapsed. - #[display("http collection deadline exceeded")] - DeadlineExceeded, /// Geo lookup failed. #[display("geo lookup error")] Geo, diff --git a/crates/trusted-server-core/src/platform/http.rs b/crates/trusted-server-core/src/platform/http.rs deleted file mode 100644 index e1fe8ff1f2..0000000000 --- a/crates/trusted-server-core/src/platform/http.rs +++ /dev/null @@ -1,687 +0,0 @@ -use std::any::Any; -use std::fmt; - -use edgezero_core::http::{Method, Request as EdgeRequest, Response as EdgeResponse}; -use error_stack::Report; -use web_time::Instant; - -use super::PlatformError; -use super::image_optimizer::PlatformImageOptimizerOptions; - -/// What the caller wants the platform's intermediary cache to do with this request. -/// -/// One enum rather than independent flags because on Fastly they are *not* independent: -/// `set_surrogate_key` and `set_ttl` each "override any previous `Request::set_pass` call" -/// (`fastly-0.12.1/src/http/request.rs:2462`, `:2381`). A pair of booleans could express -/// "bypass the cache, and tag it for purge", which on Fastly silently means "do not -/// bypass" — the opposite of how it reads. This type makes that combination unspellable. -#[derive(Debug, Clone, PartialEq, Eq, Default)] -pub enum PlatformCacheIntent { - /// Let the platform apply its default behavior, honoring origin freshness. - #[default] - Default, - /// Do not use the intermediary cache for this request. - Bypass, - /// Allow caching, tagged with a surrogate key so it can be purged. - Shared { - /// Key attached to the stored object. - surrogate_key: String, - }, -} - -impl PlatformCacheIntent { - /// Whether this intent asks the platform to skip its cache entirely. - #[must_use] - pub fn is_bypass(&self) -> bool { - matches!(self, Self::Bypass) - } - - /// The surrogate key to tag the stored object with, when there is one. - /// - /// `None` for both [`Self::Default`] and [`Self::Bypass`]: attaching a key to a - /// bypassed request would reverse the bypass on Fastly. - #[must_use] - pub fn surrogate_key(&self) -> Option<&str> { - match self { - Self::Shared { surrogate_key } => Some(surrogate_key), - Self::Default | Self::Bypass => None, - } - } -} - -/// Outbound HTTP request paired with a pre-resolved backend name. -/// -/// Uses `EdgeZero`'s neutral [`EdgeRequest`] type so adapters share one -/// HTTP/body model while still preserving backend correlation for Fastly-style -/// fan-out. -#[derive(Debug)] -pub struct PlatformHttpRequest { - /// Platform-neutral request to send upstream. - pub request: EdgeRequest, - /// Backend name resolved ahead of time via `PlatformBackend`. - pub backend_name: String, - /// Optional Image Optimizer metadata for platforms that support it. - /// - /// Adapters that cannot attach this metadata to their send path should - /// return an error rather than silently dropping transformations. - pub image_optimizer: Option, - /// What the platform's intermediary response cache should do with this request. - /// - /// Adapters without an intermediary outbound cache may ignore it. Defaults to - /// [`PlatformCacheIntent::Default`] so existing call sites preserve their behavior. - pub cache_intent: PlatformCacheIntent, - /// Whether the response body should stay streaming in the platform response. - /// - /// Adapters that cannot preserve streaming response bodies should return an - /// error rather than silently buffering large asset responses. - pub stream_response: bool, -} - -impl PlatformHttpRequest { - /// Create a new outbound request wrapper. - #[must_use] - pub fn new(request: EdgeRequest, backend_name: impl Into) -> Self { - Self { - request, - backend_name: backend_name.into(), - image_optimizer: None, - cache_intent: PlatformCacheIntent::Default, - stream_response: false, - } - } - - /// Attach Image Optimizer metadata to the request. - /// - /// The current Fastly adapter supports this on [`PlatformHttpClient::send`] - /// and rejects it on [`PlatformHttpClient::send_async`] because Fastly IO is - /// not available through the fan-out helper path. - #[must_use] - pub fn with_image_optimizer(mut self, options: PlatformImageOptimizerOptions) -> Self { - self.image_optimizer = Some(options); - self - } - - /// Bypass the platform's intermediary response cache for this request. - #[must_use] - pub fn with_cache_bypass(mut self) -> Self { - self.cache_intent = PlatformCacheIntent::Bypass; - self - } - - /// Allow the platform to cache this response, tagged for purge. - /// - /// Replaces any prior bypass rather than combining with it, because the two cannot - /// both hold: see [`PlatformCacheIntent`]. - #[must_use] - pub fn with_shared_cache(mut self, surrogate_key: impl Into) -> Self { - self.cache_intent = PlatformCacheIntent::Shared { - surrogate_key: surrogate_key.into(), - }; - self - } - - /// Preserve the upstream response body as a stream when the adapter supports it. - /// - /// Asset routes use this to avoid materializing large image/static responses - /// into WASM memory before returning them to the client. - #[must_use] - pub fn with_stream_response(mut self) -> Self { - self.stream_response = true; - self - } -} - -/// Outbound HTTP response with optional backend correlation metadata. -#[derive(Debug)] -pub struct PlatformResponse { - /// Platform-neutral HTTP response. - pub response: EdgeResponse, - /// Backend that produced the response, when known. - pub backend_name: Option, -} - -impl PlatformResponse { - /// Create a response wrapper without backend metadata. - #[must_use] - pub fn new(response: EdgeResponse) -> Self { - Self { - response, - backend_name: None, - } - } - - /// Attach backend correlation metadata to the response. - #[must_use] - pub fn with_backend_name(mut self, backend_name: impl Into) -> Self { - self.backend_name = Some(backend_name.into()); - self - } -} - -/// Opaque handle for an in-flight outbound request. -/// -/// The core stores this as an opaque support type. Adapter implementations can -/// recover their concrete runtime handle through [`Self::downcast`]. -/// -/// # `!Send` design -/// -/// `inner` is `Box` (not `Box`) because all async -/// operations in this platform layer use `#[async_trait(?Send)]`. The `?Send` -/// bound exists because [`edgezero_core::body::Body`] wraps a -/// `LocalBoxStream` that is intentionally `!Send` for wasm32 compatibility — -/// wasm32 targets are single-threaded and cannot use `Send` futures. -/// Adapter crates targeting a multi-threaded runtime (e.g. Axum with tokio) -/// would need to wrap state in `Arc` rather than relying on `Send` here. -/// -/// See [`PlatformHttpClient`] for the trait-level `?Send` design rationale, -/// including why `Send + Sync` bounds on the trait type are compatible with -/// `?Send` futures. -pub struct PlatformPendingRequest { - inner: Box, - backend_name: Option, - stream_response: bool, - request_method: Option, -} - -impl PlatformPendingRequest { - /// Wrap an adapter-specific pending request handle. - #[must_use] - pub fn new(inner: T) -> Self - where - T: Any, - { - Self { - inner: Box::new(inner), - backend_name: None, - stream_response: false, - request_method: None, - } - } - - /// Attach backend correlation metadata to the pending request. - #[must_use] - pub fn with_backend_name(mut self, backend_name: impl Into) -> Self { - self.backend_name = Some(backend_name.into()); - self - } - - /// Return the correlated backend name when it is known before completion. - #[must_use] - pub fn backend_name(&self) -> Option<&str> { - self.backend_name.as_deref() - } - - /// Retain response-conversion metadata for direct single-handle waits. - #[must_use] - pub fn with_response_handling(mut self, stream_response: bool, request_method: Method) -> Self { - self.stream_response = stream_response; - self.request_method = Some(request_method); - self - } - - /// Whether completion must preserve the response body as a stream. - #[must_use] - pub fn stream_response(&self) -> bool { - self.stream_response - } - - /// Method of the originating request, when retained by the adapter. - #[must_use] - pub fn request_method(&self) -> Option<&Method> { - self.request_method.as_ref() - } - - /// Recover the adapter-specific pending request type. - /// - /// # Errors - /// - /// Returns `Err(self)` — the original wrapper with its backend metadata - /// preserved — when `T` does not match the stored type. - pub fn downcast(self) -> Result - where - T: Any, - { - let Self { - inner, - backend_name, - stream_response, - request_method, - } = self; - - match inner.downcast::() { - Ok(inner) => Ok(*inner), - Err(inner) => Err(Self { - inner, - backend_name, - stream_response, - request_method, - }), - } - } -} - -impl fmt::Debug for PlatformPendingRequest { - fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { - f.debug_struct("PlatformPendingRequest") - .field("backend_name", &self.backend_name) - .field("stream_response", &self.stream_response) - .field("request_method", &self.request_method) - .finish() - } -} - -/// Result of waiting for one in-flight request to complete. -#[derive(Debug)] -pub struct PlatformSelectResult { - /// Completed response, or the error returned by the ready request. - pub ready: Result>, - /// Requests still in flight after the ready result is removed. - /// - /// Note: entries in this list may have `backend_name == None` on some - /// adapters (e.g., Fastly), because the adapter cannot preserve input - /// identifiers across the platform `select()` call. Orchestrators must - /// not rely on `backend_name()` being set on remaining entries. - pub remaining: Vec, - /// Backend name of the request that became ready with an error, when - /// `ready` is `Err`. `None` on the success path and when the adapter - /// cannot identify the failed backend. - pub failed_backend_name: Option, -} - -/// A [`PlatformHttpClient`] stand-in used when outbound HTTP is not available -/// on the current platform (e.g. Cloudflare Workers, where the proxy client is -/// managed by the edgezero dispatch layer instead). -/// -/// Every method returns [`PlatformError::HttpClient`], ensuring that code paths -/// that reach this stub receive a typed error. Adapter crates should use this -/// type rather than defining their own stub so the fallback behaviour is -/// consistent across all platform implementations. -pub struct UnavailableHttpClient; - -#[async_trait::async_trait(?Send)] -impl PlatformHttpClient for UnavailableHttpClient { - async fn send( - &self, - _request: PlatformHttpRequest, - ) -> Result> { - Err(Report::new(PlatformError::HttpClient) - .attach("HTTP client is unavailable on this platform")) - } - - async fn send_async( - &self, - _request: PlatformHttpRequest, - ) -> Result> { - Err(Report::new(PlatformError::HttpClient) - .attach("HTTP client is unavailable on this platform")) - } - - async fn select( - &self, - _pending_requests: Vec, - ) -> Result> { - Err(Report::new(PlatformError::HttpClient) - .attach("HTTP client is unavailable on this platform")) - } -} - -/// Outbound HTTP client abstraction. -/// -/// Supports both single-request sends ([`Self::send`]) and async fan-out -/// ([`Self::send_async`] + [`Self::select`]) so adapters can drive parallel -/// upstream requests without additional abstractions. -/// -/// Object safety is provided by `async_trait`, which boxes the returned -/// futures behind `dyn PlatformHttpClient`. -/// -/// Uses `?Send` on all targets because [`edgezero_core::body::Body`] contains -/// a `LocalBoxStream` that is `!Send` by design for wasm32 compatibility. -#[async_trait::async_trait(?Send)] -pub trait PlatformHttpClient: Send + Sync { - /// Send a single upstream request and wait for the response. - /// - /// # Errors - /// - /// Returns `PlatformError::HttpClient` when the request cannot be sent or - /// the platform client fails before a response is produced. - async fn send( - &self, - request: PlatformHttpRequest, - ) -> Result>; - - /// Starts an upstream request without waiting for its response. - /// - /// Adapters retain ownership of the request operation so it can progress - /// while the publisher origin is fetched. Dropping its pending handle - /// cancels or abandons unfinished work according to the runtime contract. - /// - /// On Cloudflare and Spin the request runs as a task on the runtime - /// executor, so [`Self::wait`] and [`Self::select_until`] must be awaited - /// on that executor. Driving them through a blocking executor such as - /// `futures::executor::block_on` blocks the only thread the task can run - /// on and hangs. - /// - /// # Errors - /// - /// Returns `PlatformError::HttpClient` when the request cannot be - /// started. - async fn send_async( - &self, - request: PlatformHttpRequest, - ) -> Result>; - - /// Whether multiple pending requests can progress concurrently. - /// - /// Fastly and Axum provide concurrent fan-out. Cloudflare and Spin retain - /// a single-provider contract and return `false`; the auction orchestrator - /// checks this capability before launching more than one request. - fn supports_concurrent_fanout(&self) -> bool { - true - } - - /// Whether [`send`](Self::send) can preserve upstream response bodies as - /// [`Body::Stream`](edgezero_core::body::Body::Stream) when requested via - /// [`PlatformHttpRequest::with_stream_response`]. - /// - /// Adapters that cannot preserve streaming response bodies must keep the - /// default `false` so callers do not request a contract the adapter will - /// reject or silently buffer. - fn supports_streaming_responses(&self) -> bool { - false - } - - /// Whether a request started by [`send_async`](Self::send_async) can be - /// completed directly by [`wait`](Self::wait) while preserving a streamed - /// response body. - /// - /// Supporting adapters must override `wait` with direct single-handle - /// completion. A pending request marked for streaming must never be passed - /// to [`select`](Self::select), whose fan-out contract remains buffered. - fn supports_pending_streaming_responses(&self) -> bool { - false - } - - /// Wait for one of the in-flight requests to complete. - /// - /// # Errors - /// - /// Returns `PlatformError::HttpClient` if the platform cannot poll the - /// pending requests at all. - async fn select( - &self, - pending_requests: Vec, - ) -> Result>; - - /// Collects an available response, waiting only until an absolute deadline. - /// - /// On expiry, implementations take one nonblocking snapshot of complete - /// available responses before relinquishing unfinished requests. This - /// preserves buffered bids when a publisher origin delays collection. - /// Subsequent calls may drain only the completed snapshot. The deadline - /// bounds network waiting, not eligibility by an upstream arrival timestamp. - /// Returns `Ok(None)` when no complete response remains. Both header waits - /// and body reads must be bounded; unfinished work is canceled or abandoned. - /// Fastly cannot cancel backend requests with the installed SDK. - /// Stream-marked requests require the separate streaming wait contract. - /// - /// # Performance - /// - /// Complete response bodies are buffered within adapter-specific limits. - /// Deadline expiry drops incomplete bodies and pending handles after the - /// nonblocking snapshot; complete buffered responses remain collectible. - /// - /// # Examples - /// - /// ```no_run - /// # use trusted_server_core::platform::{PlatformHttpClient, PlatformPendingRequest, PlatformSelectResult, PlatformError}; - /// # use error_stack::Report; - /// # async fn collect(client: &dyn PlatformHttpClient, pending: Vec) -> Result, Report> { - /// let deadline = web_time::Instant::now() + std::time::Duration::from_millis(100); - /// client.select_until(pending, deadline).await - /// # } - /// ``` - /// - /// # Errors - /// - /// Returns [`PlatformError::Unsupported`] unless the adapter implements - /// bounded collection, or an HTTP error when polling itself fails. - async fn select_until( - &self, - pending_requests: Vec, - _deadline: Instant, - ) -> Result, Report> { - drop(pending_requests); - Err(Report::new(PlatformError::Unsupported) - .attach("absolute-deadline HTTP collection is unavailable")) - } - - /// Waits for a complete buffered response before an absolute deadline. - /// - /// Uses [`Self::select_until`] so expiry also relinquishes the pending - /// request and never falls back to an unbounded wait. - /// - /// # Examples - /// - /// ```no_run - /// # use trusted_server_core::platform::{PlatformHttpClient, PlatformPendingRequest, PlatformResponse, PlatformError}; - /// # use error_stack::Report; - /// # async fn collect(client: &dyn PlatformHttpClient, pending: PlatformPendingRequest) -> Result> { - /// let deadline = web_time::Instant::now() + std::time::Duration::from_millis(100); - /// client.wait_until(pending, deadline).await - /// # } - /// ``` - /// - /// # Errors - /// - /// Returns [`PlatformError::DeadlineExceeded`] on expiry, an adapter polling - /// error, or the attributed upstream error in the selected result. - async fn wait_until( - &self, - pending: PlatformPendingRequest, - deadline: Instant, - ) -> Result> { - self.select_until(vec![pending], deadline) - .await? - .ok_or_else(|| Report::new(PlatformError::DeadlineExceeded))? - .ready - } - - /// Wait for a single in-flight request to complete. - /// - /// This is a convenience wrapper around [`select`](Self::select) for the - /// common case where only one request is in flight. - /// - /// # Errors - /// - /// Returns `PlatformError::HttpClient` if the underlying `select` fails or - /// the response itself contains an error. - async fn wait( - &self, - pending: PlatformPendingRequest, - ) -> Result> { - self.select(vec![pending]).await?.ready - } -} - -#[cfg(test)] -mod tests { - use std::sync::{ - Arc, - atomic::{AtomicUsize, Ordering}, - }; - use std::time::Duration; - - use edgezero_core::body::Body; - use edgezero_core::http::request_builder; - - use super::*; - - struct DropProbe(Arc); - - impl Drop for DropProbe { - fn drop(&mut self) { - self.0.fetch_add(1, Ordering::SeqCst); - } - } - - #[test] - fn bounded_collection_default_relinquishes_expired_handles() { - let dropped = Arc::new(AtomicUsize::new(0)); - let response = futures::executor::block_on(UnavailableHttpClient.select_until( - vec![PlatformPendingRequest::new(DropProbe(Arc::clone(&dropped)))], - Instant::now(), - )) - .expect_err("should reject unsupported collection even after expiry"); - assert!(matches!( - response.current_context(), - PlatformError::Unsupported - )); - assert_eq!( - dropped.load(Ordering::SeqCst), - 1, - "should drop abandoned state" - ); - } - - #[test] - fn bounded_collection_default_fails_closed_instead_of_unbounded_polling() { - let error = futures::executor::block_on(UnavailableHttpClient.select_until( - vec![PlatformPendingRequest::new(())], - Instant::now() + Duration::from_secs(1), - )) - .expect_err("should reject unsupported bounded collection"); - assert!( - matches!(error.current_context(), PlatformError::Unsupported), - "should not delegate to unbounded select" - ); - } - - #[test] - fn unsupported_bounded_wait_reports_unsupported() { - let error = futures::executor::block_on( - UnavailableHttpClient.wait_until(PlatformPendingRequest::new(()), Instant::now()), - ) - .expect_err("should reject an unsupported wait"); - assert!( - matches!(error.current_context(), PlatformError::Unsupported), - "should preserve the unsupported collection contract" - ); - } - - #[test] - fn platform_http_request_cache_bypass_defaults_to_false() { - let request = PlatformHttpRequest::new( - request_builder() - .body(Body::empty()) - .expect("should build request"), - "stub-backend", - ); - - assert_eq!( - request.cache_intent, - PlatformCacheIntent::Default, - "should preserve existing cache behavior by default" - ); - } - - #[test] - fn platform_http_request_cache_bypass_builder_enables_bypass() { - let request = PlatformHttpRequest::new( - request_builder() - .body(Body::empty()) - .expect("should build request"), - "stub-backend", - ) - .with_cache_bypass(); - - assert_eq!( - request.cache_intent, - PlatformCacheIntent::Bypass, - "should enable intermediary cache bypass" - ); - } - - #[test] - fn cache_intent_cannot_request_bypass_and_a_surrogate_key_at_once() { - // The whole reason this is an enum. On Fastly a surrogate key reverses a prior - // set_pass, so "bypass, and tag for purge" would silently mean "do not bypass". - let bypass = PlatformCacheIntent::Bypass; - let shared = PlatformCacheIntent::Shared { - surrogate_key: "ts-origin".to_owned(), - }; - - assert!(bypass.is_bypass()); - assert!(bypass.surrogate_key().is_none()); - assert!(!shared.is_bypass()); - assert_eq!(shared.surrogate_key(), Some("ts-origin")); - assert!(PlatformCacheIntent::Default.surrogate_key().is_none()); - assert!(!PlatformCacheIntent::Default.is_bypass()); - } - - #[test] - fn the_last_cache_builder_call_wins_rather_than_combining() { - // Builders replace rather than accumulate: a request cannot end up asking for - // both, whichever order a caller writes them in. - let base = || { - PlatformHttpRequest::new( - request_builder() - .body(Body::empty()) - .expect("should build request"), - "stub-backend", - ) - }; - - assert_eq!( - base() - .with_cache_bypass() - .with_shared_cache("ts-origin") - .cache_intent, - PlatformCacheIntent::Shared { - surrogate_key: "ts-origin".to_owned() - } - ); - assert_eq!( - base() - .with_shared_cache("ts-origin") - .with_cache_bypass() - .cache_intent, - PlatformCacheIntent::Bypass - ); - } - - // --------------------------------------------------------------------------- - // Error-correlation interim scope (before EdgeZero #213) - // --------------------------------------------------------------------------- - - #[test] - fn platform_response_default_has_no_backend_name() { - // On Axum/Cloudflare noop clients return PlatformResponse::new(response) - // with no backend_name. Core logic must not panic when backend_name is None. - let response = edgezero_core::http::response_builder() - .status(200) - .body(edgezero_core::body::Body::empty()) - .expect("should build response"); - let resp = PlatformResponse::new(response); - // PlatformResponse has a public field, not a method. - // PlatformPendingRequest has backend_name() method; PlatformResponse does not. - assert_eq!( - resp.backend_name, None, - "PlatformResponse without backend_name must have None field" - ); - } - - #[test] - fn platform_response_with_backend_name_is_some() { - // On Fastly, responses carry backend_name for error correlation. - let response = edgezero_core::http::response_builder() - .status(200) - .body(edgezero_core::body::Body::empty()) - .expect("should build response"); - let resp = PlatformResponse::new(response).with_backend_name("prebid-backend"); - assert_eq!( - resp.backend_name.as_deref(), - Some("prebid-backend"), - "with_backend_name must set backend_name field" - ); - } -} diff --git a/crates/trusted-server-core/src/platform/image_optimizer.rs b/crates/trusted-server-core/src/platform/image_optimizer.rs deleted file mode 100644 index 9a3f1709eb..0000000000 --- a/crates/trusted-server-core/src/platform/image_optimizer.rs +++ /dev/null @@ -1,181 +0,0 @@ -//! Platform-neutral Image Optimizer request metadata. -//! -//! These DTOs describe the closed image transformation set that Trusted Server -//! attaches to an outbound asset proxy request. They intentionally avoid Fastly -//! SDK types so the profile-table logic can stay in core while the Fastly -//! adapter remains responsible for translating metadata into -//! `fastly::image_optimizer::ImageOptimizerOptions`. -//! -//! Unsupported adapters should reject requests carrying this metadata rather -//! than silently dropping transformations. - -/// Platform-neutral Image Optimizer processing region. -/// -/// These variants mirror the regions currently exposed by the Fastly SDK while -/// keeping config validation in core independent from adapter-specific types. -#[derive(Debug, Clone, Copy, PartialEq, Eq)] -pub enum PlatformImageOptimizerRegion { - /// Apply image optimization in the eastern United States. - UsEast, - /// Apply image optimization in the central United States. - UsCentral, - /// Apply image optimization in the western United States. - UsWest, - /// Apply image optimization in central Europe. - EuCentral, - /// Apply image optimization in western Europe. - EuWest, - /// Apply image optimization in Asia. - Asia, - /// Apply image optimization in Australia. - Australia, -} - -impl PlatformImageOptimizerRegion { - /// Parse a configured region string into a supported Image Optimizer region. - #[must_use] - pub fn parse(region: &str) -> Option { - match region - .trim() - .to_ascii_lowercase() - .replace('-', "_") - .as_str() - { - "us_east" | "us_east_1" => Some(Self::UsEast), - "us_central" | "us_central_1" => Some(Self::UsCentral), - "us_west" | "us_west_1" | "us_west_2" => Some(Self::UsWest), - "eu_central" | "eu_central_1" => Some(Self::EuCentral), - "eu_west" | "eu_west_1" => Some(Self::EuWest), - "asia" => Some(Self::Asia), - "australia" => Some(Self::Australia), - _ => None, - } - } -} - -/// Platform-neutral Image Optimizer options for an upstream request. -/// -/// Core code stores only a closed transformation set here. The Fastly adapter is -/// responsible for translating these values to SDK-specific -/// `ImageOptimizerOptions`, while adapters without Image Optimizer support -/// should reject requests carrying this metadata. -#[derive(Debug, Clone, PartialEq, Eq)] -pub struct PlatformImageOptimizerOptions { - /// Image Optimizer processing region understood by the target adapter. - pub region: String, - /// Whether non-IO query parameters should be preserved on the origin request. - pub preserve_query_string_on_origin_request: bool, - /// Transformation parameters to apply. - pub params: PlatformImageOptimizerParams, -} - -impl PlatformImageOptimizerOptions { - /// Create Image Optimizer options for the given region and params. - #[must_use] - pub fn new(region: impl Into, params: PlatformImageOptimizerParams) -> Self { - Self { - region: region.into(), - preserve_query_string_on_origin_request: false, - params, - } - } - - /// Preserve non-IO query parameters on the origin request. - /// - /// Asset routes with profile-table IO reject arbitrary query preservation by - /// default because client query parameters can otherwise become additional - /// Image Optimizer inputs. - #[must_use] - pub fn with_preserve_query_string_on_origin_request(mut self, preserve: bool) -> Self { - self.preserve_query_string_on_origin_request = preserve; - self - } -} - -/// Platform-neutral subset of image transformation parameters. -/// -/// This intentionally mirrors only the parameters accepted by asset-route -/// profile tables: format, quality, resize filter, dimensions, and crop. Client -/// query strings are converted into this closed set before the request reaches a -/// platform adapter. -#[derive(Debug, Default, Clone, PartialEq, Eq)] -pub struct PlatformImageOptimizerParams { - /// Output format such as `auto` or `webp`. - pub format: Option, - /// Output quality from 0 to 100. - pub quality: Option, - /// Resize filter such as `bicubic`. - pub resize_filter: Option, - /// Output width in pixels. - pub width: Option, - /// Output height in pixels. - pub height: Option, - /// Crop transformation. - pub crop: Option, -} - -impl PlatformImageOptimizerParams { - /// Merge another param set into this one, with `other` taking precedence. - pub fn merge_from(&mut self, other: Self) { - if other.format.is_some() { - self.format = other.format; - } - if other.quality.is_some() { - self.quality = other.quality; - } - if other.resize_filter.is_some() { - self.resize_filter = other.resize_filter; - } - if other.width.is_some() { - self.width = other.width; - } - if other.height.is_some() { - self.height = other.height; - } - if other.crop.is_some() { - self.crop = other.crop; - } - } -} - -/// Platform-neutral crop transformation. -#[derive(Debug, Clone, PartialEq, Eq)] -pub struct PlatformImageOptimizerCrop { - /// Aspect-ratio width component. - pub width: u32, - /// Aspect-ratio height component. - pub height: u32, - /// Optional crop focus mode. - pub mode: Option, - /// Optional x-axis crop offset bucket. - pub offset_x: Option, - /// Optional y-axis crop offset bucket. - pub offset_y: Option, -} - -impl PlatformImageOptimizerCrop { - /// Create a bare aspect-ratio crop. - #[must_use] - pub fn aspect_ratio(width: u32, height: u32) -> Self { - Self { - width, - height, - mode: None, - offset_x: None, - offset_y: None, - } - } - - /// Returns true when no focus mode or explicit offsets have been applied. - #[must_use] - pub fn is_bare_aspect_ratio(&self) -> bool { - self.mode.is_none() && self.offset_x.is_none() && self.offset_y.is_none() - } -} - -/// Platform-neutral crop focus mode. -#[derive(Debug, Clone, Copy, PartialEq, Eq)] -pub enum PlatformImageOptimizerCropMode { - /// Use Fastly IO smart crop mode. - Smart, -} diff --git a/crates/trusted-server-core/src/platform/mod.rs b/crates/trusted-server-core/src/platform/mod.rs index e4733106e3..d90307dee3 100644 --- a/crates/trusted-server-core/src/platform/mod.rs +++ b/crates/trusted-server-core/src/platform/mod.rs @@ -9,8 +9,6 @@ //! - [`PlatformConfigStore`] — key-value config store access //! - [`PlatformSecretStore`] — encrypted secret store access //! - [`PlatformKvStore`] — (re-exported from `edgezero_core`) -//! - [`PlatformBackend`] — dynamic backend registration -//! - [`PlatformHttpClient`] — outbound HTTP client //! - [`PlatformGeo`] — geographic information lookup //! - [`PlatformTemplateAssembler`] — cold-response shared-template assembly //! - [`PlatformTemplateCache`] — shared transformed-template caching @@ -32,14 +30,8 @@ //! No `PlatformContentRewriter` trait exists or is needed. //! -use std::time::Duration; - -mod backend_naming; mod error; -mod http; -mod image_optimizer; mod kv; -mod pending_response; mod template_assembly; mod template_cache; #[cfg(test)] @@ -48,22 +40,11 @@ mod timer; mod traits; mod types; -pub use backend_naming::{ - AuctionTargetCapabilities, AuctionTargetDescriptor, AuctionTargetId, BackendNamingError, - BackendNamingPolicy, PredictedBackend, -}; pub use edgezero_core::key_value_store::{KvError, KvHandle, KvStore as PlatformKvStore}; +pub use edgezero_core::outbound::HttpClient; +pub use edgezero_core::time::MonotonicClock; pub use error::PlatformError; -pub use http::{ - PlatformCacheIntent, PlatformHttpClient, PlatformHttpRequest, PlatformPendingRequest, - PlatformResponse, PlatformSelectResult, UnavailableHttpClient, -}; -pub use image_optimizer::{ - PlatformImageOptimizerCrop, PlatformImageOptimizerCropMode, PlatformImageOptimizerOptions, - PlatformImageOptimizerParams, PlatformImageOptimizerRegion, -}; pub use kv::UnavailableKvStore; -pub use pending_response::PlatformPendingResponse; pub use template_assembly::{ PlatformTemplateAssembler, TemplateAssemblyError, UnavailableTemplateAssembler, contains_publisher_esi_directive, @@ -76,14 +57,8 @@ pub use template_cache::{ UnavailableTemplateCache, VaryHeaderValues, VarySpec, reader_url_surrogate_key, }; pub use timer::PlatformTimer; -pub use traits::{PlatformBackend, PlatformConfigStore, PlatformGeo, PlatformSecretStore}; -pub use types::{ - ClientInfo, GeoInfo, PlatformBackendSpec, RuntimeServices, RuntimeServicesBuilder, StoreId, - StoreName, -}; - -/// Default first-byte timeout for platform backends. -pub(crate) const DEFAULT_FIRST_BYTE_TIMEOUT: Duration = Duration::from_secs(15); +pub use traits::{PlatformConfigStore, PlatformGeo, PlatformSecretStore}; +pub use types::{ClientInfo, GeoInfo, RuntimeServices, RuntimeServicesBuilder, StoreId, StoreName}; #[cfg(test)] mod tests { @@ -149,8 +124,6 @@ mod tests { fn _assert_config_store_object_safe(_: &dyn PlatformConfigStore) {} fn _assert_secret_store_object_safe(_: &dyn PlatformSecretStore) {} fn _assert_kv_store_object_safe(_: &dyn PlatformKvStore) {} - fn _assert_backend_object_safe(_: &dyn PlatformBackend) {} - fn _assert_http_client_object_safe(_: &dyn PlatformHttpClient) {} fn _assert_geo_object_safe(_: &dyn PlatformGeo) {} // Arc requires the trait impl to be Send + Sync. The assertion // below documents that RuntimeServices itself satisfies those bounds — the @@ -251,45 +224,6 @@ mod tests { ); } - #[test] - fn platform_pending_request_downcasts_and_preserves_backend_name() { - let pending = PlatformPendingRequest::new(7_u8).with_backend_name("origin"); - let pending = pending - .downcast::() - .expect_err("should reject downcast to the wrong pending type"); - assert_eq!( - pending.backend_name(), - Some("origin"), - "should preserve backend metadata when downcast fails" - ); - - let pending = PlatformPendingRequest::new(7_u8).with_backend_name("origin"); - let value = pending - .downcast::() - .expect("should recover the stored pending request type"); - assert_eq!(value, 7, "should preserve the stored pending request"); - } - - #[test] - fn platform_pending_request_preserves_response_handling_metadata() { - let pending = PlatformPendingRequest::new(7_u8) - .with_backend_name("origin") - .with_response_handling(true, edgezero_core::http::Method::HEAD); - let pending = pending - .downcast::() - .expect_err("should reject downcast to the wrong pending type"); - - assert!( - pending.stream_response(), - "should preserve the streaming response flag" - ); - assert_eq!( - pending.request_method(), - Some(&edgezero_core::http::Method::HEAD), - "should preserve the originating request method" - ); - } - #[test] fn geo_info_coordinates_string_formats_correctly() { let geo = GeoInfo { diff --git a/crates/trusted-server-core/src/platform/pending_response.rs b/crates/trusted-server-core/src/platform/pending_response.rs deleted file mode 100644 index 9590578df5..0000000000 --- a/crates/trusted-server-core/src/platform/pending_response.rs +++ /dev/null @@ -1,323 +0,0 @@ -//! Shared ownership and deadline collection for dispatched runtime operations. - -/// Receives a dispatched operation and cancels it when its owner is dropped. -pub struct PlatformPendingResponse { - response: futures::channel::oneshot::Receiver, - abort: futures::future::AbortHandle, -} - -impl PlatformPendingResponse { - /// Starts an operation with the caller's runtime spawner. - /// - /// The operation publishes only its complete result. Dropping this owner - /// aborts queued or running guest work when the executor next polls it. - /// Runtime-specific transport cancellation remains the caller's responsibility. - /// - /// # Examples - /// - /// ```no_run - /// use futures::task::LocalSpawnExt as _; - /// use trusted_server_core::platform::PlatformPendingResponse; - /// let pool = futures::executor::LocalPool::new(); - /// let pending = PlatformPendingResponse::start(async { 42 }, |task| { - /// pool.spawner().spawn_local(task).expect("should spawn operation"); - /// }); - /// ``` - pub fn start( - operation: impl std::future::Future + 'static, - spawn: impl FnOnce(futures::future::LocalBoxFuture<'static, ()>), - ) -> Self { - let (sender, response) = futures::channel::oneshot::channel(); - let (abort, registration) = futures::future::AbortHandle::new_pair(); - spawn(Box::pin(async move { - if let Ok(result) = futures::future::Abortable::new(operation, registration).await { - let _ = sender.send(result); - } - })); - Self { response, abort } - } - - /// Waits for the complete result while retaining cancellation ownership. - /// - /// # Errors - /// - /// Returns a channel error if the dispatched operation stops without a result. - /// - /// # Examples - /// - /// ```no_run - /// # use trusted_server_core::platform::PlatformPendingResponse; - /// # async fn collect(pending: &mut PlatformPendingResponse) -> Result { - /// let result = pending.wait().await?; - /// # Ok(result) - /// # } - /// ``` - pub async fn wait(&mut self) -> Result { - (&mut self.response).await - } - - /// Collects a buffered result, waiting at most until the supplied timer fires. - /// - /// An initial nonblocking probe retains results after an expired deadline. - /// A final probe handles a result published in the same turn as timer expiry. - /// The caller supplies the runtime timer and a monotonic clock; no transport - /// is polled directly here. Returns `None` when the deadline expires without - /// a complete buffered result. The owner remains alive throughout the wait. - /// - /// # Errors - /// - /// Returns a channel error inside `Some` if the operation stops without a result. - /// - /// # Examples - /// - /// ```no_run - /// # use trusted_server_core::platform::PlatformPendingResponse; - /// # async fn collect(pending: &mut PlatformPendingResponse, timer: impl std::future::Future, deadline: web_time::Instant) { - /// let result = pending.complete_before_deadline(timer, deadline, web_time::Instant::now).await; - /// # } - /// ``` - pub async fn complete_before_deadline( - &mut self, - timer: F, - deadline: web_time::Instant, - now: N, - ) -> Option> - where - F: std::future::Future, - N: Fn() -> web_time::Instant, - { - complete_before_deadline(&mut self.response, timer, deadline, now).await - } -} - -impl Drop for PlatformPendingResponse { - fn drop(&mut self) { - self.abort.abort(); - } -} - -async fn complete_before_deadline( - response: &mut futures::channel::oneshot::Receiver, - timer: T, - deadline: web_time::Instant, - now: N, -) -> Option> -where - T: std::future::Future, - N: Fn() -> web_time::Instant, -{ - // Inspect only the task's buffered result, including when collection starts - // after publisher-origin work has exhausted the auction budget. - match response.try_recv() { - Ok(Some(ready)) => return Some(Ok(ready)), - Err(error) => return Some(Err(error)), - Ok(None) => {} - } - if now() >= deadline { - return None; - } - match futures::future::select(response, Box::pin(timer)).await { - futures::future::Either::Left((ready, _)) => Some(ready), - // The timer and task can become ready in the same executor turn. A final - // channel probe retains buffered results without polling transport I/O. - futures::future::Either::Right(((), response)) => response.try_recv().transpose(), - } -} - -#[cfg(test)] -mod deadline_tests { - use std::cell::Cell; - use std::rc::Rc; - use std::time::Duration; - - use futures::executor::{LocalPool, block_on}; - use futures::future::{pending, ready}; - use futures::task::LocalSpawnExt as _; - use web_time::Instant; - - use super::{PlatformPendingResponse, complete_before_deadline}; - - #[test] - fn dispatched_task_starts_before_collection_and_buffers_complete_body() { - let mut executor = LocalPool::new(); - let spawner = executor.spawner(); - let started = Rc::new(Cell::new(false)); - let started_operation = Rc::clone(&started); - let (finish_body, body) = futures::channel::oneshot::channel(); - let mut pending = PlatformPendingResponse::start( - async move { - started_operation.set(true); - body.await.expect("should receive complete response body") - }, - |task| { - spawner - .spawn_local(task) - .expect("should spawn outbound task") - }, - ); - - executor.run_until_stalled(); - assert!(started.get(), "should start outbound work before selection"); - assert_eq!( - pending.response.try_recv().expect("should poll result"), - None, - "should not expose headers before response body completes" - ); - finish_body.send(42).expect("should finish response body"); - executor.run_until_stalled(); - assert_eq!( - pending - .response - .try_recv() - .expect("should poll buffered result"), - Some(42), - "should preserve a complete buffered response without polling transport" - ); - } - - #[test] - fn dropping_dispatched_pending_cancels_started_body_operation() { - struct DropProbe(Rc>); - impl Drop for DropProbe { - fn drop(&mut self) { - self.0.set(true); - } - } - let mut executor = LocalPool::new(); - let spawner = executor.spawner(); - let dropped = Rc::new(Cell::new(false)); - let probe = DropProbe(Rc::clone(&dropped)); - let pending = PlatformPendingResponse::start( - async move { - let _probe = probe; - futures::future::pending::<()>().await; - }, - |task| { - spawner - .spawn_local(task) - .expect("should spawn outbound task") - }, - ); - executor.run_until_stalled(); - assert!(!dropped.get(), "should retain an active body operation"); - drop(pending); - executor.run_until_stalled(); - assert!( - dropped.get(), - "should drop spawned operation after cancellation" - ); - } - - #[test] - fn dropping_queued_dispatch_prevents_transport_poll() { - let mut executor = LocalPool::new(); - let spawner = executor.spawner(); - let started = Rc::new(Cell::new(false)); - let operation_started = Rc::clone(&started); - let pending = PlatformPendingResponse::start( - async move { - operation_started.set(true); - }, - |task| { - spawner - .spawn_local(task) - .expect("should spawn outbound task") - }, - ); - drop(pending); - executor.run_until_stalled(); - assert!( - !started.get(), - "should cancel queued work before transport starts" - ); - } - - #[test] - fn expired_deadline_preserves_buffered_response() { - let (sender, mut response) = futures::channel::oneshot::channel(); - sender.send(42).expect("should buffer response"); - let deadline = Instant::now(); - let result = block_on(complete_before_deadline( - &mut response, - pending(), - deadline, - || deadline, - )); - assert_eq!( - result, - Some(Ok(42)), - "should retain a buffered response after cutoff" - ); - } - - #[test] - fn expired_deadline_returns_without_waiting_for_response() { - let (_sender, mut response) = futures::channel::oneshot::channel::<()>(); - let deadline = Instant::now(); - let result = block_on(complete_before_deadline( - &mut response, - pending(), - deadline, - || deadline, - )); - assert_eq!(result, None, "should stop waiting at an expired deadline"); - } - - #[test] - fn timer_cuts_off_pending_response() { - let (_sender, mut response) = futures::channel::oneshot::channel::<()>(); - let now = Instant::now(); - let result = block_on(complete_before_deadline( - &mut response, - ready(()), - now + Duration::from_secs(1), - || now, - )); - assert_eq!(result, None, "should cut off an unbuffered response"); - } - - #[test] - fn closed_response_channel_reports_failure() { - let (sender, mut response) = futures::channel::oneshot::channel::<()>(); - drop(sender); - let now = Instant::now(); - let result = block_on(complete_before_deadline( - &mut response, - pending(), - now + Duration::from_secs(1), - || now, - )); - assert!( - matches!(result, Some(Err(_))), - "should preserve a dispatched task failure" - ); - } - - #[test] - fn timer_rechecks_result_published_during_cutoff() { - let (sender, mut response) = futures::channel::oneshot::channel(); - let now = Instant::now(); - let clock = Cell::new(now); - let deadline = now + Duration::from_secs(1); - let timer = async { - clock.set(deadline); - sender.send(42).expect("should publish at cutoff"); - }; - let result = block_on(complete_before_deadline( - &mut response, - timer, - deadline, - || clock.get(), - )); - assert_eq!( - result, - Some(Ok(42)), - "should retain result buffered when timer wins" - ); - assert_eq!( - clock.get(), - deadline, - "should exercise the deadline boundary" - ); - } -} diff --git a/crates/trusted-server-core/src/platform/test_support.rs b/crates/trusted-server-core/src/platform/test_support.rs index be0b3feb8f..4fa341c5a3 100644 --- a/crates/trusted-server-core/src/platform/test_support.rs +++ b/crates/trusted-server-core/src/platform/test_support.rs @@ -1,31 +1,28 @@ -use std::collections::{HashMap, VecDeque}; +use std::collections::HashMap; use std::net::IpAddr; -use std::sync::{Arc, Mutex}; -use std::time::Duration; +use std::sync::Arc; use base64::{Engine as _, engine::general_purpose}; use ed25519_dalek::SigningKey; -use error_stack::{Report, ResultExt as _}; +use error_stack::Report; use rand::rngs::OsRng; -use web_time::Instant; use super::{ - ClientInfo, GeoInfo, PlatformBackend, PlatformBackendSpec, PlatformCacheIntent, - PlatformConfigStore, PlatformError, PlatformGeo, PlatformHttpClient, PlatformHttpRequest, - PlatformImageOptimizerOptions, PlatformImageOptimizerParams, PlatformPendingRequest, - PlatformResponse, PlatformSecretStore, PlatformSelectResult, RuntimeServices, StoreId, - StoreName, + ClientInfo, GeoInfo, PlatformConfigStore, PlatformError, PlatformGeo, PlatformSecretStore, + RuntimeServices, StoreId, StoreName, }; use crate::request_signing::{JWKS_STORE_NAME, SIGNING_STORE_NAME}; +use crate::test_support::scripted_client::{ScriptedHttpClient, TestClock}; pub(crate) struct NoopConfigStore; +#[async_trait::async_trait(?Send)] impl PlatformConfigStore for NoopConfigStore { fn get(&self, _store_name: &StoreName, _key: &str) -> Result> { Err(Report::new(PlatformError::Unsupported)) } - fn put( + async fn put( &self, _store_id: &StoreId, _key: &str, @@ -34,13 +31,14 @@ impl PlatformConfigStore for NoopConfigStore { Err(Report::new(PlatformError::Unsupported)) } - fn delete(&self, _store_id: &StoreId, _key: &str) -> Result<(), Report> { + async fn delete(&self, _store_id: &StoreId, _key: &str) -> Result<(), Report> { Err(Report::new(PlatformError::Unsupported)) } } pub(crate) struct NoopSecretStore; +#[async_trait::async_trait(?Send)] impl PlatformSecretStore for NoopSecretStore { fn get_bytes( &self, @@ -50,7 +48,7 @@ impl PlatformSecretStore for NoopSecretStore { Err(Report::new(PlatformError::Unsupported)) } - fn create( + async fn create( &self, _store_id: &StoreId, _name: &str, @@ -59,7 +57,7 @@ impl PlatformSecretStore for NoopSecretStore { Err(Report::new(PlatformError::Unsupported)) } - fn delete(&self, _store_id: &StoreId, _name: &str) -> Result<(), Report> { + async fn delete(&self, _store_id: &StoreId, _name: &str) -> Result<(), Report> { Err(Report::new(PlatformError::Unsupported)) } } @@ -74,6 +72,7 @@ impl HashMapConfigStore { } } +#[async_trait::async_trait(?Send)] impl PlatformConfigStore for HashMapConfigStore { fn get(&self, _store_name: &StoreName, key: &str) -> Result> { self.data @@ -82,7 +81,7 @@ impl PlatformConfigStore for HashMapConfigStore { .ok_or_else(|| Report::new(PlatformError::ConfigStore)) } - fn put( + async fn put( &self, _store_id: &StoreId, _key: &str, @@ -91,7 +90,7 @@ impl PlatformConfigStore for HashMapConfigStore { Err(Report::new(PlatformError::Unsupported)) } - fn delete(&self, _store_id: &StoreId, _key: &str) -> Result<(), Report> { + async fn delete(&self, _store_id: &StoreId, _key: &str) -> Result<(), Report> { Err(Report::new(PlatformError::Unsupported)) } } @@ -106,6 +105,7 @@ impl HashMapSecretStore { } } +#[async_trait::async_trait(?Send)] impl PlatformSecretStore for HashMapSecretStore { fn get_bytes( &self, @@ -118,7 +118,7 @@ impl PlatformSecretStore for HashMapSecretStore { .ok_or_else(|| Report::new(PlatformError::SecretStore)) } - fn create( + async fn create( &self, _store_id: &StoreId, _name: &str, @@ -127,854 +127,11 @@ impl PlatformSecretStore for HashMapSecretStore { Err(Report::new(PlatformError::Unsupported)) } - fn delete(&self, _store_id: &StoreId, _name: &str) -> Result<(), Report> { - Err(Report::new(PlatformError::Unsupported)) - } -} - -pub(crate) struct NoopBackend; - -impl PlatformBackend for NoopBackend { - fn naming_policy(&self) -> super::BackendNamingPolicy { - super::BackendNamingPolicy::Axum - } - - fn predict_name(&self, _spec: &PlatformBackendSpec) -> Result> { - Err(Report::new(PlatformError::Unsupported)) - } - - fn ensure(&self, _spec: &PlatformBackendSpec) -> Result> { + async fn delete(&self, _store_id: &StoreId, _name: &str) -> Result<(), Report> { Err(Report::new(PlatformError::Unsupported)) } } -pub(crate) struct NoopHttpClient; - -// ?Send matches PlatformHttpClient. Body wraps LocalBoxStream which is !Send -// by design; see http.rs for the full rationale. -#[async_trait::async_trait(?Send)] -impl PlatformHttpClient for NoopHttpClient { - async fn send( - &self, - _request: PlatformHttpRequest, - ) -> Result> { - Err(Report::new(PlatformError::Unsupported)) - } - - async fn send_async( - &self, - _request: PlatformHttpRequest, - ) -> Result> { - Err(Report::new(PlatformError::Unsupported)) - } - - async fn select( - &self, - _pending_requests: Vec, - ) -> Result> { - Err(Report::new(PlatformError::Unsupported)) - } -} - -// --------------------------------------------------------------------------- -// StubBackend -// --------------------------------------------------------------------------- - -/// Test stub for [`PlatformBackend`] that returns `"stub-backend"` for any -/// spec, allowing callers to proceed past backend registration. -pub(crate) struct StubBackend; - -impl PlatformBackend for StubBackend { - fn naming_policy(&self) -> super::BackendNamingPolicy { - super::BackendNamingPolicy::Axum - } - - fn predict_name(&self, _spec: &PlatformBackendSpec) -> Result> { - Ok("stub-backend".to_owned()) - } - - fn ensure(&self, _spec: &PlatformBackendSpec) -> Result> { - Ok("stub-backend".to_owned()) - } -} - -// --------------------------------------------------------------------------- -// StubHttpClient -// --------------------------------------------------------------------------- - -/// Canned response carried by a [`PlatformPendingRequest`] through `send_async` -/// and resolved by [`StubHttpClient::select`]. -struct StubPendingResponse { - backend_name: String, - cache_intent: PlatformCacheIntent, - status: u16, - body: Vec, - headers: Vec<(String, String)>, -} - -/// Test stub for [`PlatformHttpClient`] that records call backend names and -/// returns pre-queued canned responses for `send`, `send_async`, and `select`. -/// -/// Responses are stored as status/body/header parts to remain [`Send`]. -/// [`PlatformResponse`] contains [`edgezero_core::body::Body`] which wraps a -/// `LocalBoxStream` that is `!Send`, so it cannot be stored directly in a -/// `Mutex` field. -/// -/// Use [`push_response`](Self::push_response) to enqueue responses before -/// exercising the code under test, then inspect -/// [`recorded_backend_names`](Self::recorded_backend_names) to assert call -/// sites. -/// Upper bound on the request body bytes captured per `send` call. -const MAX_RECORDED_BODY_BYTES: usize = 64 * 1024 * 1024; -type RecordedHeaderBytes = Vec)>>; - -pub(crate) struct StubHttpClient { - calls: Mutex>, - responses: Mutex>, - // Headers captured per send call, stored as (name, value) string pairs. - request_headers: Mutex>>, - // Raw header values, including invalid UTF-8 values that cannot be represented - // by `recorded_request_headers`. - request_header_bytes: Mutex, - // Queued select() outcomes; true makes that select return ready: Err. - select_errors: Mutex>, - // Queued direct wait() errors for pending-stream failure-path tests. - wait_errors: Mutex>, - // Test-only wall-clock delays applied before each direct wait result. - wait_delays: Mutex>, - // Test-only overrides for backend metadata on returned pending handles. - pending_backend_name_overrides: Mutex>>, - // Test-only wall-clock delays applied before each select result is returned. - select_delays: Mutex>, - select_count: std::sync::atomic::AtomicUsize, - wait_count: std::sync::atomic::AtomicUsize, - // Reported by supports_concurrent_fanout(); set false to emulate - // platforms whose collection contract permits only a single provider. - concurrent_fanout: std::sync::atomic::AtomicBool, - // Reported by supports_streaming_responses(); set true to emulate Fastly's - // streaming response support. - streaming_responses_supported: std::sync::atomic::AtomicBool, - pending_streaming_responses_supported: std::sync::atomic::AtomicBool, - image_optimizer_options: Mutex>>, - cache_intents: Mutex>, - stream_response_flags: Mutex>, - request_methods: Mutex>, - request_uris: Mutex>, - // Outgoing request bodies captured per send call, collected to bytes. - request_bodies: Mutex>>, -} - -struct StubHttpResponse { - status: u16, - body: Vec, - headers: Vec<(String, String)>, - stream_body: bool, -} - -impl StubHttpClient { - /// Number of calls to the unbounded selection method. - pub(crate) fn recorded_select_count(&self) -> usize { - self.select_count.load(std::sync::atomic::Ordering::Relaxed) - } - - /// Number of calls to the unbounded single-request wait method. - pub(crate) fn recorded_wait_count(&self) -> usize { - self.wait_count.load(std::sync::atomic::Ordering::Relaxed) - } - - pub fn new() -> Self { - Self { - calls: Mutex::new(Vec::new()), - responses: Mutex::new(VecDeque::new()), - request_headers: Mutex::new(Vec::new()), - request_header_bytes: Mutex::new(Vec::new()), - select_errors: Mutex::new(VecDeque::new()), - wait_errors: Mutex::new(VecDeque::new()), - wait_delays: Mutex::new(VecDeque::new()), - pending_backend_name_overrides: Mutex::new(VecDeque::new()), - select_delays: Mutex::new(VecDeque::new()), - select_count: std::sync::atomic::AtomicUsize::new(0), - wait_count: std::sync::atomic::AtomicUsize::new(0), - concurrent_fanout: std::sync::atomic::AtomicBool::new(true), - streaming_responses_supported: std::sync::atomic::AtomicBool::new(false), - pending_streaming_responses_supported: std::sync::atomic::AtomicBool::new(false), - image_optimizer_options: Mutex::new(Vec::new()), - cache_intents: Mutex::new(Vec::new()), - stream_response_flags: Mutex::new(Vec::new()), - request_methods: Mutex::new(Vec::new()), - request_uris: Mutex::new(Vec::new()), - request_bodies: Mutex::new(Vec::new()), - } - } - - /// Make `supports_concurrent_fanout()` report the given value. - pub fn set_concurrent_fanout(&self, supported: bool) { - self.concurrent_fanout - .store(supported, std::sync::atomic::Ordering::Relaxed); - } - - /// Make `supports_streaming_responses()` report the given value. - pub fn set_streaming_responses_supported(&self, supported: bool) { - self.streaming_responses_supported - .store(supported, std::sync::atomic::Ordering::Relaxed); - } - - /// Make `supports_pending_streaming_responses()` report the given value. - pub fn set_pending_streaming_responses_supported(&self, supported: bool) { - self.pending_streaming_responses_supported - .store(supported, std::sync::atomic::Ordering::Relaxed); - } - - /// Queue a canned response by status code and body bytes. - pub fn push_response(&self, status: u16, body: Vec) { - self.push_response_with_headers(status, body, Vec::<(String, String)>::new()); - } - - /// Queue a canned response with headers. - pub fn push_response_with_headers( - &self, - status: u16, - body: Vec, - headers: Vec<(impl Into, impl Into)>, - ) { - let headers = headers - .into_iter() - .map(|(name, value)| (name.into(), value.into())) - .collect(); - self.responses - .lock() - .expect("should lock responses") - .push_back(StubHttpResponse { - status, - body, - headers, - stream_body: false, - }); - } - - /// Queue a canned response whose body is returned as a lazy stream. - pub fn push_streaming_response(&self, status: u16, body: Vec) { - self.responses - .lock() - .expect("should lock responses") - .push_back(StubHttpResponse { - status, - body, - headers: Vec::new(), - stream_body: true, - }); - } - - /// Inject a `select()` error: the next call to `select()` will return - /// `ready: Err(...)` with the failed request's backend name in - /// `failed_backend_name`. The corresponding queued response is consumed. - pub fn push_select_error(&self) { - self.select_errors - .lock() - .expect("should lock select_errors") - .push_back(true); - } - - /// Make the next `select()` complete successfully before a later queued error. - pub(crate) fn push_select_success(&self) { - self.select_errors - .lock() - .expect("should lock select_errors") - .push_back(false); - } - - /// Override backend metadata on the next pending handle returned by - /// [`Self::send_async`]. `None` removes the metadata entirely. - pub(crate) fn push_pending_backend_name_override(&self, backend_name: Option<&str>) { - self.pending_backend_name_overrides - .lock() - .expect("should lock pending backend name overrides") - .push_back(backend_name.map(str::to_string)); - } - - /// Queue a wall-clock delay before the next [`Self::select`] result. - /// - /// This is test-only timing control for deadline behavior. It deliberately - /// uses a caller-selected, generous contrast with the tested budget rather - /// than relying on scheduler races. - pub(crate) fn push_select_delay(&self, delay: Duration) { - self.select_delays - .lock() - .expect("should lock select_delays") - .push_back(delay); - } - - /// Queue a wall-clock delay before the next direct pending-request `wait()`. - pub(crate) fn push_wait_delay(&self, delay: Duration) { - self.wait_delays - .lock() - .expect("should lock wait_delays") - .push_back(delay); - } - - /// Inject an error for the next direct pending-request `wait()`. - pub fn push_wait_error(&self) { - self.wait_errors - .lock() - .expect("should lock wait_errors") - .push_back(()); - } - - /// Return backend names recorded across all `send` calls, in order. - pub fn recorded_backend_names(&self) -> Vec { - self.calls.lock().expect("should lock calls").clone() - } - - /// Return the request headers captured per `send` call, in order. - /// - /// Each entry is the set of `(name, value)` pairs from one call. - pub fn recorded_request_headers(&self) -> Vec> { - self.request_headers - .lock() - .expect("should lock request_headers") - .clone() - } - - /// Return raw request header values captured per request, in order. - /// - /// Unlike [`Self::recorded_request_headers`], this includes malformed bytes. - pub(crate) fn recorded_request_header_bytes(&self) -> Vec)>> { - self.request_header_bytes - .lock() - .expect("should lock request_header_bytes") - .clone() - } - - /// Return Image Optimizer metadata captured per `send` call, in order. - pub fn recorded_image_optimizer_options(&self) -> Vec> { - self.image_optimizer_options - .lock() - .expect("should lock image optimizer options") - .clone() - } - - /// Return the cache intent captured per `send` or `send_async` call, in order. - pub(crate) fn recorded_cache_intents(&self) -> Vec { - self.cache_intents - .lock() - .expect("should lock cache intents") - .clone() - } - - /// Return streaming-response flags captured per `send` call, in order. - pub fn recorded_stream_response_flags(&self) -> Vec { - self.stream_response_flags - .lock() - .expect("should lock stream response flags") - .clone() - } - - /// Return request methods captured per `send` call, in order. - pub fn recorded_request_methods(&self) -> Vec { - self.request_methods - .lock() - .expect("should lock request methods") - .clone() - } - - /// Return request URIs captured per `send` call, in order. - pub fn recorded_request_uris(&self) -> Vec { - self.request_uris - .lock() - .expect("should lock request URIs") - .clone() - } - - /// Return request bodies captured per `send` call, in order. - /// - /// Each entry is the outgoing request body collected to bytes. Bodies are - /// only captured by [`send`](PlatformHttpClient::send). - pub fn recorded_request_bodies(&self) -> Vec> { - self.request_bodies - .lock() - .expect("should lock request bodies") - .clone() - } -} - -// ?Send matches PlatformHttpClient. See http.rs for the full rationale. -#[async_trait::async_trait(?Send)] -impl PlatformHttpClient for StubHttpClient { - fn supports_concurrent_fanout(&self) -> bool { - self.concurrent_fanout - .load(std::sync::atomic::Ordering::Relaxed) - } - - fn supports_streaming_responses(&self) -> bool { - self.streaming_responses_supported - .load(std::sync::atomic::Ordering::Relaxed) - } - - fn supports_pending_streaming_responses(&self) -> bool { - self.pending_streaming_responses_supported - .load(std::sync::atomic::Ordering::Relaxed) - } - - async fn send( - &self, - request: PlatformHttpRequest, - ) -> Result> { - let stream_response = request.stream_response && self.supports_streaming_responses(); - let request_is_head = request.request.method() == edgezero_core::http::Method::HEAD; - self.calls - .lock() - .expect("should lock calls") - .push(request.backend_name.clone()); - - self.image_optimizer_options - .lock() - .expect("should lock image optimizer options") - .push(request.image_optimizer.clone()); - self.cache_intents - .lock() - .expect("should lock cache intents") - .push(request.cache_intent.clone()); - self.stream_response_flags - .lock() - .expect("should lock stream response flags") - .push(request.stream_response); - self.request_methods - .lock() - .expect("should lock request methods") - .push(request.request.method().to_string()); - self.request_uris - .lock() - .expect("should lock request URIs") - .push(request.request.uri().to_string()); - - let headers: Vec<(String, String)> = request - .request - .headers() - .iter() - .filter_map(|(name, value)| { - value - .to_str() - .ok() - .map(|v| (name.as_str().to_owned(), v.to_owned())) - }) - .collect(); - self.request_headers - .lock() - .expect("should lock request_headers") - .push(headers); - let header_bytes = request - .request - .headers() - .iter() - .map(|(name, value)| (name.as_str().to_owned(), value.as_bytes().to_vec())) - .collect(); - self.request_header_bytes - .lock() - .expect("should lock request_header_bytes") - .push(header_bytes); - - // Capture the outgoing request body so tests can assert it is forwarded. - // Propagate collection failures instead of recording an empty body, so - // tests cannot mistake a capture failure for an intentionally empty body. - let (_, body) = request.request.into_parts(); - let body_bytes = body - .into_bytes_bounded(MAX_RECORDED_BODY_BYTES) - .await - .change_context(PlatformError::HttpClient) - .attach("failed to capture StubHttpClient request body")? - .to_vec(); - self.request_bodies - .lock() - .expect("should lock request bodies") - .push(body_bytes); - - let response = self - .responses - .lock() - .expect("should lock responses") - .pop_front() - .ok_or_else(|| Report::new(PlatformError::HttpClient))?; - - let edge_response = build_stub_pending_response( - StubPendingResponse { - backend_name: request.backend_name, - cache_intent: request.cache_intent, - status: response.status, - body: response.body, - headers: response.headers, - }, - stream_response || response.stream_body, - request_is_head, - )?; - - Ok(PlatformResponse::new(edge_response)) - } - - async fn send_async( - &self, - request: PlatformHttpRequest, - ) -> Result> { - if request.image_optimizer.is_some() { - return Err(Report::new(PlatformError::HttpClient) - .attach("Image Optimizer is not supported with StubHttpClient send_async")); - } - if request.stream_response && !self.supports_pending_streaming_responses() { - return Err(Report::new(PlatformError::HttpClient) - .attach("streaming responses are not supported with StubHttpClient send_async")); - } - - let backend_name = request.backend_name.clone(); - let stream_response = request.stream_response; - let request_method = request.request.method().clone(); - self.calls - .lock() - .expect("should lock calls") - .push(backend_name.clone()); - self.cache_intents - .lock() - .expect("should lock cache intents") - .push(request.cache_intent.clone()); - self.stream_response_flags - .lock() - .expect("should lock stream response flags") - .push(stream_response); - self.request_methods - .lock() - .expect("should lock request methods") - .push(request_method.to_string()); - self.request_uris - .lock() - .expect("should lock request URIs") - .push(request.request.uri().to_string()); - - let headers: Vec<(String, String)> = request - .request - .headers() - .iter() - .filter_map(|(name, value)| { - value - .to_str() - .ok() - .map(|v| (name.as_str().to_string(), v.to_string())) - }) - .collect(); - self.request_headers - .lock() - .expect("should lock request_headers") - .push(headers); - let header_bytes = request - .request - .headers() - .iter() - .map(|(name, value)| (name.as_str().to_owned(), value.as_bytes().to_vec())) - .collect(); - self.request_header_bytes - .lock() - .expect("should lock request_header_bytes") - .push(header_bytes); - - // Capture the outgoing request body, mirroring `send()`, so tests - // exercising the async fan-out path (`request_bids` providers) can - // assert on it via `recorded_request_bodies()` too. - let (_, body) = request.request.into_parts(); - let body_bytes = body - .into_bytes_bounded(MAX_RECORDED_BODY_BYTES) - .await - .change_context(PlatformError::HttpClient) - .attach("failed to capture StubHttpClient request body")? - .to_vec(); - self.request_bodies - .lock() - .expect("should lock request bodies") - .push(body_bytes); - - let response = self - .responses - .lock() - .expect("should lock responses") - .pop_front() - .ok_or_else(|| Report::new(PlatformError::HttpClient))?; - - if response.stream_body { - return Err(Report::new(PlatformError::Unsupported) - .attach("streaming stub responses require StubHttpClient send")); - } - - let pending = StubPendingResponse { - backend_name: backend_name.clone(), - cache_intent: request.cache_intent, - status: response.status, - body: response.body, - headers: response.headers, - }; - let override_name = self - .pending_backend_name_overrides - .lock() - .expect("should lock pending backend name overrides") - .pop_front(); - let pending = PlatformPendingRequest::new(pending) - .with_response_handling(stream_response, request_method); - Ok(match override_name { - Some(Some(name)) => pending.with_backend_name(name), - Some(None) => pending, - None => pending.with_backend_name(backend_name), - }) - } - - async fn select_until( - &self, - mut pending_requests: Vec, - deadline: Instant, - ) -> Result, Report> { - // Match Fastly's per-provider rejection before waiting on network work. - let mut unchecked = pending_requests.into_iter(); - let mut checked = Vec::with_capacity(unchecked.len()); - while let Some(pending) = unchecked.next() { - match validate_deadline_cache_bypass(pending) { - Ok(pending) => checked.push(pending), - Err((backend_name, error)) => { - checked.extend(unchecked); - return Ok(Some(PlatformSelectResult { - ready: Err(error), - remaining: checked, - failed_backend_name: backend_name, - })); - } - } - } - pending_requests = checked; - let remaining = deadline.saturating_duration_since(Instant::now()); - let delay = self - .select_delays - .lock() - .expect("should lock select delays") - .front() - .copied() - .unwrap_or_default(); - if delay.is_zero() { - return self.select(pending_requests).await.map(Some); - } - if remaining.is_zero() { - return Ok(None); - } - if delay >= remaining { - self.select_delays - .lock() - .expect("should lock select delays") - .pop_front(); - std::thread::sleep(remaining); - return Ok(None); - } - let response = self.select(pending_requests).await?; - Ok((Instant::now() < deadline).then_some(response)) - } - - async fn wait_until( - &self, - pending: PlatformPendingRequest, - deadline: Instant, - ) -> Result> { - let pending = validate_deadline_cache_bypass(pending).map_err(|(_, error)| error)?; - let remaining = deadline.saturating_duration_since(Instant::now()); - let delay = self - .wait_delays - .lock() - .expect("should lock wait delays") - .front() - .copied() - .unwrap_or_default(); - if delay.is_zero() { - return self.wait(pending).await; - } - if remaining.is_zero() || delay >= remaining { - self.wait_delays - .lock() - .expect("should lock wait delays") - .pop_front(); - std::thread::sleep(remaining); - return Err(Report::new(PlatformError::DeadlineExceeded)); - } - let response = self.wait(pending).await?; - if Instant::now() >= deadline { - return Err(Report::new(PlatformError::DeadlineExceeded)); - } - Ok(response) - } - - /// Always marks the first pending request in the input as ready (FIFO order). - /// - /// This differs from Fastly's production `select()`, which returns whichever - /// request completes first and makes no ordering guarantees. Tests that rely on - /// this stub should not depend on "first-pushed = first-ready" semantics, and - /// should document their ordering assumptions explicitly if order matters. - async fn select( - &self, - mut pending_requests: Vec, - ) -> Result> { - self.select_count - .fetch_add(1, std::sync::atomic::Ordering::Relaxed); - if pending_requests.is_empty() { - return Err(Report::new(PlatformError::HttpClient) - .attach("select called with empty pending_requests list")); - } - - if pending_requests - .iter() - .any(PlatformPendingRequest::stream_response) - { - return Err(Report::new(PlatformError::HttpClient) - .attach("stream-marked pending request requires direct wait")); - } - - let delay = self - .select_delays - .lock() - .expect("should lock select_delays") - .pop_front(); - if let Some(delay) = delay { - std::thread::sleep(delay); - } - - let ready_platform = pending_requests.remove(0); - let stub = ready_platform - .downcast::() - .map_err(|_| { - Report::new(PlatformError::HttpClient) - .attach("unexpected inner type in StubHttpClient::select") - })?; - - let ready_backend_name = stub.backend_name.clone(); - - // Strip backend names from remaining to match Fastly production behavior: - // Fastly's select() rebuilds remaining with PlatformPendingRequest::new() - // (no backend_name) — orchestrators must not rely on names being set. - let remaining: Vec = pending_requests - .into_iter() - .map(|r| match r.downcast::() { - Ok(inner) => PlatformPendingRequest::new(inner), - Err(r) => r, - }) - .collect(); - - let should_error = self - .select_errors - .lock() - .expect("should lock select_errors") - .pop_front() - .unwrap_or(false); - - if should_error { - return Ok(PlatformSelectResult { - ready: Err(Report::new(PlatformError::HttpClient).attach(format!( - "injected select error for backend '{ready_backend_name}'" - ))), - remaining, - failed_backend_name: Some(ready_backend_name), - }); - } - - let edge_response = build_stub_pending_response(stub, false, false)?; - - let ready = Ok(PlatformResponse::new(edge_response).with_backend_name(ready_backend_name)); - - Ok(PlatformSelectResult { - ready, - remaining, - failed_backend_name: None, - }) - } - - async fn wait( - &self, - pending: PlatformPendingRequest, - ) -> Result> { - self.wait_count - .fetch_add(1, std::sync::atomic::Ordering::Relaxed); - let delay = self - .wait_delays - .lock() - .expect("should lock wait_delays") - .pop_front(); - if let Some(delay) = delay { - std::thread::sleep(delay); - } - - if self - .wait_errors - .lock() - .expect("should lock wait_errors") - .pop_front() - .is_some() - { - return Err( - Report::new(PlatformError::HttpClient).attach("injected direct pending wait error") - ); - } - let stream_response = pending.stream_response(); - let request_is_head = pending.request_method() == Some(&edgezero_core::http::Method::HEAD); - let stub = pending.downcast::().map_err(|_| { - Report::new(PlatformError::HttpClient) - .attach("unexpected inner type in StubHttpClient::wait") - })?; - let backend_name = stub.backend_name.clone(); - let response = build_stub_pending_response(stub, stream_response, request_is_head)?; - Ok(PlatformResponse::new(response).with_backend_name(backend_name)) - } -} - -// Require the same request marker as Fastly while retaining wrapper metadata. -fn validate_deadline_cache_bypass( - pending: PlatformPendingRequest, -) -> Result, Report)> { - let backend_name = pending.backend_name().map(str::to_owned); - let stream_response = pending.stream_response(); - let request_method = pending.request_method().cloned(); - let stub = pending.downcast::().map_err(|_| { - ( - backend_name.clone(), - Report::new(PlatformError::HttpClient) - .attach("unexpected pending type in bounded stub collection"), - ) - })?; - if !stub.cache_intent.is_bypass() { - return Err(( - Some(stub.backend_name), - Report::new(PlatformError::Unsupported) - .attach("deadline collection requires a cache-bypass request"), - )); - } - let mut pending = PlatformPendingRequest::new(stub); - if let Some(name) = backend_name { - pending = pending.with_backend_name(name); - } - if let Some(method) = request_method { - pending = pending.with_response_handling(stream_response, method); - } - Ok(pending) -} - -fn build_stub_pending_response( - stub: StubPendingResponse, - stream_response: bool, - request_is_head: bool, -) -> Result> { - let mut builder = edgezero_core::http::response_builder().status(stub.status); - for (name, value) in stub.headers { - builder = builder.header(name, value); - } - let carries_body = !request_is_head - && !(100..200).contains(&stub.status) - && !matches!(stub.status, 204 | 205 | 304); - let body = if !carries_body { - edgezero_core::body::Body::empty() - } else if stream_response { - edgezero_core::body::Body::stream(futures::stream::iter([bytes::Bytes::from(stub.body)])) - } else { - edgezero_core::body::Body::from(stub.body) - }; - builder.body(body).change_context(PlatformError::HttpClient) -} - pub(crate) struct NoopGeo; impl PlatformGeo for NoopGeo { @@ -995,8 +152,8 @@ pub(crate) fn build_services_with_config_and_secret( .config_store(Arc::new(config_store)) .secret_store(Arc::new(secret_store)) .kv_store(Arc::new(edgezero_core::key_value_store::NoopKvStore)) - .backend(Arc::new(NoopBackend)) - .http_client(Arc::new(NoopHttpClient)) + .outbound_client(ScriptedHttpClient::forbidding().http_client()) + .clock(TestClock::new().monotonic_clock()) .geo(Arc::new(NoopGeo)) .client_info(ClientInfo::default()) .build() @@ -1011,8 +168,8 @@ pub(crate) fn build_services_with_config_and_secret_and_client_ip( .config_store(Arc::new(config_store)) .secret_store(Arc::new(secret_store)) .kv_store(Arc::new(edgezero_core::key_value_store::NoopKvStore)) - .backend(Arc::new(NoopBackend)) - .http_client(Arc::new(NoopHttpClient)) + .outbound_client(ScriptedHttpClient::forbidding().http_client()) + .clock(TestClock::new().monotonic_clock()) .geo(Arc::new(NoopGeo)) .client_info(ClientInfo { client_ip: Some(client_ip), @@ -1048,8 +205,8 @@ pub(crate) fn build_services_with_config( .config_store(Arc::new(config_store)) .secret_store(Arc::new(NoopSecretStore)) .kv_store(Arc::new(edgezero_core::key_value_store::NoopKvStore)) - .backend(Arc::new(NoopBackend)) - .http_client(Arc::new(NoopHttpClient)) + .outbound_client(ScriptedHttpClient::forbidding().http_client()) + .clock(TestClock::new().monotonic_clock()) .geo(Arc::new(NoopGeo)) .client_info(ClientInfo::default()) .build() @@ -1059,6 +216,22 @@ pub(crate) fn noop_services() -> RuntimeServices { build_services_with_config(NoopConfigStore) } +/// Build test services whose `EdgeZero` outbound requests go to `client`. +/// +/// The services clock reads `clock`, so a test that scripts time shares one +/// clock between the scripted replies and the code under test. Every other +/// service is a no-op. +pub(crate) fn build_services_with_outbound_client( + client: &ScriptedHttpClient, + clock: &TestClock, +) -> RuntimeServices { + RuntimeServices { + outbound_client: client.http_client(), + clock: clock.monotonic_clock(), + ..noop_services() + } +} + /// Build a [`RuntimeServices`] whose auction telemetry sink is the supplied /// recording (or otherwise custom) sink, so tests can assert which terminal /// auction events were emitted. @@ -1069,53 +242,21 @@ pub(crate) fn noop_services_with_telemetry_sink( .config_store(Arc::new(NoopConfigStore)) .secret_store(Arc::new(NoopSecretStore)) .kv_store(Arc::new(edgezero_core::key_value_store::NoopKvStore)) - .backend(Arc::new(NoopBackend)) - .http_client(Arc::new(NoopHttpClient)) + .outbound_client(ScriptedHttpClient::forbidding().http_client()) + .clock(TestClock::new().monotonic_clock()) .geo(Arc::new(NoopGeo)) .auction_telemetry_sink(auction_telemetry_sink) .client_info(ClientInfo::default()) .build() } -/// Build a [`RuntimeServices`] with a caller-supplied HTTP client and a [`StubBackend`]. -/// -/// Uses [`StubBackend`] (always returns `Ok("stub-backend")`) rather than -/// [`NoopBackend`] (always returns `Err(Unsupported)`) so that handlers which -/// both make HTTP calls and resolve backends don't need two separate service -/// setups. If your test must verify that a missing backend returns an error, -/// use [`noop_services`] directly. -pub(crate) fn build_services_with_http_client( - http_client: Arc, -) -> RuntimeServices { - build_services_with_secret_and_http_client(NoopSecretStore, http_client) -} - -/// Build test services that dispatch HTTP requests with an attested client IP. -pub(crate) fn build_services_with_http_client_and_client_ip( - http_client: Arc, - client_ip: IpAddr, -) -> RuntimeServices { - RuntimeServices::builder() - .config_store(Arc::new(NoopConfigStore)) - .secret_store(Arc::new(NoopSecretStore)) - .kv_store(Arc::new(edgezero_core::key_value_store::NoopKvStore)) - .backend(Arc::new(StubBackend)) - .http_client(http_client) - .geo(Arc::new(NoopGeo)) - .client_info(ClientInfo { - client_ip: Some(client_ip), - ..ClientInfo::default() - }) - .build() -} - pub(crate) fn noop_services_with_client_ip(ip: IpAddr) -> RuntimeServices { RuntimeServices::builder() .config_store(Arc::new(NoopConfigStore)) .secret_store(Arc::new(NoopSecretStore)) .kv_store(Arc::new(edgezero_core::key_value_store::NoopKvStore)) - .backend(Arc::new(NoopBackend)) - .http_client(Arc::new(NoopHttpClient)) + .outbound_client(ScriptedHttpClient::forbidding().http_client()) + .clock(TestClock::new().monotonic_clock()) .geo(Arc::new(NoopGeo)) .client_info(ClientInfo { client_ip: Some(ip), @@ -1124,415 +265,10 @@ pub(crate) fn noop_services_with_client_ip(ip: IpAddr) -> RuntimeServices { .build() } -/// Build a [`RuntimeServices`] with a caller-supplied [`PlatformBackend`] and -/// HTTP client. -/// -/// Lets auction tests inject a backend whose -/// [`PlatformBackend::canonicalize_transport_timeout_ms`] returns a controlled -/// value, so the orchestrator's transport-timeout wiring can be asserted -/// deterministically without depending on wall-clock timing. -#[allow( - dead_code, - reason = "retained for target-specific transport-timeout tests" -)] -pub(crate) fn build_services_with_backend_and_http_client( - backend: Arc, - http_client: Arc, -) -> RuntimeServices { - RuntimeServices::builder() - .config_store(Arc::new(NoopConfigStore)) - .secret_store(Arc::new(NoopSecretStore)) - .kv_store(Arc::new(edgezero_core::key_value_store::NoopKvStore)) - .backend(backend) - .http_client(http_client) - .geo(Arc::new(NoopGeo)) - .client_info(ClientInfo { - client_ip: None, - tls_protocol: None, - tls_cipher: None, - ..ClientInfo::default() - }) - .build() -} - -/// Build a [`RuntimeServices`] with a custom secret store, [`StubBackend`], and HTTP client. -pub(crate) fn build_services_with_config_secret_and_http_client( - config_store: impl PlatformConfigStore + 'static, - secret_store: impl PlatformSecretStore + 'static, - http_client: Arc, -) -> RuntimeServices { - RuntimeServices::builder() - .config_store(Arc::new(config_store)) - .secret_store(Arc::new(secret_store)) - .kv_store(Arc::new(edgezero_core::key_value_store::NoopKvStore)) - .backend(Arc::new(StubBackend)) - .http_client(http_client) - .geo(Arc::new(NoopGeo)) - .client_info(ClientInfo::default()) - .build() -} - -pub(crate) fn build_services_with_secret_http_client_and_client_ip( - secret_store: impl PlatformSecretStore + 'static, - http_client: Arc, - client_ip: Option, -) -> RuntimeServices { - RuntimeServices::builder() - .config_store(Arc::new(NoopConfigStore)) - .secret_store(Arc::new(secret_store)) - .kv_store(Arc::new(edgezero_core::key_value_store::NoopKvStore)) - .backend(Arc::new(StubBackend)) - .http_client(http_client) - .geo(Arc::new(NoopGeo)) - .client_info(ClientInfo { - client_ip, - tls_protocol: None, - tls_cipher: None, - ..ClientInfo::default() - }) - .build() -} - -/// Build test services with a custom secret store and the standard test config store. -pub(crate) fn build_services_with_secret_and_http_client( - secret_store: impl PlatformSecretStore + 'static, - http_client: Arc, -) -> RuntimeServices { - build_services_with_config_secret_and_http_client(NoopConfigStore, secret_store, http_client) -} - #[cfg(test)] mod tests { - use crate::platform::DEFAULT_FIRST_BYTE_TIMEOUT; - use edgezero_core::body::Body; - use edgezero_core::http::request_builder; - use super::*; - fn dispatch_cache_test_request( - stub: &StubHttpClient, - backend: &str, - bypass: bool, - ) -> PlatformPendingRequest { - stub.push_response(200, b"ready".to_vec()); - let mut request = PlatformHttpRequest::new( - request_builder() - .uri("https://example.com/bid") - .body(Body::empty()) - .expect("should build request"), - backend, - ); - if bypass { - request = request.with_cache_bypass(); - } - futures::executor::block_on(stub.send_async(request)).expect("should dispatch request") - } - - #[test] - fn bounded_stub_collection_rejects_missing_bypass_without_losing_siblings() { - for unsupported_first in [false, true] { - let stub = StubHttpClient::new(); - let unsupported = dispatch_cache_test_request(&stub, "unsupported", false); - let healthy = dispatch_cache_test_request(&stub, "healthy", true); - let selected = futures::executor::block_on(stub.select_until( - if unsupported_first { - vec![unsupported, healthy] - } else { - vec![healthy, unsupported] - }, - Instant::now(), - )) - .expect("should attribute unsupported request") - .expect("should return unsupported result"); - assert!( - matches!( - selected - .ready - .expect_err("should reject missing bypass") - .current_context(), - PlatformError::Unsupported - ), - "should match Fastly's cache bypass requirement" - ); - assert_eq!( - selected.failed_backend_name.as_deref(), - Some("unsupported"), - "should identify unsupported provider" - ); - let selected = - futures::executor::block_on(stub.select_until(selected.remaining, Instant::now())) - .expect("should collect sibling") - .expect("should retain ready sibling"); - assert_eq!( - selected - .ready - .expect("should accept bypassed request") - .backend_name - .as_deref(), - Some("healthy"), - "should preserve healthy provider" - ); - } - } - - #[test] - fn bounded_stub_wait_rejects_missing_bypass() { - let stub = StubHttpClient::new(); - let pending = dispatch_cache_test_request(&stub, "unsupported", false); - let error = futures::executor::block_on(stub.wait_until(pending, Instant::now())) - .expect_err("should reject missing bypass"); - assert!( - matches!(error.current_context(), PlatformError::Unsupported), - "should reject unsupported deadline wait" - ); - } - - #[test] - fn unbounded_stub_wait_accepts_default_cache_intent() { - let stub = StubHttpClient::new(); - let pending = dispatch_cache_test_request(&stub, "ordinary", false); - let response = futures::executor::block_on(stub.wait(pending)) - .expect("should preserve ordinary waits"); - assert_eq!( - response.backend_name.as_deref(), - Some("ordinary"), - "should preserve backend" - ); - } - - #[test] - fn bounded_stub_wait_preserves_an_already_ready_response_after_expiry() { - let stub = StubHttpClient::new(); - stub.push_response(200, b"ready".to_vec()); - let pending = futures::executor::block_on( - stub.send_async( - PlatformHttpRequest::new( - request_builder() - .uri("https://example.com/ready") - .body(Body::empty()) - .expect("should build request"), - "ready-backend", - ) - .with_cache_bypass(), - ), - ) - .expect("should dispatch ready response"); - let response = futures::executor::block_on(stub.wait_until(pending, Instant::now())) - .expect("should retain the ready response after expiry"); - assert_eq!( - response.response.status().as_u16(), - 200, - "should preserve ready status" - ); - } - - #[test] - fn stub_http_client_records_send_calls_and_returns_canned_response() { - let stub = StubHttpClient::new(); - stub.push_response(200, b"hello".to_vec()); - - let req = PlatformHttpRequest::new( - request_builder() - .method("GET") - .uri("https://example.com/test") - .body(Body::empty()) - .expect("should build request"), - "stub-backend", - ); - let result = futures::executor::block_on(stub.send(req)); - - assert!(result.is_ok(), "should return canned response"); - let names = stub.recorded_backend_names(); - assert_eq!( - names, - vec!["stub-backend"], - "should record the backend name" - ); - assert_eq!( - stub.recorded_cache_intents(), - vec![PlatformCacheIntent::Default], - "should record the default cache intent" - ); - } - - #[test] - fn stub_http_client_returns_error_when_no_response_queued() { - let stub = StubHttpClient::new(); - - let req = PlatformHttpRequest::new( - request_builder() - .method("GET") - .uri("https://example.com/") - .body(Body::empty()) - .expect("should build request"), - "stub-backend", - ); - let result = futures::executor::block_on(stub.send(req)); - - assert!(result.is_err(), "should return error when queue is empty"); - assert!( - matches!( - result.unwrap_err().current_context(), - PlatformError::HttpClient - ), - "should be HttpClient error" - ); - } - - #[test] - fn stub_http_client_send_async_and_select_fan_out() { - let stub = StubHttpClient::new(); - stub.push_response(200, b"provider-a".to_vec()); - stub.push_response(201, b"provider-b".to_vec()); - - let make_req = |backend: &str| { - PlatformHttpRequest::new( - request_builder() - .method("GET") - .uri("https://example.com/bid") - .body(Body::empty()) - .expect("should build request"), - backend, - ) - }; - - let pending_a = futures::executor::block_on(stub.send_async(make_req("backend-a"))) - .expect("should start request a"); - let pending_b = - futures::executor::block_on(stub.send_async(make_req("backend-b").with_cache_bypass())) - .expect("should start request b"); - - assert_eq!( - pending_a.backend_name(), - Some("backend-a"), - "should attach backend name to pending request a" - ); - assert_eq!( - pending_b.backend_name(), - Some("backend-b"), - "should attach backend name to pending request b" - ); - - let result = futures::executor::block_on(stub.select(vec![pending_a, pending_b])) - .expect("should select first ready request"); - - let ready_resp = result.ready.expect("should have a ready response"); - assert_eq!( - ready_resp.backend_name.as_deref(), - Some("backend-a"), - "should correlate ready response to backend-a" - ); - assert_eq!( - result.remaining.len(), - 1, - "should have one remaining request" - ); - assert_eq!( - result.remaining[0].backend_name(), - None, - "should strip backend name from remaining (matches Fastly production behavior)" - ); - - let names = stub.recorded_backend_names(); - assert_eq!( - names, - vec!["backend-a", "backend-b"], - "should record both send_async calls in order" - ); - assert_eq!( - stub.recorded_cache_intents(), - vec![PlatformCacheIntent::Default, PlatformCacheIntent::Bypass], - "should record both send_async cache intents in order" - ); - } - - #[test] - fn stub_http_client_send_async_rejects_image_optimizer_metadata() { - let stub = StubHttpClient::new(); - let req = PlatformHttpRequest::new( - request_builder() - .method("GET") - .uri("https://example.com/image.jpg") - .body(Body::empty()) - .expect("should build request"), - "stub-backend", - ) - .with_image_optimizer(PlatformImageOptimizerOptions::new( - "us_east", - PlatformImageOptimizerParams::default(), - )); - - let err = futures::executor::block_on(stub.send_async(req)) - .expect_err("should reject async Image Optimizer metadata"); - - assert!( - format!("{err:?}").contains("Image Optimizer"), - "should explain unsupported async IO path: {err:?}" - ); - } - - #[test] - fn stub_http_client_pending_stream_wait_is_direct_and_preserves_streaming() { - let stub = StubHttpClient::new(); - stub.set_streaming_responses_supported(true); - stub.set_pending_streaming_responses_supported(true); - stub.push_response(200, b"streamed publisher body".to_vec()); - // A direct single-handle wait must not route through select(). - stub.push_select_error(); - let req = PlatformHttpRequest::new( - request_builder() - .method("GET") - .uri("https://example.com/image.jpg") - .body(Body::empty()) - .expect("should build request"), - "stub-backend", - ) - .with_stream_response(); - - let pending = futures::executor::block_on(stub.send_async(req)) - .expect("should start async streaming-response request"); - let response = futures::executor::block_on(stub.wait(pending)) - .expect("should wait directly for streaming response"); - - assert!( - response.response.body().is_stream(), - "should preserve the pending response body as a stream" - ); - assert_eq!( - stub.recorded_stream_response_flags(), - vec![true], - "should record the streaming response request" - ); - } - - #[test] - fn stub_http_client_select_returns_error_when_empty() { - let stub = StubHttpClient::new(); - let err = futures::executor::block_on(stub.select(vec![])) - .expect_err("should return error for empty list"); - assert!( - matches!(err.current_context(), PlatformError::HttpClient), - "should be HttpClient error" - ); - } - - #[test] - fn stub_backend_returns_fixed_name() { - let stub = StubBackend; - let spec = PlatformBackendSpec { - scheme: "https".to_owned(), - host: "example.com".to_owned(), - port: None, - host_header_override: None, - certificate_check: true, - first_byte_timeout: DEFAULT_FIRST_BYTE_TIMEOUT, - between_bytes_timeout: DEFAULT_FIRST_BYTE_TIMEOUT, - discriminator: None, - }; - let name = stub.ensure(&spec).expect("should return a backend name"); - assert_eq!(name, "stub-backend", "should return fixed name"); - } - #[test] fn build_services_with_config_and_secret_uses_provided_stores() { // Arrange: noop stores diff --git a/crates/trusted-server-core/src/platform/traits.rs b/crates/trusted-server-core/src/platform/traits.rs index 6c31cd2796..c58ca38208 100644 --- a/crates/trusted-server-core/src/platform/traits.rs +++ b/crates/trusted-server-core/src/platform/traits.rs @@ -2,13 +2,16 @@ use std::net::IpAddr; use error_stack::Report; -use super::{BackendNamingPolicy, GeoInfo, PlatformBackendSpec, PlatformError, StoreId, StoreName}; +use super::{GeoInfo, PlatformError, StoreId, StoreName}; -/// Synchronous, object-safe access to a key-value config store. +/// Object-safe access to a key-value config store. /// -/// Reads use the edge-visible store name. Writes use the platform management -/// store identifier because Fastly separates the runtime store name from the +/// Reads are synchronous and use the edge-visible store name. Writes are async, +/// because a platform can make them through its management API over the +/// request's outbound client, and they use the platform management store +/// identifier, because Fastly separates the runtime store name from the /// management API store ID. +#[async_trait::async_trait(?Send)] pub trait PlatformConfigStore: Send + Sync { /// Retrieve a string value from `store_name` by `key`. /// @@ -24,7 +27,12 @@ pub trait PlatformConfigStore: Send + Sync { /// /// Returns [`PlatformError::ConfigStore`] when the write fails or the /// platform management API is unreachable. - fn put(&self, store_id: &StoreId, key: &str, value: &str) -> Result<(), Report>; + async fn put( + &self, + store_id: &StoreId, + key: &str, + value: &str, + ) -> Result<(), Report>; /// Delete a key from the management store identified by `store_id`. /// @@ -32,13 +40,15 @@ pub trait PlatformConfigStore: Send + Sync { /// /// Returns [`PlatformError::ConfigStore`] when the delete fails or the /// platform management API is unreachable. - fn delete(&self, store_id: &StoreId, key: &str) -> Result<(), Report>; + async fn delete(&self, store_id: &StoreId, key: &str) -> Result<(), Report>; } -/// Synchronous, object-safe access to a secret store. +/// Object-safe access to a secret store. /// -/// Reads use the edge-visible store name. Writes use the platform management -/// store identifier. +/// Reads are synchronous and use the edge-visible store name. Writes are async, +/// for the reason [`PlatformConfigStore`] gives, and use the platform +/// management store identifier. +#[async_trait::async_trait(?Send)] pub trait PlatformSecretStore: Send + Sync { /// Retrieve a secret value as raw bytes from `store_name` by `key`. /// @@ -76,7 +86,7 @@ pub trait PlatformSecretStore: Send + Sync { /// /// Returns [`PlatformError::SecretStore`] when the create fails or the /// platform management API is unreachable. - fn create( + async fn create( &self, store_id: &StoreId, name: &str, @@ -89,51 +99,7 @@ pub trait PlatformSecretStore: Send + Sync { /// /// Returns [`PlatformError::SecretStore`] when the delete fails or the /// platform management API is unreachable. - fn delete(&self, store_id: &StoreId, name: &str) -> Result<(), Report>; -} - -/// Synchronous, object-safe dynamic backend management. -pub trait PlatformBackend: Send + Sync { - /// Return this adapter's pure backend naming and transport timer policy. - fn naming_policy(&self) -> BackendNamingPolicy; - - /// Compute the deterministic backend name for the given spec without - /// registering anything. - /// - /// # Errors - /// - /// Returns [`PlatformError::Backend`] when the spec is invalid or the - /// name cannot be computed. - fn predict_name(&self, spec: &PlatformBackendSpec) -> Result>; - - /// Ensure a dynamic backend exists for the given spec and return its name. - /// - /// # Errors - /// - /// Returns [`PlatformError::Backend`] when the backend cannot be - /// registered on the platform. - fn ensure(&self, spec: &PlatformBackendSpec) -> Result>; - - /// Canonicalize a per-provider transport timeout for backend-name stability. - /// - /// `remaining_ms` is the wall-clock budget left in the auction and - /// `configured_ms` is the provider's own configured timeout. The returned - /// value is used both to derive the dynamic backend name and as the - /// provider's request deadline, so it must be identical for prediction and - /// registration of the same launch. - /// - /// Adapters that embed the transport timeout in the dynamic backend name - /// (Fastly) override this to round budget-derived values to a coarse - /// ladder, so per-request wall-clock jitter neither defeats cross-request - /// connection pooling nor accumulates registrations toward the per-service - /// dynamic backend limit. - /// - /// Delegates to the same pure policy used by startup validation so runtime - /// transport timers and predicted names cannot drift. - fn canonicalize_transport_timeout_ms(&self, remaining_ms: u32, configured_ms: u32) -> u32 { - self.naming_policy() - .canonicalize_transport_timeout_ms(remaining_ms, configured_ms) - } + async fn delete(&self, store_id: &StoreId, name: &str) -> Result<(), Report>; } /// Synchronous, object-safe geo lookup. diff --git a/crates/trusted-server-core/src/platform/types.rs b/crates/trusted-server-core/src/platform/types.rs index 760dba0e52..20f767fa68 100644 --- a/crates/trusted-server-core/src/platform/types.rs +++ b/crates/trusted-server-core/src/platform/types.rs @@ -1,13 +1,14 @@ use std::fmt; use std::net::IpAddr; use std::sync::Arc; -use std::time::Duration; + +use edgezero_core::outbound::HttpClient; +use edgezero_core::time::MonotonicClock; use crate::auction::telemetry::{AuctionTelemetrySink, NoopAuctionTelemetrySink}; use super::{ - PlatformBackend, PlatformConfigStore, PlatformGeo, PlatformHttpClient, PlatformKvStore, - PlatformSecretStore, PlatformTimer, + PlatformConfigStore, PlatformGeo, PlatformKvStore, PlatformSecretStore, PlatformTimer, }; /// Geographic information extracted from a request. @@ -123,38 +124,6 @@ impl AsRef for StoreId { } } -/// Input specification for a dynamic backend. -/// -/// Passed to [`PlatformBackend::predict_name`] and [`PlatformBackend::ensure`] -/// to deterministically name and register upstream origins. -#[derive(Debug, Clone)] -pub struct PlatformBackendSpec { - /// URL scheme. - pub scheme: String, - /// Hostname of the backend origin. - pub host: String, - /// Explicit port, or `None` to use the scheme default. - pub port: Option, - /// Optional outbound `Host` header override for backend registration. - pub host_header_override: Option, - /// Whether to verify the TLS certificate. - pub certificate_check: bool, - /// Maximum time to wait for the first response byte. - pub first_byte_timeout: Duration, - /// Maximum time to wait between response body bytes. - pub between_bytes_timeout: Duration, - /// Optional stable discriminator folded into the backend name. - /// - /// Two callers can target the same origin (scheme, host, port, TLS) with - /// the same transport timeout yet need distinct dynamic backends — for - /// example two auction providers behind one gateway host. Because the - /// auction orchestrator correlates responses back to providers by backend - /// name, a shared name would let one provider's response be parsed as - /// another's. Setting this to a per-provider/integration identifier keeps - /// their names distinct while remaining stable across requests. - pub discriminator: Option, -} - /// Cloneable container of platform services for a single request. #[derive(Clone)] pub struct RuntimeServices { @@ -178,10 +147,13 @@ pub struct RuntimeServices { /// Defaults to [`super::UnavailableTemplateAssembler`]. Core retains a portable /// byte-seam fallback when this service is unavailable or rejects a document. pub(crate) template_assembler: Arc, - /// Dynamic backend registration and name prediction. - pub(crate) backend: Arc, - /// Outbound HTTP client abstraction. - pub(crate) http_client: Arc, + /// `EdgeZero` outbound HTTP client for this request, with the proxy marker off. + pub(crate) outbound_client: HttpClient, + /// Monotonic clock paired with this request's admitted ingress lifetime. + /// + /// Outbound deadlines and post-delivery budgets read this clock, so they + /// share one clock domain with the outbound client. + pub(crate) clock: MonotonicClock, /// Geographic information lookup. pub(crate) geo: Arc, /// Auction telemetry sink. @@ -206,8 +178,8 @@ impl RuntimeServices { /// .config_store(Arc::new(MyConfigStore)) /// .secret_store(Arc::new(MySecretStore)) /// .kv_store(kv_store) - /// .backend(Arc::new(MyBackend)) - /// .http_client(Arc::new(MyHttpClient)) + /// .outbound_client(outbound_client.with_proxy_marker(false)) + /// .clock(ctx.monotonic_clock()) /// .geo(Arc::new(MyGeo)) /// .client_info(client_info) /// .build(); @@ -253,16 +225,16 @@ impl RuntimeServices { &*self.template_assembler } - /// Returns the dynamic backend service. + /// Returns the request's `EdgeZero` outbound HTTP client. #[must_use] - pub fn backend(&self) -> &dyn PlatformBackend { - &*self.backend + pub fn outbound_client(&self) -> &HttpClient { + &self.outbound_client } - /// Returns the outbound HTTP client service. + /// Returns the monotonic clock paired with this request. #[must_use] - pub fn http_client(&self) -> &dyn PlatformHttpClient { - &*self.http_client + pub fn clock(&self) -> &MonotonicClock { + &self.clock } /// Returns the platform geo lookup service. @@ -343,8 +315,8 @@ pub struct RuntimeServicesBuilder { kv_store: Option>, template_cache: Option>, template_assembler: Option>, - backend: Option>, - http_client: Option>, + outbound_client: Option, + clock: Option, geo: Option>, auction_telemetry_sink: Option>, client_info: Option, @@ -359,8 +331,8 @@ impl RuntimeServicesBuilder { kv_store: None, template_cache: None, template_assembler: None, - backend: None, - http_client: None, + outbound_client: None, + clock: None, geo: None, auction_telemetry_sink: None, client_info: None, @@ -413,17 +385,20 @@ impl RuntimeServicesBuilder { self } - /// Set the backend implementation. + /// Set the request's `EdgeZero` outbound HTTP client. + /// + /// Adapters pass the client from the request context with the proxy + /// marker off. #[must_use] - pub fn backend(mut self, backend: Arc) -> Self { - self.backend = Some(backend); + pub fn outbound_client(mut self, outbound_client: HttpClient) -> Self { + self.outbound_client = Some(outbound_client); self } - /// Set the HTTP client implementation. + /// Set the monotonic clock paired with the request. #[must_use] - pub fn http_client(mut self, http_client: Arc) -> Self { - self.http_client = Some(http_client); + pub fn clock(mut self, clock: MonotonicClock) -> Self { + self.clock = Some(clock); self } @@ -477,12 +452,12 @@ impl RuntimeServicesBuilder { template_assembler: self .template_assembler .unwrap_or_else(|| Arc::new(super::UnavailableTemplateAssembler)), - backend: self - .backend - .expect("should set backend before building RuntimeServices"), - http_client: self - .http_client - .expect("should set http_client before building RuntimeServices"), + outbound_client: self + .outbound_client + .expect("should set outbound_client before building RuntimeServices"), + clock: self + .clock + .expect("should set clock before building RuntimeServices"), geo: self .geo .expect("should set geo before building RuntimeServices"), diff --git a/crates/trusted-server-core/src/proxy.rs b/crates/trusted-server-core/src/proxy.rs index 70e36f10e5..7c47babace 100644 --- a/crates/trusted-server-core/src/proxy.rs +++ b/crates/trusted-server-core/src/proxy.rs @@ -1,16 +1,20 @@ use crate::http_util::{ - RequestInfo, compute_encrypted_sha256_token, ct_str_eq, enforce_max_body_size, + RequestInfo, body_as_reader, compute_encrypted_sha256_token, ct_str_eq, read_request_body, }; use edgezero_core::body::Body as EdgeBody; -use edgezero_core::http::{Uri as EdgeUri, request_builder as edge_request_builder}; +use edgezero_core::error::EdgeError; +use edgezero_core::outbound::{ + OutboundFollowFailure, OutboundImageOptimizerOptions, OutboundRedirectAction, + OutboundRedirectContext, OutboundRedirectLimit, OutboundRequest, OutboundResponse, + ResponseDecoding, +}; use error_stack::{Report, ResultExt}; -use futures::StreamExt as _; use http::{HeaderValue, Method, Request, Response, StatusCode, header}; use serde::{Deserialize, Serialize}; -use std::io::{Cursor, Write}; use std::time::Duration; use web_time::{SystemTime, UNIX_EPOCH}; +use crate::asset_image_optimizer::{image_optimizer_options, profile_for_asset_request}; use crate::cache_policy::{ CachePolicy, EdgeCacheHeader, NO_STORE_PRIVATE_CACHE_CONTROL, apply_no_store_private_to_headers, cache_control_headers_are_private_or_no_store, @@ -23,10 +27,12 @@ use crate::constants::{ use crate::creative::{CreativeCssProcessor, CreativeHtmlProcessor, first_party_url}; use crate::edge_cookie::get_ec_id; use crate::error::TrustedServerError; -use crate::platform::{ - DEFAULT_FIRST_BYTE_TIMEOUT, PlatformBackendSpec, PlatformHttpRequest, PlatformResponse, - RuntimeServices, +use crate::outbound::{ + BUFFERED_TIMEOUT, DEFAULT_BUFFERED_RESPONSE_BYTES, OutboundRequestSource, + REDIRECT_CHAIN_DEADLINE, STREAMED_TIMEOUT, buffered, outbound_error, outbound_request, + streamed, }; +use crate::platform::RuntimeServices; use crate::s3_sigv4::{self, S3Credentials}; use crate::settings::{AssetOriginAuth, OriginQueryPolicy, ProxyAssetRoute, Settings}; use crate::streaming_processor::{Compression, PipelineConfig, StreamProcessor, StreamingPipeline}; @@ -40,23 +46,6 @@ const IMAGE_FALLBACK_CONTENT_TYPE: &str = "application/octet-stream"; const SIGN_MAX_BODY_BYTES: usize = 65536; const REBUILD_MAX_BODY_BYTES: usize = 65536; -fn body_as_reader(body: EdgeBody) -> Cursor { - Cursor::new(body.into_bytes().unwrap_or_default()) -} - -fn request_body_bytes( - body: EdgeBody, - endpoint: &str, -) -> Result> { - if body.is_stream() { - return Err(Report::new(TrustedServerError::BadRequest { - message: format!("{endpoint} request body must be buffered, not streamed"), - })); - } - - Ok(body.into_bytes().unwrap_or_default()) -} - /// Headers copied from the original client request to the upstream proxy request /// when `copy_request_headers` is enabled. /// @@ -72,15 +61,6 @@ const PROXY_FORWARD_HEADERS: [header::HeaderName; 5] = [ HEADER_X_FORWARDED_FOR, ]; -/// Request headers that describe a body and must not ride on a bodyless redirect follow-up. -const REQUEST_BODY_HEADERS: [header::HeaderName; 5] = [ - header::CONTENT_TYPE, - header::CONTENT_LENGTH, - header::CONTENT_ENCODING, - header::CONTENT_LANGUAGE, - header::CONTENT_LOCATION, -]; - /// Curated request headers preserved for asset proxying. /// /// Unlike the HTML publisher fallback, asset requests need cache validation and @@ -214,24 +194,6 @@ impl AssetProxyResponse { self.cache_policy } - /// Consume this wrapper and return a buffered Fastly response. - /// - /// # Errors - /// - /// Returns [`TrustedServerError::Proxy`] when the response carries a - /// streaming body. Use [`Self::into_response_and_body`] when the caller - /// needs to handle both buffered and streaming asset responses. - pub fn into_response(self) -> Result, Report> { - if self.stream_body.is_some() { - return Err(Report::new(TrustedServerError::Proxy { - message: "streaming asset response cannot be converted into a buffered response" - .to_string(), - })); - } - - Ok(self.response) - } - /// Consume this wrapper and return the response headers plus optional stream body. #[must_use] pub fn into_response_and_body(self) -> (Response, Option) { @@ -239,95 +201,6 @@ impl AssetProxyResponse { } } -/// Convert a platform-neutral response into a buffered [`Response`] for downstream processing. -/// -/// # Errors -/// -/// Returns [`TrustedServerError::Proxy`] when `platform_resp` carries a -/// streaming body, which the buffered proxy path cannot materialize. -pub(crate) fn platform_response_to_fastly( - platform_resp: PlatformResponse, -) -> Result, Report> { - let (parts, body) = platform_resp.response.into_parts(); - let body_bytes = match body { - EdgeBody::Once(bytes) => bytes.to_vec(), - EdgeBody::Stream(_) => { - return Err(Report::new(TrustedServerError::Proxy { - message: "streaming platform response body is not supported by Fastly response conversion" - .to_string(), - })); - } - }; - Ok(Response::from_parts(parts, EdgeBody::from(body_bytes))) -} - -fn platform_response_to_fastly_asset(platform_resp: PlatformResponse) -> AssetProxyResponse { - let (parts, body) = platform_resp.response.into_parts(); - - match body { - EdgeBody::Once(bytes) => AssetProxyResponse::origin_controlled(Response::from_parts( - parts, - EdgeBody::from(bytes.to_vec()), - )), - stream_body @ EdgeBody::Stream(_) => { - let resp = Response::from_parts(parts, EdgeBody::empty()); - AssetProxyResponse::origin_controlled_stream(resp, stream_body) - } - } -} - -/// Stream a platform response body directly to a writable client stream. -/// -/// Asset routes and Fastly `EdgeZero` publisher fallback both use this bridge -/// after headers have been committed through `stream_to_client()`. -/// -/// # Errors -/// -/// Returns an error if the platform stream yields an error or if writing a -/// chunk to the client stream fails. -pub async fn stream_asset_body( - body: EdgeBody, - output: &mut W, -) -> Result<(), Report> { - match body { - EdgeBody::Once(bytes) => { - output - .write_all(bytes.as_ref()) - .change_context(TrustedServerError::Proxy { - message: "failed to write buffered platform response body".to_string(), - })?; - } - EdgeBody::Stream(mut stream) => { - while let Some(chunk) = stream.next().await { - let chunk = chunk.map_err(|err| { - Report::new(TrustedServerError::Proxy { - message: format!("streaming platform response body failed: {err}"), - }) - })?; - output - .write_all(chunk.as_ref()) - .change_context(TrustedServerError::Proxy { - message: "failed to write streaming platform response body".to_string(), - })?; - // Flush per chunk: Fastly's `StreamingBody` is a - // `BufWriter`, so a segment smaller than - // the write buffer would sit in the Wasm heap until a later - // write filled it. The publisher stream yields compressed - // segments that are commonly smaller than that buffer, and the - // generator then awaits the origin (or the auction) before - // producing the next one — without this flush the client sees - // committed headers and no body, undoing the codec's per-chunk - // sync flush and delaying first paint. - output.flush().change_context(TrustedServerError::Proxy { - message: "failed to flush streaming platform response body".to_string(), - })?; - } - } - } - - Ok(()) -} - #[derive(Deserialize)] struct ProxySignReq { url: String, @@ -356,8 +229,12 @@ pub struct ProxyRequestConfig<'a> { pub copy_request_headers: bool, /// When true, stream the origin response without HTML/CSS rewrites. pub stream_passthrough: bool, - /// When true, ask the platform adapter to preserve the upstream response body as a stream. + /// When true, stream the upstream response body instead of buffering it. pub stream_response: bool, + /// Cap on a buffered upstream response, in bytes. + /// + /// A streamed response, see [`Self::stream_response`], has no cap. + pub max_response_bytes: u64, /// Domains allowed for the initial request and any redirects. /// /// **Open mode** (`&[]`): every host is permitted. Most integration proxies pass @@ -387,6 +264,7 @@ impl<'a> ProxyRequestConfig<'a> { copy_request_headers: true, stream_passthrough: false, stream_response: false, + max_response_bytes: DEFAULT_BUFFERED_RESPONSE_BYTES, allowed_domains: &[], require_https: false, } @@ -441,12 +319,19 @@ impl<'a> ProxyRequestConfig<'a> { self } - /// Ask the platform adapter to preserve the upstream response body as a stream. + /// Stream the upstream response body instead of buffering it. #[must_use] pub fn with_stream_response(mut self) -> Self { self.stream_response = true; self } + + /// Cap a buffered upstream response at `max_response_bytes`. + #[must_use] + pub fn with_max_response_bytes(mut self, max_response_bytes: u64) -> Self { + self.max_response_bytes = max_response_bytes; + self + } } /// Encodings we support decompressing in `finalize_proxied_response`. @@ -501,7 +386,7 @@ fn process_response_with_pipeline( let mut output = Vec::new(); let mut pipeline = StreamingPipeline::new(config, processor); pipeline - .process(body_as_reader(body), &mut output) + .process(body_as_reader(body, "proxy")?, &mut output) .change_context(TrustedServerError::Proxy { message: error_context.to_string(), })?; @@ -760,20 +645,8 @@ fn finalize_response( Ok(response) } -/// Bundles per-request header configuration and [`RuntimeServices`] for the proxy redirect loop. -struct ProxyRequestHeaders<'a> { - additional_headers: &'a [(header::HeaderName, HeaderValue)], - copy_request_headers: bool, - services: &'a RuntimeServices, -} - -struct ProxyRedirectPolicy<'a> { - follow_redirects: bool, - stream_passthrough: bool, - stream_response: bool, - allowed_domains: &'a [String], - require_https: bool, -} +/// Redirects a followed proxy request takes before it returns the next one. +const MAX_REDIRECTS: usize = 4; /// Proxy a request to a clear target URL while reusing creative rewrite logic. /// @@ -781,6 +654,12 @@ struct ProxyRedirectPolicy<'a> { /// the caller's EC ID as a `ts-ec` query parameter to the target URL. /// Optional bodies/headers can be supplied via [`ProxyRequestConfig`]. /// +/// The target must pass [`check_proxy_target`] before anything is sent. A followed +/// request goes through `EdgeZero`'s redirect follower, which shows every redirect +/// to [`redirect_policy`]. Each hop gets the request's own timeout, 15 s buffered +/// or 60 s streamed, and the whole chain shares a 60 s deadline. A request that does not follow redirects is sent +/// once, and a `3xx` answer is returned as it is. +/// /// Followed `307`/`308` redirects keep the original method and body. Followed /// `301`/`302`/`303` redirects are sent as a bodyless `GET` (a `HEAD` stays /// `HEAD`) without body-describing headers such as `Content-Type`. A `301`/`302` @@ -792,6 +671,10 @@ struct ProxyRedirectPolicy<'a> { /// /// - [`TrustedServerError::Proxy`] if the target URL is invalid, uses an unsupported /// scheme, lacks a host, or the upstream fetch fails +/// - [`TrustedServerError::Forbidden`] if HTTPS is required and the target or a +/// redirect is not HTTPS +/// - [`TrustedServerError::AllowlistViolation`] if the target or a redirect leaves +/// `allowed_domains` pub async fn proxy_request( settings: &Settings, req: Request, @@ -807,40 +690,216 @@ pub async fn proxy_request( copy_request_headers, stream_passthrough, stream_response, + max_response_bytes, allowed_domains, require_https, } = config; + let failed = || TrustedServerError::Proxy { + message: "Failed to proxy".to_string(), + }; - let mut target_url_parsed = url::Url::parse(target_url).map_err(|_| { + let mut target = url::Url::parse(target_url).map_err(|_| { Report::new(TrustedServerError::Proxy { message: "invalid url".to_string(), }) })?; - if forward_ec_id { - append_ec_id(&req, &mut target_url_parsed); + append_ec_id(&req, &mut target); } + check_proxy_target(&target, allowed_domains, require_https)?; + + let request = proxy_outbound_request(&req, &target, body, &headers, copy_request_headers) + .map_err(|error| outbound_error(error, failed(), OutboundRequestSource::Internal))?; + let request = if stream_response { + streamed(request, STREAMED_TIMEOUT) + } else { + buffered( + request, + BUFFERED_TIMEOUT, + max_response_bytes, + ResponseDecoding::Original, + ) + }; + + let (response, final_uri) = if follow_redirects { + let limit = OutboundRedirectLimit::new(MAX_REDIRECTS) + .map_err(|error| outbound_error(error, failed(), OutboundRequestSource::Internal))?; + let chain_deadline = services.clock().deadline_after(REDIRECT_CHAIN_DEADLINE); + let followed = services + .outbound_client() + .follow( + request.deadline(chain_deadline), + limit, + redirect_policy(allowed_domains.to_vec(), require_https), + ) + .await + .map_err(|failure| match failure { + OutboundFollowFailure::Policy { error, .. } => Report::new(error), + OutboundFollowFailure::Hop { error, .. } => { + outbound_error(error, failed(), OutboundRequestSource::Internal) + } + })?; + if let Some(reason) = followed.stop_reason() { + log::warn!("returning an unfollowed redirect response: {reason:?}"); + } + let (response, final_uri, _redirects_followed, _stop_reason) = followed.into_parts(); + (response, final_uri) + } else { + let final_uri = request.uri().clone(); + let response = services + .outbound_client() + .send(request) + .await + .map_err(|error| outbound_error(error, failed(), OutboundRequestSource::Internal))?; + (response, final_uri) + }; + let response = response + .into_response() + .map_err(|error| outbound_error(error, failed(), OutboundRequestSource::Internal))?; - proxy_with_redirects( + finalize_response( settings, &req, - target_url_parsed, - body.as_deref(), - ProxyRequestHeaders { - additional_headers: &headers, - copy_request_headers, - services, - }, - ProxyRedirectPolicy { - follow_redirects, - stream_passthrough, - stream_response: stream_response - && services.http_client().supports_streaming_responses(), - allowed_domains, - require_https, - }, + &final_uri.to_string(), + response, + stream_passthrough, ) - .await +} + +/// Checks the first target of a proxied request before it is sent. +/// +/// The target needs an `http` or `https` scheme, HTTPS when `require_https` is +/// set, a host, and a host that `allowed_domains` permits. `EdgeZero` never shows +/// the first request to [`redirect_policy`], which checks every later hop. +/// +/// # Errors +/// +/// Returns [`TrustedServerError::Proxy`] for another scheme or a missing host, +/// [`TrustedServerError::Forbidden`] for a non-HTTPS target that must be HTTPS, +/// and [`TrustedServerError::AllowlistViolation`] for a host outside +/// `allowed_domains`. +fn check_proxy_target( + target: &url::Url, + allowed_domains: &[String], + require_https: bool, +) -> Result<(), Report> { + let scheme = target.scheme(); + if scheme != "http" && scheme != "https" { + return Err(Report::new(TrustedServerError::Proxy { + message: "unsupported scheme".to_string(), + })); + } + if require_https && scheme != "https" { + log::warn!("request to `{target}` blocked: HTTPS is required"); + return Err(Report::new(TrustedServerError::Forbidden { + message: "non-HTTPS proxy target blocked".to_string(), + })); + } + + let host = target.host_str().unwrap_or(""); + if host.is_empty() { + return Err(Report::new(TrustedServerError::Proxy { + message: "missing host".to_string(), + })); + } + if !is_host_permitted(allowed_domains, host) { + log::warn!("request to `{host}` blocked: host not in proxy allowed_domains"); + return Err(Report::new(TrustedServerError::AllowlistViolation { + host: host.to_string(), + })); + } + + Ok(()) +} + +/// Builds the first request of a proxied fetch from the client's `req`. +/// +/// With `copy_request_headers`, the request carries [`PROXY_FORWARD_HEADERS`] from +/// the client and [`SUPPORTED_ENCODINGS`] as `Accept-Encoding`. Each of the +/// caller's `headers` then replaces any value of the same name. +/// +/// # Errors +/// +/// Returns [`EdgeError::BadRequest`] when `target` is not a URI `EdgeZero` sends. +fn proxy_outbound_request( + req: &Request, + target: &url::Url, + body: Option>, + headers: &[(header::HeaderName, HeaderValue)], + copy_request_headers: bool, +) -> Result { + let mut request = outbound_request(req.method().clone(), target.as_str())?; + let outbound_headers = request.headers_mut(); + if copy_request_headers { + for header_name in PROXY_FORWARD_HEADERS { + if let Some(value) = req.headers().get(&header_name) { + outbound_headers.insert(header_name, value.clone()); + } + } + outbound_headers.insert( + HEADER_ACCEPT_ENCODING, + HeaderValue::from_static(SUPPORTED_ENCODINGS), + ); + } + for (name, value) in headers { + outbound_headers.insert(name.clone(), value.clone()); + } + + Ok(match body { + Some(body) => request.body(body), + None => request, + }) +} + +/// Builds the policy a followed proxy request applies to every redirect. +/// +/// It owns copies of `allowed_domains` and `require_https`, because `EdgeZero` +/// keeps the policy for the whole chain. For each redirect it reads the resolved +/// target and returns, in this order: +/// +/// 1. [`TrustedServerError::Forbidden`] when `require_https` is set and the target +/// is not `https` +/// 2. [`OutboundRedirectAction::Stop`] for any other scheme that is not `http` or +/// `https`, which returns the redirect response +/// 3. [`TrustedServerError::Proxy`] for a target without a host +/// 4. [`TrustedServerError::AllowlistViolation`] when `allowed_domains` does not +/// permit the host +/// 5. [`OutboundRedirectAction::FollowPrepared`] otherwise +fn redirect_policy( + allowed_domains: Vec, + require_https: bool, +) -> impl FnMut(&mut OutboundRedirectContext<'_>) -> Result ++ 'static { + move |context| { + let target = + url::Url::parse(context.resolved_target()).map_err(|_| TrustedServerError::Proxy { + message: "invalid redirect".to_string(), + })?; + let scheme = target.scheme(); + if require_https && scheme != "https" { + log::warn!("redirect to `{target}` blocked: HTTPS is required"); + return Err(TrustedServerError::Forbidden { + message: "non-HTTPS redirect blocked".to_string(), + }); + } + if scheme != "http" && scheme != "https" { + return Ok(OutboundRedirectAction::Stop); + } + + let Some(host) = target.host_str().filter(|host| !host.is_empty()) else { + return Err(TrustedServerError::Proxy { + message: "missing host in redirect location".to_string(), + }); + }; + if !is_host_permitted(&allowed_domains, host) { + log::warn!("redirect to `{host}` blocked: host not in proxy allowed_domains"); + return Err(TrustedServerError::AllowlistViolation { + host: host.to_string(), + }); + } + + Ok(OutboundRedirectAction::FollowPrepared) + } } fn default_port_for_scheme(scheme: &str) -> Option { @@ -942,62 +1001,63 @@ fn apply_asset_origin_auth( } } -fn build_asset_platform_request( - method: &Method, - target_url: &url::Url, - outbound_headers: &http::HeaderMap, - backend_name: &str, -) -> Result> { - let mut builder = edge_request_builder().method(method.clone()).uri( - target_url - .as_str() - .parse::() - .change_context(TrustedServerError::Proxy { - message: "invalid asset target URL".to_string(), - })?, - ); - - for (name, value) in outbound_headers { - builder = builder.header(name, value); - } - - let edge_req = - builder - .body(EdgeBody::from(Vec::new())) - .change_context(TrustedServerError::Proxy { - message: "failed to build asset proxy request".to_string(), - })?; - - Ok(PlatformHttpRequest::new(edge_req, backend_name)) -} - +/// Sends one request to an asset origin. +/// +/// A `GET` streams its response under the 60 s streamed timeout. Any other method, +/// the S3 `HEAD` preflight included, buffers its response under the 15 s buffered +/// timeout. Both keep the origin's content coding and the default cache policy, +/// and `image_optimizer` options go on the request as given. +/// +/// `outbound_headers` may carry the signer's `Host` input. `EdgeZero` drops that +/// header and sends the URI authority, from which [`asset_origin_host_header`] +/// built it. +/// +/// # Errors +/// +/// Returns [`TrustedServerError::Proxy`] when the request cannot be built or the +/// origin request fails. async fn send_asset_origin_request( services: &RuntimeServices, - backend_name: &str, method: &Method, target_url: &url::Url, outbound_headers: &http::HeaderMap, - stream_response: bool, + image_optimizer: Option, ) -> Result> { - let stream_response = stream_response && services.http_client().supports_streaming_responses(); - let mut platform_req = - build_asset_platform_request(method, target_url, outbound_headers, backend_name)?; - if stream_response { - platform_req = platform_req.with_stream_response(); - } - let platform_resp = services - .http_client() - .send(platform_req) - .await - .change_context(TrustedServerError::Proxy { - message: "Failed to proxy asset request".to_string(), - })?; + let failed = || TrustedServerError::Proxy { + message: "Failed to proxy asset request".to_string(), + }; + let mut request = outbound_request(method.clone(), target_url.as_str()) + .map_err(|error| outbound_error(error, failed(), OutboundRequestSource::Internal))?; + *request.headers_mut() = outbound_headers.clone(); + if let Some(options) = image_optimizer { + request = request.image_optimizer(options); + } + let request = if method == Method::GET { + streamed(request, STREAMED_TIMEOUT) + } else { + buffered( + request, + BUFFERED_TIMEOUT, + DEFAULT_BUFFERED_RESPONSE_BYTES, + ResponseDecoding::Original, + ) + }; - if stream_response { - Ok(platform_response_to_fastly_asset(platform_resp)) + let response = services + .outbound_client() + .send(request) + .await + .and_then(OutboundResponse::into_response) + .map_err(|error| outbound_error(error, failed(), OutboundRequestSource::Internal))?; + let (parts, body) = response.into_parts(); + Ok(if body.is_stream() { + AssetProxyResponse::origin_controlled_stream( + Response::from_parts(parts, EdgeBody::empty()), + body, + ) } else { - platform_response_to_fastly(platform_resp).map(AssetProxyResponse::origin_controlled) - } + AssetProxyResponse::origin_controlled(Response::from_parts(parts, body)) + }) } fn strip_asset_proxy_response_headers(response: &mut Response) { @@ -1028,7 +1088,6 @@ async fn preflight_s3_origin_for_image_optimizer( target_url: &url::Url, request_method: &Method, unsigned_headers: &http::HeaderMap, - backend_name: &str, ) -> Result, Report> { let Some(auth @ AssetOriginAuth::S3SigV4(_)) = route.auth.as_ref() else { return Ok(None); @@ -1039,22 +1098,16 @@ async fn preflight_s3_origin_for_image_optimizer( // without invoking IO on the failure path. let mut head_headers = unsigned_headers.clone(); apply_asset_origin_auth(&Method::HEAD, target_url, &mut head_headers, auth)?; - let head_response = send_asset_origin_request( - services, - backend_name, - &Method::HEAD, - target_url, - &head_headers, - false, - ) - .await?; + let head_response = + send_asset_origin_request(services, &Method::HEAD, target_url, &head_headers, None).await?; let head_status = head_response.response().status(); if head_status.is_success() || head_status == StatusCode::NOT_MODIFIED { return Ok(None); } if request_method == Method::HEAD { - let mut response = head_response.into_response()?; + // The preflight buffers its bodiless HEAD, so no stream comes back. + let (mut response, _stream_body) = head_response.into_response_and_body(); strip_asset_proxy_response_headers(&mut response); apply_no_store_cache_control(&mut response); return Ok(Some(AssetProxyResponse::no_store_private(response))); @@ -1062,15 +1115,8 @@ async fn preflight_s3_origin_for_image_optimizer( let mut get_headers = unsigned_headers.clone(); apply_asset_origin_auth(&Method::GET, target_url, &mut get_headers, auth)?; - let mut response = send_asset_origin_request( - services, - backend_name, - &Method::GET, - target_url, - &get_headers, - true, - ) - .await?; + let mut response = + send_asset_origin_request(services, &Method::GET, target_url, &get_headers, None).await?; strip_asset_proxy_response_headers(response.response_mut()); response.apply_no_store_private_policy(); @@ -1081,18 +1127,20 @@ async fn preflight_s3_origin_for_image_optimizer( /// /// This is a lean raw pass-through path: it preserves status/body/headers, /// does not follow redirects, and bypasses publisher-page processing. The flow -/// is path rewrite, profile-table Image Optimizer metadata extraction, optional -/// origin query stripping, optional origin authentication, then platform send. +/// is path rewrite, profile-table Image Optimizer options, optional origin query +/// stripping, optional origin authentication, then the origin request. /// /// The origin query policy is applied before S3 signing so the signature covers -/// the exact URL sent to the asset origin. Image Optimizer metadata remains -/// separate from the origin URL and is translated by the platform adapter. +/// the exact URL sent to the asset origin. Image Optimizer options stay separate +/// from the origin URL and go only on the final request, never on the S3 `HEAD` +/// preflight. A profile that cannot be expressed as options, which settings +/// validation rules out, serves the asset unoptimized with a warning. /// /// # Errors /// -/// Returns an error if the configured origin URL is invalid, backend -/// registration fails, S3 credentials cannot be read, signing fails, image -/// optimizer metadata cannot be built, or the upstream request cannot be sent. +/// Returns an error if the configured origin URL is invalid, the request names +/// an unknown image profile in `reject` mode, signing fails, or the upstream +/// request cannot be sent. pub async fn handle_asset_proxy_request( settings: &Settings, services: &RuntimeServices, @@ -1112,36 +1160,21 @@ pub async fn handle_asset_proxy_request( ); None } else { - crate::asset_image_optimizer::options_for_asset_request(settings, route, incoming_query)? + profile_for_asset_request(settings, route, incoming_query)?.and_then(|profile| { + image_optimizer_options(&profile) + .inspect_err(|error| { + log::warn!( + "Serving asset {incoming_path} unoptimized: its Image Optimizer profile cannot be applied: {error}" + ); + }) + .ok() + }) }; if route.origin_query_policy() == OriginQueryPolicy::Strip { target_url.set_query(None); } - let scheme = target_url.scheme(); - let host = target_url.host_str().ok_or_else(|| { - Report::new(TrustedServerError::Proxy { - message: "Missing host in asset target URL".to_string(), - }) - })?; - - let backend_name = services - .backend() - .ensure(&PlatformBackendSpec { - scheme: scheme.to_string(), - host: host.to_string(), - port: target_url.port(), - host_header_override: None, - certificate_check: settings.proxy.certificate_check, - first_byte_timeout: DEFAULT_FIRST_BYTE_TIMEOUT, - between_bytes_timeout: DEFAULT_FIRST_BYTE_TIMEOUT, - discriminator: None, - }) - .change_context(TrustedServerError::Proxy { - message: "asset backend registration failed".to_string(), - })?; - let mut outbound_headers = http::HeaderMap::new(); for header_name in ASSET_PROXY_FORWARD_HEADERS { if let Some(value) = req.headers().get(&header_name) { @@ -1157,7 +1190,6 @@ pub async fn handle_asset_proxy_request( &target_url, req.method(), &outbound_headers, - &backend_name, ) .await? { @@ -1168,29 +1200,14 @@ pub async fn handle_asset_proxy_request( apply_asset_origin_auth(req.method(), &target_url, &mut outbound_headers, auth)?; } - let mut platform_req = - build_asset_platform_request(req.method(), &target_url, &outbound_headers, &backend_name)?; - if let Some(image_optimizer) = image_optimizer { - platform_req = platform_req.with_image_optimizer(image_optimizer); - } - let stream_response = services.http_client().supports_streaming_responses(); - if stream_response { - platform_req = platform_req.with_stream_response(); - } - - let platform_resp = services - .http_client() - .send(platform_req) - .await - .change_context(TrustedServerError::Proxy { - message: "Failed to proxy asset request".to_string(), - })?; - - let mut response = if stream_response { - platform_response_to_fastly_asset(platform_resp) - } else { - platform_response_to_fastly(platform_resp).map(AssetProxyResponse::origin_controlled)? - }; + let mut response = send_asset_origin_request( + services, + req.method(), + &target_url, + &outbound_headers, + image_optimizer, + ) + .await?; strip_asset_proxy_response_headers(response.response_mut()); let status = response.response().status(); @@ -1273,335 +1290,52 @@ pub(crate) fn is_host_allowed(host: &str, pattern: &str) -> bool { } } -async fn proxy_with_redirects( +/// Unified proxy endpoint for resources referenced by ad creatives. +/// +/// Accepts: +/// - `u`: Base64 URL-safe (no padding) encoded URL of the third-party resource. +/// +/// Behavior: +/// - Proxies the decoded URL, following redirects only to hosts in +/// `proxy.allowed_domains`. +/// - If the response `Content-Type` contains `text/html`, rewrites the HTML creative +/// (img/srcset/iframe to first-party) before returning `text/html; charset=utf-8`. +/// - If the response is an image or the request `Accept` indicates images, ensures an +/// `application/octet-stream` content type if origin omitted it, and logs likely 1×1 +/// pixels using simple size/URL heuristics. No special response (still proxied). +/// +/// # Errors +/// +/// Returns an error if the signed target cannot be reconstructed or validation fails. +pub async fn handle_first_party_proxy( settings: &Settings, - req: &Request, - target_url_parsed: url::Url, - body: Option<&[u8]>, - request_headers: ProxyRequestHeaders<'_>, - redirect_policy: ProxyRedirectPolicy<'_>, + services: &RuntimeServices, + req: Request, ) -> Result, Report> { - const MAX_REDIRECTS: usize = 4; - - let mut current_url = target_url_parsed.to_string(); - let mut current_method: Method = req.method().clone(); - let mut current_body = body; - // Set once a redirect rewrites the request into a bodyless follow-up. - let mut body_dropped = false; - - for redirect_attempt in 0..=MAX_REDIRECTS { - let parsed_url = url::Url::parse(¤t_url).map_err(|_| { - Report::new(TrustedServerError::Proxy { - message: "invalid url".to_string(), - }) - })?; + // Parse, reconstruct, and validate the signed target URL + let SignedTarget { target_url, .. } = + reconstruct_and_validate_signed_target(settings, &req.uri().to_string())?; - let scheme = parsed_url.scheme().to_ascii_lowercase(); - if scheme != "http" && scheme != "https" { - return Err(Report::new(TrustedServerError::Proxy { - message: "unsupported scheme".to_string(), - })); - } - if redirect_policy.require_https && scheme != "https" { - log::warn!("request to `{}` blocked: HTTPS is required", current_url); - return Err(Report::new(TrustedServerError::Forbidden { - message: "non-HTTPS proxy target blocked".to_string(), - })); - } - - let host = parsed_url.host_str().unwrap_or(""); - if host.is_empty() { - return Err(Report::new(TrustedServerError::Proxy { - message: "missing host".to_string(), - })); - } - - if !is_host_permitted(redirect_policy.allowed_domains, host) { - log::warn!( - "request to `{}` blocked: host not in proxy allowed_domains", - host - ); - return Err(Report::new(TrustedServerError::AllowlistViolation { - host: host.to_string(), - })); - } - - let backend_name = request_headers - .services - .backend() - .ensure(&PlatformBackendSpec { - scheme: scheme.clone(), - host: host.to_string(), - port: parsed_url.port(), - host_header_override: None, - certificate_check: settings.proxy.certificate_check, - first_byte_timeout: DEFAULT_FIRST_BYTE_TIMEOUT, - between_bytes_timeout: DEFAULT_FIRST_BYTE_TIMEOUT, - discriminator: None, - }) - .change_context(TrustedServerError::Proxy { - message: "backend registration failed".to_string(), - })?; - - let mut builder = edge_request_builder().method(current_method.clone()).uri( - current_url - .parse::() - .change_context(TrustedServerError::Proxy { - message: "invalid url".to_string(), - })?, - ); - - // Collect outbound headers using insert-semantics so additional_headers override any - // header set by copy_request_headers, matching the old set_header() replace behavior. - let mut outbound_headers = http::HeaderMap::new(); - if request_headers.copy_request_headers { - for header_name in PROXY_FORWARD_HEADERS { - if let Some(v) = req.headers().get(&header_name) { - outbound_headers.insert(header_name, v.clone()); - } - } - outbound_headers.insert( - HEADER_ACCEPT_ENCODING, - HeaderValue::from_static(SUPPORTED_ENCODINGS), - ); - } - for (name, value) in request_headers.additional_headers { - // insert() replaces any existing value, matching set_header() semantics. - outbound_headers.insert(name.clone(), value.clone()); - } - if body_dropped { - for header_name in &REQUEST_BODY_HEADERS { - outbound_headers.remove(header_name); - } - } - for (name, value) in &outbound_headers { - builder = builder.header(name, value); - } - let body_bytes = current_body.map(<[u8]>::to_vec).unwrap_or_default(); - let edge_req = - builder - .body(EdgeBody::from(body_bytes)) - .change_context(TrustedServerError::Proxy { - message: "failed to build proxy request".to_string(), - })?; - - let mut platform_request = PlatformHttpRequest::new(edge_req, backend_name); - if redirect_policy.stream_response { - platform_request = platform_request.with_stream_response(); - } - - let platform_resp = request_headers - .services - .http_client() - .send(platform_request) - .await - .change_context(TrustedServerError::Proxy { - message: "Failed to proxy".to_string(), - })?; - - let beresp = platform_resp.response; - - if !redirect_policy.follow_redirects { - return finalize_response( - settings, - req, - ¤t_url, - beresp, - redirect_policy.stream_passthrough, - ); - } - - let status = beresp.status(); - let is_redirect = matches!( - status, - StatusCode::MOVED_PERMANENTLY - | StatusCode::FOUND - | StatusCode::SEE_OTHER - | StatusCode::TEMPORARY_REDIRECT - | StatusCode::PERMANENT_REDIRECT - ); - - if !is_redirect { - return finalize_response( - settings, - req, - ¤t_url, - beresp, - redirect_policy.stream_passthrough, - ); - } - - let Some(location) = beresp - .headers() - .get(header::LOCATION) - .and_then(|h| h.to_str().ok()) - .filter(|value| !value.is_empty()) - else { - return finalize_response( - settings, - req, - ¤t_url, - beresp, - redirect_policy.stream_passthrough, - ); - }; - - // Fetch keeps methods other than POST on a 301/302, but replaying the - // body would reach a target the caller did not address, and rewriting - // to GET would report success for an update that never ran. - if matches!(status, StatusCode::MOVED_PERMANENTLY | StatusCode::FOUND) - && current_method != Method::GET - && current_method != Method::HEAD - && current_method != Method::POST - { - log::warn!( - "not following {} redirect for {current_method} request to {current_url}; returning redirect response", - status.as_u16() - ); - return finalize_response( - settings, - req, - ¤t_url, - beresp, - redirect_policy.stream_passthrough, - ); - } - - if redirect_attempt == MAX_REDIRECTS { - log::warn!( - "redirect limit reached for {}; returning redirect response", - current_url - ); - return finalize_response( - settings, - req, - ¤t_url, - beresp, - redirect_policy.stream_passthrough, - ); - } - - let next_url = url::Url::parse(location) - .or_else(|_| parsed_url.join(location)) - .map_err(|_| { - Report::new(TrustedServerError::Proxy { - message: "invalid redirect".to_string(), - }) - })?; - - let next_scheme = next_url.scheme().to_ascii_lowercase(); - if redirect_policy.require_https && next_scheme != "https" { - log::warn!("redirect to `{}` blocked: HTTPS is required", next_url); - return Err(Report::new(TrustedServerError::Forbidden { - message: "non-HTTPS redirect blocked".to_string(), - })); - } - if next_scheme != "http" && next_scheme != "https" { - return finalize_response( - settings, - req, - ¤t_url, - beresp, - redirect_policy.stream_passthrough, - ); - } - - let next_host = match next_url.host_str() { - Some(h) if !h.is_empty() => h, - _ => { - return Err(Report::new(TrustedServerError::Proxy { - message: "missing host in redirect location".to_string(), - })); - } - }; - if !is_host_permitted(redirect_policy.allowed_domains, next_host) { - log::warn!( - "redirect to `{}` blocked: host not in proxy allowed_domains", - next_host - ); - return Err(Report::new(TrustedServerError::AllowlistViolation { - host: next_host.to_string(), - })); - } - - log::info!( - "following redirect {} => {} (status {})", - current_url, - next_url, - status.as_u16() - ); - - current_url = next_url.to_string(); - // 307/308 replay the original method and body by design. 301/302/303 - // become a bodyless GET (HEAD stays HEAD) so those redirects cannot - // carry the payload to a target the caller did not address. - if matches!( - status, - StatusCode::MOVED_PERMANENTLY | StatusCode::FOUND | StatusCode::SEE_OTHER - ) { - if current_body.is_some_and(|bytes| !bytes.is_empty()) { - log::warn!( - "dropping {current_method} request body on {} redirect to {current_url}", - status.as_u16() - ); - } - if current_method != Method::HEAD { - current_method = Method::GET; - } - current_body = None; - body_dropped = true; - } - } - - Err(Report::new(TrustedServerError::Proxy { - message: "redirect handling failed".to_string(), - })) -} - -/// Unified proxy endpoint for resources referenced by ad creatives. -/// -/// Accepts: -/// - `u`: Base64 URL-safe (no padding) encoded URL of the third-party resource. -/// -/// Behavior: -/// - Proxies the decoded URL via a dynamic backend derived from scheme/host/port. -/// - If the response `Content-Type` contains `text/html`, rewrites the HTML creative -/// (img/srcset/iframe to first-party) before returning `text/html; charset=utf-8`. -/// - If the response is an image or the request `Accept` indicates images, ensures an -/// `application/octet-stream` content type if origin omitted it, and logs likely 1×1 -/// pixels using simple size/URL heuristics. No special response (still proxied). -/// -/// # Errors -/// -/// Returns an error if the signed target cannot be reconstructed or validation fails. -pub async fn handle_first_party_proxy( - settings: &Settings, - services: &RuntimeServices, - req: Request, -) -> Result, Report> { - // Parse, reconstruct, and validate the signed target URL - let SignedTarget { target_url, .. } = - reconstruct_and_validate_signed_target(settings, &req.uri().to_string())?; - - proxy_request( - settings, - req, - ProxyRequestConfig { - target_url: &target_url, - follow_redirects: true, - forward_ec_id: true, - body: None, - headers: Vec::new(), - copy_request_headers: true, - stream_passthrough: false, - stream_response: false, - allowed_domains: &settings.proxy.allowed_domains, - require_https: false, - }, - services, - ) - .await -} + proxy_request( + settings, + req, + ProxyRequestConfig { + target_url: &target_url, + follow_redirects: true, + forward_ec_id: true, + body: None, + headers: Vec::new(), + copy_request_headers: true, + stream_passthrough: false, + stream_response: false, + max_response_bytes: DEFAULT_BUFFERED_RESPONSE_BYTES, + allowed_domains: &settings.proxy.allowed_domains, + require_https: false, + }, + services, + ) + .await +} /// First-party click redirect endpoint. /// @@ -1691,7 +1425,8 @@ pub async fn handle_first_party_click( /// /// # Errors /// -/// Returns an error if JSON parsing fails, the URL cannot be parsed, the URL uses an +/// Returns an error if a POST body cannot be read within 64 KiB (see +/// [`read_request_body`]), JSON parsing fails, the URL cannot be parsed, the URL uses an /// unsupported scheme, the URL lacks a host, the host is declined by /// `rewrite.exclude_domains` or `rewrite.include_domains`, the host violates /// `proxy.allowed_domains`, or request metadata needed to determine the scheme @@ -1712,7 +1447,7 @@ pub async fn handle_first_party_proxy_sign( // adapter forwards verbatim — have no scheme of their own, so parsing would // hand back the placeholder base's `https` and make an HTTP dev server sign // an HTTPS target it then proxies over TLS against a plaintext service. - let request_scheme = match crate::forwarder::public_origin(&req) + let request_scheme = match crate::forwarder::public_origin(req.extensions()) .map(|origin| origin.scheme.clone()) .or_else(|| req.uri().scheme_str().map(str::to_ascii_lowercase)) { @@ -1721,8 +1456,8 @@ pub async fn handle_first_party_proxy_sign( }; let payload = if method == Method::POST { - let body_bytes = request_body_bytes(req.into_body(), "first-party sign")?; - enforce_max_body_size(&body_bytes, SIGN_MAX_BODY_BYTES, "first-party sign")?; + let (parts, body) = req.into_parts(); + let body_bytes = read_request_body(&parts.headers, body, SIGN_MAX_BODY_BYTES).await?; let body = std::str::from_utf8(&body_bytes).change_context(TrustedServerError::InvalidUtf8 { message: "first-party sign request body should be valid UTF-8".to_string(), @@ -1912,8 +1647,8 @@ pub async fn handle_first_party_proxy_rebuild( let redirect_response = method == Method::GET || is_form_post; let payload = if method == Method::POST { - let body_bytes = request_body_bytes(req.into_body(), "first-party rebuild")?; - enforce_max_body_size(&body_bytes, REBUILD_MAX_BODY_BYTES, "first-party rebuild")?; + let (parts, body) = req.into_parts(); + let body_bytes = read_request_body(&parts.headers, body, REBUILD_MAX_BODY_BYTES).await?; let body = std::str::from_utf8(&body_bytes).change_context(TrustedServerError::InvalidUtf8 { message: "first-party rebuild request body should be valid UTF-8".to_string(), @@ -2236,32 +1971,30 @@ fn reconstruct_and_validate_signed_target( #[cfg(test)] mod tests { - use std::cell::RefCell; - use std::collections::{HashMap, VecDeque}; - use std::io; - use std::rc::Rc; + use std::collections::HashMap; use std::sync::{Arc, Mutex}; + use std::task::{Context, Poll, Waker}; use std::time::Duration; use super::{ - AssetProxyCachePolicy, IMAGE_FALLBACK_CONTENT_TYPE, ProxyRequestConfig, - SUPPORTED_ENCODINGS, asset_origin_host_header, asset_path_skips_image_optimizer, - build_asset_proxy_target_url, handle_asset_proxy_request, handle_first_party_click, - handle_first_party_proxy, handle_first_party_proxy_rebuild, handle_first_party_proxy_sign, - is_host_allowed, is_host_permitted, proxy_request, rebuild_response_with_body, - reconstruct_and_validate_signed_target, stream_asset_body, + AssetProxyCachePolicy, AssetProxyResponse, IMAGE_FALLBACK_CONTENT_TYPE, ProxyRequestConfig, + REBUILD_MAX_BODY_BYTES, SIGN_MAX_BODY_BYTES, SUPPORTED_ENCODINGS, asset_origin_host_header, + asset_path_skips_image_optimizer, build_asset_proxy_target_url, handle_asset_proxy_request, + handle_first_party_click, handle_first_party_proxy, handle_first_party_proxy_rebuild, + handle_first_party_proxy_sign, is_host_allowed, is_host_permitted, proxy_request, + rebuild_response_with_body, reconstruct_and_validate_signed_target, }; use crate::cache_policy::{CachePolicy, EdgeCacheHeader}; use crate::constants::{HEADER_ACCEPT, HEADER_X_FORWARDED_FOR}; use crate::creative; use crate::error::{IntoHttpResponse, TrustedServerError}; - use crate::platform::test_support::{ - HashMapSecretStore, StubHttpClient, build_services_with_http_client, - build_services_with_secret_and_http_client, noop_services, + use crate::outbound::{ + BUFFERED_TIMEOUT, DEFAULT_BUFFERED_RESPONSE_BYTES, REDIRECT_CHAIN_DEADLINE, + STREAMED_TIMEOUT, }; + use crate::platform::test_support::{build_services_with_outbound_client, noop_services}; use crate::platform::{ - PlatformError, PlatformHttpClient, PlatformHttpRequest, PlatformPendingRequest, - PlatformResponse, PlatformSecretStore, PlatformSelectResult, StoreId, StoreName, + PlatformError, PlatformSecretStore, RuntimeServices, StoreId, StoreName, }; use crate::redacted::Redacted; use crate::settings::{ @@ -2269,10 +2002,20 @@ mod tests { ImageOptimizerCropOffsetsConfig, ImageOptimizerProfileSet, ImageOptimizerSettings, OriginQueryPolicy, ProxyAssetRoute, S3SigV4AuthConfig, Settings, UnknownProfilePolicy, }; + use crate::test_support::body_read::body_read_failure_cases; + use crate::test_support::logging::WarningCapture; + use crate::test_support::scripted_client::{ + ExpectedTransport, RecordedRequest, ScriptedHttpClient, ScriptedReply, TestClock, + }; use crate::test_support::tests::{crate_test_settings_str, create_test_settings}; use bytes::Bytes; use edgezero_core::body::Body as EdgeBody; use edgezero_core::http::response_builder as edge_response_builder; + use edgezero_core::outbound::{ + OutboundCachePolicy, OutboundCropFocus, OutboundImageFormat, OutboundImageOptimizerOptions, + OutboundImageOptimizerRegion, OutboundResizeFilter, ResponseDecoding, ResponseMode, + }; + use edgezero_core::time::Deadline; use error_stack::Report; use http::{HeaderValue, Method, Request as HttpRequest, Response, StatusCode, header}; @@ -2340,8 +2083,13 @@ mod tests { build_http_post_json_request(uri, &serde_json::json!({ "url": target })) } - fn build_http_post_streaming_request(uri: impl AsRef) -> HttpRequest { - let stream = futures::stream::iter(vec![Bytes::from_static(b"{}")]); + /// Builds a JSON `POST` whose `body` arrives as a stream of two chunks. + fn build_http_post_streaming_request( + uri: impl AsRef, + body: &'static [u8], + ) -> HttpRequest { + let (head, tail) = body.split_at(body.len() / 2); + let stream = futures::stream::iter([Bytes::from_static(head), Bytes::from_static(tail)]); HttpRequest::builder() .method(Method::POST) .uri(uri.as_ref()) @@ -2361,79 +2109,6 @@ mod tests { .expect("response body should be valid UTF-8") } - struct QueuedHttpResponse { - status: u16, - headers: Vec<(header::HeaderName, HeaderValue)>, - body: Vec, - } - - #[derive(Default)] - struct HeaderAwareStubHttpClient { - responses: Mutex>, - } - - impl HeaderAwareStubHttpClient { - fn new() -> Self { - Self::default() - } - - fn push_response( - &self, - status: u16, - headers: Vec<(header::HeaderName, HeaderValue)>, - body: Vec, - ) { - self.responses - .lock() - .expect("should lock queued responses") - .push_back(QueuedHttpResponse { - status, - headers, - body, - }); - } - } - - #[async_trait::async_trait(?Send)] - impl PlatformHttpClient for HeaderAwareStubHttpClient { - async fn send( - &self, - _request: PlatformHttpRequest, - ) -> Result> { - let queued = self - .responses - .lock() - .expect("should lock queued responses") - .pop_front() - .ok_or_else(|| Report::new(PlatformError::HttpClient))?; - - let mut builder = edgezero_core::http::response_builder().status(queued.status); - for (name, value) in queued.headers { - builder = builder.header(name, value); - } - - let response = builder - .body(EdgeBody::from(queued.body)) - .expect("should build stub HTTP response"); - - Ok(PlatformResponse::new(response)) - } - - async fn send_async( - &self, - _request: PlatformHttpRequest, - ) -> Result> { - Err(Report::new(PlatformError::Unsupported)) - } - - async fn select( - &self, - _pending_requests: Vec, - ) -> Result> { - Err(Report::new(PlatformError::Unsupported)) - } - } - fn build_http_response(status: StatusCode, body: EdgeBody) -> Response { let mut response = Response::new(body); *response.status_mut() = status; @@ -2447,57 +2122,6 @@ mod tests { .and_then(|value| value.to_str().ok()) } - /// Test double that always returns a streaming (non-buffered) response body. - /// - /// Used to exercise the `Body::Stream` pass-through path in - /// `proxy_request`, which returns the response body without materialising it. - /// Only `send` is implemented; `send_async` and - /// `select` return `PlatformError::Unsupported`. - struct StreamingResponseHttpClient; - - #[async_trait::async_trait(?Send)] - impl PlatformHttpClient for StreamingResponseHttpClient { - fn supports_streaming_responses(&self) -> bool { - true - } - - async fn send( - &self, - _request: PlatformHttpRequest, - ) -> Result> { - let edge_response = edge_response_builder() - .status(StatusCode::OK) - .header(header::SET_COOKIE.as_str(), "asset=1; Path=/; Secure") - .header(header::SET_COOKIE.as_str(), "other=2; Path=/; Secure") - .header( - header::STRICT_TRANSPORT_SECURITY.as_str(), - "max-age=31536000; includeSubDomains; preload", - ) - .header("clear-site-data", "\"*\"") - .header(header::ETAG.as_str(), "\"stream-etag\"") - .body(EdgeBody::stream(futures::stream::iter(vec![ - Bytes::from_static(b"chunk"), - ]))) - .expect("should build streaming test response"); - - Ok(PlatformResponse::new(edge_response).with_backend_name("stub-backend")) - } - - async fn send_async( - &self, - _request: PlatformHttpRequest, - ) -> Result> { - Err(Report::new(PlatformError::Unsupported)) - } - - async fn select( - &self, - _pending_requests: Vec, - ) -> Result> { - Err(Report::new(PlatformError::Unsupported)) - } - } - #[derive(Clone)] struct CountingSecretStore { data: Arc>>, @@ -2522,6 +2146,7 @@ mod tests { } } + #[async_trait::async_trait(?Send)] impl PlatformSecretStore for CountingSecretStore { fn get_bytes( &self, @@ -2538,7 +2163,7 @@ mod tests { .ok_or_else(|| Report::new(PlatformError::SecretStore)) } - fn create( + async fn create( &self, _store_id: &StoreId, _name: &str, @@ -2547,7 +2172,11 @@ mod tests { Err(Report::new(PlatformError::Unsupported)) } - fn delete(&self, _store_id: &StoreId, _name: &str) -> Result<(), Report> { + async fn delete( + &self, + _store_id: &StoreId, + _name: &str, + ) -> Result<(), Report> { Err(Report::new(PlatformError::Unsupported)) } } @@ -3006,7 +2635,9 @@ mod tests { ); req.headers_mut() .insert("x-forwarded-proto", HeaderValue::from_static("https")); - crate::forwarder::prepare_trusted_forwarder(&mut req, &settings); + crate::test_support::tests::with_request_parts(&mut req, |parts| { + crate::forwarder::prepare_trusted_forwarder(parts, &settings); + }); let resp = handle_first_party_proxy_sign(&settings, &noop_services(), req) .await @@ -4330,213 +3961,391 @@ mod tests { ); } - // --- Platform HTTP client integration --- - - #[test] - fn proxy_request_calls_platform_http_client_send() { - futures::executor::block_on(async { - use crate::platform::test_support::StubHttpClient; - - let stub = Arc::new(StubHttpClient::new()); - stub.push_response(200, b"ok".to_vec()); - let services = build_services_with_http_client( - Arc::clone(&stub) as Arc - ); - let settings = create_test_settings(); - let req = build_http_request(Method::GET, "https://example.com/"); + // --- Outbound client integration --- - let result = proxy_request( - &settings, - req, - ProxyRequestConfig { - target_url: "https://example.com/resource", - follow_redirects: false, - forward_ec_id: false, - body: None, - headers: Vec::new(), - copy_request_headers: false, - stream_passthrough: false, - stream_response: false, - allowed_domains: &[], - require_https: false, - }, - &services, - ) - .await; + const SOURCE: &str = "https://source.example.com"; - assert!(result.is_ok(), "should proxy successfully"); - let calls = stub.recorded_backend_names(); - assert_eq!(calls.len(), 1, "should call send exactly once"); - assert_eq!( - calls[0], "stub-backend", - "should use backend name from StubBackend" - ); - }); + /// Test services whose outbound requests go to a fresh scripted upstream. + fn scripted_services() -> (TestClock, ScriptedHttpClient, RuntimeServices) { + let clock = TestClock::new(); + let client = ScriptedHttpClient::new(&clock); + let services = build_services_with_outbound_client(&client, &clock); + (clock, client, services) } - #[test] - fn proxy_request_allows_open_mode_when_settings_allowlist_is_non_empty() { - futures::executor::block_on(async { - let mut settings = create_test_settings(); - settings.proxy.allowed_domains = vec!["allowed.example".to_string()]; - - let stub = Arc::new(HeaderAwareStubHttpClient::new()); - stub.push_response(200, Vec::new(), b"ok".to_vec()); - let services = build_services_with_http_client( - Arc::clone(&stub) as Arc - ); - let req = build_http_request(Method::GET, "https://edge.example/"); - - let response = proxy_request( - &settings, - req, - ProxyRequestConfig { - target_url: "https://blocked.example/resource.js", - follow_redirects: false, - forward_ec_id: false, - body: None, - headers: Vec::new(), - copy_request_headers: false, - stream_passthrough: false, - stream_response: false, - allowed_domains: &[], - require_https: false, - }, - &services, - ) - .await - .expect("open mode should ignore settings.proxy.allowed_domains"); + fn source_url(path: &str) -> String { + format!("{SOURCE}{path}") + } - assert_eq!(response.status(), StatusCode::OK); - assert_eq!(response_body_string(response), "ok"); - }); + fn redirect_reply(status: u16, location: &str) -> ScriptedReply { + ScriptedReply::status(StatusCode::from_u16(status).expect("should be a redirect status")) + .header("location", location) } - #[test] - fn proxy_request_uses_config_allowlist_for_redirect_hops() { - futures::executor::block_on(async { - let mut settings = create_test_settings(); - settings.proxy.allowed_domains = vec!["origin.example".to_string()]; - - let stub = Arc::new(HeaderAwareStubHttpClient::new()); - stub.push_response( - 302, - vec![( - header::LOCATION, - HeaderValue::from_static("https://redirected.example/final.js"), - )], - Vec::new(), - ); - stub.push_response(200, Vec::new(), b"redirected".to_vec()); + /// Settings every buffered proxy request carries. + fn proxied_transport() -> ExpectedTransport { + ExpectedTransport { + timeout: BUFFERED_TIMEOUT, + response_mode: ResponseMode::Buffered { + max_bytes: DEFAULT_BUFFERED_RESPONSE_BYTES, + }, + response_decoding: ResponseDecoding::Original, + cache_policy: OutboundCachePolicy::PlatformDefault, + } + } - let services = build_services_with_http_client( - Arc::clone(&stub) as Arc - ); - let req = build_http_request(Method::GET, "https://edge.example/"); + /// The chain deadline a followed request takes from a clock that never moved. + fn chain_deadline(clock: &TestClock) -> Deadline { + clock + .monotonic_clock() + .deadline_after(REDIRECT_CHAIN_DEADLINE) + } - let response = proxy_request( - &settings, - req, - ProxyRequestConfig { - target_url: "https://origin.example/start.js", - follow_redirects: true, - forward_ec_id: false, - body: None, - headers: Vec::new(), - copy_request_headers: false, - stream_passthrough: false, - stream_response: false, - allowed_domains: &[], - require_https: false, - }, - &services, - ) - .await - .expect("open mode should allow redirect hops outside settings allowlist"); + fn sent_methods(client: &ScriptedHttpClient) -> Vec { + client + .requests() + .into_iter() + .map(|request| request.method) + .collect() + } - assert_eq!(response.status(), StatusCode::OK); - assert_eq!(response_body_string(response), "redirected"); - }); + fn sent_uris(client: &ScriptedHttpClient) -> Vec { + client + .requests() + .iter() + .map(|request| request.uri.to_string()) + .collect() } - const REDIRECT_TEST_BODY: &[u8] = b"fictional-payload=example"; + fn sent_bodies(client: &ScriptedHttpClient) -> Vec { + client + .requests() + .into_iter() + .map(|request| request.body) + .collect() + } - /// Send `method` with a body through one redirect of `status` and return the stub. - fn proxy_through_redirect(method: Method, status: u16) -> Arc { - proxy_through_redirect_with_response(method, status).0 + fn sent_header_names(client: &ScriptedHttpClient, index: usize) -> Vec { + client.requests()[index] + .headers + .keys() + .map(|name| name.as_str().to_owned()) + .collect() } - /// Like [`proxy_through_redirect`], but also return the proxied response. - fn proxy_through_redirect_with_response( - method: Method, - status: u16, - ) -> (Arc, Response) { - let stub = Arc::new(StubHttpClient::new()); - stub.push_response_with_headers( - status, - Vec::new(), - vec![("location", "https://redirect.example.com/final")], - ); - stub.push_response(200, b"ok".to_vec()); - let services = build_services_with_http_client( - Arc::clone(&stub) as Arc + #[test] + fn first_party_proxy_request_readback() { + let (clock, client, services) = scripted_services(); + client.respond_to( + Method::GET, + "https://cdn.example.com/creative.png", + ScriptedReply::status(StatusCode::OK) + .header("content-type", "image/png") + .body("png"), ); let settings = create_test_settings(); - let req = build_http_request(method, "https://edge.example/"); + let first_party = + creative::build_proxy_url(&settings, "https://cdn.example.com/creative.png", ""); + let req = HttpRequest::builder() + .method(Method::GET) + .uri(format!("https://edge.example.com{first_party}")) + .header(header::USER_AGENT, "fictional-agent/1.0") + .header(header::ACCEPT, "image/*") + .header(header::ACCEPT_LANGUAGE, "en-US") + .header(header::REFERER, "https://publisher.example.com/article") + .header(HEADER_X_FORWARDED_FOR, "198.51.100.10") + .header(header::ACCEPT_ENCODING, "zstd") + .header(header::COOKIE, "session=fictional") + .header(header::AUTHORIZATION, "Basic ZmljdGlvbmFsOnNlY3JldA==") + .body(EdgeBody::empty()) + .expect("should build the first-party proxy request"); - let response = futures::executor::block_on(proxy_request( - &settings, + let response = + futures::executor::block_on(handle_first_party_proxy(&settings, &services, req)) + .expect("should proxy the creative"); + + assert_eq!(response.status(), StatusCode::OK, "should relay the origin"); + let requests = client.requests(); + assert_eq!(requests.len(), 1, "should send one creative request"); + let sent = &requests[0]; + assert_eq!( + (&sent.method, sent.uri.to_string()), + ( + &Method::GET, + "https://cdn.example.com/creative.png".to_owned() + ), + "should fetch the signed target" + ); + sent.assert_headers(&[ + ("user-agent", "fictional-agent/1.0"), + ("accept", "image/*"), + ("accept-language", "en-US"), + ("referer", "https://publisher.example.com/article"), + ("x-forwarded-for", "198.51.100.10"), + ("accept-encoding", SUPPORTED_ENCODINGS), + ]); + sent.assert_followed_transport(proxied_transport(), chain_deadline(&clock)); + } + + #[test] + fn follow_disabled_uses_send_with_forward_headers() { + let (_clock, client, services) = scripted_services(); + client.respond_to( + Method::POST, + &source_url("/collect"), + redirect_reply(302, "https://redirect.example.com/final"), + ); + let settings = create_test_settings(); + let req = HttpRequest::builder() + .method(Method::POST) + .uri("https://edge.example.com/collect") + .header(header::USER_AGENT, "fictional-agent/1.0") + .header(header::ACCEPT, "text/html") + .header(header::ACCEPT_LANGUAGE, "en-US") + .header(header::REFERER, "https://publisher.example.com/article") + .header(HEADER_X_FORWARDED_FOR, "198.51.100.10") + .header(header::ACCEPT_ENCODING, "zstd") + .header(header::COOKIE, "session=fictional") + .body(EdgeBody::empty()) + .expect("should build the client request"); + let target = source_url("/collect"); + let mut config = ProxyRequestConfig::new(&target) + .without_ec_id() + .with_body(b"fictional=beacon".to_vec()) + .with_header(header::CONTENT_TYPE, HeaderValue::from_static("text/plain")) + .with_header(header::ACCEPT, HeaderValue::from_static("*/*")); + config.follow_redirects = false; + + let response = + futures::executor::block_on(proxy_request(&settings, req, config, &services)) + .expect("should return the unfollowed redirect"); + + assert_eq!( + response.status(), + StatusCode::FOUND, + "should return the redirect instead of following it" + ); + let requests = client.requests(); + assert_eq!(requests.len(), 1, "should send one request"); + let sent = &requests[0]; + assert_eq!( + (&sent.method, sent.uri.to_string(), sent.body.as_ref()), + ( + &Method::POST, + source_url("/collect"), + b"fictional=beacon".as_slice() + ), + "should send the method, target, and buffered body" + ); + sent.assert_headers(&[ + ("user-agent", "fictional-agent/1.0"), + ("accept", "*/*"), + ("accept-language", "en-US"), + ("referer", "https://publisher.example.com/article"), + ("x-forwarded-for", "198.51.100.10"), + ("accept-encoding", SUPPORTED_ENCODINGS), + ("content-type", "text/plain"), + ]); + sent.assert_transport(proxied_transport()); + } + + #[test] + fn proxy_request_streams_response_when_asked() { + let (clock, client, services) = scripted_services(); + client.respond_to( + Method::GET, + &source_url("/resource"), + ScriptedReply::status(StatusCode::OK).chunks(vec![ + Bytes::from_static(b"stream"), + Bytes::from_static(b"ed"), + ]), + ); + let settings = create_test_settings(); + let req = build_http_request(Method::GET, "https://edge.example.com/"); + + let response = futures::executor::block_on(proxy_request( + &settings, req, - ProxyRequestConfig::new("https://source.example.com/start") + ProxyRequestConfig::new(&source_url("/resource")) .without_ec_id() .without_forward_headers() + .with_streaming() + .with_stream_response(), + &services, + )) + .expect("should proxy successfully"); + + assert!( + response.body().is_stream(), + "should keep the origin body as a stream" + ); + let body = futures::executor::block_on(response.into_body().into_bytes_bounded(1024)) + .expect("should read the streamed body"); + assert_eq!(body.as_ref(), b"streamed", "should stream every chunk"); + client.requests()[0].assert_followed_transport( + ExpectedTransport { + timeout: STREAMED_TIMEOUT, + response_mode: ResponseMode::Streamed, + response_decoding: ResponseDecoding::Original, + cache_policy: OutboundCachePolicy::PlatformDefault, + }, + chain_deadline(&clock), + ); + } + + #[test] + fn proxy_target_with_fragment_is_sent_without_it() { + let (_clock, client, services) = scripted_services(); + client.respond_to( + Method::GET, + &source_url("/creative.html"), + ScriptedReply::status(StatusCode::OK).body("ok"), + ); + let settings = create_test_settings(); + let req = build_http_request(Method::GET, "https://edge.example.com/"); + + futures::executor::block_on(proxy_request( + &settings, + req, + ProxyRequestConfig::new(&source_url("/creative.html#landing")) + .without_ec_id() + .without_forward_headers(), + &services, + )) + .expect("should proxy a target that carries a fragment"); + + assert_eq!( + sent_uris(&client), + vec![source_url("/creative.html")], + "should drop the fragment from the upstream target" + ); + } + + #[test] + fn proxy_request_allows_open_mode_when_settings_allowlist_is_non_empty() { + let (_clock, client, services) = scripted_services(); + client.respond_to( + Method::GET, + "https://blocked.example.com/resource.js", + ScriptedReply::status(StatusCode::OK).body("ok"), + ); + let mut settings = create_test_settings(); + settings.proxy.allowed_domains = vec!["allowed.example.com".to_string()]; + let req = build_http_request(Method::GET, "https://edge.example.com/"); + let mut config = ProxyRequestConfig::new("https://blocked.example.com/resource.js") + .without_ec_id() + .without_forward_headers(); + config.follow_redirects = false; + + let response = + futures::executor::block_on(proxy_request(&settings, req, config, &services)) + .expect("open mode should ignore settings.proxy.allowed_domains"); + + assert_eq!(response.status(), StatusCode::OK); + assert_eq!(response_body_string(response), "ok"); + } + + #[test] + fn proxy_request_uses_config_allowlist_for_redirect_hops() { + let (_clock, client, services) = scripted_services(); + client + .respond_to( + Method::GET, + "https://origin.example.com/start.js", + redirect_reply(302, "https://redirected.example.com/final.js"), + ) + .respond_to( + Method::GET, + "https://redirected.example.com/final.js", + ScriptedReply::status(StatusCode::OK).body("redirected"), + ); + let mut settings = create_test_settings(); + settings.proxy.allowed_domains = vec!["origin.example.com".to_string()]; + let req = build_http_request(Method::GET, "https://edge.example.com/"); + + let response = futures::executor::block_on(proxy_request( + &settings, + req, + ProxyRequestConfig::new("https://origin.example.com/start.js") + .without_ec_id() + .without_forward_headers(), + &services, + )) + .expect("open mode should allow redirect hops outside settings allowlist"); + + assert_eq!(response.status(), StatusCode::OK); + assert_eq!(response_body_string(response), "redirected"); + } + + const REDIRECT_TEST_BODY: &[u8] = b"fictional-payload=example"; + + /// Sends `method` through one `status` redirect and returns the upstream and the response. + /// + /// A method that carries a body sends [`REDIRECT_TEST_BODY`] with body headers. `GET` and + /// `HEAD` send none, because `EdgeZero` rejects a body on them. + fn proxy_through_redirect( + method: &Method, + status: u16, + ) -> (ScriptedHttpClient, Response) { + let (_clock, client, services) = scripted_services(); + client.respond_to( + method.clone(), + &source_url("/start"), + redirect_reply(status, "https://redirect.example.com/final"), + ); + for follow_up in [Method::GET, Method::HEAD, method.clone()] { + client.respond_to( + follow_up, + "https://redirect.example.com/final", + ScriptedReply::status(StatusCode::OK).body("ok"), + ); + } + let settings = create_test_settings(); + let req = build_http_request(method.clone(), "https://edge.example.com/"); + let target = source_url("/start"); + let mut config = ProxyRequestConfig::new(&target) + .without_ec_id() + .without_forward_headers(); + if method != Method::GET && method != Method::HEAD { + config = config .with_body(REDIRECT_TEST_BODY.to_vec()) .with_header( header::CONTENT_TYPE, HeaderValue::from_static("application/x-www-form-urlencoded"), ) - .with_header(header::CONTENT_LANGUAGE, HeaderValue::from_static("en-US")) - .with_header( - header::CONTENT_LENGTH, - HeaderValue::from(REDIRECT_TEST_BODY.len()), - ), - &services, - )) - .expect("should follow redirect"); + .with_header(header::CONTENT_LANGUAGE, HeaderValue::from_static("en-US")); + } - (stub, response) - } + let response = + futures::executor::block_on(proxy_request(&settings, req, config, &services)) + .expect("should proxy through the redirect"); - fn recorded_header_names(stub: &StubHttpClient, index: usize) -> Vec { - stub.recorded_request_headers()[index] - .iter() - .map(|(name, _)| name.clone()) - .collect() + (client, response) } #[test] fn proxy_request_post_redirect_drops_body_for_301_302_303() { for status in [301, 302, 303] { - let stub = proxy_through_redirect(Method::POST, status); + let (client, _response) = proxy_through_redirect(&Method::POST, status); assert_eq!( - stub.recorded_request_methods(), - vec!["POST".to_string(), "GET".to_string()], + sent_methods(&client), + vec![Method::POST, Method::GET], "should follow a {status} POST redirect with GET" ); assert_eq!( - stub.recorded_request_uris()[1], + sent_uris(&client)[1], "https://redirect.example.com/final", "should send the {status} follow-up to the Location target" ); assert_eq!( - stub.recorded_request_bodies(), - vec![REDIRECT_TEST_BODY.to_vec(), Vec::new()], + sent_bodies(&client), + vec![Bytes::from_static(REDIRECT_TEST_BODY), Bytes::new()], "should send the body only on the first hop for {status}" ); - let first_hop_headers = recorded_header_names(&stub, 0); - let follow_up_headers = recorded_header_names(&stub, 1); - for body_header in ["content-type", "content-language", "content-length"] { + let first_hop_headers = sent_header_names(&client, 0); + let follow_up_headers = sent_header_names(&client, 1); + for body_header in ["content-type", "content-language"] { assert!( first_hop_headers.iter().any(|name| name == body_header), "should send {body_header} on the first {status} hop" @@ -4553,7 +4362,7 @@ mod tests { fn proxy_request_returns_301_302_unfollowed_for_non_post_body_methods() { for method in [Method::PUT, Method::PATCH, Method::DELETE] { for status in [301, 302] { - let (stub, response) = proxy_through_redirect_with_response(method.clone(), status); + let (client, response) = proxy_through_redirect(&method, status); assert_eq!( response.status().as_u16(), @@ -4561,13 +4370,13 @@ mod tests { "should return the {status} redirect for {method} to the caller" ); assert_eq!( - stub.recorded_request_methods(), - vec![method.to_string()], + sent_methods(&client), + vec![method.clone()], "should not follow a {status} redirect for {method}" ); assert_eq!( - stub.recorded_request_bodies(), - vec![REDIRECT_TEST_BODY.to_vec()], + sent_bodies(&client), + vec![Bytes::from_static(REDIRECT_TEST_BODY)], "should send the {method} body only to the addressed target on {status}" ); } @@ -4576,7 +4385,7 @@ mod tests { #[test] fn proxy_request_put_redirect_becomes_bodyless_get_for_303() { - let (stub, response) = proxy_through_redirect_with_response(Method::PUT, 303); + let (client, response) = proxy_through_redirect(&Method::PUT, 303); assert_eq!( response.status().as_u16(), @@ -4584,13 +4393,13 @@ mod tests { "should return the 303 target's response" ); assert_eq!( - stub.recorded_request_methods(), - vec!["PUT".to_string(), "GET".to_string()], + sent_methods(&client), + vec![Method::PUT, Method::GET], "should follow a 303 PUT redirect with GET" ); assert_eq!( - stub.recorded_request_bodies(), - vec![REDIRECT_TEST_BODY.to_vec(), Vec::new()], + sent_bodies(&client), + vec![Bytes::from_static(REDIRECT_TEST_BODY), Bytes::new()], "should send the PUT body only on the first hop" ); } @@ -4598,20 +4407,23 @@ mod tests { #[test] fn proxy_request_post_redirect_retains_method_and_body_for_307_308() { for status in [307, 308] { - let stub = proxy_through_redirect(Method::POST, status); + let (client, _response) = proxy_through_redirect(&Method::POST, status); assert_eq!( - stub.recorded_request_methods(), - vec!["POST".to_string(), "POST".to_string()], + sent_methods(&client), + vec![Method::POST, Method::POST], "should keep POST across a {status} redirect" ); assert_eq!( - stub.recorded_request_bodies(), - vec![REDIRECT_TEST_BODY.to_vec(), REDIRECT_TEST_BODY.to_vec()], + sent_bodies(&client), + vec![ + Bytes::from_static(REDIRECT_TEST_BODY), + Bytes::from_static(REDIRECT_TEST_BODY) + ], "should replay the exact body across a {status} redirect" ); assert!( - recorded_header_names(&stub, 1) + sent_header_names(&client, 1) .iter() .any(|name| name == "content-type"), "should keep Content-Type across a {status} redirect" @@ -4621,28 +4433,30 @@ mod tests { #[test] fn proxy_request_post_body_stays_dropped_after_302_then_307() { - let stub = Arc::new(StubHttpClient::new()); - stub.push_response_with_headers( - 302, - Vec::new(), - vec![("location", "https://redirect.example.com/second")], - ); - stub.push_response_with_headers( - 307, - Vec::new(), - vec![("location", "https://redirect.example.com/final")], - ); - stub.push_response(200, b"ok".to_vec()); - let services = build_services_with_http_client( - Arc::clone(&stub) as Arc - ); + let (_clock, client, services) = scripted_services(); + client + .respond_to( + Method::POST, + &source_url("/start"), + redirect_reply(302, "https://redirect.example.com/second"), + ) + .respond_to( + Method::GET, + "https://redirect.example.com/second", + redirect_reply(307, "https://redirect.example.com/final"), + ) + .respond_to( + Method::GET, + "https://redirect.example.com/final", + ScriptedReply::status(StatusCode::OK).body("ok"), + ); let settings = create_test_settings(); - let req = build_http_request(Method::POST, "https://edge.example/"); + let req = build_http_request(Method::POST, "https://edge.example.com/"); futures::executor::block_on(proxy_request( &settings, req, - ProxyRequestConfig::new("https://source.example.com/start") + ProxyRequestConfig::new(&source_url("/start")) .without_ec_id() .without_forward_headers() .with_body(REDIRECT_TEST_BODY.to_vec()) @@ -4650,10 +4464,6 @@ mod tests { header::CONTENT_TYPE, HeaderValue::from_static("application/x-www-form-urlencoded"), ) - .with_header( - header::CONTENT_LENGTH, - HeaderValue::from(REDIRECT_TEST_BODY.len()), - ) .with_header( header::CONTENT_ENCODING, HeaderValue::from_static("identity"), @@ -4668,19 +4478,22 @@ mod tests { .expect("should follow the 302 then 307 chain"); assert_eq!( - stub.recorded_request_methods(), - vec!["POST".to_string(), "GET".to_string(), "GET".to_string()], + sent_methods(&client), + vec![Method::POST, Method::GET, Method::GET], "should keep the 302 follow-up as GET across the later 307" ); assert_eq!( - stub.recorded_request_bodies(), - vec![REDIRECT_TEST_BODY.to_vec(), Vec::new(), Vec::new()], + sent_bodies(&client), + vec![ + Bytes::from_static(REDIRECT_TEST_BODY), + Bytes::new(), + Bytes::new() + ], "should not restore the dropped body on the 307 hop" ); - let first_hop_headers = recorded_header_names(&stub, 0); + let first_hop_headers = sent_header_names(&client, 0); for body_header in [ "content-type", - "content-length", "content-encoding", "content-language", "content-location", @@ -4691,7 +4504,7 @@ mod tests { ); for hop in [1, 2] { assert!( - !recorded_header_names(&stub, hop) + !sent_header_names(&client, hop) .iter() .any(|name| name == body_header), "should not send {body_header} on bodyless hop {hop}" @@ -4700,29 +4513,28 @@ mod tests { } } + /// Scripts a chain of five `302` answers, one more than the redirect limit follows. + fn script_redirect_limit(client: &ScriptedHttpClient, reply: impl Fn() -> ScriptedReply) { + client.respond_to(Method::GET, &source_url("/start"), reply()); + for _ in 0..4 { + client.respond_to(Method::GET, "https://redirect.example.com/next", reply()); + } + } + #[test] fn proxy_request_redirect_limit_response_strips_cors_policy() { - let stub = Arc::new(StubHttpClient::new()); - for _ in 0..5 { - stub.push_response_with_headers( - 302, - Vec::new(), - vec![ - ("location", "https://redirect.example.com/next"), - ("access-control-allow-origin", "*"), - ], - ); - } - let services = build_services_with_http_client( - Arc::clone(&stub) as Arc - ); + let (_clock, client, services) = scripted_services(); + script_redirect_limit(&client, || { + redirect_reply(302, "https://redirect.example.com/next") + .header("access-control-allow-origin", "*") + }); let settings = create_test_settings(); - let req = build_http_request(Method::GET, "https://edge.example/"); + let req = build_http_request(Method::GET, "https://edge.example.com/"); let response = futures::executor::block_on(proxy_request( &settings, req, - ProxyRequestConfig::new("https://source.example.com/start") + ProxyRequestConfig::new(&source_url("/start")) .without_ec_id() .without_forward_headers(), &services, @@ -4730,7 +4542,7 @@ mod tests { .expect("should return the redirect once the limit is reached"); assert_eq!( - stub.recorded_request_methods().len(), + client.requests().len(), 5, "should stop following after the redirect limit" ); @@ -4750,27 +4562,19 @@ mod tests { #[test] fn proxy_request_redirect_limit_response_honors_stream_passthrough() { - let stub = Arc::new(StubHttpClient::new()); - for _ in 0..5 { - stub.push_response_with_headers( - 302, - b"

moved

".to_vec(), - vec![ - ("location", "https://redirect.example.com/next"), - ("content-type", "text/html"), - ], - ); - } - let services = build_services_with_http_client( - Arc::clone(&stub) as Arc - ); + let (_clock, client, services) = scripted_services(); + script_redirect_limit(&client, || { + redirect_reply(302, "https://redirect.example.com/next") + .header("content-type", "text/html") + .body("

moved

") + }); let settings = create_test_settings(); - let req = build_http_request(Method::GET, "https://edge.example/"); + let req = build_http_request(Method::GET, "https://edge.example.com/"); let response = futures::executor::block_on(proxy_request( &settings, req, - ProxyRequestConfig::new("https://source.example.com/start") + ProxyRequestConfig::new(&source_url("/start")) .without_ec_id() .without_forward_headers() .with_streaming(), @@ -4783,21 +4587,26 @@ mod tests { Some(&HeaderValue::from_static("text/html")), "should pass the redirect-limit response through without the HTML rewrite" ); + assert_eq!( + response_body_string(response), + "

moved

", + "should keep the redirect body unchanged" + ); } #[test] fn proxy_request_head_redirect_stays_head_for_301_302_303() { for status in [301, 302, 303] { - let stub = proxy_through_redirect(Method::HEAD, status); + let (client, _response) = proxy_through_redirect(&Method::HEAD, status); assert_eq!( - stub.recorded_request_methods(), - vec!["HEAD".to_string(), "HEAD".to_string()], + sent_methods(&client), + vec![Method::HEAD, Method::HEAD], "should keep HEAD across a {status} redirect" ); assert!( - stub.recorded_request_bodies()[1].is_empty(), - "should send no body on the {status} HEAD follow-up" + sent_bodies(&client).iter().all(Bytes::is_empty), + "should send no body on either {status} HEAD hop" ); } } @@ -4805,32 +4614,29 @@ mod tests { #[test] fn proxy_request_head_redirect_keeps_header_only_compressed_response() { for (content_type, content_encoding) in [("text/html", "gzip"), ("text/css", "br")] { - let stub = Arc::new(StubHttpClient::new()); - stub.push_response_with_headers( - 303, - Vec::new(), - vec![("location", "https://redirect.example.com/final")], - ); - stub.push_response_with_headers( - 200, - Vec::new(), - vec![ - ("content-type", content_type), - ("content-encoding", content_encoding), - ("content-length", "1234"), - ("access-control-allow-origin", "*"), - ], - ); - let services = build_services_with_http_client( - Arc::clone(&stub) as Arc - ); + let (_clock, client, services) = scripted_services(); + client + .respond_to( + Method::HEAD, + &source_url("/start"), + redirect_reply(303, "https://redirect.example.com/final"), + ) + .respond_to( + Method::HEAD, + "https://redirect.example.com/final", + ScriptedReply::status(StatusCode::OK) + .header("content-type", content_type) + .header("content-encoding", content_encoding) + .header("content-length", "1234") + .header("access-control-allow-origin", "*"), + ); let settings = create_test_settings(); - let req = build_http_request(Method::HEAD, "https://edge.example/"); + let req = build_http_request(Method::HEAD, "https://edge.example.com/"); let response = futures::executor::block_on(proxy_request( &settings, req, - ProxyRequestConfig::new("https://source.example.com/start") + ProxyRequestConfig::new(&source_url("/start")) .without_ec_id() .without_forward_headers(), &services, @@ -4838,8 +4644,8 @@ mod tests { .expect("should return the header-only HEAD response"); assert_eq!( - stub.recorded_request_methods(), - vec!["HEAD".to_string(), "HEAD".to_string()], + sent_methods(&client), + vec![Method::HEAD, Method::HEAD], "should keep HEAD across the 303 redirect" ); assert_eq!( @@ -4871,206 +4677,304 @@ mod tests { #[test] fn proxy_request_post_redirect_still_enforces_per_hop_allowlist() { - futures::executor::block_on(async { - let stub = Arc::new(StubHttpClient::new()); - stub.push_response_with_headers( - 307, - Vec::new(), - vec![("location", "https://blocked.example.com/final")], - ); - let services = build_services_with_http_client( - Arc::clone(&stub) as Arc - ); - let settings = create_test_settings(); - let allowed_domains = vec!["source.example.com".to_string()]; - let req = build_http_request(Method::POST, "https://edge.example/"); + let (_clock, client, services) = scripted_services(); + client.respond_to( + Method::POST, + &source_url("/start"), + redirect_reply(307, "https://blocked.example.com/final"), + ); + let settings = create_test_settings(); + let allowed_domains = vec!["source.example.com".to_string()]; + let req = build_http_request(Method::POST, "https://edge.example.com/"); - let err = proxy_request( - &settings, - req, - ProxyRequestConfig::new("https://source.example.com/start") - .without_ec_id() - .without_forward_headers() - .with_body(REDIRECT_TEST_BODY.to_vec()) - .with_allowed_domains(&allowed_domains), - &services, - ) - .await - .expect_err("should block a redirect to a host outside the allowlist"); + let err = futures::executor::block_on(proxy_request( + &settings, + req, + ProxyRequestConfig::new(&source_url("/start")) + .without_ec_id() + .without_forward_headers() + .with_body(REDIRECT_TEST_BODY.to_vec()) + .with_allowed_domains(&allowed_domains), + &services, + )) + .expect_err("should block a redirect to a host outside the allowlist"); - assert!( - matches!( - err.current_context(), - TrustedServerError::AllowlistViolation { host } if host == "blocked.example.com" - ), - "should report the blocked redirect host" - ); - assert_eq!( - stub.recorded_request_bodies().len(), - 1, - "should never send the body to the blocked host" - ); - }); + assert!( + matches!( + err.current_context(), + TrustedServerError::AllowlistViolation { host } if host == "blocked.example.com" + ), + "should report the blocked redirect host" + ); + assert_eq!( + client.requests().len(), + 1, + "should never send the body to the blocked host" + ); } #[test] - fn proxy_request_forwards_stream_response_flag_to_platform_request() { - futures::executor::block_on(async { - use crate::platform::test_support::StubHttpClient; - - let stub = Arc::new(StubHttpClient::new()); - stub.set_streaming_responses_supported(true); - stub.push_response(200, b"ok".to_vec()); - let services = build_services_with_http_client( - Arc::clone(&stub) as Arc + fn redirect_policy_blocks_https_downgrade_before_other_checks() { + for location in [ + "http://blocked.example.com/final", + "ftp://files.example.com/final", + ] { + let (_clock, client, services) = scripted_services(); + client.respond_to( + Method::GET, + &source_url("/start"), + redirect_reply(302, location), ); let settings = create_test_settings(); - let req = build_http_request(Method::GET, "https://example.com/"); + let allowed_domains = vec!["source.example.com".to_string()]; + let req = build_http_request(Method::GET, "https://edge.example.com/"); - proxy_request( + let err = futures::executor::block_on(proxy_request( &settings, req, - ProxyRequestConfig::new("https://example.com/resource") + ProxyRequestConfig::new(&source_url("/start")) + .without_ec_id() .without_forward_headers() - .with_stream_response(), + .with_allowed_domains(&allowed_domains) + .with_https_only(), &services, - ) - .await - .expect("should proxy successfully"); + )) + .expect_err("should block a redirect that leaves HTTPS"); - assert_eq!( - stub.recorded_stream_response_flags(), - vec![true], - "should request a streaming platform response" + assert!( + matches!(err.current_context(), TrustedServerError::Forbidden { .. }), + "should forbid the {location} hop before its scheme or host is checked" ); - }); + assert_eq!(client.requests().len(), 1, "should not follow {location}"); + } } #[test] - fn proxy_request_forwards_curated_headers_when_copy_request_headers_is_true() { - futures::executor::block_on(async { - use crate::platform::test_support::StubHttpClient; - - let stub = Arc::new(StubHttpClient::new()); - stub.push_response(200, b"ok".to_vec()); - let services = build_services_with_http_client( - Arc::clone(&stub) as Arc - ); - let settings = create_test_settings(); - let mut req = HttpRequest::builder() - .method(Method::GET) - .uri("https://example.com/") - .body(EdgeBody::empty()) - .expect("should build test request"); - req.headers_mut().insert( - header::USER_AGENT, - HeaderValue::from_static("test-agent/1.0"), - ); - req.headers_mut() - .insert(header::ACCEPT, HeaderValue::from_static("text/html")); - req.headers_mut() - .insert(header::ACCEPT_LANGUAGE, HeaderValue::from_static("en-US")); - - let result = proxy_request( - &settings, - req, - ProxyRequestConfig { - target_url: "https://example.com/resource", - follow_redirects: false, - forward_ec_id: false, - body: None, - headers: Vec::new(), - copy_request_headers: true, - stream_passthrough: false, - stream_response: false, - allowed_domains: &[], - require_https: false, - }, - &services, - ) - .await; - - assert!(result.is_ok(), "should proxy successfully"); - let all_headers = stub.recorded_request_headers(); - assert_eq!(all_headers.len(), 1, "should have captured one request"); - let sent = &all_headers[0]; + fn redirect_policy_returns_the_redirect_for_other_schemes() { + let (_clock, client, services) = scripted_services(); + client.respond_to( + Method::GET, + &source_url("/start"), + redirect_reply(302, "ftp://files.example.com/final"), + ); + let settings = create_test_settings(); + let req = build_http_request(Method::GET, "https://edge.example.com/"); - let header_value = |name: &str| -> Option { - sent.iter().find(|(n, _)| n == name).map(|(_, v)| v.clone()) - }; + let response = futures::executor::block_on(proxy_request( + &settings, + req, + ProxyRequestConfig::new(&source_url("/start")) + .without_ec_id() + .without_forward_headers(), + &services, + )) + .expect("should return the redirect to a scheme it cannot follow"); - assert_eq!( - header_value("user-agent").as_deref(), - Some("test-agent/1.0"), - "should forward User-Agent" - ); - assert_eq!( - header_value("accept").as_deref(), - Some("text/html"), - "should forward Accept" - ); - assert_eq!( - header_value("accept-language").as_deref(), - Some("en-US"), - "should forward Accept-Language" - ); - assert_eq!( - header_value("accept-encoding").as_deref(), - Some(SUPPORTED_ENCODINGS), - "should override Accept-Encoding with supported encodings" - ); - }); + assert_eq!( + response.status(), + StatusCode::FOUND, + "should return the 302 itself" + ); + assert_eq!(client.requests().len(), 1, "should not follow the ftp hop"); } #[test] - fn proxy_request_passes_through_streaming_platform_response_body() { - futures::executor::block_on(async { - // HTTP types can carry streaming bodies; proxy_request returns Ok even when - // the origin sends a streaming body. - let services = build_services_with_http_client( - Arc::new(StreamingResponseHttpClient) as Arc + fn redirect_chain_past_deadline_fails_with_site_502() { + let (clock, client, services) = scripted_services(); + let hops = ["/start", "/one", "/two", "/three", "/four"]; + for pair in hops.windows(2) { + client.respond_to( + Method::GET, + &source_url(pair[0]), + redirect_reply(302, &source_url(pair[1])).after(Duration::from_secs(14)), ); - let settings = create_test_settings(); - let req = HttpRequest::builder() - .method(Method::GET) - .uri("https://example.com/") - .body(EdgeBody::empty()) - .expect("should build test request"); - - let result = proxy_request( - &settings, - req, - ProxyRequestConfig { - target_url: "https://example.com/resource", - follow_redirects: false, - forward_ec_id: false, - body: None, - headers: Vec::new(), - copy_request_headers: false, - stream_passthrough: false, - stream_response: false, - allowed_domains: &[], - require_https: false, - }, - &services, - ) - .await; + } + client.respond_to( + Method::GET, + &source_url("/four"), + ScriptedReply::status(StatusCode::OK).after(Duration::from_secs(5)), + ); + let deadline = chain_deadline(&clock); + let settings = create_test_settings(); + let req = build_http_request(Method::GET, "https://edge.example.com/"); + let target = source_url("/start"); + let mut proxied = std::pin::pin!(proxy_request( + &settings, + req, + ProxyRequestConfig::new(&target) + .without_ec_id() + .without_forward_headers(), + &services, + )); + let mut context = Context::from_waker(Waker::noop()); + // Each hop answers inside its own 15 s timeout, but the fifth answer lands at 61 s. + for step in [14, 14, 14, 14, 5] { assert!( - result.is_ok(), - "should pass streaming body through with HTTP types: {result:?}" - ); - assert_eq!( - result.expect("should succeed").status(), - StatusCode::OK, - "should preserve the origin status code" + proxied.as_mut().poll(&mut context).is_pending(), + "should wait for the next hop" ); - }); - } + clock.advance(Duration::from_secs(step)); + } + let Poll::Ready(outcome) = proxied.as_mut().poll(&mut context) else { + panic!("should finish once the chain deadline passes"); + }; - #[test] - fn rebuild_response_with_body_preserves_multiple_set_cookie_headers() { - let mut beresp = Response::new(EdgeBody::empty()); + let err = outcome.expect_err("should fail past the chain deadline"); + assert!( + matches!(err.current_context(), TrustedServerError::Proxy { .. }), + "should keep the proxy's own error" + ); + assert_eq!( + err.current_context().status_code(), + StatusCode::BAD_GATEWAY, + "should answer the expired chain with the site's 502" + ); + let requests = client.requests(); + assert_eq!( + requests.len(), + 5, + "should send every hop before the deadline" + ); + for request in &requests { + request.assert_followed_transport(proxied_transport(), deadline); + } + } + + #[test] + fn location_with_userinfo_fails_with_site_502() { + let (_clock, client, services) = scripted_services(); + client.respond_to( + Method::GET, + &source_url("/start"), + redirect_reply(302, "https://fictional:secret@redirect.example.com/final"), + ); + let settings = create_test_settings(); + let req = build_http_request(Method::GET, "https://edge.example.com/"); + + let err = futures::executor::block_on(proxy_request( + &settings, + req, + ProxyRequestConfig::new(&source_url("/start")) + .without_ec_id() + .without_forward_headers(), + &services, + )) + .expect_err("should reject a Location that carries userinfo"); + + assert_eq!( + err.current_context().status_code(), + StatusCode::BAD_GATEWAY, + "should answer the userinfo redirect with the site's 502" + ); + assert_eq!( + client.requests().len(), + 1, + "should not follow the userinfo Location" + ); + } + + #[test] + fn cross_origin_hop_strips_authorization_and_cookie() { + let (_clock, client, services) = scripted_services(); + client + .respond_to( + Method::GET, + &source_url("/start"), + redirect_reply(302, &source_url("/same-origin")), + ) + .respond_to( + Method::GET, + &source_url("/same-origin"), + redirect_reply(302, "https://other.example.com/final"), + ) + .respond_to( + Method::GET, + "https://other.example.com/final", + ScriptedReply::status(StatusCode::OK).body("ok"), + ); + let settings = create_test_settings(); + let req = build_http_request(Method::GET, "https://edge.example.com/"); + + futures::executor::block_on(proxy_request( + &settings, + req, + ProxyRequestConfig::new(&source_url("/start")) + .without_ec_id() + .without_forward_headers() + .with_header( + header::AUTHORIZATION, + HeaderValue::from_static("Bearer fictional-token"), + ) + .with_header( + header::COOKIE, + HeaderValue::from_static("session=fictional"), + ), + &services, + )) + .expect("should follow the chain"); + + let requests = client.requests(); + assert_eq!(requests.len(), 3, "should follow both redirects"); + for (index, request) in requests.iter().enumerate() { + let credentials = ( + request.headers.contains_key(header::AUTHORIZATION), + request.headers.contains_key(header::COOKIE), + ); + let expected = if index < 2 { + (true, true) + } else { + (false, false) + }; + assert_eq!( + credentials, expected, + "should keep credentials only on same-origin hops, hop {index} to {}", + request.uri + ); + } + } + + #[test] + fn non_ascii_location_is_percent_encoded_and_followed() { + let (_clock, client, services) = scripted_services(); + client + .respond_to( + Method::GET, + &source_url("/start"), + redirect_reply(302, "/caf\u{e9}?q=\u{e9}t\u{e9}"), + ) + .respond_to( + Method::GET, + &source_url("/caf%C3%A9?q=%C3%A9t%C3%A9"), + ScriptedReply::status(StatusCode::OK).body("followed"), + ); + let settings = create_test_settings(); + let req = build_http_request(Method::GET, "https://edge.example.com/"); + + let response = futures::executor::block_on(proxy_request( + &settings, + req, + ProxyRequestConfig::new(&source_url("/start")) + .without_ec_id() + .without_forward_headers(), + &services, + )) + .expect("should follow the non-ASCII Location"); + + assert_eq!( + sent_uris(&client), + vec![ + source_url("/start"), + source_url("/caf%C3%A9?q=%C3%A9t%C3%A9") + ], + "should percent-encode the Location before following it" + ); + assert_eq!(response_body_string(response), "followed"); + } + + #[test] + fn rebuild_response_with_body_preserves_multiple_set_cookie_headers() { + let mut beresp = Response::new(EdgeBody::empty()); *beresp.status_mut() = StatusCode::OK; beresp.headers_mut().append( header::SET_COOKIE, @@ -5240,238 +5144,157 @@ mod tests { ); } - #[test] - fn handle_asset_proxy_request_forwards_asset_headers_and_host() { - futures::executor::block_on(async { - use crate::platform::test_support::StubHttpClient; - - let stub = Arc::new(StubHttpClient::new()); - stub.set_streaming_responses_supported(false); - stub.push_response(200, b"ok".to_vec()); - let services = build_services_with_http_client( - Arc::clone(&stub) as Arc - ); - let settings = create_test_settings(); - let mut req = build_http_request( - Method::GET, - "https://www.example.com/.images/foo.jpg?auto=webp", - ); - req.headers_mut().insert( - header::USER_AGENT, - HeaderValue::from_static("asset-agent/1.0"), - ); - req.headers_mut().insert( - header::ACCEPT, - HeaderValue::from_static("image/avif,image/webp,image/*,*/*;q=0.8"), - ); - req.headers_mut().insert( - header::ACCEPT_ENCODING, - HeaderValue::from_static("gzip, br"), - ); - req.headers_mut() - .insert(header::ACCEPT_LANGUAGE, HeaderValue::from_static("en-US")); - req.headers_mut().insert( - header::REFERER, - HeaderValue::from_static("https://www.example.com/article"), - ); - req.headers_mut().insert( - header::IF_NONE_MATCH, - HeaderValue::from_static("\"asset-etag\""), - ); - req.headers_mut().insert( - header::IF_MODIFIED_SINCE, - HeaderValue::from_static("Thu, 13 Mar 2025 08:00:00 GMT"), - ); - req.headers_mut().insert( - header::IF_MATCH, - HeaderValue::from_static("\"asset-precondition\""), - ); - req.headers_mut().insert( - header::IF_UNMODIFIED_SINCE, - HeaderValue::from_static("Thu, 13 Mar 2025 09:00:00 GMT"), - ); - req.headers_mut() - .insert(header::RANGE, HeaderValue::from_static("bytes=0-1023")); - req.headers_mut().insert( - header::IF_RANGE, - HeaderValue::from_static("\"asset-range\""), - ); - req.headers_mut().insert( - HEADER_X_FORWARDED_FOR, - HeaderValue::from_static("198.51.100.10"), - ); - req.headers_mut().insert( - header::HeaderName::from_static("x-custom-test"), - HeaderValue::from_static("drop-me"), - ); - - let route = ProxyAssetRoute::new("/.images/", "https://assets.example.com:8443"); - let response = handle_asset_proxy_request(&settings, &services, req, &route) - .await - .expect("should proxy asset request") - .into_response() - .expect("should return buffered asset response"); - assert_eq!(response.status(), StatusCode::OK); - assert_eq!( - stub.recorded_stream_response_flags(), - vec![false], - "unsupported platforms should buffer asset proxy responses" - ); + /// Settings every asset `GET` carries: a streamed response in its original coding. + fn asset_get_transport() -> ExpectedTransport { + ExpectedTransport { + timeout: STREAMED_TIMEOUT, + response_mode: ResponseMode::Streamed, + response_decoding: ResponseDecoding::Original, + cache_policy: OutboundCachePolicy::PlatformDefault, + } + } - let all_headers = stub.recorded_request_headers(); - assert_eq!(all_headers.len(), 1, "should have captured one request"); - let sent = &all_headers[0]; - let header_value = |name: &str| -> Option { - sent.iter().find(|(n, _)| n == name).map(|(_, v)| v.clone()) - }; + /// Settings every asset `HEAD` carries, the S3 preflight included. + fn asset_head_transport() -> ExpectedTransport { + ExpectedTransport { + timeout: BUFFERED_TIMEOUT, + response_mode: ResponseMode::Buffered { + max_bytes: DEFAULT_BUFFERED_RESPONSE_BYTES, + }, + response_decoding: ResponseDecoding::Original, + cache_policy: OutboundCachePolicy::PlatformDefault, + } + } - assert_eq!( - header_value("user-agent").as_deref(), - Some("asset-agent/1.0"), - "should forward User-Agent" - ); - assert_eq!( - header_value("accept-encoding").as_deref(), - Some("gzip, br"), - "should preserve the incoming Accept-Encoding" - ); - assert_eq!( - header_value("if-none-match").as_deref(), - Some("\"asset-etag\""), - "should forward conditional ETag validation headers" - ); - assert_eq!( - header_value("if-modified-since").as_deref(), - Some("Thu, 13 Mar 2025 08:00:00 GMT"), - "should forward conditional date validation headers" - ); - assert_eq!( - header_value("if-match").as_deref(), - Some("\"asset-precondition\""), - "should forward precondition headers" - ); - assert_eq!( - header_value("if-unmodified-since").as_deref(), - Some("Thu, 13 Mar 2025 09:00:00 GMT"), - "should forward date precondition headers" - ); - assert_eq!( - header_value("range").as_deref(), - Some("bytes=0-1023"), - "should forward byte-range requests" - ); - assert_eq!( - header_value("if-range").as_deref(), - Some("\"asset-range\""), - "should forward range validators" - ); - assert_eq!( - header_value("host").as_deref(), - Some("assets.example.com:8443"), - "should override Host to the asset origin host" - ); - assert!( - header_value("x-forwarded-for").is_none(), - "should not forward client-supplied X-Forwarded-For" - ); - assert!( - header_value("x-custom-test").is_none(), - "should not forward unrelated custom headers" - ); - }); + /// Splits an asset response and reads its streamed body. + fn streamed_asset(asset_response: AssetProxyResponse) -> (Response, Vec) { + let (response, stream_body) = asset_response.into_response_and_body(); + let body = stream_body.expect("should keep the asset body"); + assert!(body.is_stream(), "should keep the asset body as a stream"); + let bytes = futures::executor::block_on(body.into_bytes_bounded(1024 * 1024)) + .expect("should stream the asset body"); + (response, bytes.to_vec()) } #[test] - fn handle_asset_proxy_request_streams_when_supported() { - futures::executor::block_on(async { - use crate::platform::test_support::StubHttpClient; - - let stub = Arc::new(StubHttpClient::new()); - stub.set_streaming_responses_supported(true); - stub.push_response(200, b"ok".to_vec()); - let services = build_services_with_http_client( - Arc::clone(&stub) as Arc - ); - let settings = create_test_settings(); - let req = build_http_request(Method::GET, "https://www.example.com/.images/foo.jpg"); - let route = ProxyAssetRoute::new("/.images/", "https://assets.example.com"); + fn asset_proxy_get_request_readback() { + let (_clock, client, services) = scripted_services(); + client.respond_to( + Method::GET, + "https://assets.example.com:8443/.images/foo.jpg?auto=webp", + ScriptedReply::status(StatusCode::OK).body("ok"), + ); + let settings = create_test_settings(); + let mut req = build_http_request( + Method::GET, + "https://www.example.com/.images/foo.jpg?auto=webp", + ); + for (name, value) in [ + (header::USER_AGENT, "asset-agent/1.0"), + (header::ACCEPT, "image/avif,image/webp,image/*,*/*;q=0.8"), + (header::ACCEPT_ENCODING, "gzip, br"), + (header::ACCEPT_LANGUAGE, "en-US"), + (header::REFERER, "https://www.example.com/article"), + (header::IF_NONE_MATCH, "\"asset-etag\""), + (header::IF_MODIFIED_SINCE, "Thu, 13 Mar 2025 08:00:00 GMT"), + (header::IF_MATCH, "\"asset-precondition\""), + (header::IF_UNMODIFIED_SINCE, "Thu, 13 Mar 2025 09:00:00 GMT"), + (header::RANGE, "bytes=0-1023"), + (header::IF_RANGE, "\"asset-range\""), + (HEADER_X_FORWARDED_FOR, "198.51.100.10"), + (header::HeaderName::from_static("x-custom-test"), "drop-me"), + ] { + req.headers_mut() + .insert(name, HeaderValue::from_static(value)); + } + let route = ProxyAssetRoute::new("/.images/", "https://assets.example.com:8443"); - handle_asset_proxy_request(&settings, &services, req, &route) - .await - .expect("should proxy streaming asset response"); + let (response, body) = streamed_asset( + futures::executor::block_on(handle_asset_proxy_request( + &settings, &services, req, &route, + )) + .expect("should proxy asset request"), + ); - assert_eq!(stub.recorded_stream_response_flags(), vec![true]); - }); + assert_eq!(response.status(), StatusCode::OK); + assert_eq!(body, b"ok", "should stream the asset body"); + let requests = client.requests(); + assert_eq!(requests.len(), 1, "should send one asset request"); + let sent = &requests[0]; + sent.assert_headers(&[ + ("user-agent", "asset-agent/1.0"), + ("accept", "image/avif,image/webp,image/*,*/*;q=0.8"), + ("accept-encoding", "gzip, br"), + ("accept-language", "en-US"), + ("referer", "https://www.example.com/article"), + ("if-none-match", "\"asset-etag\""), + ("if-modified-since", "Thu, 13 Mar 2025 08:00:00 GMT"), + ("if-match", "\"asset-precondition\""), + ("if-unmodified-since", "Thu, 13 Mar 2025 09:00:00 GMT"), + ("range", "bytes=0-1023"), + ("if-range", "\"asset-range\""), + ]); + assert_eq!( + sent.host_authority, "assets.example.com:8443", + "should send the asset origin authority, which the Host header named" + ); + sent.assert_transport(asset_get_transport()); } #[test] fn handle_asset_proxy_request_strips_unsafe_response_headers() { - futures::executor::block_on(async { - let stub = Arc::new(StubHttpClient::new()); - stub.push_response_with_headers( - 200, - Vec::new(), - vec![ - (header::SET_COOKIE.as_str(), "asset=1; Path=/; Secure"), - (header::SET_COOKIE.as_str(), "other=2; Path=/; Secure"), - ( - header::STRICT_TRANSPORT_SECURITY.as_str(), - "max-age=31536000; includeSubDomains; preload", - ), - ("clear-site-data", "\"*\""), - (header::ETAG.as_str(), "\"asset-etag\""), - ], - ); - let services = build_services_with_http_client( - Arc::clone(&stub) as Arc - ); - let settings = create_test_settings(); - let req = build_http_request(Method::GET, "https://www.example.com/.images/foo.jpg"); + let (_clock, client, services) = scripted_services(); + client.respond_to( + Method::GET, + "https://assets.example.com/.images/foo.jpg", + ScriptedReply::status(StatusCode::OK) + .header(header::SET_COOKIE.as_str(), "asset=1; Path=/; Secure") + .header(header::SET_COOKIE.as_str(), "other=2; Path=/; Secure") + .header( + header::STRICT_TRANSPORT_SECURITY.as_str(), + "max-age=31536000; includeSubDomains; preload", + ) + .header("clear-site-data", "\"*\"") + .header(header::ETAG.as_str(), "\"asset-etag\"") + .chunks(vec![Bytes::from_static(b"chunk")]), + ); + let settings = create_test_settings(); + let req = build_http_request(Method::GET, "https://www.example.com/.images/foo.jpg"); + let route = ProxyAssetRoute::new("/.images/", "https://assets.example.com"); - let route = ProxyAssetRoute::new("/.images/", "https://assets.example.com"); - let response = handle_asset_proxy_request(&settings, &services, req, &route) - .await - .expect("should proxy asset request") - .into_response() - .expect("should return buffered asset response"); + let (response, body) = streamed_asset( + futures::executor::block_on(handle_asset_proxy_request( + &settings, &services, req, &route, + )) + .expect("should proxy asset request"), + ); - assert!( - response.headers().get(header::SET_COOKIE).is_none(), - "should strip upstream Set-Cookie headers from asset responses" - ); - assert!( - response - .headers() - .get(header::STRICT_TRANSPORT_SECURITY) - .is_none(), - "should strip upstream HSTS headers from asset responses" - ); - assert!( - response.headers().get("clear-site-data").is_none(), - "should strip upstream Clear-Site-Data headers from asset responses" - ); - assert_eq!( - response_header(&response, header::ETAG), - Some("\"asset-etag\""), - "should preserve safe cache validator headers on asset responses" - ); - }); + assert!( + response.headers().get(header::SET_COOKIE).is_none(), + "should strip upstream Set-Cookie headers from asset responses" + ); + assert!( + response + .headers() + .get(header::STRICT_TRANSPORT_SECURITY) + .is_none(), + "should strip upstream HSTS headers from asset responses" + ); + assert!( + response.headers().get("clear-site-data").is_none(), + "should strip upstream Clear-Site-Data headers from asset responses" + ); + assert_eq!( + response_header(&response, header::ETAG), + Some("\"asset-etag\""), + "should preserve safe cache validator headers on asset responses" + ); + assert_eq!(body, b"chunk", "should stream the origin body"); } - #[test] - fn handle_asset_proxy_request_replaces_third_party_cache_policy_for_rehosted_asset() { - futures::executor::block_on(async { - let stub = Arc::new(StubHttpClient::new()); - stub.push_response_with_headers( - 200, - b"asset".to_vec(), - vec![(header::CACHE_CONTROL.as_str(), "no-store")], - ); - let services = build_services_with_http_client( - Arc::clone(&stub) as Arc - ); - let settings = Settings::from_toml(&format!( - r#"{} + fn fingerprinted_asset_settings() -> Settings { + Settings::from_toml(&format!( + r#"{} [[cache.asset_rules]] id = "fingerprinted-assets" @@ -5483,52 +5306,63 @@ mod tests { edge_ttl_seconds = 31536000 immutable = true "#, - crate_test_settings_str() - )) - .expect("should parse settings with cache asset rule"); - let req = build_http_request( - Method::GET, - "https://www.example.com/assets/app.0123abcd.js", - ); - let route = ProxyAssetRoute::new("/assets/", "https://assets.example.com"); - - let asset_response = handle_asset_proxy_request(&settings, &services, req, &route) - .await - .expect("should proxy asset request"); - assert_eq!( - asset_response.cache_policy(), - AssetProxyCachePolicy::Normalized(CachePolicy::public_immutable( - Duration::from_secs(31_536_000) - )), - "should carry normalized cache policy metadata" - ); + crate_test_settings_str() + )) + .expect("should parse settings with cache asset rule") + } - let mut response = asset_response - .into_response() - .expect("should return buffered asset response"); - assert_eq!( - response_header(&response, header::CACHE_CONTROL), - Some("public, max-age=31536000, immutable"), - "configured rehost policy should replace the third-party no-store directive" - ); - assert!( - response.headers().get("surrogate-control").is_none(), - "runtime-specific edge header should wait for adapter finalization" - ); + #[test] + fn handle_asset_proxy_request_replaces_third_party_cache_policy_for_rehosted_asset() { + let (_clock, client, services) = scripted_services(); + client.respond_to( + Method::GET, + "https://assets.example.com/assets/app.0123abcd.js", + ScriptedReply::status(StatusCode::OK) + .header(header::CACHE_CONTROL.as_str(), "no-store") + .body("asset"), + ); + let settings = fingerprinted_asset_settings(); + let req = build_http_request( + Method::GET, + "https://www.example.com/assets/app.0123abcd.js", + ); + let route = ProxyAssetRoute::new("/assets/", "https://assets.example.com"); + let asset_response = futures::executor::block_on(handle_asset_proxy_request( + &settings, &services, req, &route, + )) + .expect("should proxy asset request"); + assert_eq!( + asset_response.cache_policy(), AssetProxyCachePolicy::Normalized(CachePolicy::public_immutable(Duration::from_secs( - 31_536_000, - ))) - .apply_after_route_finalization(&mut response, EdgeCacheHeader::SurrogateControl); - assert_eq!( - response - .headers() - .get("surrogate-control") - .and_then(|value| value.to_str().ok()), - Some("max-age=31536000"), - "Fastly finalization should render Surrogate-Control" - ); - }); + 31_536_000 + ))), + "should carry normalized cache policy metadata" + ); + + let (mut response, _body) = streamed_asset(asset_response); + assert_eq!( + response_header(&response, header::CACHE_CONTROL), + Some("public, max-age=31536000, immutable"), + "configured rehost policy should replace the third-party no-store directive" + ); + assert!( + response.headers().get("surrogate-control").is_none(), + "runtime-specific edge header should wait for adapter finalization" + ); + + AssetProxyCachePolicy::Normalized(CachePolicy::public_immutable(Duration::from_secs( + 31_536_000, + ))) + .apply_after_route_finalization(&mut response, EdgeCacheHeader::SurrogateControl); + assert_eq!( + response + .headers() + .get("surrogate-control") + .and_then(|value| value.to_str().ok()), + Some("max-age=31536000"), + "Fastly finalization should render Surrogate-Control" + ); } #[test] @@ -5570,53 +5404,34 @@ mod tests { #[test] fn handle_asset_proxy_request_leaves_non_matching_assets_origin_controlled() { - futures::executor::block_on(async { - let stub = Arc::new(StubHttpClient::new()); - stub.push_response_with_headers( - 200, - b"asset".to_vec(), - vec![(header::CACHE_CONTROL.as_str(), "public, max-age=60")], - ); - let services = build_services_with_http_client( - Arc::clone(&stub) as Arc - ); - let settings = Settings::from_toml(&format!( - r#"{} + let (_clock, client, services) = scripted_services(); + client.respond_to( + Method::GET, + "https://assets.example.com/assets/app.js", + ScriptedReply::status(StatusCode::OK) + .header(header::CACHE_CONTROL.as_str(), "public, max-age=60") + .body("asset"), + ); + let settings = fingerprinted_asset_settings(); + let req = build_http_request(Method::GET, "https://www.example.com/assets/app.js"); + let route = ProxyAssetRoute::new("/assets/", "https://assets.example.com"); - [[cache.asset_rules]] - id = "fingerprinted-assets" - enabled = true - path_globs = ["/assets/**/*.js"] - fingerprint_style = "hex" - visibility = "public" - browser_ttl_seconds = 31536000 - edge_ttl_seconds = 31536000 - immutable = true - "#, - crate_test_settings_str() - )) - .expect("should parse settings with cache asset rule"); - let req = build_http_request(Method::GET, "https://www.example.com/assets/app.js"); - let route = ProxyAssetRoute::new("/assets/", "https://assets.example.com"); - - let asset_response = handle_asset_proxy_request(&settings, &services, req, &route) - .await - .expect("should proxy asset request"); + let asset_response = futures::executor::block_on(handle_asset_proxy_request( + &settings, &services, req, &route, + )) + .expect("should proxy asset request"); - assert_eq!( - asset_response.cache_policy(), - AssetProxyCachePolicy::OriginControlled, - "non-fingerprinted file should not receive normalized immutable policy" - ); - let response = asset_response - .into_response() - .expect("should return buffered asset response"); - assert_eq!( - response_header(&response, header::CACHE_CONTROL), - Some("public, max-age=60"), - "origin-controlled response should preserve origin cache header" - ); - }); + assert_eq!( + asset_response.cache_policy(), + AssetProxyCachePolicy::OriginControlled, + "non-fingerprinted file should not receive normalized immutable policy" + ); + let (response, _body) = streamed_asset(asset_response); + assert_eq!( + response_header(&response, header::CACHE_CONTROL), + Some("public, max-age=60"), + "origin-controlled response should preserve origin cache header" + ); } fn test_profile_set() -> ImageOptimizerProfileSet { @@ -5646,24 +5461,8 @@ mod tests { } } - fn test_s3_secrets() -> HashMap> { - HashMap::from([ - ( - "access_key_id".to_string(), - b"AKIAIOSFODNN7EXAMPLE".to_vec(), - ), - ( - "secret_access_key".to_string(), - b"wJalrXUtnFEMI/K7MDENG+bPxRfiCYEXAMPLEKEY".to_vec(), - ), - ]) - } - fn test_s3_image_optimizer_route() -> ProxyAssetRoute { - let mut route = ProxyAssetRoute::new( - "/.images/", - "https://examplebucket.s3.us-east-1.amazonaws.com", - ); + let mut route = ProxyAssetRoute::new("/.images/", "https://examplebucket.s3.example.com"); route.auth = Some(AssetOriginAuth::S3SigV4(S3SigV4AuthConfig { region: "us-east-1".to_string(), secret_store: None, @@ -5683,762 +5482,633 @@ mod tests { route } - #[test] - fn handle_asset_proxy_request_signs_s3_and_strips_transform_query() { - futures::executor::block_on(async { - let stub = Arc::new(StubHttpClient::new()); - stub.push_response(200, b"ok".to_vec()); - let mut secrets = HashMap::new(); - secrets.insert( - "access_key_id".to_string(), - b"AKIAIOSFODNN7EXAMPLE".to_vec(), - ); - secrets.insert( - "secret_access_key".to_string(), - b"wJalrXUtnFEMI/K7MDENG+bPxRfiCYEXAMPLEKEY".to_vec(), - ); - let services = build_services_with_secret_and_http_client( - HashMapSecretStore::new(secrets), - Arc::clone(&stub) as Arc, - ); - let settings = create_test_settings(); - let req = build_http_request( - Method::GET, - "https://www.example.com/.images/foo.jpg?profile=medium&ar=1-1", - ); - let mut route = ProxyAssetRoute::new( - "/.images/", - "https://examplebucket.s3.us-east-1.amazonaws.com", - ); - route.auth = Some(AssetOriginAuth::S3SigV4(S3SigV4AuthConfig { - region: "us-east-1".to_string(), - secret_store: None, - access_key_id: Redacted::new("AKIAIOSFODNN7EXAMPLE".to_string()), - secret_access_key: Redacted::new( - "wJalrXUtnFEMI/K7MDENG+bPxRfiCYEXAMPLEKEY".to_string(), - ), - session_token: None, - origin_query: Some(OriginQueryPolicy::Strip), - })); - - handle_asset_proxy_request(&settings, &services, req, &route) - .await - .expect("should proxy signed S3 asset request"); - - let uris = stub.recorded_request_uris(); - assert_eq!( - uris, - vec!["https://examplebucket.s3.us-east-1.amazonaws.com/.images/foo.jpg"], - "should strip transform query before signing and sending to S3" - ); - let headers = stub.recorded_request_headers(); - let sent = &headers[0]; - let header_value = |name: &str| -> Option { - sent.iter().find(|(n, _)| n == name).map(|(_, v)| v.clone()) - }; - assert_eq!( - header_value("host").as_deref(), - Some("examplebucket.s3.us-east-1.amazonaws.com"), - "should sign for the S3 origin host" - ); - assert!( - header_value("authorization") - .as_deref() - .is_some_and(|value| value.starts_with("AWS4-HMAC-SHA256 Credential=")), - "should add a SigV4 Authorization header" - ); - assert_eq!( - header_value("x-amz-content-sha256").as_deref(), - Some("UNSIGNED-PAYLOAD"), - "should use unsigned payload for read-only asset requests" - ); - }); - } - - #[test] - fn handle_asset_proxy_request_uses_resolved_s3_credentials_without_store_reads() { - futures::executor::block_on(async { - let stub = Arc::new(StubHttpClient::new()); - stub.push_response(200, Vec::new()); - stub.push_response(200, b"optimized".to_vec()); - let secret_store = CountingSecretStore::new(HashMap::new()); - let observed_secret_store = secret_store.clone(); - let services = build_services_with_secret_and_http_client( - secret_store, - Arc::clone(&stub) as Arc, - ); - let mut settings = create_test_settings(); - settings.image_optimizer = ImageOptimizerSettings { - profile_sets: HashMap::from([("default_images".to_string(), test_profile_set())]), - }; - let req = build_http_request( - Method::GET, - "https://www.example.com/.images/foo.jpg?profile=medium", - ); - let mut route = test_s3_image_optimizer_route(); - route.auth = Some(AssetOriginAuth::S3SigV4(S3SigV4AuthConfig { - region: "us-east-1".to_string(), - secret_store: None, - access_key_id: Redacted::new("AKIAIOSFODNN7EXAMPLE".to_string()), - secret_access_key: Redacted::new( - "wJalrXUtnFEMI/K7MDENG+bPxRfiCYEXAMPLEKEY".to_string(), - ), - session_token: Some(Redacted::new("session-token".to_string())), - origin_query: None, - })); - - handle_asset_proxy_request(&settings, &services, req, &route) - .await - .expect("should proxy optimized S3 asset request"); + const S3_OBJECT: &str = "https://examplebucket.s3.example.com/.images/foo.jpg"; - assert_eq!( - stub.recorded_request_methods(), - vec!["HEAD", "GET"], - "should sign both the S3 preflight and final request" - ); - assert_eq!( - observed_secret_store.read_count("AKIAIOSFODNN7EXAMPLE"), - 0, - "should not read S3 credentials from the runtime secret store" - ); - let headers = stub.recorded_request_headers(); - assert!( - headers - .iter() - .all(|sent| sent - .iter() - .any(|(name, value)| name == "x-amz-security-token" - && value == "session-token")), - "should sign both requests with the cached session token" - ); - }); + fn image_optimizer_settings(profile_set: ImageOptimizerProfileSet) -> Settings { + let mut settings = create_test_settings(); + settings.image_optimizer = ImageOptimizerSettings { + profile_sets: HashMap::from([("default_images".to_string(), profile_set)]), + }; + settings } - #[test] - fn handle_asset_proxy_request_attaches_image_optimizer_metadata() { - futures::executor::block_on(async { - let stub = Arc::new(StubHttpClient::new()); - stub.set_streaming_responses_supported(true); - stub.push_response(200, b"ok".to_vec()); - let services = build_services_with_http_client( - Arc::clone(&stub) as Arc - ); - let mut settings = create_test_settings(); - settings.image_optimizer = ImageOptimizerSettings { - profile_sets: HashMap::from([("default_images".to_string(), test_profile_set())]), - }; - let req = build_http_request( - Method::GET, - "https://www.example.com/.images/foo.jpg?profile=medium&ar=1-1&x=71&y=bad", - ); - let mut route = ProxyAssetRoute::new("/.images/", "https://assets.example.com"); - route.image_optimizer = Some(AssetImageOptimizerConfig { - enabled: true, - region: "us_east".to_string(), - profile_set: "default_images".to_string(), - origin_query: None, - }); - - handle_asset_proxy_request(&settings, &services, req, &route) - .await - .expect("should proxy optimized asset request"); - - let uris = stub.recorded_request_uris(); - assert_eq!( - uris, - vec!["https://assets.example.com/.images/foo.jpg"], - "should strip profile-table query from the origin request by default" - ); - assert_eq!( - stub.recorded_stream_response_flags(), - vec![true], - "final asset responses should stay streaming" - ); - let options = stub.recorded_image_optimizer_options(); - let options = options[0] - .as_ref() - .expect("should attach image optimizer metadata"); - assert_eq!(options.region, "us_east"); - assert!(!options.preserve_query_string_on_origin_request); - assert_eq!(options.params.quality, Some(70)); - assert_eq!(options.params.resize_filter.as_deref(), Some("bicubic")); - assert_eq!(options.params.format.as_deref(), Some("auto")); - assert_eq!(options.params.width, Some(828)); - let crop = options.params.crop.as_ref().expect("should set crop"); - assert_eq!((crop.width, crop.height), (1, 1)); - assert_eq!((crop.offset_x, crop.offset_y), (Some(70), Some(50))); + fn image_optimizer_route() -> ProxyAssetRoute { + let mut route = ProxyAssetRoute::new("/.images/", "https://assets.example.com"); + route.image_optimizer = Some(AssetImageOptimizerConfig { + enabled: true, + region: "us_east".to_string(), + profile_set: "default_images".to_string(), + origin_query: None, }); + route } - #[test] - fn handle_asset_proxy_request_skips_image_optimizer_for_svg_assets() { - futures::executor::block_on(async { - let stub = Arc::new(StubHttpClient::new()); - stub.push_response(200, b"ok".to_vec()); - let services = build_services_with_http_client( - Arc::clone(&stub) as Arc - ); - let mut settings = create_test_settings(); - let mut profile_set = test_profile_set(); - profile_set.unknown_profile = UnknownProfilePolicy::Reject; - settings.image_optimizer = ImageOptimizerSettings { - profile_sets: HashMap::from([("default_images".to_string(), profile_set)]), - }; - let req = build_http_request( - Method::GET, - "https://www.example.com/.images/logo.SVG?profile=unknown&ar=1-1", - ); - let mut route = ProxyAssetRoute::new("/.images/", "https://assets.example.com"); - route.image_optimizer = Some(AssetImageOptimizerConfig { - enabled: true, - region: "us_east".to_string(), - profile_set: "default_images".to_string(), - origin_query: None, - }); - - handle_asset_proxy_request(&settings, &services, req, &route) - .await - .expect("should proxy SVG asset without Image Optimizer profile parsing"); - - assert_eq!( - stub.recorded_request_uris(), - vec!["https://assets.example.com/.images/logo.SVG"], - "should still strip profile-table query from SVG origin requests" - ); - assert_eq!( - stub.recorded_image_optimizer_options(), - vec![None], - "SVG assets should bypass Image Optimizer metadata" - ); - }); + /// The options [`test_profile_set`]'s `medium` profile maps to. + fn medium_profile_options() -> OutboundImageOptimizerOptions { + OutboundImageOptimizerOptions::new() + .region(OutboundImageOptimizerRegion::UsEast) + .quality(70) + .expect("should accept quality 70") + .resize_filter(OutboundResizeFilter::Bicubic) + .format(OutboundImageFormat::Auto) + .width(828) + .expect("should accept width 828") } #[test] - fn handle_asset_proxy_request_skips_image_optimizer_for_incoming_svg() { - futures::executor::block_on(async { - let stub = Arc::new(StubHttpClient::new()); - stub.push_response(200, b"ok".to_vec()); - let services = build_services_with_http_client( - Arc::clone(&stub) as Arc - ); - let mut settings = create_test_settings(); - let mut profile_set = test_profile_set(); - profile_set.unknown_profile = UnknownProfilePolicy::Reject; - settings.image_optimizer = ImageOptimizerSettings { - profile_sets: HashMap::from([("default_images".to_string(), profile_set)]), - }; - let req = build_http_request( - Method::GET, - "https://www.example.com/.image/object-id/logo.svg?profile=unknown&ar=1-1", - ); - let mut route = ProxyAssetRoute::new("/.image/", "https://assets.example.com"); - route.path_pattern = Some(r"^/\.image/([^/]+)/[^/]+\.([^/.]+)$".to_string()); - route.target_path = Some("/image/upload/$1".to_string()); - route.image_optimizer = Some(AssetImageOptimizerConfig { - enabled: true, - region: "us_east".to_string(), - profile_set: "default_images".to_string(), - origin_query: None, - }); + fn handle_asset_proxy_request_signs_s3_and_strips_transform_query() { + let (_clock, client, services) = scripted_services(); + client.respond_to( + Method::GET, + S3_OBJECT, + ScriptedReply::status(StatusCode::OK).body("ok"), + ); + let settings = create_test_settings(); + let req = build_http_request( + Method::GET, + "https://www.example.com/.images/foo.jpg?profile=medium&ar=1-1", + ); + let mut route = ProxyAssetRoute::new("/.images/", "https://examplebucket.s3.example.com"); + route.auth = Some(AssetOriginAuth::S3SigV4(S3SigV4AuthConfig { + region: "us-east-1".to_string(), + secret_store: None, + access_key_id: Redacted::new("AKIAIOSFODNN7EXAMPLE".to_string()), + secret_access_key: Redacted::new( + "wJalrXUtnFEMI/K7MDENG+bPxRfiCYEXAMPLEKEY".to_string(), + ), + session_token: None, + origin_query: Some(OriginQueryPolicy::Strip), + })); - handle_asset_proxy_request(&settings, &services, req, &route) - .await - .expect("should proxy incoming SVG asset without Image Optimizer profile parsing"); + futures::executor::block_on(handle_asset_proxy_request( + &settings, &services, req, &route, + )) + .expect("should proxy signed S3 asset request"); - assert_eq!( - stub.recorded_request_uris(), - vec!["https://assets.example.com/image/upload/object-id"], - "should strip profile-table query even when SVG target path omits extension" - ); - assert_eq!( - stub.recorded_image_optimizer_options(), - vec![None], - "incoming SVG assets should bypass Image Optimizer metadata" - ); - }); + assert_eq!( + sent_uris(&client), + vec![S3_OBJECT], + "should strip transform query before signing and sending to S3" + ); + let requests = client.requests(); + let sent = &requests[0]; + assert_eq!( + sent.host_authority, "examplebucket.s3.example.com", + "should send the S3 origin authority it signed as Host" + ); + assert!( + sent.headers + .get(header::AUTHORIZATION) + .and_then(|value| value.to_str().ok()) + .is_some_and(|value| value + .starts_with("AWS4-HMAC-SHA256 Credential=AKIAIOSFODNN7EXAMPLE/") + && value.contains("SignedHeaders=host;x-amz-content-sha256;x-amz-date")), + "should add a SigV4 Authorization header that signs the host" + ); + assert_eq!( + sent.headers + .get("x-amz-content-sha256") + .and_then(|value| value.to_str().ok()), + Some("UNSIGNED-PAYLOAD"), + "should use unsigned payload for read-only asset requests" + ); + sent.assert_transport(asset_get_transport()); } #[test] - fn handle_asset_proxy_request_skips_image_optimizer_for_rewritten_svg_assets() { - futures::executor::block_on(async { - let stub = Arc::new(StubHttpClient::new()); - stub.push_response(200, b"ok".to_vec()); - let services = build_services_with_http_client( - Arc::clone(&stub) as Arc - ); - let mut settings = create_test_settings(); - settings.image_optimizer = ImageOptimizerSettings { - profile_sets: HashMap::from([("default_images".to_string(), test_profile_set())]), - }; - let req = build_http_request( + fn handle_asset_proxy_request_uses_resolved_s3_credentials_without_store_reads() { + let (_clock, client, outbound_services) = scripted_services(); + client + .respond_to( + Method::HEAD, + S3_OBJECT, + ScriptedReply::status(StatusCode::OK), + ) + .respond_to( Method::GET, - "https://www.example.com/.image/options/id/logo.svg?profile=medium", - ); - let mut route = ProxyAssetRoute::new("/.image/", "https://assets.example.com"); - route.path_pattern = Some(r"^/\.image/(.*)/[^/]+\.([^/.]+)$".to_string()); - route.target_path = Some("/image/upload/$1.$2".to_string()); - route.image_optimizer = Some(AssetImageOptimizerConfig { - enabled: true, - region: "us_east".to_string(), - profile_set: "default_images".to_string(), - origin_query: None, - }); + S3_OBJECT, + ScriptedReply::status(StatusCode::OK).body("optimized"), + ); + let secret_store = CountingSecretStore::new(HashMap::new()); + let observed_secret_store = secret_store.clone(); + let services = RuntimeServices { + secret_store: Arc::new(secret_store), + ..outbound_services + }; + let settings = image_optimizer_settings(test_profile_set()); + let req = build_http_request( + Method::GET, + "https://www.example.com/.images/foo.jpg?profile=medium", + ); + let mut route = test_s3_image_optimizer_route(); + route.auth = Some(AssetOriginAuth::S3SigV4(S3SigV4AuthConfig { + region: "us-east-1".to_string(), + secret_store: None, + access_key_id: Redacted::new("AKIAIOSFODNN7EXAMPLE".to_string()), + secret_access_key: Redacted::new( + "wJalrXUtnFEMI/K7MDENG+bPxRfiCYEXAMPLEKEY".to_string(), + ), + session_token: Some(Redacted::new("session-token".to_string())), + origin_query: None, + })); - handle_asset_proxy_request(&settings, &services, req, &route) - .await - .expect("should proxy rewritten SVG asset without Image Optimizer metadata"); + futures::executor::block_on(handle_asset_proxy_request( + &settings, &services, req, &route, + )) + .expect("should proxy optimized S3 asset request"); - assert_eq!( - stub.recorded_request_uris(), - vec!["https://assets.example.com/image/upload/options/id.svg"], - "should detect SVG after route path rewriting" - ); - assert_eq!( - stub.recorded_image_optimizer_options(), - vec![None], - "rewritten SVG assets should bypass Image Optimizer metadata" - ); - }); + assert_eq!( + sent_methods(&client), + vec![Method::HEAD, Method::GET], + "should sign both the S3 preflight and final request" + ); + assert_eq!( + observed_secret_store.read_count("AKIAIOSFODNN7EXAMPLE"), + 0, + "should not read S3 credentials from the runtime secret store" + ); + assert!( + client.requests().iter().all(|sent| sent + .headers + .get("x-amz-security-token") + .is_some_and(|value| value == "session-token")), + "should sign both requests with the cached session token" + ); } #[test] - fn handle_asset_proxy_request_skips_s3_preflight_for_svg_image_optimizer_routes() { - futures::executor::block_on(async { - let stub = Arc::new(StubHttpClient::new()); - stub.push_response(200, b"raw-svg".to_vec()); - let services = build_services_with_secret_and_http_client( - HashMapSecretStore::new(test_s3_secrets()), - Arc::clone(&stub) as Arc, - ); - let settings = create_test_settings(); - let req = build_http_request( - Method::GET, - "https://www.example.com/.images/logo.svg?profile=medium", - ); - let route = test_s3_image_optimizer_route(); - - let response = handle_asset_proxy_request(&settings, &services, req, &route) - .await - .expect("should proxy raw SVG asset through S3 route") - .into_response() - .expect("should return buffered asset response"); - - assert_eq!(response_body_string(response), "raw-svg"); - assert_eq!( - stub.recorded_request_methods(), - vec!["GET"], - "SVG IO bypass should not add an S3 preflight" - ); - assert_eq!( - stub.recorded_request_uris(), - vec!["https://examplebucket.s3.us-east-1.amazonaws.com/.images/logo.svg"], - "SVG IO bypass should still strip the transform query" - ); - assert_eq!( - stub.recorded_image_optimizer_options(), - vec![None], - "SVG IO bypass should not attach Image Optimizer metadata" - ); - }); - } + fn asset_proxy_image_optimizer_options_read_back_on_final_get() { + let (_clock, client, services) = scripted_services(); + client.respond_to( + Method::GET, + "https://assets.example.com/.images/foo.jpg", + ScriptedReply::status(StatusCode::OK).body("ok"), + ); + let settings = image_optimizer_settings(test_profile_set()); + let req = build_http_request( + Method::GET, + "https://www.example.com/.images/foo.jpg?profile=medium&ar=1-1&x=71&y=bad", + ); - #[test] - fn handle_asset_proxy_request_preflights_s3_before_image_optimizer() { - futures::executor::block_on(async { - let stub = Arc::new(StubHttpClient::new()); - stub.set_streaming_responses_supported(true); - stub.push_response(200, Vec::new()); - stub.push_response(200, b"optimized".to_vec()); - let services = build_services_with_secret_and_http_client( - HashMapSecretStore::new(test_s3_secrets()), - Arc::clone(&stub) as Arc, - ); - let mut settings = create_test_settings(); - settings.image_optimizer = ImageOptimizerSettings { - profile_sets: HashMap::from([("default_images".to_string(), test_profile_set())]), - }; - let req = build_http_request( - Method::GET, - "https://www.example.com/.images/foo.jpg?profile=medium", - ); - let route = test_s3_image_optimizer_route(); + futures::executor::block_on(handle_asset_proxy_request( + &settings, + &services, + req, + &image_optimizer_route(), + )) + .expect("should proxy optimized asset request"); - let (response, stream_body) = - handle_asset_proxy_request(&settings, &services, req, &route) - .await - .expect("should proxy optimized S3 asset request") - .into_response_and_body(); - - assert_eq!(response.status(), StatusCode::OK, "should proxy the asset"); - let mut output = Vec::new(); - stream_asset_body( - stream_body.expect("should preserve the asset body as a stream"), - &mut output, - ) - .await - .expect("should stream the asset body"); - assert_eq!( - output, b"optimized", - "should forward the complete asset body" - ); - assert_eq!( - stub.recorded_request_methods(), - vec!["HEAD", "GET"], - "should preflight S3 with HEAD before the optimized GET" - ); - assert_eq!( - stub.recorded_request_uris(), - vec![ - "https://examplebucket.s3.us-east-1.amazonaws.com/.images/foo.jpg", - "https://examplebucket.s3.us-east-1.amazonaws.com/.images/foo.jpg", - ], - "should strip transform query from both S3 requests" - ); - assert_eq!( - stub.recorded_stream_response_flags(), - vec![false, true], - "S3 HEAD preflight stays non-streaming, final asset GET stays streaming" - ); - let options = stub.recorded_image_optimizer_options(); - assert_eq!(options.len(), 2, "should send two origin requests"); - assert!( - options[0].is_none(), - "preflight should not attach IO metadata" - ); - assert!( - options[1].is_some(), - "final asset request should attach IO metadata" - ); - }); + assert_eq!( + sent_uris(&client), + vec!["https://assets.example.com/.images/foo.jpg"], + "should strip profile-table query from the origin request by default" + ); + let expected = medium_profile_options() + .crop(1, 1, OutboundCropFocus::Percent { x: 70, y: 50 }) + .expect("should accept the bucketed crop"); + client.requests()[0].assert_image_optimizer_transport(asset_get_transport(), &expected); } #[test] - fn handle_asset_proxy_request_returns_raw_s3_error_before_image_optimizer() { - futures::executor::block_on(async { - let stub = Arc::new(StubHttpClient::new()); - stub.set_streaming_responses_supported(true); - stub.push_response(404, Vec::new()); - stub.push_response_with_headers( - 404, - br#"NoSuchKeyimage/upload/missing.jpg"# - .to_vec(), - vec![ - (header::CONTENT_TYPE.as_str(), "application/xml"), - (header::CACHE_CONTROL.as_str(), "public, max-age=3600"), - (header::SET_COOKIE.as_str(), "asset=1; Path=/"), - ], - ); - let services = build_services_with_secret_and_http_client( - HashMapSecretStore::new(test_s3_secrets()), - Arc::clone(&stub) as Arc, - ); - let mut settings = create_test_settings(); - settings.image_optimizer = ImageOptimizerSettings { - profile_sets: HashMap::from([("default_images".to_string(), test_profile_set())]), - }; - let req = build_http_request( - Method::GET, - "https://www.example.com/.images/missing.jpg?profile=medium", - ); - let route = test_s3_image_optimizer_route(); - - let asset_response = handle_asset_proxy_request(&settings, &services, req, &route) - .await - .expect("should return raw S3 error response"); - assert_eq!( - asset_response.cache_policy(), - AssetProxyCachePolicy::NoStorePrivate, - "should carry a typed no-store policy for router finalization" - ); - let (response, stream_body) = asset_response.into_response_and_body(); + fn asset_proxy_image_optimizer_options_read_back_on_final_head() { + let (_clock, client, services) = scripted_services(); + client.respond_to( + Method::HEAD, + "https://assets.example.com/.images/foo.jpg", + ScriptedReply::status(StatusCode::OK).header("content-type", "image/jpeg"), + ); + let settings = image_optimizer_settings(test_profile_set()); + let req = build_http_request( + Method::HEAD, + "https://www.example.com/.images/foo.jpg?profile=medium", + ); - assert_eq!(response.status(), StatusCode::NOT_FOUND); - assert_eq!( - response_header(&response, header::CACHE_CONTROL), - Some("no-store, private"), - "S3 origin errors should not be cached" - ); - assert!( - response.headers().get(header::SET_COOKIE).is_none(), - "raw S3 error should still strip unsafe response headers" - ); - let mut output = Vec::new(); - stream_asset_body( - stream_body.expect("should preserve the raw S3 error body as a stream"), - &mut output, - ) - .await - .expect("should stream the raw S3 error body"); - let body = String::from_utf8(output).expect("should decode the S3 error body"); - assert!(body.contains("NoSuchKey"), "should return S3 error body"); - assert!( - body.contains("image/upload/missing.jpg"), - "should expose the missing S3 key" - ); - assert_eq!( - stub.recorded_request_methods(), - vec!["HEAD", "GET"], - "should fetch S3 error body after failed HEAD preflight" - ); - assert_eq!( - stub.recorded_stream_response_flags(), - vec![false, true], - "S3 HEAD preflight stays non-streaming, error-body GET stays streaming" - ); - let options = stub.recorded_image_optimizer_options(); - assert_eq!( - options, - vec![None, None], - "should not invoke IO for S3 errors" - ); - }); + let (response, stream_body) = futures::executor::block_on(handle_asset_proxy_request( + &settings, + &services, + req, + &image_optimizer_route(), + )) + .expect("should proxy optimized asset HEAD") + .into_response_and_body(); + + assert_eq!(response.status(), StatusCode::OK, "should relay the HEAD"); + assert!(stream_body.is_none(), "should buffer the bodiless HEAD"); + let requests = client.requests(); + assert_eq!(requests.len(), 1, "should send one HEAD"); + requests[0] + .assert_image_optimizer_transport(asset_head_transport(), &medium_profile_options()); } #[test] - fn handle_asset_proxy_request_does_not_preflight_when_io_disabled() { - futures::executor::block_on(async { - let stub = Arc::new(StubHttpClient::new()); - stub.set_streaming_responses_supported(true); - stub.push_response(200, b"raw".to_vec()); - let services = build_services_with_secret_and_http_client( - HashMapSecretStore::new(test_s3_secrets()), - Arc::clone(&stub) as Arc, - ); - let mut settings = create_test_settings(); - settings.image_optimizer = ImageOptimizerSettings { - profile_sets: HashMap::from([("default_images".to_string(), test_profile_set())]), - }; + fn asset_proxy_s3_preflight_is_buffered_without_image_optimizer_options() { + for (method, transport) in [ + (Method::GET, asset_get_transport()), + (Method::HEAD, asset_head_transport()), + ] { + let (_clock, client, services) = scripted_services(); + client + .respond_to( + Method::HEAD, + S3_OBJECT, + ScriptedReply::status(StatusCode::OK), + ) + .respond_to( + method.clone(), + S3_OBJECT, + ScriptedReply::status(StatusCode::OK).body("optimized"), + ); + let settings = image_optimizer_settings(test_profile_set()); let req = build_http_request( - Method::GET, - "https://www.example.com/.images/foo.jpg?profile=medium&_io_debug=1", + method.clone(), + "https://www.example.com/.images/foo.jpg?profile=medium", ); - let route = test_s3_image_optimizer_route(); - let (response, stream_body) = - handle_asset_proxy_request(&settings, &services, req, &route) - .await - .expect("should proxy debug S3 asset request") - .into_response_and_body(); - - assert_eq!(response.status(), StatusCode::OK, "should proxy the asset"); - let mut output = Vec::new(); - stream_asset_body( - stream_body.expect("should preserve the asset body as a stream"), - &mut output, - ) - .await - .expect("should stream the asset body"); - assert_eq!(output, b"raw", "should forward the complete asset body"); - assert_eq!( - stub.recorded_request_methods(), - vec!["GET"], - "disabled IO should not add S3 preflight" - ); + futures::executor::block_on(handle_asset_proxy_request( + &settings, + &services, + req, + &test_s3_image_optimizer_route(), + )) + .expect("should proxy optimized S3 asset request"); + + let requests = client.requests(); assert_eq!( - stub.recorded_image_optimizer_options(), - vec![None], - "debug query param should disable image optimizer metadata" + (requests.len(), &requests[0].method, &requests[1].method), + (2, &Method::HEAD, &method), + "should preflight S3 with HEAD before the optimized {method}" ); assert_eq!( - stub.recorded_stream_response_flags(), - vec![true], - "debug asset responses should still stay streaming" + sent_uris(&client), + vec![S3_OBJECT, S3_OBJECT], + "should strip transform query from both S3 requests" ); - }); + requests[0].assert_transport(asset_head_transport()); + requests[1].assert_image_optimizer_transport(transport, &medium_profile_options()); + } } #[test] - fn handle_asset_proxy_request_debug_param_disables_image_optimizer() { - futures::executor::block_on(async { - let stub = Arc::new(StubHttpClient::new()); - stub.push_response(200, b"ok".to_vec()); - let services = build_services_with_http_client( - Arc::clone(&stub) as Arc - ); - let mut settings = create_test_settings(); - settings.image_optimizer = ImageOptimizerSettings { - profile_sets: HashMap::from([("default_images".to_string(), test_profile_set())]), - }; - let req = build_http_request( + fn handle_asset_proxy_request_preflights_s3_before_image_optimizer() { + let (_clock, client, services) = scripted_services(); + client + .respond_to( + Method::HEAD, + S3_OBJECT, + ScriptedReply::status(StatusCode::OK), + ) + .respond_to( Method::GET, - "https://www.example.com/.images/foo.jpg?profile=medium&_io_debug=1", + S3_OBJECT, + ScriptedReply::status(StatusCode::OK).body("optimized"), ); - let mut route = ProxyAssetRoute::new("/.images/", "https://assets.example.com"); - route.image_optimizer = Some(AssetImageOptimizerConfig { - enabled: true, - region: "us_east".to_string(), - profile_set: "default_images".to_string(), - origin_query: None, - }); + let settings = image_optimizer_settings(test_profile_set()); + let req = build_http_request( + Method::GET, + "https://www.example.com/.images/foo.jpg?profile=medium", + ); - handle_asset_proxy_request(&settings, &services, req, &route) - .await - .expect("should proxy debug asset request"); + let (response, body) = streamed_asset( + futures::executor::block_on(handle_asset_proxy_request( + &settings, + &services, + req, + &test_s3_image_optimizer_route(), + )) + .expect("should proxy optimized S3 asset request"), + ); - let options = stub.recorded_image_optimizer_options(); - assert!( - options[0].is_none(), - "debug query param should disable image optimizer metadata" - ); - }); + assert_eq!(response.status(), StatusCode::OK, "should proxy the asset"); + assert_eq!(body, b"optimized", "should forward the complete asset body"); } #[test] - fn handle_asset_proxy_request_accepts_streaming_platform_response_body() { - futures::executor::block_on(async { - let services = build_services_with_http_client( - Arc::new(StreamingResponseHttpClient) as Arc - ); - let settings = create_test_settings(); - let req = build_http_request(Method::GET, "https://www.example.com/.images/foo.jpg"); - let route = ProxyAssetRoute::new("/.images/", "https://assets.example.com"); - - let (response, stream_body) = - handle_asset_proxy_request(&settings, &services, req, &route) - .await - .expect("should proxy a streaming asset response") - .into_response_and_body(); - assert_eq!(response.status(), StatusCode::OK); - assert!( - response.headers().get(header::SET_COOKIE).is_none(), - "should strip upstream Set-Cookie headers from streaming asset responses" - ); - assert!( - response - .headers() - .get(header::STRICT_TRANSPORT_SECURITY) - .is_none(), - "should strip upstream HSTS headers from streaming asset responses" - ); - assert!( - response.headers().get("clear-site-data").is_none(), - "should strip upstream Clear-Site-Data headers from streaming asset responses" - ); - assert_eq!( - response_header(&response, header::ETAG), - Some("\"stream-etag\""), - "should preserve safe cache validator headers on streaming asset responses" + fn handle_asset_proxy_request_returns_raw_s3_error_before_image_optimizer() { + let (_clock, client, services) = scripted_services(); + client + .respond_to( + Method::HEAD, + S3_OBJECT, + ScriptedReply::status(StatusCode::NOT_FOUND), + ) + .respond_to( + Method::GET, + S3_OBJECT, + ScriptedReply::status(StatusCode::NOT_FOUND) + .header(header::CONTENT_TYPE.as_str(), "application/xml") + .header(header::CACHE_CONTROL.as_str(), "public, max-age=3600") + .header(header::SET_COOKIE.as_str(), "asset=1; Path=/") + .body( + r#"NoSuchKeyimage/upload/missing.jpg"#, + ), ); + let settings = image_optimizer_settings(test_profile_set()); + let req = build_http_request( + Method::GET, + "https://www.example.com/.images/foo.jpg?profile=medium", + ); - let mut output = Vec::new(); - stream_asset_body( - stream_body.expect("should preserve asset body as a stream"), - &mut output, - ) - .await - .expect("should stream asset body"); + let asset_response = futures::executor::block_on(handle_asset_proxy_request( + &settings, + &services, + req, + &test_s3_image_optimizer_route(), + )) + .expect("should return raw S3 error response"); + assert_eq!( + asset_response.cache_policy(), + AssetProxyCachePolicy::NoStorePrivate, + "should carry a typed no-store policy for router finalization" + ); + let (response, body) = streamed_asset(asset_response); - assert_eq!(output, b"chunk"); - }); + assert_eq!(response.status(), StatusCode::NOT_FOUND); + assert_eq!( + response_header(&response, header::CACHE_CONTROL), + Some("no-store, private"), + "S3 origin errors should not be cached" + ); + assert!( + response.headers().get(header::SET_COOKIE).is_none(), + "raw S3 error should still strip unsafe response headers" + ); + let body = String::from_utf8(body).expect("should decode the S3 error body"); + assert!(body.contains("NoSuchKey"), "should return S3 error body"); + assert!( + body.contains("image/upload/missing.jpg"), + "should expose the missing S3 key" + ); + let requests = client.requests(); + assert_eq!( + sent_methods(&client), + vec![Method::HEAD, Method::GET], + "should fetch S3 error body after failed HEAD preflight" + ); + requests[0].assert_transport(asset_head_transport()); + requests[1].assert_transport(asset_get_transport()); } #[test] - fn stream_asset_body_reports_mid_stream_origin_errors() { - futures::executor::block_on(async { - let body = EdgeBody::from_stream(futures::stream::iter(vec![ - Ok::(Bytes::from_static(b"partial")), - Err(io::Error::other("origin stream failed")), - ])); - let mut output = Vec::new(); + fn handle_asset_proxy_request_answers_head_with_raw_s3_preflight_error() { + let (_clock, client, services) = scripted_services(); + client.respond_to( + Method::HEAD, + S3_OBJECT, + ScriptedReply::status(StatusCode::FORBIDDEN) + .header(header::CACHE_CONTROL.as_str(), "public, max-age=3600") + .header(header::SET_COOKIE.as_str(), "asset=1; Path=/"), + ); + let settings = image_optimizer_settings(test_profile_set()); + let req = build_http_request( + Method::HEAD, + "https://www.example.com/.images/foo.jpg?profile=medium", + ); - let err = stream_asset_body(body, &mut output) - .await - .expect_err("should report mid-stream origin errors"); + let asset_response = futures::executor::block_on(handle_asset_proxy_request( + &settings, + &services, + req, + &test_s3_image_optimizer_route(), + )) + .expect("should return the raw S3 preflight error"); - assert_eq!( - output, b"partial", - "should only write chunks received before the stream error" - ); - assert!( - format!("{err:?}").contains("streaming platform response body failed"), - "should describe the streaming failure: {err:?}" - ); - }); + assert_eq!( + asset_response.cache_policy(), + AssetProxyCachePolicy::NoStorePrivate, + "should carry a typed no-store policy for router finalization" + ); + let (response, stream_body) = asset_response.into_response_and_body(); + assert_eq!(response.status(), StatusCode::FORBIDDEN); + assert!(stream_body.is_none(), "should answer HEAD without a body"); + assert_eq!( + response_header(&response, header::CACHE_CONTROL), + Some("no-store, private"), + "S3 origin errors should not be cached" + ); + assert!( + response.headers().get(header::SET_COOKIE).is_none(), + "raw S3 error should still strip unsafe response headers" + ); + assert_eq!( + sent_methods(&client), + vec![Method::HEAD], + "should answer a HEAD from its failed preflight alone" + ); } - /// One observable step of [`stream_asset_body`] driving a buffered sink. - #[derive(Debug, PartialEq, Eq)] - enum StreamSinkEvent { - SourcePolled(usize), - Wrote(usize), - Flushed, + /// Proxies `uri` through `route` and returns the one origin request it sent. + fn single_asset_request( + settings: &Settings, + route: &ProxyAssetRoute, + uri: &str, + origin: &str, + ) -> RecordedRequest { + let (_clock, client, services) = scripted_services(); + client.respond_to( + Method::GET, + origin, + ScriptedReply::status(StatusCode::OK).body("raw"), + ); + let req = build_http_request(Method::GET, uri); + + let (response, body) = streamed_asset( + futures::executor::block_on(handle_asset_proxy_request( + settings, &services, req, route, + )) + .expect("should proxy the asset"), + ); + + assert_eq!(response.status(), StatusCode::OK, "should proxy the asset"); + assert_eq!(body, b"raw", "should forward the complete asset body"); + let requests = client.requests(); + assert_eq!( + requests.len(), + 1, + "should send one origin request for {uri}" + ); + requests[0].clone() } - /// Sink that records its calls so a test can assert how they interleave - /// with the source stream's polls. - struct RecordingSink(Rc>>); + fn rejecting_profile_set() -> ImageOptimizerProfileSet { + let mut profile_set = test_profile_set(); + profile_set.unknown_profile = UnknownProfilePolicy::Reject; + profile_set + } - impl io::Write for RecordingSink { - fn write(&mut self, buf: &[u8]) -> io::Result { - self.0.borrow_mut().push(StreamSinkEvent::Wrote(buf.len())); - Ok(buf.len()) - } + #[test] + fn handle_asset_proxy_request_skips_image_optimizer_for_svg_assets() { + let sent = single_asset_request( + &image_optimizer_settings(rejecting_profile_set()), + &image_optimizer_route(), + "https://www.example.com/.images/logo.SVG?profile=unknown&ar=1-1", + "https://assets.example.com/.images/logo.SVG", + ); - fn flush(&mut self) -> io::Result<()> { - self.0.borrow_mut().push(StreamSinkEvent::Flushed); - Ok(()) - } + assert_eq!( + sent.uri.to_string(), + "https://assets.example.com/.images/logo.SVG", + "should still strip profile-table query from SVG origin requests" + ); + sent.assert_transport(asset_get_transport()); } #[test] - fn stream_asset_body_flushes_each_chunk_before_polling_the_source_again() { - // Fastly's `StreamingBody` is a `BufWriter`, so a yielded segment only - // reaches the client once the sink is flushed. The publisher stream - // awaits the origin (and the auction) between segments, so a segment - // smaller than the write buffer must be flushed before the next poll or - // the client renders nothing behind committed headers. - let events = Rc::new(RefCell::new(Vec::new())); - let source_events = Rc::clone(&events); - let body = EdgeBody::from_stream(async_stream::stream! { - for (index, segment) in ["", ""].into_iter().enumerate() { - source_events - .borrow_mut() - .push(StreamSinkEvent::SourcePolled(index)); - yield Ok::(Bytes::from_static(segment.as_bytes())); - } - }); + fn handle_asset_proxy_request_skips_image_optimizer_for_incoming_svg() { + let mut route = ProxyAssetRoute::new("/.image/", "https://assets.example.com"); + route.path_pattern = Some(r"^/\.image/([^/]+)/[^/]+\.([^/.]+)$".to_string()); + route.target_path = Some("/image/upload/$1".to_string()); + route.image_optimizer = image_optimizer_route().image_optimizer; - futures::executor::block_on(stream_asset_body( - body, - &mut RecordingSink(Rc::clone(&events)), - )) - .expect("should stream asset body"); + let sent = single_asset_request( + &image_optimizer_settings(rejecting_profile_set()), + &route, + "https://www.example.com/.image/object-id/logo.svg?profile=unknown&ar=1-1", + "https://assets.example.com/image/upload/object-id", + ); assert_eq!( - *events.borrow(), - vec![ - StreamSinkEvent::SourcePolled(0), - StreamSinkEvent::Wrote(12), - StreamSinkEvent::Flushed, - StreamSinkEvent::SourcePolled(1), - StreamSinkEvent::Wrote(13), - StreamSinkEvent::Flushed, - ], - "each segment must be flushed to the client before the source is polled again" + sent.uri.to_string(), + "https://assets.example.com/image/upload/object-id", + "should strip profile-table query even when SVG target path omits extension" ); + sent.assert_transport(asset_get_transport()); } #[test] - fn asset_proxy_response_into_response_rejects_stream_body() { - futures::executor::block_on(async { - let services = build_services_with_http_client( - Arc::new(StreamingResponseHttpClient) as Arc - ); - let settings = create_test_settings(); - let req = build_http_request(Method::GET, "https://www.example.com/.images/foo.jpg"); - let route = ProxyAssetRoute::new("/.images/", "https://assets.example.com"); - let asset_response = handle_asset_proxy_request(&settings, &services, req, &route) - .await - .expect("should proxy a streaming asset response"); + fn handle_asset_proxy_request_skips_image_optimizer_for_rewritten_svg_assets() { + let mut route = ProxyAssetRoute::new("/.image/", "https://assets.example.com"); + route.path_pattern = Some(r"^/\.image/(.*)/[^/]+\.([^/.]+)$".to_string()); + route.target_path = Some("/image/upload/$1.$2".to_string()); + route.image_optimizer = image_optimizer_route().image_optimizer; - let err = asset_response - .into_response() - .expect_err("streaming asset responses should require explicit stream handling"); - assert!( - format!("{err:?}").contains("streaming asset response cannot be converted"), - "should describe the buffered conversion failure: {err:?}" - ); + let sent = single_asset_request( + &image_optimizer_settings(test_profile_set()), + &route, + "https://www.example.com/.image/options/id/logo.svg?profile=medium", + "https://assets.example.com/image/upload/options/id.svg", + ); + + assert_eq!( + sent.uri.to_string(), + "https://assets.example.com/image/upload/options/id.svg", + "should detect SVG after route path rewriting" + ); + sent.assert_transport(asset_get_transport()); + } + + #[test] + fn handle_asset_proxy_request_skips_s3_preflight_for_svg_image_optimizer_routes() { + let sent = single_asset_request( + &create_test_settings(), + &test_s3_image_optimizer_route(), + "https://www.example.com/.images/logo.svg?profile=medium", + "https://examplebucket.s3.example.com/.images/logo.svg", + ); + + assert_eq!( + (sent.method.clone(), sent.uri.to_string()), + ( + Method::GET, + "https://examplebucket.s3.example.com/.images/logo.svg".to_owned() + ), + "SVG IO bypass should send one GET without an S3 preflight or the transform query" + ); + sent.assert_transport(asset_get_transport()); + } + + #[test] + fn handle_asset_proxy_request_does_not_preflight_when_io_disabled() { + let sent = single_asset_request( + &image_optimizer_settings(test_profile_set()), + &test_s3_image_optimizer_route(), + "https://www.example.com/.images/foo.jpg?profile=medium&_io_debug=1", + S3_OBJECT, + ); + + assert_eq!( + sent.method, + Method::GET, + "disabled IO should not add S3 preflight" + ); + sent.assert_transport(asset_get_transport()); + } + + #[test] + fn handle_asset_proxy_request_debug_param_disables_image_optimizer() { + let sent = single_asset_request( + &image_optimizer_settings(test_profile_set()), + &image_optimizer_route(), + "https://www.example.com/.images/foo.jpg?profile=medium&_io_debug=1", + "https://assets.example.com/.images/foo.jpg", + ); + + sent.assert_transport(asset_get_transport()); + } + + #[test] + fn image_optimizer_builder_error_serves_unoptimized_with_warning() { + // Settings validation keeps crop offsets within 0 to 100, so only a + // profile set built past validation can make the options builder fail. + let mut profile_set = test_profile_set(); + profile_set.crop_offsets = Some(ImageOptimizerCropOffsetsConfig { + enabled: true, + x_param: "x".to_string(), + y_param: "y".to_string(), + buckets: vec![150], + default: 150, + when_missing: Default::default(), }); + let capture = WarningCapture::start(); + + let sent = single_asset_request( + &image_optimizer_settings(profile_set), + &image_optimizer_route(), + "https://www.example.com/.images/foo.jpg?profile=medium&ar=1-1&x=10&y=10", + "https://assets.example.com/.images/foo.jpg", + ); + + sent.assert_transport(asset_get_transport()); + let warnings = capture.take(); + assert!( + warnings + .iter() + .any(|warning| warning.message.contains("unoptimized")), + "should warn that the asset is served unoptimized, got {:?}", + warnings + .iter() + .map(|warning| warning.message.as_str()) + .collect::>() + ); + } + + #[test] + fn handle_asset_proxy_request_streams_a_chunked_origin_body() { + let (_clock, client, services) = scripted_services(); + client.respond_to( + Method::GET, + "https://assets.example.com/.images/foo.jpg", + ScriptedReply::status(StatusCode::OK) + .header(header::ETAG.as_str(), "\"stream-etag\"") + .chunks(vec![Bytes::from_static(b"ch"), Bytes::from_static(b"unk")]), + ); + let settings = create_test_settings(); + let req = build_http_request(Method::GET, "https://www.example.com/.images/foo.jpg"); + let route = ProxyAssetRoute::new("/.images/", "https://assets.example.com"); + + let (response, body) = streamed_asset( + futures::executor::block_on(handle_asset_proxy_request( + &settings, &services, req, &route, + )) + .expect("should proxy a streaming asset response"), + ); + + assert_eq!(response.status(), StatusCode::OK); + assert_eq!( + response_header(&response, header::ETAG), + Some("\"stream-etag\""), + "should keep the origin validators" + ); + assert_eq!(body, b"chunk", "should stream every origin chunk"); } // --- is_host_allowed --- @@ -6657,35 +6327,32 @@ mod tests { #[test] fn proxy_request_blocks_non_https_target_when_https_only() { - futures::executor::block_on(async { - let settings = create_test_settings(); - let services = crate::platform::test_support::build_services_with_http_client( - Arc::new(StreamingResponseHttpClient) as Arc, - ); - let req = build_http_request( - Method::GET, - "https://edge.example/integrations/prebid/bundle.js", - ); - let config = ProxyRequestConfig::new("http://assets.example/prebid/trusted-prebid.js") - .without_ec_id() - .without_forward_headers() - .with_streaming() - .with_https_only(); + let upstream = ScriptedHttpClient::forbidding(); + let services = build_services_with_outbound_client(&upstream, &TestClock::new()); + let settings = create_test_settings(); + let req = build_http_request( + Method::GET, + "https://edge.example.com/integrations/prebid/bundle.js", + ); + let config = ProxyRequestConfig::new("http://assets.example.com/prebid/trusted-prebid.js") + .without_ec_id() + .without_forward_headers() + .with_streaming() + .with_https_only(); - let err = proxy_request(&settings, req, config, &services) - .await - .expect_err("should block non-HTTPS target before proxying"); + let err = futures::executor::block_on(proxy_request(&settings, req, config, &services)) + .expect_err("should block non-HTTPS target before proxying"); - assert_eq!( - err.current_context().status_code(), - StatusCode::FORBIDDEN, - "HTTPS-only proxy requests should reject http targets" - ); - assert!( - matches!(err.current_context(), TrustedServerError::Forbidden { .. }), - "should return a forbidden error" - ); - }); + assert_eq!( + err.current_context().status_code(), + StatusCode::FORBIDDEN, + "HTTPS-only proxy requests should reject http targets" + ); + assert!( + matches!(err.current_context(), TrustedServerError::Forbidden { .. }), + "should return a forbidden error" + ); + upstream.assert_no_requests(); } #[test] @@ -6746,29 +6413,67 @@ mod tests { } #[test] - fn sign_rejects_streaming_body() { + fn sign_reads_a_streamed_body() { futures::executor::block_on(async { let settings = create_test_settings(); - let req = build_http_post_streaming_request("https://edge.example/first-party/sign"); - let err = handle_first_party_proxy_sign(&settings, &noop_services(), req) - .await - .expect_err("should reject streaming sign body"); - assert_eq!( - err.current_context().status_code(), - StatusCode::BAD_REQUEST, - "should return 400 for streaming sign body" + let req = build_http_post_streaming_request( + "https://edge.example/first-party/sign", + br#"{"url":"https://cdn.example/asset.js"}"#, ); + let resp = handle_first_party_proxy_sign(&settings, &noop_services(), req) + .await + .expect("should sign the URL from a streamed body"); + assert_eq!(resp.status(), StatusCode::OK); + let json = response_body_string(resp); assert!( - matches!( - err.current_context(), - TrustedServerError::BadRequest { message } - if message == "first-party sign request body must be buffered, not streamed" - ), - "should explain that sign bodies must be buffered" + json.contains("\"base\":\"https://cdn.example/asset.js\""), + "should sign the streamed target: {json}" ); }); } + #[test] + fn sign_body_read_failures_map_to_statuses() { + let settings = create_test_settings(); + for case in body_read_failure_cases(SIGN_MAX_BODY_BYTES) { + let (req, expectation) = case.into_request( + HttpRequest::builder() + .method(Method::POST) + .uri("https://edge.example/first-party/sign") + .header(header::CONTENT_TYPE, "application/json"), + ); + + let outcome = futures::executor::block_on(handle_first_party_proxy_sign( + &settings, + &noop_services(), + req, + )); + + expectation.assert_outcome("first-party sign", &outcome); + } + } + + #[test] + fn rebuild_body_read_failures_map_to_statuses() { + let settings = create_test_settings(); + for case in body_read_failure_cases(REBUILD_MAX_BODY_BYTES) { + let (req, expectation) = case.into_request( + HttpRequest::builder() + .method(Method::POST) + .uri("https://edge.example/first-party/proxy-rebuild") + .header(header::CONTENT_TYPE, "application/json"), + ); + + let outcome = futures::executor::block_on(handle_first_party_proxy_rebuild( + &settings, + &noop_services(), + req, + )); + + expectation.assert_outcome("first-party rebuild", &outcome); + } + } + #[test] fn rebuild_rejects_oversized_body() { futures::executor::block_on(async { diff --git a/crates/trusted-server-core/src/publisher.rs b/crates/trusted-server-core/src/publisher.rs index 8d26af7859..565fda36de 100644 --- a/crates/trusted-server-core/src/publisher.rs +++ b/crates/trusted-server-core/src/publisher.rs @@ -31,7 +31,11 @@ use brotli::enc::BrotliEncoderParams; use brotli::enc::writer::CompressorWriter; use chrono::{DateTime, Utc}; use cookie::CookieJar; -use edgezero_core::body::Body as EdgeBody; +use edgezero_core::body::{Body as EdgeBody, BodyPayload}; +use edgezero_core::error::EdgeError; +use edgezero_core::outbound::{ + OutboundCachePolicy, OutboundRequest, OutboundResponse, OutboundScopeOwner, ResponseDecoding, +}; use error_stack::{Report, ResultExt}; use flate2::read::ZlibDecoder; use flate2::write::{GzEncoder, ZlibEncoder}; @@ -75,12 +79,15 @@ use crate::error::TrustedServerError; use crate::html_processor::{ BodyCloseInjection, HtmlProcessorConfig, create_html_processor_with_document_state, }; -use crate::http_util::{RequestInfo, is_navigation_request, serve_static_with_etag}; +use crate::http_util::{ + RequestInfo, body_as_reader, into_body_payload, is_navigation_request, serve_static_with_etag, +}; use crate::integrations::{IntegrationDocumentState, IntegrationRegistry}; +use crate::outbound::{ + OutboundRequestSource, PUBLISHER_UPLOAD_BYTES, STREAMED_TIMEOUT, outbound_error, streamed, +}; use crate::platform::{ - ClientInfo, GeoInfo, PlatformBackendSpec, PlatformHttpRequest, RuntimeServices, - TEMPLATE_CACHE_PURGE_ALL_SURROGATE_KEY, VarySpec, contains_publisher_esi_directive, - reader_url_surrogate_key, + ClientInfo, GeoInfo, RuntimeServices, VarySpec, contains_publisher_esi_directive, }; use crate::price_bucket::{PriceGranularity, price_bucket}; use crate::response_header_rewrite::{OriginHeaderRewrite, rewrite_origin_urls_in_headers}; @@ -107,7 +114,6 @@ use html_finalization::HtmlEofDriver; include!(concat!(env!("OUT_DIR"), "/template_build_digest.rs")); const SUPPORTED_ENCODING_VALUES: [&str; 3] = ["gzip", "deflate", "br"]; -const DEFAULT_PUBLISHER_FIRST_BYTE_TIMEOUT: Duration = Duration::from_secs(15); const HEADER_X_TS_TEMPLATE_CACHE: &str = "x-ts-template-cache"; const HEADER_X_TS_ASSEMBLY: &str = "x-ts-assembly"; @@ -174,19 +180,16 @@ fn set_assembly_response_state(response: &mut Response, state: Assembl ); } -fn body_as_reader( - body: EdgeBody, -) -> Result, Report> { - let bytes = body.into_bytes().ok_or_else(|| { - Report::new(TrustedServerError::Proxy { - message: "streaming body cannot be processed by sync publisher pipeline".to_string(), - }) - })?; - Ok(std::io::Cursor::new(bytes)) -} +/// Carries a publisher origin body read error inside a processing report. +/// +/// [`publisher_stream_error`] takes the error back out, so a response stream +/// ends with the origin's own [`EdgeError`] and keeps its status class. +struct OriginBodyError(Option); struct BodyChunkSource { - body: Option, + // Converted once from the origin body: buffered bytes, or a stream that + // keeps any outbound scope owner alive until it ends. + body: Option, chunk_size: usize, max_bytes: usize, bytes_seen: usize, @@ -196,7 +199,7 @@ struct BodyChunkSource { impl BodyChunkSource { fn new(body: EdgeBody, chunk_size: usize) -> Self { Self { - body: Some(body), + body: Some(into_body_payload(body)), chunk_size, max_bytes: usize::MAX, bytes_seen: 0, @@ -215,7 +218,7 @@ impl BodyChunkSource { // silently reporting end-of-stream on the next call. let pulled = match &mut self.body { None => Ok(None), - Some(EdgeBody::Once(bytes)) => { + Some(BodyPayload::Once(bytes)) => { let end = (self.once_offset + self.chunk_size).min(bytes.len()); if self.once_offset >= end { Ok(None) @@ -225,11 +228,12 @@ impl BodyChunkSource { Ok(Some(chunk)) } } - Some(EdgeBody::Stream(stream)) => match stream.next().await { + Some(BodyPayload::Stream(stream)) => match stream.next().await { Some(Ok(chunk)) => Ok(Some(chunk)), Some(Err(err)) => Err(Report::new(TrustedServerError::Proxy { message: format!("Failed to read publisher origin body stream: {err}"), - })), + }) + .attach_opaque(OriginBodyError(Some(err)))), None => Ok(None), }, }; @@ -287,10 +291,19 @@ fn process_and_encode_chunk( Ok(Some(bytes::Bytes::from(encoded))) } +/// Turn a publisher stream failure into the error that ends the response body. +/// +/// An origin body read error passes through unchanged. Any other processing +/// failure becomes an internal error that carries the report text. // By-value signature so `map_err(publisher_stream_error)` works directly. -#[allow(clippy::needless_pass_by_value)] -fn publisher_stream_error(err: Report) -> std::io::Error { - std::io::Error::other(format!("{err:?}")) +fn publisher_stream_error(mut err: Report) -> EdgeError { + if let Some(origin) = err + .downcast_mut::() + .and_then(|carried| carried.0.take()) + { + return origin; + } + EdgeError::internal(std::io::Error::other(format!("{err:?}"))) } fn not_found_response() -> Response { @@ -787,7 +800,7 @@ fn process_response_streaming( })?; StreamingPipeline::new(config, processor) .with_max_pending_decoded_bytes(max_pending_decoded_bytes) - .process(body_as_reader(body)?, output)?; + .process(body_as_reader(body, "publisher")?, output)?; } else if is_rsc_flight { // RSC Flight responses are length-prefixed (T rows). A naive string replacement will // corrupt the stream by changing byte lengths without updating the prefixes. @@ -799,7 +812,7 @@ fn process_response_streaming( ); StreamingPipeline::new(config, processor) .with_max_pending_decoded_bytes(max_pending_decoded_bytes) - .process(body_as_reader(body)?, output)?; + .process(body_as_reader(body, "publisher")?, output)?; } else { let replacer = create_url_replacer( params.origin_host, @@ -809,7 +822,7 @@ fn process_response_streaming( ); StreamingPipeline::new(config, replacer) .with_max_pending_decoded_bytes(max_pending_decoded_bytes) - .process(body_as_reader(body)?, output)?; + .process(body_as_reader(body, "publisher")?, output)?; } Ok(()) @@ -855,7 +868,7 @@ async fn process_response_streaming_async( &mut processor, input_compression, output_compression, - settings.publisher.max_buffered_body_bytes, + params.max_input_bytes(settings), ) .await } @@ -1771,6 +1784,8 @@ pub struct OwnedProcessResponseParams { pub(crate) response_status: StatusCode, /// Whether status or headers identify a range-associated representation. pub(crate) range_associated: bool, + /// The adapter's cap on origin input, from [`PublisherOriginOptions::input_cap`]. + pub(crate) input_cap: Option, pub(crate) ad_slots_script: Option, pub(crate) ad_bids_state: AdBidsState, /// Observation context for the in-flight auction. @@ -1799,6 +1814,17 @@ pub struct OwnedProcessResponseParams { } impl OwnedProcessResponseParams { + /// Cumulative cap on the raw and decoded origin bytes the rewrite reads. + /// + /// It is the adapter's input cap where the adapter collects the response + /// before egress, and `max_buffered_body_bytes` otherwise. + fn max_input_bytes(&self, settings: &Settings) -> usize { + self.input_cap + .map_or(settings.publisher.max_buffered_body_bytes, |cap| { + usize::try_from(cap).unwrap_or(usize::MAX) + }) + } + fn is_range_associated(&self) -> bool { self.range_associated || self.response_status == StatusCode::PARTIAL_CONTENT } @@ -1821,11 +1847,15 @@ enum TemplateStoreOutcome { /// Buffers a [`PublisherResponse`] into a single [`Response`], collecting the /// dispatched server-side auction before buffering. /// -/// Handles all three variants: returns [`PublisherResponse::Buffered`] unchanged, -/// pipes [`PublisherResponse::Stream`] through the streaming pipeline into -/// memory, and reattaches [`PublisherResponse::PassThrough`] bodies directly. +/// Handles every variant: pipes [`PublisherResponse::Stream`] through the +/// streaming pipeline into memory and reattaches the raw +/// [`PublisherResponse::Buffered`] and [`PublisherResponse::PassThrough`] +/// bodies. With an `input_cap`, raw bodies are collected under it, and the +/// processed pipeline decodes at most `input_cap` origin bytes, so every +/// classification is bounded and fails before egress. Without one, which is +/// Fastly's template-cache miss, raw bodies keep streaming. /// -/// The buffered size is capped by `settings.publisher.max_buffered_body_bytes` +/// The processed output is capped by `settings.publisher.max_buffered_body_bytes` /// (16 MiB by default), so processable origin responses cannot grow the buffer /// without bound and exhaust the Wasm heap. /// @@ -1833,8 +1863,8 @@ enum TemplateStoreOutcome { /// statuses (204, 304) carry no body but may advertise the `GET` representation's /// length, so they skip the buffer and length rewrite. /// -/// Buffered adapters (Axum, Cloudflare, Spin, and non-streaming fallbacks) call -/// this: it drives +/// Axum, Cloudflare, and Spin call this for every publisher response, and +/// Fastly calls it on a template-cache miss. It drives /// [`stream_publisher_body_async`], which awaits /// [`AuctionOrchestrator::collect_dispatched_auction`], writes the winning bids /// into `ad_bids_state`, and injects them before ``. @@ -1842,7 +1872,8 @@ enum TemplateStoreOutcome { /// # Errors /// /// Returns an error if the streaming pipeline fails to process the response -/// body, if the processed body exceeds the configured buffer cap, or if async +/// body, if the processed body exceeds the configured buffer cap, if a raw +/// body is over `input_cap` or its source fails, or if async /// HTML completion lacks operator permission or has inconsistent factory/session /// capabilities. /// @@ -1857,6 +1888,7 @@ pub async fn buffer_publisher_response_async( integration_registry: &IntegrationRegistry, orchestrator: &AuctionOrchestrator, services: &RuntimeServices, + input_cap: Option, ) -> Result, Report> { if let Err(error) = integration_registry.ensure_async_html_finish_allowed(settings) { if let PublisherResponse::Stream { params, .. } = &mut publisher_response { @@ -1866,12 +1898,14 @@ pub async fn buffer_publisher_response_async( } match publisher_response { PublisherResponse::Buffered(mut response) => { - // A buffered-unmodified response can carry an origin body (a stream - // on streaming-capable adapters). A bodiless response (HEAD, 204, + // A buffered-unmodified response can carry an origin body, which + // arrives as a stream. A bodiless response (HEAD, 204, // 205, 304) must stay bodiless, so drop the body and correct its // framing (204/205 Content-Length; HEAD/304 preserved), matching // the streaming finalizer. - if !response_carries_body(method, response.status()) { + if response_carries_body(method, response.status()) { + collect_raw_body(&mut response, input_cap).await?; + } else { make_response_bodiless(&mut response); } Ok(response) @@ -1911,6 +1945,7 @@ pub async fn buffer_publisher_response_async( { abandon_response_auction(&mut params, services, "partial_response").await; *response.body_mut() = body; + collect_raw_body(&mut response, input_cap).await?; return Ok(response); } if is_html_content_type(¶ms.content_type) @@ -2092,11 +2127,40 @@ pub async fn buffer_publisher_response_async( } PublisherResponse::PassThrough { mut response, body } => { *response.body_mut() = body; + collect_raw_body(&mut response, input_cap).await?; Ok(response) } } } +/// Collects a raw origin body under the adapter's input cap, so it fails before egress. +/// +/// Without a cap the body is left as it is. +/// +/// # Errors +/// +/// Returns the publisher's 502 when the body is over `input_cap` or its source fails. +async fn collect_raw_body( + response: &mut Response, + input_cap: Option, +) -> Result<(), Report> { + let Some(cap) = input_cap else { + return Ok(()); + }; + let body = std::mem::replace(response.body_mut(), EdgeBody::empty()); + let bytes = body + .into_bytes_bounded(usize::try_from(cap).unwrap_or(usize::MAX)) + .await + .map_err(|error| { + Report::new(TrustedServerError::Proxy { + message: "publisher origin body could not be collected under its cap".to_owned(), + }) + .attach_opaque(error) + })?; + *response.body_mut() = EdgeBody::from(bytes); + Ok(()) +} + /// Splices this visitor's slots and bids into the seam, if the mode assembles. /// /// Gives the platform assembler the already-buffered cold document. Fastly resolves a @@ -2238,6 +2302,7 @@ fn build_template_assembly_params( OwnedProcessResponseParams { response_status: StatusCode::OK, range_associated: false, + input_cap: None, csp_nonce_observed: None, // Already stored; storing again on a hit would be pointless work. template_cache_key: None, @@ -2483,8 +2548,8 @@ async fn store_template_if_authorized( /// bodies where possible. /// /// Buffered adapters should keep using [`buffer_publisher_response_async`]. -/// Fastly uses this helper before the entry point commits headers, allowing the -/// response body to be pulled lazily by `stream_to_client()`. +/// Fastly uses this helper before `EdgeZero`'s delivery commits headers, +/// allowing the response body to be pulled lazily as it streams to the client. /// /// # Errors /// @@ -2538,6 +2603,7 @@ pub async fn publisher_response_into_streaming_response( integration_registry, &orchestrator, &services, + None, )) .await; } @@ -2546,10 +2612,9 @@ pub async fn publisher_response_into_streaming_response( PublisherResponse::Buffered(mut response) => { // Fastly requests the origin body as a stream before the response is // classified, so a buffered-unmodified response can still hold an - // `EdgeBody::Stream`. A bodiless response (HEAD, 204, 205, 304) must - // stay bodiless — `send_edgezero_response` streams any - // `EdgeBody::Stream` to the client — so drop the body and correct - // its framing (204/205 Content-Length; HEAD/304 preserved). + // `EdgeBody::Stream`. Delivery streams any `EdgeBody::Stream` to the + // client, so a bodiless response (HEAD, 204, 205, 304) drops its body + // and corrects its framing (204/205 Content-Length; HEAD/304 preserved). if !response_carries_body(method, response.status()) { make_response_bodiless(&mut response); } @@ -2586,7 +2651,7 @@ pub async fn publisher_response_into_streaming_response( // eagerly instead measured ~100x worse TTFB than doing nothing. let stream = async_stream::try_stream! { let (head, tail) = split_template_at_seam(&template) - .map_err(|e| std::io::Error::other(e.to_string()))?; + .map_err(|e| EdgeError::internal(std::io::Error::other(e.to_string())))?; let mut encoder = BodyStreamEncoder::new(compression); let encoded_head = encoder .encode_chunk(head.to_vec()) @@ -2637,7 +2702,7 @@ pub async fn publisher_response_into_streaming_response( yield bytes::Bytes::from(trailer); } }; - *response.body_mut() = EdgeBody::from_stream::<_, std::io::Error>(stream); + *response.body_mut() = EdgeBody::from_stream(stream); Ok(response) } PublisherResponse::PassThrough { mut response, body } => { @@ -2730,7 +2795,7 @@ pub async fn publisher_response_into_streaming_response( } } }; - *response.body_mut() = EdgeBody::from_stream::<_, std::io::Error>(stream); + *response.body_mut() = EdgeBody::from_stream(stream); return Ok(response); } @@ -2763,9 +2828,9 @@ pub async fn publisher_response_into_streaming_response( }; (DispatchedAuctionGuard::new(dispatched), telemetry) }); + let max_body_bytes = params.max_input_bytes(&settings); let stream = async_stream::try_stream! { let compression = Compression::from_content_encoding(¶ms.content_encoding); - let max_body_bytes = settings.publisher.max_buffered_body_bytes; let mut decoder = BodyStreamDecoder::new(compression, max_body_bytes); let mut encoder = BodyStreamEncoder::new(compression); let mut source = BodyChunkSource::new(body, STREAM_CHUNK_SIZE) @@ -2904,7 +2969,7 @@ pub async fn publisher_response_into_streaming_response( } } }; - *response.body_mut() = EdgeBody::from_stream::<_, std::io::Error>(stream); + *response.body_mut() = EdgeBody::from_stream(stream); Ok(response) } } @@ -2971,45 +3036,40 @@ fn apply_datadome_client_tag_cache_privacy( /// Drop a bodiless response's body and correct its framing headers. /// -/// The response keeps no body, and its `Content-Length` is corrected where the -/// origin's value would be invalid for the now-empty message: +/// The response keeps no body and never keeps `Trailer`: response egress +/// refuses to send a response that declares one. Its `Content-Length` is +/// corrected where the origin's value would be invalid for the now-empty +/// message: /// - **204 No Content**: RFC 9110 §8.6 forbids `Content-Length`; remove it. /// - **205 Reset Content**: the reset response carries nothing, so a nonzero /// origin length is wrong (RFC 9110 §15.4.6); normalize it to `0`. /// - **HEAD and 304 Not Modified**: `Content-Length` legitimately advertises /// the `GET` representation length, so it is preserved untouched. /// -/// The normalized 204/205 responses also drop chunked-framing metadata the -/// adapters would otherwise copy verbatim: RFC 9112 §6.1 forbids -/// `Transfer-Encoding` on a 204, and keeping it on a 205 alongside the +/// The normalized 204/205 responses also drop `Transfer-Encoding`: RFC 9112 +/// §6.1 forbids it on a 204, and keeping it on a 205 alongside the /// `Content-Length: 0` inserted above would advertise two conflicting framings. -/// `Trailer` describes fields that only a chunked body can carry, so it goes -/// with it. `HEAD` and 304 keep both for the same reason they keep -/// `Content-Length`: the fields describe the `GET` representation, not this -/// message. +/// `HEAD` and 304 keep it for the same reason they keep `Content-Length`: it +/// describes the `GET` representation, not this message. Response egress +/// strips `Transfer-Encoding` from every response before it is sent. fn make_response_bodiless(response: &mut Response) { *response.body_mut() = EdgeBody::empty(); + response.headers_mut().remove(header::TRAILER); match response.status() { StatusCode::NO_CONTENT => { response.headers_mut().remove(header::CONTENT_LENGTH); - remove_chunked_framing_headers(response.headers_mut()); + response.headers_mut().remove(header::TRANSFER_ENCODING); } StatusCode::RESET_CONTENT => { response .headers_mut() .insert(header::CONTENT_LENGTH, HeaderValue::from(0_u64)); - remove_chunked_framing_headers(response.headers_mut()); + response.headers_mut().remove(header::TRANSFER_ENCODING); } _ => {} } } -/// Remove the framing fields that only apply to a message with a chunked body. -fn remove_chunked_framing_headers(headers: &mut header::HeaderMap) { - headers.remove(header::TRANSFER_ENCODING); - headers.remove(header::TRAILER); -} - /// A [`Write`] sink that buffers into a `Vec` but fails once the configured /// byte limit would be exceeded. /// @@ -3057,10 +3117,10 @@ impl Write for BoundedWriter { /// Stream the publisher response body through the processing pipeline. /// -/// Called by the adapter after `stream_to_client()` has committed the response -/// headers. Runs synchronously against an already-materialised body; the async -/// I/O happened upstream in [`handle_publisher_request`]. Writes processed -/// chunks directly to `output`. +/// Called by an adapter after its response headers are committed. Runs +/// synchronously against an already-materialised body; the async I/O happened +/// upstream in [`handle_publisher_request`]. Writes processed chunks directly +/// to `output`. /// /// This helper performs synchronous transformations only. Registered HTML /// finalizers require [`stream_publisher_body_async`], @@ -3089,7 +3149,7 @@ pub fn stream_publisher_body( && is_html_content_type(¶ms.content_type) && integration_registry.has_async_html_finish() { - let mut input = body_as_reader(body)?; + let mut input = body_as_reader(body, "publisher")?; std::io::copy(&mut input, output).change_context(TrustedServerError::Proxy { message: "Failed to copy range-associated HTML".to_string(), })?; @@ -3397,6 +3457,7 @@ pub async fn stream_publisher_body_async( settings, request_origin: request_origin(¶ms.request_scheme, ¶ms.request_host), }, + max_input_bytes: params.max_input_bytes(settings), }, ) .await @@ -3465,40 +3526,10 @@ fn should_preload_ec_snapshot(input: &EcSnapshotPreloadInput) -> bool { eligible && !marker_can_skip } -/// Rewrites a downstream request into an outbound publisher-origin request. -/// -/// Restricts advertised encodings to those the rewrite pipeline can handle, -/// strips the internal `fastly-ssl` scheme signal so it never leaks to the -/// backend, and retargets the URI and `Host` header at the origin. -fn rewrite_origin_request( - req: &mut Request, - target_uri: Uri, - origin_host_header: &str, -) -> Result<(), Report> { - restrict_accept_encoding(req); - // Layering wart: `fastly-ssl` is a vendor-specific header name. Core only - // knows it because the Fastly adapter re-injects it from trusted TLS - // metadata and `detect_scheme` (see `http_util`) still reads it as a - // fallback scheme signal. The neutral signal (`ClientInfo::tls_protocol`) - // is already the primary path, so this strip — and the whole `fastly-ssl` - // coupling in core — should move behind a platform-neutral header in a - // separate change. Until then we strip it here so the edge signal never - // reaches publisher backends. - req.headers_mut().remove("fastly-ssl"); - *req.uri_mut() = target_uri; - req.headers_mut().insert( - header::HOST, - HeaderValue::from_str(origin_host_header).change_context(TrustedServerError::Proxy { - message: "invalid publisher origin host header".to_string(), - })?, - ); - Ok(()) -} - /// Build a minimal [`AuctionContext`] for the collect phase. /// /// See [`AuctionContext::request`]: the orchestrator's collect path runs -/// after `send_async` has already consumed the real client request, so this +/// after the origin request has consumed the real client request, so this /// context carries a synthetic placeholder. The orchestrator itself /// instantiates a fresh placeholder when it actually invokes a mediator — /// this argument is plumbing for the (presently unused) case where the @@ -3518,7 +3549,6 @@ fn make_collect_context<'a>( settings, request: placeholder, timeout_ms: 0, - transport_timeout_ms: 0, provider_responses: None, services, } @@ -4030,6 +4060,8 @@ struct AuctionCollectCtx<'a> { dispatched: DispatchedAuction, telemetry: AuctionTelemetryCarry, deps: AuctionCollectDeps<'a>, + /// Cumulative cap on the raw and decoded origin bytes the hold reads. + max_input_bytes: usize, } /// Borrowed dependencies of the auction collect step. @@ -4058,8 +4090,8 @@ async fn stream_html_with_auction_hold( output_compression: Compression, ctx: AuctionCollectCtx<'_>, ) -> Result<(), Report> { + let max_body_bytes = ctx.max_input_bytes; if body.is_stream() { - let max_body_bytes = ctx.deps.settings.publisher.max_buffered_body_bytes; return body_close_hold_loop_stream( body, output, @@ -4073,10 +4105,8 @@ async fn stream_html_with_auction_hold( .await; } - // Bound the gzip decode budget to the same ceiling the buffered writer - // enforces, matching the streaming arm above and the no-hold buffered path. - let max_body_bytes = ctx.deps.settings.publisher.max_buffered_body_bytes; - let body = body_as_reader(body)?; + // The gzip decode budget is the same input cap the streaming arm above uses. + let body = body_as_reader(body, "publisher")?; if output_compression == Compression::None { return match input_compression { Compression::None => { @@ -4146,11 +4176,9 @@ async fn stream_html_with_auction_hold( /// lazy streaming body built by [`publisher_response_into_streaming_response`], /// so the two async hold paths cannot drift apart. /// -/// No production caller reaches this today: it is only entered through -/// [`buffer_publisher_response_async`], and the buffered adapters (Axum, -/// Cloudflare, Spin) never produce `Body::Stream` because the publisher fetch -/// is gated on `supports_streaming_responses()`. It is groundwork for those -/// adapters' streaming cutover; Fastly uses the lazy stream instead. +/// It is entered through [`buffer_publisher_response_async`], which Axum, +/// Cloudflare, Spin, and Fastly's template-cache miss use: the publisher origin +/// body is always a stream. Fastly's ordinary path uses the lazy stream instead. #[allow( clippy::too_many_arguments, reason = "stream state remains explicit across the shared adapter driver" @@ -4169,6 +4197,7 @@ async fn body_close_hold_loop_stream( dispatched, telemetry, deps: collect_refs, + .. } = ctx; let mut decoder = BodyStreamDecoder::new(input_compression, max_body_bytes); let mut encoder = BodyStreamEncoder::new(output_compression); @@ -4390,6 +4419,7 @@ async fn body_close_hold_loop( dispatched, mut telemetry, deps, + .. } = ctx; let mut buffer = vec![0u8; STREAM_CHUNK_SIZE]; let mut hold = inline_seam_token.map(InlineBodyCloseSeam::new); @@ -4808,53 +4838,6 @@ pub(crate) fn origin_response_is_shareable(inputs: SharedRequestInputs) -> bool && !inputs.request_requires_origin } -/// Apply the origin fetch's cache intent. -/// -/// Both publisher-origin fetch paths call this rather than deciding for themselves. They -/// are alternatives for the same fetch — one inside the EC-preload fan-out, one in the -/// branch taken when that did not fire — so a condition written twice could drift and make -/// readthrough eligibility depend on whether EC preload happened, which is not a property -/// of the origin response at all. -fn apply_origin_cache_intent( - request: PlatformHttpRequest, - readthrough_enabled: bool, - origin_response_is_shareable: bool, - should_run_ad_stack: bool, - request_is_document: bool, - reader_url: &str, -) -> PlatformHttpRequest { - // With the opt-in disabled, preserve the existing policy: ad-serving requests - // bypass and other publisher requests use the platform default. Enabling the flag - // replaces that policy with request shareability, both widening eligible ad traffic - // and tightening non-ad *document* traffic that carries disqualifying reader state. - // - // Documents only. Subresources keep the platform default, because the readthrough - // gate answers a question about pages: whether the origin's HTML may be shared - // between readers. Judging them on it would bypass the edge cache for every - // cookie-bearing or conditional asset request — browsers send first-party cookies on - // subresources, so that is most repeat-visitor asset traffic — and would tag every - // cached asset with `ts-template`, turning the template rollback lever into an - // origin-wide asset flush. - let readthrough_applies = readthrough_enabled && request_is_document; - let bypass = if readthrough_applies { - !origin_response_is_shareable - } else { - should_run_ad_stack - }; - if bypass { - request.with_cache_bypass() - } else if readthrough_applies { - // The same reader-facing key and all-scope key used by the purge endpoint. - // Fastly accepts multiple space-separated surrogate keys without changing TTL. - request.with_shared_cache(format!( - "{TEMPLATE_CACHE_PURGE_ALL_SURROGATE_KEY} {}", - reader_url_surrogate_key(reader_url), - )) - } else { - request - } -} - /// Whether this request may additionally use a shared *template*. /// /// The extra conditions say whether this pipeline can assemble one, and whether this @@ -4877,6 +4860,141 @@ pub(crate) fn request_can_use_shared_template( && !reader_requires_origin } +/// Maps a failed [`OutboundScopeOwner::attach`] onto the publisher site's error. +/// +/// Each adapter attaches the owner [`handle_publisher_request`] returns to the +/// final response exactly once. The attach fails only when the response body +/// already owns an outbound scope or the owner was closed, both of which mean +/// the response cannot carry its auction and origin work to egress. +pub fn publisher_owner_error(error: EdgeError) -> Report { + Report::new(TrustedServerError::Proxy { + message: "publisher response could not own its outbound work".to_owned(), + }) + .attach_opaque(error) +} + +/// How [`handle_publisher_request`] fetches the publisher origin on one adapter. +#[derive(Debug, Clone, Copy)] +pub struct PublisherOriginOptions { + /// Shared-cache header family the publisher asset cache policy writes. + pub edge_header: EdgeCacheHeader, + /// Cap on the origin body the adapter reads before it commits the response. + /// + /// `Some` on adapters that collect the whole response before egress (Axum, + /// Cloudflare, Spin). The origin request then fails any body over the cap, + /// raw bodies are collected under it, and processed HTML is decoded under + /// it. `None` on Fastly, which streams the response. + pub input_cap: Option, + /// Whether the origin body arrives in the coding the origin sent. + /// + /// [`ResponseDecoding::Original`] keeps the origin coding, so processed HTML + /// is re-encoded with it and raw bodies relay their encoded bytes. + /// [`ResponseDecoding::Decode`] hands Trusted Server identity bytes, bounded + /// at `input_cap` once decoded. + pub response_decoding: ResponseDecoding, +} + +/// Returns the [`PublisherOriginOptions::input_cap`] for an adapter whose runtime allows +/// `platform_cap` bytes: the smaller of that and the publisher's `max_buffered_body_bytes`. +/// +/// # Examples +/// +/// ``` +/// use trusted_server_core::publisher::publisher_input_cap; +/// use trusted_server_core::settings::Settings; +/// +/// let mut settings = Settings::default(); +/// settings.publisher.max_buffered_body_bytes = 4096; +/// assert_eq!(publisher_input_cap(&settings, 1024), 1024); +/// assert_eq!(publisher_input_cap(&settings, u64::MAX), 4096); +/// ``` +#[must_use] +pub fn publisher_input_cap(settings: &Settings, platform_cap: u64) -> u64 { + u64::try_from(settings.publisher.max_buffered_body_bytes) + .unwrap_or(u64::MAX) + .min(platform_cap) +} + +/// Whether `override_authority` names a different authority than `origin`. +/// +/// Hosts compare case-insensitively and a default port counts as absent, so +/// `Example.COM` and `example.com:443` both equal `https://example.com`. An +/// override that does not parse as an authority counts as different, and the +/// request builder then rejects it. +fn host_override_differs(override_authority: &str, origin: &url::Url) -> bool { + let Ok(authority) = override_authority.parse::() else { + return true; + }; + let default_port = match origin.scheme() { + "http" => Some(80), + "https" => Some(443), + _ => None, + }; + let override_port = authority.port_u16().or(default_port); + let host_matches = origin + .host_str() + .is_some_and(|host| host.eq_ignore_ascii_case(authority.host())); + !(host_matches && override_port == origin.port_or_known_default()) +} + +/// Builds the publisher origin request from the reader's request. +/// +/// The request keeps the inbound body and its read deadline and uploads at most +/// [`PUBLISHER_UPLOAD_BYTES`]. It streams the response, in its original coding +/// with [`ResponseDecoding::Original`] or decoded with +/// [`ResponseDecoding::Decode`], and caps the encoded response at `input_cap`. +/// Where the adapter collects the response, Trusted Server bounds the decoded +/// bytes at `input_cap` too. +/// +/// # Errors +/// +/// Returns [`TrustedServerError::BadRequest`] when the forward is malformed, +/// for example a malformed `Connection` field or an origin URL with userinfo. +fn publisher_origin_request( + req: Request, + target: Uri, + host_authority_override: Option<&str>, + cache_policy: OutboundCachePolicy, + origin: PublisherOriginOptions, +) -> Result> { + let forward_error = |error| { + outbound_error( + error, + TrustedServerError::Proxy { + message: "publisher origin request could not be built".to_owned(), + }, + OutboundRequestSource::ClientForward, + ) + }; + let mut request = streamed( + OutboundRequest::from_request(req, target).map_err(forward_error)?, + STREAMED_TIMEOUT, + ) + .cache_policy(cache_policy) + .max_request_body_bytes(PUBLISHER_UPLOAD_BYTES) + .response_decoding(origin.response_decoding); + if let Some(cap) = origin.input_cap { + request = request.max_encoded_response_bytes(cap); + } + if let Some(authority) = host_authority_override { + request = request + .host_authority_override(authority) + .map_err(forward_error)?; + } + Ok(request) +} + +/// Maps a failed publisher origin fetch onto the publisher site's error. +fn origin_fetch_error(error: EdgeError) -> Report { + outbound_error( + error, + TrustedServerError::Proxy { + message: "Failed to proxy request to origin".to_owned(), + }, + OutboundRequestSource::ClientForward, + ) +} + /// Proxies requests to the publisher's origin server. /// /// Returns a [`PublisherResponse`] indicating how the response should be sent: @@ -4887,10 +5005,19 @@ pub(crate) fn request_can_use_shared_template( /// unprocessable content, `204/205`, or HTML that requires full-document /// post-processing /// +/// The response travels with the [`OutboundScopeOwner`] of its outbound work: +/// the dispatched auction batch and the origin body. The owner is created +/// before anything is dispatched, and both the auction and the origin call +/// are constructed inside [`OutboundScopeOwner::run`], so head finishing binds +/// the origin body to the shared response-body scope. The adapter attaches the +/// owner to its final response exactly once, with +/// [`OutboundScopeOwner::attach`]. Dropping the owner cancels the work. +/// /// # Errors /// -/// Returns a [`TrustedServerError`] if the request host is malformed, the proxy -/// request fails, or the origin backend is unreachable. +/// Returns a [`TrustedServerError`] if the request host is malformed, the +/// forward is malformed (400), the upload is over its cap (413), the upload +/// misses the inbound read deadline (408), or the origin fetch fails (502). pub async fn handle_publisher_request( settings: &Settings, services: &RuntimeServices, @@ -4898,8 +5025,9 @@ pub async fn handle_publisher_request( ec_context: &mut EcContext, auction: AuctionDispatch<'_>, mut req: Request, - edge_header: EdgeCacheHeader, -) -> Result> { + origin: PublisherOriginOptions, +) -> Result<(PublisherResponse, OutboundScopeOwner), Report> { + let owner = OutboundScopeOwner::response_body(); auction .integration_registry .ensure_async_html_finish_allowed(settings)?; @@ -4972,41 +5100,40 @@ pub async fn handle_publisher_request( }, )?; let origin_scheme = parsed_origin.scheme().to_string(); - let origin_host_without_port = parsed_origin.host_str().ok_or_else(|| { - Report::new(TrustedServerError::Proxy { + if parsed_origin.host_str().is_none() { + return Err(Report::new(TrustedServerError::Proxy { message: "Missing host in origin_url".to_string(), - }) - })?; - let backend_name = services - .backend() - .ensure(&PlatformBackendSpec { - scheme: origin_scheme.clone(), - host: origin_host_without_port.to_string(), - port: parsed_origin.port(), - host_header_override: settings.publisher.origin_host_header_override.clone(), - certificate_check: settings.proxy.certificate_check, - first_byte_timeout: DEFAULT_PUBLISHER_FIRST_BYTE_TIMEOUT, - between_bytes_timeout: DEFAULT_PUBLISHER_FIRST_BYTE_TIMEOUT, - discriminator: None, - }) - .change_context(TrustedServerError::Proxy { - message: "backend registration failed".to_string(), - })?; + })); + } let origin_host = settings.publisher.origin_host(); - let origin_host_header = settings.publisher.origin_host_header(); + // Only an override that names another authority changes what the origin + // receives. An equivalent one would change nothing but Fastly's backend + // identity, so it sets none. + let host_authority_override = settings + .publisher + .origin_host_header_override + .as_deref() + .filter(|authority| host_override_differs(authority, &parsed_origin)); + // The authority the origin receives as `Host`, so an equivalent override reads + // as no host change to the header rewrite and the template identity. + let origin_host_header = + host_authority_override.map_or_else(|| origin_host.clone(), str::to_owned); let origin_path_and_query = req .uri() .path_and_query() .map(http::uri::PathAndQuery::as_str) .unwrap_or("/"); let request_path_and_query = origin_path_and_query.to_string(); - let target_uri = format!("{origin_scheme}://{origin_host}{origin_path_and_query}") + // The target keeps any userinfo of `origin_url`, so the request builder + // refuses it instead of the forward silently dropping the credentials. + let origin_authority = &parsed_origin[url::Position::BeforeUsername..url::Position::AfterPort]; + let target_uri = format!("{origin_scheme}://{origin_authority}{origin_path_and_query}") .parse::() .change_context(TrustedServerError::Proxy { message: "invalid publisher origin uri".to_string(), })?; - log::debug!("Proxying request to configured publisher backend"); + log::debug!("Proxying request to the configured publisher origin"); let request_path = req.uri().path().to_string(); let is_get = req.method() == http::Method::GET; @@ -5183,8 +5310,7 @@ pub async fn handle_publisher_request( || datadome_suppression_requires_origin; let method_is_cacheable = req.method() == Method::GET; - // Read while the request is still in hand: the readthrough policy below applies to - // documents only, and the origin send consumes these headers. + // Read while the request is still in hand: the origin request consumes these headers. let request_is_document = is_html_document_request(&req); let shared_request_inputs = SharedRequestInputs { method_is_cacheable, @@ -5193,21 +5319,13 @@ pub async fn handle_publisher_request( cookie_disqualifies, request_requires_origin, }; - // Cookie variants belong to the template key only. The platform readthrough key - // cannot distinguish them, including an absent configured cookie, so these - // deployments must fetch origin bytes while retaining per-variant templates. + // Cookie variants belong to the template key only. A platform cache key cannot + // distinguish them, including an absent configured cookie, so telemetry counts + // those responses as not shareable between readers. let origin_response_is_shareable = origin_response_is_shareable(SharedRequestInputs { cookie_disqualifies: cookie_disqualifies || !key_cookie_names.is_empty(), ..shared_request_inputs }); - // Readthrough is off unless an operator turns it on. Unlike the cookie flag, which - // only ever applies to cookie-bearing requests, this gate would otherwise admit every - // cookieless request the moment this code deploys — and cookieless first-time visitors - // are exactly the readers an origin issues a session cookie to. - let origin_readthrough_enabled = settings - .creative_opportunities - .as_ref() - .is_some_and(CreativeOpportunitiesConfig::origin_readthrough_enabled); let request_can_use_shared_template = !async_html_finish_present && request_can_use_shared_template( shared_request_inputs, @@ -5253,21 +5371,12 @@ pub async fn handle_publisher_request( if reader_requires_origin && matches!(assembly_mode, AssemblyMode::Esi) { log::debug!("template_cache bypass: request cache semantics or diagnostics require origin"); } - // Capture the reader URL before origin rewriting, including its query. Inline - // assembly has no template key, but readthrough still needs both purge scopes. - let readthrough_reader_url = - if origin_readthrough_enabled && origin_response_is_shareable && request_is_document { - format!("{request_scheme}://{request_host}{request_path_and_query}") - } else { - String::new() - }; let template_cache_key = request_can_use_shared_template.then(|| crate::platform::TemplateCacheKey { url: target_uri.to_string(), request_host: request_host.to_string(), request_scheme: request_scheme.to_string(), - // Captured before `rewrite_origin_request` below replaces the URI with the - // origin target. Path *and* query: a different query is a different page, and + // The reader's own path, not the origin target. Path *and* query: a different query is a different page, and // a purge caller types the whole address. request_path: request_path_and_query.clone(), origin_identity: format!("{}\0{}", settings.publisher.origin_url, origin_host_header), @@ -5284,14 +5393,27 @@ pub async fn handle_publisher_request( }); let mut template_cache_response_state = matches!(assembly_mode, AssemblyMode::Esi) .then_some(TemplateCacheResponseState::BypassRequest); - rewrite_origin_request(&mut req, target_uri, &origin_host_header)?; let request_method = req.method().clone(); - let mut origin_request = Some(req); + // Ad-serving documents bypass the platform cache. Every other publisher + // request keeps the platform default. + let origin_cache_policy = if should_run_ad_stack { + OutboundCachePolicy::Bypass + } else { + OutboundCachePolicy::PlatformDefault + }; + let mut origin_request = Some(publisher_origin_request( + req, + target_uri, + host_authority_override, + origin_cache_policy, + origin, + )?); + let origin_client = services.outbound_client(); let eid_cookie_may_need_persistence = cookie_jar .as_ref() .is_some_and(|jar| jar.get(COOKIE_TS_EIDS).is_some() || jar.get(COOKIE_SHAREDID).is_some()); - // This decision also gates the concurrent origin send below. A marker skip + // This decision also gates the overlapped origin request below. A marker skip // cannot currently delay the origin behind an auction because marker // validation requires a non-empty registry, and every such auction sets // `auction_needs_row` and retains the preload. @@ -5309,44 +5431,33 @@ pub async fn handle_publisher_request( privacy_needs_row: crate::ec::consent::ec_consent_withdrawn(&consent_context), }); let mut pending_origin = None; - // A shared-template hit skips the origin entirely, and an in-flight request - // cannot be recalled — so when this request is eligible for one, the origin + // The overlap hides the EC snapshot read behind the origin fetch. A + // shared-template hit skips the origin entirely, and an in-flight request + // cannot be recalled, so when this request is eligible for one, the origin // waits for the lookup below. Starting it here would make every hit pay for - // a fetch whose response is discarded. Requests with no template cache key — - // every request in the default `Inline` assembly mode — keep the overlap. - if should_preload_ec - && template_cache_key.is_none() - && services.http_client().supports_concurrent_fanout() - && services - .http_client() - .supports_pending_streaming_responses() - { - let origin_req = origin_request.take().ok_or_else(|| { + // a fetch whose response is discarded. Requests with no template cache key, + // every request in the default `Inline` assembly mode included, keep the + // overlap. + if should_preload_ec && template_cache_key.is_none() { + let request = origin_request.take().ok_or_else(|| { Report::new(TrustedServerError::Proxy { message: "publisher origin request was already consumed".to_owned(), }) })?; - let mut platform_request = - PlatformHttpRequest::new(origin_req, backend_name.clone()).with_stream_response(); - // Apply the opt-in shareability policy, or preserve the existing ad-stack - // bypass policy when readthrough is disabled. - platform_request = apply_origin_cache_intent( - platform_request, - origin_readthrough_enabled, - origin_response_is_shareable, - should_run_ad_stack, - request_is_document, - &readthrough_reader_url, - ); - pending_origin = Some( - services - .http_client() - .send_async(platform_request) - .await - .change_context(TrustedServerError::Proxy { - message: "Failed to start publisher origin request".to_string(), - })?, - ); + // `begin` captures the scope when it is called, so the call is made + // inside `run` and the origin body joins the response-body scope. + let started = owner + .run(async { origin_client.begin(request).await }) + .await; + pending_origin = Some(started.map_err(|error| { + outbound_error( + error, + TrustedServerError::Proxy { + message: "Failed to start publisher origin request".to_owned(), + }, + OutboundRequestSource::ClientForward, + ) + })?); } if should_preload_ec && let (Some(graph), Some(active_ec_id)) = (kv, active_ec_id) { let refreshed = graph.load_snapshot(active_ec_id); @@ -5460,13 +5571,17 @@ pub async fn handle_publisher_request( settings, request: &auction_client_request, timeout_ms: auction_timeout_ms, - transport_timeout_ms: auction_timeout_ms, provider_responses: None, services, }; - match auction - .orchestrator - .dispatch_auction(&auction_request, &auction_context) + // The batch the auction starts captures the ambient scope, so it runs + // inside the response-body owner and lives until egress ends. + match owner + .run( + auction + .orchestrator + .dispatch_auction(&auction_request, &auction_context), + ) .await { DispatchAuctionOutcome::Dispatched(dispatched) => { @@ -5640,11 +5755,14 @@ pub async fn handle_publisher_request( params.dispatched_auction = dispatched_auction.take(); params.auction_observation = auction_observation.take(); params.auction_request = auction_request_for_telemetry.clone(); - return Ok(PublisherResponse::AssembleTemplate { - response, - template: entry.body, - params: Box::new(params), - }); + return Ok(( + PublisherResponse::AssembleTemplate { + response, + template: entry.body, + params: Box::new(params), + }, + owner, + )); } } Ok(crate::platform::TemplateCacheLookup::Reserved(reservation)) => { @@ -5673,32 +5791,20 @@ pub async fn handle_publisher_request( // Provider operations are already running while the origin loads. Collection // retains buffered results, then waits only for the original remaining budget. let origin_result = if let Some(pending) = pending_origin { - services.http_client().wait(pending).await + pending.collect().await } else { - let origin_req = origin_request.take().ok_or_else(|| { + let request = origin_request.take().ok_or_else(|| { Report::new(TrustedServerError::Proxy { message: "publisher origin request was already consumed".to_owned(), }) })?; - let mut platform_request = PlatformHttpRequest::new(origin_req, backend_name); - if services.http_client().supports_streaming_responses() { - platform_request = platform_request.with_stream_response(); - } - // Apply the opt-in shareability policy, or preserve the existing ad-stack - // bypass policy when readthrough is disabled. - platform_request = apply_origin_cache_intent( - platform_request, - origin_readthrough_enabled, - origin_response_is_shareable, - should_run_ad_stack, - request_is_document, - &readthrough_reader_url, - ); - services.http_client().send(platform_request).await + // `send` captures the scope when it is called, so the call is made + // inside `run` and the origin body joins the response-body scope. + owner.run(async { origin_client.send(request).await }).await }; - let mut response = match origin_result { - Ok(platform_response) => platform_response.response, - Err(err) => { + let mut response = match origin_result.and_then(OutboundResponse::into_response) { + Ok(response) => response, + Err(error) => { if let Some(dispatched) = dispatched_auction.take() { emit_abandoned_auction( services, @@ -5708,9 +5814,7 @@ pub async fn handle_publisher_request( ) .await; } - return Err(err.change_context(TrustedServerError::Proxy { - message: "Failed to proxy request to origin".to_string(), - })); + return Err(origin_fetch_error(error)); } }; @@ -5747,7 +5851,7 @@ pub async fn handle_publisher_request( message: "failed to build unexpected origin 304 response".to_string(), })?; enforce_terminal_private_cache_privacy(&mut response); - return Ok(PublisherResponse::Buffered(response)); + return Ok((PublisherResponse::Buffered(response), owner)); } // Map origin URLs in navigation, preload, and CSP headers to the serving host, @@ -5935,7 +6039,7 @@ pub async fn handle_publisher_request( settings, &request_path, &request_method, - edge_header, + origin.edge_header, &mut response, )?; @@ -5973,7 +6077,7 @@ pub async fn handle_publisher_request( set_response_compression(&mut response, reader_compression); } - match route { + let publisher_response = match route { ResponseRoute::PassThrough => { log::debug!( "Pass-through binary response - Content-Type: '{}', status: {}", @@ -6000,7 +6104,7 @@ pub async fn handle_publisher_request( } let (parts, body) = response.into_parts(); let response = Response::from_parts(parts, EdgeBody::empty()); - Ok(PublisherResponse::PassThrough { response, body }) + PublisherResponse::PassThrough { response, body } } ResponseRoute::BufferedUnmodified => { // Unsupported or unprocessable responses must bypass rewriting @@ -6041,7 +6145,7 @@ pub async fn handle_publisher_request( ) .await; } - Ok(PublisherResponse::Buffered(response)) + PublisherResponse::Buffered(response) } ResponseRoute::Stream => { log::debug!( @@ -6069,12 +6173,13 @@ pub async fn handle_publisher_request( guard.disarm(); } - Ok(PublisherResponse::Stream { + PublisherResponse::Stream { response, body, params: Box::new(OwnedProcessResponseParams { response_status: status, range_associated: false, + input_cap: origin.input_cap, // The transform writes here; the post-transform gate reads it. Always // present on the live path so no future caller has to remember to // supply it — the handlers themselves stay gated on authorization. @@ -6098,9 +6203,10 @@ pub async fn handle_publisher_request( gpt_diagnostics: Some(gpt_diagnostics), trace_bootstrap, }), - }) + } } - } + }; + Ok((publisher_response, owner)) } /// Bundle of the per-request creative-opportunity inputs that travel together. @@ -7882,7 +7988,6 @@ pub async fn handle_page_bids( settings, request: &req, timeout_ms, - transport_timeout_ms: timeout_ms, provider_responses: None, services, }; @@ -8060,10 +8165,168 @@ mod tests { use super::*; use crate::auction::orchestrator::OrchestrationResult; - use crate::auction::provider::{AuctionProvider, ProviderRequestOutcome}; use crate::auction::types::AuctionResponse; use crate::creative_opportunities::{CreativeOpportunityFormat, CreativeOpportunitySlot}; - use crate::platform::PlatformCacheIntent; + use crate::platform::test_support::build_services_with_outbound_client; + use crate::test_support::scripted_client::{ + RecordedRequest, ScriptedHttpClient, ScriptedReply, TestClock, + }; + + /// The input cap Axum passes with the default 16 MiB `max_buffered_body_bytes`, for + /// tests that buffer a response as the buffered adapters do. + const TEST_BUFFERED_INPUT_CAP: Option = Some(16 * 1024 * 1024); + + /// Host of the publisher origin in [`create_test_settings`]. + fn test_origin_host() -> String { + create_test_settings().publisher.origin_host() + } + + /// The scripted upstream a publisher test sends its origin request and bids through. + /// + /// The origin answers its requests in the order the test queued answers with + /// [`PublisherUpstream::queue_origin`], whatever their path, as an origin + /// serving the page under test. Bidders are scripted on + /// [`PublisherUpstream::client`] by method and URI. Clones share the script. + #[derive(Clone)] + struct PublisherUpstream { + clock: TestClock, + client: ScriptedHttpClient, + origin_host: String, + origin_replies: Arc>>, + observer: Arc>>, + } + + /// A test callback that sees each request the upstream records. + type UpstreamObserver = Arc; + + impl PublisherUpstream { + /// An upstream whose origin is the [`create_test_settings`] origin. + fn new() -> Self { + Self::for_origin(&test_origin_host()) + } + + /// An upstream whose origin answers requests to `origin_host`. + fn for_origin(origin_host: &str) -> Self { + let origin_host = origin_host.to_owned(); + let answered_host = origin_host.clone(); + let clock = TestClock::new(); + let script = ScriptedHttpClient::new(&clock); + let rearm = script.clone(); + let origin_replies: Arc>> = + Arc::default(); + let queued = Arc::clone(&origin_replies); + let observer: Arc>> = Arc::default(); + let observed = Arc::clone(&observer); + // Each origin request takes the next queued answer as it is recorded, + // before the script looks its reply up. + let client = script.with_request_observer(move |request| { + if request.uri.host() == Some(answered_host.as_str()) { + let next = queued + .lock() + .expect("should lock the queued origin replies") + .pop_front(); + if let Some(reply) = next { + rearm.respond_to(request.method.clone(), &request.uri.to_string(), reply); + } + } + let observer = observed + .lock() + .expect("should lock the upstream observer") + .clone(); + if let Some(observer) = observer { + observer(request); + } + }); + Self { + clock, + client, + origin_host, + origin_replies, + observer, + } + } + + /// Calls `observer` with each request as the upstream records it. + fn observe(&self, observer: impl Fn(&RecordedRequest) + Send + Sync + 'static) { + *self + .observer + .lock() + .expect("should lock the upstream observer") = Some(Arc::new(observer)); + } + + /// Queues the origin's answer to its next request. + fn queue_origin( + &self, + status: u16, + body: impl Into, + headers: Vec<(&str, &str)>, + ) { + let mut reply = ScriptedReply::status( + StatusCode::from_u16(status).expect("should script a valid origin status"), + ); + for (name, value) in headers { + reply = reply.header(name, value); + } + self.queue_origin_reply(reply.body(body)); + } + + /// Queues `reply` as the origin's answer to its next request. + fn queue_origin_reply(&self, reply: ScriptedReply) { + self.origin_replies + .lock() + .expect("should lock the queued origin replies") + .push_back(reply); + } + + /// Every request the origin received, in arrival order. + fn origin_requests(&self) -> Vec { + self.client + .requests() + .into_iter() + .filter(|request| request.uri.host() == Some(self.origin_host.as_str())) + .collect() + } + + /// The URI of every request the origin received. + fn origin_uris(&self) -> Vec { + self.origin_requests() + .iter() + .map(|request| request.uri.to_string()) + .collect() + } + + /// The cache policy of every request the origin received. + fn origin_cache_policies(&self) -> Vec { + self.origin_requests() + .iter() + .map(|request| request.cache_policy) + .collect() + } + + /// The headers of every request the origin received, as text pairs. + fn origin_headers(&self) -> Vec> { + self.origin_requests() + .iter() + .map(|request| { + request + .headers + .iter() + .filter_map(|(name, value)| { + value + .to_str() + .ok() + .map(|value| (name.as_str().to_owned(), value.to_owned())) + }) + .collect() + }) + .collect() + } + + /// Services whose outbound requests go to this upstream, on its clock. + fn services(&self) -> RuntimeServices { + build_services_with_outbound_client(&self.client, &self.clock) + } + } /// Every shared condition passing, as the base for single-condition negations. fn all_shareable() -> SharedRequestInputs { @@ -8150,7 +8413,9 @@ mod tests { edgezero_core::ingress::HeaderFidelity::default(), ); request.extensions_mut().insert(ingress); - crate::forwarder::prepare_trusted_forwarder(&mut request, &settings); + crate::test_support::tests::with_request_parts(&mut request, |parts| { + crate::forwarder::prepare_trusted_forwarder(parts, &settings) + }); let mut snapshot = request_head_snapshot(&request); assert!( snapshot @@ -8205,7 +8470,9 @@ mod tests { snapshot .headers_mut() .insert("x-forwarded-proto", http::HeaderValue::from_static("https")); - crate::forwarder::prepare_trusted_forwarder(&mut snapshot, &settings); + crate::test_support::tests::with_request_parts(&mut snapshot, |parts| { + crate::forwarder::prepare_trusted_forwarder(parts, &settings); + }); let info = crate::http_util::RequestInfo::from_request( &snapshot, &crate::platform::ClientInfo::default(), @@ -8372,7 +8639,7 @@ mod tests { assert_eq!(snapshot.uri(), request.uri()); assert_eq!(snapshot.headers(), request.headers()); assert!( - matches!(snapshot.body(), EdgeBody::Once(bytes) if bytes.is_empty()), + snapshot.body().as_bytes().is_some_and(<[u8]>::is_empty), "snapshot should never duplicate the request body" ); } @@ -8439,103 +8706,40 @@ mod tests { use crate::ec::kv_backend::test_support::InMemoryEcKv; use crate::ec::kv_backend::{EcKvLookup, EcKvStore, EcKvWrite, EcKvWriteOutcome}; use crate::integrations::IntegrationRegistry; - use crate::platform::test_support::{ - NoopSecretStore, StubHttpClient, build_services_with_http_client, - build_services_with_secret_http_client_and_client_ip, noop_services, - noop_services_with_telemetry_sink, - }; + use crate::platform::test_support::{noop_services, noop_services_with_telemetry_sink}; use crate::test_support::tests::{crate_test_settings_str, create_test_settings}; use edgezero_core::body::Body as EdgeBody; use http::{Method, Request as HttpRequest, StatusCode, header}; use std::sync::Arc; - /// [`EcKvStore`] that records how many HTTP calls the shared stub client had - /// made at the moment of each identity-graph lookup. This exposes the - /// interleaving between origin dispatch and the EC KV read so scheduling - /// tests can prove origin starts before the lookup only for concurrent - /// clients. + /// [`EcKvStore`] that records how many origin requests the upstream had + /// received at the moment of each identity-graph lookup. This exposes the + /// interleaving between origin dispatch and the EC KV read, so scheduling + /// tests can prove the overlapped origin request starts before the lookup. struct OrderRecordingKv { inner: InMemoryEcKv, - http: Arc, + http: PublisherUpstream, http_calls_at_lookup: Arc, lookups: Arc, } const SCHEDULING_PROVIDER: &str = "scheduling-capture"; - #[derive(Debug)] - struct CapturedSchedulingAuction { - request: AuctionRequest, - client_uri: String, - client_host: Option, - client_fastly_ssl: Option, - client_user_agent: Option, - http_calls_at_dispatch: usize, - lookups_at_dispatch: usize, - } - - struct SchedulingCaptureProvider { - captured: Arc>>, - http: Arc, - lookups: Arc, - } - - #[async_trait::async_trait(?Send)] - impl AuctionProvider for SchedulingCaptureProvider { - fn provider_name(&self) -> &'static str { - SCHEDULING_PROVIDER - } - - async fn request_bids( - &self, - request: &AuctionRequest, - context: &AuctionContext<'_>, - ) -> Result> { - let header_value = |name: &str| { - context - .request - .headers() - .get(name) - .and_then(|value| value.to_str().ok()) - .map(str::to_owned) - }; - *self.captured.lock().expect("should lock captured auction") = - Some(CapturedSchedulingAuction { - request: request.clone(), - client_uri: context.request.uri().to_string(), - client_host: header_value(header::HOST.as_str()), - client_fastly_ssl: header_value("fastly-ssl"), - client_user_agent: header_value(header::USER_AGENT.as_str()), - http_calls_at_dispatch: self.http.recorded_backend_names().len(), - lookups_at_dispatch: self.lookups.load(Ordering::SeqCst), - }); - Err(Report::new(TrustedServerError::Auction { - message: "scheduling capture only".to_owned(), - })) - } - - async fn parse_response( - &self, - _response: crate::platform::PlatformResponse, - _response_time_ms: u64, - ) -> Result> { - panic!("parse_response must not run when scheduling capture fails") - } - - fn timeout_ms(&self) -> u32 { - 100 - } + /// Endpoint of the [`SCHEDULING_PROVIDER`] that `legacy_provider_map` configures. + const SCHEDULING_ENDPOINT: &str = "https://scheduling-capture.example/openrtb2/auction"; - fn backend_name(&self, _services: &RuntimeServices, _timeout_ms: u32) -> Option { - Some("scheduling-capture-backend".to_owned()) - } + /// What had happened when the scheduling provider's bid request left. + struct BidderDispatch { + body: bytes::Bytes, + origin_calls: usize, + lookups: usize, } struct SchedulingProbeOutcome { lookups: usize, http_calls_at_lookup: usize, - stream_flags: Vec, - cache_intents: Vec, + origin_modes: Vec, + cache_policies: Vec, body_is_stream: bool, request_rewritten: bool, response_is_private: bool, @@ -8552,7 +8756,7 @@ mod tests { fn lookup(&self, _key: &str) -> Result, Report> { self.lookups.fetch_add(1, Ordering::SeqCst); self.http_calls_at_lookup - .store(self.http.recorded_backend_names().len(), Ordering::SeqCst); + .store(self.http.origin_requests().len(), Ordering::SeqCst); // Report a miss: the scheduling assertions only care about ordering. Ok(None) } @@ -8646,37 +8850,48 @@ mod tests { } } - /// Drives one EC-capable navigation and captures the complete pending-origin + /// Drives one EC-capable navigation and captures the overlapped origin /// ordering plus the origin and auction views of the request. - async fn run_scheduling_probe( - concurrent_fanout: bool, - streaming_responses: bool, - pending_streaming_responses: bool, - queue_origin: bool, - ) -> SchedulingProbeOutcome { + async fn run_scheduling_probe(request: Request) -> SchedulingProbeOutcome { let settings = scheduling_settings(); - let http = Arc::new(StubHttpClient::new()); - http.set_concurrent_fanout(concurrent_fanout); - http.set_streaming_responses_supported(streaming_responses); - http.set_pending_streaming_responses_supported(pending_streaming_responses); - if queue_origin { - http.push_response_with_headers( - 200, - b"ok".to_vec(), - vec![("content-type", "text/html; charset=utf-8")], - ); - } + let http = PublisherUpstream::new(); + http.queue_origin( + 200, + b"ok".to_vec(), + vec![("content-type", "text/html; charset=utf-8")], + ); let lookups = Arc::new(AtomicUsize::new(0)); let http_calls_at_lookup = Arc::new(AtomicUsize::new(0)); let graph = KvIdentityGraph::new(OrderRecordingKv { inner: InMemoryEcKv::new("order-store"), - http: Arc::clone(&http), + http: http.clone(), http_calls_at_lookup: Arc::clone(&http_calls_at_lookup), lookups: Arc::clone(&lookups), }); - let services = build_services_with_http_client( - Arc::clone(&http) as Arc + let bidder_dispatch = Arc::new(Mutex::new(None)); + { + let upstream = http.clone(); + let lookups = Arc::clone(&lookups); + let bidder_dispatch = Arc::clone(&bidder_dispatch); + http.observe(move |request| { + if request.uri != SCHEDULING_ENDPOINT { + return; + } + *bidder_dispatch + .lock() + .expect("should lock the bidder dispatch") = Some(BidderDispatch { + body: request.body.clone(), + origin_calls: upstream.origin_requests().len(), + lookups: lookups.load(Ordering::SeqCst), + }); + }); + } + http.client.respond_to( + Method::POST, + SCHEDULING_ENDPOINT, + ScriptedReply::status(StatusCode::NO_CONTENT), ); + let services = http.services(); let ec_id = format!("{}.CkId01", "b".repeat(64)); let mut ec_context = EcContext::new_for_test_with_ip( Some(ec_id), @@ -8687,14 +8902,8 @@ mod tests { ec_context.ec_allowed() && ec_context.ec_value().is_some(), "test precondition: an active, consent-allowed EC must exist" ); - let orchestrator = AuctionOrchestrator::new(settings.auction.clone()); - let captured_auction = Arc::new(Mutex::new(None)); - let mut orchestrator = orchestrator; - orchestrator.register_provider(Arc::new(SchedulingCaptureProvider { - captured: Arc::clone(&captured_auction), - http: Arc::clone(&http), - lookups: Arc::clone(&lookups), - })); + let orchestrator = crate::auction::build_orchestrator(&settings) + .expect("should build the auction orchestrator"); let slots = [scheduling_slot()]; let result = handle_publisher_request( @@ -8708,10 +8917,18 @@ mod tests { slots: &slots, registry: None, }, - navigation_request(), - EdgeCacheHeader::SurrogateControl, + request, + PublisherOriginOptions { + edge_header: EdgeCacheHeader::SurrogateControl, + input_cap: None, + response_decoding: ResponseDecoding::Original, + }, ) .await; + let (result, _owner) = match result { + Ok((response, owner)) => (Ok(response), Some(owner)), + Err(error) => (Err(error), None), + }; let ok = result.is_ok(); let body_is_stream = result.as_ref().ok().is_some_and(|response| match response { @@ -8732,9 +8949,13 @@ mod tests { .and_then(|value| value.to_str().ok()) == Some(crate::cache_policy::NO_STORE_PRIVATE_CACHE_CONTROL) }); - let recorded_headers = http.recorded_request_headers(); - let request_rewritten = http.recorded_request_uris() + let recorded_headers = http.origin_headers(); + let request_rewritten = http.origin_uris() == vec!["https://origin.test-publisher.com/article"] + && http + .origin_requests() + .first() + .is_some_and(|request| request.host_authority == test_origin_host()) && recorded_headers.first().is_some_and(|headers| { let header_value = |name: &str| { headers @@ -8742,31 +8963,26 @@ mod tests { .find(|(header_name, _)| header_name.eq_ignore_ascii_case(name)) .map(|(_, value)| value.as_str()) }; - header_value(header::HOST.as_str()) == Some("origin.test-publisher.com") + header_value(header::HOST.as_str()).is_none() && header_value("fastly-ssl").is_none() && header_value(header::IF_NONE_MATCH.as_str()).is_none() && header_value(header::RANGE.as_str()).is_none() }); - let captured_auction = captured_auction + let bidder_dispatch = bidder_dispatch .lock() - .expect("should lock captured auction") + .expect("should lock the bidder dispatch") .take(); - let auction_preserved_client_snapshot = captured_auction.as_ref().is_some_and(|captured| { - captured.client_uri == "https://publisher.example/article" - && captured.client_host.as_deref() == Some("publisher.example") - && captured.client_fastly_ssl.as_deref() == Some("1") - && captured.client_user_agent.as_deref() == Some("scheduling-browser") - && captured - .request - .device - .as_ref() - .and_then(|device| device.user_agent.as_deref()) - == Some("scheduling-browser") - }); - let origin_kv_auction_order = captured_auction.as_ref().is_some_and(|captured| { - captured.http_calls_at_dispatch == 1 && captured.lookups_at_dispatch == 1 + let article_page = format!("https://{}/article", settings.publisher.domain); + let auction_preserved_client_snapshot = bidder_dispatch.as_ref().is_some_and(|dispatch| { + let auction: serde_json::Value = + serde_json::from_slice(&dispatch.body).expect("should send a JSON bid request"); + auction["device"]["ua"] == "scheduling-browser" + && auction["site"]["page"] == article_page }); + let origin_kv_auction_order = bidder_dispatch + .as_ref() + .is_some_and(|dispatch| dispatch.origin_calls == 1 && dispatch.lookups == 1); let datadome_tag_suppressed = if let Ok(response) = result { let registry = IntegrationRegistry::new(&settings) .expect("should create integration registry with DataDome"); @@ -8777,6 +8993,7 @@ mod tests { ®istry, &orchestrator, &services, + TEST_BUFFERED_INPUT_CAP, ) .await .ok() @@ -8792,8 +9009,12 @@ mod tests { SchedulingProbeOutcome { lookups: lookups.load(Ordering::SeqCst), http_calls_at_lookup: http_calls_at_lookup.load(Ordering::SeqCst), - stream_flags: http.recorded_stream_response_flags(), - cache_intents: http.recorded_cache_intents(), + origin_modes: http + .origin_requests() + .iter() + .map(|request| request.mode) + .collect(), + cache_policies: http.origin_cache_policies(), body_is_stream, request_rewritten, response_is_private, @@ -8845,19 +9066,16 @@ mod tests { settings.ec.partners = vec![marker_partner("ssp.example.com")]; let registry = PartnerRegistry::from_config(&settings.ec.partners) .expect("should build pull partner registry"); - let http = Arc::new(StubHttpClient::new()); - http.set_concurrent_fanout(true); - http.push_response(200, b"ok".to_vec()); + let http = PublisherUpstream::new(); + http.queue_origin(200, b"ok".to_vec(), Vec::new()); let lookups = Arc::new(AtomicUsize::new(0)); let graph = KvIdentityGraph::new(OrderRecordingKv { inner: InMemoryEcKv::new("marker-store"), - http: Arc::clone(&http), + http: http.clone(), http_calls_at_lookup: Arc::new(AtomicUsize::new(0)), lookups: Arc::clone(&lookups), }); - let services = build_services_with_http_client( - Arc::clone(&http) as Arc - ); + let services = http.services(); let ec_id = format!("{}.CkId01", "b".repeat(64)); let marker = match probe { MarkerProbe::Valid => { @@ -8896,7 +9114,8 @@ mod tests { ec_context.set_pull_sync_marker_for_test( crate::ec::pull_sync_marker::PullSyncMarkerState::from_cookie(Some(marker)), ); - let orchestrator = AuctionOrchestrator::new(settings.auction.clone()); + let orchestrator = crate::auction::build_orchestrator(&settings) + .expect("should build the auction orchestrator"); let slots = has_matched_auction_slot .then(scheduling_slot) .into_iter() @@ -8920,7 +9139,11 @@ mod tests { registry: Some(®istry), }, request, - EdgeCacheHeader::SurrogateControl, + PublisherOriginOptions { + edge_header: EdgeCacheHeader::SurrogateControl, + input_cap: None, + response_decoding: ResponseDecoding::Original, + }, ) .await; @@ -8969,36 +9192,40 @@ mod tests { } #[tokio::test] - async fn concurrent_client_starts_origin_before_ec_lookup() { - let outcome = run_scheduling_probe(true, true, true, true).await; + async fn overlapped_navigation_starts_origin_before_ec_lookup() { + let outcome = run_scheduling_probe(navigation_request()).await; assert!(outcome.ok, "should proxy the origin response"); assert_eq!(outcome.lookups, 1, "should perform exactly one EC lookup"); assert_eq!( outcome.http_calls_at_lookup, 1, - "a concurrent client must start the origin before its EC KV lookup" + "the overlapped path must start the origin before its EC KV lookup" + ); + assert_eq!( + outcome.origin_modes, + vec![edgezero_core::outbound::OutboundSendMode::Begin], + "the overlapped path should begin the origin request" ); - assert_eq!(outcome.stream_flags, vec![true]); assert_eq!( - outcome.cache_intents, - vec![PlatformCacheIntent::Bypass], - "pending publisher origin request should bypass platform caching" + outcome.cache_policies, + vec![OutboundCachePolicy::Bypass], + "the overlapped publisher origin request should bypass platform caching" ); assert!( outcome.body_is_stream, - "pending origin wait should preserve the publisher body stream" + "the collected origin response should keep the publisher body stream" ); assert!( outcome.request_rewritten, - "pending origin path should preserve rewriting and header filtering" + "the overlapped path should retarget the request and filter its headers" ); assert!( outcome.response_is_private, - "DataDome-suppressed pending HTML should remain private" + "DataDome-suppressed overlapped HTML should remain private" ); assert!( outcome.origin_kv_auction_order, - "pending origin should start before KV lookup and auction dispatch" + "the origin should start before the KV lookup and the auction dispatch" ); assert!( outcome.auction_preserved_client_snapshot, @@ -9011,56 +9238,13 @@ mod tests { } #[tokio::test] - async fn eager_client_reads_ec_before_starting_origin() { - let outcome = run_scheduling_probe(false, false, false, true).await; - - assert!(outcome.ok, "should proxy the origin response"); - assert_eq!(outcome.lookups, 1, "should perform exactly one EC lookup"); - assert_eq!( - outcome.http_calls_at_lookup, 0, - "an eager client must not start the origin before its EC KV lookup" - ); - assert_eq!(outcome.stream_flags, vec![false]); - } + async fn overlapped_origin_start_failure_skips_ec_and_auction_work() { + // A navigation GET that carries body bytes cannot be forwarded, so + // `begin` refuses it before the EC lookup or the auction runs. + let mut request = navigation_request(); + *request.body_mut() = EdgeBody::from("example GET body"); - #[tokio::test] - async fn concurrent_send_streaming_without_pending_streaming_uses_eager_fallback() { - let outcome = run_scheduling_probe(true, true, false, true).await; - - assert!(outcome.ok, "should proxy through the eager send fallback"); - assert_eq!(outcome.lookups, 1, "should perform exactly one EC lookup"); - assert_eq!( - outcome.http_calls_at_lookup, 0, - "client without pending streaming must preload EC before eager send" - ); - assert_eq!(outcome.stream_flags, vec![true]); - assert!( - outcome.body_is_stream, - "eager fallback should retain streamed body" - ); - } - - #[tokio::test] - async fn pending_streaming_does_not_require_eager_send_streaming() { - let outcome = run_scheduling_probe(true, false, true, true).await; - - assert!(outcome.ok, "should proxy through the pending origin path"); - assert_eq!(outcome.lookups, 1, "should perform exactly one EC lookup"); - assert_eq!( - outcome.http_calls_at_lookup, 1, - "pending-stream support should start origin before the EC lookup" - ); - assert_eq!(outcome.stream_flags, vec![true]); - assert!( - outcome.body_is_stream, - "pending-stream support should retain the streamed body independently of eager send" - ); - } - - #[tokio::test] - async fn concurrent_origin_start_failure_skips_ec_and_auction_work() { - // No origin response is queued, so the concurrent `send_async` fails. - let outcome = run_scheduling_probe(true, true, true, false).await; + let outcome = run_scheduling_probe(request).await; assert!( !outcome.ok, @@ -9070,6 +9254,10 @@ mod tests { outcome.lookups, 0, "origin-start failure must occur before any EC KV work" ); + assert!( + !outcome.origin_kv_auction_order, + "origin-start failure must occur before any auction dispatch" + ); } fn make_test_bid_with_creative(creative: &str) -> Bid { @@ -9930,6 +10118,7 @@ mod tests { OwnedProcessResponseParams { response_status: StatusCode::OK, range_associated: false, + input_cap: None, csp_nonce_observed: None, template_cache_key: None, seam_ad_slots: None, @@ -10149,12 +10338,9 @@ mod tests { use std::net::{IpAddr, Ipv4Addr}; use chrono::TimeZone as _; - use edgezero_core::router::PreDispatchHook as _; use futures::executor::block_on; - use crate::platform::test_support::{ - StubHttpClient, build_services_with_http_client_and_client_ip, - }; + use crate::test_support::tests::run_pre_dispatch_hook; use crate::trace::{TracePreDispatchHook, TraceTerminalResponse}; fn gate_and_decision( @@ -10163,17 +10349,13 @@ mod tests { TraceCaptureGate, crate::integrations::gpt_diagnostics::GptDiagnosticsRequestDecision, ) { - let mut request = request("", Some("__Host-ts-console=1"), "document"); + let request = request("", Some("__Host-ts-console=1"), "document"); let hook = TracePreDispatchHook::new( Arc::new(settings.clone()), Arc::new(|_| panic!("should not query setup metadata for a publisher document")), ); - assert!( - block_on(hook.handle(&mut request)) - .expect("should freeze document gate") - .is_none(), - "should continue publisher routing" - ); + let mut request = block_on(run_pre_dispatch_hook(hook, request)) + .expect("should continue publisher routing"); let decision = crate::integrations::gpt_diagnostics::prepare_request(settings, &mut request) .expect("should prepare active navigation decision"); @@ -10245,28 +10427,27 @@ mod tests { fn render( settings: &Settings, - mut request: Request, + request: Request, content_type: &str, ) -> (Response, String) { - let stub = Arc::new(StubHttpClient::new()); - stub.push_response_with_headers(200, b"Examplepublisher ads remain".to_vec(), + let stub = PublisherUpstream::new(); + stub.queue_origin(200, b"Examplepublisher ads remain".to_vec(), vec![("content-type",content_type),("cache-control","public, max-age=300"),("etag","\"origin-example\"")]); - let services = build_services_with_http_client_and_client_ip( - stub, - IpAddr::V4(Ipv4Addr::new(192, 0, 2, 99)), - ); + let services = RuntimeServices { + client_info: ClientInfo { + client_ip: Some(IpAddr::V4(Ipv4Addr::new(192, 0, 2, 99))), + ..ClientInfo::default() + }, + ..stub.services() + }; let hook = TracePreDispatchHook::new( Arc::new(settings.clone()), Arc::new(|_| { panic!("should not request setup metadata on ordinary publisher traffic") }), ); - assert!( - block_on(hook.handle(&mut request)) - .expect("should freeze ordinary request gate") - .is_none(), - "should continue ordinary publisher routing" - ); + let mut request = block_on(run_pre_dispatch_hook(hook, request)) + .expect("should continue ordinary publisher routing"); crate::integrations::gpt_diagnostics::prepare_request(settings, &mut request) .expect("should prepare effective diagnostics decision"); assert!( @@ -10279,8 +10460,9 @@ mod tests { .any(|part| part == b"__Host-ts-console")), "should exercise sanitized cookies rather than re-reading the original session" ); - let publisher = block_on(run_publisher_proxy(settings, &services, request)); - let orchestrator = AuctionOrchestrator::new(settings.auction.clone()); + let (publisher, _owner) = block_on(run_publisher_proxy(settings, &services, request)); + let orchestrator = crate::auction::build_orchestrator(settings) + .expect("should build the auction orchestrator"); let registry = IntegrationRegistry::new(settings).expect("should build publisher registry"); let response = block_on(buffer_publisher_response_async( @@ -10290,6 +10472,7 @@ mod tests { ®istry, &orchestrator, &services, + TEST_BUFFERED_INPUT_CAP, )) .expect("should finalize publisher response"); let (parts, body) = response.into_parts(); @@ -10517,7 +10700,8 @@ mod tests { Arc::new(settings), Arc::new(|_| panic!("should not fetch setup metadata for a document")), ); - block_on(hook.handle(&mut request)).expect("should capture document gate"); + let request = block_on(run_pre_dispatch_hook(hook, request)) + .expect("should capture document gate"); let gate = request .extensions() .get::() @@ -10610,19 +10794,17 @@ mod tests { params.trace_bootstrap = Some(bootstrap); let registry = IntegrationRegistry::new(&settings).expect("should build publisher registry"); - let orchestrator = AuctionOrchestrator::new(settings.auction.clone()); - let body = EdgeBody::Stream( - futures::stream::iter([ - Ok(bytes::Bytes::from_static(b"Examplepublisher ads remain", - )), - ]) - .boxed_local(), - ); + let orchestrator = crate::auction::build_orchestrator(&settings) + .expect("should build the auction orchestrator"); + let body = EdgeBody::from_stream(futures::stream::iter([ + Ok(bytes::Bytes::from_static(b"Examplepublisher ads remain", + )), + ])); let mut output = Vec::new(); block_on(stream_publisher_body_async( body, @@ -10835,8 +11017,9 @@ mod tests { settings: &Settings, services: &RuntimeServices, req: Request, - ) -> PublisherResponse { - let orchestrator = AuctionOrchestrator::new(settings.auction.clone()); + ) -> (PublisherResponse, OutboundScopeOwner) { + let orchestrator = crate::auction::build_orchestrator(settings) + .expect("should build the auction orchestrator"); let mut ec_context = EcContext::read_from_request(settings, &req, services).expect("should read EC context"); handle_publisher_request( @@ -10851,7 +11034,11 @@ mod tests { registry: None, }, req, - EdgeCacheHeader::SurrogateControl, + PublisherOriginOptions { + edge_header: EdgeCacheHeader::SurrogateControl, + input_cap: None, + response_decoding: ResponseDecoding::Original, + }, ) .await .expect("should proxy publisher request") @@ -11533,14 +11720,7 @@ mod tests { use super::*; use crate::platform::ClientInfo; - use crate::platform::test_support::{ - NoopConfigStore, NoopGeo, NoopSecretStore, StubBackend, StubHttpClient, - }; use crate::test_support::tests::crate_test_settings_str; - use edgezero_core::ingress::{ - CapturedTarget, HeaderFidelity, InboundOrigin, OriginSource, RequestIngress, - TargetUnavailable, - }; use std::collections::HashMap; use std::sync::atomic::{AtomicBool, AtomicUsize, Ordering}; @@ -11802,20 +11982,6 @@ mod tests { settings } - /// Settings with the readthrough opt-in turned on. - /// - /// A separate helper rather than a default, because the flag being off by default - /// is the property that keeps this change inert until an operator asks for it. - fn settings_with_readthrough_enabled(mode: &str) -> Settings { - let mut settings = settings_with_mode(mode); - settings - .creative_opportunities - .as_mut() - .expect("settings_with_mode should configure creative opportunities") - .origin_readthrough_enabled = Some(true); - settings - } - fn settings_with_mode_and_template_cache_max_age(mode: &str, seconds: u32) -> Settings { let mut settings = settings_with_mode(mode); settings @@ -11827,27 +11993,21 @@ mod tests { } fn services( - http_client: Arc, + upstream: &PublisherUpstream, cache: Arc, ) -> RuntimeServices { - RuntimeServices::builder() - .config_store(Arc::new(NoopConfigStore)) - .secret_store(Arc::new(NoopSecretStore)) - .kv_store(Arc::new(edgezero_core::key_value_store::NoopKvStore)) - .backend(Arc::new(StubBackend)) - .http_client(http_client) - .geo(Arc::new(NoopGeo)) - .client_info(ClientInfo::default()) - .template_cache(cache) - .build() + RuntimeServices { + template_cache: cache, + ..upstream.services() + } } fn services_with_assembler( - http_client: Arc, + upstream: &PublisherUpstream, cache: Arc, assembler: Arc, ) -> RuntimeServices { - services(http_client, cache).with_template_assembler(assembler) + services(upstream, cache).with_template_assembler(assembler) } /// A template cache **and** a telemetry sink. @@ -11856,23 +12016,15 @@ mod tests { /// sink, and `services_with_telemetry` in the SSAT module sets the sink and no /// cache. Cache-outcome telemetry cannot be asserted end to end without both. fn services_with_cache_and_telemetry( - http_client: Arc, + upstream: &PublisherUpstream, cache: Arc, telemetry_sink: Arc, ) -> RuntimeServices { - let telemetry_sink: Arc = - telemetry_sink; - RuntimeServices::builder() - .config_store(Arc::new(NoopConfigStore)) - .secret_store(Arc::new(NoopSecretStore)) - .kv_store(Arc::new(edgezero_core::key_value_store::NoopKvStore)) - .backend(Arc::new(StubBackend)) - .http_client(http_client) - .geo(Arc::new(NoopGeo)) - .client_info(ClientInfo::default()) - .template_cache(cache) - .auction_telemetry_sink(telemetry_sink) - .build() + RuntimeServices { + template_cache: cache, + auction_telemetry_sink: telemetry_sink, + ..upstream.services() + } } /// The most recent `summary` row the sink recorded. @@ -11902,8 +12054,8 @@ mod tests { /// Shareable HTML: no `Set-Cookie`, no `Vary`, a public `Cache-Control`. Every /// condition the gate checks is satisfied, so a bypass here would be a bug in /// the wiring rather than in the fixture. - fn queue_shareable_html(stub: &StubHttpClient) { - stub.push_response_with_headers( + fn queue_shareable_html(stub: &PublisherUpstream) { + stub.queue_origin( 200, b"origin".to_vec(), vec![ @@ -11981,92 +12133,6 @@ mod tests { /// string that cannot appear in the fixture page by accident. const STUB_BID_CPM: f64 = 3.5; - /// An auction provider that returns one genuine winning bid. - /// - /// Every other fixture in this file leaves the orchestrator with no providers, so - /// every auction resolves to an empty bid map. That is exactly why a defect that - /// discarded *non-empty* maps survived: no test ever produced one. - struct WinningBidProvider { - price: f64, - } - - #[async_trait::async_trait(?Send)] - impl crate::auction::provider::AuctionProvider for WinningBidProvider { - fn provider_name(&self) -> &'static str { - STUB_BIDDER - } - - async fn request_bids( - &self, - _request: &AuctionRequest, - context: &crate::auction::types::AuctionContext<'_>, - ) -> Result> - { - let request = PlatformHttpRequest::new( - HttpRequest::builder() - .method(Method::POST) - .uri("https://bidder.example.com/openrtb2/auction") - .body(EdgeBody::empty()) - .expect("should build stub bid request"), - "stub-bidder-backend", - ); - context - .services - .http_client() - .send_async(request.with_cache_bypass()) - .await - .change_context(TrustedServerError::Auction { - message: "stub bid launch failed".to_string(), - }) - .map(crate::auction::provider::ProviderRequestOutcome::pending) - } - - async fn parse_response( - &self, - _response: crate::platform::PlatformResponse, - response_time_ms: u64, - ) -> Result> - { - Ok(crate::auction::types::AuctionResponse::success( - STUB_BIDDER, - vec![Bid { - slot_id: "test-slot".to_string(), - price: Some(self.price), - currency: "USD".to_string(), - creative: None, - adomain: None, - bidder: STUB_BIDDER.to_string(), - returned_seat: None, - width: 728, - height: 90, - nurl: None, - burl: None, - bid_id: None, - creative_id: None, - renderer: None, - ad_id: Some("stub-creative-1".to_string()), - cache_id: None, - cache_host: None, - cache_path: None, - metadata: Default::default(), - }], - response_time_ms, - )) - } - - fn timeout_ms(&self) -> u32 { - 2000 - } - - fn backend_name( - &self, - _services: &RuntimeServices, - _timeout_ms: u32, - ) -> Option { - Some("stub-bidder-backend".to_string()) - } - } - /// [`settings_with_mode`], with an auction provider that actually bids. fn settings_with_bidder(mode: &str) -> Settings { let mut settings = settings_with_mode(mode); @@ -12075,12 +12141,33 @@ mod tests { settings } - /// Queue the bidder's canned response. + /// Endpoint of the stub bidder that `legacy_provider_map` configures. + const STUB_BIDDER_ENDPOINT: &str = "https://stub-bidder.example/openrtb2/auction"; + + /// Queues one genuine winning bid at `price` for the stub bidder's next request. + /// /// - /// Must be pushed **before** the origin HTML: the stub client serves one shared - /// FIFO queue, and the auction is dispatched before the origin fetch. - fn queue_bid_response(stub: &StubHttpClient) { - stub.push_response(200, b"{}".to_vec()); + /// Every other fixture in this file leaves the auction without bids, so every + /// auction resolves to an empty bid map. That is exactly why a defect that + /// discarded *non-empty* maps survived: no test ever produced one. + fn queue_winning_bid(bidders: &ScriptedHttpClient, price: f64) { + bidders.respond_to( + Method::POST, + STUB_BIDDER_ENDPOINT, + ScriptedReply::status(StatusCode::OK).body( + serde_json::to_vec(&serde_json::json!({ + "seatbid": [{"seat": STUB_BIDDER, "bid": [{ + "impid": "test-slot", + "price": price, + "adid": "stub-creative-1", + "adm": "
stub creative
", + "w": 728, + "h": 90 + }]}] + })) + .expect("should serialize the stub bid"), + ), + ); } /// Which finalizer a request is driven through. @@ -12110,7 +12197,8 @@ mod tests { settings, services, request, - AuctionOrchestrator::new(settings.auction.clone()), + crate::auction::build_orchestrator(settings) + .expect("should build the auction orchestrator"), Finalizer::Streaming, ) .await @@ -12127,64 +12215,36 @@ mod tests { settings, services, request, - AuctionOrchestrator::new(settings.auction.clone()), + crate::auction::build_orchestrator(settings) + .expect("should build the auction orchestrator"), finalizer, ) .await } - /// [`run`], with a provider registered so the auction returns a real bid. - async fn run_bidding( - settings: &Arc, - services: &RuntimeServices, - request: Request, - ) -> Response { - run_bidding_at_price(settings, services, request, STUB_BID_CPM).await - } - - /// [`run_bidding`], with a distinct winning CPM for this request. - async fn run_bidding_at_price( - settings: &Arc, - services: &RuntimeServices, - request: Request, - price: f64, - ) -> Response { - let mut orchestrator = AuctionOrchestrator::new(settings.auction.clone()); - orchestrator.register_provider(Arc::new(WinningBidProvider { price })); - run_with_orchestrator( - settings, - services, - request, - orchestrator, - Finalizer::Streaming, - ) - .await - } - #[tokio::test] async fn html_finalization_preserves_a_winning_bid_before_the_structural_body_end() { let settings = Arc::new(crate::publisher::allow_eof_test_settings( settings_with_bidder("inline"), )); - let stub = Arc::new(StubHttpClient::new()); + let stub = PublisherUpstream::new(); let cache = Arc::new(MemoryTemplateCache::default()); - let services = services(Arc::clone(&stub), cache); + let services = services(&stub, cache); let registry = IntegrationRegistry::from_html_stream_processors(vec![Arc::new( super::TerminalTestFactory, )]); - let mut orchestrator = AuctionOrchestrator::new(settings.auction.clone()); - orchestrator.register_provider(Arc::new(WinningBidProvider { - price: STUB_BID_CPM, - })); + let orchestrator = crate::auction::build_orchestrator(&settings) + .expect("should build the auction orchestrator"); + let bidders = stub.client.clone(); + queue_winning_bid(&bidders, STUB_BID_CPM); let orchestrator = Arc::new(orchestrator); - queue_bid_response(&stub); queue_shareable_html(&stub); let consent = crate::consent::ConsentContext { jurisdiction: crate::consent::jurisdiction::Jurisdiction::NonRegulated, ..Default::default() }; let mut ec_context = EcContext::new_for_test(None, consent); - let response = handle_publisher_request( + let (response, _owner) = handle_publisher_request( &settings, &services, None, @@ -12196,7 +12256,11 @@ mod tests { registry: None, }, navigation_request(), - EdgeCacheHeader::SMaxageFallback, + PublisherOriginOptions { + edge_header: EdgeCacheHeader::SMaxageFallback, + input_cap: None, + response_decoding: ResponseDecoding::Original, + }, ) .await .expect("should launch the winning auction"); @@ -12253,10 +12317,10 @@ mod tests { crate::publisher::allow_eof_test_settings(settings_with_bidder("inline")); settings.auction.enabled = live; let settings = Arc::new(settings); - let stub = Arc::new(StubHttpClient::new()); + let stub = PublisherUpstream::new(); let sink = Arc::new(RecordingTelemetrySink::default()); let services = services_with_cache_and_telemetry( - Arc::clone(&stub), + &stub, Arc::new(MemoryTemplateCache::default()), Arc::clone(&sink), ); @@ -12265,13 +12329,10 @@ mod tests { IntegrationRegistry::from_html_stream_processors(vec![Arc::new( super::CountingTerminalFactory(Arc::clone(&calls)), )]); - let mut orchestrator = AuctionOrchestrator::new(settings.auction.clone()); - orchestrator.register_provider(Arc::new(WinningBidProvider { - price: STUB_BID_CPM, - })); - if live { - queue_bid_response(&stub); - } + let orchestrator = crate::auction::build_orchestrator(&settings) + .expect("should build the auction orchestrator"); + let bidders = stub.client.clone(); + queue_winning_bid(&bidders, STUB_BID_CPM); let raw = b"

Example partial article

"; let mut headers = vec![ ("content-type", "text/html"), @@ -12284,7 +12345,7 @@ mod tests { if content_range { headers.push(("content-range", "bytes 0-29/100")); } - stub.push_response_with_headers(status, raw.to_vec(), headers); + stub.queue_origin(status, raw.to_vec(), headers); let consent = crate::consent::ConsentContext { jurisdiction: crate::consent::jurisdiction::Jurisdiction::NonRegulated, ..Default::default() @@ -12296,7 +12357,7 @@ mod tests { .extensions_mut() .insert(crate::integrations::datadome::DataDomeClientTagSuppressed); } - let publisher = handle_publisher_request( + let (publisher, _owner) = handle_publisher_request( &settings, &services, None, @@ -12308,7 +12369,11 @@ mod tests { registry: None, }, request, - EdgeCacheHeader::SMaxageFallback, + PublisherOriginOptions { + edge_header: EdgeCacheHeader::SMaxageFallback, + input_cap: None, + response_decoding: ResponseDecoding::Original, + }, ) .await .expect("should classify partial HTML before synthesized policy"); @@ -12382,176 +12447,14 @@ mod tests { } } - #[derive(Default)] - struct RawOriginCacheClient { - cached: Mutex>>, - misses: AtomicUsize, - hits: AtomicUsize, - } - - #[async_trait::async_trait(?Send)] - impl crate::platform::PlatformHttpClient for RawOriginCacheClient { - async fn send( - &self, - request: crate::platform::PlatformHttpRequest, - ) -> Result> - { - assert!( - matches!( - request.cache_intent, - crate::platform::PlatformCacheIntent::Shared { .. } - ), - "should retain raw-origin cache eligibility" - ); - let mut cached = self.cached.lock().expect("should lock raw origin cache"); - let bytes = if let Some(bytes) = cached.as_ref() { - self.hits.fetch_add(1, Ordering::SeqCst); - bytes.clone() - } else { - self.misses.fetch_add(1, Ordering::SeqCst); - let bytes = - b"example raw article".to_vec(); - *cached = Some(bytes.clone()); - bytes - }; - Ok(crate::platform::PlatformResponse::new( - Response::builder() - .header(header::CONTENT_TYPE, "text/html") - .header(header::CACHE_CONTROL, "public, max-age=60") - .body(EdgeBody::from(bytes)) - .expect("should construct raw origin response"), - )) - } - - async fn send_async( - &self, - _: crate::platform::PlatformHttpRequest, - ) -> Result< - crate::platform::PlatformPendingRequest, - Report, - > { - panic!("should not dispatch an auction in the raw-origin cache fixture") - } - - async fn select( - &self, - _: Vec, - ) -> Result> - { - panic!("should not collect an auction in the raw-origin cache fixture") - } - } - - #[tokio::test] - async fn html_finalization_runs_again_on_a_raw_origin_cache_hit() { - let client = Arc::new(RawOriginCacheClient::default()); - let template_cache = Arc::new(MemoryTemplateCache::default()); - let mut settings = - crate::publisher::allow_eof_test_settings(settings_with_readthrough_enabled("esi")); - settings.auction.enabled = false; - let settings = Arc::new(settings); - let services = RuntimeServices::builder() - .config_store(Arc::new(NoopConfigStore)) - .secret_store(Arc::new(NoopSecretStore)) - .kv_store(Arc::new(edgezero_core::key_value_store::NoopKvStore)) - .backend(Arc::new(StubBackend)) - .http_client(client.clone()) - .geo(Arc::new(NoopGeo)) - .client_info(ClientInfo::default()) - .template_cache(template_cache.clone()) - .build(); - let registry = IntegrationRegistry::from_html_stream_processors(vec![Arc::new( - super::TerminalTestFactory, - )]); - let orchestrator = Arc::new(AuctionOrchestrator::new(settings.auction.clone())); - for _ in 0..2 { - let response = handle_publisher_request( - &settings, - &services, - None, - &mut EcContext::default(), - AuctionDispatch { - integration_registry: ®istry, - orchestrator: &orchestrator, - slots: &[], - registry: None, - }, - navigation_request(), - EdgeCacheHeader::SMaxageFallback, - ) - .await - .expect("should obtain raw origin HTML"); - let response = publisher_response_into_streaming_response( - response, - &Method::GET, - Arc::clone(&settings), - ®istry, - Arc::clone(&orchestrator), - services.clone(), - ) - .await - .expect("should finalize cached or uncached raw HTML"); - assert_eq!( - header_of(&response, header::CACHE_CONTROL), - Some(crate::cache_policy::NO_STORE_PRIVATE_CACHE_CONTROL), - "should keep both synthesized responses private" - ); - let body = body_of(response).await; - assert_eq!( - body.windows(b"".len()) - .filter(|bytes| *bytes == b"") - .count(), - 1, - "should run finalization once for each raw response" - ); - } - assert_eq!( - client.misses.load(Ordering::SeqCst), - 1, - "should fetch origin once" - ); - assert_eq!( - client.hits.load(Ordering::SeqCst), - 1, - "should serve a raw cache hit" - ); - assert!( - !client - .cached - .lock() - .expect("should lock raw origin cache") - .as_ref() - .expect("should retain raw cached HTML") - .windows(b"example-finalized".len()) - .any(|bytes| bytes == b"example-finalized"), - "should cache raw bytes without terminal replacements" - ); - assert!( - template_cache - .lookups - .lock() - .expect("should lock template lookups") - .is_empty(), - "should skip template lookup on both requests" - ); - assert!( - template_cache - .stored_keys - .lock() - .expect("should lock template stores") - .is_empty(), - "should not store finalized HTML as a shared template" - ); - } - #[tokio::test] - async fn html_finalizers_bypass_warm_and_cold_templates_but_keep_origin_readthrough() { - let stub = Arc::new(StubHttpClient::new()); + async fn html_finalizers_bypass_warm_and_cold_templates() { + let stub = PublisherUpstream::new(); let cache = Arc::new(MemoryTemplateCache::default()); let settings = Arc::new(crate::publisher::allow_eof_test_settings( - settings_with_readthrough_enabled("esi"), + settings_with_mode("esi"), )); - let services = services(Arc::clone(&stub), Arc::clone(&cache)); + let services = services(&stub, Arc::clone(&cache)); queue_shareable_html(&stub); let _ = body_of(run(&settings, &services, navigation_request()).await).await; let lookups_before = cache.lookups.lock().expect("should lock lookups").len(); @@ -12564,7 +12467,10 @@ mod tests { let registry = IntegrationRegistry::from_html_stream_processors(vec![Arc::new( super::TerminalTestFactory, )]); - let orchestrator = Arc::new(AuctionOrchestrator::new(settings.auction.clone())); + let orchestrator = Arc::new( + crate::auction::build_orchestrator(&settings) + .expect("should build the auction orchestrator"), + ); for path in ["/article", "/article-cold"] { queue_shareable_html(&stub); let mut request = navigation_request(); @@ -12572,7 +12478,7 @@ mod tests { .parse() .expect("should parse an example URL"); let mut ec_context = EcContext::default(); - let response = handle_publisher_request( + let (response, _owner) = handle_publisher_request( &settings, &services, None, @@ -12584,7 +12490,11 @@ mod tests { registry: None, }, request, - EdgeCacheHeader::SMaxageFallback, + PublisherOriginOptions { + edge_header: EdgeCacheHeader::SMaxageFallback, + input_cap: None, + response_decoding: ResponseDecoding::Original, + }, ) .await .expect("should request raw origin HTML"); @@ -12631,13 +12541,6 @@ mod tests { stores_before, "should never store terminal output" ); - assert!( - stub.recorded_cache_intents().iter().all(|intent| matches!( - intent, - crate::platform::PlatformCacheIntent::Shared { .. } - )), - "should retain configured raw-origin readthrough" - ); let _ = body_of(run(&settings, &services, navigation_request()).await).await; assert!( cache.lookups.lock().expect("should lock lookups").len() > lookups_before, @@ -12660,7 +12563,7 @@ mod tests { ..Default::default() }; let mut ec_context = EcContext::new_for_test(None, consent); - let publisher_response = handle_publisher_request( + let (publisher_response, owner) = handle_publisher_request( settings, services, None, @@ -12672,12 +12575,16 @@ mod tests { registry: None, }, request, - EdgeCacheHeader::SMaxageFallback, + PublisherOriginOptions { + edge_header: EdgeCacheHeader::SMaxageFallback, + input_cap: None, + response_decoding: ResponseDecoding::Original, + }, ) .await .expect("should proxy publisher request"); - finalize_test_publisher_response( + let response = finalize_test_publisher_response( publisher_response, settings, services, @@ -12685,7 +12592,10 @@ mod tests { orchestrator, finalizer, ) - .await + .await; + owner + .attach(response) + .expect("should attach the owner once, as the adapters do") } async fn finalize_test_publisher_response( @@ -12714,6 +12624,7 @@ mod tests { registry, &orchestrator, services, + TEST_BUFFERED_INPUT_CAP, ) .await .expect("should finalize publisher response"), @@ -12726,15 +12637,15 @@ mod tests { /// can be stored, and one it refused keeps streaming exactly like `inline`. A /// helper that assumed either would silently only ever test one of them. async fn body_of(response: Response) -> Vec { - match response.into_body() { - EdgeBody::Stream(mut stream) => { + match response.into_body().into_stream() { + Ok(mut stream) => { let mut out = Vec::new(); while let Some(chunk) = stream.next().await { out.extend_from_slice(&chunk.expect("stream chunk should read")); } out } - other => other + Err(other) => other .into_bytes() .expect("a non-stream body has its bytes in hand") .to_vec(), @@ -12743,10 +12654,10 @@ mod tests { #[tokio::test] async fn harness_emits_a_summary_row_for_an_ad_serving_navigation() { - let stub = Arc::new(StubHttpClient::new()); + let stub = PublisherUpstream::new(); let sink = Arc::new(RecordingTelemetrySink::default()); let services = services_with_cache_and_telemetry( - Arc::clone(&stub), + &stub, Arc::new(MemoryTemplateCache::default()), Arc::clone(&sink), ); @@ -12772,7 +12683,10 @@ mod tests { services: &RuntimeServices, request: Request, ) { - let orchestrator = Arc::new(AuctionOrchestrator::new(settings.auction.clone())); + let orchestrator = Arc::new( + crate::auction::build_orchestrator(settings) + .expect("should build the auction orchestrator"), + ); let kv = crate::ec::kv::KvIdentityGraph::in_memory("test-store"); let consent = crate::consent::ConsentContext { jurisdiction: crate::consent::jurisdiction::Jurisdiction::NonRegulated, @@ -12780,7 +12694,7 @@ mod tests { }; let mut ec_context = EcContext::new_for_test(Some("test-ec-id".to_owned()), consent); - let _ = handle_publisher_request( + let (_, _owner) = handle_publisher_request( settings, services, Some(&kv), @@ -12792,73 +12706,47 @@ mod tests { registry: None, }, request, - EdgeCacheHeader::SMaxageFallback, + PublisherOriginOptions { + edge_header: EdgeCacheHeader::SMaxageFallback, + input_cap: None, + response_decoding: ResponseDecoding::Original, + }, ) .await .expect("should proxy publisher request"); } #[tokio::test] - async fn the_ec_preload_fetch_path_applies_the_same_cache_gate() { - // The call site a naive revert missed. Reverting only this one previously - // passed all 2,697 tests; this is the behavioral cover for it. - let stub = Arc::new(StubHttpClient::new()); - let services = services_with_cache_and_telemetry( - Arc::clone(&stub), - Arc::new(MemoryTemplateCache::default()), - Arc::new(RecordingTelemetrySink::default()), - ); - stub.set_pending_streaming_responses_supported(true); - let settings = Arc::new(settings_with_readthrough_enabled("inline")); - queue_shareable_html(&stub); - - run_through_ec_preload(&settings, &services, navigation_request()).await; - - assert_eq!( - stub.recorded_cache_intents(), - vec![PlatformCacheIntent::Shared { - surrogate_key: format!( - "ts-template {}", - crate::platform::reader_url_surrogate_key("http://ts.example.com/article") - ), - }], - "the EC-preload path must honor the gate, not decide for itself" - ); - } - - #[tokio::test] - async fn the_ec_preload_fetch_path_still_bypasses_when_readthrough_is_off() { - let stub = Arc::new(StubHttpClient::new()); + async fn the_ec_preload_fetch_path_bypasses_for_the_ad_stack() { + let stub = PublisherUpstream::new(); let services = services_with_cache_and_telemetry( - Arc::clone(&stub), + &stub, Arc::new(MemoryTemplateCache::default()), Arc::new(RecordingTelemetrySink::default()), ); - stub.set_pending_streaming_responses_supported(true); let settings = Arc::new(settings_with_mode("inline")); queue_shareable_html(&stub); run_through_ec_preload(&settings, &services, navigation_request()).await; assert_eq!( - stub.recorded_cache_intents(), - vec![PlatformCacheIntent::Bypass], + stub.origin_cache_policies(), + vec![OutboundCachePolicy::Bypass], "the opt-in must gate both fetch paths, not only the non-preload one" ); } #[tokio::test] - async fn disabled_readthrough_preserves_non_ad_origin_caching() { + async fn non_ad_origin_requests_keep_platform_default_caching() { for without_creative_config in [false, true] { - let stub = Arc::new(StubHttpClient::new()); - let services = - services(Arc::clone(&stub), Arc::new(MemoryTemplateCache::default())); + let stub = PublisherUpstream::new(); + let services = services(&stub, Arc::new(MemoryTemplateCache::default())); let mut settings = settings_with_mode("inline"); if without_creative_config { settings.creative_opportunities = None; } let settings = Arc::new(settings); - stub.push_response_with_headers( + stub.queue_origin( 200, b"body {}".to_vec(), vec![ @@ -12875,241 +12763,40 @@ mod tests { let _ = run(&settings, &services, request).await; assert_eq!( - stub.recorded_cache_intents(), - vec![PlatformCacheIntent::Default], + stub.origin_cache_policies(), + vec![OutboundCachePolicy::PlatformDefault], "should preserve existing subresource caching with unchanged settings" ); } } #[tokio::test] - async fn enabled_readthrough_still_preserves_subresource_caching() { - // Browsers send first-party cookies on subresources, so judging assets on - // shareability would bypass the edge cache for most repeat-visitor asset - // traffic — and tag the rest with `ts-template`, so the template rollback - // purge would flush every cached asset at once. - for cookie in [None, Some("ts-ec=abc")] { - let stub = Arc::new(StubHttpClient::new()); - let services = - services(Arc::clone(&stub), Arc::new(MemoryTemplateCache::default())); - let settings = Arc::new(settings_with_readthrough_enabled("inline")); - stub.push_response_with_headers( - 200, - b"body {}".to_vec(), - vec![ - ("content-type", "text/css"), - ("cache-control", "public, max-age=300"), - ], - ); - let mut request = HttpRequest::builder() - .uri("https://ts.example.com/style.css") - .header(header::HOST, "ts.example.com") - .header("sec-fetch-dest", "style") - .body(EdgeBody::empty()) - .expect("should build an asset request"); - if let Some(cookie) = cookie { - request.headers_mut().insert( - header::COOKIE, - HeaderValue::from_str(cookie).expect("should build a cookie header"), - ); - } - - let _ = run(&settings, &services, request).await; - - assert_eq!( - stub.recorded_cache_intents(), - vec![PlatformCacheIntent::Default], - "the opt-in governs documents; a subresource keeps the platform default \ - (cookie={cookie:?})" - ); - } - } - - #[tokio::test] - async fn disabled_readthrough_preserves_non_ad_preload_caching() { - let stub = Arc::new(StubHttpClient::new()); - let services = services(Arc::clone(&stub), Arc::new(MemoryTemplateCache::default())); - stub.set_pending_streaming_responses_supported(true); + async fn non_ad_preload_requests_keep_platform_default_caching() { + let stub = PublisherUpstream::new(); + let services = services(&stub, Arc::new(MemoryTemplateCache::default())); let settings = Arc::new(settings_with_mode("inline")); queue_shareable_html(&stub); run_through_ec_preload(&settings, &services, prefetch_navigation_request()).await; assert_eq!( - stub.recorded_cache_intents(), - vec![PlatformCacheIntent::Default], + stub.origin_cache_policies(), + vec![OutboundCachePolicy::PlatformDefault], "should preserve existing non-ad caching through EC preload" ); } - #[test] - fn cache_intent_matches_the_explicit_opt_in_policy() { - let shared = PlatformCacheIntent::Shared { - surrogate_key: format!( - "ts-template {}", - reader_url_surrogate_key("https://example.com/article") - ), - }; - for (enabled, shareable, ad_stack, document, expected) in [ - (false, false, false, true, PlatformCacheIntent::Default), - (false, false, true, true, PlatformCacheIntent::Bypass), - (false, true, false, true, PlatformCacheIntent::Default), - (false, true, true, true, PlatformCacheIntent::Bypass), - (true, false, false, true, PlatformCacheIntent::Bypass), - (true, false, true, true, PlatformCacheIntent::Bypass), - (true, true, false, true, shared.clone()), - (true, true, true, true, shared), - // A subresource keeps the platform default whatever the gate says, so - // enabling readthrough never bypasses an asset fetch or tags it for the - // template rollback purge. - (true, false, false, false, PlatformCacheIntent::Default), - (true, true, false, false, PlatformCacheIntent::Default), - ] { - let request = PlatformHttpRequest::new( - HttpRequest::builder() - .body(EdgeBody::empty()) - .expect("should build request"), - "backend", - ); - assert_eq!( - apply_origin_cache_intent( - request, - enabled, - shareable, - ad_stack, - document, - "https://example.com/article" - ) - .cache_intent, - expected, - "should honor the explicit policy (enabled={enabled}, shareable={shareable}, ad_stack={ad_stack}, document={document})" - ); - } - } - - #[tokio::test] - async fn readthrough_tags_the_reader_url_before_origin_rewriting_in_inline_mode() { - for preload in [false, true] { - let stub = Arc::new(StubHttpClient::new()); - let services = - services(Arc::clone(&stub), Arc::new(MemoryTemplateCache::default())); - stub.set_pending_streaming_responses_supported(preload); - let settings = Arc::new(settings_with_readthrough_enabled("inline")); - queue_shareable_html(&stub); - let mut request = navigation_request(); - let ingress = RequestIngress::new( - CapturedTarget::Unavailable(TargetUnavailable::NotExposed), - Some( - InboundOrigin::parse("https", "ts.example.com", OriginSource::RuntimeUri) - .expect("should capture the reader transport origin"), - ), - HeaderFidelity::default(), - ); - request.extensions_mut().insert(ingress); - *request.uri_mut() = "https://ts.example.com/article?b=2&a=1" - .parse() - .expect("should parse reader URI"); - if preload { - run_through_ec_preload(&settings, &services, request).await; - } else { - let _ = run(&settings, &services, request).await; - } - let page_key = crate::platform::reader_url_surrogate_key( - "https://ts.example.com/article?a=1&b=2", - ); - assert_eq!( - stub.recorded_cache_intents(), - vec![PlatformCacheIntent::Shared { - surrogate_key: format!("ts-template {page_key}"), - }], - "should attach the URL-purge and all-purge keys independently of template eligibility (preload={preload})" - ); - } - } - - #[tokio::test] - async fn a_shareable_navigation_no_longer_forces_an_origin_miss() { - // The point of issue #852. A cookieless, ad-serving navigation used to set - // pass on every origin fetch, so every pageview paid a full origin round - // trip. - // - // This asserts the *intent* recorded on the outbound request, not a cache - // hit, because a hit is not observable here. Measured under Viceroy 0.17 with - // the gate enabled, the request judged shareable, and an origin responding - // `Cache-Control: public, max-age=60` with no `Set-Cookie`: two identical - // navigations still produced two origin fetches. Viceroy does not implement - // the readthrough cache, so the saving this gate exists for cannot be - // demonstrated locally in any form — only the decision that enables it. - let stub = Arc::new(StubHttpClient::new()); - let services = services_with_cache_and_telemetry( - Arc::clone(&stub), - Arc::new(MemoryTemplateCache::default()), - Arc::new(RecordingTelemetrySink::default()), - ); - let settings = Arc::new(settings_with_readthrough_enabled("esi")); - queue_shareable_html(&stub); - - let _ = run(&settings, &services, navigation_request()).await; - - assert_eq!( - stub.recorded_cache_intents(), - vec![PlatformCacheIntent::Shared { - surrogate_key: format!( - "ts-template {}", - crate::platform::reader_url_surrogate_key("http://ts.example.com/article") - ), - }], - "a shareable navigation must not force a MISS" - ); - } - - #[tokio::test] - async fn a_repeat_visitor_revalidating_still_gets_readthrough() { - // The conditional headers are stripped before the origin is asked, so it - // returns a full document that is shareable like any other. Judging the - // pre-strip request marked this unshareable and cost readthrough for exactly - // the repeat-visit population the change targets. - let stub = Arc::new(StubHttpClient::new()); - let services = services_with_cache_and_telemetry( - Arc::clone(&stub), - Arc::new(MemoryTemplateCache::default()), - Arc::new(RecordingTelemetrySink::default()), - ); - let settings = Arc::new(settings_with_readthrough_enabled("esi")); - queue_shareable_html(&stub); - - let mut request = navigation_request(); - request.headers_mut().insert( - header::IF_NONE_MATCH, - HeaderValue::from_static("\"cached\""), - ); - - let _ = run(&settings, &services, request).await; - - assert_eq!( - stub.recorded_cache_intents(), - vec![PlatformCacheIntent::Shared { - surrogate_key: format!( - "ts-template {}", - crate::platform::reader_url_surrogate_key("http://ts.example.com/article") - ), - }], - "a stripped conditional navigation asks the origin an unconditional \ - question, so its answer is shareable" - ); - } - #[tokio::test] async fn a_reader_asking_for_a_range_is_not_served_a_shared_template() { // The other half, and the invariant that caught an over-broad first attempt: // stripping changes what the origin is asked, not what the reader wanted. - let stub = Arc::new(StubHttpClient::new()); + let stub = PublisherUpstream::new(); let services = services_with_cache_and_telemetry( - Arc::clone(&stub), + &stub, Arc::new(MemoryTemplateCache::default()), Arc::new(RecordingTelemetrySink::default()), ); - let settings = Arc::new(settings_with_readthrough_enabled("esi")); + let settings = Arc::new(settings_with_mode("esi")); queue_shareable_html(&stub); queue_shareable_html(&stub); @@ -13123,95 +12810,19 @@ mod tests { let _ = run(&settings, &services, request).await; assert_eq!( - stub.recorded_cache_intents().len(), + stub.origin_cache_policies().len(), 2, "the range request must reach the origin rather than be answered from the \ warm shared template" ); } - #[tokio::test] - async fn unshareable_requests_still_bypass() { - // Each of these is a distinct reason the origin response cannot be held in a - // shared cache, and each must reach the same decision on its own. - for (label, request) in [ - ("cookie", navigation_request_with_cookie("ts-ec=abc")), - ("authorization", { - let mut request = navigation_request(); - request.headers_mut().insert( - header::AUTHORIZATION, - HeaderValue::from_static("Basic dXNlcjpwYXNz"), - ); - request - }), - ("non-GET", { - let mut request = navigation_request(); - *request.method_mut() = Method::POST; - request - }), - ("conditional", { - let mut request = navigation_request(); - request - .headers_mut() - .insert(header::IF_MATCH, HeaderValue::from_static("\"tag\"")); - request - }), - ] { - let stub = Arc::new(StubHttpClient::new()); - let services = services_with_cache_and_telemetry( - Arc::clone(&stub), - Arc::new(MemoryTemplateCache::default()), - Arc::new(RecordingTelemetrySink::default()), - ); - let settings = Arc::new(settings_with_readthrough_enabled("esi")); - queue_shareable_html(&stub); - - let _ = run(&settings, &services, request).await; - - assert_eq!( - stub.recorded_cache_intents(), - vec![PlatformCacheIntent::Bypass], - "a {label}-bearing request must bypass the shared cache" - ); - } - } - - #[tokio::test] - async fn a_request_that_skips_the_ad_stack_is_still_judged_on_shareability() { - // The tightening half, and the term that left the condition. A prefetch runs - // no auction, so it used to skip the bypass; whether an auction runs says - // nothing about whether the origin's response may be shared. - let stub = Arc::new(StubHttpClient::new()); - let services = services_with_cache_and_telemetry( - Arc::clone(&stub), - Arc::new(MemoryTemplateCache::default()), - Arc::new(RecordingTelemetrySink::default()), - ); - let settings = Arc::new(settings_with_readthrough_enabled("esi")); - queue_shareable_html(&stub); - - let _ = run(&settings, &services, { - let mut request = prefetch_navigation_request(); - request - .headers_mut() - .insert(header::COOKIE, HeaderValue::from_static("ts-ec=abc")); - request - }) - .await; - - assert_eq!( - stub.recorded_cache_intents(), - vec![PlatformCacheIntent::Bypass], - "an ad-stack opt-out that is also unshareable must still bypass" - ); - } - #[tokio::test] async fn navigation_records_whether_the_origin_response_was_shareable() { - let stub = Arc::new(StubHttpClient::new()); + let stub = PublisherUpstream::new(); let sink = Arc::new(RecordingTelemetrySink::default()); let services = services_with_cache_and_telemetry( - Arc::clone(&stub), + &stub, Arc::new(MemoryTemplateCache::default()), Arc::clone(&sink), ); @@ -13236,10 +12847,10 @@ mod tests { #[tokio::test] async fn cookieless_navigation_records_the_origin_response_as_shareable() { - let stub = Arc::new(StubHttpClient::new()); + let stub = PublisherUpstream::new(); let sink = Arc::new(RecordingTelemetrySink::default()); let services = services_with_cache_and_telemetry( - Arc::clone(&stub), + &stub, Arc::new(MemoryTemplateCache::default()), Arc::clone(&sink), ); @@ -13260,10 +12871,10 @@ mod tests { #[tokio::test] async fn non_document_requests_do_not_count_toward_readthrough_gate_reach() { for fetch_metadata in [true, false] { - let stub = Arc::new(StubHttpClient::new()); + let stub = PublisherUpstream::new(); let sink = Arc::new(RecordingTelemetrySink::default()); let services = services_with_cache_and_telemetry( - Arc::clone(&stub), + &stub, Arc::new(MemoryTemplateCache::default()), Arc::clone(&sink), ); @@ -13306,11 +12917,11 @@ mod tests { ), }); let settings = Arc::new(settings); - let stub = Arc::new(StubHttpClient::new()); + let stub = PublisherUpstream::for_origin("origin.example.com"); let cache = Arc::new(MemoryTemplateCache::default()); - let services = services(Arc::clone(&stub), Arc::clone(&cache)); + let services = services(&stub, Arc::clone(&cache)); for _ in 0..2 { - stub.push_response_with_headers(200, + stub.queue_origin(200, b"next".to_vec(), vec![("content-type", "text/html; charset=utf-8"), ("cache-control", "public, max-age=300")]); } @@ -13337,7 +12948,9 @@ mod tests { request .headers_mut() .insert("x-forwarded-proto", HeaderValue::from_static("https")); - crate::forwarder::prepare_trusted_forwarder(&mut request, &settings); + crate::test_support::tests::with_request_parts(&mut request, |parts| { + crate::forwarder::prepare_trusted_forwarder(parts, &settings) + }); let response = run(&settings, &services, request).await; let body = String::from_utf8(body_of(response).await).expect("should decode HTML"); assert!( @@ -13349,7 +12962,7 @@ mod tests { "should not expose transport authority in HTML" ); assert_eq!( - stub.recorded_request_uris().len(), + stub.origin_uris().len(), expected_fetches, "should hit cache only for the same public authority" ); @@ -13372,20 +12985,23 @@ mod tests { "should partition templates by public authority" ); } - for uri in stub.recorded_request_uris() { + for uri in stub.origin_uris() { assert_eq!( uri, "http://origin.example.com/article", "should connect only to configured publisher origin" ); } - for fields in stub.recorded_request_headers() { - assert!( - fields - .iter() - .any(|(name, value)| name.eq_ignore_ascii_case("host") - && value == "backend.example.com"), - "should retain configured backend Host" + for request in stub.origin_requests() { + assert_eq!( + ( + request.host_authority.as_str(), + request.has_host_authority_override + ), + ("backend.example.com", true), + "should retain the configured backend host as the authority override" ); + } + for fields in stub.origin_headers() { assert!( !fields.iter().any(|(name, _)| matches!( name.as_str(), @@ -13398,10 +13014,10 @@ mod tests { #[tokio::test] async fn a_second_request_is_served_from_the_cache_without_touching_the_origin() { - let stub = Arc::new(StubHttpClient::new()); + let stub = PublisherUpstream::new(); let cache = Arc::new(MemoryTemplateCache::default()); let settings = Arc::new(settings_with_mode("esi")); - let services = services(Arc::clone(&stub), Arc::clone(&cache)); + let services = services(&stub, Arc::clone(&cache)); // Only one origin response is queued. If the second request reached the // origin it would find the queue empty, so this fixture is itself part of @@ -13424,7 +13040,7 @@ mod tests { ); let first = body_of(cold).await; assert_eq!( - stub.recorded_request_uris().len(), + stub.origin_uris().len(), 1, "the cold request must fetch the origin" ); @@ -13444,7 +13060,7 @@ mod tests { ); let second = body_of(warm).await; assert_eq!( - stub.recorded_request_uris().len(), + stub.origin_uris().len(), 1, "the warm request must not fetch the origin — that saving is the point" ); @@ -13456,10 +13072,10 @@ mod tests { #[tokio::test] async fn a_pass_through_authorization_never_reaches_the_shared_template() { - let stub = Arc::new(StubHttpClient::new()); + let stub = PublisherUpstream::new(); let cache = Arc::new(MemoryTemplateCache::default()); let settings = Arc::new(settings_with_mode("esi")); - let services = services(Arc::clone(&stub), Arc::clone(&cache)); + let services = services(&stub, Arc::clone(&cache)); queue_shareable_html(&stub); let mut request = navigation_request(); @@ -13486,10 +13102,10 @@ mod tests { #[tokio::test] async fn repeated_authorization_never_reaches_the_shared_template() { - let stub = Arc::new(StubHttpClient::new()); + let stub = PublisherUpstream::new(); let cache = Arc::new(MemoryTemplateCache::default()); let settings = Arc::new(settings_with_mode("esi")); - let services = services(Arc::clone(&stub), Arc::clone(&cache)); + let services = services(&stub, Arc::clone(&cache)); queue_shareable_html(&stub); let mut request = navigation_request(); @@ -13525,7 +13141,7 @@ mod tests { #[tokio::test] async fn a_replaced_edge_terminated_authorization_bypasses_the_shared_template() { - let stub = Arc::new(StubHttpClient::new()); + let stub = PublisherUpstream::new(); let cache = Arc::new(MemoryTemplateCache::default()); let mut settings = settings_with_mode("esi"); settings.handlers[0] = toml::from_str( @@ -13537,7 +13153,7 @@ mod tests { ) .expect("should parse article auth handler"); let settings = Arc::new(settings); - let services = services(Arc::clone(&stub), Arc::clone(&cache)); + let services = services(&stub, Arc::clone(&cache)); queue_shareable_html(&stub); let mut request = navigation_request(); @@ -13546,9 +13162,11 @@ mod tests { HeaderValue::from_static("Basic dXNlcjpwYXNz"), ); assert!( - crate::auth::enforce_basic_auth(&settings, &mut request) - .expect("should evaluate auth") - .is_none(), + crate::test_support::tests::with_request_parts(&mut request, |parts| { + crate::auth::enforce_basic_auth(&settings, parts) + }) + .expect("should evaluate auth") + .is_none(), "the original credential should pass edge auth" ); request.headers_mut().insert( @@ -13574,10 +13192,10 @@ mod tests { #[tokio::test] async fn an_edge_terminated_authorization_still_shares_its_template() { - let stub = Arc::new(StubHttpClient::new()); + let stub = PublisherUpstream::new(); let cache = Arc::new(MemoryTemplateCache::default()); let settings = Arc::new(settings_with_mode("esi")); - let services = services(Arc::clone(&stub), Arc::clone(&cache)); + let services = services(&stub, Arc::clone(&cache)); // One origin response for two requests: the warm read is asserted by the // fixture running dry, exactly as in the unauthenticated case. @@ -13616,7 +13234,7 @@ mod tests { let second = body_of(warm).await; assert_eq!( - stub.recorded_request_uris().len(), + stub.origin_uris().len(), 1, "the warm gated request must not fetch the origin" ); @@ -13628,24 +13246,24 @@ mod tests { #[tokio::test] async fn esi_cache_hit_with_ec_snapshot_does_not_start_pending_origin() { - let stub = Arc::new(StubHttpClient::new()); - stub.set_concurrent_fanout(true); - stub.set_streaming_responses_supported(true); - stub.set_pending_streaming_responses_supported(true); + let stub = PublisherUpstream::new(); let cache = Arc::new(MemoryTemplateCache::default()); - let settings = Arc::new(settings_with_readthrough_enabled("esi")); - let services = services(Arc::clone(&stub), Arc::clone(&cache)); + let settings = Arc::new(settings_with_mode("esi")); + let services = services(&stub, Arc::clone(&cache)); let lookups = Arc::new(AtomicUsize::new(0)); let http_calls_at_lookup = Arc::new(AtomicUsize::new(0)); let graph = KvIdentityGraph::new(OrderRecordingKv { inner: InMemoryEcKv::new("esi-scheduling-store"), - http: Arc::clone(&stub), + http: stub.clone(), http_calls_at_lookup: Arc::clone(&http_calls_at_lookup), lookups: Arc::clone(&lookups), }); let registry = IntegrationRegistry::new(&settings).expect("should create integration registry"); - let orchestrator = Arc::new(AuctionOrchestrator::new(settings.auction.clone())); + let orchestrator = Arc::new( + crate::auction::build_orchestrator(&settings) + .expect("should build the auction orchestrator"), + ); // Only the cold request has an origin response available. queue_shareable_html(&stub); @@ -13655,7 +13273,7 @@ mod tests { scheduling_consent(), Some("203.0.113.7".to_owned()), ); - let response = handle_publisher_request( + let (response, _owner) = handle_publisher_request( &settings, &services, Some(&graph), @@ -13667,7 +13285,11 @@ mod tests { registry: None, }, navigation_request(), - EdgeCacheHeader::SMaxageFallback, + PublisherOriginOptions { + edge_header: EdgeCacheHeader::SMaxageFallback, + input_cap: None, + response_decoding: ResponseDecoding::Original, + }, ) .await .expect("should serve ESI navigation with an active EC"); @@ -13706,7 +13328,7 @@ mod tests { "should return the assembled publisher body" ); assert_eq!( - stub.recorded_request_uris().len(), + stub.origin_uris().len(), 1, "should fetch origin only for the cold cache miss" ); @@ -13721,35 +13343,27 @@ mod tests { "should consult the template cache on both navigations" ); assert_eq!( - stub.recorded_stream_response_flags(), - vec![true], - "should preserve streaming on the cold origin fetch" - ); - assert_eq!( - stub.recorded_cache_intents(), - vec![PlatformCacheIntent::Shared { - surrogate_key: format!( - "ts-template {}", - reader_url_surrogate_key("http://ts.example.com/article") - ), - }], - "a shareable cold fetch must stop forcing a MISS — the origin round \ - trip issue #852 exists to take off the hot path" + stub.origin_requests() + .iter() + .map(|request| (request.response_mode, request.mode)) + .collect::>(), + vec![( + edgezero_core::outbound::ResponseMode::Streamed, + edgezero_core::outbound::OutboundSendMode::Send + )], + "should stream the cold origin fetch, sent after the template lookup" ); } #[tokio::test] async fn only_the_cold_miss_uses_the_platform_assembler() { - let stub = Arc::new(StubHttpClient::new()); + let stub = PublisherUpstream::new(); let cache = Arc::new(MemoryTemplateCache::default()); let assembler = Arc::new(RecordingTemplateAssembler::default()); assembler.require_store_before_call(Arc::clone(&cache.insert_completed)); let settings = Arc::new(settings_with_mode("esi")); - let services = services_with_assembler( - Arc::clone(&stub), - Arc::clone(&cache), - Arc::clone(&assembler), - ); + let services = + services_with_assembler(&stub, Arc::clone(&cache), Arc::clone(&assembler)); queue_shareable_html(&stub); let cold = run(&settings, &services, navigation_request()).await; @@ -13777,16 +13391,13 @@ mod tests { #[tokio::test] async fn a_platform_assembly_failure_falls_back_to_a_complete_byte_seam() { - let stub = Arc::new(StubHttpClient::new()); + let stub = PublisherUpstream::new(); let cache = Arc::new(MemoryTemplateCache::default()); let assembler = Arc::new(RecordingTemplateAssembler::default()); assembler.fail.store(true, Ordering::Relaxed); let settings = Arc::new(settings_with_mode("esi")); - let services = services_with_assembler( - Arc::clone(&stub), - Arc::clone(&cache), - Arc::clone(&assembler), - ); + let services = + services_with_assembler(&stub, Arc::clone(&cache), Arc::clone(&assembler)); queue_shareable_html(&stub); let response = run(&settings, &services, navigation_request()).await; @@ -13812,16 +13423,13 @@ mod tests { "publisher", ), ] { - let stub = Arc::new(StubHttpClient::new()); + let stub = PublisherUpstream::new(); let cache = Arc::new(MemoryTemplateCache::default()); let assembler = Arc::new(RecordingTemplateAssembler::default()); let settings = Arc::new(settings_with_mode("esi")); - let services = services_with_assembler( - Arc::clone(&stub), - Arc::clone(&cache), - Arc::clone(&assembler), - ); - stub.push_response_with_headers( + let services = + services_with_assembler(&stub, Arc::clone(&cache), Arc::clone(&assembler)); + stub.queue_origin( 200, format!("{source}").into_bytes(), vec![ @@ -13865,12 +13473,12 @@ mod tests { #[tokio::test] async fn ineligible_requests_and_responses_never_use_the_platform_assembler() { - let inline_stub = Arc::new(StubHttpClient::new()); + let inline_stub = PublisherUpstream::new(); let inline_cache = Arc::new(MemoryTemplateCache::default()); let inline_assembler = Arc::new(RecordingTemplateAssembler::default()); let inline_settings = Arc::new(settings_with_mode("inline")); let inline_services = services_with_assembler( - Arc::clone(&inline_stub), + &inline_stub, Arc::clone(&inline_cache), Arc::clone(&inline_assembler), ); @@ -13880,16 +13488,16 @@ mod tests { body_of(run(&inline_settings, &inline_services, navigation_request()).await).await; assert_eq!(inline_assembler.calls.load(Ordering::Relaxed), 0); - let private_stub = Arc::new(StubHttpClient::new()); + let private_stub = PublisherUpstream::new(); let private_cache = Arc::new(MemoryTemplateCache::default()); let private_assembler = Arc::new(RecordingTemplateAssembler::default()); let esi_settings = Arc::new(settings_with_mode("esi")); let private_services = services_with_assembler( - Arc::clone(&private_stub), + &private_stub, Arc::clone(&private_cache), Arc::clone(&private_assembler), ); - private_stub.push_response_with_headers( + private_stub.queue_origin( 200, b"private origin".to_vec(), vec![ @@ -13903,11 +13511,11 @@ mod tests { body_of(run(&esi_settings, &private_services, navigation_request()).await).await; assert_eq!(private_assembler.calls.load(Ordering::Relaxed), 0); - let authenticated_stub = Arc::new(StubHttpClient::new()); + let authenticated_stub = PublisherUpstream::new(); let authenticated_cache = Arc::new(MemoryTemplateCache::default()); let authenticated_assembler = Arc::new(RecordingTemplateAssembler::default()); let authenticated_services = services_with_assembler( - Arc::clone(&authenticated_stub), + &authenticated_stub, Arc::clone(&authenticated_cache), Arc::clone(&authenticated_assembler), ); @@ -13929,12 +13537,12 @@ mod tests { .await; assert_eq!(authenticated_assembler.calls.load(Ordering::Relaxed), 0); - let failed_cache_stub = Arc::new(StubHttpClient::new()); + let failed_cache_stub = PublisherUpstream::new(); let failed_cache = Arc::new(MemoryTemplateCache::default()); failed_cache.fail_lookup.store(true, Ordering::Relaxed); let failed_cache_assembler = Arc::new(RecordingTemplateAssembler::default()); let failed_cache_services = services_with_assembler( - Arc::clone(&failed_cache_stub), + &failed_cache_stub, Arc::clone(&failed_cache), Arc::clone(&failed_cache_assembler), ); @@ -13947,15 +13555,15 @@ mod tests { #[tokio::test] async fn fastly_surrogate_control_still_allows_a_cold_fill_and_warm_hit() { - let stub = Arc::new(StubHttpClient::new()); + let stub = PublisherUpstream::new(); let cache = Arc::new(MemoryTemplateCache::default()); let settings = Arc::new(settings_with_mode_and_template_cache_max_age("esi", 1_200)); - let services = services(Arc::clone(&stub), Arc::clone(&cache)); + let services = services(&stub, Arc::clone(&cache)); // This is the policy observed on the publisher that exposed the bug. Fastly // gives the surrogate lifetime edge precedence; the configured safety // ceiling allows the origin's twenty-minute edge freshness. - stub.push_response_with_headers( + stub.queue_origin( 200, b"origin".to_vec(), vec![ @@ -13996,7 +13604,7 @@ mod tests { ); let _ = body_of(warm).await; assert_eq!( - stub.recorded_request_uris().len(), + stub.origin_uris().len(), 1, "the warm request must reuse the template authorized by Surrogate-Control" ); @@ -14004,11 +13612,11 @@ mod tests { #[tokio::test] async fn one_identity_template_is_negotiated_per_reader_on_miss_and_hit() { - let stub = Arc::new(StubHttpClient::new()); + let stub = PublisherUpstream::new(); let cache = Arc::new(MemoryTemplateCache::default()); let settings = Arc::new(settings_with_mode("esi")); - let services = services(Arc::clone(&stub), Arc::clone(&cache)); - stub.push_response_with_headers( + let services = services(&stub, Arc::clone(&cache)); + stub.queue_origin( 200, gzip_encode(b"origin"), vec![ @@ -14088,7 +13696,7 @@ mod tests { ); let warm_brotli_decoded = brotli_decode(&body_of(warm_brotli).await); - assert_eq!(stub.recorded_request_uris().len(), 1); + assert_eq!(stub.origin_uris().len(), 1); assert_eq!(cold_decoded, warm_identity_body); assert_eq!(warm_gzip_decoded, warm_identity_body); assert_eq!(warm_brotli_decoded, warm_identity_body); @@ -14096,10 +13704,10 @@ mod tests { #[tokio::test] async fn esi_unsupported_reader_encoding_still_injects_tsjs() { - let stub = Arc::new(StubHttpClient::new()); + let stub = PublisherUpstream::new(); let cache = Arc::new(MemoryTemplateCache::default()); let settings = Arc::new(settings_with_mode("esi")); - let services = services(Arc::clone(&stub), Arc::clone(&cache)); + let services = services(&stub, Arc::clone(&cache)); queue_shareable_html(&stub); let mut request = navigation_request(); request.headers_mut().insert( @@ -14110,7 +13718,7 @@ mod tests { let body = String::from_utf8(body_of(run(&settings, &services, request).await).await) .expect("should return UTF-8 HTML"); - let forwarded = stub.recorded_request_headers(); + let forwarded = stub.origin_headers(); assert!( forwarded[0].iter().any(|(name, value)| { name.eq_ignore_ascii_case("accept-encoding") && value == "identity" @@ -14135,12 +13743,12 @@ mod tests { #[tokio::test] async fn an_unsupported_origin_encoding_bypasses_template_cache_without_relabeling_the_body() { - let stub = Arc::new(StubHttpClient::new()); + let stub = PublisherUpstream::new(); let cache = Arc::new(MemoryTemplateCache::default()); let settings = Arc::new(settings_with_mode("esi")); - let services = services(Arc::clone(&stub), Arc::clone(&cache)); + let services = services(&stub, Arc::clone(&cache)); let wire_body = b"opaque-zstd-wire-bytes".to_vec(); - stub.push_response_with_headers( + stub.queue_origin( 200, wire_body.clone(), vec![ @@ -14184,10 +13792,10 @@ mod tests { (header::RANGE, "bytes=0-31"), (header::IF_NONE_MATCH, "\"reader-validator\""), ] { - let stub = Arc::new(StubHttpClient::new()); + let stub = PublisherUpstream::new(); let cache = Arc::new(MemoryTemplateCache::default()); let settings = Arc::new(settings_with_mode("esi")); - let services = services(Arc::clone(&stub), Arc::clone(&cache)); + let services = services(&stub, Arc::clone(&cache)); queue_shareable_html(&stub); queue_shareable_html(&stub); @@ -14199,7 +13807,7 @@ mod tests { let _ = body_of(run(&settings, &services, revalidating).await).await; assert_eq!( - stub.recorded_request_uris().len(), + stub.origin_uris().len(), 2, "{name}: {value} must reach the origin even when template cache is warm" ); @@ -14324,8 +13932,8 @@ mod tests { /// The marker is reserved, but publisher content can still contain it. The /// transform adds its own terminal placeholder; repeated markers then make the /// response bypass the template cache rather than requiring normalization. - fn queue_html_that_collides_with_the_marker(stub: &StubHttpClient) { - stub.push_response_with_headers( + fn queue_html_that_collides_with_the_marker(stub: &PublisherUpstream) { + stub.queue_origin( 200, format!( "origin" .to_vec(), @@ -17976,7 +17428,7 @@ for (const mode of ['ready', 'queued', 'stale', 'inactive', 'throws']) { let _ = body_of(run(&settings, &services, navigation_request()).await).await; assert_eq!( - stub.recorded_request_uris().len(), + stub.origin_uris().len(), 2, "a response-bound CSP nonce and its HTML must never be reused from template cache" ); @@ -17994,12 +17446,12 @@ for (const mode of ['ready', 'queued', 'stale', 'inactive', 'throws']) { // The response-header gate cannot see this policy at all. Storing the document // would replay one response's nonce to every later reader — the exact thing the // header check exists to prevent. - let stub = Arc::new(StubHttpClient::new()); + let stub = PublisherUpstream::new(); let cache = Arc::new(MemoryTemplateCache::default()); let settings = Arc::new(settings_with_mode("esi")); - let services = services(Arc::clone(&stub), Arc::clone(&cache)); + let services = services(&stub, Arc::clone(&cache)); for _ in 0..2 { - stub.push_response_with_headers( + stub.queue_origin( 200, br#"origin"# .to_vec(), @@ -18022,7 +17474,7 @@ for (const mode of ['ready', 'queued', 'stale', 'inactive', 'throws']) { let _ = body_of(run(&settings, &services, navigation_request()).await).await; assert_eq!( - stub.recorded_request_uris().len(), + stub.origin_uris().len(), 2, "the warm request must reach the origin, not a replayed nonce" ); @@ -18040,12 +17492,12 @@ for (const mode of ['ready', 'queued', 'stale', 'inactive', 'throws']) { async fn a_nonce_attribute_without_a_policy_bypasses_the_shared_template_cache() { // Fail closed: the attributes say the document was written for a per-response // policy, whether or not the policy itself survived to this scan. - let stub = Arc::new(StubHttpClient::new()); + let stub = PublisherUpstream::new(); let cache = Arc::new(MemoryTemplateCache::default()); let settings = Arc::new(settings_with_mode("esi")); - let services = services(Arc::clone(&stub), Arc::clone(&cache)); + let services = services(&stub, Arc::clone(&cache)); for _ in 0..2 { - stub.push_response_with_headers( + stub.queue_origin( 200, br#"origin"# .to_vec(), @@ -18059,7 +17511,7 @@ for (const mode of ['ready', 'queued', 'stale', 'inactive', 'throws']) { let _ = body_of(run(&settings, &services, navigation_request()).await).await; let _ = body_of(run(&settings, &services, navigation_request()).await).await; - assert_eq!(stub.recorded_request_uris().len(), 2); + assert_eq!(stub.origin_uris().len(), 2); assert!( cache .entries @@ -18075,11 +17527,11 @@ for (const mode of ['ready', 'queued', 'stale', 'inactive', 'throws']) { // literal, and the payload's `` then terminates the publisher's script // — in the response served cold and in the template every warm reader gets. const PUBLISHER_SCRIPT: &str = r#""#; - let stub = Arc::new(StubHttpClient::new()); + let stub = PublisherUpstream::new(); let cache = Arc::new(MemoryTemplateCache::default()); let settings = Arc::new(settings_with_mode("esi")); - let services = services(Arc::clone(&stub), Arc::clone(&cache)); - stub.push_response_with_headers( + let services = services(&stub, Arc::clone(&cache)); + stub.queue_origin( 200, format!("{PUBLISHER_SCRIPT}

article

").into_bytes(), vec![ @@ -18098,7 +17550,7 @@ for (const mode of ['ready', 'queued', 'stale', 'inactive', 'throws']) { .expect("the warm document should be UTF-8"); assert_eq!( - stub.recorded_request_uris().len(), + stub.origin_uris().len(), 1, "the document is otherwise shareable and must still be stored and reused" ); @@ -18119,11 +17571,11 @@ for (const mode of ['ready', 'queued', 'stale', 'inactive', 'throws']) { // The reverse search took the *last* `` sequence, so the seam landed // inside this comment and the assembled bids never executed. const TRAILING_COMMENT: &str = ""; - let stub = Arc::new(StubHttpClient::new()); + let stub = PublisherUpstream::new(); let cache = Arc::new(MemoryTemplateCache::default()); let settings = Arc::new(settings_with_mode("esi")); - let services = services(Arc::clone(&stub), Arc::clone(&cache)); - stub.push_response_with_headers( + let services = services(&stub, Arc::clone(&cache)); + stub.queue_origin( 200, format!("

article

{TRAILING_COMMENT}") .into_bytes(), @@ -18142,7 +17594,7 @@ for (const mode of ['ready', 'queued', 'stale', 'inactive', 'throws']) { ) .expect("the warm document should be UTF-8"); - assert_eq!(stub.recorded_request_uris().len(), 1); + assert_eq!(stub.origin_uris().len(), 1); for (label, document) in [("cold", &cold), ("warm", &warm)] { assert!( document.contains(TRAILING_COMMENT), @@ -18166,16 +17618,16 @@ for (const mode of ['ready', 'queued', 'stale', 'inactive', 'throws']) { // here for both. Serving the cached GET to the POST swallows the mutating // request entirely: the origin never sees it, the caller gets 200 and a page, // and nothing anywhere reports a problem. - let stub = Arc::new(StubHttpClient::new()); + let stub = PublisherUpstream::new(); let cache = Arc::new(MemoryTemplateCache::default()); let settings = Arc::new(settings_with_mode("esi")); - let services = services(Arc::clone(&stub), Arc::clone(&cache)); + let services = services(&stub, Arc::clone(&cache)); queue_shareable_html(&stub); queue_shareable_html(&stub); // Warm the cache with a GET. let _ = run(&settings, &services, navigation_request()).await; - assert_eq!(stub.recorded_request_uris().len(), 1); + assert_eq!(stub.origin_uris().len(), 1); let post = HttpRequest::builder() .method(Method::POST) @@ -18186,7 +17638,7 @@ for (const mode of ['ready', 'queued', 'stale', 'inactive', 'throws']) { let _ = run(&settings, &services, post).await; assert_eq!( - stub.recorded_request_uris().len(), + stub.origin_uris().len(), 2, "the POST must reach the origin rather than being answered from the \ cached GET" @@ -18203,10 +17655,10 @@ for (const mode of ['ready', 'queued', 'stale', 'inactive', 'throws']) { // The stored template is perfectly cacheable; this request is not entitled // to it. The store gate cannot express that, because it is a property of // the reader rather than of the bytes — which is why the lookup re-checks. - let stub = Arc::new(StubHttpClient::new()); + let stub = PublisherUpstream::new(); let cache = Arc::new(MemoryTemplateCache::default()); let settings = Arc::new(settings_with_mode("esi")); - let services = services(Arc::clone(&stub), Arc::clone(&cache)); + let services = services(&stub, Arc::clone(&cache)); queue_shareable_html(&stub); queue_shareable_html(&stub); @@ -18217,7 +17669,8 @@ for (const mode of ['ready', 'queued', 'stale', 'inactive', 'throws']) { "the cold request should have populated the cache" ); - let orchestrator = AuctionOrchestrator::new(settings.auction.clone()); + let orchestrator = crate::auction::build_orchestrator(&settings) + .expect("should build the auction orchestrator"); let consent = crate::consent::ConsentContext { jurisdiction: crate::consent::jurisdiction::Jurisdiction::NonRegulated, ..Default::default() @@ -18231,7 +17684,7 @@ for (const mode of ['ready', 'queued', 'stale', 'inactive', 'throws']) { .header(header::AUTHORIZATION, "Basic dXNlcjpwYXNz") .body(EdgeBody::empty()) .expect("should build authenticated request"); - let _ = handle_publisher_request( + let (_, _owner) = handle_publisher_request( &settings, &services, None, @@ -18243,13 +17696,17 @@ for (const mode of ['ready', 'queued', 'stale', 'inactive', 'throws']) { registry: None, }, authenticated, - EdgeCacheHeader::SMaxageFallback, + PublisherOriginOptions { + edge_header: EdgeCacheHeader::SMaxageFallback, + input_cap: None, + response_decoding: ResponseDecoding::Original, + }, ) .await .expect("should proxy publisher request"); assert_eq!( - stub.recorded_request_uris().len(), + stub.origin_uris().len(), 2, "an authenticated request must reach the origin rather than read a \ shared template" @@ -19292,8 +18749,8 @@ for (const mode of ['ready', 'queued', 'stale', 'inactive', 'throws']) { crate_test_settings_str() )) .expect("should parse settings with cache rule"); - let stub = Arc::new(StubHttpClient::new()); - stub.push_response_with_headers( + let stub = PublisherUpstream::new(); + stub.queue_origin( 200, b"png".to_vec(), vec![ @@ -19301,9 +18758,7 @@ for (const mode of ['ready', 'queued', 'stale', 'inactive', 'throws']) { (header::CACHE_CONTROL.as_str(), "public, max-age=60"), ], ); - let services = build_services_with_http_client( - Arc::clone(&stub) as Arc - ); + let services = stub.services(); let request = HttpRequest::builder() .method(Method::GET) .uri("https://publisher.example/assets/logo.0123abcd.png") @@ -19311,7 +18766,7 @@ for (const mode of ['ready', 'queued', 'stale', 'inactive', 'throws']) { .body(EdgeBody::empty()) .expect("should build request"); - let response = run_publisher_proxy(&settings, &services, request).await; + let (response, _owner) = run_publisher_proxy(&settings, &services, request).await; let PublisherResponse::PassThrough { response, .. } = response else { panic!("should pass through non-HTML asset response"); }; @@ -19356,8 +18811,8 @@ for (const mode of ['ready', 'queued', 'stale', 'inactive', 'throws']) { #[tokio::test] async fn transformed_page_headers_map_origin_urls_to_serving_host() { let settings = create_test_settings(); - let stub = Arc::new(StubHttpClient::new()); - stub.push_response_with_headers( + let stub = PublisherUpstream::new(); + stub.queue_origin( 302, b"go" .to_vec(), @@ -19381,11 +18836,10 @@ for (const mode of ['ready', 'queued', 'stale', 'inactive', 'throws']) { ), ], ); - let services = build_services_with_http_client( - Arc::clone(&stub) as Arc - ); + let services = stub.services(); - let response = run_publisher_proxy(&settings, &services, publisher_page_request()).await; + let (response, _owner) = + run_publisher_proxy(&settings, &services, publisher_page_request()).await; assert!( matches!(response, PublisherResponse::Stream { .. }), @@ -19425,8 +18879,8 @@ for (const mode of ['ready', 'queued', 'stale', 'inactive', 'throws']) { #[tokio::test] async fn bodiless_redirect_location_maps_origin_url_to_serving_host() { let settings = create_test_settings(); - let stub = Arc::new(StubHttpClient::new()); - stub.push_response_with_headers( + let stub = PublisherUpstream::new(); + stub.queue_origin( 301, Vec::new(), vec![( @@ -19434,11 +18888,10 @@ for (const mode of ['ready', 'queued', 'stale', 'inactive', 'throws']) { "http://origin.test-publisher.com/moved?x=1", )], ); - let services = build_services_with_http_client( - Arc::clone(&stub) as Arc - ); + let services = stub.services(); - let response = run_publisher_proxy(&settings, &services, publisher_page_request()).await; + let (response, _owner) = + run_publisher_proxy(&settings, &services, publisher_page_request()).await; assert!( matches!(response, PublisherResponse::Buffered(_)), @@ -19460,19 +18913,13 @@ for (const mode of ['ready', 'queued', 'stale', 'inactive', 'throws']) { ) -> String { let mut settings = create_test_settings(); settings.publisher.origin_url = origin_url.to_string(); - let stub = Arc::new(StubHttpClient::new()); - stub.push_response_with_headers( - 303, - Vec::new(), - vec![(header::LOCATION.as_str(), location)], - ); - let services = build_services_with_http_client( - Arc::clone(&stub) as Arc - ); + let stub = PublisherUpstream::new(); + stub.queue_origin(303, Vec::new(), vec![(header::LOCATION.as_str(), location)]); + let services = stub.services(); let mut request = publisher_page_request(); *request.method_mut() = method; - let response = run_publisher_proxy(&settings, &services, request).await; + let (response, _owner) = run_publisher_proxy(&settings, &services, request).await; proxy_response_headers(&response) .get(header::LOCATION) @@ -19534,8 +18981,8 @@ for (const mode of ['ready', 'queued', 'stale', 'inactive', 'throws']) { async fn overridden_host_redirect_with_cookie_is_not_rewritten_into_a_loop() { let mut settings = create_test_settings(); settings.publisher.origin_host_header_override = Some("publisher.example".to_string()); - let stub = Arc::new(StubHttpClient::new()); - stub.push_response_with_headers( + let stub = PublisherUpstream::new(); + stub.queue_origin( 301, Vec::new(), vec![ @@ -19546,11 +18993,10 @@ for (const mode of ['ready', 'queued', 'stale', 'inactive', 'throws']) { (header::SET_COOKIE.as_str(), "AWSALB=abc123; Path=/"), ], ); - let services = build_services_with_http_client( - Arc::clone(&stub) as Arc - ); + let services = stub.services(); - let response = run_publisher_proxy(&settings, &services, publisher_page_request()).await; + let (response, _owner) = + run_publisher_proxy(&settings, &services, publisher_page_request()).await; assert_eq!( proxy_response_headers(&response) @@ -19561,6 +19007,34 @@ for (const mode of ['ready', 'queued', 'stale', 'inactive', 'throws']) { ); } + #[tokio::test] + async fn equivalent_host_override_redirect_with_cookie_stays_on_serving_host() { + let mut settings = create_test_settings(); + settings.publisher.origin_url = "https://example.com".to_owned(); + settings.publisher.origin_host_header_override = Some("example.com:443".to_owned()); + let stub = PublisherUpstream::for_origin("example.com"); + stub.queue_origin( + 301, + Vec::new(), + vec![ + (header::LOCATION.as_str(), "https://example.com/redirect"), + (header::SET_COOKIE.as_str(), "AWSALB=abc123; Path=/"), + ], + ); + let services = stub.services(); + + let (response, _owner) = + run_publisher_proxy(&settings, &services, publisher_page_request()).await; + + assert_eq!( + proxy_response_headers(&response) + .get(header::LOCATION) + .and_then(|value| value.to_str().ok()), + Some("https://publisher.example/redirect"), + "should treat an equivalent override as no host change and keep the redirect on the serving host" + ); + } + #[tokio::test] async fn publisher_asset_policy_response_with_cookie_is_private_after_finalization() { let settings = Settings::from_toml(&format!( @@ -19579,8 +19053,8 @@ for (const mode of ['ready', 'queued', 'stale', 'inactive', 'throws']) { crate_test_settings_str() )) .expect("should parse settings with cache rule"); - let stub = Arc::new(StubHttpClient::new()); - stub.push_response_with_headers( + let stub = PublisherUpstream::new(); + stub.queue_origin( 200, b"png".to_vec(), vec![ @@ -19589,9 +19063,7 @@ for (const mode of ['ready', 'queued', 'stale', 'inactive', 'throws']) { (header::SET_COOKIE.as_str(), "viewer=example; Path=/"), ], ); - let services = build_services_with_http_client( - Arc::clone(&stub) as Arc - ); + let services = stub.services(); let request = HttpRequest::builder() .method(Method::GET) .uri("https://publisher.example/assets/logo.0123abcd.png") @@ -19599,7 +19071,7 @@ for (const mode of ['ready', 'queued', 'stale', 'inactive', 'throws']) { .body(EdgeBody::empty()) .expect("should build request"); - let response = run_publisher_proxy(&settings, &services, request).await; + let (response, _owner) = run_publisher_proxy(&settings, &services, request).await; let PublisherResponse::PassThrough { mut response, .. } = response else { panic!("should pass through non-HTML asset response"); }; @@ -19640,8 +19112,8 @@ for (const mode of ['ready', 'queued', 'stale', 'inactive', 'throws']) { crate_test_settings_str() )) .expect("should parse settings with cache rule"); - let stub = Arc::new(StubHttpClient::new()); - stub.push_response_with_headers( + let stub = PublisherUpstream::new(); + stub.queue_origin( 200, b"news".to_vec(), vec![ @@ -19649,9 +19121,7 @@ for (const mode of ['ready', 'queued', 'stale', 'inactive', 'throws']) { (header::CACHE_CONTROL.as_str(), "public, max-age=60"), ], ); - let services = build_services_with_http_client( - Arc::clone(&stub) as Arc - ); + let services = stub.services(); let request = HttpRequest::builder() .method(Method::GET) .uri("https://publisher.example/news/story.0123abcd.html") @@ -19659,7 +19129,7 @@ for (const mode of ['ready', 'queued', 'stale', 'inactive', 'throws']) { .body(EdgeBody::empty()) .expect("should build request"); - let response = run_publisher_proxy(&settings, &services, request).await; + let (response, _owner) = run_publisher_proxy(&settings, &services, request).await; let response = match response { PublisherResponse::Stream { response, .. } | PublisherResponse::Buffered(response) => { response @@ -19685,126 +19155,38 @@ for (const mode of ['ready', 'queued', 'stale', 'inactive', 'throws']) { mod ssat_cache_policy_tests { use super::*; - use crate::auction::provider::{AuctionProvider, ProviderRequestOutcome}; + use crate::auction::telemetry::{AuctionEventBatch, AuctionTelemetrySink}; use crate::creative_opportunities::{CreativeOpportunityFormat, CreativeOpportunitySlot}; - use crate::platform::test_support::{ - NoopConfigStore, NoopGeo, NoopSecretStore, StubBackend, - }; - use crate::platform::{ - ClientInfo, PlatformError, PlatformHttpClient, PlatformPendingRequest, - PlatformResponse, PlatformSelectResult, - }; use crate::test_support::tests::crate_test_settings_str; const ORIGIN_ETAG: &str = "\"origin-tag\""; const ORIGIN_LAST_MODIFIED: &str = "Wed, 21 Oct 2015 07:28:00 GMT"; const UNEXPECTED_304_PROVIDER: &str = "example-navigation-bidder"; - const UNEXPECTED_304_BACKEND: &str = "example-navigation-bidder-backend"; - - struct DispatchingTestProvider; - - struct RangeAwareHttpClient { - stub: StubHttpClient, - } - impl RangeAwareHttpClient { - fn new() -> Self { - Self { - stub: StubHttpClient::new(), + /// An upstream whose origin answers a request that carries `Range` with a + /// partial 206 and any other request with the complete document. + fn range_aware_upstream() -> PublisherUpstream { + let upstream = PublisherUpstream::new(); + let script = upstream.client.clone(); + let origin_host = upstream.origin_host.clone(); + upstream.observe(move |request| { + if request.uri.host() != Some(origin_host.as_str()) { + return; } - } - } - - #[async_trait::async_trait(?Send)] - impl PlatformHttpClient for RangeAwareHttpClient { - async fn send( - &self, - request: PlatformHttpRequest, - ) -> Result> { - if request.request.headers().contains_key(header::RANGE) { - self.stub.push_response_with_headers( - 206, - b"partial".to_vec(), - vec![ - ("content-type", "text/html; charset=utf-8"), - ("content-range", "bytes 0-18/39"), - ], - ); + let reply = if request.headers.contains_key(header::RANGE) { + ScriptedReply::status(StatusCode::PARTIAL_CONTENT) + .header("content-type", "text/html; charset=utf-8") + .header("content-range", "bytes 0-18/39") + .body("partial") } else { - self.stub.push_response_with_headers( - 200, - b"origin".to_vec(), - vec![("content-type", "text/html; charset=utf-8")], - ); - } - self.stub.send(request).await - } - - async fn send_async( - &self, - request: PlatformHttpRequest, - ) -> Result> { - self.stub.send_async(request).await - } - - async fn select( - &self, - pending_requests: Vec, - ) -> Result> { - self.stub.select(pending_requests).await - } - } - - #[async_trait::async_trait(?Send)] - impl AuctionProvider for DispatchingTestProvider { - fn provider_name(&self) -> &str { - UNEXPECTED_304_PROVIDER - } - - async fn request_bids( - &self, - _request: &AuctionRequest, - context: &AuctionContext<'_>, - ) -> Result> { - let request = PlatformHttpRequest::new( - HttpRequest::builder() - .method(Method::POST) - .uri("https://bidder.example.com/navigation-bids") - .body(EdgeBody::empty()) - .expect("should build test provider request"), - UNEXPECTED_304_BACKEND, - ); - context - .services - .http_client() - .send_async(request.with_cache_bypass()) - .await - .change_context(TrustedServerError::Auction { - message: "test provider launch failed".to_string(), - }) - .map(ProviderRequestOutcome::pending) - } - - async fn parse_response( - &self, - _response: PlatformResponse, - _response_time_ms: u64, - ) -> Result> { - panic!("parse_response must not run for an unexpected origin 304"); - } - - fn timeout_ms(&self) -> u32 { - 100 - } - - fn backend_name( - &self, - _services: &RuntimeServices, - _timeout_ms: u32, - ) -> Option { - Some(UNEXPECTED_304_BACKEND.to_string()) - } + ScriptedReply::status(StatusCode::OK) + .header("content-type", "text/html; charset=utf-8") + .body("origin") + }; + script.respond_to(request.method.clone(), &request.uri.to_string(), reply); + }); + upstream } #[derive(Default)] @@ -19871,20 +19253,13 @@ for (const mode of ['ready', 'queued', 'stale', 'inactive', 'throws']) { } fn services_with_telemetry( - http_client: Arc, + upstream: &PublisherUpstream, telemetry_sink: Arc, ) -> RuntimeServices { - let telemetry_sink: Arc = telemetry_sink; - RuntimeServices::builder() - .config_store(Arc::new(NoopConfigStore)) - .secret_store(Arc::new(NoopSecretStore)) - .kv_store(Arc::new(edgezero_core::key_value_store::NoopKvStore)) - .backend(Arc::new(StubBackend)) - .http_client(http_client) - .geo(Arc::new(NoopGeo)) - .auction_telemetry_sink(telemetry_sink) - .client_info(ClientInfo::default()) - .build() + RuntimeServices { + auction_telemetry_sink: telemetry_sink, + ..upstream.services() + } } fn article_slot() -> CreativeOpportunitySlot { @@ -19919,18 +19294,18 @@ for (const mode of ['ready', 'queued', 'stale', 'inactive', 'throws']) { } fn queue_html_response_with_cache_control( - stub: &StubHttpClient, + stub: &PublisherUpstream, cache_control: &'static str, ) { queue_html_response_with_status_and_cache_control(stub, 200, cache_control); } fn queue_html_response_with_status_and_cache_control( - stub: &StubHttpClient, + stub: &PublisherUpstream, status: u16, cache_control: &'static str, ) { - stub.push_response_with_headers( + stub.queue_origin( status, b"origin".to_vec(), vec![ @@ -19958,7 +19333,7 @@ for (const mode of ['ready', 'queued', 'stale', 'inactive', 'throws']) { services: &RuntimeServices, slots: &[CreativeOpportunitySlot], req: Request, - ) -> PublisherResponse { + ) -> (PublisherResponse, OutboundScopeOwner) { run_with_slots_and_consent(settings, services, slots, req, non_regulated_consent()) .await } @@ -19969,8 +19344,9 @@ for (const mode of ['ready', 'queued', 'stale', 'inactive', 'throws']) { slots: &[CreativeOpportunitySlot], req: Request, consent: crate::consent::ConsentContext, - ) -> PublisherResponse { - let orchestrator = AuctionOrchestrator::new(settings.auction.clone()); + ) -> (PublisherResponse, OutboundScopeOwner) { + let orchestrator = crate::auction::build_orchestrator(settings) + .expect("should build the auction orchestrator"); run_with_orchestrator_and_consent( settings, services, @@ -19988,7 +19364,7 @@ for (const mode of ['ready', 'queued', 'stale', 'inactive', 'throws']) { orchestrator: &AuctionOrchestrator, slots: &[CreativeOpportunitySlot], req: Request, - ) -> PublisherResponse { + ) -> (PublisherResponse, OutboundScopeOwner) { run_with_orchestrator_and_consent( settings, services, @@ -20007,7 +19383,7 @@ for (const mode of ['ready', 'queued', 'stale', 'inactive', 'throws']) { slots: &[CreativeOpportunitySlot], req: Request, consent: crate::consent::ConsentContext, - ) -> PublisherResponse { + ) -> (PublisherResponse, OutboundScopeOwner) { let mut ec_context = EcContext::new_for_test(None, consent); handle_publisher_request( @@ -20022,7 +19398,11 @@ for (const mode of ['ready', 'queued', 'stale', 'inactive', 'throws']) { registry: None, }, req, - EdgeCacheHeader::SMaxageFallback, + PublisherOriginOptions { + edge_header: EdgeCacheHeader::SMaxageFallback, + input_cap: None, + response_decoding: ResponseDecoding::Original, + }, ) .await .expect("should proxy publisher request") @@ -20055,19 +19435,14 @@ for (const mode of ['ready', 'queued', 'stale', 'inactive', 'throws']) { #[tokio::test] async fn pending_origin_wait_failure_abandons_dispatched_auction_once() { let settings = settings_with_dispatching_provider(); - let mut orchestrator = AuctionOrchestrator::new(settings.auction.clone()); - orchestrator.register_provider(Arc::new(DispatchingTestProvider)); + let orchestrator = crate::auction::build_orchestrator(&settings) + .expect("should build the auction orchestrator"); let telemetry_sink = Arc::new(RecordingTelemetrySink::default()); - let stub = Arc::new(StubHttpClient::new()); - stub.set_streaming_responses_supported(true); - stub.set_pending_streaming_responses_supported(true); - queue_html_response_with_cache_control(&stub, "public, max-age=300"); - stub.push_response(200, b"unused provider response".to_vec()); - stub.push_wait_error(); - let services = services_with_telemetry( - Arc::clone(&stub) as Arc, - Arc::clone(&telemetry_sink), - ); + let stub = PublisherUpstream::new(); + stub.queue_origin_reply(ScriptedReply::error(EdgeError::bad_gateway( + "example origin failure", + ))); + let services = services_with_telemetry(&stub, Arc::clone(&telemetry_sink)); let graph = KvIdentityGraph::new(InMemoryEcKv::new("wait-error-store")); let mut ec_context = active_ec_context(); let slots = [article_slot()]; @@ -20084,7 +19459,11 @@ for (const mode of ['ready', 'queued', 'stale', 'inactive', 'throws']) { registry: None, }, conditional_navigation_request(), - EdgeCacheHeader::SurrogateControl, + PublisherOriginOptions { + edge_header: EdgeCacheHeader::SurrogateControl, + input_cap: None, + response_decoding: ResponseDecoding::Original, + }, ) .await { @@ -20124,20 +19503,15 @@ for (const mode of ['ready', 'queued', 'stale', 'inactive', 'throws']) { #[tokio::test] async fn pending_origin_start_failure_skips_kv_auction_and_telemetry() { let settings = settings_with_dispatching_provider(); - let mut orchestrator = AuctionOrchestrator::new(settings.auction.clone()); - orchestrator.register_provider(Arc::new(DispatchingTestProvider)); + let orchestrator = crate::auction::build_orchestrator(&settings) + .expect("should build the auction orchestrator"); let telemetry_sink = Arc::new(RecordingTelemetrySink::default()); - let stub = Arc::new(StubHttpClient::new()); - stub.set_streaming_responses_supported(true); - stub.set_pending_streaming_responses_supported(true); - let services = services_with_telemetry( - Arc::clone(&stub) as Arc, - Arc::clone(&telemetry_sink), - ); + let stub = PublisherUpstream::new(); + let services = services_with_telemetry(&stub, Arc::clone(&telemetry_sink)); let lookups = Arc::new(AtomicUsize::new(0)); let graph = KvIdentityGraph::new(OrderRecordingKv { inner: InMemoryEcKv::new("start-error-store"), - http: Arc::clone(&stub), + http: stub.clone(), http_calls_at_lookup: Arc::new(AtomicUsize::new(0)), lookups: Arc::clone(&lookups), }); @@ -20155,16 +19529,25 @@ for (const mode of ['ready', 'queued', 'stale', 'inactive', 'throws']) { slots: &slots, registry: None, }, - conditional_navigation_request(), - EdgeCacheHeader::SurrogateControl, + { + // A GET that carries body bytes cannot be forwarded, so the + // overlapped `begin` fails before any other work starts. + let mut request = conditional_navigation_request(); + *request.body_mut() = EdgeBody::from("example GET body"); + request + }, + PublisherOriginOptions { + edge_header: EdgeCacheHeader::SurrogateControl, + input_cap: None, + response_decoding: ResponseDecoding::Original, + }, ) .await; assert!(result.is_err(), "origin start failure should be returned"); assert_eq!(lookups.load(Ordering::SeqCst), 0, "should skip KV lookup"); - assert_eq!( - stub.recorded_backend_names(), - vec!["stub-backend"], + assert!( + stub.client.requests().is_empty(), "should not dispatch the auction provider after origin start fails" ); assert!( @@ -20191,15 +19574,13 @@ for (const mode of ['ready', 'queued', 'stale', 'inactive', 'throws']) { ); let orchestrator = crate::auction::build_orchestrator_with_plan(plan, &settings) .expect("should build signed plan-backed orchestrator"); - let stub = Arc::new(StubHttpClient::new()); + let stub = PublisherUpstream::new(); queue_html_response_with_cache_control(&stub, "public, max-age=300"); - let services = services_with_telemetry( - Arc::clone(&stub) as Arc, - Arc::new(RecordingTelemetrySink::default()), - ); + let services = + services_with_telemetry(&stub, Arc::new(RecordingTelemetrySink::default())); let slots = [article_slot()]; - let response = run_with_orchestrator( + let (response, _owner) = run_with_orchestrator( &settings, &services, &orchestrator, @@ -20210,9 +19591,14 @@ for (const mode of ['ready', 'queued', 'stale', 'inactive', 'throws']) { assert_eq!(response_head(response).status, StatusCode::OK); assert_eq!( - stub.recorded_backend_names(), - vec!["stub-backend".to_string()], - "signer admission failure should skip provider I/O and still fetch the publisher origin" + stub.client.requests().len(), + stub.origin_requests().len(), + "signer admission failure should skip provider I/O" + ); + assert_eq!( + stub.origin_requests().len(), + 1, + "signer admission failure should still fetch the publisher origin" ); } @@ -20220,25 +19606,23 @@ for (const mode of ['ready', 'queued', 'stale', 'inactive', 'throws']) { async fn eligible_navigation_bypasses_cache_and_returns_non_storable_html() { // Arrange let settings = settings_with_enabled_auction_and_creative_opportunities(); - let stub = Arc::new(StubHttpClient::new()); + let stub = PublisherUpstream::new(); queue_html_response_with_cache_control(&stub, "public, max-age=300"); - let services = build_services_with_http_client( - Arc::clone(&stub) as Arc - ); + let services = stub.services(); let slots = [article_slot()]; let req = conditional_navigation_request(); // Act - let response = run_with_slots(&settings, &services, &slots, req).await; + let (response, _owner) = run_with_slots(&settings, &services, &slots, req).await; let response_head = response_head(response); // Assert assert_eq!( - stub.recorded_cache_intents(), - vec![PlatformCacheIntent::Bypass], + stub.origin_cache_policies(), + vec![OutboundCachePolicy::Bypass], "eligible publisher navigation should bypass the platform cache" ); - let recorded_requests = stub.recorded_request_headers(); + let recorded_requests = stub.origin_headers(); let outbound_headers = recorded_requests .first() .expect("should record the outbound publisher request"); @@ -20279,10 +19663,8 @@ for (const mode of ['ready', 'queued', 'stale', 'inactive', 'throws']) { async fn eligible_range_navigation_fetches_complete_html() { // Arrange let settings = settings_with_enabled_auction_and_creative_opportunities(); - let http_client = Arc::new(RangeAwareHttpClient::new()); - let services = build_services_with_http_client( - Arc::clone(&http_client) as Arc - ); + let upstream = range_aware_upstream(); + let services = upstream.services(); let slots = [article_slot()]; let mut req = conditional_navigation_request(); req.headers_mut() @@ -20291,7 +19673,7 @@ for (const mode of ['ready', 'queued', 'stale', 'inactive', 'throws']) { .insert(header::IF_RANGE, HeaderValue::from_static(ORIGIN_ETAG)); // Act - let response = run_with_slots(&settings, &services, &slots, req).await; + let (response, _owner) = run_with_slots(&settings, &services, &slots, req).await; let response_head = response_head(response); // Assert @@ -20300,7 +19682,7 @@ for (const mode of ['ready', 'queued', 'stale', 'inactive', 'throws']) { StatusCode::OK, "eligible range navigation should fetch the complete origin document" ); - let recorded_requests = http_client.stub.recorded_request_headers(); + let recorded_requests = upstream.origin_headers(); let outbound_headers = recorded_requests .first() .expect("should record the outbound publisher request"); @@ -20317,11 +19699,9 @@ for (const mode of ['ready', 'queued', 'stale', 'inactive', 'throws']) { async fn navigation_without_matched_slots_uses_short_browser_cache_policy() { // Arrange let settings = settings_with_enabled_auction_and_creative_opportunities(); - let stub = Arc::new(StubHttpClient::new()); + let stub = PublisherUpstream::new(); queue_html_response_with_cache_control(&stub, "public, max-age=300"); - let services = build_services_with_http_client( - Arc::clone(&stub) as Arc - ); + let services = stub.services(); let mut req = conditional_navigation_request(); req.headers_mut() .insert(header::RANGE, HeaderValue::from_static("bytes=0-18")); @@ -20329,16 +19709,16 @@ for (const mode of ['ready', 'queued', 'stale', 'inactive', 'throws']) { .insert(header::IF_RANGE, HeaderValue::from_static(ORIGIN_ETAG)); // Act - let response = run_with_slots(&settings, &services, &[], req).await; + let (response, _owner) = run_with_slots(&settings, &services, &[], req).await; let response_head = response_head(response); // Assert assert_eq!( - stub.recorded_cache_intents(), - vec![PlatformCacheIntent::Default], + stub.origin_cache_policies(), + vec![OutboundCachePolicy::PlatformDefault], "should preserve platform caching for non-ad requests while readthrough is disabled", ); - let recorded_requests = stub.recorded_request_headers(); + let recorded_requests = stub.origin_headers(); let outbound_headers = recorded_requests .first() .expect("should record the outbound publisher request"); @@ -20401,15 +19781,13 @@ for (const mode of ['ready', 'queued', 'stale', 'inactive', 'throws']) { let mut settings = settings_with_disabled_ad_templates(); settings.proxy.allowed_domains = vec!["*.example".to_string(), "*.example.com".to_string()]; - let stub = Arc::new(StubHttpClient::new()); + let stub = PublisherUpstream::new(); queue_html_response_with_cache_control(&stub, "public, max-age=300"); - let services = build_services_with_http_client( - Arc::clone(&stub) as Arc - ); + let services = stub.services(); let slots = [article_slot()]; // Act - let response = run_with_slots( + let (response, _owner) = run_with_slots( &settings, &services, &slots, @@ -20418,7 +19796,8 @@ for (const mode of ['ready', 'queued', 'stale', 'inactive', 'throws']) { .await; let registry = IntegrationRegistry::new(&settings).expect("should create integration registry"); - let orchestrator = AuctionOrchestrator::new(settings.auction.clone()); + let orchestrator = crate::auction::build_orchestrator(&settings) + .expect("should build the auction orchestrator"); let response = buffer_publisher_response_async( response, &Method::GET, @@ -20426,6 +19805,7 @@ for (const mode of ['ready', 'queued', 'stale', 'inactive', 'throws']) { ®istry, &orchestrator, &services, + TEST_BUFFERED_INPUT_CAP, ) .await .expect("should buffer disabled-template response"); @@ -20439,8 +19819,8 @@ for (const mode of ['ready', 'queued', 'stale', 'inactive', 'throws']) { // Assert assert_eq!( - stub.recorded_cache_intents(), - vec![PlatformCacheIntent::Default], + stub.origin_cache_policies(), + vec![OutboundCachePolicy::PlatformDefault], "should preserve platform caching with disabled ad templates and readthrough" ); assert_eq!( @@ -20490,14 +19870,12 @@ for (const mode of ['ready', 'queued', 'stale', 'inactive', 'throws']) { async fn disabled_auction_uses_private_browser_cache_policy() { // Arrange let settings = settings_with_disabled_auction(); - let stub = Arc::new(StubHttpClient::new()); + let stub = PublisherUpstream::new(); queue_html_response_with_cache_control(&stub, "no-cache"); - let services = build_services_with_http_client( - Arc::clone(&stub) as Arc - ); + let services = stub.services(); // Act - let response = run_with_slots( + let (response, _owner) = run_with_slots( &settings, &services, &[article_slot()], @@ -20523,14 +19901,12 @@ for (const mode of ['ready', 'queued', 'stale', 'inactive', 'throws']) { for cache_control in ["no-cache", "max-age=0", "must-revalidate", "s-maxage=0"] { // Arrange - let stub = Arc::new(StubHttpClient::new()); + let stub = PublisherUpstream::new(); queue_html_response_with_cache_control(&stub, cache_control); - let services = build_services_with_http_client( - Arc::clone(&stub) as Arc - ); + let services = stub.services(); // Act - let response = + let (response, _owner) = run_with_slots(&settings, &services, &[], conditional_navigation_request()) .await; let response_head = response_head(response); @@ -20553,14 +19929,12 @@ for (const mode of ['ready', 'queued', 'stale', 'inactive', 'throws']) { for cache_control in ["private, max-age=0", "No-Store"] { // Arrange - let stub = Arc::new(StubHttpClient::new()); + let stub = PublisherUpstream::new(); queue_html_response_with_cache_control(&stub, cache_control); - let services = build_services_with_http_client( - Arc::clone(&stub) as Arc - ); + let services = stub.services(); // Act - let response = + let (response, _owner) = run_with_slots(&settings, &services, &[], conditional_navigation_request()) .await; let response_head = response_head(response); @@ -20633,14 +20007,12 @@ for (const mode of ['ready', 'queued', 'stale', 'inactive', 'throws']) { ), ] { // Arrange - let stub = Arc::new(StubHttpClient::new()); + let stub = PublisherUpstream::new(); queue_html_response_with_cache_control(&stub, "no-cache"); - let services = build_services_with_http_client( - Arc::clone(&stub) as Arc - ); + let services = stub.services(); // Act - let response = + let (response, _owner) = run_with_slots_and_consent(&settings, &services, &slots, request, consent) .await; let response_head = response_head(response); @@ -20661,14 +20033,12 @@ for (const mode of ['ready', 'queued', 'stale', 'inactive', 'throws']) { async fn absent_creative_opportunities_use_short_browser_cache_policy() { // Arrange let settings = settings_without_creative_opportunities(); - let stub = Arc::new(StubHttpClient::new()); + let stub = PublisherUpstream::new(); queue_html_response_with_cache_control(&stub, "no-cache"); - let services = build_services_with_http_client( - Arc::clone(&stub) as Arc - ); + let services = stub.services(); // Act - let response = + let (response, _owner) = run_with_slots(&settings, &services, &[], conditional_navigation_request()).await; let response_head = response_head(response); @@ -20689,14 +20059,12 @@ for (const mode of ['ready', 'queued', 'stale', 'inactive', 'throws']) { for status in [206, 404, 500, 503] { // Arrange - let stub = Arc::new(StubHttpClient::new()); + let stub = PublisherUpstream::new(); queue_html_response_with_status_and_cache_control(&stub, status, "no-cache"); - let services = build_services_with_http_client( - Arc::clone(&stub) as Arc - ); + let services = stub.services(); // Act - let response = run_with_slots( + let (response, _owner) = run_with_slots( &settings, &services, &[article_slot()], @@ -20725,11 +20093,9 @@ for (const mode of ['ready', 'queued', 'stale', 'inactive', 'throws']) { .integrations .insert_config("gpt_diagnostics", &serde_json::json!({ "enabled": true })) .expect("should enable GPT diagnostics"); - let stub = Arc::new(StubHttpClient::new()); + let stub = PublisherUpstream::new(); queue_html_response_with_cache_control(&stub, "no-cache"); - let services = build_services_with_http_client( - Arc::clone(&stub) as Arc - ); + let services = stub.services(); let request = HttpRequest::builder() .method(Method::GET) .uri("https://ts.example.com/article?ts_console=1") @@ -20739,7 +20105,8 @@ for (const mode of ['ready', 'queued', 'stale', 'inactive', 'throws']) { .expect("should build GPT diagnostics request"); // Act - let response = run_with_slots(&settings, &services, &[article_slot()], request).await; + let (response, _owner) = + run_with_slots(&settings, &services, &[article_slot()], request).await; let response_head = response_head(response); // Assert @@ -20774,14 +20141,12 @@ for (const mode of ['ready', 'queued', 'stale', 'inactive', 'throws']) { .expect("should build non-document request"), ] { // Arrange - let stub = Arc::new(StubHttpClient::new()); + let stub = PublisherUpstream::new(); queue_html_response_with_cache_control(&stub, "no-cache"); - let services = build_services_with_http_client( - Arc::clone(&stub) as Arc - ); + let services = stub.services(); // Act - let response = + let (response, _owner) = run_with_slots(&settings, &services, &[article_slot()], request).await; let response_head = response_head(response); @@ -20802,14 +20167,11 @@ for (const mode of ['ready', 'queued', 'stale', 'inactive', 'throws']) { for content_type in [None, Some("text/html; charset=utf-8")] { // Arrange let settings = settings_with_dispatching_provider(); - let mut orchestrator = AuctionOrchestrator::new(settings.auction.clone()); - orchestrator.register_provider(Arc::new(DispatchingTestProvider)); + let orchestrator = crate::auction::build_orchestrator(&settings) + .expect("should build the auction orchestrator"); let telemetry_sink = Arc::new(RecordingTelemetrySink::default()); - let stub = Arc::new(StubHttpClient::new()); + let stub = PublisherUpstream::new(); - // `send_async` consumes the first response before the publisher - // origin request consumes the second response. - stub.push_response(200, b"unused provider response".to_vec()); let mut origin_headers = vec![ ("cache-control", "public, max-age=300"), ("etag", ORIGIN_ETAG), @@ -20820,15 +20182,12 @@ for (const mode of ['ready', 'queued', 'stale', 'inactive', 'throws']) { if let Some(content_type) = content_type { origin_headers.push(("content-type", content_type)); } - stub.push_response_with_headers(304, Vec::new(), origin_headers); - let services = services_with_telemetry( - Arc::clone(&stub) as Arc, - Arc::clone(&telemetry_sink), - ); + stub.queue_origin(304, Vec::new(), origin_headers); + let services = services_with_telemetry(&stub, Arc::clone(&telemetry_sink)); let slots = [article_slot()]; // Act - let response = run_with_orchestrator( + let (response, _owner) = run_with_orchestrator( &settings, &services, &orchestrator, @@ -20909,8 +20268,8 @@ for (const mode of ['ready', 'queued', 'stale', 'inactive', 'throws']) { async fn inactive_ad_stack_304_uses_short_browser_cache_policy() { // Arrange let settings = settings_with_enabled_auction_and_creative_opportunities(); - let stub = Arc::new(StubHttpClient::new()); - stub.push_response_with_headers( + let stub = PublisherUpstream::new(); + stub.queue_origin( 304, Vec::new(), vec![ @@ -20921,12 +20280,10 @@ for (const mode of ['ready', 'queued', 'stale', 'inactive', 'throws']) { ("fastly-surrogate-control", "max-age=300"), ], ); - let services = build_services_with_http_client( - Arc::clone(&stub) as Arc - ); + let services = stub.services(); // Act - let response = + let (response, _owner) = run_with_slots(&settings, &services, &[], conditional_navigation_request()).await; // Assert @@ -20973,11 +20330,11 @@ for (const mode of ['ready', 'queued', 'stale', 'inactive', 'throws']) { ); } assert_eq!( - stub.recorded_cache_intents(), - vec![PlatformCacheIntent::Default], + stub.origin_cache_policies(), + vec![OutboundCachePolicy::PlatformDefault], "should preserve platform revalidation for non-ad requests with readthrough disabled" ); - let recorded_requests = stub.recorded_request_headers(); + let recorded_requests = stub.origin_headers(); let outbound_headers = recorded_requests .first() .expect("should record the outbound publisher request"); @@ -20999,18 +20356,16 @@ for (const mode of ['ready', 'queued', 'stale', 'inactive', 'throws']) { for cache_control in ["private, max-age=0", "No-Store"] { // Arrange - let stub = Arc::new(StubHttpClient::new()); - stub.push_response_with_headers( + let stub = PublisherUpstream::new(); + stub.queue_origin( 304, Vec::new(), vec![("cache-control", cache_control), ("etag", ORIGIN_ETAG)], ); - let services = build_services_with_http_client( - Arc::clone(&stub) as Arc - ); + let services = stub.services(); // Act - let response = + let (response, _owner) = run_with_slots(&settings, &services, &[], conditional_navigation_request()) .await; let response_head = response_head(response); @@ -21029,13 +20384,11 @@ for (const mode of ['ready', 'queued', 'stale', 'inactive', 'throws']) { } #[tokio::test] - async fn publisher_request_uses_platform_http_client_with_http_types() { + async fn publisher_request_reaches_the_origin_through_the_outbound_client() { let settings = create_test_settings(); - let stub = Arc::new(StubHttpClient::new()); - stub.push_response(200, b"origin response".to_vec()); - let services = build_services_with_http_client( - Arc::clone(&stub) as Arc - ); + let stub = PublisherUpstream::new(); + stub.queue_origin(200, b"origin response".to_vec(), Vec::new()); + let services = stub.services(); let req = HttpRequest::builder() .method(Method::GET) .uri("https://publisher.example/page") @@ -21043,7 +20396,8 @@ for (const mode of ['ready', 'queued', 'stale', 'inactive', 'throws']) { .body(EdgeBody::empty()) .expect("should build request"); - let response = match run_publisher_proxy(&settings, &services, req).await { + let (publisher_response, _owner) = run_publisher_proxy(&settings, &services, req).await; + let response = match publisher_response { PublisherResponse::Buffered(r) => r, PublisherResponse::PassThrough { mut response, body } => { *response.body_mut() = body; @@ -21056,24 +20410,22 @@ for (const mode of ['ready', 'queued', 'stale', 'inactive', 'throws']) { assert_eq!(response.status(), StatusCode::OK); assert_eq!(response_body_string(response), "origin response"); assert_eq!( - stub.recorded_backend_names(), - vec!["stub-backend".to_string()], - "should proxy through the platform http client" + stub.origin_requests().len(), + 1, + "should proxy through the outbound client" ); } #[tokio::test] async fn suppressed_navigation_removes_conditional_and_range_headers() { let settings = create_test_settings(); - let stub = Arc::new(StubHttpClient::new()); - stub.push_response_with_headers( + let stub = PublisherUpstream::new(); + stub.queue_origin( 200, b"origin".to_vec(), vec![("content-type", "text/html; charset=utf-8")], ); - let services = build_services_with_http_client( - Arc::clone(&stub) as Arc - ); + let services = stub.services(); let mut req = HttpRequest::builder() .method(Method::GET) .uri("https://publisher.example/page") @@ -21088,10 +20440,10 @@ for (const mode of ['ready', 'queued', 'stale', 'inactive', 'throws']) { req.extensions_mut() .insert(crate::integrations::datadome::DataDomeClientTagSuppressed); - let _response = run_publisher_proxy(&settings, &services, req).await; + let (_response, _owner) = run_publisher_proxy(&settings, &services, req).await; let headers = stub - .recorded_request_headers() + .origin_headers() .into_iter() .next() .expect("should record one outbound request"); @@ -21113,15 +20465,13 @@ for (const mode of ['ready', 'queued', 'stale', 'inactive', 'throws']) { #[tokio::test] async fn suppressed_iframe_removes_conditional_and_range_headers() { let settings = create_test_settings(); - let stub = Arc::new(StubHttpClient::new()); - stub.push_response_with_headers( + let stub = PublisherUpstream::new(); + stub.queue_origin( 200, b"frame".to_vec(), vec![("content-type", "text/html; charset=utf-8")], ); - let services = build_services_with_http_client( - Arc::clone(&stub) as Arc - ); + let services = stub.services(); let mut req = HttpRequest::builder() .method(Method::GET) .uri("https://publisher.example/frame") @@ -21136,10 +20486,10 @@ for (const mode of ['ready', 'queued', 'stale', 'inactive', 'throws']) { req.extensions_mut() .insert(crate::integrations::datadome::DataDomeClientTagSuppressed); - let _response = run_publisher_proxy(&settings, &services, req).await; + let (_response, _owner) = run_publisher_proxy(&settings, &services, req).await; let headers = stub - .recorded_request_headers() + .origin_headers() .into_iter() .next() .expect("should record one outbound request"); @@ -21161,15 +20511,9 @@ for (const mode of ['ready', 'queued', 'stale', 'inactive', 'throws']) { #[tokio::test] async fn suppressed_subresource_preserves_conditional_and_range_headers() { let settings = create_test_settings(); - let stub = Arc::new(StubHttpClient::new()); - stub.push_response_with_headers( - 200, - b"video".to_vec(), - vec![("content-type", "video/mp4")], - ); - let services = build_services_with_http_client( - Arc::clone(&stub) as Arc - ); + let stub = PublisherUpstream::new(); + stub.queue_origin(200, b"video".to_vec(), vec![("content-type", "video/mp4")]); + let services = stub.services(); let mut req = HttpRequest::builder() .method(Method::GET) .uri("https://publisher.example/video.mp4") @@ -21184,10 +20528,10 @@ for (const mode of ['ready', 'queued', 'stale', 'inactive', 'throws']) { req.extensions_mut() .insert(crate::integrations::datadome::DataDomeClientTagSuppressed); - let _response = run_publisher_proxy(&settings, &services, req).await; + let (_response, _owner) = run_publisher_proxy(&settings, &services, req).await; let headers = stub - .recorded_request_headers() + .origin_headers() .into_iter() .next() .expect("should record one outbound request"); @@ -21208,63 +20552,6 @@ for (const mode of ['ready', 'queued', 'stale', 'inactive', 'throws']) { } } - #[tokio::test] - async fn publisher_origin_fetch_leaves_stream_response_disabled_when_unsupported() { - let settings = create_test_settings(); - let stub = Arc::new(StubHttpClient::new()); - stub.push_response_with_headers( - 200, - b"origin".to_vec(), - vec![("content-type", "text/html; charset=utf-8")], - ); - let services = build_services_with_http_client( - Arc::clone(&stub) as Arc - ); - let req = HttpRequest::builder() - .method(Method::GET) - .uri("https://publisher.example/page") - .header(header::HOST, "publisher.example") - .body(EdgeBody::empty()) - .expect("should build request"); - - let _ = run_publisher_proxy(&settings, &services, req).await; - - assert_eq!( - stub.recorded_stream_response_flags(), - vec![false], - "publisher origin fetch must not request streams when the platform does not support them" - ); - } - - #[tokio::test] - async fn publisher_origin_fetch_sets_stream_response_when_supported() { - let settings = create_test_settings(); - let stub = Arc::new(StubHttpClient::new()); - stub.set_streaming_responses_supported(true); - stub.push_response_with_headers( - 200, - b"origin".to_vec(), - vec![("content-type", "text/html; charset=utf-8")], - ); - let services = build_services_with_http_client( - Arc::clone(&stub) as Arc - ); - let req = HttpRequest::builder() - .method(Method::GET) - .uri("https://publisher.example/page") - .header(header::HOST, "publisher.example") - .body(EdgeBody::empty()) - .expect("should build request"); - - let _ = run_publisher_proxy(&settings, &services, req).await; - - assert_eq!( - stub.recorded_stream_response_flags(), - vec![true], - "publisher origin fetch should request streams when the platform supports them" - ); - } - #[tokio::test] async fn handle_publisher_request_does_not_self_generate_ec() { // EC generation is the adapter's real-browser-gated responsibility. This @@ -21275,11 +20562,9 @@ for (const mode of ['ready', 'queued', 'stale', 'inactive', 'throws']) { // exactly the conditions under which the old inline call would have // generated one. let settings = create_test_settings(); - let stub = Arc::new(StubHttpClient::new()); - stub.push_response(200, b"ok".to_vec()); - let services = build_services_with_http_client( - Arc::clone(&stub) as Arc - ); + let stub = PublisherUpstream::new(); + stub.queue_origin(200, b"ok".to_vec(), Vec::new()); + let services = stub.services(); let consent = crate::consent::ConsentContext { jurisdiction: crate::consent::jurisdiction::Jurisdiction::NonRegulated, @@ -21292,7 +20577,8 @@ for (const mode of ['ready', 'queued', 'stale', 'inactive', 'throws']) { "test precondition: consent must allow EC creation" ); - let orchestrator = AuctionOrchestrator::new(settings.auction.clone()); + let orchestrator = crate::auction::build_orchestrator(&settings) + .expect("should build the auction orchestrator"); let req = HttpRequest::builder() .method(Method::GET) .uri("https://publisher.example/article") @@ -21301,7 +20587,7 @@ for (const mode of ['ready', 'queued', 'stale', 'inactive', 'throws']) { .body(EdgeBody::empty()) .expect("should build request"); - let _ = handle_publisher_request( + let (_, _owner) = handle_publisher_request( &settings, &services, None, @@ -21313,7 +20599,11 @@ for (const mode of ['ready', 'queued', 'stale', 'inactive', 'throws']) { registry: None, }, req, - EdgeCacheHeader::SMaxageFallback, + PublisherOriginOptions { + edge_header: EdgeCacheHeader::SMaxageFallback, + input_cap: None, + response_decoding: ResponseDecoding::Original, + }, ) .await .expect("should proxy publisher request"); @@ -21343,17 +20633,19 @@ for (const mode of ['ready', 'queued', 'stale', 'inactive', 'throws']) { .expect("should configure DataDome integration"); let registry = IntegrationRegistry::new(&settings) .expect("should create integration registry with DataDome"); - let stub = Arc::new(StubHttpClient::new()); - stub.push_response_with_headers( + let stub = PublisherUpstream::new(); + stub.queue_origin( 200, b"content".to_vec(), vec![("content-type", "text/html; charset=utf-8")], ); - let services = build_services_with_secret_http_client_and_client_ip( - NoopSecretStore, - Arc::clone(&stub) as Arc, - Some("192.0.2.10".parse().expect("should parse client IP")), - ); + let services = RuntimeServices { + client_info: ClientInfo { + client_ip: Some("192.0.2.10".parse().expect("should parse client IP")), + ..ClientInfo::default() + }, + ..stub.services() + }; let mut req = HttpRequest::builder() .method(Method::GET) .uri("https://publisher.example/page") @@ -21375,14 +20667,16 @@ for (const mode of ['ready', 'queued', 'stale', 'inactive', 'throws']) { filter_outcome, crate::integrations::RequestFilterRegistryOutcome::Continue(_) )); - let publisher_response = run_publisher_proxy(&settings, &services, req).await; + let (publisher_response, _owner) = run_publisher_proxy(&settings, &services, req).await; let response = buffer_publisher_response_async( publisher_response, &Method::GET, &settings, ®istry, - &AuctionOrchestrator::new(settings.auction.clone()), + &crate::auction::build_orchestrator(&settings) + .expect("should build the auction orchestrator"), &services, + TEST_BUFFERED_INPUT_CAP, ) .await .expect("should buffer publisher response"); @@ -21391,7 +20685,7 @@ for (const mode of ['ready', 'queued', 'stale', 'inactive', 'throws']) { assert!(!html.contains("window.ddjskey")); assert!(!html.contains("/integrations/datadome/tags.js")); assert_eq!( - stub.recorded_backend_names().len(), + stub.client.requests().len(), 1, "only the publisher origin should be called" ); @@ -21685,14 +20979,10 @@ for (const mode of ['ready', 'queued', 'stale', 'inactive', 'throws']) { } fn response_body_string(response: http::Response) -> String { - String::from_utf8( - response - .into_body() - .into_bytes() - .unwrap_or_default() - .to_vec(), - ) - .expect("response body should be valid UTF-8") + let bytes = + futures::executor::block_on(response.into_body().into_bytes_bounded(usize::MAX)) + .expect("should read the response body"); + String::from_utf8(bytes.to_vec()).expect("response body should be valid UTF-8") } #[test] @@ -21757,7 +21047,8 @@ for (const mode of ['ready', 'queued', 'stale', 'inactive', 'throws']) { async fn parser_seam_loop_collects_before_writing_post_body_chunks() { let settings = create_test_settings(); let services = noop_services(); - let orchestrator = AuctionOrchestrator::new(settings.auction.clone()); + let orchestrator = crate::auction::build_orchestrator(&settings) + .expect("should build the auction orchestrator"); let dispatched = DispatchedAuction::empty_for_test(test_auction_request(), 500); let read_count = Arc::new(AtomicUsize::new(0)); let body_close_processed_at = Arc::new(AtomicUsize::new(0)); @@ -21789,6 +21080,7 @@ for (const mode of ['ready', 'queued', 'stale', 'inactive', 'throws']) { settings: &settings, request_origin: String::new(), }, + max_input_bytes: settings.publisher.max_buffered_body_bytes, }; let mut output = Vec::new(); @@ -21841,7 +21133,8 @@ for (const mode of ['ready', 'queued', 'stale', 'inactive', 'throws']) { let settings = create_test_settings(); let sink = Arc::new(RecordingTelemetrySink::default()); let services = noop_services_with_telemetry_sink(Arc::clone(&sink) as _); - let orchestrator = AuctionOrchestrator::new(settings.auction.clone()); + let orchestrator = crate::auction::build_orchestrator(&settings) + .expect("should build the auction orchestrator"); let ad_bids_state = AdBidsState::default(); let ec_context = EcContext::new_for_test(None, ConsentContext::default()); let ctx = AuctionCollectCtx { @@ -21865,6 +21158,7 @@ for (const mode of ['ready', 'queued', 'stale', 'inactive', 'throws']) { settings: &settings, request_origin: String::new(), }, + max_input_bytes: settings.publisher.max_buffered_body_bytes, }; let mut processor = RecordingProcessor { read_count: Arc::new(AtomicUsize::new(0)), @@ -21921,7 +21215,8 @@ for (const mode of ['ready', 'queued', 'stale', 'inactive', 'throws']) { let settings = create_test_settings(); let sink = Arc::new(RecordingTelemetrySink::default()); let services = noop_services_with_telemetry_sink(Arc::clone(&sink) as _); - let orchestrator = AuctionOrchestrator::new(settings.auction.clone()); + let orchestrator = crate::auction::build_orchestrator(&settings) + .expect("should build the auction orchestrator"); let ad_bids_state = AdBidsState::default(); let ec_context = EcContext::new_for_test(None, ConsentContext::default()); let ctx = AuctionCollectCtx { @@ -21945,6 +21240,7 @@ for (const mode of ['ready', 'queued', 'stale', 'inactive', 'throws']) { settings: &settings, request_origin: String::new(), }, + max_input_bytes: settings.publisher.max_buffered_body_bytes, }; let mut processor = RecordingProcessor { read_count: Arc::new(AtomicUsize::new(0)), @@ -22017,7 +21313,8 @@ for (const mode of ['ready', 'queued', 'stale', 'inactive', 'throws']) { // remains pending. let settings = create_test_settings(); let services = noop_services(); - let orchestrator = AuctionOrchestrator::new(settings.auction.clone()); + let orchestrator = crate::auction::build_orchestrator(&settings) + .expect("should build the auction orchestrator"); let ad_bids_state = AdBidsState::default(); let token = b""; let mut state = AuctionHoldState::new( @@ -22913,26 +22210,21 @@ for (const mode of ['ready', 'queued', 'stale', 'inactive', 'throws']) { settings.publisher.origin_url = "https://origin.example.com".to_owned(); let registry = IntegrationRegistry::new(&settings) .expect("should create integration registry"); - let stub = Arc::new(StubHttpClient::new()); - stub.push_response_with_headers( + let stub = PublisherUpstream::for_origin("origin.example.com"); + stub.queue_origin( 200, b"Article".to_vec(), vec![("content-type", "text/html; charset=utf-8")], ); // Unauthenticated forwarded fields no longer set the scheme, so // HTTPS comes from the platform TLS facts. - let services = crate::platform::RuntimeServices::builder() - .config_store(Arc::new(crate::platform::test_support::NoopConfigStore)) - .secret_store(Arc::new(crate::platform::test_support::NoopSecretStore)) - .kv_store(Arc::new(edgezero_core::key_value_store::NoopKvStore)) - .backend(Arc::new(crate::platform::test_support::StubBackend)) - .http_client(Arc::clone(&stub) as Arc) - .geo(Arc::new(crate::platform::test_support::NoopGeo)) - .client_info(ClientInfo { + let services = RuntimeServices { + client_info: ClientInfo { tls_protocol: Some("TLSv1.3".to_owned()), ..ClientInfo::default() - }) - .build(); + }, + ..stub.services() + }; let mut req = HttpRequest::builder() .version(version) .uri(uri) @@ -22957,14 +22249,17 @@ for (const mode of ['ready', 'queued', 'stale', 'inactive', 'throws']) { "should have no nonempty Host header" ); - let publisher_response = run_publisher_proxy(&settings, &services, req).await; + let (publisher_response, _owner) = + run_publisher_proxy(&settings, &services, req).await; let response = buffer_publisher_response_async( publisher_response, &Method::GET, &settings, ®istry, - &AuctionOrchestrator::new(settings.auction.clone()), + &crate::auction::build_orchestrator(&settings) + .expect("should build the auction orchestrator"), &services, + TEST_BUFFERED_INPUT_CAP, ) .await .expect("should buffer processed publisher response"); @@ -22987,40 +22282,6 @@ for (const mode of ['ready', 'queued', 'stale', 'inactive', 'throws']) { } } - #[tokio::test] - async fn publisher_request_sends_configured_host_header_override() { - let mut settings = create_test_settings(); - settings.publisher.origin_host_header_override = Some("www.example.com".to_string()); - let stub = Arc::new(StubHttpClient::new()); - stub.push_response(200, b"origin response".to_vec()); - let services = build_services_with_http_client( - Arc::clone(&stub) as Arc - ); - let req = HttpRequest::builder() - .method(Method::GET) - .uri("https://publisher.example/page") - .header(header::HOST, "publisher.example") - .body(EdgeBody::empty()) - .expect("should build request"); - - let _ = run_publisher_proxy(&settings, &services, req).await; - - let recorded_headers = stub.recorded_request_headers(); - let outbound_headers = recorded_headers - .first() - .expect("should record one outbound request"); - let outbound_host = outbound_headers - .iter() - .find(|(name, _)| name.eq_ignore_ascii_case("host")) - .map(|(_, value)| value.as_str()); - - assert_eq!( - outbound_host, - Some("www.example.com"), - "should send configured host override to outbound request" - ); - } - #[test] fn stream_publisher_body_preserves_gzip_round_trip() { use flate2::write::GzEncoder; @@ -23050,6 +22311,7 @@ for (const mode of ['ready', 'queued', 'stale', 'inactive', 'throws']) { let params = OwnedProcessResponseParams { response_status: StatusCode::OK, range_associated: false, + input_cap: None, csp_nonce_observed: None, template_cache_key: None, seam_ad_slots: None, @@ -23112,6 +22374,7 @@ for (const mode of ['ready', 'queued', 'stale', 'inactive', 'throws']) { let params = OwnedProcessResponseParams { response_status: StatusCode::OK, range_associated: false, + input_cap: None, csp_nonce_observed: None, template_cache_key: None, seam_ad_slots: None, @@ -23163,6 +22426,7 @@ for (const mode of ['ready', 'queued', 'stale', 'inactive', 'throws']) { let params = OwnedProcessResponseParams { response_status: StatusCode::OK, range_associated: false, + input_cap: None, csp_nonce_observed: None, template_cache_key: None, seam_ad_slots: None, @@ -23183,7 +22447,7 @@ for (const mode of ['ready', 'queued', 'stale', 'inactive', 'throws']) { trace_bootstrap: None, suppress_datadome_client_side_tag: false, }; - let body = EdgeBody::from_stream(futures::stream::iter(vec![Ok::<_, io::Error>( + let body = EdgeBody::from_external_stream(futures::stream::iter(vec![Ok::<_, io::Error>( bytes::Bytes::from_static(b"live"), )])); let mut output = Vec::new(); @@ -23287,11 +22551,13 @@ for (const mode of ['ready', 'queued', 'stale', 'inactive', 'throws']) { ), ) .expect("should create integration registry"); - let orchestrator = AuctionOrchestrator::new(settings.auction.clone()); + let orchestrator = crate::auction::build_orchestrator(&settings) + .expect("should build the auction orchestrator"); let services = noop_services(); let mut params = OwnedProcessResponseParams { response_status: StatusCode::OK, range_associated: false, + input_cap: None, csp_nonce_observed: None, template_cache_key: None, seam_ad_slots: None, @@ -23354,11 +22620,13 @@ for (const mode of ['ready', 'queued', 'stale', 'inactive', 'throws']) { ), ) .expect("should create integration registry"); - let orchestrator = AuctionOrchestrator::new(settings.auction.clone()); + let orchestrator = crate::auction::build_orchestrator(&settings) + .expect("should build the auction orchestrator"); let services = noop_services(); let mut params = OwnedProcessResponseParams { response_status: StatusCode::OK, range_associated: false, + input_cap: None, csp_nonce_observed: None, template_cache_key: None, seam_ad_slots: None, @@ -23424,11 +22692,13 @@ for (const mode of ['ready', 'queued', 'stale', 'inactive', 'throws']) { ), ) .expect("should create integration registry"); - let orchestrator = AuctionOrchestrator::new(settings.auction.clone()); + let orchestrator = crate::auction::build_orchestrator(&settings) + .expect("should build the auction orchestrator"); let services = noop_services(); let mut params = OwnedProcessResponseParams { response_status: StatusCode::OK, range_associated: false, + input_cap: None, csp_nonce_observed: None, template_cache_key: None, seam_ad_slots: None, @@ -23494,11 +22764,13 @@ for (const mode of ['ready', 'queued', 'stale', 'inactive', 'throws']) { ), ) .expect("should create integration registry"); - let orchestrator = AuctionOrchestrator::new(settings.auction.clone()); + let orchestrator = crate::auction::build_orchestrator(&settings) + .expect("should build the auction orchestrator"); let services = noop_services(); let mut params = OwnedProcessResponseParams { response_status: StatusCode::OK, range_associated: false, + input_cap: None, csp_nonce_observed: None, template_cache_key: None, seam_ad_slots: None, @@ -23564,11 +22836,13 @@ for (const mode of ['ready', 'queued', 'stale', 'inactive', 'throws']) { ), ) .expect("should create integration registry"); - let orchestrator = AuctionOrchestrator::new(settings.auction.clone()); + let orchestrator = crate::auction::build_orchestrator(&settings) + .expect("should build the auction orchestrator"); let services = noop_services(); let mut params = OwnedProcessResponseParams { response_status: StatusCode::OK, range_associated: false, + input_cap: None, csp_nonce_observed: None, template_cache_key: None, seam_ad_slots: None, @@ -23621,6 +22895,7 @@ for (const mode of ['ready', 'queued', 'stale', 'inactive', 'throws']) { OwnedProcessResponseParams { response_status: StatusCode::OK, range_associated: false, + input_cap: None, csp_nonce_observed: None, template_cache_key: None, seam_ad_slots: None, @@ -23655,7 +22930,8 @@ for (const mode of ['ready', 'queued', 'stale', 'inactive', 'throws']) { ), ) .expect("should create integration registry"); - let orchestrator = AuctionOrchestrator::new(settings.auction.clone()); + let orchestrator = crate::auction::build_orchestrator(&settings) + .expect("should build the auction orchestrator"); let services = noop_services(); let mut params = non_html_stream_params("gzip"); let compressed = @@ -23698,7 +22974,8 @@ for (const mode of ['ready', 'queued', 'stale', 'inactive', 'throws']) { ), ) .expect("should create integration registry"); - let orchestrator = AuctionOrchestrator::new(settings.auction.clone()); + let orchestrator = crate::auction::build_orchestrator(&settings) + .expect("should build the auction orchestrator"); let services = noop_services(); let mut params = non_html_stream_params("deflate"); let compressed = @@ -23746,7 +23023,8 @@ for (const mode of ['ready', 'queued', 'stale', 'inactive', 'throws']) { ), ) .expect("should create integration registry"); - let orchestrator = AuctionOrchestrator::new(settings.auction.clone()); + let orchestrator = crate::auction::build_orchestrator(&settings) + .expect("should build the auction orchestrator"); let services = noop_services(); let mut params = non_html_stream_params("gzip"); let compressed = gzip_encode(&vec![b'a'; 64 * 1024]); @@ -23796,7 +23074,7 @@ for (const mode of ['ready', 'queued', 'stale', 'inactive', 'throws']) { b"chunk", )))) }); - let body = EdgeBody::from_stream(stream); + let body = EdgeBody::from_external_stream(stream); let mut source = BodyChunkSource::new(body, STREAM_CHUNK_SIZE); { @@ -23835,12 +23113,14 @@ for (const mode of ['ready', 'queued', 'stale', 'inactive', 'throws']) { ), ) .expect("should create integration registry"); - let orchestrator = AuctionOrchestrator::new(settings.auction.clone()); + let orchestrator = crate::auction::build_orchestrator(&settings) + .expect("should build the auction orchestrator"); let services = noop_services(); let state = AdBidsState::default(); let mut params = OwnedProcessResponseParams { response_status: StatusCode::OK, range_associated: false, + input_cap: None, csp_nonce_observed: None, template_cache_key: None, seam_ad_slots: None, @@ -23913,12 +23193,14 @@ for (const mode of ['ready', 'queued', 'stale', 'inactive', 'throws']) { ), ) .expect("should create integration registry"); - let orchestrator = AuctionOrchestrator::new(settings.auction.clone()); + let orchestrator = crate::auction::build_orchestrator(&settings) + .expect("should build the auction orchestrator"); let services = noop_services(); let state = AdBidsState::default(); let mut params = OwnedProcessResponseParams { response_status: StatusCode::OK, range_associated: false, + input_cap: None, csp_nonce_observed: None, template_cache_key: None, seam_ad_slots: None, @@ -23994,11 +23276,13 @@ for (const mode of ['ready', 'queued', 'stale', 'inactive', 'throws']) { ), ) .expect("should create integration registry"); - let orchestrator = AuctionOrchestrator::new(settings.auction.clone()); + let orchestrator = crate::auction::build_orchestrator(&settings) + .expect("should build the auction orchestrator"); let services = noop_services(); let mut params = OwnedProcessResponseParams { response_status: StatusCode::OK, range_associated: false, + input_cap: None, csp_nonce_observed: None, template_cache_key: None, seam_ad_slots: None, @@ -24063,7 +23347,10 @@ for (const mode of ['ready', 'queued', 'stale', 'inactive', 'throws']) { ) .expect("should create integration registry"), ); - let orchestrator = Arc::new(AuctionOrchestrator::new(settings.auction.clone())); + let orchestrator = Arc::new( + crate::auction::build_orchestrator(&settings) + .expect("should build the auction orchestrator"), + ); let services = noop_services(); let response = Response::builder() .status(StatusCode::OK) @@ -24073,6 +23360,7 @@ for (const mode of ['ready', 'queued', 'stale', 'inactive', 'throws']) { let params = OwnedProcessResponseParams { response_status: StatusCode::OK, range_associated: false, + input_cap: None, csp_nonce_observed: None, template_cache_key: None, seam_ad_slots: None, @@ -24110,7 +23398,7 @@ for (const mode of ['ready', 'queued', 'stale', 'inactive', 'throws']) { .expect("should build streaming response"); assert!( - matches!(response.body(), EdgeBody::Stream(_)), + response.body().is_stream(), "streaming finalize should keep a lazy Body::Stream" ); @@ -24147,7 +23435,7 @@ for (const mode of ['ready', 'queued', 'stale', 'inactive', 'throws']) { /// the document head but not yet finished the response. fn origin_chunk_then_pending(chunk: bytes::Bytes) -> EdgeBody { let mut sent = false; - EdgeBody::from_stream(futures::stream::poll_fn(move |_cx| { + EdgeBody::from_external_stream(futures::stream::poll_fn(move |_cx| { if sent { return std::task::Poll::Pending; } @@ -24256,7 +23544,8 @@ for (const mode of ['ready', 'queued', 'stale', 'inactive', 'throws']) { &mut params, &settings, ®istry, - &AuctionOrchestrator::new(settings.auction.clone()), + &crate::auction::build_orchestrator(&settings) + .expect("should build the auction orchestrator"), &noop_services(), )) .expect_err("should reject inconsistent EOF declarations before input effects"); @@ -24309,7 +23598,8 @@ for (const mode of ['ready', 'queued', 'stale', 'inactive', 'throws']) { bytes: Arc::clone(&bytes), flushed: Arc::clone(&flushed), }; - let orchestrator = AuctionOrchestrator::new(settings.auction.clone()); + let orchestrator = crate::auction::build_orchestrator(&settings) + .expect("should build the auction orchestrator"); let mut params = html_stream_params("", None); let services = noop_services(); let mut work = Box::pin(stream_publisher_body_async( @@ -24386,11 +23676,14 @@ for (const mode of ['ready', 'queued', 'stale', 'inactive', 'throws']) { &Method::GET, Arc::clone(&settings), ®istry, - Arc::new(AuctionOrchestrator::new(settings.auction.clone())), + Arc::new( + crate::auction::build_orchestrator(&settings) + .expect("should build the auction orchestrator"), + ), noop_services(), )) .expect("should create lazy limited response"); - let EdgeBody::Stream(mut stream) = response.into_body() else { + let Ok(mut stream) = response.into_body().into_stream() else { panic!("should return lazy output") }; let mut failed = false; @@ -24454,11 +23747,14 @@ for (const mode of ['ready', 'queued', 'stale', 'inactive', 'throws']) { &Method::GET, Arc::clone(&settings), ®istry, - Arc::new(AuctionOrchestrator::new(settings.auction.clone())), + Arc::new( + crate::auction::build_orchestrator(&settings) + .expect("should build the auction orchestrator"), + ), noop_services(), )) .expect("should prepare lazy EOF output"); - let EdgeBody::Stream(mut stream) = response.into_body() else { + let Ok(mut stream) = response.into_body().into_stream() else { panic!("should return a lazy body"); }; let first = futures::executor::block_on(stream.next()) @@ -24497,8 +23793,8 @@ for (const mode of ['ready', 'queued', 'stale', 'inactive', 'throws']) { let settings = crate::publisher::allow_eof_test_settings(create_test_settings()); let registry = IntegrationRegistry::from_html_stream_processors(vec![Arc::new(TerminalTestFactory)]); - let http = Arc::new(StubHttpClient::new()); - http.push_response_with_headers( + let http = PublisherUpstream::new(); + http.queue_origin( 404, b"Example missing article".to_vec(), vec![ @@ -24507,10 +23803,9 @@ for (const mode of ['ready', 'queued', 'stale', 'inactive', 'throws']) { ("etag", "example-tag"), ], ); - let services = build_services_with_http_client( - Arc::clone(&http) as Arc - ); - let orchestrator = AuctionOrchestrator::new(settings.auction.clone()); + let services = http.services(); + let orchestrator = crate::auction::build_orchestrator(&settings) + .expect("should build the auction orchestrator"); let request = Request::builder() .method(Method::GET) .uri("https://proxy.example.com/article") @@ -24523,7 +23818,7 @@ for (const mode of ['ready', 'queued', 'stale', 'inactive', 'throws']) { .header(header::IF_MATCH, "example-precondition") .body(EdgeBody::empty()) .expect("should build a navigation"); - let result = futures::executor::block_on(handle_publisher_request( + let (result, _owner) = futures::executor::block_on(handle_publisher_request( &settings, &services, None, @@ -24535,7 +23830,11 @@ for (const mode of ['ready', 'queued', 'stale', 'inactive', 'throws']) { integration_registry: ®istry, }, request, - EdgeCacheHeader::SurrogateControl, + PublisherOriginOptions { + edge_header: EdgeCacheHeader::SurrogateControl, + input_cap: None, + response_decoding: ResponseDecoding::Original, + }, )) .expect("should fetch the origin"); let PublisherResponse::Stream { response, .. } = result else { @@ -24558,7 +23857,7 @@ for (const mode of ['ready', 'queued', 'stale', 'inactive', 'throws']) { !response.headers().contains_key(header::ETAG), "should strip origin validators" ); - let requests = http.recorded_request_headers(); + let requests = http.origin_headers(); let forwarded = &requests[0]; for name in ["if-none-match", "if-modified-since", "range", "if-range"] { assert!( @@ -24593,7 +23892,8 @@ for (const mode of ['ready', 'queued', 'stale', 'inactive', 'throws']) { &mut params, &settings, ®istry, - &AuctionOrchestrator::new(settings.auction.clone()), + &crate::auction::build_orchestrator(&settings) + .expect("should build the auction orchestrator"), &noop_services(), )) .expect("should finish the document"); @@ -24724,8 +24024,10 @@ for (const mode of ['ready', 'queued', 'stale', 'inactive', 'throws']) { body, params: Box::new(params), }; - let orchestrator = - Arc::new(AuctionOrchestrator::new(settings.auction.clone())); + let orchestrator = Arc::new( + crate::auction::build_orchestrator(&settings) + .expect("should build the auction orchestrator"), + ); let response = if buffered { futures::executor::block_on(buffer_publisher_response_async( publisher, @@ -24734,6 +24036,7 @@ for (const mode of ['ready', 'queued', 'stale', 'inactive', 'throws']) { ®istry, &orchestrator, &services, + TEST_BUFFERED_INPUT_CAP, )) } else { futures::executor::block_on( @@ -24813,15 +24116,18 @@ for (const mode of ['ready', 'queued', 'stale', 'inactive', 'throws']) { let registry = IntegrationRegistry::from_html_stream_processors(vec![Arc::new( CountingTerminalFactory(Arc::clone(&calls)), )]); - let http = Arc::new(StubHttpClient::new()); + let http = PublisherUpstream::new(); let raw = b"

Example fragment

"; let mut headers = vec![("content-type", "text/html")]; if content_range { headers.push(("content-range", "bytes 0-22/100")); } - http.push_response_with_headers(status, raw.to_vec(), headers); - let services = build_services_with_http_client(http.clone()); - let orchestrator = Arc::new(AuctionOrchestrator::new(settings.auction.clone())); + http.queue_origin(status, raw.to_vec(), headers); + let services = http.services(); + let orchestrator = Arc::new( + crate::auction::build_orchestrator(&settings) + .expect("should build the auction orchestrator"), + ); let request = Request::builder() .method(Method::GET) .uri("https://proxy.example.com/article") @@ -24831,7 +24137,7 @@ for (const mode of ['ready', 'queued', 'stale', 'inactive', 'throws']) { .header(header::RANGE, "bytes=0-22") .body(EdgeBody::empty()) .expect("should build a ranged fetch"); - let publisher = futures::executor::block_on(handle_publisher_request( + let (publisher, _owner) = futures::executor::block_on(handle_publisher_request( &settings, &services, None, @@ -24843,7 +24149,11 @@ for (const mode of ['ready', 'queued', 'stale', 'inactive', 'throws']) { registry: None, }, request, - EdgeCacheHeader::SurrogateControl, + PublisherOriginOptions { + edge_header: EdgeCacheHeader::SurrogateControl, + input_cap: None, + response_decoding: ResponseDecoding::Original, + }, )) .expect("should fetch a range-associated response"); let response = @@ -24866,7 +24176,7 @@ for (const mode of ['ready', 'queued', 'stale', 'inactive', 'throws']) { ); assert_eq!(actual.as_ref(), raw, "should preserve the origin fragment"); assert_eq!( - http.recorded_request_headers()[0] + http.origin_headers()[0] .iter() .any(|(name, _)| name == "range"), destination == "empty", @@ -24906,7 +24216,8 @@ for (const mode of ['ready', 'queued', 'stale', 'inactive', 'throws']) { StatusCode::PARTIAL_CONTENT }; params.range_associated = streamed; - let orchestrator = AuctionOrchestrator::new(settings.auction.clone()); + let orchestrator = crate::auction::build_orchestrator(&settings) + .expect("should build the auction orchestrator"); let mut output = Vec::new(); futures::executor::block_on(stream_publisher_body_async( body, @@ -24969,7 +24280,8 @@ for (const mode of ['ready', 'queued', 'stale', 'inactive', 'throws']) { let mut source = b"

".to_vec(); source.extend(vec![b'x'; 5 * STREAM_CHUNK_SIZE]); source.extend_from_slice(b"

"); - let orchestrator = AuctionOrchestrator::new(settings.auction.clone()); + let orchestrator = crate::auction::build_orchestrator(&settings) + .expect("should build the auction orchestrator"); let mut output = FlushCounter { bytes: Vec::new(), flushes: 0, @@ -25033,15 +24345,14 @@ for (const mode of ['ready', 'queued', 'stale', 'inactive', 'throws']) { crate::auction::AuctionConfig::legacy_provider_map(&["seam-test"]); settings.auction.timeout_ms = 60_000; settings.auction.mediator = None; - let client = Arc::new(GatedAuctionHttpClient { - inner: StubHttpClient::new(), - released: AtomicBool::new(false), - collections: AtomicUsize::new(0), - }); - client.inner.push_response(200, Vec::new()); - let services = build_services_with_http_client(client.clone()); - let mut orchestrator = AuctionOrchestrator::new(settings.auction.clone()); - orchestrator.register_provider(Arc::new(GatedAuctionProvider)); + let clock = TestClock::new(); + let bidders = ScriptedHttpClient::new(&clock); + // The held bid keeps the auction unresolved until the test releases it. + let (bid, release) = ScriptedReply::status(StatusCode::NO_CONTENT).held(); + bidders.respond_to(Method::POST, SEAM_TEST_BIDDER, bid); + let services = build_services_with_outbound_client(&bidders, &clock); + let orchestrator = crate::auction::build_orchestrator(&settings) + .expect("should build the auction orchestrator"); let request = Request::new(EdgeBody::empty()); let dispatched = futures::executor::block_on(orchestrator.dispatch_auction( &test_auction_request(), @@ -25049,7 +24360,6 @@ for (const mode of ['ready', 'queued', 'stale', 'inactive', 'throws']) { settings: &settings, request: &request, timeout_ms: 60_000, - transport_timeout_ms: 60_000, provider_responses: None, services: &services, }, @@ -25111,12 +24421,7 @@ for (const mode of ['ready', 'queued', 'stale', 'inactive', 'throws']) { 1, "should use one pre-auction flush barrier" ); - assert_eq!( - client.collections.load(Ordering::SeqCst), - 1, - "should poll the pending auction once" - ); - client.released.store(true, Ordering::SeqCst); + release.release(); futures::executor::block_on(work) .expect("should complete once the auction is released"); assert_eq!( @@ -25189,35 +24494,43 @@ for (const mode of ['ready', 'queued', 'stale', 'inactive', 'throws']) { let settings = Arc::new(crate::publisher::allow_eof_test_settings( create_test_settings(), )); - let http = Arc::new(StubHttpClient::new()); - http.push_response_with_headers( + let http = PublisherUpstream::new(); + http.queue_origin( status, b"Example complete response".to_vec(), vec![("content-type", "text/html")], ); - let services = build_services_with_http_client(http); - let orchestrator = Arc::new(AuctionOrchestrator::new(settings.auction.clone())); + let services = http.services(); + let orchestrator = Arc::new( + crate::auction::build_orchestrator(&settings) + .expect("should build the auction orchestrator"), + ); let request = Request::builder() .uri("https://proxy.example.com/article") .header(header::HOST, "proxy.example.com") .header(header::ACCEPT, "text/html") .body(EdgeBody::empty()) .expect("should build a complete document request"); - let mut publisher = futures::executor::block_on(handle_publisher_request( - &settings, - &services, - None, - &mut EcContext::default(), - AuctionDispatch { - integration_registry: ®istry, - orchestrator: &orchestrator, - slots: &[], - registry: None, - }, - request, - EdgeCacheHeader::SurrogateControl, - )) - .expect("should fetch complete success or error HTML"); + let (mut publisher, _owner) = + futures::executor::block_on(handle_publisher_request( + &settings, + &services, + None, + &mut EcContext::default(), + AuctionDispatch { + integration_registry: ®istry, + orchestrator: &orchestrator, + slots: &[], + registry: None, + }, + request, + PublisherOriginOptions { + edge_header: EdgeCacheHeader::SurrogateControl, + input_cap: None, + response_decoding: ResponseDecoding::Original, + }, + )) + .expect("should fetch complete success or error HTML"); if let PublisherResponse::Stream { response, .. } = &mut publisher { response.headers_mut().remove(header::CACHE_CONTROL); } @@ -25229,6 +24542,7 @@ for (const mode of ['ready', 'queued', 'stale', 'inactive', 'throws']) { ®istry, &orchestrator, &services, + TEST_BUFFERED_INPUT_CAP, )) } else { futures::executor::block_on(publisher_response_into_streaming_response( @@ -25286,8 +24600,8 @@ for (const mode of ['ready', 'queued', 'stale', 'inactive', 'throws']) { let registry = IntegrationRegistry::from_html_stream_processors(vec![Arc::new( CountingTerminalFactory(Arc::clone(&calls)), )]); - let http = Arc::new(StubHttpClient::new()); - http.push_response_with_headers( + let http = PublisherUpstream::new(); + http.queue_origin( status, b"Example body".to_vec(), vec![ @@ -25295,8 +24609,11 @@ for (const mode of ['ready', 'queued', 'stale', 'inactive', 'throws']) { ("content-encoding", encoding), ], ); - let services = build_services_with_http_client(http); - let orchestrator = Arc::new(AuctionOrchestrator::new(settings.auction.clone())); + let services = http.services(); + let orchestrator = Arc::new( + crate::auction::build_orchestrator(&settings) + .expect("should build the auction orchestrator"), + ); let request = Request::builder() .method(method.clone()) .uri("https://proxy.example.com/article") @@ -25304,7 +24621,7 @@ for (const mode of ['ready', 'queued', 'stale', 'inactive', 'throws']) { .header(header::ACCEPT, "text/html") .body(EdgeBody::empty()) .expect("should build a fixture request"); - let response = futures::executor::block_on(handle_publisher_request( + let (response, _owner) = futures::executor::block_on(handle_publisher_request( &settings, &services, None, @@ -25316,7 +24633,11 @@ for (const mode of ['ready', 'queued', 'stale', 'inactive', 'throws']) { registry: None, }, request, - EdgeCacheHeader::SurrogateControl, + PublisherOriginOptions { + edge_header: EdgeCacheHeader::SurrogateControl, + input_cap: None, + response_decoding: ResponseDecoding::Original, + }, )) .expect("should classify the origin response"); let response = futures::executor::block_on(publisher_response_into_streaming_response( @@ -25367,7 +24688,10 @@ for (const mode of ['ready', 'queued', 'stale', 'inactive', 'throws']) { )), params: Box::new(html_stream_params("", None)), }; - let orchestrator = Arc::new(AuctionOrchestrator::new(settings.auction.clone())); + let orchestrator = Arc::new( + crate::auction::build_orchestrator(&settings) + .expect("should build the auction orchestrator"), + ); let response = futures::executor::block_on(publisher_response_into_streaming_response( response, &Method::GET, @@ -25464,7 +24788,10 @@ for (const mode of ['ready', 'queued', 'stale', 'inactive', 'throws']) { body, params: Box::new(html_stream_params(encoding, None)), }; - let orchestrator = Arc::new(AuctionOrchestrator::new(settings.auction.clone())); + let orchestrator = Arc::new( + crate::auction::build_orchestrator(&settings) + .expect("should build the auction orchestrator"), + ); let response = futures::executor::block_on(publisher_response_into_streaming_response( response, @@ -25578,7 +24905,10 @@ for (const mode of ['ready', 'queued', 'stale', 'inactive', 'throws']) { )), )), }; - let orchestrator = Arc::new(AuctionOrchestrator::new(settings.auction.clone())); + let orchestrator = Arc::new( + crate::auction::build_orchestrator(&settings) + .expect("should build the auction orchestrator"), + ); let response = futures::executor::block_on(publisher_response_into_streaming_response( response, &Method::GET, @@ -25663,7 +24993,8 @@ for (const mode of ['ready', 'queued', 'stale', 'inactive', 'throws']) { )]); let sink = Arc::new(RecordingTelemetrySink::default()); let services = noop_services_with_telemetry_sink(Arc::clone(&sink) as _); - let orchestrator = AuctionOrchestrator::new(settings.auction.clone()); + let orchestrator = crate::auction::build_orchestrator(&settings) + .expect("should build the auction orchestrator"); let encoding = if failure == "decode" { "gzip" } else { "" }; let mut params = html_stream_params( encoding, @@ -25682,11 +25013,12 @@ for (const mode of ['ready', 'queued', 'stale', 'inactive', 'throws']) { )); let source = b"Example"; let body = match failure { - "read" => { - EdgeBody::from_stream(futures::stream::iter(vec![Err::( - io::Error::other("example origin read failure"), - )])) - } + "read" => EdgeBody::from_external_stream(futures::stream::iter(vec![Err::< + bytes::Bytes, + _, + >( + io::Error::other("example origin read failure"), + )])), "decode" => { let mut encoded = gzip_encode(source); encoded.truncate(encoded.len() / 2); @@ -25742,7 +25074,8 @@ for (const mode of ['ready', 'queued', 'stale', 'inactive', 'throws']) { )]); let sink = Arc::new(RecordingTelemetrySink::default()); let services = noop_services_with_telemetry_sink(Arc::clone(&sink) as _); - let orchestrator = AuctionOrchestrator::new(settings.auction.clone()); + let orchestrator = crate::auction::build_orchestrator(&settings) + .expect("should build the auction orchestrator"); let mut params = html_stream_params( "", Some(DispatchedAuction::empty_for_test( @@ -25908,8 +25241,10 @@ for (const mode of ['ready', 'queued', 'stale', 'inactive', 'throws']) { IntegrationRegistry::from_html_stream_processors(vec![Arc::new( TerminalTestFactory, )]); - let orchestrator = - Arc::new(AuctionOrchestrator::new(settings.auction.clone())); + let orchestrator = Arc::new( + crate::auction::build_orchestrator(&settings) + .expect("should build the auction orchestrator"), + ); let body = if streamed { let chunks: Vec<_> = encoded .chunks(7) @@ -25917,7 +25252,7 @@ for (const mode of ['ready', 'queued', 'stale', 'inactive', 'throws']) { Ok::<_, io::Error>(bytes::Bytes::copy_from_slice(chunk)) }) .collect(); - EdgeBody::from_stream(futures::stream::iter(chunks)) + EdgeBody::from_external_stream(futures::stream::iter(chunks)) } else { EdgeBody::from(encoded.clone()) }; @@ -25953,6 +25288,7 @@ for (const mode of ['ready', 'queued', 'stale', 'inactive', 'throws']) { ®istry, &orchestrator, &noop_services(), + TEST_BUFFERED_INPUT_CAP, )) .expect("should build buffered response") }; @@ -26006,7 +25342,10 @@ for (const mode of ['ready', 'queued', 'stale', 'inactive', 'throws']) { ) .expect("should create integration registry"), ); - let orchestrator = Arc::new(AuctionOrchestrator::new(settings.auction.clone())); + let orchestrator = Arc::new( + crate::auction::build_orchestrator(&settings) + .expect("should build the auction orchestrator"), + ); let services = noop_services(); let response = Response::builder() .status(StatusCode::OK) @@ -26037,6 +25376,7 @@ for (const mode of ['ready', 'queued', 'stale', 'inactive', 'throws']) { OwnedProcessResponseParams { response_status: StatusCode::OK, range_associated: false, + input_cap: None, csp_nonce_observed: None, template_cache_key: None, seam_ad_slots: None, @@ -26227,126 +25567,8 @@ for (const mode of ['ready', 'queued', 'stale', 'inactive', 'throws']) { ); } - // Manually polled fixture for publisher injection order, not timer scheduling. - // Adapter runtime tests cover autonomous deadline wakeups and cancellation. - struct GatedAuctionHttpClient { - inner: StubHttpClient, - released: std::sync::atomic::AtomicBool, - collections: AtomicUsize, - } - - #[async_trait::async_trait(?Send)] - impl crate::platform::PlatformHttpClient for GatedAuctionHttpClient { - async fn send( - &self, - request: crate::platform::PlatformHttpRequest, - ) -> Result> - { - self.inner.send(request).await - } - - async fn send_async( - &self, - request: crate::platform::PlatformHttpRequest, - ) -> Result> - { - self.inner.send_async(request).await - } - - async fn select_until( - &self, - pending: Vec, - deadline: web_time::Instant, - ) -> Result< - Option, - Report, - > { - self.collections.fetch_add(1, Ordering::SeqCst); - let ready = futures::future::poll_fn(|_| { - // The gate models completion, so probe it before checking expiry. - if self.released.load(Ordering::SeqCst) { - std::task::Poll::Ready(true) - } else if web_time::Instant::now() >= deadline { - std::task::Poll::Ready(false) - } else { - std::task::Poll::Pending - } - }) - .await; - if ready { - self.inner.select_until(pending, deadline).await - } else { - Ok(None) - } - } - - async fn select( - &self, - pending: Vec, - ) -> Result> - { - self.collections.fetch_add(1, Ordering::SeqCst); - futures::future::poll_fn(|_| { - if self.released.load(Ordering::SeqCst) { - std::task::Poll::Ready(()) - } else { - std::task::Poll::Pending - } - }) - .await; - self.inner.select(pending).await - } - } - - struct GatedAuctionProvider; - - #[async_trait::async_trait(?Send)] - impl AuctionProvider for GatedAuctionProvider { - fn provider_name(&self) -> &'static str { - "seam-test" - } - - async fn request_bids( - &self, - _request: &AuctionRequest, - context: &AuctionContext<'_>, - ) -> Result> { - context - .services - .http_client() - .send_async( - crate::platform::PlatformHttpRequest::new( - Request::builder() - .uri("https://bidder.example.com/bid") - .body(EdgeBody::empty()) - .expect("should build test bid request"), - "seam-test", - ) - .with_cache_bypass(), - ) - .await - .change_context(TrustedServerError::Auction { - message: "Failed to dispatch test auction".to_string(), - }) - .map(ProviderRequestOutcome::pending) - } - - async fn parse_response( - &self, - _response: crate::platform::PlatformResponse, - response_time_ms: u64, - ) -> Result> { - Ok(AuctionResponse::success( - "seam-test", - Vec::new(), - response_time_ms, - )) - } - - fn timeout_ms(&self) -> u32 { - 60_000 - } - } + /// Endpoint of the `seam-test` bidder that `legacy_provider_map` configures. + const SEAM_TEST_BIDDER: &str = "https://seam-test.example/openrtb2/auction"; #[test] fn parser_confirmed_auction_seam_streams_nextjs_for_every_encoding() { @@ -26367,15 +25589,14 @@ for (const mode of ['ready', 'queued', 'stale', 'inactive', 'throws']) { }), ) .expect("should enable Next.js"); - let client = Arc::new(GatedAuctionHttpClient { - inner: StubHttpClient::new(), - released: std::sync::atomic::AtomicBool::new(false), - collections: AtomicUsize::new(0), - }); - client.inner.push_response(200, Vec::new()); - let services = build_services_with_http_client(Arc::clone(&client) as _); - let mut orchestrator = AuctionOrchestrator::new(settings.auction.clone()); - orchestrator.register_provider(Arc::new(GatedAuctionProvider)); + let clock = TestClock::new(); + let bidders = ScriptedHttpClient::new(&clock); + // The held bid keeps the auction unresolved until the test releases it. + let (bid, release) = ScriptedReply::status(StatusCode::NO_CONTENT).held(); + bidders.respond_to(Method::POST, SEAM_TEST_BIDDER, bid); + let services = build_services_with_outbound_client(&bidders, &clock); + let orchestrator = crate::auction::build_orchestrator(&settings) + .expect("should build the auction orchestrator"); let request = Request::new(EdgeBody::empty()); let dispatched = futures::executor::block_on(orchestrator.dispatch_auction( &test_auction_request(), @@ -26383,7 +25604,6 @@ for (const mode of ['ready', 'queued', 'stale', 'inactive', 'throws']) { settings: &settings, request: &request, timeout_ms: 60_000, - transport_timeout_ms: 60_000, provider_responses: None, services: &services, }, @@ -26470,11 +25690,6 @@ for (const mode of ['ready', 'queued', 'stale', 'inactive', 'throws']) { !prefix.contains("var b=JSON.parse("), "should hold bids until auction completes" ); - assert_eq!( - client.collections.load(Ordering::SeqCst), - 1, - "should begin collection at the real seam" - ); assert!( state .script_cell() @@ -26483,7 +25698,7 @@ for (const mode of ['ready', 'queued', 'stale', 'inactive', 'throws']) { .is_none(), "should keep auction unresolved at seam" ); - client.released.store(true, Ordering::SeqCst); + release.release(); futures::executor::block_on(async { while let Some(chunk) = futures::StreamExt::next(&mut stream).await { output.extend_from_slice(&chunk.expect("should stream completed auction")); @@ -26515,22 +25730,17 @@ for (const mode of ['ready', 'queued', 'stale', 'inactive', 'throws']) { !html.contains("ts-inline-body-close-") && !html.contains("__ts_rsc_"), "should remove internal markers for {encoding}: {html}" ); - assert_eq!( - client.collections.load(Ordering::SeqCst), - 1, - "should collect once" - ); } } // (method, status, expected Content-Length, expected Transfer-Encoding) // after bodiless normalization. 204 forbids Content-Length (removed); 205 // must advertise a zero-length body; HEAD and 304 legitimately advertise the - // GET representation length. Chunked framing is stripped from 204 (forbidden - // by RFC 9112 §6.1) and 205 (would conflict with the inserted + // GET representation length. `Transfer-Encoding` is stripped from 204 + // (forbidden by RFC 9112 §6.1) and 205 (would conflict with the inserted // `Content-Length: 0`), but preserved for HEAD and 304, whose message length // is fixed by the header section regardless of framing fields - // (RFC 9112 §6.3, rule 1). + // (RFC 9112 §6.3, rule 1). `Trailer` goes in every case. const BODILESS_FRAMING_CASES: [(Method, StatusCode, Option<&str>, Option<&str>); 4] = [ (Method::HEAD, StatusCode::OK, Some("42"), Some("chunked")), (Method::GET, StatusCode::NO_CONTENT, None, None), @@ -26569,14 +25779,43 @@ for (const mode of ['ready', 'queued', 'stale', 'inactive', 'throws']) { "bodiless {method} {status} must carry the corrected Transfer-Encoding" ); assert_eq!( - header_value(header::TRAILER).as_deref(), - // `Trailer` only describes fields a chunked body can carry, so it - // survives exactly where chunked framing does. - expected_transfer_encoding.map(|_| "expires"), - "bodiless {method} {status} must drop Trailer with its chunked framing" + header_value(header::TRAILER), + None, + "bodiless {method} {status} must not carry Trailer" ); } + #[test] + fn make_response_bodiless_removes_trailer_for_head_and_304() { + // A HEAD answer keeps its origin status, so 200 stands for HEAD here. + for status in [StatusCode::OK, StatusCode::NOT_MODIFIED] { + let mut response = Response::builder() + .status(status) + .header(header::CONTENT_LENGTH, "42") + .header(header::TRANSFER_ENCODING, "chunked") + .header(header::TRAILER, "expires") + .body(EdgeBody::from(b"origin body".to_vec())) + .expect("should build response"); + + make_response_bodiless(&mut response); + + assert_eq!( + response.headers().get(header::TRAILER), + None, + "{status} must lose Trailer, which egress would refuse to send" + ); + assert_eq!( + response.headers().get(header::CONTENT_LENGTH), + Some(&HeaderValue::from_static("42")), + "{status} must keep the GET representation length" + ); + assert!( + response.body().as_bytes().is_some_and(<[u8]>::is_empty), + "{status} must carry no body" + ); + } + } + #[test] fn publisher_response_streaming_finalize_normalizes_bodiless_framing() { // Fastly requests the origin body as a stream before classification, so @@ -26594,7 +25833,10 @@ for (const mode of ['ready', 'queued', 'stale', 'inactive', 'throws']) { ) .expect("should create integration registry"), ); - let orchestrator = Arc::new(AuctionOrchestrator::new(settings.auction.clone())); + let orchestrator = Arc::new( + crate::auction::build_orchestrator(&settings) + .expect("should build the auction orchestrator"), + ); for (method, status, expected_length, expected_transfer_encoding) in BODILESS_FRAMING_CASES { @@ -26620,7 +25862,7 @@ for (const mode of ['ready', 'queued', 'stale', 'inactive', 'throws']) { .expect("should finalize buffered response"); assert!( - !matches!(response.body(), EdgeBody::Stream(_)), + !response.body().is_stream(), "bodiless {method} {status} must not carry a streaming body" ); assert_bodiless_framing( @@ -26659,7 +25901,8 @@ for (const mode of ['ready', 'queued', 'stale', 'inactive', 'throws']) { ), ) .expect("should create integration registry"); - let orchestrator = AuctionOrchestrator::new(settings.auction.clone()); + let orchestrator = crate::auction::build_orchestrator(&settings) + .expect("should build the auction orchestrator"); let services = noop_services(); for (method, status, expected_length, expected_transfer_encoding) in BODILESS_FRAMING_CASES @@ -26682,6 +25925,7 @@ for (const mode of ['ready', 'queued', 'stale', 'inactive', 'throws']) { ®istry, &orchestrator, &services, + TEST_BUFFERED_INPUT_CAP, )) .expect("should finalize buffered response"); @@ -26745,7 +25989,10 @@ for (const mode of ['ready', 'queued', 'stale', 'inactive', 'throws']) { ), ) .expect("should create integration registry"); - let orchestrator = Arc::new(AuctionOrchestrator::new(settings.auction.clone())); + let orchestrator = Arc::new( + crate::auction::build_orchestrator(&settings) + .expect("should build the auction orchestrator"), + ); let make_params = || { let ec_context = @@ -26753,6 +26000,7 @@ for (const mode of ['ready', 'queued', 'stale', 'inactive', 'throws']) { OwnedProcessResponseParams { response_status: StatusCode::OK, range_associated: false, + input_cap: None, csp_nonce_observed: None, template_cache_key: None, seam_ad_slots: None, @@ -26838,6 +26086,7 @@ for (const mode of ['ready', 'queued', 'stale', 'inactive', 'throws']) { ®istry, orchestrator.as_ref(), &buffered_services, + TEST_BUFFERED_INPUT_CAP, )) .expect("buffered finalize should succeed"); assert_eq!(response.status(), StatusCode::NOT_MODIFIED); @@ -26929,7 +26178,10 @@ for (const mode of ['ready', 'queued', 'stale', 'inactive', 'throws']) { ) .expect("should create integration registry"), ); - let orchestrator = Arc::new(AuctionOrchestrator::new(settings.auction.clone())); + let orchestrator = Arc::new( + crate::auction::build_orchestrator(&settings) + .expect("should build the auction orchestrator"), + ); let services = noop_services(); let response = Response::builder() .status(StatusCode::OK) @@ -26951,6 +26203,7 @@ for (const mode of ['ready', 'queued', 'stale', 'inactive', 'throws']) { let params = OwnedProcessResponseParams { response_status: StatusCode::OK, range_associated: false, + input_cap: None, csp_nonce_observed: None, template_cache_key: None, seam_ad_slots: None, @@ -27035,6 +26288,7 @@ for (const mode of ['ready', 'queued', 'stale', 'inactive', 'throws']) { let params = OwnedProcessResponseParams { response_status: StatusCode::OK, range_associated: false, + input_cap: None, csp_nonce_observed: None, template_cache_key: None, seam_ad_slots: None, @@ -27102,6 +26356,7 @@ for (const mode of ['ready', 'queued', 'stale', 'inactive', 'throws']) { let params = OwnedProcessResponseParams { response_status: StatusCode::OK, range_associated: false, + input_cap: None, csp_nonce_observed: None, template_cache_key: None, seam_ad_slots: None, @@ -27223,6 +26478,7 @@ for (const mode of ['ready', 'queued', 'stale', 'inactive', 'throws']) { let params = OwnedProcessResponseParams { response_status: StatusCode::OK, range_associated: false, + input_cap: None, csp_nonce_observed: None, template_cache_key: None, seam_ad_slots: None, @@ -27293,6 +26549,7 @@ for (const mode of ['ready', 'queued', 'stale', 'inactive', 'throws']) { let params = OwnedProcessResponseParams { response_status: StatusCode::OK, range_associated: false, + input_cap: None, csp_nonce_observed: None, template_cache_key: None, seam_ad_slots: None, @@ -28992,109 +28249,59 @@ for (const mode of ['ready', 'queued', 'stale', 'inactive', 'throws']) { mod page_bids_no_match_tests { use super::super::*; - use super::build_services_with_http_client; use crate::auction::AuctionOrchestrator; - use crate::auction::provider::{AuctionProvider, ProviderRequestOutcome}; - use crate::auction::types::{AuctionRequest, AuctionResponse, Bid}; use crate::creative_opportunities::{CreativeOpportunityFormat, CreativeOpportunitySlot}; - use crate::platform::test_support::{StubHttpClient, noop_services}; - use crate::platform::{PlatformHttpRequest, PlatformResponse}; + use crate::platform::test_support::{build_services_with_outbound_client, noop_services}; + use crate::test_support::scripted_client::{ScriptedHttpClient, ScriptedReply, TestClock}; use crate::test_support::tests::crate_test_settings_str; - use error_stack::{Report, ResultExt}; + use http::Method; - use std::sync::{Arc, Mutex}; + use std::sync::Arc; - const AUCTION_ID_TEST_PROVIDER: &str = "auction_id_test_provider"; - const AUCTION_ID_TEST_BACKEND: &str = "auction-id-test-backend"; + const AUCTION_ID_TEST_PROVIDER: &str = "auction-id-test"; + /// Endpoint of the [`AUCTION_ID_TEST_PROVIDER`] that `legacy_provider_map` configures. + const AUCTION_ID_TEST_ENDPOINT: &str = "https://auction-id-test.example/openrtb2/auction"; - struct AuctionIdTestProvider { - captured_request: Arc>>, + /// The plan-backed orchestrator for `settings`, with services whose test bidder + /// answers one winning `atf` bid, or no bid. + fn auction_id_test_auction( + settings: &Settings, winning_bid: bool, - creative: Option, + creative: &str, + ) -> (AuctionOrchestrator, RuntimeServices, ScriptedHttpClient) { + let clock = TestClock::new(); + let bidder = ScriptedHttpClient::new(&clock); + let reply = if winning_bid { + ScriptedReply::status(StatusCode::OK).body( + serde_json::to_vec(&serde_json::json!({ + "seatbid": [{"seat": AUCTION_ID_TEST_PROVIDER, "bid": [{ + "impid": "atf", + "price": 1.50, + "adid": "winner-123", + "adm": creative, + "w": 300, + "h": 250 + }]}] + })) + .expect("should serialize the winning bid"), + ) + } else { + ScriptedReply::status(StatusCode::NO_CONTENT) + }; + bidder.respond_to(Method::POST, AUCTION_ID_TEST_ENDPOINT, reply); + let services = build_services_with_outbound_client(&bidder, &clock); + let orchestrator = crate::auction::build_orchestrator(settings) + .expect("should build the auction orchestrator"); + (orchestrator, services, bidder) } - #[async_trait::async_trait(?Send)] - impl AuctionProvider for AuctionIdTestProvider { - fn provider_name(&self) -> &str { - AUCTION_ID_TEST_PROVIDER - } - - async fn request_bids( - &self, - request: &AuctionRequest, - context: &AuctionContext<'_>, - ) -> Result> { - *self - .captured_request - .lock() - .expect("should lock captured auction request") = Some(request.clone()); - let request = PlatformHttpRequest::new( - Request::builder() - .method(Method::POST) - .uri("https://bidder.example.test/bids") - .body(EdgeBody::empty()) - .expect("should build test bidder request"), - AUCTION_ID_TEST_BACKEND, - ); - context - .services - .http_client() - .send_async(request.with_cache_bypass()) - .await - .change_context(TrustedServerError::Auction { - message: "test bidder launch failed".to_string(), - }) - .map(ProviderRequestOutcome::pending) - } - - async fn parse_response( - &self, - _response: PlatformResponse, - response_time_ms: u64, - ) -> Result> { - let bids = if self.winning_bid { - vec![Bid { - slot_id: "atf".to_string(), - price: Some(1.50), - currency: "USD".to_string(), - creative: self.creative.clone(), - adomain: None, - bidder: AUCTION_ID_TEST_PROVIDER.to_string(), - returned_seat: None, - width: 300, - height: 250, - nurl: None, - burl: None, - bid_id: None, - creative_id: None, - renderer: None, - ad_id: Some("winner-123".to_string()), - cache_id: None, - cache_host: None, - cache_path: None, - metadata: Default::default(), - }] - } else { - Vec::new() - }; - Ok(AuctionResponse::success( - AUCTION_ID_TEST_PROVIDER, - bids, - response_time_ms, - )) - } - - fn timeout_ms(&self) -> u32 { - 100 - } - - fn backend_name( - &self, - _services: &RuntimeServices, - _timeout_ms: u32, - ) -> Option { - Some(AUCTION_ID_TEST_BACKEND.to_string()) - } + /// The `OpenRTB` request the test bidder received. + fn sent_auction(bidder: &ScriptedHttpClient) -> serde_json::Value { + let requests = bidder.requests(); + let request = requests + .first() + .expect("should dispatch the auction request"); + serde_json::from_slice(&request.body).expect("should send a JSON bid request") } fn settings_with_co() -> Settings { @@ -29195,7 +28402,8 @@ for (const mode of ['ready', 'queued', 'stale', 'inactive', 'throws']) { #[tokio::test] async fn page_bids_format_absent_or_json_returns_json() { let settings = settings_with_co(); - let orchestrator = AuctionOrchestrator::new(settings.auction.clone()); + let orchestrator = crate::auction::build_orchestrator(&settings) + .expect("should build the auction orchestrator"); for path_and_format in ["/2024/article", "/2024/article&format=json"] { let response = run_page_bids_response( &settings, @@ -29247,20 +28455,25 @@ for (const mode of ['ready', 'queued', 'stale', 'inactive', 'throws']) { .expect("should compile actual provider plan"), ); let orchestrator = AuctionOrchestrator::from_plan(plan, None); - let http = Arc::new(StubHttpClient::new()); - if winning_bid { - http.push_response(200, serde_json::to_vec(&serde_json::json!({"seatbid":[{"seat":"bidder","bid":[{"id":"bid","impid":"atf","price":1.5,"w":300,"h":250,"adm":"
Example creative
"}]}]})).expect("should encode actual winning response")); + let clock = TestClock::new(); + let bidder = ScriptedHttpClient::new(&clock); + let reply = if winning_bid { + ScriptedReply::status(StatusCode::OK).body( + serde_json::to_vec(&serde_json::json!({"seatbid":[{"seat":"bidder","bid":[{"id":"bid","impid":"atf","price":1.5,"w":300,"h":250,"adm":"
Example creative
"}]}]})) + .expect("should encode actual winning response"), + ) } else { - http.push_response(204, Vec::new()); - } - let services = build_services_with_http_client(Arc::clone(&http) as Arc<_>); + ScriptedReply::status(StatusCode::NO_CONTENT) + }; + bidder.respond_to(Method::POST, "https://bidder.example.com/auction", reply); + let services = build_services_with_outbound_client(&bidder, &clock); let mut request = make_page_bids_request_on(endpoint, "/2024/article"); set_test_header(&mut request, "cookie", "__Host-ts-console=1"); let hook = crate::trace::TracePreDispatchHook::new( Arc::clone(&settings), Arc::new(|_| panic!("should not load SPA setup metadata")), ); - edgezero_core::router::PreDispatchHook::handle(&hook, &mut request) + let request = crate::test_support::tests::run_pre_dispatch_hook(hook, request) .await .expect("should freeze SPA gate"); let mut ec_context = consent_allowing_ec_context(); @@ -29332,12 +28545,9 @@ for (const mode of ['ready', 'queued', 'stale', 'inactive', 'throws']) { "should share opportunity and evidence token" ); } - assert_eq!( - http.recorded_request_bodies().len(), - 1, - "should execute one real provider launch" - ); - let source = String::from_utf8(http.recorded_request_bodies()[0].clone()) + let sent = bidder.requests(); + assert_eq!(sent.len(), 1, "should execute one real provider launch"); + let source = String::from_utf8(sent[0].body.to_vec()) .expect("should serialize ordinary provider JSON"); assert!( !source.contains("trace_slot_ref") && !source.contains(&token), @@ -29359,7 +28569,8 @@ for (const mode of ['ready', 'queued', 'stale', 'inactive', 'throws']) { ) .expect("should enable request-local trace evidence"); let settings = Arc::new(settings); - let orchestrator = AuctionOrchestrator::new(settings.auction.clone()); + let orchestrator = crate::auction::build_orchestrator(&settings) + .expect("should build the auction orchestrator"); let mut request = make_page_bids_request("/2024/article"); if active { set_test_header(&mut request, "cookie", "__Host-ts-console=1"); @@ -29368,13 +28579,9 @@ for (const mode of ['ready', 'queued', 'stale', 'inactive', 'throws']) { Arc::clone(&settings), Arc::new(|_| panic!("should not load setup metadata on SPA capture")), ); - assert!( - edgezero_core::router::PreDispatchHook::handle(&hook, &mut request) - .await - .expect("should freeze SPA cookies") - .is_none(), - "should keep ordinary SPA routing" - ); + let request = crate::test_support::tests::run_pre_dispatch_hook(hook, request) + .await + .expect("should keep ordinary SPA routing"); let response = run_page_bids_response(&settings, &orchestrator, &[], request).await; @@ -29420,7 +28627,8 @@ for (const mode of ['ready', 'queued', 'stale', 'inactive', 'throws']) { #[tokio::test] async fn page_bids_format_rejects_removed_unknown_and_empty_values() { let settings = settings_with_co(); - let orchestrator = AuctionOrchestrator::new(settings.auction.clone()); + let orchestrator = crate::auction::build_orchestrator(&settings) + .expect("should build the auction orchestrator"); for format in ["fragment", "scrpit", ""] { let response = run_page_bids_response( &settings, @@ -29495,20 +28703,6 @@ for (const mode of ['ready', 'queued', 'stale', 'inactive', 'throws']) { .expect("should return ok response") } - fn auction_id_test_orchestrator( - settings: &Settings, - captured_request: Arc>>, - winning_bid: bool, - ) -> AuctionOrchestrator { - let mut orchestrator = AuctionOrchestrator::new(settings.auction.clone()); - orchestrator.register_provider(Arc::new(AuctionIdTestProvider { - captured_request, - winning_bid, - creative: None, - })); - orchestrator - } - #[tokio::test] async fn public_origin_spa_creatives_use_public_port_and_preserve_publisher_identity() { let mut settings = settings_with_co(); @@ -29524,16 +28718,10 @@ for (const mode of ['ready', 'queued', 'stale', 'inactive', 'throws']) { ), }); for endpoint in [PAGE_BIDS_PATH, PAGE_BIDS_LEGACY_PATH] { - let captured = Arc::new(Mutex::new(None)); - let mut orchestrator = AuctionOrchestrator::new(settings.auction.clone()); - orchestrator.register_provider(Arc::new(AuctionIdTestProvider { - captured_request: Arc::clone(&captured), winning_bid: true, - creative: Some("".to_owned()), - })); - let stub = Arc::new(StubHttpClient::new()); - stub.push_response(200, b"winner".to_vec()); - let services = build_services_with_http_client( - Arc::clone(&stub) as Arc + let (orchestrator, services, bidder) = auction_id_test_auction( + &settings, + true, + "", ); let mut request = Request::builder() .uri(format!( @@ -29549,7 +28737,9 @@ for (const mode of ['ready', 'queued', 'stale', 'inactive', 'throws']) { .header("x-forwarded-proto", "https") .body(EdgeBody::empty()) .expect("should construct SPA request"); - crate::forwarder::prepare_trusted_forwarder(&mut request, &settings); + crate::test_support::tests::with_request_parts(&mut request, |parts| { + crate::forwarder::prepare_trusted_forwarder(parts, &settings) + }); let mut ec_context = EcContext::new_for_test( None, crate::consent::ConsentContext { @@ -29596,17 +28786,13 @@ for (const mode of ['ready', 'queued', 'stale', 'inactive', 'throws']) { !creative.contains("internal.example.com"), "should not leak transport authority into SPA creative" ); - let auction = captured - .lock() - .expect("should lock captured request") - .clone() - .expect("should dispatch SPA auction"); + let auction = sent_auction(&bidder); assert_eq!( - auction.publisher.domain, "publisher.example.com", + auction["site"]["publisher"]["domain"], "publisher.example.com", "should retain configured publisher identity" ); assert_eq!( - auction.publisher.page_url.as_deref(), + auction["site"]["page"].as_str(), Some("https://publisher.example.com/2024/article"), "should use public scheme with configured publisher page identity" ); @@ -29623,14 +28809,8 @@ for (const mode of ['ready', 'queued', 'stale', 'inactive', 'throws']) { .insert_config("gpt_diagnostics", &serde_json::json!({ "enabled": true })) .expect("should enable diagnostics"); let slots = article_slot(); - let winning_stub = Arc::new(StubHttpClient::new()); - winning_stub.push_response(200, b"winner".to_vec()); - let winning_services = build_services_with_http_client( - Arc::clone(&winning_stub) as Arc - ); - let winning_request = Arc::new(Mutex::new(None)); - let winning_orchestrator = - auction_id_test_orchestrator(&settings, Arc::clone(&winning_request), true); + let (winning_orchestrator, winning_services, winning_bidder) = + auction_id_test_auction(&settings, true, "
winner
"); let mut ec_context = EcContext::new_for_test( Some("page-auction-example-123".to_string()), crate::consent::ConsentContext { @@ -29661,14 +28841,13 @@ for (const mode of ['ready', 'queued', 'stale', 'inactive', 'throws']) { .expect("should read winning page-bids response body"), ) .expect("should serialize winning page-bids response as JSON"); - let auction_request = winning_request - .lock() - .expect("should lock captured winning request") - .clone() - .expect("should dispatch a winning auction request"); + let auction_request = sent_auction(&winning_bidder); + let auction_request_id = auction_request["id"] + .as_str() + .expect("should send the auction request ID"); assert_eq!( - auction_request.id, "ts-page-auction-example-123", + auction_request_id, "ts-page-auction-example-123", "test EC ID should produce a deterministic auction request ID" ); let winning_auction_id = winning_body["bids"]["atf"]["hb_auction_id"] @@ -29680,7 +28859,7 @@ for (const mode of ['ready', 'queued', 'stale', 'inactive', 'throws']) { "page-bids should expose a freshly minted diagnostics token, got `{winning_auction_id}`" ); assert_ne!( - winning_auction_id, auction_request.id, + winning_auction_id, auction_request_id, "browser-visible auction ID must not be the EC-derived request ID" ); assert!( @@ -29688,13 +28867,8 @@ for (const mode of ['ready', 'queued', 'stale', 'inactive', 'throws']) { "browser-visible auction ID must not embed the EC ID" ); - let no_winner_stub = Arc::new(StubHttpClient::new()); - no_winner_stub.push_response(200, b"no-bid".to_vec()); - let no_winner_services = build_services_with_http_client( - Arc::clone(&no_winner_stub) as Arc - ); - let no_winner_orchestrator = - auction_id_test_orchestrator(&settings, Arc::new(Mutex::new(None)), false); + let (no_winner_orchestrator, no_winner_services, _no_winner_bidder) = + auction_id_test_auction(&settings, false, "
winner
"); let no_winner_response = handle_page_bids( &settings, &no_winner_services, @@ -29734,13 +28908,8 @@ for (const mode of ['ready', 'queued', 'stale', 'inactive', 'throws']) { async fn page_bids_auction_id_is_per_auction_and_gated_on_diagnostics() { async fn winning_auction_id(settings: &Settings) -> Option { let slots = article_slot(); - let stub = Arc::new(StubHttpClient::new()); - stub.push_response(200, b"winner".to_vec()); - let services = build_services_with_http_client( - Arc::clone(&stub) as Arc - ); - let orchestrator = - auction_id_test_orchestrator(settings, Arc::new(Mutex::new(None)), true); + let (orchestrator, services, _bidder) = + auction_id_test_auction(settings, true, "
winner
"); let mut ec_context = EcContext::new_for_test( Some("page-auction-example-123".to_string()), crate::consent::ConsentContext { @@ -29816,7 +28985,8 @@ for (const mode of ['ready', 'queued', 'stale', 'inactive', 'throws']) { #[tokio::test] async fn deprecated_alias_response_matches_canonical_path() { let settings = settings_with_co(); - let orchestrator = AuctionOrchestrator::new(settings.auction.clone()); + let orchestrator = crate::auction::build_orchestrator(&settings) + .expect("should build the auction orchestrator"); let canonical = run_page_bids_response( &settings, @@ -29853,7 +29023,8 @@ for (const mode of ['ready', 'queued', 'stale', 'inactive', 'throws']) { #[tokio::test] async fn deprecated_alias_response_is_marked_deprecated() { let settings = settings_with_co(); - let orchestrator = AuctionOrchestrator::new(settings.auction.clone()); + let orchestrator = crate::auction::build_orchestrator(&settings) + .expect("should build the auction orchestrator"); let canonical = run_page_bids_response( &settings, @@ -29897,7 +29068,8 @@ for (const mode of ['ready', 'queued', 'stale', 'inactive', 'throws']) { settings.creative_opportunities.is_none(), "test settings should have no creative opportunities configured" ); - let orchestrator = AuctionOrchestrator::new(settings.auction.clone()); + let orchestrator = crate::auction::build_orchestrator(&settings) + .expect("should build the auction orchestrator"); let response = run_page_bids_response( &settings, @@ -29924,7 +29096,8 @@ for (const mode of ['ready', 'queued', 'stale', 'inactive', 'throws']) { #[tokio::test] async fn cross_site_request_is_denied_before_configuration_is_revealed() { let settings = settings_without_co(); - let orchestrator = AuctionOrchestrator::new(settings.auction.clone()); + let orchestrator = crate::auction::build_orchestrator(&settings) + .expect("should build the auction orchestrator"); let mut req = Request::builder() .method(Method::GET) .uri(format!("https://test-publisher.com{PAGE_BIDS_PATH}?path=/")) @@ -29944,7 +29117,8 @@ for (const mode of ['ready', 'queued', 'stale', 'inactive', 'throws']) { #[tokio::test] async fn cross_site_fetch_metadata_is_rejected() { let settings = settings_with_co(); - let orchestrator = AuctionOrchestrator::new(settings.auction.clone()); + let orchestrator = crate::auction::build_orchestrator(&settings) + .expect("should build the auction orchestrator"); let mut req = make_page_bids_request("/2024/01/my-article/"); set_test_header(&mut req, "sec-fetch-site", "cross-site"); @@ -29961,7 +29135,8 @@ for (const mode of ['ready', 'queued', 'stale', 'inactive', 'throws']) { #[tokio::test] async fn missing_fetch_metadata_without_tsjs_header_is_rejected() { let settings = settings_with_co(); - let orchestrator = AuctionOrchestrator::new(settings.auction.clone()); + let orchestrator = crate::auction::build_orchestrator(&settings) + .expect("should build the auction orchestrator"); let mut req = make_page_bids_request("/2024/01/my-article/"); req.headers_mut().remove("sec-fetch-site"); @@ -29978,7 +29153,8 @@ for (const mode of ['ready', 'queued', 'stale', 'inactive', 'throws']) { #[tokio::test] async fn missing_fetch_metadata_with_tsjs_header_is_allowed() { let settings = settings_with_co(); - let orchestrator = AuctionOrchestrator::new(settings.auction.clone()); + let orchestrator = crate::auction::build_orchestrator(&settings) + .expect("should build the auction orchestrator"); let mut req = make_page_bids_request("/2024/01/my-article/"); req.headers_mut().remove("sec-fetch-site"); set_test_header(&mut req, "x-tsjs-page-bids", "1"); @@ -29996,7 +29172,8 @@ for (const mode of ['ready', 'queued', 'stale', 'inactive', 'throws']) { #[tokio::test] async fn same_site_fetch_metadata_is_rejected() { let settings = settings_with_co(); - let orchestrator = AuctionOrchestrator::new(settings.auction.clone()); + let orchestrator = crate::auction::build_orchestrator(&settings) + .expect("should build the auction orchestrator"); let mut req = make_page_bids_request("/2024/01/my-article/"); // `same-site` admits sibling origins under the same registrable // domain — not trusted to spend SSP quota. @@ -30017,7 +29194,8 @@ for (const mode of ['ready', 'queued', 'stale', 'inactive', 'throws']) { // Spec §8 kill-switch: creative-opportunities.toml with zero slots disables // all server-side auction activity and injection. let settings = settings_with_co(); - let orchestrator = AuctionOrchestrator::new(settings.auction.clone()); + let orchestrator = crate::auction::build_orchestrator(&settings) + .expect("should build the auction orchestrator"); let req = make_page_bids_request("/2024/01/my-article/"); let body = run_page_bids(&settings, &orchestrator, &[], req).await; @@ -30046,7 +29224,8 @@ for (const mode of ['ready', 'queued', 'stale', 'inactive', 'throws']) { // but the server must not burn SSP request quota running a real auction // for them. Same gate the publisher path applies. let settings = settings_with_co(); - let orchestrator = AuctionOrchestrator::new(settings.auction.clone()); + let orchestrator = crate::auction::build_orchestrator(&settings) + .expect("should build the auction orchestrator"); let slots = article_slot(); let mut req = make_page_bids_request("/2024/01/my-article/"); set_test_header( @@ -30080,7 +29259,8 @@ for (const mode of ['ready', 'queued', 'stale', 'inactive', 'throws']) { // Navigations triggered by Sec-Purpose=prefetch should not fire real // SSP auctions — the user has not yet visited the page. let settings = settings_with_co(); - let orchestrator = AuctionOrchestrator::new(settings.auction.clone()); + let orchestrator = crate::auction::build_orchestrator(&settings) + .expect("should build the auction orchestrator"); let slots = article_slot(); let mut req = make_page_bids_request("/2024/01/my-article/"); set_test_header(&mut req, "sec-purpose", "prefetch"); @@ -30108,7 +29288,8 @@ for (const mode of ['ready', 'queued', 'stale', 'inactive', 'throws']) { #[tokio::test] async fn page_bids_omits_only_over_limit_dynamic_slot() { let settings = settings_with_co(); - let orchestrator = AuctionOrchestrator::new(settings.auction.clone()); + let orchestrator = crate::auction::build_orchestrator(&settings) + .expect("should build the auction orchestrator"); let mut over_limit = article_slot() .into_iter() .next() @@ -30149,7 +29330,8 @@ for (const mode of ['ready', 'queued', 'stale', 'inactive', 'throws']) { async fn url_not_matching_any_pattern_returns_empty_response() { // Slots exist but request path does not match — no auction, no injection. let settings = settings_with_co(); - let orchestrator = AuctionOrchestrator::new(settings.auction.clone()); + let orchestrator = crate::auction::build_orchestrator(&settings) + .expect("should build the auction orchestrator"); let slots = article_slot(); // slot matches /20** only let req = make_page_bids_request("/about"); // does not match @@ -30211,7 +29393,8 @@ for (const mode of ['ready', 'queued', 'stale', 'inactive', 'throws']) { // the auction is off. Consent is allowed here so the test isolates // the kill switch. let settings = settings_with_co_auction_disabled(); - let orchestrator = AuctionOrchestrator::new(settings.auction.clone()); + let orchestrator = crate::auction::build_orchestrator(&settings) + .expect("should build the auction orchestrator"); let slots = article_slot(); let req = make_page_bids_request("/2024/01/my-article/"); @@ -30240,7 +29423,8 @@ for (const mode of ['ready', 'queued', 'stale', 'inactive', 'throws']) { // The dedicated template switch must suppress publisher/page-bids // delivery without using the global auction switch. let settings = settings_with_co_templates_disabled(); - let orchestrator = AuctionOrchestrator::new(settings.auction.clone()); + let orchestrator = crate::auction::build_orchestrator(&settings) + .expect("should build the auction orchestrator"); let slots = article_slot(); let req = make_page_bids_request("/2024/01/my-article/"); @@ -30271,7 +29455,8 @@ for (const mode of ['ready', 'queued', 'stale', 'inactive', 'throws']) { // hook does not create GPT slots client-side — matching the publisher // navigation path's shared `evaluate_ad_stack_gate` gate. let settings = settings_with_co(); - let orchestrator = AuctionOrchestrator::new(settings.auction.clone()); + let orchestrator = crate::auction::build_orchestrator(&settings) + .expect("should build the auction orchestrator"); let slots = article_slot(); let req = make_page_bids_request("/2024/01/my-article/"); @@ -30343,15 +29528,10 @@ for (const mode of ['ready', 'queued', 'stale', 'inactive', 'throws']) { /// the handler emitted. mod navigation_publisher_domain_tests { use super::*; - use crate::auction::provider::{AuctionProvider, ProviderRequestOutcome}; + use crate::auction::telemetry::{AuctionEventBatch, AuctionTelemetrySink}; - use crate::auction::types::AuctionRequest; - use crate::auction::{AuctionContext, AuctionOrchestrator}; + use crate::creative_opportunities::{CreativeOpportunityFormat, CreativeOpportunitySlot}; - use crate::platform::test_support::{ - NoopConfigStore, NoopGeo, NoopSecretStore, StubBackend, - }; - use crate::platform::{ClientInfo, PlatformResponse}; use crate::test_support::tests::crate_test_settings_str; use std::sync::Mutex; @@ -30364,49 +29544,17 @@ for (const mode of ['ready', 'queued', 'stale', 'inactive', 'throws']) { const CAPTURING_PROVIDER: &str = "request-capturing-provider"; - /// Records the [`AuctionRequest`] the orchestrator dispatched, then - /// fails its launch so no real transport handle is needed. - struct RequestCapturingProvider { - captured: Arc>>, - } - - #[async_trait::async_trait(?Send)] - impl AuctionProvider for RequestCapturingProvider { - fn provider_name(&self) -> &str { - CAPTURING_PROVIDER - } - - async fn request_bids( - &self, - request: &AuctionRequest, - _context: &AuctionContext<'_>, - ) -> Result> { - *self.captured.lock().expect("should lock captured request") = - Some(request.clone()); - Err(Report::new(TrustedServerError::Auction { - message: "capture only".to_string(), - })) - } - - async fn parse_response( - &self, - _response: PlatformResponse, - _response_time_ms: u64, - ) -> Result> { - panic!("parse_response must not run when the launch fails"); - } + /// Endpoint of the [`CAPTURING_PROVIDER`] in [`settings_with_capturing_provider`]. + const CAPTURING_ENDPOINT: &str = "https://capture.example/openrtb2/auction"; - fn timeout_ms(&self) -> u32 { - 100 - } - - fn backend_name( - &self, - _services: &RuntimeServices, - _timeout_ms: u32, - ) -> Option { - Some("capture-backend".to_string()) - } + /// The `OpenRTB` request the capturing provider's bidder received. + fn sent_auction(bidder: &ScriptedHttpClient) -> serde_json::Value { + let requests = bidder.requests(); + let request = requests + .iter() + .find(|request| request.uri == CAPTURING_ENDPOINT) + .expect("should dispatch an auction request"); + serde_json::from_slice(&request.body).expect("should send a JSON bid request") } #[derive(Default)] @@ -30477,15 +29625,14 @@ for (const mode of ['ready', 'queued', 'stale', 'inactive', 'throws']) { vec![over_limit, valid_static] } - fn assert_only_renderable_slot_was_auctioned( - captured: &Arc>>, - ) { - let request = captured - .lock() - .expect("should lock captured request") - .clone() - .expect("should dispatch an auction request"); - let slot_ids: Vec<_> = request.slots.iter().map(|slot| slot.id.as_str()).collect(); + fn assert_only_renderable_slot_was_auctioned(bidder: &ScriptedHttpClient) { + let request = sent_auction(bidder); + let slot_ids: Vec<_> = request["imp"] + .as_array() + .expect("should send impressions") + .iter() + .filter_map(|imp| imp["id"].as_str()) + .collect(); assert_eq!(slot_ids, vec!["valid_static_sibling"]); } @@ -30498,63 +29645,46 @@ for (const mode of ['ready', 'queued', 'stale', 'inactive', 'throws']) { EcContext::new_for_test(None, consent) } + /// Services on `upstream` whose capturing bidder answers no bid, returned with + /// the script that records the bid request. fn services_with( - http_client: Arc, + upstream: &PublisherUpstream, telemetry_sink: Arc, - ) -> RuntimeServices { - let telemetry_sink: Arc = telemetry_sink; - RuntimeServices::builder() - .config_store(Arc::new(NoopConfigStore)) - .secret_store(Arc::new(NoopSecretStore)) - .kv_store(Arc::new(edgezero_core::key_value_store::NoopKvStore)) - .backend(Arc::new(StubBackend)) - .http_client(http_client) - .geo(Arc::new(NoopGeo)) - .auction_telemetry_sink(telemetry_sink) - .client_info(ClientInfo::default()) - .build() - } - - fn orchestrator_capturing_request( - settings: &Settings, - captured: &Arc>>, - ) -> AuctionOrchestrator { - let mut orchestrator = AuctionOrchestrator::new(settings.auction.clone()); - orchestrator.register_provider(Arc::new(RequestCapturingProvider { - captured: Arc::clone(captured), - })); - orchestrator + ) -> (RuntimeServices, ScriptedHttpClient) { + upstream.client.respond_to( + Method::POST, + CAPTURING_ENDPOINT, + ScriptedReply::status(StatusCode::NO_CONTENT), + ); + let services = RuntimeServices { + auction_telemetry_sink: telemetry_sink, + ..upstream.services() + }; + (services, upstream.client.clone()) } /// Assert the dispatched request and every emitted telemetry row carry /// the configured publisher domain rather than the edge host. fn assert_configured_domain( - captured: &Arc>>, + bidder: &ScriptedHttpClient, telemetry_sink: &RecordingTelemetrySink, ) { - let request = captured - .lock() - .expect("should lock captured request") - .clone() - .expect("should dispatch an auction request"); + let request = sent_auction(bidder); assert_eq!( - request.publisher.domain, CONFIGURED_DOMAIN, + request["site"]["publisher"]["domain"], CONFIGURED_DOMAIN, "publisher.domain should be the configured publisher domain, not the edge host" ); - let site = request.site.expect("should populate site metadata"); assert_eq!( - site.domain, CONFIGURED_DOMAIN, + request["site"]["domain"], CONFIGURED_DOMAIN, "site.domain should be the configured publisher domain, not the edge host" ); // Only the host is asserted: the two `AuctionRequest` builders still // disagree on where the scheme comes from (edge-detected here, // canonical `https` in `convert_tsjs_to_auction_request`), which is // tracked separately. - let page_url = request - .publisher - .page_url - .as_deref() - .expect("should populate page_url"); + let page_url = request["site"]["page"] + .as_str() + .expect("should populate site.page"); let page_url_host = url::Url::parse(page_url) .expect("should build a parseable page_url") .host_str() @@ -30564,10 +29694,6 @@ for (const mode of ['ready', 'queued', 'stale', 'inactive', 'throws']) { page_url_host, CONFIGURED_DOMAIN, "page_url host should be the configured publisher domain, not the edge host" ); - assert_eq!( - site.page, page_url, - "site.page should mirror page_url, so it carries the configured publisher domain too" - ); let batches = telemetry_sink .batches @@ -30586,15 +29712,16 @@ for (const mode of ['ready', 'queued', 'stale', 'inactive', 'throws']) { #[tokio::test] async fn initial_navigation_advertises_configured_publisher_domain() { let settings = settings_with_capturing_provider(); - let captured = Arc::new(Mutex::new(None)); - let orchestrator = orchestrator_capturing_request(&settings, &captured); + let orchestrator = crate::auction::build_orchestrator(&settings) + .expect("should build the auction orchestrator"); let telemetry_sink = Arc::new(RecordingTelemetrySink::default()); - let stub = Arc::new(StubHttpClient::new()); - stub.push_response(200, b"ok".to_vec()); - let services = services_with( - Arc::clone(&stub) as Arc, - Arc::clone(&telemetry_sink), + let stub = PublisherUpstream::new(); + stub.queue_origin( + 200, + b"ok".to_vec(), + Vec::new(), ); + let (services, bidder) = services_with(&stub, Arc::clone(&telemetry_sink)); let mut ec_context = consent_allowing_ec_context(); let req = HttpRequest::builder() .method(Method::GET) @@ -30604,7 +29731,7 @@ for (const mode of ['ready', 'queued', 'stale', 'inactive', 'throws']) { .body(EdgeBody::empty()) .expect("should build test request"); - let _ = handle_publisher_request( + let (_, _owner) = handle_publisher_request( &settings, &services, None, @@ -30616,24 +29743,26 @@ for (const mode of ['ready', 'queued', 'stale', 'inactive', 'throws']) { registry: None, }, req, - EdgeCacheHeader::SMaxageFallback, + PublisherOriginOptions { + edge_header: EdgeCacheHeader::SMaxageFallback, + input_cap: None, + response_decoding: ResponseDecoding::Original, + }, ) .await .expect("should proxy publisher request"); - assert_configured_domain(&captured, &telemetry_sink); + assert_configured_domain(&bidder, &telemetry_sink); } #[tokio::test] async fn page_bids_advertises_configured_publisher_domain() { let settings = settings_with_capturing_provider(); - let captured = Arc::new(Mutex::new(None)); - let orchestrator = orchestrator_capturing_request(&settings, &captured); + let orchestrator = crate::auction::build_orchestrator(&settings) + .expect("should build the auction orchestrator"); let telemetry_sink = Arc::new(RecordingTelemetrySink::default()); - let services = services_with( - Arc::new(crate::platform::test_support::NoopHttpClient), - Arc::clone(&telemetry_sink), - ); + let (services, bidder) = + services_with(&PublisherUpstream::new(), Arc::clone(&telemetry_sink)); let mut ec_context = consent_allowing_ec_context(); let mut req = HttpRequest::builder() .method(Method::GET) @@ -30664,21 +29793,22 @@ for (const mode of ['ready', 'queued', 'stale', 'inactive', 'throws']) { .await .expect("should return ok response"); - assert_configured_domain(&captured, &telemetry_sink); + assert_configured_domain(&bidder, &telemetry_sink); } #[tokio::test] async fn initial_navigation_auctions_only_renderable_slots() { let settings = settings_with_capturing_provider(); - let captured = Arc::new(Mutex::new(None)); - let orchestrator = orchestrator_capturing_request(&settings, &captured); + let orchestrator = crate::auction::build_orchestrator(&settings) + .expect("should build the auction orchestrator"); let telemetry_sink = Arc::new(RecordingTelemetrySink::default()); - let stub = Arc::new(StubHttpClient::new()); - stub.push_response(200, b"ok".to_vec()); - let services = services_with( - Arc::clone(&stub) as Arc, - telemetry_sink, + let stub = PublisherUpstream::new(); + stub.queue_origin( + 200, + b"ok".to_vec(), + Vec::new(), ); + let (services, bidder) = services_with(&stub, telemetry_sink); let mut ec_context = consent_allowing_ec_context(); let request_path = format!("/{}", "a".repeat(60)); let req = HttpRequest::builder() @@ -30690,7 +29820,7 @@ for (const mode of ['ready', 'queued', 'stale', 'inactive', 'throws']) { .expect("should build test request"); let slots = slots_with_over_limit_dynamic_sibling(); - let _ = handle_publisher_request( + let (_, _owner) = handle_publisher_request( &settings, &services, None, @@ -30702,24 +29832,25 @@ for (const mode of ['ready', 'queued', 'stale', 'inactive', 'throws']) { registry: None, }, req, - EdgeCacheHeader::SMaxageFallback, + PublisherOriginOptions { + edge_header: EdgeCacheHeader::SMaxageFallback, + input_cap: None, + response_decoding: ResponseDecoding::Original, + }, ) .await .expect("should proxy publisher request"); - assert_only_renderable_slot_was_auctioned(&captured); + assert_only_renderable_slot_was_auctioned(&bidder); } #[tokio::test] async fn page_bids_auctions_only_renderable_slots() { let settings = settings_with_capturing_provider(); - let captured = Arc::new(Mutex::new(None)); - let orchestrator = orchestrator_capturing_request(&settings, &captured); + let orchestrator = crate::auction::build_orchestrator(&settings) + .expect("should build the auction orchestrator"); let telemetry_sink = Arc::new(RecordingTelemetrySink::default()); - let services = services_with( - Arc::new(crate::platform::test_support::NoopHttpClient), - telemetry_sink, - ); + let (services, bidder) = services_with(&PublisherUpstream::new(), telemetry_sink); let mut ec_context = consent_allowing_ec_context(); let request_path = format!("/{}", "a".repeat(60)); let mut req = HttpRequest::builder() @@ -30752,7 +29883,730 @@ for (const mode of ['ready', 'queued', 'stale', 'inactive', 'throws']) { .await .expect("should return ok response"); - assert_only_renderable_slot_was_auctioned(&captured); + assert_only_renderable_slot_was_auctioned(&bidder); + } + } + + /// The publisher origin fetched through `EdgeZero`'s client inside the response scope. + mod publisher_origin_tests { + use super::*; + + use edgezero_core::outbound::{ + HttpClient, OutboundCachePolicy, OutboundRequest, OutboundSendMode, ResponseDecoding, + ResponseMode, + }; + + use crate::outbound::{PUBLISHER_UPLOAD_BYTES, STREAMED_TIMEOUT, streamed}; + use crate::test_support::logging::WarningCapture; + use crate::test_support::scripted_client::ExpectedTransport; + + /// Origin of the scope control, which uses no publisher settings. + const CONTROL_ORIGIN: &str = "https://origin.example.com"; + + /// Host the reader addresses. + const EDGE_HOST: &str = "ts.example.com"; + + /// Input cap the buffered cases pass, the Cloudflare default. + const CAP: u64 = 10 * 1024 * 1024; + + /// One publisher setup: settings, the scripted origin, and the services that reach it. + struct OriginFixture { + settings: Arc, + origin: ScriptedHttpClient, + services: RuntimeServices, + orchestrator: Arc, + registry: IntegrationRegistry, + } + + impl OriginFixture { + fn new(settings: Settings) -> Self { + Self::with_registry(settings, IntegrationRegistry::empty_for_tests()) + } + + fn with_registry(settings: Settings, registry: IntegrationRegistry) -> Self { + let clock = TestClock::new(); + let origin = ScriptedHttpClient::new(&clock); + let services = build_services_with_outbound_client(&origin, &clock); + let orchestrator = Arc::new( + crate::auction::build_orchestrator(&settings) + .expect("should build the auction orchestrator"), + ); + Self { + settings: Arc::new(settings), + origin, + services, + orchestrator, + registry, + } + } + + /// Scripts the origin's answer to one `method` request for `path`. + fn reply(&self, method: Method, path: &str, reply: ScriptedReply) { + self.origin.respond_to( + method, + &format!("{}{path}", self.settings.publisher.origin_url), + reply, + ); + } + + fn dispatch(&self) -> AuctionDispatch<'_> { + AuctionDispatch { + integration_registry: &self.registry, + orchestrator: &self.orchestrator, + slots: &[], + registry: None, + } + } + + /// Sends `request` through the publisher handler, which takes the plain path. + async fn publish( + &self, + request: Request, + input_cap: Option, + ) -> Result<(PublisherResponse, OutboundScopeOwner), Report> + { + let mut ec_context = + EcContext::new_for_test(None, crate::consent::ConsentContext::default()); + handle_publisher_request( + &self.settings, + &self.services, + None, + &mut ec_context, + self.dispatch(), + request, + PublisherOriginOptions { + edge_header: EdgeCacheHeader::SMaxageFallback, + input_cap, + response_decoding: ResponseDecoding::Original, + }, + ) + .await + } + + /// Sends a navigation that preloads an EC snapshot, which takes the overlapped path. + async fn publish_overlapped( + &self, + request: Request, + ) -> Result<(PublisherResponse, OutboundScopeOwner), Report> + { + let graph = KvIdentityGraph::new(InMemoryEcKv::new("origin-tests")); + let mut ec_context = EcContext::new_for_test_with_ip( + Some(format!("{}.CkId01", "c".repeat(64))), + crate::consent::ConsentContext { + jurisdiction: crate::consent::jurisdiction::Jurisdiction::NonRegulated, + ..Default::default() + }, + Some("203.0.113.9".to_owned()), + ); + handle_publisher_request( + &self.settings, + &self.services, + Some(&graph), + &mut ec_context, + self.dispatch(), + request, + PublisherOriginOptions { + edge_header: EdgeCacheHeader::SurrogateControl, + input_cap: None, + response_decoding: ResponseDecoding::Original, + }, + ) + .await + } + + /// Buffers a publisher response as Axum, Cloudflare, and Spin do, then attaches its owner. + async fn buffer( + &self, + published: (PublisherResponse, OutboundScopeOwner), + method: &Method, + input_cap: u64, + ) -> Result, Report> { + let (response, owner) = published; + let response = buffer_publisher_response_async( + response, + method, + &self.settings, + &self.registry, + &self.orchestrator, + &self.services, + Some(input_cap), + ) + .await?; + owner.attach(response).map_err(publisher_owner_error) + } + + /// Streams a publisher response as Fastly does, then attaches its owner. + async fn stream( + &self, + published: (PublisherResponse, OutboundScopeOwner), + method: &Method, + ) -> Result, Report> { + let (response, owner) = published; + let response = publisher_response_into_streaming_response( + response, + method, + Arc::clone(&self.settings), + &self.registry, + Arc::clone(&self.orchestrator), + self.services.clone(), + ) + .await?; + owner.attach(response).map_err(publisher_owner_error) + } + + /// Sends `request` on the plain path and buffers the response under `input_cap`. + async fn buffered( + &self, + request: Request, + input_cap: u64, + ) -> Result, Report> { + let method = request.method().clone(); + let published = self.publish(request, Some(input_cap)).await?; + self.buffer(published, &method, input_cap).await + } + } + + fn request(method: Method, path: &str) -> Request { + HttpRequest::builder() + .method(method) + .uri(format!("https://{EDGE_HOST}{path}")) + .header(header::HOST, EDGE_HOST) + .body(EdgeBody::empty()) + .expect("should build the publisher request") + } + + fn navigation(path: &str) -> Request { + let mut request = request(Method::GET, path); + request + .headers_mut() + .insert("sec-fetch-dest", HeaderValue::from_static("document")); + request + .headers_mut() + .insert("sec-fetch-mode", HeaderValue::from_static("navigate")); + request + } + + fn post(path: &str, body: EdgeBody) -> Request { + let mut request = request(Method::POST, path); + request.headers_mut().insert( + header::CONTENT_TYPE, + HeaderValue::from_static("application/octet-stream"), + ); + *request.body_mut() = body; + request + } + + async fn body_bytes(response: Response) -> Vec { + response + .into_body() + .into_bytes_bounded(64 * 1024 * 1024) + .await + .expect("should read the response body") + .to_vec() + } + + fn chunked(bytes: &[u8], chunk: usize) -> EdgeBody { + EdgeBody::stream(futures::stream::iter( + bytes + .chunks(chunk) + .map(bytes::Bytes::copy_from_slice) + .collect::>(), + )) + } + + fn only_request( + origin: &ScriptedHttpClient, + ) -> crate::test_support::scripted_client::RecordedRequest { + let mut requests = origin.requests(); + assert_eq!(requests.len(), 1, "should send exactly one origin request"); + requests.remove(0) + } + + fn status_of(report: &Report) -> StatusCode { + crate::error::IntoHttpResponse::status_code(report.current_context()) + } + + /// The raw variants test 20 covers: origin path, status, content type, bytes. + const RAW_VARIANTS: [(&str, StatusCode, &str, &[u8]); 3] = [ + ( + "/pass-through.png", + StatusCode::OK, + "image/png", + b"example pass-through bytes", + ), + ( + "/buffered.png", + StatusCode::NOT_FOUND, + "image/png", + b"example buffered bytes", + ), + ( + "/range.html", + StatusCode::PARTIAL_CONTENT, + "text/html", + b"example range", + ), + ]; + + #[tokio::test] + async fn raw_buffered_passthrough_and_range_attach_owner_once_when_origin_call_is_inside_scope() + { + for overlapped in [false, true] { + for buffered_finalizer in [false, true] { + let fixture = OriginFixture::with_registry( + crate::publisher::allow_eof_test_settings(create_test_settings()), + IntegrationRegistry::from_html_stream_processors(vec![Arc::new( + super::TerminalTestFactory, + )]), + ); + for (path, status, content_type, bytes) in RAW_VARIANTS { + let mut reply = + ScriptedReply::status(status).header("content-type", content_type); + if status == StatusCode::PARTIAL_CONTENT { + reply = reply.header("content-range", "bytes 0-38/100"); + } + fixture.reply(Method::GET, path, reply.body(bytes)); + let published = if overlapped { + fixture.publish_overlapped(navigation(path)).await + } else { + fixture.publish(navigation(path), Some(CAP)).await + } + .expect("should fetch the raw origin response"); + assert!( + matches!( + published.0, + PublisherResponse::PassThrough { .. } + | PublisherResponse::Buffered(_) + ), + "should classify {path} as a raw variant" + ); + + let response = if buffered_finalizer { + fixture.buffer(published, &Method::GET, CAP).await + } else { + fixture.stream(published, &Method::GET).await + } + .expect("should attach the owner once to the raw response"); + + assert_eq!(response.status(), status, "should keep the {path} status"); + assert_eq!( + body_bytes(response).await, + bytes, + "should keep every {path} byte through the attached owner" + ); + } + let expected_mode = if overlapped { + OutboundSendMode::Begin + } else { + OutboundSendMode::Send + }; + assert!( + fixture + .origin + .requests() + .iter() + .all(|request| request.mode == expected_mode), + "should fetch every raw variant with {expected_mode:?}" + ); + } + } + } + + #[tokio::test] + async fn origin_call_outside_scope_fails_attach_with_already_owns_scope() { + let clock = TestClock::new(); + let origin = ScriptedHttpClient::new(&clock); + let client: HttpClient = origin.http_client(); + for begin in [false, true] { + origin.respond_to( + Method::GET, + &format!("{CONTROL_ORIGIN}/control.png"), + ScriptedReply::status(StatusCode::OK) + .header("content-type", "image/png") + .body("example control bytes"), + ); + let request = streamed( + OutboundRequest::get(format!("{CONTROL_ORIGIN}/control.png")) + .expect("should build the control request"), + STREAMED_TIMEOUT, + ); + let owner = OutboundScopeOwner::response_body(); + + // The call is made before `run` activates the owner, so it captures no + // response-body scope and head finishing gives the body an owner of its own. + let response = if begin { + owner + .run(client.begin(request)) + .await + .expect("should begin the control request") + .collect() + .await + } else { + owner.run(client.send(request)).await + } + .expect("should fetch the control response") + .into_response() + .expect("should convert the control response"); + let error = owner + .attach(response) + .expect_err("should refuse a body that already owns an outbound scope"); + + assert!( + error.inner().is_some_and(|inner| inner + .to_string() + .contains("response body already owns an outbound scope")), + "should name the second owner when begin is {begin}, got {error:?}" + ); + } + } + + #[tokio::test] + async fn origin_request_carries_inbound_body_and_upload_cap() { + let fixture = OriginFixture::new(create_test_settings()); + fixture.reply( + Method::POST, + "/form", + ScriptedReply::status(StatusCode::OK) + .header("content-type", "application/octet-stream") + .body("example accepted"), + ); + let sent: Vec = (0..=u8::MAX).cycle().take(4096).collect(); + + let response = fixture + .buffered(post("/form", chunked(&sent, 1000)), CAP) + .await + .expect("should forward the publisher POST"); + + assert_eq!( + response.status(), + StatusCode::OK, + "should answer from the origin" + ); + let forwarded = only_request(&fixture.origin); + assert_eq!( + (forwarded.method.clone(), forwarded.uri.to_string()), + ( + Method::POST, + format!("{}/form", fixture.settings.publisher.origin_url) + ), + "should forward the POST to the origin path" + ); + assert_eq!( + forwarded.body.as_ref(), + sent.as_slice(), + "should forward every inbound body byte" + ); + assert_eq!( + forwarded.max_request_body_bytes, PUBLISHER_UPLOAD_BYTES, + "should cap the upload at the publisher upload cap" + ); + assert_eq!( + PUBLISHER_UPLOAD_BYTES, + 16 * 1024 * 1024, + "should keep the publisher upload cap at 16 MiB" + ); + assert_eq!( + forwarded.max_encoded_response_bytes, + Some(CAP), + "should bound the origin body at the adapter's input cap" + ); + forwarded.assert_transport(ExpectedTransport { + timeout: STREAMED_TIMEOUT, + response_mode: ResponseMode::Streamed, + response_decoding: ResponseDecoding::Original, + cache_policy: OutboundCachePolicy::PlatformDefault, + }); + } + + #[tokio::test] + async fn upload_over_16_mib_returns_413() { + let fixture = OriginFixture::new(create_test_settings()); + fixture.reply( + Method::POST, + "/upload", + ScriptedReply::status(StatusCode::OK), + ); + let limit = usize::try_from(PUBLISHER_UPLOAD_BYTES) + .expect("should fit the upload cap in usize"); + let upload = vec![b'x'; limit + 1]; + + let error = fixture + .publish(post("/upload", chunked(&upload, 64 * 1024)), None) + .await + .err() + .expect("should refuse an upload over the publisher cap"); + + assert_eq!( + status_of(&error), + StatusCode::PAYLOAD_TOO_LARGE, + "should answer an oversized upload with 413" + ); + } + + #[tokio::test] + async fn host_override_differing_from_authority_sets_host_authority_override() { + let mut settings = create_test_settings(); + settings.publisher.origin_host_header_override = Some("www.example.com".to_owned()); + let fixture = OriginFixture::new(settings); + fixture.reply( + Method::GET, + "/page", + ScriptedReply::status(StatusCode::OK).header("content-type", "image/png"), + ); + + fixture + .buffered(request(Method::GET, "/page"), CAP) + .await + .expect("should fetch the origin with the override"); + + let forwarded = only_request(&fixture.origin); + assert_eq!( + ( + forwarded.uri.to_string(), + forwarded.host_authority.as_str(), + forwarded.has_host_authority_override + ), + ( + format!("{}/page", fixture.settings.publisher.origin_url), + "www.example.com", + true + ), + "should connect to the origin URI and send the override as its authority" + ); + } + + #[tokio::test] + async fn equivalent_host_override_sets_none() { + for equivalent in ["Example.COM", "example.com:443"] { + let mut settings = create_test_settings(); + settings.publisher.origin_url = "https://example.com".to_owned(); + settings.publisher.origin_host_header_override = Some(equivalent.to_owned()); + let fixture = OriginFixture::new(settings); + fixture.reply( + Method::GET, + "/page", + ScriptedReply::status(StatusCode::OK).header("content-type", "image/png"), + ); + + fixture + .buffered(request(Method::GET, "/page"), CAP) + .await + .expect("should fetch the origin"); + + let forwarded = only_request(&fixture.origin); + assert_eq!( + ( + forwarded.host_authority.as_str(), + forwarded.has_host_authority_override + ), + ("example.com", false), + "should set no override for the equivalent authority {equivalent}" + ); + } + } + + #[tokio::test] + async fn non_utf8_cookie_is_dropped_and_request_forwarded() { + let fixture = OriginFixture::new(create_test_settings()); + fixture.reply( + Method::GET, + "/page", + ScriptedReply::status(StatusCode::OK).header("content-type", "image/png"), + ); + let mut request = request(Method::GET, "/page"); + request.headers_mut().insert( + header::COOKIE, + HeaderValue::from_bytes(b"session=example-caf\xe9") + .expect("should build a non-UTF-8 cookie value"), + ); + let capture = WarningCapture::start(); + + let response = fixture + .buffered(request, CAP) + .await + .expect("should forward a request whose cookie is not UTF-8"); + + assert_eq!( + response.status(), + StatusCode::OK, + "should answer from the origin" + ); + let forwarded = only_request(&fixture.origin); + assert!( + !forwarded.headers.contains_key(header::COOKIE), + "should reach the origin with no Cookie field" + ); + let warnings = capture.take(); + assert!( + warnings + .iter() + .any(|warning| warning.message.contains("cookie")), + "should warn that the cookie field was dropped" + ); + assert!( + warnings + .iter() + .all(|warning| !warning.message.contains("example-caf")), + "should never log the cookie value" + ); + } + + #[tokio::test] + async fn runtime_treats_readthrough_as_false_with_warning() { + let mut settings = create_test_settings(); + settings.creative_opportunities = Some( + serde_json::from_value(serde_json::json!({ + "gam_network_id": "12345", + "origin_readthrough_enabled": true, + })) + .expect("should build a readthrough creative opportunities config"), + ); + let capture = WarningCapture::start(); + settings.warn_ignored_origin_readthrough(); + assert!( + capture.take().iter().any(|warning| { + warning.message.contains("origin_readthrough_enabled") + && warning.message.contains("ignored") + }), + "should warn that origin readthrough is ignored" + ); + let fixture = OriginFixture::new(settings); + fixture.reply( + Method::GET, + "/article", + ScriptedReply::status(StatusCode::OK) + .header("content-type", "text/html") + .header("cache-control", "public, max-age=300") + .body("example"), + ); + + fixture + .buffered(navigation("/article"), CAP) + .await + .expect("should serve the page without readthrough"); + + assert_eq!( + only_request(&fixture.origin).cache_policy, + OutboundCachePolicy::PlatformDefault, + "should fetch a shareable document with the platform default, never a shared cache" + ); + } + + #[tokio::test] + async fn source_error_mid_raw_body_fails_502_before_egress() { + let fixture = OriginFixture::new(create_test_settings()); + fixture.reply( + Method::GET, + "/broken.png", + ScriptedReply::status(StatusCode::OK) + .header("content-type", "image/png") + .body("example partial bytes") + .body_error(EdgeError::bad_gateway("example origin reset")), + ); + + let error = fixture + .buffered(request(Method::GET, "/broken.png"), CAP) + .await + .expect_err("should fail a raw body whose source fails"); + + assert_eq!( + status_of(&error), + StatusCode::BAD_GATEWAY, + "should answer a failed raw body with the publisher's 502" + ); + only_request(&fixture.origin); + } + + #[tokio::test] + async fn gzip_html_under_cap_encoded_over_cap_decoded_fails_502() { + let cap: u64 = 1024 * 1024; + let decoded_len = usize::try_from(cap).expect("should fit the cap in usize") + 1; + let mut document = b"".to_vec(); + document.resize(decoded_len - b"".len(), b'a'); + document.extend_from_slice(b""); + let mut encoder = GzEncoder::new(Vec::new(), flate2::Compression::best()); + std::io::Write::write_all(&mut encoder, &document) + .expect("should compress the document"); + let encoded = encoder.finish().expect("should finish the gzip stream"); + assert!( + (encoded.len() as u64) < cap && (document.len() as u64) > cap, + "should build gzip HTML under the cap encoded and over it decoded" + ); + let fixture = OriginFixture::new(create_test_settings()); + fixture.reply( + Method::GET, + "/article", + ScriptedReply::status(StatusCode::OK) + .header("content-type", "text/html") + .header("content-encoding", "gzip") + .body(encoded), + ); + let mut request = navigation("/article"); + request + .headers_mut() + .insert(header::ACCEPT_ENCODING, HeaderValue::from_static("gzip")); + + let error = fixture + .buffered(request, cap) + .await + .expect_err("should fail HTML whose decoded size exceeds the input cap"); + + assert_eq!( + status_of(&error), + StatusCode::BAD_GATEWAY, + "should answer decoded HTML over the input cap with the publisher's 502" + ); + only_request(&fixture.origin); + } + + #[tokio::test] + async fn rewrite_growing_past_output_cap_fails() { + let mut settings = create_test_settings(); + settings.publisher.max_buffered_body_bytes = 4096; + let mut document = b"".to_vec(); + document.resize(4096 - b"".len(), b'a'); + document.extend_from_slice(b""); + let fixture = OriginFixture::new(settings); + fixture.reply( + Method::GET, + "/article", + ScriptedReply::status(StatusCode::OK) + .header("content-type", "text/html") + .body(document), + ); + + let error = fixture + .buffered(navigation("/article"), 4096) + .await + .expect_err("should fail output that grows past the output cap"); + + assert_eq!( + status_of(&error), + StatusCode::BAD_GATEWAY, + "should answer output over its cap with the publisher's 502" + ); + only_request(&fixture.origin); + } + + #[tokio::test] + async fn origin_url_with_userinfo_answers_400() { + let mut settings = create_test_settings(); + settings.publisher.origin_url = + "https://example-user:example-pass@origin.example.com".to_owned(); + let fixture = OriginFixture::new(settings); + + let error = fixture + .publish(request(Method::GET, "/page"), Some(CAP)) + .await + .err() + .expect("should refuse an origin URL with userinfo"); + + assert_eq!( + status_of(&error), + StatusCode::BAD_REQUEST, + "should answer a publisher forward to an origin with userinfo with 400" + ); + fixture.origin.assert_no_requests(); } } } diff --git a/crates/trusted-server-core/src/publisher/html_finalization.rs b/crates/trusted-server-core/src/publisher/html_finalization.rs index 49ad33a5f8..5950af97ff 100644 --- a/crates/trusted-server-core/src/publisher/html_finalization.rs +++ b/crates/trusted-server-core/src/publisher/html_finalization.rs @@ -122,7 +122,7 @@ impl HtmlEofDriver { })?; processor.set_finish_marker(marker.as_deref()); let compression = Compression::from_content_encoding(¶ms.content_encoding); - let limit = settings.publisher.max_buffered_body_bytes; + let limit = params.max_input_bytes(settings); let input = TerminalInput::new(body, compression, limit); let hold = params.dispatched_auction.take().map(|dispatched| { AuctionHoldState::new( diff --git a/crates/trusted-server-core/src/request_signing/endpoints.rs b/crates/trusted-server-core/src/request_signing/endpoints.rs index d35e51a151..736eda87ba 100644 --- a/crates/trusted-server-core/src/request_signing/endpoints.rs +++ b/crates/trusted-server-core/src/request_signing/endpoints.rs @@ -9,7 +9,7 @@ use http::{Request, Response, StatusCode, header}; use serde::{Deserialize, Serialize}; use crate::error::{IntoHttpResponse, TrustedServerError}; -use crate::http_util::enforce_max_body_size; +use crate::http_util::read_request_body; use crate::platform::RuntimeServices; use crate::request_signing::discovery::TrustedServerDiscovery; use crate::request_signing::rotation::KeyRotationManager; @@ -24,19 +24,6 @@ fn json_response(status: StatusCode, body: String) -> Response { .expect("should build json response") } -fn request_body_bytes( - body: EdgeBody, - endpoint: &str, -) -> Result> { - if body.is_stream() { - return Err(Report::new(TrustedServerError::BadRequest { - message: format!("{endpoint} request body must be buffered, not streamed"), - })); - } - - Ok(body.into_bytes().unwrap_or_default()) -} - /// Retrieves and returns the trusted-server discovery document. /// /// This endpoint provides a standardized discovery mechanism following the IAB @@ -106,15 +93,16 @@ const ADMIN_MAX_BODY_BYTES: usize = 4096; /// /// # Errors /// -/// Returns an error if the request body cannot be parsed as JSON or if the -/// response body cannot be serialized. -pub fn handle_verify_signature( +/// Returns an error if the request body cannot be read within 4096 bytes (see +/// [`read_request_body`]), cannot be parsed as JSON, or if the response body +/// cannot be serialized. +pub async fn handle_verify_signature( _settings: &Settings, services: &RuntimeServices, req: Request, ) -> Result, Report> { - let body = request_body_bytes(req.into_body(), "verify-signature")?; - enforce_max_body_size(&body, VERIFY_MAX_BODY_BYTES, "verify-signature")?; + let (parts, body) = req.into_parts(); + let body = read_request_body(&parts.headers, body, VERIFY_MAX_BODY_BYTES).await?; let verify_req: VerifySignatureRequest = serde_json::from_slice(&body).change_context(TrustedServerError::Configuration { message: "invalid JSON request body".into(), @@ -292,8 +280,9 @@ pub fn kid_is_creatable(kid: &str) -> bool { /// /// # Errors /// -/// Returns an error if the request signing settings are missing or JSON parsing fails. -pub fn handle_rotate_key( +/// Returns an error if the request signing settings are missing, the request body +/// cannot be read within 4096 bytes (see [`read_request_body`]), or JSON parsing fails. +pub async fn handle_rotate_key( settings: &Settings, services: &RuntimeServices, req: Request, @@ -303,8 +292,8 @@ pub fn handle_rotate_key( secret_store_id, } = signing_store_ids(settings)?; - let body = request_body_bytes(req.into_body(), "rotate-key")?; - enforce_max_body_size(&body, ADMIN_MAX_BODY_BYTES, "rotate-key")?; + let (parts, body) = req.into_parts(); + let body = read_request_body(&parts.headers, body, ADMIN_MAX_BODY_BYTES).await?; let rotate_req: RotateKeyRequest = if body.is_empty() { RotateKeyRequest { kid: None } } else { @@ -319,7 +308,10 @@ pub fn handle_rotate_key( } else { Ok(()) }; - let result = validation_result.and_then(|()| manager.rotate_key(services, rotate_req.kid)); + let result = match validation_result { + Ok(()) => manager.rotate_key(services, rotate_req.kid).await, + Err(error) => Err(error), + }; match result { Ok(result) => { @@ -411,8 +403,9 @@ pub struct DeactivateKeyResponse { /// /// # Errors /// -/// Returns an error if the request signing settings are missing or JSON parsing fails. -pub fn handle_deactivate_key( +/// Returns an error if the request signing settings are missing, the request body +/// cannot be read within 4096 bytes (see [`read_request_body`]), or JSON parsing fails. +pub async fn handle_deactivate_key( settings: &Settings, services: &RuntimeServices, req: Request, @@ -422,8 +415,8 @@ pub fn handle_deactivate_key( secret_store_id, } = signing_store_ids(settings)?; - let body = request_body_bytes(req.into_body(), "deactivate-key")?; - enforce_max_body_size(&body, ADMIN_MAX_BODY_BYTES, "deactivate-key")?; + let (parts, body) = req.into_parts(); + let body = read_request_body(&parts.headers, body, ADMIN_MAX_BODY_BYTES).await?; let deactivate_req: DeactivateKeyRequest = serde_json::from_slice(&body).change_context(TrustedServerError::Configuration { message: "invalid JSON request body".into(), @@ -435,13 +428,11 @@ pub fn handle_deactivate_key( // validation rules (e.g. digit- or uppercase-leading) can still be // deactivated or deleted. The stricter validate_kid only gates new key // creation/rotation. - let result = validate_kid_format(&deactivate_req.kid).and_then(|()| { - if deactivate_req.delete { - manager.delete_key(services, &deactivate_req.kid) - } else { - manager.deactivate_key(services, &deactivate_req.kid) - } - }); + let result = match validate_kid_format(&deactivate_req.kid) { + Ok(()) if deactivate_req.delete => manager.delete_key(services, &deactivate_req.kid).await, + Ok(()) => manager.deactivate_key(services, &deactivate_req.kid).await, + Err(error) => Err(error), + }; match result { Ok(()) => { @@ -511,6 +502,7 @@ mod tests { }; use super::*; + use crate::test_support::body_read::body_read_failure_cases; fn build_request(method: Method, uri: &str, body: Option<&str>) -> HttpRequest { let body = match body { @@ -525,8 +517,11 @@ mod tests { .expect("should build request") } - fn build_streaming_request(method: Method, uri: &str) -> HttpRequest { - let stream = futures::stream::iter(vec![Bytes::from_static(b"{}")]); + /// Builds a request whose `body` arrives as a stream of two chunks. + fn build_streaming_request(method: Method, uri: &str, body: &[u8]) -> HttpRequest { + let (head, tail) = body.split_at(body.len() / 2); + let stream = + futures::stream::iter([Bytes::copy_from_slice(head), Bytes::copy_from_slice(tail)]); HttpRequest::builder() .method(method) .uri(uri) @@ -559,6 +554,7 @@ mod tests { /// Config store stub that returns a minimal JWKS with one Ed25519 key. struct StubJwksConfigStore; + #[async_trait::async_trait(?Send)] impl PlatformConfigStore for StubJwksConfigStore { fn get(&self, _store_name: &StoreName, key: &str) -> Result> { match key { @@ -571,11 +567,11 @@ mod tests { } } - fn put(&self, _: &StoreId, _: &str, _: &str) -> Result<(), Report> { + async fn put(&self, _: &StoreId, _: &str, _: &str) -> Result<(), Report> { Err(Report::new(PlatformError::Unsupported)) } - fn delete(&self, _: &StoreId, _: &str) -> Result<(), Report> { + async fn delete(&self, _: &StoreId, _: &str) -> Result<(), Report> { Err(Report::new(PlatformError::Unsupported)) } } @@ -605,7 +601,7 @@ mod tests { Some(&body), ); - let resp = handle_verify_signature(&settings, &services, req) + let resp = futures::executor::block_on(handle_verify_signature(&settings, &services, req)) .expect("should handle verification request"); assert_eq!(resp.status(), StatusCode::OK); assert_json_content_type(&resp); @@ -644,7 +640,7 @@ mod tests { Some(&body), ); - let resp = handle_verify_signature(&settings, &services, req) + let resp = futures::executor::block_on(handle_verify_signature(&settings, &services, req)) .expect("should handle verification request"); assert_eq!(resp.status(), StatusCode::OK); assert_json_content_type(&resp); @@ -679,7 +675,7 @@ mod tests { ); let services = noop_services(); - let resp = handle_verify_signature(&settings, &services, req) + let resp = futures::executor::block_on(handle_verify_signature(&settings, &services, req)) .expect("should return a verification response for internal errors"); assert_eq!(resp.status(), StatusCode::OK, "should return 200 OK"); @@ -715,7 +711,8 @@ mod tests { Some("not valid json"), ); - let result = handle_verify_signature(&settings, &noop_services(), req); + let result = + futures::executor::block_on(handle_verify_signature(&settings, &noop_services(), req)); assert!(result.is_err(), "Malformed JSON should error"); } @@ -724,7 +721,7 @@ mod tests { let settings = crate::test_support::tests::create_test_settings(); let req = build_request(Method::POST, "https://test.com/admin/keys/rotate", None); - let resp = handle_rotate_key(&settings, &noop_services(), req) + let resp = futures::executor::block_on(handle_rotate_key(&settings, &noop_services(), req)) .expect("should return a response even when stores are unavailable"); assert_eq!( @@ -762,7 +759,7 @@ mod tests { Some(&body_json), ); - let resp = handle_rotate_key(&settings, &noop_services(), req) + let resp = futures::executor::block_on(handle_rotate_key(&settings, &noop_services(), req)) .expect("should return a response even when stores are unavailable"); assert_eq!( @@ -794,7 +791,8 @@ mod tests { Some("invalid json"), ); - let result = handle_rotate_key(&settings, &noop_services(), req); + let result = + futures::executor::block_on(handle_rotate_key(&settings, &noop_services(), req)); assert!(result.is_err(), "Invalid JSON should return error"); } @@ -813,7 +811,7 @@ mod tests { Some(&body_json), ); - let resp = handle_rotate_key(&settings, &noop_services(), req) + let resp = futures::executor::block_on(handle_rotate_key(&settings, &noop_services(), req)) .expect("should return a response for invalid kid"); assert_eq!( @@ -856,8 +854,9 @@ mod tests { Some(&body_json), ); - let resp = handle_deactivate_key(&settings, &noop_services(), req) - .expect("should return a response even when stores are unavailable"); + let resp = + futures::executor::block_on(handle_deactivate_key(&settings, &noop_services(), req)) + .expect("should return a response even when stores are unavailable"); assert_eq!( resp.status(), @@ -896,8 +895,9 @@ mod tests { Some(&body_json), ); - let resp = handle_deactivate_key(&settings, &noop_services(), req) - .expect("should return a response even when stores are unavailable"); + let resp = + futures::executor::block_on(handle_deactivate_key(&settings, &noop_services(), req)) + .expect("should return a response even when stores are unavailable"); assert_eq!( resp.status(), @@ -932,7 +932,8 @@ mod tests { Some("invalid json"), ); - let result = handle_deactivate_key(&settings, &noop_services(), req); + let result = + futures::executor::block_on(handle_deactivate_key(&settings, &noop_services(), req)); assert!(result.is_err(), "Invalid JSON should return error"); } @@ -953,8 +954,9 @@ mod tests { Some(&body_json), ); - let resp = handle_deactivate_key(&settings, &noop_services(), req) - .expect("should return a response for invalid kid"); + let resp = + futures::executor::block_on(handle_deactivate_key(&settings, &noop_services(), req)) + .expect("should return a response for invalid kid"); assert_eq!( resp.status(), @@ -988,8 +990,9 @@ mod tests { "https://test.com/verify-signature", Some(&oversized), ); - let err = handle_verify_signature(&settings, &noop_services(), req) - .expect_err("should reject oversized body"); + let err = + futures::executor::block_on(handle_verify_signature(&settings, &noop_services(), req)) + .expect_err("should reject oversized body"); assert_eq!( err.current_context().status_code(), StatusCode::PAYLOAD_TOO_LARGE, @@ -998,26 +1001,56 @@ mod tests { } #[test] - fn verify_signature_rejects_streaming_body() { + fn verify_signature_reads_a_streamed_body() { let settings = crate::test_support::tests::create_test_settings(); - let req = build_streaming_request(Method::POST, "https://test.com/verify-signature"); - let err = handle_verify_signature(&settings, &noop_services(), req) - .expect_err("should reject streaming verify body"); - assert_eq!( - err.current_context().status_code(), - StatusCode::BAD_REQUEST, - "should return 400 for streaming verify body" + let services = build_request_signing_services(); + let signer = crate::request_signing::RequestSigner::from_services(&services) + .expect("should create signer from services"); + let verify_req = VerifySignatureRequest { + payload: "test message".to_string(), + signature: signer.sign(b"test message").expect("should sign payload"), + kid: signer.kid.clone(), + }; + let body = serde_json::to_vec(&verify_req).expect("should serialize verify request"); + let req = build_streaming_request( + Method::POST, + "https://edge.example.com/verify-signature", + &body, ); + + let resp = futures::executor::block_on(handle_verify_signature(&settings, &services, req)) + .expect("should verify a streamed body"); + + assert_eq!(resp.status(), StatusCode::OK); + let verify_resp: VerifySignatureResponse = + serde_json::from_str(&response_body_string(resp)) + .expect("should deserialize verify response"); assert!( - matches!( - err.current_context(), - TrustedServerError::BadRequest { message } - if message == "verify-signature request body must be buffered, not streamed" - ), - "should explain that verify bodies must be buffered" + verify_resp.verified, + "should verify the signature read from a streamed body" ); } + #[test] + fn verify_signature_body_read_failures_map_to_statuses() { + let settings = crate::test_support::tests::create_test_settings(); + for case in body_read_failure_cases(VERIFY_MAX_BODY_BYTES) { + let (req, expectation) = case.into_request( + HttpRequest::builder() + .method(Method::POST) + .uri("https://edge.example.com/verify-signature"), + ); + + let outcome = futures::executor::block_on(handle_verify_signature( + &settings, + &noop_services(), + req, + )); + + expectation.assert_outcome("verify-signature", &outcome); + } + } + #[test] fn rotate_key_rejects_oversized_body() { let settings = crate::test_support::tests::create_test_settings(); @@ -1027,7 +1060,7 @@ mod tests { "https://test.com/admin/keys/rotate", Some(&oversized), ); - let err = handle_rotate_key(&settings, &noop_services(), req) + let err = futures::executor::block_on(handle_rotate_key(&settings, &noop_services(), req)) .expect_err("should reject oversized body"); assert_eq!( err.current_context().status_code(), @@ -1037,26 +1070,61 @@ mod tests { } #[test] - fn rotate_key_rejects_streaming_body() { + fn rotate_key_reads_a_streamed_body() { let settings = crate::test_support::tests::create_test_settings(); - let req = build_streaming_request(Method::POST, "https://test.com/admin/keys/rotate"); - let err = handle_rotate_key(&settings, &noop_services(), req) - .expect_err("should reject streaming rotate body"); + let req = build_streaming_request( + Method::POST, + "https://edge.example.com/admin/keys/rotate", + br#"{"kid":"bad,kid"}"#, + ); + + let resp = futures::executor::block_on(handle_rotate_key(&settings, &noop_services(), req)) + .expect("should return a response for the streamed kid"); + assert_eq!( - err.current_context().status_code(), + resp.status(), StatusCode::BAD_REQUEST, - "should return 400 for streaming rotate body" - ); - assert!( - matches!( - err.current_context(), - TrustedServerError::BadRequest { message } - if message == "rotate-key request body must be buffered, not streamed" - ), - "should explain that rotate bodies must be buffered" + "should validate the kid read from a streamed body" ); } + #[test] + fn rotate_key_body_read_failures_map_to_statuses() { + let settings = crate::test_support::tests::create_test_settings(); + for case in body_read_failure_cases(ADMIN_MAX_BODY_BYTES) { + let (req, expectation) = case.into_request( + HttpRequest::builder() + .method(Method::POST) + .uri("https://edge.example.com/admin/keys/rotate"), + ); + + let outcome = + futures::executor::block_on(handle_rotate_key(&settings, &noop_services(), req)); + + expectation.assert_outcome("rotate-key", &outcome); + } + } + + #[test] + fn deactivate_key_body_read_failures_map_to_statuses() { + let settings = crate::test_support::tests::create_test_settings(); + for case in body_read_failure_cases(ADMIN_MAX_BODY_BYTES) { + let (req, expectation) = case.into_request( + HttpRequest::builder() + .method(Method::POST) + .uri("https://edge.example.com/admin/keys/deactivate"), + ); + + let outcome = futures::executor::block_on(handle_deactivate_key( + &settings, + &noop_services(), + req, + )); + + expectation.assert_outcome("deactivate-key", &outcome); + } + } + #[test] fn deactivate_key_rejects_oversized_body() { let settings = crate::test_support::tests::create_test_settings(); @@ -1066,8 +1134,9 @@ mod tests { "https://test.com/admin/keys/deactivate", Some(&oversized), ); - let err = handle_deactivate_key(&settings, &noop_services(), req) - .expect_err("should reject oversized body"); + let err = + futures::executor::block_on(handle_deactivate_key(&settings, &noop_services(), req)) + .expect_err("should reject oversized body"); assert_eq!( err.current_context().status_code(), StatusCode::PAYLOAD_TOO_LARGE, @@ -1159,8 +1228,9 @@ mod tests { Some(&body_json), ); - let resp = handle_deactivate_key(&settings, &noop_services(), req) - .expect("should return a response for a legacy digit-leading kid"); + let resp = + futures::executor::block_on(handle_deactivate_key(&settings, &noop_services(), req)) + .expect("should return a response for a legacy digit-leading kid"); assert_ne!( resp.status(), diff --git a/crates/trusted-server-core/src/request_signing/jwks.rs b/crates/trusted-server-core/src/request_signing/jwks.rs index 7fd7790fe9..af36decde9 100644 --- a/crates/trusted-server-core/src/request_signing/jwks.rs +++ b/crates/trusted-server-core/src/request_signing/jwks.rs @@ -105,6 +105,7 @@ mod tests { struct FailingConfigStore; + #[async_trait::async_trait(?Send)] impl PlatformConfigStore for FailingConfigStore { fn get( &self, @@ -114,7 +115,7 @@ mod tests { Err(Report::new(PlatformError::ConfigStore)) } - fn put( + async fn put( &self, _store_id: &StoreId, _key: &str, @@ -123,7 +124,11 @@ mod tests { Err(Report::new(PlatformError::Unsupported)) } - fn delete(&self, _store_id: &StoreId, _key: &str) -> Result<(), Report> { + async fn delete( + &self, + _store_id: &StoreId, + _key: &str, + ) -> Result<(), Report> { Err(Report::new(PlatformError::Unsupported)) } } diff --git a/crates/trusted-server-core/src/request_signing/rotation.rs b/crates/trusted-server-core/src/request_signing/rotation.rs index 2d7222b705..37e8848fdd 100644 --- a/crates/trusted-server-core/src/request_signing/rotation.rs +++ b/crates/trusted-server-core/src/request_signing/rotation.rs @@ -60,7 +60,7 @@ impl KeyRotationManager { /// # Errors /// /// Returns an error if key storage or update operations fail. - pub fn rotate_key( + pub async fn rotate_key( &self, services: &RuntimeServices, kid: Option, @@ -86,14 +86,16 @@ impl KeyRotationManager { let jwk = keypair.get_jwk(new_kid.clone()); // Step 1: write private key. Nothing to roll back on failure. - self.store_private_key(services, &new_kid, &keypair.signing_key)?; + self.store_private_key(services, &new_kid, &keypair.signing_key) + .await?; // Step 2: write public JWK. Roll back the private key on failure so no // orphaned key material is left in the secret store. - if let Err(err) = self.store_public_jwk(services, &new_kid, &jwk) { + if let Err(err) = self.store_public_jwk(services, &new_kid, &jwk).await { if let Err(rollback_err) = services .secret_store() .delete(&self.secret_store_id, &new_kid) + .await { log::warn!( "rotate_key: rollback of private key '{new_kid}' failed after JWK write error: {rollback_err}" @@ -110,10 +112,11 @@ impl KeyRotationManager { // Step 3: publish the new kid in active-kids BEFORE flipping current-kid. // Roll back both artifacts on failure so the new kid never appears in JWKS // without a reachable private key. - if let Err(err) = self.update_active_kids(services, &active_kids) { + if let Err(err) = self.update_active_kids(services, &active_kids).await { if let Err(rollback_err) = services .config_store() .delete(&self.config_store_id, &new_kid) + .await { log::warn!( "rotate_key: rollback of JWK '{new_kid}' failed after active-kids write error: {rollback_err}" @@ -122,6 +125,7 @@ impl KeyRotationManager { if let Err(rollback_err) = services .secret_store() .delete(&self.secret_store_id, &new_kid) + .await { log::warn!( "rotate_key: rollback of private key '{new_kid}' failed after active-kids write error: {rollback_err}" @@ -131,8 +135,8 @@ impl KeyRotationManager { } // Step 4: flip current-kid last. A failure here leaves the old kid still - // active and the new kid visible in JWKS but unused — a recoverable state. - self.update_current_kid(services, &new_kid)?; + // active and the new kid visible in JWKS but unused, which is recoverable. + self.update_current_kid(services, &new_kid).await?; Ok(KeyRotationResult { new_kid, @@ -160,7 +164,7 @@ impl KeyRotationManager { format!("{base_kid}-{}", Uuid::new_v4().simple()) } - fn store_private_key( + async fn store_private_key( &self, services: &RuntimeServices, kid: &str, @@ -174,12 +178,13 @@ impl KeyRotationManager { services .secret_store() .create(&self.secret_store_id, kid, &key_b64) + .await .change_context(TrustedServerError::Configuration { message: format!("failed to store private key '{kid}'"), }) } - fn store_public_jwk( + async fn store_public_jwk( &self, services: &RuntimeServices, kid: &str, @@ -194,12 +199,13 @@ impl KeyRotationManager { services .config_store() .put(&self.config_store_id, kid, &jwk_json) + .await .change_context(TrustedServerError::Configuration { message: format!("failed to store public JWK '{kid}'"), }) } - fn update_current_kid( + async fn update_current_kid( &self, services: &RuntimeServices, kid: &str, @@ -207,12 +213,13 @@ impl KeyRotationManager { services .config_store() .put(&self.config_store_id, "current-kid", kid) + .await .change_context(TrustedServerError::Configuration { message: "failed to update current-kid".into(), }) } - fn update_active_kids( + async fn update_active_kids( &self, services: &RuntimeServices, active_kids: &[String], @@ -222,6 +229,7 @@ impl KeyRotationManager { services .config_store() .put(&self.config_store_id, "active-kids", &active_kids_str) + .await .change_context(TrustedServerError::Configuration { message: "failed to update active-kids".into(), }) @@ -244,7 +252,7 @@ impl KeyRotationManager { /// # Errors /// /// Returns an error if this would deactivate the last active key, or if the update fails. - pub fn deactivate_key( + pub async fn deactivate_key( &self, services: &RuntimeServices, kid: &str, @@ -260,7 +268,7 @@ impl KeyRotationManager { })); } - self.update_active_kids(services, &active_kids) + self.update_active_kids(services, &active_kids).await } /// Deletes a key by deactivating it and removing it from storage. @@ -268,13 +276,13 @@ impl KeyRotationManager { /// # Errors /// /// Returns an error if deactivation fails or if the key cannot be deleted from storage. - pub fn delete_key( + pub async fn delete_key( &self, services: &RuntimeServices, kid: &str, ) -> Result<(), Report> { self.ensure_not_current_key(services, kid, "delete")?; - self.deactivate_key(services, kid)?; + self.deactivate_key(services, kid).await?; // Delete the private key first. A failure here leaves the JWK in the // config store but no private key — the key is verifiable but cannot @@ -283,6 +291,7 @@ impl KeyRotationManager { services .secret_store() .delete(&self.secret_store_id, kid) + .await .change_context(TrustedServerError::Configuration { message: "failed to delete signing key from secret store".into(), })?; @@ -290,6 +299,7 @@ impl KeyRotationManager { services .config_store() .delete(&self.config_store_id, kid) + .await .change_context(TrustedServerError::Configuration { message: "failed to delete JWK from config store".into(), })?; @@ -332,6 +342,7 @@ mod tests { use std::sync::{Arc, Mutex}; use error_stack::Report; + use futures::executor::block_on; use crate::platform::test_support::build_services_with_config_and_secret; use crate::platform::{ @@ -396,6 +407,7 @@ mod tests { } } + #[async_trait::async_trait(?Send)] impl PlatformConfigStore for SpyConfigStore { fn get(&self, _: &StoreName, key: &str) -> Result> { self.inner @@ -407,7 +419,7 @@ mod tests { .ok_or_else(|| Report::new(PlatformError::ConfigStore)) } - fn put( + async fn put( &self, store_id: &StoreId, key: &str, @@ -433,7 +445,7 @@ mod tests { Ok(()) } - fn delete(&self, store_id: &StoreId, key: &str) -> Result<(), Report> { + async fn delete(&self, store_id: &StoreId, key: &str) -> Result<(), Report> { self.inner .deletes .lock() @@ -500,12 +512,13 @@ mod tests { } } + #[async_trait::async_trait(?Send)] impl PlatformSecretStore for SpySecretStore { fn get_bytes(&self, _: &StoreName, _: &str) -> Result, Report> { Err(Report::new(PlatformError::SecretStore)) } - fn create( + async fn create( &self, store_id: &StoreId, name: &str, @@ -528,7 +541,11 @@ mod tests { Ok(()) } - fn delete(&self, store_id: &StoreId, name: &str) -> Result<(), Report> { + async fn delete( + &self, + store_id: &StoreId, + name: &str, + ) -> Result<(), Report> { self.inner .deletes .lock() @@ -574,7 +591,7 @@ mod tests { let services = build_services_with_config_and_secret(config_store, secret_store); let manager = KeyRotationManager::new("cfg-id", "sec-id"); - let result = manager.rotate_key(&services, Some("new-kid".to_owned())); + let result = block_on(manager.rotate_key(&services, Some("new-kid".to_owned()))); assert!(result.is_ok(), "should succeed when stores accept writes"); let rotation = result.expect("should produce rotation result"); @@ -596,8 +613,7 @@ mod tests { let services = build_services_with_config_and_secret(config_store, secret_store); let manager = KeyRotationManager::new("cfg-id", "sec-id"); - let rotation = manager - .rotate_key(&services, Some("kid-c".to_owned())) + let rotation = block_on(manager.rotate_key(&services, Some("kid-c".to_owned()))) .expect("should rotate key successfully"); assert_eq!( @@ -627,8 +643,7 @@ mod tests { let services = build_services_with_config_and_secret(config_store, secret_store); let manager = KeyRotationManager::new("cfg-id", "sec-id"); - let rotation = manager - .rotate_key(&services, Some("kid-c".to_owned())) + let rotation = block_on(manager.rotate_key(&services, Some("kid-c".to_owned()))) .expect("should rotate key successfully"); assert_eq!( @@ -650,7 +665,7 @@ mod tests { build_services_with_config_and_secret(config_store.clone(), secret_store.clone()); let manager = KeyRotationManager::new("cfg-id", "sec-id"); - let result = manager.rotate_key(&services, Some("kid-a".to_owned())); + let result = block_on(manager.rotate_key(&services, Some("kid-a".to_owned()))); assert!( result.is_err(), @@ -678,8 +693,7 @@ mod tests { let services = build_services_with_config_and_secret(config_store, secret_store); let manager = KeyRotationManager::new("cfg-id", "sec-id"); - let rotation = manager - .rotate_key(&services, None) + let rotation = block_on(manager.rotate_key(&services, None)) .expect("should rotate with a uniquified generated kid"); assert_ne!( @@ -709,7 +723,7 @@ mod tests { let services = build_services_with_config_and_secret(config_store, secret_store); let manager = KeyRotationManager::new("cfg-id", "sec-id"); - let result = manager.deactivate_key(&services, "only-key"); + let result = block_on(manager.deactivate_key(&services, "only-key")); assert!( result.is_err(), @@ -740,7 +754,7 @@ mod tests { let services = build_services_with_config_and_secret(config_store, secret_store); let manager = KeyRotationManager::new("cfg-id", "sec-id"); - let result = manager.rotate_key(&services, Some("new-kid".to_owned())); + let result = block_on(manager.rotate_key(&services, Some("new-kid".to_owned()))); assert!( result.is_err(), @@ -756,7 +770,7 @@ mod tests { build_services_with_config_and_secret(config_store.clone(), secret_store.clone()); let manager = KeyRotationManager::new("cfg-id", "sec-id"); - let result = manager.rotate_key(&services, Some("rollback-kid".to_owned())); + let result = block_on(manager.rotate_key(&services, Some("rollback-kid".to_owned()))); assert!(result.is_err(), "should fail when JWK write fails"); assert_eq!( @@ -778,7 +792,7 @@ mod tests { build_services_with_config_and_secret(config_store.clone(), secret_store.clone()); let manager = KeyRotationManager::new("cfg-id", "sec-id"); - let result = manager.rotate_key(&services, Some("rollback-kid".to_owned())); + let result = block_on(manager.rotate_key(&services, Some("rollback-kid".to_owned()))); assert!(result.is_err(), "should fail when active-kids write fails"); assert_eq!( @@ -805,7 +819,7 @@ mod tests { build_services_with_config_and_secret(config_store.clone(), secret_store.clone()); let manager = KeyRotationManager::new("cfg-id", "sec-id"); - let result = manager.deactivate_key(&services, "kid-a"); + let result = block_on(manager.deactivate_key(&services, "kid-a")); assert!(result.is_err(), "should reject deactivating current-kid"); assert!( @@ -830,7 +844,7 @@ mod tests { build_services_with_config_and_secret(config_store.clone(), secret_store.clone()); let manager = KeyRotationManager::new("cfg-id", "sec-id"); - let result = manager.delete_key(&services, "kid-a"); + let result = block_on(manager.delete_key(&services, "kid-a")); assert!(result.is_err(), "should reject deleting current-kid"); assert!( @@ -857,9 +871,7 @@ mod tests { let services = build_services_with_config_and_secret(config_store, secret_store); let manager = KeyRotationManager::new("cfg-id", "sec-id"); - manager - .delete_key(&services, "kid-a") - .expect("should delete key successfully"); + block_on(manager.delete_key(&services, "kid-a")).expect("should delete key successfully"); // After deletion, the JWK entry should be gone from the config store. let jwk_gone = services diff --git a/crates/trusted-server-core/src/response_privacy.rs b/crates/trusted-server-core/src/response_privacy.rs index 46ff4bf5b4..79f57720d7 100644 --- a/crates/trusted-server-core/src/response_privacy.rs +++ b/crates/trusted-server-core/src/response_privacy.rs @@ -26,6 +26,34 @@ pub struct TerminalPrivateResponse; const INACTIVE_AD_STACK_BROWSER_CACHE_CONTROL: &str = "private, max-age=60"; +/// Operator `response_headers` names that Trusted Server never sends. +/// +/// Response egress owns message framing: it refuses a response that declares +/// `Trailer`, frames the body itself, and strips the hop-by-hop fields. A +/// `Connection` value also removes the fields it names, and a malformed one +/// fails the response with a 500 before commit. +const FORBIDDEN_OPERATOR_RESPONSE_HEADERS: [&str; 10] = [ + "trailer", + "content-length", + "connection", + "keep-alive", + "te", + "transfer-encoding", + "upgrade", + "proxy-connection", + "proxy-authenticate", + "proxy-authorization", +]; + +/// Whether `name` is an operator response header that Trusted Server never sends. +/// +/// Matches [`FORBIDDEN_OPERATOR_RESPONSE_HEADERS`] case-insensitively. +pub(crate) fn is_forbidden_operator_response_header(name: &str) -> bool { + FORBIDDEN_OPERATOR_RESPONSE_HEADERS + .iter() + .any(|forbidden| name.eq_ignore_ascii_case(forbidden)) +} + #[derive(Debug, Clone, Copy)] struct GeneratedInactiveAdStackBrowserCachePolicy; @@ -156,8 +184,9 @@ pub fn enforce_set_cookie_cache_privacy(response: &mut Response) { /// configured `Set-Cookie` combined with public edge-cache headers cannot /// produce a shared-cacheable cookie-bearing response. /// -/// Invalid header names/values are logged and skipped rather than panicking, so -/// a misconfigured operator header can never take down a request. +/// The entries [`warn_skipped_operator_response_headers`] reports are skipped +/// here without a log, so a misconfigured operator header never takes down a +/// request and a request that applies the headers twice still warns once. pub fn apply_response_headers_with_cache_privacy(settings: &Settings, response: &mut Response) { enforce_set_cookie_cache_privacy(response); @@ -165,27 +194,15 @@ pub fn apply_response_headers_with_cache_privacy(settings: &Settings, response: enforce_uncacheable_cache_privacy(response); for (key, value) in &settings.response_headers { + let Ok((name, value)) = operator_response_header(key, value) else { + continue; + }; if response_is_uncacheable - && (key.eq_ignore_ascii_case(header::CACHE_CONTROL.as_str()) - || is_edge_cache_header_name(key)) + && (name == header::CACHE_CONTROL || is_edge_cache_header_name(name.as_str())) { continue; } - let header_name = match HeaderName::from_bytes(key.as_bytes()) { - Ok(name) => name, - Err(_) => { - log::warn!("Skipping invalid configured response header name {key}"); - continue; - } - }; - let header_value = match HeaderValue::from_str(value) { - Ok(value) => value, - Err(_) => { - log::warn!("Skipping invalid configured response header value for {key}"); - continue; - } - }; - response.headers_mut().insert(header_name, header_value); + response.headers_mut().insert(name, value); } enforce_uncacheable_cache_privacy(response); @@ -197,6 +214,55 @@ pub fn apply_response_headers_with_cache_privacy(settings: &Settings, response: enforce_set_cookie_cache_privacy(response); } +/// Logs one warning for each configured response header that +/// [`apply_response_headers_with_cache_privacy`] skips. +/// +/// `Trailer`, `Content-Length`, and the hop-by-hop fields are skipped because +/// response egress owns them, and an invalid name or value because it cannot be +/// sent. Validation rejects them all, but a config that predates the rejection +/// can still carry them. Each adapter calls this once per request, so a skipped +/// entry warns once however often the request applies the operator headers. +pub fn warn_skipped_operator_response_headers(settings: &Settings) { + for (key, value) in &settings.response_headers { + match operator_response_header(key, value) { + Ok(_) => {} + Err(SkippedOperatorHeader::Forbidden) => log::warn!( + "Skipping configured response header `{key}`: framing and hop-by-hop fields belong to response egress" + ), + Err(SkippedOperatorHeader::InvalidName) => { + log::warn!("Skipping invalid configured response header name {key}"); + } + Err(SkippedOperatorHeader::InvalidValue) => { + log::warn!("Skipping invalid configured response header value for {key}"); + } + } + } +} + +/// Why a configured response header entry is never sent. +enum SkippedOperatorHeader { + /// A framing or hop-by-hop field, which response egress owns. + Forbidden, + /// A name that is not a valid header name. + InvalidName, + /// A value that is not a valid header value. + InvalidValue, +} + +/// Parses the configured response header entry `key: value`, or says why it is skipped. +fn operator_response_header( + key: &str, + value: &str, +) -> Result<(HeaderName, HeaderValue), SkippedOperatorHeader> { + if is_forbidden_operator_response_header(key) { + return Err(SkippedOperatorHeader::Forbidden); + } + let name = + HeaderName::from_bytes(key.as_bytes()).map_err(|_| SkippedOperatorHeader::InvalidName)?; + let value = HeaderValue::from_str(value).map_err(|_| SkippedOperatorHeader::InvalidValue)?; + Ok((name, value)) +} + #[cfg(test)] mod tests { use super::*; @@ -204,6 +270,7 @@ mod tests { use edgezero_core::http::response_builder; use crate::cache_policy::EDGE_CACHE_HEADER_NAMES; + use crate::test_support::logging::WarningCapture; fn settings_with_response_headers(headers: &[(&str, &str)]) -> Settings { let mut s = Settings::from_toml( @@ -610,4 +677,82 @@ mod tests { "operator headers should still apply to cacheable responses" ); } + + #[test] + fn skips_operator_framing_and_hop_by_hop_entries_and_warns_once_each() { + let forbidden = [ + "Trailer", + "Content-Length", + "Connection", + "Keep-Alive", + "TE", + "Transfer-Encoding", + "Upgrade", + "Proxy-Connection", + "Proxy-Authenticate", + "Proxy-Authorization", + ]; + let mut entries: Vec<(&str, &str)> = forbidden + .iter() + .map(|name| (*name, "example-value")) + .collect(); + entries.push(("x-example", "example-value")); + entries.push(("x example", "example-value")); + entries.push(("x-invalid-value", "example\nvalue")); + let settings = settings_with_response_headers(&entries); + let mut response = response_builder() + .body(edgezero_core::body::Body::empty()) + .expect("should build response"); + let capture = WarningCapture::start(); + + apply_response_headers_with_cache_privacy(&settings, &mut response); + let applying = capture.take(); + warn_skipped_operator_response_headers(&settings); + let warnings = capture.take(); + + assert!( + applying.is_empty(), + "should skip the entries without logging while applying them" + ); + for name in forbidden { + assert!( + !response.headers().contains_key(name), + "should skip the operator {name} entry" + ); + assert_eq!( + warnings + .iter() + .filter(|warning| warning.message.contains(&format!("`{name}`"))) + .count(), + 1, + "should warn once that the operator {name} entry is skipped" + ); + } + assert!( + !response.headers().contains_key("x-invalid-value"), + "should skip the operator entry with an invalid value" + ); + for message in [ + "Skipping invalid configured response header name x example", + "Skipping invalid configured response header value for x-invalid-value", + ] { + assert!( + warnings.iter().any(|warning| warning.message == message), + "should warn `{message}`" + ); + } + assert_eq!( + warnings.len(), + forbidden.len() + 2, + "should warn about the skipped entries only" + ); + assert_eq!( + response + .headers() + .get("x-example") + .and_then(|value| value.to_str().ok()), + Some("example-value"), + "should still apply the other operator entries" + ); + } } diff --git a/crates/trusted-server-core/src/s3_sigv4.rs b/crates/trusted-server-core/src/s3_sigv4.rs index dc4dc25ad2..59135a1f38 100644 --- a/crates/trusted-server-core/src/s3_sigv4.rs +++ b/crates/trusted-server-core/src/s3_sigv4.rs @@ -301,6 +301,82 @@ mod tests { ); } + #[test] + fn received_request_check_matches_only_the_signed_host_path_and_query() { + let url = Url::parse("https://assets.example.com:8443/photos/cat%20one.jpg?v=2&b=a%2Fb") + .expect("should parse URL"); + let mut headers = HeaderMap::new(); + headers.insert( + header::HOST, + HeaderValue::from_static("assets.example.com:8443"), + ); + let credentials = S3Credentials { + access_key_id: "AKIAIOSFODNN7EXAMPLE".to_string(), + secret_access_key: Redacted::new( + "wJalrXUtnFEMI/K7MDENG+bPxRfiCYEXAMPLEKEY".to_string(), + ), + session_token: None, + }; + sign_headers( + &Method::HEAD, + &url, + &mut headers, + "us-east-1", + &credentials, + DateTime::parse_from_rfc3339("2026-11-09T12:00:00Z") + .expect("should parse fixture date") + .with_timezone(&Utc) + .into(), + ) + .expect("should sign request"); + let signed = headers + .get(header::AUTHORIZATION) + .and_then(|value| value.to_str().ok()) + .expect("should set authorization header"); + let amz_date = headers + .get("x-amz-date") + .and_then(|value| value.to_str().ok()) + .expect("should set x-amz-date"); + let recompute = |host: &str, target: &str| { + crate::test_support::s3_sigv4_authorization( + &Method::HEAD, + host, + target, + amz_date, + "us-east-1", + "AKIAIOSFODNN7EXAMPLE", + "wJalrXUtnFEMI/K7MDENG+bPxRfiCYEXAMPLEKEY", + ) + .expect("should recompute the authorization") + }; + + assert_eq!( + recompute( + "assets.example.com:8443", + "/photos/cat%20one.jpg?v=2&b=a%2Fb" + ), + signed, + "should reproduce the signature for the signed host, path, and query" + ); + for (host, target) in [ + ("assets.example.com", "/photos/cat%20one.jpg?v=2&b=a%2Fb"), + ( + "assets.example.com:8443", + "/photos/cat%20two.jpg?v=2&b=a%2Fb", + ), + ( + "assets.example.com:8443", + "/photos/cat%20one.jpg?v=3&b=a%2Fb", + ), + ] { + assert_ne!( + recompute(host, target), + signed, + "should not reproduce the signature for {host}{target}" + ); + } + } + #[test] fn credentials_debug_redacts_secret_material() { let credentials = S3Credentials { diff --git a/crates/trusted-server-core/src/secret_resolution.rs b/crates/trusted-server-core/src/secret_resolution.rs index f362a0b7b2..6f1bde8a89 100644 --- a/crates/trusted-server-core/src/secret_resolution.rs +++ b/crates/trusted-server-core/src/secret_resolution.rs @@ -210,6 +210,7 @@ mod tests { values: BTreeMap>, } + #[async_trait::async_trait(?Send)] impl PlatformSecretStore for MemorySecretStore { fn get_bytes( &self, @@ -222,7 +223,7 @@ mod tests { }) } - fn create( + async fn create( &self, _store_id: &StoreId, _name: &str, @@ -231,7 +232,11 @@ mod tests { Ok(()) } - fn delete(&self, _store_id: &StoreId, _name: &str) -> Result<(), Report> { + async fn delete( + &self, + _store_id: &StoreId, + _name: &str, + ) -> Result<(), Report> { Ok(()) } } diff --git a/crates/trusted-server-core/src/settings.rs b/crates/trusted-server-core/src/settings.rs index 3150ead138..6113701e6f 100644 --- a/crates/trusted-server-core/src/settings.rs +++ b/crates/trusted-server-core/src/settings.rs @@ -15,6 +15,7 @@ use subtle::ConstantTimeEq as _; use url::Url; use validator::{Validate, ValidationError}; +use crate::asset_image_optimizer::image_optimizer_region; use crate::auction_config_types::AuctionConfig; use crate::cache_policy::{CachePolicy, CacheVisibility}; use crate::consent_config::ConsentConfig; @@ -24,7 +25,6 @@ use crate::error::TrustedServerError; use crate::host_header::validate_host_header_override_value; use crate::http_util::SPOOFABLE_FORWARDED_HEADERS; use crate::integrations::datadome::HEADER_DATADOME_TEST_BYPASS; -use crate::platform::PlatformImageOptimizerRegion; use crate::proxy::{is_host_allowed, is_host_permitted}; use crate::redacted::Redacted; @@ -178,6 +178,19 @@ impl Publisher { "replace-with-random-proxy-secret", ]; + /// Drops `origin_host_header_override` on Spin, with a warning. + /// + /// Spin sends the origin URI authority as `Host` and cannot override it, so + /// the Spin adapter calls this when it loads settings. A config that sets + /// the override keeps serving there, with the URI authority. + pub fn drop_origin_host_header_override_for_spin(&mut self) { + if self.origin_host_header_override.take().is_some() { + log::warn!( + "publisher.origin_host_header_override is ignored on Spin; the origin receives its URI authority as Host" + ); + } + } + /// Returns the EC cookie domain, computed as `.{domain}`. /// /// Per spec §5.2, EC cookies derive their domain from @@ -298,14 +311,6 @@ impl Publisher { .clone() .unwrap_or_else(|| format!("https://{}", self.domain)) } - - /// Returns the outbound Host header for proxied publisher-origin requests. - #[must_use] - pub fn origin_host_header(&self) -> String { - self.origin_host_header_override - .clone() - .unwrap_or_else(|| self.origin_host()) - } } #[derive(Default, Clone, Deserialize, Serialize)] @@ -1191,7 +1196,7 @@ impl AssetImageOptimizerConfig { .to_string(), })); } - if PlatformImageOptimizerRegion::parse(&self.region).is_none() { + if image_optimizer_region(&self.region).is_none() { return Err(Report::new(TrustedServerError::Configuration { message: format!( "proxy.asset_routes image_optimizer region `{}` is not supported", @@ -1967,9 +1972,7 @@ impl ProxyAssetRoute { #[derive(Debug, Clone, Deserialize, Serialize)] #[serde(deny_unknown_fields)] pub struct Proxy { - /// Enable TLS certificate verification when proxying to HTTPS origins. - /// Defaults to true for secure production use. - /// Set to false for local development with self-signed certificates. + /// Must be true. Origin TLS is always verified. #[serde(default = "default_certificate_check")] pub certificate_check: bool, /// Permitted signing, initial fetch, and redirect target domains for the @@ -3510,6 +3513,24 @@ impl Settings { Ok(()) } + /// Warns that a live `creative_opportunities.origin_readthrough_enabled = true` is ignored. + /// + /// Origin readthrough is unavailable: the publisher origin is always fetched + /// with the ad-stack bypass or the platform default cache policy, as if the + /// setting were `false`. Validation rejects the setting, and a config that + /// predates the rejection keeps serving. + pub fn warn_ignored_origin_readthrough(&self) { + if self + .creative_opportunities + .as_ref() + .is_some_and(|config| config.origin_readthrough_enabled == Some(true)) + { + log::warn!( + "creative_opportunities.origin_readthrough_enabled = true is ignored; origin readthrough is unavailable. Remove the setting before the next config push" + ); + } + } + /// Returns compiled creative opportunity slots when template delivery is enabled. #[must_use] pub fn creative_opportunity_slots( @@ -6534,8 +6555,8 @@ origin_host_header_override = "www.example.com:8443""#, let settings = Settings::from_toml(&toml_str).expect("should accept host header override"); assert_eq!( - settings.publisher.origin_host_header(), - "www.example.com:8443", + settings.publisher.origin_host_header_override.as_deref(), + Some("www.example.com:8443"), "should use configured host header override" ); } @@ -7394,7 +7415,6 @@ source_domain = "partner.example.com" settings.publisher.origin_host_header_override.as_deref(), Some("www.example.com") ); - assert_eq!(settings.publisher.origin_host_header(), "www.example.com"); }); } @@ -7504,40 +7524,6 @@ source_domain = "partner.example.com" assert_eq!(publisher.origin_host(), "[::1]:8080"); } - #[test] - fn test_publisher_origin_host_header_defaults_to_origin_host() { - let publisher = Publisher { - domain: "example.com".to_string(), - cookie_domain: ".example.com".to_string(), - origin_url: "https://origin.example.com:8443".to_string(), - public_origin: None, - origin_host_header_override: None, - proxy_secret: Redacted::new("test-secret".to_string()), - max_buffered_body_bytes: 16 * 1024 * 1024, - html_finalization_timeout_ms: default_html_finalization_timeout_ms(), - allow_async_html_finish: false, - }; - - assert_eq!(publisher.origin_host_header(), "origin.example.com:8443"); - } - - #[test] - fn test_publisher_origin_host_header_uses_override() { - let publisher = Publisher { - domain: "example.com".to_string(), - cookie_domain: ".example.com".to_string(), - origin_url: "https://origin.example.com".to_string(), - public_origin: None, - origin_host_header_override: Some("www.example.com".to_string()), - proxy_secret: Redacted::new("test-secret".to_string()), - max_buffered_body_bytes: 16 * 1024 * 1024, - html_finalization_timeout_ms: default_html_finalization_timeout_ms(), - allow_async_html_finish: false, - }; - - assert_eq!(publisher.origin_host_header(), "www.example.com"); - } - #[test] fn publisher_default_max_buffered_body_bytes_matches_config_default() { // The manual `Default` impl must agree with the serde default applied diff --git a/crates/trusted-server-core/src/settings_data.rs b/crates/trusted-server-core/src/settings_data.rs index 86616ad69d..f438ab7be9 100644 --- a/crates/trusted-server-core/src/settings_data.rs +++ b/crates/trusted-server-core/src/settings_data.rs @@ -218,6 +218,7 @@ mod tests { entries: BTreeMap, } + #[async_trait::async_trait(?Send)] impl PlatformConfigStore for MemoryConfigStore { fn get(&self, _store_name: &StoreName, key: &str) -> Result> { self.entries.get(key).cloned().ok_or_else(|| { @@ -225,7 +226,7 @@ mod tests { }) } - fn put( + async fn put( &self, _store_id: &StoreId, _key: &str, @@ -234,13 +235,18 @@ mod tests { Ok(()) } - fn delete(&self, _store_id: &StoreId, _key: &str) -> Result<(), Report> { + async fn delete( + &self, + _store_id: &StoreId, + _key: &str, + ) -> Result<(), Report> { Ok(()) } } struct EchoSecretStore; + #[async_trait::async_trait(?Send)] impl PlatformSecretStore for EchoSecretStore { fn get_bytes( &self, @@ -254,7 +260,7 @@ mod tests { Ok(value.as_bytes().to_vec()) } - fn create( + async fn create( &self, _store_id: &StoreId, _name: &str, @@ -263,7 +269,11 @@ mod tests { Ok(()) } - fn delete(&self, _store_id: &StoreId, _name: &str) -> Result<(), Report> { + async fn delete( + &self, + _store_id: &StoreId, + _name: &str, + ) -> Result<(), Report> { Ok(()) } } diff --git a/crates/trusted-server-core/src/test_support.rs b/crates/trusted-server-core/src/test_support.rs index 37d54614c0..fa938d35d0 100644 --- a/crates/trusted-server-core/src/test_support.rs +++ b/crates/trusted-server-core/src/test_support.rs @@ -1,12 +1,202 @@ #[cfg(test)] -pub(crate) mod logging; +pub(crate) mod body_read; +#[cfg(any(test, feature = "test-utils"))] +pub mod logging; +#[cfg(any(test, feature = "test-utils"))] +pub mod scripted_client; + +#[cfg(any(test, feature = "test-utils"))] +use std::io::Write as _; + +#[cfg(any(test, feature = "test-utils"))] +use edgezero_core::http::Request; + +#[cfg(any(test, feature = "test-utils"))] +use crate::platform::HttpClient; +#[cfg(any(test, feature = "test-utils"))] +use crate::{ + error::TrustedServerError, + redacted::Redacted, + s3_sigv4::{self, S3Credentials}, +}; +#[cfg(any(test, feature = "test-utils"))] +use error_stack::Report; +#[cfg(any(test, feature = "test-utils"))] +use http::{HeaderMap, HeaderValue, Method, header}; + +/// Inserts `client` into `request`'s extensions, where `ctx.http_client()` reads it. +/// +/// [`RouterService::oneshot`](edgezero_core::router::RouterService::oneshot) skips the +/// adapter's request conversion, which is what installs a default client, so a `oneshot` +/// route test calls this before dispatch. Pass a client built with +/// [`HttpClient::with_client`], which keeps the default proxy marker, so the adapter's own +/// `.with_proxy_marker(false)` stays observable. +/// +/// # Examples +/// +/// ```ignore +/// insert_outbound_client( +/// &mut request, +/// HttpClient::with_client(ScriptedHttpClient::forbidding()), +/// ); +/// let response = router.oneshot(request).await?; +/// ``` +#[cfg(any(test, feature = "test-utils"))] +pub fn insert_outbound_client(request: &mut Request, client: HttpClient) { + request.extensions_mut().insert(client); +} + +/// Returns `bytes` gzip-encoded, for an origin fixture that serves a gzip body. +/// +/// # Examples +/// +/// ```ignore +/// let body = gzip(b""); +/// assert_eq!(&body[..2], &[0x1f, 0x8b]); +/// ``` +/// +/// # Panics +/// +/// Panics if in-memory gzip encoding fails. +#[cfg(any(test, feature = "test-utils"))] +#[must_use] +pub fn gzip(bytes: &[u8]) -> Vec { + let mut encoder = flate2::write::GzEncoder::new(Vec::new(), flate2::Compression::default()); + encoder + .write_all(bytes) + .expect("should encode the gzip fixture"); + encoder.finish().expect("should finish the gzip fixture") +} + +/// Recomputes the S3 `SigV4` `Authorization` value for a request an origin received. +/// +/// An origin stub passes the method, the `Host`, request target, and `x-amz-date` it +/// received, and the route's region and credentials. The result equals the request's +/// `Authorization` only when Trusted Server signed exactly that `Host`, path, and query. +/// +/// # Errors +/// +/// Returns [`TrustedServerError::Proxy`] when +/// `amz_date` is not a `SigV4` timestamp or the received values cannot be signed. +#[cfg(any(test, feature = "test-utils"))] +pub fn s3_sigv4_authorization( + method: &Method, + host: &str, + request_target: &str, + amz_date: &str, + region: &str, + access_key_id: &str, + secret_access_key: &str, +) -> Result> { + let invalid = |message: String| Report::new(TrustedServerError::Proxy { message }); + let signed_at = chrono::NaiveDateTime::parse_from_str(amz_date, "%Y%m%dT%H%M%SZ") + .map_err(|_| invalid(format!("invalid x-amz-date `{amz_date}`")))? + .and_utc(); + let url = url::Url::parse(&format!("https://{host}{request_target}")) + .map_err(|_| invalid(format!("invalid received target `{host}{request_target}`")))?; + let mut headers = HeaderMap::new(); + headers.insert( + header::HOST, + HeaderValue::from_str(host) + .map_err(|_| invalid(format!("invalid received Host `{host}`")))?, + ); + s3_sigv4::sign_headers( + method, + &url, + &mut headers, + region, + &S3Credentials { + access_key_id: access_key_id.to_owned(), + secret_access_key: Redacted::new(secret_access_key.to_owned()), + session_token: None, + }, + std::time::SystemTime::from(signed_at), + )?; + headers + .get(header::AUTHORIZATION) + .and_then(|value| value.to_str().ok()) + .map(str::to_owned) + .ok_or_else(|| invalid("signer set no Authorization header".to_owned())) +} #[cfg(test)] pub mod tests { + use std::sync::{Arc, Mutex}; + + use bytes::Bytes; + use edgezero_core::body::Body; + use edgezero_core::context::RequestContext; + use edgezero_core::error::EdgeError; + use edgezero_core::http::RequestParts; + use edgezero_core::router::{PreDispatchHook, RouterService}; + use http::{Request, Response}; + use crate::consent::jurisdiction::Jurisdiction; use crate::consent::types::{ConsentContext, ConsentSource, TcfConsent}; use crate::settings::Settings; + /// Run `edit` against the head of `request` and return its result. + /// + /// The body is left in place. This lets tests apply a head-only helper, + /// such as forwarder preparation or basic auth, to a whole request. + pub fn with_request_parts( + request: &mut Request, + edit: impl FnOnce(&mut RequestParts) -> T, + ) -> T { + let (mut parts, body) = std::mem::take(request).into_parts(); + let result = edit(&mut parts); + *request = Request::from_parts(parts, body); + result + } + + /// Drive `hook` through a router and report the outcome. + /// + /// The route matches every path for the request's method. + /// + /// # Errors + /// + /// Returns the router's response when the hook answers instead of + /// continuing. `Ok` carries the request the route received, with the hook's + /// head and the buffered body bytes. + /// + /// # Panics + /// + /// Panics if the router cannot render its response. + pub async fn run_pre_dispatch_hook( + hook: H, + request: Request, + ) -> Result, Response> { + let received: Arc>> = Arc::default(); + let sink = Arc::clone(&received); + let record = move |context: RequestContext| { + let sink = Arc::clone(&sink); + async move { + let body = context.body_bytes(usize::MAX).await?; + let (parts, _) = context.into_request()?.into_parts(); + *sink.lock().expect("should lock the recorded request") = Some((parts, body)); + Ok::<_, EdgeError>(Response::new(Body::empty())) + } + }; + let method = request.method().clone(); + let router = RouterService::builder() + .pre_dispatch_hook(hook) + .route("/", method.clone(), record.clone()) + .route("/{*rest}", method, record) + .build(); + let response = router + .oneshot(request) + .await + .expect("should render the router response"); + let recorded = received + .lock() + .expect("should lock the recorded request") + .take(); + match recorded { + Some((parts, body)) => Ok(Request::from_parts(parts, Body::from(body))), + None => Err(response), + } + } + #[must_use] pub fn crate_test_settings_str() -> String { r#" @@ -104,30 +294,30 @@ pub mod tests { /// drive byte-identical input. #[cfg(any(test, feature = "test-utils"))] pub mod nextjs_auction { - use std::io::{Read as _, Write as _}; + use std::io::Read as _; use std::net::IpAddr; use std::sync::Arc; use std::sync::atomic::{AtomicUsize, Ordering}; use bytes::Bytes; use error_stack::Report; - use flate2::{read::GzDecoder, write::GzEncoder}; + use flate2::read::GzDecoder; + use http::{Method, StatusCode}; use crate::geo::GeoInfo; use crate::platform::{ - BackendNamingPolicy, ClientInfo, PlatformBackend, PlatformBackendSpec, PlatformConfigStore, - PlatformError, PlatformGeo, PlatformHttpClient, PlatformHttpRequest, - PlatformPendingRequest, PlatformResponse, PlatformSecretStore, PlatformSelectResult, + ClientInfo, PlatformConfigStore, PlatformError, PlatformGeo, PlatformSecretStore, RuntimeServices, StoreId, StoreName, UnavailableKvStore, }; use crate::settings::Settings; + use crate::test_support::scripted_client::{ScriptedHttpClient, ScriptedReply, TestClock}; /// Publisher host the fixture settings serve. pub const PUBLISHER_HOST: &str = "test-publisher.example.com"; /// Upstream host the fixture origin answers for. pub const ORIGIN_HOST: &str = "origin.test-publisher.example.com"; - /// Auction endpoint host the fixture bidder answers for. - pub const AUCTION_HOST: &str = "auction.example.com"; + /// Bid endpoint of the fixture auction provider, where its scripted bidder answers. + pub const AUCTION_ENDPOINT: &str = "https://auction.example.com/bid"; /// Settings that enable the Next.js integration and a single auction provider. /// @@ -169,7 +359,7 @@ pub mod nextjs_auction { settings.auction.providers = serde_json::from_value(serde_json::json!({ "fixture": { "protocol": "openrtb-2.6", - "endpoint": "https://auction.example.com/bid", + "endpoint": AUCTION_ENDPOINT, "routing": "all_eligible", "timeout_ms": 5000 } @@ -354,148 +544,114 @@ pub mod nextjs_auction { ); } - /// Upstream that serves [`origin_html`] and one deterministic bid. - #[derive(Default)] + /// Publisher origin that serves [`origin_html`], with a scripted bidder that + /// answers every bid request at [`AUCTION_ENDPOINT`] with one deterministic bid. + /// + /// Both answer through one scripted client: every GET to [`ORIGIN_HOST`] gets the + /// page, whatever its path. [`services`] sends the publisher origin and the + /// auction through it and reads time from the clock its replies arrive on. pub struct NextJsAuctionOrigin { - auction_requests: AtomicUsize, - html_response: Option>, - gzip: bool, - streaming: bool, + clock: TestClock, + client: ScriptedHttpClient, + auction_requests: Arc, + } + + impl Default for NextJsAuctionOrigin { + fn default() -> Self { + Self::serving(Bytes::from(origin_html()), false, false) + } + } + + /// Queues the fixture bid for the next bid request `bidder` receives. + fn queue_fixture_bid(bidder: &ScriptedHttpClient) { + let bid = serde_json::json!({ + "id": "fixture-auction", + "seatbid": [{"seat": "example", "bid": [{ + "id": "fixture-bid", "impid": "fixture-slot", "price": 1.25, + "adm": "
fixture-creative
", "w": 300, "h": 250, + "crid": "example-creative", "adomain": ["advertiser.example.com"] + }]}] + }); + bidder.respond_to( + Method::POST, + AUCTION_ENDPOINT, + ScriptedReply::status(StatusCode::OK) + .header("content-type", "application/json") + .body(bid.to_string()), + ); + } + + /// The origin's answer to one page request: `html`, gzip-coded when `gzip`, and + /// delivered in 97-byte chunks when `chunked`. + fn origin_reply(html: &Bytes, gzip: bool, chunked: bool) -> ScriptedReply { + let mut reply = ScriptedReply::status(StatusCode::OK).header("content-type", "text/html"); + if gzip { + reply = reply.header("content-encoding", "gzip"); + } + if chunked { + reply.chunks(html.chunks(97).map(Bytes::copy_from_slice).collect()) + } else { + reply.body(html.clone()) + } } impl NextJsAuctionOrigin { /// Serve canned HTML instead of the default fixture, optionally gzip-encoded - /// and streamed in small chunks. The default fixture remains unchanged. + /// and delivered in small chunks. The default fixture remains unchanged. /// /// # Panics /// /// Panics if in-memory gzip encoding fails. #[must_use] - pub fn with_html_response(html: &str, gzip: bool, streaming: bool) -> Self { + pub fn with_html_response(html: &str, gzip: bool, chunked: bool) -> Self { let bytes = if gzip { - let mut encoder = GzEncoder::new(Vec::new(), flate2::Compression::default()); - encoder - .write_all(html.as_bytes()) - .expect("should encode fixture gzip"); - encoder.finish().expect("should finish fixture gzip") + super::gzip(html.as_bytes()) } else { html.as_bytes().to_vec() }; + Self::serving(Bytes::from(bytes), gzip, chunked) + } + + fn serving(html: Bytes, gzip: bool, chunked: bool) -> Self { + let clock = TestClock::new(); + let auction_requests = Arc::new(AtomicUsize::new(0)); + let script = ScriptedHttpClient::new(&clock); + let rearm = script.clone(); + let counted = Arc::clone(&auction_requests); + // The scripted client spends each reply on one request, so each request + // queues its own answer as it is recorded, before its reply is taken. + let client = script.with_request_observer(move |request| { + if request.method == Method::POST && request.uri == AUCTION_ENDPOINT { + counted.fetch_add(1, Ordering::SeqCst); + queue_fixture_bid(&rearm); + } else if request.method == Method::GET && request.uri.host() == Some(ORIGIN_HOST) { + rearm.respond_to( + Method::GET, + &request.uri.to_string(), + origin_reply(&html, gzip, chunked), + ); + } + }); + queue_fixture_bid(&client); Self { - html_response: Some(bytes), - gzip, - streaming, - ..Self::default() + clock, + client, + auction_requests, } } - /// Number of auction requests this fixture has answered. + /// Number of bid requests the fixture bidder has received. #[must_use] pub fn auction_requests(&self) -> usize { self.auction_requests.load(Ordering::SeqCst) } - /// Forget the recorded auction requests. + /// Forget the recorded bid requests. pub fn reset_auction_requests(&self) { self.auction_requests.store(0, Ordering::SeqCst); } } - #[async_trait::async_trait(?Send)] - impl PlatformHttpClient for NextJsAuctionOrigin { - fn supports_streaming_responses(&self) -> bool { - self.streaming - } - - async fn send( - &self, - request: PlatformHttpRequest, - ) -> Result> { - let (content_type, body) = match request.request.uri().host() { - Some(ORIGIN_HOST) => ("text/html", origin_html()), - Some(AUCTION_HOST) => { - self.auction_requests.fetch_add(1, Ordering::SeqCst); - ( - "application/json", - serde_json::json!({ - "id": "fixture-auction", - "seatbid": [{"seat": "example", "bid": [{ - "id": "fixture-bid", "impid": "fixture-slot", "price": 1.25, - "adm": "
fixture-creative
", "w": 300, "h": 250, - "crid": "example-creative", "adomain": ["advertiser.example.com"] - }]}] - }) - .to_string(), - ) - } - host => { - return Err(Report::new(PlatformError::HttpClient) - .attach(format!("unexpected fixture upstream: {host:?}"))); - } - }; - let canned_origin = - request.request.uri().host() == Some(ORIGIN_HOST) && self.html_response.is_some(); - let bytes = if canned_origin { - self.html_response - .as_ref() - .expect("should have canned response") - .clone() - } else { - body.into_bytes() - }; - let mut response = http::Response::builder() - .status(200) - .header("content-type", content_type); - if canned_origin && self.gzip { - response = response.header("content-encoding", "gzip"); - } - let body = if request.stream_response && self.streaming { - let chunks: Vec = bytes.chunks(97).map(Bytes::copy_from_slice).collect(); - edgezero_core::body::Body::stream(futures::stream::iter(chunks)) - } else { - edgezero_core::body::Body::from(bytes) - }; - Ok(PlatformResponse::new( - response - .body(body) - .expect("should build deterministic upstream response"), - ) - .with_backend_name(request.backend_name)) - } - - async fn send_async( - &self, - request: PlatformHttpRequest, - ) -> Result> { - let backend = request.backend_name.clone(); - Ok(PlatformPendingRequest::new(request).with_backend_name(backend)) - } - - async fn select_until( - &self, - pending_requests: Vec, - _deadline: web_time::Instant, - ) -> Result, Report> { - // All fixture responses are deterministic and already available. - self.select(pending_requests).await.map(Some) - } - - async fn select( - &self, - mut pending_requests: Vec, - ) -> Result> { - let request = pending_requests - .remove(0) - .downcast::() - .expect("should recover fixture pending request"); - Ok(PlatformSelectResult { - ready: self.send(request).await, - remaining: pending_requests, - failed_backend_name: None, - }) - } - } - struct FixtureGeo; impl PlatformGeo for FixtureGeo { @@ -518,6 +674,7 @@ pub mod nextjs_auction { struct FixtureStore; + #[async_trait::async_trait(?Send)] impl PlatformConfigStore for FixtureStore { fn get( &self, @@ -527,7 +684,7 @@ pub mod nextjs_auction { Err(Report::new(PlatformError::Unsupported)) } - fn put( + async fn put( &self, _store_id: &StoreId, _key: &str, @@ -536,11 +693,16 @@ pub mod nextjs_auction { Err(Report::new(PlatformError::Unsupported)) } - fn delete(&self, _store_id: &StoreId, _key: &str) -> Result<(), Report> { + async fn delete( + &self, + _store_id: &StoreId, + _key: &str, + ) -> Result<(), Report> { Err(Report::new(PlatformError::Unsupported)) } } + #[async_trait::async_trait(?Send)] impl PlatformSecretStore for FixtureStore { fn get_bytes( &self, @@ -550,7 +712,7 @@ pub mod nextjs_auction { Err(Report::new(PlatformError::Unsupported)) } - fn create( + async fn create( &self, _store_id: &StoreId, _name: &str, @@ -559,44 +721,34 @@ pub mod nextjs_auction { Err(Report::new(PlatformError::Unsupported)) } - fn delete(&self, _store_id: &StoreId, _name: &str) -> Result<(), Report> { - Err(Report::new(PlatformError::Unsupported)) - } - } - - impl PlatformBackend for FixtureStore { - fn naming_policy(&self) -> BackendNamingPolicy { - BackendNamingPolicy::Axum - } - - fn predict_name( + async fn delete( &self, - _spec: &PlatformBackendSpec, - ) -> Result> { - Ok("fixture-backend".to_owned()) - } - - fn ensure(&self, _spec: &PlatformBackendSpec) -> Result> { - Ok("fixture-backend".to_owned()) + _store_id: &StoreId, + _name: &str, + ) -> Result<(), Report> { + Err(Report::new(PlatformError::Unsupported)) } } - /// Runtime services wired to `client`, with every other platform capability - /// stubbed out. + /// Runtime services wired to `client`, with its scripted origin and bidder as the + /// outbound client and every other platform capability stubbed out. #[must_use] - pub fn services(client: Arc) -> RuntimeServices { - services_with_http_client(client) + pub fn services(client: &NextJsAuctionOrigin) -> RuntimeServices { + services_with_outbound_client(&client.client, &client.clock) } - /// Build fixture runtime services with a custom asynchronous upstream. + /// Fixture runtime services whose outbound requests go to `client`, on `clock`. #[must_use] - pub fn services_with_http_client(client: Arc) -> RuntimeServices { + pub fn services_with_outbound_client( + client: &ScriptedHttpClient, + clock: &TestClock, + ) -> RuntimeServices { RuntimeServices::builder() .config_store(Arc::new(FixtureStore)) .secret_store(Arc::new(FixtureStore)) .kv_store(Arc::new(UnavailableKvStore)) - .backend(Arc::new(FixtureStore)) - .http_client(client) + .outbound_client(client.http_client()) + .clock(clock.monotonic_clock()) .geo(Arc::new(FixtureGeo)) .client_info(ClientInfo::default()) .build() diff --git a/crates/trusted-server-core/src/test_support/async_html_finalization.rs b/crates/trusted-server-core/src/test_support/async_html_finalization.rs index 82cc898ebc..bf346ecfc7 100644 --- a/crates/trusted-server-core/src/test_support/async_html_finalization.rs +++ b/crates/trusted-server-core/src/test_support/async_html_finalization.rs @@ -1,33 +1,47 @@ //! Fictional terminal HTTP consumer, available only with `test-utils`. +//! +//! It also answers [`FRAMING_TRAILER_PATH`] with a `Trailer` field, which +//! `EdgeZero`'s response framing rejects, so the runtime checks can reach a +//! framing fallback that Trusted Server itself never produces. use std::io; use std::sync::Arc; use std::time::Duration; -use edgezero_core::body::Body; -use edgezero_core::http::{Uri, request_builder}; +use async_trait::async_trait; +use edgezero_core::body::Body as EdgeBody; +use edgezero_core::error::EdgeError; +use edgezero_core::outbound::{OutboundCachePolicy, ResponseDecoding}; use error_stack::{Report, ResultExt as _}; use futures::future::LocalBoxFuture; +use http::{HeaderValue, Request, Response, header}; use serde::Deserialize; use serde_json::json; use sha2::{Digest as _, Sha256}; -use url::Url; use validator::Validate; use crate::error::TrustedServerError; use crate::integrations::{ - IntegrationHtmlStreamContext, IntegrationHtmlStreamProcessorFactory, IntegrationRegistration, + IntegrationEndpoint, IntegrationHtmlStreamContext, IntegrationHtmlStreamProcessorFactory, + IntegrationProxy, IntegrationRegistration, }; -use crate::platform::{PlatformBackendSpec, PlatformHttpRequest}; +use crate::outbound::{buffered, outbound_request}; +use crate::platform::RuntimeServices; use crate::settings::{IntegrationConfig, Settings}; use crate::streaming_processor::StreamProcessor; use crate::streaming_processor::finish::{StreamFinish, StreamFinishError, StreamFinishInput}; pub(crate) const ID: &str = "html_finalization_fixture"; +/// Path the fixture answers with a `Trailer` field. +pub(crate) const FRAMING_TRAILER_PATH: &str = + "/integrations/html_finalization_fixture/framing-trailer"; const EOF_SENTINEL: &[u8] = b""; const ORIGINAL: &[u8] = b"EXAMPLE_UNSIGNED"; const REPLACEMENT: &[u8] = b"EXAMPLE_FINALIZED"; +/// Largest lookup reply the fixture reads. +const MAX_LOOKUP_RESPONSE_BYTES: u64 = 64; + #[derive(Clone, Deserialize, Validate)] #[serde(deny_unknown_fields)] pub(crate) struct FixtureConfig { @@ -66,6 +80,7 @@ pub(crate) fn register( IntegrationRegistration::builder(ID) .without_js() .with_html_stream_processor(Arc::new(FixtureFactory(config))) + .with_proxy(Arc::new(FramingTrailerProxy)) .build(), )) } @@ -124,71 +139,57 @@ impl StreamProcessor for FixtureSession { if !self.saw_eof_sentinel || !self.synchronous_eof { return Err(Report::new(StreamFinishError::Rejected)); } - let url = - Url::parse(&self.config.lookup_url).change_context(StreamFinishError::Rejected)?; - // Fastly cannot interrupt a blocking host call. Bound its transport - // by the remaining chain budget; timer adapters deliberately test - // cancellation against a longer transport timeout. - let timeout = if input.deadline.has_cooperative_timer() { - Duration::from_millis(u64::from(self.config.transport_timeout_ms)) + // Fastly cannot interrupt a blocking host call, so its request also + // carries the remaining chain budget as a deadline. The constant + // timeout keeps the backend identity stable; the reported timeout is + // the bound the request actually runs under. + let transport_timeout = + Duration::from_millis(u64::from(self.config.transport_timeout_ms)); + let cooperative = input.deadline.has_cooperative_timer(); + let reported_timeout = if cooperative { + transport_timeout } else { - input - .deadline - .remaining() - .min(Duration::from_millis(u64::from( - self.config.transport_timeout_ms, - ))) + input.deadline.remaining().min(transport_timeout) }; - let backend = input - .services - .backend() - .ensure(&PlatformBackendSpec { - scheme: url.scheme().to_string(), - host: url - .host_str() - .ok_or_else(|| Report::new(StreamFinishError::Rejected))? - .to_string(), - port: url.port(), - host_header_override: None, - certificate_check: true, - first_byte_timeout: timeout, - between_bytes_timeout: timeout, - discriminator: Some(ID.to_string()), - }) - .change_context(StreamFinishError::Unavailable)?; let payload = serde_json::to_vec(&json!({ "complete": true, "sha256": hex::encode(self.digest.clone().finalize()), "bytes": self.bytes, - "transport_timeout_ms": timeout.as_millis(), + "transport_timeout_ms": reported_timeout.as_millis(), })) .change_context(StreamFinishError::Rejected)?; - let request = request_builder() - .method(http::Method::POST) - .uri( - self.config - .lookup_url - .parse::() - .change_context(StreamFinishError::Rejected)?, - ) - .header(http::header::CONTENT_TYPE, "application/json") - .body(Body::from(payload)) - .change_context(StreamFinishError::Rejected)?; + let mut request = buffered( + outbound_request(http::Method::POST, &self.config.lookup_url) + .map_err(|error| fixture_error(error, StreamFinishError::Rejected))?, + transport_timeout, + MAX_LOOKUP_RESPONSE_BYTES, + ResponseDecoding::Decode, + ) + .cache_policy(OutboundCachePolicy::Bypass) + .header(http::header::CONTENT_TYPE, "application/json") + .map_err(|error| fixture_error(error, StreamFinishError::Rejected))? + .body(payload); + if !cooperative { + request = request.deadline( + input + .services + .clock() + .deadline_after(input.deadline.remaining()), + ); + } let response = input .services - .http_client() - .send(PlatformHttpRequest::new(request, backend).with_cache_bypass()) + .outbound_client() + .send(request) .await - .change_context(StreamFinishError::Unavailable)?; - if !response.response.status().is_success() { + .map_err(|error| fixture_error(error, StreamFinishError::Unavailable))?; + if !response.is_success() { return Err(Report::new(StreamFinishError::Unavailable)); } let reply = response - .response - .into_body() - .into_bytes_bounded(64) + .into_bytes_bounded(MAX_LOOKUP_RESPONSE_BYTES) .await - .change_context(StreamFinishError::Rejected)?; + .map_err(|error| fixture_error(error, StreamFinishError::Rejected))?; if reply.as_ref() != b"EXAMPLE_APPROVED" { return Err(Report::new(StreamFinishError::Rejected)); } @@ -214,6 +215,41 @@ impl StreamProcessor for FixtureSession { } } +/// Answers [`FRAMING_TRAILER_PATH`] with a `Trailer` field. +/// +/// `EdgeZero` rejects any `Trailer` before commit and delivers its framing +/// fallback 500 in place of this response. +struct FramingTrailerProxy; + +#[async_trait(?Send)] +impl IntegrationProxy for FramingTrailerProxy { + fn integration_name(&self) -> &'static str { + ID + } + + fn routes(&self) -> Vec { + vec![IntegrationEndpoint::get(FRAMING_TRAILER_PATH)] + } + + async fn handle( + &self, + _settings: &Settings, + _services: &RuntimeServices, + _req: Request, + ) -> Result, Report> { + let mut response = Response::new(EdgeBody::from("example body with a trailer")); + response + .headers_mut() + .insert(header::TRAILER, HeaderValue::from_static("x-example")); + Ok(response) + } +} + +/// Reports an outbound failure as `context`, carrying the `EdgeError` for diagnosis. +fn fixture_error(error: EdgeError, context: StreamFinishError) -> Report { + Report::new(context).attach_opaque(error) +} + #[cfg(test)] mod tests { use super::*; @@ -239,6 +275,27 @@ mod tests { !registry.js_module_ids().contains(&ID), "should never add a fictional browser bundle" ); + assert_eq!( + registry.has_route(&http::Method::GET, FRAMING_TRAILER_PATH), + enabled, + "should route the framing fixture only when enabled" + ); } } + + #[test] + fn framing_route_answers_with_a_trailer_field() { + let response = futures::executor::block_on(FramingTrailerProxy.handle( + &create_test_settings(), + &crate::platform::test_support::noop_services(), + Request::new(EdgeBody::empty()), + )) + .expect("should answer the framing route"); + + assert_eq!( + response.headers().get(header::TRAILER), + Some(&HeaderValue::from_static("x-example")), + "should carry the Trailer field EdgeZero rejects" + ); + } } diff --git a/crates/trusted-server-core/src/test_support/body_read.rs b/crates/trusted-server-core/src/test_support/body_read.rs new file mode 100644 index 0000000000..30d7715f90 --- /dev/null +++ b/crates/trusted-server-core/src/test_support/body_read.rs @@ -0,0 +1,177 @@ +//! Request bodies that drive each failure [`read_request_body`] maps, for reader route tests. +//! +//! [`read_request_body`]: crate::http_util::read_request_body + +use std::cell::Cell; +use std::rc::Rc; +use std::task::Poll; + +use bytes::Bytes; +use edgezero_core::body::Body; +use edgezero_core::error::EdgeError; +use error_stack::Report; +use http::{Request, Response, StatusCode, header}; + +use crate::error::{IntoHttpResponse as _, TrustedServerError}; + +/// One body-read failure that a reader route must answer with a fixed status. +pub(crate) struct BodyReadCase { + name: &'static str, + content_length: Option, + body: Body, + expected: StatusCode, + polled: Option>>, +} + +impl BodyReadCase { + /// Sets this case's body and declared length on `builder` and builds the request. + /// + /// # Panics + /// + /// Panics when `builder` cannot build a request. + pub(crate) fn into_request( + self, + builder: http::request::Builder, + ) -> (Request, BodyReadExpectation) { + let builder = match &self.content_length { + Some(declared) => builder.header(header::CONTENT_LENGTH, declared.as_str()), + None => builder, + }; + let request = builder + .body(self.body) + .expect("should build the body-read case request"); + let expectation = BodyReadExpectation { + name: self.name, + expected: self.expected, + polled: self.polled, + }; + (request, expectation) + } +} + +/// What a reader route must answer for one [`BodyReadCase`]. +pub(crate) struct BodyReadExpectation { + name: &'static str, + expected: StatusCode, + polled: Option>>, +} + +impl BodyReadExpectation { + /// Names the case, for assertion messages that callers add. + pub(crate) fn name(&self) -> &'static str { + self.name + } + + /// Returns the status this case must answer with. + pub(crate) fn expected(&self) -> StatusCode { + self.expected + } + + /// Asserts that the route answered `status`, and never polled a declared overflow. + /// + /// # Panics + /// + /// Panics when the status differs or the declared-overflow body was polled. + pub(crate) fn assert_status(&self, route: &str, status: StatusCode) { + assert_eq!( + status, self.expected, + "{route} should answer the {} case with {}", + self.name, self.expected + ); + if let Some(polled) = &self.polled { + assert!( + !polled.get(), + "{route} should reject the {} case without polling the body", + self.name + ); + } + } + + /// Asserts on the status of a handler outcome, whether a response or an error. + /// + /// # Panics + /// + /// Panics as [`Self::assert_status`] does. + pub(crate) fn assert_outcome( + &self, + route: &str, + outcome: &Result, Report>, + ) { + self.assert_status(route, outcome_status(outcome)); + } +} + +/// Returns the status a handler outcome answers with, as an adapter renders it. +pub(crate) fn outcome_status( + outcome: &Result, Report>, +) -> StatusCode { + match outcome { + Ok(response) => response.status(), + Err(report) => report.current_context().status_code(), + } +} + +/// Returns the five body-read failures every reader route maps, for a reader capped at `max`. +/// +/// The cases are a declared length over `max` (413, never polled), a streamed +/// body that crosses `max` (413), a read past the inbound deadline (408), a +/// client-caused stream error (400), and a host read error (502). +pub(crate) fn body_read_failure_cases(max: usize) -> Vec { + let polled = Rc::new(Cell::new(false)); + let poll_flag = Rc::clone(&polled); + let unpolled = Body::from_stream(futures::stream::poll_fn(move |_| { + poll_flag.set(true); + Poll::Ready(None::>) + })); + + vec![ + BodyReadCase { + name: "declared over cap", + content_length: Some((max + 1).to_string()), + body: unpolled, + expected: StatusCode::PAYLOAD_TOO_LARGE, + polled: Some(polled), + }, + BodyReadCase { + name: "streamed over cap", + content_length: None, + body: streamed_over(max), + expected: StatusCode::PAYLOAD_TOO_LARGE, + polled: None, + }, + failing_case( + "read deadline", + EdgeError::request_timeout("expired"), + StatusCode::REQUEST_TIMEOUT, + ), + failing_case( + "client stream error", + EdgeError::bad_request("malformed chunk framing"), + StatusCode::BAD_REQUEST, + ), + failing_case( + "host read error", + EdgeError::bad_gateway("host read failed"), + StatusCode::BAD_GATEWAY, + ), + ] +} + +/// Returns a streamed body of `max` bytes followed by one more, with no declared length. +pub(crate) fn streamed_over(max: usize) -> Body { + let chunks: [Result; 2] = [ + Ok(Bytes::from(vec![b'x'; max])), + Ok(Bytes::from_static(b"x")), + ]; + Body::from_stream(futures::stream::iter(chunks)) +} + +fn failing_case(name: &'static str, error: EdgeError, expected: StatusCode) -> BodyReadCase { + BodyReadCase { + name, + content_length: None, + body: Body::from_stream(futures::stream::iter([Err::(error)])), + expected, + polled: None, + } +} diff --git a/crates/trusted-server-core/src/test_support/logging.rs b/crates/trusted-server-core/src/test_support/logging.rs index 9c0f09657b..e98fbdc237 100644 --- a/crates/trusted-server-core/src/test_support/logging.rs +++ b/crates/trusted-server-core/src/test_support/logging.rs @@ -1,4 +1,4 @@ -//! Thread-local warning capture behind the single process-wide test logger. +//! Thread-local warning and error capture behind the single process-wide test logger. use std::cell::RefCell; use std::sync::Once; @@ -7,8 +7,13 @@ thread_local! { static WARNINGS: RefCell>> = const { RefCell::new(None) }; } -pub(crate) struct CapturedWarning { +/// One warning or error a [`WarningCapture`] recorded. +pub struct CapturedWarning { + /// Level of the record, [`log::Level::Warn`] or [`log::Level::Error`]. + pub level: log::Level, + /// Log target of the warning. pub target: String, + /// Rendered warning message. pub message: String, } @@ -16,7 +21,7 @@ struct TestLogger; impl log::Log for TestLogger { fn enabled(&self, metadata: &log::Metadata<'_>) -> bool { - metadata.level() == log::Level::Warn + metadata.level() <= log::Level::Warn } fn log(&self, record: &log::Record<'_>) { @@ -24,6 +29,7 @@ impl log::Log for TestLogger { WARNINGS.with(|warnings| { if let Some(warnings) = warnings.borrow_mut().as_mut() { warnings.push(CapturedWarning { + level: record.level(), target: record.target().to_owned(), message: record.args().to_string(), }); @@ -35,9 +41,20 @@ impl log::Log for TestLogger { fn flush(&self) {} } -pub(crate) struct WarningCapture; +/// Records the warnings and errors logged on the current thread until it is dropped. +/// +/// The first capture installs one process-wide logger that keeps only warnings and errors, +/// so a test binary that installs another logger cannot use it. Captures do not nest. +pub struct WarningCapture; impl WarningCapture { + /// Starts capturing the warnings and errors logged on this thread. + /// + /// # Panics + /// + /// Panics if another logger is already installed or a capture is already active on this + /// thread. + #[must_use] pub fn start() -> Self { static INIT: Once = Once::new(); INIT.call_once(|| { @@ -55,6 +72,12 @@ impl WarningCapture { Self } + /// Returns the warnings and errors captured since the start or the last take. + /// + /// # Panics + /// + /// Panics if the capture is no longer active on this thread. + #[must_use] pub fn take(&self) -> Vec { WARNINGS.with(|warnings| { std::mem::take( diff --git a/crates/trusted-server-core/src/test_support/scripted_client.rs b/crates/trusted-server-core/src/test_support/scripted_client.rs new file mode 100644 index 0000000000..80fa8cf705 --- /dev/null +++ b/crates/trusted-server-core/src/test_support/scripted_client.rs @@ -0,0 +1,2208 @@ +//! Scripted `EdgeZero` outbound client and test clock. +//! +//! [`ScriptedHttpClient`] implements `EdgeZero`'s [`OutboundHttpClient`] for tests. It records +//! every request the [`HttpClient`] facade dispatches, as [`RecordedRequest`], and answers each +//! one from a first-in, first-out script kept per method and URI. +//! +//! Replies arrive at instants on a [`TestClock`] that only the test moves. A reply scripted +//! with [`ScriptedReply::after`] arrives that long after its request entered dispatch, which for a +//! batch is the batch start. A reply that has not arrived by its request's deadline times out at +//! that deadline. A batch drops a slot that ends at or after its cutoff, and the slot is +//! `Unresolved`, as `EdgeZero`'s batch ledger does. A test that waits on a future reply advances +//! the clock between polls, or lets [`TestClock::drive`] advance it to each instant something +//! waits on. +//! +//! The double's [`OutboundBatch::check`](edgezero_core::outbound::OutboundBatch::check) reports a +//! reply as soon as the test clock reaches its +//! arrival, which is more eager than Fastly's, where a check reports only what an earlier +//! `EdgeZero` poll recorded. + +// This module builds under the `test-utils` feature, which the crate-root +// `cfg(test)` allow does not cover, and it buffers scripted bodies. +#![allow(clippy::disallowed_methods)] + +use std::collections::{BTreeMap, HashMap, VecDeque}; +use std::future::Future; +use std::io; +use std::mem; +use std::pin::pin; +use std::rc::Rc; +use std::sync::atomic::{AtomicBool, AtomicUsize, Ordering}; +use std::sync::{Arc, Mutex, MutexGuard, PoisonError}; +use std::task::{Context, Poll, Wake, Waker}; +use std::time::Duration; + +use bytes::{Bytes, BytesMut}; +use edgezero_core::body::{Body, BodyPayload, BodyStream}; +use edgezero_core::error::{EdgeError, RequestBodyOverflowOrigin}; +use edgezero_core::http::{HeaderMap, HeaderName, HeaderValue, Method, StatusCode, Uri}; +use edgezero_core::outbound::{ + ContentTypeProvenance, HttpClient, IndependentOutboundProgress, OutboundBatch, + OutboundBatchDriverEvent, OutboundCachePolicy, OutboundDispatchRequest, OutboundExchangeOwner, + OutboundFuture, OutboundHead, OutboundHeaderFidelity, OutboundHeaderInput, OutboundHttpClient, + OutboundImageOptimizerOptions, OutboundPending, OutboundPreparedBatch, OutboundProgressRuntime, + OutboundProxyMarker, OutboundRequest, OutboundRequestParts, OutboundResponse, + OutboundResponseContext, OutboundSendMode, OutboundTime, ResponseDecoding, ResponseMode, +}; +use edgezero_core::time::{Deadline, DispatchBudget, MonotonicClock, MonotonicInstant}; +use futures::FutureExt as _; +use futures::future::poll_fn; +use futures::stream::{self, StreamExt as _}; + +/// Adapter identity the scripted client reports for a proxy marker a request turns on. +const ADAPTER: &str = "scripted"; + +/// Hand-advanced monotonic time for scripted outbound tests. +/// +/// Clones share one instant, which starts an hour after the process clock's current instant and +/// moves only through [`TestClock::advance`]. A deadline taken from the process clock, such as one +/// from `MonotonicClock::default()`, is therefore already expired on this clock, so code that +/// mixes clocks fails its tests. Advancing wakes every timer and scripted reply waiting on the +/// clock, as `EdgeZero` requires of a fake clock. +#[derive(Clone)] +pub struct TestClock { + state: Arc>, +} + +/// How far a new [`TestClock`] starts ahead of the process clock. +const PROCESS_CLOCK_OFFSET: Duration = Duration::from_hours(1); + +struct ClockState { + now: MonotonicInstant, + /// Wakers waiting for the clock, each with the instant it waits for. + waiters: Vec<(MonotonicInstant, Waker)>, +} + +impl TestClock { + /// Creates a clock that stands still until advanced. + /// + /// # Panics + /// + /// Panics if the process clock cannot be offset by an hour. + #[must_use] + pub fn new() -> Self { + Self { + state: Arc::new(Mutex::new(ClockState { + now: MonotonicInstant::now() + .checked_add(PROCESS_CLOCK_OFFSET) + .expect("should offset the test clock from process time"), + waiters: Vec::new(), + })), + } + } + + /// Returns a [`MonotonicClock`] that reads this clock. + #[must_use] + pub fn monotonic_clock(&self) -> MonotonicClock { + let state = Arc::clone(&self.state); + MonotonicClock::new(move || lock(&state).now) + } + + /// Returns an [`OutboundTime`] on this clock. + /// + /// Its timers resolve once [`TestClock::advance`] brings the clock to their deadline. + #[must_use] + pub fn outbound_time(&self) -> OutboundTime { + let clock = self.clone(); + OutboundTime::new(self.monotonic_clock(), move |deadline, _paired| { + let clock = clock.clone(); + Box::pin(poll_fn(move |context| { + clock + .poll_reached(deadline.instant(), context.waker()) + .map(Ok) + })) + }) + } + + /// Moves the clock forward by `by` and wakes everything waiting on it. + /// + /// # Panics + /// + /// Panics if the new instant does not fit the platform's monotonic clock. + pub fn advance(&self, by: Duration) { + let waiters = { + let mut state = lock(&self.state); + state.now = state + .now + .checked_add(by) + .expect("should advance the test clock within the platform clock range"); + mem::take(&mut state.waiters) + }; + waiters + .into_iter() + .for_each(|(_instant, waker)| waker.wake()); + } + + /// Polls `future` to completion, advancing the clock whenever it can make no progress. + /// + /// Each time `future` is pending and has not woken itself, the clock moves to the earliest + /// instant a timer or scripted reply waits on. Time therefore moves only as far as the code + /// under test waits, and no test sleeps. + /// + /// # Panics + /// + /// Panics if `future` is pending while nothing waits on the clock, such as a future that only + /// waits on a held reply, because no amount of time would finish it. + pub fn drive(&self, future: F) -> F::Output { + let woken = Arc::new(WakeFlag(AtomicBool::new(true))); + let waker = Waker::from(Arc::clone(&woken)); + let mut context = Context::from_waker(&waker); + let mut future = pin!(future); + loop { + if woken.0.swap(false, Ordering::SeqCst) { + if let Poll::Ready(output) = future.as_mut().poll(&mut context) { + return output; + } + continue; + } + let next = self + .next_waited_instant() + .expect("should only drive a future that waits on the test clock"); + self.advance(next.saturating_duration_since(self.now())); + } + } + + fn now(&self) -> MonotonicInstant { + lock(&self.state).now + } + + /// Returns the earliest instant a registered waker waits for. + fn next_waited_instant(&self) -> Option { + lock(&self.state) + .waiters + .iter() + .map(|(instant, _waker)| *instant) + .min() + } + + /// Reports whether the clock has reached `instant`, or arranges a wakeup when it does. + fn poll_reached(&self, instant: MonotonicInstant, waker: &Waker) -> Poll<()> { + let mut state = lock(&self.state); + if state.now >= instant { + return Poll::Ready(()); + } + if !state + .waiters + .iter() + .any(|(waited, waiter)| *waited == instant && waiter.will_wake(waker)) + { + state.waiters.push((instant, waker.clone())); + } + Poll::Pending + } +} + +impl Default for TestClock { + fn default() -> Self { + Self::new() + } +} + +/// One scripted answer to a request. +/// +/// A status reply has no headers and an empty body until the builder methods add them. A reply +/// built with [`ScriptedReply::error`] ignores header and body settings. +pub struct ScriptedReply { + outcome: Result, + headers: HeaderMap, + chunks: Vec, + delay: Duration, + hold: Option, + body_polls: Option>, + body_error: Option, + fails_batch: bool, +} + +impl ScriptedReply { + /// A response with `status`. + #[must_use] + pub fn status(status: StatusCode) -> Self { + Self::with_outcome(Ok(status)) + } + + /// A failure that the request returns unchanged instead of a response. + #[must_use] + pub fn error(error: EdgeError) -> Self { + Self::with_outcome(Err(error)) + } + + /// A failure of the whole batch driver, at this slot's arrival. + /// + /// In a batch, the driver reports `error` when this reply would arrive and observes nothing + /// after it, as a provider whose batch driver breaks. Outside a batch the request fails with + /// `error`, as [`ScriptedReply::error`] does. + #[must_use] + pub fn failing_batch(error: EdgeError) -> Self { + Self { + fails_batch: true, + ..Self::with_outcome(Err(error)) + } + } + + /// Appends one response header. + /// + /// # Panics + /// + /// Panics if `name` or `value` is not a valid header field. + #[must_use] + pub fn header(mut self, name: &str, value: &str) -> Self { + let name = + HeaderName::from_bytes(name.as_bytes()).expect("should parse the scripted header name"); + let value = HeaderValue::from_str(value).expect("should parse the scripted header value"); + self.headers.append(name, value); + self + } + + /// Sets the body, delivered as one chunk. + #[must_use] + pub fn body(self, bytes: impl Into) -> Self { + self.chunks(vec![bytes.into()]) + } + + /// Sets the body as `chunks`, delivered as one stream item each. + #[must_use] + pub fn chunks(mut self, chunks: Vec) -> Self { + self.chunks = chunks; + self + } + + /// Ends the body with `error` after its chunks, as an upstream that fails mid-body. + #[must_use] + pub fn body_error(mut self, error: EdgeError) -> Self { + self.body_error = Some(error); + self + } + + /// Makes the reply arrive `delay` after its request enters dispatch, on the test clock. + /// + /// A batch slot's request enters dispatch at the batch start. + #[must_use] + pub fn after(mut self, delay: Duration) -> Self { + self.delay = delay; + self + } + + /// Holds the reply until the returned [`ReplyRelease`] is released. + /// + /// A held reply that is still held at its request's deadline times out there. + pub fn held(mut self) -> (Self, ReplyRelease) { + let hold = Hold::default(); + self.hold = Some(hold.clone()); + (self, ReplyRelease { hold }) + } + + /// Counts in `polls` each poll of the body stream, the poll that ends it included. + /// + /// A test reads the count to prove how much of the body the code under test read. + #[must_use] + pub fn counting_body_polls(mut self, polls: &Arc) -> Self { + self.body_polls = Some(Arc::clone(polls)); + self + } + + fn with_outcome(outcome: Result) -> Self { + Self { + outcome, + headers: HeaderMap::new(), + chunks: Vec::new(), + delay: Duration::ZERO, + hold: None, + body_polls: None, + body_error: None, + fails_batch: false, + } + } + + /// Builds the provider head `EdgeZero` finalizes, or returns the scripted error. + fn into_head(self, context: OutboundResponseContext) -> Result { + let status = self.outcome?; + let mut chunks = stream::iter( + self.chunks + .into_iter() + .map(Ok::<_, EdgeError>) + .chain(self.body_error.map(Err)), + ); + let body = match self.body_polls { + None => chunks.boxed_local(), + Some(polls) => stream::poll_fn(move |context| { + polls.fetch_add(1, Ordering::SeqCst); + chunks.poll_next_unpin(context) + }) + .boxed_local(), + }; + OutboundHead::new( + status, + OutboundHeaderInput::new( + ADAPTER, + self.headers, + OutboundHeaderFidelity::AdapterVisibleFields, + ), + body, + context, + OutboundExchangeOwner::new(|| {}), + ) + } +} + +/// Releases one held [`ScriptedReply`]. +#[must_use = "a held reply never arrives unless it is released"] +pub struct ReplyRelease { + hold: Hold, +} + +impl ReplyRelease { + /// Lets the held reply arrive now, or at its [`ScriptedReply::after`] delay if that is later. + pub fn release(self) { + self.hold.release(); + } + + /// Reports whether the held reply is still queued or still awaited by a scheduled exchange. + /// + /// It turns false once the request was dispatched and its exchange dropped, as when the + /// code under test cancels the request. + #[must_use] + pub fn is_awaited(&self) -> bool { + Arc::strong_count(&self.hold.0) > 1 + } +} + +/// One request as the scripted client received it from the facade. +/// +/// The fields come from [`OutboundRequestParts`] after preparation, so the proxy marker is +/// resolved to on or off and the headers are normalized. +#[derive(Clone, Debug)] +pub struct RecordedRequest { + /// Request method. + pub method: Method, + /// Canonical request URI. + pub uri: Uri, + /// Normalized request headers. + pub headers: HeaderMap, + /// Whole request body. + /// + /// A streamed upload is recorded when its request enters dispatch, before its body is + /// collected, so this stays empty for an upload that fails or that the facade never let + /// finish. + pub body: Bytes, + /// Cap on the request body the request may upload. + pub max_request_body_bytes: u64, + /// Per-call timeout. + pub timeout: Option, + /// Connect timeout. + pub connect_timeout: Option, + /// Caller-supplied absolute deadline. + pub deadline: Option, + /// Intermediary cache policy. + pub cache_policy: OutboundCachePolicy, + /// Whether response content codings are decoded. + pub response_decoding: ResponseDecoding, + /// Whether the response is buffered or streamed. + pub response_mode: ResponseMode, + /// Cap on decoded response bytes. + pub max_decoded_response_bytes: Option, + /// Cap on encoded response bytes. + pub max_encoded_response_bytes: Option, + /// Image Optimizer options. + pub image_optimizer: Option, + /// Resolved proxy marker setting. + pub proxy_marker: OutboundProxyMarker, + /// Authority sent upstream, from [`OutboundRequest::host_authority`]. + pub host_authority: String, + /// Whether the request overrides its URI authority upstream. + pub has_host_authority_override: bool, + /// The facade operation that dispatched the request. + pub mode: OutboundSendMode, + /// The effective budget the facade selected, a batch cutoff included. + pub budget: DispatchBudget, +} + +/// Transport settings a readback test expects one recorded request to carry. +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +pub struct ExpectedTransport { + /// Per-call timeout. + pub timeout: Duration, + /// Buffered under its cap, or streamed. + pub response_mode: ResponseMode, + /// Whether response content codings are decoded. + pub response_decoding: ResponseDecoding, + /// Intermediary cache policy. + pub cache_policy: OutboundCachePolicy, +} + +impl RecordedRequest { + /// Asserts the transport settings every Trusted Server request carries. + /// + /// Checks a one second connect timeout, no caller deadline, no Image Optimizer options, the + /// proxy marker off, the URI authority sent upstream with no override, and the timeout, + /// response mode, decoding, and cache policy in `expected`. A failure names the request by + /// its method only, never its URI or the authority it sent. + /// + /// # Panics + /// + /// Panics when any of those settings differs. + pub fn assert_transport(&self, expected: ExpectedTransport) { + self.assert_transport_settings(expected, None, None); + } + + /// Asserts [`RecordedRequest::assert_transport`]'s settings for one hop of a followed chain. + /// + /// Every hop of a followed redirect chain carries the chain's `deadline`, so this checks that + /// deadline in place of the absent one. + /// + /// # Panics + /// + /// Panics when any of those settings differs. + pub fn assert_followed_transport(&self, expected: ExpectedTransport, deadline: Deadline) { + self.assert_transport_settings(expected, Some(deadline), None); + } + + /// Asserts [`RecordedRequest::assert_transport`]'s settings for an Image Optimizer request. + /// + /// This checks that the request carries exactly `options` in place of no options. + /// + /// # Panics + /// + /// Panics when any of those settings differs. + pub fn assert_image_optimizer_transport( + &self, + expected: ExpectedTransport, + options: &OutboundImageOptimizerOptions, + ) { + self.assert_transport_settings(expected, None, Some(options)); + } + + fn assert_transport_settings( + &self, + expected: ExpectedTransport, + deadline: Option, + image_optimizer: Option<&OutboundImageOptimizerOptions>, + ) { + // The request is named by its method only: its URI can carry a credential. + let request = format!("the {} request", self.method); + assert_eq!( + self.connect_timeout, + Some(Duration::from_secs(1)), + "should bound the connect of {request} at one second" + ); + assert_eq!( + self.timeout, + Some(expected.timeout), + "should give {request} its constant timeout" + ); + assert_eq!( + self.deadline.map(|recorded| recorded.instant()), + deadline.map(|expected| expected.instant()), + "should give {request} only the chain deadline it expects, never one from a remaining budget" + ); + assert_eq!( + self.image_optimizer.as_ref(), + image_optimizer, + "should give {request} exactly the Image Optimizer options it expects" + ); + assert_eq!( + ( + self.response_mode, + self.response_decoding, + self.cache_policy + ), + ( + expected.response_mode, + expected.response_decoding, + expected.cache_policy + ), + "should give {request} its response mode, decoding, and cache policy" + ); + assert_eq!( + self.proxy_marker, + OutboundProxyMarker::Off, + "should send {request} with the proxy marker off" + ); + assert!( + !self.has_host_authority_override + && self.uri.authority().map(http::uri::Authority::as_str) + == Some(self.host_authority.as_str()), + "should send {request} to its URI authority with no override" + ); + } + + /// Asserts the recorded headers are exactly `expected`, in any order. + /// + /// Names compare case-insensitively. `Host`, `Content-Length`, and hop-by-hop fields never + /// appear here, because `EdgeZero` strips them before dispatch. A failure names the missing, + /// unexpected, and differing headers only. It never prints a header value or the request + /// URI, because either can carry a credential. + /// + /// # Panics + /// + /// Panics when a header is missing, extra, or carries other values, or when a recorded + /// value is not UTF-8. + pub fn assert_headers(&self, expected: &[(&str, &str)]) { + assert_eq!( + self.header_mismatch(expected), + HeaderMismatch::default(), + "should send exactly the expected headers" + ); + } + + /// Compares the recorded headers with `expected` by name, in any order. + fn header_mismatch(&self, expected: &[(&str, &str)]) -> HeaderMismatch { + let mut sent: BTreeMap> = BTreeMap::new(); + for (name, value) in &self.headers { + let value = value.to_str().expect("should record UTF-8 header values"); + sent.entry(name.as_str().to_owned()) + .or_default() + .push(value); + } + let mut wanted: BTreeMap> = BTreeMap::new(); + for (name, value) in expected { + wanted + .entry(name.to_ascii_lowercase()) + .or_default() + .push(value); + } + for values in sent.values_mut().chain(wanted.values_mut()) { + values.sort_unstable(); + } + + HeaderMismatch { + missing: wanted + .keys() + .filter(|name| !sent.contains_key(*name)) + .cloned() + .collect(), + unexpected: sent + .keys() + .filter(|name| !wanted.contains_key(*name)) + .cloned() + .collect(), + differing: wanted + .iter() + .filter(|(name, values)| sent.get(*name).is_some_and(|sent| sent != *values)) + .map(|(name, _)| name.clone()) + .collect(), + } + } +} + +/// The header names a [`RecordedRequest`] got wrong, without their values. +#[derive(Debug, Default, PartialEq, Eq)] +struct HeaderMismatch { + /// Expected names the request did not send. + missing: Vec, + /// Sent names the test did not expect. + unexpected: Vec, + /// Names sent with other values than the test expected. + differing: Vec, +} + +/// A scripted `EdgeZero` outbound client that records requests and answers them from a script. +/// +/// Clones share the script, the record, and the clock. Requests that a test never scripted fail +/// with an internal error and are still recorded. +#[derive(Clone)] +pub struct ScriptedHttpClient { + clock: TestClock, + forbid: bool, + state: Arc>, + observer: Option, +} + +/// A test callback that sees each request as the scripted client records it. +type RequestObserver = Arc; + +#[derive(Default)] +struct ScriptState { + replies: HashMap<(Method, String), VecDeque>, + requests: Vec, +} + +impl ScriptedHttpClient { + /// Creates a client with an empty script whose replies arrive on `clock`. + #[must_use] + pub fn new(clock: &TestClock) -> Self { + Self { + clock: clock.clone(), + forbid: false, + state: Arc::default(), + observer: None, + } + } + + /// Calls `observer` with each request as the client records it. + /// + /// A test uses it to see what else had happened when a request left, such as the calls + /// another double had received by then. The observer runs outside the client's lock, so it + /// may read this client. + #[must_use] + pub fn with_request_observer( + mut self, + observer: impl Fn(&RecordedRequest) + Send + Sync + 'static, + ) -> Self { + self.observer = Some(Arc::new(observer)); + self + } + + /// Creates a client that fails every request it receives. + /// + /// Each failure is internal, with a message that starts with `should not send + /// `, and each request is still recorded. Call + /// [`ScriptedHttpClient::assert_no_requests`] to fail the test on any of them. + #[must_use] + pub fn forbidding() -> Self { + Self { + forbid: true, + ..Self::new(&TestClock::new()) + } + } + + /// Queues `reply` after any replies already queued for `method` requests to `uri`. + /// + /// # Panics + /// + /// Panics if `uri` is not an absolute HTTP or HTTPS URI. + // Queuing the reply is the effect. The returned reference only lets calls chain. + #[allow(clippy::must_use_candidate)] + pub fn respond_to(&self, method: Method, uri: &str, reply: ScriptedReply) -> &Self { + let canonical = OutboundRequest::get(uri) + .expect("should parse the scripted URI") + .uri() + .to_string(); + if let Some(hold) = &reply.hold { + hold.attach(&self.clock); + } + lock(&self.state) + .replies + .entry((method, canonical)) + .or_default() + .push_back(reply); + self + } + + /// Returns a facade over this client with the proxy marker off. + #[must_use] + pub fn http_client(&self) -> HttpClient { + HttpClient::with_client(self.clone()).with_proxy_marker(false) + } + + /// Returns every request received so far, in arrival order. + #[must_use] + pub fn requests(&self) -> Vec { + lock(&self.state).requests.clone() + } + + /// Fails the test if any request was received. + /// + /// # Panics + /// + /// Panics with `should not send `, naming every received request, if there are + /// any. + pub fn assert_no_requests(&self) { + let sent: Vec = lock(&self.state) + .requests + .iter() + .map(|request| format!("{} {}", request.method, request.uri)) + .collect(); + assert!(sent.is_empty(), "should not send {}", sent.join(", ")); + } + + /// Takes the reply for a recorded request and builds the head it arrives with. + fn schedule( + &self, + recorded: &RecordedRequest, + context: OutboundResponseContext, + ) -> Scheduled { + let reply = self.take_reply(recorded); + let budget = context.budget(); + let arrival = Arrival { + at: context + .entered_at() + .checked_add(reply.delay) + .unwrap_or_else(|| budget.deadline.instant()), + budget, + hold: reply.hold.clone(), + fails_batch: reply.fails_batch, + }; + Scheduled { + arrival, + outcome: reply.into_head(context), + } + } + + /// Schedules a buffered request and finalizes its response at once, as eager work does. + fn schedule_eager(&self, request: OutboundDispatchRequest) -> Scheduled { + let Dispatched { + mut recorded, + body, + context, + .. + } = Dispatched::new(request); + // The facade hands begin and batch only buffered bodies. It rejects streamed uploads + // there, and settlement drains a GET or HEAD stream before the provider sees it. + recorded.body = body + .into_bytes() + .expect("should receive a buffered body for begin or batch"); + self.record(&recorded); + let Scheduled { arrival, outcome } = self.schedule(&recorded, context); + Scheduled { + arrival, + outcome: outcome.and_then(finish_now), + } + } + + /// Records a request as it enters dispatch and returns its position in the record. + /// + /// The observer sees the request as recorded, so a streamed upload's body is still empty. + fn record(&self, recorded: &RecordedRequest) -> usize { + let mut state = lock(&self.state); + state.requests.push(recorded.clone()); + let position = state.requests.len() - 1; + drop(state); + if let Some(observer) = &self.observer { + observer(recorded); + } + position + } + + /// Fills in the body of the request recorded at `position` once its upload is collected. + fn record_upload(&self, position: usize, body: &Bytes) { + if let Some(recorded) = lock(&self.state).requests.get_mut(position) { + recorded.body = body.clone(); + } + } + + fn take_reply(&self, recorded: &RecordedRequest) -> ScriptedReply { + let RecordedRequest { method, uri, .. } = recorded; + if self.forbid { + return ScriptedReply::error(internal(format!("should not send {method} {uri}"))); + } + lock(&self.state) + .replies + .get_mut(&(method.clone(), uri.to_string())) + .and_then(VecDeque::pop_front) + .unwrap_or_else(|| { + ScriptedReply::error(internal(format!("unscripted request {method} {uri}"))) + }) + } +} + +impl OutboundHttpClient for ScriptedHttpClient { + fn begin(&self, request: OutboundDispatchRequest) -> Result { + let scheduled = self.schedule_eager(request); + Ok(OutboundPending::new( + Box::pin(scheduled.wait(self.clock.clone())), + OutboundExchangeOwner::new(|| {}), + )) + } + + fn dispatch_head(&self, request: OutboundDispatchRequest) -> OutboundFuture { + let client = self.clone(); + let Dispatched { + mut recorded, + body, + context, + } = Dispatched::new(request); + let (payload, _egress_owner) = body.into_parts(); + let upload = match payload { + BodyPayload::Once(bytes) => { + recorded.body = bytes; + None + } + BodyPayload::Stream(source) => Some(source), + }; + let position = self.record(&recorded); + Box::pin(async move { + if let Some(source) = upload { + let budget = context.budget(); + recorded.body = collect_upload( + source, + recorded.max_request_body_bytes, + budget, + &client.clock, + ) + .await?; + client.record_upload(position, &recorded.body); + } + let scheduled = client.schedule(&recorded, context); + scheduled.wait(client.clock).await + }) + } + + fn progress(&self) -> Rc { + Rc::new(IndependentOutboundProgress) + } + + fn start_batch_until(&self, batch: OutboundPreparedBatch) -> OutboundBatch { + let (context, requests) = batch.into_slots(); + if requests.is_empty() { + return OutboundBatch::cutoff(context); + } + let mut driver = BatchDriver { + clock: self.clock.clone(), + cutoff: context.observation_cutoff(), + slots: requests + .into_iter() + .map(|(index, request)| (index, self.schedule_eager(request))) + .collect(), + late: false, + }; + OutboundBatch::from_driver( + context, + stream::poll_fn(move |context| driver.poll_event(context)), + OutboundExchangeOwner::new(|| {}), + ) + } + + fn time(&self) -> OutboundTime { + self.clock.outbound_time() + } + + fn validate_supported_options( + &self, + _request: &OutboundRequest, + _mode: OutboundSendMode, + ) -> Result<(), EdgeError> { + Ok(()) + } +} + +/// A prepared request taken apart for recording. +/// +/// `recorded.body` stays empty until the body is collected. +struct Dispatched { + recorded: RecordedRequest, + body: Body, + context: OutboundResponseContext, +} + +impl Dispatched { + fn new(request: OutboundDispatchRequest) -> Self { + let host_authority = request.request().host_authority().to_owned(); + let has_host_authority_override = request.request().has_host_authority_override(); + let (parts, context) = request.into_parts(); + let mode = context.mode(); + let budget = context.budget(); + let OutboundRequestParts { + body, + cache_policy, + connect_timeout, + deadline, + headers, + image_optimizer, + max_decoded_response_bytes, + max_encoded_response_bytes, + max_request_body_bytes, + method, + proxy_marker, + response_decoding, + response_mode, + timeout, + uri, + .. + } = parts; + Self { + recorded: RecordedRequest { + method, + uri, + headers, + body: Bytes::new(), + max_request_body_bytes, + timeout, + connect_timeout, + deadline, + cache_policy, + response_decoding, + response_mode, + max_decoded_response_bytes, + max_encoded_response_bytes, + image_optimizer, + proxy_marker, + host_authority, + has_host_authority_override, + mode, + budget, + }, + body, + context, + } + } +} + +/// A scheduled exchange: when its reply arrives, and what it arrives with. +struct Scheduled { + arrival: Arrival, + outcome: Result, +} + +impl Scheduled { + /// Waits on `clock` for the reply or the deadline, whichever comes first. + async fn wait(self, clock: TestClock) -> Result { + let terminal = poll_fn(|context| self.arrival.poll(&clock, context.waker())).await; + self.settle(terminal) + } + + fn settle(self, terminal: Terminal) -> Result { + match terminal { + Terminal::Arrived(_) => self.outcome, + Terminal::TimedOut(_) => Err(EdgeError::gateway_timeout_caused( + "scripted reply did not arrive before its deadline", + self.arrival.budget.cause, + )), + } + } +} + +/// When one scripted reply becomes observable on the test clock. +struct Arrival { + at: MonotonicInstant, + budget: DispatchBudget, + hold: Option, + /// Whether the reply's arrival fails the batch driver instead of resolving its slot. + fails_batch: bool, +} + +/// The first event that ends a scripted exchange, at its test-clock instant. +#[derive(Clone, Copy)] +enum Terminal { + Arrived(MonotonicInstant), + TimedOut(MonotonicInstant), +} + +impl Terminal { + fn instant(self) -> MonotonicInstant { + match self { + Self::Arrived(instant) | Self::TimedOut(instant) => instant, + } + } +} + +impl Arrival { + /// Reports the terminal event once the clock reaches it, or arranges a wakeup. + /// + /// A reply arriving at or after the budget deadline times out at the deadline, as `EdgeZero` + /// treats a deadline at or before now as expired. + fn poll(&self, clock: &TestClock, waker: &Waker) -> Poll { + let arrived = match &self.hold { + None => Some(self.at), + Some(hold) => match hold.poll_release(waker) { + Poll::Pending => None, + Poll::Ready(released_at) => { + Some(released_at.map_or(self.at, |released| released.max(self.at))) + } + }, + }; + let deadline = self.budget.deadline.instant(); + let terminal = match arrived { + Some(at) if at < deadline => Terminal::Arrived(at), + _ => Terminal::TimedOut(deadline), + }; + clock + .poll_reached(terminal.instant(), waker) + .map(|()| terminal) + } +} + +/// Emits batch slot outcomes in test-clock order, as `EdgeZero`'s Axum batch driver does. +struct BatchDriver { + clock: TestClock, + cutoff: Deadline, + slots: Vec<(usize, Scheduled)>, + late: bool, +} + +impl BatchDriver { + fn poll_event(&mut self, context: &mut Context<'_>) -> Poll> { + let mut earliest: Option<(usize, Terminal)> = None; + for (position, (_, slot)) in self.slots.iter().enumerate() { + if let Poll::Ready(terminal) = slot.arrival.poll(&self.clock, context.waker()) + && earliest.is_none_or(|(_, first)| terminal.instant() < first.instant()) + { + earliest = Some((position, terminal)); + } + } + let Some((position, terminal)) = earliest else { + if !self.slots.is_empty() { + return Poll::Pending; + } + // The ledger drops a slot that ends at or after the cutoff, so a batch that lost one + // must end with an explicit cutoff rather than a clean end of stream. + return Poll::Ready( + mem::take(&mut self.late).then_some(OutboundBatchDriverEvent::Cutoff), + ); + }; + let (index, slot) = self.slots.remove(position); + let completed_at = terminal.instant(); + if slot.arrival.fails_batch && matches!(terminal, Terminal::Arrived(_)) { + self.slots.clear(); + let error = slot + .settle(terminal) + .expect_err("should script a batch failure as an error"); + return Poll::Ready(Some(OutboundBatchDriverEvent::Failed(error))); + } + self.late |= self.cutoff.is_expired_at(completed_at) + && !slot + .arrival + .budget + .own_deadline_expired_before(self.cutoff, completed_at); + Poll::Ready(Some(OutboundBatchDriverEvent::Item { + index, + completed_at, + outcome: slot.settle(terminal), + })) + } +} + +/// Shared state between a held reply and its [`ReplyRelease`]. +#[derive(Clone, Default)] +struct Hold(Arc>); + +#[derive(Default)] +struct HoldState { + clock: Option, + release: Release, + waiters: Vec, +} + +#[derive(Clone, Copy, Default)] +enum Release { + #[default] + Held, + /// Released at this instant, or before the reply was scripted on any clock. + Released(Option), +} + +impl Hold { + fn attach(&self, clock: &TestClock) { + let mut state = lock(&self.0); + if state.clock.is_none() { + state.clock = Some(clock.clone()); + } + } + + fn release(&self) { + let waiters = { + let mut state = lock(&self.0); + state.release = Release::Released(state.clock.as_ref().map(TestClock::now)); + mem::take(&mut state.waiters) + }; + waiters.into_iter().for_each(Waker::wake); + } + + /// Reports when the reply was released, or arranges a wakeup for its release. + fn poll_release(&self, waker: &Waker) -> Poll> { + let mut state = lock(&self.0); + match state.release { + Release::Released(released_at) => Poll::Ready(released_at), + Release::Held => { + register(&mut state.waiters, waker); + Poll::Pending + } + } + } +} + +/// Records whether a future woke itself while being polled. +struct WakeFlag(AtomicBool); + +impl Wake for WakeFlag { + fn wake(self: Arc) { + self.wake_by_ref(); + } + + fn wake_by_ref(self: &Arc) { + self.0.store(true, Ordering::SeqCst); + } +} + +/// Finalizes a scripted head at the current test-clock instant, without an executor. +/// +/// Scripted bodies are in memory, so the finalizer only ever waits on itself, yielding between +/// chunks and waking itself. Polling it until it stops waking itself finishes it the way a real +/// provider's eagerly scheduled task would. +fn finish_now(head: OutboundHead) -> Result { + let woken = Arc::new(WakeFlag(AtomicBool::new(true))); + let waker = Waker::from(Arc::clone(&woken)); + let mut context = Context::from_waker(&waker); + let mut finish = head.finish(); + while woken.0.swap(false, Ordering::SeqCst) { + if let Poll::Ready(outcome) = finish.as_mut().poll(&mut context) { + return outcome; + } + } + Err(internal( + "scripted reply finalizer waited on something outside the script".to_owned(), + )) +} + +/// Collects a streamed upload, enforcing its cap and its budget as providers do. +/// +/// Each read races the budget deadline on the test clock, as Axum's provider does, so an upload +/// that stalls past the deadline times out there. The facade already checked a buffered body +/// against the cap. +async fn collect_upload( + mut source: BodyStream, + max_request_body_bytes: u64, + budget: DispatchBudget, + clock: &TestClock, +) -> Result { + let mut collected = BytesMut::new(); + loop { + let mut next = source.next(); + let read = poll_fn(|context| { + if let Poll::Ready(item) = next.poll_unpin(context) { + return Poll::Ready(Some(item)); + } + clock + .poll_reached(budget.deadline.instant(), context.waker()) + .map(|()| None) + }) + .await; + let Some(item) = read else { + return Err(EdgeError::gateway_timeout_caused( + "scripted upload did not finish before its deadline", + budget.cause, + )); + }; + let Some(chunk) = item else { + return Ok(collected.freeze()); + }; + collected.extend_from_slice(&chunk?); + if u64::try_from(collected.len()).unwrap_or(u64::MAX) > max_request_body_bytes { + return Err(EdgeError::payload_too_large( + RequestBodyOverflowOrigin::OutboundUpload, + )); + } + } +} + +/// Wraps `response` as the [`OutboundResponse`] a POST that `EdgeZero` sent would return. +/// +/// Parser tests use it to hand a provider a response without dispatching a request. The status, +/// headers, and body are kept as given, and the response's clock is a fresh [`TestClock`]. +#[must_use] +pub fn outbound_response(response: edgezero_core::http::Response) -> OutboundResponse { + let (parts, body) = response.into_parts(); + OutboundResponse::new( + Method::POST, + parts.status, + parts.headers, + body, + TestClock::new().monotonic_clock(), + ContentTypeProvenance::Unknown, + ) +} + +fn internal(message: String) -> EdgeError { + EdgeError::internal(io::Error::other(message)) +} + +/// Adds `waker` to `waiters` unless an equivalent waker is already there. +fn register(waiters: &mut Vec, waker: &Waker) { + if !waiters.iter().any(|waiter| waiter.will_wake(waker)) { + waiters.push(waker.clone()); + } +} + +/// Locks `mutex`, recovering the data if a panicking test poisoned it. +fn lock(mutex: &Mutex) -> MutexGuard<'_, T> { + mutex.lock().unwrap_or_else(PoisonError::into_inner) +} + +#[cfg(test)] +mod tests { + use std::sync::atomic::{AtomicUsize, Ordering}; + use std::task::Wake; + + use edgezero_core::error::BudgetSource; + use edgezero_core::outbound::{ + OutboundBatchCheck, OutboundBatchNext, OutboundBatchTermination, OutboundCollectedSlot, + OutboundImageFormat, OutboundImageOptimizerRegion, + }; + use futures::executor::block_on; + + use super::*; + + const ORIGIN: &str = "https://example.com"; + + /// Counts wakeups, so tests can prove that advancing or releasing wakes a waiter. + struct WakeCount(AtomicUsize); + + impl Wake for WakeCount { + fn wake(self: Arc) { + self.0.fetch_add(1, Ordering::SeqCst); + } + } + + fn uri(path: &str) -> String { + format!("{ORIGIN}{path}") + } + + fn get(path: &str) -> OutboundRequest { + OutboundRequest::get(uri(path)).expect("should build the GET request") + } + + fn poll_once(future: &mut OutboundFuture, waker: &Waker) -> Poll> { + future.as_mut().poll(&mut Context::from_waker(waker)) + } + + fn body_of(response: &OutboundResponse) -> &[u8] { + response + .body() + .as_bytes() + .expect("should buffer the response body") + } + + #[test] + fn records_method_uri_headers_and_options() { + let clock = TestClock::new(); + let client = ScriptedHttpClient::new(&clock); + client + .respond_to( + Method::POST, + &uri("/bid"), + ScriptedReply::status(StatusCode::OK), + ) + .respond_to( + Method::GET, + &uri("/image.png"), + ScriptedReply::status(StatusCode::OK), + ); + let http = client.http_client(); + let deadline = clock + .monotonic_clock() + .deadline_after(Duration::from_secs(60)); + let bid = OutboundRequest::post(uri("/bid")) + .expect("should build the POST request") + .header("x-example", "one") + .expect("should add the request header") + .host_authority_override("origin.example.com") + .expect("should override the host authority") + .body(Bytes::from_static(b"{\"id\":1}")) + .timeout(Duration::from_secs(2)) + .connect_timeout(Duration::from_secs(1)) + .deadline(deadline) + .cache_policy(OutboundCachePolicy::Bypass) + .response_decoding(ResponseDecoding::Original) + .max_decoded_response_bytes(4096) + .max_encoded_response_bytes(2048); + let options = OutboundImageOptimizerOptions::new() + .width(320) + .expect("should set the image width") + .format(OutboundImageFormat::WebP) + .region(OutboundImageOptimizerRegion::UsEast); + let image = get("/image.png") + .image_optimizer(options.clone()) + .stream_response(); + + block_on(http.send(bid)).expect("should answer the bid request"); + block_on(http.send(image)).expect("should answer the image request"); + + assert!( + !http.proxy_marker_enabled(), + "should hand out a client with the proxy marker off" + ); + let recorded = client.requests(); + assert_eq!(recorded.len(), 2, "should record both requests"); + let bid = &recorded[0]; + assert_eq!( + ( + &bid.method, + bid.uri.to_string(), + bid.headers.get("x-example"), + &bid.body + ), + ( + &Method::POST, + uri("/bid"), + Some(&HeaderValue::from_static("one")), + &Bytes::from_static(b"{\"id\":1}") + ), + "should record the method, URI, headers, and body" + ); + assert_eq!( + bid.max_request_body_bytes, + OutboundRequest::post(uri("/bid")) + .expect("should build a default POST request") + .into_parts() + .max_request_body_bytes, + "should record the default upload cap" + ); + assert_eq!( + ( + bid.timeout, + bid.connect_timeout, + bid.deadline.map(|recorded| recorded.instant()) + ), + ( + Some(Duration::from_secs(2)), + Some(Duration::from_secs(1)), + Some(deadline.instant()) + ), + "should record the timeouts and deadline" + ); + assert_eq!( + ( + bid.cache_policy, + bid.response_decoding, + bid.max_decoded_response_bytes, + bid.max_encoded_response_bytes + ), + ( + OutboundCachePolicy::Bypass, + ResponseDecoding::Original, + Some(4096), + Some(2048) + ), + "should record the cache policy, decoding mode, and response caps" + ); + assert!( + matches!(bid.response_mode, ResponseMode::Buffered { .. }), + "should record a buffered response mode" + ); + assert_eq!( + ( + bid.proxy_marker, + &bid.image_optimizer, + bid.host_authority.as_str(), + bid.has_host_authority_override + ), + (OutboundProxyMarker::Off, &None, "origin.example.com", true), + "should record the resolved marker, no image options, and the authority override" + ); + let image = &recorded[1]; + assert_eq!( + ( + &image.method, + image.response_mode, + &image.image_optimizer, + image.host_authority.as_str(), + image.has_host_authority_override + ), + ( + &Method::GET, + ResponseMode::Streamed, + &Some(options), + "example.com", + false + ), + "should record the image mode, its Image Optimizer options, and the URI authority" + ); + } + + #[test] + fn replies_in_fifo_order_per_method_and_uri() { + let clock = TestClock::new(); + let client = ScriptedHttpClient::new(&clock); + client + .respond_to( + Method::GET, + &uri("/a"), + ScriptedReply::status(StatusCode::OK).body("first"), + ) + .respond_to( + Method::GET, + &uri("/a"), + ScriptedReply::status(StatusCode::CREATED).body("second"), + ) + .respond_to( + Method::POST, + &uri("/a"), + ScriptedReply::status(StatusCode::ACCEPTED), + ) + .respond_to( + Method::GET, + &uri("/b"), + ScriptedReply::status(StatusCode::OK).header("x-reply", "b"), + ); + let http = client.http_client(); + let post = OutboundRequest::post(uri("/a")).expect("should build the POST request"); + + let first = block_on(http.send(get("/a"))).expect("should answer the first GET"); + let posted = block_on(http.send(post)).expect("should answer the POST"); + let other = block_on(http.send(get("/b"))).expect("should answer the other URI"); + let second = block_on(http.send(get("/a"))).expect("should answer the second GET"); + let unscripted = + block_on(http.send(get("/a"))).expect_err("should fail once the script runs out"); + + assert_eq!( + first.status(), + StatusCode::OK, + "should reply first in first out" + ); + assert_eq!(body_of(&first), b"first", "should send the first body"); + assert_eq!( + posted.status(), + StatusCode::ACCEPTED, + "should keep a separate queue per method" + ); + assert_eq!( + other.headers().get("x-reply").map(HeaderValue::as_bytes), + Some(&b"b"[..]), + "should keep a separate queue per URI and send scripted headers" + ); + assert_eq!( + second.status(), + StatusCode::CREATED, + "should send the second reply next" + ); + assert_eq!(body_of(&second), b"second", "should send the second body"); + assert!( + matches!(unscripted, EdgeError::Internal { .. }), + "should fail an unscripted request as internal, got {unscripted:?}" + ); + assert!( + unscripted + .inner() + .is_some_and(|inner| inner.to_string().starts_with("unscripted request")), + "should name the failure, got {unscripted}" + ); + let recorded = client.requests(); + assert_eq!( + recorded.len(), + 5, + "should record every request, unscripted too" + ); + assert_eq!( + recorded[4].uri.to_string(), + uri("/a"), + "should record the unscripted request" + ); + } + + #[test] + fn chunked_reply_streams_each_chunk() { + let clock = TestClock::new(); + let client = ScriptedHttpClient::new(&clock); + let chunks = vec![ + Bytes::from_static(b"one"), + Bytes::from_static(b"two"), + Bytes::from_static(b"three"), + ]; + for _ in 0..2 { + client.respond_to( + Method::GET, + &uri("/stream"), + ScriptedReply::status(StatusCode::OK).chunks(chunks.clone()), + ); + } + let http = client.http_client(); + + let streamed = block_on(http.send(get("/stream").stream_response())) + .expect("should answer the streamed request"); + let Ok(stream) = streamed.into_body().into_stream() else { + panic!("should stream the response body"); + }; + let received: Vec = block_on( + stream + .map(|chunk| chunk.expect("should read each chunk")) + .collect(), + ); + let buffered = + block_on(http.send(get("/stream"))).expect("should answer the buffered request"); + + assert_eq!( + received, chunks, + "should deliver each scripted chunk as its own stream item" + ); + assert_eq!( + body_of(&buffered), + b"onetwothree", + "should join the chunks for a buffered response" + ); + } + + #[test] + fn held_reply_resolves_only_after_release() { + let clock = TestClock::new(); + let client = ScriptedHttpClient::new(&clock); + let (begun_reply, begun_release) = + ScriptedReply::status(StatusCode::OK).body("late").held(); + let (sent_reply, sent_release) = ScriptedReply::status(StatusCode::OK).held(); + client + .respond_to(Method::GET, &uri("/begun"), begun_reply) + .respond_to(Method::GET, &uri("/sent"), sent_reply); + let http = client.http_client(); + let wakes = Arc::new(WakeCount(AtomicUsize::new(0))); + let waker = Waker::from(Arc::clone(&wakes)); + + let mut pending = block_on(http.begin(get("/begun"))).expect("should begin the request"); + let mut sent = http.send(get("/sent")); + clock.advance(Duration::from_millis(10)); + let begun_while_held = pending.is_ready(); + let sent_while_held = poll_once(&mut sent, &waker); + let wakes_while_held = wakes.0.load(Ordering::SeqCst); + begun_release.release(); + sent_release.release(); + + assert!( + !begun_while_held && sent_while_held.is_pending(), + "should keep held requests pending, even as time passes" + ); + assert!( + wakes.0.load(Ordering::SeqCst) > wakes_while_held, + "should wake the waiting send when released" + ); + assert!( + pending.is_ready(), + "should resolve the begun request once released" + ); + let begun = block_on(pending.collect()).expect("should collect the released reply"); + assert_eq!(body_of(&begun), b"late", "should deliver the held body"); + assert!( + matches!(poll_once(&mut sent, &waker), Poll::Ready(Ok(_))), + "should resolve the send once released" + ); + } + + #[test] + fn batch_drops_reply_arriving_after_cutoff() { + let clock = TestClock::new(); + let client = ScriptedHttpClient::new(&clock); + client + .respond_to( + Method::GET, + &uri("/fast"), + ScriptedReply::status(StatusCode::OK).after(Duration::from_millis(100)), + ) + .respond_to( + Method::GET, + &uri("/slow"), + ScriptedReply::status(StatusCode::OK).after(Duration::from_millis(600)), + ); + let cutoff = clock + .monotonic_clock() + .deadline_after(Duration::from_millis(500)); + + let mut batch = client + .http_client() + .start_batch_until(vec![get("/fast"), get("/slow")], cutoff); + let before = batch.check().expect("should check the batch"); + clock.advance(Duration::from_millis(600)); + let (slots, termination) = block_on(batch.collect()) + .expect("should collect the batch") + .into_parts(); + + assert!( + matches!(before, OutboundBatchCheck::Pending), + "should observe nothing before a reply arrives, got {before:?}" + ); + assert_eq!( + termination, + OutboundBatchTermination::Cutoff, + "should end the batch at the cutoff" + ); + let [ + OutboundCollectedSlot::Resolved(fast), + OutboundCollectedSlot::Unresolved, + ] = &slots[..] + else { + panic!("should resolve the fast slot and leave the slow one unresolved: {slots:?}"); + }; + assert_eq!( + fast.elapsed, + Duration::from_millis(100), + "should time the slot from batch start to its arrival, not its collection" + ); + assert_eq!( + fast.outcome + .as_ref() + .expect("should answer the fast slot") + .status(), + StatusCode::OK, + "should deliver the fast reply" + ); + } + + #[test] + fn error_reply_fails_only_its_slot() { + let clock = TestClock::new(); + let client = ScriptedHttpClient::new(&clock); + client + .respond_to( + Method::GET, + &uri("/ok"), + ScriptedReply::status(StatusCode::OK).body("bid"), + ) + .respond_to( + Method::GET, + &uri("/broken"), + ScriptedReply::error(EdgeError::bad_gateway("scripted bidder failure")), + ); + let cutoff = clock + .monotonic_clock() + .deadline_after(Duration::from_secs(1)); + + let (slots, termination) = block_on( + client + .http_client() + .send_all_until(vec![get("/ok"), get("/broken")], cutoff), + ) + .expect("should collect the batch") + .into_parts(); + + assert_eq!( + termination, + OutboundBatchTermination::Completed, + "should complete once every slot resolves" + ); + let [ + OutboundCollectedSlot::Resolved(ok), + OutboundCollectedSlot::Resolved(broken), + ] = &slots[..] + else { + panic!("should resolve both slots: {slots:?}"); + }; + assert_eq!( + body_of(ok.outcome.as_ref().expect("should answer the healthy slot")), + b"bid", + "should deliver the healthy slot's body" + ); + let error = broken + .outcome + .as_ref() + .expect_err("should fail the broken slot"); + assert!( + matches!(error, EdgeError::BadGateway { .. }), + "should keep the scripted error, got {error:?}" + ); + assert_eq!( + error.message(), + "scripted bidder failure", + "should keep the scripted error message" + ); + } + + #[test] + fn forbidding_client_fails_and_records_every_request() { + let client = ScriptedHttpClient::forbidding(); + client.respond_to( + Method::GET, + &uri("/forbidden"), + ScriptedReply::status(StatusCode::OK), + ); + + let error = block_on(client.http_client().send(get("/forbidden"))) + .expect_err("should fail every request"); + + assert!( + matches!(error, EdgeError::Internal { .. }), + "should fail as internal, got {error:?}" + ); + assert!( + error.inner().is_some_and(|inner| inner + .to_string() + .starts_with("should not send GET https://example.com/forbidden")), + "should name the forbidden request, got {error}" + ); + let recorded = client.requests(); + assert_eq!(recorded.len(), 1, "should record the forbidden request"); + assert_eq!(recorded[0].method, Method::GET, "should record its method"); + assert_eq!( + recorded[0].uri.to_string(), + uri("/forbidden"), + "should record its URI" + ); + } + + #[test] + fn header_mismatch_names_headers_without_their_values() { + let client = ScriptedHttpClient::forbidding(); + let request = get("/headers?example=query-value") + .header("authorization", "Bearer sent-value") + .expect("should add the authorization header") + .header("x-extra", "extra-value") + .expect("should add the extra header"); + block_on(client.http_client().send(request)).expect_err("should fail every request"); + let recorded = client.requests(); + + let mismatch = recorded[0].header_mismatch(&[ + ("Authorization", "Bearer expected-value"), + ("accept", "application/json"), + ]); + + assert_eq!( + mismatch, + HeaderMismatch { + missing: vec!["accept".to_owned()], + unexpected: vec!["x-extra".to_owned()], + differing: vec!["authorization".to_owned()], + }, + "should name each wrong header" + ); + let rendered = format!("{mismatch:?}"); + for fragment in [ + "sent-value", + "expected-value", + "extra-value", + "application/json", + "query-value", + "example.com", + ] { + assert!( + !rendered.contains(fragment), + "should leave `{fragment}` out of the failure output" + ); + } + } + + #[test] + fn header_mismatch_is_empty_for_the_same_headers_in_any_order_and_case() { + let client = ScriptedHttpClient::forbidding(); + let request = get("/headers") + .header("x-example", "one") + .expect("should add the first value") + .header("accept", "application/json") + .expect("should add the accept header") + .header("x-example", "two") + .expect("should add the second value"); + block_on(client.http_client().send(request)).expect_err("should fail every request"); + + client.requests()[0].assert_headers(&[ + ("Accept", "application/json"), + ("x-example", "two"), + ("X-Example", "one"), + ]); + } + + #[test] + fn forbidding_client_records_an_upload_that_fails_before_it_is_collected() { + let client = ScriptedHttpClient::forbidding(); + let upload = Body::from_stream(stream::iter([ + Ok(Bytes::from_static(b"partial")), + Err(EdgeError::bad_request("example upload source failed")), + ])); + + let error = block_on( + client.http_client().send( + OutboundRequest::post(uri("/upload")) + .expect("should build the POST request") + .body(upload), + ), + ) + .expect_err("should fail the upload"); + + assert!( + matches!(error, EdgeError::BadRequest { .. }), + "should fail with the upload source error, got {error:?}" + ); + let recorded = client.requests(); + assert_eq!( + recorded + .iter() + .map(|request| (request.method.clone(), request.uri.to_string())) + .collect::>(), + vec![(Method::POST, uri("/upload"))], + "should record the request whose upload failed" + ); + assert!( + recorded[0].body.is_empty(), + "should leave the failed upload's body empty" + ); + } + + #[test] + fn streamed_upload_over_its_cap_is_recorded_with_the_cap_and_fails_413() { + let clock = TestClock::new(); + let client = ScriptedHttpClient::new(&clock); + let upload = Body::from_stream(stream::iter([ + Ok(Bytes::from_static(b"1234")), + Ok(Bytes::from_static(b"5")), + ])); + + let error = block_on( + client.http_client().send( + OutboundRequest::post(uri("/upload")) + .expect("should build the POST request") + .max_request_body_bytes(4) + .body(upload), + ), + ) + .expect_err("should fail the upload over its cap"); + + assert_eq!( + error.status(), + StatusCode::PAYLOAD_TOO_LARGE, + "should fail an upload past its cap with 413, got {error:?}" + ); + let recorded = client.requests(); + assert_eq!(recorded.len(), 1, "should record the over-cap upload"); + assert_eq!( + recorded[0].max_request_body_bytes, 4, + "should record the upload cap" + ); + } + + #[test] + fn streamed_upload_that_stalls_past_its_deadline_times_out() { + let clock = TestClock::new(); + let client = ScriptedHttpClient::new(&clock); + let upload = Body::from_stream(stream::pending::>()); + + let mut sent = client.http_client().send( + OutboundRequest::post(uri("/upload")) + .expect("should build the POST request") + .timeout(Duration::from_secs(1)) + .body(upload), + ); + let before = poll_once(&mut sent, Waker::noop()); + clock.advance(Duration::from_secs(1)); + let after = poll_once(&mut sent, Waker::noop()); + + assert!(before.is_pending(), "should wait for the upload"); + assert!( + matches!( + after, + Poll::Ready(Err(EdgeError::GatewayTimeout { + cause: BudgetSource::PerCallTimeout, + .. + })) + ), + "should time out the stalled upload at its deadline, got {after:?}" + ); + assert_eq!( + client.requests().len(), + 1, + "should record the request before its upload stalls" + ); + } + + #[test] + fn forbidding_client_without_requests_passes_its_check() { + let client = ScriptedHttpClient::forbidding(); + + client.assert_no_requests(); + + assert!(client.requests().is_empty(), "should record nothing"); + } + + #[test] + fn send_times_out_when_its_reply_arrives_after_the_deadline() { + let clock = TestClock::new(); + let client = ScriptedHttpClient::new(&clock); + client.respond_to( + Method::GET, + &uri("/slow"), + ScriptedReply::status(StatusCode::OK).after(Duration::from_secs(2)), + ); + + let mut sent = client + .http_client() + .send(get("/slow").timeout(Duration::from_secs(1))); + let before = poll_once(&mut sent, Waker::noop()); + clock.advance(Duration::from_secs(1)); + let after = poll_once(&mut sent, Waker::noop()); + + assert!(before.is_pending(), "should wait for the reply"); + assert!( + matches!( + after, + Poll::Ready(Err(EdgeError::GatewayTimeout { + cause: BudgetSource::PerCallTimeout, + .. + })) + ), + "should time out at the per-call deadline, got {after:?}" + ); + } + + #[test] + fn test_clock_wakes_timers_when_advanced() { + let clock = TestClock::new(); + let time = clock.outbound_time(); + let mut wait = time.wait_until(time.clock().deadline_after(Duration::from_millis(100))); + let wakes = Arc::new(WakeCount(AtomicUsize::new(0))); + let waker = Waker::from(Arc::clone(&wakes)); + + let before = poll_once(&mut wait, &waker); + clock.advance(Duration::from_millis(99)); + let woken_early = wakes.0.load(Ordering::SeqCst); + let early = poll_once(&mut wait, &waker); + clock.advance(Duration::from_millis(1)); + let at_deadline = poll_once(&mut wait, &waker); + + assert!(before.is_pending(), "should wait before the deadline"); + assert!(woken_early > 0, "should wake the timer when advanced"); + assert!( + early.is_pending(), + "should keep waiting short of the deadline" + ); + assert!( + matches!(at_deadline, Poll::Ready(Ok(()))), + "should resolve once the clock reaches the deadline, got {at_deadline:?}" + ); + } + + #[test] + fn per_call_timeout_before_cutoff_stays_a_gateway_timeout_in_a_completed_batch() { + let clock = TestClock::new(); + let client = ScriptedHttpClient::new(&clock); + client.respond_to( + Method::GET, + &uri("/slow"), + ScriptedReply::status(StatusCode::OK).after(Duration::from_millis(300)), + ); + let cutoff = clock + .monotonic_clock() + .deadline_after(Duration::from_millis(500)); + + let batch = client.http_client().start_batch_until( + vec![get("/slow").timeout(Duration::from_millis(100))], + cutoff, + ); + let (slots, termination) = clock + .drive(batch.collect()) + .expect("should collect the batch") + .into_parts(); + + assert_eq!( + termination, + OutboundBatchTermination::Completed, + "should complete when the only slot ends on its own timeout before the cutoff" + ); + let [OutboundCollectedSlot::Resolved(slow)] = &slots[..] else { + panic!("should resolve the timed-out slot: {slots:?}"); + }; + assert!( + matches!( + slow.outcome, + Err(EdgeError::GatewayTimeout { + cause: BudgetSource::PerCallTimeout, + .. + }) + ), + "should keep the per-call timeout as the slot's outcome, got {:?}", + slow.outcome + ); + assert_eq!( + slow.elapsed, + Duration::from_millis(100), + "should time the slot to its own deadline" + ); + } + + #[test] + fn check_reports_a_fast_arrival_then_pending_before_the_cutoff() { + let clock = TestClock::new(); + let client = ScriptedHttpClient::new(&clock); + client + .respond_to( + Method::GET, + &uri("/fast"), + ScriptedReply::status(StatusCode::OK).after(Duration::from_millis(100)), + ) + .respond_to( + Method::GET, + &uri("/slow"), + ScriptedReply::status(StatusCode::OK).after(Duration::from_millis(600)), + ); + let cutoff = clock + .monotonic_clock() + .deadline_after(Duration::from_millis(500)); + + let mut batch = client + .http_client() + .start_batch_until(vec![get("/fast"), get("/slow")], cutoff); + clock.advance(Duration::from_millis(100)); + let first = batch.check().expect("should check the batch"); + let second = batch.check().expect("should check the batch again"); + + let OutboundBatchCheck::Item(item) = first else { + panic!("should report the fast arrival first, got {first:?}"); + }; + assert_eq!(item.index(), 0, "should report the fast slot"); + assert!( + matches!(second, OutboundBatchCheck::Pending), + "should report pending while the slow slot is still in flight, got {second:?}" + ); + } + + #[test] + fn held_slot_released_before_the_cutoff_resolves() { + let clock = TestClock::new(); + let client = ScriptedHttpClient::new(&clock); + let (reply, release) = ScriptedReply::status(StatusCode::OK).body("bid").held(); + client.respond_to(Method::GET, &uri("/held"), reply); + let cutoff = clock + .monotonic_clock() + .deadline_after(Duration::from_millis(500)); + + let mut batch = client + .http_client() + .start_batch_until(vec![get("/held")], cutoff); + clock.advance(Duration::from_millis(200)); + let while_held = batch.check().expect("should check the held batch"); + release.release(); + let next = clock + .drive(batch.next()) + .expect("should observe the released slot"); + let finished = clock.drive(batch.next()).expect("should finish the batch"); + + assert!( + matches!(while_held, OutboundBatchCheck::Pending), + "should keep the held slot pending, got {while_held:?}" + ); + let OutboundBatchNext::Item(item) = next else { + panic!("should resolve the released slot, got {next:?}"); + }; + let (index, result) = item.into_parts(); + assert_eq!( + (index, result.elapsed), + (0, Duration::from_millis(200)), + "should time the released slot to its release" + ); + assert_eq!( + body_of(&result.outcome.expect("should answer the released slot")), + b"bid", + "should deliver the released body" + ); + assert!( + matches!( + finished, + OutboundBatchNext::Finished(OutboundBatchTermination::Completed) + ), + "should complete once the released slot resolves, got {finished:?}" + ); + } + + #[test] + fn held_reply_is_no_longer_awaited_once_its_batch_is_cancelled() { + let clock = TestClock::new(); + let client = ScriptedHttpClient::new(&clock); + let (reply, release) = ScriptedReply::status(StatusCode::OK).held(); + client.respond_to(Method::GET, &uri("/held"), reply); + let cutoff = clock + .monotonic_clock() + .deadline_after(Duration::from_secs(1)); + + let batch = client + .http_client() + .start_batch_until(vec![get("/held")], cutoff); + let awaited_in_flight = release.is_awaited(); + let undelivered = batch.cancel(); + + assert!( + awaited_in_flight, + "should await the held reply while its slot is in flight" + ); + assert_eq!(undelivered, vec![0], "should report the undelivered slot"); + assert!( + !release.is_awaited(), + "should drop the exchange that awaited the held reply" + ); + } + + #[test] + fn batch_requests_record_batch_mode_and_cutoff_capped_budget() { + let clock = TestClock::new(); + let client = ScriptedHttpClient::new(&clock); + client.respond_to( + Method::GET, + &uri("/bid"), + ScriptedReply::status(StatusCode::OK), + ); + let cutoff = clock + .monotonic_clock() + .deadline_after(Duration::from_millis(500)); + + let batch = client + .http_client() + .start_batch_until(vec![get("/bid").timeout(Duration::from_secs(2))], cutoff); + drop( + clock + .drive(batch.collect()) + .expect("should collect the batch"), + ); + + let recorded = client.requests(); + assert_eq!(recorded.len(), 1, "should record the batch request"); + assert_eq!( + ( + recorded[0].mode, + recorded[0].budget.cause, + recorded[0].budget.deadline.instant() + ), + ( + OutboundSendMode::Batch, + BudgetSource::BatchCutoff, + cutoff.instant() + ), + "should record the batch mode and the cutoff that caps the longer timeout" + ); + } + + #[test] + fn request_observer_sees_each_request_as_it_is_recorded() { + let clock = TestClock::new(); + let seen = Arc::new(Mutex::new(Vec::new())); + let client = { + let seen = Arc::clone(&seen); + ScriptedHttpClient::new(&clock).with_request_observer(move |request| { + lock(&seen).push(request.uri.to_string()); + }) + }; + client.respond_to( + Method::GET, + &uri("/observed"), + ScriptedReply::status(StatusCode::OK), + ); + + block_on(client.http_client().send(get("/observed"))) + .expect("should answer the observed request"); + + assert_eq!( + *lock(&seen), + vec![uri("/observed")], + "should call the observer once with the recorded request" + ); + } + + #[test] + fn drive_advances_the_clock_only_as_far_as_the_future_waits() { + let clock = TestClock::new(); + let client = ScriptedHttpClient::new(&clock); + client.respond_to( + Method::GET, + &uri("/delayed"), + ScriptedReply::status(StatusCode::OK).after(Duration::from_secs(2)), + ); + let started = clock.now(); + + let response = clock + .drive( + client + .http_client() + .send(get("/delayed").timeout(Duration::from_secs(5))), + ) + .expect("should answer once the reply arrives"); + + assert_eq!( + response.status(), + StatusCode::OK, + "should deliver the reply" + ); + assert_eq!( + clock.now().saturating_duration_since(started), + Duration::from_secs(2), + "should advance exactly to the reply's arrival" + ); + } + + #[test] + fn process_clock_deadline_is_already_expired_on_the_test_clock() { + let clock = TestClock::new(); + let test_now = clock.monotonic_clock().now(); + + let process_deadline = MonotonicClock::default().deadline_after(Duration::from_secs(60)); + let test_deadline = clock + .monotonic_clock() + .deadline_after(Duration::from_secs(60)); + + assert!( + process_deadline.is_expired_at(test_now), + "should expire a process-clock deadline on the test clock, so mixed clocks fail tests" + ); + assert!( + !test_deadline.is_expired_at(test_now), + "should keep a test-clock deadline open on the test clock" + ); + } +} diff --git a/crates/trusted-server-core/src/trace/actions.rs b/crates/trusted-server-core/src/trace/actions.rs index 2343e6def1..ab89b7a384 100644 --- a/crates/trusted-server-core/src/trace/actions.rs +++ b/crates/trusted-server-core/src/trace/actions.rs @@ -1,9 +1,11 @@ //! Deliberate, authenticated trace cookie mutations. use edgezero_core::body::Body as EdgeBody; +use edgezero_core::error::EdgeError; +use edgezero_core::http::RequestParts; use edgezero_core::ingress::RequestIngress; -use futures::StreamExt as _; -use http::{HeaderMap, HeaderName, HeaderValue, Request, Response, StatusCode, header}; +use edgezero_core::pre_dispatch::PreDispatchRequest; +use http::{HeaderMap, HeaderName, HeaderValue, Response, StatusCode, header}; use url::Url; use crate::forwarder::{PublicOrigin, public_origin}; @@ -21,11 +23,12 @@ const FETCH_SITE_HEADER: HeaderName = HeaderName::from_static("sec-fetch-site"); /// /// # Performance /// -/// Rejected controls and framing never poll the body. Streaming validation -/// skips empty chunks and stops on the first nonempty chunk or read error; -/// it does not allocate a body buffer or introduce a transport deadline. +/// Rejected controls and framing never poll the body. The body is read with a +/// zero-byte limit, which skips empty chunks and stops on the first nonempty +/// chunk or read error. It keeps no body bytes and adds no deadline beyond the +/// admitted inbound read deadline. pub(crate) async fn action_response( - request: &mut Request, + request: &PreDispatchRequest, route: TraceRoute, ) -> Response { let (action, cookie_action) = match route { @@ -38,16 +41,15 @@ pub(crate) async fn action_response( != Some(action) || single_header(request.headers(), FETCH_SITE_HEADER).map(HeaderValue::as_bytes) != Some(b"same-origin".as_slice()) - || !matches_trusted_origin(request) + || !matches_trusted_origin(request.parts()) { return failure(StatusCode::FORBIDDEN); } if !accepts_empty_framing(request.headers()) { return failure(StatusCode::PAYLOAD_TOO_LARGE); } - let body = std::mem::replace(request.body_mut(), EdgeBody::empty()); - if let Some(status) = empty_body_rejection(body).await { - return failure(status); + if let Err(error) = request.body_bytes(0).await { + return failure(body_rejection_status(&error)); } let Some(cookie) = cookie_action.set_cookie_header() else { return failure(StatusCode::INTERNAL_SERVER_ERROR); @@ -61,7 +63,9 @@ fn failure(status: StatusCode) -> Response { let body: &'static [u8] = match status { StatusCode::FORBIDDEN => br#"{"error":"trace action rejected"}"#, StatusCode::PAYLOAD_TOO_LARGE => br#"{"error":"trace body must be empty"}"#, - StatusCode::BAD_REQUEST => br#"{"error":"trace body unavailable"}"#, + StatusCode::BAD_REQUEST | StatusCode::REQUEST_TIMEOUT => { + br#"{"error":"trace body unavailable"}"# + } _ => br#"{"error":"trace action unavailable"}"#, }; let mut response = Response::new(EdgeBody::from(body)); @@ -75,9 +79,9 @@ fn single_header(headers: &HeaderMap, name: HeaderName) -> Option<&HeaderValue> fields.next().is_none().then_some(value) } -fn matches_trusted_origin(request: &Request) -> bool { - let Some(trusted) = request - .extensions() +fn matches_trusted_origin(parts: &RequestParts) -> bool { + let Some(trusted) = parts + .extensions .get::() .and_then(RequestIngress::origin) else { @@ -86,31 +90,31 @@ fn matches_trusted_origin(request: &Request) -> bool { let Some(expected) = canonical_authority_origin(trusted.scheme(), trusted.authority()) else { return false; }; - let Some(origin) = single_header(request.headers(), header::ORIGIN) + let Some(origin) = single_header(&parts.headers, header::ORIGIN) .and_then(|origin| core::str::from_utf8(origin.as_bytes()).ok()) .and_then(canonical_origin) else { return false; }; let public_expected = - public_origin(request).map_or_else(|| expected.clone(), PublicOrigin::serialized); + public_origin(&parts.extensions).map_or_else(|| expected.clone(), PublicOrigin::serialized); // Public origin authorizes the browser. The received URI and Host must // still agree with immutable transport ingress below. if origin != public_expected - || request - .uri() + || parts + .uri .scheme_str() .is_some_and(|scheme| !scheme.eq_ignore_ascii_case(trusted.scheme())) { return false; } - let authority = request.uri().authority(); + let authority = parts.uri.authority(); if authority.is_some_and(|authority| { canonical_authority_origin(trusted.scheme(), authority.as_str()).as_ref() != Some(&expected) }) { return false; } - match single_header(request.headers(), header::HOST) { + match single_header(&parts.headers, header::HOST) { Some(host) => { core::str::from_utf8(host.as_bytes()) .ok() @@ -118,8 +122,8 @@ fn matches_trusted_origin(request: &Request) -> bool { .as_ref() == Some(&expected) } - None if request.headers().contains_key(header::HOST) => false, - None => request.uri().scheme().is_some() && authority.is_some(), + None if parts.headers.contains_key(header::HOST) => false, + None => parts.uri.scheme().is_some() && authority.is_some(), } } @@ -183,20 +187,17 @@ fn accepts_empty_framing(headers: &HeaderMap) -> bool { && length.as_bytes().iter().all(|byte| *byte == b'0') } -async fn empty_body_rejection(body: EdgeBody) -> Option { - match body { - EdgeBody::Once(bytes) if bytes.is_empty() => None, - EdgeBody::Once(_) => Some(StatusCode::PAYLOAD_TOO_LARGE), - EdgeBody::Stream(mut stream) => { - while let Some(chunk) = stream.next().await { - match chunk { - Ok(bytes) if bytes.is_empty() => {} - Ok(_) => return Some(StatusCode::PAYLOAD_TOO_LARGE), - Err(_) => return Some(StatusCode::BAD_REQUEST), - } - } - None - } +/// Map a failed zero-byte body read to its bounded trace error status. +/// +/// Any body byte is 413, and an expired read is 408. On canonical dispatch an +/// expired inbound read deadline is a fixed input outcome, so `EdgeZero` +/// replaces this response with its own 408 after the hook returns. Every other +/// read failure keeps the existing 400 for an unreadable body. +fn body_rejection_status(error: &EdgeError) -> StatusCode { + match error.status() { + StatusCode::PAYLOAD_TOO_LARGE => StatusCode::PAYLOAD_TOO_LARGE, + StatusCode::REQUEST_TIMEOUT => StatusCode::REQUEST_TIMEOUT, + _ => StatusCode::BAD_REQUEST, } } @@ -205,11 +206,13 @@ mod tests { use std::cell::Cell; use std::io; use std::rc::Rc; + use std::sync::Arc; use std::task::Poll; use base64::{Engine as _, engine::general_purpose::STANDARD}; use bytes::Bytes; use edgezero_core::body::Body as EdgeBody; + use edgezero_core::error::EdgeError; use edgezero_core::ingress::{ CapturedTarget, HeaderFidelity, InboundOrigin, OriginSource, RequestIngress, TargetUnavailable, @@ -219,8 +222,10 @@ mod tests { use serde_json::{Value, json}; use crate::settings::{Handler, Settings}; - use crate::test_support::tests::create_test_settings; - use crate::trace::{TracePreflight, TraceTerminalResponse, inspect_cookies, preflight}; + use crate::test_support::tests::{ + create_test_settings, run_pre_dispatch_hook, with_request_parts, + }; + use crate::trace::{TracePreDispatchHook, TraceTerminalResponse, inspect_cookies}; fn settings(enabled: bool, auth: bool) -> Settings { let mut settings = create_test_settings(); @@ -265,15 +270,13 @@ mod tests { request } - fn respond(settings: &Settings, mut request: Request) -> Response { - match preflight(settings, &mut request) { - TracePreflight::Response(response) => response, - TracePreflight::Ready(context) => { - let response = block_on(super::super::action_response(&mut request, context.route)); - context.respond(response) - } - TracePreflight::NotTrace => panic!("should preflight every exact action route locally"), - } + fn respond(settings: &Settings, request: Request) -> Response { + let hook = TracePreDispatchHook::new( + Arc::new(settings.clone()), + Arc::new(|_| panic!("should not load setup metadata for an action")), + ); + block_on(run_pre_dispatch_hook(hook, request)) + .expect_err("should answer every exact action route locally") } fn private_error(response: Response, expected: StatusCode) { @@ -421,7 +424,9 @@ mod tests { request.headers(), request.extensions().get::(), ); - crate::forwarder::prepare_trusted_forwarder(&mut request, &settings); + with_request_parts(&mut request, |parts| { + crate::forwarder::prepare_trusted_forwarder(parts, &settings); + }); assert_eq!( inspect_cookies( request.headers(), @@ -452,7 +457,9 @@ mod tests { HeaderName::from_bytes(name.as_bytes()).expect("should validate field name"), HeaderValue::from_static(value), ); - crate::forwarder::prepare_trusted_forwarder(&mut request, &settings); + with_request_parts(&mut request, |parts| { + crate::forwarder::prepare_trusted_forwarder(parts, &settings); + }); private_error(respond(&settings, request), StatusCode::FORBIDDEN); } for target in [ @@ -464,7 +471,9 @@ mod tests { *request.uri_mut() = target .parse() .expect("should construct conflicting transport target"); - crate::forwarder::prepare_trusted_forwarder(&mut request, &settings); + with_request_parts(&mut request, |parts| { + crate::forwarder::prepare_trusted_forwarder(parts, &settings); + }); private_error(respond(&settings, request), StatusCode::FORBIDDEN); } } @@ -937,7 +946,7 @@ mod tests { let polls = Rc::new(Cell::new(0)); let captured = Rc::clone(&polls); let mut request = request("end"); - *request.body_mut() = EdgeBody::from_stream(stream::poll_fn( + *request.body_mut() = EdgeBody::from_external_stream(stream::poll_fn( move |_| -> Poll>> { captured.set(captured.get() + 1); Poll::Ready(Some(match captured.get() { @@ -958,6 +967,28 @@ mod tests { ); } + #[test] + fn trace_actions_stream_read_timeout_is_bounded_request_timeout_without_later_polling() { + let polls = Rc::new(Cell::new(0)); + let captured = Rc::clone(&polls); + let mut request = request("end"); + *request.body_mut() = EdgeBody::from_stream(stream::poll_fn( + move |_| -> Poll>> { + captured.set(captured.get() + 1); + Poll::Ready(Some(match captured.get() { + 1 => Ok(Bytes::new()), + 2 => Err(EdgeError::request_timeout("fictional-secret-expired-read")), + _ => panic!("should not poll after the read timeout"), + })) + }, + )); + private_error( + respond(&settings(true, false), request), + StatusCode::REQUEST_TIMEOUT, + ); + assert_eq!(polls.get(), 2, "should stop reading at the expired read"); + } + #[test] fn trace_actions_preflight_authentication_precedes_controls_flag_and_body() { for enabled in [false, true] { diff --git a/crates/trusted-server-core/src/trace/dispatch.rs b/crates/trusted-server-core/src/trace/dispatch.rs index 183704ac29..5fc23734f5 100644 --- a/crates/trusted-server-core/src/trace/dispatch.rs +++ b/crates/trusted-server-core/src/trace/dispatch.rs @@ -5,9 +5,11 @@ use async_trait::async_trait; use chrono::{DateTime, Utc}; use edgezero_core::body::Body; use edgezero_core::error::EdgeError; +use edgezero_core::http::RequestParts; use edgezero_core::ingress::RequestIngress; +use edgezero_core::pre_dispatch::PreDispatchRequest; use edgezero_core::router::PreDispatchHook; -use http::{HeaderMap, Method, Request, Response, StatusCode}; +use http::{HeaderMap, Method, Response, StatusCode}; use trusted_server_js::trace_assets::trace_asset; use super::actions::action_response; @@ -87,7 +89,7 @@ pub struct TraceMetadata { pub geo: Option, } -type MetadataSupplier = dyn Fn(&Request) -> TraceMetadata + Send + Sync; +type MetadataSupplier = dyn Fn(&RequestParts) -> TraceMetadata + Send + Sync; /// Intercept the trace namespace before ordinary router lookup and middleware. pub struct TracePreDispatchHook { @@ -117,7 +119,7 @@ impl TracePreDispatchHook { impl PreDispatchHook for TracePreDispatchHook { async fn handle( &self, - request: &mut Request, + request: &mut PreDispatchRequest, ) -> Result>, EdgeError> { self.handle_with_clock(request, Utc::now).await } @@ -126,15 +128,15 @@ impl PreDispatchHook for TracePreDispatchHook { impl TracePreDispatchHook { async fn handle_with_clock( &self, - request: &mut Request, + request: &mut PreDispatchRequest, clock: impl FnOnce() -> DateTime, ) -> Result>, EdgeError> { // Resolve before origin preparation removes headers that may also be // configured as client-IP credentials. Retain only the authenticated IP. let forwarded_client_ip = authenticated_client_ip(request.headers(), self.settings.trusted_client_ip.as_ref()); - prepare_trusted_forwarder(request, &self.settings); - let dispatch = match preflight(&self.settings, request) { + prepare_trusted_forwarder(request.parts_mut(), &self.settings); + let dispatch = match preflight(&self.settings, request.parts_mut()) { TracePreflight::NotTrace => { let enabled = self .settings @@ -187,7 +189,7 @@ impl TracePreDispatchHook { geo: None, } } else { - let mut metadata = (self.metadata)(request); + let mut metadata = (self.metadata)(request.parts()); if let Some(client_ip) = forwarded_client_ip { if metadata.client_info.client_ip != Some(client_ip) { // Runtime geolocation describes the forwarder when @@ -246,6 +248,7 @@ fn unavailable_response() -> Response { #[cfg(test)] mod tests { + use std::sync::Mutex; use std::sync::atomic::{AtomicUsize, Ordering}; use chrono::TimeZone as _; @@ -253,15 +256,60 @@ mod tests { use edgezero_core::middleware::{Middleware, Next}; use edgezero_core::router::RouterService; use futures::executor::block_on; - use http::{HeaderValue, Method, StatusCode, header}; + use http::{HeaderValue, Method, Request, StatusCode, header}; use serde_json::json; use crate::redacted::Redacted; - use crate::test_support::tests::create_test_settings; + use crate::test_support::tests::{create_test_settings, run_pre_dispatch_hook}; use crate::trace::TraceTerminalResponse; use super::*; + /// Runs the trace hook at a fixed clock and records the head it leaves. + /// + /// The router drops the request after a terminal response, so tests that + /// check sanitation on every outcome read the recorded head instead. + struct ObservedHook { + hook: TracePreDispatchHook, + clock: DateTime, + headers: Arc>>, + } + + #[async_trait(?Send)] + impl PreDispatchHook for ObservedHook { + async fn handle( + &self, + request: &mut PreDispatchRequest, + ) -> Result>, EdgeError> { + let clock = self.clock; + let outcome = self.hook.handle_with_clock(request, move || clock).await; + *self.headers.lock().expect("should lock the observed head") = + Some(request.headers().clone()); + outcome + } + } + + fn observed( + hook: TracePreDispatchHook, + clock: DateTime, + ) -> (ObservedHook, Arc>>) { + let headers = Arc::new(Mutex::new(None)); + let observed = ObservedHook { + hook, + clock, + headers: Arc::clone(&headers), + }; + (observed, headers) + } + + fn observed_headers(headers: &Mutex>) -> HeaderMap { + headers + .lock() + .expect("should lock the observed head") + .take() + .expect("should observe the hook request") + } + fn hook(enabled: bool, calls: Arc) -> TracePreDispatchHook { let mut settings = create_test_settings(); settings @@ -351,15 +399,15 @@ mod tests { asn: None, }), }); - let mut request = Request::builder() + let request = Request::builder() .uri("/_ts/trace") .header("x-example-client-ip", "198.51.100.7") .header("x-ts-forwarder-auth", "fictional-client-secret-0123456789") .body(Body::empty()) .expect("should build setup request"); - let response = block_on(hook.handle(&mut request)) - .expect("should dispatch trace") - .expect("should intercept setup"); + let (hook, headers) = observed(hook, Utc::now()); + let response = + block_on(run_pre_dispatch_hook(hook, request)).expect_err("should intercept setup"); let bytes = response .into_body() .into_bytes() @@ -375,7 +423,7 @@ mod tests { "should retain geo only when it describes the authenticated reader" ); assert!( - !request.headers().contains_key("x-ts-forwarder-auth"), + !observed_headers(&headers).contains_key("x-ts-forwarder-auth"), "should still strip forwarding credentials" ); } @@ -384,14 +432,13 @@ mod tests { #[test] fn forwarding_credentials_are_removed_before_every_preflight_outcome() { let calls = Arc::new(AtomicUsize::new(0)); - let hook = hook(false, calls); for path in [ "/_ts/trace/enable", "/_ts/trace/trace.js", "/publisher", "/_ts/trace/missing", ] { - let mut request = Request::builder() + let request = Request::builder() .uri(path) .header( "x-ts-forwarder-auth", @@ -399,10 +446,16 @@ mod tests { ) .body(Body::empty()) .expect("should construct a preflight request"); - let _response = - block_on(hook.handle(&mut request)).expect("should complete local preflight"); + let (hook, headers) = observed(hook(false, Arc::clone(&calls)), Utc::now()); + let outcome = block_on(run_pre_dispatch_hook(hook, request)); + if let Ok(continued) = &outcome { + assert!( + !continued.headers().contains_key("x-ts-forwarder-auth"), + "should route the sanitized request" + ); + } assert!( - !request.headers().contains_key("x-ts-forwarder-auth"), + !observed_headers(&headers).contains_key("x-ts-forwarder-auth"), "should sanitize credentials before any preflight return" ); } @@ -411,7 +464,6 @@ mod tests { #[test] fn trace_shell_head_matches_get_status_with_unrepresentable_clock() { let calls = Arc::new(AtomicUsize::new(0)); - let hook = hook(true, Arc::clone(&calls)); let clock = Utc .with_ymd_and_hms(10_000, 1, 1, 0, 0, 0) .single() @@ -420,14 +472,14 @@ mod tests { for method in [Method::GET, Method::HEAD] { calls.store(0, Ordering::SeqCst); let is_head = method == Method::HEAD; - let mut request = Request::builder() + let request = Request::builder() .method(method) .uri("/_ts/trace") .body(Body::empty()) .expect("should construct a shell request"); - let response = block_on(hook.handle_with_clock(&mut request, || clock)) - .expect("should dispatch the trace shell") - .expect("should intercept the shell"); + let (hook, _headers) = observed(hook(true, Arc::clone(&calls)), clock); + let response = block_on(run_pre_dispatch_hook(hook, request)) + .expect_err("should intercept the shell"); assert_eq!( response.status(), StatusCode::INTERNAL_SERVER_ERROR, @@ -456,7 +508,6 @@ mod tests { #[test] fn trace_dispatch_terminal_routes_only_project_get_shell_metadata() { let calls = Arc::new(AtomicUsize::new(0)); - let hook = hook(true, Arc::clone(&calls)); for (method, path, status, projected) in [ (Method::GET, "/_ts/trace", StatusCode::OK, 1), (Method::HEAD, "/_ts/trace", StatusCode::OK, 0), @@ -476,14 +527,16 @@ mod tests { ] { calls.store(0, Ordering::SeqCst); let is_head = method == Method::HEAD; - let mut request = Request::builder() + let request = Request::builder() .method(method) .uri(path) .body(Body::empty()) .expect("should build trace request"); - let response = block_on(hook.handle(&mut request)) - .expect("should return local policy response") - .expect("should intercept namespace before lookup"); + let response = block_on(run_pre_dispatch_hook( + hook(true, Arc::clone(&calls)), + request, + )) + .expect_err("should intercept namespace before lookup"); assert_eq!( response.status(), status, @@ -523,13 +576,12 @@ mod tests { fn trace_dispatch_disabled_namespace_avoids_metadata() { let calls = Arc::new(AtomicUsize::new(0)); let hook = hook(false, Arc::clone(&calls)); - let mut request = Request::builder() + let request = Request::builder() .uri("/_ts/trace") .body(Body::empty()) .expect("should build disabled request"); - let response = block_on(hook.handle(&mut request)) - .expect("should produce local response") - .expect("should reserve disabled namespace"); + let response = block_on(run_pre_dispatch_hook(hook, request)) + .expect_err("should reserve disabled namespace"); assert_eq!( response.status(), StatusCode::NOT_FOUND, @@ -557,17 +609,13 @@ mod tests { ] { let calls = Arc::new(AtomicUsize::new(0)); let hook = hook(enabled, Arc::clone(&calls)); - let mut request = Request::builder() + let request = Request::builder() .uri("/article") .header(header::COOKIE, cookie) .body(Body::empty()) .expect("should build ordinary request"); - assert!( - block_on(hook.handle(&mut request)) - .expect("should continue ordinary dispatch") - .is_none(), - "should continue ordinary requests" - ); + let mut request = block_on(run_pre_dispatch_hook(hook, request)) + .expect("should continue ordinary requests"); let gate = request.extensions().get::().copied(); assert_eq!( gate.map(|gate| gate.base_active()), @@ -674,6 +722,7 @@ mod tests { ), ] { let hook = hook(true, Arc::new(AtomicUsize::new(0))); + let settings = Arc::clone(&hook.settings); let mut request = Request::builder() .uri(format!("https://publisher.example.com/article{query}")) .header("accept", "text/html") @@ -685,15 +734,14 @@ mod tests { http::HeaderValue::from_str(cookie).expect("should parse example cookie"), ); } - let _ = - block_on(hook.handle(&mut request)).expect("should continue ordinary navigation"); + let mut request = block_on(run_pre_dispatch_hook(hook, request)) + .expect("should continue ordinary navigation"); let gate = *request .extensions() .get::() .expect("should freeze incoming gate"); - let _ = - crate::integrations::gpt_diagnostics::prepare_request(&hook.settings, &mut request) - .expect("should preserve console preparation"); + let _ = crate::integrations::gpt_diagnostics::prepare_request(&settings, &mut request) + .expect("should preserve console preparation"); let decision = request .extensions() .get::() diff --git a/crates/trusted-server-core/src/trace/mod.rs b/crates/trusted-server-core/src/trace/mod.rs index ac3ca75902..a6e9dd3377 100644 --- a/crates/trusted-server-core/src/trace/mod.rs +++ b/crates/trusted-server-core/src/trace/mod.rs @@ -13,8 +13,6 @@ mod routes; mod shell; mod types; -#[cfg(test)] -pub(crate) use actions::action_response; pub use auction::{ DiagnosticAuctionId, TraceAuctionEvidenceV1, TraceAuctionSource, TraceAuctionTerminalReason, TraceAuctionTerminalStatus, TraceAuctionTransportV1, TraceProviderRole, TraceProviderStatus, diff --git a/crates/trusted-server-core/src/trace/routes.rs b/crates/trusted-server-core/src/trace/routes.rs index 147d2755a2..669877fa3f 100644 --- a/crates/trusted-server-core/src/trace/routes.rs +++ b/crates/trusted-server-core/src/trace/routes.rs @@ -1,7 +1,8 @@ use std::borrow::Cow; use edgezero_core::body::Body as EdgeBody; -use http::{HeaderValue, Method, Request, Response, StatusCode, header}; +use edgezero_core::http::RequestParts; +use http::{HeaderValue, Method, Response, StatusCode, header}; use sha2::{Digest as _, Sha256}; use crate::auth::{TraceAuthLogPolicy, enforce_basic_auth}; @@ -232,20 +233,20 @@ pub fn is_trace_path(path: &str) -> bool { /// # Examples /// /// ```ignore -/// match trusted_server_core::trace::preflight(&settings, &mut request) { +/// match trusted_server_core::trace::preflight(&settings, request.parts_mut()) { /// trusted_server_core::trace::TracePreflight::Response(response) => return response, /// trusted_server_core::trace::TracePreflight::NotTrace => continue_dispatch(request), /// trusted_server_core::trace::TracePreflight::Ready(context) => handle_trace(context, request), /// } /// ``` #[must_use] -pub fn preflight(settings: &Settings, request: &mut Request) -> TracePreflight { - let classification = classify_path(request.uri().path()); +pub fn preflight(settings: &Settings, request: &mut RequestParts) -> TracePreflight { + let classification = classify_path(request.uri.path()); if classification == Classification::NotTrace { return TracePreflight::NotTrace; } - let head = request.method() == Method::HEAD; - request.extensions_mut().insert(TraceAuthLogPolicy); + let head = request.method == Method::HEAD; + request.extensions.insert(TraceAuthLogPolicy); match enforce_basic_auth(settings, request) { Ok(Some(mut response)) => { *response.body_mut() = error_response(StatusCode::UNAUTHORIZED).into_body(); @@ -268,14 +269,14 @@ pub fn preflight(settings: &Settings, request: &mut Request) -> TraceP Classification::Rejected(status) => return reject(error_response(status), head), Classification::NotTrace => return TracePreflight::NotTrace, }; - if !route.allows(request.method()) { + if !route.allows(&request.method) { let mut response = error_response(StatusCode::METHOD_NOT_ALLOWED); response .headers_mut() .insert(header::ALLOW, HeaderValue::from_static(route.allow())); return reject(response, head); } - let protected = match settings.handler_for_path(request.uri().path()) { + let protected = match settings.handler_for_path(request.uri.path()) { Ok(handler) => handler.is_some(), Err(_) => return reject(error_response(StatusCode::INTERNAL_SERVER_ERROR), head), }; @@ -404,14 +405,14 @@ mod tests { use super::*; use base64::{Engine as _, engine::general_purpose::STANDARD}; - use http::{HeaderValue, header}; + use http::{HeaderValue, Request, header}; use serde_json::json; use crate::auth::EdgeTerminatedAuthorization; use crate::cache_policy::EDGE_CACHE_HEADER_NAMES; use crate::settings::Handler; use crate::test_support::logging::WarningCapture; - use crate::test_support::tests::create_test_settings; + use crate::test_support::tests::{create_test_settings, with_request_parts}; use crate::trace::TraceTerminalResponse; fn handler(pattern: &str, username: &str, password: &str) -> Handler { @@ -452,14 +453,14 @@ mod tests { } fn rejected(settings: &Settings, request: &mut Request) -> Response { - match preflight(settings, request) { + match with_request_parts(request, |parts| preflight(settings, parts)) { TracePreflight::Response(response) => response, _ => panic!("should terminate the rejected trace request"), } } fn ready(settings: &Settings, request: &mut Request) -> TraceDispatch { - match preflight(settings, request) { + match with_request_parts(request, |parts| preflight(settings, parts)) { TracePreflight::Ready(dispatch) => dispatch, _ => panic!("should dispatch the accepted trace request"), } @@ -808,9 +809,11 @@ mod tests { request.headers()[header::AUTHORIZATION].clone(), ); assert!( - enforce_basic_auth(&settings(true, Some("^/")), &mut ordinary) - .expect("should evaluate ordinary auth") - .is_some(), + with_request_parts(&mut ordinary, |parts| { + enforce_basic_auth(&settings(true, Some("^/")), parts) + }) + .expect("should evaluate ordinary auth") + .is_some(), "should retain ordinary credential rejection" ); assert!( @@ -888,7 +891,10 @@ mod tests { let before_uri = request.uri().clone(); let before_headers = request.headers().clone(); assert!( - matches!(preflight(&settings, &mut request), TracePreflight::NotTrace), + matches!( + with_request_parts(&mut request, |parts| preflight(&settings, parts)), + TracePreflight::NotTrace + ), "should continue ordinary dispatch without parsing trace configuration" ); assert_eq!(request.uri(), &before_uri, "should retain the ordinary URI"); diff --git a/crates/trusted-server-integration-tests/Cargo.toml b/crates/trusted-server-integration-tests/Cargo.toml index 070e60a7b9..e712e839a2 100644 --- a/crates/trusted-server-integration-tests/Cargo.toml +++ b/crates/trusted-server-integration-tests/Cargo.toml @@ -41,23 +41,24 @@ axum = { workspace = true } base64 = { workspace = true } bytes = { workspace = true } derive_more = { workspace = true } -edgezero-adapter-axum = { workspace = true, features = ["axum"] } +edgezero-adapter-axum = { workspace = true, features = ["axum", "test-utils"] } +edgezero-adapter-cloudflare = { workspace = true } +edgezero-adapter-spin = { workspace = true } env_logger = { workspace = true } error-stack = { workspace = true } futures = { workspace = true } http = { workspace = true } -http-body-util = { workspace = true } libc = { workspace = true } log = { workspace = true } +rcgen = { workspace = true } reqwest = { workspace = true, features = ["blocking", "cookies"] } +rustls = { workspace = true, features = ["ring"] } scraper = { workspace = true } testcontainers = { workspace = true } temp-env = { workspace = true } tempfile = { workspace = true } tokio = { workspace = true, features = ["net", "rt-multi-thread", "signal"] } -tower = { workspace = true, features = ["util"] } trusted-server-adapter-axum = { path = "../trusted-server-adapter-axum" } trusted-server-adapter-cloudflare = { path = "../trusted-server-adapter-cloudflare" } trusted-server-adapter-spin = { path = "../trusted-server-adapter-spin" } trusted-server-js = { workspace = true } -urlencoding = { workspace = true } diff --git a/crates/trusted-server-integration-tests/README.md b/crates/trusted-server-integration-tests/README.md index 2d3981721e..0a5ba222e3 100644 --- a/crates/trusted-server-integration-tests/README.md +++ b/crates/trusted-server-integration-tests/README.md @@ -5,11 +5,14 @@ end-to-end publisher behavior. It is not linked into an adapter artifact. ## Test surfaces -- `tests/parity.rs` calls Axum, Cloudflare, and Spin routers in process and - compares their shared route behavior. +- `tests/parity.rs` serves the Axum, Cloudflare, and Spin apps on EdgeZero's + `AxumTestServer`, a loopback listener that applies each app's production + policies, and compares their shared route behavior. A scripted outbound + client in router state answers their upstream calls. - `tests/documentation_snippets.rs` extracts the checked integration-guide fixture and compiles it in an isolated offline crate. -- `tests/integration.rs` exercises the Fastly/Viceroy and Axum paths against the +- `tests/integration.rs` exercises the Fastly (Viceroy), Axum, Cloudflare + (workerd), and Spin runtimes against local origins, loopback bidders, and the WordPress and Next.js fixture containers. - `browser/` uses Playwright and Chromium to verify script loading, navigation, rewriting, APS rendering, and GPT diagnostics in real pages. diff --git a/crates/trusted-server-integration-tests/browser/tests/nextjs/mobile-trace-live.spec.ts b/crates/trusted-server-integration-tests/browser/tests/nextjs/mobile-trace-live.spec.ts index c6ef1bb6a7..0cb64724d2 100644 --- a/crates/trusted-server-integration-tests/browser/tests/nextjs/mobile-trace-live.spec.ts +++ b/crates/trusted-server-integration-tests/browser/tests/nextjs/mobile-trace-live.spec.ts @@ -385,6 +385,7 @@ for (const mode of ['selected', 'empty'] as const) { 'example-creative', 'Trace fixture creative', 'bidder.example.com', + '/api/trace-bidder', ]) expect(publicEvidence).not.toContain(forbidden) if (mode === 'selected') { diff --git a/crates/trusted-server-integration-tests/fixtures/configs/trusted-server.integration.toml b/crates/trusted-server-integration-tests/fixtures/configs/trusted-server.integration.toml index c3d9ff53d8..240b694dcd 100644 --- a/crates/trusted-server-integration-tests/fixtures/configs/trusted-server.integration.toml +++ b/crates/trusted-server-integration-tests/fixtures/configs/trusted-server.integration.toml @@ -93,9 +93,6 @@ rewrite_script = true [integrations.gpt_diagnostics] enabled = true -[proxy] -certificate_check = false - [auction] enabled = false timeout_ms = 2000 diff --git a/crates/trusted-server-integration-tests/fixtures/configs/trusted-server.trace-auth.toml b/crates/trusted-server-integration-tests/fixtures/configs/trusted-server.trace-auth.toml index a94f7c515b..573b831794 100644 --- a/crates/trusted-server-integration-tests/fixtures/configs/trusted-server.trace-auth.toml +++ b/crates/trusted-server-integration-tests/fixtures/configs/trusted-server.trace-auth.toml @@ -33,8 +33,5 @@ secret_store_id = "secrets" enabled = true trace_page_enabled = true -[proxy] -certificate_check = false - [auction] enabled = false diff --git a/crates/trusted-server-integration-tests/fixtures/configs/trusted-server.trace.toml b/crates/trusted-server-integration-tests/fixtures/configs/trusted-server.trace.toml index dcc6b9b1c8..27f494007d 100644 --- a/crates/trusted-server-integration-tests/fixtures/configs/trusted-server.trace.toml +++ b/crates/trusted-server-integration-tests/fixtures/configs/trusted-server.trace.toml @@ -34,9 +34,6 @@ gam_attribution_enabled = false script_url = "https://ads.example.com/gpt.js" rewrite_script = false -[proxy] -certificate_check = false - [auction] enabled = true timeout_ms = 1000 diff --git a/crates/trusted-server-integration-tests/fixtures/configs/viceroy-template.toml b/crates/trusted-server-integration-tests/fixtures/configs/viceroy-template.toml index 551d275d98..97734071fe 100644 --- a/crates/trusted-server-integration-tests/fixtures/configs/viceroy-template.toml +++ b/crates/trusted-server-integration-tests/fixtures/configs/viceroy-template.toml @@ -81,6 +81,14 @@ [[local_server.secret_stores.trusted_server_secrets]] key = "integration_partner_token_bravo" data = "integration-test-token-bravo-32-bytes-ok" + # Example AWS credentials for the S3 SigV4 asset routes. They sign + # requests to a loopback test origin only. + [[local_server.secret_stores.trusted_server_secrets]] + key = "integration_s3_access_key_id" + data = "AKIAIOSFODNN7EXAMPLE" + [[local_server.secret_stores.trusted_server_secrets]] + key = "integration_s3_secret_access_key" + data = "wJalrXUtnFEMI/K7MDENG+bPxRfiCYEXAMPLEKEY" [local_server.config_stores] # Generated integration configs inject the manifest-default app-config blob diff --git a/crates/trusted-server-integration-tests/src/bin/generate-viceroy-config.rs b/crates/trusted-server-integration-tests/src/bin/generate-viceroy-config.rs index 8bd86d68aa..0d25cfdfa3 100644 --- a/crates/trusted-server-integration-tests/src/bin/generate-viceroy-config.rs +++ b/crates/trusted-server-integration-tests/src/bin/generate-viceroy-config.rs @@ -1,15 +1,13 @@ -use std::collections::BTreeSet; use std::env; use std::error::Error; use std::fs; use std::path::PathBuf; -use std::time::Duration; use edgezero_core::blob_envelope::BlobEnvelope; use trusted_server_core::config::TrustedServerAppConfig; use trusted_server_core::config_payload::{CONFIG_BLOB_KEY, DEFAULT_CONFIG_STORE_ID}; -use trusted_server_core::platform::{BackendNamingPolicy, PlatformBackendSpec}; -use url::{Host, Url}; +use trusted_server_core::settings::Settings; +use url::Url; const GENERATED_AT: &str = "2026-06-23T00:00:00Z"; const GENERATED_STORES_MARKER: &str = " # GENERATED_TRUSTED_SERVER_CONFIG_STORES"; @@ -23,7 +21,7 @@ struct Args { output: PathBuf, origin_url: Option, public_origin: Option, - bidder_origin_url: Option, + provider_endpoint_origin: Option, } fn main() -> Result<(), DynError> { @@ -48,13 +46,9 @@ fn run(args: &Args) -> Result<(), DynError> { &app_config, args.origin_url.as_deref(), args.public_origin.as_deref(), + args.provider_endpoint_origin.as_deref(), )?; - let generated_config = inject_generated_config_stores(&template, &envelope_json)?; - if let Some(origin) = &args.bidder_origin_url { - inject_controlled_bidder_backends(&generated_config, &app_config, origin)? - } else { - generated_config - } + inject_generated_config_stores(&template, &envelope_json)? } else { retarget_generated_config( &template, @@ -87,7 +81,7 @@ fn parse_args(args: impl IntoIterator) -> Result let mut output = None; let mut origin_url = None; let mut public_origin = None; - let mut bidder_origin_url = None; + let mut provider_endpoint_origin = None; let mut iter = args.into_iter(); while let Some(arg) = iter.next() { @@ -99,8 +93,9 @@ fn parse_args(args: impl IntoIterator) -> Result } "--output" => output = Some(next_path_arg(&mut iter, "--output")?), "--origin-url" => origin_url = Some(next_string_arg(&mut iter, "--origin-url")?), - "--bidder-origin-url" => { - bidder_origin_url = Some(next_string_arg(&mut iter, "--bidder-origin-url")?); + "--provider-endpoint-origin" => { + provider_endpoint_origin = + Some(next_string_arg(&mut iter, "--provider-endpoint-origin")?); } "--help" | "-h" => return Err(error_box(usage())), other => { @@ -112,8 +107,10 @@ fn parse_args(args: impl IntoIterator) -> Result } } - if bidder_origin_url.is_some() && app_config.is_none() { - return Err(error_box("--bidder-origin-url requires --app-config")); + if provider_endpoint_origin.is_some() && app_config.is_none() { + return Err(error_box( + "--provider-endpoint-origin requires --app-config", + )); } Ok(Args { @@ -123,97 +120,37 @@ fn parse_args(args: impl IntoIterator) -> Result output: output.ok_or_else(|| error_box(format!("missing --output\n\n{}", usage())))?, origin_url, public_origin, - bidder_origin_url, + provider_endpoint_origin, }) } -fn inject_controlled_bidder_backends( - runtime_config: &str, - app_config: &str, - origin: &str, -) -> Result { - let origin = Url::parse(origin)?; - let local = match origin.host() { - Some(Host::Domain("localhost")) => true, - Some(Host::Ipv4(address)) => address.is_loopback(), - Some(Host::Ipv6(address)) => address.is_loopback(), - _ => false, - }; - if !local - || origin.scheme() != "http" - || !origin.username().is_empty() +/// Points every auction provider at `origin`, keeping each endpoint's path and query. +/// +/// `origin` must be a bare origin: a scheme and authority with no credentials, path, query, +/// or fragment. Settings validation then decides whether the rewritten endpoints are allowed, +/// which admits plain HTTP only for the loopback literals. +fn rewrite_provider_endpoints(settings: &mut Settings, origin: &str) -> Result<(), DynError> { + let origin = Url::parse(origin) + .map_err(|error| error_box(format!("invalid provider endpoint origin: {error}")))?; + if !origin.username().is_empty() || origin.password().is_some() || origin.path() != "/" || origin.query().is_some() || origin.fragment().is_some() { return Err(error_box( - "bidder fixture requires a plain loopback HTTP origin", + "provider endpoint origin must be a scheme and authority only", )); } - let settings = toml::from_str::(app_config)?.into_settings(); - let logical_budget = settings.auction.timeout_ms.max( - settings - .creative_opportunities - .as_ref() - .and_then(|config| config.auction_timeout_ms) - .unwrap_or(settings.auction.timeout_ms), - ); - if logical_budget > 60_000 { - return Err(error_box( - "controlled bidder fixture budget exceeds 60 seconds", - )); - } - let plan = trusted_server_core::auction::compile_auction_plan(&settings) - .map_err(|_| error_box("invalid controlled bidder auction plan"))?; - let mut runtime: toml::Value = toml::from_str(runtime_config)?; - let backends = runtime - .get_mut("local_server") - .and_then(|server| server.get_mut("backends")) - .and_then(toml::Value::as_table_mut) - .ok_or_else(|| error_box("runtime template requires a backend table"))?; - for provider in plan.providers() { - let endpoint = Url::parse(provider.endpoint.as_str())?; - // Reuse production policies for every reachable timer/name, rather - // than duplicating Fastly's quantization or backend-name algorithm. - let timers = (1..=provider.timeout_ms.min(logical_budget)) - .map(|remaining| { - BackendNamingPolicy::Fastly - .canonicalize_transport_timeout_ms(remaining, provider.timeout_ms) - }) - .filter(|timer| *timer != 0) - .collect::>(); - for timer in timers { - let duration = Duration::from_millis(u64::from(timer)); - let spec = PlatformBackendSpec { - scheme: endpoint.scheme().to_owned(), - host: endpoint - .host_str() - .ok_or_else(|| error_box("missing bidder host"))? - .to_owned(), - port: endpoint.port(), - host_header_override: None, - certificate_check: true, - first_byte_timeout: duration, - between_bytes_timeout: duration, - discriminator: Some(provider.id.as_str().to_owned()), - }; - let name = BackendNamingPolicy::Fastly - .predict(&spec) - .map_err(|_| error_box("invalid controlled bidder backend name"))? - .name; - let alias = toml::Value::Table(toml::Table::from_iter([( - "url".to_owned(), - toml::Value::String(origin.as_str().to_owned()), - )])); - if backends.insert(name, alias).is_some() { - return Err(error_box( - "controlled bidder alias collides with template backend", - )); - } - } + for provider in settings.auction.providers.values_mut() { + let endpoint = Url::parse(&provider.endpoint) + .map_err(|error| error_box(format!("invalid auction provider endpoint: {error}")))?; + let mut rewritten = origin.clone(); + rewritten.set_path(endpoint.path()); + rewritten.set_query(endpoint.query()); + provider.endpoint = rewritten.into(); } - Ok(toml::to_string(&runtime)?) + Ok(()) } fn next_path_arg( @@ -232,23 +169,30 @@ fn next_string_arg( } fn usage() -> String { - "usage: generate-viceroy-config --template [--app-config ] --output [--origin-url ] [--public-origin ] [--bidder-origin-url (requires --app-config)]\nWithout --app-config, retarget the blob in an already generated Viceroy config.".to_string() + "usage: generate-viceroy-config --template [--app-config ] --output [--origin-url ] [--public-origin ] [--provider-endpoint-origin (requires --app-config)]\nWithout --app-config, retarget the blob in an already generated Viceroy config.".to_string() } fn build_app_config_envelope( app_config_toml: &str, origin_url: Option<&str>, public_origin: Option<&str>, + provider_endpoint_origin: Option<&str>, ) -> Result { let app_config: TrustedServerAppConfig = toml::from_str(app_config_toml) .map_err(|error| error_box(format!("invalid Trusted Server app config: {error}")))?; - serialize_app_config_envelope(app_config, origin_url, public_origin) + serialize_app_config_envelope( + app_config, + origin_url, + public_origin, + provider_endpoint_origin, + ) } fn serialize_app_config_envelope( app_config: TrustedServerAppConfig, origin_url: Option<&str>, public_origin: Option<&str>, + provider_endpoint_origin: Option<&str>, ) -> Result { let mut settings = app_config.into_settings(); if let Some(origin_url) = origin_url { @@ -257,6 +201,9 @@ fn serialize_app_config_envelope( if let Some(public_origin) = public_origin { settings.publisher.public_origin = Some(public_origin.to_string()); } + if let Some(origin) = provider_endpoint_origin { + rewrite_provider_endpoints(&mut settings, origin)?; + } let app_config = TrustedServerAppConfig::new(settings) .map_err(|report| error_box(format!("invalid Trusted Server app config: {report:?}")))?; @@ -290,7 +237,7 @@ fn retarget_generated_config( let envelope: BlobEnvelope = serde_json::from_str(envelope_json)?; envelope.verify()?; let app_config = serde_json::from_value(envelope.data)?; - let envelope_json = serialize_app_config_envelope(app_config, origin_url, public_origin)?; + let envelope_json = serialize_app_config_envelope(app_config, origin_url, public_origin, None)?; contents[CONFIG_BLOB_KEY] = toml::Value::String(envelope_json); Ok(toml::to_string(&config)?) } @@ -326,7 +273,6 @@ mod tests { use super::*; use error_stack::Report; use std::collections::HashMap; - use tempfile::tempdir; use trusted_server_core::config_payload::settings_from_config_blob; use trusted_server_core::platform::{PlatformError, PlatformSecretStore, StoreId, StoreName}; @@ -337,6 +283,7 @@ mod tests { values: HashMap>, } + #[async_trait::async_trait(?Send)] impl PlatformSecretStore for IntegrationSecretStore { fn get_bytes( &self, @@ -349,7 +296,7 @@ mod tests { .ok_or_else(|| Report::new(PlatformError::SecretStore)) } - fn create( + async fn create( &self, _store_id: &StoreId, _name: &str, @@ -358,7 +305,11 @@ mod tests { Ok(()) } - fn delete(&self, _store_id: &StoreId, _name: &str) -> Result<(), Report> { + async fn delete( + &self, + _store_id: &StoreId, + _name: &str, + ) -> Result<(), Report> { Ok(()) } } @@ -407,138 +358,116 @@ mod tests { ); } + const TRACE_APP_CONFIG: &str = include_str!("../../fixtures/configs/trusted-server.trace.toml"); + #[test] - fn parse_args_accepts_explicit_controlled_bidder_origin() { - assert!( - parse_args([ - "--template".to_string(), - "template.toml".to_string(), - "--app-config".to_string(), - "trusted-server.toml".to_string(), - "--output".to_string(), - "generated.toml".to_string(), - "--bidder-origin-url".to_string(), - "http://127.0.0.1:8888".to_string(), - ]) - .is_ok(), - "should accept an explicit local bidder fixture without changing defaults" + fn parse_args_accepts_provider_endpoint_origin() { + let args = parse_args([ + "--template".to_string(), + "template.toml".to_string(), + "--app-config".to_string(), + "trusted-server.toml".to_string(), + "--output".to_string(), + "generated.toml".to_string(), + "--provider-endpoint-origin".to_string(), + "http://127.0.0.1:8888".to_string(), + ]) + .expect("should accept a provider endpoint origin"); + + assert_eq!( + args.provider_endpoint_origin.as_deref(), + Some("http://127.0.0.1:8888"), + "should keep the provider endpoint origin" ); } - fn run_bidder_fixture(origin: &str) -> Result { - let directory = tempdir().expect("should create isolated generator fixture"); - let template = directory.path().join("template.toml"); - let app_config = directory.path().join("app.toml"); - let output = directory.path().join("output.toml"); - fs::write(&template, TEMPLATE).expect("should write runtime template"); - fs::write( - &app_config, - include_str!("../../fixtures/configs/trusted-server.trace.toml"), - ) - .expect("should write controlled trace app config"); - run(&Args { - template, - app_config: Some(app_config), - output: output.clone(), - origin_url: None, - public_origin: Some("http://127.0.0.1:43130".to_string()), - bidder_origin_url: Some(origin.to_string()), - })?; - Ok(toml::from_str(&fs::read_to_string(output)?)?) + fn provider_endpoint(envelope: &str, provider: &str) -> String { + let envelope: serde_json::Value = + serde_json::from_str(envelope).expect("should parse the generated envelope"); + envelope["data"]["auction"]["providers"][provider]["endpoint"] + .as_str() + .expect("should keep the provider endpoint") + .to_owned() } #[test] - fn controlled_bidder_uses_backend_aliases_and_preserves_https_app_config() { - let generated = run_bidder_fixture("http://127.0.0.1:8888") - .expect("should generate explicit local bidder aliases"); - let aliases = generated["local_server"]["backends"] - .as_table() - .expect("should retain backend table"); - assert!( - !aliases.is_empty(), - "should route the real Rust bidder HTTP client to the controlled local origin" - ); - for alias in aliases.values() { - assert_eq!( - alias["url"].as_str(), - Some("http://127.0.0.1:8888/"), - "should keep every runtime-only alias local" - ); - } - let retargeted = retarget_generated_config( - &toml::to_string(&generated).expect("should serialize generated config"), - None, - Some("http://127.0.0.1:43131"), - ) - .expect("should retarget controlled bidder config"); - let retargeted: toml::Value = - toml::from_str(&retargeted).expect("should parse retargeted bidder config"); - assert_eq!( - retargeted["local_server"]["backends"], generated["local_server"]["backends"], - "should preserve controlled bidder aliases when retargeting the runtime listener" + fn provider_endpoint_origin_replaces_scheme_and_authority_and_keeps_path_and_query() { + let app_config = TRACE_APP_CONFIG.replace( + "endpoint = \"https://bidder.example.com/api/trace-bidder\"", + "endpoint = \"https://bidder.example.com/api/trace-bidder?route=example\"", ); - let envelope: serde_json::Value = serde_json::from_str( - generated["local_server"]["config_stores"][DEFAULT_CONFIG_STORE_ID]["contents"] - [CONFIG_BLOB_KEY] - .as_str() - .expect("should retain validated app envelope"), - ) - .expect("should read unchanged app config envelope"); + + let envelope = + build_app_config_envelope(&app_config, None, None, Some("http://127.0.0.1:8888")) + .expect("should rewrite the provider endpoint onto the loopback origin"); + assert_eq!( - envelope["data"]["publisher"]["public_origin"], "http://127.0.0.1:43130", - "should support a runtime public origin alongside controlled bidder aliases" + provider_endpoint(&envelope, "example"), + "http://127.0.0.1:8888/api/trace-bidder?route=example", + "should replace the scheme and authority and keep the path and query" ); + } + + #[test] + fn provider_endpoints_are_unchanged_without_provider_endpoint_origin() { + let envelope = build_app_config_envelope(TRACE_APP_CONFIG, None, None, None) + .expect("should build the trace envelope"); + assert_eq!( - envelope["data"]["auction"]["providers"]["example"]["endpoint"], + provider_endpoint(&envelope, "example"), "https://bidder.example.com/api/trace-bidder", - "should preserve production HTTPS and certificate admission" + "should keep the configured endpoint" ); } #[test] - fn controlled_bidder_rejects_non_loopback_or_ambiguous_origins() { + fn provider_endpoint_origin_rejects_non_origin_and_non_loopback_urls() { for origin in [ - "http://bidder.example.com", "http://127.0.0.1:8888/path", "http://127.0.0.1:8888/?query=value", "http://user:password@127.0.0.1:8888", + "http://localhost:8888", + "http://bidder.example.com", ] { assert!( - run_bidder_fixture(origin).is_err(), - "should reject a non-local or non-origin bidder fixture URL" + build_app_config_envelope(TRACE_APP_CONFIG, None, None, Some(origin)).is_err(), + "should reject `{origin}` as a provider endpoint origin" ); } } #[test] - fn controlled_bidder_omits_unreachable_large_provider_timer_aliases() { - let app_config = include_str!("../../fixtures/configs/trusted-server.trace.toml").replace( - "[auction.providers.example]", - "[auction.providers.example]\ntimeout_ms = 100000", - ); - let generated = - inject_controlled_bidder_backends(TEMPLATE, &app_config, "http://127.0.0.1:8888") - .expect("should bound the configured provider by reachable logical budgets"); - let generated: toml::Value = - toml::from_str(&generated).expect("should generate valid bounded runtime aliases"); + fn retargeting_keeps_rewritten_provider_endpoints() { + let envelope = build_app_config_envelope( + TRACE_APP_CONFIG, + None, + Some("http://127.0.0.1:43130"), + Some("http://127.0.0.1:8888"), + ) + .expect("should build a trace envelope with a public origin and a provider origin"); + let generated = inject_generated_config_stores(TEMPLATE, &envelope) + .expect("should inject generated stores"); + let generated_data: serde_json::Value = + serde_json::from_str(&envelope).expect("should parse the generated envelope"); assert_eq!( - generated["local_server"]["backends"] - .as_table() - .expect("should retain aliases") - .len(), - 8, - "should generate only eight reachable timers for the 1000 ms fixture budget" + generated_data["data"]["publisher"]["public_origin"], "http://127.0.0.1:43130", + "should support a runtime public origin alongside a provider endpoint origin" ); - } - #[test] - fn controlled_bidder_rejects_excessive_fixture_logical_budget() { - let app_config = include_str!("../../fixtures/configs/trusted-server.trace.toml") - .replace("auction_timeout_ms = 1000", "auction_timeout_ms = 100000"); - assert!( - inject_controlled_bidder_backends(TEMPLATE, &app_config, "http://127.0.0.1:8888") - .is_err(), - "should reject fixture budgets beyond the bounded 60 second timer enumeration" + let retargeted = + retarget_generated_config(&generated, None, Some("http://127.0.0.1:43131")) + .expect("should retarget the generated trace config"); + let retargeted: toml::Value = + toml::from_str(&retargeted).expect("should parse the retargeted trace config"); + let retargeted_envelope = retargeted["local_server"]["config_stores"] + [DEFAULT_CONFIG_STORE_ID]["contents"][CONFIG_BLOB_KEY] + .as_str() + .expect("should contain a blob"); + + assert_eq!( + provider_endpoint(retargeted_envelope, "example"), + "http://127.0.0.1:8888/api/trace-bidder", + "should keep the rewritten provider endpoint when retargeting the runtime listener" ); } @@ -564,25 +493,25 @@ mod tests { output: PathBuf::from("generated.toml"), origin_url: Some("http://127.0.0.1:9999".to_string()), public_origin: None, - bidder_origin_url: None, + provider_endpoint_origin: None, }, "should parse expected args" ); } #[test] - fn parse_args_rejects_bidder_injection_without_app_config() { + fn parse_args_rejects_provider_endpoint_origin_without_app_config() { let result = parse_args([ "--template".to_string(), "generated.toml".to_string(), "--output".to_string(), "runtime.toml".to_string(), - "--bidder-origin-url".to_string(), + "--provider-endpoint-origin".to_string(), "http://127.0.0.1:8888".to_string(), ]); assert!( result.is_err(), - "should not silently ignore bidder injection when retargeting an existing blob" + "should not silently ignore a provider endpoint origin when retargeting an existing blob" ); } @@ -615,6 +544,7 @@ mod tests { APP_CONFIG, Some("http://127.0.0.1:9999"), Some("http://127.0.0.1:43126"), + None, ) .expect("should build runtime-specific config"); let envelope: BlobEnvelope = @@ -634,7 +564,7 @@ mod tests { #[test] fn retargeting_generated_config_preserves_resources_and_app_policy() { - let json = build_app_config_envelope(APP_CONFIG, Some("http://127.0.0.1:9999"), None) + let json = build_app_config_envelope(APP_CONFIG, Some("http://127.0.0.1:9999"), None, None) .expect("should build original envelope"); let generated = inject_generated_config_stores(TEMPLATE, &json) .expect("should generate original config"); @@ -673,7 +603,7 @@ mod tests { #[test] fn generated_config_contains_blob_without_removed_rollout_flags() { let envelope = - build_app_config_envelope(APP_CONFIG, None, None).expect("should build envelope"); + build_app_config_envelope(APP_CONFIG, None, None, None).expect("should build envelope"); let generated = inject_generated_config_stores(TEMPLATE, &envelope) .expect("should inject generated stores"); @@ -700,7 +630,7 @@ mod tests { #[test] fn generated_config_is_valid_toml() { let envelope = - build_app_config_envelope(APP_CONFIG, None, None).expect("should build envelope"); + build_app_config_envelope(APP_CONFIG, None, None, None).expect("should build envelope"); let generated = inject_generated_config_stores(TEMPLATE, &envelope) .expect("should inject generated stores"); let parsed: toml::Value = toml::from_str(&generated).expect("should parse as TOML"); @@ -716,8 +646,9 @@ mod tests { #[test] fn generated_blob_verifies_and_applies_origin_override() { - let envelope = build_app_config_envelope(APP_CONFIG, Some("http://127.0.0.1:9999"), None) - .expect("should build envelope"); + let envelope = + build_app_config_envelope(APP_CONFIG, Some("http://127.0.0.1:9999"), None, None) + .expect("should build envelope"); let settings = settings_from_config_blob( &envelope, &integration_secret_store(), @@ -733,7 +664,7 @@ mod tests { #[test] fn invalid_app_config_fails() { - let result = build_app_config_envelope("not valid toml", None, None); + let result = build_app_config_envelope("not valid toml", None, None, None); assert!(result.is_err(), "should reject invalid app config"); } @@ -742,7 +673,7 @@ mod tests { fn invalid_non_secret_app_config_fails_before_envelope_generation() { let invalid = APP_CONFIG.replace("domain = \"localhost\"", "domain = \"invalid/domain\""); - let err = build_app_config_envelope(&invalid, None, None) + let err = build_app_config_envelope(&invalid, None, None, None) .expect_err("should reject invalid non-secret config before creating an envelope"); assert!( diff --git a/crates/trusted-server-integration-tests/tests/common/config.rs b/crates/trusted-server-integration-tests/tests/common/config.rs index ea48162c8b..fdab8e6103 100644 --- a/crates/trusted-server-integration-tests/tests/common/config.rs +++ b/crates/trusted-server-integration-tests/tests/common/config.rs @@ -24,6 +24,89 @@ pub fn integration_app_config_envelope_with_trace( app_config_envelope(origin_port, trace, |_| Ok(())) } +/// Builds the integration app-config envelope with `proxy.allowed_domains` set. +/// +/// The first-party proxy then checks its signed target and every redirect hop +/// against `allowed_domains`. +/// +/// # Errors +/// +/// Returns a fixture-generation error when configuration validation fails. +pub fn proxy_allowlist_app_config_envelope( + origin_port: u16, + allowed_domains: &[&str], +) -> TestResult { + app_config_envelope(origin_port, false, |table| { + let allowed_domains = allowed_domains + .iter() + .map(|domain| toml::Value::String((*domain).to_owned())) + .collect(); + let mut proxy = toml::Table::new(); + proxy.insert( + "allowed_domains".to_owned(), + toml::Value::Array(allowed_domains), + ); + table.insert("proxy".to_owned(), toml::Value::Table(proxy)); + Ok(()) + }) +} + +/// Builds the integration app-config envelope with two S3 `SigV4` asset routes +/// to `asset_origin`. +/// +/// `/.sigv4-assets/` signs plain requests and keeps their query. +/// `/.sigv4-images/` adds Image Optimizer, so each request first sends a signed +/// S3 `HEAD` preflight. Both routes sign for `us-east-1` with the +/// `integration_s3_*` secrets of the Viceroy template. +/// +/// # Errors +/// +/// Returns a fixture-generation error when configuration validation fails. +pub fn sigv4_asset_app_config_envelope(origin_port: u16, asset_origin: &str) -> TestResult { + let assets: toml::Table = toml::from_str(&format!( + r#" +[[proxy.asset_routes]] +prefix = "/.sigv4-assets/" +origin_url = "{asset_origin}" + +[proxy.asset_routes.auth] +type = "s3_sigv4" +region = "us-east-1" +access_key_id = "integration_s3_access_key_id" +secret_access_key = "integration_s3_secret_access_key" + +[[proxy.asset_routes]] +prefix = "/.sigv4-images/" +origin_url = "{asset_origin}" + +[proxy.asset_routes.auth] +type = "s3_sigv4" +region = "us-east-1" +access_key_id = "integration_s3_access_key_id" +secret_access_key = "integration_s3_secret_access_key" + +[proxy.asset_routes.image_optimizer] +enabled = true +region = "us_east" +profile_set = "sigv4_images" + +[image_optimizer.profile_sets.sigv4_images] +default_profile = "default" + +[image_optimizer.profile_sets.sigv4_images.profiles] +default = "width=640" +"# + )) + .map_err(|error| { + Report::new(TestError::ConfigGeneration) + .attach(format!("invalid SigV4 asset route fixture: {error}")) + })?; + app_config_envelope(origin_port, false, |table| { + table.extend(assets); + Ok(()) + }) +} + /// Builds the integration app-config envelope after `customize` edits the /// parsed fixture. fn app_config_envelope( @@ -95,6 +178,108 @@ pub fn cloudflare_auction_config_json(origin_port: u16) -> TestResult { cloudflare_binding_json(&serde_json::json!({ CONFIG_BLOB_KEY: envelope })) } +/// One standard `OpenRTB` provider an auction fixture routes every eligible slot to. +pub struct AuctionBidderFixture { + /// Provider ID. + pub id: String, + /// Provider endpoint, such as a loopback bidder's `http://127.0.0.1:/bid`. + pub endpoint: String, + /// Configured provider timeout. + pub timeout_ms: u32, +} + +/// Builds the integration app-config envelope with the auction enabled over `bidders`. +/// +/// The fixture's own providers and bidder routes are replaced, so only `bidders` take part. +/// +/// # Errors +/// +/// Returns a fixture-generation error when configuration validation fails. +pub fn auction_app_config_envelope( + origin_port: u16, + bidders: &[AuctionBidderFixture], + auction_timeout_ms: u32, +) -> TestResult { + app_config_envelope(origin_port, false, |table| { + enable_auction(table, bidders, auction_timeout_ms) + }) +} + +/// Enables the auction in `table` over `bidders` alone, with `auction_timeout_ms`. +fn enable_auction( + table: &mut toml::Table, + bidders: &[AuctionBidderFixture], + auction_timeout_ms: u32, +) -> TestResult<()> { + let mut providers = toml::Table::new(); + for bidder in bidders { + let provider: toml::Table = toml::from_str(&format!( + "protocol = \"openrtb-2.6\"\nprofile = \"standard\"\nendpoint = \"{}\"\ntimeout_ms = {}\nrouting = \"all_eligible\"\n", + bidder.endpoint, bidder.timeout_ms + )) + .map_err(|error| { + Report::new(TestError::ConfigGeneration) + .attach(format!("invalid auction bidder fixture: {error}")) + })?; + providers.insert(bidder.id.clone(), toml::Value::Table(provider)); + } + let auction = table_mut(table, "auction")?; + auction.insert("enabled".to_string(), toml::Value::Boolean(true)); + auction.insert( + "timeout_ms".to_string(), + toml::Value::Integer(i64::from(auction_timeout_ms)), + ); + auction.insert("providers".to_string(), toml::Value::Table(providers)); + auction.remove("bidders"); + Ok(()) +} + +/// Builds [`auction_app_config_envelope`] with server-side ad templates on `path`. +/// +/// Each slot is a 300x250 creative opportunity on `path`, and navigations to `path` run +/// their auction with `auction_timeout_ms`. +/// +/// # Errors +/// +/// Returns a fixture-generation error when configuration validation fails. +pub fn navigation_auction_app_config_envelope( + origin_port: u16, + bidders: &[AuctionBidderFixture], + auction_timeout_ms: u32, + path: &str, + slots: &[&str], +) -> TestResult { + let mut slot_tables = Vec::new(); + for slot in slots { + let slot: toml::Table = toml::from_str(&format!( + "id = \"{slot}\"\npage_patterns = [\"{path}\"]\nformats = [{{ width = 300, height = 250 }}]\n" + )) + .map_err(|error| { + Report::new(TestError::ConfigGeneration) + .attach(format!("invalid creative opportunity slot fixture: {error}")) + })?; + slot_tables.push(toml::Value::Table(slot)); + } + app_config_envelope(origin_port, false, |table| { + enable_auction(table, bidders, auction_timeout_ms)?; + let mut creative_opportunities = toml::Table::new(); + creative_opportunities.insert( + "gam_network_id".to_string(), + toml::Value::String("12345".to_owned()), + ); + creative_opportunities.insert( + "auction_timeout_ms".to_string(), + toml::Value::Integer(i64::from(auction_timeout_ms)), + ); + creative_opportunities.insert("slot".to_string(), toml::Value::Array(slot_tables)); + table.insert( + "creative_opportunities".to_string(), + toml::Value::Table(creative_opportunities), + ); + Ok(()) + }) +} + /// Retargets an existing fixture without changing its policies or secret references. pub fn envelope_with_public_origin(envelope_json: &str, public_origin: &str) -> TestResult { let envelope: BlobEnvelope = serde_json::from_str(envelope_json) @@ -175,6 +360,31 @@ fn cloudflare_binding_json(binding: &serde_json::Value) -> TestResult { mod tests { use super::*; + #[test] + fn sigv4_asset_fixture_validates_both_signed_routes() { + let envelope: serde_json::Value = serde_json::from_str( + &sigv4_asset_app_config_envelope(8888, "https://127.0.0.1:8443") + .expect("should generate the SigV4 asset fixture"), + ) + .expect("should parse the SigV4 asset envelope"); + let routes = envelope["data"]["proxy"]["asset_routes"] + .as_array() + .expect("should carry the asset routes"); + + assert_eq!( + routes + .iter() + .map(|route| route["prefix"].as_str()) + .collect::>(), + vec![Some("/.sigv4-assets/"), Some("/.sigv4-images/")], + "should keep the plain and the Image Optimizer route" + ); + assert_eq!( + routes[1]["image_optimizer"]["enabled"], true, + "should enable Image Optimizer on the preflight route" + ); + } + #[test] fn trace_browser_fixture_keeps_auctions_and_shared_assembly_disabled() { let envelope: serde_json::Value = serde_json::from_str( diff --git a/crates/trusted-server-integration-tests/tests/common/ec.rs b/crates/trusted-server-integration-tests/tests/common/ec.rs index 0a1f149c3b..14af19952f 100644 --- a/crates/trusted-server-integration-tests/tests/common/ec.rs +++ b/crates/trusted-server-integration-tests/tests/common/ec.rs @@ -276,7 +276,9 @@ fn mappings_to_json(mappings: &[BatchMapping]) -> Vec { /// /// The `EdgeZero` router returns a router-level `405` for methods outside its /// registered set (e.g. `TRACE`). This canary catches stale binaries or broken -/// fixtures that fail before reaching the router. +/// fixtures that fail before reaching the router. The canonical 405 must also +/// carry `x-ts-version` and `x-ts-fastly-version`, from Viceroy's +/// `FASTLY_SERVICE_VERSION` (spec test 11). pub fn assert_edgezero_entry_point(base_url: &str) -> TestResult<()> { let client = Client::builder() .redirect(reqwest::redirect::Policy::none()) @@ -287,7 +289,21 @@ pub fn assert_edgezero_entry_point(base_url: &str) -> TestResult<()> { .send() .change_context(TestError::HttpRequest) .attach("TRACE / (EdgeZero entry-point canary)")?; - assert_status(&response, 405).attach("EdgeZero canary: TRACE should return a router-level 405") + assert_status(&response, 405) + .attach("EdgeZero canary: TRACE should return a router-level 405")?; + for name in ["x-ts-version", "x-ts-fastly-version"] { + if response + .headers() + .get(name) + .is_none_or(reqwest::header::HeaderValue::is_empty) + { + return Err(Report::new(TestError::MissingHeader { + name: name.to_owned(), + }) + .attach("EdgeZero canary: the canonical 405 should carry the version headers")); + } + } + Ok(()) } pub fn assert_status(resp: &Response, expected: u16) -> TestResult<()> { diff --git a/crates/trusted-server-integration-tests/tests/common/runtime.rs b/crates/trusted-server-integration-tests/tests/common/runtime.rs index 048c76d44b..dbb2ebcf21 100644 --- a/crates/trusted-server-integration-tests/tests/common/runtime.rs +++ b/crates/trusted-server-integration-tests/tests/common/runtime.rs @@ -52,6 +52,9 @@ pub enum TestError { #[display("JSON field assertion failed: {field}")] JsonFieldMismatch { field: String }, + #[display("Response header `{name}` is missing or empty")] + MissingHeader { name: String }, + // Resource/config errors #[display("Failed to generate runtime configuration")] ConfigGeneration, diff --git a/crates/trusted-server-integration-tests/tests/common/trace_boundary.rs b/crates/trusted-server-integration-tests/tests/common/trace_boundary.rs index c54ab734a2..40bfd19ca6 100644 --- a/crates/trusted-server-integration-tests/tests/common/trace_boundary.rs +++ b/crates/trusted-server-integration-tests/tests/common/trace_boundary.rs @@ -18,11 +18,11 @@ use trusted_server_core::config_payload::{CONFIG_BLOB_KEY, DEFAULT_CONFIG_STORE_ use crate::common::config::integration_app_config_envelope_with_trace; use crate::common::runtime::TestError; -use crate::common::runtime::{ - RuntimeEnvironment as _, RuntimeProcess, RuntimeProcessHandle, origin_port, wasm_binary_path, -}; +use crate::common::runtime::{RuntimeProcess, RuntimeProcessHandle, origin_port, wasm_binary_path}; use crate::environments::{ - axum::AxumDevServer, cloudflare::CloudflareWorkers, fastly::FastlyViceroy, + axum::AxumDevServer, + cloudflare::CloudflareWorkers, + fastly::{FastlyViceroy, GuestStdout}, }; use crate::frameworks::{FrontendFramework as _, nextjs::NextJs}; @@ -92,21 +92,30 @@ impl Drop for ChildHandle { fn drop(&mut self) { #[cfg(unix)] if self.process_group { - // Give Playwright's signal handlers time to close its browsers - // before escalating the owned Node and worker process group. + // Give the owned group's signal handlers time to exit, such as + // Playwright closing its browsers, before escalating the group. + let group = self.child.id() as libc::pid_t; unsafe { - libc::killpg(self.child.id() as libc::pid_t, libc::SIGTERM); + libc::killpg(group, libc::SIGTERM); } let deadline = Instant::now() + Duration::from_secs(3); + let mut group_exited = false; while Instant::now() < deadline { - let group_exists = unsafe { libc::killpg(self.child.id() as libc::pid_t, 0) == 0 }; - if !group_exists { + // An unreaped leader still counts as a group member on Linux, + // so reap it before probing. Only ESRCH means the group is gone. + let _ = self.child.try_wait(); + let probe = unsafe { libc::killpg(group, 0) }; + if probe != 0 && std::io::Error::last_os_error().raw_os_error() == Some(libc::ESRCH) + { + group_exited = true; break; } std::thread::sleep(Duration::from_millis(100)); } - unsafe { - libc::killpg(self.child.id() as libc::pid_t, libc::SIGKILL); + if !group_exited { + unsafe { + libc::killpg(group, libc::SIGKILL); + } } } let _ = self.child.kill(); @@ -157,16 +166,25 @@ fn exercise_case(runtime: TraceRuntime, enabled: bool, auth_pattern: Option<&str with_trace_runtime(runtime, &envelope, None, check); } -fn with_trace_runtime( +/// Runs `check` against `runtime` serving the app config `envelope` in an isolated fixture. +/// +/// On Spin, `publisher_origin` is the only outbound host the component may reach, and it +/// may carry a port wildcard such as `http://127.0.0.1:*`. +/// +/// # Panics +/// +/// Panics when the runtime's artifact or CLI is missing or the runtime never loads the +/// configuration. +pub(crate) fn with_trace_runtime( runtime: TraceRuntime, envelope: &str, publisher_origin: Option<&str>, check: impl FnOnce(&str), ) { - let fixture = IsolatedFixture::new(); let root = Path::new(env!("CARGO_MANIFEST_DIR")).join("../.."); match runtime { TraceRuntime::Cloudflare => { + let fixture = IsolatedFixture::new(); let original = root.join("crates/trusted-server-adapter-cloudflare"); copy_directory(&original.join("build"), &fixture.0.join("build")); let template = fs::read_to_string(original.join("wrangler.ci.toml")) @@ -192,19 +210,9 @@ fn with_trace_runtime( }, ); } - TraceRuntime::Fastly => { - let template = include_str!("../../fixtures/configs/viceroy-template.toml"); - let config=template.replace(" # GENERATED_TRUSTED_SERVER_CONFIG_STORES",&format!(" [local_server.config_stores.{DEFAULT_CONFIG_STORE_ID}]\n format = \"inline-toml\"\n [local_server.config_stores.{DEFAULT_CONFIG_STORE_ID}.contents]\n {CONFIG_BLOB_KEY} = '''{envelope}'''")); - let path = fixture.0.join("viceroy.toml"); - fs::write(&path, config).expect("should write isolated Viceroy config"); - temp_env::with_var("VICEROY_CONFIG_PATH", Some(path.as_os_str()), || { - let process = FastlyViceroy - .spawn(&wasm_binary_path()) - .expect("should spawn isolated Fastly trace runtime"); - check(&process.base_url); - }); - } + TraceRuntime::Fastly => with_fastly_runtime(envelope, |base_url, _stdout| check(base_url)), TraceRuntime::Spin => { + let fixture = IsolatedFixture::new(); let artifact = root.join("target/wasm32-wasip1/release/trusted_server_adapter_spin.wasm"); assert!( @@ -256,7 +264,8 @@ fn with_trace_runtime( fs::write(&manifest, config).expect("should write isolated Spin manifest"); let port = crate::environments::find_available_port() .expect("should reserve Spin runtime port"); - let child = Command::new("spin") + let mut command = Command::new("spin"); + command .arg("up") .arg("--from") .arg(&manifest) @@ -267,14 +276,17 @@ fn with_trace_runtime( .arg("--key-value") .arg(format!("{CONFIG_BLOB_KEY}={envelope}")) .stdout(Stdio::null()) - .stderr(Stdio::inherit()) - .spawn() - .expect("should spawn isolated Spin runtime"); + .stderr(Stdio::inherit()); + // `spin up` serves through a `spin trigger http` child, so the + // runtime owns a process group that the handle stops as a whole. + #[cfg(unix)] + command.process_group(0); + let child = command.spawn().expect("should spawn isolated Spin runtime"); let process = RuntimeProcess { inner: Box::new(ChildHandle { child, #[cfg(unix)] - process_group: false, + process_group: true, }), base_url: format!("http://127.0.0.1:{port}"), }; @@ -285,6 +297,26 @@ fn with_trace_runtime( } } +/// Runs `check` against Viceroy serving the app config `envelope` in an isolated +/// fixture, with what the guest has printed to stdout. +/// +/// # Panics +/// +/// Panics when the Fastly artifact or Viceroy is missing or the runtime never becomes ready. +pub(crate) fn with_fastly_runtime(envelope: &str, check: impl FnOnce(&str, &GuestStdout)) { + let fixture = IsolatedFixture::new(); + let template = include_str!("../../fixtures/configs/viceroy-template.toml"); + let config=template.replace(" # GENERATED_TRUSTED_SERVER_CONFIG_STORES",&format!(" [local_server.config_stores.{DEFAULT_CONFIG_STORE_ID}]\n format = \"inline-toml\"\n [local_server.config_stores.{DEFAULT_CONFIG_STORE_ID}.contents]\n {CONFIG_BLOB_KEY} = '''{envelope}'''")); + let path = fixture.0.join("viceroy.toml"); + fs::write(&path, config).expect("should write isolated Viceroy config"); + temp_env::with_var("VICEROY_CONFIG_PATH", Some(path.as_os_str()), || { + let (process, stdout) = FastlyViceroy + .spawn_capturing_stdout(&wasm_binary_path()) + .expect("should spawn isolated Fastly trace runtime"); + check(&process.base_url, &stdout); + }); +} + /// Exercise a real host-only cookie and publisher-to-viewer browser workflow. /// /// Auction and shared assembly stay disabled. Only GPT callbacks are mocked; @@ -310,7 +342,7 @@ pub(crate) fn exercise_browser(runtime: TraceBrowserRuntime) { match runtime { TraceBrowserRuntime::Axum => { let process = AxumDevServer - .spawn_with_app_config(&envelope) + .spawn_with_app_config(&envelope, &[]) .expect("should spawn the real native trace runtime"); wait_for_trace_fixture_ready(&process.base_url) .expect("should load native trace configuration"); diff --git a/crates/trusted-server-integration-tests/tests/documentation_snippets.rs b/crates/trusted-server-integration-tests/tests/documentation_snippets.rs index bac7d0b4a0..1e5026f798 100644 --- a/crates/trusted-server-integration-tests/tests/documentation_snippets.rs +++ b/crates/trusted-server-integration-tests/tests/documentation_snippets.rs @@ -14,6 +14,20 @@ fn integration_guide_html_eof_fixture_compiles() { check_guide_snippet("html-eof"); } +/// Returns the version requirement the workspace manifest gives `dependency`. +fn workspace_requirement(workspace_manifest: &str, dependency: &str) -> String { + workspace_manifest + .lines() + .find_map(|line| { + let assignment = line.trim().strip_prefix(dependency)?.trim_start(); + let value = assignment.strip_prefix('=')?; + let start = value.find('"')? + 1; + let end = start + value[start..].find('"')?; + Some(value[start..end].to_owned()) + }) + .expect("should find the workspace dependency requirement") +} + fn check_guide_snippet(name: &str) { let start = format!(""); let end = format!(""); @@ -47,18 +61,10 @@ fn check_guide_snippet(name: &str) { let core_path = repository_root.join("crates/trusted-server-core"); let workspace_manifest = fs::read_to_string(repository_root.join("Cargo.toml")) .expect("should read the workspace manifest"); - let error_stack_requirement = workspace_manifest - .lines() - .find_map(|line| { - let assignment = line.trim().strip_prefix("error-stack")?.trim_start(); - let value = assignment.strip_prefix('=')?; - let start = value.find('"')? + 1; - let end = start + value[start..].find('"')?; - Some(value[start..end].to_owned()) - }) - .expect("should find the workspace error-stack requirement"); + let async_trait_requirement = workspace_requirement(&workspace_manifest, "async-trait"); + let error_stack_requirement = workspace_requirement(&workspace_manifest, "error-stack"); let manifest = format!( - "[package]\nname = \"documentation-snippet\"\nversion = \"0.0.0\"\nedition = \"2024\"\n\n[workspace]\n\n[dependencies]\nerror-stack = \"{error_stack_requirement}\"\nfutures = \"0.3\"\ntrusted-server-core = {{ path = {:?} }}\n", + "[package]\nname = \"documentation-snippet\"\nversion = \"0.0.0\"\nedition = \"2024\"\n\n[workspace]\n\n[dependencies]\nasync-trait = \"{async_trait_requirement}\"\nerror-stack = \"{error_stack_requirement}\"\nfutures = \"0.3\"\ntrusted-server-core = {{ path = {:?} }}\n", core_path ); fs::write(fixture_root.join("Cargo.toml"), manifest) diff --git a/crates/trusted-server-integration-tests/tests/environments/axum.rs b/crates/trusted-server-integration-tests/tests/environments/axum.rs index 98b56ff73c..e8cd4eec06 100644 --- a/crates/trusted-server-integration-tests/tests/environments/axum.rs +++ b/crates/trusted-server-integration-tests/tests/environments/axum.rs @@ -56,7 +56,7 @@ impl RuntimeEnvironment for AxumDevServer { fn spawn(&self, _wasm_path: &Path) -> TestResult { let app_config = integration_app_config_envelope(origin_port())?; - self.spawn_with_app_config(&app_config) + self.spawn_with_app_config(&app_config, &[]) } fn health_check_path(&self) -> &str { @@ -67,25 +67,27 @@ impl RuntimeEnvironment for AxumDevServer { impl AxumDevServer { /// Spawn the real native binary with an isolated app-config envelope. /// - /// Normal [`RuntimeEnvironment::spawn`] retains its disabled baseline config. + /// `env` adds environment variables, such as `EDGEZERO__ADAPTER__INGRESS__*` + /// limits. Normal [`RuntimeEnvironment::spawn`] retains its disabled + /// baseline config. /// /// # Errors /// /// Returns a spawn or readiness error if the native process is unavailable. - pub(crate) fn spawn_with_app_config(&self, app_config: &str) -> TestResult { + pub(crate) fn spawn_with_app_config( + &self, + app_config: &str, + env: &[(&str, &str)], + ) -> TestResult { let binary = self.binary_path(); let port = super::find_available_port().unwrap_or(AXUM_DEFAULT_PORT); let base_url = format!("http://127.0.0.1:{port}"); let app_config = envelope_with_public_origin(app_config, &base_url)?; - let store_name = default_config_store_name(); - let config_key = default_config_key(); - let config_variable = config_env_var(store_name.as_ref(), &config_key); - let mut child = Command::new(&binary) - .env("PORT", port.to_string()) - .env(config_variable, app_config) - .envs(INTEGRATION_SECRET_ENV.iter().copied()) + let mut child = self + .command(&app_config, port) + .envs(env.iter().copied()) .stdout(Stdio::null()) .stderr(Stdio::piped()) .spawn() @@ -118,6 +120,23 @@ impl AxumDevServer { }) } + /// The command that runs the binary on `port` with `app_config` and the + /// integration secrets. + /// + /// The caller chooses where its output goes. + pub(crate) fn command(&self, app_config: &str, port: u16) -> Command { + let store_name = default_config_store_name(); + let config_key = default_config_key(); + let config_variable = config_env_var(store_name.as_ref(), &config_key); + + let mut command = Command::new(self.binary_path()); + command + .env("PORT", port.to_string()) + .env(config_variable, app_config) + .envs(INTEGRATION_SECRET_ENV.iter().copied()); + command + } + /// Resolve the path to the compiled `trusted-server-axum` binary. /// /// Respects the `AXUM_BINARY_PATH` environment variable for CI overrides. diff --git a/crates/trusted-server-integration-tests/tests/environments/fastly.rs b/crates/trusted-server-integration-tests/tests/environments/fastly.rs index 66e92e1758..195c93f23e 100644 --- a/crates/trusted-server-integration-tests/tests/environments/fastly.rs +++ b/crates/trusted-server-integration-tests/tests/environments/fastly.rs @@ -6,6 +6,7 @@ use error_stack::{Report, ResultExt as _}; use std::io::{BufRead as _, BufReader}; use std::path::Path; use std::process::{Child, Command, Stdio}; +use std::sync::{Arc, Mutex, PoisonError}; /// Fastly Compute runtime using Viceroy local simulator. /// @@ -14,12 +15,45 @@ use std::process::{Child, Command, Stdio}; /// blob). pub struct FastlyViceroy; +/// The lines a Viceroy guest printed to stdout, in order. +/// +/// Viceroy prints each line written to a log endpoint as ` :: `. +#[derive(Clone, Default)] +pub(crate) struct GuestStdout(Arc>>); + +impl GuestStdout { + /// Returns every line printed so far. + pub(crate) fn lines(&self) -> Vec { + self.0 + .lock() + .unwrap_or_else(PoisonError::into_inner) + .clone() + } +} + impl RuntimeEnvironment for FastlyViceroy { fn id(&self) -> &'static str { "fastly" } fn spawn(&self, wasm_path: &Path) -> TestResult { + self.spawn_capturing_stdout(wasm_path) + .map(|(process, _stdout)| process) + } +} + +impl FastlyViceroy { + /// Spawns Viceroy as [`RuntimeEnvironment::spawn`] does, and also returns + /// what its guest prints to stdout. + /// + /// # Errors + /// + /// Returns [`TestError::RuntimeSpawn`] if the process cannot be started, and + /// [`TestError::RuntimeNotReady`] if the health check times out. + pub(crate) fn spawn_capturing_stdout( + &self, + wasm_path: &Path, + ) -> TestResult<(RuntimeProcess, GuestStdout)> { let port = super::find_available_port()?; let base_url = format!("http://127.0.0.1:{port}"); @@ -51,7 +85,7 @@ impl RuntimeEnvironment for FastlyViceroy { .arg(runtime_config.path()) .arg("--addr") .arg(format!("127.0.0.1:{port}")) - .stdout(Stdio::null()) + .stdout(Stdio::piped()) .stderr(Stdio::piped()) .spawn() .change_context(TestError::RuntimeSpawn) @@ -68,6 +102,21 @@ impl RuntimeEnvironment for FastlyViceroy { }); } + // Drained for the whole run, so the guest never blocks on a full pipe. + let stdout = GuestStdout::default(); + if let Some(pipe) = child.stdout.take() { + let recorded = stdout.clone(); + std::thread::spawn(move || { + for line in BufReader::new(pipe).lines().map_while(Result::ok) { + recorded + .0 + .lock() + .unwrap_or_else(PoisonError::into_inner) + .push(line); + } + }); + } + // Wrap immediately so Drop::drop kills the process if readiness check fails let handle = ViceroyHandle { child, @@ -78,14 +127,15 @@ impl RuntimeEnvironment for FastlyViceroy { // adds redundant requests while the runtime is still starting up. super::wait_for_ready(&base_url, self.health_check_path(), false)?; - Ok(RuntimeProcess { - inner: Box::new(handle), - base_url, - }) + Ok(( + RuntimeProcess { + inner: Box::new(handle), + base_url, + }, + stdout, + )) } -} -impl FastlyViceroy { /// Path to the generated Viceroy configuration. /// /// This contains `[local_server]` configuration (backends, KV stores, diff --git a/crates/trusted-server-integration-tests/tests/html_eof_runtime.rs b/crates/trusted-server-integration-tests/tests/html_eof_runtime.rs index 6004da4fd8..f3dcf661a0 100644 --- a/crates/trusted-server-integration-tests/tests/html_eof_runtime.rs +++ b/crates/trusted-server-integration-tests/tests/html_eof_runtime.rs @@ -511,6 +511,79 @@ fn capture_body(reader: &mut impl std::io::Read) -> CapturedBody { } } +/// Operator response header the fixture config adds, which synthesized sets carry. +const OPERATOR_HEADER: (&str, &str) = ("x-example-operator", "fixture"); + +/// Route the test-utils fixture answers with a `Trailer` field. +const FRAMING_TRAILER_PATH: &str = "/integrations/html_finalization_fixture/framing-trailer"; + +/// Checks that `EdgeZero`'s detached and fallback responses carry the request's synthesized headers. +/// +/// A target of 8193 bytes gets 414, about 72 KiB of headers in 60 lines get +/// 431, and the fixture's `Trailer` route gets the framing 500. Each carries the +/// `x-ts-version` and `x-ts-fastly-version` the `routed` page carried, a geo +/// header, and the operator header. Viceroy answers 101 header lines with its +/// own 431 before the guest runs, so the case stays at 60. +fn head_limit_414_431_and_framing_500_carry_synthesized_headers( + client: &Client, + base_url: &str, + routed: &reqwest::header::HeaderMap, +) -> HarnessResult<()> { + // Both version headers must be on the routed page, so the comparisons below + // cannot pass on a header that is absent everywhere. + for name in ["x-ts-version", "x-ts-fastly-version"] { + assert!( + routed.get(name).is_some_and(|value| !value.is_empty()), + "should carry {name} on the routed page" + ); + } + let long_target = format!("{base_url}/{}", "a".repeat(8192)); + let mut large_head = client.get(format!("{base_url}/article")); + for index in 0..60 { + large_head = large_head.header(format!("x-example-{index:02}"), "a".repeat(1200)); + } + for (label, request, status) in [ + ("head-limit 414", client.get(long_target), 414), + ("head-limit 431", large_head, 431), + ( + "framing 500", + client.get(format!("{base_url}{FRAMING_TRAILER_PATH}")), + 500, + ), + ] { + let response = request.send().change_context(HarnessError::Http)?; + let headers = response.headers(); + assert_eq!( + response.status().as_u16(), + status, + "should answer the {label}" + ); + for name in ["x-ts-version", "x-ts-fastly-version"] { + assert_eq!( + headers.get(name), + routed.get(name), + "should carry the routed {name} on the {label}" + ); + } + assert!( + headers.contains_key("x-geo-info-available"), + "should carry the geo headers on the {label}" + ); + assert_eq!( + headers + .get(OPERATOR_HEADER.0) + .and_then(|value| value.to_str().ok()), + Some(OPERATOR_HEADER.1), + "should carry the operator header on the {label}" + ); + assert!( + !headers.contains_key("trailer"), + "should never send a Trailer field on the {label}" + ); + } + Ok(()) +} + #[test] fn truncated_response_preserves_received_bytes() { struct TruncatedResponse(usize); @@ -764,6 +837,12 @@ fn actual_runtime() -> HarnessResult<()> { "should prohibit shared caching" ); } + // Spec test 11's Viceroy legs need the fixture's test-utils route. + if adapter == "fastly" && case == "success" { + head_limit_414_431_and_framing_500_carry_synthesized_headers( + &client, &base_url, &headers, + )?; + } let events = origin.events(); let lookup = origin::lookup_for_request(&events, started, completed, case != "disabled")?; if let Some(lookup) = lookup { diff --git a/crates/trusted-server-integration-tests/tests/html_eof_runtime/config.rs b/crates/trusted-server-integration-tests/tests/html_eof_runtime/config.rs index 127ae83c40..2fc54a7ffe 100644 --- a/crates/trusted-server-integration-tests/tests/html_eof_runtime/config.rs +++ b/crates/trusted-server-integration-tests/tests/html_eof_runtime/config.rs @@ -79,6 +79,16 @@ pub(super) fn fixture_config( "html_finalization_timeout_ms".to_owned(), Value::Integer(750), ); + let (operator_name, operator_value) = super::OPERATOR_HEADER; + let mut response_headers = toml::map::Map::new(); + response_headers.insert( + operator_name.to_owned(), + Value::String(operator_value.to_owned()), + ); + table(&mut value, &[])?.insert( + "response_headers".to_owned(), + Value::Table(response_headers), + ); replace( &mut value, &["auction", "enabled"], diff --git a/crates/trusted-server-integration-tests/tests/integration.rs b/crates/trusted-server-integration-tests/tests/integration.rs index f99f8f6c0e..33846a5eda 100644 --- a/crates/trusted-server-integration-tests/tests/integration.rs +++ b/crates/trusted-server-integration-tests/tests/integration.rs @@ -2,14 +2,24 @@ mod common; mod environments; mod frameworks; +use base64::Engine as _; use common::ec::CookieVaryingOrigin; -use common::runtime::{RuntimeEnvironment, TestError, origin_port, wasm_binary_path}; +use common::runtime::{ + RuntimeEnvironment, RuntimeProcess, TestError, origin_port, wasm_binary_path, +}; use environments::{RUNTIME_ENVIRONMENTS, ReadyCheckOptions, wait_for_http_ready}; use error_stack::ResultExt as _; use frameworks::scenarios::EcScenario; use frameworks::{FRAMEWORKS, FrontendFramework}; +use reqwest::header::CONTENT_TYPE; +use std::io::{Read as _, Write as _}; +use std::net::{TcpListener, TcpStream}; +use std::process::{Child, ExitStatus, Stdio}; +use std::sync::{Arc, Mutex, MutexGuard, PoisonError, mpsc}; +use std::thread::{self, JoinHandle}; use std::time::Duration; use testcontainers::runners::SyncRunner as _; +use trusted_server_core::config_payload::{CONFIG_BLOB_KEY, DEFAULT_CONFIG_STORE_ID}; /// Initialize the logger once for the test binary. /// @@ -319,7 +329,7 @@ fn test_ec_lifecycle_fastly() { // Verify the post-cutover router is active before route-level scenarios. common::ec::assert_edgezero_entry_point(&process.base_url) - .expect("EdgeZero entry-point canary failed: TRACE did not return a router-level 405"); + .expect("should answer TRACE with a router-level 405 carrying the version headers"); for scenario in EcScenario::all() { log::info!(" Running EC scenario: {scenario:?}"); @@ -380,3 +390,2639 @@ fn trace_browser_workflow_spin() { init_logger(); common::trace_boundary::exercise_browser(common::trace_boundary::TraceBrowserRuntime::Spin); } + +// --------------------------------------------------------------------------- +// Runtime ingress baseline for the EdgeZero cut +// --------------------------------------------------------------------------- +// +// These cases record how each runtime hands request bodies and methods to +// Trusted Server today, so later changes can show exactly what moved. They +// all bind the fixed origin port, so each one holds `ORIGIN_PORT_LOCK` while +// it runs and stays correct without `--test-threads=1`. + +/// Largest `/auction` body the handler accepts. +/// +/// Mirrors `MAX_AUCTION_BODY_SIZE` in `trusted-server-core`. +const AUCTION_BODY_CAP: usize = 256 * 1024; + +/// Status the recording origin returns when a case compares statuses. +/// +/// Neither Trusted Server nor `EdgeZero` synthesizes 203, so a 203 at the +/// client can only have come from the origin. +const ORIGIN_PASS_THROUGH_STATUS: u16 = 203; + +/// Publisher path the byte-for-byte POST case sends its body to. +const INGRESS_FORM_PATH: &str = "/ingress-form"; + +/// Serializes the ingress cases that bind the fixed origin port. +static ORIGIN_PORT_LOCK: Mutex<()> = Mutex::new(()); + +/// Takes the origin-port lock for the rest of the calling case. +fn lock_origin_port() -> MutexGuard<'static, ()> { + // A failed case poisons the lock but frees the port, so later cases can + // still run. + ORIGIN_PORT_LOCK + .lock() + .unwrap_or_else(PoisonError::into_inner) +} + +/// A runtime the ingress cases start against the fixed origin port. +#[derive(Clone, Copy, Debug)] +enum IngressRuntime { + Axum, + Cloudflare, + Fastly, +} + +impl IngressRuntime { + /// Starts the runtime with the shared integration app config. + /// + /// # Panics + /// + /// Panics when the runtime's artifact or CLI is missing or it never + /// becomes ready. + fn spawn(self) -> RuntimeProcess { + match self { + Self::Axum => environments::axum::AxumDevServer + .spawn(&wasm_binary_path()) + .expect("should start the Axum dev server"), + Self::Cloudflare => { + let config_json = common::config::cloudflare_config_json(origin_port()) + .expect("should build the Cloudflare config binding"); + environments::cloudflare::CloudflareWorkers + .spawn_with_config_json(&config_json) + .expect("should start the Cloudflare Worker") + } + Self::Fastly => environments::fastly::FastlyViceroy + .spawn(&wasm_binary_path()) + .expect("should start Viceroy"), + } + } +} + +/// One request as the recording origin received it. +#[derive(Clone, Debug)] +struct OriginRequest { + method: String, + path: String, + body: Vec, + /// The `Content-Length` value the request was framed with, if any. + content_length: Option, + /// The `Transfer-Encoding` value the request was framed with, if any. + transfer_encoding: Option, + /// Every header field, names lowercased, in arrival order. + headers: Vec<(String, String)>, +} + +impl OriginRequest { + /// Returns the value of the first `name` field, compared case-insensitively. + fn header(&self, name: &str) -> Option<&str> { + self.headers + .iter() + .find(|(field, _)| field.eq_ignore_ascii_case(name)) + .map(|(_, value)| value.as_str()) + } +} + +/// Builds the complete HTTP response a [`RecordingOrigin`] sends for one request. +type OriginResponder = Arc Vec + Send + Sync>; + +/// Loopback origin that records every request and answers it from a responder. +/// +/// Unlike [`common::ec::MinimalOrigin`], it reads the whole request body and +/// decodes chunked framing, so a case can compare what reached the origin +/// with what the client sent. +struct RecordingOrigin { + port: u16, + requests: Arc>>, + shutdown_tx: mpsc::Sender<()>, + handle: Option>, +} + +impl RecordingOrigin { + /// Starts the origin on `127.0.0.1:{port}`, answering every request with + /// `status` and a small HTML page. + /// + /// # Panics + /// + /// Panics if the port is already in use. + fn start(port: u16, status: u16) -> Self { + let listener = TcpListener::bind(("127.0.0.1", port)).expect("should bind origin port"); + Self::serve( + listener, + Arc::new(move |request| html_response(status, request.method == "HEAD")), + ) + } + + /// Serves `listener`, answering each request with what `respond` builds. + /// + /// # Panics + /// + /// Panics if the listener cannot be configured. + fn serve(listener: TcpListener, respond: OriginResponder) -> Self { + let port = listener + .local_addr() + .expect("should read the origin address") + .port(); + listener + .set_nonblocking(true) + .expect("should set listener nonblocking"); + let requests = Arc::new(Mutex::new(Vec::new())); + let recorded = Arc::clone(&requests); + let (shutdown_tx, shutdown_rx) = mpsc::channel::<()>(); + + let handle = thread::spawn(move || { + while matches!(shutdown_rx.try_recv(), Err(mpsc::TryRecvError::Empty)) { + match listener.accept() { + Ok((stream, _addr)) => { + let recorded = Arc::clone(&recorded); + let respond = Arc::clone(&respond); + // One thread per connection, so an idle pooled + // connection cannot hold up the next request. + thread::spawn(move || { + serve_origin_connection(stream, respond.as_ref(), &recorded); + }); + } + Err(error) if error.kind() == std::io::ErrorKind::WouldBlock => { + thread::sleep(Duration::from_millis(10)); + } + Err(_) => break, + } + } + }); + + Self { + port, + requests, + shutdown_tx, + handle: Some(handle), + } + } + + /// Returns every request recorded so far, in arrival order. + fn requests(&self) -> Vec { + self.requests + .lock() + .unwrap_or_else(PoisonError::into_inner) + .clone() + } +} + +impl Drop for RecordingOrigin { + fn drop(&mut self) { + let _ = self.shutdown_tx.send(()); + if let Some(handle) = self.handle.take() { + let _ = handle.join(); + } + } +} + +/// Reads one request from `stream`, records it, and answers with what `respond` builds. +fn serve_origin_connection( + mut stream: TcpStream, + respond: &(dyn Fn(&OriginRequest) -> Vec + Send + Sync), + recorded: &Mutex>, +) { + // Some platforms hand out accepted sockets in the listener's nonblocking + // mode; the request read below must block, with a bound. + if stream.set_nonblocking(false).is_err() + || stream + .set_read_timeout(Some(Duration::from_secs(10))) + .is_err() + { + return; + } + let Some(request) = read_origin_request(&mut stream) else { + return; + }; + let response = respond(&request); + recorded + .lock() + .unwrap_or_else(PoisonError::into_inner) + .push(request); + + let _ = stream.write_all(&response); + let _ = stream.flush(); +} + +/// A `status` answer with a small HTML page, or only its head for `HEAD`. +fn html_response(status: u16, head_only: bool) -> Vec { + let body = b"

Example origin

"; + let reason = http::StatusCode::from_u16(status) + .ok() + .and_then(|code| code.canonical_reason()) + .unwrap_or("Example"); + let mut response = format!( + "HTTP/1.1 {status} {reason}\r\nContent-Type: text/html; charset=utf-8\r\nContent-Length: {}\r\nConnection: close\r\n\r\n", + body.len() + ) + .into_bytes(); + if !head_only { + response.extend_from_slice(body); + } + response +} + +/// Reads one HTTP/1.1 request, decoding a `Content-Length` or chunked body. +/// +/// Returns `None` when the connection closes, times out, or sends a malformed +/// head before the request is complete. +fn read_origin_request(stream: &mut TcpStream) -> Option { + let mut received = Vec::new(); + let head_end = loop { + if let Some(position) = find_subslice(&received, b"\r\n\r\n") { + break position; + } + read_more(stream, &mut received)?; + }; + let head = std::str::from_utf8(&received[..head_end]).ok()?; + let mut lines = head.split("\r\n"); + let mut request_line = lines.next()?.split(' '); + let method = request_line.next()?.to_owned(); + let path = request_line.next()?.to_owned(); + let mut content_length = 0; + let mut chunked = false; + let mut declared_length = None; + let mut transfer_encoding = None; + let mut headers = Vec::new(); + for line in lines { + let (name, value) = line.split_once(':')?; + let value = value.trim(); + headers.push((name.to_ascii_lowercase(), value.to_owned())); + if name.eq_ignore_ascii_case("content-length") { + content_length = value.parse().ok()?; + declared_length = Some(value.to_owned()); + } else if name.eq_ignore_ascii_case("transfer-encoding") { + chunked = value.eq_ignore_ascii_case("chunked"); + transfer_encoding = Some(value.to_owned()); + } + } + + let mut body = received.split_off(head_end + 4); + let body = if chunked { + loop { + if let Some(decoded) = decode_chunked(&body) { + break decoded; + } + read_more(stream, &mut body)?; + } + } else { + while body.len() < content_length { + read_more(stream, &mut body)?; + } + body.truncate(content_length); + body + }; + + Some(OriginRequest { + method, + path, + body, + content_length: declared_length, + transfer_encoding, + headers, + }) +} + +/// Appends the next read from `stream` to `buffer`. +/// +/// Returns `None` at end of stream or on a read error, including a timeout. +fn read_more(stream: &mut TcpStream, buffer: &mut Vec) -> Option<()> { + let mut chunk = [0_u8; 8192]; + let read = stream.read(&mut chunk).ok()?; + if read == 0 { + return None; + } + buffer.extend_from_slice(&chunk[..read]); + Some(()) +} + +/// Decodes a complete chunked body, or returns `None` while it is incomplete. +fn decode_chunked(mut bytes: &[u8]) -> Option> { + let mut body = Vec::new(); + loop { + let line_end = find_subslice(bytes, b"\r\n")?; + let size_line = std::str::from_utf8(&bytes[..line_end]).ok()?; + let size = usize::from_str_radix(size_line.split(';').next()?.trim(), 16).ok()?; + bytes = &bytes[line_end + 2..]; + if size == 0 { + // Trailer fields, if any, end at an empty line. + loop { + let field_end = find_subslice(bytes, b"\r\n")?; + if field_end == 0 { + return Some(body); + } + bytes = &bytes[field_end + 2..]; + } + } + let data_end = size.checked_add(2)?; + if bytes.get(size..data_end)? != b"\r\n" { + return None; + } + body.extend_from_slice(&bytes[..size]); + bytes = &bytes[data_end..]; + } +} + +/// Returns the offset of the first `needle` in `haystack`. +fn find_subslice(haystack: &[u8], needle: &[u8]) -> Option { + haystack + .windows(needle.len()) + .position(|window| window == needle) +} + +/// Builds a valid auction request exactly one byte over [`AUCTION_BODY_CAP`]. +/// +/// The padding rides in `config`, which the handler accepts and filters. A +/// runtime that skipped the cap would answer 200, and one that delivered an +/// empty body would answer 400, so only an enforced cap answers 413. +fn over_cap_auction_body() -> Vec { + let prefix = r#"{"adUnits":[{"code":"ingress-slot","mediaTypes":{"banner":{"sizes":[[300,250]]}}}],"config":{"padding":""#; + let suffix = r#""}}"#; + let padding = "a".repeat(AUCTION_BODY_CAP + 1 - prefix.len() - suffix.len()); + let body = format!("{prefix}{padding}{suffix}").into_bytes(); + assert_eq!( + body.len(), + AUCTION_BODY_CAP + 1, + "should build a body one byte over the auction cap" + ); + body +} + +/// A `POST /auction` with a JSON body reaches the handler and gets an +/// auction response, not the parse error an empty body would cause. +fn post_auction_with_json_body_reaches_handler(runtime: IngressRuntime) { + init_logger(); + let _port = lock_origin_port(); + let _origin = RecordingOrigin::start(origin_port(), 200); + let process = runtime.spawn(); + let request_body = serde_json::json!({ + "adUnits": [{ + "code": "ingress-slot", + "mediaTypes": { "banner": { "sizes": [[300, 250]] } } + }] + }); + + let response = reqwest::blocking::Client::new() + .post(format!("{}/auction", process.base_url)) + .json(&request_body) + .send() + .expect("should send the auction request"); + + let status = response.status().as_u16(); + let body = response.text().expect("should read the auction response"); + assert_eq!( + status, 200, + "{runtime:?} should hand the JSON body to the auction handler: {body}" + ); + let auction_response: serde_json::Value = + serde_json::from_str(&body).expect("should return an OpenRTB JSON response"); + assert!( + auction_response["id"] + .as_str() + .is_some_and(|id| !id.is_empty()), + "{runtime:?} auction response should carry the auction id: {body}" + ); +} + +/// A publisher `POST` body reaches the origin unchanged, every byte value +/// included. +fn publisher_post_body_reaches_origin_byte_for_byte(runtime: IngressRuntime) { + init_logger(); + let _port = lock_origin_port(); + let origin = RecordingOrigin::start(origin_port(), 200); + let process = runtime.spawn(); + let sent: Vec = (0..=u8::MAX).cycle().take(4096).collect(); + + let response = reqwest::blocking::Client::new() + .post(format!("{}{INGRESS_FORM_PATH}", process.base_url)) + .header(CONTENT_TYPE, "application/octet-stream") + .body(sent.clone()) + .send() + .expect("should send the publisher POST"); + + assert_eq!( + response.status().as_u16(), + 200, + "{runtime:?} should return the origin's answer to the publisher POST" + ); + let received = origin + .requests() + .into_iter() + .find(|request| request.method == "POST" && request.path == INGRESS_FORM_PATH) + .expect("should record the publisher POST at the origin"); + let first_difference = received + .body + .iter() + .zip(&sent) + .position(|(received, sent)| received != sent); + assert!( + received.body == sent, + "{runtime:?} should forward all {} body bytes unchanged; the origin received {} bytes, first difference at {first_difference:?}", + sent.len(), + received.body.len() + ); +} + +/// A request for `/` returns the status the origin answered with. +fn root_returns_origin_status(runtime: IngressRuntime, method: &reqwest::Method) { + init_logger(); + let _port = lock_origin_port(); + let _origin = RecordingOrigin::start(origin_port(), ORIGIN_PASS_THROUGH_STATUS); + let process = runtime.spawn(); + + let response = reqwest::blocking::Client::new() + .request(method.clone(), format!("{}/", process.base_url)) + .send() + .expect("should send the root request"); + + assert_eq!( + response.status().as_u16(), + ORIGIN_PASS_THROUGH_STATUS, + "{runtime:?} {method} / should return the origin's status" + ); +} + +/// A chunked `POST /auction` one byte over the cap gets a `text/plain` 413. +fn chunked_post_auction_over_cap_returns_413(runtime: IngressRuntime) { + init_logger(); + let _port = lock_origin_port(); + let _origin = RecordingOrigin::start(origin_port(), 200); + let process = runtime.spawn(); + + // A reader body has no known length, so reqwest frames it as chunked and + // sends no `Content-Length`. The declared-length check cannot reject it; + // only reading the body can. + let response = reqwest::blocking::Client::new() + .post(format!("{}/auction", process.base_url)) + .header(CONTENT_TYPE, "application/json") + .body(reqwest::blocking::Body::new(std::io::Cursor::new( + over_cap_auction_body(), + ))) + .send() + .expect("should send the chunked auction request"); + + let status = response.status().as_u16(); + let content_type = response + .headers() + .get(CONTENT_TYPE) + .and_then(|value| value.to_str().ok()) + .unwrap_or_default() + .to_owned(); + let body = response.text().expect("should read the rejection body"); + assert_eq!( + status, 413, + "{runtime:?} should reject a chunked auction body over the cap: {body}" + ); + assert!( + content_type.starts_with("text/plain"), + "{runtime:?} should answer the over-cap body with text/plain, got `{content_type}`" + ); + assert!( + !body.is_empty(), + "{runtime:?} should explain the over-cap rejection in the body" + ); +} + +#[test] +#[ignore = "requires Viceroy, the generated Viceroy config, and the release Fastly WASM"] +fn post_auction_with_json_body_reaches_handler_fastly() { + post_auction_with_json_body_reaches_handler(IngressRuntime::Fastly); +} + +#[test] +#[ignore = "requires the pre-built trusted-server-axum binary"] +fn post_auction_with_json_body_reaches_handler_axum() { + post_auction_with_json_body_reaches_handler(IngressRuntime::Axum); +} + +#[test] +#[ignore = "requires Viceroy, the generated Viceroy config, and the release Fastly WASM"] +fn publisher_post_body_reaches_origin_byte_for_byte_fastly() { + publisher_post_body_reaches_origin_byte_for_byte(IngressRuntime::Fastly); +} + +#[test] +#[ignore = "requires the `wrangler` CLI in $PATH and a prebuilt Cloudflare Workers bundle (run build.sh first); the test starts wrangler dev automatically"] +fn publisher_post_body_reaches_origin_byte_for_byte_cloudflare() { + publisher_post_body_reaches_origin_byte_for_byte(IngressRuntime::Cloudflare); +} + +#[test] +#[ignore = "requires the pre-built trusted-server-axum binary"] +fn publisher_post_body_reaches_origin_byte_for_byte_axum() { + publisher_post_body_reaches_origin_byte_for_byte(IngressRuntime::Axum); +} + +#[test] +#[ignore = "requires Viceroy, the generated Viceroy config, and the release Fastly WASM"] +fn head_root_returns_origin_status_fastly() { + root_returns_origin_status(IngressRuntime::Fastly, &reqwest::Method::HEAD); +} + +#[test] +#[ignore = "requires the `wrangler` CLI in $PATH and a prebuilt Cloudflare Workers bundle (run build.sh first); the test starts wrangler dev automatically"] +fn head_root_returns_origin_status_cloudflare() { + root_returns_origin_status(IngressRuntime::Cloudflare, &reqwest::Method::HEAD); +} + +#[test] +#[ignore = "requires the pre-built trusted-server-axum binary"] +fn head_root_returns_origin_status_axum() { + root_returns_origin_status(IngressRuntime::Axum, &reqwest::Method::HEAD); +} + +#[test] +#[ignore = "requires Viceroy, the generated Viceroy config, and the release Fastly WASM"] +fn options_root_returns_origin_status_fastly() { + root_returns_origin_status(IngressRuntime::Fastly, &reqwest::Method::OPTIONS); +} + +#[test] +#[ignore = "requires the `wrangler` CLI in $PATH and a prebuilt Cloudflare Workers bundle (run build.sh first); the test starts wrangler dev automatically"] +fn options_root_returns_origin_status_cloudflare() { + root_returns_origin_status(IngressRuntime::Cloudflare, &reqwest::Method::OPTIONS); +} + +#[test] +#[ignore = "requires the pre-built trusted-server-axum binary"] +fn options_root_returns_origin_status_axum() { + root_returns_origin_status(IngressRuntime::Axum, &reqwest::Method::OPTIONS); +} + +#[test] +#[ignore = "requires Viceroy, the generated Viceroy config, and the release Fastly WASM"] +fn chunked_post_auction_over_cap_returns_413_fastly() { + chunked_post_auction_over_cap_returns_413(IngressRuntime::Fastly); +} + +#[test] +#[ignore = "requires the `wrangler` CLI in $PATH and a prebuilt Cloudflare Workers bundle (run build.sh first); the test starts wrangler dev automatically"] +fn chunked_post_auction_over_cap_returns_413_cloudflare() { + chunked_post_auction_over_cap_returns_413(IngressRuntime::Cloudflare); +} + +#[test] +#[ignore = "requires the pre-built trusted-server-axum binary"] +fn chunked_post_auction_over_cap_returns_413_axum() { + chunked_post_auction_over_cap_returns_413(IngressRuntime::Axum); +} + +// --------------------------------------------------------------------------- +// First-party proxy redirect policy on the Axum binary +// --------------------------------------------------------------------------- +// +// The scripted redirect tests in `trusted-server-core` cover the policy on +// every adapter's code path. This case runs it again on the real Axum binary, +// whose `EdgeZero` client follows the chain against a loopback origin. + +/// Answers the paths of the first-party proxy redirect chain. +/// +/// `/start` redirects to `/final` on the same origin, `/to-blocked-host` to the +/// same origin under `localhost`, a host outside the proxy allowlist, and +/// `/to-userinfo` to a `Location` that carries userinfo. A policy that let the +/// blocked hop through would reach this origin at `/blocked`. +fn redirect_chain_response(origin_port: u16, request: &OriginRequest) -> Vec { + let redirect = |location: &str| { + format!( + "HTTP/1.1 302 Found\r\nLocation: {location}\r\nContent-Length: 0\r\nConnection: close\r\n\r\n" + ) + .into_bytes() + }; + match origin_request_path(request) { + "/start" => redirect("/final"), + "/to-blocked-host" => redirect(&format!("http://localhost:{origin_port}/blocked")), + "/to-userinfo" => redirect(&format!( + "http://fictional:secret@127.0.0.1:{origin_port}/userinfo" + )), + "/final" => b"HTTP/1.1 200 OK\r\nContent-Type: text/plain\r\nContent-Length: 14\r\nConnection: close\r\n\r\nfollowed-final".to_vec(), + _ => html_response(404, request.method == "HEAD"), + } +} + +/// The path of `request`'s target, without its query. +fn origin_request_path(request: &OriginRequest) -> &str { + request.path.split('?').next().unwrap_or_default() +} + +/// The path and query of a signed `href` from `/first-party/sign`. +/// +/// The route answers with an absolute URL on `publisher.public_origin`, which +/// need not be the runtime's listener, so callers resend it to their own base URL. +fn signed_href_path_and_query(href: &str) -> String { + let url = url::Url::parse(href).expect("should return an absolute signed href"); + url[url::Position::BeforePath..url::Position::AfterQuery].to_owned() +} + +/// Signs `target` through the binary's `/first-party/sign` route and fetches +/// the first-party proxy URL it returns. +fn fetch_signed_proxy_url( + client: &reqwest::blocking::Client, + base_url: &str, + target: &str, +) -> reqwest::blocking::Response { + let signed: serde_json::Value = client + .get(format!("{base_url}/first-party/sign")) + .query(&[("url", target)]) + .send() + .expect("should send the sign request") + .error_for_status() + .expect("should sign the loopback target") + .json() + .expect("should return the signed proxy URL"); + let href = signed["href"] + .as_str() + .expect("should return the first-party proxy href"); + client + .get(format!("{base_url}{}", signed_href_path_and_query(href))) + .send() + .expect("should send the first-party proxy request") +} + +#[test] +#[ignore = "requires the pre-built trusted-server-axum binary"] +fn first_party_proxy_redirect_policy_runs_on_axum_binary() { + init_logger(); + let listener = + TcpListener::bind(("127.0.0.1", 0)).expect("should bind the redirect chain origin"); + let port = listener + .local_addr() + .expect("should read the redirect chain origin address") + .port(); + let origin = RecordingOrigin::serve( + listener, + Arc::new(move |request| redirect_chain_response(port, request)), + ); + let envelope = + common::config::proxy_allowlist_app_config_envelope(origin_port(), &["127.0.0.1"]) + .expect("should build the proxy allowlist app config"); + let process = environments::axum::AxumDevServer + .spawn_with_app_config(&envelope, &[]) + .expect("should start the Axum dev server"); + let client = reqwest::blocking::Client::builder() + .redirect(reqwest::redirect::Policy::none()) + .build() + .expect("should build a client that returns redirects"); + let target = |path: &str| format!("http://127.0.0.1:{}{path}", origin.port); + + let followed = fetch_signed_proxy_url(&client, &process.base_url, &target("/start")); + assert_eq!( + followed.status().as_u16(), + 200, + "should follow a permitted hop to the final origin" + ); + assert_eq!( + followed.text().expect("should read the followed body"), + "followed-final", + "should serve the final origin's body" + ); + let blocked = fetch_signed_proxy_url(&client, &process.base_url, &target("/to-blocked-host")); + assert_eq!( + blocked.status().as_u16(), + 403, + "should answer a hop outside proxy.allowed_domains with the allowlist violation" + ); + let userinfo = fetch_signed_proxy_url(&client, &process.base_url, &target("/to-userinfo")); + assert_eq!( + userinfo.status().as_u16(), + 502, + "should answer a Location with userinfo with the proxy's 502" + ); + + let requests = origin.requests(); + let paths: Vec<&str> = requests.iter().map(origin_request_path).collect(); + assert_eq!( + paths, + vec!["/start", "/final", "/to-blocked-host", "/to-userinfo"], + "should send only the permitted follow-up hop" + ); +} + +// --------------------------------------------------------------------------- +// SigV4 asset host check for the EdgeZero cut +// --------------------------------------------------------------------------- +// +// `EdgeZero` drops the `Host` header Trusted Server signs and sends the URI +// authority instead, so these cases check that the authority, path, and query +// an S3 origin receives are exactly what Trusted Server signed. S3 routes need +// an `https` origin, so the origin is a loopback TLS server behind a test CA, +// and `SSL_CERT_FILE` points Viceroy's native root store at that CA. + +/// Region the `SigV4` asset routes sign for. +const SIGV4_REGION: &str = "us-east-1"; + +/// Access key ID behind the routes' `integration_s3_access_key_id` secret. +const SIGV4_ACCESS_KEY_ID: &str = "AKIAIOSFODNN7EXAMPLE"; + +/// Secret access key behind the routes' `integration_s3_secret_access_key` secret. +const SIGV4_SECRET_ACCESS_KEY: &str = "wJalrXUtnFEMI/K7MDENG+bPxRfiCYEXAMPLEKEY"; + +/// One request head as the TLS asset origin received it. +#[derive(Clone, Debug)] +struct SignedAssetRequest { + method: String, + target: String, + headers: Vec<(String, String)>, +} + +impl SignedAssetRequest { + fn header(&self, name: &str) -> Option<&str> { + self.headers + .iter() + .find(|(header_name, _)| header_name.eq_ignore_ascii_case(name)) + .map(|(_, value)| value.as_str()) + } +} + +/// Loopback HTTPS origin, trusted through a test CA, that records every request +/// head and answers each with a small image. +struct TlsAssetOrigin { + port: u16, + ca_file: tempfile::NamedTempFile, + requests: Arc>>, + shutdown_tx: mpsc::Sender<()>, + handle: Option>, +} + +impl TlsAssetOrigin { + /// Starts the origin on an ephemeral loopback port. + /// + /// # Panics + /// + /// Panics if the test certificates or the listener cannot be set up. + fn start() -> Self { + let ca_key = rcgen::KeyPair::generate().expect("should generate the test CA key"); + let mut ca_params = rcgen::CertificateParams::new(Vec::::new()) + .expect("should create the test CA parameters"); + ca_params + .distinguished_name + .push(rcgen::DnType::CommonName, "Example test CA"); + ca_params.is_ca = rcgen::IsCa::Ca(rcgen::BasicConstraints::Unconstrained); + ca_params.key_usages = vec![rcgen::KeyUsagePurpose::KeyCertSign]; + let ca_cert = ca_params + .self_signed(&ca_key) + .expect("should self-sign the test CA"); + let leaf_key = rcgen::KeyPair::generate().expect("should generate the origin key"); + let leaf_cert = rcgen::CertificateParams::new(vec!["127.0.0.1".to_owned()]) + .expect("should create the origin certificate parameters") + .signed_by(&leaf_key, &ca_cert, &ca_key) + .expect("should sign the origin certificate"); + let provider = Arc::new(rustls::crypto::ring::default_provider()); + let tls_config = Arc::new( + rustls::ServerConfig::builder_with_provider(provider) + .with_safe_default_protocol_versions() + .expect("should select TLS versions") + .with_no_client_auth() + .with_single_cert( + vec![leaf_cert.der().clone()], + rustls::pki_types::PrivateKeyDer::try_from(leaf_key.serialize_der()) + .expect("should encode the origin key"), + ) + .expect("should build the origin TLS configuration"), + ); + let mut ca_file = tempfile::NamedTempFile::new().expect("should create the CA file"); + ca_file + .write_all(ca_cert.pem().as_bytes()) + .expect("should write the CA certificate"); + + let listener = TcpListener::bind(("127.0.0.1", 0)).expect("should bind the TLS origin"); + let port = listener + .local_addr() + .expect("should read the TLS origin address") + .port(); + listener + .set_nonblocking(true) + .expect("should set listener nonblocking"); + let requests = Arc::new(Mutex::new(Vec::new())); + let recorded = Arc::clone(&requests); + let (shutdown_tx, shutdown_rx) = mpsc::channel::<()>(); + let handle = thread::spawn(move || { + while matches!(shutdown_rx.try_recv(), Err(mpsc::TryRecvError::Empty)) { + match listener.accept() { + Ok((stream, _addr)) => { + let recorded = Arc::clone(&recorded); + let tls_config = Arc::clone(&tls_config); + thread::spawn(move || { + serve_tls_asset_connection(stream, tls_config, &recorded); + }); + } + Err(error) if error.kind() == std::io::ErrorKind::WouldBlock => { + thread::sleep(Duration::from_millis(10)); + } + Err(_) => break, + } + } + }); + + Self { + port, + ca_file, + requests, + shutdown_tx, + handle: Some(handle), + } + } + + /// Returns every request recorded so far, in arrival order. + fn requests(&self) -> Vec { + self.requests + .lock() + .unwrap_or_else(PoisonError::into_inner) + .clone() + } +} + +impl Drop for TlsAssetOrigin { + fn drop(&mut self) { + let _ = self.shutdown_tx.send(()); + if let Some(handle) = self.handle.take() { + let _ = handle.join(); + } + } +} + +/// Completes the TLS handshake, records one request head, and answers it. +fn serve_tls_asset_connection( + stream: TcpStream, + tls_config: Arc, + recorded: &Mutex>, +) { + if stream.set_nonblocking(false).is_err() + || stream + .set_read_timeout(Some(Duration::from_secs(10))) + .is_err() + { + return; + } + let Ok(connection) = rustls::ServerConnection::new(tls_config) else { + return; + }; + let mut tls = rustls::StreamOwned::new(connection, stream); + let mut received = Vec::new(); + let head_end = loop { + if let Some(position) = find_subslice(&received, b"\r\n\r\n") { + break position; + } + let mut chunk = [0_u8; 4096]; + match tls.read(&mut chunk) { + Ok(read) if read > 0 => received.extend_from_slice(&chunk[..read]), + _ => return, + } + }; + let Ok(head) = std::str::from_utf8(&received[..head_end]) else { + return; + }; + let mut lines = head.split("\r\n"); + let mut request_line = lines.next().unwrap_or_default().split(' '); + let (Some(method), Some(target)) = (request_line.next(), request_line.next()) else { + return; + }; + let headers = lines + .filter_map(|line| line.split_once(':')) + .map(|(name, value)| (name.trim().to_owned(), value.trim().to_owned())) + .collect(); + let head_only = method == "HEAD"; + recorded + .lock() + .unwrap_or_else(PoisonError::into_inner) + .push(SignedAssetRequest { + method: method.to_owned(), + target: target.to_owned(), + headers, + }); + + let body = b"example-image"; + let response_head = format!( + "HTTP/1.1 200 OK\r\nContent-Type: image/jpeg\r\nContent-Length: {}\r\nConnection: close\r\n\r\n", + body.len() + ); + let _ = tls.write_all(response_head.as_bytes()); + if !head_only { + let _ = tls.write_all(body); + } + tls.conn.send_close_notify(); + let _ = tls.flush(); +} + +/// The Viceroy integration config with `envelope` as its app config. +fn viceroy_config_with_app_config(envelope: &str) -> String { + include_str!("../fixtures/configs/viceroy-template.toml").replace( + " # GENERATED_TRUSTED_SERVER_CONFIG_STORES", + &format!( + " [local_server.config_stores.{DEFAULT_CONFIG_STORE_ID}]\n format = \"inline-toml\"\n [local_server.config_stores.{DEFAULT_CONFIG_STORE_ID}.contents]\n {CONFIG_BLOB_KEY} = '''{envelope}'''" + ), + ) +} + +/// Asserts that `request` carries the `Host`, path, and query Trusted Server signed. +/// +/// The recorded `Host` and request target must equal the expected values, and +/// recomputing the signature from what the origin received must reproduce the +/// request's `Authorization`, which holds only if those are the signed values. +fn assert_sent_as_signed(request: &SignedAssetRequest, expected_host: &str, expected_target: &str) { + let host = request + .header("host") + .expect("should record the Host header"); + assert_eq!( + (host, request.target.as_str()), + (expected_host, expected_target), + "should send the {} to the signed authority and target", + request.method + ); + let authorization = request + .header("authorization") + .expect("should send a SigV4 Authorization header"); + let amz_date = request + .header("x-amz-date") + .expect("should send the signing timestamp"); + let method = + http::Method::from_bytes(request.method.as_bytes()).expect("should record a valid method"); + let recomputed = trusted_server_core::test_support::s3_sigv4_authorization( + &method, + host, + &request.target, + amz_date, + SIGV4_REGION, + SIGV4_ACCESS_KEY_ID, + SIGV4_SECRET_ACCESS_KEY, + ) + .expect("should recompute the signature from the received request"); + assert_eq!( + authorization, recomputed, + "should sign exactly the Host, path, and query the {} reached the origin with", + request.method + ); +} + +#[test] +#[ignore = "requires Viceroy and the release Fastly WASM"] +fn sigv4_signed_asset_reaches_origin_with_signed_host_path_and_query() { + init_logger(); + let origin = TlsAssetOrigin::start(); + let expected_host = format!("127.0.0.1:{}", origin.port); + let envelope = common::config::sigv4_asset_app_config_envelope( + origin_port(), + &format!("https://{expected_host}"), + ) + .expect("should build the SigV4 asset app config"); + let config_dir = tempfile::tempdir().expect("should create the Viceroy config directory"); + let config_path = config_dir.path().join("viceroy.toml"); + std::fs::write(&config_path, viceroy_config_with_app_config(&envelope)) + .expect("should write the Viceroy config"); + + let image_status = temp_env::with_vars( + [ + ("SSL_CERT_FILE", Some(origin.ca_file.path().as_os_str())), + ("VICEROY_CONFIG_PATH", Some(config_path.as_os_str())), + ], + || { + let process = environments::fastly::FastlyViceroy + .spawn(&wasm_binary_path()) + .expect("should start Viceroy"); + let client = reqwest::blocking::Client::new(); + let plain = client + .get(format!( + "{}/.sigv4-assets/photos/cat%20one.jpg?version=2&name=a%2Fb", + process.base_url + )) + .send() + .expect("should send the plain asset request"); + assert_eq!( + plain.status().as_u16(), + 200, + "should serve the signed asset from the origin" + ); + // Viceroy cannot run Image Optimizer, so only the preflight before it is checked. + client + .get(format!( + "{}/.sigv4-images/photos/dog.jpg?profile=default", + process.base_url + )) + .send() + .expect("should send the optimized asset request") + .status() + }, + ); + log::info!("optimized SigV4 asset answered {image_status}"); + + let requests = origin.requests(); + let plain = requests + .iter() + .find(|request| request.method == "GET" && request.target.starts_with("/.sigv4-assets/")) + .expect("should record the signed asset GET"); + assert_sent_as_signed( + plain, + &expected_host, + "/.sigv4-assets/photos/cat%20one.jpg?version=2&name=a%2Fb", + ); + let preflight = requests + .iter() + .find(|request| request.method == "HEAD" && request.target.starts_with("/.sigv4-images/")) + .expect("should record the signed S3 HEAD preflight"); + assert_sent_as_signed(preflight, &expected_host, "/.sigv4-images/photos/dog.jpg"); +} + +// --------------------------------------------------------------------------- +// Auction fan-out against loopback bidders (spec test 5) +// --------------------------------------------------------------------------- + +/// Impression the runtime auction cases request. +const AUCTION_SLOT: &str = "auction-slot"; + +/// A TCF v2 `euconsent-v2` cookie that grants Purpose 1. +/// +/// Runtimes without a geo source treat the jurisdiction as unknown, where the +/// server-side auction needs Purpose 1. The construction matches the HTML +/// end-of-file harness's consent cookie, which lives in another test binary. +fn tcf_purpose_one_cookie() -> String { + let mut bits = [0_u8; 29]; + let now = std::time::SystemTime::now() + .duration_since(std::time::UNIX_EPOCH) + .expect("should have a current fixture clock") + .as_millis() + / 100; + for (offset, width, value) in [ + (0, 6, 2), + (6, 36, now), + (42, 36, now), + (108, 6, 4), + (114, 6, 13), + (152, 1, 1), + ] { + for index in 0..width { + if value & (1 << (width - index - 1)) != 0 { + bits[(offset + index) / 8] |= 1 << (7 - (offset + index) % 8); + } + } + } + format!( + "euconsent-v2={}", + base64::engine::general_purpose::URL_SAFE_NO_PAD.encode(bits) + ) +} + +/// A loopback bidder on its own port that answers each bid request with one bid. +/// +/// It waits `delay` before answering and records every request it receives, +/// framing included. +fn start_loopback_bidder(seat: &str, price: f64, delay: Duration) -> RecordingOrigin { + let body = serde_json::to_vec(&serde_json::json!({ + "seatbid": [{"seat": seat, "bid": [{ + "id": format!("{seat}-bid"), + "impid": AUCTION_SLOT, + "price": price, + "adm": "
loopback bid
", + "w": 300, + "h": 250 + }]}] + })) + .expect("should serialize the loopback bid"); + let listener = TcpListener::bind(("127.0.0.1", 0)).expect("should bind a loopback bidder port"); + RecordingOrigin::serve( + listener, + Arc::new(move |_request| { + thread::sleep(delay); + let mut response = format!( + "HTTP/1.1 200 OK\r\nContent-Type: application/json\r\nContent-Length: {}\r\nConnection: close\r\n\r\n", + body.len() + ) + .into_bytes(); + response.extend_from_slice(&body); + response + }), + ) +} + +/// Configures one standard provider per loopback bidder, at `/bid` on its port. +fn loopback_bidder_fixtures( + bidders: &[(&str, &RecordingOrigin, Duration)], +) -> Vec { + bidders + .iter() + .map( + |(id, bidder, timeout)| common::config::AuctionBidderFixture { + id: (*id).to_owned(), + endpoint: format!("http://127.0.0.1:{}/bid", bidder.port), + timeout_ms: u32::try_from(timeout.as_millis()) + .expect("should fit the provider timeout in u32"), + }, + ) + .collect() +} + +/// Sends one `/auction` request for [`AUCTION_SLOT`] and returns its `OpenRTB` response. +fn post_auction(base_url: &str) -> serde_json::Value { + let response = reqwest::blocking::Client::new() + .post(format!("{base_url}/auction")) + .header("cookie", tcf_purpose_one_cookie()) + .header("cf-ipcountry", "US") + .json(&serde_json::json!({ + "adUnits": [{ + "code": AUCTION_SLOT, + "mediaTypes": { "banner": { "sizes": [[300, 250]] } } + }] + })) + .timeout(Duration::from_secs(30)) + .send() + .expect("should send the auction request"); + let status = response.status().as_u16(); + let body = response.text().expect("should read the auction response"); + assert_eq!(status, 200, "should complete the auction: {body}"); + serde_json::from_str(&body).expect("should return an OpenRTB JSON response") +} + +/// The provider summary the auction response carries for `provider`. +fn provider_detail<'a>(response: &'a serde_json::Value, provider: &str) -> &'a serde_json::Value { + let detail = response["ext"]["orchestrator"]["provider_details"] + .as_array() + .expect("should list provider details") + .iter() + .find(|detail| detail["name"] == provider); + assert!( + detail.is_some(), + "should report provider {provider}: {response}" + ); + detail.expect("should have found the provider detail") +} + +/// Asserts every bidder received `auctions` requests with a JSON bid request body, +/// and logs the framing each request arrived with. +fn assert_bidders_received(runtime: &str, bidders: &[&RecordingOrigin], auctions: usize) { + for bidder in bidders { + let requests = bidder.requests(); + assert_eq!( + requests.len(), + auctions, + "{runtime} should send one bid request per auction to port {}", + bidder.port + ); + for request in &requests { + log::info!( + "{runtime} bid request framing: content-length {:?}, transfer-encoding {:?}", + request.content_length, + request.transfer_encoding + ); + assert_eq!( + (request.method.as_str(), request.path.as_str()), + ("POST", "/bid"), + "{runtime} should POST the bid request to the provider path" + ); + let bid_request = serde_json::from_slice::(&request.body); + assert!( + bid_request.is_ok(), + "{runtime} should deliver the whole JSON bid request body: {bid_request:?}" + ); + } + } +} + +/// Runs a two-provider auction on `runtime` and asserts both bids arrive. +fn multi_provider_auction_runs(runtime: common::trace_boundary::TraceRuntime, name: &str) { + init_logger(); + let first = start_loopback_bidder("first-seat", 1.25, Duration::ZERO); + let second = start_loopback_bidder("second-seat", 2.5, Duration::ZERO); + let envelope = common::config::auction_app_config_envelope( + origin_port(), + &loopback_bidder_fixtures(&[ + ("first", &first, Duration::from_millis(1500)), + ("second", &second, Duration::from_millis(1500)), + ]), + 2000, + ) + .expect("should build the multi-provider auction config"); + + common::trace_boundary::with_trace_runtime( + runtime, + &envelope, + Some("http://127.0.0.1:*"), + |base_url| { + let response = post_auction(base_url); + assert_eq!( + ( + &response["ext"]["orchestrator"]["providers"], + &response["ext"]["orchestrator"]["total_bids"] + ), + (&serde_json::json!(2), &serde_json::json!(2)), + "{name} should fan out to both providers and collect both bids: {response}" + ); + assert_eq!( + response["seatbid"][0]["bid"][0]["price"], + serde_json::json!(2.5), + "{name} should deliver the higher bid: {response}" + ); + }, + ); + + assert_bidders_received(name, &[&first, &second], 1); +} + +#[test] +#[ignore = "requires Viceroy and the release Fastly WASM"] +fn twelve_bidder_auction_registers_twelve_backends_fastly() { + init_logger(); + let bidders: Vec = (1..=12) + .map(|index| { + start_loopback_bidder( + &format!("seat-{index:02}"), + f64::from(index), + Duration::from_millis(20), + ) + }) + .collect(); + let ids: Vec = (1..=12).map(|index| format!("bidder-{index:02}")).collect(); + let fixtures = loopback_bidder_fixtures( + &ids.iter() + .zip(&bidders) + .map(|(id, bidder)| (id.as_str(), bidder, Duration::from_millis(1500))) + .collect::>(), + ); + let envelope = common::config::auction_app_config_envelope(origin_port(), &fixtures, 2000) + .expect("should build the twelve-bidder auction config"); + + common::trace_boundary::with_fastly_runtime(&envelope, |base_url, stdout| { + // Each bidder listens on its own port, so each needs its own backend. + // Two auctions show the backends serve every later auction too. + for auction in 1..=2 { + let response = post_auction(base_url); + assert_eq!( + ( + &response["ext"]["orchestrator"]["providers"], + &response["ext"]["orchestrator"]["total_bids"] + ), + (&serde_json::json!(12), &serde_json::json!(12)), + "auction {auction} should collect one bid from each of the twelve bidders: {response}" + ); + for id in &ids { + assert_eq!( + provider_detail(&response, id)["bid_count"], + serde_json::json!(1), + "auction {auction} should parse the bid from {id}" + ); + } + assert_eq!( + response["seatbid"][0]["bid"][0]["price"], + serde_json::json!(12.0), + "auction {auction} should deliver the highest of the twelve bids" + ); + } + // Each callback has its own observation, so this shows that each + // auction configures and registers exactly one backend per bidder + // and no other outbound backend. + assert_eq!( + wait_for_outbound_observations(stdout, 2), + [(12, 12), (12, 12)], + "each auction should configure and register exactly one backend per bidder" + ); + }); + + assert_bidders_received("Fastly", &bidders.iter().collect::>(), 2); +} + +/// Waits up to five seconds for the guest to log `expected` outbound +/// observations, and returns the configuration and registration counts of each. +/// +/// The callback logs its observation after delivery, so the line can trail +/// the response the client already read. Viceroy prints a line once for the +/// `tslog` endpoint and once more as the guest's stdout echo, so only the +/// endpoint copy counts. +fn wait_for_outbound_observations( + stdout: &environments::fastly::GuestStdout, + expected: usize, +) -> Vec<(u64, u64)> { + let count = |line: &str, field: &str| -> u64 { + line.split_whitespace() + .find_map(|entry| entry.strip_prefix(field)) + .and_then(|value| value.parse().ok()) + .expect("should log every outbound observation count") + }; + let deadline = std::time::Instant::now() + Duration::from_secs(5); + loop { + let observations: Vec<(u64, u64)> = stdout + .lines() + .iter() + .filter_map(|line| line.strip_prefix("tslog :: ")) + .filter(|line| line.contains("[outbound_observation]")) + .map(|line| { + ( + count(line, "configurations="), + count(line, "registrations="), + ) + }) + .collect(); + if observations.len() >= expected || std::time::Instant::now() >= deadline { + return observations; + } + thread::sleep(Duration::from_millis(100)); + } +} + +#[test] +#[ignore = "requires the `wrangler` CLI in $PATH and a prebuilt Cloudflare Workers bundle (run build.sh first); the test starts wrangler dev automatically"] +fn multi_provider_auction_runs_cloudflare() { + multi_provider_auction_runs( + common::trace_boundary::TraceRuntime::Cloudflare, + "Cloudflare", + ); +} + +#[test] +#[ignore = "requires Spin and the current production Spin WASM artifact"] +fn multi_provider_auction_runs_spin() { + multi_provider_auction_runs(common::trace_boundary::TraceRuntime::Spin, "Spin"); +} + +#[test] +#[ignore = "requires Spin and the current production Spin WASM artifact"] +fn spin_runtime_auction_reports_one_timeout_and_cutoff() { + init_logger(); + // The first bidder's own 200 ms timeout ends before the 1000 ms auction + // cutoff. The second bidder has time left but answers after the cutoff. + let early_timeout = start_loopback_bidder("early-seat", 1.0, Duration::from_millis(600)); + let after_cutoff = start_loopback_bidder("late-seat", 2.0, Duration::from_millis(1600)); + let envelope = common::config::auction_app_config_envelope( + origin_port(), + &loopback_bidder_fixtures(&[ + ("early-timeout", &early_timeout, Duration::from_millis(200)), + ("after-cutoff", &after_cutoff, Duration::from_secs(2)), + ]), + 1000, + ) + .expect("should build the timeout and cutoff auction config"); + + common::trace_boundary::with_trace_runtime( + common::trace_boundary::TraceRuntime::Spin, + &envelope, + Some("http://127.0.0.1:*"), + |base_url| { + let response = post_auction(base_url); + let early = provider_detail(&response, "early-timeout"); + let late = provider_detail(&response, "after-cutoff"); + assert_eq!( + ( + &early["metadata"]["error_type"], + &late["metadata"]["error_type"] + ), + (&serde_json::json!("timeout"), &serde_json::json!("timeout")), + "should report the provider timeout and the cutoff as timeouts, not a failed batch: {response}" + ); + let early_ms = early["time_ms"] + .as_u64() + .expect("should time the early timeout"); + let late_ms = late["time_ms"].as_u64().expect("should time the cutoff"); + assert!( + early_ms < 1000 && late_ms >= 1000, + "should end the first bidder at its own timeout and the second at the cutoff, got {early_ms} ms and {late_ms} ms: {response}" + ); + assert!( + response["seatbid"].as_array().is_none_or(Vec::is_empty), + "should deliver no bid: {response}" + ); + }, + ); +} + +// --------------------------------------------------------------------------- +// Publisher origin through EdgeZero's client (spec tests 1, 2, and 6) +// --------------------------------------------------------------------------- + +/// A runtime the publisher-origin cases serve an isolated app config on. +#[derive(Clone, Copy, Debug)] +enum OriginRuntime { + Axum, + Cloudflare, + Fastly, + Spin, +} + +impl OriginRuntime { + /// Serves `envelope` on this runtime and runs `check` against its base URL. + /// + /// Spin may reach any loopback port, so loopback origins and bidders work there. + /// + /// # Panics + /// + /// Panics when the runtime's artifact or CLI is missing or it never becomes ready. + fn serve(self, envelope: &str, check: impl FnOnce(&str)) { + use common::trace_boundary::{TraceRuntime, with_trace_runtime}; + match self { + Self::Axum => { + let process = environments::axum::AxumDevServer + .spawn_with_app_config(envelope, &[]) + .expect("should start the Axum dev server"); + check(&process.base_url); + } + Self::Cloudflare => with_trace_runtime(TraceRuntime::Cloudflare, envelope, None, check), + Self::Fastly => with_trace_runtime(TraceRuntime::Fastly, envelope, None, check), + Self::Spin => with_trace_runtime( + TraceRuntime::Spin, + envelope, + Some("http://127.0.0.1:*"), + check, + ), + } + } +} + +/// A `GET /` that carries body bytes returns 400 and never reaches the origin +/// (EZ-1 criterion 13). +fn get_root_with_body_returns_400(runtime: IngressRuntime) { + init_logger(); + let _port = lock_origin_port(); + let origin = RecordingOrigin::start(origin_port(), ORIGIN_PASS_THROUGH_STATUS); + let process = runtime.spawn(); + // Readiness probes may already have fetched the root page from the origin. + let probes = origin.requests().len(); + + let response = reqwest::blocking::Client::new() + .get(format!("{}/", process.base_url)) + .header(CONTENT_TYPE, "text/plain") + .body("example GET body") + .send() + .expect("should send the GET with a body"); + + assert_eq!( + response.status().as_u16(), + 400, + "{runtime:?} should refuse to forward a GET that carries a body" + ); + let forwarded = origin.requests(); + assert_eq!( + forwarded.len(), + probes, + "{runtime:?} should not forward the GET with a body to the origin: {:?}", + &forwarded[probes..] + ); +} + +#[test] +#[ignore = "requires Viceroy, the generated Viceroy config, and the release Fastly WASM"] +fn get_root_with_body_returns_400_fastly() { + get_root_with_body_returns_400(IngressRuntime::Fastly); +} + +#[test] +#[ignore = "requires the `wrangler` CLI in $PATH and a prebuilt Cloudflare Workers bundle (run build.sh first); the test starts wrangler dev automatically"] +fn get_root_with_body_returns_400_cloudflare() { + get_root_with_body_returns_400(IngressRuntime::Cloudflare); +} + +#[test] +#[ignore = "requires the pre-built trusted-server-axum binary"] +fn get_root_with_body_returns_400_axum() { + get_root_with_body_returns_400(IngressRuntime::Axum); +} + +/// Serves the shared integration app config on Spin, with the recording origin +/// on the fixed origin port, and runs `check` against its base URL. +fn with_spin_ingress(status: u16, check: impl FnOnce(&str, &RecordingOrigin)) { + init_logger(); + let _port = lock_origin_port(); + let origin = RecordingOrigin::start(origin_port(), status); + let envelope = common::config::integration_app_config_envelope(origin_port()) + .expect("should build the integration app config"); + OriginRuntime::Spin.serve(&envelope, |base_url| check(base_url, &origin)); +} + +#[test] +#[ignore = "requires Spin and the current production Spin WASM artifact"] +fn spin_runtime_post_auction_with_json_body() { + with_spin_ingress(200, |base_url, _origin| { + let response = reqwest::blocking::Client::new() + .post(format!("{base_url}/auction")) + .json(&serde_json::json!({ + "adUnits": [{ + "code": "ingress-slot", + "mediaTypes": { "banner": { "sizes": [[300, 250]] } } + }] + })) + .send() + .expect("should send the auction request"); + let status = response.status().as_u16(); + let body = response.text().expect("should read the auction response"); + assert_eq!( + status, 200, + "Spin should hand the JSON body to the auction handler: {body}" + ); + let auction: serde_json::Value = + serde_json::from_str(&body).expect("should return an OpenRTB JSON response"); + assert!( + auction["id"].as_str().is_some_and(|id| !id.is_empty()), + "Spin auction response should carry the auction id: {body}" + ); + }); +} + +#[test] +#[ignore = "requires Spin and the current production Spin WASM artifact"] +fn spin_runtime_publisher_post_body_reaches_origin() { + with_spin_ingress(200, |base_url, origin| { + let sent: Vec = (0..=u8::MAX).cycle().take(4096).collect(); + let response = reqwest::blocking::Client::new() + .post(format!("{base_url}{INGRESS_FORM_PATH}")) + .header(CONTENT_TYPE, "application/octet-stream") + .body(sent.clone()) + .send() + .expect("should send the publisher POST"); + assert_eq!( + response.status().as_u16(), + 200, + "Spin should return the origin's answer to the publisher POST" + ); + let received = origin + .requests() + .into_iter() + .find(|request| request.method == "POST" && request.path == INGRESS_FORM_PATH) + .expect("should record the publisher POST at the origin"); + assert!( + received.body == sent, + "Spin should forward all {} body bytes unchanged; the origin received {} bytes", + sent.len(), + received.body.len() + ); + }); +} + +#[test] +#[ignore = "requires Spin and the current production Spin WASM artifact"] +fn spin_runtime_head_root() { + with_spin_ingress(ORIGIN_PASS_THROUGH_STATUS, |base_url, _origin| { + let response = reqwest::blocking::Client::new() + .head(format!("{base_url}/")) + .send() + .expect("should send the HEAD request"); + assert_eq!( + response.status().as_u16(), + ORIGIN_PASS_THROUGH_STATUS, + "Spin HEAD / should return the origin's status" + ); + }); +} + +// --------------------------------------------------------------------------- +// Bodiless publisher responses +// --------------------------------------------------------------------------- + +/// Publisher path the bodiless-status cases request. +const BODILESS_PAGE_PATH: &str = "/bodiless-page"; + +/// Entity tag for which the origin answers a conditional GET with 304. +const BODILESS_PAGE_ETAG: &str = "\"example-etag\""; + +/// Answers a HEAD or a matching conditional GET for [`BODILESS_PAGE_PATH`] +/// with a bodiless head that declares `Trailer`, and anything else with a +/// small HTML page. +fn bodiless_page_response(request: &OriginRequest) -> Vec { + let head_only = request.method == "HEAD"; + let not_modified = request.header("if-none-match") == Some(BODILESS_PAGE_ETAG); + if request.path != BODILESS_PAGE_PATH || !(head_only || not_modified) { + return html_response(200, head_only); + } + let status = if head_only { + ORIGIN_PASS_THROUGH_STATUS + } else { + 304 + }; + raw_response( + status, + &[ + ("Content-Type", "text/html; charset=utf-8".to_owned()), + ("ETag", BODILESS_PAGE_ETAG.to_owned()), + ("Trailer", "expires".to_owned()), + ], + b"", + ) +} + +/// A publisher HEAD and a conditional GET that the origin answers with 304 +/// return their real status, with no `Trailer`, although the origin declares one. +fn publisher_head_and_conditional_304_return_real_status(runtime: OriginRuntime) { + init_logger(); + let _port = lock_origin_port(); + let origin = RecordingOrigin::serve( + TcpListener::bind(("127.0.0.1", origin_port())).expect("should bind origin port"), + Arc::new(bodiless_page_response), + ); + let envelope = common::config::integration_app_config_envelope(origin_port()) + .expect("should build the integration app config"); + + runtime.serve(&envelope, |base_url| { + let url = format!("{base_url}{BODILESS_PAGE_PATH}"); + let head = raw_client() + .head(&url) + .send() + .expect("should send the publisher HEAD"); + let conditional = raw_client() + .get(&url) + .header("if-none-match", BODILESS_PAGE_ETAG) + .send() + .expect("should send the conditional publisher GET"); + for (request, response, expected) in [ + ("HEAD", &head, ORIGIN_PASS_THROUGH_STATUS), + ("conditional GET", &conditional, 304), + ] { + assert_eq!( + response.status().as_u16(), + expected, + "{runtime:?} {request} should return the origin's status" + ); + assert!( + response.headers().get("trailer").is_none(), + "{runtime:?} {request} should not send Trailer" + ); + } + }); + + assert!( + origin.requests().iter().any(|request| { + request.method == "GET" + && request.path == BODILESS_PAGE_PATH + && request.header("if-none-match") == Some(BODILESS_PAGE_ETAG) + }), + "{runtime:?} should forward the validator, so the 304 is the origin's" + ); +} + +#[test] +#[ignore = "requires Viceroy and the release Fastly WASM"] +fn publisher_head_and_conditional_304_return_real_status_fastly() { + publisher_head_and_conditional_304_return_real_status(OriginRuntime::Fastly); +} + +#[test] +#[ignore = "requires the pre-built trusted-server-axum binary"] +fn publisher_head_and_conditional_304_return_real_status_axum() { + publisher_head_and_conditional_304_return_real_status(OriginRuntime::Axum); +} + +#[test] +#[ignore = "requires the `wrangler` CLI in $PATH and a prebuilt Cloudflare Workers bundle (run build.sh first); the test starts wrangler dev automatically"] +fn publisher_head_and_conditional_304_return_real_status_cloudflare() { + publisher_head_and_conditional_304_return_real_status(OriginRuntime::Cloudflare); +} + +#[test] +#[ignore = "requires Spin and the current production Spin WASM artifact"] +fn publisher_head_and_conditional_304_return_real_status_spin() { + publisher_head_and_conditional_304_return_real_status(OriginRuntime::Spin); +} + +// --------------------------------------------------------------------------- +// Axum ingress variables +// --------------------------------------------------------------------------- + +/// Raw-head limit the raised case sets, twice the listener's 64 KiB default. +const RAISED_RAW_HEAD_BYTES: &str = "131072"; + +/// A `GET /health` whose raw head exceeds the Axum listener's default 64 KiB +/// limit while its header fields stay under `EdgeZero`'s 64 KiB header limit. +/// +/// `EdgeZero` counts only header fields toward its limit, so the request line +/// carries the excess, with a target under `EdgeZero`'s 8192 bytes. +fn oversized_head_request() -> Vec { + let mut request = format!( + "GET /health?padding={} HTTP/1.1\r\nHost: 127.0.0.1\r\nConnection: close\r\n", + "a".repeat(7000) + ); + // 58 fields of 1,040 bytes plus `Host` and `Connection`: 60 field lines + // of 60,356 bytes. With the 7,016-byte target the raw head is 67,389 + // bytes (about 66 KiB), 1,853 over the default 65,536. + for index in 0..58 { + request.push_str(&format!("x-example-{index:02}: {}\r\n", "b".repeat(1024))); + } + request.push_str("\r\n"); + request.into_bytes() +} + +/// Sends `request` to the runtime at `base_url` and returns the response +/// status, or `None` when the runtime closed the connection without one. +fn raw_request_status(base_url: &str, request: &[u8]) -> Option { + let address = base_url + .strip_prefix("http://") + .expect("should serve plain HTTP"); + let mut stream = TcpStream::connect(address).expect("should connect to the runtime"); + stream + .set_read_timeout(Some(Duration::from_secs(10))) + .expect("should set a read timeout"); + // A runtime that refuses the head may close before reading all of it. + let _ = stream.write_all(request); + let mut response = Vec::new(); + let _ = stream.read_to_end(&mut response); + std::str::from_utf8(&response) + .ok()? + .strip_prefix("HTTP/1.1 ")? + .get(..3)? + .parse() + .ok() +} + +#[test] +#[ignore = "requires the pre-built trusted-server-axum binary"] +fn axum_ingress_variable_raises_limit() { + init_logger(); + let envelope = common::config::integration_app_config_envelope(origin_port()) + .expect("should build the integration app config"); + let request = oversized_head_request(); + + let default = environments::axum::AxumDevServer + .spawn_with_app_config(&envelope, &[]) + .expect("should start the Axum dev server"); + let default_status = raw_request_status(&default.base_url, &request); + drop(default); + let raised = environments::axum::AxumDevServer + .spawn_with_app_config( + &envelope, + &[( + "EDGEZERO__ADAPTER__INGRESS__MAX_RAW_HEAD_BYTES", + RAISED_RAW_HEAD_BYTES, + )], + ) + .expect("should start the Axum dev server with a raised raw-head limit"); + let raised_status = raw_request_status(&raised.base_url, &request); + + assert_ne!( + default_status, + Some(200), + "should refuse the head under the default raw-head limit, or the case proves nothing" + ); + assert_eq!( + raised_status, + Some(200), + "should admit the same head once the variable raises the raw-head limit" + ); +} + +/// Waits up to `deadline` for `child` to exit, killing it if it does not. +/// +/// Returns `None` when the child was still running at the deadline. +fn wait_for_exit(child: &mut Child, deadline: Duration) -> Option { + let started = std::time::Instant::now(); + loop { + if let Some(status) = child.try_wait().expect("should poll the child process") { + return Some(status); + } + if started.elapsed() >= deadline { + child.kill().expect("should stop the child process"); + child.wait().expect("should reap the child process"); + return None; + } + thread::sleep(Duration::from_millis(20)); + } +} + +#[test] +#[ignore = "requires the pre-built trusted-server-axum binary"] +fn axum_invalid_ingress_variable_fails_startup() { + init_logger(); + let envelope = common::config::integration_app_config_envelope(origin_port()) + .expect("should build the integration app config"); + let port = environments::find_available_port().expect("should reserve a port"); + let mut child = environments::axum::AxumDevServer + .command(&envelope, port) + .env("EDGEZERO__ADAPTER__INGRESS__MAX_HEADER_COUNT", "0") + .env("RUST_LOG", "info") + .stdin(Stdio::null()) + .stdout(Stdio::piped()) + .stderr(Stdio::piped()) + .spawn() + .expect("should spawn the Axum binary"); + + let status = wait_for_exit(&mut child, Duration::from_secs(10)); + let mut output = String::new(); + if let Some(mut stdout) = child.stdout.take() { + stdout + .read_to_string(&mut output) + .expect("should read the Axum binary stdout"); + } + if let Some(mut stderr) = child.stderr.take() { + stderr + .read_to_string(&mut output) + .expect("should read the Axum binary stderr"); + } + + let status = status.unwrap_or_else(|| { + panic!("should exit instead of serving with an invalid ingress variable; output:\n{output}") + }); + assert!( + !status.success(), + "should exit non-zero with an invalid ingress variable, got {status}; output:\n{output}" + ); + assert!( + output.contains("invalid Axum ingress setting `max_header_count`"), + "should name the invalid ingress setting; output:\n{output}" + ); + assert!( + !output.contains("Listening on"), + "should not reach the listener; output:\n{output}" + ); +} + +/// Page the navigation bid cases load. +const NAVIGATION_PATH: &str = "/article"; + +/// When the navigation origin's head arrives. +const NAVIGATION_ORIGIN_DELAY: Duration = Duration::from_millis(700); + +/// The navigation auction's deadline. +const NAVIGATION_AUCTION_TIMEOUT_MS: u32 = 500; + +/// Pre-seeded EC row of the Viceroy template, so the Fastly navigation +/// preloads an EC snapshot and takes the overlapped path. +const NAVIGATION_EC_ID: &str = + "eeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeee.test05"; + +/// The creative a navigation bidder answering after `delay` delivers. +fn navigation_creative(delay: Duration) -> String { + format!("example-creative-{}", delay.as_millis()) +} + +/// A loopback bidder that waits `delay` and bids on its own slot. +fn start_navigation_bidder(delay: Duration) -> RecordingOrigin { + let slot = format!("slot-{}", delay.as_millis()); + let body = serde_json::to_vec(&serde_json::json!({ + "seatbid": [{"seat": format!("seat-{}", delay.as_millis()), "bid": [{ + "id": format!("bid-{}", delay.as_millis()), + "impid": slot, + "price": 1.5, + "adm": navigation_creative(delay), + "w": 300, + "h": 250 + }]}] + })) + .expect("should serialize the navigation bid"); + let listener = TcpListener::bind(("127.0.0.1", 0)).expect("should bind a bidder port"); + RecordingOrigin::serve( + listener, + Arc::new(move |_request| { + thread::sleep(delay); + let mut response = format!( + "HTTP/1.1 200 OK\r\nContent-Type: application/json\r\nContent-Length: {}\r\nConnection: close\r\n\r\n", + body.len() + ) + .into_bytes(); + response.extend_from_slice(&body); + response + }), + ) +} + +/// The publisher origin of the navigation cases: its head arrives after +/// [`NAVIGATION_ORIGIN_DELAY`]. +fn start_slow_navigation_origin() -> RecordingOrigin { + let listener = + TcpListener::bind(("127.0.0.1", origin_port())).expect("should bind the origin port"); + RecordingOrigin::serve( + listener, + Arc::new(|_request| { + thread::sleep(NAVIGATION_ORIGIN_DELAY); + let body = b"Example
example article
"; + let mut response = format!( + "HTTP/1.1 200 OK\r\nContent-Type: text/html; charset=utf-8\r\nContent-Length: {}\r\nConnection: close\r\n\r\n", + body.len() + ) + .into_bytes(); + response.extend_from_slice(body); + response + }), + ) +} + +/// Loads a navigation whose 500 ms auction races bidders at 180, 320, and 600 ms +/// and an origin whose head arrives at 700 ms (EZ-6). +fn navigation_bids_include_180_and_320_ms_and_exclude_600_ms(runtime: OriginRuntime) { + init_logger(); + let _port = lock_origin_port(); + let _origin = start_slow_navigation_origin(); + let delays = [180, 320, 600].map(Duration::from_millis); + let bidders = delays.map(start_navigation_bidder); + let ids = delays.map(|delay| format!("bidder-{}", delay.as_millis())); + let fixtures = loopback_bidder_fixtures( + &ids.iter() + .zip(&bidders) + .map(|(id, bidder)| (id.as_str(), bidder, Duration::from_millis(1500))) + .collect::>(), + ); + let slots = delays.map(|delay| format!("slot-{}", delay.as_millis())); + let envelope = common::config::navigation_auction_app_config_envelope( + origin_port(), + &fixtures, + NAVIGATION_AUCTION_TIMEOUT_MS, + NAVIGATION_PATH, + &slots.iter().map(String::as_str).collect::>(), + ) + .expect("should build the navigation auction config"); + + runtime.serve(&envelope, |base_url| { + let response = reqwest::blocking::Client::new() + .get(format!("{base_url}{NAVIGATION_PATH}")) + .header( + "cookie", + format!("{}; ts-ec={NAVIGATION_EC_ID}", tcf_purpose_one_cookie()), + ) + .header("cf-ipcountry", "US") + .header("accept", "text/html") + .header("sec-fetch-dest", "document") + .header("sec-fetch-mode", "navigate") + .timeout(Duration::from_secs(30)) + .send() + .expect("should load the navigation"); + let status = response.status().as_u16(); + let html = response.text().expect("should read the navigation page"); + assert_eq!(status, 200, "{runtime:?} should serve the page: {html}"); + for delay in [180, 320].map(Duration::from_millis) { + assert!( + html.contains(&navigation_creative(delay)), + "{runtime:?} should inject the bid that answered at {} ms: {html}", + delay.as_millis() + ); + } + assert!( + !html.contains(&navigation_creative(Duration::from_millis(600))), + "{runtime:?} should drop the bid that answered after the 500 ms cutoff: {html}" + ); + }); + + assert_bidders_received( + &format!("{runtime:?}"), + &bidders.iter().collect::>(), + 1, + ); +} + +#[test] +#[ignore = "requires Viceroy and the release Fastly WASM"] +fn navigation_bids_include_180_and_320_ms_and_exclude_600_ms_fastly() { + navigation_bids_include_180_and_320_ms_and_exclude_600_ms(OriginRuntime::Fastly); +} + +#[test] +#[ignore = "requires the pre-built trusted-server-axum binary"] +fn navigation_bids_include_180_and_320_ms_and_exclude_600_ms_axum() { + navigation_bids_include_180_and_320_ms_and_exclude_600_ms(OriginRuntime::Axum); +} + +#[test] +#[ignore = "requires the `wrangler` CLI in $PATH and a prebuilt Cloudflare Workers bundle (run build.sh first); the test starts wrangler dev automatically"] +fn navigation_bids_include_180_and_320_ms_and_exclude_600_ms_cloudflare() { + navigation_bids_include_180_and_320_ms_and_exclude_600_ms(OriginRuntime::Cloudflare); +} + +#[test] +#[ignore = "requires Spin and the current production Spin WASM artifact"] +fn navigation_bids_include_180_and_320_ms_and_exclude_600_ms_spin() { + navigation_bids_include_180_and_320_ms_and_exclude_600_ms(OriginRuntime::Spin); +} + +// --------------------------------------------------------------------------- +// Encoded passthrough on Viceroy (spec test 7, EZ-5 criterion 8) +// --------------------------------------------------------------------------- + +/// Passes every chunk through unchanged, so a pipeline only re-encodes. +struct IdentityProcessor; + +impl trusted_server_core::streaming_processor::StreamProcessor for IdentityProcessor { + fn process_chunk(&mut self, chunk: &[u8], _is_last: bool) -> std::io::Result> { + Ok(chunk.to_vec()) + } +} + +/// Decodes `bytes` coded with `compression` through Trusted Server's own codec. +fn decode( + bytes: &[u8], + compression: trusted_server_core::streaming_processor::Compression, +) -> Vec { + use trusted_server_core::streaming_processor::{ + Compression, PipelineConfig, StreamingPipeline, + }; + let mut decoded = Vec::new(); + StreamingPipeline::new( + PipelineConfig { + input_compression: compression, + output_compression: Compression::None, + chunk_size: 8192, + }, + IdentityProcessor, + ) + .process(bytes, &mut decoded) + .expect("should decode the body"); + decoded +} + +/// A complete HTTP/1.1 response with `status`, `headers`, and `body`. +fn raw_response(status: u16, headers: &[(&str, String)], body: &[u8]) -> Vec { + let reason = http::StatusCode::from_u16(status) + .ok() + .and_then(|code| code.canonical_reason()) + .unwrap_or("Example"); + let mut response = format!("HTTP/1.1 {status} {reason}\r\n"); + for (name, value) in headers { + response.push_str(&format!("{name}: {value}\r\n")); + } + response.push_str("Connection: close\r\n\r\n"); + let mut response = response.into_bytes(); + response.extend_from_slice(body); + response +} + +/// HTML the publisher origin serves for the encoded passthrough cases. +const PASSTHROUGH_HTML: &[u8] = + b"Examplenext"; + +/// [`PASSTHROUGH_HTML`] as `gzip -n -9` codes it. +const PASSTHROUGH_HTML_GZIP: &[u8] = &[ + 31, 139, 8, 0, 0, 0, 0, 0, 2, 3, 179, 201, 40, 201, 205, 177, 179, 201, 72, 77, 76, 177, 179, + 41, 201, 44, 201, 73, 181, 115, 173, 72, 204, 45, 200, 73, 181, 209, 135, 112, 109, 244, 33, + 146, 73, 249, 41, 149, 118, 54, 137, 10, 25, 69, 169, 105, 182, 74, 250, 121, 169, 21, 37, 74, + 118, 32, 210, 70, 63, 17, 168, 8, 34, 173, 15, 54, 14, 0, 15, 183, 7, 35, 85, 0, 0, 0, +]; + +/// [`PASSTHROUGH_HTML`] as the `brotli` command line tool codes it. +const PASSTHROUGH_HTML_BROTLI: &[u8] = &[ + 161, 160, 2, 0, 224, 56, 176, 99, 52, 211, 83, 219, 181, 121, 205, 227, 195, 214, 54, 195, 208, + 232, 226, 197, 49, 14, 216, 239, 6, 154, 116, 31, 23, 222, 115, 88, 126, 55, 165, 56, 183, 208, + 95, 189, 160, 80, 215, 28, 134, 101, 217, 51, 196, 199, 81, 240, 80, 58, +]; + +/// The SVG the publisher origin serves br-coded. +const PASSTHROUGH_SVG: &[u8] = b""; + +/// [`PASSTHROUGH_SVG`] as the `brotli` command line tool codes it. +const PASSTHROUGH_SVG_BROTLI: &[u8] = &[ + 161, 64, 1, 192, 47, 122, 178, 212, 191, 206, 71, 69, 171, 214, 109, 14, 27, 112, 34, 148, 4, + 22, 143, 114, 199, 160, 75, 55, 49, 249, 186, 223, 115, 79, 163, 115, 36, 153, 82, 40, 72, 58, + 194, 7, +]; + +/// The bid response the gzip bidder serves. +const GZIP_BID_JSON: &[u8] = br#"{"seatbid":[{"seat":"gzip-seat","bid":[{"id":"gzip-bid","impid":"auction-slot","price":2.25,"adm":"
gzip bid
","w":300,"h":250}]}]}"#; + +/// [`GZIP_BID_JSON`] as `gzip -n -9` codes it. +const GZIP_BID_JSON_GZIP: &[u8] = &[ + 31, 139, 8, 0, 0, 0, 0, 0, 2, 3, 171, 86, 42, 78, 77, 44, 73, 202, 76, 81, 178, 138, 174, 6, + 179, 149, 172, 148, 210, 171, 50, 11, 116, 193, 108, 29, 37, 152, 20, 136, 130, 72, 128, 68, + 116, 148, 50, 115, 11, 192, 66, 137, 165, 201, 37, 153, 249, 121, 186, 197, 57, 249, 32, 229, + 5, 69, 153, 201, 169, 74, 86, 70, 122, 70, 166, 58, 74, 137, 41, 185, 64, 21, 54, 41, 153, 101, + 118, 32, 157, 10, 64, 157, 54, 250, 32, 30, 80, 97, 185, 146, 149, 177, 129, 129, 142, 82, 6, + 80, 177, 169, 65, 109, 44, 16, 2, 0, 181, 125, 165, 97, 140, 0, 0, 0, +]; + +/// The gzip-stored asset the range case slices: every byte value once, which +/// `gzip -n -9` stores in one uncoded deflate block, so it is 279 bytes long. +fn range_asset() -> Vec { + let mut asset = vec![31, 139, 8, 0, 0, 0, 0, 0, 2, 3, 1, 0, 1, 255, 254]; + asset.extend(0..=u8::MAX); + asset.extend([115, 140, 5, 41, 0, 1, 0, 0]); + asset +} + +/// The `Range` the range case requests. +const RANGE: &str = "bytes=0-99"; + +/// The length of the part [`RANGE`] selects. +const RANGE_LENGTH: usize = 100; + +#[test] +fn encoded_fixtures_decode_to_their_plain_bodies() { + use trusted_server_core::streaming_processor::Compression; + assert_eq!( + ( + decode(PASSTHROUGH_HTML_GZIP, Compression::Gzip), + decode(PASSTHROUGH_HTML_BROTLI, Compression::Brotli), + decode(PASSTHROUGH_SVG_BROTLI, Compression::Brotli), + decode(GZIP_BID_JSON_GZIP, Compression::Gzip), + decode(&range_asset(), Compression::Gzip), + ), + ( + PASSTHROUGH_HTML.to_vec(), + PASSTHROUGH_HTML.to_vec(), + PASSTHROUGH_SVG.to_vec(), + GZIP_BID_JSON.to_vec(), + (0..=u8::MAX).collect::>(), + ), + "should decode every precomputed fixture to its plain body" + ); + assert!( + range_asset().len() > RANGE_LENGTH, + "should keep the range inside the stored asset" + ); +} + +/// Answers the encoded passthrough paths of the publisher origin. +fn encoded_passthrough_response(request: &OriginRequest) -> Vec { + match origin_request_path(request) { + "/gzip-page" => raw_response( + 200, + &[ + ("Content-Type", "text/html; charset=utf-8".to_owned()), + ("Content-Encoding", "gzip".to_owned()), + ("Content-Length", PASSTHROUGH_HTML_GZIP.len().to_string()), + ], + PASSTHROUGH_HTML_GZIP, + ), + "/image.svg" => raw_response( + 200, + &[ + ("Content-Type", "image/svg+xml".to_owned()), + ("Content-Encoding", "br".to_owned()), + ("Content-Length", PASSTHROUGH_SVG_BROTLI.len().to_string()), + ("ETag", "\"example-svg\"".to_owned()), + ], + PASSTHROUGH_SVG_BROTLI, + ), + "/asset.bin" => { + let asset = range_asset(); + raw_response( + 206, + &[ + ("Content-Type", "application/octet-stream".to_owned()), + ("Content-Encoding", "gzip".to_owned()), + ("Content-Range", format!("bytes 0-99/{}", asset.len())), + ("Content-Length", RANGE_LENGTH.to_string()), + ], + &asset[..RANGE_LENGTH], + ) + } + "/app.js" if request.header("if-none-match") == Some("\"example-js\"") => { + raw_response(304, &[("ETag", "\"example-js\"".to_owned())], b"") + } + "/creative.html" => raw_response( + 200, + &[ + ("Content-Type", "text/html; charset=utf-8".to_owned()), + ("Content-Encoding", "br".to_owned()), + ("Content-Length", PASSTHROUGH_HTML_BROTLI.len().to_string()), + ], + PASSTHROUGH_HTML_BROTLI, + ), + _ => html_response(200, request.method == "HEAD"), + } +} + +/// Serves the shared integration app config on Viceroy, with the encoded +/// passthrough origin on the fixed origin port. +fn with_encoded_passthrough(check: impl FnOnce(&str, &RecordingOrigin)) { + init_logger(); + let _port = lock_origin_port(); + let listener = + TcpListener::bind(("127.0.0.1", origin_port())).expect("should bind the origin port"); + let origin = RecordingOrigin::serve(listener, Arc::new(encoded_passthrough_response)); + let process = IngressRuntime::Fastly.spawn(); + check(&process.base_url, &origin); +} + +/// A client that neither decodes responses nor follows redirects. +fn raw_client() -> reqwest::blocking::Client { + reqwest::blocking::Client::builder() + .no_gzip() + .redirect(reqwest::redirect::Policy::none()) + .build() + .expect("should build a raw client") +} + +#[test] +#[ignore = "requires Viceroy, the generated Viceroy config, and the release Fastly WASM"] +fn gzip_html_page_keeps_gzip_and_carries_tsjs_fastly() { + with_encoded_passthrough(|base_url, _origin| { + let response = raw_client() + .get(format!("{base_url}/gzip-page")) + .header("accept-encoding", "gzip") + .send() + .expect("should load the gzip page"); + assert_eq!( + response.status().as_u16(), + 200, + "should serve the gzip page" + ); + assert_eq!( + response + .headers() + .get("content-encoding") + .and_then(|value| value.to_str().ok()), + Some("gzip"), + "should keep the gzip coding" + ); + let html = String::from_utf8(decode( + &response.bytes().expect("should read the gzip page"), + trusted_server_core::streaming_processor::Compression::Gzip, + )) + .expect("should decode to UTF-8 HTML"); + assert!( + html.contains("/static/tsjs="), + "should decode to the rewritten document with the TSJS tag: {html}" + ); + }); +} + +#[test] +#[ignore = "requires Viceroy, the generated Viceroy config, and the release Fastly WASM"] +fn br_svg_relays_byte_identical_with_length_and_etag_fastly() { + with_encoded_passthrough(|base_url, _origin| { + let response = raw_client() + .get(format!("{base_url}/image.svg")) + .header("accept-encoding", "br") + .send() + .expect("should load the SVG"); + let header = |name: &str| { + response + .headers() + .get(name) + .and_then(|value| value.to_str().ok()) + .map(str::to_owned) + }; + let head = ( + response.status().as_u16(), + header("content-encoding"), + header("content-length"), + header("etag"), + ); + assert_eq!( + response.bytes().expect("should read the SVG").as_ref(), + PASSTHROUGH_SVG_BROTLI, + "should relay the br SVG byte for byte" + ); + assert_eq!( + head, + ( + 200, + Some("br".to_owned()), + Some(PASSTHROUGH_SVG_BROTLI.len().to_string()), + Some("\"example-svg\"".to_owned()) + ), + "should relay the br SVG with its origin length and ETag" + ); + }); +} + +#[test] +#[ignore = "requires Viceroy, the generated Viceroy config, and the release Fastly WASM"] +fn gzip_asset_range_returns_206_with_origin_content_range_fastly() { + let asset = range_asset(); + with_encoded_passthrough(|base_url, origin| { + let response = raw_client() + .get(format!("{base_url}/asset.bin")) + .header("range", RANGE) + .header("accept-encoding", "gzip") + .send() + .expect("should request the range"); + let header = |name: &str| { + response + .headers() + .get(name) + .and_then(|value| value.to_str().ok()) + .map(str::to_owned) + }; + let head = ( + response.status().as_u16(), + header("content-range"), + header("content-length"), + ); + assert_eq!( + response.bytes().expect("should read the range").as_ref(), + &asset[..RANGE_LENGTH], + "should relay the encoded range unchanged" + ); + assert!( + origin + .requests() + .iter() + .any(|request| request.header("range") == Some(RANGE)), + "should forward the Range header to the origin" + ); + assert_eq!( + head, + ( + 206, + Some(format!("bytes 0-99/{}", asset.len())), + Some(RANGE_LENGTH.to_string()) + ), + "should return the origin range with its Content-Range" + ); + }); +} + +#[test] +#[ignore = "requires Viceroy, the generated Viceroy config, and the release Fastly WASM"] +fn js_asset_revalidation_returns_304_fastly() { + with_encoded_passthrough(|base_url, _origin| { + let response = raw_client() + .get(format!("{base_url}/app.js")) + .header("if-none-match", "\"example-js\"") + .send() + .expect("should revalidate the script"); + assert_eq!( + response.status().as_u16(), + 304, + "should return the origin's 304 for a matching validator" + ); + }); +} + +#[test] +#[ignore = "requires Viceroy and the release Fastly WASM"] +fn first_party_proxy_html_creative_keeps_br_fastly() { + init_logger(); + let listener = TcpListener::bind(("127.0.0.1", 0)).expect("should bind the creative origin"); + let creative_origin = RecordingOrigin::serve(listener, Arc::new(encoded_passthrough_response)); + let envelope = + common::config::proxy_allowlist_app_config_envelope(origin_port(), &["127.0.0.1"]) + .expect("should build the proxy allowlist app config"); + common::trace_boundary::with_trace_runtime( + common::trace_boundary::TraceRuntime::Fastly, + &envelope, + None, + |base_url| { + let client = raw_client(); + let signed: serde_json::Value = client + .get(format!("{base_url}/first-party/sign")) + .query(&[( + "url", + format!("http://127.0.0.1:{}/creative.html", creative_origin.port), + )]) + .send() + .expect("should send the sign request") + .error_for_status() + .expect("should sign the creative URL") + .json() + .expect("should return the signed proxy URL"); + let href = signed["href"] + .as_str() + .expect("should return the first-party proxy href"); + let response = client + .get(format!("{base_url}{}", signed_href_path_and_query(href))) + .header("accept-encoding", "br") + .send() + .expect("should fetch the proxied creative"); + assert_eq!( + response.status().as_u16(), + 200, + "should serve the proxied creative" + ); + assert_eq!( + response + .headers() + .get("content-encoding") + .and_then(|value| value.to_str().ok()), + Some("br"), + "should keep the creative's br coding" + ); + }, + ); +} + +#[test] +#[ignore = "requires Viceroy and the release Fastly WASM"] +fn gzip_json_bid_response_is_decoded_fastly() { + init_logger(); + let listener = TcpListener::bind(("127.0.0.1", 0)).expect("should bind the gzip bidder"); + let bidder = RecordingOrigin::serve( + listener, + Arc::new(|_request| { + raw_response( + 200, + &[ + ("Content-Type", "application/json".to_owned()), + ("Content-Encoding", "gzip".to_owned()), + ("Content-Length", GZIP_BID_JSON_GZIP.len().to_string()), + ], + GZIP_BID_JSON_GZIP, + ) + }), + ); + let envelope = common::config::auction_app_config_envelope( + origin_port(), + &loopback_bidder_fixtures(&[("gzip", &bidder, Duration::from_millis(1500))]), + 2000, + ) + .expect("should build the gzip bidder auction config"); + + common::trace_boundary::with_trace_runtime( + common::trace_boundary::TraceRuntime::Fastly, + &envelope, + None, + |base_url| { + let response = post_auction(base_url); + assert_eq!( + response["seatbid"][0]["bid"][0]["price"], + serde_json::json!(2.25), + "should decode the gzip bid response: {response}" + ); + }, + ); +} + +#[test] +#[ignore = "requires Viceroy, the generated Viceroy config, and the release Fastly WASM"] +fn origin_records_identity_accept_encoding_when_none_was_set_fastly() { + with_encoded_passthrough(|base_url, origin| { + let response = raw_client() + .get(format!("{base_url}/plain")) + .send() + .expect("should load the page without Accept-Encoding"); + assert_eq!(response.status().as_u16(), 200, "should serve the page"); + let forwarded = origin + .requests() + .into_iter() + .find(|request| origin_request_path(request) == "/plain") + .expect("should forward the page request"); + assert_eq!( + forwarded.header("accept-encoding"), + Some("identity"), + "should send identity when Trusted Server set no Accept-Encoding" + ); + }); +} + +// --------------------------------------------------------------------------- +// Head limits fit realistic traffic (spec test 11) +// --------------------------------------------------------------------------- + +/// Bytes of the creative URL the realistic first-party proxy case wraps. +const REALISTIC_CREATIVE_URL_BYTES: usize = 4096; + +/// Request headers the realistic first-party proxy case sends, each of [`REALISTIC_HEADER_BYTES`]. +const REALISTIC_HEADER_COUNT: usize = 40; + +/// Bytes of each request header value the realistic first-party proxy case sends. +const REALISTIC_HEADER_BYTES: usize = 1024; + +/// Fetches a signed first-party proxy URL that wraps a 4 KiB creative URL, with 40 headers of 1 KiB each. +/// +/// `EdgeZero`'s default head limits must admit it, so it reaches `creative_origin`. +fn realistic_first_party_proxy_request_is_admitted( + base_url: &str, + creative_origin: &RecordingOrigin, +) { + let client = raw_client(); + let prefix = format!("http://127.0.0.1:{}/creative?ctx=", creative_origin.port); + let creative_url = format!( + "{prefix}{}", + "a".repeat(REALISTIC_CREATIVE_URL_BYTES - prefix.len()) + ); + let signed: serde_json::Value = client + .get(format!("{base_url}/first-party/sign")) + .query(&[("url", creative_url.as_str())]) + .send() + .expect("should send the sign request") + .error_for_status() + .expect("should sign the creative URL") + .json() + .expect("should return the signed proxy URL"); + let href = signed["href"] + .as_str() + .expect("should return the first-party proxy href"); + let mut request = client.get(format!("{base_url}{}", signed_href_path_and_query(href))); + for index in 0..REALISTIC_HEADER_COUNT { + request = request.header( + format!("x-example-{index:02}"), + "b".repeat(REALISTIC_HEADER_BYTES), + ); + } + + let response = request + .send() + .expect("should send the realistic first-party proxy request"); + + assert_eq!( + response.status().as_u16(), + 200, + "should admit and serve the realistic first-party proxy request" + ); + assert!( + creative_origin + .requests() + .iter() + .any(|forwarded| origin_request_path(forwarded) == "/creative"), + "should reach the creative origin" + ); +} + +/// Starts a creative origin and the proxy allowlist app config for the realistic case. +fn realistic_creative_origin() -> (RecordingOrigin, String) { + let listener = TcpListener::bind(("127.0.0.1", 0)).expect("should bind the creative origin"); + let origin = RecordingOrigin::serve( + listener, + Arc::new(|request| html_response(200, request.method == "HEAD")), + ); + let envelope = + common::config::proxy_allowlist_app_config_envelope(origin_port(), &["127.0.0.1"]) + .expect("should build the proxy allowlist app config"); + (origin, envelope) +} + +#[test] +#[ignore = "requires Viceroy and the release Fastly WASM"] +fn realistic_first_party_proxy_request_is_admitted_fastly() { + init_logger(); + let (origin, envelope) = realistic_creative_origin(); + common::trace_boundary::with_trace_runtime( + common::trace_boundary::TraceRuntime::Fastly, + &envelope, + None, + |base_url| realistic_first_party_proxy_request_is_admitted(base_url, &origin), + ); +} + +#[test] +#[ignore = "requires the `wrangler` CLI in $PATH and a prebuilt Cloudflare Workers bundle (run build.sh first); the test starts wrangler dev automatically"] +fn realistic_first_party_proxy_request_is_admitted_cloudflare() { + init_logger(); + let (origin, envelope) = realistic_creative_origin(); + common::trace_boundary::with_trace_runtime( + common::trace_boundary::TraceRuntime::Cloudflare, + &envelope, + None, + |base_url| realistic_first_party_proxy_request_is_admitted(base_url, &origin), + ); +} + +#[test] +#[ignore = "requires the pre-built trusted-server-axum binary"] +fn realistic_first_party_proxy_request_is_admitted_axum() { + init_logger(); + let (origin, envelope) = realistic_creative_origin(); + let process = environments::axum::AxumDevServer + .spawn_with_app_config(&envelope, &[]) + .expect("should start the Axum dev server"); + realistic_first_party_proxy_request_is_admitted(&process.base_url, &origin); +} + +// --------------------------------------------------------------------------- +// Response write deadline (spec test 17) +// --------------------------------------------------------------------------- + +/// How long the slow publisher origin streams its page. +const SLOW_PAGE_STREAM: Duration = Duration::from_secs(35); + +/// Marker the slow publisher page ends with. +const SLOW_PAGE_END: &str = "

example slow page end

"; + +/// Serves one chunked HTML page on `listener` whose body takes [`SLOW_PAGE_STREAM`] to arrive. +fn serve_slow_streaming_page(listener: TcpListener) -> JoinHandle<()> { + thread::spawn(move || { + let Ok((mut stream, _addr)) = listener.accept() else { + return; + }; + let _ = stream.set_read_timeout(Some(Duration::from_secs(10))); + let Some(_request) = read_origin_request(&mut stream) else { + return; + }; + let write_chunk = |stream: &mut TcpStream, chunk: &str| { + stream + .write_all(format!("{:x}\r\n{chunk}\r\n", chunk.len()).as_bytes()) + .and_then(|()| stream.flush()) + }; + if stream + .write_all(b"HTTP/1.1 200 OK\r\nContent-Type: text/html; charset=utf-8\r\nTransfer-Encoding: chunked\r\nConnection: close\r\n\r\n") + .is_err() + || write_chunk(&mut stream, "Example").is_err() + { + return; + } + let started = std::time::Instant::now(); + while started.elapsed() < SLOW_PAGE_STREAM { + thread::sleep(Duration::from_secs(1)); + if write_chunk(&mut stream, "

example slow chunk

").is_err() { + return; + } + } + let _ = write_chunk(&mut stream, &format!("{SLOW_PAGE_END}")) + .and_then(|()| stream.write_all(b"0\r\n\r\n")) + .and_then(|()| stream.flush()); + }) +} + +#[test] +#[ignore = "requires Viceroy, the generated Viceroy config, and the release Fastly WASM"] +fn publisher_page_streaming_35_seconds_completes() { + init_logger(); + let _port = lock_origin_port(); + let listener = + TcpListener::bind(("127.0.0.1", origin_port())).expect("should bind the origin port"); + let origin = serve_slow_streaming_page(listener); + let process = IngressRuntime::Fastly.spawn(); + let started = std::time::Instant::now(); + + let response = reqwest::blocking::Client::builder() + .no_gzip() + .timeout(Duration::from_secs(90)) + .build() + .expect("should build a client that waits for the slow page") + .get(format!("{}/slow-page", process.base_url)) + .header("accept", "text/html") + .header("accept-encoding", "identity") + .send() + .expect("should start the slow page"); + let status = response.status().as_u16(); + let body = response.text().expect("should read the whole slow page"); + + assert_eq!(status, 200, "should serve the slow page"); + assert!( + started.elapsed() >= SLOW_PAGE_STREAM, + "should stream for the origin's full {SLOW_PAGE_STREAM:?}" + ); + assert!( + body.contains(SLOW_PAGE_END), + "should deliver the page past EdgeZero's 30 s default write deadline" + ); + drop(process); + let _ = origin.join(); +} diff --git a/crates/trusted-server-integration-tests/tests/parity.rs b/crates/trusted-server-integration-tests/tests/parity.rs index 60d4f44955..a192140397 100644 --- a/crates/trusted-server-integration-tests/tests/parity.rs +++ b/crates/trusted-server-integration-tests/tests/parity.rs @@ -1,39 +1,64 @@ -//! Cross-adapter parity tests: Axum vs Cloudflare vs Spin in-process. +//! Cross-adapter parity tests: Axum vs Cloudflare vs Spin on host. //! -//! Sends identical requests to all host-runnable adapters and asserts that: +//! Each adapter's app is assembled from shared test settings under its own +//! platform metadata, with its production `configure` policies, and served on +//! `EdgeZero`'s Axum test server over loopback TCP. Every request therefore +//! passes the same ingress, admission, and egress as production Axum. A +//! scripted client that fails every request sits in router state, so no real +//! `EdgeZero` transport starts. Provider request contexts are absent, as the +//! test server documents. +//! +//! The suite sends identical requests to all host-runnable adapters and asserts that: //! - Response status codes match //! - Critical headers (X-Geo-Info-Available, WWW-Authenticate on 401) match //! //! Fastly parity is verified via cargo test-fastly + Viceroy in CI. -// Both adapters define `TrustedServerApp` — alias both to avoid name collision. -// axum::http re-exports from the `http` crate, so HeaderMap types are identical. -use error_stack::Report; -use std::sync::Arc; +// Test crates may buffer bodies with `Body::into_bytes`. +#![allow(clippy::disallowed_methods)] + +// All three adapters define `TrustedServerApp`, so alias each to avoid name collisions. +// reqwest and axum::http re-export the `http` crate, so HeaderMap types are identical. +use std::net::SocketAddr; use std::sync::atomic::{AtomicUsize, Ordering}; -use trusted_server_core::platform::{ - PlatformError, PlatformHttpClient, PlatformHttpRequest, PlatformPendingRequest, - PlatformResponse, PlatformSelectResult, -}; +use std::sync::{Arc, Mutex}; +use std::time::Duration; +use tokio::io::{AsyncReadExt as _, AsyncWriteExt as _}; +use trusted_server_core::platform::HttpClient; use axum::body::Body as AxumBody; use axum::http::Request as AxumRequest; -use edgezero_adapter_axum::service::EdgeZeroAxumService; +use base64::Engine as _; +use edgezero_adapter_axum::AXUM_PLATFORM; +use edgezero_adapter_axum::dev_server::{AxumDevServer, AxumDevServerConfig}; +use edgezero_adapter_axum::outbound::AxumOutboundClient; +use edgezero_adapter_axum::request::into_core_request; +use edgezero_adapter_axum::test_utils::AxumTestServer; +use edgezero_adapter_cloudflare::CLOUDFLARE_PLATFORM; +use edgezero_adapter_spin::SPIN_PLATFORM; +use edgezero_core::app::App; +use edgezero_core::error::EdgeError; use edgezero_core::http::request_builder; use edgezero_core::router::RouterService; use http::HeaderMap; -use tower::{Service as _, ServiceExt as _}; use trusted_server_adapter_axum::app::TrustedServerApp as AxumApp; use trusted_server_adapter_cloudflare::app::TrustedServerApp as CloudflareApp; use trusted_server_adapter_spin::app::TrustedServerApp as SpinApp; +use trusted_server_core::constants::{HEADER_X_TS_VERSION, TS_GIT_VERSION}; use trusted_server_core::settings::Settings; use trusted_server_core::test_support::nextjs_auction; +use trusted_server_core::test_support::scripted_client::{ + ScriptedHttpClient, ScriptedReply, TestClock, +}; /// Shared test settings for all adapters. /// /// The settings baked into the binaries contain placeholder secrets that -/// `get_settings()` rejects by design, so all routers are built through -/// their `routes_with_settings` testing seams from this known-good config. +/// `get_settings()` rejects by design, so every app and router here is built +/// from this known-good config through an adapter testing seam: +/// `routes_with_settings_and_outbound_client` for served apps, +/// `routes_with_settings` for route-table checks, and +/// `routes_with_settings_and_services` for fixed-service fixtures. /// The handler regex is the production-shaped `^/_ts/admin`, matching /// `Settings::ADMIN_ENDPOINTS` and the default config, so the canonical /// `/_ts/admin/keys/*` routes are auth-gated exactly as in production. @@ -58,12 +83,6 @@ fn test_settings() -> Settings { .expect("should parse parity test settings") } -/// Build the Axum adapter router from the shared test settings. -fn axum_router() -> RouterService { - AxumApp::routes_with_settings(test_settings()) - .expect("should build Axum router from parity test settings") -} - /// Build the Cloudflare adapter router from the shared test settings. fn cf_router() -> RouterService { CloudflareApp::routes_with_settings(test_settings()) @@ -76,50 +95,123 @@ fn spin_router() -> RouterService { .expect("should build Spin router from parity test settings") } -/// Send a GET request to the Axum adapter and return (status, headers). -async fn axum_get(uri: &str) -> (u16, HeaderMap) { - let mut svc = EdgeZeroAxumService::new(axum_router()); - let req = AxumRequest::builder() - .method("GET") - .uri(uri) - .body(AxumBody::empty()) - .expect("should build GET request"); - let resp = svc - .ready() +/// A host-runnable adapter whose app the parity suite serves. +#[derive(Clone, Copy)] +enum Adapter { + Axum, + Cloudflare, + Spin, +} + +impl Adapter { + /// Assembles this adapter's app from `settings` under its own platform + /// metadata, with `client` in router state. + fn app(self, settings: Settings, client: HttpClient) -> App { + match self { + Self::Axum => App::from_router::( + AxumApp::routes_with_settings_and_outbound_client(settings, client) + .expect("should build the Axum router from parity test settings"), + AXUM_PLATFORM, + ), + Self::Cloudflare => App::from_router::( + CloudflareApp::routes_with_settings_and_outbound_client(settings, client) + .expect("should build the Cloudflare router from parity test settings"), + CLOUDFLARE_PLATFORM, + ), + Self::Spin => App::from_router::( + SpinApp::routes_with_settings_and_outbound_client(settings, client) + .expect("should build the Spin router from parity test settings"), + SPIN_PLATFORM, + ), + } + .expect("should assemble the app with its production policies") + } +} + +/// Serves `app` on `EdgeZero`'s Axum test server at a free loopback port. +/// +/// The app reads no `EdgeZero` store: its settings are injected, so no store +/// is attached. +async fn serve(app: App) -> AxumTestServer { + let config = AxumDevServerConfig { + addr: SocketAddr::from(([127, 0, 0, 1], 0)), + enable_ctrl_c: false, + ..AxumDevServerConfig::default() + }; + AxumTestServer::start(AxumDevServer::with_config(app, config)) .await - .expect("should be ready") - .call(req) + .expect("should start the Axum test server") +} + +/// Sends one request over TCP to `adapter`'s app and returns (status, headers, body bytes). +/// +/// Each call serves a fresh app built from the shared test settings, with a +/// script that fails every outbound request in router state. +async fn send( + adapter: Adapter, + method: &str, + uri: &str, + headers: &[(&str, &str)], + body: Option<&str>, +) -> (u16, HeaderMap, bytes::Bytes) { + let server = serve(adapter.app( + test_settings(), + HttpClient::with_client(ScriptedHttpClient::forbidding()), + )) + .await; + // No proxy and no redirect following: the tests assert on exactly what + // the listener sends. + let client = reqwest::Client::builder() + .no_proxy() + .redirect(reqwest::redirect::Policy::none()) + .build() + .expect("should build the test HTTP client"); + let method = + reqwest::Method::from_bytes(method.as_bytes()).expect("should parse the test method"); + let mut request = client.request(method, format!("http://{}{uri}", server.address())); + for (name, value) in headers { + request = request.header(*name, *value); + } + if let Some(body) = body { + request = request.body(body.to_owned()); + } + let response = tokio::time::timeout(Duration::from_secs(10), request.send()) .await + .expect("should answer promptly") .expect("should respond"); - (resp.status().as_u16(), resp.headers().clone()) + let status = response.status().as_u16(); + let response_headers = response.headers().clone(); + let response_body = response + .bytes() + .await + .expect("should read the response body"); + server + .shutdown() + .await + .expect("should stop the Axum test server"); + (status, response_headers, response_body) +} + +/// Header that marks a request body as JSON. +const JSON_CONTENT_TYPE: (&str, &str) = ("content-type", "application/json"); + +/// Header carrying the shared test settings' admin credentials. +const ADMIN_AUTHORIZATION: (&str, &str) = ("authorization", "Basic YWRtaW46YWRtaW4tcGFzcw=="); + +/// Send a GET request to the Axum adapter and return (status, headers). +async fn axum_get(uri: &str) -> (u16, HeaderMap) { + let (s, h, _) = send(Adapter::Axum, "GET", uri, &[], None).await; + (s, h) +} + +/// Send a GET request to the Axum adapter and return (status, headers, body bytes). +async fn axum_get_body(uri: &str) -> (u16, HeaderMap, bytes::Bytes) { + send(Adapter::Axum, "GET", uri, &[], None).await } /// Send a POST request to the Axum adapter and return (status, headers, body bytes). async fn axum_post(uri: &str, body: &str) -> (u16, HeaderMap, bytes::Bytes) { - use http_body_util::BodyExt as _; - let mut svc = EdgeZeroAxumService::new(axum_router()); - let req = AxumRequest::builder() - .method("POST") - .uri(uri) - .header("content-type", "application/json") - .body(AxumBody::from(body.to_owned())) - .expect("should build POST request"); - let resp = svc - .ready() - .await - .expect("should be ready") - .call(req) - .await - .expect("should respond"); - let status = resp.status().as_u16(); - let headers = resp.headers().clone(); - let body_bytes = resp - .into_body() - .collect() - .await - .expect("should collect body") - .to_bytes(); - (status, headers, body_bytes) + send(Adapter::Axum, "POST", uri, &[JSON_CONTENT_TYPE], Some(body)).await } /// Convenience wrapper for tests that don't need body. @@ -130,30 +222,25 @@ async fn axum_post_headers(uri: &str, body: &str) -> (u16, HeaderMap) { /// Send a GET request to the Cloudflare adapter and return (status, headers). async fn cf_get(uri: &str) -> (u16, HeaderMap) { - let router = cf_router(); - let req = request_builder() - .method("GET") - .uri(uri) - .body(edgezero_core::body::Body::empty()) - .expect("should build GET request"); - let resp = router.oneshot(req).await.expect("should respond"); - (resp.status().as_u16(), resp.headers().clone()) + let (s, h, _) = cf_get_body(uri).await; + (s, h) +} + +/// Send a GET request to the Cloudflare adapter and return (status, headers, body bytes). +async fn cf_get_body(uri: &str) -> (u16, HeaderMap, bytes::Bytes) { + send(Adapter::Cloudflare, "GET", uri, &[], None).await } /// Send a POST request to the Cloudflare adapter and return (status, headers, body bytes). async fn cf_post(uri: &str, body: &str) -> (u16, HeaderMap, bytes::Bytes) { - let router = cf_router(); - let req = request_builder() - .method("POST") - .uri(uri) - .header("content-type", "application/json") - .body(edgezero_core::body::Body::from(body.to_owned())) - .expect("should build POST request"); - let resp = router.oneshot(req).await.expect("should respond"); - let status = resp.status().as_u16(); - let headers = resp.headers().clone(); - let body_bytes = resp.into_body().into_bytes().unwrap_or_default(); - (status, headers, body_bytes) + send( + Adapter::Cloudflare, + "POST", + uri, + &[JSON_CONTENT_TYPE], + Some(body), + ) + .await } /// Convenience wrapper for tests that don't need body. @@ -164,17 +251,7 @@ async fn cf_post_headers(uri: &str, body: &str) -> (u16, HeaderMap) { /// Send a GET request to the Spin adapter and return (status, headers, body bytes). async fn spin_get_body(uri: &str) -> (u16, HeaderMap, bytes::Bytes) { - let router = spin_router(); - let req = request_builder() - .method("GET") - .uri(uri) - .body(edgezero_core::body::Body::empty()) - .expect("should build GET request"); - let resp = router.oneshot(req).await.expect("should respond"); - let status = resp.status().as_u16(); - let headers = resp.headers().clone(); - let body_bytes = resp.into_body().into_bytes().unwrap_or_default(); - (status, headers, body_bytes) + send(Adapter::Spin, "GET", uri, &[], None).await } /// Send a GET request to the Spin adapter and return (status, headers). @@ -185,18 +262,7 @@ async fn spin_get(uri: &str) -> (u16, HeaderMap) { /// Send a POST request to the Spin adapter and return (status, headers, body bytes). async fn spin_post(uri: &str, body: &str) -> (u16, HeaderMap, bytes::Bytes) { - let router = spin_router(); - let req = request_builder() - .method("POST") - .uri(uri) - .header("content-type", "application/json") - .body(edgezero_core::body::Body::from(body.to_owned())) - .expect("should build POST request"); - let resp = router.oneshot(req).await.expect("should respond"); - let status = resp.status().as_u16(); - let headers = resp.headers().clone(); - let body_bytes = resp.into_body().into_bytes().unwrap_or_default(); - (status, headers, body_bytes) + send(Adapter::Spin, "POST", uri, &[JSON_CONTENT_TYPE], Some(body)).await } /// Convenience wrapper for tests that don't need body. @@ -211,109 +277,62 @@ async fn spin_post_with_headers( body: &str, extra_headers: &[(&str, &str)], ) -> (u16, HeaderMap) { - let router = spin_router(); - let mut builder = request_builder() - .method("POST") - .uri(uri) - .header("content-type", "application/json"); - for (name, value) in extra_headers { - builder = builder.header(*name, *value); - } - let req = builder - .body(edgezero_core::body::Body::from(body.to_owned())) - .expect("should build POST request"); - let resp = router.oneshot(req).await.expect("should respond"); - (resp.status().as_u16(), resp.headers().clone()) + let headers: Vec<(&str, &str)> = std::iter::once(JSON_CONTENT_TYPE) + .chain(extra_headers.iter().copied()) + .collect(); + let (s, h, _) = send(Adapter::Spin, "POST", uri, &headers, Some(body)).await; + (s, h) +} + +/// Send an authorized JSON request to `adapter` and return (status, headers). +async fn authorized_json( + adapter: Adapter, + method: &str, + uri: &str, + body: &str, +) -> (u16, HeaderMap) { + let (s, h, _) = send( + adapter, + method, + uri, + &[ADMIN_AUTHORIZATION, JSON_CONTENT_TYPE], + Some(body), + ) + .await; + (s, h) } /// Send an authorized JSON request to the Axum adapter and return (status, headers). async fn axum_authorized_json(method: &str, uri: &str, body: &str) -> (u16, HeaderMap) { - let mut svc = EdgeZeroAxumService::new(axum_router()); - let req = AxumRequest::builder() - .method(method) - .uri(uri) - .header("authorization", "Basic YWRtaW46YWRtaW4tcGFzcw==") - .header("content-type", "application/json") - .body(AxumBody::from(body.to_owned())) - .expect("should build authorized JSON request"); - let resp = svc - .ready() - .await - .expect("should be ready") - .call(req) - .await - .expect("should respond"); - (resp.status().as_u16(), resp.headers().clone()) + authorized_json(Adapter::Axum, method, uri, body).await } /// Send an authorized JSON request to the Cloudflare adapter and return (status, headers). async fn cf_authorized_json(method: &str, uri: &str, body: &str) -> (u16, HeaderMap) { - let router = cf_router(); - let req = request_builder() - .method(method) - .uri(uri) - .header("authorization", "Basic YWRtaW46YWRtaW4tcGFzcw==") - .header("content-type", "application/json") - .body(edgezero_core::body::Body::from(body.to_owned())) - .expect("should build authorized JSON request"); - let resp = router.oneshot(req).await.expect("should respond"); - (resp.status().as_u16(), resp.headers().clone()) + authorized_json(Adapter::Cloudflare, method, uri, body).await } /// Send an authorized JSON request to the Spin adapter and return (status, headers). async fn spin_authorized_json(method: &str, uri: &str, body: &str) -> (u16, HeaderMap) { - let router = spin_router(); - let req = request_builder() - .method(method) - .uri(uri) - .header("authorization", "Basic YWRtaW46YWRtaW4tcGFzcw==") - .header("content-type", "application/json") - .body(edgezero_core::body::Body::from(body.to_owned())) - .expect("should build authorized JSON request"); - let resp = router.oneshot(req).await.expect("should respond"); - (resp.status().as_u16(), resp.headers().clone()) + authorized_json(Adapter::Spin, method, uri, body).await } /// Send an OPTIONS request to the Axum adapter and return (status, headers). async fn axum_options(uri: &str) -> (u16, HeaderMap) { - let mut svc = EdgeZeroAxumService::new(axum_router()); - let req = AxumRequest::builder() - .method("OPTIONS") - .uri(uri) - .body(AxumBody::empty()) - .expect("should build OPTIONS request"); - let resp = svc - .ready() - .await - .expect("should be ready") - .call(req) - .await - .expect("should respond"); - (resp.status().as_u16(), resp.headers().clone()) + let (s, h, _) = send(Adapter::Axum, "OPTIONS", uri, &[], None).await; + (s, h) } /// Send an OPTIONS request to the Cloudflare adapter and return (status, headers). async fn cf_options(uri: &str) -> (u16, HeaderMap) { - let router = cf_router(); - let req = request_builder() - .method("OPTIONS") - .uri(uri) - .body(edgezero_core::body::Body::empty()) - .expect("should build OPTIONS request"); - let resp = router.oneshot(req).await.expect("should respond"); - (resp.status().as_u16(), resp.headers().clone()) + let (s, h, _) = send(Adapter::Cloudflare, "OPTIONS", uri, &[], None).await; + (s, h) } /// Send an OPTIONS request to the Spin adapter and return (status, headers). async fn spin_options(uri: &str) -> (u16, HeaderMap) { - let router = spin_router(); - let req = request_builder() - .method("OPTIONS") - .uri(uri) - .body(edgezero_core::body::Body::empty()) - .expect("should build OPTIONS request"); - let resp = router.oneshot(req).await.expect("should respond"); - (resp.status().as_u16(), resp.headers().clone()) + let (s, h, _) = send(Adapter::Spin, "OPTIONS", uri, &[], None).await; + (s, h) } // --------------------------------------------------------------------------- @@ -341,46 +360,10 @@ async fn discovery_route_body_is_json_parity() { // return an error body. Assert that whichever body type each returns (JSON or // not) is consistent: if the Cloudflare adapter returns valid JSON then the // Axum adapter must also return valid JSON for the same route. - use http_body_util::BodyExt as _; use serde_json::Value; - let (axum_status, axum_body_bytes) = { - let mut svc = EdgeZeroAxumService::new(axum_router()); - let req = AxumRequest::builder() - .method("GET") - .uri("/.well-known/trusted-server.json") - .body(AxumBody::empty()) - .expect("should build GET request"); - let resp = svc - .ready() - .await - .expect("should be ready") - .call(req) - .await - .expect("should respond"); - let status = resp.status().as_u16(); - let body = resp - .into_body() - .collect() - .await - .expect("should collect body") - .to_bytes(); - (status, body) - }; - - let (cf_status, cf_body_bytes) = { - let router = cf_router(); - let req = request_builder() - .method("GET") - .uri("/.well-known/trusted-server.json") - .body(edgezero_core::body::Body::empty()) - .expect("should build GET request"); - let resp = router.oneshot(req).await.expect("should respond"); - let status = resp.status().as_u16(); - let body = resp.into_body().into_bytes().unwrap_or_default(); - (status, body) - }; - + let (axum_status, _, axum_body_bytes) = axum_get_body("/.well-known/trusted-server.json").await; + let (cf_status, _, cf_body_bytes) = cf_get_body("/.well-known/trusted-server.json").await; let (spin_status, _, spin_body_bytes) = spin_get_body("/.well-known/trusted-server.json").await; // This endpoint serves a static config document, so the bodies must be @@ -1021,12 +1004,266 @@ async fn admin_cache_purge_rejects_credential_forwarding_methods() { } } +// --------------------------------------------------------------------------- +// Production ingress on the Axum test server (EZ-23 criterion 13) +// --------------------------------------------------------------------------- + +/// Environment variables that route reqwest's outbound requests through a proxy. +const PROXY_VARIABLES: [&str; 6] = [ + "HTTP_PROXY", + "HTTPS_PROXY", + "ALL_PROXY", + "http_proxy", + "https_proxy", + "all_proxy", +]; + +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn axum_test_server_parity_rejects_9000_byte_target_with_414() { + // Trusted Server keeps EdgeZero's default 8192-byte request-target limit, + // so a 9000-byte target fails admission on every app before any route runs. + let target = format!("/{}", "a".repeat(8999)); + assert_eq!( + target.len(), + 9000, + "should build a 9000-byte request target" + ); + + let (axum_status, axum_headers) = axum_get(&target).await; + let (cf_status, cf_headers) = cf_get(&target).await; + let (spin_status, spin_headers) = spin_get(&target).await; + + for (adapter, status, headers) in [ + ("Axum", axum_status, axum_headers), + ("Cloudflare", cf_status, cf_headers), + ("Spin", spin_status, spin_headers), + ] { + assert_eq!( + status, 414, + "{adapter} should reject a 9000-byte request target with 414" + ); + // The assembled app renders the rejection under its production + // policies, so it carries Trusted Server's synthesized headers. A + // bare router dispatch would answer 414 without them. + assert_eq!( + headers + .get(HEADER_X_TS_VERSION) + .and_then(|value| value.to_str().ok()), + TS_GIT_VERSION, + "{adapter} should add the synthesized x-ts-version to its 414" + ); + } +} + +#[test] +fn bodiless_get_with_unread_stream_reaches_origin_without_body() { + // EdgeZero's Axum client honors the proxy variables, and so does the Axum + // app's publisher client. Unset them for this test only, through the + // harness's process-wide environment lock. + temp_env::with_vars_unset(PROXY_VARIABLES, || { + tokio::runtime::Builder::new_multi_thread() + .worker_threads(2) + .enable_all() + .build() + .expect("should build the test runtime") + .block_on(assert_bodiless_get_reaches_origin_without_body()); + }); +} + +/// Forwards a bodiless GET whose inbound body is an unread stream and checks +/// that the origin received no body. +async fn assert_bodiless_get_reaches_origin_without_body() { + let origin = RecordingOrigin::start().await; + let mut settings = test_settings(); + settings.publisher.origin_url = format!("http://{}", origin.address); + // EdgeZero's own Axum client goes in router state instead of a script, and + // the publisher forward runs through it. + // `bodiless_get_with_a_forbidding_router_client_never_reaches_the_origin` + // proves the router-state client is the one that carries the forward. + let client = HttpClient::with_client( + AxumOutboundClient::try_new().expect("should build EdgeZero's Axum outbound client"), + ); + let server = serve(Adapter::Axum.app(settings, client)).await; + + // A chunked body that holds only its terminating chunk is bodiless, but + // the listener learns that only by reading it, so the app receives the + // body as an unread stream. + let response = send_raw( + server.address(), + "GET / HTTP/1.1\r\nHost: test-publisher.example.com\r\n\ + Transfer-Encoding: chunked\r\nConnection: close\r\n\r\n0\r\n\r\n", + ) + .await; + assert!( + response.starts_with("HTTP/1.1 200"), + "should relay the origin's 200, got: {response}" + ); + + let records = origin.records(); + assert_eq!(records.len(), 1, "should reach the origin exactly once"); + let record = &records[0]; + assert_eq!(record.method, http::Method::GET, "should forward a GET"); + assert!( + record.body.is_empty(), + "should forward no request body, got {} bytes", + record.body.len() + ); + assert!( + !record.headers.contains_key(http::header::TRANSFER_ENCODING), + "should forward no Transfer-Encoding, got {:?}", + record.headers.get(http::header::TRANSFER_ENCODING) + ); + // EZ-1 allows a bodiless forward to carry `Content-Length: 0`, though it + // should not, so the header must be absent or zero. + let content_length = record.headers.get(http::header::CONTENT_LENGTH); + assert!( + content_length.is_none_or(|value| value == "0"), + "should forward no Content-Length or `Content-Length: 0`, got {content_length:?}" + ); + server + .shutdown() + .await + .expect("should stop the Axum test server"); +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn bodiless_get_with_a_forbidding_router_client_never_reaches_the_origin() { + let origin = RecordingOrigin::start().await; + let mut settings = test_settings(); + settings.publisher.origin_url = format!("http://{}", origin.address); + let server = serve(Adapter::Axum.app( + settings, + HttpClient::with_client(ScriptedHttpClient::forbidding()), + )) + .await; + + // The forbidding script fails the forward wherever it ends, so the request + // leaves only through the router-state client and never succeeds. + + let response = send_raw( + server.address(), + "GET / HTTP/1.1\r\nHost: test-publisher.example.com\r\n\ + Transfer-Encoding: chunked\r\nConnection: close\r\n\r\n0\r\n\r\n", + ) + .await; + + let status = response + .split_whitespace() + .nth(1) + .and_then(|code| code.parse::().ok()) + .expect("should answer with an HTTP status line"); + assert!( + !(200..300).contains(&status), + "should fail the forward inside the router-state client, got: {response}" + ); + assert!( + origin.records().is_empty(), + "should never reach the loopback origin through another client" + ); + server + .shutdown() + .await + .expect("should stop the Axum test server"); +} + +/// What the loopback origin received for one request. +#[derive(Clone)] +struct OriginRecord { + method: http::Method, + headers: HeaderMap, + body: bytes::Bytes, +} + +/// A loopback origin that answers every request with 200 and records what it received. +struct RecordingOrigin { + address: SocketAddr, + records: Arc>>, + task: tokio::task::JoinHandle<()>, +} + +impl RecordingOrigin { + /// Binds `127.0.0.1:0` and starts serving. + async fn start() -> Self { + let records = Arc::new(Mutex::new(Vec::new())); + let recorder = Arc::clone(&records); + let app = axum::Router::new().fallback(move |request: axum::extract::Request| { + let recorder = Arc::clone(&recorder); + async move { + let (parts, body) = request.into_parts(); + let body = axum::body::to_bytes(body, 1024 * 1024) + .await + .expect("should read the origin request body"); + recorder + .lock() + .expect("should lock the origin records") + .push(OriginRecord { + method: parts.method, + headers: parts.headers, + body, + }); + ( + [(http::header::CONTENT_TYPE, "text/plain")], + "example origin", + ) + } + }); + let listener = tokio::net::TcpListener::bind("127.0.0.1:0") + .await + .expect("should bind the loopback origin"); + let address = listener + .local_addr() + .expect("should read the loopback origin address"); + let task = tokio::spawn(async move { + axum::serve(listener, app) + .await + .expect("should serve the loopback origin"); + }); + Self { + address, + records, + task, + } + } + + /// Returns every request received so far, in arrival order. + fn records(&self) -> Vec { + self.records + .lock() + .expect("should lock the origin records") + .clone() + } +} + +impl Drop for RecordingOrigin { + fn drop(&mut self) { + self.task.abort(); + } +} + +/// Writes `request` to a fresh connection to `address` and returns everything the server sends +/// before it closes the connection. +async fn send_raw(address: SocketAddr, request: &str) -> String { + let mut stream = tokio::net::TcpStream::connect(address) + .await + .expect("should connect to the Axum test server"); + stream + .write_all(request.as_bytes()) + .await + .expect("should write the raw request"); + let mut response = Vec::new(); + tokio::time::timeout(Duration::from_secs(10), stream.read_to_end(&mut response)) + .await + .expect("should close the connection promptly") + .expect("should read the raw response"); + String::from_utf8_lossy(&response).into_owned() +} + /// A known non-regulated location permits the fixture's server-side auction. #[tokio::test(flavor = "multi_thread", worker_threads = 2)] async fn adapter_buffers_nextjs_auction_output() { - let client = Arc::new(nextjs_auction::NextJsAuctionOrigin::default()); + let client = nextjs_auction::NextJsAuctionOrigin::default(); let settings = nextjs_auction::settings(); - let services = nextjs_auction::services(Arc::clone(&client)); + let services = nextjs_auction::services(&client); let routers = [ ( "Axum", @@ -1152,74 +1389,72 @@ async fn adapter_buffers_nextjs_auction_output() { } } -#[derive(Default)] +/// Scripted publisher origin and EOF lookup for the HTML finalization parity case. +/// +/// Every request queues its own answer as the script records it: the lookup +/// approves or fails, and every other request gets the example document. struct TerminalFixtureOrigin { - lookup_calls: AtomicUsize, - saw_range: AtomicUsize, - fail_lookup: bool, + clock: TestClock, + client: ScriptedHttpClient, + lookup_calls: Arc, + saw_range: Arc, } -#[async_trait::async_trait(?Send)] -impl PlatformHttpClient for TerminalFixtureOrigin { - async fn send( - &self, - request: PlatformHttpRequest, - ) -> Result> { - let (content_type, body) = if request.request.uri().host() == Some("lookup.example.com") { - tokio::task::yield_now().await; - let payload = request - .request - .into_body() - .into_bytes() - .expect("should send a bounded fixture payload"); - let payload: serde_json::Value = serde_json::from_slice(&payload) - .expect("should encode completed-document metadata"); - assert_eq!( - payload["complete"], true, - "should observe the EOF sentinel before I/O" - ); - assert_eq!( - payload["sha256"] - .as_str() - .expect("should hash transformed HTML") - .len(), - 64, - "should send a complete digest" - ); - self.lookup_calls.fetch_add(1, Ordering::SeqCst); - if self.fail_lookup { - return Err(Report::new(PlatformError::HttpClient)); - } - ("text/plain", "EXAMPLE_APPROVED") - } else { - if request.request.headers().contains_key(http::header::RANGE) { - self.saw_range.fetch_add(1, Ordering::SeqCst); +impl TerminalFixtureOrigin { + fn new(fail_lookup: bool) -> Self { + let clock = TestClock::new(); + let script = ScriptedHttpClient::new(&clock); + let rearm = script.clone(); + let lookup_calls = Arc::new(AtomicUsize::new(0)); + let saw_range = Arc::new(AtomicUsize::new(0)); + let (lookups, ranges) = (Arc::clone(&lookup_calls), Arc::clone(&saw_range)); + let client = script.with_request_observer(move |request| { + let uri = request.uri.to_string(); + if request.uri.host() == Some("lookup.example.com") { + let payload: serde_json::Value = serde_json::from_slice(&request.body) + .expect("should encode completed-document metadata"); + assert_eq!( + payload["complete"], true, + "should observe the EOF sentinel before I/O" + ); + assert_eq!( + payload["sha256"] + .as_str() + .expect("should hash transformed HTML") + .len(), + 64, + "should send a complete digest" + ); + lookups.fetch_add(1, Ordering::SeqCst); + let reply = if fail_lookup { + ScriptedReply::error(EdgeError::bad_gateway("example lookup failure")) + } else { + ScriptedReply::status(http::StatusCode::OK) + .header("content-type", "text/plain") + .header("cache-control", "public, max-age=60") + .body("EXAMPLE_APPROVED") + }; + rearm.respond_to(request.method.clone(), &uri, reply); + } else { + if request.headers.contains_key(http::header::RANGE) { + ranges.fetch_add(1, Ordering::SeqCst); + } + rearm.respond_to( + request.method.clone(), + &uri, + ScriptedReply::status(http::StatusCode::OK) + .header("content-type", "text/html") + .header("cache-control", "public, max-age=60") + .body("

EXAMPLE_UNSIGNED

"), + ); } - ( - "text/html", - "

EXAMPLE_UNSIGNED

", - ) - }; - Ok(PlatformResponse::new( - http::Response::builder() - .status(200) - .header("content-type", content_type) - .header("cache-control", "public, max-age=60") - .body(edgezero_core::body::Body::from(body)) - .expect("should build an example upstream response"), - )) - } - async fn send_async( - &self, - _: PlatformHttpRequest, - ) -> Result> { - Err(Report::new(PlatformError::Unsupported)) - } - async fn select( - &self, - _: Vec, - ) -> Result> { - Err(Report::new(PlatformError::Unsupported)) + }); + Self { + clock, + client, + lookup_calls, + saw_range, + } } } @@ -1233,14 +1468,8 @@ async fn asynchronous_html_finalization_and_privacy_match_host_adapters() { settings.integrations.insert_config("html_finalization_fixture", &serde_json::json!({ "enabled": enabled, "lookup_url": format!("https://lookup.example.com/{case}"), "transport_timeout_ms": 1000, })).expect("should enable the fictional terminal consumer"); - let client = Arc::new(TerminalFixtureOrigin { - lookup_calls: AtomicUsize::new(0), - saw_range: AtomicUsize::new(0), - fail_lookup: case == "failed", - }); - let services = nextjs_auction::services_with_http_client( - Arc::clone(&client) as Arc - ); + let client = TerminalFixtureOrigin::new(case == "failed"); + let services = nextjs_auction::services_with_outbound_client(&client.client, &client.clock); let routers = [ ( "Axum", @@ -1356,11 +1585,107 @@ async fn asynchronous_html_finalization_and_privacy_match_host_adapters() { } } +/// A TCF v2 `euconsent-v2` cookie that grants Purpose 1. +/// +/// The test server has no geo source, so the jurisdiction is unknown and the +/// server-side auction needs Purpose 1. The construction matches the +/// integration suite's consent cookie, which lives in another test binary. +fn tcf_purpose_one_cookie() -> String { + let mut bits = [0_u8; 29]; + let now = std::time::SystemTime::now() + .duration_since(std::time::UNIX_EPOCH) + .expect("should have a current fixture clock") + .as_millis() + / 100; + for (offset, width, value) in [ + (0, 6, 2), + (6, 36, now), + (42, 36, now), + (108, 6, 4), + (114, 6, 13), + (152, 1, 1), + ] { + for index in 0..width { + if value & (1 << (width - index - 1)) != 0 { + bits[(offset + index) / 8] |= 1 << (7 - (offset + index) % 8); + } + } + } + format!( + "euconsent-v2={}", + base64::engine::general_purpose::URL_SAFE_NO_PAD.encode(bits) + ) +} + +/// Only Fastly sends Tinybird telemetry: the other adapters keep the no-op +/// sink, so an auction with Tinybird enabled makes no Events API request. +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn axum_cloudflare_and_spin_send_no_telemetry() { + let mut settings = nextjs_auction::settings(); + settings.tinybird.enabled = true; + settings.tinybird.api_host = "api.tinybird.example.com".to_owned(); + settings.tinybird.auction_token_secret = Some(trusted_server_core::redacted::Redacted::new( + "example-append-token".to_owned(), + )); + + for (name, adapter) in [ + ("Axum", Adapter::Axum), + ("Cloudflare", Adapter::Cloudflare), + ("Spin", Adapter::Spin), + ] { + let scripted = ScriptedHttpClient::new(&TestClock::new()); + let server = serve(adapter.app(settings.clone(), scripted.http_client())).await; + + let response = reqwest::Client::builder() + .no_proxy() + .build() + .expect("should build the test HTTP client") + .post(format!("http://{}/auction", server.address())) + .header("cookie", tcf_purpose_one_cookie()) + .header("content-type", "application/json") + .body( + serde_json::json!({ + "adUnits": [{ + "code": "fixture-slot", + "mediaTypes": { "banner": { "sizes": [[300, 250]] } } + }] + }) + .to_string(), + ) + .send() + .await + .expect("should answer the auction"); + let status = response.status().as_u16(); + let body: serde_json::Value = response + .json() + .await + .expect("should answer with the auction's JSON"); + server + .shutdown() + .await + .expect("should stop the Axum test server"); + + assert_eq!(status, 200, "{name} should complete the auction"); + assert_eq!( + body["ext"]["orchestrator"]["providers"], 1, + "{name} should run the auction, which emits its telemetry rows: {body}" + ); + let requested: Vec = scripted + .requests() + .iter() + .map(|request| request.uri.to_string()) + .collect(); + assert!( + !requested.iter().any(|uri| uri.contains("tinybird")), + "{name} should send no Tinybird request: {requested:?}" + ); + } +} + #[allow(clippy::panic)] #[cfg(test)] mod trace_parity { use std::net::IpAddr; - use std::sync::Mutex; use std::sync::atomic::{AtomicUsize, Ordering}; use std::time::Duration; @@ -1376,10 +1701,9 @@ mod trace_parity { use trusted_server_core::auction::telemetry::{AuctionEventBatch, AuctionTelemetrySink}; use trusted_server_core::error::TrustedServerError; use trusted_server_core::platform::{ - BackendNamingPolicy, ClientInfo, GeoInfo, PlatformBackend, PlatformBackendSpec, - PlatformConfigStore, PlatformError, PlatformGeo, PlatformHttpClient, PlatformHttpRequest, - PlatformKvStore, PlatformPendingRequest, PlatformResponse, PlatformSecretStore, - PlatformSelectResult, RuntimeServices, StoreId, StoreName, UnavailableKvStore, + ClientInfo, GeoInfo, HttpClient, PlatformConfigStore, PlatformError, PlatformGeo, + PlatformKvStore, PlatformSecretStore, RuntimeServices, StoreId, StoreName, + UnavailableKvStore, }; use trusted_server_core::redacted::Redacted; use trusted_server_core::settings::TrustedForwarderConfig; @@ -1387,13 +1711,38 @@ mod trace_parity { use super::*; - struct ForbiddenLifecycle; + /// Platform services that fail the test when a trace or forwarder request reaches them. + /// + /// Its outbound client is a forbid-mode script, which fails each request instead of + /// panicking. Each test checks [`ForbiddenLifecycle::script`] at its end, so the test + /// fails if any outbound request was sent. + struct ForbiddenLifecycle { + outbound: ScriptedHttpClient, + } + impl ForbiddenLifecycle { + fn new() -> Self { + Self { + outbound: ScriptedHttpClient::forbidding(), + } + } + + fn outbound_client(&self) -> HttpClient { + self.outbound.http_client() + } + + /// Returns a handle on the script that records every outbound request. + fn script(&self) -> ScriptedHttpClient { + self.outbound.clone() + } + } + + #[async_trait(?Send)] impl PlatformConfigStore for ForbiddenLifecycle { fn get(&self, _store: &StoreName, _key: &str) -> Result> { panic!("should not read trace config per request"); } - fn put( + async fn put( &self, _store: &StoreId, _key: &str, @@ -1401,10 +1750,11 @@ mod trace_parity { ) -> Result<(), Report> { panic!("should not write trace config"); } - fn delete(&self, _store: &StoreId, _key: &str) -> Result<(), Report> { + async fn delete(&self, _store: &StoreId, _key: &str) -> Result<(), Report> { panic!("should not delete trace config"); } } + #[async_trait(?Send)] impl PlatformSecretStore for ForbiddenLifecycle { fn get_bytes( &self, @@ -1413,7 +1763,7 @@ mod trace_parity { ) -> Result, Report> { panic!("should not read trace secrets"); } - fn create( + async fn create( &self, _store: &StoreId, _key: &str, @@ -1421,24 +1771,10 @@ mod trace_parity { ) -> Result<(), Report> { panic!("should not write trace secrets"); } - fn delete(&self, _store: &StoreId, _key: &str) -> Result<(), Report> { + async fn delete(&self, _store: &StoreId, _key: &str) -> Result<(), Report> { panic!("should not delete trace secrets"); } } - impl PlatformBackend for ForbiddenLifecycle { - fn naming_policy(&self) -> BackendNamingPolicy { - panic!("should not resolve trace backends"); - } - fn predict_name( - &self, - _spec: &PlatformBackendSpec, - ) -> Result> { - panic!("should not predict trace backends"); - } - fn ensure(&self, _spec: &PlatformBackendSpec) -> Result> { - panic!("should not register trace backends"); - } - } #[async_trait(?Send)] impl PlatformKvStore for ForbiddenLifecycle { async fn get_bytes(&self, _key: &str) -> Result, KvError> { @@ -1468,27 +1804,6 @@ mod trace_parity { } } #[async_trait(?Send)] - impl PlatformHttpClient for ForbiddenLifecycle { - async fn send( - &self, - _request: PlatformHttpRequest, - ) -> Result> { - panic!("should not contact trace origin"); - } - async fn send_async( - &self, - _request: PlatformHttpRequest, - ) -> Result> { - panic!("should not launch trace auctions"); - } - async fn select( - &self, - _requests: Vec, - ) -> Result> { - panic!("should not collect trace auctions"); - } - } - #[async_trait(?Send)] impl AuctionTelemetrySink for ForbiddenLifecycle { fn is_enabled(&self) -> bool { panic!("should not inspect trace auction telemetry"); @@ -1527,11 +1842,15 @@ mod trace_parity { } } + /// Trace routers for each adapter over shared forbidden-lifecycle services, with the + /// script that records their outbound requests. + type ForbiddenRouters = (Vec<(&'static str, RouterService)>, ScriptedHttpClient); + fn routers( enabled: bool, auth_pattern: Option<&str>, calls: Arc, - ) -> Vec<(&'static str, RouterService)> { + ) -> ForbiddenRouters { routers_with_client( enabled, auth_pattern, @@ -1549,7 +1868,7 @@ mod trace_parity { auth_pattern: Option<&str>, calls: Arc, client_ip: Option, - ) -> Vec<(&'static str, RouterService)> { + ) -> ForbiddenRouters { let mut settings = test_settings(); settings .integrations @@ -1561,13 +1880,14 @@ mod trace_parity { if let Some(pattern) = auth_pattern { settings.handlers.insert(0,serde_json::from_value(serde_json::json!({"path":pattern,"username":"example-user","password":"example-password"})).expect("should insert trace auth rule")); } - let forbidden = Arc::new(ForbiddenLifecycle); + let forbidden = Arc::new(ForbiddenLifecycle::new()); + let script = forbidden.script(); let services = RuntimeServices::builder() .config_store(forbidden.clone()) .secret_store(forbidden.clone()) .kv_store(forbidden.clone()) - .backend(forbidden.clone()) - .http_client(forbidden.clone()) + .outbound_client(forbidden.outbound_client()) + .clock(TestClock::new().monotonic_clock()) .auction_telemetry_sink(forbidden) .geo(Arc::new(CountingGeo(calls))) .client_info(ClientInfo { @@ -1575,7 +1895,7 @@ mod trace_parity { ..ClientInfo::default() }) .build(); - routers_with_services(settings, services) + (routers_with_services(settings, services), script) } fn routers_with_services( @@ -1607,7 +1927,8 @@ mod trace_parity { #[tokio::test] async fn trace_dispatch_parity_missing_client_ip_skips_geo() { let calls = Arc::new(AtomicUsize::new(0)); - for (adapter, router) in routers_with_client(true, None, Arc::clone(&calls), None) { + let (adapters, outbound) = routers_with_client(true, None, Arc::clone(&calls), None); + for (adapter, router) in adapters { let response = RouterService::oneshot( &router, request_builder() @@ -1628,6 +1949,7 @@ mod trace_parity { 0, "should not synthesize a geo lookup without trusted client IP" ); + outbound.assert_no_requests(); } #[tokio::test] @@ -1656,7 +1978,8 @@ mod trace_parity { }, ), ] { - for (adapter, router) in routers(enabled, pattern, Arc::clone(&calls)) { + let (adapters, outbound) = routers(enabled, pattern, Arc::clone(&calls)); + for (adapter, router) in adapters { let request = request_builder() .method("EXAMPLE-METHOD") .uri(path) @@ -1688,6 +2011,7 @@ mod trace_parity { "{adapter} should not write rejected cookies" ); } + outbound.assert_no_requests(); } } assert_eq!( @@ -1700,7 +2024,8 @@ mod trace_parity { #[tokio::test] async fn trace_dispatch_parity_read_only_setup_and_verified_assets() { let calls = Arc::new(AtomicUsize::new(0)); - for (adapter, router) in routers(true, None, Arc::clone(&calls)) { + let (adapters, outbound) = routers(true, None, Arc::clone(&calls)); + for (adapter, router) in adapters { for (method, path, expected_calls) in [ (Method::HEAD, "/_ts/trace", 0), (Method::GET, "/_ts/trace/state", 0), @@ -1776,6 +2101,7 @@ mod trace_parity { } } } + outbound.assert_no_requests(); } fn trusted_server_js_asset(path: &str) -> &'static [u8] { @@ -1787,7 +2113,8 @@ mod trace_parity { #[tokio::test] async fn trace_dispatch_parity_stream_actions_preserve_cookie_policy_and_separate_state() { let calls = Arc::new(AtomicUsize::new(0)); - for (adapter, router) in routers(true, None, Arc::clone(&calls)) { + let (adapters, outbound) = routers(true, None, Arc::clone(&calls)); + for (adapter, router) in adapters { for action in ["enable", "end"] { let mut request = request_builder() .method("POST") @@ -1797,7 +2124,7 @@ mod trace_parity { .header("sec-fetch-site", "same-origin") .header("x-ts-trace-action", action) .header("cookie", "__Host-ts-console=1, unrelated=value") - .body(edgezero_core::body::Body::from_stream( + .body(edgezero_core::body::Body::from_external_stream( futures_trace_empty_stream(), )) .expect("should build no-content-type streamed action"); @@ -1875,6 +2202,7 @@ mod trace_parity { 0, "should not lookup metadata for actions or state" ); + outbound.assert_no_requests(); } const FORWARDER_HEADER: &str = "x-example-forwarder-key"; @@ -1923,24 +2251,26 @@ mod trace_parity { request } - fn forwarder_trace_routers(settings: Settings) -> Vec<(&'static str, RouterService)> { - let forbidden = Arc::new(ForbiddenLifecycle); + fn forwarder_trace_routers(settings: Settings) -> ForbiddenRouters { + let forbidden = Arc::new(ForbiddenLifecycle::new()); + let script = forbidden.script(); let services = RuntimeServices::builder() .config_store(forbidden.clone()) .secret_store(forbidden.clone()) .kv_store(forbidden.clone()) - .backend(forbidden.clone()) - .http_client(forbidden.clone()) + .outbound_client(forbidden.outbound_client()) + .clock(TestClock::new().monotonic_clock()) .auction_telemetry_sink(forbidden) .geo(Arc::new(CountingGeo(Arc::new(AtomicUsize::new(0))))) .client_info(ClientInfo::default()) .build(); - routers_with_services(settings, services) + (routers_with_services(settings, services), script) } #[tokio::test] async fn trusted_forwarder_parity_trace_actions_use_public_origin() { - for (adapter, router) in forwarder_trace_routers(forwarder_settings()) { + let (adapters, outbound) = forwarder_trace_routers(forwarder_settings()); + for (adapter, router) in adapters { for action in ["enable", "end"] { let mut request = forwarded_request(&format!("/_ts/trace/{action}")); request.headers_mut().insert( @@ -1986,18 +2316,24 @@ mod trace_parity { ); } } + outbound.assert_no_requests(); } #[tokio::test(flavor = "multi_thread", worker_threads = 2)] async fn trusted_forwarder_axum_boundary_requires_transport_binding() { - let router = AxumApp::routes_with_settings(forwarder_settings()) - .expect("should build native Axum forwarded routes"); - let mut service = EdgeZeroAxumService::new(router); + let outbound = ScriptedHttpClient::forbidding(); + let router = AxumApp::routes_with_settings_and_outbound_client( + forwarder_settings(), + HttpClient::with_client(outbound.clone()), + ) + .expect("should build native Axum forwarded routes"); for duplicate in [false, true] { let request = forwarded_request("/_ts/trace/enable"); let (mut parts, _) = request.into_parts(); // Without the private binding from a real HTTP listener, the SDK // must leave transport origin unavailable even for an absolute URI. + // Every request on the Axum test server carries that binding, so + // this case converts through the SDK's public request converter. parts.extensions.remove::(); if duplicate { parts.headers.append( @@ -2005,13 +2341,12 @@ mod trace_parity { http::HeaderValue::from_static(FORWARDER_SECRET), ); } - let response = service - .ready() + let request = into_core_request(AxumRequest::from_parts(parts, AxumBody::empty())) .await - .expect("should ready native Axum service") - .call(AxumRequest::from_parts(parts, AxumBody::empty())) + .expect("should convert the native Axum request"); + let response = RouterService::oneshot(&router, request) .await - .expect("should convert and dispatch forwarded action"); + .expect("should dispatch forwarded action"); assert_eq!( response.status(), StatusCode::FORBIDDEN, @@ -2022,11 +2357,13 @@ mod trace_parity { "should not mutate cookies without transport binding" ); } + outbound.assert_no_requests(); } #[tokio::test] async fn trusted_forwarder_parity_rejects_untrusted_and_ambiguous_actions() { - for (adapter, router) in forwarder_trace_routers(forwarder_settings()) { + let (adapters, outbound) = forwarder_trace_routers(forwarder_settings()); + for (adapter, router) in adapters { for case in [ "missing-auth", "wrong-auth", @@ -2117,13 +2454,15 @@ mod trace_parity { ); } } + outbound.assert_no_requests(); } #[tokio::test] async fn trusted_forwarder_parity_default_config_keeps_transport_authority() { let mut settings = forwarder_settings(); settings.trusted_forwarder = None; - for (adapter, router) in forwarder_trace_routers(settings) { + let (adapters, outbound) = forwarder_trace_routers(settings); + for (adapter, router) in adapters { for origin in [PUBLIC_ORIGIN, "http://upstream.example.com:8080"] { let mut request = forwarded_request("/_ts/trace/enable"); request.headers_mut().insert( @@ -2144,11 +2483,13 @@ mod trace_parity { ); } } + outbound.assert_no_requests(); } #[tokio::test] async fn trusted_forwarder_parity_preserves_cookie_fidelity() { - for (adapter, router) in forwarder_trace_routers(forwarder_settings()) { + let (adapters, outbound) = forwarder_trace_routers(forwarder_settings()); + for (adapter, router) in adapters { for (preservation, expected) in [ (Preservation::Unknown, false), (Preservation::Preserved, true), @@ -2199,93 +2540,68 @@ mod trace_parity { ); } } + outbound.assert_no_requests(); } - struct ForwarderOrigin { - base: RuntimeServices, - requests: Mutex>, - } - - impl PlatformBackend for ForwarderOrigin { - fn naming_policy(&self) -> BackendNamingPolicy { - self.base.backend().naming_policy() - } - - fn predict_name( - &self, - spec: &PlatformBackendSpec, - ) -> Result> { - self.base.backend().predict_name(spec) - } + /// Geo lookup that knows no location. + struct NoGeo; - fn ensure(&self, spec: &PlatformBackendSpec) -> Result> { - self.base.backend().ensure(spec) - } - } - - impl PlatformGeo for ForwarderOrigin { - fn lookup(&self, ip: Option) -> Result, Report> { - self.base.geo().lookup(ip) - } - } - - #[async_trait(?Send)] - impl PlatformHttpClient for ForwarderOrigin { - async fn send( - &self, - request: PlatformHttpRequest, - ) -> Result> { - self.requests - .lock() - .expect("should lock recorded origin requests") - .push(( - request.request.uri().to_string(), - request.request.headers().clone(), - )); - if request.request.uri().host() == Some("vendor.example.com") { - return Ok(PlatformResponse::new( - http::Response::builder() - .header(header::CONTENT_TYPE, "application/json") - .body(edgezero_core::body::Body::from( - serde_json::json!({"example":true}).to_string(), - )) - .expect("should answer fictional vendor request"), - )); - } - self.base.http_client().send(request).await - } - - async fn send_async( - &self, - _request: PlatformHttpRequest, - ) -> Result> { - panic!("should not launch auctions from forwarder fixture"); - } - - async fn select( - &self, - _requests: Vec, - ) -> Result> { - panic!("should not collect auctions from forwarder fixture"); + impl PlatformGeo for NoGeo { + fn lookup(&self, _ip: Option) -> Result, Report> { + Ok(None) } } #[tokio::test] async fn trusted_forwarder_parity_proxy_strips_credentials_and_retains_backend_targets() { - let origin = Arc::new(ForwarderOrigin { - base: nextjs_auction::services( - Arc::new(nextjs_auction::NextJsAuctionOrigin::default()), - ), - requests: Mutex::new(Vec::new()), + let forbidden = Arc::new(ForbiddenLifecycle::new()); + // The publisher origin and the Lockr leg both reach this script. Every + // origin request queues the Next.js page for itself as it is recorded, + // and the vendor answers once per adapter. + let clock = TestClock::new(); + let script = ScriptedHttpClient::new(&clock); + let rearm = script.clone(); + let upstream = script.with_request_observer(move |request| { + if request.uri.host() == Some(nextjs_auction::ORIGIN_HOST) { + rearm.respond_to( + request.method.clone(), + &request.uri.to_string(), + ScriptedReply::status(StatusCode::OK) + .header("content-type", "text/html") + .body(nextjs_auction::origin_html()), + ); + } }); - let forbidden = Arc::new(ForbiddenLifecycle); + for _ in 0..3 { + upstream.respond_to( + Method::GET, + "https://vendor.example.com/example", + ScriptedReply::status(StatusCode::OK) + .header("content-type", "application/json") + .body(serde_json::json!({"example":true}).to_string()), + ); + } + let origin_requests = || { + upstream + .requests() + .into_iter() + .filter(|request| request.uri.host() == Some(nextjs_auction::ORIGIN_HOST)) + .collect::>() + }; + let vendor_requests = || { + upstream + .requests() + .into_iter() + .filter(|request| request.uri.host() == Some("vendor.example.com")) + .collect::>() + }; let services = RuntimeServices::builder() .config_store(forbidden.clone()) .secret_store(forbidden) .kv_store(Arc::new(UnavailableKvStore)) - .backend(origin.clone()) - .http_client(origin.clone()) - .geo(origin.clone()) + .outbound_client(upstream.http_client()) + .clock(clock.monotonic_clock()) + .geo(Arc::new(NoGeo)) .client_info(ClientInfo::default()) .build(); let mut settings = forwarder_settings(); @@ -2298,11 +2614,8 @@ mod trace_parity { .expect("should enable public URL rewriting"); settings.integrations.insert_config("lockr", &serde_json::json!({"enabled":true,"app_id":"fictional-app","api_endpoint":"https://vendor.example.com","sdk_url":"https://vendor.example.com/sdk.js"})).expect("should configure fictional integration upstream"); for (adapter, router) in routers_with_services(settings, services) { - origin - .requests - .lock() - .expect("should reset recorded origin requests") - .clear(); + let origin_sent = origin_requests().len(); + let vendor_sent = vendor_requests().len(); for (method, path, target) in [ ( Method::GET, @@ -2368,12 +2681,17 @@ mod trace_parity { "{adapter} should not generate internal transport URLs" ); } - let requests = origin - .requests - .lock() - .expect("should inspect actual outbound request"); - let (actual_target, headers) = - requests.last().expect("should reach configured upstream"); + let (actual_target, headers) = if path.starts_with("/integrations/") { + let request = vendor_requests() + .pop() + .expect("should reach the integration upstream"); + (request.uri.to_string(), request.headers) + } else { + let request = origin_requests() + .pop() + .expect("should reach configured upstream"); + (request.uri.to_string(), request.headers) + }; assert_eq!( actual_target, target, "{adapter} should keep configured backend target for {method} {path}" @@ -2393,13 +2711,12 @@ mod trace_parity { ); } assert_eq!( - origin - .requests - .lock() - .expect("should count outbound requests") - .len(), - 3, - "{adapter} should dispatch exactly the three intended upstream requests" + ( + origin_requests().len() - origin_sent, + vendor_requests().len() - vendor_sent + ), + (2, 1), + "{adapter} should dispatch exactly the two publisher and one integration requests" ); for method in [Method::GET, Method::PUT, Method::DELETE, Method::OPTIONS] { let mut request = forwarded_request("/_ts/trace/enable"); @@ -2418,12 +2735,11 @@ mod trace_parity { ); } assert_eq!( - origin - .requests - .lock() - .expect("should count requests after early returns") - .len(), - 3, + ( + origin_requests().len() - origin_sent, + vendor_requests().len() - vendor_sent + ), + (2, 1), "{adapter} should keep forwarding credentials away from all upstreams on trace early returns" ); } diff --git a/docs/guide/ad-serving.md b/docs/guide/ad-serving.md index 7cd3b03b78..95b8d99b97 100644 --- a/docs/guide/ad-serving.md +++ b/docs/guide/ad-serving.md @@ -14,7 +14,7 @@ publisher proxy path. `POST /auction` accepts the documented auction request shape. When `[auction].enabled = true`, the compiled `AuctionPlan` selects configured -providers, routes bidder codes, executes supported provider fan-out, applies an +providers, routes bidder codes, sends every provider request in one batch, applies an optional mediator, and returns the winning bids. When auctions are disabled, the endpoint returns a no-bid response without contacting a provider. @@ -45,8 +45,8 @@ routing = "explicit" provider = "pbs-main" ``` -Deploy validation compiles this configuration before publication. Adapter -startup adds target-specific provider-count and backend-name checks; see +Deploy validation compiles this configuration before publication. Fastly +startup adds its bound of 160 provider backends; see [Auction orchestration](/guide/auction-orchestration) for the complete schema and adapter limits. @@ -68,10 +68,11 @@ assets at runtime. end-to-end checks. - Confirm provider authorization and test inventory with each upstream before enabling traffic. -- Collection stops network waiting at the auction deadline measured from dispatch. - Complete responses already available remain eligible; unfinished providers time - out and mediation is skipped when no budget remains. Fastly abandons unfinished - requests; other adapters cancel owned request tasks. See +- Bidder requests leave together as one batch whose cutoff is the auction + deadline, measured from auction start. A bid that has not arrived by the + cutoff becomes a provider timeout, and mediation is skipped when no budget + remains. Dropping the batch cancels in-flight requests on every adapter; + Fastly host cancellation is best effort. See [Timeout behavior](/guide/auction-orchestration#timeout-behavior). See also [Creative processing](/guide/creative-processing) and the diff --git a/docs/guide/api-reference.md b/docs/guide/api-reference.md index a84707f492..42fcc7b2e2 100644 --- a/docs/guide/api-reference.md +++ b/docs/guide/api-reference.md @@ -18,10 +18,10 @@ Quick reference for all Trusted Server HTTP endpoints. | Adapter | Release status | Health | Startup status | Startup health | Provider fan-out | Trusted-client-IP handling | Request normalization | | ------------ | -------------- | ---------- | -------------- | -------------- | ---------------- | ------------------------------ | -------------------------------- | -| `axum` | development | real | `500` | no | multiple | outermost sanitize | none | -| `cloudflare` | development | absent | `500` | no | single | outermost sanitize | none | +| `axum` | development | real | exit 1 | no | multiple | outermost sanitize | none | +| `cloudflare` | development | absent | `500` | no | multiple | outermost sanitize | none | | `fastly` | production | pre router | `500` | yes | multiple | entry-point resolve + sanitize | none | -| `spin` | experimental | real | `503` | yes | single | outermost sanitize | innermost Spin-header derivation | +| `spin` | experimental | real | `503` | no | multiple | outermost sanitize | innermost Spin-header derivation | `trusted client IP handling` describes the optional `[trusted_client_ip]` configuration. Fastly resolves the authenticated header @@ -62,14 +62,16 @@ authority, scheme, and client-address headers in the innermost middleware. | normal | `/{*rest}` | `DELETE`, `GET`, `HEAD`, `OPTIONS`, `PATCH`, `POST`, `PUT` | template | `publisher_fallback` | publisher fallback | publisher fallback | publisher fallback | publisher fallback | | normal | `/` | `DELETE`, `GET`, `HEAD`, `OPTIONS`, `PATCH`, `POST`, `PUT` | literal | `publisher_fallback` | publisher fallback | publisher fallback | publisher fallback | publisher fallback | | normal | `{*rest}` | `GET`, `HEAD` | config derived | `settings.proxy.asset_routes[]` | real | — | — | — | -| startup | `/health` | `GET` | literal | `startup_error` | real | — | — | real | -| startup | `/{*rest}` | `DELETE`, `GET`, `HEAD`, `OPTIONS`, `PATCH`, `POST`, `PUT` | template | `startup_error` | startup error (`500`) | startup error (`500`) | startup error (`500`) | startup error (`503`) | -| startup | `/` | `DELETE`, `GET`, `HEAD`, `OPTIONS`, `PATCH`, `POST`, `PUT` | literal | `startup_error` | startup error (`500`) | startup error (`500`) | startup error (`500`) | startup error (`503`) | +| startup | `/health` | `GET` | literal | `startup_error` | real | exits with status `1` | startup error (`500`) | startup error (`503`) | +| startup | `/{*rest}` | `DELETE`, `GET`, `HEAD`, `OPTIONS`, `PATCH`, `POST`, `PUT` | template | `startup_error` | startup error (`500`) | exits with status `1` | startup error (`500`) | startup error (`503`) | +| startup | `/` | `DELETE`, `GET`, `HEAD`, `OPTIONS`, `PATCH`, `POST`, `PUT` | literal | `startup_error` | startup error (`500`) | exits with status `1` | startup error (`500`) | startup error (`503`) | -`real`, `unsupported`, `guarded`, `publisher fallback`, and `startup error` are -distinct observable dispositions. An em dash means that the adapter has no -local route; the publisher fallback can therefore receive that path. The -startup rows are a separate degraded router, not additional normal routes. +`real`, `unsupported`, `guarded`, `publisher fallback`, `startup error`, and +`exits with status 1` are distinct observable dispositions. An em dash means +that the adapter has no local route; the publisher fallback can therefore +receive that path. The startup rows show what each path answers when app assembly fails, +in place of every normal route. Fastly answers `/health` before it assembles +the app, and Axum exits with status 1 before it listens. ## Contract conventions @@ -86,8 +88,14 @@ that is not a blanket CORS grant. There is no repository-wide error body schema. Some handlers return structured JSON, some return an empty body, and shared adapter errors are plain text with -safe client-facing messages. Treat the status and body documented for each -endpoint as authoritative. +safe client-facing messages. EdgeZero's own errors, such as a canonical 404 or +405, a 414 or 431 head-limit rejection, a 400 for ambiguous request framing, or +a startup failure, are JSON with redacted text. Treat the status and body +documented for each endpoint as authoritative. + +An endpoint that reads a request body returns `413` when the declared or +received length passes its cap, and `408` when the body is still unread 30 s +after the request arrived. ## Utility Endpoints @@ -1006,10 +1014,15 @@ effects. CORS is therefore the sanitized publisher-origin policy, not a global Trusted Server policy. There is no built-in endpoint rate limiter. Example: `curl -i https://edge.example.com/article`. -When startup fails, the degraded router replaces normal routing: Fastly, -Axum, and Cloudflare return `500` for both fallback shapes and all seven -methods; Spin returns a generic `503`. Only the health behavior shown in the -table survives. +The request body streams to the origin under a 16 MiB cap, and a larger upload +returns `413`. A `GET` or `HEAD` that carries body bytes returns `400`, and so +does a forward the origin request cannot carry, such as a target with a +fragment or a backslash, or a malformed `Connection` header. + +When startup fails, Fastly and Cloudflare answer both fallback shapes and all +seven methods with EdgeZero's JSON error at the startup failure's status, +normally `500`; Spin answers a generic `503`; Axum exits at startup and serves +nothing. Only the health behavior shown in the table survives. --- @@ -1136,7 +1149,7 @@ available when a deployment needs either. | DataDome signals | `GET` or `POST /integrations/datadome/js/` and `/js/*`; method, query, bounded POST body, and selected browser headers forwarded to `api_origin` | Upstream status/body/headers preserved; transport or size failures use shared integration errors; no added cache/CORS policy | Browser SDK traffic; manual payload is upstream-schema-specific | | Didomi consent | `GET` or `POST` under the configured prefix (default `/integrations/didomi/consent/*`); path selects SDK or API origin; query and bounded POST body forwarded | Upstream status/body preserved; SDK responses receive the integration's CORS headers; API responses retain selected upstream headers; no local cache policy | `curl -i https://edge.example.com/integrations/didomi/consent/loader.js` | | GTM/gtag scripts | `GET` the generated `gtm.js`, `gtag.js`, or `gtag/js` paths; query forwarded or configured container ID supplied; successful script is rewritten | Non-success upstream status preserved; rewritten scripts use `cache_max_age`; oversized rewritten upstream bodies use shared integration errors | `curl -i 'https://edge.example.com/integrations/google_tag_manager/gtm.js?id=GTM-XXXX'` | -| Google collect | `GET` or `POST` the generated `collect` or `g/collect` paths; query, selected headers, and bounded body proxy to the configured Google origin | Malformed `Content-Length` returns `400`; body over `max_beacon_body_size` returns `413`; stream-read failure returns `502`; upstream response otherwise preserved | Browser beacon; body schema belongs to Google Analytics | +| Google collect | `GET` or `POST` the generated `collect` or `g/collect` paths; query, selected headers, and bounded body proxy to the configured Google origin | Malformed `Content-Length` returns `400`; body over `max_beacon_body_size` returns `413`; body unread at 30 s returns `408`; failed read returns `502`; upstream response otherwise preserved | Browser beacon; body schema belongs to Google Analytics | | JS asset proxy | `GET` each configured `[[integrations.js_asset_proxy.assets]]` path whose `proxy = "enabled"`; the exact `origin_url` is fetched and served first-party | Upstream failures use shared integration errors; successful responses honor the per-asset or integration `cache_ttl_seconds`; `blocked` assets register no route and strip matching tags | Path is operator-configured, for example `curl -i https://edge.example.com/js/vendor-tag.js` | | GPT | `GET` `/script`, `/pagead/*`, or `/tag/*`; path/query proxy to the configured GPT origins and script content can be rewritten | Upstream status is preserved; successful scripts/assets apply integration cache rules; selected upstream CORS is preserved | `curl -i https://edge.example.com/integrations/gpt/script` | | Lockr SDK | `GET /integrations/lockr/sdk`; no body; fetches and returns the configured SDK as JavaScript | Successful SDK uses `cache_ttl_seconds`; upstream/transport failures follow integration mapping; no added CORS policy | `curl -i https://edge.example.com/integrations/lockr/sdk` | diff --git a/docs/guide/architecture.md b/docs/guide/architecture.md index fb02fe2e68..262b3a81ac 100644 --- a/docs/guide/architecture.md +++ b/docs/guide/architecture.md @@ -59,7 +59,8 @@ Native Axum dev/test adapter (native binary): | Geo lookup | Always returns `None` | | Config/secret-store writes | Return an error (read-only via env vars) | | Admin key management (`/_ts/admin/keys/*`) | Returns 501 Not Implemented. Retired `/admin/keys` aliases, including trailing, descendant, and percent-encoded forms, are denied locally with 404 and are not proxied to the publisher fallback | -| Auction fan-out ordering | Requests run concurrently via `tokio::spawn`; `select` returns first-to-complete but does not replicate Fastly's priority-queue tie-breaking | +| Auction fan-out timing | Bidder requests run as an EdgeZero batch on a local task, so they leave at the next await instead of during batch setup, and their timing does not represent Fastly | +| Startup failure | The process exits non-zero before it listens | ### trusted-server-adapter-spin @@ -67,8 +68,10 @@ Fermyon Spin adapter (`wasm32-wasip1` component): - Production-capable deployment target for the Spin runtime - Platform services (config store, secret store, KV) backed by Spin component variables and the EdgeZero KV handle -- Outbound HTTP via `spin_sdk::http::send` — no configurable per-request timeout (see rustdoc) -- Single auction provider only; enabled multi-provider plans fail target validation at startup +- Outbound HTTP through EdgeZero's Spin client, which cannot override the + `Host` authority, so Spin ignores `publisher.origin_host_header_override` + with a warning +- Multi-provider auctions fan out as one EdgeZero request batch ```bash # Check (native) @@ -95,14 +98,17 @@ cargo clippy-spin-wasm Core request and response values use the standard `http` types with `edgezero_core::body::Body`. Runtime-specific capabilities are supplied through `RuntimeServices`, which holds trait objects for configuration and secret -stores, KV access, dynamic backends, outbound HTTP, geographic lookup, template -assembly, and template caching. - -The contracts live under `trusted_server_core::platform`, including -`PlatformConfigStore`, `PlatformSecretStore`, `PlatformBackend`, -`PlatformHttpClient`, `PlatformGeo`, `PlatformTemplateAssembler`, and -`PlatformTemplateCache`. Each adapter constructs the services it supports; -unavailable capabilities fail through explicit unavailable implementations. +stores, KV access, geographic lookup, template assembly, and template caching, +and the request's EdgeZero `HttpClient` and `MonotonicClock`. + +The store and lookup contracts live under `trusted_server_core::platform`, +including `PlatformConfigStore`, `PlatformSecretStore`, `PlatformGeo`, +`PlatformTemplateAssembler`, and `PlatformTemplateCache`. Outbound HTTP has no +Trusted Server contract: each adapter passes the client that EdgeZero installs +for the request, with the proxy marker off, and Fastly work after delivery +builds a fresh EdgeZero client. Each adapter constructs the services it +supports; unavailable capabilities fail through explicit unavailable +implementations. ### Settings-Driven Configuration @@ -168,13 +174,12 @@ Page content and request bodies are processed in-flight and are not persisted. E The workspace has multiple WASM runtimes with runtime-specific SDKs. Use target-matched clippy aliases (`cargo clippy-fastly`, `cargo clippy-spin-native`, etc.) rather than broad `--all-features` workspace clippy — the latter is not a reliable gate across adapters. -Fastly and Axum support concurrent auction provider fan-out. Cloudflare and Spin -currently accept at most one provider in an enabled auction. Every adapter runs -target-aware fan-out and backend-name checks at startup. Auction collection uses -an absolute cutoff for network waiting across headers and complete response bodies. -A nonblocking snapshot retains complete results after a slow origin. Fastly -abandons pending backend requests because its SDK cannot cancel them; other -adapters cancel their owned operations at cutoff. See +Every adapter sends an auction's bidder requests as one EdgeZero request batch +whose cutoff is the auction deadline, so a multi-provider auction fans out on +Fastly, Axum, Cloudflare, and Spin alike. Fastly startup also rejects an enabled +plan that could need more than 160 provider backends. A bid that has not arrived +by the cutoff becomes a provider timeout. Dropping the batch cancels in-flight +bidder and mediator requests; Fastly host cancellation is best effort. See [Auction timeout behavior](/guide/auction-orchestration#timeout-behavior) for scheduling, worker-runtime dispatch, and local processing limitations. diff --git a/docs/guide/asset-routes.md b/docs/guide/asset-routes.md index 405bc1d1bb..9d69d332c7 100644 --- a/docs/guide/asset-routes.md +++ b/docs/guide/asset-routes.md @@ -16,7 +16,7 @@ For a matching `GET` or `HEAD` request, Trusted Server runs this sequence: 6. Send the request to the resolved backend origin. 7. Return the origin status, body, and safe headers. -The origin query policy runs before S3 signing, so the signature covers the exact URL sent to S3. Image Optimizer metadata stays separate from the origin URL and is translated by the Fastly adapter. +The origin query policy runs before S3 signing, so the signature covers the exact URL sent to S3. Image Optimizer options stay separate from the origin URL. They go on the final `GET` or `HEAD` as EdgeZero Image Optimizer options, never on the S3 preflight `HEAD`. EdgeZero's Fastly client applies them; the Axum, Cloudflare, and Spin clients reject them before sending. ## Basic route @@ -196,7 +196,7 @@ The asset route still matches. Path rewrite, origin query policy, and S3 signing ## Local testing notes -Viceroy does not perform real Fastly Image Optimizer transformations. Local tests can verify routing, query stripping, SigV4 headers, and metadata attachment. End-to-end image transformation verification requires a deployed Fastly Compute service with Image Optimizer enabled. +Viceroy does not perform Fastly Image Optimizer transformations. When the host cannot run Image Optimizer, EdgeZero's Fastly client fails the request and the asset proxy answers 502. Local tests verify routing, query stripping, SigV4 headers, and the options the request carries, read back through the scripted client. End-to-end image transformation verification requires a deployed Fastly Compute service with Image Optimizer enabled. ## Security checklist diff --git a/docs/guide/auction-orchestration.md b/docs/guide/auction-orchestration.md index 2dd9d3016b..3e5294a55b 100644 --- a/docs/guide/auction-orchestration.md +++ b/docs/guide/auction-orchestration.md @@ -8,7 +8,7 @@ The auction orchestrator is the core system that manages server-side ad auctions Key capabilities: -- **Parallel execution** — Bid requests to all providers launch concurrently using Fastly's `select()` API +- **Parallel execution** — Bid requests to all providers leave together as one EdgeZero request batch on every adapter - **Strategy-based winner selection** — Automatic strategy detection based on configuration - **Mediator support** — Optional external mediator for final winner selection and unified floor pricing - **Provider abstraction** — Pluggable provider interface for adding new demand sources @@ -185,7 +185,7 @@ POST /auction (AdRequest in Prebid.js format) AuctionOrchestrator.run_auction() │ ├─ Detect strategy (parallel_only or parallel_mediation) - ├─ Launch all providers in parallel via select() + ├─ Send all bidder requests as one EdgeZero batch ├─ Collect responses as they complete │ ├─[parallel_only]─── Select highest decoded CPM per slot @@ -330,15 +330,17 @@ Mediation is optional for APS. APS reduces to one candidate per impression befor Demand sources implement the async, platform-neutral [`AuctionProvider`](https://github.com/IABTechLab/trusted-server/blob/main/crates/trusted-server-core/src/auction/provider.rs). -The trait receives an `AuctionRequest` and `AuctionContext`, launches a request -as a `ProviderRequestOutcome`, and parses a `PlatformResponse` into an -`AuctionResponse`. It also supplies capability, timeout, enablement, and -platform-backend metadata. Providers that need request-local response state use -the context-aware parsing hooks instead of storing mutable state on the shared -provider instance. - -The orchestrator launches every request before collecting pending responses, so -providers can run concurrently without depending on a Fastly-specific API. +The trait receives an `AuctionRequest` and `AuctionContext` and returns a +`ProviderRequestOutcome`: `Ready` with an unsent EdgeZero `OutboundRequest`, or +an immediate response that needs no upstream call. It parses the +`OutboundResponse` into an `AuctionResponse`, and also supplies media-type +support, its timeout, and enablement. Providers that need request-local +response state use the context-aware parsing hooks instead of storing mutable +state on the shared provider instance. + +The orchestrator builds and signs every bidder request before it sends any, then +sends them together as one batch, so providers run concurrently without +depending on a Fastly-specific API. ### Prebid Provider @@ -767,7 +769,7 @@ timeout_ms = 500 ``` `[auction.providers]` is a map, not a provider-name list. Each provider ID owns -endpoint/backend correlation and telemetry. `[auction.bidders]` maps each +endpoint correlation and telemetry. `[auction.bidders]` maps each client-visible bidder ID to one provider. The mediator remains a separately registered integration selected by `[auction].mediator`. @@ -783,6 +785,9 @@ Common provider fields and defaults: | `profile_config` | `{}` | Profile-owned typed settings | | `notifications` | No suppression | Common `nurl`/`burl` suppression by all bids or returned seats | +A local test bidder's `endpoint` may instead be plain HTTP to `127.0.0.1` or +`[::1]`. + APS normally uses `all_eligible`, which sends every compatible banner slot but never another provider's bidder parameters. `explicit` providers receive only centrally routed or trusted stored-request demand. The `prebid-server` profile @@ -812,58 +817,54 @@ are each limited to 16 KiB, eight container levels, and 256 keys at one object level. Notification seat lists are limited to 128 unique entries of at most 128 UTF-8 bytes each. -### Validation and target capability +### Validation and the Fastly backend bound Target-independent `ts config validate` compiles profiles, defaults, routes, endpoints, bounds, signing structure, and mediator selection. Every adapter -startup compiles the same plan and then validates backend-name prediction, -fan-out capability, and target resource limits. Fastly and Axum allow -multi-provider fan-out; Cloudflare and Spin currently reject enabled auctions -with more than one provider. Fastly reserves 40 of its default 200 dynamic -backend names for non-auction traffic and rejects auction plans whose provider -IDs and reachable timeout buckets could require more than the remaining 160. - -This tree does not yet have the EdgeZero callback required to run target-aware -validation before `ts config push --adapter ` performs remote work. -Until that callback lands, startup remains the mandatory target-aware gate. +startup compiles the same plan, and every adapter runs multi-provider auctions. +Fastly startup adds one check: an enabled plan may need at most 160 Fastly +backend configurations, which leaves 40 of Fastly's 200 dynamic backends for +other outbound traffic. Each distinct tuple of provider scheme, host, port, and +timeout counts as one configuration. Config validation and push do not run this +check, so Fastly startup is where it applies. ### Timeout behavior -Auction requests start at dispatch and progress while the publisher origin loads. -Response collection waits for headers and complete upstream bodies only until the -original absolute deadline. At expiry, one nonblocking snapshot preserves complete -responses already available; unfinished providers become timeouts. A slow origin -never restarts the budget or causes an already-buffered bid to be discarded. -Fastly has no upstream arrival timestamp, so this is a bound on network waiting, -not bid eligibility based on an exact server-side arrival time. A mediator starts -only with remaining budget and uses the earlier of the auction cutoff and its own -logical timeout. - -Provider bids that are already complete at the final nonblocking snapshot remain -eligible even after `tmax`; mediation starts only with remaining budget and -rejects responses collected after its cutoff. - -Personalized auction and mediator requests explicitly bypass intermediary caches. -Fastly uses the SDK's direct pending-request handles and body readiness handles in -a timed wait. Retained partial payloads share a 10 MiB budget. Under memory pressure, -the largest partial response is rejected with its own backend attribution so a -large stalled provider cannot make a small healthy sibling take the failure. -Completed payloads move directly into the platform body without a host-buffer -round trip. This cap covers retained payloads, not every allocator or host buffer. The final nonblocking body drain is bounded by this payload limit; its local CPU work can extend beyond the network cutoff. - -On expiry Fastly closes acquired response bodies and abandons unfinished requests; -the installed SDK has no pending-request cancellation API. Axum aborts outstanding -request tasks. Cloudflare and Spin start owned background tasks at dispatch and -retain abort handles until response collection completes. Cloudflare also aborts -its fetch through an AbortController. Both workers retain their single-provider -restriction. Dropping a pending handle cancels queued or running guest work; -upstream side effects that already happened cannot be undone. - -Backend first-byte and between-byte timeouts remain transport safeguards rather -than total deadlines. Timer granularity and runtime scheduling can add a small -overshoot. Parsing, ranking, HTML processing, and delivery are local work outside -network collection, so this cutoff is not a hard ceiling on the whole publisher -response time. +The orchestrator builds and signs every bidder request, then sends them together +as one EdgeZero request batch whose cutoff is the auction deadline, measured +from auction start. The bids progress while the publisher origin loads. Each +bidder request carries the provider's configured timeout as its total timeout +and a 1 s connect timeout, and the batch cutoff bounds it by the remaining +auction budget. Collection waits on the batch until every bidder has answered +or the cutoff passes. A bid that has not arrived by the cutoff becomes a +provider timeout. A slow origin never restarts the budget or discards a bid that +already arrived. The mediator starts after the batch finishes, only with +remaining budget, and uses the earlier of the auction deadline and its own +timeout. + +A provider's reported response time runs from the batch start to its bid's +arrival, so a collection that waits after an on-time bid does not increase it. + +Personalized auction and mediator requests explicitly bypass intermediary +caches. Each bidder response is capped at 1 MiB and the mediator response at +2 MiB. No cap applies across all bidder responses together. + +On Fastly, EdgeZero records a bid's arrival whenever one of its polls services +the provider, including the origin body reads while a page streams. A bid that +completes while nothing polls, and is first observed after the cutoff, is +dropped. + +Dropping the batch cancels its in-flight requests on every adapter. An abandoned +auction, an origin error, a non-HTML response, or a client that disconnects +mid-stream therefore cancels the bidder and mediator requests. Only guest-side +cancellation is guaranteed: Fastly host cancellation is best effort, so a bidder +may still receive and answer a cancelled request. Upstream side effects that +already happened cannot be undone. + +Timer granularity and runtime scheduling can add a small overshoot. Parsing, +ranking, HTML processing, and delivery are local work outside network +collection, so this cutoff is not a hard ceiling on the whole publisher response +time. Browser Prebid `timeout_ms` and `debug` stay under `[integrations.prebid]` and are independent of all server provider values. Server endpoint, timeout, @@ -915,7 +916,7 @@ Floor prices can be set per-slot in the auction request. The orchestrator enforc The orchestrator is designed to be resilient: -- **Provider launch failure** — The provider records a `launch_failed` outcome and other providers continue. If every eligible provider fails before producing a pending or immediate outcome, direct `/auction` execution returns `502 Bad Gateway`. Split publisher execution records `dispatch_failed` telemetry and continues the origin response without bids. +- **Provider launch failure** — The provider records a `launch_failed` outcome when its request cannot be built or Fastly cannot register its backend, and other providers continue. If every routed provider fails to launch, direct `/auction` execution returns `502 Bad Gateway`. Split publisher execution records `dispatch_failed` telemetry and continues the origin response without bids. - **Provider parse failure** — If a response can't be parsed, an `AuctionResponse::error()` is recorded. Other results are unaffected. - **No providers configured** — Completes as a no-bid without provider I/O. - **No provider produces a valid bid** — Returns an empty `OrchestrationResult` with zero winning bids after recording provider outcomes. @@ -930,9 +931,9 @@ The auction system logs at multiple levels throughout execution: | Level | Examples | | ------- | --------------------------------------------------------------------------------------- | -| `info` | Auction request received, provider launch, bid counts, winner selection, total timing | +| `info` | Auction request received, in-flight bidders, bid counts, winner selection, total timing | | `debug` | Bid-drop reasons, mediation restoration notes, creative processing mode and byte counts | -| `warn` | Provider launch failures, parse failures, mediator bids without decoded prices | +| `warn` | Provider request failures, parse failures, mediator bids without decoded prices | ### Response Metadata diff --git a/docs/guide/auction-testing.md b/docs/guide/auction-testing.md index b317c584d3..a9b00bef8c 100644 --- a/docs/guide/auction-testing.md +++ b/docs/guide/auction-testing.md @@ -62,9 +62,9 @@ cargo test-spin auction cargo test --manifest-path crates/trusted-server-integration-tests/Cargo.toml --test parity auction ``` -Fastly and Axum support multi-provider fan-out. Cloudflare and Spin reject an -enabled plan with more than one provider at startup, so their negative tests -are part of the contract rather than skipped coverage. +Every adapter runs a multi-provider auction as one EdgeZero request batch. +The Cloudflare and Spin tests load a multi-provider plan and run its auction. +Fastly's tests also cover its startup bound of 160 provider backends. See [Auction orchestration](/guide/auction-orchestration) for configuration and response details. @@ -73,20 +73,23 @@ response details. Core tests cover split dispatch/collection and synchronous execution, pending provider attribution, preserved on-time bids, late collection after an origin -wait, and exhausted-budget mediation. Axum loopback tests separately delay -headers and trickle bodies, and verify outstanding task cancellation. Worker -adapter tests use controlled futures and timers. CI also runs the actual worker -HTTP clients under local workerd and Spin 4.1, covering eager dispatch, buffered -bids after a slow origin, delayed headers, and trickling bodies. Run +wait, and exhausted-budget mediation. The integration suite loads a navigation +on Fastly, Axum, Cloudflare, and Spin whose 500 ms auction races bidders at 180, +320, and 600 ms behind a 700 ms origin, and checks that the first two bids are +injected and the third is dropped. CI also runs EdgeZero's Cloudflare and +Spin HTTP clients as request batches under local workerd and Spin 4.1, covering +a fast bid, buffered bids after slow local work, delayed headers, and trickling +bodies. Run `node scripts/test-worker-auction-deadlines.mjs --help` for the same local fixtures. -Fastly also has Viceroy network fixtures for stalled headers, trickling bodies, -bodyless responses, and competing bidders. Its local origin harness builds the +Fastly also has Viceroy network fixtures for stalled headers, trickling bodies, a +HEAD response's declared length, a fast bid beside a trickling one, and replies +that land before or after the cutoff. Its local origin harness builds the adapter test binary itself: ```bash node crates/trusted-server-adapter-fastly/tests/deadline-origin.mjs "$(command -v viceroy)" ``` -The harness binds a local ephemeral port and runs the five network tests +The harness binds a local ephemeral port and runs the six network tests excluded from the standard suite; CI runs the same command. To reuse a prebuilt test artifact, pass its `.wasm` path as a second argument. diff --git a/docs/guide/axum-dev.md b/docs/guide/axum-dev.md index 82c18cd5d8..8b6d123d1c 100644 --- a/docs/guide/axum-dev.md +++ b/docs/guide/axum-dev.md @@ -8,11 +8,11 @@ production deployment target. | Adapter | Release status | Health | Startup status | Startup health | Provider fan-out | Trusted-client-IP handling | Request normalization | | ------- | -------------- | ------ | -------------- | -------------- | ---------------- | -------------------------- | --------------------- | -| `axum` | development | real | `500` | no | multiple | outermost sanitize | none | +| `axum` | development | real | exit 1 | no | multiple | outermost sanitize | none | This row is the canonical status summary. Axum supports multiple -auction providers and exposes a real `/health` handler. A startup failure -replaces every route with a 500 response, including `/health`. +auction providers and exposes a real `/health` handler. A startup failure stops +the process with exit status 1 before it listens, so no route answers. ## Verify the complete local handoff @@ -36,15 +36,37 @@ are: - `TRUSTED_SERVER_SECRET_TRUSTED_SERVER_SECRETS_EC_PASSPHRASE` Before the positive request, the smoke omits the config and each secret in -separate process starts. Each request must return 500 and its log must identify -the missing environment variable or exact setting path. This prevents a port, -launcher, or origin error from satisfying a negative case. +separate process starts. Each start must fail before Axum listens: the process +must exit non-zero and its log must identify the missing environment variable +or exact setting path. This prevents a port, launcher, or origin error from +satisfying a negative case. The final publisher request must return 200, include the stub-origin sentinel, rewrite its origin URL to the Axum listener, and remove the original URL. The exit trap stops the Axum process and stub origin, then removes the temporary config, local-push output, bodies, headers, and logs. +## Ingress limits + +The listener reads its HTTP/1 ingress limits from the environment at startup. +An unset variable keeps its default. A set value must be a positive integer in +range. An invalid value stops startup with exit status 1 and logs +`invalid Axum ingress setting` with the setting's name, never its value. + +| Variable | Default | Range | +| -------------------------------------------------- | ------- | ---------------- | +| `EDGEZERO__ADAPTER__INGRESS__MAX_CONNECTIONS` | 256 | 1 to 4096 | +| `EDGEZERO__ADAPTER__INGRESS__MAX_RAW_HEAD_BYTES` | 65536 | 8192 to 1048576 | +| `EDGEZERO__ADAPTER__INGRESS__MAX_HEADER_COUNT` | 100 | 1 to 1024 | +| `EDGEZERO__ADAPTER__INGRESS__HEAD_READ_TIMEOUT_MS` | 10000 | 1 to 120000 (ms) | + +The raw-head limit covers the request line and every header field together, +and response headers are held to the same byte limit. A request head over the +byte limit or the field limit gets 431. A connection beyond `MAX_CONNECTIONS` +is closed without a response, and so is a connection whose request head does +not arrive within the head read timeout, which also bounds an idle keep-alive +connection. + ## Logging Axum logs to standard output at `info` by default. Set `RUST_LOG` to a single diff --git a/docs/guide/cli.md b/docs/guide/cli.md index 099c046341..5e275c1e30 100644 --- a/docs/guide/cli.md +++ b/docs/guide/cli.md @@ -228,9 +228,14 @@ adapter flags directly need the `--` added. Trusted Server declares Config, KV, and Secret Stores, so EdgeZero treats every Fastly deploy as managed and requires a verified application release root via `--application-release`; a bare `ts deploy --adapter fastly` without a release is accepted only for -store-free applications. The CLI loads `edgezero.toml` from the working -directory unless `EDGEZERO_MANIFEST` names another file, and it rejects a -manifest outside the release root, so select the release's own manifest. +store-free applications. + +`ts build`, `ts serve`, and `ts deploy` find the manifest in this order: the +file `EDGEZERO_MANIFEST` names, the nearest `edgezero.toml` in the working +directory or one of its parents, and then the nearest `edgezero.toml` under +the workspace root. Two manifests equally near the working directory are an +error. A managed deploy rejects a manifest outside the release root, so set +`EDGEZERO_MANIFEST` to the release's own manifest. Managed deployment also requires a nonempty `FASTLY_API_TOKEN` in the process environment; signing in with `ts auth login` or a Fastly CLI profile alone diff --git a/docs/guide/cloudflare.md b/docs/guide/cloudflare.md index 73448d8cf1..12255a7a81 100644 --- a/docs/guide/cloudflare.md +++ b/docs/guide/cloudflare.md @@ -9,11 +9,11 @@ config that `ts config push` writes into KV to the nested | Adapter | Release status | Health | Startup status | Startup health | Provider fan-out | Trusted-client-IP handling | Request normalization | | ------------ | -------------- | ------ | -------------- | -------------- | ---------------- | -------------------------- | --------------------- | -| `cloudflare` | development | absent | `500` | no | single | outermost sanitize | none | +| `cloudflare` | development | absent | `500` | no | multiple | outermost sanitize | none | This row is the canonical status summary. Cloudflare has no -`/health` route, supports one enabled auction provider, and returns 500 on -every route when startup fails. +`/health` route and supports multiple auction providers. When startup fails, +every route answers 500 with EdgeZero's JSON error body. ## Verify the complete local handoff diff --git a/docs/guide/configuration.md b/docs/guide/configuration.md index d60fb7f810..056e436ee6 100644 --- a/docs/guide/configuration.md +++ b/docs/guide/configuration.md @@ -432,9 +432,16 @@ silently ignored. The example TOML above includes that leaf. - ✅ `https://origin.publisher.com:8080` - ✅ `http://192.168.1.1:9000` - ❌ `origin.publisher.com` (missing protocol) +- ❌ `https://user:secret@origin.example.com` (userinfo) **Port Handling**: Includes port if non-standard (not 80/443). +**Userinfo**: `ts config validate` and `ts config push` reject an `origin_url` +with a username or password, and the error does not echo the URL. A live +config that still has one answers every publisher request with 400, because +the origin request cannot carry userinfo. Send origin credentials in a header +instead. + #### `public_origin` {#public_origin} **Purpose**: Browser-facing Trusted Server origin used for first-party URLs emitted into rewritten creatives, including proxy, click, and injected TSJS URLs. @@ -525,6 +532,10 @@ connecting to the host in `origin_url`. **Default**: When omitted, Trusted Server sends the host from `origin_url`. +**Spin**: EdgeZero's Spin client cannot send a `Host` that differs from the +connection target, so Spin ignores this setting and logs a warning at settings +load. Fastly, Axum, and Cloudflare send it. + #### `proxy_secret` **Purpose**: Secret-store key name for the HMAC-SHA256 value used to sign proxy URLs. @@ -582,7 +593,7 @@ a cooperative timer that can cancel yielding callbacks. Fastly has no timer: cor checks the budget only before and after each callback, so an integration that does not bound its own blocking host calls can hold the response past this value. Opted-in processed HTML bypasses shared-template caching and uses private/no-store -policy; raw-origin readthrough remains separate. Opted-in HTML with 206 or any +policy. Opted-in HTML with 206 or any `Content-Range` bypasses rewriting and EOF work while preserving encoded bytes and range/framing metadata. Full error HTML remains eligible and exposes actual status. See [Asynchronous HTML processor completion](/guide/integration-guide#asynchronous-html-processor-completion). @@ -996,8 +1007,22 @@ Edit TOML, validate, and push again for those changes. - Debugging identifiers - CORS headers (if needed) +**Rejected names**: Response egress owns message framing, so these ten names +cannot be set here: `Trailer`, `Content-Length`, and the hop-by-hop fields +`Connection`, `Keep-Alive`, `TE`, `Transfer-Encoding`, `Upgrade`, +`Proxy-Connection`, `Proxy-Authenticate`, and `Proxy-Authorization`. Names +match case-insensitively. `ts config validate` and `ts config push` reject an +entry with any of them. A live config that predates the rule keeps loading, and +each request skips such an entry with one warning that names it. + +Because the `Connection` entry is skipped, egress never sees its value, so a +field that value names still reaches the client. For example, an operator +`X-Example` entry stays on the response even when the skipped `Connection` +entry names `X-Example`. + ::: warning Header Precedence -Custom headers may be overwritten by application logic. Standard headers (`Content-Type`, `Content-Length`) are controlled by the application. +Custom headers may be overwritten by application logic. `Content-Type` is +controlled by the application. ::: ### Origin URLs in proxied response headers @@ -1305,9 +1330,9 @@ include_domains = ["assets.example.com", "*.img.example.com"] **Environment Override**: -EdgeZero v0.0.4 cannot replace these arrays or address their elements by index. -Edit `exclude_domains` and `include_domains` in TOML, then validate and push the -file again. +The pinned EdgeZero loader cannot replace these arrays or address their elements +by index. Edit `exclude_domains` and `include_domains` in TOML, then validate and +push the file again. ### Pattern Matching @@ -1422,7 +1447,7 @@ Controls first-party proxy security settings and path-based asset routes. | Field | Type | Required | Description | | ------------------- | ------------- | -------------------- | ----------------------------------------------------------- | | `allowed_domains` | Array[String] | No (default: `[]`) | Hosts permitted for signing, initial fetches, and redirects | -| `certificate_check` | Boolean | No (default: `true`) | Verify TLS certificates when proxying HTTPS origins | +| `certificate_check` | Boolean | No (default: `true`) | Must be `true`; origin TLS is always verified | | `asset_routes` | Array[Table] | No (default: `[]`) | Path prefixes proxied directly to configured origins | **Example**: @@ -1437,8 +1462,8 @@ allowed_domains = [ **Environment Override**: -EdgeZero v0.0.4 cannot replace this array or address its elements by index. Edit -`allowed_domains` in TOML, then validate and push the file again. +The pinned EdgeZero loader cannot replace this array or address its elements by +index. Edit `allowed_domains` in TOML, then validate and push the file again. ### Field Details @@ -1480,11 +1505,16 @@ See [First-Party Proxy](/guide/first-party-proxy#proxy-allowlist) for usage deta #### `certificate_check` -**Purpose**: Control TLS certificate verification for HTTPS proxy and asset-route origins. +**Purpose**: None to configure. Every adapter always verifies origin TLS. **Default**: `true` -Set this to `false` only for local development with self-signed certificates. +`ts config validate` and `ts config push` reject `certificate_check = false`. A +live config that still sets it keeps loading: settings load logs a warning, +ignores the setting, and origin TLS stays verified. Remove the setting before +the next push. An origin whose certificate fails verification fails the +request with the route's 502. For local development, serve the origin over +plain HTTP instead of with a self-signed certificate. ### `[[proxy.asset_routes]]` @@ -2407,7 +2437,8 @@ inventory, debug, and creative controls to the `aps` `profile_config`. Only bidder codes listed in `[auction.bidders]` are folded into Trusted Server requests. Unlisted publisher bids remain native browser demand. All provider -endpoints must be absolute HTTPS URLs. +endpoints must be absolute HTTPS URLs, except plain HTTP to `127.0.0.1` or +`[::1]` for a local test bidder. The old and new blobs are mutually incompatible. Activate the new binary and map-shaped config together. A binary-first or config-first rolling deployment @@ -2415,8 +2446,8 @@ will put one version on a schema it rejects. Roll back by restoring the old binary and old-schema blob together. ::: -Each table name is the provider ID used for configuration, backend correlation, -health, response metadata, and telemetry. Provider IDs must match +Each table name is the provider ID used for configuration, health, response +metadata, and telemetry. Provider IDs must match `^[a-z][a-z0-9-]{0,62}$`. Multiple instances may select the same profile and endpoint because the provider ID remains their distinct runtime identity. @@ -2476,6 +2507,9 @@ timeout_ms = 500 | `profile_config` | No | `{}` | Typed object owned by the selected profile | | `notifications` | No | No suppression | Common `nurl`/`burl` suppression after response normalization | +A local test bidder's `endpoint` may instead be plain HTTP to `127.0.0.1` or +`[::1]`. + ### Profile configuration The table reflects the typed profile schemas. `Required` refers to the selected @@ -2542,36 +2576,28 @@ UTF-8 bytes and without ASCII control characters. `ts config validate` and ordinary deploy validation compile the complete target-independent plan: profiles and defaults, routes, endpoint ownership, extension bounds, notifications, signing structure, and mediator selection. -Target-specific checks are deferred to adapter startup. Startup uses the same -compiled plan and additionally validates backend-name prediction/collisions and -provider fan-out capability. - -Fastly and Axum support multiple configured providers. Cloudflare and Spin -currently reject an enabled auction with more than one provider because those -adapters do not support concurrent provider fan-out. Disabled auctions may keep -dormant multi-provider maps without target rejection. +Every adapter startup compiles the same plan, and every adapter runs +multi-provider auctions. -A target-aware pre-write `ts config push --adapter ` callback is **not -available in this tree** because the required EdgeZero callback is not yet -available. Until it lands, push performs target-independent validation and -adapter startup is the mandatory target-aware gate. Do not treat a successful -push as proof that a Cloudflare or Spin multi-provider plan can start. +Fastly startup adds one target check. An enabled plan may need at most 160 +Fastly backend configurations, counted as distinct tuples of provider scheme, +host, port, and timeout; Fastly keeps the other 40 of its 200 dynamic backends +for non-auction traffic. Config validation and push do not run this check, so +Fastly startup is where it applies. Disabled auctions skip it. ### Deadline behavior -Auction response collection waits only until an absolute deadline measured from -dispatch, including headers and complete response bodies. One nonblocking snapshot -retains complete available responses after a slow origin; unfinished providers -become timeouts and mediation cannot start with an exhausted budget. Backend -timeout rounding does not extend network waiting. -Provider bids that are already complete at the final nonblocking snapshot remain -eligible even after `tmax`; mediation starts only with remaining budget and -rejects responses collected after its cutoff. +All bidder requests leave together as one EdgeZero request batch whose cutoff +is the auction deadline, measured from auction start. A bid that has not +arrived by the cutoff becomes a provider timeout, and mediation cannot start +with an exhausted budget. Each bidder request carries its provider timeout as a +total timeout, and the cutoff bounds it by the remaining budget. -Fastly abandons pending requests; Axum, Cloudflare, and Spin cancel outstanding -operations where supported. Local processing and delivery can finish after the -network cutoff. See [Timeout behavior](/guide/auction-orchestration#timeout-behavior) -for runtime semantics and limitations. +Dropping the batch cancels in-flight bidder and mediator requests on every +adapter; Fastly host cancellation is best effort. Local processing and delivery +can finish after the network cutoff. See +[Timeout behavior](/guide/auction-orchestration#timeout-behavior) for runtime +semantics and limitations. Creative sanitization is opt-in. `sanitize_creatives = true` strips executable markup before delivery. `rewrite_creatives = false` leaves asset URLs @@ -2915,9 +2941,7 @@ Rollback must preserve configuration compatibility: services (`localhost`, `127.0.0.1`, or `::1`). The admin endpoint `POST /_ts/admin/cache/purge` is the same operation for a CMS webhook. Either clears the `ts-template` surrogate key; waiting out the bounded origin-derived lifetime also - works. With readthrough caching enabled, `--all` also purges tagged origin - documents, so the next requests refetch those documents from the origin. Check - whether the origin can absorb that load before purging during a traffic peak. + works. Run `scripts/template-cache-local-test.sh esi` before a rollout and `scripts/template-cache-local-test.sh inline` as its control. The harness uses a temporary @@ -2929,164 +2953,17 @@ versus empty buckets, ignored compact JSON and comma-list cookies, and session bypass on warm and cold URLs. Each request checks the selected HTML, cache diagnostics, private response policy, winning-bid assembly, and origin fetch count. -### Origin readthrough caching +### Origin readthrough -`origin_readthrough_enabled` controls a **different cache** from everything above. -The template cache stores Trusted Server's own transformed HTML. Readthrough is the -platform's own cache sitting in front of the publisher origin, and it stores the -origin's bytes. +Origin readthrough is unavailable. `ts config validate` and `ts config push` +reject `origin_readthrough_enabled = true`. A live config that still sets it +keeps loading: settings load logs a warning and treats it as `false`. Remove +the setting before the next push. -```toml -[creative_opportunities] -# Default false. Enable only after `ts origin probe-shareability` passes on every -# axis and every verdict. -origin_readthrough_enabled = true -``` - -Left at the default, the existing caching policy is preserved: ad-serving requests -bypass the origin cache, while other publisher requests (including ordinary assets) -keep the platform's default caching behavior. Setting it to `true` applies request -shareability instead: eligible ad-serving requests can use the cache, while -ineligible non-ad requests bypass it. Eligible requests are `GET`s with a `Host`, -no disqualifying authorization or cookie, and no remaining conditional or range -semantics. - -**Document requests only.** The gate answers a question about pages — whether the -origin's HTML may be shared between readers — so it applies to document requests -(`Sec-Fetch-Dest: document` and equivalents, or a navigation when that header is -absent). Subresources keep the platform default whether the flag is on or off. -Judging them on shareability would bypass the edge cache for every cookie-bearing -or conditional asset request, which is most repeat-visitor asset traffic, and would -tag every cached asset with `ts-template`, turning the template rollback purge into -an origin-wide asset flush. - -#### This cache has far weaker guarantees than the template cache - -Read this before enabling it. The template cache refuses storage on inspection of -the origin's _response_ — `Set-Cookie`, a CSP nonce, missing positive freshness, an -uncovered `Vary`, and the rest of the list above. **Readthrough has none of those -refusals**, and cannot: the decision is made before the origin replies, and no -post-response hook is reachable on the Fastly adapter. - -What that means concretely, for each refusal the template cache performs: - -| Template-cache refusal | Covered on readthrough? | -| --------------------------- | ------------------------------------------------- | -| No positive freshness | **No** — probe verdict only | -| Origin `Set-Cookie` | **No** — probe verdict only | -| Response CSP nonce | **No** — probe verdict only | -| Origin marks it unshareable | Yes — the platform honours `private` / `no-store` | -| Non-`200` status | Yes — the platform honours status | -| Uncovered `Vary` | Yes — the platform keys on the origin's `Vary` | -| Not HTML | Not applicable; readthrough caches per origin | - -Every row marked **No** is an accepted risk carried by the operator, not by the -code. An origin that personalises HTML without saying so in its headers can -cross-serve one reader's page to another, including session fixation through a -cached `Set-Cookie`. That last case is the sharpest: readthrough admits requests -carrying _no_ cookie, which is exactly the first-time visitor an origin issues a -session cookie to. - -`origin_is_cookie_independent = true` also widens this gate: cookie-bearing -requests with unlisted cookies can become readthrough-eligible. Named bypass -cookies and malformed cookie policies still refuse admission. Configuring any -`template_cache_key_cookies` disables readthrough, even when those cookies are -absent: the platform cache does not include their variant values in its key. -The template cache continues to separate those variants. On the template cache, -an origin's `Vary: Cookie` still overrides the independence assertion. On readthrough there is no such -response-side guard. Setting both flags is the highest-risk configuration and -requires a cookie-axis probe pass specifically. - -#### You cannot verify this locally - -Viceroy does not implement the readthrough cache. Measured with the gate enabled, the -request judged shareable, and a stub origin answering `Cache-Control: public, max-age=60` -with no `Set-Cookie`, two identical navigations still produced two origin fetches. The -local harness can therefore show the _decision_ this gate makes, and never its effect. - -The first evidence either way comes from a deployed service. Treat any local timing as -saying nothing about this setting. - -#### Enablement - -1. Run `ts origin probe-shareability --url `. Publisher cookies a - real reader carries go in `TRUSTED_SERVER_PROBE_COOKIES` as one cookie header value - (`name=value; name=value`), and a bot-wall admission cookie in - `TRUSTED_SERVER_PROBE_ADMISSION_COOKIE`. Both are environment-only, never flags: - these are credentials, and an argument is visible to every process on the host - through `ps` and lands in shell history. Each `--url` must be HTTPS; plain HTTP is - accepted only for a loopback development origin. Admission-cookie runs are - diagnostic only: every request carries that cookie, so cookieless responses remain - untested and the safety gate fails. Rerun against the origin without it before - enabling caching. -2. **Every axis and every verdict must pass.** Do not enable on a partial pass. - The probe checks status and safety headers on every sampled response, including - repeats. Each axis compares what a cache would store — the body **and** the policy - headers replayed with it, such as `Content-Security-Policy` — so an origin that - serves one document under two policies fails just as a varying document does. - Crawler user agents and prefetch requests are their own axes: neither - classification blocks readthrough, so an origin that answers a bot or a prefetch - with a different document without declaring `Vary` would otherwise have that - document cross-served to a reader. Any `Age` header, including `Age: 0`, blocks the verdict because a - fresh cached response can hide origin personalization. Pass `--vary-header =` - for each representative value of an additional request header, for example - `--vary-header x-exp-variant=A --vary-header x-exp-variant=B`. Each value is compared - against the absent baseline and the other supplied values, both with and without RSC. - Bare `--vary-header ` samples `1` for compatibility; it cannot establish safety - for categorical values the origin actually uses. Supply every relevant variant. - A declared `Vary` can explain a user-agent, RSC, - or custom-header difference only when every response declares it. Cookie - differences and different decoded gzip/identity documents always fail, because - the template cache requires those representations to be identical. - Navigation samples send HTML `Accept` and navigation Fetch Metadata and must - return `text/html`. RSC uses an explicit same-origin fetch profile, permitting - HTML fallback or `text/x-component`. A separate fetch control keeps `RSC` - variation independent of `Accept` and Fetch Metadata changes. If navigation - and fetch controls differ, all changed profile headers must be declared in - `Vary`; this conservative check cannot attribute a combined-profile difference - to one header. `Vary: *`, revalidation directives, and unreadable safety headers - always fail. - The probe is the only response-safety control on the readthrough path. -3. Read the probe's stated limits. It runs from one client address, so - personalisation keyed on the reader's IP — geo, rate class — is invisible to - it, as are `Accept-Language` and client-hint variants it does not vary. -4. Set `origin_readthrough_enabled = true` and push the configuration. -5. Watch the `origin_cache_shareable` breakdown in publisher summary telemetry. - Its denominator is matching-slot candidates, including skipped auctions; it - does not measure every publisher origin fetch or a site-wide admission rate. - See the [telemetry population and query](https://github.com/IABTechLab/trusted-server/blob/main/tinybird/README.md#the-denominator-is-matching-slot-candidates-not-all-requests). - The predicate estimates eligibility in that population before or after enablement, - not actual cache hits. -6. Confirm the origin's own hit rate and page correctness before widening to more - URLs. - -#### Rollback - -1. Set `origin_readthrough_enabled = false` and push. This takes effect on the - next request with no deploy and restores the previous policy: ad-serving - requests bypass, while non-ad traffic keeps the platform default. It does not - disable origin caching globally. -2. Purge tagged objects with `ts cache purge --service --all`, - or `--page ` for one exact reader-facing URL. Both the template cache - and opted-in origin readthrough objects carry the page key and `ts-template` - purge-all key. The readthrough tags use the original reader URL, before origin - rewriting, and work in both inline and ESI assembly modes. -3. Objects stored by older versions without readthrough tags remain unreachable - through these purge keys and must expire on the origin's TTL. Changing the - origin's TTL does not shorten an already-cached object's lifetime. -4. **Use `--all` on multi-host or dual-scheme deployments.** The template cache keys - on scheme and host, so purging each spelling you serve covers it. Readthrough does - not line up the same way: reader URLs that rewrite to one origin URL — `http://` - and `https://`, or `www.` and the apex on one service — share a single stored - object, tagged with the reader URL of whichever request filled it first. A - `--page https://example.com/a` can therefore leave an `http://`-tagged object in - place, and the next template miss refetches through it and re-stores the stale page - into the freshly purged template cache. If you serve one page under more than one - reader-facing spelling, purge with `--all`. - -The Fastly SDK attaches these tags to cached objects; production hit and purge -behavior still requires validation on a deployed service, since Viceroy does not -implement readthrough caching. +Publisher origin requests keep the default policy. Ad-serving requests bypass +the platform's origin cache, and other publisher requests, ordinary assets +included, keep the platform's default caching. The template cache above is +unaffected. ### `gam_unit_path` templating @@ -3218,9 +3095,9 @@ inject_adm_for_testing = false ## Fastly Runtime Config Store -After the EdgeZero cutover, the Fastly adapter always dispatches through the -EdgeZero entry point. The former `edgezero_enabled` and `edgezero_rollout_pct` -canary keys are no longer read. +The Fastly adapter serves every request through EdgeZero's canonical Fastly +lifecycle. The former `edgezero_enabled` and `edgezero_rollout_pct` canary keys +are no longer read. `[stores.config].default` in `edgezero.toml` supplies the logical config store ID, currently `trusted_server_config`. The Fastly runtime opens the store by @@ -3302,7 +3179,7 @@ Configuration is validated at startup: **Error Format**: ``` -Configuration error: provider `pbs-main` endpoint must be an absolute HTTPS URL +Configuration error: provider `pbs-main` endpoint must be absolute HTTPS, or plain HTTP to 127.0.0.1 or [::1], with a host and no credentials or fragment ``` ## Best Practices diff --git a/docs/guide/edgezero.md b/docs/guide/edgezero.md index 2107f95f56..01bdea4760 100644 --- a/docs/guide/edgezero.md +++ b/docs/guide/edgezero.md @@ -1,9 +1,18 @@ # EdgeZero Lifecycle and Stores -EdgeZero is the deployment and configuration layer used by the `ts` CLI. +EdgeZero is the deployment and configuration layer used by the `ts` CLI, and +the runtime layer under every adapter: each adapter serves requests through +EdgeZero's request lifecycle and sends outbound HTTP through EdgeZero's client. The repository's `edgezero.toml` declares one application, three logical stores, and the Fastly, Axum, Cloudflare, and Spin adapter commands. +## Pinned version + +The workspace `Cargo.toml` pins all six EdgeZero crates to one git revision of +`IABTechLab/edgezero`: `14a6ca8bb5a5cbe055a1f725bdab8f42569a3bf0`, the head of +EdgeZero pull request 275. The pin moves with that pull request's head until +it merges, and then to an EdgeZero release tag. + ## Lifecycle Use `ts config init` to create a file, edit the generated @@ -61,6 +70,49 @@ actual deletion requires `--yes --older-than WINDOW`. The age assertion applies to the whole physical store, not one root key; verify the reported store before deletion, especially when using `--store` or `--no-env`. +## Request handling + +These rules hold on Fastly, Axum, Cloudflare, and Spin. + +**Inbound requests.** + +- A request target over 8192 bytes returns 414. More than 100 header fields, + or more than 64 KiB of header bytes, returns 431. +- Ambiguous request framing returns 400: a duplicate or malformed + `Content-Length`, `Content-Length` with `Transfer-Encoding`, or a + `Transfer-Encoding` that is not chunked. +- Each endpoint reads its body under its own cap and returns 413 above it. A + declared `Content-Length` over the cap returns 413 before any body byte is + read. A body still unread 30 s after the request arrived returns 408. +- A publisher request streams its body to the origin under a 16 MiB cap, and a + larger upload returns 413. A forwarded `GET` or `HEAD` that carries body bytes + returns 400. + +**Outbound requests.** + +- Every request has a 1 s connect timeout and a constant total timeout: the + configured timeout where a setting names one, such as an auction provider's + `timeout_ms`, 2 s for Tinybird, and otherwise 15 s for a buffered response or + 60 s for a streamed one. A redirect chain also has a 60 s deadline. +- Origin TLS is always verified. +- A request that sets no `Accept-Encoding` carries `Accept-Encoding: identity`. +- Hop-by-hop fields, and fields that `Connection` names, never reach an + upstream or come back from one. A request or response header whose value is + not UTF-8 is dropped whole, with a warning that names the header. +- The request URI is normalized as a WHATWG URL. A publisher request whose + target has a fragment, a backslash, or userinfo returns 400. +- On Fastly, EdgeZero registers each backend at request time and names it + `ez_b31v1_` followed by a hash of the backend configuration. + +**Responses.** + +- Response egress strips hop-by-hop fields and never sends `Trailer`. A framing + error answers 500 before the response is committed. +- A response body must finish within 60 s after egress starts. +- Framework errors, such as a 404, 405, 414, or 431, render as EdgeZero JSON + and carry `x-ts-version`. A startup failure response carries no + `x-ts-version`. + ## Safe operator sequence 1. Put required secret values in the physical store mapped from diff --git a/docs/guide/error-reference.md b/docs/guide/error-reference.md index 07d170861e..851b39b8ae 100644 --- a/docs/guide/error-reference.md +++ b/docs/guide/error-reference.md @@ -84,13 +84,14 @@ TOML file. **Error Message:** ``` -provider `pbs-main` endpoint must be an absolute HTTPS URL +provider `pbs-main` endpoint must be absolute HTTPS, or plain HTTP to 127.0.0.1 or [::1], with a host and no credentials or fragment ``` **Cause:** Malformed auction provider endpoint. **Solution:** Configure an absolute HTTPS endpoint with a host and no embedded -credentials or fragment: +credentials or fragment. Plain HTTP is accepted only for `127.0.0.1` and +`[::1]`, which local test bidders use: ```toml # ❌ Wrong @@ -168,24 +169,31 @@ TRUSTED_SERVER__EC__PASSPHRASE=ec_passphrase --- -### Backend not found +### Provider request failed **Error Message:** ``` -Backend not found: prebid-server +Planned provider 'pbs-main' request failed: ... ``` -**Cause:** Dynamic backend creation for a configured provider endpoint failed. -Provider backends are derived from `[auction.providers.]`; they are not -manually named static Fastly backends. +**Cause:** The bidder request did not complete: the connection or TLS +handshake failed, the provider answered with a protocol error, or Fastly could +not register the provider's backend. EdgeZero derives Fastly backends from the +provider endpoint; they are not static Fastly backends. A registration failure +records `launch_failed` for the provider, and other failures record +`transport_failed`. The preceding `Outbound request failed` warning carries the +provider failure detail, which holds closed failure codes and no URL. **Solution:** -- Verify the provider endpoint is canonical HTTPS and reachable from the edge -- Check the provider ID and target-specific backend-name validation error -- Check platform backend-count limits -- Run `ts config validate`, then verify target-aware startup validation on the selected adapter +- Verify the provider endpoint is HTTPS, reachable from the edge, and serves a + valid certificate; origin TLS is always verified +- Read the provider failure detail in the `Outbound request failed` warning +- On Fastly, check the service's dynamic backend use. Fastly allows 200 + dynamic backends per service; the auction may use up to 160, and the + publisher origin, integrations, and other outbound sites share the rest +- Run `ts config validate`, then restart the adapter and check its startup log --- @@ -220,19 +228,34 @@ curl "https://edge.example.com/first-party/sign?url=https://external.com/pixel.g **Error Message:** ``` -Upstream request timeout after 1000ms +Planned provider 'pbs-main' exhausted its response budget +Planned provider 'pbs-main' exhausted its dispatched auction collection budget ``` -**Cause:** Upstream service (Prebid Server, Permutive, etc.) didn't respond in time +Both lines log at `debug`, and the auction response reports the provider as +timed out. + +**Cause:** An auction provider didn't answer in time: + +- `exhausted its response budget`: the provider's own `timeout_ms` passed + before it answered. +- `exhausted its dispatched auction collection budget`: the auction deadline, + `[auction].timeout_ms` from auction start, passed before it answered. **Solution:** -1. Increase the affected server provider timeout: +1. Match the remedy to the log line: + - For `exhausted its response budget`, raise that provider's timeout: -```toml -[auction.providers.pbs-main] -timeout_ms = 2000 -``` + ```toml + [auction.providers.pbs-main] + timeout_ms = 2000 + ``` + + - For `exhausted its dispatched auction collection budget`, raise + `[auction].timeout_ms` or check the bidder's latency. A provider timeout + longer than the auction timeout does not help, because the auction + deadline ends collection first. Browser `[integrations.prebid].timeout_ms` is independent and does not control Prebid Server transport. @@ -243,7 +266,10 @@ Prebid Server transport. curl -w "%{time_total}\n" https://upstream-service.example.com ``` -3. Check Fastly backend timeout settings +3. Integration proxies such as Permutive use fixed timeouts that settings do + not change: a 1 s connect timeout, and a 15 s total timeout for a buffered + call or 60 s for a streamed one. Fastly backends have no timeout settings + of their own to tune. --- @@ -580,7 +606,6 @@ fastly profile create **Service validation errors:** - Check Fastly dashboard for validation messages -- Verify all required backends are configured - Ensure KV/Config/Secret stores are linked --- diff --git a/docs/guide/fastly.md b/docs/guide/fastly.md index 6cf1e5e38c..dd1051b06c 100644 --- a/docs/guide/fastly.md +++ b/docs/guide/fastly.md @@ -39,20 +39,14 @@ before it constructs the application. - Add your domain of the website you'll be testing or using - Click **Update** -## Configure Origins +## Origins -Origins are the backend servers that Trusted Server will communicate with (ad servers, SSPs, etc.). - -1. In your Compute service, click on the **Origins** section -2. For each backend you need to add: - - Enter the FQDN or IP address - - Click **Add** - - Enter a **Name** in the first field - this name will be referenced in your code (e.g., `my_ad_integration_1`) - - Configure port numbers and TLS settings as needed - -::: tip -After saving origin information, you can select port numbers and toggle TLS on/off. -::: +Trusted Server needs no static origins on the Compute service. For every +upstream it calls, such as the publisher origin, an auction provider, or an +integration, EdgeZero registers a dynamic backend at request time from the +request URL. Each backend is named `ez_b31v1_` followed by a hash of its +configuration, so dashboards that group by backend name see those names. TLS +to an HTTPS upstream is always verified. ## Configure Fastly CLI Profile @@ -447,8 +441,8 @@ through Viceroy. The required keys are `handler_password`, `publisher_proxy_secret`, and `ec_passphrase`. The check deliberately proves both halves of startup. With no config entry, it -requires `/health` to return 200 while the publisher route returns 500 with the -missing `trusted_server_config` diagnostic. It then removes each required +requires `/health` to return 200 while the publisher route returns 500 and the +log names the missing `trusted_server_config` entry. It then removes each required secret independently and requires the corresponding setting path to fail. Finally, the publisher request must return 200, retain the stub-origin sentinel, rewrite an origin URL to the Fastly listener, and omit the original diff --git a/docs/guide/first-party-proxy.md b/docs/guide/first-party-proxy.md index 4e94b7ef41..2cc5b0c390 100644 --- a/docs/guide/first-party-proxy.md +++ b/docs/guide/first-party-proxy.md @@ -371,11 +371,22 @@ The proxy automatically follows HTTP redirects: **Behavior**: -1. Follow up to **4 redirect hops** -2. Re-apply `ts-ec` on each hop -3. Switch to `GET` after `303` response -4. Log when redirect limit reached -5. Preserve request headers across hops +1. Follow up to **4 redirect hops** through EdgeZero's redirect follower. +2. Check each redirect target before it is followed. A non-HTTPS target on a + route that requires HTTPS fails with `403`, a target without a host fails + with `502`, and a host outside `proxy.allowed_domains` fails with `403`. A + target whose scheme is neither `http` nor `https` is not followed, and its + redirect response is returned. +3. Send `301`, `302`, and `303` hops as a bodiless `GET` (a `HEAD` stays + `HEAD`); `307` and `308` keep the method and body. A `301` or `302` that + answers a method other than `GET`, `HEAD`, or `POST` is returned, not + followed. +4. Give each hop the request's own timeout, 15 s for a buffered response or 60 s + for a streamed one, and the whole chain a 60 s deadline. +5. Strip `Authorization` and `Cookie` on a hop to another origin. +6. Fail a `Location` that carries userinfo with `502 Bad Gateway`. +7. Log each hop at `debug` without its query, and log a warning when a redirect + response is returned instead of followed. **Example Flow**: @@ -410,19 +421,9 @@ Proxied backend request: ### Redirect Propagation -EC IDs are re-applied on **every redirect hop**: - -``` -/first-party/proxy?tsurl=https://redirect1.com&tstoken=sig - → https://redirect1.com?ts-ec=user123 - → 302 to https://redirect2.com - → https://redirect2.com?ts-ec=user123 - → 302 to https://final.com - → https://final.com?ts-ec=user123 - → 200 response -``` - -This ensures downstream endpoints receive consistent IDs even through redirect chains. +The `ts-ec` parameter goes on the first target only. A redirect hop requests the +`Location` the upstream returned, so a later endpoint receives the EC ID only +when the redirecting server carries it into that `Location`. ### Click ID Forwarding @@ -671,10 +672,10 @@ Missing required parameter: tsurl When redirect limit (4 hops) is reached: ``` -Log: proxy: redirect limit reached for url=https://... +Log: returning an unfollowed redirect response: LimitReached ``` -Response: Returns last redirect response (302/307/308) without following. +Response: Returns the last redirect response (302/307/308) without following. **Solutions**: diff --git a/docs/guide/getting-started.md b/docs/guide/getting-started.md index 9e759697c5..9496c9cd21 100644 --- a/docs/guide/getting-started.md +++ b/docs/guide/getting-started.md @@ -201,10 +201,9 @@ ts config validate ts config push --adapter fastly ``` -This command performs target-independent plan validation. Each adapter performs -mandatory target-aware fan-out and backend-name validation at startup. The -EdgeZero callback needed for target-aware pre-write push validation is not yet -available in this tree, so startup remains the final target gate. +This command performs target-independent plan validation. Fastly startup also +checks that an enabled auction fits its bound of 160 provider backends; push +does not run that check. Restart or redeploy instances after secret rotation. See [Configuration](/guide/configuration) and [Trusted Server CLI](/guide/cli) for details. diff --git a/docs/guide/integration-guide.md b/docs/guide/integration-guide.md index 9368cd8f93..179a7c0916 100644 --- a/docs/guide/integration-guide.md +++ b/docs/guide/integration-guide.md @@ -24,7 +24,8 @@ namespaced and bounded; do not introduce a general outbound proxy. ## Compiling core-neutral fixture The fixture below registers an attribute rewriter and constructs every -required `RuntimeServices` service without an adapter dependency. The +required `RuntimeServices` service without an adapter dependency. The adapter +supplies the request's outbound client and monotonic clock. The documentation test extracts this exact fence and compiles it as an isolated crate. @@ -40,14 +41,14 @@ use trusted_server_core::integrations::{ IntegrationAttributeRewriter, IntegrationRegistration, }; use trusted_server_core::platform::{ - BackendNamingPolicy, ClientInfo, GeoInfo, PlatformBackend, - PlatformBackendSpec, PlatformConfigStore, PlatformError, PlatformGeo, - PlatformSecretStore, RuntimeServices, StoreId, StoreName, - UnavailableHttpClient, UnavailableKvStore, + ClientInfo, GeoInfo, HttpClient, MonotonicClock, PlatformConfigStore, + PlatformError, PlatformGeo, PlatformSecretStore, RuntimeServices, StoreId, + StoreName, UnavailableKvStore, }; struct ReadOnlyStore; +#[async_trait::async_trait(?Send)] impl PlatformConfigStore for ReadOnlyStore { fn get( &self, @@ -57,7 +58,7 @@ impl PlatformConfigStore for ReadOnlyStore { Err(Report::new(PlatformError::Unsupported)) } - fn put( + async fn put( &self, _store: &StoreId, _key: &str, @@ -66,7 +67,7 @@ impl PlatformConfigStore for ReadOnlyStore { Err(Report::new(PlatformError::Unsupported)) } - fn delete( + async fn delete( &self, _store: &StoreId, _key: &str, @@ -75,6 +76,7 @@ impl PlatformConfigStore for ReadOnlyStore { } } +#[async_trait::async_trait(?Send)] impl PlatformSecretStore for ReadOnlyStore { fn get_bytes( &self, @@ -84,7 +86,7 @@ impl PlatformSecretStore for ReadOnlyStore { Err(Report::new(PlatformError::Unsupported)) } - fn create( + async fn create( &self, _store: &StoreId, _key: &str, @@ -93,7 +95,7 @@ impl PlatformSecretStore for ReadOnlyStore { Err(Report::new(PlatformError::Unsupported)) } - fn delete( + async fn delete( &self, _store: &StoreId, _key: &str, @@ -102,32 +104,6 @@ impl PlatformSecretStore for ReadOnlyStore { } } -struct FixtureBackend; - -impl PlatformBackend for FixtureBackend { - fn naming_policy(&self) -> BackendNamingPolicy { - BackendNamingPolicy::Axum - } - - fn predict_name( - &self, - spec: &PlatformBackendSpec, - ) -> Result> { - if spec.host.is_empty() { - Err(Report::new(PlatformError::Backend)) - } else { - Ok("fixture-origin".to_owned()) - } - } - - fn ensure( - &self, - spec: &PlatformBackendSpec, - ) -> Result> { - self.predict_name(spec) - } -} - struct FixtureGeo; impl PlatformGeo for FixtureGeo { @@ -169,14 +145,21 @@ pub fn registration() -> IntegrationRegistration { .build() } -pub fn runtime_services() -> RuntimeServices { +/// Builds request services around the adapter's outbound client and clock. +/// +/// Every outbound request goes through `outbound_client`, with the proxy +/// marker off, and outbound deadlines read `clock`. +pub fn runtime_services( + outbound_client: HttpClient, + clock: MonotonicClock, +) -> RuntimeServices { let store = Arc::new(ReadOnlyStore); RuntimeServices::builder() .config_store(store.clone()) .secret_store(store) .kv_store(Arc::new(UnavailableKvStore)) - .backend(Arc::new(FixtureBackend)) - .http_client(Arc::new(UnavailableHttpClient)) + .outbound_client(outbound_client.with_proxy_marker(false)) + .clock(clock) .geo(Arc::new(FixtureGeo)) .client_info(ClientInfo::default()) .build() @@ -192,16 +175,24 @@ is omitted, so adapter startup must construct the complete service graph. ## Proxy implementation rules An `IntegrationProxy::handle` implementation receives the complete runtime -service graph. Register or predict backends through `services.backend()`, -send through `services.http_client()`, bound request and response bodies, and -return `Report` with integration context. The registry -strips internal identity headers before dispatch; an integration must opt into -any explicit forwarding behavior. - -Streaming support is an adapter capability. Request -`PlatformHttpRequest::with_stream_response()` only when the caller has -checked `supports_streaming_responses()`; unsupported adapters must reject -the request instead of silently buffering it. +service graph. Send every outbound request through +`services.outbound_client()`, EdgeZero's `HttpClient`, and take deadlines from +`services.clock()`. Build each request with a helper from +`trusted_server_core::outbound`: `buffered` sets a response cap and the +response decoding, and `streamed` streams the response in the coding the +upstream sent. Both set a 1 s connect timeout and a constant total timeout. +Never compute a timeout from a remaining budget; use a request deadline for +that, so Fastly backend identity stays stable. Pass client failures to +`outbound_error`, which keeps the integration's own error and its 502. + +Read an inbound body with `http_util::read_request_body` and a cap. It answers +413 when the body is over the cap and 408 when the read passes EdgeZero's 30 s +inbound read deadline. Return `Report` with integration +context. The registry strips internal identity headers before dispatch; an +integration must opt into any explicit forwarding behavior. + +Every adapter streams responses and runs request batches, so an integration +checks no adapter capability. ## Asynchronous HTML processor completion @@ -224,8 +215,9 @@ A non-retaining session can make a KV lookup at EOF and append a fixed footer. The generic KV interface has no hard timeout, and Fastly installs no cooperative timer, so an unbounded KV call can hold the response indefinitely. The example skips the lookup when `input.deadline.has_cooperative_timer()` is false. On -Fastly, use an HTTP lookup whose transport timeouts are capped with -`input.deadline.remaining()` instead. +Fastly, use an HTTP lookup instead. Give it a constant `timeout` and a request +deadline at +`input.services.clock().deadline_after(input.deadline.remaining())`. ::: @@ -304,9 +296,9 @@ their entire lifetimes. Their capabilities must match; a mismatch fails the requ Leave `buffer_output_until_eof()` at its default `false` for this append operation. KV must be available on the runtime; unavailable lookup falls back to the synchronous tail. Fastly consumers must use host operations with independently -bounded latency and cap configurable transport timeouts with `input.deadline.remaining()`; -this generic KV interface does not provide a hard timeout. Checking the remaining -budget alone cannot interrupt a blocking KV call. +bounded latency. An HTTP lookup keeps a constant `timeout` and adds a request deadline +at the remaining budget; this generic KV interface does not provide a hard timeout. +Checking the remaining budget alone cannot interrupt a blocking KV call. Operators must explicitly accept the possible delivery cost: @@ -354,7 +346,8 @@ sink errors remain fatal. Callback panics propagate. Cancellation does not undo writes or guarantee that transport work is cancelled. Axum, Workers and Spin provide cooperative timers. Fastly has none because its host -calls block; bound those calls with `deadline.remaining()` and transport timeouts. +calls block; bound an HTTP call there with a request deadline at the remaining budget, +beside its constant `timeout`. A timer cannot preempt CPU work or blocking host calls. Avoid logging HTML, lookup URLs or external response contents. @@ -379,7 +372,7 @@ paths that can await emit abandonment telemetry. Opted-in HTML bypasses shared-template reads/reservations/stores and receives `Cache-Control: no-store, private`, including unchanged, failed and overflow-bypassed -work. Raw-origin readthrough remains independent. Cacheable asynchronous output needs +work. Cacheable asynchronous output needs separate rules for output identity, external-data freshness, expiry, invalidation, failure caching and callback execution on hits; this is tracked in [#1397](https://github.com/IABTechLab/trusted-server/issues/1397). @@ -420,6 +413,9 @@ untouched. See [Trusted Server JavaScript](/guide/tsjs) and integration is intentionally universal. 2. Register the exact capability predicates and route methods. 3. Add source and behavior parity records. -4. Add positive, negative, body-bound, header, and adapter-capability tests. +4. Add positive, negative, body-bound, header, and outbound request readback + tests. The scripted client in + `trusted_server_core::test_support::scripted_client`, behind the + `test-utils` feature, records each request the integration sends. 5. Document configuration, failure behavior, and runtime limitations. 6. Run the target aliases from [Testing](/guide/testing). diff --git a/docs/guide/integrations-overview.md b/docs/guide/integrations-overview.md index f14a5b2f90..c695c4b376 100644 --- a/docs/guide/integrations-overview.md +++ b/docs/guide/integrations-overview.md @@ -10,13 +10,14 @@ loaded by a separate tag. | Adapter | Release status | Health | Startup status | Startup health | Provider fan-out | Trusted-client-IP handling | Request normalization | | ------------ | -------------- | ---------- | -------------- | -------------- | ---------------- | ------------------------------ | -------------------------------- | -| `axum` | development | real | `500` | no | multiple | outermost sanitize | none | -| `cloudflare` | development | absent | `500` | no | single | outermost sanitize | none | +| `axum` | development | real | exit 1 | no | multiple | outermost sanitize | none | +| `cloudflare` | development | absent | `500` | no | multiple | outermost sanitize | none | | `fastly` | production | pre router | `500` | yes | multiple | entry-point resolve + sanitize | none | -| `spin` | experimental | real | `503` | yes | single | outermost sanitize | innermost Spin-header derivation | +| `spin` | experimental | real | `503` | no | multiple | outermost sanitize | innermost Spin-header derivation | -This table summarizes the adapter implementations. “Startup health” means the -degraded startup router still serves `/health`; it does not +This table summarizes the adapter implementations. “Startup health” means +`/health` still answers 200 when startup fails. Only Fastly does, because it +answers `/health` before it builds the application, so that answer does not mean configuration loaded. “Provider fan-out” describes whether the adapter can dispatch one auction to multiple configured providers. See the individual [Fastly](/guide/fastly), [Axum](/guide/axum-dev), diff --git a/docs/guide/integrations/aps.md b/docs/guide/integrations/aps.md index 5e1b90e6c3..54c7b9d4a3 100644 --- a/docs/guide/integrations/aps.md +++ b/docs/guide/integrations/aps.md @@ -130,7 +130,8 @@ inventory, CSP, iframe/script creative behavior, impression reporting, and click-through behavior with the APS account team before production rollout. The common provider `endpoint` is required and must be an absolute HTTPS URL -with a host and no credentials or fragment. The legacy `/e/dtb/bid` path is +with a host and no credentials or fragment; plain HTTP is accepted only for +`127.0.0.1` and `[::1]`, which local test bidders use. The legacy `/e/dtb/bid` path is rejected. `timeout_ms` belongs beside `endpoint`; when omitted, the `aps` profile default is 800 ms. Runtime caps it by the remaining auction budget. diff --git a/docs/guide/integrations/datadome.md b/docs/guide/integrations/datadome.md index f289a77b98..b8720f55f3 100644 --- a/docs/guide/integrations/datadome.md +++ b/docs/guide/integrations/datadome.md @@ -78,7 +78,7 @@ patterns = ["(?i)\\.(avi|flv|mka|mkv|mov|mp4|mpeg|mpg|mp3|flac|ogg|ogm|opus|wav| | `enable_protection` | boolean | `false` | Call the Protection API before route matching | | `server_side_key_secret_name` | string | none | Default-store secret reference required when protection is enabled | | `protection_api_origin` | string | `https://api-fastly.datadome.co` | Protection API origin | -| `timeout_ms` | integer | `1500` | Dynamic backend first-byte timeout for Protection API calls | +| `timeout_ms` | integer | `1500` | Total timeout for each Protection API call | | `protection_excluded_methods` | array | `["OPTIONS"]` | HTTP methods skipped before the Protection API call | | `protection_excluded_asns` | array | `[]` | Client autonomous system numbers skipped before the Protection API call | | `protection_excluded_ip_cidrs` | array | `[]` | Inline client IP CIDR ranges skipped before the Protection API call | diff --git a/docs/guide/integrations/google_tag_manager.md b/docs/guide/integrations/google_tag_manager.md index fcada96646..468299e067 100644 --- a/docs/guide/integrations/google_tag_manager.md +++ b/docs/guide/integrations/google_tag_manager.md @@ -251,22 +251,26 @@ POST /integrations/google_tag_manager/g/collect?v=2&... **POST Request Handling**: -The endpoint validates POST request sizes to prevent memory pressure: - -- If `Content-Length` header is present and valid: - - Requests exceeding `max_beacon_body_size` are rejected early with `413 Payload Too Large` - - Valid requests proceed normally -- If `Content-Length` header is malformed: `400 Bad Request` -- If `Content-Length` header is missing: Request is accepted (HTTP/2 compatible) - - Body is read in 8KB chunks with size validation - - Reading stops immediately if `max_beacon_body_size` is exceeded - - Oversized bodies return `413 Payload Too Large` without buffering the full payload +The endpoint bounds POST bodies to prevent memory pressure: + +- A declared `Content-Length` over `max_beacon_body_size` returns + `413 Payload Too Large` before any body byte is read. +- A malformed or duplicate `Content-Length`, or `Content-Length` together with + `Transfer-Encoding`, returns `400 Bad Request` from EdgeZero's ingress check + before the route runs. +- A body with no `Content-Length`, or under it, is read through a bounded + reader. The read stops as soon as the body would pass + `max_beacon_body_size` and returns `413 Payload Too Large`. +- A body still unread 30 s after the request arrived returns + `408 Request Timeout`. +- Any other failed body read returns `502 Bad Gateway`. **Memory Protection**: -The implementation uses chunked reading to prevent unbounded memory allocation. Bodies are read in small chunks (8KB), and size is validated incrementally. This ensures that even if a client sends a malicious multi-gigabyte POST (with no Content-Length or an incorrect one), the server will reject it after reading at most `max_beacon_body_size + 8KB` into memory. - -This approach maintains compatibility with HTTP/2 and HTTP/3 clients while providing robust protection against memory exhaustion attacks. +The bounded reader checks the cap before it appends each chunk, so it never +holds more than `max_beacon_body_size` bytes of a body, whatever length the +client declares. Bodies without a `Content-Length`, as HTTP/2 and HTTP/3 clients +send them, are accepted under the same bound. ## Performance & Caching diff --git a/docs/guide/key-rotation.md b/docs/guide/key-rotation.md index 3f8b6f4d96..cfaf4143e4 100644 --- a/docs/guide/key-rotation.md +++ b/docs/guide/key-rotation.md @@ -305,18 +305,21 @@ curl -X POST https://your-domain/_ts/admin/keys/rotate \ ```rust use trusted_server_core::request_signing::KeyRotationManager; -// Initialize rotation manager -let manager = KeyRotationManager::new("jwks_store", "signing_keys")?; +// `services` is the request's `&RuntimeServices`. The two arguments are the +// management API IDs of the JWKS config store and the signing secret store. +let manager = KeyRotationManager::new("example-config-store-id", "example-secret-store-id"); // Rotate with automatic kid -let result = manager.rotate_key(None)?; +let result = manager.rotate_key(services, None).await?; println!("New key: {}", result.new_kid); println!("Previous key: {:?}", result.previous_kid); println!("Active keys: {:?}", result.active_kids); // Or rotate with custom kid -let custom_result = manager.rotate_key(Some("my-custom-key".to_string()))?; +let custom_result = manager + .rotate_key(services, Some("my-custom-key".to_string())) + .await?; ``` ## Managing Active Keys @@ -326,8 +329,8 @@ let custom_result = manager.rotate_key(Some("my-custom-key".to_string()))?; **Rust API**: ```rust -let manager = KeyRotationManager::new("jwks_store", "signing_keys")?; -let active_keys = manager.list_active_keys()?; +let manager = KeyRotationManager::new("example-config-store-id", "example-secret-store-id"); +let active_keys = manager.list_active_keys(services)?; for kid in active_keys { println!("Active key: {}", kid); @@ -417,13 +420,13 @@ curl -X POST https://your-domain/_ts/admin/keys/deactivate \ ### Using the Rust API ```rust -let manager = KeyRotationManager::new("jwks_store", "signing_keys")?; +let manager = KeyRotationManager::new("example-config-store-id", "example-secret-store-id"); // Deactivate (keep in storage) -manager.deactivate_key("ts-2024-01-15")?; +manager.deactivate_key(services, "ts-2024-01-15").await?; // Delete completely -manager.delete_key("ts-2024-01-15")?; +manager.delete_key(services, "ts-2024-01-15").await?; ``` ### Safety Checks @@ -599,7 +602,11 @@ Test rotation in staging first: ### Rotation Failed -**Error**: `Failed to create KeyRotationManager` +**Error**: the rotate endpoint answers 500 with `"message": "Key rotation failed"` +and `"error": "Configuration error: failed to store private key ''"`. + +The first store write failed, so nothing was rotated. The usual causes are a +missing Fastly API token and a Fastly API call that the API refused. **Solutions**: diff --git a/docs/guide/onboarding.md b/docs/guide/onboarding.md index 1310484eff..21525655eb 100644 --- a/docs/guide/onboarding.md +++ b/docs/guide/onboarding.md @@ -28,17 +28,19 @@ The adapters share the core publisher handler, but their entry points and streaming capabilities differ. This trace uses Fastly to show the streaming path. Navigate by function name; line numbers change. -| Step | Function or area | Responsibility | -| ---- | ---------------------------------------------------- | --------------------------------------------------------------------------------------------------------- | -| 1 | `adapter-fastly/src/main.rs` — `main` | Answer `/health` immediately; otherwise initialize logging and call `edgezero_main` | -| 2 | `main.rs` — `edgezero_main` | Load runtime configuration, sanitize forwarded client data, capture TLS/device signals, and dispatch | -| 3 | EdgeZero router | Select a named Trusted Server route or the fallback handler | -| 4 | `adapter-fastly/src/app.rs` — `dispatch_fallback` | Build EC request state, run pre-route integration filters, and distinguish asset/integration traffic | -| 5 | `dispatch_fallback` | For an eligible publisher navigation, generate an EC ID when needed and call the shared publisher handler | -| 6 | `core/src/publisher.rs` — `handle_publisher_request` | Use the consent context, match configured slots, and evaluate the server-side ad-stack gate | -| 7 | `handle_publisher_request` | Dispatch eligible bidder requests before sending the request to the publisher origin | -| 8 | Publisher response pipeline | Classify the origin response, then rewrite or inject into eligible HTML | -| 9 | Fastly entry point | Apply final EC and filter effects, then send the buffered or streaming response | +| Step | Function or area | Responsibility | +| ---- | --------------------------------------------------- | -------------------------------------------------------------------------------------------------------------------------------------------------------- | +| 1 | `adapter-fastly/src/main.rs`: `main` | Hand the request to EdgeZero's Fastly lifecycle (`run_custom`, or the bounded `serve_custom` loop with `reusable-sandbox`), which calls `handle_request` | +| 2 | `main.rs`: `handle_request` | Answer `/health` immediately; otherwise initialize logging, assemble the app once per sandbox from runtime configuration, and serve | +| 3 | `main.rs`: `serve` | Sanitize forwarded client data, capture TLS/device signals and geo, and send the request through EdgeZero's canonical lifecycle | +| 4 | EdgeZero router | Select a named Trusted Server route or the fallback handler | +| 5 | `adapter-fastly/src/app.rs`: `dispatch_fallback` | Build EC request state, run pre-route integration filters, and distinguish asset/integration traffic | +| 6 | `dispatch_fallback` | For an eligible publisher navigation, generate an EC ID when needed and call the shared publisher handler | +| 7 | `core/src/publisher.rs`: `handle_publisher_request` | Use the consent context, match configured slots, and evaluate the server-side ad-stack gate | +| 8 | `handle_publisher_request` | Dispatch eligible bidder requests before sending the request to the publisher origin | +| 9 | Publisher response pipeline | Classify the origin response, then rewrite or inject into eligible HTML | +| 10 | `main.rs`: `finalize_response` | Apply final EC and filter effects in EdgeZero's `finalize` hook; EdgeZero then delivers the response | +| 11 | `adapter-fastly/src/delivery.rs`: `handle_delivery` | After a committed delivery, send queued auction telemetry and run EC pull sync | Two ordering details matter when debugging: diff --git a/docs/guide/spin.md b/docs/guide/spin.md index 1a0b5c5a7f..1972e9594b 100644 --- a/docs/guide/spin.md +++ b/docs/guide/spin.md @@ -8,12 +8,12 @@ through Spin variables. | Adapter | Release status | Health | Startup status | Startup health | Provider fan-out | Trusted-client-IP handling | Request normalization | | ------- | -------------- | ------ | -------------- | -------------- | ---------------- | -------------------------- | -------------------------------- | -| `spin` | experimental | real | `503` | yes | single | outermost sanitize | innermost Spin-header derivation | +| `spin` | experimental | real | `503` | no | multiple | outermost sanitize | innermost Spin-header derivation | -This row is the canonical status summary. Spin supports one enabled -auction provider. When startup fails, its hardened router preserves `/health` -but answers publisher routes with 503; neither response proves successful -configuration. +This row is the canonical status summary. Spin supports multiple auction +providers. When startup fails, every route, `/health` included, answers 503 +with a generic `Service Unavailable` error, and the full report goes only to +the log. ## Prerequisites diff --git a/docs/guide/telemetry.md b/docs/guide/telemetry.md index a4ad08b93a..6778ef1823 100644 --- a/docs/guide/telemetry.md +++ b/docs/guide/telemetry.md @@ -10,14 +10,16 @@ The request entry point derives coarse `DeviceSignals`. Auction execution combines those signals with request and navigation facts in an `AuctionObservationContext`. The terminal auction outcome is converted into an `AuctionEventBatch` containing summary, provider-call, and bid rows. -Fastly serializes the batch as NDJSON and starts one Tinybird Events API -request to the configured auction datasource. - -Emission is best effort. Failure to construct or start telemetry is logged and -must not change the customer response. The sink drops the in-flight handle -after dispatch; it does not wait for Tinybird's response. Each batch is limited -to 512 rows and `tinybird.max_body_bytes` bytes. The Fastly backend uses -two-second first-byte and between-byte timeouts. +Fastly serializes the batch as NDJSON and queues it for the request. After the +response is delivered, the entry point sends one Tinybird Events API request +per queued batch to the configured auction datasource. It sends nothing when +delivery did not commit the application's response body. + +Emission is best effort. Failure to construct, queue, or send telemetry is +logged and does not change the customer response, which the client already +has. Each batch is limited to 512 rows and `tinybird.max_body_bytes` bytes. +Each request has a 2 s total timeout and a 1 s connect timeout, and bypasses +the cache. ## Privacy boundary diff --git a/docs/internal/EDGEZERO_MIGRATION.md b/docs/internal/EDGEZERO_MIGRATION.md index 9543f5b4ba..23502e9e4f 100644 --- a/docs/internal/EDGEZERO_MIGRATION.md +++ b/docs/internal/EDGEZERO_MIGRATION.md @@ -9,8 +9,10 @@ epic [#480](https://github.com/IABTechLab/trusted-server/issues/480)). ## Current Status The Fastly legacy cleanup has removed `legacy_main()` and the runtime canary -flag plumbing. The Fastly entry point now always dispatches through -`edgezero_main()`. +flag plumbing. The Fastly entry point serves every request through EdgeZero's +canonical Fastly lifecycle: `handle_request` assembles the app once per +sandbox, and `serve` sends each request through +`send_request_with_registries_and_hooks`. The historical canary keys are no longer read: @@ -25,17 +27,21 @@ Do not use those keys for rollout or rollback after the legacy cleanup. Config store name: **`trusted_server_config`** (Fastly service config store) -The store must exist because the Fastly entry point opens it before EdgeZero -dispatch and passes the handle to EdgeZero-backed platform services. The store -may be empty unless another feature adds keys to it. +App assembly loads settings from the app-config entry in this store, under the +`trusted_server_config` key. A missing store or entry fails settings load, +which surfaces as an app assembly error. -| Condition | Effective behaviour | -| --------------------------------- | -------------------------------------------- | -| `trusted_server_config` exists | Request dispatches through `edgezero_main()` | -| `trusted_server_config` is absent | Request returns `500 Internal Server Error` | +| Condition | Effective behaviour | +| -------------------------------------------- | ------------------------------------------------------------------------------------------------- | +| `trusted_server_config` holds the app config | The app assembles, and requests run through EdgeZero's lifecycle | +| The store or its entry is absent | Each request answers `500` with EdgeZero's redacted JSON error; the next request retries assembly | -Local Viceroy templates must define an inline `trusted_server_config` store even -when no keys are present. +`/health` answers 200 in both cases, because the entry point serves it before it +assembles the app. + +Local Viceroy configurations must define an inline `trusted_server_config` +store; `ts config push --adapter fastly --local` writes the app-config entry +into it. --- diff --git a/docs/superpowers/plans/2026-11-08-edgezero-pr275-hard-cutover.md b/docs/superpowers/plans/2026-11-08-edgezero-pr275-hard-cutover.md new file mode 100644 index 0000000000..1bad1696c0 --- /dev/null +++ b/docs/superpowers/plans/2026-11-08-edgezero-pr275-hard-cutover.md @@ -0,0 +1,1201 @@ +# EdgeZero PR 275 Hard Cutover Implementation Plan + +> **For agentic workers:** REQUIRED SUB-SKILL: Use +> superpowers:subagent-driven-development (recommended) or +> superpowers:executing-plans to implement this plan task-by-task. Steps use +> checkbox (`- [ ]`) syntax for tracking. + +**Goal:** Move Trusted Server onto EdgeZero PR 275 in PR #1506 (issue #1507), +with no compatibility layer. Every outbound request uses EdgeZero's +`HttpClient`, Fastly runs EdgeZero's canonical lifecycle, and every inbound +body goes through a bounded reader. + +**Architecture:** The plan repins the six EdgeZero crates to PR 275's head and +fixes the nine API changes. It then adds a scripted `OutboundHttpClient` and +two `RuntimeServices` fields. The outbound sites are ported in four groups +while the old layer still compiles: integrations, proxies, auction, then +publisher. Fastly then moves onto `send_request_with_registries_and_hooks`. +Last, the old layer, its doubles, and every harness that depends on it are +deleted, and the docs are updated. Tasks follow the spec's commit sequence, +steps 1 to 11, with steps 2, 3, and 8 split. + +**Tech Stack:** Rust 1.98.1 (2024 edition), EdgeZero PR 275 (`edgezero-core`, +the Fastly, Axum, Cloudflare, and Spin adapters, `edgezero-cli`), Fastly +Compute on `wasm32-wasip1` with Viceroy 0.21.0, Cloudflare Workers through +wrangler, Spin 4.1.0, Axum, `error-stack`, and Node 24.20.0 for the harness +scripts. + +**Spec:** +[`docs/superpowers/specs/2026-10-17-edgezero-pr275-hard-cutover-design.md`](../specs/2026-10-17-edgezero-pr275-hard-cutover-design.md), +cited below as "Spec §N" and "D1" to "D20". Its requirements are in +[`docs/superpowers/specs/2026-10-17-edgezero-pr275-requirements.md`](../specs/2026-10-17-edgezero-pr275-requirements.md). +Read the spec section a task names before starting the task. The spec holds +the per-site detail, and this plan does not repeat it. + +## How to read the citations + +- Spec citations of the form `crates/...:` are Trusted Server at + `51730bdaf`. `main` has moved one commit since then, `4d55a3c6d` (#1405, + SDK URL ownership). That commit touches the Lockr and Permutive integrations + and adds `integrations/sdk_url_test_support.rs`, so line numbers in those + two files can be off by a few lines. Search by function name. +- `edgezero:` citations are EdgeZero at H, `9b31a697`. The public API of the + runtime crates at PR 275's head on 2026-11-08, `5ea8d93c`, matches H for + every item this plan uses. Read EdgeZero from a scratch clone checked out at + the pinned rev, never from the user's EdgeZero checkout. + +## Working rules + +- Worktree: `.claude/worktrees/spec-edgezero-pr275` (`$W`), branch + `spec/edgezero-pr275-migration`, PR #1506. Source + `$(git -C "$W" rev-parse --git-dir)/prfection-lib.sh` and run every + command that writes files through `prf_run "$W" `. +- Use the worktree's own `target/`. Never share a `CARGO_TARGET_DIR` with + another worktree. +- A failing verification step stops the task. Report the command, its exit + code, and the output. Never weaken a test or a CI step to get green. + +## Global Constraints + +- EdgeZero source: `git = "https://github.com/IABTechLab/edgezero"`, + `rev = ""`, which was `5ea8d93c940bc021c552d29ddd86dd14bdadcd3e` + when this plan was written. Features and `default-features` stay as they are + (Spec §1, D5). +- `spin-sdk`: `version = "=7.0.0"` from + `git = "https://github.com/stackpop/spin-rust-sdk.git"` at + `rev = "6c59ed5b4fcb377072763aee0b27a3abdd8edb72"`, with the features + EdgeZero's `Cargo.toml:106` uses (Spec §1). +- No shim, dual path, or feature flag remains after Task 9 (D1). The old + `http_client` and the new `outbound_client` exist side by side only from + Task 3B to Task 9. +- Every outbound request sets `.connect_timeout(Duration::from_secs(1))` and a + constant `.timeout(..)`: the site's configured timeout where the site map + names one, 2 s for Tinybird, and otherwise 60 s streamed or 15 s buffered. + Redirect chains add `.deadline(clock.deadline_after(60 s))`. No timeout is + ever computed from a remaining budget (D7). +- Buffered sites keep the response caps in Spec §2.3. A site with no Trusted + Server cap today sets 10 MiB (Spec §2.2). +- Every client has the proxy marker off: `.with_proxy_marker(false)` (D6, + EZ-7). Production code installs no client in router state. +- The response write deadline is 60 s after egress starts. The publisher upload + cap is 16 MiB. The inbound read deadline stays at EdgeZero's default of 30 s + (D7, Q2). +- Only the statuses the cut adds on purpose change: 413, 408, and the + publisher-forward 400 (D17). Every other upstream failure keeps the site's 502. +- Trusted Server never sends `Trailer`, an operator `Content-Length`, or an + operator hop-by-hop field (D10). +- Errors use `error-stack` (`Report`) and + `derive_more::Display`. No `thiserror`. No `anyhow` except the Spin entry + point. +- Code rules from `AGENTS.md`: + - `expect("should ...")`; no `unwrap()` in production code. + - `log` macros only. + - Comments go above the code. + - No em dashes in comments or test names. + - Only `example.com` hosts and fictional data. + - No function takes more than 7 arguments. +- No new crate dependencies. The only additions are dev-dependency features: + `test-utils` on `edgezero-adapter-axum`, `edgezero-adapter-fastly`, and + `edgezero-core`, where a task names them. +- Never change the EdgeZero repository. +- Commit messages are in sentence case and the imperative, with no prefix. They + carry no `Co-Authored-By`, "Generated with", or other AI trailer. The user's + rule overrides any tool reminder. +- Push only at a task's push step. PR #1506 stays a draft until the user says + otherwise. + +## Review Focus + +These input classes are implied by the spec, but no spec test exercises them. +Each one has a test in the task named. + +1. **Loopback lookalikes.** Plain `http` provider endpoints on port 8080 whose + host is `localhost`, the IPv4 address one above the loopback literal, or the + unspecified IPv4 address are still rejected. Only the literals `127.0.0.1` + and `[::1]` pass, with or without a port (D18). Test in Task 6. +2. **Equivalent host override.** Some `origin_host_header_override` values + equal the origin authority, compared case-insensitively with default ports + removed. Examples are `Example.COM` and `example.com:443` for + `https://example.com`. They set no `host_authority_override`, so Fastly + backend identity is unchanged and Spin does not reject the request. Test in + Task 7. +3. **Empty auction.** Consent or media type can filter out every provider. + The auction then returns today's no-bid result at once, without waiting for + the auction deadline, and starts no batch slot. Test in Task 6. +4. **Raw-byte cookie.** A publisher request whose `Cookie` value holds a byte + that is not UTF-8 is still forwarded, not answered with 400 or 500. It + reaches the origin with no `Cookie` field. The warning names `cookie` and + never logs the value. Test in Task 7. +5. **Odd `Content-Length`.** On a reader route, a duplicate or non-numeric + `Content-Length` skips the declared-length precheck. The bounded read then + governs, with no panic. Test in Task 2D. + +## Task map + +| Task | Spec step | Content | Push | +| ---- | --------- | ----------------------------------------------------------- | ---- | +| 0 | none | Merge `main`, confirm the pin, baseline suite | no | +| 1 | 1 | Baseline ingress cases and the Fastly response-diff script | yes | +| 2A | 2 | Pin, lockfile, manifests, and core API fixes | no | +| 2B | 2 | Fastly and Axum API fixes, Axum tests on `AxumTestServer` | no | +| 2C | 2 | Cloudflare and Spin API fixes, `configure` policies | no | +| 2D | 2 | `read_request_body` and every inbound reader | yes | +| 3A | 3 | Scripted client and test clock | no | +| 3B | 3 | `RuntimeServices` fields, request helpers, `outbound_error` | no | +| 3C | 3 | `oneshot` client helper, parity suite on `AxumTestServer` | yes | +| 4 | 4 | Integrations O10 to O15, `certificate_check` rule | no | +| 5 | 5 | Proxies O6 to O9: redirects, asset proxy, Image Optimizer | no | +| 6 | 6 | Auction O1 to O3, loopback endpoints, fan-out everywhere | yes | +| 7 | 7 | Publisher origin O4 and O5, scope owner, input caps, O19 | yes | +| 8A | 8 | Async store writes and the management API | no | +| 8B | 8 | Fastly canonical lifecycle and post-delivery work | yes | +| 9 | 9 | Delete the old layer, port the deadline harnesses, guard | yes | +| 10 | 10 | Egress validation, Axum ingress variables, observation log | no | +| 11 | 11 | Documentation and `CHANGELOG.md` | yes | +| 12 | none | Full verification and the PR handoff | yes | + +After each spec step, only the tests that the spec lists as red may fail. After +Task 2D those are the `cargo test` job's template-cache harnesses, +`test-fastly-eof`, the Viceroy browser and trace jobs, the Fastly EC lifecycle +job, and the publisher-forward integration cases. They turn green in Task 7 +(Spec "Commit sequence" step 2). The Task 2A to 2C commits compile only the +crates they name. Task 2D restores every target. PR #1506 is squash-merged, so +these intermediate commits never reach `main` on their own. + +--- + +### Task 0: Prepare the branch + +**Files:** none. The task merges `main` into the branch. + +**Interfaces:** Produces the pinned rev `$EZ_REV` that Task 2A writes, and the +baseline results that Task 12 compares against. + +- [ ] **Step 1: Merge `main`.** Run + `prf_run "$W" git fetch origin && prf_run "$W" git merge --no-edit origin/main`. + Expected: a clean merge that brings `4d55a3c6d` in, with no conflict, + since the branch changes only documentation. +- [ ] **Step 2: Confirm the PR 275 head.** Run + `gh pr view 275 --repo IABTechLab/edgezero --json headRefOid --jq .headRefOid`. + Set `EZ_REV` to the value. If it is not `5ea8d93c…`, diff the public API + of the runtime crates between `5ea8d93c` and `EZ_REV` in the scratch + clone. Use + `git diff 5ea8d93c "$EZ_REV" -- crates/edgezero-core/src crates/edgezero-adapter-*/src | grep -E '^[+-]\s*pub (async )?(fn|struct|enum|trait|type|const|use)'`. + Then record every changed item the spec cites in the PR body's + verification section. Expected: the diff is empty, or every changed item + is listed. +- [ ] **Step 3: Baseline suite.** Run every CI gate in `AGENTS.md` ("CI + Gates", 1 to 15) at the merged head and save each command's exit code + and summary line to `$SCRATCH/baseline.md`. Expected: every gate exits 0. + Report any failure to the user before Task 1, together with a + reproduction at `origin/main`. +- [ ] **Step 4: Commit.** The merge commit is the commit. Do not push yet. + +### Task 1: Baseline ingress cases and the response-diff script (Spec "Commit sequence" step 1) + +**Files:** + +- Modify: `crates/trusted-server-integration-tests/tests/integration.rs` +- Create: `scripts/fastly-response-diff.sh`, + `scripts/fastly-response-diff.mjs`, + `scripts/fastly-response-diff.requests.json` +- Modify: `scripts/README.md` (one entry for the script) + +**Interfaces:** + +- Produces the ignored runtime tests below, which Tasks 2D and 7 extend. +- Produces the diff script, which Task 12 runs with + `scripts/fastly-response-diff.sh --base
--head HEAD`. + +- [ ] **Step 1: Write the ingress cases.** Follow the pattern of + `test_cloudflare_enabled_auction_reaches_telemetry`: `#[ignore]` with the + runtime requirement, then `RuntimeEnvironment` and the origin helpers in + `common/runtime`. Add these tests: + - `post_auction_with_json_body_reaches_handler_fastly` and `_axum`. A + `POST /auction` with a valid JSON body gets 200 and an auction response, + not the empty-body parse error. + - `publisher_post_body_reaches_origin_byte_for_byte_fastly`, `_cloudflare`, + and `_axum`. The origin stub records a body equal to the 4096 sent bytes, + including byte values 0 to 255. + - `head_root_returns_origin_status_{fastly,cloudflare,axum}` and + `options_root_returns_origin_status_{fastly,cloudflare,axum}`. The status + equals the origin stub's. + - `chunked_post_auction_over_cap_returns_413_{fastly,cloudflare,axum}`. The + body is 256 KiB plus 1 byte, chunked, with no `Content-Length`. Expected: + status 413 with a `text/plain` body. +- [ ] **Step 2: Write the response-diff script.** + - `fastly-response-diff.sh --base --head ` builds both revs in two + temporary detached worktrees under `$TMPDIR` and serves each under Viceroy + on its own port with the integration fixture config. It runs the `.mjs` + and removes the worktrees on exit, through a `trap`. + - `fastly-response-diff.mjs base_url head_url requests.json expected.json` + sends each request to both servers. It normalizes `x-ts-version`, + `x-ts-sandbox-*`, `date`, `age`, and generated IDs, and with `--staging` + also geo, `x-ts-fastly-version`, and `x-ts-env`. It compares the status, + the sorted headers, and the SHA-256 of the body. It exits 1 when a + difference is not listed in `expected.json`, and also when an expected + difference does not occur. + - `requests.json` holds the Spec "Runtime" list: + - a publisher page + - `HEAD /` and a conditional `GET /` that gets 304 + - `POST /auction` + - a first-party proxy URL + - an asset + - a tsjs bundle + - an admin endpoint + - `TRACE` + - a trace action + - Each request carries an `id`. `expected.json` maps an `id` to the Fastly + differences items it exercises, and starts empty. +- [ ] **Step 3: Run them against `v0.0.11`.** + 1. Build what the cases need: `cargo build-fastly --release` and the Axum + binary, plus `crates/trusted-server-adapter-cloudflare/build.sh`. + 2. Run + `cargo test --manifest-path crates/trusted-server-integration-tests/Cargo.toml --test integration -- --include-ignored post_auction_with_json publisher_post_body head_root options_root chunked_post_auction`. + Expected: every case passes. + 3. Run `scripts/fastly-response-diff.sh --base HEAD --head HEAD`. Expected: + exit 0 and no differences. +- [ ] **Step 4: Commit and push.** Commit with the message "Add baseline ingress + cases and a Fastly response diff for the EdgeZero cut", then run + `prf_run "$W" git push`. + +### Task 2A: Pin, lockfile, manifests, and core API fixes (Spec §1, §6, §7) + +**Files:** + +- Modify: `Cargo.toml` (the six EdgeZero entries and `spin-sdk`), `Cargo.lock`, + `edgezero.toml` (`[capabilities.outbound] hosts = ["*"]`) +- Modify: `crates/trusted-server-adapter-cloudflare/wrangler.toml:12`, + `wrangler.ci.toml:6` (add the `enable_weak_ref` and `enable_request_signal` + compatibility flags) +- Modify, in core: `src/integrations/mod.rs`, `integrations/google_tag_manager.rs`, + `proxy.rs`, `publisher.rs`, `trace/actions.rs`, `trace/dispatch.rs`, + `trace/routes.rs`, `forwarder.rs`, `auth.rs`, `http_util.rs`, and the core + test files in `$S/inventory.tsv`. `$S` is the session scratch `ez275v2/` + directory, or regenerate the inventory with the probe tools. + +**Interfaces:** + +- Produces `TracePreDispatchHook::handle(&self, request: &mut PreDispatchRequest)`. +- Produces `prepare_trusted_forwarder(parts: &mut RequestParts, ..)`, + `action_response(request: &PreDispatchRequest, ..)`, + `matches_trusted_origin(parts: &RequestParts, ..)`, and + `MetadataSupplier::supply(&self, parts: &RequestParts, ..)`. The rest of each + signature is unchanged. +- Produces `enforce_basic_auth(parts: &mut RequestParts, ..)` and + `sanitize_trusted_client_ip_headers(parts: &mut RequestParts)`. + +- [ ] **Step 1: Repin.** Set the six EdgeZero entries to `rev = "$EZ_REV"` and + `spin-sdk` to the Global Constraints spec. Run + `cargo update -p edgezero-core`, then + `cargo update -p 'registry+https://github.com/rust-lang/crates.io-index#spin-sdk@7.0.0'`. + Restore the `colored` 3.1.1 and `winapi-util` 0.1.11 edges to + `windows-sys 0.61.2` by hand. Expected: running `cargo metadata --locked` + exits 0, and the lock moves only the entries in the Spec §1 table and the + git `spin-sdk` stack. +- [ ] **Step 2: Add the manifest and wrangler changes.** Add the + `edgezero.toml` and wrangler entries from Spec §1, then run + `cargo check-fastly 2>&1 | tail -3`. Expected: the 24 core lib errors of + Spec "Problem" (`Body` variants, `from_stream`, `PreDispatchHook`). +- [ ] **Step 3: Fix the core lib.** + - `Body` matches move to `into_parts()`, `into_stream()`, `as_bytes()`, and + `is_stream()` (Spec §7). + - `BodyChunkSource::next_chunk` converts once to a `BodyStream` or `Bytes` + and holds that. + - Publisher body streams use `Body::from_stream` with `EdgeError` items that + pass the origin error through. Foreign `io::Error` streams use + `Body::from_external_stream`. + - Retype the hook and its helpers (Spec §6). The trace action body read + becomes the `Content-Length` check followed by + `request.body_bytes(0).await`, mapped to the trace JSON errors. + - Run `cargo check -p trusted-server-core --target wasm32-wasip1`. Expected: + exit 0. +- [ ] **Step 4: Fix the core tests.** + - The 17 hook test calls drive the hook through + `RouterService::builder().pre_dispatch_hook(..).build().oneshot(request)`. + - A test that checks a continue outcome adds a route whose handler records + the request. + - Run + `cargo check -p trusted-server-core --all-targets --all-features --target wasm32-wasip1`. + Expected: exit 0. +- [ ] **Step 5: Commit.** Commit with "Repin EdgeZero to PR 275 and port core + to its body and hook APIs". Adapters do not compile yet. + +### Task 2B: Fastly and Axum API fixes, Axum tests on `AxumTestServer` (Spec §7, §8) + +**Files:** + +- Modify: `crates/trusted-server-adapter-fastly/src/{app.rs,middleware.rs,compat.rs,main.rs,platform.rs}` +- Modify: `crates/trusted-server-adapter-axum/{Cargo.toml,src/main.rs,src/app.rs,src/middleware.rs,src/platform.rs,tests/routes.rs}` +- Modify: `crates/trusted-server-cli/tests/proxy_trusted_server.rs`, + `crates/trusted-server-cli/Cargo.toml` (adds the dev-dependency feature) + +**Interfaces:** + +- Consumes the Task 2A signatures. +- Produces `TrustedServerApp::routes() -> Result` on + each adapter. +- Produces `fn startup_error(report: &Report) -> EdgeError` + in `crates/trusted-server-core/src/error.rs`. It maps the root status to + `EdgeError::internal`, `service_unavailable`, `bad_gateway`, or + `bad_request` (Spec §7, "Startup failures"). + +- [ ] **Step 1: Write the failing Axum startup test.** Add + `axum_main_exits_non_zero_when_settings_fail` to + `crates/trusted-server-adapter-axum/tests/routes.rs`. It runs the binary + with an invalid `TRUSTED_SERVER__` setting and asserts a non-zero exit + and the logged error line. Run `cargo test-axum` and expect a failure, + because the binary keeps serving 500s. +- [ ] **Step 2: Fix the API sites.** + - `into_request()?` at the 15 roots. + - `parts()`, `headers()`, and `extensions()` in place of + `request()`/`request_mut()`. + - `routes()` returns `Err(startup_error(&report))` after + `log::error!("{report:?}")`. Delete the Axum `startup_error_router`, but + keep Fastly's until Task 8B. + - Fastly `run_custom` and `serve_custom` callbacks return `Ok(())` on every + path. That stays a stopgap until Task 8B. +- [ ] **Step 3: Assemble the Axum dev server.** In Axum `main`, call + `App::build::(AXUM_PLATFORM)`. Pass it to + `AxumDevServer::with_config(app, config)`. On error, log it and + `std::process::exit(1)`. +- [ ] **Step 4: Move the Axum route tests and the CLI proxy test to + `AxumTestServer`.** Build each app with + `App::from_router::(routes_with_settings(..), AXUM_PLATFORM)`, + listen on `127.0.0.1:0`, and attach the stores the test needs. Add + `features = ["test-utils"]` on `edgezero-adapter-axum` in both crates' + dev-dependencies. +- [ ] **Step 5: Verify.** Run `cargo check-fastly`, + `cargo check-fastly --all-targets --all-features`, `cargo test-axum`, and + `./scripts/test-cli.sh`. Expected: all exit 0, and the new startup test + passes. +- [ ] **Step 6: Commit.** Commit with "Port the Fastly and Axum adapters to + EdgeZero PR 275 APIs". + +### Task 2C: Cloudflare and Spin API fixes and `configure` policies (Spec §5, §7, §8) + +**Files:** + +- Modify: `crates/trusted-server-adapter-cloudflare/src/{app.rs,middleware.rs,platform.rs}`, + `tests/routes.rs` +- Modify: `crates/trusted-server-adapter-spin/src/{lib.rs,app.rs,middleware.rs,platform.rs}`, + `tests/routes.rs` +- Modify: the `Hooks::configure` implementation in each of the four adapters' + `app.rs` +- Create: `crates/trusted-server-core/src/edge_app.rs` with + `pub const RESPONSE_WRITE_DEADLINE: Duration = Duration::from_secs(60);`, + declared in `crates/trusted-server-core/src/lib.rs` + +**Interfaces:** + +- Produces + `pub fn install_edge_policies(app: &mut App) -> Result<(), EdgeError>` in + `crates/trusted-server-core/src/edge_app.rs`. Every adapter's `configure` + calls it. + - It calls `app.set_response_egress_policy(..)`, whose deadline is egress + start plus `RESPONSE_WRITE_DEADLINE`. + - It also calls + `app.set_synthesized_response_headers(SynthesizedResponseHeaders::new(..)?)` + with `x-ts-version`. + - Fastly's `configure` adds `x-ts-fastly-version` when it is known. + +- [ ] **Step 1: Write the failing policy test.** Add + `installed_egress_policy_sets_write_deadline_sixty_seconds_after_egress_start` + to `edge_app.rs`. It builds an `App`, calls `install_edge_policies`, and + asserts the policy's deadline equals the egress start plus 60 s (test 17, + unit part). Run `cargo test-fastly -- edge_app::tests`. Expected: a + compile failure, because the function is not defined yet. +- [ ] **Step 2: Implement `install_edge_policies`** and call it from all four + `configure` implementations. +- [ ] **Step 3: Fix the Cloudflare and Spin sites.** + - `ConfigStore::get` converts the `ConfigValue` to `String` at the boundary. + The Spin test store implements `get_bounded`. + - `SpinConfigStore::for_label` replaces `open`. + - The Spin entry point returns + `anyhow::Result`. + - Delete the Cloudflare and Spin `startup_error_router`s and the Spin + `/health` startup test at `app.rs:1531`. `routes()` returns `Err` as in + Task 2B, and Spin always uses `service_unavailable`. +- [ ] **Step 4: Add the synthesized-header tests.** Add + `canonical_not_found_carries_x_ts_version` and + `canonical_method_not_allowed_carries_allow_and_x_ts_version` to each + adapter's route tests. They assert a 404 for an unknown path, and a 405 + with a sorted `Allow` for a wrong method on a known path. Both carry + `x-ts-version`. +- [ ] **Step 5: Verify.** Run `cargo check-cloudflare`, `cargo check-spin`, + `cargo test-cloudflare`, `cargo test-spin`, `cargo test-axum`, and the + core test above. Expected: all exit 0. +- [ ] **Step 6: Commit.** Commit with "Port the Cloudflare and Spin adapters to + EdgeZero PR 275 and install egress policies". + +### Task 2D: `read_request_body` and every inbound reader (Spec §4) + +**Files:** + +- Modify: `crates/trusted-server-core/src/http_util.rs`, `error.rs` +- Modify the readers: `auction/endpoints.rs`, `ec/batch_sync.rs`, + `cache_purge.rs`, `proxy.rs` (sign and rebuild), + `request_signing/endpoints.rs`, `integrations/mod.rs`, the six integration + proxies, and `integrations/google_tag_manager.rs` (beacon) +- Modify: `crates/trusted-server-integration-tests/tests/integration.rs` + +**Interfaces:** + +- Produces + `pub async fn read_request_body(headers: &HeaderMap, body: Body, max: usize) -> Result>` + in `http_util.rs`. +- Produces `TrustedServerError::RequestTimeout { message: String }`, with + status 408. +- Deletes `request_body_bytes` and `collect_body_bounded`. + +- [ ] **Step 1: Write the failing helper tests** in `http_util.rs`: + - `declared_length_over_max_returns_413_without_polling`. The body is a + stream that panics when polled. + - `stream_crossing_max_returns_413`. + - `read_after_deadline_returns_408`, which uses `Body::from_stream` with an + item of `EdgeError::request_timeout("expired")`. + - `client_caused_stream_error_returns_400`, with `EdgeError::bad_request`. + - `host_read_error_returns_502`, with `EdgeError::bad_gateway`. + - `duplicate_content_length_falls_through_to_bounded_read` and + `non_numeric_content_length_falls_through_to_bounded_read` (Review Focus 5). + Both bodies are under `max` and return their bytes. + - `body_under_max_returns_exact_bytes`. + + Each test asserts the `TrustedServerError` variant and its `status_code()`. + Run `cargo test-fastly -- http_util::tests` and expect a compile failure, + because the helper is not defined yet. + +- [ ] **Step 2: Implement it.** First, a single parsable `Content-Length` + over `max` returns `RequestTooLarge` without touching the body. + Otherwise await `body.into_bytes_bounded(max)` and map the `EdgeError` + status: + - 413 becomes `RequestTooLarge`. + - 408 becomes `RequestTimeout`. + - Another 4xx becomes `BadRequest`. + - A 5xx becomes `Proxy`, logged at error. + - 408 and 413 are logged at warn. + + Body bytes are never logged. + +- [ ] **Step 3: Move every reader to the helper.** Use the cap in the Spec §4 + table for each reader. Keep the `text/plain` 413 body on `/auction` and + the `body_too_large` error body on batch sync. +- [ ] **Step 4: Write the reader tests (test 8).** For each reader, add a + declared-over-cap 413, a streamed-over-cap 413, a 408, a 400, and a 502 + through its route. Add + `batch_sync_stream_past_declared_length_and_cap_returns_413`. In + `integration.rs`, add `get_root_with_body_returns_400_{fastly,cloudflare,axum}` + (EZ-1 criterion 13). +- [ ] **Step 5: Verify.** Run `cargo test-fastly`, `cargo test-axum`, + `cargo test-cloudflare`, `cargo test-spin`, and all eight clippy + aliases. Run the integration cases from Task 1 on Cloudflare and Axum, + plus the new GET case. Expected: all exit 0. The Fastly + publisher-forward cases are allowed red until Task 7. Also run + `./scripts/async-html-finalization-local-test.sh fastly --release` and + record whether Fastly hands a GET a body handle (Spec step 2's open + check). +- [ ] **Step 6: Commit and push.** Commit with "Read every inbound body through + a bounded EdgeZero reader", then push. + +### Task 3A: Scripted client and test clock (D12, Spec test 3) + +**Files:** + +- Create: `crates/trusted-server-core/src/test_support/scripted_client.rs` +- Modify: `crates/trusted-server-core/src/test_support.rs` + (`#[cfg(any(test, feature = "test-utils"))] pub mod scripted_client;`) +- Modify: `crates/trusted-server-core/Cargo.toml`, enabling + `edgezero-core/test-utils` from the core `test-utils` feature if the build + needs it + +**Interfaces:** + +- `pub struct TestClock`: + - `new() -> Self` + - `monotonic_clock(&self) -> MonotonicClock` + - `outbound_time(&self) -> OutboundTime`. Its wait factory resolves when + `advance` moves the clock past the deadline. + - `advance(&self, by: Duration)` +- `pub struct ScriptedHttpClient` (`Clone`, `Send + Sync`): + - `new(clock: &TestClock) -> Self` + - `forbidding() -> Self`. It panics with `"should not send "` + on any request. + - `respond_to(&self, method: Method, uri: &str, reply: ScriptedReply) -> &Self`. + Replies queue first in, first out per method and URI. An unmatched request + fails with `EdgeError::internal("unscripted request")` and is still + recorded. + - `http_client(&self) -> HttpClient`, which is + `HttpClient::with_client(self.clone()).with_proxy_marker(false)`. + - `requests(&self) -> Vec` +- `pub struct ScriptedReply`: + - `status(StatusCode) -> Self` + - `header(self, name: &str, value: &str) -> Self` + - `body(self, bytes: impl Into) -> Self` + - `chunks(self, chunks: Vec) -> Self` + - `after(self, delay: Duration) -> Self`, the arrival time on the test clock + - `held(self) -> (Self, ReplyRelease)` + - `error(EdgeError) -> Self` +- `pub struct ReplyRelease` with `release(self)`. +- `pub struct RecordedRequest`. Its public fields are copied from + `OutboundDispatchRequest::into_parts()` (`edgezero:crates/edgezero-core/src/outbound.rs:330-362`): + - `method`, `uri`, `headers`, and `body: Bytes` + - `timeout`, `connect_timeout`, `deadline`, and `cache_policy` + - `response_decoding` and `response_mode` + - `max_decoded_response_bytes` and `max_encoded_response_bytes` + - `image_optimizer` and `proxy_marker` + - `host_authority: String` and `has_host_authority_override: bool`, read + from the `OutboundRequest` before `into_parts`. + +Use the EdgeZero Axum adapter as the model for the provider half: +`schedule_batch` and `schedule_response` +(`edgezero:crates/edgezero-adapter-axum/src/outbound.rs:220-320`) and its test +`ResponseClient` (`edgezero:crates/edgezero-adapter-axum/src/outbound/testing.rs`). +The progress runtime is `IndependentOutboundProgress`. + +- [ ] **Step 1: Write the failing client tests** in `scripted_client.rs`: + - `records_method_uri_headers_and_options` + - `replies_in_fifo_order_per_method_and_uri` + - `chunked_reply_streams_each_chunk` + - `held_reply_resolves_only_after_release` + - `batch_drops_reply_arriving_after_cutoff`. Two slots answer at 100 ms and + 600 ms, and the cutoff is 500 ms. The first is resolved and the second is + `Unresolved`. + - `error_reply_fails_only_its_slot` + - `forbidding_client_panics_on_any_request`, with + `#[should_panic(expected = "should not send")]` + + Run `cargo test-fastly -- scripted_client` and expect a compile failure. + +- [ ] **Step 2: Implement the client and the clock.** +- [ ] **Step 3: Verify.** Run the tests from Step 1. Expected: pass. Then run + `cargo clippy-fastly`. Expected: exit 0. +- [ ] **Step 4: Commit.** Commit with "Add a scripted EdgeZero outbound client + for tests". + +### Task 3B: `RuntimeServices` fields, request helpers, and `outbound_error` (Spec §2.1, §2.2, §2.11) + +**Files:** + +- Modify: `crates/trusted-server-core/src/platform/types.rs`, + `platform/mod.rs` (re-export `HttpClient` and `MonotonicClock`) +- Create: `crates/trusted-server-core/src/outbound.rs`, declared in `lib.rs` +- Modify: every `RuntimeServices::builder()` caller (41 calls in 16 files, found + with `git grep -n 'RuntimeServices::builder()'`) +- Modify: the four adapters' per-request service builders + +**Interfaces:** + +- `RuntimeServices` gains: + - `outbound_client(&self) -> &HttpClient` and + `clock(&self) -> &MonotonicClock` + - the builder methods `.outbound_client(HttpClient)` and + `.clock(MonotonicClock)`, both required by `build()`, as the other + required services are. +- `outbound.rs` holds these constants: + - `CONNECT_TIMEOUT` (1 s) + - `STREAMED_TIMEOUT` (60 s) + - `BUFFERED_TIMEOUT` (15 s) + - `REDIRECT_CHAIN_DEADLINE` (60 s) + - `DEFAULT_BUFFERED_RESPONSE_BYTES: u64` (10 MiB) + - `PUBLISHER_UPLOAD_BYTES: u64` (16 MiB) +- It also holds these functions: + - `pub fn buffered(request: OutboundRequest, timeout: Duration, max_response_bytes: u64, decoding: ResponseDecoding) -> OutboundRequest`. + It sets the connect timeout, the timeout, the cap, and the decoding. + - `pub fn streamed(request: OutboundRequest, timeout: Duration) -> OutboundRequest`. + It sets the connect timeout, the timeout, `.stream_response()`, and + `ResponseDecoding::Original`. + - `pub enum OutboundRequestSource { ClientForward, Internal }` + - `pub fn outbound_error(error: EdgeError, context: TrustedServerError, source: OutboundRequestSource) -> Report`. + The spec's two-argument signature cannot tell the publisher-forward 400 + from a request Trusted Server built itself, so the source argument is + added. + +- [ ] **Step 1: Write the failing tests (test 18).** + - `outbound_error_maps_payload_too_large_to_413` + - `outbound_error_maps_request_timeout_to_408` + - `outbound_error_maps_bad_request_to_400_for_client_forward` + - `outbound_error_maps_bad_request_to_site_502_for_internal` + - `outbound_error_keeps_site_502_for_gateway_timeout_bad_gateway_response_too_large_and_internal` + - `outbound_error_attaches_edge_error_detail` + - `buffered_sets_one_second_connect_timeout_and_constant_timeout` + - `streamed_sets_original_decoding_and_stream_response` + + Each one asserts the status and the attached `EdgeError` display. Run + `cargo test-fastly -- outbound::tests` and expect a compile failure. + +- [ ] **Step 2: Implement the helpers and the fields.** + - `outbound_error` attaches the `EdgeError`, its budget source, and + `provider_failure_detail()`. It logs the detail with `{:?}` at warn, never + provider text or URLs (Spec §9). + - The adapters set `.outbound_client(ctx.http_client().ok_or_else(..)?.with_proxy_marker(false))` + and `.clock(ctx.monotonic_clock())`. + - A missing client is `EdgeError::internal("request has no outbound client")`, + not a panic. + - Test fixtures use `ScriptedHttpClient`. + - `ForbiddenLifecycle` in `parity.rs` uses `ScriptedHttpClient::forbidding()`. +- [ ] **Step 3: Verify.** Run `cargo test-fastly`, `cargo test-axum`, + `cargo test-cloudflare`, `cargo test-spin`, and the eight clippy + aliases. Expected: exit 0. +- [ ] **Step 4: Commit.** Commit with "Carry EdgeZero's outbound client and + clock in RuntimeServices". + +### Task 3C: `oneshot` client helper and the parity suite on `AxumTestServer` (Spec §2.1, §8) + +**Files:** + +- Modify: `crates/trusted-server-core/src/test_support.rs` +- Modify: `crates/trusted-server-adapter-fastly/src/app.rs` (the `route` test + helper) +- Modify: `crates/trusted-server-adapter-cloudflare/tests/routes.rs`, + `crates/trusted-server-adapter-spin/tests/routes.rs` +- Modify: `crates/trusted-server-integration-tests/tests/parity.rs`, + `crates/trusted-server-integration-tests/Cargo.toml` (dev-dependency + features) + +**Interfaces:** + +- Produces + `pub fn insert_outbound_client(request: &mut Request, client: HttpClient)` in + `test_support.rs`. It inserts into the request extensions, where + `ctx.http_client()` reads. + +- [ ] **Step 1: Write the failing parity test (test 19).** Add + `axum_test_server_parity_rejects_9000_byte_target_with_414` for Axum, + Cloudflare, and Spin. Add + `bodiless_get_with_unread_stream_reaches_origin_without_body`. That test + supplies EdgeZero's Axum client, `AxumOutboundClient`, against a + loopback origin that records the request body. It runs with + `HTTP_PROXY`, `HTTPS_PROXY`, and `ALL_PROXY` unset, and asserts an empty + recorded body. +- [ ] **Step 2: Implement the helper.** Use it in the Fastly, Cloudflare, and + Spin `route` helpers. Move the `cf_*` and `spin_*` parity helpers and + `discovery_route_body_is_json_parity` to `AxumTestServer`. Assemble each + app under its own `PlatformMetadata` and put a `ScriptedHttpClient` in + router state. +- [ ] **Step 3: Verify.** Run + `cargo test --manifest-path crates/trusted-server-integration-tests/Cargo.toml --test parity`, + `cargo test-fastly`, `cargo test-cloudflare`, and `cargo test-spin`. + Expected: exit 0. +- [ ] **Step 4: Commit and push.** Commit with "Run the parity suite on + EdgeZero's Axum test server", then push. + +### Task 4: Integrations O10 to O15 and the `certificate_check` rule (Spec §1, §2.3, D9) + +**Files:** + +- Modify: `crates/trusted-server-core/src/integrations/{sourcepoint.rs,lockr.rs,permutive.rs,datadome.rs,datadome/protection.rs,didomi.rs}` +- Modify: `crates/trusted-server-core/src/config.rs` + (`validate_settings_for_deploy`), `settings.rs:1916` (doc comment) +- Modify: `crates/trusted-server-adapter-fastly/src/app.rs:241` (replace + `warn_if_certificate_check_disabled`) +- Modify: the three integration fixture configs, `scripts/integration-tests.sh:76`, + `scripts/integration-tests-browser.sh:39`, `.github/workflows/test.yml:282`, + and `.github/actions/setup-integration-test-env/action.yml:94`, `:104`, and + `:135` + +**Interfaces:** + +- Consumes `outbound::{buffered, streamed, outbound_error}` and + `services.outbound_client()`. +- Produces + `fn reject_disabled_certificate_check(settings: &Settings) -> Result<(), Report>`, + called from `validate_settings_for_deploy`. + +- [ ] **Step 1: Write the failing tests.** + - Readback tests (test 3) for each of the eight sites, with + `ScriptedHttpClient`. Each asserts the method, URI, and headers, the + 1 s connect timeout, the timeout from Spec §2.3, the cap, the decoding, + the cache policy, and `proxy_marker == OutboundProxyMarker::Inherit` + under a client with the marker off. `Accept-Encoding` must match what + Trusted Server set, and be absent when it set none. + - `sourcepoint_drops_non_utf8_request_header` + - `validate_rejects_certificate_check_false` + - `runtime_ignores_certificate_check_false_with_warning` +- [ ] **Step 2: Port each site** to `services.outbound_client()`, keeping + today's method and headers. Copied client headers go through + `headers_mut()`, never `header()`. +- [ ] **Step 3: Remove `certificate_check = false`** from the fixtures, the + scripts, and CI (Spec §1). +- [ ] **Step 4: Verify.** Run `cargo test-fastly`, `cargo test-axum`, + `cargo test-cloudflare`, `cargo test-spin`, and the clippy aliases. + Expected: exit 0. +- [ ] **Step 5: Commit.** Commit with "Send integration requests through + EdgeZero's client". + +### Task 5: Proxies O6 to O9 (Spec §2.6, §2.7) + +**Files:** + +- Modify: `crates/trusted-server-core/src/proxy.rs` (`proxy_request`, + `send_asset_origin_request`, `handle_asset_proxy_request`, the S3 + preflight) +- Modify: `integrations/{google_tag_manager.rs,prebid.rs,testlight.rs,gpt.rs,js_asset_proxy.rs}` +- Modify: `crates/trusted-server-core/src/asset_image_optimizer.rs` + +**Interfaces:** + +- Produces + `fn redirect_policy(allowed_domains: Vec, require_https: bool) -> impl FnMut(&mut OutboundRedirectContext<'_>) -> Result + 'static` + in `proxy.rs`. +- Produces + `fn image_optimizer_options(profile: &ImageOptimizerProfile, request: &AssetRequest) -> Result` + in `asset_image_optimizer.rs`. It maps Trusted Server's region aliases to + EdgeZero's seven regions. + +- [ ] **Step 1: Port the redirect tests and add the new ones (test 4).** Port + today's redirect, allowlist, and initial-target tests to the scripted + client. Add: + - `redirect_chain_past_deadline_fails_with_site_502`, which advances the + test clock 61 s across hops. + - `location_with_userinfo_fails_with_site_502` + - `cross_origin_hop_strips_authorization_and_cookie` + - `non_ascii_location_is_percent_encoded_and_followed` + - `follow_disabled_uses_send_with_forward_headers` +- [ ] **Step 2: Add the asset and Image Optimizer readback tests (test 3, + EZ-8).** Image Optimizer options read back on a final GET and on a final + HEAD, and never on the S3 preflight. The preflight is buffered with + `Original`. Also add `image_optimizer_builder_error_serves_unoptimized_with_warning`. +- [ ] **Step 3: Implement.** + - `proxy_request` checks the initial target as today. + - Then it calls + `client.follow(request, OutboundRedirectLimit::new(4)?, redirect_policy(..))` + with `.timeout(BUFFERED_TIMEOUT)` and + `.deadline(clock.deadline_after(REDIRECT_CHAIN_DEADLINE))`. + - `OutboundFollowFailure::Policy(e)` maps back to `e`, and `Hop` goes + through `outbound_error`. + - `finalize_response` receives `final_uri()`. + - Delete `proxy_with_redirects` and `ProxyRedirectPolicy` here, with their + tests. +- [ ] **Step 4: Write the SigV4 host test (test 12).** Add + `sigv4_signed_asset_reaches_origin_with_signed_host_path_and_query` to + `integration.rs`, under Viceroy, for a GET without Image Optimizer and a + HEAD preflight with it. +- [ ] **Step 5: Verify.** Run the adapter tests and clippy aliases, and the + SigV4 case. Expected: exit 0. +- [ ] **Step 6: Commit.** Commit with "Follow redirects and proxy assets + through EdgeZero's client". + +### Task 6: Auction O1 to O3 and the provider contract (Spec §2.4, D18) + +**Files:** + +- Modify: `crates/trusted-server-core/src/auction/{provider.rs,orchestrator.rs,types.rs,plan.rs,mod.rs,endpoints.rs,telemetry.rs,test_support.rs,openrtb/test_executor.rs,openrtb/tests.rs}` +- Modify: `crates/trusted-server-core/src/integrations/{adserver_mock.rs,aps.rs,prebid.rs}` +- Modify: the four adapters' state builders, at the `validate_for_target` + calls +- Modify: `crates/trusted-server-integration-tests/src/bin/generate-viceroy-config.rs`, + `scripts/generate-integration-viceroy-configs.sh`, + `scripts/integration-tests-browser.sh`, + `.github/workflows/integration-tests.yml:39-40`, + `scripts/template-cache-local-test.sh`, and + `browser/tests/nextjs/mobile-trace-live.spec.ts:381-389` + +**Interfaces:** + +- `ProviderRequestOutcome::Ready { request: OutboundRequest, parse_state: Option> }`. + The `parse_state` type is unchanged. +- `AuctionProvider::parse_response`, `parse_response_with_context`, and + `parse_response_with_context_and_state` take `response: OutboundResponse`. +- `DispatchedAuction` holds `batch: OutboundBatch` and + `slot_providers: Vec`, an index into the provider list. +- In `plan.rs`, `fn validate_fastly_backend_bound(plan: &AuctionPlan) -> Result<(), Report>` + counts distinct `(scheme, host, port, timeout_ms)` tuples and rejects more + than 160 when the auction is enabled. The limits 200 and 40 move with it. +- In `plan.rs`, `fn is_loopback_http_endpoint(url: &Url) -> bool` is true only + for `http` with host `127.0.0.1` or `[::1]`. + +- [ ] **Step 1: Write the failing unit tests.** + - `bidders_leave_together_in_one_batch` + - `bid_after_cutoff_becomes_provider_timeout` + - `mediator_runs_after_batch_with_auction_deadline` + - `bidder_timeout_is_configured_value_not_remaining_budget` + - `two_auctions_with_different_budgets_record_identical_request_options` + - `abandoned_auction_cancels_pending_slots`, which asserts the slot's + `ReplyRelease` is never awaited. + - `delayed_collection_keeps_slot_elapsed_and_response_time` + - `empty_auction_returns_without_waiting_for_deadline` (Review Focus 3). + It asserts the clock did not advance and no request was recorded. + - `bidder_requests_carry_consent_and_attested_client_ip_headers` + - `fastly_backend_bound_accepts_160_and_rejects_161_tuples` + - `providers_sharing_a_tuple_count_once` + - `disabled_auction_skips_backend_bound` + - `loopback_http_endpoints_are_accepted` + - `loopback_lookalike_endpoints_are_rejected` (Review Focus 1), with the + `localhost`, `127.0.0.2`, and `0.0.0.0` cases. The URL-host linter + rejects the last two hosts, so each of those lines carries an + `// allow-domain: ` comment. Name those suppressions in the PR body + for review. + - `cloudflare_and_spin_load_multi_provider_plan`, which replaces + `startup_rejects_multi_provider_auction_plan`. +- [ ] **Step 2: Implement.** + - Providers return `Ready`. + - `dispatch_planned_auction` calls + `client.start_batch_until(requests, auction_deadline)`, where + `auction_deadline = services.clock().deadline_after(timeout)`. + - Collection loops `batch.next()` until `Finished`, using `batch.check()` + where today's code polls without waiting. + - `OutboundBatchFailure` marks every unresolved slot as a provider error. + - The mediator uses `send` with `.timeout(mediator)` and + `.deadline(auction_deadline)`. + - Delete `validate_for_target`, `AuctionTargetId`, the target descriptors, + the fan-out gate, `canonicalize_transport_timeout_ms`, the name + prediction, and `AuctionContext::transport_timeout_ms` (Spec §2.4). + - `parse_planned_aps_response` takes an `OutboundResponse`, as the Prebid + parser does. + - The cfg(test) paths `run_routed`, `run_parallel_mediation`, + `run_parallel_only`, and `run_providers_parallel`, and the cfg(test) + `AuctionProvider` impls for APS and Prebid, move to the scripted client. + Delete any path whose tests a ported test already covers. +- [ ] **Step 3: Switch the test bidders to loopback.** + - `--bidder-origin-url` becomes `--provider-endpoint-origin`. It rewrites + the scheme and authority of every provider endpoint. + - Delete `inject_controlled_bidder_backends` and its tests. + - Strip TLS from `template-cache-local-test.sh` (Spec §1). + - Add `/api/trace-bidder` to the forbidden evidence list. +- [ ] **Step 4: Write the runtime auction tests (test 5).** Add + `twelve_bidder_auction_registers_twelve_backends_fastly` to + `integration.rs`, with 12 loopback bidders on 12 ports. Assert the bids + and the observation's configuration count across two auctions. Add + `multi_provider_auction_runs_{cloudflare,spin}`, and + `spin_runtime_auction_reports_one_timeout_and_cutoff`. The loopback + bidders record request framing. +- [ ] **Step 5: Verify.** Run the adapter tests and clippy aliases, + `scripts/template-cache-local-test.sh`, and the new runtime cases. + Expected: exit 0. +- [ ] **Step 6: Commit and push.** Commit with "Run auctions as EdgeZero + batches on every adapter", then push. + +### Task 7: Publisher origin O4 and O5 (Spec §2.5, D9) + +**Files:** + +- Modify: `crates/trusted-server-core/src/publisher.rs` + (`handle_publisher_request`, `rewrite_origin_request`, + `buffer_publisher_response_async`, + `publisher_response_into_streaming_response`, the HTML decode bound) +- Modify: `crates/trusted-server-core/src/settings.rs` (readthrough, `origin_url` + userinfo, the Spin warning), `creative_opportunities.rs` +- Modify: the four adapters' publisher call sites +- Modify: `crates/trusted-server-core/src/test_support/async_html_finalization.rs` + (O19) +- Modify: `.github/workflows/integration-tests.yml:230-231` (the + `spin_runtime_` filter) + +**Interfaces:** + +- `pub struct PublisherOriginOptions { pub edge_header: EdgeCacheHeader, pub input_cap: Option }` + replaces the `edge_header` argument, which keeps the signature at 7 + arguments. +- `handle_publisher_request(settings, services, kv, ec_context, auction, req, origin: PublisherOriginOptions) -> Result<(PublisherResponse, OutboundScopeOwner), Report>` +- `buffer_publisher_response_async(publisher_response, method, settings, integration_registry, orchestrator, services, input_cap: u64)` +- The adapters' `input_cap` values: + - Cloudflare: `Some(min(max_buffered_body_bytes, 10 MiB))` + - Spin: `Some(min(max_buffered_body_bytes, 8 MiB))` + - Axum: `Some(max_buffered_body_bytes)` + - Fastly: `None` +- Each adapter calls `owner.attach(response)?` exactly once, after the + response function returns. + +- [ ] **Step 1: Write the failing ownership tests (test 20).** + - `raw_buffered_passthrough_and_range_attach_owner_once_when_origin_call_is_inside_scope`, + for `send`, and for `begin` followed by `collect`. + - `origin_call_outside_scope_fails_attach_with_already_owns_scope` +- [ ] **Step 2: Write the failing cap tests (test 21).** These run with the + 16 MiB default for `max_buffered_body_bytes`: + - `raw_body_at_cap_passes_and_one_over_fails_502_{axum,cloudflare,spin}` + - `source_error_mid_raw_body_fails_502_before_egress` + - `identity_html_at_cap_passes_and_one_over_fails_502_{cloudflare,spin}` + - `gzip_html_under_cap_encoded_over_cap_decoded_fails_502` + - `rewrite_growing_past_output_cap_fails` +- [ ] **Step 3: Write the failing request tests.** + - `origin_request_carries_inbound_body_and_upload_cap` + - `upload_over_16_mib_returns_413` + - `host_override_differing_from_authority_sets_host_authority_override` + - `equivalent_host_override_sets_none` (Review Focus 2) + - `spin_ignores_host_override_with_warning` + - `non_utf8_cookie_is_dropped_and_request_forwarded` (Review Focus 4) + - `validate_rejects_origin_url_with_userinfo` + - `validate_rejects_origin_readthrough_enabled` + - `runtime_treats_readthrough_as_false_with_warning` +- [ ] **Step 4: Implement (Spec §2.5).** + - The owner is created before dispatch. + - The auction and both origin calls are constructed inside `owner.run`. + - `rewrite_origin_request` stops writing `Host`. + - Requests use `streamed(..)` with `STREAMED_TIMEOUT` and the mapped cache + policy. With an `input_cap`, they also set `.max_encoded_response_bytes`. + - The HTML decode is bounded cumulatively at `input_cap`. + - Raw bodies are collected with `into_bytes_bounded(cap)`. + - Until Task 8B, Trusted Server's Fastly delivery converts the response body + with `into_stream()`, so the attached owner lives until the body ends. + - O19 uses `buffered(..)` with a 64-byte cap and `Bypass`. On Fastly it + also sets `.deadline`. +- [ ] **Step 5: Write the runtime tests.** Test 2 adds + `spin_runtime_post_auction_with_json_body`, + `spin_runtime_publisher_post_body_reaches_origin`, and + `spin_runtime_head_root`. Test 6 adds + `navigation_bids_include_180_and_320_ms_and_exclude_600_ms_{fastly,axum,cloudflare,spin}`, + with the overlapped path on Fastly and the plain path elsewhere. Test 7 + adds the seven Viceroy encoded-passthrough cases. +- [ ] **Step 6: Verify.** Run every adapter test and the clippy aliases. Run + the integration suite with `--include-ignored`, the HTML EOF checks in + Spec "Runtime", and `scripts/template-cache-local-test.sh`. Expected: + exit 0. The Task 2D red set is now green. +- [ ] **Step 7: Commit and push.** Commit with "Fetch the publisher origin + through EdgeZero's client inside the response scope", then push. + +### Task 8A: Async store writes and the management API (Spec §2.9) + +**Files:** + +- Modify: `crates/trusted-server-core/src/platform/traits.rs` + (`PlatformConfigStore::put` and `delete`, `PlatformSecretStore::create` and + `delete` become `async` under `#[async_trait(?Send)]`) +- Modify: all 37 implementers, found with + `git grep -nE 'impl (PlatformConfigStore|PlatformSecretStore) for'`, and + their callers in `request_signing/rotation.rs` and the admin handlers +- Modify: `crates/trusted-server-adapter-fastly/src/management_api.rs`, + `platform.rs` +- Modify: `docs/guide/integration-guide.md` (the snippet's store writes), + `crates/trusted-server-integration-tests/tests/documentation_snippets.rs` + (adds `async-trait` at the workspace requirement) + +**Interfaces:** + +- `FastlyManagementApiClient::make_request(&self, client: &HttpClient, method: Method, path: &str, body: Option) -> Result>`, + with `buffered(.., BUFFERED_TIMEOUT, 1 MiB, Decode)`. + +- [ ] **Step 1: Write the failing test (test 14).** + `rotation_sends_expected_method_headers_and_body` uses the scripted client. + `non_2xx_management_response_truncates_body_to_200_chars`. +- [ ] **Step 2: Implement.** Make the four trait methods `async`. Update + every implementer and caller with `.await`. The Fastly implementations + hold the request's `HttpClient`. +- [ ] **Step 3: Verify.** Run every adapter test, the clippy aliases, and + `cargo test --manifest-path crates/trusted-server-integration-tests/Cargo.toml --test documentation_snippets`. + Expected: exit 0. +- [ ] **Step 4: Commit.** Commit with "Make platform store writes async and + send management calls through EdgeZero". + +### Task 8B: Fastly canonical lifecycle and post-delivery work (Spec §3, §2.8) + +**Files:** + +- Modify: `crates/trusted-server-adapter-fastly/src/main.rs` (the callback) +- Create: `crates/trusted-server-adapter-fastly/src/delivery.rs`, + `crates/trusted-server-adapter-fastly/src/post_delivery.rs` +- Modify: `app.rs` (delete the hand-built `App` and `startup_error_router`), + `middleware.rs` (the per-request synthesized set), `tinybird.rs`, `compat.rs` + (delete `to_fastly_response*` and `stream_to_client`) +- Modify: `crates/trusted-server-core/src/proxy.rs` (delete + `stream_asset_body`) +- Modify: `crates/trusted-server-adapter-fastly/Cargo.toml`, adding the + `test-utils` dev feature on `edgezero-adapter-fastly` + +**Interfaces:** + +- `pub(crate) struct FinalizeState { partner_registry: Arc, ec_state: EcFinalizeState }` +- `pub(crate) fn handle_delivery(delivery: FastlyRequestDelivery, counters: &SandboxCounters, post: impl FnOnce(S))`. + It implements the Spec §3 outcome table, with one arm per row. +- `pub(crate) type TelemetryQueue = Arc>>` +- `pub(crate) struct QueueingTelemetrySink`, which implements today's + telemetry sink trait by pushing NDJSON rows. +- `pub(crate) fn run_post_delivery(state: FinalizeState, queue: TelemetryQueue, settings: &Settings)`. + It sends the telemetry rows first, then runs pull sync with + `HttpClient::with_client(FastlyOutboundClient::new()).with_proxy_marker(false)` + and `MonotonicClock::default()`, under `futures::executor::block_on`. +- `fn per_request_synthesized_headers(settings: &Settings, geo: Option<&GeoInfo>) -> SynthesizedResponseHeaders` + in `middleware.rs`. It leaves out invalid values with a warning, and falls + back to the version headers on a bounds error. + +- [ ] **Step 1: Write the failing outcome tests (test 9).** With + `delivery::testing::{deliver_with_hooks, response_delivery}`, add one + test per outcome-table row, named + `delivery__`, for example + `delivery_routed_committed_application_body_runs_post_delivery`. Also + add `assembly_error_delivery_is_handled_like_any_other`, and + `unconvertible_request_head_gets_native_500` (EZ-17 criterion 7), if + Viceroy can deliver such a head. If it cannot, record that in the PR + body. +- [ ] **Step 2: Write the failing post-delivery tests (test 13).** + - `telemetry_rows_queue_during_request_and_send_after_delivery` + - `no_post_delivery_when_not_committed_or_fallback_body` + - `pull_sync_group_ends_at_fifteen_second_cutoff` + - `axum_cloudflare_and_spin_send_no_telemetry`, using the no-op sinks +- [ ] **Step 3: Implement the callback.** + - Native short-circuits. + - Assembly with `App::from_router`. + - The native request work. + - `send_request_with_registries_and_hooks(app, TrustedServerApp::stores(), req, prepare, finalize)`. + - Outcome handling. + - Post-delivery work. + - `prepare` and `finalize` follow Spec §3 exactly, including the + `TraceTerminalResponse` early return. + - Delete the `oneshot` dispatch, `into_core_request`, + `edge_error_response`, `send_edgezero_response`, + `send_response_to_client`, `open_trusted_server_config_store`, and + `ConfigStoreHandle`. +- [ ] **Step 4: Write the Viceroy tests.** These go in `integration.rs` or + `cargo test-fastly`, as the existing EC lifecycle tests do: + - `finalize_sets_ec_cookies_geo_headers_cache_policy_and_counters` + - `trace_response_passes_untouched` + - `reused_sandbox_keeps_serving_after_delivery_failure`, under + `cargo test-fastly-reuse` + - `missing_config_blob_gets_503_then_recovers_in_same_sandbox` (EZ-22 + criterion 9), under `cargo test-fastly-reuse` + - `head_limit_414_431_and_framing_500_carry_synthesized_headers` (test 11). + It uses a `test-utils`-only route that answers with `Trailer`, and its + 431 case uses 60 header lines totaling about 72 KiB. + - `realistic_first_party_proxy_request_is_admitted_{fastly,cloudflare,axum}` + (test 11) + - `publisher_page_streaming_35_seconds_completes` (test 17) + - `pull_sync_reaches_partner_after_origin_aborts_mid_body` + - `framing_fallback_delivery_makes_no_pull_sync_call` (EZ-17 criterion 7) +- [ ] **Step 5: Verify.** + - Run `cargo test-fastly`, `cargo test-fastly-reuse`, `cargo clippy-fastly`, + and the integration suite. + - Run `./scripts/async-html-finalization-local-test.sh fastly --release --auction`. + - Run `scripts/fastly-response-diff.sh --base --head HEAD`, + with `expected.json` filled in from Fastly differences items 1 to 24 for + the requests that exercise them. + + Expected: exit 0. + +- [ ] **Step 6: Commit and push.** Commit with "Run Fastly on EdgeZero's + canonical lifecycle", then push. + +### Task 9: Delete the old outbound layer and port the deadline harnesses (Spec §2.10, §8) + +**Files:** + +- Delete: `crates/trusted-server-core/src/platform/{http.rs,backend_naming.rs,pending_response.rs,image_optimizer.rs}`, + the HTTP doubles in `platform/test_support.rs`, + `crates/trusted-server-adapter-fastly/src/{backend.rs,platform/deadline.rs}`, + the four adapters' outbound clients, the Spin decoder, and the Cloudflare + encoding strip +- Modify: `crates/trusted-server-core/src/platform/{traits.rs,types.rs,mod.rs}`, + removing `http_client`, `backend`, and the three capability methods with + their 39 call lines +- Create: `crates/trusted-server-adapter-fastly/src/deadline_network.rs` +- Modify: `crates/trusted-server-adapter-fastly/tests/deadline-origin.mjs` + (the inline `deadline_fixture` config store, the filter, and the count) +- Modify: `crates/trusted-server-adapter-cloudflare/tests/runtime/deadline.rs`, + `crates/trusted-server-adapter-spin/tests/runtime/deadline.rs` +- Modify: `clippy.toml` (the `disallowed-methods` entry for + `edgezero_core::body::Body::into_bytes`), the crate roots (the `cfg_attr` + allow), and each `tests/*.rs` root +- Modify: `crates/trusted-server-core/src/integrations/registry.rs:264` + (the doc comment) +- Modify: `docs/guide/integration-guide.md` (the snippet uses + `outbound_client` and `clock`) + +**Interfaces:** Deletes everything in the Spec "Dependents inventory" pattern +except the four guide documents. + +- [ ] **Step 1: Port the Fastly deadline cases** into `deadline_network::tests` + with these names: + - `network_stalled_headers_end_at_cutoff` + - `network_trickling_body_ends_at_cutoff` + - `network_head_skips_declared_length` + - `network_fast_sibling_completes` + - `network_expired_snapshot_drops_reply_landed_during_sleep` + - `network_reply_observed_during_wait_before_cutoff_is_kept` + + Each case builds `HttpClient::with_client(FastlyOutboundClient::new())` with + `http://127.0.0.1:` URIs read from the `deadline_fixture` store. + +- [ ] **Step 2: Port the Cloudflare and Spin `deadline-runtime` examples** to + `start_batch_until` and `send`. Keep every case in + `scripts/test-worker-auction-deadlines.mjs`. +- [ ] **Step 3: Delete the old layer and its doubles,** then add the clippy + guard. First move every remaining user in the Spec "Ported tests" + paragraph to the scripted client. That covers the users of + `StubHttpClient`, `NoopHttpClient`, and `StubBackend`, and the bespoke + doubles it lists by line. `body_as_reader` returns an error for a stream, and the two + production uses get a scoped `#[allow]` with a comment (Spec §4, + "Guard"). +- [ ] **Step 4: Verify.** + - Run the Spec "Dependents inventory" command. Expected: it prints only + `docs/guide/architecture.md`, `docs/guide/auction-orchestration.md`, + `docs/guide/integration-guide.md`, and + `crates/trusted-server-core/src/auction/README.md`. + - Run the scoped `transport_timeout_ms` grep. Expected: it prints nothing. + - Run every adapter test and clippy alias, `deadline-origin.mjs`, the + worker deadline job's build and run commands, and + `cargo test --manifest-path crates/trusted-server-integration-tests/Cargo.toml --test documentation_snippets`. + Expected: exit 0. +- [ ] **Step 5: Commit and push.** Commit with "Delete Trusted Server's outbound + layer", then push. + +### Task 10: Egress validation, Axum ingress variables, and the observation log (Spec §5, §8, §3, D10, D16) + +**Files:** + +- Modify: `crates/trusted-server-core/src/config.rs`, + `response_privacy.rs` (`apply_response_headers_with_cache_privacy`), + `publisher.rs` (`make_response_bodiless`, and its test at `:26560-26575`) +- Modify: `crates/trusted-server-adapter-axum/src/main.rs` (pass + `AxumIngressConfig::from_env(&EnvConfig::from_env())?` as `ingress`) +- Modify: `scripts/smoke-axum.sh:101-111` +- Modify: `crates/trusted-server-adapter-fastly/src/main.rs` (log + `FastlyOutboundObservation::current()` at info on the target + `trusted_server::outbound_observation`, after post-delivery work) + +**Interfaces:** + +- `const FORBIDDEN_OPERATOR_RESPONSE_HEADERS: [&str; 10]` in + `response_privacy.rs`: `trailer`, `content-length`, `connection`, + `keep-alive`, `te`, `transfer-encoding`, `upgrade`, `proxy-connection`, + `proxy-authenticate`, and `proxy-authorization`. + +- [ ] **Step 1: Write the failing tests (tests 15 and 16).** + - `make_response_bodiless_removes_trailer_for_head_and_304` + - `validate_rejects_operator_trailer_and_connection_entries` + - `runtime_skips_malformed_connection_entry_and_serves_real_status` + - `runtime_skip_of_connection_x_example_serves_x_example` + - `publisher_head_and_conditional_304_return_real_status_{fastly,axum,cloudflare,spin}` + - `axum_ingress_variable_raises_limit` + - `axum_invalid_ingress_variable_fails_startup` +- [ ] **Step 2: Implement** the changes above. +- [ ] **Step 3: Verify.** Run every adapter test and clippy alias, + `scripts/smoke-axum.sh`, and the integration cases. Expected: exit 0. +- [ ] **Step 4: Commit.** Commit with "Reject operator framing headers and + read Axum ingress limits from the environment". + +### Task 11: Documentation and `CHANGELOG.md` (Spec §10) + +**Files:** Modify the Spec §10 list: + +- `docs/guide/architecture.md` +- `docs/guide/auction-orchestration.md` +- the prose after the snippet in `docs/guide/integration-guide.md` +- the guide pages on the EdgeZero version, outbound behavior, redirects, + timeouts, header rules, inbound limits, and CLI manifest discovery +- `crates/trusted-server-core/src/auction/README.md` +- `scripts/README.md` +- `CHANGELOG.md` + +- [ ] **Step 1: Rewrite the documents.** +- [ ] **Step 2: Verify.** The Spec "Dependents inventory" command prints + nothing. Run `cd docs && npm run lint && npm run format && npm run build`, + the Markdown Prettier gate (CI gate 8), and + `ts dev lint domains --changed-vs origin/main`. Expected: all exit 0. +- [ ] **Step 3: Commit and push.** Commit with "Document the EdgeZero PR 275 + cut", then push. + +### Task 12: Full verification and the PR handoff + +**Files:** The PR #1506 body. + +- [ ] **Step 1: Run the full suite** at the head: every `AGENTS.md` CI gate + (1 to 15), the Spec "Runtime" list, the HTML EOF checks, the response + diff against the Task 0 merge commit, and the four smoke scripts. + Record each command, exit code, and summary line. +- [ ] **Step 2: Update the PR body.** Add `PRfection phase: implementation`, + `Closes #1507` (with `Closes #1513` kept), the plan link, the pinned + `EZ_REV`, the verification record, and the hosted legs the maintainer + runs by hand (Spec "EdgeZero qualification"). Also list the Spec "Before + deploy" steps the maintainer runs before production: `ts config validate` + for each environment, and the response diff's read-only staging subset. +- [ ] **Step 3: Read the PR back.** Confirm the head SHA, that `isDraft` is + unchanged, and that every check is reported. Ask the user before marking + the PR ready for review. diff --git a/docs/superpowers/specs/2026-10-17-edgezero-pr275-hard-cutover-design.md b/docs/superpowers/specs/2026-10-17-edgezero-pr275-hard-cutover-design.md new file mode 100644 index 0000000000..d9d8ccc00f --- /dev/null +++ b/docs/superpowers/specs/2026-10-17-edgezero-pr275-hard-cutover-design.md @@ -0,0 +1,2749 @@ +# EdgeZero PR 275 Hard Cutover + +**Date:** 2026-10-17, rewritten 2026-11-05, updated 2026-11-06, 2026-11-07, +2026-11-08 and 2026-11-11 + +**Status:** Approved at `85f5ca3d9`, with the corrections the implementation +found folded in since. Five agent review rounds, then a 2026-11-06 update +against PR 275 head `9b31a697` and its review, and the 2026-11-07 fixes for +GitHub reviews 5810751514 and 5815168335. The maintainer answered Q1 to Q7 on +2026-11-08 ([Maintainer answers](#maintainer-answers)). + +**EdgeZero target:** `IABTechLab/edgezero` PR 275 (branch +`docs/outbound-http-spec`). This spec was checked against head +`9b31a69760255c279f163a3e56bc95a7eed228e3`, called **H** below. Earlier drafts +used `8f82547b`. The commits since then change Cloudflare response egress, +Cloudflare tests, documentation, and CI. They change no public API and no line +an earlier draft cited. The Fastly guide's same-name paragraph, which Fastly +differences item 12 cites, is new in `9b31a697`. The cut is stacked on PR 275 +and follows its head, and `main` pins the head the cut was last rebased onto +(D5, [Merge gate and pin](#merge-gate-and-pin)). On 2026-11-08 the head was +`5ea8d93c`, and the 2026-11-08 revision rechecked at `5ea8d93c` only the facts +the answers rely on. The cut now pins `14a6ca8b` +(`14a6ca8bb5a5cbe055a1f725bdab8f42569a3bf0`), PR 275's head on 2026-11-11. It +is four commits after `a7d7fd0f`, the previous pin, which was five commits +after `dcbf622`, the cut's first pin, and changed only the absolute target of +the Axum native timer in production code. The four commits change production +code in two places: Spin frames an empty outbound body only for `POST`, `PUT`, +and `PATCH`, and EdgeZero adds a received-head preparer that no adapter calls +yet. Every other citation is still at H. + +**Trusted Server base:** `main` at `51730bdaf`, which pins EdgeZero `v0.0.11`. + +**Requirements:** +[EdgeZero requirements for the PR 275 hard cutover](2026-10-17-edgezero-pr275-requirements.md), +requirements EZ-1 to EZ-25 and the gate EZ-Q1. That file is the copy Trusted +Server handed over. PR 275 carries EdgeZero's copy, which has been amended since, +including its 2026-10-24 Scoped Amendment and later criteria changes. EdgeZero +is measured against its own copy, and this spec cites that copy's criterion +numbers. + +**Tracking:** #1505, the Auctioneer integration epic linked with +IABTechLab/auctioneer#8. This cut is its first scope item. #840 tracks the +wider EdgeZero migration. + +Citations: `crates/...:` is Trusted Server at `51730bdaf`. +`edgezero::` is EdgeZero at H. `edgezero@5ea8d93c:` is PR 275's +head on 2026-11-08. `edgezero@v0.0.11:` is the current pin. + +## Problem + +Trusted Server pins the six EdgeZero crates to `tag = "v0.0.11"` +(`Cargo.toml:59-64`). PR 275 replaces EdgeZero's request lifecycle and adds a +portable outbound HTTP client, with no compatibility layer. Between the earlier +head `f6c708cc` and H, EdgeZero built the nine blocking requirements Trusted +Server asked for (EZ-1 to EZ-9) and most of EZ-10 and EZ-16 to EZ-23. A +requirement-by-requirement review of H on 2026-11-06 found five blocking +defects, 30 should-fix findings, and 5 nits, mostly documentation +([Open EdgeZero defects at H](#open-edgezero-defects-at-h)). + +Repinning `main` to H without code changes fails to compile on every target. A +compile probe at `8f82547b` found 167 error sites under nine API changes: 63 in +core, 38 in the Fastly adapter, 34 in Spin, 22 in Cloudflare, and 10 in Axum. That count is +a floor, because rustc stops at type checking. + +Three behavior changes survive the mechanical fixes and break Trusted Server at +runtime. + +- Inbound bodies arrive as deadline-bounded streams on every adapter, Fastly + included (`edgezero:crates/edgezero-adapter-fastly/src/request.rs:857-876`). + `into_bytes().unwrap_or_default()` then reads an empty body, so + `POST /auction` and EC batch sync parse nothing + (`crates/trusted-server-core/src/auction/endpoints.rs:153`, + `crates/trusted-server-core/src/ec/batch_sync.rs:152`). `request_body_bytes` + rejects the stream with 400, which breaks first-party sign and rebuild, + request signing, and cache purge. +- The Fastly, Cloudflare, and Spin outbound clients reject streamed request + bodies (`crates/trusted-server-adapter-fastly/src/platform.rs:366-368`, + `crates/trusted-server-adapter-cloudflare/src/platform.rs:330-335`, + `crates/trusted-server-adapter-spin/src/platform.rs:514-519`), so publisher + forwards fail on three adapters. Axum's client buffers them. +- Every response passes EdgeZero's egress framing, Fastly included once it runs + the canonical lifecycle. Any `Trailer` header becomes a 500 + (`edgezero:crates/edgezero-core/src/response_egress_framing.rs:127-128`). + `make_response_bodiless` keeps an origin `Trailer` on some HEAD and 304 + publisher responses (`crates/trusted-server-core/src/publisher.rs:2970-3010`). + After the cut, H strips `Trailer` from upstream heads + (`edgezero:crates/edgezero-core/src/outbound.rs:2211-2228`), so the remaining + sources are operator config and Trusted Server's own code. + +The maintainer decided the cut adopts PR 275 completely. Trusted Server deletes +its own outbound layer, Fastly moves onto EdgeZero's lifecycle, and EdgeZero +follows redirects. This spec maps every site onto H. It names the places where +H cannot express today's behavior and says what Trusted Server gives up there +until EdgeZero's follow-ups land. + +## Goals + +- Move all six EdgeZero crates to PR 275 in one pull request, with no shim, + dual path, or feature flag. +- Send every outbound request through EdgeZero's `HttpClient`. Delete + `PlatformHttpClient`, `PlatformBackend`, backend naming, the four outbound + clients, the Spin decoder, the Cloudflare encoding strip, the redirect loop, + and Trusted Server's Fastly response delivery. +- Run Fastly on EdgeZero's canonical lifecycle inside Trusted Server's sandbox + loop, with Trusted Server logic around and inside the `prepare` and + `finalize` hooks. +- Keep Fastly responses, auction semantics, consent, and identity as they are + today, apart from the changes in [Fastly differences](#fastly-differences). +- Read every inbound body through a bounded EdgeZero reader, with today's + per-endpoint caps. +- Produce Trusted Server's share of EdgeZero's qualification evidence (EZ-Q1). + The maintainer runs the hosted legs by hand. Neither the merge nor production + deploys wait for EdgeZero's EZ-Q1 record (D5). + +## Non-goals + +- Building what H lacks: surrogate keys (EZ-11), a certificate-check mode + (EZ-12), a Spin host override (EZ-13), an idle bound for streamed bodies + (EZ-14), fire-and-forget sends (EZ-15), and a published test double (EZ-24). + Trusted Server takes the interim each requirement names. It does not build + its own version. +- Reusable sandboxes in production. The `reusable-sandbox` feature stays off by + default (`crates/trusted-server-adapter-fastly/Cargo.toml:13-18`). EZ-2's + reuse-mode parts are follow-ups. +- Streaming publisher responses on Axum, Cloudflare, and Spin. They keep + collecting every publisher response variant before egress, raw ones included, + with `buffer_publisher_response_async` + (`crates/trusted-server-core/src/publisher.rs:1853`) (2.5). +- Declaring stores on Cloudflare and Spin. Their apps declare none today, and + their config and KV fallbacks stay (section 7). +- Toolchain changes. Rust 1.98.1, Viceroy 0.21.0, Fastly CLI 16.0.0, Node + 24.20.0, and Spin 4.1.0 already match H. +- Moving Spin from `wasm32-wasip1` to `wasm32-wasip2`. +- Changing consent, identity, auction, or HTML rewriting behavior. +- Replacing `PlatformTimer` with EdgeZero's clock. The HTML end-of-file budget + keeps using it + (`crates/trusted-server-core/src/publisher/html_finalization.rs:315`). + +## Decisions + +| # | Decision | Owner | +| --- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ | -------------------------------------------------- | +| D1 | Hard cut onto PR 275's APIs in one pull request. No compatibility layer, shim, dual path, or feature flag. | Maintainer | +| D2 | No Trusted Server transport plumbing. Every outbound request uses EdgeZero's `HttpClient`. Where H lacks a feature, Trusted Server takes the requirement's interim. | Maintainer | +| D3 | Fastly calls `send_request_with_registries_and_hooks` inside the `run_custom` and `serve_custom` loop. Trusted Server logic moves into the callback around it and into `prepare` and `finalize`. | Maintainer | +| D4 | Redirect following uses `HttpClient::follow` (EZ-9). | Maintainer | +| D5 | The cut is stacked on PR 275. It pins PR 275's head and repins at each rebase. `main` merges it without waiting for EdgeZero's EZ-Q1 record and pins the head the cut was last rebased onto. Production deploys do not wait for qualification. | Maintainer (Q1, Q5) | +| D6 | Requests use the adapter's default client with the proxy marker off. Fastly post-delivery work uses a fresh EdgeZero Fastly client. | Spec author | +| D7 | Every request sets a constant 1 s connect timeout and a per-call `timeout` that never depends on a remaining budget. It is the site's configured timeout where the site map names one, 2 s for Tinybird, and otherwise 60 s streamed and 15 s buffered. Redirect chains get a 60 s deadline. The response write deadline is 60 s after egress starts. Each value is Trusted Server's to set, and EdgeZero exposes every knob for it. | Spec author, confirmed by the maintainer (Q2) | +| D8 | Sites that relay bytes or run Trusted Server's own rewrite codec use `ResponseDecoding::Original`. Sites that parse a body Trusted Server does not decode itself use `Decode`. One deviation: the Cloudflare publisher origin uses `Decode` (2.5). | Spec author; Cloudflare deviation: maintainer | +| D9 | `ts config validate` and `push` reject `certificate_check = false` and `origin_readthrough_enabled = true`. At runtime both are ignored with a warning. Spin ignores `origin_host_header_override` with a warning, which deviates from the EZ-13 interim (Configuration explains why). | Spec author, from the interims, with one deviation | +| D10 | Trusted Server never sends `Trailer`, and never sends `Content-Length` or a hop-by-hop field from operator config. Validate and push reject such operator entries. Runtime skips them with a warning. | Spec author | +| D11 | Fastly auction telemetry is queued during the request and sent after delivery inside the serving callback (EZ-15 interim). | Spec author, from the EZ-15 interim | +| D12 | Tests use a scripted `OutboundHttpClient` in Trusted Server test code until EZ-24 lands. | Spec author, from the EZ-24 interim | +| D13 | EZ-2 measurements baseline on the `main` commit the cut branch is based on when the maintainer runs them, instead of `a9037770a` plus #1123. EdgeZero acknowledges the change before its hosted phase. | Spec author | +| D14 | A Spin component that fails to instantiate blocks the cut. CI's Spin job proves it. | Spec author | +| D15 | Framework responses carry `x-ts-version` through EdgeZero's synthesized response headers (EZ-19), where EdgeZero applies them. On Fastly, `prepare` supplies a per-request set that adds the service version, environment, geo, and operator headers (EZ-19.2). | Spec author | +| D16 | The Axum dev server reads `EDGEZERO__ADAPTER__INGRESS__*` from the environment. | Spec author | +| D17 | Upstream failures keep today's statuses. Only the statuses the cut introduces on purpose (413, 408, and the publisher-forward 400) change. | Spec author | +| D18 | Provider endpoint validation also accepts plain `http` for the loopback IP literals `127.0.0.1` and `[::1]`, so test fixtures reach local bidder stubs. | Maintainer (Q6) | +| D19 | Trusted Server confirms as present the follow-ups the cut uses: EZ-10, EZ-16, EZ-17, EZ-18, EZ-19, EZ-20, EZ-22, and EZ-23. The criteria of their present parts join the gate (EZ-Q1.6), and the record maps conditional legs (EZ-Q1.8). EZ-18.4's bodiless part follows D20. EZ-22.6's Cloudflare and Spin retention recipe is not adopted, so EZ-22 criterion 10 is deferred. EZ-13 and EZ-21 are not confirmed, so their criteria stay out of the gate. EZ-21's interim applies, and D9 replaces EZ-13's. | Spec author | +| D20 | Accept EdgeZero's strict suppression of bodiless Fastly responses (HEAD, 204, 205, 304) in place of SDK framing parity. Trusted Server withdraws the bodiless part of EZ-18.4. EdgeZero amended EZ-18.4, EZ-18.6, and EZ-18 criteria 3 and 6 to match in `dcbf622`. | Maintainer (Q7) | + +D1 to D4 are settled and not open for review. The rationale for each other +decision sits in the section that implements it. + +## Design + +### 1. Pin, lockfile, manifests, and fixtures + +**Pin.** The six EdgeZero crates move from `tag = "v0.0.11"` to a `rev` on the +same `IABTechLab/edgezero` git source. Features and `default-features` stay as +they are. The rev is PR 275's head, `14a6ca8b` at the 2026-11-11 repin, and +the cut repins it at each rebase. At merge, `main` pins the head the cut was +last rebased onto (D5). + +**spin-sdk.** Trusted Server's `spin-sdk` must come from the same source as +EdgeZero's: `version = "=7.0.0"` from the `stackpop/spin-rust-sdk` git fork at +rev `6c59ed5b4fcb377072763aee0b27a3abdd8edb72` (`edgezero:Cargo.toml:106`). +With crates.io `~7.0` (`Cargo.toml:111`) the graph holds two `spin-sdk` 7.0.0 +packages, and `check-spin` fails with E0308 at +`crates/trusted-server-adapter-spin/src/lib.rs:18` and E0277 at `:15`. The +probe confirmed that both errors clear after the switch. + +**Lockfile.** H's 26 exact pins move these entries: + +| Crate | Lock today | After | Why | +| --------------------- | ---------- | ------ | ------------------------------------- | +| `async-compression` | 0.4.42 | 0.4.43 | `=0.4.43` | +| `brotli-decompressor` | 5.0.3 | 5.0.1 | `=5.0.1`, a downgrade | +| `bytes` | 1.12.0 | 1.12.1 | `=1.12.1` | +| `hyper` | 1.10.1 | 1.12.0 | `=1.12.0` | +| `want` | 0.3.1 | 0.3.2 | required by hyper 1.12.0 | +| `syn` | 3.0.4 | 3.0.9 | required by the fork's `wit-bindgen` | +| `spin-sdk` stack | crates.io | git | the fork brings `wasip3` 0.10.0 along | + +A scoped `cargo update -p edgezero-core` also rewrote the `colored` 3.1.1 and +`winapi-util` 0.1.11 edges from `windows-sys` 0.61.2 to 0.48.0, which no pin +requires. Restore those by hand and validate with `--locked`. The probe's final +lock removes 25 packages, adds 27, and changes 3 edges, all explained. The +`esi` patch stays valid, because it accepts `fastly ^0.13`. + +While Trusted Server pins PR 275, H's exact pins are version ceilings. They +include `serde_json`, `bytes`, `http`, `url`, `hyper`, the `fastly` crates at +0.13.1, `worker` 0.8.5, `reqwest` 0.13.4, and the compression crates. +Dependabot updates of those fail resolution. Ignore rules are a follow-up. + +**`edgezero.toml`.** Add: + +```toml +[capabilities.outbound] +hosts = ["*"] +``` + +Without it, `ts build`, `ts serve`, and `ts deploy --adapter spin` fail with +"Spin outbound host drift". An absent list means HTTPS only +(`edgezero:crates/edgezero-core/src/manifest.rs:1152-1169`), while +`crates/trusted-server-adapter-spin/spin.toml:43` allows HTTP and HTTPS. `"*"` +expands to both (`manifest.rs:1106-1120`). Declare no required or optional +capabilities. The manifest now rejects unknown top-level keys +(`manifest.rs:343-376`), and Trusted Server's has none. + +**Wrangler files.** Add `enable_weak_ref` and `enable_request_signal` to +`compatibility_flags` in `crates/trusted-server-adapter-cloudflare/wrangler.toml:12` +and `wrangler.ci.toml:6`, as H's template does +(`edgezero:crates/edgezero-adapter-cloudflare/src/templates/wrangler.toml.hbs:4`). +EdgeZero's guide says its bounded config and KV reads need the first +(`edgezero:docs/guide/adapters/cloudflare.md:38-42`). Disconnect observation +needs the second. `cache_option_enabled` stays, because H's Cloudflare client +sets the fetch `cache` field for `Bypass`. + +**Certificate check in fixtures and CI.** The three integration fixture configs +(`crates/trusted-server-integration-tests/fixtures/configs/trusted-server.integration.toml:96-97`, +`trusted-server.trace.toml:38`, `trusted-server.trace-auth.toml:37`), the two +integration scripts (`scripts/integration-tests.sh:76`, +`scripts/integration-tests-browser.sh:39`), and the CI build steps +(`.github/workflows/test.yml:282`, +`.github/actions/setup-integration-test-env/action.yml:94`, `:104`, `:135`) +drop `certificate_check = false`. Their local origins use plain HTTP, so the +setting has no effect there today, and nothing reads it at build time. D9 +rejects it. The doc comment at `crates/trusted-server-core/src/settings.rs:1916` +changes to "Must be true. Origin TLS is always verified." + +**Test bidders (D18).** Today CI reaches local bidder stubs through a Viceroy +alias. `generate-viceroy-config` predicts Trusted Server's Fastly backend names +and registers static backends under them that point at the loopback origin +(`crates/trusted-server-integration-tests/src/bin/generate-viceroy-config.rs:118-196`). +Trusted Server's dynamic registration then gets `NameInUse` and treats it as +reuse (`crates/trusted-server-adapter-fastly/src/backend.rs:262-265`). On H the +alias cannot work. Backend names hash private identity fields +(`edgezero:crates/edgezero-adapter-fastly/src/outbound.rs:1009-1023`), an +unknown `NameInUse` fails the request (`outbound.rs:1132-1134`), the naming code +the generator compiles against is deleted, and HTTPS is always verified. +Provider endpoints must be HTTPS today +(`crates/trusted-server-core/src/auction/plan.rs:565-584`). + +- Endpoint validation also accepts `http://127.0.0.1:` and + `http://[::1]:`. Hosted runtimes cannot reach loopback, so production + behavior does not change. +- `--bidder-origin-url` becomes `--provider-endpoint-origin `. When it is + given, the generator replaces the scheme and authority of every + `auction.providers.*.endpoint` and keeps the path and query. It stops + injecting backend aliases, so `inject_controlled_bidder_backends`, its + `BackendNamingPolicy` import, and its tests + (`generate-viceroy-config.rs:370-468`) are deleted. + `INTEGRATION_BIDDER_ORIGIN_URL` stays and sets the flag. Only the two trace + runs set it (`.github/workflows/integration-tests.yml:39-40`, + `scripts/generate-integration-viceroy-configs.sh:45-46`, + `scripts/integration-tests-browser.sh:45-50`), so the integration config's + providers keep their endpoints. +- The Next.js trace browser test runs on Viceroy only. It adds + `/api/trace-bidder` to its forbidden evidence list + (`crates/trusted-server-integration-tests/browser/tests/nextjs/mobile-trace-live.spec.ts:381-389`), + because the rewritten endpoint no longer contains `bidder.example.com`. +- `scripts/template-cache-local-test.sh`, which the required `cargo test` job + runs (`.github/workflows/test.yml:69-76`), serves its stub bidder over plain + HTTP on `127.0.0.1` and sets the `pbs-main` endpoint to + `http://127.0.0.1:{bid_port}/bid`. Its CA generation, `openssl` check, TLS + socket wrap, and backend pre-registration under Trusted Server's predicted + name (`:82-120`, `:239`, `:350-391`) are deleted. The `hb_pb` assertions stay. +- Tests 5 and 6, EZ-3 criterion 6, and EZ-6's Trusted Server leg configure + loopback bidders the same way on every runtime. + +### 2. Outbound HTTP on EdgeZero's client + +#### 2.1 One client per request + +`RuntimeServices` drops its `http_client` and `backend` fields and their +builders (`crates/trusted-server-core/src/platform/types.rs:182-184`, +`:258-268`, `:418-430`). It gains two required fields, +`outbound_client: HttpClient` and `clock: MonotonicClock`. A new field name +keeps the old and new clients apart while sites are ported (commit sequence). +Both types are `Send + Sync` +(`edgezero:crates/edgezero-core/src/outbound.rs:419-422`, +`edgezero:crates/edgezero-core/src/time.rs:28-31`), which matters because +`RuntimeServices` travels in response extensions. + +Each adapter's per-request service builder fills them from +`ctx.http_client()` (`edgezero:crates/edgezero-core/src/context.rs:570`) with +`.with_proxy_marker(false)` (`outbound.rs:806`), and from +`ctx.monotonic_clock()` (`context.rs:691`) (D6). Every adapter inserts a default +client (`edgezero:crates/edgezero-adapter-fastly/src/request.rs:851-853`, +`edgezero:crates/edgezero-adapter-axum/src/request.rs:93`, +`edgezero:crates/edgezero-adapter-cloudflare/src/request.rs:380`, +`edgezero:crates/edgezero-adapter-spin/src/request.rs:164`). A missing client is +an internal error, not a panic. + +Tests that drive a router with `oneshot` have no adapter to insert a client. +`ctx.http_client()` reads only request extensions (`context.rs:568-572`), and +`oneshot` adds none (`edgezero:crates/edgezero-core/src/router.rs:662-668`). +H's Cloudflare and Spin clients are wasm-only, so native tests have no default +either. `test_support.rs` gains `insert_outbound_client`, which inserts any +`HttpClient` the test passes into a request's extensions. The route helpers +pass a scripted client, which fails every request the test did not script. +Every `oneshot` route helper uses it: the Fastly `route` helper +(`crates/trusted-server-adapter-fastly/src/app.rs:1497-1510`) and the Cloudflare +and Spin `route` helpers in `tests/routes.rs`. The parity `cf_*` and `spin_*` +helpers (`crates/trusted-server-integration-tests/tests/parity.rs:132-315`) and +the inline Cloudflare request in `discovery_route_body_is_json_parity` +(`parity.rs:371-378`) leave `oneshot` for `AxumTestServer` instead (section 8). +Tests that inject `RuntimeServices` through `routes_with_settings_and_services` +already carry their client. + +Fastly post-delivery work (2.8) runs after the request is gone. It builds a +fresh `HttpClient::with_client(FastlyOutboundClient::new()).with_proxy_marker(false)` +and uses `MonotonicClock::default()`. The adapter default is the same client +type over the app's clock +(`edgezero:crates/edgezero-adapter-fastly/src/request.rs:851-853`), the app +clock is the default because Trusted Server never sets one, and +`FastlyOutboundClient::new()` uses `MonotonicClock::default()` +(`edgezero:crates/edgezero-adapter-fastly/src/outbound.rs:601-611`). Request and +post-delivery deadlines are therefore in one clock domain. EdgeZero's own +post-delivery fixture builds its client the same way +(`edgezero:crates/edgezero-test-harness/fixtures/reusable-app/crates/fixture-fastly/src/bin/outbound_progress/post_delivery.rs:86-103`). + +The marker is off for every call. It is a response header that a relay would +pass to browsers, and Trusted Server never reads it on JSON calls, so one +setting covers both cases (EZ-7). Production code installs no client in router +state, which would override the adapter default +(`edgezero:crates/edgezero-core/src/router.rs:410-416`). Tests on the Axum test +server do, so no real transport starts (EZ-23.7). The exception is test 19's +bodiless GET, which supplies EdgeZero's Axum client. + +#### 2.2 Settings every request carries + +| Today | On H | +| ------------------------------------------------------------------------------------------------ | ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| Backend first-byte and between-bytes timers | `.timeout(d)`, a total budget (D7). Fastly rounds `d` up to a power of two of milliseconds for the provider timer, and the guest enforces `d` itself (`edgezero:crates/edgezero-adapter-fastly/src/outbound.rs:555-563`, `:1025-1031`). A request with only a deadline spreads over up to 31 backend configurations per origin, so every request sets `timeout`, and the value is a constant per site, never a remaining budget. | +| Fixed 1 s Fastly connect timeout (`crates/trusted-server-adapter-fastly/src/backend.rs:240`) | `.connect_timeout(Duration::from_secs(1))` on every request (EZ-4, `outbound.rs:1082`). Fastly backend identity includes the exact connect value in milliseconds (`edgezero:crates/edgezero-adapter-fastly/src/outbound.rs:988-992`, `:1018-1021`), so it never depends on a budget. A 1 s connect bound over a shorter total is harmless, because the guest budget takes the earliest bound (`edgezero:crates/edgezero-core/src/time.rs:186-197`). | +| Transport-timeout canonicalization (`crates/trusted-server-core/src/platform/traits.rs:117-136`) | Deleted. Constant per-site timeouts and a constant connect timeout keep backend identity stable, which is the reuse canonicalization protected. | +| `with_cache_bypass()` | `.cache_policy(OutboundCachePolicy::Bypass)`. Fastly maps it to pass (`edgezero:crates/edgezero-adapter-fastly/src/outbound.rs:1089-1091`). | +| `with_stream_response()` | `.stream_response()`. Sites that only set `with_streaming()` (passthrough) stay buffered, as today. | +| Platform response cap of 10 MiB on Fastly and Cloudflare | `.max_response_bytes(n)` on every buffered site. H defaults to 1 MiB (`outbound.rs:77`). Sites with no Trusted Server cap today set 10 MiB, so they keep today's limit. | +| Encoded bytes on Fastly and Axum | `.response_decoding(..)` per D8. `Original` leaves `Accept-Encoding` as Trusted Server set it (`outbound.rs:1459`). | +| Explicit `Host` header | H strips it before dispatch (`edgezero:crates/edgezero-core/src/outbound/dispatch.rs:101`, `outbound.rs:2229-2232`). A `Host` that differs from the URI authority, compared case-insensitively with default ports removed, becomes `.host_authority_override(..)` (`outbound.rs:1239`). On Fastly an override is part of backend identity (`outbound.rs:999`, `:1011`). Spin rejects any override (`edgezero:crates/edgezero-adapter-spin/src/outbound.rs:1526-1530`). | +| Backend `certificate_check` | No equivalent. Verification is always strict (`edgezero:crates/edgezero-adapter-fastly/src/outbound.rs:650-655`). D9 rejects `false`. | +| Backend names, `ensure`, `predict_name` | Deleted. EdgeZero names Fastly backends `ez_b31v1_` (`outbound.rs:1009-1023`), and batch results are positional. | + +H applies these rules to every request and response at dispatch. They are +listed in [Fastly differences](#fastly-differences). + +- `Host` and `Content-Length` are dropped from requests, and H frames the body. +- The adapter clients add `Accept-Encoding: identity` when the request has none + (`edgezero:crates/edgezero-adapter-fastly/src/outbound.rs:688-692`; Axum, + Cloudflare, and Spin do the same). Core dispatch adds nothing. +- Header values that are not UTF-8 are dropped from requests and responses, + with a warning that names the header (`outbound.rs:2192-2209`, called at + `:2075` and `:1758`). The whole field goes, not one value inside it. +- Hop-by-hop fields, and fields that `Connection` names, are stripped from + requests and responses, `Trailer` included. A malformed `Connection` field is + a 400 on a request and a 502 on a response (`outbound.rs:2080-2100`, + `:2211-2234`). +- The URI is canonicalized as a WHATWG URL. Dot segments are removed, `%2e%2e` + included, and default ports are dropped. Userinfo, fragments, and backslashes + are rejected with 400 (`outbound.rs:1953-1986`, `:2248-2279`). Raw-string + constructors such as `OutboundRequest::get` reject `#` anywhere + (`outbound.rs:2248-2253`), so proxies build their requests with + `OutboundRequest::new(method, uri)` from a parsed `http::Uri`, which drops a + fragment while parsing. Creative click URLs with fragments keep working. + +Sites that copy client headers write into `OutboundRequest::headers_mut()` +(`outbound.rs:1220`), never `header()`, because `header()` rejects a value that +is not UTF-8 and would fail the whole call (`outbound.rs:1191-1211`). + +#### 2.3 Site map + +| Site | Trusted Server code | Call on H | Response | Timeout | +| ---------------------------------- | ----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | -------------------------------------------- | ------------------------------------------------------------------------------ | ------------------------------------------------------------------------------------------------------ | +| O1 Auction bidders | `GenericOpenRtbProvider::request_bids_routed` (`crates/trusted-server-core/src/auction/provider.rs:273-400`) | One `start_batch_until` per auction (2.4) | Buffered, 1 MiB (`provider.rs:30`), Decode | Configured provider timeout | +| O2 Auction mediator | `AdServerMockProvider::request_bids` (`crates/trusted-server-core/src/integrations/adserver_mock.rs:435-523`) | `send` after the batch (2.4) | Buffered 2 MiB (`UPSTREAM_RTB_MAX_RESPONSE_BYTES`, unchanged), Decode | Configured mediator timeout, plus the auction deadline | +| O3 Auction entry points | `/auction` `handle_auction` (`crates/trusted-server-core/src/auction/endpoints.rs:407`), page bids `handle_page_bids` (`crates/trusted-server-core/src/publisher.rs:7634`), and the publisher page path | O1 and O2 | As O1 and O2 | As O1 and O2 | +| O4 Publisher, overlapped | `publisher.rs:5318-5352`, `:5676` | `begin` (2.5) | Streamed, Original | 60 s | +| O5 Publisher, plain | `publisher.rs:5683-5697` | `send` with `from_request` (2.5) | Streamed, Original | 60 s | +| O6 Redirecting proxies | `proxy_request` (`crates/trusted-server-core/src/proxy.rs:795-843`) from the first-party proxy (`proxy.rs:1577-1610`), GTM (`integrations/google_tag_manager.rs:1081`), the Prebid bundle (`integrations/prebid.rs:1281-1288`), and testlight (`integrations/testlight.rs:199-216`) | `follow` (2.6), or `send` when not following | Buffered 10 MiB (testlight 2 MiB), Original | 15 s per hop, 60 s chain deadline | +| O7 GPT | `integrations/gpt.rs:282` | `send` | Buffered 10 MiB, Original | 15 s | +| O8 JS asset proxy | `integrations/js_asset_proxy.rs:509` | `send` | Streamed, Original | 60 s | +| O9 Asset proxy | `handle_asset_proxy_request` (`proxy.rs:1096-1220`), S3 preflight (`proxy.rs:1025-1094`) | `send` (2.7) | Final GET streamed, Original. Final HEAD and preflight HEAD buffered, Original | 60 s GET, 15 s HEAD | +| O10 Sourcepoint | `integrations/sourcepoint.rs:817-960` | `send` | Streamed, Original | 60 s | +| O11 Lockr | SDK GET (`integrations/lockr.rs:108-170`), API proxy (`lockr.rs:179-260`) | `send` | SDK buffered 16 MiB, Decode. API buffered 10 MiB, Original | 15 s | +| O12 Permutive | SDK GET (`integrations/permutive.rs:108-170`), proxy (`permutive.rs:183-260`) | `send` | As Lockr | 15 s | +| O13 DataDome | `tags.js` (`integrations/datadome.rs:633-700`), `js_api` (`datadome.rs:716-800`) | `send` | `tags.js` buffered 16 MiB, Decode. `js_api` 10 MiB, Original | 15 s | +| O14 DataDome Protection | `filter_protection_request_inner` (`integrations/datadome/protection.rs:78-135`) | `send` | Buffered 10 MiB, Decode | `config.timeout_ms` | +| O15 Didomi | `integrations/didomi.rs:407-510` | `send` | Buffered 10 MiB, Original. Bypass or `PlatformDefault` as today | 15 s | +| O16 EC pull sync | `dispatch_pull_sync` (`crates/trusted-server-core/src/ec/pull_sync.rs:116-300`) | `send_all_until` per group (2.8) | Buffered 64 KiB (`pull_sync.rs:410`), Decode | 15 s | +| O17 Tinybird | `crates/trusted-server-adapter-fastly/src/tinybird.rs:133-182` | `send` after delivery (2.8) | Ignored | 2 s, today's timers (`tinybird.rs:19-20`) | +| O18 Fastly management API | `FastlyManagementApiClient::make_request` (`crates/trusted-server-adapter-fastly/src/management_api.rs:158-188`) | `send` from the admin handler (2.9) | Buffered 1 MiB, Decode | 15 s | +| O19 EOF fixture, `test-utils` only | `crates/trusted-server-core/src/test_support/async_html_finalization.rs:126-181` | `send` from the end-of-file finish hook | Buffered 64 bytes, Decode, Bypass | `.timeout(transport_timeout_ms)`. On Fastly also `.deadline` at the remaining chain budget (see below) | + +GTM asks for `identity`, so `Original` and `Decode` behave the same for it +when the origin honors that. The requirements doc lists it under decoding. + +The EOF fixture keeps reporting `min(remaining, transport_timeout_ms)` on +Fastly and `transport_timeout_ms` elsewhere, because +`crates/trusted-server-integration-tests/tests/html_eof_runtime.rs:753-766` +asserts those values. Its phase timers come from the constant timeout, so its +backend identity stays stable. + +Every row keeps today's method and request headers, with the dispatch rules in +2.2 applied. Bidder requests keep their consent and attested client-IP +headers. The DataDome Protection filter runs before routing, and it gets the +client from `RuntimeServices` like every handler. ESI assembly and the template +cache make no HTTP calls. + +#### 2.4 Auction fan-out + +- `ProviderRequestOutcome::Pending { request: PlatformPendingRequest, parse_state }` + (`provider.rs:47-57`) becomes `Ready { request: OutboundRequest, parse_state }`. + Providers build the request and return it unsent. `request_bids_routed` + (`provider.rs:384-393`) and the mediator's `AuctionProvider::request_bids` + (`adserver_mock.rs:513-523`) return `Ready`. + `parse_response_with_context_and_state` takes an `OutboundResponse` + (`crates/trusted-server-core/src/auction/orchestrator.rs:2627-2634`). + `AuctionContext::transport_timeout_ms` is deleted, with its declaration + (`crates/trusted-server-core/src/auction/types.rs:156`) and the test + constructor that sets it (`auction/test_support.rs:30`). APS's planned + parser, `parse_planned_aps_response` + (`crates/trusted-server-core/src/integrations/aps.rs:879-884`), takes an + `OutboundResponse`, like the Prebid parser. +- `dispatch_planned_auction` (`orchestrator.rs:1686`) puts the bidder requests + in `client.start_batch_until(requests, auction_deadline)` (`outbound.rs:734`). + On Fastly the call is synchronous and must hand every slot to the SDK before + it returns (EZ-6.7). At H it stops at the first slot that completes during + setup, so later slots wait for the next EdgeZero poll + ([open defect 1](#open-edgezero-defects-at-h)). Fixed at the pin: EdgeZero's + fix is in `598f0a2a`, an ancestor of `14a6ca8b`, and Trusted Server keeps no + workaround for it. Axum, Cloudflare, and Spin + spawn a local task + (`edgezero:crates/edgezero-adapter-axum/src/outbound.rs:268`, + `edgezero:crates/edgezero-adapter-cloudflare/src/outbound.rs:537`, + `edgezero:crates/edgezero-adapter-spin/src/outbound.rs:622-625`), so their + bidder requests leave at Trusted Server's next await. + `DispatchedAuction` holds the `OutboundBatch` and a slot-to-provider map. + Bidder requests now leave together after all are built and signed. Today each + leaves as soon as it is built (`orchestrator.rs:1839-1850`). +- Bidders set `.timeout` to the configured provider timeout, not to today's + `remaining_budget_ms(..).min(provider.timeout_ms())` + (`orchestrator.rs:1817-1818`). The batch cutoff bounds them by the remaining + budget, so the effective bound is the same and backend identity stays stable + (D7). +- `auction_deadline` is `services.clock.deadline_after(timeout)`, taken at + auction start. It replaces `collection_deadline(auction_start, timeout_ms)` + (`orchestrator.rs:258`). +- `collect_dispatched_auction` replaces its `select_until` loop + (`orchestrator.rs:2286-2289`) with `batch.next()` until the batch reports + `Finished`. Where today's code polls without waiting before the deadline, it + calls `batch.check()` (`edgezero:crates/edgezero-core/src/outbound/batch.rs:430`), + which reports only outcomes an earlier EdgeZero poll recorded. Collection + after the cutoff calls `batch.next()`, which returns `Finished(Cutoff)` + without a host wait. At the pin under Viceroy 0.21.0, a `check` after the + cutoff returns `Finished(Cutoff)` too (section 8). +- A `Cutoff` termination leaves slots `Unresolved`. Those become provider + timeouts in the auction result, as expiry does today since #1433. Collection + maps an `OutboundBatchFailure` to a provider error for every slot it leaves + unresolved. On Spin at H, a batch whose earlier-deadline slot is still + unobserved when the cutoff passes fails that way with an internal error + ([open defect 3](#open-edgezero-defects-at-h)). Trusted Server adds no + workaround. Fixed at the pin: EdgeZero's fix is in `598f0a2a`, an ancestor + of `14a6ca8b`. +- The mediator goes out after the batch finishes, with `client.send(..)`, + `.timeout(mediator timeout)`, and `.deadline(auction_deadline)`. The guest + budget takes the earlier of the two + (`edgezero:crates/edgezero-core/src/time.rs:160-200`). +- Dropping the batch or a pending request cancels it + (`batch.rs:414-423`, `edgezero:crates/edgezero-core/src/outbound/progress.rs:94-104`). + `DispatchedAuction::abandon` (`orchestrator.rs:95-131`), an origin error + (`publisher.rs:5700-5714`), a non-HTML response, or a client that disconnects + mid-stream therefore cancels in-flight bidder and mediator requests. Today + Fastly lets them run to completion. Only the guest side is guaranteed. Fastly + host cancellation is best effort and unqualified + (`edgezero:docs/guide/capabilities.md:285`, `:352`), so a bidder may still receive + and answer a cancelled request. +- `AuctionPlan::validate_for_target` (`crates/trusted-server-core/src/auction/plan.rs:417-466`), + `AuctionTargetId`, and the target descriptors are deleted with + `backend_naming.rs`, with their calls in the four adapters' state builders + (`crates/trusted-server-adapter-fastly/src/app.rs:225`, + `crates/trusted-server-adapter-cloudflare/src/app.rs:208`, + `crates/trusted-server-adapter-axum/src/app.rs:95`, + `crates/trusted-server-adapter-spin/src/app.rs:137`) and the test at + `crates/trusted-server-core/src/auction/mod.rs:198-202`. Settings load no + longer rejects a multi-provider plan on Cloudflare or Spin. The planned + auction also drops its refusal when `!supports_concurrent_fanout()` + (`orchestrator.rs:1700-1709`). All four adapters run batches (EZ-6), so + Cloudflare and Spin now fan out, which closes issue #1484. +- The Fastly state builder keeps today's startup bound of 160 provider + backends. It replaces the call at + `crates/trusted-server-adapter-fastly/src/app.rs:225` with core's + `validate_fastly_backend_bound` (`plan.rs:455`). The limits 200 and 40 live + beside it in core (`crates/trusted-server-core/src/auction/plan.rs:21-23`). + It counts distinct tuples of provider scheme, host, port, and configured + timeout, because each tuple is at most one EdgeZero backend configuration. + The count can only overstate. EdgeZero keys identity on the timer rung, so + two timeouts that round to one rung share a configuration but count twice. + Bidders set no host override, and the connect timeout is constant. + The predicted-name collision check (`plan.rs:451-464`) goes, because EdgeZero + owns names. The Cloudflare and Spin tests that rejected the plan became + `cloudflare_and_spin_load_multi_provider_plan` + (`crates/trusted-server-adapter-cloudflare/src/app.rs:941`, + `crates/trusted-server-adapter-spin/src/app.rs:1105`), which assert that the + plan loads. The `validate_for_target` tests (`plan.rs:695-830`) and the four + target-descriptor tests (`platform.rs` in each adapter) are deleted. +- `canonicalize_transport_timeout_ms`, the backend-name prediction and + uniqueness checks (`provider.rs:310-344`), and `reserved_backend_names` are + deleted. +- Batch slots must be buffered both ways (`outbound.rs:1924-1934`). Bidder + requests are built fresh with buffered JSON bodies, so they qualify. + +`OutboundSlotResult::elapsed` runs from the batch's shared start to the slot's +recorded terminal observation, or its logical expiry for a classified timeout +(`edgezero:crates/edgezero-core/src/outbound/batch.rs:653-674`). Today each +provider's `response_time_ms` runs from its own launch, including request +construction and signing (`orchestrator.rs:1839`), until publisher-side +collection (`orchestrator.rs:2375`, `publisher.rs:4717-4723`). The new metric +includes launch skew from the shared start but excludes the wait between a +successful response's arrival and collection. Delaying collection after a +timely response must not increase its reported slot elapsed time. + +On Fastly, EdgeZero records arrivals in any EdgeZero poll that services the +provider. That covers outbound waits, body reads, EdgeZero's timers, and every +poll of the canonical handler and egress drivers +(`edgezero:crates/edgezero-adapter-fastly/src/request.rs:427`, +`edgezero:crates/edgezero-adapter-fastly/src/response.rs:573`, through +`edgezero:crates/edgezero-adapter-fastly/src/outbound/progress.rs:352` and +`edgezero:crates/edgezero-adapter-fastly/src/outbound/scope.rs:207-224`). +`batch.check()` and `is_ready()` record nothing. They report only outcomes an +earlier poll recorded, and never wait +(`edgezero:crates/edgezero-adapter-fastly/src/outbound.rs:5456-5494`). +Cross-cutting requirement 5 and EZ-6.11 require a check to record, so this is +[open defect 2](#open-edgezero-defects-at-h). Fixed at the pin: EdgeZero's fix +is in `6f0cc74c`, an ancestor of `14a6ca8b`. At the pin under Viceroy 0.21.0, +a `check` after the cutoff returns `Finished(Cutoff)` (section 8). Collection +after the cutoff calls `batch.next()`, which also returns `Finished(Cutoff)` +without a host wait. The +publisher path reads the origin body through EdgeZero while it streams, and +those reads keep arrivals current. EdgeZero's `body-read-arrivals` harness case +covers that +(`edgezero:crates/edgezero-test-harness/src/net/wire/lifetimes.rs:2246`). A long +CPU stretch with no EdgeZero call can still push a bid's observation past the +cutoff, and the ledger then drops it (`batch.rs:645-650`). Test 6 checks the +publisher path end to end. + +The cfg(test) orchestration paths (`run_routed`, `run_parallel_mediation`, +`run_parallel_only`, and `run_providers_parallel`, at `orchestrator.rs:356-840`, +`:1015`, `:1193`, `:1216`) and the cfg(test) `AuctionProvider` impls for APS +and Prebid (`integrations/aps.rs:1675`, `integrations/prebid.rs:3495`) move to +the scripted client (D12) or are deleted with their tests. + +#### 2.5 Publisher origin + +The overlapped path runs today when +`should_preload_ec && template_cache_key.is_none() && supports_concurrent_fanout && supports_pending_streaming_responses` +(`publisher.rs:5318-5325`). The two capability terms are deleted, and +`should_preload_ec` stays. The overlap exists to hide the EC KV snapshot read +behind the origin fetch, and `should_preload_ec` requires a KV graph +(`publisher.rs:5298-5310`, `:3459-3466`). Axum, Cloudflare, and Spin pass no +KV graph to `handle_publisher_request` +(`crates/trusted-server-adapter-axum/src/app.rs:256-264`, +`crates/trusted-server-adapter-cloudflare/src/app.rs:564-572`, +`crates/trusted-server-adapter-spin/src/app.rs:846-854`), and the cut keeps +their unavailable-KV fallbacks (section 7). So the overlap still runs only on +Fastly navigation GETs that preload an EC snapshot. Everywhere else the plain +path dispatches the auction first and then sends the origin request while the +bids are in flight (`publisher.rs:5467-5470`, `:5673-5697`). EZ-6 describes +the overlapped path the same way, so its Trusted Server check holds on both +paths. + +The origin request: + +- It is built with `OutboundRequest::from_request(req, origin_uri)?` (EZ-1, + `outbound.rs:1154`), which keeps the inbound body and the admitted read + deadline. `restrict_accept_encoding` (`publisher.rs:302`) and the + `fastly-ssl` strip stay. +- `rewrite_origin_request` (`publisher.rs:3474-3495`) stops writing `Host`. + Today it writes `settings.publisher.origin_host_header()`, which is + `origin_host_header_override` when set and the origin host otherwise + (`crates/trusted-server-core/src/settings.rs:257-261`), and the backend + repeats the override (`publisher.rs:4978-4994`). On H, an override that + differs from the origin URI authority becomes `.host_authority_override(..)` + on Fastly, Axum, and Cloudflare. Spin sets none (D9). +- Options are `.stream_response()`, `Original`, the D7 timeouts, and the cache + policy. `Bypass` maps to `Bypass`, and `Default` to `PlatformDefault`. + `Shared` with a surrogate key is unreachable once D9 disables origin + readthrough. `with_shared_cache` has one production caller + (`publisher.rs:4849`). + +**Cloudflare deviation from D8.** Workers rewrites the inbound +`Accept-Encoding` to `br, gzip` whether the client sent none or `identity`, so +`restrict_accept_encoding` forwards a coding the client may not accept, and the +origin answers in it. The Cloudflare publisher origin therefore sets +`ResponseDecoding::Decode`, raw passthrough included, and serves identity bytes +as Trusted Server's Cloudflare client did. A direct workerd probe (Wrangler +4.83.0 and 4.131.0) showed that EdgeZero's outbound `Accept-Encoding: identity` +is not replaced, so EdgeZero meets EZ-5 and the cause is Cloudflare's inbound +normalization. EZ-5's rules still apply under `Decode`: a 206 keeps the origin +coding, and HEAD and 304 responses keep their `Content-Encoding`. Fastly, Axum, +and Spin keep `Original`. + +Both paths dispatch the auction before the origin response is read, and the +batch must outlive the handler. + +1. `handle_publisher_request` creates `owner = OutboundScopeOwner::response_body()` + (`edgezero:crates/edgezero-core/src/outbound/scope.rs:223`) before the + auction is dispatched, and returns `(PublisherResponse, OutboundScopeOwner)`. + Every variant of `PublisherResponse` (`publisher.rs:1598-1660`) travels with + it, `AssembleTemplate` on a template-cache hit included. +2. It awaits `owner.run(orchestrator.dispatch_auction(..))` (`scope.rs:253-264`) + in place of today's plain await (`publisher.rs:5467-5470`). The batch that + `dispatch_planned_auction` starts captures the ambient scope at call time + (`outbound.rs:739-741`), so the response body owns it. +3. Both origin calls are also constructed inside `owner.run`: + `owner.run(async { client.send(origin).await })` on the plain path, and + `owner.run(async { client.begin(origin).await })` on the overlapped path. H + captures the ambient scope when `send` or `begin` is called, not when the + future is first polled (`outbound.rs:578-583`, `:700-705`), so + `owner.run(client.send(origin))` still captures the callback scope. + Outside the response-body scope, head finishing gives a streamed body its + own response-body owner (`edgezero:crates/edgezero-core/src/outbound/head.rs:296-302`, + `:384-400`). The raw `Buffered` and `PassThrough` conversions keep that body + (`publisher.rs:1868-1877`, `:2093-2095`, `:2546-2556`), so the attach below + would fail with "response body already owns an outbound scope". Inside the + scope, head finishing binds the body to the shared scope and adds no owner + (`head.rs:388-393`). Test 20 covers both calls and every raw variant. +4. The adapter call site attaches the owner exactly once, after + `publisher_response_into_streaming_response` (`publisher.rs:2500`, Fastly) or + `buffer_publisher_response_async` (the other three) returns: + `owner.attach(response)?` (`scope.rs:164`). Neither function attaches, + because the Fastly function calls the buffering one itself on a + template-cache miss (`publisher.rs:2530-2546`), and `attach` fails on a body + that already has an owner (`scope.rs:164-175`). +5. EdgeZero's egress releases the work when egress ends. Attaching to a + buffered body is harmless. A handler error drops `owner`, which cancels the + batch. Code that rebuilds a response body after the attach converts with + `into_stream()`, which carries the owner. It never splits with + `into_parts()` and rebuilds with `from_stream`, because dropping the + returned owner closes the batch. + +On Axum, Cloudflare, and Spin, `buffer_publisher_response_async` collects every +variant before it returns, so nothing streams to egress there (Non-goals). H +collects only `ResponseMode::Buffered` bodies (`head.rs:288-301`), and a +streamed response has no byte cap by default (`outbound.rs:1514-1515`, +`:1554-1556`). Without collection, the raw `Buffered` and `PassThrough` +variants, which the function returns unchanged today (`publisher.rs:1868-1877`, +`:2093-2095`), would stream, lose today's client caps, and fail during egress +instead of before it. + +Today's clients bound the origin input before the publisher classifies the +response (`publisher.rs:5944-5964`, `:6046-6075`), so the bound covers HTML as +well as raw responses. Cloudflare's client caps the declared length and the +received bytes at 10 MiB +(`crates/trusted-server-adapter-cloudflare/src/platform.rs:187`, `:385-395`, +`:422-440`). Spin's client caps the received bytes at 8 MiB and separately +bounds gzip and br expansion at 8 MiB +(`crates/trusted-server-adapter-spin/src/platform.rs:29-33`, `:344-372`, +`:541-543`). Axum's client has no cap +(`crates/trusted-server-adapter-axum/src/platform.rs:364-366`). The cut keeps +those input bounds for every publisher classification, raw and processed +alike: + +- Each adapter has an origin input cap `cap`: the smaller of + `publisher.max_buffered_body_bytes` and its current client cap, so 10 MiB on + Cloudflare and 8 MiB on Spin by default. Axum has no client cap, so only the + setting applies there. Each adapter passes `cap` to + `handle_publisher_request` and `buffer_publisher_response_async`, because + the client modules that hold those constants are deleted. +- The origin request sets `.max_encoded_response_bytes(cap)`, so H fails any + origin body over `cap` whatever its classification. These adapters collect + before egress, so that failure arrives before the response is committed. +- Raw `Buffered` and `PassThrough` bodies are collected with + `Body::into_bytes_bounded(cap)`. +- Processed HTML is decoded by Trusted Server's rewrite codec, because the + origin request uses `Original`. Its cumulative decoded bytes are bounded by + `cap`, as the old clients bound the bytes they returned. The streaming path + already applies a cumulative decoded bound, so this uses the same mechanism + with `cap` (`crates/trusted-server-core/src/settings.rs:70-75`). +- The processed output keeps its separate `publisher.max_buffered_body_bytes` + cap (`publisher.rs:1921`). An input bound and an output bound are different + limits, and both apply. +- Overflow and source errors fail before egress with the site's 502. That is + `TrustedServerError::Proxy`, or H's `ResponseTooLarge` through 2.11, as the + client caps fail today. None of these go through the `PayloadTooLarge` rule + in 2.11. + +Test 21 covers this. + +The overlapped path (O4) adds the origin `begin`: + +1. `owner.run(async { client.begin(origin).await }).await?` (`outbound.rs:578`) + before the auction is dispatched. The request is on its way once this future + resolves. A `begin` future that is never awaited sends nothing. +2. Dispatch the auction under `owner.run`, as above. +3. `pending.collect().await` returns the streamed origin response + (`edgezero:crates/edgezero-core/src/outbound/progress.rs:106`). A collected + streamed response owns its own transport + (`edgezero:docs/guide/proxying.md:83-88`). + +`begin` accepts a bodiless GET or HEAD, or a buffered body +(`outbound.rs:1916-1923`). The overlapped path is GET navigation only. On +Fastly, EZ-1 settles the inbound body by reading it to the end, which is +immediate for a bodiless GET. A forward that starts after the inbound read +deadline fails with 408 even when bodiless +(`edgezero:crates/edgezero-core/src/outbound/dispatch.rs:160-175`). The origin +fetch starts milliseconds after the request arrives, so only pathological +requests reach that. + +The plain path (O5) also covers POST, PUT, PATCH, DELETE, and OPTIONS. It calls +`client.send(..)` inside `owner.run`, as above, with the same request and +options, and hands the inbound body stream upward with +`.max_request_body_bytes(16 MiB)`. Overflow returns 413 +(EZ-10). Axum and Cloudflare prebuffer the upload under the cap before they +contact the origin (EZ-1.11, EZ-10.6), so their origin sees no request on +overflow. Fastly and Spin stream it. On Fastly the overflow is found after +`send_async_streaming` has started +(`edgezero:crates/edgezero-adapter-fastly/src/outbound.rs:1555`, `:1462-1470`), +so the origin sees the head and up to 16 MiB of an aborted upload. Only a +buffered body is checked before sending +(`edgezero:crates/edgezero-core/src/outbound.rs:1903-1908`). Today Fastly +reads the whole inbound body into memory with no cap. + +#### 2.6 Redirect following + +`proxy_with_redirects` (`proxy.rs:1276-1560`) and `ProxyRedirectPolicy` +(`proxy.rs:770`) are replaced by +`client.follow(request, OutboundRedirectLimit::new(4)?, policy)` +(`outbound.rs:611-636`, +`edgezero:crates/edgezero-core/src/outbound/redirect.rs:320-340`). Today's +`MAX_REDIRECTS` of 4 (`proxy.rs:1284`) stays. + +- `proxy_request` checks the initial target before `follow` or `send`, as today: + scheme, `require_https`, host, and allowlist (`proxy.rs:1293-1327`). H never + shows the first request to the policy (`redirect.rs:64`). +- The policy is a `'static` closure (`outbound.rs:623-635`) that owns a copy of + `allowed_domains` and `require_https`, since `ProxyRedirectPolicy` borrows + them today (`proxy.rs:770-776`). For each redirect it reads + `resolved_target()` and returns, in today's order (`proxy.rs:1492-1524`): + - `Err(Forbidden)` when `require_https` is set and the scheme is not `https` + - `Stop` for any other scheme that is not `http` or `https`, which returns + the 3xx as today + - `Err(Proxy)` for a missing host, as today + - `Err(AllowlistViolation)` when `is_host_permitted` rejects the host + - `FollowPrepared` otherwise +- Followed requests set `.timeout(15 s)` per hop and + `.deadline(clock.deadline_after(60 s))` for the whole chain. Per-call timeouts + restart on every hop, and only a deadline holds across the chain + (`outbound.rs:613-617`). Today each hop has 15 s timers and the chain has no + bound. +- When `follow_redirects` is false, the caller uses `send`. The request keeps + `PROXY_FORWARD_HEADERS` (`proxy.rs:67-73`), the fixed `Accept-Encoding` + (`proxy.rs:454`), caller headers, and buffered body bytes. +- `finalize_response` (`proxy.rs:747`) decodes and re-encodes on its own + (`proxy.rs:482-512`), so follow requests set `Original`. It receives the + final URL from `OutboundFollowResult::final_uri()` (`redirect.rs:226-229`), + which it uses to rewrite relative creative URLs (`proxy.rs:1426-1431`). +- `OutboundFollowFailure::Policy` maps back to the policy's Trusted Server + error. `OutboundFollowFailure::Hop` goes through the error mapping in 2.11. + +H matches Trusted Server on the rest (`redirect.rs:388-533`). 301, 302, and 303 +rewrite to GET, except HEAD, and drop the body and its headers. 307 and 308 +replay buffered bodies. A method that cannot be rewritten, the limit, or a +missing `Location` returns the 3xx. + +Accepted differences: + +- Cross-origin hops strip `Authorization`, `Cookie`, and the host authority + override (`redirect.rs:392-407`). Trusted Server's proxies set none of them + by default, so only caller-set values are affected. +- A `Location` with userinfo fails as a protocol error, which becomes the site's + 502 (`redirect.rs:479-492`). So do multiple or non-text `Location` values on + Fastly, Axum, and Spin. Cloudflare never sees them: workerd merges duplicate + fields and maps raw bytes to text, and EdgeZero follows the merged value + (`edgezero:crates/edgezero-adapter-cloudflare/src/outbound.rs:648-652`, + `:857-875`). Trusted Server took the first value and followed userinfo. +- A UTF-8 `Location` with non-ASCII characters is percent-encoded and followed. + Trusted Server returned the 3xx, because `to_str` rejects non-ASCII bytes + (`proxy.rs:1434-1438`). Locations with spaces were already percent-encoded + and followed. +- A 3xx that is returned instead of followed now gets the call's ordinary body + handling, so it can fail on the response cap or on decoding (EZ-9.11, + EZ-9.12, `edgezero:crates/edgezero-core/src/outbound/head.rs:227-317`). +- `Expect` is removed on every body-dropping rewrite. +- Hops log at debug level without the query (`redirect.rs:212-222`). + +#### 2.7 Asset proxy and Image Optimizer + +- `send_asset_origin_request` (`proxy.rs:974-1001`) and + `handle_asset_proxy_request` build `OutboundRequest::new(method, uri)?` with + `ASSET_PROXY_FORWARD_HEADERS` (`proxy.rs:90`), `.stream_response()` on the + GET, `Original`, the D7 timeouts, and the default cache policy. The S3 HEAD + preflight stays buffered, as today (`proxy.rs:1037-1044`), and also sets + `Original`. At H, `Decode` leaves a HEAD's encoded `Content-Encoding` and + `Content-Length` alone only because EZ-5.9 is not adopted. `Original` keeps + that metadata stable if EdgeZero adopts it. +- `asset_origin_host_header` (`proxy.rs:893-916`) stays. It is the signer's + `host` input, and `s3_sigv4::sign_headers` fails without it + (`crates/trusted-server-core/src/s3_sigv4.rs:128-147`). H strips `Host` + before dispatch and sends the URI authority, so the header never reaches the + wire. Test 12 checks that the authority, path, and query H sends equal what + Trusted Server signed. H re-parses the URI as a WHATWG URL + (`outbound.rs:1953-1989`), so they must agree byte for byte. +- Image Optimizer options go on with `.image_optimizer(opts)` + (`outbound.rs:1263`), on the final client GET or HEAD, never on the diagnostic + S3 HEAD preflight. H requires GET or HEAD, `PlatformDefault` caching, and a + region, and returns 400 otherwise + (`edgezero:crates/edgezero-core/src/outbound/image_optimizer.rs:197-217`). + The asset route admits HEAD, a successful preflight proceeds to the final + request, and the final request keeps the client's method and attaches the + options today (`proxy.rs:1052-1057`, `:1171-1174`), so a final HEAD keeps its + options. Test 3 and the EZ-8 hosted leg cover the final HEAD. + `begin` and batches reject the options. Axum, Cloudflare, and Spin reject + them before sending (`edgezero:crates/edgezero-adapter-axum/src/outbound.rs:153-160`, + `edgezero:crates/edgezero-adapter-spin/src/outbound.rs:1520-1525`), as + Trusted Server's own clients do today. +- Settings load already accepts only values H's types can express: known + regions (`settings.rs:1131-1142`), H's format and filter sets plus aliases + (`:1530-1560`), quality 0 to 100 (`:1497-1505`), positive dimensions, and crop + offsets 0 to 100 that never combine with smart crop (`:1630-1640`, + `crates/trusted-server-core/src/asset_image_optimizer.rs:330-343`). No new + validation is needed. +- `PlatformImageOptimizerOptions` and its Fastly mapping + (`crates/trusted-server-core/src/platform/image_optimizer.rs`, + `crates/trusted-server-adapter-fastly/src/platform.rs:238-322`) are deleted. + Per request, the asset proxy maps the validated profile plus the request's + aspect ratio and offsets (`asset_image_optimizer.rs:88-89`, `:125-188`) to + `OutboundImageOptimizerOptions`, with Trusted Server's region aliases mapped + to H's seven regions. `preserve_query_string_on_origin_request` maps to + `preserve_origin_query`. A builder error, which validation makes unreachable, + serves the image unoptimized with a warning. +- Viceroy answers Image Optimizer requests with 502 + `ImageOptimizerUnavailable` + (`edgezero:crates/edgezero-adapter-fastly/src/outbound.rs:88-91`), unverified + by a run. Proof needs an IO-entitled staging service (EZ-8). The synchronous + optimizer call has no approved blocking bound yet, because EdgeZero's + G4-OPTIMIZER-HORIZON approval is pending. + +#### 2.8 Post-delivery work on Fastly + +Both items run inside the serving callback after +`send_request_with_registries_and_hooks` returns `Routed`, and only when the +delivery committed with the application's body (section 3). Today pull sync +runs after every send (`crates/trusted-server-adapter-fastly/src/main.rs:547-556`). +They use the fresh client and clock from 2.1. EdgeZero releases +callback-owned batch drivers when the callback returns +(`edgezero:crates/edgezero-adapter-fastly/src/lifecycle.rs:145-146`), so both +finish before it does. Neither delays the client, which already has its +response. Each extends the sandbox's lifetime, as pull sync does today. + +One outer `futures::executor::block_on` in `run_post_delivery` +(`crates/trusted-server-adapter-fastly/src/post_delivery.rs:45-61`) drives +both items, which await their sends inside it. That call is safe only here. +Inside the helper, dispatch already runs under `block_on` +(`edgezero:crates/edgezero-adapter-fastly/src/request.rs:427-431`), and nesting +panics. EdgeZero's fixture uses `block_on` in the same position +(`post_delivery.rs:93-178`). + +- **Telemetry (D11).** `send_fire_and_forget` drops a pending request, and + Fastly keeps it running (`tinybird.rs:133-148`). On H, dropping an + `OutboundPending` cancels it (`progress.rs:94-110`). The sink is shared app + state today (`crates/trusted-server-adapter-fastly/src/app.rs:229`, `:289`). + After the cut, the callback creates a queue, an `Arc>>`, + and `prepare` inserts a clone into the request extensions. + `build_per_request_services` wraps the clone from `ctx.extensions()` in a + per-request sink that queues NDJSON instead of sending. After delivery the + callback drains the queue and awaits `client.send(..)` for each row, with + today's headers, `.timeout(2 s)`, the 1 s connect timeout, and `Bypass`. Each + row is one short request. Axum, Cloudflare, and Spin keep their no-op sink. +- **EC pull sync.** `dispatch_pull_sync` takes the post-delivery client and + clock instead of the ones in `ec_state.services` + (`crates/trusted-server-adapter-fastly/src/main.rs:679-686`). Each group of + `ec.pull_sync_concurrency` requests (`pull_sync.rs:184`) goes out with + `client.send_all_until(group, cutoff).await` (`outbound.rs:722`). + Requests carry `.timeout(15 s)` and `.max_response_bytes(64 KiB)`, and the + cutoff is `clock.deadline_after(15 s)` per group. Today each request has 15 s + idle timers and no total bound, so the per-group cutoff is new. + +Telemetry goes first, because its rows are short and pull sync can take a full +group budget. + +#### 2.9 Fastly management API + +`FastlyManagementApiClient::make_request` is a synchronous raw +`fastly::Request::send`. Key rotation reaches it through synchronous store-write +methods (`crates/trusted-server-core/src/request_signing/rotation.rs:96-285`, +`crates/trusted-server-adapter-fastly/src/platform.rs:78`, `:83`, `:144`, +`:149`). A synchronous wrapper around `HttpClient::send` would need `block_on` +inside a handler, which panics (2.8). + +`PlatformConfigStore::put` and `delete`, and `PlatformSecretStore::create` and +`delete`, become async with `#[async_trait(?Send)]`, like the other platform +traits (`crates/trusted-server-core/src/platform/http.rs:301`), because +`RuntimeServices` holds both stores as `Arc` (`platform/types.rs:162-164`). +The traits have no default bodies (`platform/traits.rs:12-36`, `:42-93`), so all +37 implementers in `crates/` change in step 8, and so does the guide snippet. That includes the test stores in +`config_payload.rs`, `request_signing/jwks.rs`, `secret_resolution.rs`, and +`settings_data.rs`. The Fastly +implementations hold the request's `HttpClient` and send +`OutboundRequest::new(method, uri)?` with `Fastly-Key`, +`Accept: application/json`, `Content-Type` (`management_api.rs:170-178`), and +the body, with a 1 MiB response cap and the D7 15 s timeout. Error bodies keep +today's 200-character truncation (`management_api.rs:33`, `:79-102`). The other +adapters keep returning their current unsupported error. Rotation and its admin +handlers await the calls. + +#### 2.10 Capability methods + +`supports_concurrent_fanout`, `supports_streaming_responses`, and +`supports_pending_streaming_responses` are deleted, with their 16, 16, and 7 +call lines. On H every adapter streams and runs batches. + +#### 2.11 Outbound errors + +Today every site wraps transport errors in its own context, and `Proxy`, +`Integration`, `Auction`, and `Prebid` all map to 502 +(`crates/trusted-server-core/src/error.rs:116-135`). Trusted Server has no 504 +variant. One helper, `outbound_error(error: EdgeError, context: TrustedServerError)`, +maps every `HttpClient` failure and every request-construction error +(`from_request`, `host_authority_override`, and the builders) (D17): + +- `PayloadTooLarge` becomes `RequestTooLarge` (413). +- `RequestTimeout` becomes `RequestTimeout` (408, section 4). It comes from the + inbound read deadline while a forwarded body is settled or uploaded. +- `BadRequest` on the publisher forward becomes `BadRequest` (400). The client's + request usually causes it: a GET with a body, a malformed `Connection`, or a + URI with a fragment or backslash. A `publisher.origin_url` with userinfo also + causes it, so validation rejects such a URL (Configuration). +- Everything else keeps the site's context, which is 502 as today. That covers + timeouts, connect failures, protocol errors, a Fastly `NameInUse`, and a + `BadRequest` from a request Trusted Server built itself. + +The helper attaches the `EdgeError`, its budget source, and its +`provider_failure_detail()` to the report. + +### 3. Fastly on the canonical lifecycle + +`main` keeps choosing `run_custom` for single-request sandboxes and +`serve_custom` for reuse (`crates/trusted-server-adapter-fastly/src/main.rs:96-152`). +The callback now returns `Result<(), Infallible>` +(`edgezero:crates/edgezero-adapter-fastly/src/lifecycle.rs:153-181`) and +always returns `Ok(())`. An `Err` would end the reuse loop with no response +(`lifecycle.rs:139-149`). + +The callback runs these steps in order. + +1. **Native short-circuits, unchanged.** The trace-path check, health probe, + `setup_once(init_logger)`, request id, sandbox metrics probe (reuse only), + and JA4 debug probe (`main.rs:172-208`, `:342`, `:362`) answer with native + responses before EdgeZero runs. The metrics probe must stay ahead of app + assembly. The callback takes `clock = MonotonicClock::default()` and + `request_start = clock.now()` at entry. Only `send_assembly_error` uses them. +2. **Assembly.** `sandbox.initialize` builds the router and calls + `App::from_router::(router, FASTLY_PLATFORM)` + (`edgezero:crates/edgezero-core/src/app.rs:347`). `from_router` runs + `Hooks::configure` once, so the hand-built `App` and its + `Self::configure(&mut app);` call, which would drop a `Result` at H + (`crates/trusted-server-adapter-fastly/src/app.rs:1280-1287`), go away. On + error, the callback calls `send_assembly_error(req, error, request_start, clock)` + (`edgezero:crates/edgezero-adapter-fastly/src/request.rs:627-645`) and + handles its result like any other delivery. A failed assembly is not + retained, as today. `startup_error_router` (`app.rs:1047`) is deleted, + because `send_assembly_error` replaces it. +3. **Native request work.** The callback runs today's native mutations on the + owned request before the send: `compat::capture_fastly_forwarder` + (`main.rs:433-435`), client-IP resolution and sanitization, and re-inserting + `fastly-ssl` and `x-ts-tls-*` with their trace-path conditions + (`main.rs:439-472`). It computes `client_info_from_request` and + `derive_device_signals` (`main.rs:476-478`) and keeps the resolved client IP. + It runs the geo lookup on that IP once, for both `prepare` and `finalize`. + It also captures `SandboxCounters` (`main.rs:425-426`) and creates the + telemetry queue (2.8). `RequestIngress` capture reads only the URL and + `Host` (`edgezero:crates/edgezero-adapter-fastly/src/request.rs:772-782`), + and none of these mutations touch either + (`crates/trusted-server-adapter-fastly/src/compat.rs:66-130`). +4. **Send.** + `send_request_with_registries_and_hooks(app, TrustedServerApp::stores(), req, prepare, finalize)` + (`edgezero:crates/edgezero-adapter-fastly/src/request.rs:594-631`). +5. **Outcome** handling, post-delivery work (2.8), and the observation log. + +`prepare(&mut FastlyRequest, &mut Extensions)` moves the computed values into +the extensions that `into_core_request` carried (`main.rs:484`): the forwarder, +device signals, client info, and the telemetry queue. The helper captures +`RequestIngress` before `prepare` and removes any copy `prepare` inserts +(`request.rs:466-468`), so `prepare` must not insert one. `prepare` also inserts +the request's synthesized header set, which the helper takes out before head +limits are checked (EZ-19.2, `request.rs:469`). Section 5 lists its fields. +`open_trusted_server_config_store` and its `ConfigStoreHandle` extension +(`main.rs:65-70`, `:384-394`, `:489`) are deleted, because nothing reads it. A +missing config store now fails settings load and surfaces as an assembly error. + +The `finalize` hook calls +`finalize_response(&mut Response, &FinalizeInputs) -> Option` +(`crates/trusted-server-adapter-fastly/src/main.rs:414-479`). +`FinalizeInputs { settings, geo, counters }` carries the settings, the +request's one geo lookup, and the sandbox counters. It takes over today's +post-routing steps, all synchronous: + +1. If `TraceTerminalResponse` is present, return without touching the response. + Today it is sent raw (`main.rs:508-516`). Now it passes EdgeZero's egress, + which strips hop-by-hop fields. Whether any trace header changes is + unverified. The response diff (Runtime) covers a trace request. +2. Remove `EcFinalizeState`, `AssetProxyCachePolicy`, and `RequestFilterEffects` + (`main.rs:518-520`). +3. Apply finalize headers unless `x-ts-finalized` is set, with the geo result + the callback computed in its step 3 (`main.rs:522-541`, `:618-637`). Then the asset cache + policy (`main.rs:541-543`), then EC finalize (`main.rs:545-547`). +4. Apply the terminal privacy effects that `send_edgezero_response` applies + today (`main.rs:776-808`), and the sandbox counter headers. +5. Return the pull-sync inputs: the partner registry and the EC state. It + returns `None` for a trace response, for a response that carries no EC + state to finalize, and when the partner registry cannot be built, which is + logged at error. + +`finalize` runs for every routed response, including handler errors and +canonical 404 and 405, before egress policy and framing +(`edgezero:crates/edgezero-adapter-fastly/src/delivery.rs:313-334`). Detached +responses skip it: head-limit 414 and 431, framing rejections at admission, and +admitted source errors. They carry the request's synthesized set (section 5), +which holds the version, environment, geo, and operator headers. They carry no +EC cookies, cookie cache privacy, or sandbox counters. A framing fallback +replaces a routed response after `finalize` ran, so its finalize side effects, +such as EC KV writes, have happened, but its cookies never reach the client. It +carries the synthesized set in place of the finalize headers. + +Outcomes of `FastlyRequestDelivery` (`delivery.rs:183-312`). The callback +handles every delivery the same way, including the one `send_assembly_error` +returns. Every failure log carries the sandbox counter context +(`[instance= ordinal= request=]`, as at `main.rs:726-768`). + +| Outcome | Callback action | +| ----------------------------------------------------------------------------- | ------------------------------------------------------------------------------ | +| `NoAttempt` with `RequestHeadConversion` | Native 500 with fixed text, as today (`main.rs:499-505`) and EZ-17 criterion 7 | +| `NoAttempt` with `RequiredStore` | Unreachable, because Trusted Server declares no required store. Native 500 | +| `Routed`, committed, application body | Post-delivery work with `state`, even when `delivery.error()` is set | +| `Routed` or `Detached` with `may_send_native_response()` true (not committed) | Log at error, send a native 500 | +| Any other `Routed` (unknown commit state, or committed with a fallback body) | Log at error. No post-delivery work and no second response | +| Any other `Detached` | Log the report at debug, or at warn for a source error | +| `Aborted` | Unreachable, because Trusted Server installs no abort policy. Log at error | +| Any other variant (`FastlyRequestDelivery` is `#[non_exhaustive]`) | Log at error | + +Outcome handling is one function generic over `FastlyRequestDelivery`. Only +the routed post-delivery arm needs `FinalizeState`. The assembly-error result +has `S = ()` (`request.rs:634-644`), and its `Routed` arm cannot occur. Tests +drive every row with EdgeZero's `delivery::testing` helpers (test 9). They build +envelopes with the public `App::assembly_error_egress`, +`App::admitted_error_egress`, and `App::dispatch_admitted`, and detached +outcomes with `App::detached_ingress_error_egress` +(`edgezero:crates/edgezero-core/src/app.rs:94`, `:120-125`, `:221-226`, +`:276`). Viceroy cannot produce `NoAttempt`, `Unknown`, or +`NotCommitted` on demand. + +After post-delivery work, the callback logs a `FastlyOutboundObservation::current()` +snapshot at info on a dedicated target: configuration count, registration +count, and total and maximum registration time +(`edgezero:crates/edgezero-adapter-fastly/src/context.rs:202-265`). Hosted +Fastly logs at info (`crates/trusted-server-adapter-fastly/src/logging.rs:19-62`), +and EZ-2's criteria 9 and 15 read it. Taking it after post-delivery work +includes pull-sync and telemetry registrations. + +Deleted from the Fastly adapter: the `oneshot` dispatch in `handle_request` +(`main.rs:494`), the `into_core_request` call (`main.rs:484`), +`edge_error_response` (`main.rs:597`), `send_edgezero_response` and +`send_response_to_client` (`main.rs:723-808`), and `compat::to_fastly_response*` +and `stream_to_client` (`compat.rs:16-64`). Core's `stream_asset_body` +(`crates/trusted-server-core/src/proxy.rs:288`) goes with its tests. The sandbox +counters and retirement log stay +(`crates/trusted-server-adapter-fastly/src/sandbox.rs`). + +### 4. Inbound request bodies + +`crates/trusted-server-core/src/http_util.rs` gains one helper, +`read_request_body(headers: &HeaderMap, body: Body, max: usize) -> Result>`. + +- It returns `RequestTooLarge` without polling the body when a single valid + `Content-Length` exceeds `max`. +- Otherwise it awaits `Body::into_bytes_bounded(max)` + (`edgezero:crates/edgezero-core/src/body.rs:163`), which checks the cap before + each append and passes source errors through unchanged. +- It maps errors by status. 413 becomes `RequestTooLarge`. 408 becomes a new + `TrustedServerError::RequestTimeout`, mapped to 408. Another 4xx becomes + `BadRequest` (400). A 5xx becomes `Proxy` (502). A Fastly + host read error is a 5xx (`edgezero:crates/edgezero-adapter-fastly/src/request.rs:917-921`). + Today `collect_body_bounded` maps chunk errors to a 502 too + (`crates/trusted-server-core/src/integrations/mod.rs:198-204`). +- It does not log, and body bytes never reach its report. Each adapter renders + the report through core's `error::http_error`, which logs it once: at warn + for a 4xx, which the client caused, and at error for a 5xx. The auction and + batch-sync routes answer their own 413 and log the report at warn. + +It replaces `request_body_bytes` (`proxy.rs:47`, +`crates/trusted-server-core/src/request_signing/endpoints.rs:27`) and +`collect_body_bounded` (`integrations/mod.rs:181-215`). Core handlers take a +`Request`, not a `RequestContext`, so the helper wraps the `Body` reader instead +of `RequestContext::body_bytes`. + +| Reader | Cap, unchanged | Change | +| -------------------------------------------------------------------------------------------------------- | ----------------------------------- | ----------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| `/auction` (`auction/endpoints.rs:140-160`) | 256 KiB (`endpoints.rs:47`) | Helper replaces `into_bytes().unwrap_or_default()`. The `text/plain` 413 body stays | +| EC batch sync (`ec/batch_sync.rs:140-160`) | 2 MB | Helper. The `body_too_large` error body stays | +| Cache purge (`cache_purge.rs:135-142`) | 4096 (`cache_purge.rs:30`) | Helper replaces the stream rejection | +| First-party sign and rebuild (`proxy.rs:1724-1725`, `:1909-1910`) | 64 KiB (`proxy.rs:40-41`) | Helper | +| Request signing verify, rotate, deactivate (`request_signing/endpoints.rs:116`, `:306`, `:425`) | 4096 | Helper | +| Integration proxies: Sourcepoint, Lockr, Permutive, DataDome, Didomi, testlight | 256 KiB (`integrations/mod.rs:166`) | Helper | +| GTM beacon (`google_tag_manager.rs:802-830`) | `max_beacon_body_size` | Helper | +| Trace actions in the pre-dispatch hook (`crates/trusted-server-core/src/trace/actions.rs:40-55`, `:186`) | empty only | The helper's `Content-Length` check, then `PreDispatchRequest::body_bytes(0)` (`edgezero:crates/edgezero-core/src/pre_dispatch.rs:21`), mapped to the trace JSON errors | +| Publisher forward (O5) | none today | `from_request` streams it, with a 16 MiB `max_request_body_bytes` | + +At H, EdgeZero's own readers drain up to the cap before they return 413, even +for a declared length over the cap. That is EZ-10.4, an open should-fix +(`edgezero:crates/edgezero-core/src/context.rs:338-358`). Trusted Server's +helper checks the declared length itself, and the trace action makes the same +check before `body_bytes(0)`, so neither polls the body for a declared +overflow. Trusted Server uses none of the `Json` or `Form` extractors, so their +new 8 MiB and 1 MiB default caps do not apply. + +EdgeZero's default admission sets the inbound read deadline to request start +plus 30 s (`edgezero:crates/edgezero-core/src/ingress.rs:34`, `:730-736`), and +reads after it fail with 408. Trusted Server keeps the default (Q2), which a +16 MiB upload meets at about 4.5 Mbit/s. If uploads need more later, +`Hooks::configure` installs an admission policy (`App::set_ingress_admission_policy`) +that gives the publisher fallback routes a longer read deadline +(`ingress.rs:743-763`). On Fastly the deadline is checked between +blocking reads (`edgezero:crates/edgezero-adapter-fastly/src/request.rs:864-925`), +so a read that blocks is bounded by the host, not by 30 s. + +**Guard.** `clippy.toml` gains a `disallowed-methods` entry for +`edgezero_core::body::Body::into_bytes`. It returns `None` for a stream +(`body.rs:141`), which is how the empty-body bug compiles. The reason names the +bounded readers: `read_request_body` for inbound bodies and +`OutboundResponse::into_bytes_bounded` (`outbound.rs:864`) for outbound +responses. It does not name `Body::as_bytes`, which also returns `None` for a +stream (`body.rs:41-44`). + +- The core and adapter crate roots add + `#![cfg_attr(test, allow(clippy::disallowed_methods))]`, so lib targets check + production code and unit tests stay free to buffer. +- A `tests/*.rs` crate root adds `#![allow(clippy::disallowed_methods)]` only + when its tests buffer a `Body`: the Cloudflare and Spin `tests/routes.rs` and + the parity suite. Those are separate crates. A test crate that later buffers + one adds the allow when clippy asks for it. +- The one `test-utils` module that buffers bodies, + `test_support/scripted_client.rs`, carries a module-level allow. The + crate-root `cfg_attr` must not widen to `test-utils`, because `clippy-fastly` + builds with `--all-features` (`.cargo/config.toml:31`) and would then skip + production code. +- Production code carries no allow. The proxy and publisher readers share one + `pub(crate)` helper, `http_util::body_as_reader`, which matches on + `into_body_payload` and returns an error for a stream, so the first-party + proxy keeps buffering its responses. DataDome Protection no longer calls + `into_bytes`. Ported outbound reads such as `into_body().into_bytes()` + (`ec/pull_sync.rs:468-472`) move to `OutboundResponse::into_bytes_bounded`. + +### 5. Response egress + +These rules now apply on all four adapters. + +**`Trailer` and `Content-Length` (D10).** Framing rejects any `Trailer` +(`response_egress_framing.rs:127-128`), and a buffered body whose +`Content-Length` disagrees with its length, as a 500 before commit. + +- H strips `Trailer` from upstream response heads (`outbound.rs:2211-2228`, + applied at `:929` and `edgezero:crates/edgezero-core/src/outbound/head.rs:166`), + so relays never see one. +- `make_response_bodiless` (`publisher.rs:2970-3010`) removes `Trailer` in every + branch. Today it keeps `Trailer` wherever chunked framing survives, and the + test at `publisher.rs:26560-26575` asserts that. The test changes with it. +- `validate_settings_for_deploy` (`crates/trusted-server-core/src/config.rs:258`) + rejects an operator `response_headers` entry named `trailer`, + `content-length`, or any hop-by-hop field: `connection`, `keep-alive`, `te`, + `transfer-encoding`, `upgrade`, `proxy-connection`, `proxy-authenticate`, or + `proxy-authorization`. `ts config validate` and `ts config push` then fail + with the header name. At runtime, `apply_response_headers_with_cache_privacy` + (`crates/trusted-server-core/src/response_privacy.rs:161-188`) skips such an + entry with a warning, so a live config that has one keeps loading. +- Integration response headers already drop both names + (`crates/trusted-server-core/src/integrations/registry.rs:450-462`). + +**Stripped fields.** Egress removes `Connection` and the fields it names, +`Transfer-Encoding`, `Keep-Alive`, `TE`, `Upgrade`, `Proxy-Connection`, +`Proxy-Authenticate`, and `Proxy-Authorization` +(`response_egress_framing.rs:320-345`). A malformed `Connection` value, such as +a token that is not a field name (for example `x example`), is not stripped. +It is a framing error and a 500 before commit +(`response_egress_framing.rs:320-335`). Since EdgeZero `e65394a1`, egress +ignores up to 32 empty `Connection` members, so an empty token or a trailing +comma is not malformed. Operator +`response_headers` are Trusted Server's only source of such a value. Settings +accept any valid `HeaderName` and `HeaderValue` (`settings.rs:3437-3449`), and +the response-header helper applies the value unchanged +(`response_privacy.rs:167-188`). So D10 rejects and skips every operator +hop-by-hop entry. + +Skipping is not wire-neutral for `Connection`. A valid `Connection` value names +fields, and egress removes every field it names from the response, whatever set +that field (`response_egress_framing.rs:320-333`). Skipping an operator +`Connection` entry also drops that removal. A field it names, such as an +operator `x-example`, then reaches the client, where H would have removed it. +The ban accepts that loss of nomination semantics instead of re-implementing +it. `ts config validate` and `push` reject every such config, so only a live +config that predates the cut can carry one, and the pre-deploy check (Before +deploy) finds it. Test 16 asserts the retained field. Skipping the other +hop-by-hop names changes nothing on the wire, because egress strips them +anyway. + +**Write deadline (D7).** `Hooks::configure` installs +`App::set_response_egress_policy` (`app.rs:553-560`), whose policy receives the +egress start time, with a write deadline of 60 s after egress starts. The +default is 30 s from egress start +(`edgezero:crates/edgezero-core/src/response_egress.rs:22`, `:1231-1241`). +Without the policy, a streamed page ends at 30 s whatever its origin timeout. + +**Head limits.** The defaults are 8192 target bytes, 100 header fields, and +64 KiB of header bytes (`ingress.rs:35-37`). An absolute URI counts its scheme +and host (`ingress.rs:845`). Adapters check them before admission, and the +router checks again after the pre-dispatch hook. Violations return 414 or 431. +Admission also rejects ambiguous request framing with 400: a duplicate or +malformed `Content-Length`, `Content-Length` with `Transfer-Encoding`, or a +`Transfer-Encoding` that is not chunked (`ingress.rs:887-942`). Fastly enforces +all of this now too. Trusted Server keeps the defaults, and test 11 checks that +realistic traffic fits. If it does not, `Hooks::configure` raises them with +`App::set_ingress_head_limits` (`app.rs:523`). + +**Synthesized headers (D15).** `Hooks::configure` calls +`App::set_synthesized_response_headers` (`app.rs:570`) with `x-ts-version` +(`crates/trusted-server-core/src/constants.rs:36`), and on Fastly also the +service-version header that the finalize step sets. EdgeZero adds them only +where absent (`edgezero:crates/edgezero-core/src/response_headers.rs:51-62`), +to head-limit errors, framing rejections, admitted source errors, fallbacks, +and canonical 404 and 405 (`app.rs:221-252`, `:276-307`). + +On Fastly, `prepare` replaces that set for each request (EZ-19.2). The +per-request set holds the finalize headers that do not depend on the response: +`x-ts-version`, `x-ts-fastly-version`, `x-ts-env` on staging, the geo headers or +`x-geo-info-available: false`, and the operator's `response_headers` +(`crates/trusted-server-adapter-fastly/src/middleware.rs:187-237`). Operator +entries whose names EZ-19.5 forbids are left out. This meets EZ-19 criterion 6. +`SynthesizedResponseHeaders::new` (`response_headers.rs:70`) rejects a set over +32 field values or 4096 bytes (`:6-9`), or one with a value outside visible +ASCII and interior tabs (`:13-14`, `:97-108`), such as a geo city with accents. +`prepare` leaves out each such geo or operator value with a warning before it +builds the set, as the geo headers already skip invalid values one by one +(`crates/trusted-server-core/src/geo.rs:51-59`). Only a bounds failure then +falls back to the version headers alone, with the error category logged at +warn. Axum, Cloudflare, and Spin have no per-request hook and keep the +app-level set. + +Admission refusals are not decorated (EZ-19.3), and assembly errors carry no set +(EZ-19.7, `app.rs:563-565`). Trusted Server installs no admission policy, so +refusals do not occur, but a startup failure now carries no `x-ts-version`. + +**Error text.** EdgeZero renders `EdgeError::Internal` as +`internal server error` instead of `internal error: {source}`. 405 responses +carry a sorted `Allow` header, and overflow responses carry no `Retry-After` +(`edgezero:crates/edgezero-core/src/error.rs:452-461`, `:557-559`, `:822-845`). +No Trusted Server test matches the old text. + +### 6. Pre-dispatch hook and request ingress + +`TracePreDispatchHook::handle` (`crates/trusted-server-core/src/trace/dispatch.rs:117-127`) +takes `&mut PreDispatchRequest` (`router.rs:43`). Helpers that took +`&mut Request` or `&Request` take `&mut RequestParts`, `&RequestParts`, or the +hook request: `prepare_trusted_forwarder` +(`crates/trusted-server-core/src/forwarder.rs:63`), `trace/routes.rs:242`, +`action_response` (`trace/actions.rs:27`), `matches_trusted_origin` +(`trace/actions.rs:78`), and `MetadataSupplier` (`trace/dispatch.rs:90`) with +its four adapter suppliers. + +- The hook can no longer replace the body. `mem::replace(request.body_mut())` + (`trace/actions.rs:48-49`) becomes `request.body_bytes(0)` (section 4). It + returns 413 for any byte and caches the body instead of consuming it. +- On canonical paths the hook runs after admission (`router.rs:579-599`). + Refusals and aborts never reach it, a changed method or path is a 500, and + the head is validated again after a hook that continues. The trace hook + answers or continues and is not expected to change method or path. Test 9 + checks the trace routes through canonical dispatch. +- The `PreDispatchRequest` constructor is crate-private (`pre_dispatch.rs:93`). + The 17 hook test calls drive the hook through + `RouterService::builder().pre_dispatch_hook(..).build().oneshot(request)`. + Tests that check a continue outcome add a route whose handler records the + request, since `oneshot` hides the mutated request. `oneshot` runs the hook + before route lookup (`router.rs:429-452`), unlike canonical paths, so the + integration suite covers the canonical order on Viceroy, workerd, and Axum. +- `RequestIngress` and the fidelity types are unchanged. The H file is + byte-identical to the `v0.0.11` one, and `edgezero_core::ingress` still + exports them. + +### 7. Mechanical API changes + +| Change at H | Sites | Fix | +| ------------------------------------------------------------------------------------------------ | ------------------------------------------------------ | ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| `Body` is an opaque struct (`body.rs:24`) | 46 | Use `into_parts()`, `into_stream()`, `as_bytes()`, `is_stream()`. `BodyChunkSource::next_chunk` (`publisher.rs:218`, `:228`) converts once to a `BodyStream` or `Bytes` and holds that. Dropping a body that owns an outbound scope ends the scope (`body.rs:359-369`), so streamed outbound bodies stay alive until egress. | +| `Body::from_stream` takes `EdgeError` items (`body.rs:126`) | 17 | Publisher body streams pass the origin's `EdgeError` through, so origin failures keep their status class. Foreign `io::Error` streams, mostly in tests, use `from_external_stream` (`body.rs:103`), which maps to 500. | +| `RequestContext::into_request` returns `Result` (`context.rs:579`) | 46, from 15 roots | Propagate with `?`. | +| `RequestContext::request` and `request_mut` removed | 29 | `parts()`, `headers()`, `extensions()` and their `_mut` forms. `enforce_basic_auth` (`crates/trusted-server-core/src/auth.rs:83`), `sanitize_trusted_client_ip_headers` (`http_util.rs:60`), and `normalize_spin_request` (`crates/trusted-server-adapter-spin/src/app.rs:296`) take `&mut RequestParts`. Adapter context lookups read `ctx.extensions()`. | +| `Hooks::routes` and `Hooks::configure` return `Result` (`app.rs:672`, `:733`) | 4, plus each `configure` | Return `Err(EdgeError)` on a settings failure, as below. Delete the Axum, Cloudflare, and Spin `startup_error_router`s (`crates/trusted-server-adapter-axum/src/app.rs:583`, `crates/trusted-server-adapter-cloudflare/src/app.rs:411`, `crates/trusted-server-adapter-spin/src/app.rs:509`). Fastly's goes with section 3. | +| `ConfigStore::get` returns `ConfigValue`, `get_bounded` required (`config_store.rs:409`, `:416`) | 4 | Convert to `String` at the boundary (`crates/trusted-server-adapter-cloudflare/src/platform.rs:99`, `crates/trusted-server-adapter-spin/src/platform.rs:141`). The Spin test store implements `get_bounded`. | +| `SpinConfigStore::open` replaced by `for_label` | 1 (`crates/trusted-server-adapter-spin/src/app.rs:91`) | Use `for_label`. Open failures surface at the first read, through the existing error branch. | +| `run_custom` and `serve_custom` callbacks return `Result` | 2 (`main.rs:113`, `:138`) | Section 3. | +| `PreDispatchHook::handle` takes `PreDispatchRequest` | 1 impl and 17 test calls | Section 6. | + +**Startup failures.** Today the startup routers answer with the root error's +`status_code()` (`crates/trusted-server-adapter-axum/src/app.rs:584-585`, +`crates/trusted-server-adapter-cloudflare/src/app.rs:412-413`, +`crates/trusted-server-adapter-fastly/src/app.rs:1048-1049`), which is 500 for +settings errors, 503 for `KvStore`, 502 for upstream errors, and 400 for +`BadRequest` (`crates/trusted-server-core/src/error.rs:116-135`). Spin answers +503 for every failure (`crates/trusted-server-adapter-spin/src/app.rs:509-540`). +The cut keeps those statuses. A startup failure maps to the `EdgeError` +constructor for the root error's status: `internal` (500), +`service_unavailable` (503), `bad_gateway` (502), or `bad_request` (400) +(`edgezero:crates/edgezero-core/src/error.rs:225-596`). Spin always uses +`service_unavailable`. Trusted Server logs the report at error before +returning it. + +- The body becomes EdgeZero's redacted JSON with no `x-ts-version`. Today it is + Trusted Server's text, and the finalize step adds `x-ts-version` + (`main.rs:523-538`). +- Spin's `/health` no longer answers 200 while startup fails + (`crates/trusted-server-adapter-spin/src/app.rs:530-534`). The test at + `:1531` goes with the router. +- The Axum dev server now takes an assembled `App`, so a settings failure stops + it before it listens. Axum `main` returns `()` today + (`crates/trusted-server-adapter-axum/src/main.rs:13`). It logs the error and + exits non-zero. +- The Fastly, Cloudflare, and Spin smoke scripts keep their status and log + checks, since they already read the diagnostic from the log + (`scripts/smoke-common.sh:127-134`). Only `scripts/smoke-axum.sh:101-111` + changes. It asserts a non-zero exit and the log line instead of an HTTP 500. + +**Config and KV fallbacks on Cloudflare and Spin.** Trusted Server's Cloudflare +and Spin apps declare no `stores()` +(`crates/trusted-server-adapter-cloudflare/src/app.rs:445-461`, +`crates/trusted-server-adapter-spin/src/app.rs:549-565`). Only Fastly does +(`crates/trusted-server-adapter-fastly/src/app.rs:1376-1393`). At H those +adapters build a config registry only from declared metadata +(`edgezero:crates/edgezero-adapter-cloudflare/src/request.rs:770-775`, +`edgezero:crates/edgezero-adapter-spin/src/request.rs:568-572`), so +`ctx.config_store_default()` returns `None`. The `NoopConfigStore` and +`UnavailableKvStore` fallbacks +(`crates/trusted-server-adapter-cloudflare/src/platform.rs:636-644`, +`crates/trusted-server-adapter-spin/src/platform.rs:738-746`) are the live path +and stay. + +### 8. Axum, Cloudflare, and Spin entry points + +**Axum.** + +- `AxumDevServer::with_config(app, config)` now takes an assembled `App` + (`edgezero:crates/edgezero-adapter-axum/src/dev_server.rs:166-173`). + `crates/trusted-server-adapter-axum/src/main.rs:41-42` passes + `App::build::(AXUM_PLATFORM)` and exits non-zero on error + (section 7). Every policy that `configure` installs now applies over real TCP + (EZ-23). +- `AxumDevServerConfig` gains `ingress` (`dev_server.rs:58-64`), set from + `AxumIngressConfig::from_env(&EnvConfig::from_env())` (D16). An invalid value + fails startup. The variables are in [Configuration](#configuration). +- A request head over the raw-head byte limit or the field limit gets 431 + (`edgezero:crates/edgezero-adapter-axum/src/dev_server.rs:2069-2088`). Only a + connection over `MAX_CONNECTIONS`, which is dropped on accept + (`edgezero:crates/edgezero-adapter-axum/src/dev_server.rs:434-437`), or a + head that misses its read timeout is closed. Response headers share the + raw-head budget + (`edgezero:crates/edgezero-adapter-axum/src/ingress_config.rs:89-102`). + Accepted for the dev server. +- `EdgeZeroAxumService` is private (`edgezero:crates/edgezero-adapter-axum/src/lib.rs:29-30`). + The Axum route tests (`crates/trusted-server-adapter-axum/tests/routes.rs`), + the Axum parity leg (`crates/trusted-server-integration-tests/tests/parity.rs:21`, + `:81`, `:100`), and the CLI proxy test + (`crates/trusted-server-cli/tests/proxy_trusted_server.rs:84-92`) move to + `AxumTestServer` (`edgezero:crates/edgezero-adapter-axum/src/test_utils.rs:57-72`). + It needs the adapter's `test-utils` feature in those dev-dependencies. The + tests build the app with + `App::from_router::(routes_with_settings(..), AXUM_PLATFORM)` + so they can inject test settings, listen on loopback port 0, and attach + stores themselves, because the helper does not create them from metadata. +- The parity suite's Cloudflare and Spin helpers move to `AxumTestServer` too, + with each app assembled under its own platform metadata (EZ-23.8) and the + scripted client supplied through router state (EZ-23.7). Their production + policies then apply, which EZ-23 criterion 13 needs (test 19). For that + criterion's bodiless GET, the suite supplies EdgeZero's Axum client + (`edgezero:crates/edgezero-adapter-axum/src/outbound.rs:55`) instead and + records what a loopback origin receives. That client honors `HTTP_PROXY`, so + the test runs with the proxy variables unset. The Cloudflare and Spin route tests + in `tests/routes.rs` stay on `oneshot` (2.1). + +**Cloudflare.** + +- `run_app` keeps its signature + (`edgezero:crates/edgezero-adapter-cloudflare/src/lib.rs:157-181`). It builds + the app per request and returns assembly failures as an `Ok` response. +- An admission abort returns `Err` (`request.rs:299-310`). Trusted Server + installs no abort policy, so the existing 500 fallback + (`crates/trusted-server-adapter-cloudflare/src/lib.rs:36-39`) stays as a + guard. +- Trusted Server's client is deleted with its encoding strip + (`crates/trusted-server-adapter-cloudflare/src/platform.rs:400-415`), manual + redirect (`:267`), and 10 MiB cap (`:187`). H's Cloudflare client honors the + host override (`edgezero:crates/edgezero-adapter-cloudflare/src/outbound.rs:336-340`). +- Settings load from the `TRUSTED_SERVER_CONFIG` variable + (`crates/trusted-server-adapter-cloudflare/src/lib.rs:29-31`), not the config + store. Cloudflare declares no stores (section 7), so `services.config_store()` + is `NoopConfigStore` at `v0.0.11` and at H, and the `block_on(get)` bridge at + `platform.rs:99` is unused. Request signing on Cloudflare reads the no-op + store and fails, as today. The cut leaves both unchanged. + +**Spin.** + +- `run_app` (`edgezero:crates/edgezero-adapter-spin/src/lib.rs:173`) returns + `anyhow::Result`. `SpinResponse` is a streamed response that an + adapter task writes (`lib.rs:83-85`). The entry point declares + `anyhow::Result`, as H's template does. + The `anyhow` exception in `AGENTS.md` still applies. +- `SpinPlatformHttpClient` and `decompress_body` + (`crates/trusted-server-adapter-spin/src/platform.rs:372-690`) are deleted. + That also removes the deprecated `spin_sdk::wasip3::spawn` call (`:579`). +- The fork keeps the `spin-sdk` APIs Trusted Server calls (`variables::get`, + `time::sleep`, and `http_service`, at `platform.rs:701`, `:816`, and + `lib.rs`). The probe's `check-spin` reported no error at those uses. +- Spin ignores `origin_host_header_override` today and sends the URI authority + (`platform.rs:449-456`). Whether WASI hosts forbid an explicit `Host` is + unverified (EZ-13). H's Spin client rejects any override, so Spin's settings + load ignores the setting with a warning (D9). +- At H, Spin's client could send every buffered request body, bidder JSON + included, chunked with no `Content-Length`. Another reviewer raised this as + an open question on PR 275. At `14a6ca8b` the client sets `Content-Length` + on a buffered body for every method but GET and HEAD, and frames an empty + body only for POST, PUT, and PATCH + (`edgezero:crates/edgezero-adapter-spin/src/outbound.rs:900-915`). Test 5's Spin + leg records the framing a loopback bidder receives. Task 6 observed + `Content-Length: 428` with no `Transfer-Encoding`. +- Spin config reads go through `block_on(handle.get(..))` with + `SpinConfigStore::for_label` (`crates/trusted-server-adapter-spin/src/app.rs:86-108`). + Whether the fork's key-value future completes there is unverified. The Spin + cases in CI prove it (D14). +- Trusted Server builds `wasm32-wasip1`, and EdgeZero validates `wasm32-wasip2` + (`edgezero:crates/edgezero-adapter-spin/src/cli.rs:129`). Only a running + component proves it instantiates (D14). + +**Worker deadline job.** `.github/workflows/worker-auction-deadlines.yml` runs +on every pull request and is `AGENTS.md` gate 15. It is not a required status +check. It builds the Cloudflare and Spin `deadline-runtime` examples +(`crates/trusted-server-adapter-cloudflare/Cargo.toml:52-56`, +`crates/trusted-server-adapter-spin/Cargo.toml:49-53`), which include each +adapter's production `platform.rs` and call `PlatformHttpClient` +(`crates/trusted-server-adapter-cloudflare/tests/runtime/deadline.rs:1-11`). +`scripts/test-worker-auction-deadlines.mjs` drives them against delayed headers, +trickling bodies, and slow publisher work. The cut ports both examples to the +adapter's EdgeZero `HttpClient`, through `start_batch_until` and `collect`, +against the same delayed origins, asserting the same outcomes. Every case models +a bid, so the examples call no single `send`. The runtime integration tests in +`crates/trusted-server-integration-tests/tests/integration.rs`, which CI runs +with `--include-ignored` and the Spin filter, cover the real client's `send` on +Cloudflare and Spin. The script keeps its cases. Gate 15's text stays, because +its build and run commands do not change. + +**Fastly deadline harness.** The required `cargo test` job runs +`crates/trusted-server-adapter-fastly/tests/deadline-origin.mjs` +(`.github/workflows/test.yml:58-59`, `AGENTS.md` gate 14). It runs the ignored +tests `platform::deadline::tests::network_*` +(`crates/trusted-server-adapter-fastly/src/platform/deadline.rs:795-946`) under +Viceroy against a slow local origin, and asserts an exact pass count +(`deadline-origin.mjs:148-151`). Those tests reach the origin through a static +`deadline_origin` backend, which is the only place the harness writes the +origin's random port (`deadline-origin.mjs:103-112`), and the file is deleted. +H derives backends from the request URI and cannot target a named static +backend (`edgezero:crates/edgezero-adapter-fastly/src/outbound.rs:977-1023`). + +- The cut moves the cases to a new test module in the Fastly adapter binary. + Each case builds `HttpClient::with_client(FastlyOutboundClient::new())` and + calls `start_batch_until` with `http://127.0.0.1:` URIs. The module path + must not start with `platform::deadline`, because the dependents inventory + searches that string. A module at the crate root, such as + `deadline_network`, gives the filter `deadline_network::tests::network_`. +- The harness writes the port into an inline Viceroy config store named + `deadline_fixture`, in place of the static backend, and the tests read it + with `fastly::ConfigStore::open`. Viceroy 0.21.0's `run` and `serve` both + build their context through `create_execution_context`, which installs the + config's dictionaries, and `config_stores` is a serde alias for + `dictionaries` (Viceroy `cli/src/execute_ctx.rs:96-113`, `src/config.rs:270`). + The current harness already relies on the same path for + `local_server.backends`. +- Three cases keep their outcomes: stalled headers and a trickling body end at + the cutoff, and a HEAD skips its declared length. +- The fast-sibling case drops its missing-cache-bypass slot, because H accepts + any cache policy in a batch (`outbound.rs:1082-1091`). +- The expired-snapshot case changes. After the sleep it calls `batch.next()`. + That call expires both slots at the cutoff and returns `Finished(Cutoff)` + without a host wait, so H drops the fast reply that landed during the sleep + (`outbound.rs:921-926`, + `edgezero:crates/edgezero-core/src/outbound/batch.rs:645-650`). The case + asserts that drop. At the pin under Viceroy 0.21.0, a `batch.check()` at the + same point also returns `Finished(Cutoff)`, so the case asserts only + `batch.next()`. A new case asserts that a reply observed + during an EdgeZero wait before the cutoff is kept. +- An EdgeZero wait drives a batch with no request lifecycle. With no canonical + owner, the batch gets a caller-owned scope + (`edgezero:crates/edgezero-core/src/outbound/scope.rs:314-338`), and + `batch.next()` reaches `fastly_async_io::select` through the native runtime + (`edgezero:crates/edgezero-adapter-fastly/src/outbound/progress.rs:315-383`). + Every ported case depends on that path. +- The harness filter changes to the new module path, and the count assertion + matches the number of ported cases. Gate 14's text stays, because it names + only the harness path. + +### 9. Logging + +Trusted Server keeps `owns_logging` false. Its Fastly and Axum loggers already +admit target `edgezero::boot` at warn +(`edgezero:crates/edgezero-core/src/logging.rs:10-12`), where assembly failures +log once (`app.rs:113-146`). Outbound failures log +`provider_failure_detail()` (`error.rs:490`) with `{:?}` at warn through the +helper in 2.11. The detail holds closed enums and numeric codes, never provider +text or URLs (EZ-16). + +### 10. Documentation + +- Guide pages that name the EdgeZero version, or describe outbound behavior, + redirects, timeouts, header rules, or inbound body limits. +- The rejected settings (D9, D10, D18) and the doc comment on + `certificate_check`. +- The Axum ingress variables (D16) and the Axum startup-failure exit. +- CLI manifest discovery. `build`, `serve`, and `deploy` resolve + `EDGEZERO_MANIFEST`, then the nearest ancestor `edgezero.toml`, then a + workspace scan (`edgezero:crates/edgezero-cli/src/manifest_source.rs:117-126`). +- `docs/guide/architecture.md`, which names the deleted platform types. +- `docs/guide/auction-orchestration.md`, which describes + `ProviderRequestOutcome`, `PlatformResponse`, and fan-out through Fastly + `select()` (`:11`, `:188`, `:334`). +- The `runtime-services` snippet in `docs/guide/integration-guide.md` + (`:31-186`). `crates/trusted-server-integration-tests/tests/documentation_snippets.rs:5-10` + compiles it, and the integration-tests job runs that test. The snippet + implements `PlatformBackend`, the synchronous store writes, and + `.http_client(..)`, so it changes in the same commits as those APIs: step 8 + makes its store writes async, and step 9 removes its backend and HTTP client + and builds `RuntimeServices` with `outbound_client` and `clock`. The guide's + prose after the snippet (`:195-203`) changes in step 11. The test writes the + snippet into a fresh crate whose manifest lists only `error-stack`, + `futures`, and core (`documentation_snippets.rs:60-63`). So the test adds + `async-trait` to that manifest at the workspace requirement, read the way it + reads `error-stack` (`:50-59`), and `trusted_server_core::platform` + re-exports `HttpClient` and `MonotonicClock`, as it re-exports the KV types + (`platform/mod.rs:54`). The snippet's `runtime_services` takes the client and + clock as arguments, as adapters pass `ctx.http_client()` and + `ctx.monotonic_clock()`. +- `crates/trusted-server-core/src/auction/README.md` and `scripts/README.md`, + which describe the deleted layer and the changed harnesses. +- A `CHANGELOG.md` entry. + +## Adapter impact + +| Adapter | Compile changes | Runtime changes | +| ---------- | ------------------------------------------------------------------------------------------------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| Fastly | 38 sites, the lifecycle rewrite, deletion of the client, backends, and delivery code | [Fastly differences](#fastly-differences) | +| Axum | 10 sites, the dev server `App` and `ingress` field, route and parity tests on `AxumTestServer` | EdgeZero client in place of Trusted Server's. Ingress caps. Framing, head limits, and `configure` policies apply. Inbound 30 s read deadline. Publisher origin input is capped at `publisher.max_buffered_body_bytes` for every classification, its first input cap, and raw responses are collected before egress (2.5). Startup failure exits the process | +| Cloudflare | 22 sites, deletion of the client, two compatibility flags, the deadline example | Multi-provider auctions load and fan out. EdgeZero client in place of the encoding strip (`Decode`, 2.5) and manual redirect. Publisher origin input still capped at 10 MiB for every classification, and raw responses collected before egress (2.5). Strict framing. Head limits. Inbound 30 s read deadline. Startup failure answers with the assembly error | +| Spin | 34 sites, the `spin-sdk` fork, `SpinResponse`, deletion of the client and decoder, the deadline example | Multi-provider auctions load and fan out. Publisher origin input and HTML expansion still capped at 8 MiB for every classification, and raw responses collected before egress (2.5). Streamed egress. Head limits. Inbound 30 s read deadline. Startup failure answers 503 with the assembly error. Config open failure at first read. Outbound-host manifest check | + +Core carries 63 sites plus the outbound migration. The CLI library compiles +against H unchanged, and its tests change only for the Axum harness. + +### Fastly differences + +Intended changes to Fastly behavior. The response diff's expected set lists +each one a request exercises. + +Inbound: + +1. A forwarded GET or HEAD that carries body bytes returns 400 (EZ-1). +2. A body read or forwarded more than 30 s after the request arrived returns 408. +3. A publisher upload over 16 MiB returns 413. The origin sees the head and up + to 16 MiB of an aborted upload (2.5). +4. A request target over 8192 bytes returns 414. More than 100 header fields or + 64 KiB of headers returns 431. Ambiguous or malformed request framing + returns 400. + +Responses: + +5. Framework errors render as EdgeZero JSON. Internal text is redacted. 405 + carries `Allow`. Detached responses other than startup failures carry the + version, environment, geo, and operator headers through the synthesized set, + or only the version headers when that set fails its bounds. They carry no EC + cookies, cookie cache privacy, or sandbox counters (section 5). +6. Hop-by-hop fields are stripped, and `Trailer` is never sent. +7. Bodiless responses (HEAD, 204, 205, 304) skip the SDK's bulk framing + (`edgezero:crates/edgezero-adapter-fastly/src/response.rs:399-405`), so their + `Content-Length` and `Transfer-Encoding` can differ from today's (D20). +8. A startup failure answers with EdgeZero's redacted JSON and carries no + `x-ts-version`, `x-ts-fastly-version`, or finalize headers (EZ-19.7). Today + it carries `x-ts-version`. Its status is unchanged. +9. A response body must finish within 60 s after egress starts. + +Outbound: + +10. A streamed origin that stalls fails at the 60 s total instead of the 15 s + first-byte or between-bytes timer, and a healthy streamed transfer longer + than 60 s is cut. There is no idle bound until EZ-14. +11. Buffered calls have a 15 s total instead of 15 s idle timers. Redirect + chains have a 60 s deadline. +12. Backend names are `ez_b31v1_*`, and provider timers round up to a power of + two of milliseconds. A provider `NameInUse` fails the request with the + site's 502. EdgeZero reports it as an internal error + (`edgezero:crates/edgezero-adapter-fastly/src/outbound.rs:1132-1134`, + reached through `finish_backend_creation` at `:1261-1291`). Today Trusted + Server treats `NameInUse` as reuse + (`crates/trusted-server-adapter-fastly/src/backend.rs:262-265`). Fastly + documents that re-registering a name with identical properties succeeds, + and that conflicting properties fail. EdgeZero hashes the whole backend + configuration into the name + (`edgezero:docs/guide/adapters/fastly.md:733-737`), so a `NameInUse` means + a real collision. The reviewer who asked about this accepted that on the + PR 275 thread. The hosted same-name controls stay in EdgeZero's Fastly + qualification. +13. TLS is always verified. `certificate_check = false` is ignored (D9). +14. A request or response header field whose value is not UTF-8 is dropped, + with a warning that names the header. That includes `Cookie` on the + publisher forward and origin `Set-Cookie`. Today they pass as raw bytes. +15. Upstream requests with no `Accept-Encoding` carry `identity`. +16. Hop-by-hop request fields, and fields that `Connection` names, no longer + reach upstreams. A malformed request `Connection` returns 400 on the + publisher forward. A malformed response `Connection` returns the site's 502. +17. The origin path is WHATWG-normalized. A request target with a fragment, a + backslash, or userinfo returns 400 on the publisher forward. +18. Redirects follow the differences in section 2.6, including a 502 for a + `Location` with userinfo. +19. An auction that is abandoned, or whose page response ends early, cancels its + in-flight bidder and mediator requests on the guest side. Host cancellation + is best effort, so a bidder may still see the whole request. Today they run + to completion. +20. Bidder requests leave together after all are built, not one at a time. +21. Telemetry goes out after delivery. Pull sync and telemetry run only after a + committed application body. +22. A bid that completes while no EdgeZero poll services the provider, and is + first observed after the cutoff, is dropped. At H a `batch.check()` does not + observe it (2.4). Today a snapshot after the deadline keeps it (#1433). +23. Pull sync has a 15 s cutoff per group. +24. Origin readthrough is unavailable (D9). + +Responses, continued: + +25. A streamed relay whose origin declares `Content-Length` keeps it instead of + chunked framing (test 7). + +Outbound, continued: + +26. Bidder responses have no aggregate buffer cap. Today Trusted Server's + Fastly collector caps the partial bid bodies it retains at 10 MiB together + (`MAX_TOTAL_BUFFERED_BODY_BYTES`). After the cut each bidder response keeps + its own 1 MiB cap, across up to 160 provider backends. The maintainer + accepted the per-bidder cap as enough, so the cut adds no aggregate cap and + asks EdgeZero for none. + +Unchanged, and checked by the response diff: routes, other statuses, other +headers, auction semantics, consent handling, EC cookies, upstream failures as +502 (D17), and `x-compress-hint` on tsjs bundles (`publisher.rs:597`). + +## Configuration + +**No new configuration keys.** Validation changes: + +| Setting | Today | After the cut | +| ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ | -------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| `proxy.certificate_check = false` | Disables origin TLS checks for the publisher, asset, redirect, and pull-sync backends (`crates/trusted-server-adapter-fastly/src/backend.rs:244-250`). Default `true` (`settings.rs:1937`) | `ts config validate` and `push` reject it. Runtime ignores it with a warning, which replaces `warn_if_certificate_check_disabled` (`crates/trusted-server-adapter-fastly/src/app.rs:241`). Only origins with invalid certificates then fail. | +| `creative_opportunities.origin_readthrough_enabled = true` | Shared-cache origin fetch with a surrogate key. Off by default (`crates/trusted-server-core/src/creative_opportunities.rs:424-425`) | Validate and push reject it. Runtime treats it as `false` with a warning. | +| `publisher.origin_host_header_override` on Spin | Ignored. Spin sends the URI authority (`crates/trusted-server-adapter-spin/src/platform.rs:449-456`) | Still ignored, now with a warning at settings load. Fastly, Axum, and Cloudflare send it through `host_authority_override`. | +| `publisher.origin_url` with userinfo | Accepted | Validate and push reject it. At runtime H rejects the request (2.11), so publisher requests answer 400. | +| `response_headers` naming `trailer`, `content-length`, or a hop-by-hop field (`connection`, `keep-alive`, `te`, `transfer-encoding`, `upgrade`, `proxy-connection`, `proxy-authenticate`, `proxy-authorization`) | Applied | Validate and push reject the entry. Runtime skips it with a warning. A malformed `Connection` value would otherwise fail H's egress framing (section 5). | +| Multi-provider `auction.providers` on Cloudflare or Spin | Startup fails (`crates/trusted-server-core/src/auction/plan.rs:417-466`) | Loads and fans out (2.4). | +| Provider endpoint `http://127.0.0.1:` or `http://[::1]:` | Rejected. Endpoints must be HTTPS (`plan.rs:565-584`) | Accepted (D18). Other `http` endpoints are still rejected. | + +Validation rejects at deploy time and degrades at runtime. A live config that +predates the cut keeps serving, and the pre-deploy check finds every entry to +fix. + +The Spin row deviates from EZ-13's interim, which asked for rejection at +validation on Spin builds. `ts config validate` takes no adapter +(`edgezero:crates/edgezero-cli/src/args.rs:609-626`), so it cannot reject a +Spin-only setting. Spin's own settings load could reject it, but that would fail +startup for configs that serve today, which contradicts the rule above. The +requirements doc keeps its text. This spec records the deviation. EZ-13 is not +confirmed present (D19), so its criteria stay out of the gate, and its +Trusted Server criterion 3, which expects validation to fail, is not run. + +Image Optimizer profiles need no change. Settings load already rejects +everything H's types cannot express (section 2.7). + +**Constants, not configuration (D7).** The connect timeout is 1 s. Per-call +timeouts are the site's configured timeout where one exists, 2 s for Tinybird, +and otherwise 60 s streamed and 15 s buffered. Redirect chains get a 60 s +deadline. The response write deadline is 60 s after egress starts, and the +publisher upload cap is 16 MiB. Existing endpoint caps are unchanged. + +**Axum ingress variables (D16).** Values must be positive integers in range, +and an invalid value fails startup +(`edgezero:crates/edgezero-adapter-axum/src/ingress_config.rs:28-58`, +`:185-199`). + +| Variable | Default | Range | +| -------------------------------------------------- | ------- | -------------- | +| `EDGEZERO__ADAPTER__INGRESS__MAX_CONNECTIONS` | 256 | 1 to 4096 | +| `EDGEZERO__ADAPTER__INGRESS__MAX_RAW_HEAD_BYTES` | 65536 | 8 KiB to 1 MiB | +| `EDGEZERO__ADAPTER__INGRESS__MAX_HEADER_COUNT` | 100 | 1 to 1024 | +| `EDGEZERO__ADAPTER__INGRESS__HEAD_READ_TIMEOUT_MS` | 10000 | 1 ms to 120 s | + +**Cloudflare publisher origin decoding.** No setting changes. The Cloudflare +publisher origin decodes origin bodies (D8 deviation, 2.5), as Trusted Server's +Cloudflare client did, so a Cloudflare response carries the origin's +`Content-Encoding` only on 206, HEAD, and 304. + +**Unchanged for operators.** The app-config envelope, the canonical hash, the +Fastly chunk pointer format (`fastly_config_chunks` version 1, +`edgezero:crates/edgezero-adapter-fastly/src/chunked_config.rs:11-12`), store +links, secrets, and deploy flags are unchanged. No re-push is needed. + +## Consent and privacy + +- Bidder requests carry the same consent and attested client-IP headers as + today. Test 3 asserts the headers on every bidder request. +- A header field whose value is not UTF-8 is dropped whole on the publisher + forward and from origin responses. One raw-byte cookie in a combined `Cookie` + header therefore drops every cookie to the origin, consent cookies included, + and a raw-byte `Set-Cookie` no longer reaches the browser. The warning names + the header, never its value. +- Detached responses fail closed. Head-limit and framing rejections happen + before any handler, so they set no EC cookie and do no consent work. +- EC finalize and cookie writes run in `finalize` for every routed response, as + they run after routing today. A framing fallback after `finalize` keeps its + EC KV writes but drops its cookies, so the browser and the store can + disagree for that one response. Framing fallbacks are unreachable from + Trusted Server's sources after D10. +- Pull sync runs only after a committed application response. Its consent + gating is unchanged. +- An abandoned auction cancels its bidder requests, so bidders may see a + request cut off mid-response. Nothing extra is sent. +- Cross-origin redirects strip `Authorization` and `Cookie`, which is stricter + than today. +- The proxy marker is off, so browsers see no new header. +- TLS is always verified, which is stricter than a config with + `certificate_check = false`. +- EdgeZero logs redirect hops at debug without the query. Provider failure + details hold no provider text, URLs, or header values. Trusted Server never + logs body bytes. +- Telemetry rows have the same content, are sent later, and measure bidder time + from the batch start to the recorded arrival, not to collection (2.4). + +## Failure handling + +| Failure | Behavior | +| ------------------------------------------------------------------------------------ | -------------------------------------------------------------------------------------------------------------------------------------- | +| Inbound body over the endpoint cap, declared or streamed | 413 `RequestTooLarge`, logged at warn | +| Inbound read after the 30 s read deadline | 408 `RequestTimeout`, logged at warn | +| Ambiguous or malformed request framing | 400 from EdgeZero's ingress, before any handler runs | +| Any other inbound read failure mid-body, client-caused or host-side | 502 `Proxy`, logged at error. Every adapter maps the source error to `EdgeError::internal` | +| Forwarded GET or HEAD with body bytes, or an invalid forwarded URI | 400 `BadRequest` (2.11) | +| Publisher upload over 16 MiB | 413 `RequestTooLarge` (2.11) | +| Origin, integration, or connect timeout | The site's 502, as today. The budget source is attached to the report | +| Bidder still pending at the cutoff | Provider timeout in the auction result | +| Auction abandoned or page response ends early | In-flight bidder and mediator requests are cancelled | +| Redirect policy rejection | Today's `Forbidden`, `AllowlistViolation`, or `Proxy` error | +| Redirect hop failure, a bad `Location`, or a redirect chain past 60 s | The site's 502 | +| Image Optimizer without entitlement, or on Viceroy | The asset proxy's 502, from `ImageOptimizerUnavailable` | +| Fastly `NameInUse`, a backend name collision | The site's 502 | +| App assembly fails | EdgeZero's assembly error response with today's status (503 on Spin). Not retained, so the next request retries. Axum exits at startup | +| Fastly delivery not committed | Native 500 | +| Fastly delivery unknown or committed with a fallback body | Logged. No post-delivery work | +| Fastly request head cannot be converted | Native 500, as today | +| Framing violation | EdgeZero 500 fallback. Unreachable from Trusted Server sources after D10 | +| Telemetry or pull-sync send fails | Logged at warn. The client already has its response | +| `certificate_check = false`, readthrough, or forbidden `response_headers` at runtime | Warning, and the setting is ignored or skipped | +| `origin_host_header_override` on Spin | Warning, and the setting is ignored, as today | +| Invalid `EDGEZERO__ADAPTER__INGRESS__*` on Axum | Startup fails with the parse error | +| Spin manifest outbound hosts disagree with `spin.toml` | `ts build`, `ts serve`, and `ts deploy --adapter spin` fail with "Spin outbound host drift" | +| Spin component fails to instantiate | CI's Spin job fails. Blocks the cut (D14) | + +## Compatibility and rollout + +### Merge gate and pin + +`main` merges the cut without waiting for EdgeZero's EZ-Q1 record, and +production deploys do not wait for qualification (D5, Q1). The maintainer +qualifies Trusted Server's hosted legs by hand +([EdgeZero qualification, Trusted Server's part](#edgezero-qualification-trusted-servers-part)). + +The cut is stacked on PR 275, because its Axum changes need it (Q5). The cut +pins PR 275's head, repins whenever that head moves, and rebases onto `main` +regularly. At each repin it reruns the compile probe's tools to re-diff the +public API, and reruns the dependents inventory (Static). `main` pins the head +the cut was last rebased onto. + +This accepts two risks: + +- `main` and production can run a PR 275 head that EdgeZero has not + qualified. PR 275 is open with change requests from two reviewers. + EdgeZero's qualification rule says a defect forces a new candidate and a full + rerun + (`edgezero:docs/superpowers/plans/2026-10-18-trusted-server-outbound-05-qualification.md:416-417`). + Without the gate, Trusted Server gets such fixes at its next repin. +- A force-push to the PR branch can orphan the pinned commit, and an orphaned + commit breaks `--locked` builds once it is collected. The pin then moves to + the new head. + +Once PR 275 merges, Trusted Server moves from the PR 275 commit to an EdgeZero +release tag (Follow-ups). + +Auctioneer pins EdgeZero `775366fd`, an earlier PR 275 commit. #1505 needs both +projects on one revision, so Auctioneer moves to Trusted Server's PR 275 commit +after the cut. + +### Open EdgeZero defects at H + +Review 5798502238 on PR 275 (2026-11-06, at H) checked every requirement +against EdgeZero's own copy of the contract. It found five blocking defects. +Trusted Server adds no workaround (D2). Re-reviews verified EdgeZero's fixes +for all five: review 5810546608 at `598f0a2a` for defects 1, 3, 4, and 5, and +review 5815197468 at `6f0cc74c` for defect 2. Because the cut follows PR 275's +head (D5), it picks those fixes up. + +1. **Fastly batch setup (EZ-6.7, EZ-6.8).** Setup stops at the first slot that + completes synchronously, such as a slot whose deadline passed during another + slot's cold registration. Later bidders go out at the next EdgeZero poll, and + a batch that is never polled never sends them (2.4). +2. **Fastly checks (EZ-6.11, cross-cutting requirement 5).** `check()` and + `is_ready()` record no arrivals (2.4). +3. **Spin cutoff (EZ-6.14, EZ-6.16).** Spin can report the cutoff before it + observes an expired earlier-deadline slot, and the whole batch then fails + with an internal error. On Spin every bidder in that batch then fails, + instead of one timing out (2.4). +4. **Qualification build (EZ-Q1.4).** EdgeZero has no single-request + `run_custom` Fastly build for the hosted criteria, and no frozen-record + export. Step 4 of the run below cannot start without them. +5. **Inventory (EZ-Q1.6, EZ-Q1.8).** The inventory cannot bring EZ-16 to EZ-22 + into the gate. Their 57 EdgeZero-owned local legs are planned and name no + test, and 30 of them use a command the harness rejects. + +Three of its should-fix findings change what Trusted Server sees. Fastly batch +slots and single sends take different SDK cache paths (EZ-2.9). EdgeZero's own +readers drain the body before a declared-length 413 (EZ-10.4, section 4). The +Cloudflare inbound read deadline can fire up to 1 ms early. + +Other reviewers' open threads also reach Trusted Server: + +- At H, Spin could send buffered request bodies chunked with no + `Content-Length`. The pin sets the length (section 8). +- Every non-batch Fastly send waits for its response head in 1 ms polls, which + costs guest CPU while an origin is slow. +- An upstream response with both `Transfer-Encoding` and `Content-Length` keeps + the stale length. +- On Spin, request headers that wasmtime forbids, such as `HTTP2-Settings`, fail + a forwarded request with 400. +- On Axum and Spin, a client-caused inbound body error, such as malformed + chunked framing or an aborted upload, is an internal error + (`edgezero:crates/edgezero-adapter-axum/src/request.rs:185`, + `edgezero:crates/edgezero-adapter-spin/src/request.rs:227`). Trusted + Server's body helper then answers 502 and logs at error. At the pin, Fastly + and Cloudflare map such an error the same way, so Failure handling lists 502 + for every adapter. + +The earlier question on Fastly `NameInUse` is answered (Fastly differences item +12). + +### EdgeZero qualification, Trusted Server's part + +At H no qualification record exists. EdgeZero's inventory lists 261 required +results, all missing (217 local, 44 external). No release leg is accepted, and +the approvals G4-FASTLY-POLICY, G4-OPTIMIZER-HORIZON, G4-APPLICABILITY, and +G4-MEASUREMENTS are pending +(`edgezero:docs/audits/2026-10-18-outbound-api-approval.md:30-41`). Since +`6f0cc74c`, its Strict Suppression And SDK Deviation Record (lines 795 to 809 +at `5ea8d93c`) records the requester's choice to keep strict suppression, +without waiving EZ-18.4. D20 withdraws EZ-18.4's bodiless part, and EdgeZero +amended the contract to match in `dcbf622` +([PR 275 thread r4504573926](https://github.com/IABTechLab/edgezero/pull/275#discussion_r4504573926)): +EZ-18.4's SDK parity now covers only responses that can carry content, and +HEAD, 204, 205, and 304 follow EZ-18.6 strict suppression. +Trusted Server owns 24 of the 44 external legs: 15 local consumer legs, 4 hosted +legs on its own build, and the 5 consumer reviews. It runs the other 20 on its +staging service. The inventory assigns all 20 to EdgeZero's qualification +build, EZ-2 criteria 7 to 10 and 13a included, while the requirements doc and +the table below give those five to Trusted Server's build. EdgeZero reconciles +the two before the hosted phase. + +These counts predate D19. Its confirmed follow-ups add 11 Trusted Server legs, +the EZ-10 to EZ-23 rows below. They also bring in five hosted legs that the +inventory gives EdgeZero: EZ-16 criterion 3, and EZ-18 criteria 3 and 3b on +HTTP/1.1 and HTTP/2, all on Trusted Server staging. EZ-Q1.4 does not list them +for EdgeZero's qualification build, so they fall to Trusted Server's build, and +EdgeZero reconciles them with EZ-2's five. + +Trusted Server's inputs to the hosted legs: + +- The maintainer is the operator and runs Trusted Server's hosted legs by hand + for now (Q3). For each run, the maintainer chooses the staging service, its + Image Optimizer entitlement, and the thresholds and sample counts for EZ-2's + M1 to M7, and the run's record states them. +- HTTP/3 is qualified per adapter, not through Trusted Server's staging + service (Q4). EdgeZero amended EZ-1 criterion 2 to match in `dcbf622` + ([PR 275 thread r4504554519](https://github.com/IABTechLab/edgezero/pull/275#discussion_r4504554519)). + Each adapter now has its own HTTP/3 leg that needs a disposition, and none + is conditional on `hosted HTTP3 enabled`. +- The baseline commit for EZ-2 (D13). EZ-2's text and the runbook field + (`edgezero:crates/edgezero-test-harness/acceptance/ts-outbound-qualification-runbook.md:69`) + change to match. + +The run: + +1. At each repin, the cut branch runs Trusted Server's local legs. +2. The maintainer deploys Trusted Server's build of the pinned head to staging + and runs its hosted legs by hand. +3. When EdgeZero's qualification build exists for the pinned head, the + maintainer deploys it to the same staging service for EdgeZero's hosted + legs. At H that build does not exist (open defect 4). Review 5810546608 + verified it at `598f0a2a`. +4. Trusted Server records its consumer review. EZ-5.9 and EZ-5.10 are not + needed, because relays and the S3 preflight use `Original`. EZ-11 and EZ-14 + are not needed for the cut, because D9 and D7 cover them. Trusted Server + confirms the follow-ups in D19 as present, so their criteria join the gate + (EZ-Q1.6), with the Trusted Server legs in the table below. Reuse mode is + not adopted in production. EZ-22 criterion 9 still joins the gate, because + CI builds the `reusable-sandbox` feature. +5. None of these steps gates the merge or production deploys (D5). A failed + leg goes to EdgeZero on PR 275, and Trusted Server takes the fix at its next + repin. + +Trusted Server's working list of legs follows. EdgeZero's inventory +(`edgezero:crates/edgezero-test-harness/acceptance/outbound-cutover-cases.json`) +is authoritative. At H it lists the confirmed follow-ups' legs only as +`follow-up`, with no `present` decision that brings them into the gate (open +defect 5). + +Stock Viceroy 0.21.0 cannot supply timing evidence (Runtime), so the Viceroy legs +Trusted Server adds to the record run on the repaired Viceroy that EdgeZero's +CI builds (`edgezero:scripts/ci_viceroy.sh`, `repair` mode, +`edgezero:.github/workflows/test.yml:387`, `:504`). The record names that build +as repaired (EZ-Q1.3), as EdgeZero's patch README requires +(`edgezero:patches/viceroy/README.md:3-5`). Trusted Server's review under EZ-Q1 +criterion 3 accepts that build as EZ-Q1.3's Viceroy 0.21.0 only once EdgeZero +records the repaired-versus-stock disposition under G4-FASTLY-POLICY +(`edgezero:docs/audits/2026-10-18-outbound-api-approval.md:50-51`). + +| Req | Local, Trusted Server build | Hosted, Trusted Server build | Hosted, EdgeZero build on Trusted Server staging | +| ----- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------- | -------------------------------------------------------------------------------------------------------------------------------------------------------- | ------------------------------------------------ | +| EZ-1 | Criterion 13 on Viceroy, workerd, and Axum (test 1) | Criterion 5: an HTTP/2 GET with a DATA frame and no `Content-Length` gets 400, and the origin records nothing | Criteria 2 and 6 | +| EZ-2 | Criterion 15 on Viceroy | Criteria 7 to 10, 10b (EZ-16 is present), and 13a | Criteria 11, 12, 13b, 14a, and 14b | +| EZ-3 | Criterion 6, a 12-bidder auction on Viceroy (test 5) | Criterion 7 (M7) | | +| EZ-4 | Criterion 10, a connect-timeout readback unit test (test 3) | Criterion 9 | Criterion 8 (M8) | +| EZ-5 | Criterion 8 on Viceroy (test 7) | | | +| EZ-6 | End to end on Viceroy, Axum, workerd, and Spin (test 6) | | Scenarios 1, 2, 6, 7, 8, and 11 | +| EZ-7 | Criterion 5 on Viceroy (test 3) | | | +| EZ-8 | Criterion 9, a profile readback unit test (test 3) | The Trusted Server criterion on the IO-entitled service, with a final HEAD's metadata compared with the GET's | Criterion 8 on the IO-entitled service | +| EZ-9 | Criterion 20, unit and Axum (test 4) | | | +| EZ-10 | Criterion 9: today's read-cap 413 tests unchanged, an oversized chunked `POST /auction` (test 1), and a batch-sync stream past its declared length and cap (test 8) | Criterion 4: an HTTP/2 POST with no `content-length` over the cap gets 413 | | +| EZ-16 | | Criterion 3, and criterion 8: a provider on a closed port logs one line with the provider and the refused cause, and the auction error body stays static | | +| EZ-17 | Criterion 7 on Viceroy (tests 9 and 13) | | | +| EZ-18 | Criterion 6 on Viceroy, through the response diff. Publisher HEAD and 304 follow D20 | Criteria 3 and 6 on hosted Fastly. HEAD, 204, and 304 follow D20 | | +| EZ-19 | Criterion 6 on Viceroy (test 11) | | | +| EZ-20 | Criterion 5 on Viceroy, through the response diff | | | +| EZ-22 | Criterion 9 on Viceroy with the `reusable-sandbox` feature (test 9), with today's 500, not 503, for the missing blob. Criterion 10 deferred (D19) | | | +| EZ-23 | Criterion 13 on the Axum test server (test 19) | | | +| EZ-Q1 | The five consumer reviews (step 5) | | | + +### Before deploy + +- Run the new `ts config validate` against each environment's operator-owned + `trusted-server.toml`. A failure names a setting from the Configuration + table. +- Run the Viceroy response diff. On Fastly staging, run its read-only subset + against production, with no admin writes or key rotation, and also normalize + geo headers, `x-ts-fastly-version`, and `x-ts-env`. No read-only requests + file exists. Build the subset from `scripts/fastly-response-diff.requests.json` + by dropping `trace-enable-action`, the `POST /_ts/trace/enable` request, and + `admin-eids`, the `GET /_ts/admin/eids` request with fixture credentials. +- Then deploy to production. + +### Rollback + +Revert the pull request and redeploy the previous build. Every config change the +new validation asks for removes or tightens a setting that the old build also +accepts, so rollback needs no config change. Two exceptions run the other way. +A multi-provider Cloudflare or Spin config accepted after the cut fails startup +again on the old build. A loopback `http` provider endpoint fails validation +again, which affects only test fixtures. `[capabilities.outbound]` and the +wrangler flags revert with the pull request. + +### Operator-visible changes + +- Fastly dynamic backend names become `ez_b31v1_*`. Dashboards keyed on backend + names change. +- The template-cache build digest hashes `Cargo.lock` and `edgezero.toml` + (`crates/trusted-server-core/build.rs:26`), so the first deploy misses shared + template-cache entries once. +- One info log line per request carries the outbound observation counts + (section 3). +- The Axum dev server has ingress caps and its own variables, and exits on a + startup failure. + +### In-flight work + +A scan of the 118 open pull requests on 2026-11-05 found 50 with heavy overlap, +14 with moderate overlap, and 10 with light overlap. Heavy means the PR edits +files the cut deletes or rewrites, or changes the EdgeZero pin. Moderate means +it adds outbound sites or platform-type uses that need porting. + +| PR | Conflict | +| ------------------------------------------------------------ | -------------------------------------------------------------------------------------------------------------------------------- | +| #1490 November RC, #1424 AWS RC | Pin other EdgeZero revs and touch the platform HTTP layer. Rebuild on the cut | +| #1453 credential bindings | Pins another rev. Touches Axum and Spin `platform.rs`, test support, `proxy.rs`, and `PlatformBackend` | +| #1261 request timing | Pins another rev. Its hooks sit in the Fastly delivery code the cut replaces | +| #1487, #1488, #1489 server-side EIDs | New fan-out on `send_async` and `select`. Must use `start_batch_until` | +| #884, and #894 stacked on it | #884 pins another rev and edits Fastly `main.rs`. #894 uses `PlatformHttpClient` and `proxy.rs` | +| #1053 Prebid latency spec | Pins a `stackpop/edgezero` branch. Cargo treats another git URL as another source, so it must move to `IABTechLab/edgezero` | +| #1193 Fastly TTFB | Edits Fastly post-send work and `platform/types.rs` | +| #917 absolute creative URLs | Textual conflict in the `proxy.rs` rebuild handler and Fastly `app.rs` tests | +| #839 EdgeZero Phase 1 | Superseded by the cut | +| #1343 to #1448 stack (55 PRs) | 36 heavy, 9 moderate. The worst (#1448, #1390, #1348, #1399) touch the HTTP layer, backend naming, and the Image Optimizer types | +| #1310 ID5, #1325 and #1437 instream video, #1460 CWV loading | New outbound calls or uses of `StubHttpClient`, `PlatformBackendSpec`, and `BackendNamingPolicy` | +| #1499 MaxMind geo (base `rc/edgezero-431-demo`) | Imports `edgezero_adapter_axum::geo`, which H does not have | +| #1516 S3 preflight cache bypass | Merges first. The port sets `Bypass` on the signed S3 HEAD and the raw error GET; the optimized GET stays `PlatformDefault` | + +While the cut was open, #917, #1435, #1504, #1493, and #1455 merged to `main`, +and the cut's Task 12 merges ported them. #1516 is still open. + +PR #1516 runs on the old API, so the cut's next merge of `main` ports it onto +`OutboundCachePolicy::Bypass` and keeps the optimized GET's Image Optimizer +options. EdgeZero `c8fd1939` documents that split +(`edgezero@14a6ca8b:docs/guide/proxying.md:320-323`). + +Issue #1484 asks for multi-provider fan-out on Cloudflare and Spin. The cut +closes it. + +The cut lands first, and later pull requests rebase onto it. New uses of +`PlatformHttpClient`, `PlatformBackend`, and `StubHttpClient` are not frozen +while the cut is open (Q5). Pull requests that merge before the cut are ported +by the cut at its next rebase onto `main`, and the dependents inventory +(Static) finds their new uses. + +## Verification + +### Static + +Every CI gate in `AGENTS.md`, and `--locked` builds after the lockfile review +in section 1 and the compile-probe re-diff at each repin. The Fastly and Axum adapters' +dev-dependencies enable EdgeZero's `test-utils` features +(`edgezero:crates/edgezero-adapter-fastly/Cargo.toml:23`, +`edgezero:crates/edgezero-adapter-axum/src/lib.rs:35-36`). + +**Dependents inventory.** Each commit that deletes a symbol ports or deletes +every user of it in the same commit: code, tests, scripts, CI, and compiled doc +snippets alike. This command lists the remaining users. At `51730bdaf` it +prints 62 files, which this spec's sections and Expected files cover. After +step 9 it prints only `docs/guide/architecture.md`, +`docs/guide/auction-orchestration.md`, `docs/guide/integration-guide.md`, and +`crates/trusted-server-core/src/auction/README.md`, which step 11 rewrites. +After step 11 it prints nothing. Historical plan and archive documents are +excluded, because they stay as written records. The pattern avoids `\b`, +which POSIX extended regular expressions do not define. On macOS Git it matches +nothing. `request_body_bytes` must follow a non-identifier character, because +EdgeZero's `max_request_body_bytes` builder stays in use. + +```sh +git grep -lE 'PlatformHttpClient|PlatformHttpRequest|PlatformResponse([^A-Za-z0-9_]|$)|PlatformPendingRequest|PlatformPendingResponse|PlatformSelectResult|PlatformBackend|BackendNamingPolicy|PlatformCacheIntent|PlatformImageOptimizer|StubHttpClient|NoopHttpClient|StubBackend|NoopBackend|UnavailableHttpClient|canonicalize_transport_timeout_ms|supports_concurrent_fanout|supports_streaming_responses|supports_pending_streaming_responses|proxy_with_redirects|ProxyRedirectPolicy|startup_error_router|validate_for_target|AuctionTargetId|platform::deadline|build_services_with_.*http_client|collect_body_bounded|(^|[^A-Za-z0-9_])request_body_bytes|send_edgezero_response|stream_asset_body|EdgeZeroAxumService|predict_name' \ + -- ':!docs/superpowers/specs' ':!docs/superpowers/plans' ':!docs/superpowers/archive' ':!CHANGELOG.md' +``` + +Two deleted uses share a name with code that stays, so scoped checks cover +them. The `AuctionContext` field shares its name with the EOF fixture's config +key. The Fastly adapter's `into_core_request` call shares its name with +EdgeZero's public Axum function, which the parity suite's transport-binding +test keeps using. Each prints nothing after step 9: + +```sh +git grep -n transport_timeout_ms -- crates/trusted-server-core/src/auction \ + crates/trusted-server-core/src/integrations ':!crates/trusted-server-core/src/integrations/registry.rs' +git grep -n into_core_request -- crates/trusted-server-adapter-fastly +``` + +### New tests + +1. **Real ingress.** In + `crates/trusted-server-integration-tests/tests/integration.rs`, which CI runs + with `--include-ignored` against Viceroy, workerd, and the Axum binary: + `POST /auction` with a JSON body on Fastly and Axum, since workerd already + has one. A publisher `POST` whose body reaches the origin stub byte for byte + on Fastly, Cloudflare, and Axum. `HEAD /`, `OPTIONS /`, and a `GET /` with a + body, which returns 400 (EZ-1 criterion 13). A chunked `POST /auction` over + 256 KiB with no `Content-Length` returns 413 (EZ-10 criterion 9). +2. **Spin.** Only the `trace-runtime-tests` job installs Spin 4.1.0 and builds + the production component (`.github/workflows/integration-tests.yml:203-231`). + Test 1's `POST /auction`, publisher `POST`, and `HEAD /` cases join as Spin + cases named `spin_runtime_*`, and that job gains a third filter for them + (`:230-231`). The `--include-ignored` job skips them (`:105-107`). Test 6's + Spin leg, `navigation_bids_include_180_and_320_ms_and_exclude_600_ms_spin`, + does not use the prefix, so both jobs name it on its own: the + `--include-ignored` job skips it and the Spin filter runs it. A component + that fails to instantiate fails CI (D14). `scripts/smoke-spin.sh` stays a + manual check. +3. **Request readback.** The scripted client (D12) lives in + `crates/trusted-server-core/src/test_support.rs` under + `#[cfg(any(test, feature = "test-utils"))]`, as the EZ-24 interim says, so + the Fastly adapter tests can use it. It implements EdgeZero's six-method + head-first `OutboundHttpClient` trait + (`edgezero:crates/edgezero-core/src/outbound.rs:426-446`) and supplies an + `OutboundTime` whose timer factory wakes pending timers when the test clock + advances (`edgezero:docs/guide/proxying.md:46-51`). It records every request + and answers from a script with the arrival-time rules EdgeZero's batch uses. For each + site in 2.3 it asserts the method, URI, and headers, `host_authority()` and + `has_host_authority_override()`, a set `timeout`, a connect timeout of + exactly 1 s (EZ-4 criterion 10), `Accept-Encoding` exactly as Trusted Server + set it and absent when it set none, the decoding mode, the marker off + through `effective_proxy_marker_enabled` (EZ-7), the response cap, the cache + policy, and the Image Optimizer options read back from the request (EZ-8). + On the asset proxy the options read back on a final GET and on a final + HEAD, and never on the S3 preflight. For bidders it asserts the consent and + attested client-IP headers. A header value that is not UTF-8 is dropped. +4. **Redirect policy (EZ-9 criterion 20).** The existing redirect, allowlist, + and initial-target tests are ported to the scripted client and run again on + the Axum binary against local origins. A redirect chain past its deadline + fails, and a `Location` with userinfo fails. +5. **Auction.** A 12-bidder auction on Viceroy against 12 loopback `http` + bidders on 12 distinct ports (D18), so each bidder needs its own cold + backend registration (EZ-3 criterion 6). Backend identity leaves out the + path, so bidders on one port would share a backend. A bid after the + cutoff is dropped. The mediator runs after the batch. Two auctions with + different remaining budgets each configure and register exactly one backend + per bidder in their own callback's observation. The observation is per + callback, so it cannot show that the second auction reuses the first one's + backend names; that needs reuse mode (EZ-2 criterion 21, deferred). + Cloudflare and Spin load a + multi-provider config and run the auction. On Spin, one bidder's provider + timeout ends before the cutoff while a sibling answers after the cutoff, + and the auction reports one provider timeout and the cutoff. The leg checks + that outcome on a real runtime. It cannot force the event order behind open + defect 3, so EdgeZero's fix carries the deterministic regression. The + loopback bidders record the request framing they receive (section 8). An + abandoned auction cancels its pending bidder requests. A bid that answers in + time reports the same slot `elapsed` and telemetry `response_time_ms` when + collection is delayed after its arrival (2.4). +6. **Publisher origin and bids (EZ-6).** On Viceroy, Axum, workerd, and Spin, + with loopback bidders and an origin whose head arrives at 700 ms, a + navigation with a 500 ms auction contains the bids that answered at 180 and + 320 ms and not the one that answered at 600 ms, as EZ-6's Trusted Server + check states. A batch cutoff built from `ctx.monotonic_clock()` expires on + time. The two paths differ (2.5): + - On Viceroy, a navigation GET that preloads an EC snapshot takes the + overlapped path. The origin request is dispatched before bid + construction, bids are collected at the `` hold, and a bid that + arrives while the origin body streams is observed before the cutoff. + - On Axum, workerd, and Spin, which have no KV graph, the plain path + dispatches the auction first and sends the origin request while the bids + are in flight. + Gap: the tests do not assert the Viceroy overlapped-path sub-bullet. The + Viceroy leg checks the bid outcome only. +7. **Encoded passthrough (EZ-5 criterion 8).** On Viceroy, against origin + fixtures: + - A gzip HTML page reaches the client with `Content-Encoding: gzip` and + decodes to the rewritten document with the TSJS tag. + - A br SVG relay is byte-identical, with the origin `Content-Length` and + `ETag`. + - `Range: bytes=0-99` against a gzip-stored asset returns 206 with the origin + `Content-Range` and `Content-Length: 100`. + - A JS asset revalidation with `If-None-Match` returns 304. + - A `/first-party/proxy` HTML creative keeps `Content-Encoding: br`. + - A gzip JSON bidder response is decoded. + - An origin stub records `Accept-Encoding: identity` from a site that sets + none. +8. **Inbound readers.** For each reader in section 4, a declared + `Content-Length` over the cap returns 413 without polling the body, a stream + that crosses the cap returns 413, a read past the deadline returns 408, a + stream error with a 4xx status returns 400, and a host-side read error + returns 502. The 400 case covers EdgeZero's ingress framing error. A + client-caused error in the middle of a body reaches the helper as an + internal error on every adapter, so it answers 502 (Failure handling). + The trace action's check covers its declared case. A batch-sync stream that + delivers more bytes than both its declared length and its cap returns 413 + (EZ-10 criterion 9). +9. **Fastly lifecycle.** Under Viceroy: `finalize` sets EC cookies, finalize + headers with the geo lookup, the asset cache policy, and sandbox counters. A + trace response passes untouched, and the trace routes answer through + canonical dispatch. The callback returns `Ok` after a delivery failure, and + a reused sandbox keeps serving. With `delivery::testing::{deliver_with_hooks, response_delivery}`, + every row of the outcome table, including the result of + `send_assembly_error`, takes its listed action. A request head that cannot + be converted gets the native 500, if Viceroy can deliver one (EZ-17 + criterion 7). The reuse leg is + `missing_config_blob_gets_500_then_recovers_in_same_sandbox` + (`crates/trusted-server-adapter-fastly/src/app.rs:2110`), which runs without + a feature gate. Assembly from a linked store with no app config blob fails, + and EdgeZero's assembly error answers 500, today's startup status (EZ-22 + criterion 9's text says 503), with an EdgeZero JSON body and no + `x-ts-version`. With the blob restored, the next initialization of the same + `Sandbox` succeeds, and its `SandboxCounters` show one more build and the + same instance (EZ-22 criterion 9). +10. **Validation.** `validate_settings_for_deploy` rejects each setting the + Configuration table rejects, and each degrades as listed at runtime. Spin + ignores `origin_host_header_override` with a warning. Settings load accepts + a loopback `http` provider endpoint and still rejects other `http` + endpoints. The Fastly state builder rejects an enabled plan with more than + 160 distinct provider tuples of scheme, host, port, and configured timeout, + and accepts 160. Two providers that share a tuple count once. A disabled + auction skips the check, as today (`plan.rs:421-423`). +11. **Head limits.** Through Viceroy, workerd, and the Axum binary, a signed + `/first-party/proxy` URL that wraps a 4 KiB creative URL, sent with 40 + headers of 1 KiB each, is admitted. On Viceroy, a target of 8193 bytes + returns 414, and about 72 KiB of headers in 60 lines returns 431. Viceroy's + own HTTP parser answers 101 header lines with its own 431 before the guest + runs, so the case avoids that (EZ-19 criterion 1). On Viceroy the 414, the + 431, and a framing 500 carry `x-ts-version`, `x-ts-fastly-version`, geo, and + operator headers (EZ-19 criterion 6). D10 leaves Trusted Server no source of + a framing 500, so that case uses a test-only route that answers with a + `Trailer` field. The route exists only in the `test-utils` artifact the + HTML end-of-file checks already build + (`scripts/async-html-finalization-local-test.sh:2`). Canonical 404 and 405 + carry `x-ts-version` on every runtime. +12. **SigV4 host.** Under Viceroy, an S3-signed asset GET on a route without + Image Optimizer reaches a local origin that records `Host`, path, and + query, and each equals the signer's value. On a route with Image Optimizer, + the HEAD preflight reaches it with the same checks. +13. **Post-delivery work.** Through `delivery::testing`, Fastly sends Tinybird + rows and runs pull sync after a committed application delivery, and does + neither when delivery did not commit or committed with a fallback body. A + pull-sync group whose requests outlive `clock.deadline_after(15 s)` ends at + the cutoff. The other adapters send no telemetry. Under Viceroy, pull sync + reaches its partner stub after the origin aborts mid-body, and makes no call + when a framing failure delivers the fallback (EZ-17 criterion 7). +14. **Management API.** Key rotation through the scripted client sends the + expected method, headers, and body. A non-2xx answer yields an error with + the body truncated to 200 characters. +15. **Axum ingress variables.** A variable raises its limit, and an invalid + value fails startup. +16. **Response egress.** A unit test feeds `make_response_bodiless` a HEAD and a + 304 response that carry `Trailer`, and the output has none. On all four + adapters, a publisher `HEAD` and a conditional `GET` that returns 304 return + their real status. Validation rejects an operator `trailer` entry and an + operator `connection` entry. A live config whose operator `connection` + entry predates the cut loads, whether the value is `keep-alive,`, empty, or + the malformed `x example`. The entry is skipped with a warning, and the + response is served with its real status. Without the skip, `x example` + fails egress with the framing 500 (section 5). A live config with + `connection: x-example` and `x-example: example-value` serves `x-example` + after the skip, which records that the ban drops nomination semantics. +17. **Write deadline.** A unit test calls the installed egress policy and gets a + deadline 60 s after egress starts. Under Viceroy, a publisher page whose + origin streams for 35 s reaches the client complete. +18. **Outbound error mapping.** `outbound_error` maps `PayloadTooLarge` to 413, + `RequestTimeout` to 408, `BadRequest` to 400 on the publisher forward and to + 502 elsewhere, and `GatewayTimeout`, `BadGateway`, `ResponseTooLarge`, and + `Internal` to the site's 502. +19. **Parity on the Axum test server (EZ-23 criterion 13).** The Axum, + Cloudflare, and Spin apps run on `AxumTestServer` with production policies + (section 8). A 9000-byte request target gets 414 from all three. A bodiless + GET whose inbound body is an unread stream reaches a loopback origin with no + request body. +20. **Publisher ownership (2.5).** These run with the scripted client behind + H's real facade and head finishing, for `send` and for `begin` followed by + `collect`. Each raw `PublisherResponse` variant (`Buffered`, `PassThrough`, + and a 206 range response) attaches its owner once and keeps its bytes when + the origin call is constructed inside `owner.run`. A control that + constructs the call outside the scope gets "response body already owns an + outbound scope" from `attach`. This pins the construction rule, not just + the outcome. +21. **Publisher input caps (2.5).** On Axum, Cloudflare, and Spin, + `buffer_publisher_response_async` returns raw `Buffered` and `PassThrough` + responses with buffered bodies. With `max_buffered_body_bytes` at its 16 MiB + default, each adapter's `cap` holds for every classification: + - A raw body at `cap` passes. One byte over it fails with 502 before + egress, and so does a source error mid-body. + - Identity HTML that is processed passes at `cap` and fails with 502 before + egress one byte over it: 10 MiB on Cloudflare and 8 MiB on Spin. + - Gzip HTML whose encoded size is under `cap` but whose decoded size + exceeds it fails with 502 before egress. + - The processed output still fails at its own `max_buffered_body_bytes` + cap when a rewrite grows past it. + Gap: coverage is partial. Axum covers only the raw case. The decoded size + over `cap` is tested only on Cloudflare and in core, a source error + mid-body only in core, and no test covers a raw `Buffered` body at `cap`. + +Fastly differences items 16 and 17 are EdgeZero ingress and forwarding +behavior. No Trusted Server test covers them. + +Ported tests: every user of `StubHttpClient`, `NoopHttpClient`, and +`StubBackend` moved to the scripted client or was deleted with the code it +tested, and the three doubles are deleted (`StubHttpClient` with its +self-tests in step 8, the other two in step 9). The bespoke doubles moved too: +`parity.rs:1137`, `:1402`, `:1445`, `:2183`, and `:2207`; +`crates/trusted-server-core/src/test_support.rs:405` and `:567`; +Fastly `app.rs:2934`, `:2954`, and `:3111`; `tinybird.rs:281` and `:318`; +`orchestrator.rs:3674` to `:4439`; `prebid.rs:3871`; `sourcepoint.rs:1175`; +`proxy.rs:2391` and `:2452`; `publisher.rs:12393`, `:19720`, and `:26239`; and +`auction/openrtb/test_executor.rs`. `ForbiddenLifecycle` (`parity.rs:1366-1400`) +also implements the store writes that 2.9 makes async. The scripted client +therefore supports streamed response bodies sent in chunks, a response held +until the test releases it, an error for one slot, and a forbid mode that fails +the test on any request. The hook tests move to `oneshot` (section 6). The Axum +route tests, the whole parity suite, and the CLI proxy test move to +`AxumTestServer` (section 8). The generator's bidder-alias tests +(`generate-viceroy-config.rs:370-468`) are deleted with the alias. + +### Runtime + +- `cargo test-fastly` and `cargo test-fastly-reuse` on stock Viceroy 0.21.0. + Stock Viceroy reports some failed body reads as a clean end of stream. It + also reports empty frames as ready while the next read may still wait + (`edgezero:patches/viceroy/README.md:14-19`). So Trusted Server tests do not + assert error preservation on body reads under stock Viceroy, and CI runs + there are not timing evidence. CI keeps stock Viceroy. The qualification legs + run on the repaired build + ([EdgeZero qualification](#edgezero-qualification-trusted-servers-part)). +- `cargo test --manifest-path crates/trusted-server-integration-tests/Cargo.toml -- --include-ignored` + as CI runs it, which covers `integration` and `parity`, and the + `trace-runtime-tests` job's filters, which cover Spin. +- The worker deadline job, on workerd and Spin. +- The HTML end-of-file checks, because the publisher path and HTML finalization + change: the `html_eof_runtime` test and its clippy run, + `./scripts/async-html-finalization-local-test.sh axum --auction`, and + `./scripts/async-html-finalization-local-test.sh fastly --release --auction`. +- `./scripts/integration-tests-browser.sh`, with loopback bidders. +- `scripts/smoke-fastly.sh`, `scripts/smoke-axum.sh`, + `scripts/smoke-cloudflare.sh`, and `scripts/smoke-spin.sh`, with the + `smoke-axum.sh` change in section 7. +- **Fastly response diff.** A script under `scripts/` sends a fixed request list + to a `main` build and to the branch build, both under Viceroy. It diffs + status, headers, and a body hash. The list covers a publisher page, `HEAD` and + a conditional `GET` of a page, `POST /auction`, a first-party proxy URL, an + asset, a tsjs bundle, an admin endpoint, `TRACE`, and a trace action. It + normalizes `x-ts-version`, sandbox counter headers, dates, and generated IDs. + The expected differences are exactly the [Fastly differences](#fastly-differences) + the list exercises. Reviewers can rerun it. +- The hosted legs in the qualification table, on Fastly staging. + +## Commit sequence + +1. On `main` at `v0.0.11`, add test 1's cases except the `GET` with a body, + and the response-diff script. They pass on `v0.0.11` and record the + baseline. +2. Repin, update the lockfile and `spin-sdk`, add `[capabilities.outbound]` and + the wrangler flags. Make the mechanical fixes (section 7), retype the hook + (section 6), add `read_request_body` and move every reader to it (section 4), + and port the Axum tests to `AxumTestServer`. Delete the Axum, Cloudflare, + and Spin `startup_error_router`s, and return `Err` from `routes`. Fastly + keeps its router until step 8. `Hooks::configure` installs the write policy + and the synthesized headers on every adapter (section 5), because Axum, + Cloudflare, and Spin run EdgeZero egress from this step. Trusted Server's + outbound layer is still present and compiles, because H keeps + `into_core_request` and `oneshot` public. Every target compiles. Fastly + publisher forwards fail until step 7, GET included, because the inbound body + is a stream and Trusted Server's Fastly client rejects streams. Spin + publisher forwards also fail, GET included, because H streams every Spin + request body that lacks an explicit `Content-Length: 0` + (`edgezero:crates/edgezero-adapter-spin/src/request.rs:96-112`). Cloudflare + publisher forwards fail only with a body. Red until step 7: the + `cargo test` job's template-cache harnesses, `test-fastly-eof`, the Viceroy + browser and trace jobs, the Fastly EC lifecycle job, and the + publisher-forward integration cases. Whether fastly 0.13.1 always hands a + GET a body handle is unverified. Running + `./scripts/async-html-finalization-local-test.sh fastly --release` on this + commit shows it. +3. Add the scripted client (D12), the `outbound_client` and `clock` fields of + `RuntimeServices`, the outbound error helper (2.11), and the test helper that + inserts a scripted client for `oneshot` route tests (2.1), with its uses in + the Fastly, Cloudflare, and Spin route helpers. Move the parity suite's + Cloudflare and Spin helpers to `AxumTestServer`, with the scripted client in + router state (section 8). Every + `RuntimeServices::builder()` call (41 in 16 files) sets both new fields in + this step, because `build()` panics on a missing required service + (`crates/trusted-server-core/src/platform/types.rs:454-470`). The adapters + set them from the request context. The test helpers and fixtures use the + scripted client, and `ForbiddenLifecycle` uses a forbid-mode script. The + guide snippet waits for step 9, because `cargo check` never runs `build`. +4. Port the integrations (O10 to O15), with the `certificate_check` rule and the + fixture and CI changes in section 1. +5. Port the proxies (O6 to O9), with redirect following and the asset proxy. +6. Port the auction (O1 to O3) and the provider contract. Delete the fan-out + gate and `validate_for_target`, keep the Fastly backend bound, and add the + loopback endpoint rule with the generator, fixture, browser-test, and + `template-cache-local-test.sh` changes (D18). +7. Port the publisher origin (O4, O5), with the scope owner and both origin + calls constructed inside it, raw collection on Axum, Cloudflare, and Spin, + the readthrough rule, the host override, the `origin_url` rule, and the Spin + warning (tests 20 and 21). Until + step 8, Trusted Server's Fastly delivery converts the response body with + `into_stream()`, so the attached owner lives until the end of the body. Port + the EOF fixture (O19), and run the HTML end-of-file checks in Runtime. The + integration cases turn green. +8. Move Fastly onto the canonical lifecycle (section 3), with post-delivery + telemetry and pull sync (2.8), the management API (2.9), and the + integration-guide snippet's async store writes with its test manifest. + Fastly's error router goes here. +9. Delete Trusted Server's outbound layer and its test doubles (2.10, section + 8). Port the worker deadline examples and the Fastly deadline harness + (section 8), update the integration-guide snippet and its test manifest, and + add the `clippy.toml` guard. Rewrite the `IntegrationProxy` doc comment at + `integrations/registry.rs:264`, which links to `PlatformPendingRequest`. The + dependents inventory (Static) then prints only the four documents step 11 + rewrites. +10. Add the remaining egress rules (D10 validation, `make_response_bodiless`), + the Axum ingress variables, the `smoke-axum.sh` change, and the observation + log. +11. Update the rest of the documentation and `CHANGELOG.md`. The dependents + inventory then prints nothing. + +In steps 4 to 7, ported sites use `outbound_client` while the rest still use the +old layer. That transient dual path exists only inside the pull request. + +## Follow-ups + +- EZ-11 surrogate keys, to restore origin readthrough. +- EZ-12 certificate-check mode, if an operator still needs to skip + verification. +- EZ-13 Spin host override, to honor `origin_host_header_override` on Spin. +- EZ-14 idle bound, to restore 15 s stall detection on streamed sites. +- EZ-15 fire-and-forget, to send telemetry during the request and on every + adapter. +- EZ-24 published test double, to delete Trusted Server's scripted client. +- An EdgeZero ask for a version header on assembly-error responses (EZ-19.7). +- EZ-2 reuse-mode parts, before reusable sandboxes reach production. +- Streaming publisher responses on Axum, Cloudflare, and Spin. +- Declaring stores on Cloudflare and Spin, including whether the Cloudflare + `block_on(get)` config bridge completes on workerd once it is live. +- Retained app instances on Cloudflare and Spin. H already has EZ-22's + retention recipe, which retains only a successful assembly, so per-request + assembly in the cut is a choice, not a forced interim. Adopting it brings + EZ-22 criterion 10 into the gate (D19). +- `PlatformTimer` and EdgeZero's clock as one time source. +- An error renderer that brands framework errors. +- Dependabot ignore rules for EdgeZero-pinned crates. +- Moving the pin from the PR 275 commit to an EdgeZero release tag once PR 275 + merges (D5). +- Spin on `wasm32-wasip2`. +- The rest of #1505: Auctioneer as an in-process auction provider. + +## Maintainer answers + +The maintainer answered the open questions on 2026-11-08. The decisions they +change are D5, D7, D13, D18, and D20. + +- **Q1. Merge gate (D5).** Do not wait for EdgeZero's EZ-Q1 record. `main` + merges the cut pinned to PR 275, and production deploys do not wait for + qualification ([Merge gate and pin](#merge-gate-and-pin)). +- **Q2. Timeouts (D7).** Each app that uses EdgeZero sets its own limits. + EdgeZero already lets it set every one, so EdgeZero needs no change: + - the per-request `timeout`, `connect_timeout`, and `deadline` + (`edgezero@5ea8d93c:crates/edgezero-core/src/outbound.rs:1087`, `:1100`, + `:1495`). The deadline also bounds a redirect chain under `follow` + (`outbound.rs:615-623`). + - the redirect count (`OutboundRedirectLimit::new`, + `edgezero@5ea8d93c:crates/edgezero-core/src/outbound/redirect.rs:332`). + - the response write deadline (`App::set_response_egress_policy`, + `edgezero@5ea8d93c:crates/edgezero-core/src/app.rs:553`), 30 s by default + (`response_egress.rs:22`). + - the inbound read deadline, per request (`App::set_ingress_admission_policy`, + `app.rs:514`), 30 s by default (`ingress.rs:34`). + - the head limits (`App::set_ingress_head_limits`, `app.rs:523`). + + D7's values stand: a 1 s connect timeout, 60 s for streamed calls, 15 s for + buffered calls, 60 s for redirect chains, a 60 s write deadline, and + EdgeZero's default 30 s inbound read deadline for publisher uploads. + +- **Q3. Hosted qualification.** The maintainer qualifies by hand for now, as + the operator. For each run the maintainer chooses the staging service, its + Image Optimizer entitlement, and EZ-2's M1 to M7 thresholds + ([EdgeZero qualification, Trusted Server's part](#edgezero-qualification-trusted-servers-part)). +- **Q4. HTTP/3.** Qualified per adapter. EdgeZero amended EZ-1 criterion 2 in + `dcbf622`, so its HTTP/3 legs no longer depend on Trusted Server's staging + service + ([PR 275 thread r4504554519](https://github.com/IABTechLab/edgezero/pull/275#discussion_r4504554519)). +- **Q5. In-flight pull requests.** The cut is stacked on PR 275, because its + Axum changes need it. New uses of `PlatformHttpClient`, `PlatformBackend`, + and `StubHttpClient` are not frozen. The cut ports them when it rebases + ([In-flight work](#in-flight-work)). +- **Q6. Loopback bidder endpoints (D18).** Accepted. Provider endpoint + validation accepts `http` on `127.0.0.1` and `[::1]`. +- **Q7. Bodiless Fastly framing (D20).** Strict suppression. + - HEAD, 204, and 304 responses never include content (RFC 9110 §6.4.1). On + HTTP/1.1 such a response ends at the blank line after its headers + (RFC 9112 §6.3). On HTTP/2 a body there makes it malformed, and clients + must not accept it (RFC 9113 §8.1.1). A server must not send content in a + 205 (RFC 9110 §15.3.6). + - EdgeZero's local SDK controls show `send_to_client` breaking HTTP/2 clients + on HEAD, 204, and 304, and sending a 205 body on both HTTP/1.1 and HTTP/2 + (`edgezero:docs/audits/evidence/2026-10-23-buffered-response-framing.json`, + the nine `sdk_parity.failed_cases`). + - Trusted Server withdraws EZ-18.4's bodiless part, and EdgeZero amended + EZ-18.4, EZ-18.6, and EZ-18 criteria 3 and 6 to match in `dcbf622` + ([PR 275 thread r4504573926](https://github.com/IABTechLab/edgezero/pull/275#discussion_r4504573926)). + The response diff's expected set + (`scripts/fastly-response-diff.expected.json`) cites Fastly differences + items 5, 6, and 25. Bodiless framing, item 7, shows no difference on the + diff's requests: the HEAD and 304 cases, `head-root` and + `conditional-get-root`, expect only item 6's hop-by-hop fields. + - The amendment keeps the nine cases as recorded reference differences, not + parity failures. Hosted qualification of EZ-18 is still open, and it does + not block the cut (D5). + +## Expected files + +- `Cargo.toml`, `Cargo.lock`, `edgezero.toml`, `clippy.toml` +- `crates/trusted-server-core/src/platform/` (delete `http.rs`, + `backend_naming.rs`, `pending_response.rs`, `image_optimizer.rs`, and the HTTP + doubles in `test_support.rs`; change `traits.rs`, `types.rs`, `mod.rs`) +- `crates/trusted-server-core/src/test_support.rs`, + `test_support/scripted_client.rs` (scripted client), `test_support/body_read.rs`, + `test_support/logging.rs`, and `test_support/async_html_finalization.rs` +- `crates/trusted-server-core/src/http_util.rs`, `error.rs`, `config.rs`, + `settings.rs`, `response_privacy.rs`, `auth.rs`, `forwarder.rs`, + `publisher.rs`, `proxy.rs`, `cache_purge.rs`, `creative_opportunities.rs`, + `asset_image_optimizer.rs`, `html_processor/finish.rs`, `config_payload.rs`, + `secret_resolution.rs`, `settings_data.rs`, `outbound.rs`, + `edge_app.rs` +- `crates/trusted-server-core/src/auction/` (`mod.rs`, `orchestrator.rs`, + `provider.rs`, `plan.rs`, `endpoints.rs`, `telemetry.rs`, `types.rs`, + `test_support.rs`, `openrtb/test_executor.rs`, `openrtb/tests.rs`, + `README.md`) +- `crates/trusted-server-core/src/ec/` (`pull_sync.rs`, `batch_sync.rs`) +- `crates/trusted-server-core/src/request_signing/` (`endpoints.rs`, + `rotation.rs`) +- `crates/trusted-server-core/src/trace/` (`dispatch.rs`, `actions.rs`, + `routes.rs`) +- `crates/trusted-server-core/src/integrations/` (`mod.rs`, `registry.rs`, + `aps.rs`, and each integration in section 2.3), and + `request_signing/jwks.rs` +- `crates/trusted-server-adapter-fastly/` (`Cargo.toml`, `src/main.rs`, + `app.rs`, `middleware.rs`, `platform.rs`, `tinybird.rs`, `management_api.rs`, + `compat.rs`, `delivery.rs`, `post_delivery.rs`, `sandbox.rs`, a new deadline + test module, `tests/deadline-origin.mjs`; delete `backend.rs` and + `platform/deadline.rs`) +- `crates/trusted-server-adapter-axum/` (`Cargo.toml`, `src/main.rs`, `app.rs`, + `middleware.rs`, `platform.rs`, `tests/routes.rs`) +- `crates/trusted-server-adapter-cloudflare/` (`src/app.rs`, `middleware.rs`, + `platform.rs`, `tests/routes.rs`, `tests/runtime/deadline.rs`, + `wrangler.toml`, `wrangler.ci.toml`) +- `crates/trusted-server-adapter-spin/` (`src/lib.rs`, `app.rs`, + `middleware.rs`, `platform.rs`, `tests/routes.rs`, + `tests/runtime/deadline.rs`) +- `crates/trusted-server-integration-tests/` (`src/bin/generate-viceroy-config.rs`, + `tests/parity.rs`, `tests/integration.rs`, `tests/documentation_snippets.rs`, + `tests/html_eof_runtime.rs`, `tests/common/ec.rs`, `tests/common/runtime.rs`, + environment helpers, the three + fixture configs, and `browser/tests/nextjs/mobile-trace-live.spec.ts` if its + bidder URL changes) +- `crates/trusted-server-cli/tests/proxy_trusted_server.rs` +- `.github/workflows/integration-tests.yml`, `.github/workflows/test.yml`, + `.github/workflows/format.yml`, + `.github/actions/setup-integration-test-env/action.yml`, and + `.github/workflows/worker-auction-deadlines.yml` if its build steps change +- `scripts/integration-tests.sh`, `scripts/integration-tests-browser.sh`, + `scripts/generate-integration-viceroy-configs.sh`, + `scripts/template-cache-local-test.sh`, `scripts/smoke-axum.sh`, + `scripts/README.md`, `scripts/test-worker-auction-deadlines.mjs` if its cases + change, and the new response-diff script +- `docs/guide/integration-guide.md`, `docs/guide/architecture.md`, the other + guide pages from section 10, and `CHANGELOG.md` +- `AGENTS.md` + +## Review log + +- **2026-11-05 rewrite.** This replaces the 2026-10-17 draft, which targeted + `f6c708cc`, kept Trusted Server's outbound layer, and kept Fastly's custom + lifecycle. The maintainer then chose D1 to D4, and EdgeZero implemented the + requirements at H. Inputs were four read-only investigations at H: the + blocking requirements and EZ-Q1 state, the follow-ups and the `v0.0.11` to H + API diff, a compile probe of `51730bdaf` repinned to H, and a site-by-site + migration map with an open pull request scan. +- **Round 1 (Claude Code, Opus, two fresh sessions with split scopes, verified + against both trees).** Four blocking findings, all fixed. A connect timeout + of `min(1 s, timeout)` and budget-derived timeouts would have split one + origin across many Fastly backends, so D7 now uses constants and the batch + cutoff carries the remaining budget. Deleting `asset_origin_host_header` + would have broken S3 signing, so it stays. No rule mapped EdgeZero errors to + Trusted Server errors and the 504 claims were false, so 2.11 and D17 keep + today's 502s. The Cloudflare and Spin config and KV fallbacks are the live + path, so they stay. Should-fix items adopted: H's request and response header + and URI rules, the initial redirect check and policy order, `owner.run` and + the owner in `PublisherResponse`, the provider contract, corrected response + modes, a per-request telemetry sink, the Spin override as a warning, no new + Image Optimizer validation, fixture configs, the EOF fixture site, + `wrangler.ci.toml`, D8's wording, test 7 from EZ-5 criterion 8, the native + request work before the send, the outcome table for detached and unknown + deliveries, assembly errors without `x-ts-version`, startup-failure statuses + and smoke scripts, the clippy guard for `tests/` crates, `body_as_reader`, + the `Content-Length` precheck in the helper, the upload deadline question, + the Cloudflare bridge reasoning, the qualification leg table and ownership + counts, Spin in CI, and the observation log at info. Nits adopted throughout. +- **Round 2 (Claude Code, Opus, two fresh sessions: a cold whole-spec read and + a check of every round-1 fix against both trees).** The fix check found 60 + of 64 round-1 findings fixed and 4 partly fixed. Three blocking findings, all + fixed. Local bidder stubs were reachable only through a Viceroy backend-name + alias that H cannot support, so D18 accepts loopback `http` endpoints and the + generator rewrites them. The worker deadline CI job compiled against the + deleted clients, so section 8 ports it. `validate_for_target` still rejected + multi-provider plans on Cloudflare and Spin, so 2.4 deletes it and keeps the + Fastly backend bound. Should-fix items adopted: the owner returned beside + `PublisherResponse` and attached once at the call site, today's + startup-failure statuses kept per error class, the Spin warning recorded as a + deviation from the EZ-13 interim, five more Fastly differences (cancelled + bidder requests, request hop-by-hop fields, `Location` userinfo, + post-delivery gating, batched dispatch), tests for the write deadline, error + mapping, post-delivery work, and synthesized headers, the unreachable + Cloudflare config bridge and its test removed, Spin cases in the job that + installs Spin, the EOF fixture's budget-derived values kept, and the scripted + client's `Accept-Encoding` assertion. Nits adopted throughout, including + citation corrections and the `outbound_client` field name for the transition. +- **Round 3 (Claude Code, Opus, one fresh session, blocking and should-fix + only).** Four blocking findings, all fixed, and all of one kind: harnesses + that depend on deleted code and that no section named. The Fastly deadline + harness in the required `cargo test` job runs tests in `platform/deadline.rs`, + so section 8 ports them. `template-cache-local-test.sh` reached its bidder + through a predicted backend name and a private CA, so it moves to loopback + `http`. `oneshot` route tests on Fastly, Cloudflare, Spin, and parity get no + client, so 2.1 adds a helper that inserts a scripted one. The + integration-guide snippet compiles the deleted APIs, so it changes in steps 8 + and 9. To stop finding these one round at a time, Verification now carries a + dependents inventory command that must print nothing after step 9. Should-fix + items adopted: gate 15 is already listed and not required, the generator's + `--provider-endpoint-origin` flag and trace-only scope, the EOF fixture port + in step 7, the bespoke test doubles and the scripted client's features, 12 + distinct bidder ports, the full red set in step 2, and the new home of the + Fastly backend bound with the tests that flip. +- **Round 4 (Claude Code, Opus, one fresh session: round-3 fixes, the + dependents inventory file by file, and the commit sequence).** All round-3 + findings fixed or partly fixed. One blocking finding, fixed with the + reviewer's text. The Fastly deadline harness could not keep all five + outcomes on H. It loses its only route to the origin port, a reply first + polled after the cutoff is now dropped, and H has no missing-cache-bypass + error. Section 8 now ports it through a config-store port and states the + changed cases, and Fastly differences gains item 22. Should-fix items + adopted: a portable inventory pattern (`\b` matched nothing on macOS Git) + with the true count of 62 and the four documents step 11 rewrites, + `auction-orchestration.md`, the `registry.rs` doc link, + `AuctionContext::transport_timeout_ms` with a scoped check and APS's planned + parser, the async store writes under `async_trait` with their 38 + implementers, the guide snippet's test manifest and re-exports, every + `RuntimeServices::builder()` call setting the new fields in step 3, the Spin + GET reason in step 2, and a test for the moved Fastly backend bound. +- **Round 5 (Claude Code, Opus, one fresh session: round-4 fixes, both + inventory commands, and cross-section consistency).** No blocking findings. + The config-store route to the origin port works, because Viceroy 0.21.0 + loads `local_server.config_stores` under `viceroy run`, and an EdgeZero wait + drives a batch with no request lifecycle. Both "unverified" sentences now + carry that evidence. Two should-fix items adopted. `batch.check()` never + polls the SDK on Fastly, so 2.4, Fastly differences item 22, and the + expired-snapshot harness case now use `batch.next()` after the cutoff. The + new deadline test module must not sit under `platform::deadline`, which the + inventory searches. Nits adopted: 37 store implementers in `crates/` plus the + guide snippet, and a citation fix. +- **2026-11-06 update (Claude Code, Opus).** H moves from `8f82547b` to + `9b31a697`, which changes no public API and no line an earlier draft cited. + Inputs were a + source check of every EZ requirement at `b56fccd2` by seven fresh sessions, + review 5798502238 on PR 275 at `9b31a697` (5 blocking, 30 should-fix, 5 nits), + and the other reviewers' open threads. Contradicted claims fixed: + - Fastly batch setup and arrival recording (2.4). + - Location handling on Cloudflare, percent-encoded locations, and the + returned 3xx's body handling (2.6). + - Streamed upload overflow (2.5, Fastly differences item 3). + - `NameInUse`, now a documented collision (item 12). + - The public envelope constructors (section 3). + - EZ-19.3 versus EZ-19.7 (section 5). + - The trace reader's declared-length case (section 4). + - The unverified WASI `Host` claim and the Spin `run_app` citation (section 8). + - The stock Viceroy limits (Runtime). + - The required leg count, now 261. + + Gaps against Trusted Server's own criteria fixed: + - `prepare` installs a per-request synthesized set (EZ-19 criterion 6). + - The head-conversion outcome goes back to today's 500 (EZ-17 criterion 7). + Fastly differences item 5 is removed, items 6 and 7 move up, and + bodiless framing becomes item 7, so items 8 to 24 keep their numbers. + - The confirmed follow-ups and their legs (D19). + - The parity suite on the Axum test server (EZ-23 criterion 13). + - The S3 preflight sets `Original`. + - Bodiless Fastly framing is D20 and Q7. + - A new section lists the open EdgeZero defects at H. + +- **Round 6 (Claude Code, Opus, one fresh session: every changed claim against + both trees, the new legs against EdgeZero's criteria text, numbering and + anchors, and the hard-cut policy).** No blocking findings. 13 should-fix + items adopted: + - The handoff status of the requirements copy. + - D19's scope under EZ-Q1.6 and EZ-Q1.8, and EZ-13's interim. + - How collection handles a batch failure. + - A misattributed core citation. + - Spaces in `Location`, which already worked. + - A self-referencing step. + - Client body errors on Axum and Spin. + - The five hosted follow-up legs and the post-D19 leg counts. + - The inventory's `follow-up` legs. + - The repaired Viceroy against EZ-Q1.3. + - What test 5's Spin leg can prove. + - Q7 reworded so neither option asks EdgeZero to rebuild old behavior. + - The renumbering note. + + Nits adopted throughout, including per-value filtering of the synthesized + set. + +- **Round 7 (GitHub review 5810751514 on PR #1506, 2026-11-07, at + `854fe2e36`).** One blocking finding and five should-fix, all adopted. + - Blocking: the origin `send` and `begin` are now constructed inside + `owner.run`. Constructed outside it, head finishing gives a raw origin body + its own owner and the attach fails (2.5, test 20). + - Axum, Cloudflare, and Spin now collect every raw publisher variant before + egress, under today's caps (2.5, test 21). + - The overlap claim and test 6 are narrowed to where the overlap runs: + Fastly navigation GETs with an EC preload (2.5). + - D10 now covers operator hop-by-hop fields, because a malformed + `Connection` fails H's egress framing (section 5). + - The final client HEAD keeps its Image Optimizer options (2.7). + - The telemetry latency's changed stop point is stated, with a + delayed-collection test (2.4, test 5). + +- **Round 8 (GitHub review 5815168335 on PR #1506, 2026-11-07, at + `5bd113408`).** Four round-7 fixes were accepted and resolved, and two were + partly fixed. Both remainders are adopted: + - The adapter input cap now covers every publisher classification, processed + HTML included, through H's encoded-byte cap and a cumulative decoded-byte + bound. The processed-output cap stays separate (2.5, test 21). + - The claim that skipping operator hop-by-hop entries is wire-neutral is + removed. Skipping `Connection` drops its nomination semantics, so a field + it names reaches the client. Validation finds such configs before deploy + (section 5, test 16). + +- **Round 9 (GitHub review 5815676577 on PR #1506, 2026-11-08, at + `a83dc4d89`).** The reviewer accepted and resolved the two round-8 fixes and + found no new defect. Its verdict was BLOCKED on Q1 to Q7. The maintainer's + answers are recorded in [Maintainer answers](#maintainer-answers): + - The cut is stacked on PR 275 and follows its head. `main` merges without + waiting for EdgeZero's EZ-Q1 record, and production deploys do not wait + for qualification (D5, Merge gate and pin). + - Limits are app-owned. EdgeZero already exposes every one, and D7's values + stand. + - Hosted qualification is manual, and the maintainer is the operator. + - HTTP/3 is qualified per adapter, and EZ-18.4's bodiless part is withdrawn. + Both are asked of EdgeZero in PR 275 threads. + - New uses of the outbound layer are not frozen, and loopback `http` + endpoints are accepted. + - The five blocking defects found at H were verified fixed at later PR 275 + heads (Open EdgeZero defects at H). diff --git a/docs/superpowers/specs/2026-10-17-edgezero-pr275-requirements.md b/docs/superpowers/specs/2026-10-17-edgezero-pr275-requirements.md new file mode 100644 index 0000000000..3ce5163065 --- /dev/null +++ b/docs/superpowers/specs/2026-10-17-edgezero-pr275-requirements.md @@ -0,0 +1,2197 @@ +# EdgeZero Requirements for the Trusted Server Cutover + +**Date:** 2026-10-17 + +**Status:** Handed to EdgeZero on 2026-10-18. PR 275 added its own edited copy in commit `bc300ace` and has amended it since, including its 2026-10-24 Scoped Amendment. EdgeZero's copy is the contract EdgeZero is measured against. This file is the text Trusted Server handed over. + +**EdgeZero base:** PR 275 head `f6c708ccb5be19ba9fa92575db3c08c0f17e0d72` (branch `docs/outbound-http-spec`), called **F** below + +**Consumer:** Trusted Server `main` at `a9037770a`, which pins EdgeZero `v0.0.9` today + +**Companion:** [EdgeZero PR 275 hard cutover](2026-10-17-edgezero-pr275-hard-cutover-design.md), the Trusted Server side of the cut. It is being revised to the decisions in this document. + +## Purpose + +Trusted Server (TS) is moving onto EdgeZero PR 275 in one hard cut. TS will use EdgeZero's +canonical request lifecycle on all four adapters and EdgeZero's outbound HTTP API for every +outbound request, and it will delete its own transport layer. TS keeps no HTTP or transport +plumbing of its own. Where F cannot do what TS does in production today, the gap is a requirement +in this document, not a TS workaround. + +This document states what EdgeZero must do, not how. Each requirement gives the Trusted Server need, +the gap at F with code citations, normative requirements on observable behavior, acceptance +criteria, what is out of scope, and open questions for the EdgeZero maintainers. API shape, internal +design, and test harnesses are the EdgeZero implementers' choice. + +## Decisions behind these requirements + +The TS maintainer made these decisions. The requirements assume them. + +1. **Hard cut.** TS moves to the PR 275 APIs with no backward compatibility, no shims, and no dual + paths. +2. **No TS plumbing.** TS deletes its own outbound HTTP layer, backend naming, raw-SDK clients, + response decoder, encoding strip, redirect loop, and Fastly response delivery code. Every + outbound call goes through EdgeZero's outbound API. +3. **Canonical Fastly lifecycle.** TS Fastly moves onto EdgeZero's canonical send path inside its + custom entry point, with TS logic in the lifecycle's prepare and finalize hooks. Production runs + single-request sandboxes. Reusable-sandbox mode stays off by default + (`ts:crates/trusted-server-adapter-fastly/Cargo.toml:13-18`), so requirements specific to reuse + mode are follow-ups needed before TS enables it. +4. **Redirects move into EdgeZero.** TS's per-hop redirect loop is deleted in favor of EZ-9. +5. **Blocking set.** The cut cannot ship to production without the blocking parts of EZ-1 through + EZ-9 and a passing EZ-Q1 record. The record covers every acceptance criterion of those blocking + parts, each in the environment it names, together with the criteria of any promoted requirement + and of follow-ups present at the pinned commit (EZ-Q1.6). Criteria of reuse-mode follow-ups gate + only before TS enables reuse mode. Everything else is follow-up unless the maintainers accept a + change from [Proposed priority changes](#proposed-priority-changes). + +## How Trusted Server will use EdgeZero after the cut + +This is context for the requirements, not a requirement on EdgeZero's design. + +- **Lifecycle.** Fastly runs EdgeZero's canonical send path with TS's client-IP sanitization and TLS + signals in the prepare hook, and TS response finalization, EC finalization, privacy guards, and + sandbox counters in the finalize hook. The state finalize returns drives post-send EC pull sync. + Axum, Cloudflare, and Spin run EdgeZero's standard entry points. TS sets the response egress + policy, head limits, and error renderer through `Hooks::configure`. +- **Auction.** Bidders become outbound requests that run as one batch per auction with the auction + deadline as the cutoff. The mediator is one ordinary send after the bidders. +- **Publisher origin.** On the overlapped path (a navigation `GET` with EC preload, + `ts:crates/trusted-server-core/src/publisher.rs:4821-4846`), the origin fetch begins before bid + construction, runs alongside the bidder batch, and streams its response (EZ-6). Bids are collected + at the `` hold while the page streams. Every other origin forward, including `POST`, + `PUT`, `PATCH`, `DELETE`, and `OPTIONS`, is an ordinary send with the inbound body streamed + upstream. Both are built from the inbound request (EZ-1). +- **Proxies and integrations.** About twenty call sites send through EdgeZero. They include the + first-party proxy, asset routes (with Fastly Image Optimizer and S3 SigV4 signing, EZ-8), GTM, + GPT, the JS asset proxy, the Prebid bundle, Lockr, Permutive, DataDome, Sourcepoint, Didomi, and + testlight. Redirects follow through EZ-9. Encoded bodies pass through undecoded where TS relays + bytes (EZ-5). +- **Out-of-request work.** EC pull sync runs after delivery, inside the same lifecycle callback, + with the request's outbound client (cross-cutting requirement 4). + Tinybird telemetry is fire-and-forget (EZ-15). The Fastly management API runs inside admin + handlers. +- **Inbound bodies.** Endpoints read bodies with EdgeZero's bounded readers at their own caps + (EZ-10 restores 413). + +## Requirement index + +| ID | Title | Priority | Adapters | +| ----- | --------------------------------------------------------------------------------- | ---------------- | ------------------------------------------ | +| EZ-1 | Bodiless GET/HEAD forwarding | Blocking | all four | +| EZ-2 | Fastly connection reuse across requests | Blocking in part | Fastly | +| EZ-3 | Cold backend registration never fails a request on its own | Blocking | Fastly | +| EZ-4 | Connect timeout independent of the total budget | Blocking | all four | +| EZ-5 | Encoded response passthrough | Blocking | all four | +| EZ-6 | Begin a request without waiting, and observe a batch without blocking | Blocking | all four | +| EZ-7 | Keep the `x-edgezero-proxy` header off relayed responses | Blocking | all four | +| EZ-8 | Fastly Image Optimizer on outbound requests | Blocking | Fastly; others reject | +| EZ-9 | Redirect following with the application's target policy at every hop | Blocking | all four | +| EZ-10 | 413 Payload Too Large for request-body overflow | Follow-up | all four | +| EZ-11 | Cache tagging with surrogate keys | Follow-up | Fastly, Axum, Spin; Cloudflare unsupported | +| EZ-12 | TLS certificate verification mode | Follow-up | Fastly; others unsupported | +| EZ-13 | Spin host authority override | Follow-up | Spin | +| EZ-14 | Idle-time bound for streamed response bodies | Follow-up | all four | +| EZ-15 | Explicit fire-and-forget send | Follow-up | all four | +| EZ-16 | Provider failure detail available to logs | Follow-up | all four | +| EZ-17 | Fastly send outcome tells the application what was committed | Follow-up | Fastly | +| EZ-18 | `Content-Length` on buffered responses | Follow-up | all four | +| EZ-19 | Application headers on detached and egress-fallback responses | Follow-up | all four | +| EZ-20 | A declared store that cannot open fails at use, not on every request | Follow-up | Fastly; Cloudflare and Spin should | +| EZ-21 | Framing failure reason in logs | Follow-up | all four | +| EZ-22 | Fallible application state built and retained with the routes | Follow-up | all four | +| EZ-23 | Axum dev server applies application policies, and tests can run real Axum ingress | Follow-up | Axum | +| EZ-24 | Published outbound test double that behaves like production | Follow-up | core | +| EZ-25 | Documentation drift fixes | Follow-up | docs | +| EZ-Q1 | Recorded passing qualification run of the canonical Fastly composition | Gate | all four | + +## Cross-cutting requirements + +These apply to every requirement in this document. + +1. **Every send mode honors every option.** An outbound option (decoding mode, proxy marker, connect + timeout, idle bound, cache intent, TLS verification, Image Optimizer) behaves the same on a + single send, a begun request (EZ-6), a batch slot, and every redirect hop (EZ-9), and on a + fire-and-forget send once EZ-15 lands. Where a requirement marks a send mode Unsupported for an + option, that send mode rejects it (requirement 2). No send mode silently drops an option. +2. **One rejection for unsupported options.** An option whose requirement marks an adapter or send + mode Unsupported is rejected there before anything is sent, with one documented error category + that is the same on every adapter and consistent with F's other outbound preflight rejections. + This rejection comes before every other outbound preflight check. An option marked BestEffort is + accepted, and its weaker guarantee is documented in the capability matrix. +3. **Bounded memory.** Every response EdgeZero holds or drains on the application's behalf, + including responses that arrive while the application is busy elsewhere, stays within that + request's encoded, decoded, and final caps. +4. **Request lifetime.** For outbound purposes an inbound request's lifetime ends when the + application's per-request lifecycle callback returns, not when its response is delivered. Work the + application does inside that callback after delivery, such as TS's EC pull sync, can use the + request's outbound client. Anything still in flight when the callback returns is released, except a fire-and-forget send, whose lifetime EZ-15 defines. +5. **Arrival decides lateness.** A response arrives when its outcome is final. For a buffered batch + slot or buffered begun request that is when its last body byte was received, or when the error or + cap failure that ended it occurred. For a streamed begun request it is when its response head was + received. Lateness is judged at the earliest instant EdgeZero observed the arrival, never at when + the application collected the result. Where an adapter can observe arrivals only during EdgeZero + operations (an outbound wait, a body read, or a non-blocking check), it records each arrival no + later than the next such operation, and the requirement states that adapter's capability level. +6. **Abandon releases; fire-and-forget is explicit.** Dropping or cancelling an in-flight request or + batch releases it on every adapter as far as the host allows. Sending without waiting for the + result is a separate, explicit capability (EZ-15). This document calls that a fire-and-forget + send. "Detached" keeps EdgeZero's meaning of a response produced outside routing. +7. **Diagnostics stay off the wire.** Provider failure detail, framing failure reasons, and payload + limit reasons are available to application logs and never reach a response. Error text that + applications and clients see today does not change. +8. **No silent regressions.** Behavior TS relies on at F (deadline enforcement, admission and + framing checks, reusable-sandbox serving, existing public APIs) keeps working unless a + requirement changes it explicitly. +9. **Capability matrix.** Each requirement's per-adapter capability level (Native, BestEffort, + Unsupported) appears in EdgeZero's capability matrix. Fixing guide text that is wrong at F is part + of the requirement it concerns. Requirements do not ask EdgeZero to edit its immutable + `docs/superpowers/specs/`. +10. **Checkable where it matters.** Each acceptance criterion names the environment where the + behavior must hold, from among unit, Viceroy 0.21.0, hosted Fastly, workerd, Spin 4.1, and Axum. + Viceroy 0.21.0 enforces first-byte and between-bytes timers on dynamic backends and reports their + expiry as a generic host error (`viceroy:src/upstream.rs:351-372`, `viceroy:src/body.rs:235-244`). + It stores the connect timeout without enforcing it, so connect-time behavior is checked on hosted + Fastly. A reused Viceroy guest serves at most a handful of requests, Viceroy reports a refused + connection without detail, and its HTTP layer rejects more than 100 header lines, so criteria + that depend on those facts name hosted Fastly or are written around them. "Every adapter" in a + criterion means Axum, workerd, Spin 4.1, and Viceroy 0.21.0. A criterion has a hosted Fastly leg + only where it names hosted Fastly. + +## Dependencies between requirements + +- EZ-6's overlapped origin fetch is a GET built from the inbound request, so it needs EZ-1. On + Fastly its timelines rely on EZ-2's exact deadline enforcement. +- EZ-9's hops rewritten to GET or HEAD are bodiless, so it needs EZ-1. Each new redirect origin is a + new Fastly backend, so it interacts with EZ-2. +- EZ-3 depends on EZ-2. A cold registration can only stop failing requests on its own once exact + deadlines are enforced independently of registration timing. +- EZ-4's connect timeout and EZ-14's idle bound change how Fastly backends are configured, so they + interact with EZ-2's connection reuse. +- EZ-5 does not depend on EZ-18. A preserved upstream `Content-Length` is the application's to + relay, and EZ-18 separately governs lengths on buffered responses EdgeZero frames. +- EZ-8 and EZ-11 conflict on Fastly. The platform disables Image Optimizer under cache bypass and + ignores surrogate keys with Image Optimizer, so those combinations are rejected (cross-cutting + requirement 2). +- EZ-15 and EZ-6 share abandonment semantics (cross-cutting requirement 6). +- EZ-16, EZ-17, and EZ-21 share the log-only diagnostics of cross-cutting requirement 7. +- EZ-20's "nothing was sent" outcome is the one EZ-17 reports, once EZ-17 lands. +- EZ-22 changes what canonical Fastly entry points do when assembly fails, EZ-17 reports it, and + EZ-23's in-process tests build on EZ-22's state. +- EZ-24 follows EZ-6 and EZ-9 semantics, so it changes with them. +- EZ-Q1 runs at the rev that contains the blocking parts of EZ-1 through EZ-9 and any promoted + requirement. + +## Landing and pinning + +Whether the blocking requirements land in PR 275 before it merges or in follow-up EdgeZero pull +requests is the EdgeZero maintainers' call. TS needs three things from the rev it pins: + +- It contains the blocking parts of EZ-1 through EZ-9 and any requirement promoted to blocking. +- It has a passing EZ-Q1 record for that exact commit. EdgeZero produces the results for criteria + in unit, Viceroy 0.21.0, workerd, Spin 4.1, and Axum that do not need TS, and assembles and keeps + the record. TS produces its end-to-end checks and all hosted Fastly results, running an + EdgeZero-provided qualification build for criteria TS configuration cannot express and its own + build for the rest, in single-request mode unless a criterion says otherwise. +- It stays reachable. A tag at that commit is enough, so a later force-push or squash merge cannot + leave TS pinned to an unreachable commit. + +## Proposed priority changes + +The EdgeZero and TS maintainers decide these. + +| ID | Change | Reason | +| ----- | ---------------------------------------- | --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| EZ-2 | Split into blocking and reuse-mode parts | Production runs single-request sandboxes. Connection reuse across those sandboxes (EZ-2.1 to 2.3, 2.6 to 2.10, and 2.13 to 2.17) blocks the cut. Behavior specific to reusable-sandbox mode (EZ-2.4, 2.5, 2.11, 2.12, and 2.18) is needed only before TS enables reuse mode, so it is a follow-up. | +| EZ-15 | Stays Follow-up (recommended) | TS emits Tinybird telemetry inside the `` hold today. Until EZ-15 lands, TS sends the telemetry rows with an ordinary send after delivery, inside the same lifecycle callback (cross-cutting requirement 4), so the page tail is not held. Promote the Fastly part (EZ-15.1 to 15.8, 15.10, and the Fastly clause of 15.11) only if that later emission is not acceptable. | +| EZ-24 | Minimal core Blocking (recommended) | The cut deletes TS's 22 outbound test doubles. Their replacement must match EdgeZero's begin, non-blocking batch checks, arrival-time completion, cutoff, and redirect behavior exactly, and a TS-written copy would drift and be thrown away later. The minimal core is EZ-24.1 to 24.10, with EZ-24.2 limited to the host and wasm32-wasip1 targets. | +| EZ-18 | Stays Follow-up | At F every buffered Fastly response goes out chunked with no `Content-Length`, where TS sends a fixed length today. Header parity is the only reason to promote it, since the tsjs bundles carry no `Content-Length` and host compression is unaffected. TS lists the change as an accepted Fastly difference until EZ-18 lands. | +| EZ-23 | Stays Follow-up | TS's Axum binary moves to `run_app`, which applies `Hooks::configure`. Only programmatic servers and in-process tests lose the policies, and TS keeps the ingress-level checks in its integration suite, which runs the real binary. | +| EZ-13 | Close as documentation and evidence | WASI HTTP 0.3 in spin-sdk 7.0.0 has one authority for routing and `Host`, so the override cannot be built. EdgeZero documents `Unsupported` as final with the evidence, and TS rejects `origin_host_header_override` on Spin at config validation. | + +## Verification + +Each requirement lists acceptance criteria and the TS check that proves it end to end. EdgeZero's +evidence comes from unit and adapter tests, Viceroy 0.21.0 with Fastly SDK 0.13.1, workerd, and +`spin up` on Spin 4.1, at Rust 1.98.1. TS reaches the same toolchain through TS PR #1123, which the +cut depends on. EZ-Q1 states what the qualification record must show before production. The hosted +measurements in EZ-2 to EZ-4 compare against a pre-cut TS build from `main` at `a9037770a` +with TS PR #1123. TS also runs its integration suite (Viceroy, workerd, and the Axum binary), its +browser suite, the Spin smoke, and a Fastly response diff against the same pre-cut build. + +## Related EdgeZero work outside these requirements + +These EdgeZero pull requests are not needed for the cut, but TS work that lands after it depends on +them, and each needs a rebase onto PR 275: + +- **EdgeZero PR #403** (pre-dispatch hooks and request ingress fidelity). TS #1107 and #1251 use it + heavily. F's ingress admission policy is synchronous and read-only, so it does not replace #403. +- **EdgeZero PR #389** (request timing). TS #1261 uses it. +- **EdgeZero PR #409** (trusted proxy metadata) touches the same request path. + +## Citation roots + +- `edgezero:` EdgeZero at F, `f6c708ccb5be19ba9fa92575db3c08c0f17e0d72`. +- `edgezero@v0.0.9:` EdgeZero at tag `v0.0.9`. +- `edgezero-ci:` GitHub Actions run 38934086784 on F. +- `ts:` Trusted Server `main` at `a9037770a`. +- `ts@pr1123:` Trusted Server at the head of PR #1123, commit `4c5ca8ae5`, by file and line. +- `fastly-sdk:` the `fastly` 0.13.1 crate. `fastly-sys:` and `fastly-shared:` the 0.13.1 crates. +- `viceroy:` `viceroy-lib` 0.21.0. `viceroy-0.17.0:` `viceroy-lib` 0.17.0, cited only to show + history. `hyper-0.14.26:` the hyper version Viceroy 0.21.0 uses. +- `spin-sdk:` spin-sdk 7.0.0. `worker:` worker 0.8.5. `reqwest:` reqwest 0.13.4. +- `dep:-/` a registry crate at the version EdgeZero pins. + +Claims marked **Unverified** say what would verify them. + +## Blocking requirements + +### EZ-1. Bodiless GET/HEAD forwarding + +**Priority:** Blocking +**Adapters:** Fastly, Axum, Cloudflare, Spin. The outcome must be the same everywhere. Adapters may differ only in whether the inbound body has to be read to learn that it is empty (EZ-1.2). +**Related:** EZ-6, whose begin call must accept the same bodiless request. EZ-9, which owns resolving the body once per redirect chain (EZ-9.7). EZ-10, which leaves a GET carrying a body at 400 and moves only size overflow to 413. EZ-Q1, which requires a passing result for every criterion here in the environment it names. EZ-25, whose EZ-25.1 owns the forwarding-example change. + +#### Need + +- TS proxies every request that no TS route claims to the publisher origin. Fastly registers that fallback for seven methods (`ts:crates/trusted-server-adapter-fastly/src/app.rs:291-301`), GET, POST, HEAD, OPTIONS, PUT, PATCH and DELETE, on every route path and on `/` and `/{*rest}` (`:1319-1331`). Axum, Cloudflare and Spin call the same core handler (`ts:crates/trusted-server-adapter-axum/src/app.rs:253`, `ts:crates/trusted-server-adapter-cloudflare/src/app.rs:562`, `ts:crates/trusted-server-adapter-spin/src/app.rs:843`). +- The handler rewrites the inbound request in place (`ts:crates/trusted-server-core/src/publisher.rs:3123-3146`, called at `:4810`) and moves it, body included, into the origin request (`:4812-4813`). +- After the cut TS forwards with `OutboundRequest::from_request(req, origin_uri)` and `HttpClient::send`, streaming the inbound body upstream. Only the overlapped origin fetch uses EZ-6's begin call (`publisher.rs:4847-4872`). TS overlaps only when all of these hold. The request is a navigation GET with an EC id and a KV store, and the EC snapshot is needed (`:3109-3116`, `:4821-4833`). The request has no shared-template cache key. The client supports concurrent fan-out and pending streamed responses (`:4840-4846`). +- Without EZ-1 every forwarded GET and HEAD fails with 400. That is every page navigation, every HEAD, and every asset that falls through to the publisher, which is most TS traffic. The TS integration scenarios that load a page through the fallback (`ts:crates/trusted-server-integration-tests/tests/frameworks/scenarios.rs:68-69`) fail on every runtime. +- TS cannot work around it without the body handling that decision 2 forbids. Forcing an empty body on every GET silently drops a body a client really sent, and checking emptiness means TS reads the inbound body itself. +- Today TS on EdgeZero v0.0.9 buffers the client body on Fastly (`edgezero@v0.0.9:crates/edgezero-adapter-fastly/src/request.rs:520-525`) and forwards any non-empty body, GET included (`ts:crates/trusted-server-adapter-fastly/src/platform.rs:330-340`). After EZ-1 a GET that carries body bytes gets 400. Cloudflare's `fetch` forbids a GET body, and EdgeZero already normalizes to that rule (`edgezero:docs/superpowers/specs/2026-05-21-outbound-http-design.md:1588-1591`). TS accepts the change. + +#### Gap at F + +- Every adapter hands the application a streamed body for every admitted request, bodiless or not. See Fastly `edgezero:crates/edgezero-adapter-fastly/src/request.rs:498-507`, Axum `edgezero:crates/edgezero-adapter-axum/src/request.rs:43-48`, Cloudflare `edgezero:crates/edgezero-adapter-cloudflare/src/request.rs:370-385` and `:657-658`, and Spin `edgezero:crates/edgezero-adapter-spin/src/request.rs:574-579`. The stream constructor always yields a stream (`edgezero:crates/edgezero-core/src/body.rs:63-68`). Cloudflare used to keep an empty body when the Worker request reported none. Commit 86f6e834 replaced that with a lazy read, because the browser test runtime can report no body stream for a request that does carry bytes. +- `OutboundRequest::from_request` keeps that stream and drops `Content-Length` and `Transfer-Encoding` at once (`edgezero:crates/edgezero-core/src/outbound.rs:927-934`, `:1858`, `:1863-1866`). +- Preflight rejects every GET/HEAD with a streamed body with 400 (`outbound.rs:1561-1575`). EdgeZero's outbound design calls that a deliberate false positive (`outbound-http-design.md:1619-1639`), and it predates lazy ingress. +- EdgeZero's own forwarding example (`edgezero:docs/guide/proxying.md:21-54`) and the scaffold and demo `proxy_demo` routes, declared for GET (`edgezero:crates/edgezero-cli/src/templates/root/edgezero.toml.hbs:64-67`, `edgezero:examples/app-demo/edgezero.toml:70-76`), fail the same way. This was found by reading the code, not by running it. The guide example also does not compile, because it passes `ctx.into_request()` to `from_request` without unwrapping the `Result` it returns (`proxying.md:32`, `edgezero:crates/edgezero-core/src/context.rs:381`). No test catches the failure, because every `from_request` GET test supplies an empty buffered body (`outbound.rs:4391-4403`, `edgezero:crates/edgezero-cli/src/templates/core/src/handlers.rs.hbs:580-603`, `edgezero:examples/app-demo/crates/app-demo-core/src/handlers.rs:1411-1415`). +- Fastly, Cloudflare and Spin do not carry the client's HTTP version into the core request (Fastly `request.rs:711-719`, Cloudflare `request.rs:327-342`, and Spin, whose WASI request has no version, `dep:wasip3-0.7.1+wasi-0.3.0/wit/deps/http.wit:266-343`). Request headers alone therefore cannot tell an HTTP/1.1 bodiless GET from an HTTP/2 or HTTP/3 GET that carries DATA frames with no `content-length`. +- Hosts differ in whether they say a body is empty before it is read. hyper marks a zero-length HTTP/1 body as ended (`dep:hyper-1.12.0/src/proto/h1/dispatch.rs:313-314`). Fastly reports a known length only in some cases, and its docs list chunked HTTP/1.1 and HTTP/2-or-later without `content-length` as unknown (`fastly-sdk:src/http/body.rs:339-352`). Spin knows only an explicit `Content-Length` (`dep:wasip3-0.7.1+wasi-0.3.0/src/http_compat/mod.rs:148-153`, `:278-296`). F declares every adapter's framing host-managed (`edgezero:docs/guide/capabilities.md:98-102`). +- On the outbound side, Spin hands the host a body stream for every request, even an empty buffered GET (`edgezero:crates/edgezero-adapter-spin/src/outbound.rs:362-364`). Whether the Spin host then frames an empty GET as chunked is **Unverified**. Fastly sends an empty buffered body as an empty set body (`edgezero:crates/edgezero-adapter-fastly/src/outbound.rs:445-453`) and a still-streamed body through its streaming send (`:1170-1187`). Whether the Fastly host frames a zero-byte streamed upload as chunked is **Unverified**. + +#### Requirements + +- **EZ-1.1** An inbound GET or HEAD is bodiless when the client sent no body bytes, whatever the protocol. For HTTP/1.1 that includes no `Content-Length` and no `Transfer-Encoding`, `Content-Length: 0`, and a chunked body that holds only the terminating zero-length chunk. For HTTP/2 and HTTP/3 it means the stream ended with the request headers, or the DATA frames held zero bytes, with or without a `content-length` field. A bodiless GET or HEAD built with `OutboundRequest::from_request` MUST forward successfully on all four adapters, through `HttpClient::send`, through EZ-6's begin call, and through EZ-9's redirect following. The upstream request MUST carry no body bytes and no `Transfer-Encoding`, and SHOULD NOT carry `Content-Length: 0`. +- **EZ-1.2** Where the host already knows the inbound body is empty, EdgeZero MUST NOT read it. A bodiless forward on that path can never fail with the inbound read deadline, and `RequestContext::take_body` and `into_request` yield an empty buffered body. These levels go in the capability matrix: + - Axum. Native. + - Cloudflare. Native when workerd reports no request body, and BestEffort if acceptance criterion 3 shows it does not. **Unverified** on workerd. Treating a request that reports no body as empty MUST NOT drop a body in a runtime that reports no body stream while still carrying bytes (the browser test runtime case behind commit 86f6e834). + - Fastly. BestEffort. It depends on whether the host reports a zero length for HTTP/1.1, HTTP/2 and, where the service serves it, HTTP/3 bodiless requests. **Unverified** on hosted Fastly (acceptance criterion 2). + - Spin. Unsupported, except for an explicit `Content-Length: 0`. + + On every other path EZ-1.4 and EZ-1.5 apply. + +- **EZ-1.3** EdgeZero MUST NOT decide emptiness from request headers alone. A GET whose headers declare no body, but which carries data (for example HTTP/2 DATA frames with no `content-length`), MUST be treated as carrying a body. The framing EdgeZero reports to the application's admission policy (`IngressHead::framing()`) MUST stay `IngressFraming::HostManaged`. Learning emptiness is not raw framing validation. +- **EZ-1.4** When the adapter does not know whether a forwarded GET/HEAD body is empty, EdgeZero MUST resolve it before sending, reading no more than it takes to see the first body byte or the end of the body. Zero-length chunks do not count as body bytes. An empty body makes the forward bodiless (EZ-1.1). A body with bytes makes the forward fail as in EZ-1.8. An error while reading the inbound body MUST surface unchanged. An expired inbound read deadline stays 408, and a failed host read stays in the existing internal-error category. +- **EZ-1.5** The time spent resolving the body MUST count against the outbound request's own timeout or deadline. If the client neither sends a byte nor ends the body, the forward MUST fail no later than the first of two limits. The outbound budget fails with the gateway-timeout category (504). The inbound read deadline fails with 408. This holds for single send, the begin call (EZ-6), and redirect following (EZ-9). Axum, Cloudflare and Spin MUST enforce the limit while waiting. Fastly MAY observe it only after a blocked host read returns. That is BestEffort, the same limitation as F's `inbound-read-deadlines` cell (`capabilities.md:74`, `:78-83`). +- **EZ-1.6** Only GET and HEAD get this treatment. Other methods, and GET/HEAD requests whose body is already buffered, MUST behave as at F and MUST NOT cause an extra read. The single unsupported-option rejection (cross-cutting requirement 2) MUST come before any resolving, so a GET/HEAD carrying an option the adapter or send mode marks Unsupported is rejected without its body being read. EdgeZero MAY resolve emptiness before its other outbound preflight checks. If it does, a GET/HEAD with a streamed body and some other invalid setting may wait for the inbound body before that error comes back, and the guide MUST say so. +- **EZ-1.7** Batches keep EZ-6's rule against streamed request bodies. A forwarded bodiless GET/HEAD MUST be accepted into a batch after the application has read the inbound body with `RequestContext::body_bytes`. It SHOULD be accepted without that step wherever EZ-1.2 holds. +- **EZ-1.8** A forwarded GET/HEAD that carries body bytes MUST fail with the same 400 F returns today for a GET with a non-empty buffered body (`outbound.rs:1564-1567`). It MUST fail before any backend or connection is set up, and no body byte may reach the upstream. +- **EZ-1.9** The existing `RequestContext` body semantics MUST NOT change. `body_kind()` reports the body as not yet read until it is read or taken. `body_bytes` on a bodiless request returns empty bytes and caches them. A body that was already read forwards its cached bytes and is not read again. A body whose read failed forwards the same error (`context.rs:381-394`). A body that was taken forwards empty. +- **EZ-1.10** Adapters MUST NOT frame an outbound GET/HEAD with an empty body as chunked. That means no `Transfer-Encoding` and no zero-length chunk terminator. For other methods with an empty buffered body, adapters SHOULD send no chunked framing either. +- **EZ-1.11** A forwarded OPTIONS, DELETE, POST, PUT or PATCH built with `OutboundRequest::from_request` MUST succeed through `HttpClient::send` on all four adapters, with the inbound body streamed upstream byte for byte. That includes a bodiless one, such as a CORS preflight OPTIONS with no `Content-Length` on Spin. Where EZ-1.2 holds, a bodiless one SHOULD go out with no body framing. Elsewhere an adapter MAY send it as a zero-length chunked body. EZ-6's begin call rejects these streamed uploads before sending, so TS uses `send` for them. +- **EZ-1.12** Redirect following resolves an inbound GET/HEAD body once per chain (EZ-9.7). +- **EZ-1.13** These invariants MUST hold. This requirement owns the rule on unread bytes, and EZ-10.11 refers to it. + - A GET/HEAD body is never forwarded. + - Outbound framing comes only from the body EdgeZero actually sends, never from inbound framing headers. + - On connections EdgeZero owns (Axum HTTP/1), request bytes left unread after any rejection, including EZ-1.8's 400 and EZ-10's 413, are never parsed as a new request. Closing the connection is acceptable. + - Polling the body of a GET that sent `Expect: 100-continue` may cause a `100 Continue` before the 400 (`dep:hyper-1.12.0/src/proto/h1/conn.rs:411-416`). That is acceptable. +- **EZ-1.14** EZ-25.1 owns the change to the `docs/guide/proxying.md` forwarding example. The example MUST forward a bodiless GET, a HEAD and a POST with a body correctly and MUST compile as a doc test. The capability guide MUST carry the EZ-1.2 levels and note that a late forward on a read path can fail with 408. + +#### Acceptance criteria + +1. Environment is Axum, Viceroy 0.21.0, workerd and Spin 4.1. Given a route that forwards the inbound request with `from_request` and `send` to an origin that records what it receives, when a client sends a bodiless HTTP/1.1 GET, then the client gets the origin's status and the origin saw a GET with zero body bytes and no `Transfer-Encoding`. Record whether `Content-Length: 0` appeared. The same holds for HEAD, and for a GET whose chunked body holds only the terminator. +2. Environment is hosted Fastly, Trusted Server staging. Repeat criterion 1 with HTTP/1.1 and HTTP/2 clients that send no `content-length`. Repeat it with an HTTP/3 client where the staging service has HTTP/3 enabled. Whether a Fastly Compute service serves HTTP/3 to clients is **Unverified**. Where it does not, the record states that HTTP/3 was not available and marks the HTTP/3 leg not applicable. Record per protocol whether a bodiless forward still succeeds after the inbound read deadline has passed, which shows whether EZ-1.2 held. +3. Environment is workerd. The run first records whether workerd reports no request body for a bodiless GET. Where it does, a handler that takes the body of a bodiless GET observes an empty buffered body. Where it does not, the run records that, and Cloudflare's EZ-1.2 level is BestEffort. +4. Environment is Axum, Viceroy 0.21.0, workerd and Spin 4.1. When a client sends a GET with `Content-Length: 5` and five bytes, then the client gets the EZ-1.8 400 and the origin records nothing. On workerd, record whether the body reached the guest at all. On Axum, a request pipelined after it is either refused by a connection close or answered from the correct offset. +5. Environment is hosted Fastly, Trusted Server staging. An HTTP/2 GET with a DATA frame and no `content-length` gets the EZ-1.8 400, and the origin records nothing. +6. Environment is Axum. Given an outbound timeout of 200 ms, when a client sends a chunked GET and never sends a chunk, then the forward fails with 504 within 205 ms. With an inbound read deadline of 100 ms, it fails with 408 within 105 ms. On hosted Fastly (Trusted Server staging) the same scenario records the observed outcome and time as BestEffort. +7. Environment is unit. A GET whose streamed body yields only zero-length chunks and then ends is sent bodiless. A GET with a streamed body and an already expired deadline fails with 504, and the body is not read. A POST with a streamed body, and a GET with an empty buffered body, read nothing extra. A taken body forwards empty. A body whose read failed forwards the same error. +8. Environment is Axum and workerd. A bodiless GET forwarded after the inbound read deadline has passed still succeeds (EZ-1.2). On workerd this holds where criterion 3 showed that workerd reports no request body. Otherwise the run records the outcome, and Cloudflare is BestEffort. On Spin 4.1 it succeeds when the request had `Content-Length: 0`. +9. Environment is Axum, Viceroy 0.21.0, workerd and Spin 4.1. A forwarded POST with a 3-byte body arrives at the origin with exactly those 3 bytes. A forwarded bodiless OPTIONS and a bodiless DELETE with no `Content-Length` succeed, and the origin sees zero body bytes. Record whether chunked framing was used. +10. Environment is Axum, Viceroy 0.21.0, workerd and Spin 4.1, once EZ-6 is present. A bodiless navigation GET begun with EZ-6's begin call reaches the origin with no body, under EZ-6's rule for when a begun request is on its way. A begun POST with a streamed body is rejected before anything is sent. +11. Environment is Axum and Viceroy 0.21.0, once EZ-9 is present. A forwarded bodiless GET whose origin answers 307 to a second URL makes two requests, both bodiless, and the inbound body is read at most once. +12. Environment is Axum, Viceroy 0.21.0, workerd and Spin 4.1. After the handler calls `RequestContext::body_bytes` on a bodiless GET, the forwarded request is accepted into a batch. +13. Trusted Server end to end. `cargo test --manifest-path crates/trusted-server-integration-tests/Cargo.toml --test integration -- --include-ignored` passes `test_wordpress_fastly`, `test_nextjs_fastly`, `test_wordpress_cloudflare`, `test_nextjs_cloudflare`, `test_wordpress_axum` and `test_nextjs_axum` (`ts:crates/trusted-server-integration-tests/tests/integration.rs:137-167`, `:272-285`) with TS forwarding through `from_request` and no TS-side body handling. Those tests are ignored by default, so the flag is required, as TS CI passes it (`ts:.github/workflows/integration-tests.yml:106`, `:148`). New TS cases pass for `HEAD /` (origin status, no body), a forwarded `OPTIONS /` (origin status), and `GET /` with a body (400). TS's in-process parity suite does not cover adapter ingress, so it is not enough on its own. +14. Environment is Axum, Viceroy 0.21.0, workerd and Spin 4.1. A GET carrying an option the adapter marks Unsupported, with a streamed inbound body whose client never sends a byte, is rejected at once with the unsupported-option error, and the forward does not wait for the body. + +#### Out of scope + +- Forwarding a GET or HEAD body. +- Raw framing validation. F's `raw-ingress-framing-validation` capability stays as it is. +- Resolving emptiness for methods other than GET and HEAD, beyond what EZ-1.11 asks. +- Changing the batch rule on streamed bodies. EZ-6 owns that rule. + +#### Open questions + +1. Should a bodiless OPTIONS or DELETE that the adapter cannot recognize as empty without reading be sent without chunked framing, which means reading before sending? TS prefers yes only if hosted Fastly turns out not to report a zero length for HTTP/2 and HTTP/3 bodiless requests (acceptance criterion 2). TS forwards CORS preflight OPTIONS, and some origins handle a chunked OPTIONS badly. Otherwise TS prefers no. + +### EZ-2. Fastly connection reuse across requests + +**Priority:** Blocking for EZ-2.1 to EZ-2.3, EZ-2.6 to EZ-2.10, and EZ-2.13 to EZ-2.17. Follow-up for EZ-2.4, EZ-2.5, EZ-2.11, EZ-2.12, and EZ-2.18, which apply to reusable-sandbox mode and are required before Trusted Server enables it. Trusted Server production runs single-request sandboxes (`ts:crates/trusted-server-adapter-fastly/Cargo.toml:13-18`). +**Adapters:** Fastly (the behavior that changes). Axum, Cloudflare, Spin (client precedence only; Axum keeps its existing cross-request pooling). +**Related:** EZ-3 (removing the dispatch slack depends on EZ-2.6). EZ-4 (connect timeout is part of reuse eligibility). EZ-6 (begun requests and batch observation inherit EZ-2; EZ-6.16 owns a slot whose deadline is the cutoff). EZ-8 (Image Optimizer exception to EZ-2.6). EZ-9 and EZ-15 (count toward the limit). EZ-11 and EZ-12 (settings that affect eligibility). EZ-17 (owns the helpers' shape). EZ-Q1 (gates these criteria). + +#### Need + +Trusted Server registers Fastly dynamic backends itself today, and their names repeat across requests and sandboxes. The name covers scheme, host, port, certificate check, Host override, a per-provider discriminator, and the first-byte and between-bytes timers, and no counter or request value (`ts:crates/trusted-server-core/src/platform/backend_naming.rs:304-339`, `:352-421`). TS floors remaining budgets onto a ladder so timer values repeat (`:18-22`, `:482-502`), and canonicalizes the mediator's remaining time so "the backend name remains stable across equivalent budget values" (`ts:crates/trusted-server-core/src/auction/orchestrator.rs:958-966`). Registration treats `NameInUse` as reuse (`ts:crates/trusted-server-adapter-fastly/src/backend.rs:256-273`). Config validation budgets provider backends against Fastly's limit of 200, reserving 40 for other traffic (`ts:crates/trusted-server-core/src/platform/backend_naming.rs:23-24`, `:105-111`; `ts:crates/trusted-server-core/src/auction/plan.rs:434-447`). Because a fresh single-request sandbox presents the same backend settings as the last one, Fastly can pool the TLS connections to the publisher origin and to each bidder across sandboxes on a node. + +After the cut TS deletes that layer. It sends origin, integration, and mediator calls with `HttpClient::send`, and bidder calls with `HttpClient::start_batch_until`, each request carrying `OutboundRequest::timeout` and the batch carrying a cutoff at auction start plus the auction timeout (2000 ms in the example config, `ts:trusted-server.example.toml:282`). Provider timeouts default to the auction timeout for the standard profile, 1000 ms for `prebid-server`, and 800 ms for APS (`ts:crates/trusted-server-core/src/auction/profile.rs:170-186`). Batch results carry their input index (`edgezero:crates/edgezero-core/src/outbound.rs:121-128`), so TS no longer needs per-provider backends, and two providers on one host may share connections. + +What regresses at F: + +- In a single-request sandbox, the TS production mode, reuse across sandboxes happens only when the effective budget repeats to the millisecond. The mediator, any provider launched with less than its timeout left (`ts:crates/trusted-server-core/src/auction/orchestrator.rs:453-462`), and any origin fetch bounded by a request deadline lose it. Each of those requests pays a new TCP and TLS handshake and a new backend registration. +- In a reused sandbox (TS feature `reusable-sandbox`, serving loop `ts:crates/trusted-server-adapter-fastly/src/main.rs:123-150`) no request reuses a connection an earlier request opened, and each request adds registrations the sandbox holds until it ends. A sandbox that serves 10 auctions with 15 bidder hosts holds more than 150, close to the platform limit of 200. +- The handshake cost on Fastly for TS's bidders has not been measured. Staging measurements M1 and M2 below measure it. + +#### Gap at F + +- Each `FastlyOutboundClient` takes a new value from a process-global counter and appends it to every backend name (`edgezero:crates/edgezero-adapter-fastly/src/outbound.rs:35-36`, `:57-65`, `:307-317`). The canonical lifecycle builds a new client for every request (`edgezero:crates/edgezero-adapter-fastly/src/request.rs:446`, `:707-734`). The counter restarts with each wasm instance, so a reused sandbox gives each request a fresh suffix. +- Backend identity includes the effective budget rounded up to the millisecond (`edgezero:crates/edgezero-adapter-fastly/src/outbound.rs:254-262`, `:664-685`, `:762-765`), and that budget is `deadline - now` at nanosecond precision (`edgezero:crates/edgezero-core/src/time.rs:138-187`). Any deadline- or cutoff-bound budget therefore gives a new backend per request. +- The backend cache lives in the client and is discarded with it (`edgezero:crates/edgezero-adapter-fastly/src/outbound.rs:290-295`, `:319-377`). +- Platform timers are connect at a quarter of the budget, first byte at the rest, and between bytes at the whole budget (`:718-733`). The wait for response headers, batch observation, and body reads have no guest-side limit (`:443-453`, `:1279-1290`, `:784-825`, `:1203-1238`). Viceroy 0.21.0 enforces the first-byte timer, which there covers connect plus the response head, and the between-bytes timer, and does not enforce connect timeouts (`viceroy:src/wiggle_abi/req_impl.rs:617-644`; `viceroy:src/upstream.rs:351-375`, `:412-414`). It reports a timer expiry as a generic host error (`viceroy:src/error.rs:327-329`), which the SDK reports as `SendErrorCause::InternalError` (`fastly-sdk:src/http/request.rs:3353-3364`) and F maps to `EdgeError::Internal` when the deadline has not passed (`edgezero:crates/edgezero-adapter-fastly/src/outbound.rs:73-75`, `:95-97`, `:959`). With F's three-quarter first-byte share, a slow response head is cut off before the budget, as a 500. +- The outbound design states that a newly constructed client does not intentionally reuse a previous client's connection and that the client must not outlive its request context (`edgezero:docs/superpowers/specs/2026-05-21-outbound-http-design.md:5139-5146`, `:5341-5352`). It assumes each request starts with an empty backend namespace (`:5413-5466`). +- The platform does not match that model. Viceroy creates its backend table once per sandbox and keeps it across the up to six requests a reused guest serves (`viceroy:src/sandbox.rs:54`, `:106-109`, `:170`), rejects any repeated name (`:698-707`), and builds a new HTTP client for every send, so it never pools connections (`viceroy:src/upstream.rs:356-368`). The SDK pools only for the same backend name with exactly the same settings, across sessions (`fastly-sdk:src/backend/builder.rs:281-294`). Fastly's documentation, fetched 2026-10-17 through a summarizing fetch tool and to be re-checked against the live pages, says "Compute registers dynamic backends at the node level." (Fastly's developer guide to backends, under Integrations and Non-Fastly services). It says a registration at the limit "blocks execution" until "another sandbox of the same service terminates and releases its backends" (Fastly's Compute errors reference). It lists "Maximum number of dynamic backends | 200 | per service" (Fastly's Compute resource limits page). +- Axum already pools across requests through one shared transport (`edgezero:crates/edgezero-adapter-axum/src/service.rs:73`; `edgezero:crates/edgezero-adapter-axum/src/request.rs:68-74`). + +#### Requirements + +Reuse (blocking): + +- **EZ-2.1** On Fastly, two outbound requests MUST be eligible to share a pooled connection when they target the same scheme, host, and port, with the same Host authority, the same TLS settings, and the same connect timeout (EZ-4), subject only to EZ-2.2. This MUST hold for two requests made while serving one inbound request, and for requests made by different single-request sandboxes of one service wherever Fastly pools across sandboxes. It MUST hold regardless of path, query, method, headers, body, cache policy, response mode, and send mode. +- **EZ-2.2** Reuse eligibility MUST NOT depend on the exact remaining budget. Requests with equal per-call timeouts MUST always be eligible with each other, even when a batch cutoff or caller deadline makes their effective budgets differ. For requests bounded only by a deadline or cutoff, the number of distinct backend configurations per origin MUST stay within a small documented bound however many requests are served. Requests whose budgets differ only by scheduling jitter MUST be eligible with each other except at documented boundaries. +- **EZ-2.3** Across sandboxes, requests with equal origin settings MUST be poolable wherever Fastly pools across sandboxes. Nothing request-, sandbox-, client-, or deployment-specific may prevent that within one EdgeZero version. When two EdgeZero versions run side by side during a rolling deploy, neither MAY fail a request because the other registered the same origin with different settings. + +Reuse in a reused sandbox (follow-up): + +- **EZ-2.4** In a reused sandbox, reuse eligibility MUST NOT depend on which client instance sends. The adapter's per-request default client, a client the application keeps for the sandbox, and a `FastlyOutboundClient::new()` client used after delivery inside the lifecycle callback (cross-cutting requirement 4) MUST all reuse the same connections. +- **EZ-2.5** In a reused sandbox the number of backend registrations MUST grow only with the number of distinct origin configurations used, not with the number of requests served. + +Deadlines (blocking): + +- **EZ-2.6** The per-call deadline MUST still be exact. On Fastly, on a backend used before and on a backend registered for this request, waiting for response headers and each response-body read MUST end no later than the request's deadline plus 5 ms, failing with `EdgeError::GatewayTimeout` (504) and the `BudgetSource` that selected the deadline. This MUST hold under Viceroy 0.21.0 and on hosted Fastly. The only exceptions are operations the host cannot interrupt. Those are first-time backend registration, writing a request body, reading the inbound body to settle a forwarded GET/HEAD (EZ-1.5), the Image Optimizer send of EZ-8, and a response-body read if the host cannot report body readiness (Open question 6). An overrun in one of those MUST be reported as the same 504 when the operation returns, and the capability footnote MUST name each exception that applies. +- **EZ-2.7** A platform-side timer MUST NOT fail a request before its own deadline for any budget up to Fastly's 2-minute runtime limit. A platform-side timer that fires at or after the deadline MUST be reported as the EZ-2.6 504 with the deadline's `BudgetSource`, never as an internal error. This holds under Viceroy 0.21.0 and on hosted Fastly. +- **EZ-2.8** In a batch, a slot whose own deadline passes before the observation cutoff MUST yield its 504 item at that deadline, without waiting for a sibling to finish or for a platform timer. A slot whose deadline is the cutoff follows EZ-6.16. The behavior when the observation cutoff passes is otherwise unchanged from F (`edgezero:crates/edgezero-adapter-fastly/src/outbound.rs:621-651`). + +Caching and safety (blocking): + +- **EZ-2.9** A single send and a one-slot batch of the same request MUST be indistinguishable in caching, validation, and errors. Cross-cutting requirement 1 owns the rule for the other send modes. Image Optimizer requests are governed by EZ-8. +- **EZ-2.10** EdgeZero MUST NOT send a request over a connection whose settings differ from what the request asks for (TLS settings, Host authority, connect timeout), or whose platform timers could end it before its deadline (EZ-2.7). If it cannot get such a connection, the request MUST fail before sending with a 500-class internal error, and the reason MUST be logged and kept off the wire. + +Backend limit and retirement (follow-up): + +- **EZ-2.11** One reused sandbox MUST NOT block without end on its own registrations. When the sandbox already holds 200 distinct backend configurations and a request needs another, that request MUST fail promptly with `EdgeError::ServiceUnavailable` (503) and nothing sent, and the sandbox MUST be retired after the current request. A registration that returns after the deadline follows EZ-3.4. +- **EZ-2.12** An application serving through the reusable Fastly lifecycle (`edgezero_adapter_fastly::lifecycle::serve_custom`, `edgezero_adapter_fastly::serve_app`, `edgezero_adapter_fastly::serve_app_with_hooks`) MUST be able to set a threshold of distinct backend configurations at which the sandbox stops being reused, with a documented default, and MUST be able to retire the sandbox from its own callback. Retirement takes effect after the current request completes. It MUST NOT produce an SDK-generated error response. The serving result MUST tell retirement apart from an error, and MUST keep reporting the request count that TS logs today (`ts:crates/trusted-server-adapter-fastly/src/main.rs:144-149`). A client the application keeps for the sandbox MUST NOT carry any request's in-flight exchange into the next request. + +Observation (blocking): + +- **EZ-2.13** An application MUST be able to read, for the inbound request it is serving, the number of backend registrations EdgeZero performed, the number of distinct backend configurations it used, and the total and longest registration time. A registration that takes 50 ms or more MUST be logged at `warn`. None of this reaches the wire. + +Client supply (blocking): + +- **EZ-2.14** EdgeZero MUST document and keep this precedence for the `HttpClient` a handler receives from `RequestContext::http_client` (`edgezero:crates/edgezero-core/src/context.rs:372-374`). A client registered with `RouterBuilder::with_state` wins (`edgezero:crates/edgezero-core/src/router.rs:348-354`, `:407-411`). On Fastly, a client inserted through the `prepare` hook of `send_request_with_registries_and_hooks` comes next (`edgezero:crates/edgezero-adapter-fastly/src/request.rs:446-448`). The adapter default comes last. The router-state rule MUST hold on all four adapters. A client inserted through `prepare` MUST keep reaching the handler. EZ-17 owns the helpers' shape. + +Invariants and documentation (blocking): + +- **EZ-2.15** Axum MUST keep its cross-request pooling. Cloudflare and Spin keep their platform-managed connections. No new capability row is required for pooling, which no adapter can promise. +- **EZ-2.16** EdgeZero MUST NOT retain paths, queries, headers, or bodies across requests. In a reused sandbox it MAY retain origin host names and backend configuration. Its guide MUST tell applications not to let untrusted input choose outbound hosts, because every distinct origin consumes service-wide backend capacity. +- **EZ-2.17** EdgeZero's Fastly guide and capability matrix MUST describe the reuse behavior, the deadline guarantee of EZ-2.6 and its exceptions (Fastly `outbound-deadlines` stays `BestEffort`), the limit and retirement behavior once EZ-2.11 and EZ-2.12 land, and the observation of EZ-2.13. The guide MUST record that the outbound design's dynamic-backend model is superseded, without editing `docs/superpowers/specs/`. The CHANGELOG MUST list each behavior change. + +Per-sandbox accounting (follow-up): + +- **EZ-2.18** In a reused sandbox, an application MUST be able to read the number of distinct backend configurations the sandbox holds and the number of registrations it performed since it started. + +#### Acceptance criteria + +Blocking criteria come first. Viceroy 0.21.0 never pools connections, so reuse is visible there only as registration counts. + +Unit (any host): + +1. Given a client in router state, another inserted through Fastly `prepare`, and the adapter default, when a handler calls `RequestContext::http_client`, then it receives the router-state client. Given only the `prepare` client and the default, it receives the `prepare` client. The router-state case holds on Axum, Cloudflare, and Spin against their defaults. + +Viceroy 0.21.0, one inbound request per case: + +2. Given one inbound request that runs two batches to one loopback origin with per-call timeouts of 800 ms and cutoffs leaving 640 ms and 655 ms, when both complete, then the EZ-2.13 observation reports one registration and one distinct configuration. +3. Given one inbound request whose sends differ only in scheme, port, Host authority, TLS settings, or connect timeout, then the observation reports one distinct configuration per variant. +4. Given a loopback origin that delays its response head by 500 ms and a request with a 100 ms timeout, when sent, then it fails with 504 and `BudgetSource::PerCallTimeout` between 100 and 105 ms after dispatch. (At F, by reading the code, Viceroy's 75 ms first-byte timer fires first and the request fails with an internal error at about 75 ms.) +5. Given an origin that sends its head at once and then one body byte every 300 ms, and a buffered request with a 500 ms timeout, then it fails with 504 between 500 and 505 ms. Where EdgeZero documents that the host cannot report body readiness, it fails at the first read that returns after 500 ms instead. The run records which. +6. Given a batch with a 1 s cutoff, one slot to an origin that never answers with a 200 ms timeout, and two fast slots, then the silent slot's 504 item arrives between 200 and 205 ms and the batch completes. + +Hosted Fastly, single-request mode. Who produces each result follows EZ-Q1.4. EdgeZero produces criteria 1 to 6. Criteria 11, 12, 13b and 14 need settings Trusted Server's configuration cannot express (a controllable slow origin, a 90 s timeout, a one-slot batch to a test origin, a second EdgeZero commit), so EdgeZero provides a qualification build at the pinned commit and TS deploys it to a TS staging service. TS produces criteria 7 to 10 and 13a with its own builds, and criterion 15. The baseline is a Trusted Server build of `main` at `a9037770a` with TS PR #1123, which registers its own stable backends. The candidate is TS's cut build at the EdgeZero commit TS pins. Criteria 7 to 10 replay a fixed page-view mix at a fixed rate with build order randomized, and report p50, p95, p99, and sample counts per POP. A bound measured inside the guest from dispatch or a deadline keeps its stated tolerance on hosted Fastly. The thresholds in criteria 7 to 10 are Trusted Server's proposed product thresholds (Open question 8). + +7. **M1, handshakes per bidder per auction.** TS-controlled staging endpoints for one provider and the origin log new TCP connections and full versus resumed TLS handshakes. After a 5-minute warm-up, the candidate's new handshakes per request to the origin and to that provider MUST NOT exceed the baseline's by more than 10 percent. +8. **M2, latency.** Per bidder, the candidate's p95 slot latency and p95 response-head latency MUST each be no more than 10 ms above the baseline's. +9. **M3, registration.** In the candidate, the EZ-2.13 observation for each inbound request MUST show no more registrations than the distinct origin configurations that request used, The total registration time per inbound request is recorded with its distribution and serves as EZ-3's reference. It has no pass condition, because the pre-cut build does not time its own registrations and the platform sets that latency. +10. **M5, pool exhaustion.** At peak replay rate, the candidate's gateway failures, as a share of that origin's requests, MUST be no more than 10 percent above the baseline's share or 0.1 percentage points above it, whichever is larger. Where EZ-16 has landed, the same bound applies to failures whose provider cause is a connection limit. +11. **M6, deadline in production.** Criteria 4 and 5 pass against a staging origin with the same delays, with the windows they state. +12. **Long budgets.** A request with a 90 s per-call timeout to an origin that sends its head at 80 s succeeds (EZ-2.7). +13. **Rolling deploy.** + - 13a. While the baseline and the candidate are active together during a staged rollout to one TS staging service, both serve the same origins, the candidate reports zero EZ-2.10 internal errors, and neither build's requests fail at registration. + - 13b. When the pinned commit changes backend configuration relative to the EdgeZero commit of the previous passing record, the qualification builds of the two commits active together on one staging service both serve the same origins with zero registration failures. The first record has no previous commit, and 13a alone applies. +14. **Caching parity.** A cacheable GET sent twice with `send`, and twice as a one-slot batch, produces the same number of origin receipts in each mode (one per pair), and the second response of each pair is served from cache in both modes. This runs on hosted Fastly only, because Viceroy's plain send path does not cache (`viceroy:src/wiggle_abi/req_impl.rs:946-977`). + +Trusted Server end-to-end check (Viceroy 0.21.0, single-request mode): + +15. One auction with three bidder hosts, two providers configured on one of them, and one origin fetch reports four registrations for the inbound request. + +Follow-up criteria (reusable-sandbox mode). Viceroy 0.21.0 serves at most six requests per reused guest. Cases 16 and 17 stay on one guest. Cases 18 and 19 expect a new guest. + +16. Given a reused guest and three sequential requests to the same loopback origin, the EZ-2.18 accounting reports one registration. A fourth request to a second loopback port raises it by one. (At F it reports three after the first three requests.) +17. Given one reused guest where a request on the adapter default client, a request on a client kept for the sandbox, and a `FastlyOutboundClient::new()` send after delivery all target one origin, the accounting reports one registration. +18. Given a reused guest whose first requests send to 200 distinct loopback ports, when a later request needs a 201st, then it fails with 503 within 5 ms and nothing is sent, and the next request is served by a new guest. +19. Given a retirement threshold of 2, when two requests use two loopback ports, then the third request is served by a new guest. Given a callback that asks for retirement on its first request, a new guest serves the second request. In both cases the serving result reports retirement, no error, and the request count served. +20. **M4, limit model, hosted Fastly in reuse mode.** Report distinct backends against sandbox ordinal, retirements, 503s from EZ-2.11, and registrations of 50 ms or more. Then run K concurrent sandboxes with the same N origins, and again with disjoint origins, to learn whether identical configurations in different sandboxes share capacity and whether the 200 applies per node or per service. M1 is repeated in reuse mode, and the candidate's new handshakes per request after each sandbox's first 5 requests MUST NOT exceed the baseline's. +21. Trusted Server end to end, Viceroy 0.21.0 in reuse mode. Two sequential auctions with three bidder hosts and one origin register four configurations in total, and the second auction registers none. + +#### Out of scope + +- Guaranteeing connection reuse where Fastly does not pool, or across services. +- Per-request control of pool size, keepalive, or connection lifetime. +- Bounding first-time registration or request-body writes in time (EZ-2.6 covers only how overruns are reported). +- Exact deadlines for Image Optimizer requests (EZ-8). +- A capability row for pooling. + +#### Open questions + +1. **Default retirement threshold (EZ-2.12).** TS prefers 100 distinct backends until M4 reports the capacity model. If identical configurations share capacity across sandboxes, the default can rise toward 160. If not, it should fall, and the guide should say reuse suits only applications with few distinct origins. +2. **Overrun during uninterruptible operations.** How far past its deadline can a deadline-only request run during an operation EdgeZero cannot interrupt (EZ-2.6 exceptions)? TS prefers at most twice its budget. +3. **Operator visibility.** Should operators be able to tell from Fastly's own tooling which origin host a registered backend serves? TS prefers yes. It makes M3 and M4 attributable, and origin hosts are not secret. +4. Removed. +5. **Per-backend connection cap.** Fastly's SDK defaults each backend pool to 200 connections (`fastly-sdk:src/backend/builder.rs:132`). Sharing connections across requests makes that cap reachable. TS prefers keeping the default and revisiting it if M5 shows connection-limit failures. +6. **Body readiness.** Can EdgeZero on Fastly SDK 0.13.1 wait for response-body data with a timeout? EZ-2.6 and EZ-6.21 both depend on the answer. TS expects yes. The SDK's async I/O interface lists normal bodies as objects whose readiness can be awaited with a timeout (`fastly-sys:src/service0_1_0.rs:317-325`, `:460-471`), and Viceroy 0.21.0 implements timed waits over body items (`viceroy:src/async_io.rs:14-46`; `viceroy:src/sandbox/async_item.rs:154`). Hosted behavior is **Unverified** until M6 runs. +7. **Proving EZ-2.10.** How will EdgeZero show that a request is never sent over a connection with the wrong settings, given that the clash is not observable from outside except through criterion 13? TS prefers that EdgeZero's own tests cover it and that TS's hosted runs report zero EZ-2.10 errors. +8. **Staging thresholds for TS to confirm.** The proposed product thresholds are M1 handshakes at most 10 percent above the pre-cut build, M2 p95 slot and response-head latency at most 10 ms above it, M3 registrations per inbound request no more than its distinct origin configurations, and M5 gateway-failure share at most 10 percent or 0.1 percentage points above it, whichever is larger. TS prefers these because they allow measurement noise while still catching a build that loses pooling or adds a registration round trip to every request. + +#### Interim for Trusted Server (follow-ups only) + +Until EZ-2.4, EZ-2.5, EZ-2.11, EZ-2.12, and EZ-2.18 land, Trusted Server keeps `reusable-sandbox` off in production, as it is today (`ts:crates/trusted-server-adapter-fastly/Cargo.toml:13-18`), and serves single-request sandboxes. + +### EZ-3. Cold backend registration never fails a request on its own + +**Priority:** Blocking +**Adapters:** Fastly for the slack. EZ-3.3 holds on all four adapters. Axum, Cloudflare, and Spin have no slack and otherwise do not change. +**Related:** EZ-2 (EZ-2.6 must hold before the slack is removed, so both ship together). EZ-1 (the 408 of EZ-1.5). EZ-6 (begun requests and non-blocking batch observation follow the same rule). EZ-Q1 (gates these criteria). + +#### Need + +Trusted Server has no dispatch slack today. Its auction launches each provider and then collects completions, turning late ones into timeouts where its deadline policy says so (`ts:crates/trusted-server-core/src/auction/orchestrator.rs:440-613`, `:558-570`). After the cut every bidder goes through one `start_batch_until`. In a single-request sandbox no backend is registered yet, so the batch performs one first-time registration per bidder, one after another. F charges all of them, plus each earlier slot's preparation and send, against one 25 ms allowance measured from batch entry. A 12-bidder auction in which each registration costs a few milliseconds (**Unverified**, measured by M3) exhausts it partway through the list. The remaining slots fail with an internal error while more than 1900 ms of a 2000 ms auction remain. In TS's post-cut design an internal error is a transport error, so those bids are lost without a timeout signal. + +Single sends are exposed too. DataDome protection, off by default (`ts:trusted-server.example.toml:599`), runs with a 1500 ms default budget (`ts:crates/trusted-server-core/src/integrations/datadome.rs:284-286`) and fails open on error (`ts:trusted-server.example.toml:598`). If its registration takes more than 25 ms, F returns an internal error and TS skips bot protection for that request. + +#### Gap at F + +- `BATCH_DISPATCH_SLACK_MAX` is 25 ms (`edgezero:crates/edgezero-core/src/time.rs:8-9`), exported publicly (`edgezero:crates/edgezero-core/src/lib.rs:112-115`). +- Before every send, and before each first-time registration in a batch, F fails the request with `EdgeError::Internal` if more than 25 ms passed since the call began, even when the deadline is far away (`edgezero:crates/edgezero-adapter-fastly/src/outbound.rs:1338-1357`, `:449`, `:521`, `:1179`, `:767-780`). In a batch the 25 ms is shared by all slots (`:576`, `:585-588`). For a single send it includes the registration (`:565-566`). +- A public test hook exists only to exercise this (`:224-252`, `:2531-2532`), and a Viceroy test asserts the internal error (`edgezero:crates/edgezero-adapter-fastly/tests/contract.rs:1295-1321`). +- The outbound design justifies the slack as protection against platform timers armed with a stale budget, and concedes it detects rather than prevents overruns when a backend is new (`edgezero:docs/superpowers/specs/2026-05-21-outbound-http-design.md:5191-5269`). +- The other adapters size their timers from the budget remaining at dispatch and have no slack (`edgezero:crates/edgezero-adapter-axum/src/outbound.rs:478-484`; `edgezero:crates/edgezero-adapter-cloudflare/src/outbound.rs:636-646`; `edgezero:crates/edgezero-adapter-spin/src/outbound.rs:468-488`). + +#### Requirements + +- **EZ-3.1** A Fastly request or batch slot MUST NOT fail because EdgeZero's preparation or backend registration took time while the request's deadline was still in the future. The dispatch-slack internal error MUST no longer occur. +- **EZ-3.2** Before a response arrives, the only time-based failures MUST be `EdgeError::GatewayTimeout` (504) with the `BudgetSource` that selected the deadline or the connect-timeout cause of EZ-4.6, and the 408 of EZ-1.5 when the inbound read deadline expires while a forwarded GET/HEAD body is being settled. +- **EZ-3.3** On every adapter, a request whose deadline has passed before it is sent MUST fail with 504 and its `BudgetSource` without being sent, and on Fastly without a new backend registration. +- **EZ-3.4** A backend registration that returns after the deadline MUST fail its request with 504 and the request's `BudgetSource`, and the registered backend MUST remain usable by later requests. +- **EZ-3.5** EZ-2.6 MUST hold with the slack removed. +- **EZ-3.6** In a batch, a slot whose backend was registered before SHOULD NOT wait behind other slots' first-time registrations. Results keep input order, and each slot's elapsed time is still measured from batch entry. +- **EZ-3.7** `BATCH_DISPATCH_SLACK_MAX` and the slack test hook MAY be removed from the public API. Nothing in TS uses them. +- **EZ-3.8** EdgeZero's capability matrix footnote on Fastly deadlines (`edgezero:docs/guide/capabilities.md:260-266`) and the CHANGELOG MUST stop describing the 25 ms bound and MUST describe the EZ-2.6 guarantee instead. + +#### Acceptance criteria + +Unit (any host). How a test makes time pass inside EdgeZero is EdgeZero's choice. + +1. Given a 500 ms budget, when more than 25 ms passes inside the send call before dispatch, then the request is sent and no internal error occurs. +2. Given a batch of 20 slots to 20 new origins whose registrations each take 5 ms, and a 1 s cutoff, then all 20 are sent and none fails with an internal error. +3. Given a batch with a 1 s cutoff where the first slot's registration runs first and takes 20 ms, and the second slot has a 10 ms timeout, then the second slot yields 504 with `BudgetSource::PerCallTimeout`, is not sent, causes no registration, and the batch continues. +4. Given a batch ordered new, known, new, known, then both known-origin slots SHOULD be sent before the first registration, the run records the order, and results come back in input order. + +Viceroy 0.21.0: + +5. Given a 1 s budget, more than 25 ms passing before sending, and a loopback origin that answers 200, when the request is sent, then it returns 200. (At F it fails with the dispatch-slack internal error.) +6. Trusted Server end to end, single-request mode. A 12-bidder auction against 12 loopback ports completes with no internal-error slots. + +Hosted Fastly, single-request mode, with the same baseline and candidate as EZ-2. EdgeZero produces criteria 1 to 5. Trusted Server produces criteria 6 and 7 with its own builds (EZ-Q1.4). + +7. **M7, slack failures.** On the candidate, the number of slots and single sends that fail with an internal error before sending, including DataDome calls, MUST be zero across the replay, reported per 1000 auctions by bidder count. The candidate's per-slot 504 rate MUST NOT exceed the baseline's by more than 1 percentage point. Both bounds MUST hold for a 12-bidder auction profile. + +#### Out of scope + +- Making first-time registration faster or interruptible. +- Bounding request-body writes. + +#### Open questions + +1. **Cap on first-time registrations per batch.** TS prefers no cap. EZ-3.3 already stops registering for slots whose deadline passed, and EZ-2.11 bounds a reused sandbox's total. A per-batch cap would fail live slots for the same reason the slack did. +2. **M7 threshold for TS to confirm.** The proposed bound on the per-slot 504 rate is 1 percentage point above the pre-cut build. TS prefers it because a larger rise would mean bids lost to timeouts that the pre-cut build received. + +### EZ-4. Connect timeout independent of the total budget + +**Priority:** Blocking +**Adapters:** All four. Fastly Native. Spin BestEffort. Axum and Cloudflare BestEffort, with the setting accepted and the connect phase bounded by the total budget (cross-cutting requirement 2). +**Related:** EZ-2 (requests that differ in connect timeout do not share connections). EZ-6, EZ-9, EZ-15 (the connect timeout travels with begun requests, redirect hops, and, once EZ-15 lands, fire-and-forget sends). EZ-16 (logs tell a connect timeout from a DNS failure). + +#### Need + +Trusted Server sets connect to 1 s on every Fastly backend (`ts:crates/trusted-server-adapter-fastly/src/backend.rs:238-242`) and sets a provider's first-byte and between-bytes timers to its whole transport budget (`ts:crates/trusted-server-core/src/auction/plan.rs:385-398`). DataDome protection passes its 1500 ms as both body timers (`ts:crates/trusted-server-core/src/integrations/datadome/protection.rs:238-239`). Publisher origin fetches use a 15 s first-byte timeout (`ts:crates/trusted-server-core/src/publisher.rs:98`). An unreachable origin therefore fails in about 1 s, and bidders get their full budget to answer. + +F gives the connection a quarter of the budget and the response head the remaining three quarters. A 1000 ms bidder call gets 250 ms to connect and 750 ms to first byte, so a bidder loses a quarter of its think time even on a reused connection, and a slow TLS handshake fails at 250 ms with most of the budget left. DataDome gets 375 ms and 1125 ms. An origin fetch migrated with a 15 s timeout gets 3.75 s to connect, so an unreachable origin holds the page for 3.75 s where TS fails in 1 s today. + +TS needs the response head to get the whole remaining budget, and it needs to keep a 1 s connect bound on origin, integration, and bidder requests. + +#### Gap at F + +- `OutboundRequest` has a total timeout and a deadline, and no connect setting (`edgezero:crates/edgezero-core/src/outbound.rs:49-67`, `:71-90`). +- Fastly reserves a quarter of the budget for connect (`edgezero:crates/edgezero-adapter-fastly/src/outbound.rs:718-733`). A connect timeout is reported with the budget's `BudgetSource`, as if the whole budget were spent (`:77`, `:931-933`). Viceroy 0.21.0 stores no connect timeout for dynamic backends and enforces F's first-byte share, so at F a 1000 ms request whose head arrives at 900 ms fails at about 750 ms with an internal error (`viceroy:src/wiggle_abi/req_impl.rs:617-644`; `viceroy:src/upstream.rs:370-375`). +- Spin sets connect, first byte, and between bytes to the remaining budget (`edgezero:crates/edgezero-adapter-spin/src/outbound.rs:67-74`, `:468-488`). Only a `NotSupported` refusal is ignored with a `debug` log. `Immutable` and `Other` refusals fail the request as an internal error (`:490-505`). A connect timeout before the deadline is reported with an unspecified source (`:1341-1357`). +- Axum bounds the whole request with one timeout (`edgezero:crates/edgezero-adapter-axum/src/outbound.rs:478-484`), and reqwest 0.13.4 offers a connect timeout only per client (reqwest documentation for `RequestBuilder` and `ClientBuilder`, fetched 2026-10-17). Cloudflare races the fetch against the total budget (`edgezero:crates/edgezero-adapter-cloudflare/src/outbound.rs:636-646`). +- The capability matrix marks `outbound-flexible-phase-budget` `BestEffort` on Fastly and Spin (`edgezero:docs/guide/capabilities.md:244`, `:281-282`). The outbound design defers a configurable split "pending a real use case" (`edgezero:docs/superpowers/specs/2026-05-21-outbound-http-design.md:7213-7221`). TS is that use case. + +#### Requirements + +- **EZ-4.1** An application MUST be able to set a connect timeout on an `OutboundRequest`, independently of `OutboundRequest::timeout` and `OutboundRequest::deadline`. The setting MUST survive `OutboundRequest::into_parts` and `from_parts`, and MUST apply on every send mode (cross-cutting requirement 1). A request built with `OutboundRequest::from_request` has none. +- **EZ-4.2** The connect timeout bounds establishing a new connection. Each adapter's documentation MUST say whether DNS resolution and the TLS handshake fall inside it. It MUST NOT extend the total budget. It MUST NOT reduce the time allowed for the response head. It does not apply when the request uses an existing pooled connection. +- **EZ-4.3** Without a connect timeout, no adapter may bound the connect phase more tightly than the total budget. On Fastly the response head MUST get the whole remaining budget. A request that connects in 10 ms and receives its head at 90 percent of its budget MUST succeed. +- **EZ-4.4** With a 1 s connect timeout on Fastly, a request to an unreachable origin MUST fail within 1 s plus 50 ms whatever its total budget. This restores TS's current fast failure. +- **EZ-4.5** A zero connect timeout MUST be rejected before sending with F's standard outbound preflight rejection (400). This deliberately differs from a zero total timeout, which F reports as 504 with `BudgetSource::PerCallTimeout` (`edgezero:crates/edgezero-core/src/time.rs:176-181`), because a zero connect timeout is invalid input rather than an exhausted budget. A connect timeout longer than the total budget MUST behave as if unset. +- **EZ-4.6** When the connect phase times out before the deadline and a connect timeout was set, the request MUST fail with `EdgeError::GatewayTimeout` (504) and a timeout cause the application can tell apart from the per-call timeout, the deadline, and the cutoff. When the deadline has passed, the deadline's `BudgetSource` MUST win. When no connect timeout was set and the platform reports a connect timeout before the deadline, the source MUST be `BudgetSource::Unspecified`. +- **EZ-4.7** In a batch, each slot's connect timeout MUST be independent of the cutoff and of other slots. +- **EZ-4.8** On Fastly the connect timeout MUST be passed to the platform in milliseconds, and a connect failure MUST occur within the timeout plus 50 ms on hosted Fastly (Native). Spin MUST pass it to the host and, for any refusal of the setting, fall back to the total budget without an error (BestEffort). This changes F's handling of `Immutable` and `Other` refusals. Axum and Cloudflare MUST accept the setting without an error, with the connect phase bounded by the total budget (BestEffort). Axum MAY honor it exactly and then report Native. +- **EZ-4.9** EdgeZero's capability matrix MUST gain a row for the connect timeout with the levels of EZ-4.8, and Fastly's `outbound-flexible-phase-budget` MUST become Native. The guides MUST say that connect timeouts should be configuration constants, because on Fastly requests with different connect timeouts do not share connections (EZ-2.1). The CHANGELOG MUST list the removal of Fastly's fixed connect share. +- **EZ-4.10** Removed. +- **EZ-4.11** An application MUST be able to read, from a built request, the connect timeout it carries. + +#### Acceptance criteria + +Unit (any host): + +1. Given a request with a 1 s connect timeout, when it goes through `into_parts` and `from_parts` and through a redirect-hop rebuild, then the connect timeout read back under EZ-4.11 is 1 s. +2. Given a connect timeout of zero, when the request is sent on any adapter, then it fails with 400 before anything is sent. +3. Given each kind of Spin host refusal of the setting, the request completes within its budget and reports no error for the refusal. +4. Each adapter's capability metadata reports the EZ-4.9 levels. + +Axum and workerd: + +5. Given a request with a 1 ms connect timeout to a loopback origin that accepts at once, the request succeeds (BestEffort). + +Spin 4.1: + +6. Given a 200 ms connect timeout to a non-routable address and a 2 s budget, the request fails before the budget with the connect-timeout cause, or, if the host ignores the setting, at the budget with the budget's source. The run records which. + +Viceroy 0.21.0: + +7. Given a loopback origin that accepts at once and sends its head after 900 ms, a request with a 1000 ms timeout and no connect timeout succeeds. (At F it fails at about 750 ms with an internal error.) + +Hosted Fastly, single-request mode. Viceroy 0.21.0 does not enforce connect timeouts, so these criteria need hosted Fastly. Criterion 8 targets an address Trusted Server's configuration cannot express, so it runs on EdgeZero's qualification build deployed by TS. TS produces criterion 9 with its own builds, against the EZ-2 baseline (EZ-Q1.4). The bounds in criterion 8 are measured inside the guest and keep their stated tolerance. + +8. **M8, connect.** A request to the non-routable `192.0.2.1` with a 200 ms connect timeout and a 2 s total budget fails with 504 and the connect-timeout cause within 250 ms of dispatch. Without a connect timeout it fails with the budget's source between 2000 and 2050 ms after dispatch. +9. **Bidder timeouts.** Across the EZ-2 replay, no bidder slot in the candidate fails with a per-call, deadline, or cutoff timeout before its provider timeout has elapsed. The candidate's connect-timeout failures per integration, as a share of that integration's requests, MUST be no more than 0.1 percentage points above the baseline's connection-failure share for the same integration (Open question 3). + +Trusted Server end-to-end check: + +10. A TS unit test over its request builders reads, under EZ-4.11, a 1 s connect timeout on every origin, integration, and bidder `OutboundRequest`. No client double is needed. + +#### Out of scope + +- A DNS timeout setting. +- Separate first-byte or between-bytes settings (EZ-14 covers idle time). +- Exact connect bounds on Axum, Cloudflare, and Spin. + +#### Open questions + +1. **Default.** TS prefers unset, meaning bounded by the total budget as on Axum and Cloudflare, with TS setting 1 s explicitly. A 1 s default would match Fastly's SDK default (`fastly-sdk:src/backend/builder.rs:153`) and TS today, but would apply on Fastly alone. +2. **Axum exactness.** TS prefers BestEffort. Axum is a development target, and honoring per-request connect timeouts there adds connection pools for little gain. +3. **Criterion 9 threshold for TS to confirm.** The proposed bound is 0.1 percentage points above the pre-cut build's connection-failure share per integration. TS prefers it because the cut keeps TS's 1 s connect bound, so a larger rise would point to a regression rather than to origin behavior. + +### EZ-5. Encoded response passthrough + +**Priority:** Blocking +**Adapters:** Fastly, Axum, Cloudflare, Spin. All four must behave the same. Spin may be marked Unsupported only if its host decodes responses itself. +**Related:** EZ-6 (a begun request keeps the choice), EZ-9 (every hop keeps the choice), EZ-15 (a fire-and-forget send keeps it once EZ-15 lands), EZ-14 (an idle bound applies to the stream as delivered, encoded or decoded), EZ-25.1 (owns the forwarding example in the guide) + +#### Need + +Today TS receives upstream bytes still encoded on Fastly and Axum, which are its production and dev targets. The Fastly adapter copies every upstream header (`ts:crates/trusted-server-adapter-fastly/src/platform.rs:410-411`) and never turns on SDK decompression, which is off by default (`fastly-sdk:src/http/request.rs:131`). The Axum adapter builds `reqwest` without its decompression features (`ts:Cargo.toml:99`, `ts:crates/trusted-server-adapter-axum/src/platform.rs:292-299`). TS's Cloudflare and Spin adapters already decode (`ts:crates/trusted-server-adapter-cloudflare/src/platform.rs:395-415`, `ts:crates/trusted-server-adapter-spin/src/platform.rs:338-349`). So the regression lands on Fastly. + +TS relies on the upstream coding in these places: + +1. **Publisher origin.** TS narrows the client `Accept-Encoding` to gzip, deflate, and br (`ts:crates/trusted-server-core/src/publisher.rs:290-326`, called at `:3128` and `:4740`). It then routes on the response `Content-Encoding` (`:1622-1650`, `:6760`). It relays non-rewritten bodies raw (`:5420-5422`, `:5459`). It decodes rewritten bodies with the upstream coding and re-encodes them with the same coding (`:2602-2605`, `:2923-2928`). The one exception is a template-cache response, which TS emits as identity (`:813-820`). +2. **First-party proxy and creative rewrite.** The redirect loop asks for `gzip, deflate, br` (`ts:crates/trusted-server-core/src/proxy.rs:454`, `:1363-1366`). HTML and CSS are re-encoded with the input coding, and `Content-Encoding` is kept (`:653-699`, `:487-515`, `:459-481`). +3. **Asset proxy.** It forwards the client `Accept-Encoding` together with `If-None-Match`, `If-Modified-Since`, `If-Match`, `If-Unmodified-Since`, `Range`, and `If-Range` (`proxy.rs:90-102`, `:1146-1150`). It streams the response with the upstream representation headers intact. It strips only a short first-party denylist (`:1176-1194`, `:1003-1009`) and may then apply its own `Cache-Control` policy (`:1196-1200`). Origins that store objects with `Content-Encoding` metadata, such as S3, return that coding whatever the request asks for. So a range request returns a 206 whose `Content-Range` counts encoded bytes. +4. **Relays that keep the coding.** + - GPT forwards the client `Accept-Encoding` and rebuilds the response from `Content-Encoding`, `ETag`, and `Last-Modified`. It adds `Vary: Accept-Encoding` only when the response is encoded (`ts:crates/trusted-server-core/src/integrations/gpt.rs:159-166`, `:194-253`). + - The JS asset proxy does the same and also forwards `If-None-Match` and `If-Modified-Since` (`ts:crates/trusted-server-core/src/integrations/js_asset_proxy.rs:290-299`, `:365-410`). + - The Prebid bundle keeps the upstream `Content-Encoding` (`ts:crates/trusted-server-core/src/integrations/prebid.rs:1218-1237`). + - Sourcepoint forwards the client `Accept-Encoding` for paths it does not rewrite, and it skips rewriting encoded bodies (`ts:crates/trusted-server-core/src/integrations/sourcepoint.rs:862-875`, `:945-953`). + - Lockr API, Permutive, and DataDome `/js/*` forward the client `Accept-Encoding` and return the upstream response unchanged (`ts:crates/trusted-server-core/src/integrations/lockr.rs:237`, `:264`; `permutive.rs:255`, `:269`; `datadome.rs:766`, `:789`). + - Didomi forwards the client `Accept-Encoding` and returns the upstream body and representation headers, adding CORS or private cache headers (`didomi.rs:278-290`, `:506-516`). + +On PR 275 every one of these sites receives decoded bytes, with `Content-Encoding` and `Content-Length` removed. The regressions on Fastly are these: + +- Reader HTML ships to browsers uncompressed. TS sees no coding, so its encoder emits identity. +- Creative HTML and CSS, and every relayed JS, JSON, SVG, or font body, also ship uncompressed. Their `Content-Length` is gone, so the downstream response becomes chunked. +- A 206 over an encoded representation cannot be decoded. Asset range requests fail with a gateway error or return a corrupt prefix. +- Strong validators no longer describe the bytes served. GPT, the JS asset proxy, and the asset proxy forward the upstream `ETag` next to a decoded body. + +After the cut, TS asks for the original coding on every site that relays bytes or runs its own rewrite codec. Those are the publisher origin, every first-party proxy hop, the asset proxy, GPT, the JS asset proxy, the Prebid bundle, Sourcepoint, Lockr API, Permutive, DataDome `/js/*`, and Didomi. TS's rewrite pipeline then sees the real upstream coding again. TS keeps decoding on for sites that consume decoded text or JSON. These are the auction providers, EC pull sync, the Fastly management API, DataDome protection, the GTM script rewrite (which asks for identity, `ts:crates/trusted-server-core/src/integrations/google_tag_manager.rs:825-830`), the Lockr and Permutive SDK fetches, and DataDome `tags.js`. + +#### Gap at F + +- EdgeZero decodes a single bare `gzip`, `deflate`, or `br` unconditionally, then removes `Content-Encoding` and `Content-Length` (`edgezero:crates/edgezero-core/src/outbound.rs:1252-1297`, removal at `:1280-1286`; classification at `edgezero:crates/edgezero-core/src/compression.rs:84-126`). No request option turns this off (`outbound.rs:49-67`). +- Unknown and stacked codings, and malformed `Content-Encoding` values, already pass through encoded. Both headers are kept and only the decoded cap is skipped (`outbound.rs:1278`, `:1294`, `:1397-1401`; `compression.rs:94-105`; `edgezero:docs/guide/capabilities.md:395-398`, `:415-420`). Passthrough for recognized codings should behave the same way. +- Every adapter already asks its provider for raw bytes. Axum disables `reqwest` decoding (`edgezero:crates/edgezero-adapter-axum/src/outbound.rs:145-154`). Cloudflare asks fetch for manual response encoding (`edgezero:crates/edgezero-adapter-cloudflare/src/outbound.rs:620-634`). Fastly never enables SDK decompression. **Unverified** whether the Spin host ever decodes. A Spin 4.1 run against a gzip origin would verify it. TS's own Spin adapter decodes in the guest, which is consistent with a raw host. +- A 206 is treated as a normal payload (`outbound.rs:1475-1484`), so a 206 with a recognized coding reaches the decoder. +- HEAD and 304 responses keep `Content-Encoding` and the encoded `Content-Length` (`outbound.rs:1472-1474`). In decoding mode, a HEAD therefore describes the encoded representation while the matching GET describes the decoded one. +- EdgeZero adds `Accept-Encoding: identity` only when the request has none, for example at `edgezero:crates/edgezero-adapter-fastly/src/outbound.rs:752-760`. `OutboundRequest::from_request` keeps the browser's value (`outbound.rs:927-934`), so a forwarding app makes EdgeZero decode what the browser negotiated. The guide's forwarding example does exactly that (`edgezero:docs/guide/proxying.md:12-52`). + +#### Requirements + +- **EZ-5.1** An application MUST be able to choose, per request, whether EdgeZero decodes a recognized response content coding or delivers the response with its original coding. Decoding stays the default. In decoding mode, behavior is unchanged from F except EZ-5.8 and, if adopted, EZ-5.9 and EZ-5.10. The CHANGELOG MUST list those changes. +- **EZ-5.2** When the application asks for the original coding, a gzip, deflate, or br response MUST arrive byte for byte as the upstream sent it, with `Content-Encoding` and `Content-Length` unchanged. EdgeZero MUST NOT spend decoder memory on it. +- **EZ-5.3** Identity responses, and responses whose coding EdgeZero does not recognize, MUST behave identically under both choices. +- **EZ-5.4** Caps MUST still apply when the original coding is kept: + - The encoded cap (`max_encoded_response_bytes`) MUST apply to the declared `Content-Length` before the body is read and to the bytes as they stream. + - In buffered mode, the final cap (`max_response_bytes`) MUST apply to the encoded bytes. + - The decoded cap MUST NOT apply to a body EdgeZero does not decode, which matches F's rule for unknown codings. + - A request whose `max_brotli_window_bits` is invalid MUST be rejected the same way under either choice. + - Chunk shaping (`max_chunk_bytes`) MUST apply unchanged. +- **EZ-5.5** Keeping the original coding MUST leave `ETag`, `Last-Modified`, `Vary`, `Content-Range`, `Accept-Ranges`, `Cache-Control`, and digest fields exactly as received. +- **EZ-5.6** The choice is an option under cross-cutting requirement 1, in buffered and streamed mode alike. While EdgeZero holds a begun or batched response that kept its coding, cross-cutting requirement 3 bounds it by the caps as EZ-5.4 states them. +- **EZ-5.7** `Accept-Encoding` handling MUST NOT depend on the choice. EdgeZero adds `identity` only when the request carries no `Accept-Encoding`. A value the application set is forwarded unchanged. +- **EZ-5.8** EdgeZero MUST NOT decode a 206 response under either choice. It MUST deliver a 206 with its original `Content-Encoding`, `Content-Length`, and `Content-Range`. +- **EZ-5.9** In decoding mode, HEAD and 304 responses SHOULD carry the same representation metadata the matching GET would. A recognized coding and its encoded length SHOULD therefore be removed from them. +- **EZ-5.10** In decoding mode, when EdgeZero decodes, it SHOULD weaken a strong `ETag` and remove `Accept-Ranges` and `Content-Digest`, because those describe the encoded representation. RFC 9110 sections 8.8.1 and 8.8.3.3 treat content coding as part of the selected representation. +- **EZ-5.11** An application MUST be able to tell, before sending, which choice a request carries, so its tests can check what each call site asked for. +- **EZ-5.12** Parsing JSON from a response that kept a recognized coding MUST parse the bytes as delivered, without decoding them. A parse failure is F's JSON decode gateway error (`BadGatewayDecodeReason::Json`, `outbound.rs:1688-1693`). Its internal message SHOULD say that the body is content-encoded. +- **EZ-5.13** Where the capability matrix marks an adapter Unsupported for this choice, a request that asks for the original coding MUST be rejected under cross-cutting requirement 2. +- **EZ-5.14** The capability matrix MUST list encoded response passthrough per adapter (cross-cutting requirement 9). The forwarding example that EZ-25.1 owns MUST show a relay that keeps the original coding. + +#### Acceptance criteria + +1. **Unit.** Given gzip, deflate, and br fixtures that declare `Content-Length`, when they are delivered with the original coding and a decoded cap of 1 byte, then the bytes and both headers are unchanged and no cap error occurs. +2. **Unit.** Given the original coding is kept, when the declared length exceeds the encoded cap, then the request fails with the encoded-cap limit before the body is read. When the encoded bytes exceed the final cap in buffered mode, it fails with the final-cap limit (`ResponseLimitReason::BufferedBody` at F). +3. **Unit.** Given a 206 with `Content-Encoding: gzip` and `Content-Range: bytes 10-19/100`, in decoding mode, then the response arrives undecoded with all three headers unchanged. +4. **Axum, Viceroy 0.21.0, workerd, Spin 4.1.** Given an origin that returns a gzip body with `Content-Length`, a strong `ETag`, and `Vary: Accept-Encoding`, when the application asks for the original coding, buffered and streamed, then the bytes and all four headers match the origin. In decoding mode, the bytes are decoded. On Spin 4.1 this holds where the run shows the host passes encoded bodies through. If the run shows the host decodes, the run records that, Spin is marked Unsupported, and the check becomes that a request asking for the original coding is rejected before sending under cross-cutting requirement 2 (EZ-5.13). +5. **Same environments, with the same Spin condition as criterion 4.** Given the request sets no `Accept-Encoding`, the origin sees `identity`. Given the request sets `gzip, br`, the origin sees exactly that, under both choices. Where Spin is Unsupported, the Spin run checks decoding mode only. +6. **Viceroy 0.21.0.** Given a two-slot batch where one slot keeps the original coding, then that slot's body is encoded and the other slot's body is decoded. +7. **workerd.** Given the application relays a passthrough response to its client, then the client receives the exact compressed bytes and `Content-Encoding`, and Workers does not re-encode them. The run records whether the origin `Content-Length` reached the client, and the caveat at `edgezero:docs/guide/capabilities.md:422-426` is updated to match. +8. **Trusted Server, Viceroy 0.21.0.** Run the TS integration suite against origin fixtures. + - A gzip HTML page reaches the client with `Content-Encoding: gzip` and decodes to the rewritten document with the TSJS tag. + - A br SVG relay is byte-identical, with the origin `Content-Length` and `ETag`. + - `Range: bytes=0-99` against a gzip-stored asset returns 206 with the origin `Content-Range` and `Content-Length: 100`. + - A JS asset revalidation with `If-None-Match` returns 304. + - A `/first-party/proxy` HTML creative keeps `Content-Encoding: br`. + +#### Out of scope + +- Re-encoding at response egress. EdgeZero compressing a response from the client's `Accept-Encoding` would not replace this requirement. A relay would still lose `ETag`, `Content-Range`, and `Content-Length` fidelity, and it would pay decode and encode cost on bodies it never reads. +- Deciding to decode after seeing the response headers. TS keeps its own rewrite codec for this cut. +- Any change to request-body encoding. F already sends request bodies with their declared coding. + +#### Open questions + +1. Should forwarding an inbound request keep the original coding by default, since forwarding is a proxy role? Trusted Server's preferred answer is no. One default everywhere is easier to reason about, and the forwarding example can set the choice explicitly. +2. Should EZ-5.9 and EZ-5.10 become MUST? Trusted Server's preferred answer is to keep them SHOULD. TS keeps the original coding on every relay, so it does not depend on them, but other applications that relay in decoding mode would benefit. + +### EZ-6. Begin a request without waiting, and observe a batch without blocking + +**Priority:** Blocking, except EZ-6.31, which is Follow-up and required before TS enables reusable sandboxes in production. +**Adapters:** Axum, Cloudflare, Fastly, Spin. The behavior must hold on all four. Fastly may be BestEffort where EZ-6.21 says so. +**Related:** EZ-1 (a begun GET/HEAD built from an inbound request is accepted the same way), EZ-2 (no Fastly platform timer fires before a request's deadline, and EZ-2.8 defers to EZ-6.16 when a slot's deadline is the cutoff), EZ-3 (EZ-3.3 owns the passed-deadline rule), EZ-4, EZ-5, EZ-7, EZ-11, EZ-12, EZ-14 (their per-request options apply to begun requests and batch slots), EZ-8 (Image Optimizer is Unsupported on begun requests and batches), EZ-9 (begun requests and batches never follow redirects), EZ-14 (lifetime of a streamed origin body), EZ-15 (fire-and-forget is separate from abandoning a begun request), EZ-24 (the test double models arrival-time judgment) + +#### Need + +On every auctioned navigation TS runs the origin fetch and the bidder fan-out at the same time, and it keeps every bid that arrived on time even when it looks at the bid late. + +Today on Fastly the publisher does this: + +1. On the overlapped path it starts the origin fetch, with a streamed response, before EC KV preload and bid construction (`ts:crates/trusted-server-core/src/publisher.rs:4834-4873`). The overlapped path is a navigation GET with EC preload and no template-cache key (`ts:crates/trusted-server-core/src/publisher.rs:4821-4846`, `:3109-3116`). Only the Fastly adapter supports it (`ts:crates/trusted-server-adapter-fastly/src/platform.rs:503-505`). +2. It reads the EC snapshot from KV (`ts:crates/trusted-server-core/src/publisher.rs:4874-4888`). +3. It builds, signs, and sends every bidder request without waiting (`ts:crates/trusted-server-core/src/auction/orchestrator.rs:1586-1834`). The in-flight bids are carried across the origin wait (`:27-48`). Auctions run only for GET navigations (`ts:crates/trusted-server-core/src/publisher.rs:3215-3222`). +4. It waits for the origin response head (`ts:crates/trusted-server-core/src/publisher.rs:5153-5154`). When the overlap does not apply, it sends the origin afterwards, and on an auctioned GET the bids are still in flight during that send (`:5155-5176`). +5. It streams the page and holds the tail at `` (`ts:crates/trusted-server-core/src/publisher.rs:3780`, `:3944`), where it collects the auction (`:4017`, `:4058`, `:4120`). The collection loop drains every in-flight bid (`ts:crates/trusted-server-core/src/auction/orchestrator.rs:2127-2297`). Non-HTML responses collect before their body streams (`ts:crates/trusted-server-core/src/publisher.rs:4195-4239`). A template-cache hit streams the cached template and collects at the seam (`:5108-5125`). +6. Collection keeps bids that arrived while the origin was slow, on purpose. The code says dropping them "would discard SSP responses that already arrived" (`ts:crates/trusted-server-core/src/auction/orchestrator.rs:2290-2296`). +7. After the bidders it makes one mediator call with the smaller of the remaining auction time and the mediator timeout, and skips mediation when no time remains (`ts:crates/trusted-server-core/src/auction/orchestrator.rs:2349-2364`). +8. After the response is delivered, inside the same Fastly serving callback, it runs EC pull sync, which sends partner requests concurrently and waits for them (`ts:crates/trusted-server-core/src/ec/pull_sync.rs:225-300`, called from `ts:crates/trusted-server-adapter-fastly/src/main.rs:618-643`). + +The numbers come from the example config. The navigation auction has 500 ms (`ts:trusted-server.example.toml:340`), and the `/auction` endpoint 2000 ms (`:282`). The commented-out mock mediator example uses 1000 ms (`:674`). The origin has 15 s first-byte and between-bytes timers (`ts:crates/trusted-server-core/src/publisher.rs:98`, `:4517-4518`). + +On PR 275's APIs, TS will begin the origin request for the overlapped navigation GET only. Every other origin forward, including POST, PUT, PATCH, DELETE, and OPTIONS, uses `HttpClient::send` with the inbound body streamed upstream, and those methods never run an auction. TS sends the bidders as one `start_batch_until` batch with the auction deadline as the cutoff, keeps working (KV, origin wait, page streaming), checks the batch at body-chunk boundaries, collects it at ``, and then makes the mediator call with one `HttpClient::send`. Pull sync after delivery becomes a batch or plain sends inside the serving callback. + +Without this requirement TS has two bad choices. It can serialize the origin behind the auction, which adds the auction time to TTFB, or it can lose every bid whenever the origin head or the `` chunk arrives after the cutoff. On Axum, Cloudflare, and Spin it would also send no bid at all unless it polled the batch continuously. + +#### Gap at F + +- There is no way to begin one request without waiting. On Fastly `send` waits for the response head (`edgezero:crates/edgezero-adapter-fastly/src/outbound.rs:450-453`, `:1279-1290`). Batches refuse streamed responses (`edgezero:crates/edgezero-adapter-fastly/src/outbound.rs:2580-2584`), and the origin must stream. +- On Axum, Cloudflare, and Spin a batch sends nothing until it is polled. Each slot is an unpolled future (`edgezero:crates/edgezero-adapter-axum/src/outbound.rs:200-206`; `edgezero:crates/edgezero-adapter-cloudflare/src/outbound.rs:389-395`; `edgezero:crates/edgezero-adapter-spin/src/outbound.rs:434-440`), although the trait promises that every eligible request is attempted first (`edgezero:crates/edgezero-core/src/outbound.rs:428`). +- Completion is judged when the application looks, not when the response arrived. On Fastly, once the cutoff has passed the driver does not examine pending slots and reports every one as unresolved (`edgezero:crates/edgezero-adapter-fastly/src/outbound.rs:621`, `:649-651`). A slot examined after its own budget becomes a timeout even if the host received it in time (`:1002`). On the other adapters the completion instant is taken when the slot's future is polled to completion (`edgezero:crates/edgezero-adapter-axum/src/outbound.rs:203`; `edgezero:crates/edgezero-adapter-cloudflare/src/outbound.rs:392`; `edgezero:crates/edgezero-adapter-spin/src/outbound.rs:437`). +- There is no non-blocking way to observe a batch. `OutboundBatch::next`, `collect`, and `HttpClient::send_all_until` all wait (`edgezero:crates/edgezero-core/src/outbound.rs:229-252`, `:343-399`, `:588-596`). Repeated termination exists in code and tests (`:349-351`, `:3035-3040`). +- On Fastly a buffered slot's body is read to the end before any other slot can be observed (`edgezero:crates/edgezero-adapter-fastly/src/outbound.rs:838-875`, `:1069-1072`; `edgezero:docs/guide/capabilities.md:276-279`). +- The guide says elapsed time includes "any delayed guest observation" (`edgezero:docs/guide/capabilities.md:442-444`), and EdgeZero's design spec records that a provider-ready response can stay unresolved (`edgezero:docs/superpowers/specs/2026-05-21-outbound-http-design.md:1821-1826`). + +#### Requirements + +Beginning one request. + +- **EZ-6.1** An application MUST be able to begin one outbound request, keep working, and collect its response later. A request is on its way once EdgeZero has handed it to the host, the network stack, or the adapter's runtime, so that it proceeds with no further EdgeZero call from the application. When begin returns, the request MUST be on its way. Beginning MUST NOT wait for connection establishment or response headers. This MUST hold on all four adapters. +- **EZ-6.2** A begun request MUST support streamed response mode, where collection returns at the response head with a lazily read body, and buffered response mode, where collection returns the whole body under the request's response caps. +- **EZ-6.3** A begun request MUST accept a buffered request body and a bodiless GET/HEAD built from an inbound request (EZ-1). Any other streamed upload MUST be rejected before anything is sent, as an invalid request (400 class), the category F uses for a streamed upload in a batch. `HttpClient::send` MUST keep accepting streamed uploads. +- **EZ-6.4** For the same request and the same origin behavior, a begun request MUST produce the same status, headers, body, error category, and timeout cause as `HttpClient::send`. Options and client defaults apply to a begun request as cross-cutting requirements 1 and 2 say, including options accepted at BestEffort. +- **EZ-6.5** A begun request's time budget MUST be computed from the moment begin is called, with the same rules as `send`. Time spent settling a bodiless GET/HEAD (EZ-1) counts against it. +- **EZ-6.6** The application MUST be able to ask whether a begun request's response is ready without blocking. + +Batches. + +- **EZ-6.7** When `start_batch_until` returns, every eligible slot MUST be on its way (EZ-6.1), on all four adapters. A batch that is never polled MUST still send every eligible request. +- **EZ-6.8** If an adapter configuration cannot put a request on its way without being polled, beginning the request or the batch slot MUST fail with an internal error (500 class) before anything is sent. It MUST NOT silently defer sending. +- **EZ-6.9** The application MUST be able to check a batch without blocking. A check returns a newly finished slot, the batch termination, or "nothing new". It MUST work inside a handler and inside a streamed response body that EdgeZero is writing to the client, on all four adapters. It MUST follow the same rules as `OutboundBatch::next`, including at most one result per slot, a clean end before `Completed`, and repeated termination. +- **EZ-6.10** `OutboundBatch::next` MUST stay safe to cancel. A `next` that is waiting MUST return no later than the cutoff plus 5 ms, even when no slot finishes. + +Arrival-time judgment. + +- **EZ-6.11** Arrival is as cross-cutting requirement 5 defines it. While the application waits on any EdgeZero operation (a begun request's response head, a read from a response body EdgeZero produced, a single send, a batch wait, or a non-blocking check), EdgeZero MUST record the arrival of every other in-flight request in the same request lifetime (cross-cutting requirement 4). Axum, Cloudflare, and Spin MUST also record arrivals whenever the application awaits anything else. While the application does CPU work without awaiting, an adapter records an arrival no later than the application's next await, and the guide MUST say so. A buffered slot whose body has not ended by the cutoff is unresolved at the cutoff, and EdgeZero MUST NOT wait past the cutoff to finish reading it. +- **EZ-6.12** A slot is on time if and only if its recorded arrival is strictly before the observation cutoff. An arrival at the cutoff is late. +- **EZ-6.13** A slot's or begun request's own budget MUST be judged at its recorded arrival. A response that finished within its budget MUST be delivered as received, even when the application collects it after the budget and after the cutoff. Collection itself MUST NOT turn such a response into a timeout. +- **EZ-6.14** When EdgeZero first observes a slot at or after the slot's budget deadline, whether or not a response is waiting, it MUST treat the slot as a timeout at the budget deadline, because an earlier arrival cannot be proven. EZ-6.16 decides the outcome when that deadline is the cutoff. +- **EZ-6.15** Finished slots MUST be delivered in order of recorded arrival, with ties in input order. A slot's elapsed time MUST be its arrival minus the batch start. +- **EZ-6.16** After the cutoff, the batch MUST first deliver every slot that arrived on time, then report `Cutoff` if any slot is unresolved and `Completed` otherwise. Nothing first observed at or after the cutoff may be delivered, except a timeout whose budget deadline is before the cutoff. A slot whose budget deadline equals the cutoff has one outcome. It is not delivered, and the batch reports `Cutoff` with that slot unresolved. EZ-2.8's per-slot timeout item applies only to a slot whose deadline is before the cutoff. + +Bounded background work. + +- **EZ-6.17** Responses that arrive while the application is busy elsewhere follow cross-cutting requirement 3. A cap failure MUST become that slot's outcome at the moment the cap was exceeded. +- **EZ-6.18** Receiving one slot's body MUST NOT stall the application. While a bidder's body trickles in, a streamed origin body that the application is reading MUST keep flowing, and other slots MUST still be recorded at arrival. The only exception is the Fastly fallback in EZ-6.21. + +Cancellation and request lifetime. + +- **EZ-6.19** Dropping or cancelling a begun request or a batch releases it as cross-cutting requirement 6 says. Cancelling a batch MUST report the slots it never delivered. Results recorded but not yet delivered are discarded by cancellation, so an application that wants them checks the batch first. +- **EZ-6.20** Outbound work the application starts inside its per-request lifecycle callback after delivery MUST behave as it does during dispatch, including deadlines, batches, non-blocking checks, and arrival recording. This holds with the per-request client, a client the application keeps for the sandbox, or a newly built client. Every begun request and batch slot still in flight when the callback returns MUST be released. The one exception is a fire-and-forget send, whose lifetime EZ-15 defines (cross-cutting requirement 4). +- **EZ-6.21** Per-adapter levels. Axum, Cloudflare, and Spin MUST be Native for beginning a request, checking a batch without blocking, and arrival-time recording. Fastly is BestEffort only in these respects. A cold backend registration may block begin. Arrivals are recorded only during EdgeZero operations. If the host does not report response-body readiness, Fastly MAY delay the application and other arrivals by up to one slot body's transfer time, and the capability note MUST state that bound. Every other requirement in EZ-6 holds on Fastly as written. + +Interaction with response writing. + +- **EZ-6.22** A begun request and a batch MUST be usable from inside a streamed response body for as long as that body lives. +- **EZ-6.23** When EdgeZero stops writing a response (client disconnect, write deadline, body error, or request cancellation), every begun request and batch owned by that response body MUST be released per EZ-6.19. +- **EZ-6.24** On Fastly, a batch wait inside a response body that EdgeZero is writing SHOULD return by the response write deadline, so a cutoff set past that deadline cannot hold the response. The response then ends with the existing write-deadline outcome. +- **EZ-6.25** A streamed response from a begun request MUST keep enforcing its own deadline on each body read. The guide MUST state that a streamed page lives until the earlier of the origin request's deadline (from begin) and the response write deadline (30 s from the start of writing by default, `edgezero:crates/edgezero-core/src/response_egress.rs:14`), and that a hold inside the body counts against both. + +Sequential calls and invariants. + +- **EZ-6.26** `HttpClient::send` and begin MUST work while a batch from the same request lifetime is in flight and after it finishes, including inside a streamed response body. A request whose deadline has already passed follows EZ-3.3 on Fastly and fails the same way, with a gateway timeout naming the request deadline and nothing sent, on the other adapters, as F already does (`edgezero:crates/edgezero-core/src/time.rs:176-182`). +- **EZ-6.27** Batches MUST keep refusing streamed uploads and streamed response mode. Begun requests and batches MUST NOT follow redirects; a 3xx is returned as the final response. +- **EZ-6.28** Image Optimizer is Unsupported on begun requests and batch slots. Those send modes MUST reject it as cross-cutting requirement 2 says, before any other outbound preflight check. +- **EZ-6.29** Removed. +- **EZ-6.30** EdgeZero's capability matrix MUST gain rows for beginning a request, checking a batch without blocking, and arrival-time recording, at the levels in EZ-6.21. The guide text that says elapsed time includes delayed observation (`edgezero:docs/guide/capabilities.md:442-444`) and that ready handles can stay unobserved (`:276-279`) MUST be corrected. +- **EZ-6.31** Follow-up, required before TS enables reusable sandboxes. On a reused Fastly sandbox, abandoned requests MUST NOT accumulate. A request still in flight when its callback returned MUST NOT be observed by a later request in the same sandbox, the next request MUST still send, and abandoned work MUST be gone once its backend answers or its platform timer fires. + +#### Acceptance criteria + +Unless a criterion says otherwise, the scenarios use three bidders A, B, and C, a 500 ms auction, bidder requests with a 1,000 ms per-call timeout, and an origin request with a 15 s timeout. The batch starts at 34 with cutoff 534, so each bidder's budget is 500 ms, set by the cutoff. Times are milliseconds from the inbound request. Elapsed values hold within 5 ms on Axum, workerd, Spin 4.1, and Viceroy 0.21.0, and exactly in unit runs with an injected `MonotonicClock`. On hosted Fastly the origins apply the stated delays after receiving each request. A time measured inside the guest from a deadline or the cutoff (scenario 2's gateway timeout at 534, scenario 7's `Cutoff` by 539) keeps its 5 ms tolerance. Otherwise the pass condition is the set and order of delivered slots and the `Cutoff` or `Completed` outcome. Each elapsed value MUST be no less than the stated value minus 5 ms, and no more than the stated value plus that slot's measured connection setup and round trip plus 5 ms. The run reports both measurements for every slot. Origins are loopback or staging servers that behave as each scenario states. + +1. Slow origin. Axum, workerd, Spin 4.1, Viceroy 0.21.0, hosted Fastly. Given the origin is begun at 2 and its head arrives at 700, the page body arrives in chunks at 700, 900, and 1,300, the batch starts at 34 with cutoff 534, A answers at 180, B at 320, and C at 900, when the application waits for the origin head and checks the batch at 701, then it gets A with elapsed 146, B with elapsed 286, and `Cutoff` with C unresolved. At F, the same flow loses A and B on Fastly and sends no bidder request on the other adapters. +2. Fast origin with mediator. Axum, workerd, Spin 4.1, Viceroy 0.21.0, hosted Fastly. Given the origin head at 120, a body chunk at 260, and A, B, C answering at 180, 320, and 400, when the application waits on the batch at 260, then it gets A, then B, then C and `Completed` at 400. A `send` at 400 built with `OutboundRequest::deadline` set to the cutoff and `OutboundRequest::timeout` set to 1,000 ms gets a 134 ms budget with the request deadline as its cause. If that origin stays silent, the send fails with a gateway timeout at 534. +3. CPU stretch. Axum, workerd, Spin 4.1, Viceroy 0.21.0. Given the application does CPU work from 520 to 560 without awaiting and B answers at 530, then B is either delivered with elapsed 496, where the adapter recorded its arrival during the CPU work, or not delivered, with the batch reporting `Cutoff` and B unresolved. B is never delivered with an elapsed time of 500 or more. The run records which. +4. Begin puts the request on its way. Every adapter. Given begin returns and the application then awaits an unrelated operation (a timer or a KV read) for 100 ms without any other EdgeZero outbound call, then the origin receives the request within that window. +5. Unpolled batch. Every adapter. Given a batch that the application does not check for 300 ms while it awaits an unrelated EdgeZero operation, then every bidder receives its request within that window. +6. Arrival during origin body reads. Viceroy 0.21.0 and hosted Fastly. Given the origin sends body chunks at 100 and 800 and B answers at 320, when the application only reads the origin body, then B is delivered with elapsed 286. +7. Trickling slot. Viceroy 0.21.0 and hosted Fastly. Given a bidder sends its head at 100 and its body at 600 while the origin streams a chunk every 50 ms, then the origin chunks keep reaching the application between 100 and 600, or, where the capability note records the body-readiness fallback, they stall for at most the bidder body's transfer time. The run records which. With a 534 cutoff, that bidder is unresolved and the batch reports `Cutoff` by 539. +8. Send during a live batch. Viceroy 0.21.0 and hosted Fastly. Same as scenario 1, but the origin is forwarded with `send` after the batch starts. A and B are delivered. +9. Caps while waiting. Every adapter. Given a bidder slot with a 1 MiB decoded cap and a 64 KiB buffered cap, whose gzip body decodes to 128 KiB and trickles in over 300 ms, then the slot fails with the buffered-cap error before the body ends. +10. Abandonment on a reused sandbox (EZ-6.31). Viceroy 0.21.0. Given a reused guest serving 6 requests, each beginning a request to an origin that answers after 2 s and dropping it, then request 6 still sends, no request observes another's arrivals, and no abandoned work remains 2 s after the last request. Hosted Fastly. With reuse limits raised so one sandbox can serve 1,000 requests at about 50 per second, the same holds for request 1,001. The run records how many requests each sandbox served. +11. Write deadline. Axum, workerd, Spin 4.1. Given a response with a 1 s write deadline whose body waits on a batch with a 3 s cutoff, then the response ends with the write-deadline outcome, and the batch's in-flight requests are released. On Viceroy 0.21.0 and hosted Fastly the run records the outcome, because EZ-6.24 is a SHOULD there. +12. Parity. Every adapter. For buffered uploads, a begun request and `send` produce the same outcomes on EdgeZero's existing outbound contract cases, including client defaults. +13. Upload rules. Unit and every adapter. Begin rejects a POST with a streamed body as an invalid request before sending. Begin accepts a bodiless GET built from an inbound request. `send` forwards a POST with a streamed body. +14. Timeout before the cutoff. Every adapter. Given a fourth bidder D in the scenario 1 batch with a 300 ms per-call timeout that never answers, then the batch delivers D as a gateway timeout naming the per-call timeout, with elapsed 300, before it reports `Cutoff`. +15. Arrival at the cutoff. Unit, injected clock. Given a slot whose response is complete exactly at the cutoff, then it is unresolved. +16. Waiting `next` ends at the cutoff. Every adapter. Given a batch with a 200 ms cutoff and no slot answering, then a waiting `next` returns `Cutoff` within 205 ms. +17. Cancellation report. Unit and every adapter. Given a batch with one slot recorded but not delivered and one still in flight, when it is cancelled, then both indices are reported as undelivered. +18. Check inside a streamed body. Every adapter. Given a streamed response body that checks the batch between chunks while two slots are still in flight, then each check returns "nothing new" in under 5 ms. +19. Work after delivery. Viceroy 0.21.0. Given a serving callback that, after delivery, starts a three-slot batch with a client it keeps and collects it, then all three results are delivered with arrival-based elapsed values, and nothing from that batch is still in flight after the callback returns. + +The Trusted Server end-to-end check runs in TS's integration suite on Viceroy 0.21.0, the Axum binary, and workerd, and in the Spin smoke (`ts:scripts/smoke-spin.sh`) on Spin 4.1. A navigation whose origin head arrives at 700 ms with a 500 ms auction contains the bids from bidders that answered at 180 and 320 ms and not one that answered at 600 ms. A page whose `` arrives at 1,300 ms gives the same result, and TS ranks locally because no mediator time remains. TTFB tracks origin latency plus less than 20 ms. A POST navigation forwards its body through `send` and runs no auction. On Fastly, pull sync after delivery completes inside the serving callback. + +#### Out of scope + +- Streamed responses in batches. One begun request per streamed exchange is enough. +- Streamed uploads on a begun request. TS forwards those with `send`. +- Redirect following on begun requests and batches (EZ-9). +- Fire-and-forget sends (EZ-15). +- An idle policy for long streamed bodies (EZ-14). + +#### Open questions + +1. When EdgeZero first observes a slot late, should it credit the arrival to the previous time it looked? TS prefers no. Crediting the earlier time would admit responses that arrived after the cutoff, and TS treats a delivered result as on time. +2. Should a Fastly batch wait inside a response body be bounded by the write deadline as a MUST rather than the SHOULD in EZ-6.24? TS prefers SHOULD. TS's cutoff (500 ms) is far inside the 30 s write budget, so TS does not depend on it. +3. Does hosted Fastly report when a response body has data, so that a body read can be bounded and a trickling slot need not stall the application? TS prefers that EdgeZero establish the answer on hosted Fastly before the cut and record it, because EZ-2.6, EZ-6.18, and EZ-6.21's fallback all depend on it. + +### EZ-7. Keep the `x-edgezero-proxy` header off relayed responses + +**Priority:** Blocking +**Adapters:** Fastly, Axum, Cloudflare, Spin. Identical behavior on all four. +**Related:** EZ-6 and EZ-9 (begun requests and the final response of a followed chain honor the setting), EZ-15 (a fire-and-forget send honors it once EZ-15 lands), EZ-5 (same relay paths), EZ-24 (its test double applies the same default and opt-out), EZ-25.1 (owns the forwarding example) + +#### Need + +TS has no global response-header filter. Its Fastly adapter copies every header of a TS response to the client (`ts:crates/trusted-server-adapter-fastly/src/compat.rs:13-19`, `:49-55`). These paths return upstream headers wholesale, or minus a small denylist: + +- The publisher fallback, on every route (`ts:crates/trusted-server-core/src/publisher.rs:5337-5370`, `:5420-5422`, `:5459`). +- The asset proxy, minus set-cookie, HSTS, and clear-site-data (`ts:crates/trusted-server-core/src/proxy.rs:1194`, `:1003-1009`, `:110-111`). +- `/first-party/proxy`, minus the CORS headers (`proxy.rs:747-761`, `:727-731`). +- GTM beacons and non-2xx GTM script responses, minus the first-party denylist (`ts:crates/trusted-server-core/src/integrations/google_tag_manager.rs:968-987`, strip at `:976`). +- Testlight (`ts:crates/trusted-server-core/src/integrations/testlight.rs:115-130`). +- Sourcepoint, after its own cache and `Location` adjustments (`ts:crates/trusted-server-core/src/integrations/sourcepoint.rs:934`, `:995`, `:1027`, `:1042`, `:1046`). +- Lockr API (`lockr.rs:237`), Permutive (`permutive.rs:255`), DataDome non-200 `tags.js` and `/js/*` (`datadome.rs:668-674`, `:789`), and Didomi (`didomi.rs:506-516`), which return the upstream response with its headers. + +GPT, the JS asset proxy, the Prebid bundle, and the SDK fetches rebuild their responses from allowlists, so they would not leak (`gpt.rs:194-253`, `js_asset_proxy.rs:365-443`, `prebid.rs:1218-1253`). + +On PR 275, every response on those paths gains `x-edgezero-proxy: fastly`. Every proxied page and asset then tells browsers which edge runtime served it, and any downstream cache that stores origin headers keeps the field too. Under the hard-cut decision TS does not strip headers itself, so EdgeZero must not add the field. Two secondary effects follow: + +- The marker consumes response header budget. It costs 20 to 26 bytes depending on the adapter, plus one field (`edgezero:crates/edgezero-core/src/outbound.rs:3361-3403`). So a header-count cap of N admits only N-1 upstream fields. +- Removing it after the fact would mark the response's headers as modified (`outbound.rs:631-639`, `:653-656`). TS does not read that provenance today. + +#### Gap at F + +- Every adapter adds `x-edgezero-proxy: ` to every outbound response after header normalization, bodyless responses included. It overwrites any upstream field of that name and is charged to the header caps (`outbound.rs:1407-1422`; Fastly `edgezero:crates/edgezero-adapter-fastly/src/outbound.rs:1016-1020`; Axum `edgezero:crates/edgezero-adapter-axum/src/outbound.rs:335-339`; Cloudflare `edgezero:crates/edgezero-adapter-cloudflare/src/outbound.rs:526-530`; Spin `edgezero:crates/edgezero-adapter-spin/src/outbound.rs:660-664`). +- `into_response` carries the field into the application's response (`outbound.rs:723-742`). +- No option on `OutboundRequest` or `HttpClient` turns it off. +- Only tests read the field. Those are the core unit test (`outbound.rs:3375`), an Axum contract test (`edgezero:crates/edgezero-adapter-axum/tests/contract.rs:56-79`), and the docs contract check (`edgezero:scripts/check_outbound_docs_contract.mjs:219`, `:224`). EdgeZero's design says the marker is not a loop-prevention mechanism (`edgezero:docs/superpowers/specs/2026-05-21-outbound-http-design.md:43-45`). +- The marker predates PR 275. TS never received it, because TS did not use EdgeZero's proxy client. + +#### Requirements + +- **EZ-7.1** An application MUST be able to receive outbound responses that carry no EdgeZero-added `x-edgezero-proxy` field. The setting is an option under cross-cutting requirement 1. For a followed redirect chain, it applies to the final response. +- **EZ-7.2** The application MUST be able to set this once for every request it sends through one client handle, including requests it begins, batches, or follows and, once EZ-15 lands, fire-and-forget sends. A per-request setting MUST override the client-wide setting. +- **EZ-7.3** When the marker is off, it MUST NOT be charged to `max_response_header_bytes` or `max_response_header_count`. Those caps then count only upstream fields. +- **EZ-7.4** When the marker is off, EdgeZero MUST neither add nor remove an `x-edgezero-proxy` field. An upstream field of that name is an ordinary upstream header. +- **EZ-7.5** When the marker is on, its value, overwrite behavior, and cap accounting MUST stay as at F. +- **EZ-7.6** An application SHOULD still be able to learn which adapter completed an exchange without the header. +- **EZ-7.7** An application MUST be able to tell, before sending, whether a request will carry the marker, after any client-wide setting is applied. Its tests can then check each call site. +- **EZ-7.8** The guide text that describes the marker (`edgezero:docs/guide/adapters/overview.md:78`, `edgezero:docs/guide/capabilities.md:389-390`) MUST describe how to keep it off. The forwarding example that EZ-25.1 owns MUST show it off for responses that reach browsers. + +#### Acceptance criteria + +1. **Unit.** Given an upstream response with N fields and a header-count cap of exactly N, when the marker is off, then the response succeeds. When it is on, it fails with the header-count limit. The same holds for header bytes. +2. **Unit.** Given a client handle set to omit the marker, when the application sends one request with no override, one with an explicit "on", one begun request, and one two-slot batch, then only the explicit "on" request carries the marker. +3. **Unit.** Given the marker is off and the upstream sends `x-edgezero-proxy: upstream`, then the field arrives unchanged. +4. **Axum, Viceroy 0.21.0, workerd, Spin 4.1.** Given 200, 204, HEAD 200, 304, and 302 responses from an origin, when the marker is off, then none carries the field. Under F's behavior, all of them carry it. +5. **Trusted Server, Viceroy 0.21.0.** Browser-facing responses from the publisher fallback, asset proxy, `/first-party/proxy`, GTM collect, Testlight, Lockr API, Permutive, DataDome `/js/*`, Sourcepoint, and Didomi carry no `x-edgezero-proxy`. TS's request client asks for the marker to be off once, and no TS call site sets it per request. + +#### Out of scope + +- Any loop-prevention or `Via` behavior. +- Filtering other upstream headers. TS's own allowlists and denylists stay TS business logic. + +#### Open questions + +1. Should the marker be off by default? Trusted Server's preferred answer is yes. PR 275 is already a hard migration with no compatibility aliases (`CHANGELOG.md:7-10`). No EdgeZero code outside tests consumes the field. Any forwarding application leaks it to browsers, and it silently consumes header caps. If the default stays on, EZ-7.2 still gives TS one place to turn it off. + +### EZ-8. Fastly Image Optimizer on outbound requests + +**Priority:** Blocking +**Adapters:** Fastly single sends must honor the options. Axum, Cloudflare and Spin are Unsupported, and so are Fastly begun requests, batches and fire-and-forget sends. +**Related:** EZ-1 (a bodiless GET/HEAD from an inbound request may carry the options, and the rejection comes before body settling), EZ-3 (EZ-3.3 owns the deadline-passed-before-sending rule), EZ-6 and EZ-15 (begun requests and, once EZ-15 lands, fire-and-forget sends reject the options), EZ-11 (EZ-8.6 rejects the options combined with cache tagging), EZ-2 (the options must not reduce connection reuse, and EZ-2's eligibility groups bound the uninterruptible window), EZ-5 (asset routes forward the client's `Accept-Encoding`, so unoptimized assets need encoded passthrough), EZ-9 (the options carry to redirect hops), EZ-16 (provider causes from this path reach logs), EZ-Q1 (owns the hosted Fastly criterion) + +#### Need + +Trusted Server serves publisher images through asset routes and asks Fastly Image Optimizer to transform them. `handle_asset_proxy_request` (`ts:crates/trusted-server-core/src/proxy.rs:1096-1204`) does this in order. + +1. It builds the options from the route's profile table unless the incoming or rewritten path is SVG (`proxy.rs:1105-1117`, `ts:crates/trusted-server-core/src/asset_image_optimizer.rs:43-97`). The closed set covers region, a preserve-origin-query flag, format, quality, resize filter, width, height and crop (`ts:crates/trusted-server-core/src/platform/image_optimizer.rs:12-181`). +2. It strips the origin query by default (`proxy.rs:1118-1120`, `ts:crates/trusted-server-core/src/settings.rs:1738-1758`). Config validation rejects preserving the query on Image Optimizer routes (`settings.rs:1709-1719`) because Image Optimizer reads client query parameters as transformation inputs (`settings.rs:965-967`). +3. It sets `Host` to the URI authority (`proxy.rs:1151`, `:893-917`). S3 SigV4 signs `host`, `x-amz-content-sha256`, `x-amz-date` and an optional `x-amz-security-token` (`ts:crates/trusted-server-core/src/s3_sigv4.rs:62-149`, header set `:128-133`). +4. Only when the route has options, S3 auth, and the request is GET or HEAD (`proxy.rs:1015-1023`), it first sends a signed HEAD without Image Optimizer (`:1025-1078`), because Image Optimizer can mask S3 errors (`:1037-1039`). A failed HEAD answers a HEAD client directly, or triggers a signed streamed GET without Image Optimizer that returns the raw S3 error (`:1056-1077`). +5. It then signs the final request (`proxy.rs:1167-1169`), attaches Image Optimizer (`:1173-1175`), asks for a streamed response (`:1176-1179`) and sends (`:1181-1187`). + +The Trusted Server Fastly adapter translates the options to the SDK (`ts:crates/trusted-server-adapter-fastly/src/platform.rs:190-305`), sends with the blocking `fastly::Request::send` (`:521-523`), and refuses Image Optimizer on its async path (`:527-536`). + +After the cut Trusted Server deletes that adapter layer. The asset route builds an `OutboundRequest` with `stream_response()`, a `timeout`, the SigV4 headers set through `header(..)`, and the Image Optimizer options, then calls `HttpClient::send`. The S3 HEAD preflight and the raw-error GET stay Trusted Server logic and are plain signed sends without options. Without EZ-8 the options cannot be attached, so every optimized route serves the untransformed original at full size and in its original format. A local operator configuration (not named here) enables an Image Optimizer route with S3 SigV4 today, so this blocks the cut. + +#### Gap at F + +`OutboundRequest` carries a fixed portable option set and no way to attach provider options (`edgezero:crates/edgezero-core/src/outbound.rs:48-67`, parts `:69-89`). The Fastly SDK constrains what EdgeZero can offer. + +- Image Optimizer works only on the blocking `Request::send` (`fastly-sdk:src/http/request.rs:606-635`, `send_with_image_opto` `:668-792`). `send_async` refuses it (`:861-865`). The streaming send never looks at it (`:918-947`). Converting a request to handles drops it (`:148-152`, `:2853-2865`). +- Cache pass silently disables it (`:641-660`), and it silently overrides surrogate keys (`:3134-3141`). +- It sends no request body (`:696`, `:761-763`). + +At F the Fastly single send uses `send_async` (`edgezero:crates/edgezero-adapter-fastly/src/outbound.rs:443-454`), and batch slots use handles (`:519-529`). The error mapper folds `ImageOptimizerUnsupported` into a generic 502 with the message "Fastly outbound request failed" (`:960-965`, `:91-94`). Viceroy cannot run Image Optimizer and answers its host call with `Unsupported` (`viceroy:src/component/image_optimizer.rs:7-15`, `viceroy:src/error.rs:248`), which the SDK reports as `ImageOptimizerUnsupported` (`fastly-sdk:src/http/request.rs:714-719`). The SDK's own `send_async` produces the same cause without reaching the host (`:861-865`). + +#### Requirements + +- **EZ-8.1** An application MUST be able to attach Fastly Image Optimizer options to an individual outbound request from portable code, without depending on the Fastly SDK and without compiling Fastly-only code for other targets. The options MUST stay attached wherever EdgeZero rebuilds or forwards the request, including redirect hops (EZ-9). +- **EZ-8.2** On every adapter and send mode that EZ-8.14 marks Unsupported, a request carrying the options MUST get cross-cutting requirement 2's rejection. The rejection MUST come before anything is sent and before any other outbound preflight check, including EZ-1's settling of an inbound body. It MUST name the option and the adapter. The options MUST NOT be dropped silently. +- **EZ-8.3** The options MUST express at least what Trusted Server uses. That is the seven SDK regions, the preserve-origin-query flag, output format (auto, AVIF, GIF, JPEG, JPEG XL, MP4, PNG, WebP), quality in 0 to 100, resize filter (nearest, bilinear, bicubic, Lanczos 2, Lanczos 3), width and height in pixels, and a crop to an aspect ratio with a default, smart, or explicit x and y percentage focus. Invalid values MUST be rejected when the application builds the options, as a client-class error. Invalid means quality above 100, a zero dimension, an offset above 100, smart focus combined with offsets, or only one offset. Trusted Server enforces all five rules today, at config load (`ts:crates/trusted-server-core/src/settings.rs:1429`, `:1485-1509`) and at runtime (`ts:crates/trusted-server-core/src/asset_image_optimizer.rs:229`, `:275-300`, `:316-343`). +- **EZ-8.4** On Fastly, a single send carrying the options MUST be processed by Image Optimizer, so the response body is the transformed image. +- **EZ-8.5** Only GET and HEAD may carry the options. A request with any other method, or with a streamed upload that would be uploaded, MUST get cross-cutting requirement 2's rejection before backend registration, because Fastly cannot honor the options there. A GET or HEAD whose body EZ-1 settles as bodiless is accepted. One that carries body bytes is rejected as EZ-1 requires. +- **EZ-8.6** A request carrying the options that also asks for cache bypass or cache tagging (EZ-11) MUST get cross-cutting requirement 2's rejection before sending. Fastly silently disables Image Optimizer under bypass and ignores tags under Image Optimizer. Only the default cache policy is valid with the options. This requirement owns that rule. +- **EZ-8.7** In a batch, a slot carrying the options MUST fail as that slot's own result with the EZ-8.2 rejection, and its siblings MUST proceed. F already isolates streamed slots this way (`edgezero:crates/edgezero-adapter-fastly/src/outbound.rs:2573-2586`, `edgezero:crates/edgezero-adapter-fastly/tests/contract.rs:1262-1279`). +- **EZ-8.8** Begun requests (EZ-6) and, once EZ-15 lands, fire-and-forget sends MUST give a request carrying the options the EZ-8.2 rejection on every adapter. This requirement owns that rule. +- **EZ-8.9** An optimized response MUST get the same response handling as any other Fastly response. That covers response header caps, normalization, the bodyless rules for HEAD, 204 and 304, the proxy marker (EZ-7), the decoding policy (EZ-5), and buffered or streamed mode. A streamed optimized body MUST stay lazy, so the first bytes reach the application before the transform output ends. +- **EZ-8.10** A deadline that has passed before sending follows EZ-3.3. A response that completes after the deadline MUST be a gateway timeout with the budget's cause. EdgeZero may be unable to interrupt Image Optimizer while it works. An optimized request against a stalled origin MUST still fail no later than a documented bound of at most twice its budget, consistent with how EZ-2.2 bounds backend configurations. The capability footnote MUST record that bound and the hosted measurement. +- **EZ-8.11** A service without Image Optimizer, or Viceroy, MUST produce a gateway-class 502. Its diagnostic naming the missing entitlement MUST be available to logs and as a typed reason the application can match, so the application can tell this case from other upstream failures without parsing provider text. The error's display text does not change (cross-cutting requirement 7). If Fastly refuses options that EdgeZero itself built, the result MUST be an internal error. +- **EZ-8.12** Attaching the options MUST NOT reduce origin connection reuse. Optimized and unoptimized requests to the same origin with the same transport settings MUST be eligible to share connections under EZ-2.1. +- **EZ-8.13** On every send mode, and for every request whether or not it carries options, the URI an application reads back from a built `OutboundRequest` (`OutboundRequest::uri`) MUST be the request target sent. EdgeZero MUST NOT add, remove or reorder query parameters, and MUST NOT change application-set headers beyond its documented normalization (`edgezero:crates/edgezero-adapter-fastly/src/outbound.rs:752-760`). `Host` stays derived from the URI authority unless the application sets a host authority override. Signatures computed after building a request, such as Trusted Server's signed HEAD preflight, raw-error GET and optimized GET, therefore stay valid. This is the single statement of that invariant in the document. +- **EZ-8.14** Capability levels. Fastly single send is BestEffort, because it needs a service entitlement and Viceroy cannot exercise it. It becomes Native when the EZ-Q1 record shows the hosted criterion below passing. Axum, Cloudflare and Spin are Unsupported on every send mode. Fastly begun requests, batches and fire-and-forget sends are Unsupported. +- **EZ-8.15** When EZ-24's published test double lands, it MUST record attached options and accept them, so application tests can assert them. +- **EZ-8.16** Documentation MUST add the capability row with its footnote, a guide section on attaching options, the GET/HEAD and default-cache restrictions, the uninterruptible window and its bound, the entitlement prerequisite, and a warning that Image Optimizer also reads transformation parameters from the request query, with options taking precedence (`fastly-sdk:src/http/request.rs:3094-3096`). EdgeZero does not filter the query. +- **EZ-8.17** An application MUST be able to read, from a built request, the Image Optimizer options attached to it, so its tests can check each call site. + +#### Acceptance criteria + +- Unit. Given options with quality 101, a zero width, offset 101, smart focus with offsets, or only an x offset, when the application builds them, then it gets a client-class error. Quality 0 and 100 succeed. +- Unit. Given a request with options attached, the application reads back options equal to the ones it attached. The same holds for the next request of a followed redirect (EZ-9). +- Axum. Given a local origin, when a GET with options is sent, then the result is cross-cutting requirement 2's rejection with status 400 naming the option and `axum`, and the origin accepts no connection. Given a GET built from an inbound request whose body is an unread stream, with options, then the result is the same rejection and the inbound body is not read. Given a two-slot batch with options on slot 0, then slot 0 fails with the rejection, slot 1 returns 200, and the batch completes. +- workerd and Spin 4.1. A single send and a batch slot with options each return the rejection, and no fetch or WASI request is made. +- Viceroy 0.21.0. Given a GET with options to a reachable origin, then the result is a 502 with the EZ-8.11 reason and the origin receives no request. This shows the options reached the platform, not that Image Optimizer ran, and the hosted criterion is the proof. Given POST with options, GET with a body, or GET with cache bypass, then each gets the rejection, and EZ-2.13's statistics show no registration. Once EZ-11 lands, the same holds for GET with cache tagging. Given a two-slot batch with options on slot 0, then slot 0 fails and slot 1 reaches the origin. Given an optimized GET and a plain GET to one origin with the same transport settings, then EZ-2.13's statistics report one distinct backend configuration. +- Viceroy 0.21.0. Given a plain send whose URI has percent-encoded path bytes and a query, then the origin records a request target equal to `OutboundRequest::uri`, and every header the application set arrives unchanged. +- Hosted Fastly, run under EZ-Q1 on Trusted Server's staging service with Image Optimizer enabled. Given a known JPEG and options for width 100 and WebP, when sent as GET, then the response is `image/webp` with a decoded width of 100 and the body streams. The same request as HEAD succeeds, and its status and headers are recorded in the capability footnote. Given an origin that accepts the connection and never answers, and a 1 s timeout, the request fails within the EZ-8.10 bound, and the measured time is recorded in the footnote. +- Trusted Server end to end, on a Fastly staging service. An optimized asset path returns a transformed image, the HEAD preflight carries no options, and a missing S3 object returns the raw S3 status from the preflight. Trusted Server unit tests read back the options on the final GET and find them equal to the profile-table expectation (EZ-8.17). + +#### Out of scope + +- Cloudflare Image Resizing (`worker:src/request_init.rs:139-140`). A later Cloudflare-specific requirement can add it. +- The full SDK option set (`fastly-sdk:src/image_optimizer.rs:672-760`). +- Filtering or rewriting the query for Image Optimizer. +- The S3 HEAD preflight, which stays Trusted Server logic. + +#### Open questions + +- **Q8.1** Should the option set grow beyond Trusted Server's subset now? Trusted Server prefers no. Ship the subset it uses and leave room to add fields when a consumer needs them. +- **Q8.2** Removed. +- **Q8.3** What should HEAD with options do if Image Optimizer treats it differently from GET? Trusted Server prefers passing HEAD through unchanged and documenting the hosted result. + +### EZ-9. Redirect following with the application's target policy at every hop + +**Priority:** Blocking +**Adapters:** All four, with identical behavior. On Spin, plain-HTTP origins and origins outside a narrowed host list must be declared. +**Related:** EZ-1 (EZ-9.7 owns settling the body once per chain), EZ-2 (each new origin adds a backend configuration on Fastly), EZ-3 (EZ-3.3 owns failing a hop whose deadline has passed), EZ-6 and EZ-15 (begun requests, batches and, once EZ-15 lands, fire-and-forget sends never follow), EZ-4, EZ-5, EZ-7, EZ-8, EZ-11, EZ-12 and EZ-14 (their options carry to every hop), EZ-10 (a replayed body is re-checked against the upload cap), EZ-24 (each hop reaches a test client as an ordinary send) + +#### Need + +Trusted Server follows redirects in `proxy_with_redirects` (`ts:crates/trusted-server-core/src/proxy.rs:1276-1559`). The maintainer decided to delete that loop and move following into EdgeZero. EdgeZero must reproduce this behavior or improve it deliberately. + +- Limit. `MAX_REDIRECTS = 4` (`:1284`), and the loop runs `0..=MAX_REDIRECTS` (`:1292`), so at most 5 requests go out. At the limit it returns the last 3xx as the result, not an error (`:1470-1481`, test `:4305-4352`). +- Statuses. 301, 302, 303, 307 and 308 are followed (`:1415-1422`). Anything else is final. +- `Location`. A missing or empty value, or one that is not visible ASCII, returns the 3xx (`:1434-1446`). The value is parsed as an absolute URL and otherwise joined against the current URL (`:1484-1490`). A value that fails both is a 502 "invalid redirect". +- Methods. A 301 or 302 answering anything but GET, HEAD or POST is not followed (`:1452-1468`). 301, 302 and 303 become a bodyless GET, and HEAD stays HEAD (`:1534-1553`). 307 and 308 keep the method and replay the body (`:1380`). +- Body headers. Once the body is dropped, `Content-Type`, `Content-Length`, `Content-Encoding`, `Content-Language` and `Content-Location` are removed (`:76-82`, `:1372-1376`) and stay removed after a later 307 (test `:4223-4302`). +- Per-hop policy. The scheme must be http or https (`:1299-1304`). With `require_https` a non-HTTPS target is rejected (`:1305-1310`, redirect `:1492-1498`). A redirect to another scheme returns the 3xx (`:1499-1507`). A host is required (`:1312-1317`, `:1509-1516`). The domain allowlist rejects disallowed hosts (`:1319-1327`, `:1517-1526`). These rejections reach the client as 403 on the first-party proxy and the Prebid bundle (`ts:crates/trusted-server-core/src/error.rs:128-131`). GTM, GPT and testlight wrap them as integration errors, which are 502 (`ts:crates/trusted-server-core/src/integrations/google_tag_manager.rs:968-970`, `ts:crates/trusted-server-core/src/integrations/gpt.rs:282-284`, `ts:crates/trusted-server-core/src/integrations/testlight.rs:208-210`, status `error.rs:127`). +- Headers. The same curated set is rebuilt on every hop (`proxy.rs:1356-1371`). It contains no `Authorization` or `Cookie` today (`:67-73`). +- Budget. Each hop gets a fresh 15 s first-byte timer (`:1329-1344`, `ts:crates/trusted-server-core/src/platform/mod.rs:82`). There is no chain budget. +- Final response. `finalize_response` uses the final URL for its origin log line and pixel heuristics (`proxy.rs:747-761`, `:528-590`, `:633-642`). + +| Caller | Follows | Target policy | Body | Response | +| ----------------------------------------------------------------------------------------------------- | ------- | ------------------------------------------------------- | --------------------- | -------------------- | +| First-party proxy (`proxy.rs:1577-1603`) | yes | `proxy.allowed_domains` (open when empty), HTTP allowed | none | buffered, rewritten | +| GTM (`ts:crates/trusted-server-core/src/integrations/google_tag_manager.rs:794-833`, send `:968-970`) | yes | HTTPS only, GTM domains | POST beacon, buffered | buffered, rewritten | +| GPT (`ts:crates/trusted-server-core/src/integrations/gpt.rs:142-153`, send `:273-287`) | no | none | none | buffered passthrough | +| Testlight (`ts:crates/trusted-server-core/src/integrations/testlight.rs:199-210`) | yes | open | POST JSON, buffered | passthrough | +| JS asset proxy (`ts:crates/trusted-server-core/src/integrations/js_asset_proxy.rs:279-306`) | no | none | none | streamed passthrough | +| Prebid bundle (`ts:crates/trusted-server-core/src/integrations/prebid.rs:1279-1286`) | yes | HTTPS only, `proxy.allowed_domains` | none | passthrough | + +After the cut, GPT and the JS asset proxy use a plain `HttpClient::send`. The other four ask EdgeZero to follow, with a target policy that applies Trusted Server's scheme, HTTPS and allowlist rules. Without EZ-9, first-party creative proxying, GTM, testlight and the Prebid bundle return raw 3xx responses to browsers and creatives, and GTM and Prebid script loading breaks whenever the upstream redirects. + +#### Gap at F + +EdgeZero never follows redirects, and its outbound spec calls that a security property. Following requires a fresh request, so the application's target policy runs again (`edgezero:docs/superpowers/specs/2026-05-21-outbound-http-design.md:1454-1470`). Target policy is an explicit non-goal (`:40-42`). Axum disables reqwest redirects (`edgezero:crates/edgezero-adapter-axum/src/outbound.rs:145-155`), Cloudflare sets manual redirect mode (`edgezero:crates/edgezero-adapter-cloudflare/src/outbound.rs:426-429`), and Spin and Fastly do not follow (spec `:1466-1467`). An application cannot write a correct loop on F's API alone. `OutboundRequest` cannot be copied for replay, and `OutboundRequestParts` is non-exhaustive (`edgezero:crates/edgezero-core/src/outbound.rs:69-70`). A fragment in a raw-string target fails canonicalization (`:914-916`, `:1880-1884`), while parsing a `Uri` drops it silently (`http:src/uri/path.rs:28-30`). + +#### Requirements + +- **EZ-9.1** Following MUST be an explicit per-call choice. A plain `HttpClient::send`, a begun request (EZ-6), a batch, and, once EZ-15 lands, a fire-and-forget send MUST keep returning 3xx responses unfollowed. +- **EZ-9.2** A call that follows MUST supply a hop limit from 1 to 20, pending Q9.3, and a per-hop target policy. There MUST be no default that follows everything. A limit outside the range MUST be a client-class error when the call is configured. +- **EZ-9.3** Exactly 301, 302, 303, 307 and 308 are followable. Every other status is final. +- **EZ-9.4** `Location` handling. + - A missing or empty `Location` makes the 3xx final, reported as "no location". + - More than one `Location` field (pending Q9.1), a value that is not a URL reference, a resolved URL with user information, or a resolved http or https target EdgeZero cannot send for any other reason MUST fail the chain as a gateway-class 502 protocol failure, not a client error. A request the policy itself returns is not reclassified, so its own errors come back unchanged. + - A relative value resolves against the URI of the hop that returned it. A fragment never reaches the wire. + - A resolved target with a scheme other than http or https is not a resolution failure. It reaches the policy with no prepared next request. The policy can stop, reject, or return a request of its own. If the policy asks to follow a prepared request that does not exist, the chain fails as the 502 protocol failure above. +- **EZ-9.5** The policy MUST run once per redirect, before any network work for that hop. It MUST see the redirect number, the status, the URI that answered, the resolved target, the 3xx headers, and whether the hop is cross-origin. It MUST be able to follow with the next request EdgeZero prepared, when one exists, or with a request it builds or modifies, to stop and return the 3xx as final, or to reject with the application's own error value, after which nothing more is sent. The initial request is not passed to the policy, so the application checks it before the call. The policy MUST NOT be able to change how a hop's response body is handled (EZ-9.11). +- **EZ-9.6** Method and body rules, applied to the request of the hop that got the redirect. + + | Status | Method of that hop | Next method | Next body | + | -------- | ------------------------------------- | -------------------------------------------------------- | --------- | + | 301, 302 | GET, HEAD | same | empty | + | 301, 302 | POST | GET | empty | + | 301, 302 | any method other than GET, HEAD, POST | not followed, final, reported as "method not rewritable" | none | + | 303 | HEAD | HEAD | empty | + | 303 | any other | GET | empty | + | 307, 308 | any | same | replayed | + +- **EZ-9.7** Replay. A buffered body MUST be replayed byte for byte. A streamed upload cannot be replayed, so a 307 or 308 after one is final, reported as "body not replayable", and the policy is not consulted. When the method rule drops the body, `Content-Type`, `Content-Length`, `Content-Encoding`, `Content-Language` and `Content-Location` MUST NOT be sent, and they stay unsent on later hops because the dropped body stays dropped. A bodiless GET/HEAD whose inbound body is an unread stream MUST be settled once for the whole chain, before the first hop and within the request's budget, the way EZ-1 settles it for a single send. A later 307 or 308 then replays an empty body. This requirement owns the once-per-chain rule. +- **EZ-9.8** Cross-origin hops. When the next target differs from the previous hop in scheme, host or effective port, `Authorization` and `Cookie` MUST NOT be sent to it, and a host authority override (`OutboundRequest::host_authority_override`) MUST NOT carry over. On a same-origin hop both carry over. EdgeZero MUST NOT add `Referer`, `Origin` or any other header beyond its documented normalization. Other credential-bearing headers, such as `x-amz-*` signatures or custom API-key headers, are not removed, and the documentation MUST say the policy must remove them. A request the policy returns is sent as the policy built it. This matches reqwest's cross-host rule for the two headers EdgeZero strips (`reqwest:src/redirect.rs:239-252`). +- **EZ-9.9** Every other request option MUST carry to every hop unchanged. That covers `timeout`, `deadline`, cache policy including tagging, the upload cap, every response cap, decoding settings, the proxy marker, Image Optimizer options, TLS verification mode, connect timeout and stream idle policy. +- **EZ-9.10** Budget. The per-call `timeout` MUST apply to each hop, which keeps Trusted Server's per-hop behavior. An absolute `deadline` MUST bound the whole chain, and a hop whose deadline has already passed is not sent (EZ-3.3). Each hop is an ordinary send, so on Fastly a chain waits hop by hop. On Fastly a chain's hops MUST be eligible to share connections exactly as single sends with the same origin and per-call timeout are (EZ-2.1, EZ-2.2). +- **EZ-9.11** Intermediate bodies. The body of a followed 3xx MUST NOT be delivered, MUST NOT be decoded, MUST NOT affect the chain's outcome, and MUST NOT cost more time or memory than a documented bound. EdgeZero MAY read a bounded, documented amount of it to keep the connection reusable. Such a body MUST NOT fail the chain for any reason. That covers malformed or empty compressed bodies, such as `Content-Encoding: br` with no bytes, which fails Brotli decoding at F (`edgezero:crates/edgezero-core/src/compression.rs:325-333`), and an oversized declared `Content-Length`. Response header caps still apply to each hop's headers. The caller's buffered cap and streamed or buffered mode apply to the final response only. For a chain with no redirect, the result MUST be indistinguishable from a plain send of the same request, with the same status, headers, bytes, and limit error. +- **EZ-9.12** At most the limit's number of redirects MUST be followed, so at most limit plus one requests are sent. A followable response that would exceed the limit MUST be returned as a successful final result, reported as "limit reached". +- **EZ-9.13** A successful result MUST report the final response, the final request URI, the number of redirects followed, and, when the final status is followable, why it was not followed. The reasons are no location, method not rewritable, body not replayable, limit reached, and stopped by policy. The reason is present exactly when the final status is one of the five followable statuses. +- **EZ-9.14** A hop failure MUST return that hop's error unchanged, with which request failed (0 for the initial request). A policy rejection MUST return the application's value unchanged, with the redirect number. No error is turned into a 3xx response. A following call MUST send each hop as an ordinary send through the client it was called on, including an application-supplied client, so a recording client sees every hop's method, URI, headers and body. No hop number or chain state reaches the client. +- **EZ-9.15** Abandoning a following call MUST release the in-flight hop as a plain send would, and nothing more is sent. +- **EZ-9.16** EdgeZero SHOULD log each followed hop at debug level with status, scheme, host, port and path, and MUST NOT log query strings, which can carry identifiers. Trusted Server appends `ts-ec` to proxied URLs (`ts:crates/trusted-server-core/src/proxy.rs:1206-1242`). +- **EZ-9.17** Removed. +- **EZ-9.18** When several rules could end the chain at one redirect, the first in this order wins. The order follows Trusted Server's loop (`ts:crates/trusted-server-core/src/proxy.rs:1415-1526`). Followable status (EZ-9.3), then `Location` presence ("no location"), then the method rule ("method not rewritable"), then the replay rule ("body not replayable"), then the limit ("limit reached"), then `Location` resolution (502 protocol failure, except a non-HTTP scheme, which goes on to the policy), then the policy (stop or reject). +- **EZ-9.19** Documentation MUST add a guide section on following redirects with the method table, the credential rules and what they leave in place, the budget rules, and a security note. The note says the policy is the only target policy, and an open policy also follows redirects to private or link-local addresses. The guide MUST record that this supersedes the outbound spec's "read `Location` and issue a new request" guidance for applications that use it, while plain sends keep that rule (`edgezero:docs/superpowers/specs/2026-05-21-outbound-http-design.md:1469-1470`). The capability matrix MUST list redirect following as Native on all four adapters, with a note that Spin needs plain-HTTP origins and origins outside a narrowed host list declared (`edgezero:docs/guide/capabilities.md:362-377`). + +#### Acceptance criteria + +- Unit, with a scripted upstream. For every status from 300 to 308 and each of GET, HEAD, POST, PUT, PATCH, DELETE and OPTIONS with a 5-byte buffered body (empty for GET and HEAD), the number of requests and the second request's method, body and body headers match EZ-9.6. +- Unit. A POST answered by 302 and then 307 sends POST, GET, GET, with the body only on the first request and no body headers on the later two. A POST with an empty buffered body and `Content-Type` answered by 307 sends `Content-Type` on both requests. +- Unit. A streamed upload answered by 307 sends one request and reports "body not replayable". +- Unit. A GET whose body is an empty unread stream, answered by 307, sends two requests with empty bodies and reports no stop reason. +- Unit. With limit 4 and five 302 responses to GET, five requests are sent and the result is a 302 with "limit reached" and four redirects followed. A PUT answered by 301 at the limit reports "method not rewritable". A 302 with no `Location` answering a PUT at the limit reports "no location". Limits 0 and 21 are rejected. +- Unit. `Authorization` and `Cookie` reach a same-origin hop and do not reach a hop that changes host, port, or only the scheme. A host authority override reaches a same-origin hop only. An `x-amz-date` header reaches both. +- Unit. Relative, `../x`, `//b.example/x` and absolute `Location` values follow. A fragment never reaches the wire. A missing or empty value reports "no location". Two `Location` fields, user information, or an unusable http target give a 502 protocol failure. +- Unit. A policy rejection returns the application's value with redirect number 1 and no second request. A policy stop returns the 3xx with "stopped by policy". A modified next request is sent exactly as returned. A `javascript:` target reaches the policy with no prepared request. A policy that asks to follow a prepared request there gets a 502 protocol failure. +- Unit. A 302 whose body errors when read, a 302 with `Content-Encoding: br` and an empty body, and a 302 declaring a 10 MiB `Content-Length` under a 1 MiB cap each lead to the final 200. +- Unit. A deadline that passes after the first hop fails the second hop as a gateway timeout attributed to the deadline, and the second hop is not sent. Every hop sees the original per-call timeout. +- Unit. Dropping a following call while hop 1 is in flight sends no hop 2. +- Unit. With logging captured at trace level, a two-hop chain whose URIs carry `?ts-ec=abc` logs neither `ts-ec` nor `abc`. +- Unit. A following call made on an application-supplied recording client records one ordinary send per hop, with each hop's method, URI, headers and body. +- Axum, workerd, Spin 4.1 and Viceroy 0.21.0. For each adapter, with a 200 and no redirect under a 10-byte buffered cap, bodies of 10 and 11 bytes and a declared `Content-Length: 11` give the same status, headers, bytes or limit error from a plain send and from a following call, for identity, gzip and passthrough encodings. +- Axum and Viceroy 0.21.0, with two local origins where A answers 302 to B and B answers 200. A plain send returns 302. A following call returns 200 with B as the final URI and one redirect followed. +- Viceroy 0.21.0. A two-hop chain to two loopback origins with one per-call timeout leaves EZ-2.13's statistics showing the same two backend configurations as two single sends to those origins with that timeout. +- Spin 4.1. A redirect to a plain-HTTP host that is not declared fails at that hop with Spin's denial, which F maps to 400 (`edgezero:crates/edgezero-adapter-spin/src/outbound.rs:1358-1364`). That Spin 4.1 reports this case as `HttpRequestDenied` is **Unverified**, and this criterion records what it reports. +- Trusted Server end to end. Port every redirect test in `ts:crates/trusted-server-core/src/proxy.rs:3978-4515`, the allowlist tests at `:6114-6258`, and the initial-target policy tests at `:6259-6325` to the following call, with identical recorded methods, URIs, bodies, header names and request counts. Use EZ-24 if it has landed and Trusted Server's own recording client otherwise (EZ-9.14). On the Axum dev server, a signed first-party proxy URL whose origin 302s to an allowed host returns the final 200, and one that 302s to a disallowed host returns 403. + +#### Out of scope + +- Loop detection beyond the hop limit (`edgezero:docs/superpowers/specs/2026-05-21-outbound-http-design.md:43-45`) and blocking private addresses (`:41-42`). Both stay application policy. +- Buffering a streamed upload so it can be replayed. +- An asynchronous policy. +- Following inside begun requests, batches or fire-and-forget sends. + +#### Open questions + +- **Q9.1** Should more than one `Location` field fail or use the first? Trusted Server prefers failing. `Location` is a singleton field, and a well-formed upstream never sends two. Trusted Server uses the first value today (`ts:crates/trusted-server-core/src/proxy.rs:1434-1438`). +- **Q9.2** Should EdgeZero strip credentials again when the policy's modified request changes origin? Trusted Server prefers no. The policy is application code, and stripping again would stop it from re-signing for a new origin. +- **Q9.3** Is 20 the right maximum limit? Trusted Server prefers 20. It bounds backend configurations per chain on Fastly, and Trusted Server uses 4. + +## Follow-up requirements + +### EZ-10. 413 Payload Too Large for request-body overflow + +**Priority:** Follow-up +**Adapters:** Fastly, Axum, Cloudflare, Spin. The status and error category must be the same everywhere. Adapters differ only in what the origin may already have received when a streamed upload overflows (EZ-10.6). +**Related:** EZ-1, whose GET/HEAD body rejection stays 400 and whose EZ-1.13 owns the rule on unread bytes. EZ-16, whose provider detail shares EZ-10's log-only treatment (cross-cutting requirement 7). EZ-17, which must report an upload that overflowed after the request head left as sent. EZ-25, which owns the scaffold and guide fixes. + +#### Need + +TS answers request-body overflow with 413 on every endpoint that reads a body. + +| TS endpoint | Cap | Behavior today | Source | +| ------------------------------------------------------------------------------------- | -------------- | ------------------------------------------------------------------------------- | -------------------------------------------------------------------------------------------------------------------------------------------- | +| `POST /auction` | 256 KiB | Early `Content-Length` check, then a post-read check. 413 `text/plain` | `ts:crates/trusted-server-core/src/auction/endpoints.rs:46`, `:126-159`. Test `:1771-1791` | +| First-party sign and rebuild | 64 KiB each | 413 | `ts:crates/trusted-server-core/src/proxy.rs:40-41`, `:1720-1721`, `:1909-1910`. Tests `:6343`, `:6389` | +| Verify signature, rotate key, deactivate key | 4 KiB | 413 | `ts:crates/trusted-server-core/src/request_signing/endpoints.rs:101-102`, `:116-117`, `:306-307`, `:425-426`. Tests `:995`, `:1034`, `:1073` | +| Cache purge | 4 KiB | 413 | `ts:crates/trusted-server-core/src/cache_purge.rs:30`, `:141-142` | +| Batch sync | 2 MiB | Early `Content-Length` check, then a post-read check. 413 JSON `body_too_large` | `ts:crates/trusted-server-core/src/ec/batch_sync.rs:143-158`, `:188-194` | +| Integration POST proxies (datadome, didomi, permutive, sourcepoint, lockr, testlight) | 256 KiB | 413 | `ts:crates/trusted-server-core/src/integrations/mod.rs:167`, `:174-222` | +| GTM beacons | 64 KiB default | Early `Content-Length` check and a read cap. 413 | `ts:crates/trusted-server-core/src/integrations/google_tag_manager.rs:186-188`, `:290-292`, `:689-728`. Read-cap test `:2763-2817` | + +TS's request-too-large error maps to 413 (`ts:crates/trusted-server-core/src/error.rs:48-50`, `:129`, `ts:crates/trusted-server-core/src/http_util.rs:465-484`). After the cut TS reads every body through EdgeZero's readers, and each returns 400 on overflow. Without EZ-10: + +- Every endpoint above answers an undeclared or understated overflow with 400 instead of 413. Tests that go through TS's early `Content-Length` checks keep passing, but the read-cap tests fail. +- After `RequestContext::json_within`, TS cannot tell overflow from malformed JSON, because both are 400 (`edgezero:crates/edgezero-core/src/context.rs:401-408`). +- A route that uses EdgeZero's extractors cannot fix the status with `App::set_error_response_renderer`, because EdgeZero reapplies the error's own status after the renderer runs (`edgezero:crates/edgezero-core/src/app.rs:334-344`, `edgezero:docs/guide/handlers.md:397-399`). +- A client POST forwarded to the publisher origin over TS's `max_request_body_bytes` cap fails with 400, where 413 is the accurate status. + +#### Gap at F + +- Every inbound reader returns 400 on overflow. That covers `Body::into_bytes_bounded` (`edgezero:crates/edgezero-core/src/body.rs:81-111`), `RequestContext::body_bytes` (`context.rs:202-258`, `:621-626`, `:663-668`), `json_within` and `form_within` (`:345-357`, `:396-408`), and the extractors built on them (`edgezero:crates/edgezero-core/src/extractor.rs:33-44`, `:69-84`, `:86-102`, `:430-439`, `:466`, `:507-516`). EdgeZero's inbound design states 400 for overflow and gives no reason against 413 (`edgezero:docs/superpowers/specs/2026-08-22-inbound-body-design.md:663-664`). +- No declared length is checked early. A 10 MiB declared upload against a 256 KiB cap is read up to the cap before it is refused. +- `max_request_body_bytes` overflow returns 400 on every adapter. See Axum `edgezero:crates/edgezero-adapter-axum/src/outbound.rs:229-235` and `:253-262`, Cloudflare `edgezero:crates/edgezero-adapter-cloudflare/src/outbound.rs:671-677` and `:871-878`, Fastly `edgezero:crates/edgezero-adapter-fastly/src/outbound.rs:1137-1146` and `:1269-1277`, and Spin `edgezero:crates/edgezero-adapter-spin/src/outbound.rs:582-600`. On Fastly a streamed overflow is found after the request head has been sent (`:1180-1182`), and the upload is abandoned (`outbound-http-design.md:5713-5716`). On Spin even a buffered body over the cap is found only after the request has been issued (`outbound.rs:362-377`, `:565-567`), so Spin does not meet EZ-10.6 at F. +- EdgeZero has no error category with status 413 (`edgezero:crates/edgezero-core/src/error.rs:452-474`). +- The scaffold and demo fallback policies answer an over-cap unmatched request with 400 (`edgezero:crates/edgezero-cli/src/templates/core/src/lib.rs.hbs:104-107`, `edgezero:examples/app-demo/crates/app-demo-core/src/lib.rs:196-199`). EdgeZero's own unit test uses 413 (`edgezero:crates/edgezero-core/src/ingress.rs:980-999`). + +#### Requirements + +- **EZ-10.1** EdgeZero MUST give request-body overflow its own error category with status 413. Applications MUST be able to tell it apart from malformed payloads (400), the inbound read deadline (408) and every other error without matching message text. Its client-visible body MUST be stable, MUST NOT carry provider or internal detail, and MUST NOT depend on which cap fired. +- **EZ-10.2** Every inbound body cap EdgeZero enforces MUST report overflow in that category. That covers `RequestContext::body_bytes`, `json_within`, `form_within`, the `Json`, `ValidatedJson`, `ValidatedJsonWithin`, `Form`, `ValidatedForm` and `ValidatedFormWithin` extractors, and `Body::into_bytes_bounded`. It MUST hold for a declared overflow (`Content-Length` above the cap) and for an undeclared one, such as chunked HTTP/1.1, HTTP/2 or HTTP/3 with no length, or a length that understates the body. Malformed JSON or form data under the cap stays 400. +- **EZ-10.3** The existing sticky read semantics of the `RequestContext` readers MUST NOT change. Once a read overflows while reading the body, every later read of that body returns the same 413 and reads nothing more. A stricter cap checked against a body already read returns 413 and leaves the body usable at a larger cap. +- **EZ-10.4** When the admitted request's single normalized `Content-Length` exceeds a reader's cap, the reader SHOULD fail with 413 without reading any body byte. An inbound read deadline that has already passed still wins, with 408. Such an early failure is not a read, so it MUST NOT stop a later read at a larger cap from succeeding, and EZ-10.3's sticky rule does not apply to it. +- **EZ-10.5** Overflow of an outbound request's `max_request_body_bytes` MUST report the same 413 category on all four adapters, for buffered and streamed bodies. An upstream that answers 413 MUST stay an ordinary response with status 413, never this error. +- **EZ-10.6** These rules hold for what the origin may see: + - A buffered body over the cap MUST fail before anything is sent, on every adapter. + - On Axum and Cloudflare a streamed upload over the cap MUST fail before the origin is contacted. + - On Fastly and Spin the origin may already have the request head and part of the body. The upload MUST then be aborted so the origin never receives a complete, well-formed request with a truncated body. EZ-17 reports such a request as sent. +- **EZ-10.7** Applications MUST be able to learn whether an overflow came from an inbound read or an outbound upload without parsing message text. That detail MUST NOT reach the wire, and the text and `Display` of existing errors MUST NOT change (cross-cutting requirement 7). Where EZ-16 has landed, the detail SHOULD be read the same way as EZ-16's provider detail. +- **EZ-10.8** An application's `App::set_error_response_renderer` MUST receive the overflow error, and the response status after rendering MUST be 413. +- **EZ-10.9** The scaffold and demo `AdmissionDecision::ReadBodyBeforeFallback` policies MUST answer an over-cap fallback request with 413. +- **EZ-10.10** A 413 response MUST NOT carry `Retry-After`, because the condition is not temporary (RFC 9110 section 15.5.14). +- **EZ-10.11** Unread request bytes after a 413 on connections EdgeZero owns follow EZ-1.13. +- **EZ-10.12** The handler and capability guides MUST give the statuses for inbound bodies, which are 413 for overflow, 400 for malformed payloads, and 408 for the read deadline. They MUST also state where Fastly and Spin streamed overflow happens relative to sending. + +#### Acceptance criteria + +1. Environment is unit. Given a 9-byte body with no declared length, when it is read with a cap of 8 through `body_bytes`, `json_within`, `form_within`, an extractor, or `Body::into_bytes_bounded`, then the result is the 413 category and the response status is 413. For the `RequestContext` readers, a second read at any cap returns the same 413 without reading. Malformed JSON under the cap stays 400. +2. Environment is unit. Given a request declaring `Content-Length: 10` with a 10-byte body, when it is read with a cap of 5, then the result is 413. With the inbound read deadline already passed, the result is 408. Record whether any body byte was read before the 413 (EZ-10.4 is SHOULD). Where none was, a later read of that request with a cap of 10 succeeds. +3. Environment is Axum, Viceroy 0.21.0, workerd and Spin 4.1. A chunked POST of 2 KiB to a route capped at 1 KiB gets 413. On Axum, an HTTP/1 request pipelined after it is either refused by a connection close or answered from the correct offset. +4. Environment is hosted Fastly, Trusted Server staging. An HTTP/2 POST with no `content-length` over the cap gets 413. +5. Environment is Axum, Viceroy 0.21.0, workerd and Spin 4.1. A forwarded 9-byte buffered body with `max_request_body_bytes(8)` gets 413, and the origin records no request. +6. Environment is Axum and workerd. A streamed forwarded upload over the cap gets 413, and the origin records no request. On Viceroy 0.21.0 and Spin 4.1, the origin never records a complete request. +7. Environment is Axum. An extractor route with a custom error renderer answers an oversized body with 413, and the response has no `Retry-After`. +8. Environment is Axum, using the generated scaffold and the demo app. An over-cap request to an unmatched route gets 413. +9. Trusted Server end to end. With TS reading bodies through EdgeZero's readers and mapping the overflow category to its request-too-large error, the read-cap 413 tests pass unchanged. They are `ts:crates/trusted-server-core/src/auction/endpoints.rs:1771-1791`, `proxy.rs:6343` and `:6389`, `request_signing/endpoints.rs:995`, `:1034` and `:1073`, and `integrations/google_tag_manager.rs:2763-2817`. New TS cases get 413 for an oversized chunked `POST /auction` with no `Content-Length` and for an understated length on batch sync. + +#### Out of scope + +- Changing default caps. +- Changing how malformed or duplicate `Content-Length` is handled. EdgeZero rejects those with 400 before admission (`ingress.rs:892-921`). +- Changing how an upstream's own 413 response is surfaced. +- Making a streamed upload overflow invisible to the origin on Fastly and Spin. + +#### Open questions + +1. Should an outbound upload overflow be 413, or a server-error status? TS prefers 413. The documented use of the upload cap is bounding a forwarded client upload (`edgezero:docs/guide/proxying.md:32-35`), where the client is at fault. An application that builds its own oversized body can tell the two apart through EZ-10.7 and remap. +2. Should readers reject a declared oversized length before reading (EZ-10.4 is SHOULD)? TS prefers yes. It saves reading up to the cap, and with it TS can drop its own early checks in the auction, batch-sync and GTM handlers. + +#### Interim for Trusted Server + +Until EZ-10 lands, TS maps the 400 from `RequestContext::body_bytes` and `Body::into_bytes_bounded` to its own 413. Today those two calls return 400 only for overflow, because read failures pass through with their own categories (`body.rs:99`, `context.rs:662`). TS reads JSON with `body_bytes` and then parses it, so malformed input stays a separate 400. TS keeps its early `Content-Length` checks, which are header checks, not transport plumbing. + +### EZ-11. Cache tagging with surrogate keys + +**Priority:** Follow-up +**Adapters:** Fastly must honor tags. Axum and Spin accept them with no effect, since they have no intermediary cache. Cloudflare is Unsupported. +**Related:** EZ-8 (EZ-8.6 rejects tags combined with Image Optimizer), EZ-9 (tags carry to redirect hops), EZ-6 and EZ-15 (EZ-11.4 owns the rule that tags behave the same on begun requests and, once EZ-15 lands, fire-and-forget sends) + +#### Need + +Trusted Server can let publisher origin HTML enter Fastly's cache and tag it for purge. `apply_origin_cache_intent` (`ts:crates/trusted-server-core/src/publisher.rs:4356-4394`) bypasses for non-shareable documents and tags shareable ones with `ts-template ts-template-readerurl-` (`:4384-4391`, keys `ts:crates/trusted-server-core/src/platform/template_cache.rs:46`, `:234-239`). Both origin send sites use it (`publisher.rs:4852-4872`, `:5160-5177`). The cache intent is one enum on purpose, because on Fastly a surrogate key overrides pass, so "bypass and tag" would silently mean "do not bypass" (`ts:crates/trusted-server-core/src/platform/http.rs:10-29`). On an invalid key the Fastly adapter falls back to bypass rather than store an object no purge can reach (`ts:crates/trusted-server-adapter-fastly/src/platform.rs:452-473`). It never sets a TTL (`:448-451`). + +After the cut Trusted Server sets a cache policy of "default plus these tags" on the origin `OutboundRequest`, and falls back to `OutboundCachePolicy::Bypass` if the tags are invalid. Without EZ-11 it can only pick `Bypass` or the default, and the default would cache pages that no purge can remove. + +#### Gap at F + +`OutboundCachePolicy` has only `Bypass` and `PlatformDefault` (`edgezero:crates/edgezero-core/src/outbound.rs:109-115`). Fastly maps `Bypass` to cache pass (`edgezero:crates/edgezero-adapter-fastly/src/outbound.rs:735-750`) and Cloudflare maps it to no-store (`edgezero:crates/edgezero-adapter-cloudflare/src/outbound.rs:419-436`). Axum and Spin have no intermediary cache (spec `edgezero:docs/superpowers/specs/2026-05-21-outbound-http-design.md:1045-1051`). The Fastly SDK supports tagging. A surrogate key overrides pass and extends backend `Surrogate-Key` headers (`fastly-sdk:src/http/request.rs:2481-2501`). Keys are printable ASCII without spaces, and invalid keys are silently ignored (`:2486-2490`). A cache override sets TTL and stale-while-revalidate only when they are provided (`fastly-sdk:src/http/request/handle.rs:1834-1845`), so a key alone sets neither. Whether the override's documented effect on cache directive headers (`:939-942`) changes origin freshness for a key-only override is **Unverified**. The tag reaches the host on handle-level sends too (`:943-954`, `:1815-1866`). Workers fetch has no cache-tag field in worker 0.8.5 (`worker:src/request_init.rs:116-166`). + +#### Requirements + +- **EZ-11.1** An application MUST be able to ask, per request, for the default cache behavior plus one or more purge tags. Asking for bypass and tags together MUST be impossible to express or MUST be rejected before sending. +- **EZ-11.2** Tags MUST be validated when the application sets them. There is at least one tag, each tag is non-empty, and each contains only bytes `0x21` to `0x7E`. Invalid tags MUST be a client-class error at that point and never silently ignored. Duplicates collapse while keeping order. +- **EZ-11.3** Tagging MUST behave exactly like the default policy plus tags on any object the intermediary cache stores for the exchange. It MUST NOT force caching, set a TTL, or add any origin-visible request header. +- **EZ-11.4** On Fastly, tags MUST take effect on every send mode, including single send, begun request, batch slot, redirect hop and, once EZ-15 lands, fire-and-forget send, and a purge by any one tag MUST remove the stored object. Capability Native. This requirement owns the same-semantics rule for tagging across send modes. +- **EZ-11.5** On Axum and Spin, tags MUST be accepted and have no effect. Capability Native, because no untagged object can be stored. +- **EZ-11.6** On Cloudflare, tags MUST get cross-cutting requirement 2's rejection, first in preflight and before any fetch, naming the option and the adapter. Capability Unsupported, because degrading to the default would store untagged objects. +- **EZ-11.7** Removed. +- **EZ-11.8** Tags MUST carry to every redirect hop (EZ-9.9). +- **EZ-11.9** Documentation MUST add the capability row, describe the three cache intents, and state that tagging never forces caching. + +#### Acceptance criteria + +- Unit. No tags, an empty tag, `a b`, a non-ASCII tag and `0x7F` are client-class errors. `ts-template` with a reader-URL tag is accepted. `a`, `a` collapses to `a`. +- Viceroy 0.21.0. A tagged single send and a tagged two-slot batch complete without error. +- workerd. A tagged request returns the rejection, and no fetch is made. +- Axum. A tagged request succeeds, and the origin receives no `surrogate-key` or `cache-control` header added by EdgeZero. +- Hosted Fastly. Given a cacheable origin path that counts the requests it receives, two fetches tagged `probe-key` leave the origin count at one. After a purge of `probe-key`, a third fetch raises it to two. The same holds when the fetches are begun requests and when they are batch slots. +- Trusted Server end to end. With readthrough enabled on a Fastly staging service, purging `ts-template` turns a cached page into a miss, and the origin sees the request again. Trusted Server unit tests assert the tagged policy at both origin send sites, using EZ-24 if it has landed or Trusted Server's own double otherwise. + +#### Out of scope + +- TTL overrides and forced caching. +- A purge API. +- Cloudflare cache tags. + +#### Open questions + +- **Q11.1** Which length limits apply to a tag and to the whole set? Trusted Server prefers 1024 bytes per tag and 16 KiB in total, rejected when set. These are **Unverified** Fastly limits. A hosted probe with an oversized key, or Fastly's surrogate key documentation, would confirm them. The SDK documents only the byte set (`fastly-sdk:src/http/request.rs:2486-2490`). +- **Q11.2** Should Cloudflare support tags through a raw `cf.cacheTags` property? Trusted Server prefers not now. worker 0.8.5 has no typed field, and whether the runtime honors it on subrequests is **Unverified**. + +#### Interim for Trusted Server + +Until EZ-11 lands, Trusted Server rejects `origin_readthrough_enabled = true` at config validation. Readthrough is off by default (`ts:crates/trusted-server-core/src/creative_opportunities.rs:419-426`), and no local operator configuration enables it. Document fetches keep today's bypass-or-default behavior. + +### EZ-12. TLS certificate verification mode + +**Priority:** Follow-up +**Adapters:** Fastly must honor the mode. It is Unsupported on HTTPS for Axum, Cloudflare and Spin. +**Related:** EZ-2 (EZ-2.1 requires matching TLS settings, and the mode is one of them), EZ-9 (the mode carries to redirect hops) + +#### Need + +`proxy.certificate_check` defaults to `true` and is documented as the switch "for local development with self-signed certificates" (`ts:crates/trusted-server-core/src/settings.rs:1762-1790`). It reaches four sites. Those are asset routes (`ts:crates/trusted-server-core/src/proxy.rs:1136`), every redirect hop (`:1337`), the publisher origin (`ts:crates/trusted-server-core/src/publisher.rs:4516`) and pull sync (`ts:crates/trusted-server-core/src/ec/pull_sync.rs:237`). Integrations and auction providers always verify (`ts:crates/trusted-server-core/src/integrations/mod.rs:53-66`, `ts:crates/trusted-server-core/src/auction/plan.rs:396`). On Fastly, `false` omits the certificate hostname check and keeps TLS, and keeps SNI for hostnames only (`ts:crates/trusted-server-adapter-fastly/src/backend.rs:243-254`, SNI `:245-247`). Fastly logs an INSECURE warning at startup (`ts:crates/trusted-server-adapter-fastly/src/app.rs:240-246`). Axum, Cloudflare and Spin ignore the setting (`ts:crates/trusted-server-adapter-axum/src/platform.rs:290-301`). + +Fastly still validates the chain without the hostname check. That validation "is not optional" (`fastly-sdk:src/backend/builder.rs:176-190`, `:219-229`). So the setting's real effect is that an origin is accepted when its certificate chains to a public root but names a different host. The common case is an origin reached by IP or an internal name. Viceroy is stricter. With no certificate host given it checks the SNI hostname (`viceroy:src/wiggle_abi/req_impl.rs:535-569`), or the backend URI host when there is no SNI either (`viceroy:src/upstream.rs:188-192`), so locally the setting changes nothing. + +After the cut Trusted Server asks for this mode at the four sites when `certificate_check = false`. Without EZ-12, such a Fastly deployment fails every request to an origin whose certificate names another host with a 502 (`edgezero:crates/edgezero-adapter-fastly/src/outbound.rs:935-944`). No local operator configuration sets `false`. + +#### Gap at F + +Every HTTPS request verifies the certificate against the URI host (`edgezero:crates/edgezero-core/src/outbound.rs:869-873`, Fastly backend `edgezero:crates/edgezero-adapter-fastly/src/outbound.rs:349-357`). For an IP-literal URI F sends no SNI (`edgezero:crates/edgezero-core/src/outbound.rs:1189-1192`). There is no per-request choice. reqwest 0.13.4 with rustls allows skipping only the hostname check when the client also uses an explicit root set instead of platform roots, and fails to build otherwise (`reqwest:src/async_impl/client.rs:734-747`). Workers fetch has no TLS control (`worker:src/request_init.rs:12-27`). WASI HTTP request options have only three timeouts (`spin-sdk:wit/deps/http@0.3.0/types.wit:384-416`). + +#### Requirements + +- **EZ-12.1** An application MUST be able to choose, per request, between full verification (the default) and chain-only verification. Chain-only still validates chain and expiry against the platform's trust roots and skips only the hostname match. A mode MUST NOT disable chain validation. +- **EZ-12.2** On `http` URIs the mode MUST have no effect on any adapter. SNI MUST be unchanged by the mode. +- **EZ-12.3** On Fastly, a chain-only request MUST succeed against a publicly trusted certificate for another name, and MUST fail with a gateway-class 502 against a self-signed, expired or otherwise invalid certificate. Capability Native. +- **EZ-12.4** On Fastly the mode is one of the TLS settings that EZ-2.1 requires to match for connection sharing. EZ-2.1 owns that rule. +- **EZ-12.5** On Fastly, EdgeZero MUST log a warning naming the host the first time a chain-only connection setup is used in a sandbox. +- **EZ-12.6** On Axum, Cloudflare and Spin, chain-only on an `https` URI MUST get cross-cutting requirement 2's rejection, first in preflight and before sending. Capability Unsupported. +- **EZ-12.7** The mode MUST carry to every redirect hop (EZ-9.9). +- **EZ-12.8** Documentation MUST add the capability row, state exactly what chain-only checks, and warn that any publicly trusted certificate can then impersonate the origin. + +#### Acceptance criteria + +- Unit. Chain-only on an `http` URI sends normally on every adapter. +- Axum, workerd and Spin 4.1. Chain-only on `https` returns the rejection and nothing is sent. +- Hosted Fastly. An origin reached by IP whose certificate is valid for a hostname fails with full verification (502) and succeeds with chain-only. An origin with a self-signed certificate fails in both modes. Given an origin that logs the connection each request arrives on, a full and a chain-only request to it in one sandbox arrive on different connections. Viceroy cannot show the mode difference, because it checks the SNI hostname or the URI host whenever no certificate host is given. +- Trusted Server end to end. With `certificate_check = false` on a Fastly staging service whose origin is an IP-literal HTTPS URL, publisher pages load. + +#### Out of scope + +- Disabling chain validation. +- Custom trust roots. +- Chain-only on Axum. + +#### Open questions + +- **Q12.1** Should Axum support chain-only? Trusted Server prefers not now. reqwest needs an explicit root set for it (`reqwest:src/async_impl/client.rs:734-741`), which means a new bundled root dependency or new configuration, and Trusted Server's Axum adapter has always verified in full. +- **Q12.2** Should EdgeZero add custom trust roots for self-signed development origins? Trusted Server prefers a separate follow-up. Fastly supports a CA certificate per backend (`fastly-sdk:src/backend/builder.rs:231-241`), and that is the safe answer to what `certificate_check = false` was meant to do. + +#### Interim for Trusted Server + +Until EZ-12 lands, Trusted Server rejects `certificate_check = false` at config validation on every adapter, because F always verifies the hostname and Axum, Cloudflare and Spin never honored the setting. Trusted Server also corrects the setting's doc comment, which claims self-signed support (`ts:crates/trusted-server-core/src/settings.rs:1765-1767`). + +### EZ-13. Spin host authority override + +**Priority:** Follow-up (Trusted Server recommends closing it as documentation and evidence) +**Adapters:** Spin +**Related:** EZ-9 (a cross-origin redirect hop drops the override), EZ-25 (documentation) + +#### Need + +`publisher.origin_host_header_override` sets the `Host` sent while connecting to `origin_url` (`ts:crates/trusted-server-core/src/settings.rs:44-47`, `:209-216`). The publisher backend passes it as the host override (`ts:crates/trusted-server-core/src/publisher.rs:4510-4520`). After the cut it becomes `OutboundRequest::host_authority_override` on the publisher fetch. Axum and Fastly support that natively, and Cloudflare is BestEffort (`edgezero:docs/guide/capabilities.md:236`). + +On Spin the setting has never worked. Trusted Server's Spin client strips `host` as forbidden by WASI HTTP (`ts:crates/trusted-server-adapter-spin/src/platform.rs:410-428`, `:482`), so the override is dropped and the URI authority is sent. EdgeZero rejects the override on Spin instead, so with the setting present every publisher request on Spin would fail. One deployment's local configuration sets the override, and that deployment runs on Fastly. + +#### Gap at F + +Spin rejects any override with a 400 before body polling (`edgezero:crates/edgezero-adapter-spin/src/outbound.rs:1433-1441`, batches `:1417-1431`). It sets one authority from the URI (`:509-535`). The spec and the matrix record Spin as Unsupported (`edgezero:docs/superpowers/specs/2026-05-21-outbound-http-design.md:1056-1060`, `edgezero:docs/guide/capabilities.md:236`, `:255-258`). WASI HTTP 0.3 in spin-sdk 7.0.0 offers no way to send a `Host` other than the connection authority. + +- A request has one authority, set as "the authority of the Request's target URI" (`spin-sdk:wit/deps/http@0.3.0/types.wit:336-344`). +- Request options carry only three timeouts. There is no connect-target, proxy or address override (`:384-416`). +- Whether a `host` field may be set is implementation-defined. Fields can be forbidden (`:112-114`, `:209-210`), and the host must reject requests with headers "not permitted to be sent" (`:300-304`). Trusted Server's comment says WASI HTTP 0.2 forbids `host` (`ts:crates/trusted-server-adapter-spin/src/platform.rs:412-413`). That is **Unverified** for Spin's host. The source to check is wasmtime's `wasi-http` crate, `crates/wasi-http/src/types.rs`, its forbidden-header check, which is not available locally. +- Even if `host` were accepted, nothing in the interface says the host would send it instead of the authority. That is also **Unverified**. + +#### Requirements + +- **EZ-13.1** Spin MUST keep rejecting a host authority override before anything is sent, on every send mode. It MUST NOT drop the override silently, and MUST NOT send the connection to the override's host. +- **EZ-13.2** The rejection MUST be cross-cutting requirement 2's rejection, first in preflight, naming the option and the adapter. The status stays 400. +- **EZ-13.3** EdgeZero MUST establish and record, on Spin 4.1, whether a component can set a `host` request field and what `Host` an origin receives for a request. +- **EZ-13.4** The capability matrix footnote MUST state that Unsupported is final for spin-sdk 7.0.0 and WASI HTTP 0.3.0, cite the interface evidence and the EZ-13.3 results, and name the change that would reopen it. That change is a WASI request option for the connect target or a proxy, or a Spin runtime setting that maps an authority to an address. The Spin guide (`edgezero:docs/guide/adapters/spin.md:92-94`) and the proxying guide (`edgezero:docs/guide/proxying.md:89-92`) MUST say the same. + +#### Acceptance criteria + +- Spin 4.1. A request with an override, sent alone or in a batch, returns the rejection with status 400, and no WASI request is made. +- Spin 4.1. A component that sets a `host` field records whether the host forbids it. A request to a local origin records the `Host` received, which is expected to equal the authority. Both results appear in the footnote. +- Trusted Server end to end, Spin 4.1. A Spin build whose config sets `publisher.origin_host_header_override` fails config validation with a message pointing to this limitation. A Spin build without it serves publisher pages. + +#### Out of scope + +- Implementing the override on Spin. +- A fallback that silently sends the URI authority instead of the override. + +#### Open questions + +- **Q13.1** Should an override equal to the URI authority be treated as absent on every adapter? Trusted Server prefers yes. It removes a Spin failure with no semantic content. The comparison must ignore a default port, as target canonicalization does (`edgezero:crates/edgezero-core/src/outbound.rs:1622-1627`). If adopted, EZ-13.1 applies only to an override that differs from the URI authority. +- **Q13.2** Should EZ-13 stay on the follow-up list? Trusted Server prefers closing it as documentation and evidence, since spin-sdk 7.0.0 offers no way to implement it. + +#### Interim for Trusted Server + +Trusted Server rejects `publisher.origin_host_header_override` at config validation on Spin builds. Fastly and Axum honor it natively. + +### EZ-14. Idle-time bound for streamed response bodies + +**Priority:** Follow-up +**Adapters:** Fastly, Axum, Cloudflare, Spin. Axum must be Native. The others may be BestEffort where host timers or cancellation cannot be shown to meet the requirement. +**Related:** EZ-2 (exact deadlines on Fastly, which this requirement must not undo for headers), EZ-4 (the connect bound stays separate), EZ-6 (a begun streamed request), EZ-6.27 (batches refuse streamed responses), EZ-9 (only the final response body of a followed chain is read), EZ-5 (the idle bound applies to the stream as delivered, encoded or decoded) + +#### Need + +TS today bounds connection setup and silence, not total stream time. Connect is 1 s (`ts:crates/trusted-server-adapter-fastly/src/backend.rs:240`). First byte and between bytes are 15 s each. The publisher origin sets them at `ts:crates/trusted-server-core/src/publisher.rs:98` and `:4517-4518`. The asset proxy and redirect loop set them at `ts:crates/trusted-server-core/src/proxy.rs:1137-1138` and `:1338-1339`. Integrations take the same 15 s default (`ts:crates/trusted-server-core/src/platform/mod.rs:82`, applied at `ts:crates/trusted-server-core/src/integrations/mod.rs:65` and copied into between bytes at `:157-158`). A streamed page or asset can take as long as it needs while bytes keep arriving. + +On PR 275 a streamed body shares the request's total budget, measured from the start of the send. That budget is the per-call timeout, the deadline, or 30 s by default. Without this requirement: + +- A streamed page, asset, or relayed script that takes longer than the budget is cut mid-body with a gateway timeout, however steadily the origin sends. +- Time the application spends between reads counts too, which includes TS's own rewrite work and a slow client draining the response. So a fast origin feeding a slow client is cut. +- A stalled origin is noticed only when the whole budget runs out. With a budget large enough for long transfers, for example 60 s, a stall ties up the request for 60 s instead of TS's 15 s today. + +TS will ask for a header budget of about 15 s, a connect bound of 1 s (EZ-4), and an idle bound of 15 s on streamed bodies, optionally with an absolute deadline as a hard total. + +#### Gap at F + +- The budget is computed once from the send start (`edgezero:crates/edgezero-core/src/time.rs:138-187`, 30 s default at `:13`). Every body read checks it: + - Fastly checks before and after each read (`edgezero:crates/edgezero-adapter-fastly/src/outbound.rs:1203-1267`). + - Axum checks per read and through `reqwest`'s per-request timeout, which covers the body (`edgezero:crates/edgezero-adapter-axum/src/outbound.rs:270-306`, `:478-484`; `edgezero:docs/superpowers/specs/2026-05-21-outbound-http-design.md:2281`). + - Cloudflare checks per read (`edgezero:crates/edgezero-adapter-cloudflare/src/outbound.rs:439-496`). + - Spin checks per read (`edgezero:crates/edgezero-adapter-spin/src/outbound.rs:829-870`). +- At F, Fastly sets the platform between-bytes timer to the whole budget (`edgezero:crates/edgezero-adapter-fastly/src/outbound.rs:718-733`), and Spin sets every phase timer to the remaining budget (`edgezero:crates/edgezero-adapter-spin/src/outbound.rs:67-74`). +- Viceroy 0.21.0 enforces that between-bytes timer. It measures the gap since the last chunk Viceroy delivered to the guest, not since the last byte the origin sent (`viceroy:src/body.rs:235-260`, reset at `:310`). So a guest that pauses reading longer than the timer can fail its next read even when the origin is healthy. When the timer fires during a body read, F reports a 502 transport error, or a 504 if the budget has also passed (`edgezero:crates/edgezero-adapter-fastly/src/outbound.rs:1219-1228`). Viceroy's first-byte timer covers connect and the response head, and its expiry reaches the application as an internal error (`viceroy:src/upstream.rs:351-375`; `viceroy:src/error.rs:327-329`; `edgezero:crates/edgezero-adapter-fastly/src/outbound.rs:959`). +- No streamed-response option separates the header phase from the body phase (`edgezero:crates/edgezero-core/src/outbound.rs:91-95`, `:1200-1203`). +- EZ-2 makes the total deadline exact on Fastly, and EZ-4 adds a connect bound. Neither releases body reads from the total budget. + +#### Requirements + +- **EZ-14.1** An application MUST be able to ask, per streamed request, that the body be bounded by idle time instead of the request's total budget. The default MUST stay as at F. +- **EZ-14.2** The idle bound MUST apply only to streamed responses. A single send or begun request that asks for it in buffered mode MUST be rejected before sending as an invalid request (400 class). A batch slot that asks for it is refused by the batch's streamed-response rule (EZ-6.27) before the idle bound is considered. A zero idle duration MUST be rejected as an invalid request. A duration beyond EdgeZero's far-future clamp MUST be clamped, as F clamps other durations. +- **EZ-14.3** With an idle bound, the request's budget (per-call timeout, deadline, or default) MUST still bound everything up to the arrival of response headers, exactly as without it. +- **EZ-14.4** After headers arrive, each body read MUST produce data, end, or fail within the idle duration, measured from when that read starts waiting for upstream data. Time the application spends before its next read MUST NOT count toward the idle bound. +- **EZ-14.5** With an idle bound, an application-set absolute deadline MUST still end the body as a hard total. The per-call timeout and the 30 s default MUST NOT end the body. +- **EZ-14.6** An idle expiry MUST surface as a gateway timeout (504 class) whose timeout cause is distinguishable from the per-call timeout, the deadline, and the batch cutoff. A failure the platform reports after the idle duration has passed MUST be reported as that idle timeout, not as a 502 or an internal error. +- **EZ-14.7** The idle bound MUST track upstream data arrival. Decoding (EZ-5) or chunk shaping MUST NOT make a progressing stream look idle, and MUST NOT hide a stall. +- **EZ-14.8** On every adapter, a body that keeps producing data at intervals shorter than the idle duration MUST NOT be ended by any EdgeZero or platform timer. That holds even when the idle duration is longer than the per-call timeout, and even when the application pauses between reads. The two limits that still apply are the application's deadline and any platform execution limit the capability matrix documents, such as Fastly's runtime limit (EZ-2.7). On Fastly, asking for an idle bound no longer than the per-call timeout SHOULD NOT reduce connection reuse to that origin compared with the same request without it (EZ-2). +- **EZ-14.9** An application MUST be able to tell, before sending, whether a request carries an idle bound and its duration. +- **EZ-14.10** Requests that do not ask for an idle bound MUST behave exactly as at F on every adapter, apart from EZ-2's exact deadlines. +- **EZ-14.11** On a begun request (EZ-6), the idle bound MUST govern the body once the application reads it. With redirect following (EZ-9), it MUST govern the final response's body. Intermediate redirect bodies are never delivered. +- **EZ-14.12** The capability matrix MUST list idle-bounded streamed bodies per adapter (cross-cutting requirement 9). Axum is Native. Cloudflare and Spin are BestEffort until cancellation is shown to be bounded. Fastly is BestEffort until hosted Fastly shows that a slow reader does not trigger a platform timeout. The streaming guide MUST say that an idle bound without a deadline places no EdgeZero limit on total stream time, MUST name each adapter's platform execution limit, and MUST recommend a deadline for untrusted origins. + +#### Acceptance criteria + +1. **Unit.** Given an idle bound on a buffered single send, or a zero idle duration, then each request is rejected before sending as an invalid request. Given an idle bound on a batch slot, then the slot is refused by the batch's streamed-response rule. +2. **Axum.** Given an origin that sends headers and then 8 chunks 250 ms apart, a per-call timeout of 500 ms, and an idle bound of 400 ms, then the full body arrives in about 2 s. +3. **Axum.** Given the same setup with one 600 ms gap, then the read fails with the idle timeout about 400 ms after the previous chunk, and the cause differs from a per-call timeout. +4. **Axum.** Given a fast origin and an application that waits 1 s between reads, with a 400 ms idle bound, then no idle error occurs. +5. **Axum.** Given an idle bound and a deadline 1 s after the send, then a body still flowing at 1 s ends with a deadline timeout. +6. **Viceroy 0.21.0, workerd, Spin 4.1.** Run scenarios 2, 3, and 4 against an origin that trickles its body. Viceroy enforces the platform between-bytes timer and measures it from delivery to the guest, so scenario 2 shows that EdgeZero sets no platform timer that ends a progressing body early, and scenario 4 shows that a paused reader is not failed (EZ-14.8). +7. **Hosted Fastly.** Run scenarios 2 to 4 against a real origin. Add an idle bound longer than the per-call timeout, with chunks spaced between the two. No platform timer may end the body early. +8. **Trusted Server, Viceroy 0.21.0 and hosted Fastly.** An origin streams an HTML page over 20 s in chunks 2 s apart, and the client receives the full rewritten document. A 16 s stall mid-body ends the client body at about 15 s, and TS logs an idle timeout. + +#### Out of scope + +- An idle mode for buffered responses. A buffered send returns only after the whole body, so a total budget is the natural contract, and batches rely on it. +- EdgeZero's response write deadline, which the lifecycle requirements own. + +#### Open questions + +None. + +#### Interim for Trusted Server (follow-ups only) + +TS sets an explicit per-call timeout on each streamed site, sized for its longest expected transfer, together with EZ-4's 1 s connect bound. It raises its response write deadline through `App::set_response_egress_policy` (`edgezero:crates/edgezero-core/src/app.rs:466-476`) to match. TS accepts two costs until this lands. A mid-body stall is noticed only when that timeout expires, and a transfer longer than the timeout is cut. + +### EZ-15. Explicit fire-and-forget send + +**Priority:** Follow-up +**Adapters:** Axum, Cloudflare, Fastly, Spin. Lifetime after the callback returns may differ per platform, as EZ-15.11 says. +**Related:** EZ-6 (dropping a begun request cancels it, so fire-and-forget must be explicit; EZ-6.1 defines "on its way"), EZ-1 (a bodiless GET/HEAD built from an inbound request is accepted), EZ-2 (Fastly backend capacity), EZ-3 (EZ-3.3 owns the passed-deadline rule), EZ-8 (Image Optimizer is Unsupported on this send mode), EZ-16 (provider detail in logs) + +#### Need + +TS's Fastly Tinybird telemetry sends one NDJSON POST per auction and never waits for the answer (`ts:crates/trusted-server-adapter-fastly/src/tinybird.rs:133-147`). The backend has 2 s first-byte and between-bytes timers (`:19-20`). The request carries a bearer token (`:109-115`). TS awaits the sink (`ts:crates/trusted-server-core/src/auction/telemetry.rs:526-542`). Calls sit inside the `` hold and the non-HTML collection, before the page tail is released (`ts:crates/trusted-server-core/src/publisher.rs:4283`, `:4227`). Others run on the dispatch-failure, skip, and abandon paths (`:4177`, `:4979`, `:4995`, `:5025`). Only Fastly has a sink today. The no-op sink (`ts:crates/trusted-server-core/src/auction/telemetry.rs:500`) is the default on the other adapters (`ts:crates/trusted-server-core/src/platform/types.rs:473`). + +On PR 275's APIs, TS wants one telemetry sink on every adapter that sends the POST and returns at once, wherever the row is produced. Without this requirement, a normal `send` before delivery holds the page tail for the Tinybird round trip. Dropping a begun request is a cancellation (cross-cutting requirement 6), so it cannot stand in for fire-and-forget. + +#### Gap at F + +- `HttpClient` has no fire-and-forget operation (`edgezero:crates/edgezero-core/src/outbound.rs:567-615`). +- Fastly `send` waits for the response head (`edgezero:crates/edgezero-adapter-fastly/src/outbound.rs:450-453`). On Axum, Cloudflare, and Spin a dropped, unpolled batch sends nothing (`edgezero:crates/edgezero-adapter-axum/src/outbound.rs:200-206`; `edgezero:crates/edgezero-adapter-cloudflare/src/outbound.rs:389-395`; `edgezero:crates/edgezero-adapter-spin/src/outbound.rs:434-440`). +- EdgeZero's guide warns that request cancellation does not cancel application-spawned background work (`edgezero:docs/guide/adapters/overview.md:252-255`), and EdgeZero offers no bounded alternative. + +#### Requirements + +- **EZ-15.1** An application MUST be able to send a request whose response no one observes. The call MUST return without waiting for the response, and when it returns the request MUST be on its way (EZ-6.1). +- **EZ-15.2** The call MUST reject, before sending and with an immediate error, a streamed request body that would be uploaded, and Image Optimizer options (Unsupported on this send mode, rejected as cross-cutting requirement 2 says, before any other preflight check). A GET or HEAD body that EZ-1 settles as bodiless is accepted. A request whose deadline has already passed fails as EZ-3.3 and EZ-6.26 say. +- **EZ-15.3** The request's budget MUST follow the same rules as `send`, with a 30 s default when the request sets no timeout or deadline. Each adapter MUST cap it at a documented platform lifetime, and the outcome log MUST say when that cap ended the request. +- **EZ-15.4** The request MUST keep running after the application's per-request lifecycle callback returns, until it finishes or its budget ends, wherever the platform allows. This is the lifetime cross-cutting requirement 4 leaves to EZ-15. +- **EZ-15.5** EdgeZero MUST NOT hold more than 64 KiB of the response. It reads and discards at most that much, then releases the response. +- **EZ-15.6** EdgeZero MUST log each outcome, success at `debug` and failure at `warn`, with the target host, the method, the error category, and EdgeZero's own gateway reason or timeout source. EZ-16 provider detail follows EZ-16.7. The log MUST NOT contain the path, the query, any header value, or body bytes. +- **EZ-15.7** In-flight fire-and-forget sends MUST be bounded, by default to 16 per request lifetime and 64 per sandbox or server instance. A call over either bound MUST fail immediately with a service-unavailable error (503 class) and send nothing. The application MUST be able to change both bounds. +- **EZ-15.8** The call MUST work inside a streamed response body that EdgeZero is writing, and after delivery inside the same lifecycle callback. +- **EZ-15.9** Follow-up within EZ-15, required before TS enables reusable sandboxes. On a reused Fastly sandbox, fire-and-forget sends MUST NOT accumulate beyond the per-instance bound, and finished ones MUST stop counting against it. +- **EZ-15.10** EdgeZero's documentation MUST say that dropping a begun request or a batch cancels it and that fire-and-forget uses this capability. +- **EZ-15.11** Per-adapter levels. Axum MUST be Native, and server shutdown MUST stop outstanding sends without waiting for them. Cloudflare, Fastly, and Spin MAY be BestEffort, with the capability matrix stating each platform's lifetime after the callback returns. + +#### Acceptance criteria + +1. Unit. A fire-and-forget POST with a streamed body fails immediately and sends nothing. A fire-and-forget GET built from a bodiless inbound GET is sent. With a per-request bound of 2, the third call fails with the service-unavailable error and only two requests go out. +2. Axum. Given a handler that makes a fire-and-forget call to an origin that answers after 200 ms and then returns 200, the client gets its response in under 50 ms, and the origin receives the request within 1 s. Given a server shutdown with one such send in flight, shutdown returns within 1 s. +3. workerd, Spin 4.1, and Viceroy 0.21.0. Given a handler that makes a fire-and-forget call to an origin that holds the request for 500 ms before answering, and then returns, the client's response completes before the origin answers, and the origin still receives the whole request. The Spin result sets the Spin capability level. +4. Viceroy 0.21.0 and hosted Fastly. Under Viceroy, given a reused guest serving 6 requests, each making one fire-and-forget call to an origin that answers after 1 s, every call reaches the origin and the sixth request still sends. On hosted Fastly, given 500 sequential requests, each making one such call, every call reaches the origin. +5. Axum, workerd, Spin 4.1, and hosted Fastly. A fire-and-forget call to a refused port logs one `warn` line with the host and an unreachable-category error. On Viceroy 0.21.0 the line carries the category Viceroy produces, and the run records it. No line contains a path, query, or header value. + +The Trusted Server end-to-end check runs on the Axum binary, workerd, Spin 4.1, and Fastly staging. It points Tinybird at a mock that answers after 1 s. An auctioned page on each adapter finishes its tail within 50 ms of the auction finishing, and the mock receives one NDJSON POST per auction. + +#### Out of scope + +- A completion callback. TS needs only the log line. +- Running arbitrary application work after the callback returns. Only outbound requests are covered, within the budget and bounds above. + +#### Open questions + +1. Should the Fastly part be Blocking? The Fastly part is EZ-15.1 to EZ-15.8 and EZ-15.10 as they apply on Fastly, plus the Fastly clause of EZ-15.11. EZ-15.9 stays tied to reusable sandboxes. TS prefers that it stay Follow-up. Its interim sends the telemetry row with an ordinary `send` after delivery on Fastly, which holds nothing the client waits on. Promotion matters only if telemetry must leave before delivery. +2. Are 16 per request lifetime and 64 per instance the right defaults? TS prefers them. TS sends a few telemetry rows per request, and the per-instance bound limits Fastly sandboxes and Axum servers. +3. Is 64 KiB the right response read limit? TS prefers it. Tinybird answers with a small JSON document. + +#### Interim for Trusted Server + +Until this lands, TS on Fastly carries the auction telemetry row out of the `` hold and sends it with an ordinary `HttpClient::send` after delivery, inside its serving callback, as it does pull sync (cross-cutting requirement 4). On Axum, Cloudflare, and Spin TS keeps the no-op sink they use today. TS neither holds the page tail nor relies on dropping a begun request. + +### EZ-16. Provider failure detail available to logs + +**Priority:** Follow-up +**Adapters:** Fastly, Axum, Cloudflare, Spin. Fastly and Spin MUST be Native. Axum and Cloudflare may be BestEffort, because their providers expose less. +**Related:** EZ-17, EZ-21, EZ-10 (their not-sent source, framing reason, and payload limit reason should be read the same way once EZ-16 has landed), EZ-14 (an idle timeout can carry the same detail), cross-cutting requirement 7 (diagnostics stay off the wire) + +#### Need + +TS today logs the provider's own cause. Its Fastly client keeps the SDK's send error as the source of its error report on `send` (`ts:crates/trusted-server-adapter-fastly/src/platform.rs:520-523`, `:540-543`). On the select path that auction providers take, it formats the SDK error into the report's text (`:606-614`). TS logs the full report on auction provider failures (`ts:crates/trusted-server-core/src/auction/orchestrator.rs:1422-1446`) and on every request error (`ts:crates/trusted-server-adapter-fastly/src/app.rs:1003-1005`). The SDK reports the specific cause, for example a DNS error with its rcode and info code, a TLS alert with its id, an HTTP/2 stream error with frame type and error code, a refused connection, or an I/O error (`fastly-sdk:src/http/request.rs:3215-3349`). + +On PR 275, TS can log only EdgeZero's category. Operators can no longer tell an NXDOMAIN from a refused connection, an expired certificate, or a TLS alert, because all four become one "destination could not be reached" category. Timeouts and transport failures still classify correctly, which is why this is a follow-up. + +#### Gap at F + +- Fastly reduces every SDK cause to eight failure kinds with six distinct messages, and it discards body-read, upload, and backend-registration error detail (`edgezero:crates/edgezero-adapter-fastly/src/outbound.rs:40-49`, `:68-99`, `:877-899`, `:929-966`, `:1189-1201`, `:1219`). +- Axum keeps only `reqwest`'s yes/no predicates (`edgezero:crates/edgezero-adapter-axum/src/outbound.rs:413-432`, `:457-464`). +- Cloudflare replaces a rejected fetch with one generic failure (`edgezero:crates/edgezero-adapter-cloudflare/src/outbound.rs:648`, `:1488-1493`). +- Spin drops every error payload (`edgezero:crates/edgezero-adapter-spin/src/outbound.rs:1342-1410`). +- `EdgeError` carries no detail for these failures (`edgezero:crates/edgezero-core/src/error.rs:132-192`). The wire is category-only for gateway, timeout, limit, and internal errors (`error.rs:555-575`, `:599-650`). EdgeZero's design forbids putting provider text into outbound diagnostics (`edgezero:docs/superpowers/specs/2026-05-21-outbound-http-design.md:3036-3040`). +- Provider text really does carry request data. `reqwest`'s error `Display` appends ` for url ()` (`reqwest:src/error.rs:279-281`). Spin's TLS alert payload carries a free-text `alert-message`, and its DNS payload carries the rcode as text (`spin-sdk:wit/deps/http@0.3.0/types.wit:84-94`). +- Viceroy 0.21.0 ignores send error detail on the pending-request wait (`viceroy:src/wiggle_abi/req_impl.rs:1141-1149`). Under Viceroy a refused connection reaches F as an internal error with no cause (`edgezero:crates/edgezero-adapter-fastly/src/outbound.rs:95-97`, `:959`). + +#### Requirements + +- **EZ-16.1** When EdgeZero reports an outbound failure that came from the provider, the application MUST be able to read structured detail about it from the error, without parsing any text. That covers backend registration, sending, waiting for headers, request-body upload, and response-body reads. This is a MUST on Fastly and Spin. On Axum and Cloudflare it holds as far as the provider exposes the cause. +- **EZ-16.2** The detail MUST name the adapter, the phase in which the failure occurred, and the provider's cause from a closed set that EdgeZero documents per adapter. It MUST include the provider's numeric codes where the provider supplies them. These are DNS rcode and info code, TLS alert id, HTTP/2 frame type and error code, platform status, and I/O error kind. Where the provider supplies a code only as text, such as Spin's DNS rcode, the detail MUST carry it only when it matches a documented closed set of values, and otherwise MUST record it as unknown. +- **EZ-16.3** The detail MUST NOT contain provider-supplied text, URLs, header values, or body bytes. That includes Spin's TLS `alert-message` and `reqwest`'s formatted error. +- **EZ-16.4** Cross-cutting requirement 7 applies to the detail. Error responses that EdgeZero renders MUST be byte-identical with and without it. +- **EZ-16.5** Existing error text MUST NOT change. That covers each error's `Display`, `message()`, `kind()`, `status()`, and its existing reason or cause, for every variant, including `EdgeError::Internal`. +- **EZ-16.6** The detail MUST stay with the error wherever F lets the error travel. That includes body-stream errors after decoding and limits, batch slot results and batch failures, begun requests, redirect hops, and, once EZ-15 lands, fire-and-forget sends. It also covers an error the request context reports again later. +- **EZ-16.7** EdgeZero MUST NOT log the detail itself above debug level. Log policy and request correlation belong to the application. +- **EZ-16.8** Once EZ-16 has landed, the not-sent source (EZ-17), the framing reason (EZ-21), and the payload limit reason (EZ-10) SHOULD be readable from an error in the same way as this detail. +- **EZ-16.9** The capability matrix MUST list failure diagnostics per adapter (cross-cutting requirement 9). Fastly and Spin are Native. Axum and Cloudflare are BestEffort. + +#### Acceptance criteria + +1. **Unit.** Given a gateway, timeout, and internal error, each with detail attached, then rendering each as a response produces the same bytes as without detail. Its `Display` is byte-identical. Its status, kind, and reason are unchanged. +2. **Unit.** Given a body stream that fails with detail, when the error passes through decoding, limits, chunk shaping, and buffered collection, then the final error still carries the detail. +3. **Hosted Fastly.** Given a request to a closed port on a reachable host, then the error is a gateway error with reason `Unreachable`, and its detail names Fastly, the send phase, and the SDK's connection-refused cause. +4. **Viceroy 0.21.0.** Given the same request, then the error is an internal error, because Viceroy ignores send error detail. Its detail names Fastly, the send phase, and an internal cause. +5. **Spin 4.1.** Given a TLS alert with an id and a message, then the detail carries the id and not the message. Given a DNS failure whose rcode text is outside the documented set, then the detail records the rcode as unknown. Given an internal error that carries text, then no text appears in the detail. +6. **Axum.** Given a refused connection to a URL that contains `?token=secret`, then the detail names a connect failure and the refused I/O kind, and `secret` appears in neither the detail nor the rendered error. +7. **workerd.** Given a rejected fetch, then the detail names Cloudflare and the send phase. Its cause is the exception kind only when that kind is in the documented set, and otherwise it is unknown. +8. **Trusted Server, hosted Fastly staging.** Given an auction with one provider pointed at a closed port on a reachable host, then TS logs one line with the provider name and the refused cause, and the client-facing auction error body stays static. + +#### Out of scope + +- Changing any error category, status, or wire message. +- Exposing provider free text in any form. + +#### Open questions + +1. Should Cloudflare report the JavaScript exception kind (for example abort versus type error) when it is in a documented allowlist? Trusted Server's preferred answer is yes. That one distinction lets operators separate EdgeZero's own aborts from network failures. + +#### Interim for Trusted Server (follow-ups only) + +TS logs EdgeZero's category, the gateway reason, and the timeout cause for each failure. It accepts losing the DNS, TLS, and refused distinctions until this lands. + +### EZ-17. Fastly send outcome tells the application what was committed + +**Priority:** Follow-up +**Adapters:** Fastly. Axum, Cloudflare and Spin expose no request-level send helper with preparation and finalization hooks, so the requirement does not apply to them. +**Related:** EZ-18 (decides which body is delivered for buffered responses). EZ-19 (the fallback the report names carries the application's headers). EZ-20 (removes declared-KV open failure as a "no response attempted" cause). EZ-21 (the report carries the framing reason). EZ-16 (cause detail is read the same way once it lands). EZ-2 and EZ-22 (change standard entry-point serving behavior; this requirement does not). EZ-2.14 (refers to EZ-17.7 for the helpers' inputs). + +#### Need + +TS's Fastly entry point sends every response itself and never fails after sending. A post-commit streaming failure is logged and the stream dropped (`ts:crates/trusted-server-adapter-fastly/src/main.rs:668-724`, failure branch `:710-717`). Partner pull sync then runs for real browsers whatever happened to the stream (`ts:crates/trusted-server-adapter-fastly/src/main.rs:512-517`, `:536-541`, `:618-643`). When request conversion fails, TS sends a plain 500 itself (`ts:crates/trusted-server-adapter-fastly/src/main.rs:472-477`). In reuse mode, which is off in production (`ts:crates/trusted-server-adapter-fastly/Cargo.toml:13-18`), the sandbox keeps serving after a post-commit failure, because the callback returns `()` (`ts:crates/trusted-server-adapter-fastly/src/main.rs:131-142`). + +On PR 275 TS calls `send_request_with_registries_and_hooks` from inside `serve_custom` or `run_custom`, and runs pull sync after it returns, inside the same callback, with the state `finalize` returned. Without this requirement: + +1. A client that disconnects mid-page also cancels partner pull sync, although its EC `Set-Cookie` was already committed in the head. +2. TS cannot answer a request EdgeZero never attempted, because the same error also means "delivery failed after commitment". If TS sends nothing, the client gets a host-produced response. In reuse mode the sandbox also stops at the next wait (`fastly-sdk:src/http/serve.rs:357-364`, `:616-619`). +3. In reuse mode, TS cannot keep serving after a post-commit failure while stopping after a pre-send fault, short of matching error strings. +4. When framing fails after `finalize` added the EC cookie, EdgeZero delivers its 500 fallback and reports success, so TS would run pull sync for a cookie the browser never received. + +#### Gap at F + +- The helper returns `Result, fastly::Error>` (`edgezero:crates/edgezero-adapter-fastly/src/request.rs:557-586`). A declared-KV open failure and a head-conversion failure return `Err` before any response attempt (`:570`, `:632-652`, `:445-447`, `:904-906`). An admission abort returns `Err("ingress admission aborted")` (`:383`). +- A routed response runs `finalize`, then delivery, and a delivery error drops the state (`edgezero:crates/edgezero-adapter-fastly/src/request.rs:388-391`). +- Delivery errors collapse to four fixed messages with no commit information (`edgezero:crates/edgezero-adapter-fastly/src/response.rs:115-122`, `:252-257`, `:281-286`). A failure before the head reaches the host is reported as post-commit in one path (`:397-406`), and a host rejection of the head is treated as post-commit with no fallback (`:408-419`; test `:1547-1578`). +- A delivered fallback returns success (`edgezero:crates/edgezero-adapter-fastly/src/response.rs:281-284`, `:335`), although its terminal report says the fallback was delivered (`edgezero:crates/edgezero-core/src/response_egress.rs:846-867`). +- Detached responses run without the application's response-egress observer (`edgezero:crates/edgezero-core/src/app.rs:178-181`; `edgezero:crates/edgezero-core/src/response_egress.rs:661-676`). Only the admitted source-error path uses it (`edgezero:crates/edgezero-core/src/app.rs:86-102`, `:346-361`). An admission-policy error and an unsupported admission outcome also become detached responses (`edgezero:crates/edgezero-adapter-fastly/src/request.rs:486-495`). +- EdgeZero's own fixture labels every error as post-commit, including a declared-KV failure that sent nothing (`edgezero:tests/fixtures/reusable-app/crates/fixture-fastly/src/lib.rs:232-237`). The Fastly guide warns that an escaping error "is not proof that any response was sent" (`edgezero:docs/guide/adapters/fastly.md:444-446`, `:513-515`). + +#### Requirements + +- **EZ-17.1** After `send_request_with_registries_and_hooks`, `send_with_registries_and_hooks` or `FastlyService::send_request_with_hooks` returns, the application MUST be able to tell which of four things happened. No response was attempted. The application's admission or detached-egress policy chose to abort. A detached response was attempted, for a head limit, a framing violation, an admission refusal, an admission-policy error, or a source error. A routed response was attempted. +- **EZ-17.2** For a routed response, the value `finalize` returned MUST reach the caller in every case, including a delivery failure after commitment and a delivered fallback. +- **EZ-17.3** For every attempted response, the caller MUST learn the commit state of the body EdgeZero last tried to send. That state is one of not committed (the head never reached the host), unknown (the host rejected the head, and whether the client saw anything is unknown), or committed (the host accepted the head). The caller MUST also learn which body that was, the application's or EdgeZero's fallback, and the terminal outcome. The terminal outcome MUST equal the `ResponseEgressReport` that the request's `ResponseEgressCompletion` received for the same attempt and, for routed responses and admitted source errors, the report the application's observer received. Detached responses keep F's rule that the application's observer is not invoked. +- **EZ-17.4** "No response was attempted" MUST be reported if and only if EdgeZero started no response attempt, invoked no observer or completion, and sent nothing. The caller MAY then send its own native response, and in reuse mode the sandbox MUST be able to keep serving afterwards. +- **EZ-17.5** After an attempted response that was not committed, the caller MAY send its own native response. EdgeZero MUST document that the attempt's terminal report has already reached the completion, and the observer where EZ-17.3 says it does, and that the native response is not accounted by EdgeZero. After an unknown or committed outcome, EdgeZero MUST document that the caller must not send another response. +- **EZ-17.6** Two "no response attempted" causes MUST be distinguishable as categories. One is an explicitly required store that could not be opened. The other is a request head that could not be converted. The human-readable text MUST be fixed and contain no URI, header value or store name. Cause detail MUST be readable by the application without parsing text and MUST NOT reach the wire (cross-cutting requirement 7). Where EZ-16 has landed, it SHOULD be read the same way as EZ-16's detail. +- **EZ-17.7** `prepare` and `finalize` closures written against F MUST keep working with the helpers. Only what the helpers report changes. +- **EZ-17.8** `run_app`, `run_app_with_hooks`, `serve_app`, `serve_app_with_hooks` and `FastlyService::send` MUST keep F's serving behavior for every scenario in EZ-17.1, except where EZ-2 (sandbox retirement) and EZ-22 (initialization failure) change it. +- **EZ-17.9** Applications MUST be able to exercise every outcome in EZ-17.1 and EZ-17.3 in their own tests without a live host, through EdgeZero's `test-utils` lifecycle support at F (`FastlyService::run_request_with_hooks_for_test`, `edgezero:crates/edgezero-adapter-fastly/src/request.rs:231-249`) or its successor. +- **EZ-17.10** The Fastly guide's custom-lifecycle section and its complete example MUST describe the outcomes, the send permissions in EZ-17.4 and EZ-17.5, and a reuse-mode serving policy that keeps serving after post-commit failures. + +#### Acceptance criteria + +1. Viceroy 0.21.0. Given a routed handler whose body stream fails after its first chunk, and a `finalize` that returns a state value carrying the request's outbound client, when the request is delivered, then the client sees a partial body, and the caller learns routed, application body, committed, source error, and receives the state. An outbound request the callback then makes with that client completes before the callback returns (cross-cutting requirement 4). +2. Viceroy 0.21.0. Given a routed response that `finalize` gives a `Trailer` field, when it is delivered, then the client receives the 500 fallback, and the caller learns routed, fallback body, committed, fallback completed, and receives the `finalize` value. +3. Viceroy 0.21.0. Given an admission policy that refuses, the caller learns detached, application body, committed, `Completed`, and `finalize` never runs. Given a policy that aborts, the caller learns aborted, no response is sent, and `finalize` never runs. +4. Unit. For each failure point (deadline before the head is prepared, head preparation rejected, deadline before commit, host rejects the head, failure after the host accepted the head, fallback cannot start, fallback rejected by the host), the reported commit state, body and outcome match EZ-17.3, and the completion saw exactly one report equal to the reported one. +5. Viceroy 0.21.0. Given `FastlyService::new(&app).with_kv(name).require_kv()` on a store Viceroy does not link, the caller learns no response attempted, store category, sends a native 500, and the client receives it. +6. Unit and Viceroy 0.21.0. The standard entry points produce F's serving results for the scenarios above, apart from EZ-2 and EZ-22. +7. Trusted Server, Viceroy 0.21.0. Pull sync reaches its partner stub after a mid-body origin abort. No pull-sync call is made when a framing failure delivers the fallback. A request-conversion failure gets TS's own 500, if Viceroy can deliver such a request. + +#### Out of scope + +- Changing the standard entry points' policy of ending the sandbox on post-commit failures. +- Supplying or retaining the outbound client (EZ-2). +- Any response-returning or whole-body compatibility path. + +#### Open questions + +1. Should the standard entry points keep serving after a post-commit failure now that the cause is known? TS's preferred answer is no change in this requirement. TS uses the custom path and sets its own policy. +2. Should "unknown" be folded into "committed"? TS's preferred answer is no. Production behavior after a host rejection of the head is unverified, and folding it in would let a caller double-send or wrongly skip a send. +3. Should only EZ-17.2 be promoted to Blocking? TS's preferred answer is no. It changes the same report, so a partial version would be an API TS migrates twice. If TS needs pull sync after disconnects at the cut, promote all of EZ-17. + +#### Interim for Trusted Server + +TS logs every helper error and returns `Ok(())` from its callback. In reuse mode the sandbox then keeps serving after post-commit failures and stops at the next wait when nothing was sent. TS skips pull sync when the helper returns an error and runs it when the helper reports success. It accepts that pull sync may run after a delivered fallback, and that a head-conversion failure gets the host's response instead of TS's 500. + +### EZ-18. `Content-Length` on buffered responses + +**Priority:** Follow-up. TS's preferred answer on promotion is in open question 1. +**Adapters:** all four. Axum already meets EZ-18.1 at F. Fastly has the added parity requirement EZ-18.4. +**Related:** EZ-5 (F already sends an application- or upstream-supplied length, which EZ-18.2 and EZ-18.3 keep, so EZ-5 does not wait for EZ-18). EZ-17 (reports which body was delivered). EZ-19 (decorated responses get the same framing). EZ-Q1 (its criteria join the record when EZ-18 is present at the pinned commit). + +#### Need + +TS sends every buffered Fastly response with the SDK's `Response::send_to_client` (`ts:crates/trusted-server-adapter-fastly/src/main.rs:720-722`, after `ts:crates/trusted-server-adapter-fastly/src/compat.rs:13-43`). In the SDK's default Automatic framing mode, "a `Content-Length` is used when the size of the body can be determined before it is sent" (`fastly-shared:src/lib.rs:271-272`), and `send_to_client` sends the body as one unit (`fastly-sdk:src/http/response.rs:1735-1738`, `:1816-1821`). Under Viceroy that yields an exact-size body (`viceroy:src/wiggle_abi/resp_impl.rs:164-168`; `viceroy:src/body.rs:54-70`, `:412-427`), which hyper frames with `content-length` (`hyper-0.14.26:src/proto/h1/role.rs:840-883`). So today TS's auction JSON (`ts:crates/trusted-server-core/src/auction/formats.rs:516-527`), its 413 bodies (`ts:crates/trusted-server-core/src/auction/endpoints.rs:136-158`), discovery JSON, error pages and tsjs bundles carry a fixed length. HEAD and 304 responses go out the same way. + +TS's cut spec keeps "same response status and headers" on Fastly (`ts:docs/superpowers/specs/2026-10-17-edgezero-pr275-hard-cutover-design.md:49-52`), and its Fastly response-diff gate allows only the listed Fastly differences (`:712-719`). + +The tsjs bundles set no `Content-Length` (`ts:crates/trusted-server-core/src/http_util.rs:330-336`) and ask the Fastly host to compress them with `x-compress-hint: on` (`ts:crates/trusted-server-core/src/publisher.rs:597`). They are not at risk at F, where they go out chunked in Automatic mode. The hint matters only as a constraint on how buffered framing is restored (EZ-18.5). + +The only reason left to promote the buffered part is header parity. TS knows of no functional breakage on its path from chunked buffered responses. Without EZ-18, TS lists "buffered responses lose `Content-Length` on Fastly" as an intended difference and normalizes it in its response diff. + +#### Gap at F + +- Core never derives a length from a buffered body. It only canonicalizes a length the application set (`edgezero:crates/edgezero-core/src/response_egress_framing.rs:80`, `:115-138`). +- Fastly commits the head (`edgezero:crates/edgezero-adapter-fastly/src/response.rs:408`) before it writes any body bytes (`:434`, `:486`), using the streaming send (`edgezero:crates/edgezero-adapter-fastly/src/fastly_response_abi.rs:64-70`). It selects manual framing only when the body is transmitted and a `Content-Length` header is already present (`edgezero:crates/edgezero-adapter-fastly/src/response.rs:159-161`), so HEAD, 204, 205 and 304 stay in Automatic mode even with a length. In Automatic mode a streamed response "will receive a `Transfer-Encoding: chunked`" (`fastly-shared:src/lib.rs:272-274`). Under Viceroy the body size is unknown, the guest's framing headers are stripped, and hyper sends `transfer-encoding: chunked` (`viceroy:src/body.rs:412-417`; `viceroy:src/http/framing.rs:68-70`; `viceroy:src/http/filtering.rs:3-8`; `hyper-0.14.26:src/proto/h1/role.rs:840-854`). +- EdgeZero's 500 and 504 fallbacks are buffered bodies without a length, so they are chunked too (`edgezero:crates/edgezero-adapter-fastly/src/response.rs:342-351`). +- HEAD and 304 at F use the same commit as other responses, while TS uses `send_to_client`. Under Viceroy hyper writes the same head for both (no `content-length` for an empty HEAD or 304, `hyper-0.14.26:src/proto/h1/role.rs:476-503`, `:840-892`), so only hosted Fastly can show whether the two differ. hyper does write a length for a HEAD whose body has a known non-zero size (`hyper-0.14.26:src/proto/h1/role.rs:490-499`, `:869-881`). +- A write deadline is checked before the head is prepared and before commit (`edgezero:crates/edgezero-adapter-fastly/src/response.rs:364`, `:389`). The only window in which a deadline truncates a buffered response is between commit and the body write (`:423-431`). +- In manual framing mode a `Content-Length` that disagrees with the body truncates it or hangs up the connection (`fastly-shared:src/lib.rs:285-287`). How manual framing interacts with host compression is unverified. +- Cloudflare sends a buffered body without a declared length as an unsized stream (`edgezero:crates/edgezero-adapter-cloudflare/src/response.rs:518-530`; test `:1050-1090`). Spin forwards headers and streams the body (`edgezero:crates/edgezero-adapter-spin/src/response.rs:367-389`). Axum already sends the length (`edgezero:crates/edgezero-adapter-axum/src/response.rs:278-299`). +- The streaming guide says EdgeZero "does not guarantee a particular HTTP/1 transfer coding" (`edgezero:docs/guide/streaming.md:167-169`). + +#### Requirements + +- **EZ-18.1** On every adapter, a transmitted response whose body is fully buffered MUST reach the client with `Content-Length` equal to the body's byte length, including `0`, and without `Transfer-Encoding`, on HTTP/1.1. On HTTP/2 it MUST carry `content-length`. This applies to application responses, rendered error responses, detached responses and EdgeZero's fallbacks. +- **EZ-18.2** An application `Content-Length` that matches a buffered body MUST be kept. A mismatch MUST keep F's behavior, a 500 fallback before commit. +- **EZ-18.3** Streamed bodies keep F's behavior. Without a declared length they use the provider's transfer framing. With a declared length, that length is sent and enforced, and a stream that runs long or short is aborted. +- **EZ-18.4** On Fastly, the framing the client receives for buffered responses and for bodiless responses (HEAD, 204, 205, 304) MUST match what the Fastly SDK's `Response::send_to_client` produces for the same status, headers and body, on HTTP/1.1 and HTTP/2. That is TS's pre-cut behavior, so the HEAD and 304 `Content-Length` behavior is whatever the host does for TS today. +- **EZ-18.5** When a response carries `x-compress-hint` and the Fastly host compresses it, the client MUST receive valid framing and a body that decompresses to the original bytes. EdgeZero MUST NOT cause the host to send a length that disagrees with the compressed body. This holds for buffered responses and for streamed responses with a declared length. +- **EZ-18.6** Except where EZ-18.4 requires otherwise on Fastly, EdgeZero MUST NOT add a length to a bodiless response (HEAD, 204, 205, 304) that the application did not supply. On Fastly, a HEAD whose application body has a known size gets the framing EZ-18.4's reference produces for that body. An application-supplied length on HEAD or 304 is passed to the host as at F. +- **EZ-18.7** On Fastly, a write deadline that expires after the head is committed and before a buffered body has been handed to the host SHOULD NOT truncate the response. The head and body SHOULD reach the host together, so an expiry before that point produces the 504 fallback. +- **EZ-18.8** The capability matrix MUST gain a row for buffered-response `Content-Length`. Axum is Native. Fastly is BestEffort until criterion 3 passes on hosted Fastly over HTTP/1.1 and HTTP/2, then Native. Cloudflare is BestEffort until a deployed Workers check shows the header reaching the client. Spin is BestEffort until a Spin 4.1 check shows it. `docs/guide/streaming.md` MUST state that buffered bodies carry `Content-Length`. + +#### Acceptance criteria + +1. Viceroy 0.21.0, HTTP/1.1. Given a buffered 200 route, a streamed route without a length, a rendered 404, and a routed response with a `Trailer` field, then the buffered route, the 404 and the 500 fallback (21 bytes, `internal server error`) carry `content-length` and no `transfer-encoding`, and the streamed route carries `transfer-encoding: chunked`. +2. Viceroy 0.21.0, HTTP/2 with prior knowledge, which Viceroy's hyper accepts by default (`hyper-0.14.26:src/server/conn.rs:950-955`). A buffered route carries `content-length`. +3. Hosted Fastly, HTTP/1.1 and HTTP/2, on TS's staging service. For a buffered 200, a buffered 200 with `x-compress-hint: on` requested with `Accept-Encoding: gzip`, HEAD and conditional-GET 304 of a buffered route, and a 204, the framing equals that of TS's pre-cut build. The compressed body decompresses to the expected SHA-256. +4. Viceroy 0.21.0. Given a buffered response whose response-owned `ResponseEgressDeadline` is already in the past, the client receives the 504 fallback (32 bytes, `response write deadline exceeded`) with `content-length: 32`. +5. workerd. A buffered route carries `content-length`. Spin 4.1. The same. Axum. Unchanged from F. EdgeZero records a passing run of each for the pinned commit. Its CI runs only the Fastly fixture smoke at F (`edgezero:.github/workflows/test.yml:399-401`). +6. Trusted Server, Viceroy 0.21.0 and hosted Fastly. Auction JSON, discovery JSON, a tsjs bundle and an error page match the pre-cut build's `content-length` and `transfer-encoding`. Publisher HEAD and 304 match the pre-cut build. + +#### Out of scope + +- Manual framing for HEAD and 304 beyond the parity in EZ-18.4. +- Any change to how streamed bodies are framed. +- Guessing a length for a streamed body. + +#### Open questions + +1. Promote the buffered part to Blocking? TS's preferred answer is Follow-up with the interim below. The only reason to promote is header parity and the response-diff gate. No functional breakage is known. If the TS maintainer holds the parity goal for the cut, promote EZ-18.1 and EZ-18.4. +2. Should the HEAD and 304 parity in EZ-18.4 be a MUST without hosted evidence? TS's preferred answer is yes. It is TS's current wire behavior, and criterion 3 checks it on hosted Fastly. + +#### Interim for Trusted Server + +TS lists "buffered responses lose `Content-Length` on Fastly, chunked on HTTP/1.1" as an intended Fastly difference in its cut spec, and its response diff normalizes `content-length` and `transfer-encoding` for buffered routes. + +### EZ-19. Application headers on detached and egress-fallback responses + +**Priority:** Follow-up +**Adapters:** all four, Native. The per-request form (EZ-19.2) needs a hook that runs before ingress checks, which only Fastly's canonical path has (`prepare`). +**Related:** EZ-17 (reports when these responses were delivered). EZ-18 (they still carry a length). EZ-21 (a set that breaks framing is diagnosable). EZ-22 (an initialization-failure response has no configured set). + +#### Need + +Every Fastly response TS sends today goes through TS's finalize steps, because TS sends every response itself (`ts:crates/trusted-server-adapter-fastly/src/main.rs:486-506`, `:557`, `:668-689`). Those steps add geo headers or `X-Geo-Info-Available: false`, `X-TS-Version`, `X-TS-Fastly-Version`, `X-TS-ENV: staging`, cookie cache privacy, and the operator's `response_headers` (`ts:crates/trusted-server-adapter-fastly/src/middleware.rs:187-237`), plus sandbox counter headers on `private, no-store` responses (`ts:crates/trusted-server-adapter-fastly/src/main.rs:238-269`). + +On PR 275's canonical path, responses EdgeZero synthesizes skip `finalize`. These are 414 (absolute URL over 8192 bytes), 431 (over 100 header values or 64 KiB of headers), 400 framing rejections (`edgezero:crates/edgezero-core/src/ingress.rs:21-23`), admitted source errors, and the 500 and 504 egress fallbacks. A 500 caused by a bad operator header then carries no `X-TS-Version`, so an operator cannot tell which build produced it, and 414 and 431 lack the operator's headers. TS's spec accepted this on Axum, Cloudflare and Spin (`ts:docs/superpowers/specs/2026-10-17-edgezero-pr275-hard-cutover-design.md:408-411`) before Fastly moved onto the canonical lifecycle. + +#### Gap at F + +- `finalize` runs for routed responses only (`edgezero:crates/edgezero-adapter-fastly/src/request.rs:384-387`, `:475-477`, `:481-483`, `:498-502`). +- `App::set_error_response_renderer` receives only the error, so it cannot add per-request values such as geo, and egress fallbacks bypass it (`edgezero:crates/edgezero-core/src/app.rs:410-423`). Its doc says fallbacks bypass the renderer because "the application supplied those responses directly" (`:415-416`). That is true of admission refusals but not of fallbacks, which the adapter builds. +- Each adapter builds the fallback with no extension point (`edgezero:crates/edgezero-adapter-fastly/src/response.rs:342-351`; `edgezero:crates/edgezero-adapter-cloudflare/src/response.rs:418-427`; `edgezero:crates/edgezero-adapter-spin/src/response.rs:792-801`; `edgezero:crates/edgezero-adapter-axum/src/response.rs:704-732`, which also holds Axum's second fallback builder). +- The fallback runs under a separate 1 s budget and must not run application policy (`edgezero:crates/edgezero-core/src/response_egress.rs:17`; `edgezero:docs/superpowers/specs/2026-09-08-response-egress-design.md:454-461`). + +#### Requirements + +- **EZ-19.1** An application MUST be able to configure, once per app, header fields that EdgeZero adds to every response it synthesizes outside routed dispatch. These are detached ingress errors (head limits, framing, admission-start errors), admitted source errors, and egress fallbacks, including a HEAD fallback, which gets the headers and no body. +- **EZ-19.2** On Fastly's canonical path, the application MUST be able to supply the set for one request from `prepare`, before head limits are checked. A per-request set replaces the app-level set for that request. +- **EZ-19.3** The set MUST NOT be applied to routed responses, admission refusals (the policy builds those), or responses the host produces before the guest runs. +- **EZ-19.4** A set field whose name is already present on the synthesized response MUST be skipped, so a value EdgeZero or the renderer set (for example `Allow` on a 405, `Retry-After`, or `Cache-Control`) stays the only value of that name. Several values of one name within the set are all added, in order. +- **EZ-19.5** The set MUST NOT contain `Content-Length`, `Transfer-Encoding`, `Trailer`, `TE`, `Connection`, `Keep-Alive`, `Proxy-Connection`, `Upgrade`, `Proxy-Authenticate`, `Proxy-Authorization`, `Content-Type` or `Content-Encoding`, and MUST be bounded in field count and bytes. A violating set MUST be rejected when the application configures it, never at request time. EdgeZero documents the bounds. +- **EZ-19.6** Application code MUST NOT run while a fallback is built or sent. The fallback's status, body and 1 s budget MUST NOT change, and adding the set MUST NOT make a deliverable response undeliverable. +- **EZ-19.7** When application configuration itself fails (EZ-22), no configured set exists, and the initialization-failure response carries none. EdgeZero MUST document this next to that response. +- **EZ-19.8** The renderer's doc MUST distinguish admission refusals (application-built) from fallbacks (adapter-built). + +#### Acceptance criteria + +1. Viceroy 0.21.0. Given an app-level set, when a request arrives with a 9 KiB absolute URL, then the 414 carries the set. When a request arrives with about 72 KiB of headers in 60 lines, over EdgeZero's 64 KiB limit and under hyper's 100-line limit and 417,792-byte read buffer (`hyper-0.14.26:src/proto/h1/role.rs:27`; `hyper-0.14.26:src/proto/h1/io.rs:30`; Viceroy does not lower either), then the 431 carries the set. Do not use 101 header lines, because Viceroy's hyper answers those with its own 431 before the guest runs (`hyper-0.14.26:src/error.rs:564`; `hyper-0.14.26:src/proto/h1/role.rs:443-453`). +2. Viceroy 0.21.0. Given a routed response that carries a `Trailer` field, the 500 fallback carries the set and its body is unchanged. A HEAD request to the same route gets the fallback's headers with the set and no body. +3. Viceroy 0.21.0. Given a per-request set supplied in `prepare`, the 414 carries that set and not the app-level one. +4. Unit, each adapter. A fallback carries the app-level set. A rendered 405 keeps its own `Allow` and no second one. A set holding `Cache-Control` leaves a synthesized response's own `Cache-Control` as its only value. +5. Unit. Configuring a set containing any name in EZ-19.5, or exceeding the bounds, fails at configuration time. +6. Trusted Server, Viceroy 0.21.0. The 414 and 431 carry `X-TS-Version`, `X-TS-Fastly-Version`, geo and operator headers. The framing-failure 500 carries the same headers. + +#### Out of scope + +- Running application code for synthesized responses. +- Changing the fallback's status or body. +- Decorating admission refusals. + +#### Open questions + +1. Should a per-request set replace the app-level set or merge with it by name? TS's preferred answer is replace. One rule is easier to reason about, and TS builds the full set per request anyway. +2. Should admission refusals be decorated? TS's preferred answer is no. The policy builds them and can add headers itself. + +#### Interim for Trusted Server + +TS accepts that detached and fallback responses carry no TS headers on Fastly, as it already accepts on the other three adapters. TS may add `X-TS-Version` to detached errors through `App::set_error_response_renderer`. Fallbacks stay bare. + +### EZ-20. A declared store that cannot open fails at use, not on every request + +**Priority:** Follow-up +**Adapters:** Fastly MUST. Cloudflare and Spin SHOULD. Axum keeps its boot-time failure, which happens before it accepts any request. +**Related:** EZ-17 (an explicitly required store keeps the "no response attempted" outcome). + +#### Need + +TS's Fastly `Hooks::stores()` declares the KV store `trusted_server_kv` (`ts:crates/trusted-server-adapter-fastly/src/app.rs:1347-1362`; `ts:edgezero.toml:23-25`). Neither TS's `fastly.toml` (`ts:fastly.toml:25-42`) nor its Viceroy template (`ts:crates/trusted-server-integration-tests/fixtures/configs/viceroy-template.toml:9-59`) links it, and TS reads no EdgeZero KV store on Fastly (only Cloudflare and Spin call `kv_store_default()`, `ts:crates/trusted-server-adapter-cloudflare/src/platform.rs:613`, `ts:crates/trusted-server-adapter-spin/src/platform.rs:721`). Passing `stores()` to the canonical helper therefore fails every request except TS's health probe, which short-circuits first (`ts:crates/trusted-server-adapter-fastly/src/main.rs:173-176`). One unlinked store in a deployment would take down every route, including routes that never touch KV. + +#### Gap at F + +- Fastly opens every declared KV id per request and returns an error, with no response, on the first failure (`edgezero:crates/edgezero-adapter-fastly/src/request.rs:570`, `:632-652`, `:908-923`). +- Declared config stores that fail to open become failing handles, classified as invalid configuration, and the request proceeds (`edgezero:crates/edgezero-adapter-fastly/src/request.rs:654-677`; `edgezero:crates/edgezero-core/src/config_store.rs:503-510`). The capability guide describes that for config only (`edgezero:docs/guide/capabilities.md:192-194`). +- F's Fastly KV open maps a store that does not exist to `KvError::Unavailable` (503) and an SDK open error to `KvError::Internal` (500) (`edgezero:crates/edgezero-adapter-fastly/src/key_value_store.rs:30-35`; `edgezero:crates/edgezero-core/src/key_value_store.rs:823-844`). EZ-20.2 reverses that. +- Cloudflare and Spin also fail the request on a declared KV store that cannot open (`edgezero:crates/edgezero-adapter-cloudflare/src/request.rs:758-783`; `edgezero:crates/edgezero-adapter-spin/src/request.rs:690-716`). Axum fails at boot (`edgezero:crates/edgezero-adapter-axum/src/dev_server.rs:491-496`). + +#### Requirements + +- **EZ-20.1** On Fastly's canonical path with declared store metadata, a declared KV store that cannot be opened MUST NOT fail the request, skip `prepare`, or prevent dispatch. Handlers MUST still find the store by its declared id, and its operations MUST fail when used. +- **EZ-20.2** An unlinked or missing declared store MUST fail its operations with an internal-class error (500 when rendered), because it is a deployment error. A provider failure to open MUST fail them with an unavailable-class error (503). Error text MUST NOT contain the store name or provider text. +- **EZ-20.3** A request that never uses a declared KV store SHOULD incur no extra provider work for it. +- **EZ-20.4** EdgeZero MUST log an open failure at most once per store id per sandbox, as it does today for config. +- **EZ-20.5** Store availability MUST NOT be cached across requests. The once-per-sandbox log state of EZ-20.4 is the only state kept. +- **EZ-20.6** A store the application explicitly requires (`FastlyService::new(&app).with_kv(name).require_kv()`) MUST keep failing the request before anything is sent, reported as EZ-17's "no response attempted". +- **EZ-20.7** Cloudflare and Spin SHOULD follow EZ-20.1 to EZ-20.5. +- **EZ-20.8** The capability guide MUST describe declared-KV failure behavior next to config's. + +#### Acceptance criteria + +1. Viceroy 0.21.0. Given an app declaring a KV id Viceroy does not link, a route that ignores KV returns 200, and a route that reads the store returns 500, in the same request sequence. +2. Unit. A failed open of each class yields the error class in EZ-20.2, and neither message contains the store name. +3. Viceroy 0.21.0. Given the setup in criterion 1 in one reused sandbox, two requests to the KV route produce one open-failure log line in total, and requests to the KV-free route produce none. +4. workerd and Spin 4.1, if EZ-20.7 is adopted. Same as criterion 1. +5. Trusted Server, Viceroy 0.21.0. With `TrustedServerApp::stores()` passed to the helper against TS's Viceroy template, every route serves the pre-cut responses. + +#### Out of scope + +- Changing Axum's boot-time failure. +- Changing config or secret store behavior. + +#### Open questions + +1. Should Cloudflare and Spin change in the same release? TS's preferred answer is yes, so the three serverless adapters share one rule for declared KV, as they already do for config. + +#### Interim for Trusted Server + +TS passes `StoresMetadata::default()` to the helper on Fastly and keeps `Hooks::stores()` only for its manifest-parity test. + +### EZ-21. Framing failure reason in logs + +**Priority:** Follow-up +**Adapters:** all four, Native. +**Related:** EZ-16 and EZ-10 (read the same way once they land). EZ-17 (the report it returns carries the reason). EZ-19 (a set that breaks framing is reported the same way). + +#### Need + +TS's framing failures come from operator `response_headers` or integration headers that name `trailer` or `content-length`. TS's framing-header guard (D8 in TS's cut spec) rejects those at deploy and skips them at runtime (`ts:docs/superpowers/specs/2026-10-17-edgezero-pr275-hard-cutover-design.md:88`, `:529-541`, `:604-605`). TS sets lengths from final bytes on its rewrite paths (`:399-403`), so a wrong length there would be a TS bug that also needs diagnosing. Each of these becomes a 500 fallback. An operator who adds a header and starts serving 500s needs to know which rule fired. + +#### Gap at F + +- The log line says only "response-egress framing validation failed before commit" (`edgezero:crates/edgezero-core/src/response_egress.rs:650-656`), and the report records only a conversion error (`edgezero:crates/edgezero-core/src/response_egress.rs:291-301`; test `edgezero:crates/edgezero-adapter-fastly/src/response.rs:1310-1317`). +- A policy panic, a deadline normalization failure and a Fastly head-preparation failure produce the same outcome (`edgezero:crates/edgezero-core/src/response_egress.rs:620-638`; `edgezero:crates/edgezero-adapter-fastly/src/response.rs:375-385`). +- F defines six framing rules with fixed text, `ResponseEgressFramingError` (`edgezero:crates/edgezero-core/src/response_egress_framing.rs:23-39`), but every adapter discards the rule, including when the fallback itself fails framing (`edgezero:crates/edgezero-adapter-fastly/src/response.rs:316-322`; `edgezero:crates/edgezero-adapter-cloudflare/src/response.rs:392-396`; `edgezero:crates/edgezero-adapter-spin/src/response.rs:766-770`; `edgezero:crates/edgezero-adapter-axum/src/response.rs:635-636`, `:683-686`). + +#### Requirements + +- **EZ-21.1** When a response fails framing validation before commit, the rule that failed MUST be available to the application as one stable name per rule F defines (content-length mismatch, invalid connection header, invalid content-length, unsupported status, unsupported trailers, unsupported tunnel), and per rule added later. It MUST appear in the terminal report that the completion receives, and that the observer receives where EZ-17.3 says it does, and in EdgeZero's log line for that event. +- **EZ-21.2** Applications MUST be able to read the reason from the terminal report without parsing message text. It MUST NOT reach the wire, and existing error `Display` text and the fallback body MUST NOT change (cross-cutting requirement 7). Where EZ-10 or EZ-16 has landed, the reason SHOULD be read the same way as their detail. +- **EZ-21.3** This MUST hold for routed responses, detached responses and the fallback itself, on all four adapters. +- **EZ-21.4** A policy panic, a deadline normalization failure and a host rejection of the head MUST remain distinguishable from framing failures in the report or the log. +- **EZ-21.5** The reason MUST NOT include header values, `Connection` tokens or body bytes. The log line MAY include the route's method and registered pattern, which are bounded (`edgezero:crates/edgezero-core/src/router.rs:40-68`; `edgezero:docs/superpowers/specs/2026-09-08-response-egress-design.md:851-853`). + +#### Acceptance criteria + +1. Unit, each adapter. For each framing rule, a response that breaks it yields a report naming that rule, and a policy panic yields a report naming none. +2. Viceroy 0.21.0. Given a routed response with a `Trailer` field, the runtime log line and the report both name "unsupported trailers", and the client receives the unchanged 500 fallback. +3. Trusted Server, Viceroy 0.21.0. With TS's framing-header guard disabled, an operator `trailer` header produces the 500 fallback and TS's completion log names "unsupported trailers". + +#### Out of scope + +- Naming the offending header in the reason. +- Reasons for post-commit failures, which F already logs by category (`edgezero:crates/edgezero-core/src/response_egress_framing.rs:212-256`). + +#### Open questions + +1. At what level does EdgeZero log the framing reason? TS's preferred answer is `error`, the level F already uses for this event (`edgezero:crates/edgezero-core/src/response_egress.rs:651`). The names are EdgeZero's own rules with no provider data, so a debug-only rule for provider detail need not apply. + +#### Interim for Trusted Server + +TS's framing-header guard prevents the known sources. TS accepts undiagnosed framing 500s from any other source. + +### EZ-22. Fallible application state built and retained with the routes + +**Priority:** Follow-up +**Adapters:** core behavior on all four adapters. The outcome of a failed assembly is specified per entry point. +**Related:** EZ-23 (TS assembles its test apps from explicit settings with the same policies), EZ-17 (reports the failure response as a detached response), EZ-19 (the failure response carries no app-configured headers) + +#### Need + +TS builds one application state per application lifetime. On Fastly it holds `settings`, `orchestrator`, `registry`, `default_kv_store`, and `auction_telemetry_sink` (`ts:crates/trusted-server-adapter-fastly/src/app.rs:186-192`). The auction plan is compiled at `:223` and held inside the orchestrator and registry. Building the state can fail, because it reads the config and secret stores and validates the plan for the target (`ts:crates/trusted-server-adapter-fastly/src/app.rs:200-239`). + +`Hooks::routes()` cannot fail, so every TS adapter turns a state failure into a router that answers every route with the error status (`ts:crates/trusted-server-adapter-fastly/src/app.rs:1042-1068`, `ts:crates/trusted-server-adapter-cloudflare/src/app.rs:410-435`, `ts:crates/trusted-server-adapter-axum/src/app.rs:579-603`). Spin's version always answers 503 and keeps `/health` at 200 (`ts:crates/trusted-server-adapter-spin/src/app.rs:508-538`, `:529-531`). EdgeZero's retention paths treat such a router as a successful app, so retaining it would pin one transient config-store failure for a whole sandbox or isolate. TS avoids this only on Fastly, with its own retained pair of app and state (`ts:crates/trusted-server-adapter-fastly/src/sandbox.rs:57-73`) and a failed build that serves one request and is dropped (`ts:crates/trusted-server-adapter-fastly/src/main.rs:382-412`). + +TS's Fastly entry point reads the state outside the router. It reads `settings` before dispatch to resolve the trusted client IP (`ts:crates/trusted-server-adapter-fastly/src/main.rs:414-427`), after dispatch for finalize headers and EC finalization (`:482-555`), and after the response is sent for pull sync (`:517`). When state is missing it reloads settings from the config store again in the same request (`:489-498`, `:526-553`). + +Cloudflare and Spin rebuild everything on every request, because `run_app` assembles the app per request (`edgezero:crates/edgezero-adapter-cloudflare/src/lib.rs:172-183`, `edgezero:crates/edgezero-adapter-spin/src/lib.rs:187-201`) and TS builds its state inside `routes()` (`ts:crates/trusted-server-adapter-cloudflare/src/app.rs:449-458`, `ts:crates/trusted-server-adapter-spin/src/app.rs:553-562`). Each Cloudflare request re-parses `TRUSTED_SERVER_CONFIG`, re-reads secrets, recompiles the auction plan, and rebuilds the integration registry (`ts:crates/trusted-server-adapter-cloudflare/src/app.rs:129-160`, `:196-217`). Cloudflare state needs the request's `worker::Env`, which TS passes to the input-less `routes()` through a thread-local (`ts:crates/trusted-server-adapter-cloudflare/src/app.rs:52-70`, `ts:crates/trusted-server-adapter-cloudflare/src/lib.rs:27-30`). Spin state comes from asynchronous store reads that TS blocks on inside `routes()` (`ts:crates/trusted-server-adapter-spin/src/app.rs:84-107`). + +On PR 275's APIs TS wants one shape on every adapter. It builds its state, which may fail and may need provider bindings or asynchronous reads, assembles an app with the same name and `configure` policies `App::build` applies, retains it where the adapter allows, and reads the state from the retained app in entry-point code. TS serves Fastly through `serve_custom` with `lifecycle::Sandbox`, so a failed assembly on that path must also answer the current request and must not be retained. + +#### Gap at F + +- `App::build::(platform)` takes the router from `A::routes()`, runs `A::configure`, and rejects changed platform metadata (`edgezero:crates/edgezero-core/src/app.rs:151-164`). Its doc calls it the sole adapter-facing assembly path (`:143-146`). `routes()` returns a bare `RouterService` and cannot fail (`:610-611`). Only `configure` can fail (`:554-556`). +- There is no way to assemble an app from a router the application built itself with `configure` and the platform check applied. The only alternative, `App::with_name_and_platform` (`:488-514`), skips both. +- State registered with `RouterBuilder::with_state` lives in the router (`edgezero:crates/edgezero-core/src/router.rs:337-355`) and reaches handlers through `State` (`edgezero:crates/edgezero-core/src/extractor.rs:619-653`), but neither `RouterService` (`edgezero:crates/edgezero-core/src/router.rs:458-600`) nor `App` (`edgezero:crates/edgezero-core/src/app.rs:38-50`) lets entry-point code read it. +- The `app!` macro registers only an infallible state expression (`edgezero:crates/edgezero-macros/src/app.rs:61-66`, `:358-360`). The demo works around per-request assembly with a process-wide `OnceLock` (`edgezero:examples/app-demo/crates/app-demo-core/src/lib.rs:280-298`). +- When assembly fails, Fastly `serve_app_with_hooks` ends the serving loop without a response for the current request (`edgezero:crates/edgezero-adapter-fastly/src/lib.rs:255`, `edgezero:docs/guide/adapters/fastly.md:439-446`). On the custom path, `Sandbox::initialize` returns the error to the callback and leaves how to respond to the caller (`edgezero:crates/edgezero-adapter-fastly/src/lifecycle.rs:67-85`), and EdgeZero offers no way to answer without an assembled app, so the fixture sends its own native 503 (`edgezero:tests/fixtures/reusable-app/crates/fixture-fastly/src/lib.rs:203-215`). +- Cloudflare `run_app` returns a generic `worker::Error` on a failed assembly (`edgezero:crates/edgezero-adapter-cloudflare/src/lib.rs:66-74`), and Spin returns an untyped error (`edgezero:crates/edgezero-adapter-spin/src/lib.rs:102-108`, `:199`). Neither sends the error's own status. +- The retention guidance for Cloudflare and Spin keeps only a successfully assembled app (`edgezero:docs/guide/adapters/cloudflare.md:342-378`, `edgezero:docs/guide/adapters/spin.md:338-375`), which is correct and must stay true. + +#### Requirements + +- **EZ-22.1** An application's route building MUST be able to fail with a typed `EdgeError`. When it fails, `App::build` MUST return that error unchanged, MUST NOT apply `configure`, and MUST NOT produce an app. +- **EZ-22.2** App state MUST have one source. The value an application registers with `RouterBuilder::with_state` MUST be the value handlers receive through `State` and the value entry-point code reads from the assembled app, with shared interiors shared. +- **EZ-22.3** Code that holds an assembled app, such as a Fastly custom serving callback or a Cloudflare or Spin retention wrapper, MUST be able to read registered state of a given type without a request, and MUST get nothing when no such state was registered. +- **EZ-22.4** An application's `configure` MUST be able to read the registered state when it chooses policies, for example ingress head limits taken from settings. +- **EZ-22.5** An application MUST be able to assemble an app from a router it built itself. That assembly MUST apply the same three things `App::build` applies to a router from `routes()`, which are the application's name, its `configure` policies, and the platform-metadata check. This covers state that needs provider bindings, asynchronous store reads, or explicit test settings. +- **EZ-22.6** Every EdgeZero entry point or documented recipe that retains an app MUST retain only an app whose route building and `configure` both succeeded. A failure MUST leave nothing retained, and the next request MUST retry assembly. +- **EZ-22.7** Fastly `serve_app` and `serve_app_with_hooks` MUST answer the current request when assembly fails, with the error's HTTP status (for example 503 for `EdgeError::service_unavailable`) and EdgeZero's standard error body, through EdgeZero's response delivery, and MUST keep serving so the next request retries. EZ-17 reports this response as a detached response. Because `configure` did not run, the response carries none of the application's configured error rendering or EZ-19 headers, and the guide MUST say so. +- **EZ-22.8** Fastly `run_app`, `run_app_with_hooks`, and `run_app_with_config`, Cloudflare `run_app`, and Spin `run_app` and `run_app_with_platform` SHOULD answer a failed assembly as EZ-22.7 does instead of failing with a host-generic error. Axum `run_app` MUST keep failing startup before it binds the listener. +- **EZ-22.9** The failure response MUST NOT expose internal error detail. Internal errors MUST render as F renders them today, "internal server error" (`edgezero:crates/edgezero-core/src/error.rs:555-575`). EdgeZero MUST log the full error once per failed attempt on its boot log target. +- **EZ-22.10** Applications declared through the `app!` macro SHOULD be able to declare fallible state with the outcomes of EZ-22.1 and EZ-22.6. +- **EZ-22.11** An assembled app MUST stay safe to retain and share across overlapping requests, as it is at F (`edgezero:crates/edgezero-core/src/app.rs:987-995`). +- **EZ-22.12** A Fastly custom serving callback whose assembly failed MUST be able to answer the current request with that error's HTTP status and EdgeZero's standard error body through EdgeZero's response delivery, without an assembled app, and keep serving so the next request retries. EZ-17 reports this response as a detached response, and EZ-22.9 applies to it. + +#### Acceptance criteria + +1. Environment is unit, host. Given an application whose route building fails with a 503 error and whose `configure` counts calls, when `App::build` runs, then it returns the 503 error and the count is 0. +2. Environment is unit, host. Given state registered as a shared value, when a handler extracts it through `State` during canonical dispatch and entry-point code reads it from the assembled app, then both see the same shared value. +3. Environment is unit, host. Given a `configure` that sets ingress head limits from registered state, when the app is assembled, then the app's head limits are the ones in the state. +4. Environment is unit, host. Given a router the application built itself, when it is assembled through the EZ-22.5 path, then `configure` runs once, the name is the application's, and a `configure` that changes platform metadata fails exactly as it does under `App::build`. The existing cases at `edgezero:crates/edgezero-core/src/app.rs:1022-1127` pass for both paths. +5. Environment is unit, host. Given `lifecycle::Sandbox` with an assembly that fails once and then succeeds, when two requests arrive, then after the first nothing is retained and one build was attempted, and after the second the app is retained and two builds were attempted. +6. Environment is Viceroy 0.21.0. Given TS's composition, `serve_custom` with `lifecycle::Sandbox`, and an application whose assembly fails while a flag is set, when request one arrives with the flag set and request two arrives in the same sandbox with the flag cleared, then request one gets 503 with EdgeZero's JSON error body and nothing is retained, and request two gets 200 with the same instance identity, the next ordinal, and two builds recorded. +7. Environment is workerd and Spin 4.1. Given the documented retention recipe and a build that fails once, when two requests arrive in one instance, then the first gets the failure, nothing is retained, and the second succeeds and is retained (EZ-22.6). Only if EZ-22.8 is adopted, given a failing route build, when `run_app` serves a request, then the client gets 503 with the JSON error body, not a host-generic 500. EdgeZero records a passing run of this criterion for the pinned commit. +8. Environment is unit, host, only if EZ-22.10 is adopted. Given an `app!` application with fallible state that fails, when `App::build` runs, then it fails and nothing is assembled. +9. Environment is Viceroy 0.21.0 with TS's reusable-sandbox feature. Given a config key pointing at a missing blob, when request one arrives, then EdgeZero's 503 is served through EZ-22.12. Given the blob restored, when request two arrives in the same sandbox, then it gets 200 with the same `x-ts-sandbox-instance` and `x-ts-sandbox-builds` one higher (`ts:crates/trusted-server-adapter-fastly/src/sandbox.rs:21-31`). TS no longer keeps its own app-and-state pair or its startup-error router. +10. Environment is workerd. Given TS's Cloudflare entry with the retention recipe, when ten sequential requests report the same isolate identity, then the state is built once. TS no longer keeps the Cloudflare thread-local. +11. Environment is Viceroy 0.21.0. Given `serve_app` and an assembly that fails while a flag is set, when request one arrives with the flag set and request two arrives in the same sandbox with the flag cleared, then request one gets 503 with the JSON error body and request two gets 200 with two builds recorded. + +#### Out of scope + +- Refreshing retained state when configuration changes. Staleness stays an application decision, as the guides already say (`edgezero:docs/guide/adapters/overview.md:229-233`). +- Asynchronous route building inside EdgeZero's entry points. EZ-22.5 covers asynchronous state, because the application can finish its own reads before assembling. +- A framework-owned retention cell for Cloudflare and Spin. The documented recipe stays application-owned. +- Passing platform metadata to route building. + +#### Open questions + +1. Should route building see the platform metadata, as `configure` does? TS prefers no for now. Nothing in TS needs it, and `configure` can choose platform-dependent policy after state exists. +2. Should Fastly `serve_app` stop serving after some number of consecutive assembly failures? TS prefers no. `Serve` request and lifetime limits already bound the loop, and stopping only moves the retry to a cold sandbox. + +#### Interim for Trusted Server + +TS keeps its own retained app-and-state pair on Fastly. It assembles with `App::with_name_and_platform(router, name, FASTLY_PLATFORM)` followed by an explicit `TrustedServerApp::configure(&mut app)?`, because `App::build` cannot return state, and it keeps the startup-error router as the failed-initialization value that serves one request. On Cloudflare and Spin TS keeps assembling per request through `run_app`, as it does at v0.0.9, and does not adopt the retention recipe, because the startup-error router would be retained. No transport code is involved. + +### EZ-23. Axum dev server applies application policies, and tests can run real Axum ingress + +**Priority:** Follow-up +**Adapters:** Axum. The test facility may serve an app assembled for another adapter's platform, without that adapter's provider request context. +**Related:** EZ-22 (TS assembles test apps from explicit settings with production policies), EZ-2 (router state already decides which outbound client a handler gets), EZ-24 (route tests supply the scripted client), EZ-6 (handlers that begin outbound requests must work under the test facility), EZ-1, EZ-10, EZ-18 (TS checks bodiless forwarding, 413, and `Content-Length` framing on host here) + +#### Need + +TS's Axum route tests drive the full router through `EdgeZeroAxumService::new(router)` and `tower::ServiceExt::ready().call(request)` (`ts:crates/trusted-server-adapter-axum/tests/routes.rs:1-11`, `:42-49`, `:86-100`), 28 tests in 972 lines. The cross-adapter parity suite uses the same service for Axum (`ts:crates/trusted-server-integration-tests/tests/parity.rs:15`, helpers at `:74-124`, `:224-243`, `:272-289`) and sends Cloudflare and Spin requests straight to `router.oneshot` (`:126-222`, `:244-271`, `:290-316`). TS's Axum binary hands a router to `AxumDevServer::with_config(router, config)` (`ts:crates/trusted-server-adapter-axum/src/main.rs:29-45`) so it binds the `PORT` its integration runner allocates (`ts:crates/trusted-server-integration-tests/tests/environments/axum.rs:59-67`). + +On PR 275 TS wants its route and parity tests to run through real Axum ingress, meaning admission, head limits, framing, and streamed bodies, with production `configure` policies and a scripted outbound client in place of the network. Without that the tests either stop compiling or move to `router.oneshot`, which never exercises admission, head limits, response framing, or egress policy on any adapter, and TS loses its host-side check for EZ-1, EZ-10, and EZ-18. + +The binary does not need this requirement. EdgeZero's `run_app` assembles through `App::build` and reads its bind address from `EDGEZERO__ADAPTER__HOST` and `EDGEZERO__ADAPTER__PORT` (`edgezero:crates/edgezero-adapter-axum/src/dev_server.rs:498-502`, `:687-689`). Any programmatically configured server, though, silently drops `configure` today. + +#### Gap at F + +- The adapter's request service is private (`edgezero:crates/edgezero-adapter-axum/src/lib.rs:29-30`, `edgezero:crates/edgezero-adapter-axum/src/service.rs:45-77`), and its tower entry point exists only in the adapter's own tests (`:337-355`). At v0.0.9 it was public (`edgezero@v0.0.9:crates/edgezero-adapter-axum/src/service.rs:24-37`, `:126`). +- `AxumDevServer::new` and `with_config` take a `RouterService` (`edgezero:crates/edgezero-adapter-axum/src/dev_server.rs:86-102`, `:160-168`) and serve it as `App::new(router)` (`:133-140`), which installs default policies and the default name (`edgezero:crates/edgezero-core/src/app.rs:323-326`) and never runs `configure`. Only `run_app` assembles through `App::build` (`edgezero:crates/edgezero-adapter-axum/src/dev_server.rs:440-445`, `:491`). +- Normalized head limits come from the app (`edgezero:crates/edgezero-adapter-axum/src/service.rs:225`), so a programmatic server always enforces the defaults of 65536 header bytes, 100 headers, and 8192 target bytes (`edgezero:crates/edgezero-core/src/ingress.rs:21-23`, `:409-418`). The adapter's raw ingress settings still apply, with `AxumIngressConfig` defaults of 64 KiB raw head, 100 fields, a 10 s head timeout, and 256 connections (`edgezero:crates/edgezero-adapter-axum/src/ingress_config.rs:28-38`). +- Serving on an already-bound listener exists but only for the adapter's own tests (`edgezero:crates/edgezero-adapter-axum/src/dev_server.rs:143-158`). The `test-utils` feature is empty, and the test module holds only an environment mutex (`edgezero:crates/edgezero-adapter-axum/src/lib.rs:35-36`, `edgezero:crates/edgezero-adapter-axum/src/test_utils.rs:1-14`). +- A client registered with `RouterBuilder::with_state` already wins over the adapter's per-request outbound client (EZ-2.14, `edgezero:crates/edgezero-core/src/router.rs:407-411`). The adapter still builds a real outbound client during request conversion even then, and fails the request if that build fails (`edgezero:crates/edgezero-adapter-axum/src/request.rs:67-75`). +- Handlers run on a local executor inside the serving thread (`edgezero:crates/edgezero-adapter-axum/src/dev_server.rs:371-422`). In-process dispatch outside that thread has no local executor for work a handler spawns. + +#### Requirements + +- **EZ-23.1** Every way to run an Axum dev server programmatically, including a custom bind address and custom store wiring, MUST serve an assembled application with all of its `configure` policies applied. A construction path MUST NOT drop them silently. +- **EZ-23.2** The policies in EZ-23.1 are at least ingress head limits, the admission policy, the error renderer, the response egress policy and observer, the detached-response decision, config extraction limits, the application clock, and the application name. +- **EZ-23.3** For the same application and the same configuration, `run_app` and the programmatic path MUST behave identically. +- **EZ-23.4** Under the adapter's test utilities, an application MUST be able to run its assembled app in-process, from a standard `#[tokio::test]`, through the same ingress, admission, head limits, inbound read deadlines, response framing, streaming, write deadlines, and egress completion as the production server. Work a handler spawns, including EZ-6's begun requests and the chunks of a streamed response they feed, MUST complete as in production. +- **EZ-23.5** A test MUST be able to learn the server's address, stop it without blocking or leaking connections, and read the serving result. +- **EZ-23.6** Any form of the facility that bypasses part of the production path MUST document what it skips. +- **EZ-23.7** When router state supplies an outbound client (EZ-2.14), the real outbound transport MUST NOT be needed or initialized. +- **EZ-23.8** The dev server and the test facility MUST accept an app assembled with any platform metadata, so host parity tests can run a Cloudflare or Spin app's router and policies through Axum ingress. The docs MUST say that provider request contexts are absent in that mode. +- **EZ-23.9** The dev server SHOULD log one startup warning when the app's normalized head limits exceed the raw ingress budget, because the raw parser rejects first. +- **EZ-23.10** Removed. +- **EZ-23.11** Removed. + +#### Acceptance criteria + +1. Environment is Axum, native. Given an application whose `configure` limits the request target to 64 bytes, when a 100-byte target arrives through the programmatic dev server and through the test facility, then both answer 414. Given the same router served without `configure` at F, the request succeeds, which shows the gap. +2. Environment is Axum, native. Given a `configure` that installs an error renderer adding a marker header, when an unmatched path arrives, then the 404 carries the marker. +3. Environment is Axum, native. Given an admission policy that aborts, when a request arrives, then the connection closes with no response bytes and no handler or middleware runs. +4. Environment is Axum, native. Given a client that uploads 1 MiB with chunked framing, when a handler reads the body, then it sees every byte. Given a client that sends headers and then stops sending the body past the inbound read budget, then it gets 408. +5. Environment is Axum, native. Given a handler that streams three chunks 50 ms apart, when a client reads, then the first chunk arrives before the third is produced and the wire uses chunked transfer coding. +6. Environment is Axum, native. Given a buffered 5-byte response, when it is served, then it carries `content-length: 5`. +7. Environment is Axum, native. Given a HEAD request to a route that returns a body, then the wire carries headers and no body bytes. +8. Environment is Axum, native. Given raw limits of 4 header fields, when a request with 5 fields arrives, then it is rejected before admission and the server keeps serving. +9. Environment is Axum, native. Given a handler mid-stream, when the test stops the server, then the handler's resources are released and the serving result reports success. +10. Environment is Axum, native. Given an outbound client supplied through router state on a host with no outbound network access, when a handler sends a request, then the supplied client sees it, no socket opens to the target, and no request fails because the real transport could not start. +11. Environment is Axum, native. Given an app assembled with Cloudflare's platform metadata, when it runs on the test facility, then its `configure` policies apply. +12. Environment is Axum, native. Given a plain `#[tokio::test]`, when a handler spawns local work (and, once EZ-6 lands, begins an outbound request against a supplied client), then it completes, and a streamed body fed by that work delivers every chunk. +13. Environment is Axum, native. Given TS's route tests and parity suite on the test facility with apps assembled from explicit settings with production policies (EZ-22.5), and Cloudflare and Spin parity moved off `router.oneshot` per EZ-23.8, then a 9000-byte request target, which TS does not override, gets 414 from all three apps, and a bodiless GET whose inbound body is an unread stream reaches a local origin with no request body, which is TS's host check for EZ-1. + +#### Out of scope + +- A general embeddable request service outside test utilities, which would bypass the bounded HTTP/1 ingress the adapter owns (`edgezero:docs/guide/adapters/axum.md:82-88`). +- A host test facility for Cloudflare or Spin. EZ-23.8 covers host parity through Axum ingress. +- Any change to the capability matrix. `raw-ingress-head-limits` and `raw-ingress-framing-validation` stay Unsupported for Axum (`edgezero:docs/guide/capabilities.md:75-76`). + +#### Open questions + +1. Should the test facility be usable outside the adapter's test utilities, for example to run an app under a different supervisor? TS prefers no until someone asks for it outside tests. + +#### Interim for Trusted Server + +TS moves its Axum route tests and all three adapters' parity helpers onto `router.oneshot`, which compiles at F and covers routing, middleware, and handlers but not admission, head limits, framing, or egress. TS accepts that loss in unit tests and keeps the ingress-level checks for EZ-1, EZ-10, and EZ-18 in its integration suite, which already launches the real Axum binary over TCP (`ts:crates/trusted-server-integration-tests/tests/environments/axum.rs:59-67`). The binary moves to `run_app::()` with `owns_logging` set, and the runner sets `EDGEZERO__ADAPTER__PORT`. No transport code is added. + +### EZ-24. Published outbound test double that behaves like production + +**Priority:** Follow-up +**Adapters:** core. The double must behave the same on host, `wasm32-wasip1`, and `wasm32-unknown-unknown`. +**Related:** EZ-6 (begun requests, non-blocking checks, and arrival-time rules must hold through the double), EZ-9 (redirect following must reach the double as one ordinary send per hop), EZ-15 (fire-and-forget sends must be recorded once they exist), EZ-1, EZ-5, EZ-7, EZ-10 (request checks, response handling, and the marker must match adapters as those requirements change them), EZ-23 (route tests supply the double). No acceptance criterion of EZ-1 to EZ-9 depends on this double. + +#### Need + +TS deletes its outbound layer and every test double built on it. Test code holds 23 `impl PlatformHttpClient` or `impl PlatformBackend` blocks across nine files, among them `ts:crates/trusted-server-core/src/platform/test_support.rs:136`, `:155`, `:186`, `:496`, six in the orchestrator's tests (`ts:crates/trusted-server-core/src/auction/orchestrator.rs:3029-3681`), and others in `proxy.rs:2391-2452`, `publisher.rs:16332` and `:20834`, `integrations/prebid.rs:3720`, `integrations/sourcepoint.rs:1175`, and `test_support.rs:252-381` under `ts:crates/trusted-server-core/src/`, plus `app.rs:2717-2894` and `tinybird.rs:281-318` under `ts:crates/trusted-server-adapter-fastly/src/`. All of them move onto one scripted `OutboundHttpClient`. + +TS's tests need that client to: + +- record each request's parts, so one test per call site pins method, URI, timeout, deadline, response mode and cap, cache policy, authority override, request body cap, and the options other EZ requirements add; +- script responses, transport errors, timeouts with the right budget source, and response-limit errors; +- script batch completions at arrival times, including cutoff and a mid-batch driver failure, so the auction's rule that a bid arriving at deadline minus 1 ms is kept and a bid arriving at the deadline is dropped is tested exactly as production applies it; +- overlap an origin request begun under EZ-6 with an auction batch checked without blocking at body-chunk boundaries, and show that a late check never turns an on-time bid into a cutoff; +- script the hops of a redirect-following call (EZ-9) and observe the per-hop target policy through the order of recorded requests; +- reject what adapters reject before sending, such as a GET or HEAD carrying body bytes, a streamed response in a batch, or an option a send mode does not support. + +Without a published double TS writes its own, as EdgeZero's scaffold does (`edgezero:crates/edgezero-cli/src/templates/core/src/handlers.rs.hbs:332-444`), and the copy drifts from adapter behavior. A hand-written double that does not apply the shared request checks and response policy behaves differently from every adapter for a GET with a streamed body or a gzip response. + +#### Gap at F + +- The core test-utilities feature exposes environment guards and in-memory store fixtures (`NoopKvStore`, `InMemorySecretStore`) only (`edgezero:crates/edgezero-core/src/lib.rs:47-50`, `edgezero:crates/edgezero-core/Cargo.toml:57-60`, `edgezero:crates/edgezero-core/src/key_value_store.rs:951`, `edgezero:crates/edgezero-core/src/secret_store.rs:177`). No outbound double and no controllable clock are published, although `MonotonicClock::new` accepts any time source (`edgezero:crates/edgezero-core/src/time.rs:46-53`). +- The public pieces an application can assemble a double from exist, for example `OutboundBatch::from_driver` (`edgezero:crates/edgezero-core/src/outbound.rs:268-287`), `OutboundResponse::new` (`:801-828`), `OutboundRequest::into_parts` (`:1025-1047`), `validate_for_dispatch` (`:1544-1583`), `apply_response_body_policy` (`:1252-1300`), and `finish_batch_item` (`:1307-1330`). Adapter-identical behavior from them requires repeating adapter-private steps, such as the batch eligibility checks copied into all four adapters (`edgezero:crates/edgezero-adapter-axum/src/outbound.rs:104-121`, `edgezero:crates/edgezero-adapter-cloudflare/src/outbound.rs:1511-1516`, `edgezero:crates/edgezero-adapter-spin/src/outbound.rs:1421-1426`, `edgezero:crates/edgezero-adapter-fastly/src/outbound.rs:2577-2582`) and each adapter's response handling order (`edgezero:crates/edgezero-adapter-axum/src/outbound.rs:307-397`). +- At F, lazy adapters take a slot's completion time when its exchange future is polled to completion (`edgezero:crates/edgezero-adapter-axum/src/outbound.rs:200-203`), and core drains such futures only when the batch is polled (`edgezero:crates/edgezero-core/src/outbound.rs:313-329`). EZ-6 replaces that with arrival time. A double modeled on F would pin the old behavior. +- The scaffold's double emits every batch item at once with fixed elapsed values and empty bodies (`edgezero:crates/edgezero-cli/src/templates/core/src/handlers.rs.hbs:337-380`). It cannot express latency, cutoff, driver failure, or streaming. +- CI runs no core tests on a wasm target. The wasm matrix runs adapter contract tests only (`edgezero:.github/workflows/test.yml:220-241`, `:366`). + +#### Requirements + +- **EZ-24.1** EdgeZero MUST publish, behind an opt-in that production builds do not enable, a scripted outbound client usable wherever an `HttpClient` is accepted, and a controllable clock that the application, its requests, and the scripted client share. +- **EZ-24.2** The scripted client MUST behave identically on host, `wasm32-wasip1`, and `wasm32-unknown-unknown`, and MUST NOT depend on real time, threads, or a particular async runtime. +- **EZ-24.3** It MUST record every dispatched request, in dispatch order, with every field `OutboundRequest::into_parts` exposes (including fields other EZ requirements add), the request body bytes, the dispatch instant, the effective budget and its source, and the send mode, meaning single send, begun request, batch slot with its batch and index, or fire-and-forget send once EZ-15 exists. +- **EZ-24.4** It MUST reject before sending exactly what adapters reject before sending, with the same error category and status. That includes the GET and HEAD body rules of EZ-1, batch eligibility, the unsupported-option rejection of cross-cutting requirement 2, request body caps, and an already-exhausted budget. When another requirement changes these rules for adapters, the scripted client MUST follow without separate work by the application. +- **EZ-24.5** For a scripted upstream response it MUST give the application what an adapter gives for the same upstream bytes. That means the same header limits and hop-by-hop normalization, bodyless and 205 handling, decoding with encoded, decoded, and final caps, and streamed or buffered delivery, with the same statuses, headers, bytes, and `ResponseLimitReason`. It MUST apply EZ-7's marker default and opt-out exactly as adapters do, with the value `test` when the marker is present. +- **EZ-24.6** A test MUST be able to script, per request, a response (status, headers, and a buffered or chunked body with per-chunk timing), a transport error, or no answer. It MUST be told about requests nothing scripted and scripts nothing consumed. An unscripted request MUST fail with an internal error naming its method and URI, without a panic. +- **EZ-24.7** Time in the double MUST be deterministic. A scripted reply arrives at its dispatch instant plus its latency. Lateness, budgets, the cutoff boundary, delivery order, and elapsed time MUST follow cross-cutting requirement 5 and EZ-6.11 to EZ-6.16 as adapters apply them to recorded arrivals. A non-blocking check MUST return only what has already arrived and MUST NOT move the clock. A streamed body chunk that would arrive after the budget deadline MUST end the body with a gateway timeout carrying that budget's source. How the clock advances while the application waits is EdgeZero's choice, provided every wait ends. +- **EZ-24.8** A test MUST be able to script a batch driver failure after a given number of items, delivered to the application as the batch failure F already defines. +- **EZ-24.9** The scripted client MUST NOT follow redirects itself. A redirect-following call (EZ-9) MUST reach it as one ordinary send per hop, with the application's target policy applied between hops, so tests observe each hop. Begun requests and batches never follow redirects. +- **EZ-24.10** A test MUST keep access to the recordings and the script after handing the client to the application. +- **EZ-24.11** Dropping or cancelling a begun request or a batch through the double MUST behave as cross-cutting requirement 6 requires of adapters. Scripts for abandoned requests MUST NOT be reported as unconsumed. +- **EZ-24.12** EdgeZero's generated application template MUST use the published double instead of its own. + +#### Acceptance criteria + +All criteria run on host. EdgeZero records passing runs for the pinned commit on `wasm32-wasip1` under a WASI preview 1 runner (the pinned wasmtime or Viceroy 0.21.0) and, for criteria 3 to 8, on `wasm32-unknown-unknown` under a JavaScript host such as workerd or Node. Criterion 10 runs natively. + +1. Environment is as above. Given a GET with every request option set, when it is sent, then the recording shows identical parts. Given a POST with a 3-byte body, then the recording shows those bytes. +2. Environment is as above. Given a GET whose inbound body carries bytes, when it is forwarded, then it is rejected with the same error as on an adapter. Given a batch slot that asks for a streamed response, then that slot fails with the adapters' batch-eligibility error and its siblings proceed. +3. Environment is as above. Given a 100 ms timeout and a reply scripted at 150 ms, when the request is sent, then it ends as a gateway timeout with per-call-timeout source, recorded at 100 ms. Given no answer and a 2 s deadline, then it ends as a gateway timeout with request-deadline source at 2 s. +4. Environment is as above. Given three batch slots arriving at 30, 10, and 20 ms, when the batch is collected, then items come in arrival order, indices 1, 2, 0, with elapsed 10, 20, and 30 ms, followed by normal completion. +5. Environment is as above. Given a 100 ms cutoff, a slot arriving at 99 ms, and a slot arriving at 100 ms, when the batch is collected, then the first is kept with elapsed 99 ms, the second is unresolved, and the batch ends at the cutoff. Given the 99 ms slot observed only by a check at 101 ms, then it is still kept with elapsed 99 ms. +6. Environment is as above. Given a driver failure scripted after one item, when the batch is collected, then the application gets one item and then the failure, with exactly one slot filled. +7. Environment is as above, once EZ-6 lands. Given a slot arriving at 10 ms, when the application checks without blocking at 0 ms, then nothing is returned. When the clock is advanced to 15 ms and it checks again, then the slot is returned with elapsed 10 ms. +8. Environment is as above, once EZ-6 lands. Given an origin request begun under EZ-6 whose head arrives at 300 ms, and a batch with a 250 ms cutoff whose slots arrive at 50, 100, and 200 ms, when the application checks the batch at 60, 120, and 210 ms, then all three bids are kept. Given a single check at 260 ms instead, then the same three bids are still kept. When the application then waits for the origin, the wait ends with the origin's head. +9. Environment is as above, once EZ-9 lands. Given a 302 then a 200 scripted for a redirect-following call, then two requests are recorded in hop order. Given a target policy that rejects the second target, then one request is recorded and the call returns EZ-9's rejection. +10. Environment is Axum, native. Given the same gzip body served by a local origin to the Axum adapter and scripted for the double, then status, decoded bytes, and headers match except the proxy marker value, and an over-limit body fails with the same `ResponseLimitReason` on both. +11. Environment is as above. Given an unscripted request, when it is sent, then it fails with an internal error, the test is told about it, and nothing panics. +12. Environment is as above. Given a begun request dropped before its reply arrives, then the test is not told about an unconsumed script for it. +13. Environment is host, Trusted Server. TS has no scripted client of its own. Its per-call-site option tests, batch outcome mapping tests, and cutoff boundary tests run on the published double, and its orchestrator tests that used stub clients migrate to it. + +#### Out of scope + +- Emulating a specific adapter's capability rejections, such as Spin refusing an authority override (`edgezero:crates/edgezero-adapter-spin/src/outbound.rs:1433-1441`). +- Returning hand-built responses that skip the shared response handling. +- Real-time scheduling or any dependency on wall-clock sleeps. + +#### Open questions + +1. Should the double's marker value be `test`? TS prefers yes, so tests can tell double-produced responses from adapter ones. If EZ-7.6 exposes which adapter completed an exchange, the double reports `test` there too. +2. Which part of EZ-24 is the minimal core the head proposes to promote? TS's answer is EZ-24.1 to EZ-24.10, with EZ-24.2 limited to host and `wasm32-wasip1`. EZ-24.11, EZ-24.12, and the `wasm32-unknown-unknown` part of EZ-24.2 stay follow-up. + +#### Interim for Trusted Server + +TS writes its own scripted `OutboundHttpClient` as test code in `ts:crates/trusted-server-core/src/test_support.rs`, following the same arrival-time rules, and deletes it when EZ-24 lands. It is test code, not transport code. + +### EZ-25. Documentation drift fixes + +**Priority:** Follow-up +**Adapters:** documentation for core, Fastly, Axum, Cloudflare, and Spin +**Related:** EZ-1, EZ-5, EZ-7 (EZ-25.1 owns the forwarding example and must show what EZ-1.14, EZ-5.14, and EZ-7.8 require), EZ-22 and EZ-23 (EZ-25.24, EZ-25.25, and EZ-25.37 change again when they land), EZ-2 (EZ-25.7 describes backend identity at F) + +#### Need + +TS ports four adapters by reading these guides, and several of the items below produce a wrong port. The forwarding example in `proxying.md` is the template for TS's publisher origin fetch. As written it does not compile, and once fixed it returns 400 for every publisher GET (EZ-1). The reusable-app README shows a six-argument lifecycle helper that does not exist. The Fastly guide describes delivery through the SDK's `stream_to_client`, which the same guide says the standard path cannot use (`edgezero:docs/guide/adapters/fastly.md:347-349`), so a reader cannot tell which commit behavior applies to TS's finalizer and post-send work. The CHANGELOG says Rust 1.95.0 is still the workspace toolchain while `.tool-versions` pins 1.98.1, and TS pins 1.95.0 (`ts:rust-toolchain.toml:2`). The capability declaration example cannot pass the gate on Fastly, Cloudflare, or Spin, and it is the first thing TS copies into its manifest. + +#### Gap at F + +EdgeZero checks some documentation in CI with `scripts/check_outbound_docs_contract.mjs` (`.github/workflows/test.yml:78`, `.github/workflows/format.yml:228`) and `scripts/check_outbound_legacy_api.sh` (`.github/workflows/test.yml:75`). The docs contract check pins the stale phrase "accounts synchronous body-handle writes through `stream_to_client` close" (`edgezero:scripts/check_outbound_docs_contract.mjs:1610`), pins the Limits table rows exactly (`expectedLimitRows`, `:199-240`, failing on any difference at `:1670-1673`), and requires `axum.md` to contain `EdgeZeroApp::build::(edgezero_adapter_axum::AXUM_PLATFORM)` (`:1069-1078`), which appears only in the Testing sample (`edgezero:docs/guide/adapters/axum.md:280`). Nothing compiles the guides' Rust snippets, none of which is marked `ignore`, which is why EZ-25.2, EZ-25.14, EZ-25.16, and EZ-25.17 are compile errors no check caught. Each item below cites the code that contradicts the guide. + +#### Requirements + +Each item gives the location at F, the current text, and the evidence. The requirement is that the guide states what the evidence shows. The suggested wording is TS's proposal, and EdgeZero may word it differently, provided the result passes this requirement's acceptance criteria and EdgeZero's documentation checks. No item edits `docs/superpowers/specs/`. + +Outbound guides: + +- **EZ-25.1** `docs/guide/proxying.md:9-53`, the "Forward An Inbound Request" example, which this item owns. At F it fails for GET and HEAD, because ingress always hands the router a lazy `Body::Stream` (for example `edgezero:crates/edgezero-adapter-axum/src/request.rs:43-49`), `from_request` keeps that body (`edgezero:crates/edgezero-core/src/outbound.rs:927-934`), and `validate_for_dispatch` rejects any GET or HEAD `Body::Stream` (`:1561-1574`). The example MUST compile, MUST forward a bodiless GET or HEAD once EZ-1 lands, and MUST show what EZ-1.14, EZ-5.14, and EZ-7.8 require. Until EZ-1 lands, the guide MUST say that the example works only for methods that carry a body, and it MUST NOT show a workaround that drops a GET or HEAD body that carries bytes (EZ-1.8). Suggested wording, in both cases, for the sentence at `:9-10` is "This example preserves the inbound method, forwards the inbound body for methods that carry one, replaces the target URI, removes a private header, and converts the outbound response back into a core response. GET and HEAD requests are forwarded without a body." Until EZ-1 lands, add after it "At this release, use it only for methods that carry a body. A GET or HEAD forwarded this way fails with 400." +- **EZ-25.2** `docs/guide/proxying.md:32`. Now `OutboundRequest::from_request(ctx.into_request(), target)?`. `RequestContext::into_request` returns `Result` (`edgezero:crates/edgezero-core/src/context.rs:381`), so the sample does not compile. Suggested replacement is `OutboundRequest::from_request(ctx.into_request()?, target)?`. +- **EZ-25.3** `docs/guide/proxying.md:144-145`. Replace the first sentence, through `results.termination`. Now "`client.send_all_until(requests, observation_cutoff).await` collects the same driver into an index-aligned `Vec>` plus `results.termination`." The return type is `OutboundBatchResults { slots, termination }` (`edgezero:crates/edgezero-core/src/outbound.rs:133-140`). Suggested replacement is "`client.send_all_until(requests, observation_cutoff).await` collects the same driver into `OutboundBatchResults`, whose `slots` field is an index-aligned `Vec>` and whose `termination` field records why collection stopped." +- **EZ-25.4** `docs/guide/streaming.md:116-128` and `docs/guide/capabilities.md:415-420`. The decompression text never says that every adapter sends `accept-encoding: identity` when the request sets none (`edgezero:crates/edgezero-adapter-axum/src/outbound.rs:59-61`, `edgezero:crates/edgezero-adapter-cloudflare/src/outbound.rs:266-268`, `edgezero:crates/edgezero-adapter-fastly/src/outbound.rs:754-757`, `edgezero:crates/edgezero-adapter-spin/src/outbound.rs:356-358`), so the example at `streaming.md:122-128` asks for identity and rarely exercises decompression. After the sentence ending at `streaming.md:118` and after the paragraph ending at `capabilities.md:420`, suggested addition is "Every adapter sends `accept-encoding: identity` when the request has no `accept-encoding` header, so set one explicitly to receive a compressed response." In the example, after `OutboundRequest::get("https://api.example.com/data")?` at `streaming.md:123`, insert `.header("accept-encoding", "gzip, br")?`. EZ-5.14 governs this text once EZ-5 lands. +- **EZ-25.5** `docs/guide/streaming.md:58-60`. Now "Spin's `dispatch_app::(&app, req)` returns a raw `SpinResponse` whose body and transmission lifetime remain owned by the adapter coordinator." The signature is `-> anyhow::Result` (`edgezero:crates/edgezero-adapter-spin/src/lib.rs:215`). Suggested replacement is "Spin's `dispatch_app::(&app, req)` returns `anyhow::Result`, and the response body and transmission lifetime remain owned by the adapter coordinator." +- **EZ-25.6** The guides and CHANGELOG MUST NOT describe Fastly delivery as `stream_to_client`. They MUST describe what EdgeZero guarantees about commit, framing selection, and body-handle close. Naming the hostcalls is optional. At F the adapter commits through a raw status-returning hostcall, not the SDK's `stream_to_client` (`edgezero:crates/edgezero-adapter-fastly/src/fastly_response_abi.rs:64-70`, called from `edgezero:crates/edgezero-adapter-fastly/src/response.rs:128-137`), appends every header value (`fastly_response_abi.rs:19-34`), and selects manual framing when a transmitted body declares `Content-Length` (`response.rs:159-161`). Locations and suggested wording follow. + - `docs/guide/adapters/fastly.md:468-470`. Now "append every header value, commit once through `stream_to_client`, account accepted body bytes, and finish or abandon the body handle exactly once." Suggested replacement is "append every header value, select manual framing when a transmitted body declares `Content-Length`, commit the head once, account accepted body bytes, and finish or abandon the body handle exactly once." + - `docs/guide/adapters/fastly.md:69`. Replace "owns `stream_to_client` through final body-handle close." with "owns the streaming response through final body-handle close." + - `docs/guide/adapters/fastly.md:201`. Replace "EdgeZero owns the low-level `stream_to_client` lifetime and writes portable" with "EdgeZero owns the streaming response lifetime and writes portable". + - `docs/guide/streaming.md:45-46`. Replace "Fastly `stream_to_client`," with "Fastly an adapter-owned streaming body handle,". + - `docs/guide/capabilities.md:223`. Replace "Fastly accounts synchronous body-handle writes through `stream_to_client` close" with "Fastly accounts synchronous body-handle writes through close". EZ-25.44 covers the docs contract check that pins the old phrase. + - `docs/guide/capabilities.md:284-285`. Replace "the low-level `stream_to_client` response lifetime" with "owns the streaming response lifetime". + - `CHANGELOG.md:253` is EZ-25.32. + - Keep `fastly.md:347-349`, which correctly says the SDK API is incompatible. +- **EZ-25.7** `docs/guide/adapters/fastly.md:194-195`. Now "derived from the canonical target, TLS identity, and provider timer budget". At F the backend identity also includes the wire authority (`edgezero:crates/edgezero-adapter-fastly/src/outbound.rs:254-262`). Suggested replacement is "Outbound HTTP uses deterministic **dynamic backends** derived from the canonical target, TLS identity, wire authority (including any `host_authority_override`), and provider timer budget." EZ-2's documentation replaces this text when EZ-2 lands. +- **EZ-25.8** `docs/guide/capabilities.md:11-12`. Now `required = ["outbound-deadlines"]`. The gate rejects a required capability that is only BestEffort (`edgezero:crates/edgezero-cli/src/adapter.rs:423-428`), and Cloudflare, Fastly, and Spin report `OutboundDeadlines` as BestEffort (`edgezero:crates/edgezero-adapter-cloudflare/src/cli.rs:146`, `edgezero:crates/edgezero-adapter-fastly/src/cli.rs:2052`, `edgezero:crates/edgezero-adapter-spin/src/cli.rs:153`), so the example passes only on Axum. Suggested replacement for the two lines is `required = ["outbound-cache-bypass"]` and `optional = ["outbound-deadlines", "outbound-http"]`. `outbound-cache-bypass` is Native everywhere (`edgezero:docs/guide/capabilities.md:240`). +- **EZ-25.9** `docs/guide/capabilities.md:236` and footnote ⁸ at `:294-299`. The Cloudflare `outbound-authority-override` cell points at ⁸, which never mentions authority override. Suggested addition to ⁸ is "Cloudflare sends the authority override as the `Host` header, but no deployed wire probe yet proves that the final request keeps that authority separate from the connection target, so `outbound-authority-override` also remains `BestEffort`." The code sets `Host` at `edgezero:crates/edgezero-adapter-cloudflare/src/outbound.rs:269-273`, and `proxying.md:89-92` already says the cell awaits a deployed wire probe. +- **EZ-25.10** `docs/guide/capabilities.md:383-393`, "Limits And Accounting". The 30 s default budget is not listed. `DEFAULT_NO_DEADLINE_BUDGET` is 30 s (`edgezero:crates/edgezero-core/src/time.rs:13`) and applies with `BudgetSource::Default` when neither timeout nor deadline is set, and the earliest candidate wins (`:150-166`). Suggested row is "| `timeout` / `deadline` | Per-request budget. The earliest of timeout, deadline, and batch cutoff wins | 30 s when neither is set |". EZ-25.44 covers the docs contract check that pins the table. + +Fastly lifecycle guide: + +- **EZ-25.11** `docs/guide/adapters/fastly.md:400-401`. Now "`FnMut(&mut edgezero_core::Response) -> State`". The crate root does not export `Response` (the root re-exports start at `edgezero:crates/edgezero-core/src/lib.rs:53`), and the type is `edgezero_core::http::Response` (`edgezero:crates/edgezero-core/src/http.rs:36`). Suggested replacement is "`FnMut(&mut edgezero_core::http::Response) -> State`". +- **EZ-25.12** `docs/guide/adapters/fastly.md:460-461`. Now "injects fresh `Kv`, `Config`, `Secrets`, and `AppConfig` registries". Only config, KV, and secret registries are inserted (`edgezero:crates/edgezero-adapter-fastly/src/request.rs:395-415`), and `AppConfig` is an extractor over the config registry. Suggested replacement is "injects fresh KV, config, and secret registries, which back the `Kv`, `Config`, `Secrets`, and `AppConfig` extractors,". +- **EZ-25.13** `docs/guide/adapters/fastly.md:588-590`. Now "Use `lifecycle::run_custom(Request::from_client(), callback)` for the single-request branch, initializing the Fastly ABI before receiving that request." `run_custom` initializes the ABI inside its body, after its argument has already received the request (`edgezero:crates/edgezero-adapter-fastly/src/lifecycle.rs:148-156`), and the fixture calls `fastly::init()` first (`edgezero:tests/fixtures/reusable-app/crates/fixture-fastly/src/lib.rs:277-282`). Suggested replacement is "For the single-request branch, call `fastly::init()` and then `lifecycle::run_custom(Request::from_client(), callback)`, because the argument receives the request before `run_custom` runs. It creates". Whether `from_client` needs prior initialization is **Unverified**. A Viceroy run that omits `fastly::init()` would settle it. +- **EZ-25.14** `docs/guide/adapters/fastly.md:330-343`. The sample calls `ctx.request()`, which `RequestContext` does not have (its public methods at `edgezero:crates/edgezero-core/src/context.rs:262-616` include `extensions` at `:336` but no `request`), imports neither `Response` nor `EdgeError`, and returns nothing. `FastlyRequestContext` lives in request extensions (`edgezero:crates/edgezero-adapter-fastly/src/context.rs:11-15`). Suggested replacement for the whole sample is + + ```rust + use edgezero_adapter_fastly::context::FastlyRequestContext; + use edgezero_core::context::RequestContext; + use edgezero_core::http::Response; + use edgezero_core::{Body, EdgeError}; + + async fn handler(ctx: RequestContext) -> Result { + let client_ip = ctx + .extensions() + .get::() + .and_then(|fastly_ctx| fastly_ctx.client_ip); + Ok(Response::new(Body::from(format!("{client_ip:?}")))) + } + ``` + + `Body` is a root re-export (`edgezero:crates/edgezero-core/src/lib.rs:56`) and converts from `String` (`edgezero:crates/edgezero-core/src/body.rs:216`). + +- **EZ-25.15** `tests/fixtures/reusable-app/README.md:201`. Now "`send_request_with_registries_and_hooks(app, stores, req, &runtime.env, prepare, finalize)`". The helper takes five arguments (`edgezero:crates/edgezero-adapter-fastly/src/request.rs:557-563`). Suggested replacement is "`send_request_with_registries_and_hooks(app, stores, req, prepare, finalize)`". + +Core and cross-adapter guides: + +- **EZ-25.16** `docs/guide/handlers.md:389`. Now `use edgezero_core::{Body, EdgeError, Response};`. `Response` is not a root export (`edgezero:crates/edgezero-core/src/lib.rs:53-115`). Suggested replacement is `use edgezero_core::http::Response;` and `use edgezero_core::{Body, EdgeError};`. +- **EZ-25.17** `docs/guide/handlers.md:229-242`. The sample imports `use edgezero_core::{EdgeError, Text};`, but `Text` lives at `edgezero_core::response::Text` (`edgezero:crates/edgezero-core/src/response.rs:50`) and is not re-exported. It also uses `#[action]` without importing it and an application type `MyAppConfig` the guide never defines. Suggested replacement for the imports is `use edgezero_core::action;`, `use edgezero_core::extractor::{AppConfig, Kv};`, `use edgezero_core::response::Text;`, and `use edgezero_core::EdgeError;`. Because `MyAppConfig` is the application's own type, mark the fence `rust,ignore` and add the line `// MyAppConfig is the application's own config type, defined with #[derive(AppConfig)].` as its first line. +- **EZ-25.18** `docs/guide/middleware.md:61`. Now "Middleware are applied in order before routes are matched." The route is resolved before admission (`edgezero:crates/edgezero-core/src/app.rs:116-118`), the chain is built only for a matched entry (`edgezero:crates/edgezero-core/src/router.rs:417-418`), and 404 and 405 return without it (`:370-381`). Suggested replacement is "Middleware run in registration order after the route is resolved and the request is admitted, wrapping the matched handler. Unmatched paths (404) and unsupported methods (405) return without running middleware." +- **EZ-25.19** `docs/guide/architecture.md:104` and `:113-116`. The diagram shows "into_core_request(): Provider Request → Core Request" and "RouterService matches routes ... Response built and returned". Canonical runners convert the head first and attach the body after admission (`edgezero:crates/edgezero-adapter-fastly/src/request.rs:446-508`), and the core returns an owned `ResponseEgressEnvelope` (`edgezero:crates/edgezero-core/src/app.rs:111-145`, `:236-256`). Suggested replacement for line 104 is `│ - Head normalization: Provider head → Core request head │`, and for lines 113-116 is + ``` + │ - App resolves the route and runs ingress admission │ + │ - Middleware chain executes (matched routes only) │ + │ - Handler runs with extracted params │ + │ - Response returned in an owned egress envelope │ + ``` + Each replacement line is 63 characters, the box width. +- **EZ-25.20** `docs/guide/routing.md:187-190`. Now "the Fastly, Cloudflare, and Spin entry points call `App::build::(platform)` per request". That holds only for the default `run_app` (`edgezero:crates/edgezero-adapter-fastly/src/lib.rs:205`, `edgezero:crates/edgezero-adapter-cloudflare/src/lib.rs:181`, `edgezero:crates/edgezero-adapter-spin/src/lib.rs:199`). `serve_app` builds once per sandbox (`edgezero:crates/edgezero-adapter-fastly/src/lib.rs:255`), and `dispatch_app` never builds. Suggested replacement for lines 187-190 is "When that happens depends on the adapter. Axum builds the router once at startup, so the conflict surfaces before any traffic. The default Fastly, Cloudflare, and Spin `run_app` entry points call `App::build::(platform)` per request, and Fastly's retained `serve_app` builds on the first request in each sandbox, so a conflicting route deploys successfully and then panics on the first request. With `dispatch_app`, the panic happens wherever the caller builds the app." +- **EZ-25.21** `docs/guide/getting-started.md:94-105`. The tree shows `src/main.rs` for Cloudflare and Spin. Cloudflare scaffolds `src/lib.rs` plus a host stub `src/main.rs` (`edgezero:crates/edgezero-adapter-cloudflare/src/cli.rs:85-95`), and Spin scaffolds `src/lib.rs` and `runtime-config.toml` with no `main.rs` (`edgezero:crates/edgezero-adapter-spin/src/cli.rs:90-106`). Suggested replacement shows the Cloudflare subtree as `Cargo.toml`, `wrangler.toml`, and `src/` holding `lib.rs` (Workers fetch entrypoint) and `main.rs` (host-only stub), and the Spin subtree as `Cargo.toml`, `runtime-config.toml`, `spin.toml`, and `src/lib.rs`. +- **EZ-25.22** `docs/guide/adapters/cloudflare.md:230-231`. Now "Missing bindings log a one-time warning and the id is dropped from the registry." The id keeps a failing handle (`edgezero:crates/edgezero-adapter-cloudflare/src/request.rs:855-862`). Suggested replacement is "Missing bindings log a one-time warning, and the id keeps a failing handle whose reads return `ConfigStoreError::Unavailable`." +- **EZ-25.23** `docs/guide/adapters/overview.md:195`. Now "when `configure` or `app_state` reads values". No `app_state` API exists. The macro argument is `state = ` (`edgezero:crates/edgezero-macros/src/app.rs:61-66`). Suggested replacement is "when `configure` or the `app!` `state` expression reads values". + +Axum guide: + +- **EZ-25.24** `docs/guide/adapters/axum.md:259-261`. The text says wiring `App::build::(edgezero_adapter_axum::AXUM_PLATFORM)` and `AxumDevServer` by hand remains the fallback. At F `AxumDevServer` takes a `RouterService` and wraps it in `App::new`, so application policies are discarded (`edgezero:crates/edgezero-adapter-axum/src/dev_server.rs:93-168`), while `AxumDevServerConfig::ingress` still applies (`edgezero:docs/guide/adapters/axum.md:104-113`). Until EZ-23 lands, suggested replacement is "`AxumDevServer::with_config(router, config)` accepts a router, not an `App`, and serves it through `App::new` with default application policies, so policies installed by `Hooks::configure`, such as `App::set_ingress_head_limits`, the admission policy, and the error renderer, do not apply. `AxumDevServerConfig::ingress` still applies. Use `run_app` when those policies matter." After EZ-23 lands, the text MUST name the delivered way to run a programmatically configured Axum server and state that every `configure` policy, including ingress head limits, applies. +- **EZ-25.25** `docs/guide/adapters/axum.md:266-299`, Testing. The sample calls `app.router()` and `router.oneshot`, which skip admission, head limits, response framing, and egress policy (`edgezero:crates/edgezero-core/src/router.rs:562-569` against `edgezero:crates/edgezero-core/src/app.rs:267-284`), and it relies on `use super::*` for `Body`. Until EZ-23 lands, suggested addition after the block is "`router.oneshot` exercises routing, middleware, and handlers only. It skips ingress admission, head limits, response framing, and egress policy." After EZ-23 lands, the sample MUST run the request through real Axum ingress using the delivered test facility, and MUST still contain `EdgeZeroApp::build::(edgezero_adapter_axum::AXUM_PLATFORM)` (EZ-25.44). +- **EZ-25.26** `docs/guide/adapters/axum.md:488-489`. Now "constructs one app per server startup and serves clones of its router." The service holds one `Arc` (`edgezero:crates/edgezero-adapter-axum/src/service.rs:45-77`). Suggested replacement is "constructs one app per server startup and serves every connection from that shared `Arc`, including its policies." + +Toolchain and CHANGELOG: + +- **EZ-25.27** `CHANGELOG.md:408-409`. Now "Rust `1.95.0`, Node `24.12.0`, and Fastly CLI `15.1.0` remain the workspace toolchain." `.tool-versions` pins `rust 1.98.1`, `nodejs 24.20.0`, and `fastly 16.0.0` (`edgezero:.tool-versions:1-3`), the scaffold pins `rust 1.98.1` (`edgezero:crates/edgezero-cli/src/generator.rs:629`), and the same CHANGELOG says main's 1.98.1 toolchain was merged (`edgezero:CHANGELOG.md:16-17`). Suggested replacement is "The workspace toolchain is Rust `1.98.1`, Node `24.20.0`, and Fastly CLI `16.0.0`, matching `.tool-versions`. See the dependency-majors migration guide for the moved pins." +- **EZ-25.28** `docs/guide/adapters/axum.md:435`. Now `FROM rust:1.95.0 as builder`. Suggested replacement is `FROM rust:1.98.1 as builder`. Whether the `rust:1.98.1` image is built on the same Debian release as the `debian:bookworm-slim` runtime stage is **Unverified**. `docker run --rm rust:1.98.1 cat /etc/os-release` would settle it, and `rust:1.98.1-bookworm` keeps the stages matched either way. +- **EZ-25.29** `docs/guide/adapters/axum.md:162`, `:403`, and `crates/edgezero-adapter-axum/src/ingress_config.rs:150`, `:170`. Each says an allocation audit was done against Rust 1.95.0, while the workspace builds on 1.98.1. These are audit records. EdgeZero SHOULD re-run the receive-scratch, header-map, and config-decoder audits on 1.98.1 and change "1.95.0" to "1.98.1". If it does not, the suggested change in all four places is to replace "and Rust 1.95.0" with "and Rust 1.95.0 (not yet repeated on the workspace's Rust 1.98.1)". + +Migration guides, CHANGELOG, and links: + +- **EZ-25.30** `CHANGELOG.md:338-342`. Now "Fastly `runtime_env_config` now returns `FastlyRuntimeConfig` ... call `emit_boot_diagnostics()`". None of the three names exists in `crates/` (`git grep -n 'runtime_env_config\|FastlyRuntimeConfig\|emit_boot_diagnostics' -- crates` is empty), and `CHANGELOG.md:28-30` already says selector-store setup is retired. Suggested replacement for the paragraph is "Fastly `runtime_env_config`, `FastlyRuntimeConfig`, and `emit_boot_diagnostics()` are removed with no compatibility alias. Custom entrypoints open logical store aliases directly and take logging policy from baked `Hooks::logging_for("fastly")`. Managed runners initialize logging before application configuration. Axum now returns logger-setup errors instead of ignoring them, and preinstalled backends must use `owns_logging = true`." +- **EZ-25.31** `CHANGELOG.md:385`, the migration table row "Fastly `runtime_env_config(...) -> EnvConfig` | `FastlyRuntimeConfig`, using `.env` and `emit_boot_diagnostics()` after logger setup". Suggested right cell is "Removed. Use `run_app`, `run_app_with_hooks`, or `request::send_request_with_registries_and_hooks` with baked `Hooks::stores()` and `Hooks::logging_for("fastly")`". +- **EZ-25.32** `CHANGELOG.md:253`. Now "Fastly's manual `stream_to_client`". Suggested replacement is "Fastly's adapter-owned streaming body handle", for the reason in EZ-25.6. +- **EZ-25.33** `docs/guide/manifest-store-migration.md:173-174`. Now "Fastly additionally exposes a public `request::dispatch_with_registries` for custom entry points." No public `dispatch_with_registries` exists. Cloudflare and Spin keep crate-private ones (`edgezero:crates/edgezero-adapter-cloudflare/src/request.rs:671`, `edgezero:crates/edgezero-adapter-spin/src/request.rs:622`), and Fastly's public helpers are at `edgezero:crates/edgezero-adapter-fastly/src/request.rs:534` and `:557`. Suggested replacement is "Fastly additionally exposes public `request::send_with_registries_and_hooks` and `request::send_request_with_registries_and_hooks` for custom entry points." +- **EZ-25.34** `docs/guide/dependency-majors-migration.md:34-35`. Now "including `worker = "=0.8.5"`". The workspace requirement is `worker = { version = "=0.8.5", default-features = false, features = ["http"] }` (`edgezero:Cargo.toml:120`), and the scaffold and fixture match it. Suggested replacement is "Keep the audited SDK requirements exact, including `worker = { version = "=0.8.5", default-features = false, features = ["http"] }` for Cloudflare applications." +- **EZ-25.35** `docs/guide/dependency-majors-migration.md:65-70`. The scaffold pin table has no `spin` row, but the generator now emits `spin 4.1.0` (`edgezero:crates/edgezero-cli/src/generator.rs:626-628`). Suggested row is "| `spin` | not pinned | **4.1.0** | Spin adapter |". +- **EZ-25.36** `docs/guide/dependency-majors-migration.md:72-75`. Replace the whole paragraph, which says "the deploy action's `install-fastly.sh` fails closed when `.tool-versions` and its own `versions.json` disagree, so an app that moves the Rust pin and leaves the Fastly one behind gets a red deploy". The script compares `versions.json` with the EdgeZero action checkout's `.tool-versions` (`edgezero:.github/actions/deploy-fastly/scripts/install-fastly.sh:42-47`, root set at `edgezero:.github/actions/deploy-fastly/action.yml:182`), not the application's. Suggested replacement is "The deploy action installs the Fastly CLI pinned by its own `versions.json` and fails closed when that pin disagrees with the EdgeZero action repository's `.tool-versions`. It does not read the application's `fastly` line, so keep that line equal to the action's pin for local runs to match deployment." +- **EZ-25.37** `docs/guide/configuration.md:698-703`, the `app!` argument table. It has no `configure` row although the macro accepts `configure = ` (`edgezero:crates/edgezero-macros/src/app.rs:42-49`). Suggested row is "| `configure` | `configure = ` | Emits `Hooks::configure`, calling `(expr)(&mut App) -> Result<(), EdgeError>`. Errors propagate through `App::build`. Defaults to `Ok(())` |". When EZ-22.10 lands, the table MUST also describe fallible state. +- **EZ-25.38** `docs/guide/configuration.md:547`. Now "| `echo_stdout` | Fastly, Axum | Mirror logs to stdout |". The Axum runner builds its logger from the level only (`edgezero:crates/edgezero-adapter-axum/src/dev_server.rs:447-451`), and `axum.md:236` already says `echo_stdout` has no effect there. Suggested replacement is "| `echo_stdout` | Fastly | Mirror logs to stdout. The Axum runner ignores it |". +- **EZ-25.39** `docs/guide/blob-app-config-migration.md:191`, `:377`, and `docs/guide/configuration.md:443`. They name `ConfigOutOfDate`, but extraction raises `EdgeError::StoreExtraction`, whose wire kind for a missing blob, missing secret, deserialization, or validation failure is `config_out_of_date` (`edgezero:crates/edgezero-core/src/error.rs:84-101`, `:184-186`). Suggested replacements are "a `config_out_of_date` error (`EdgeError::StoreExtraction` with reason `MissingSecret`) at runtime" for "`ConfigOutOfDate` at runtime" at `:191`, the heading "### `config_out_of_date` at runtime" at `:377`, and "is a `config_out_of_date` error" for "is a `ConfigOutOfDate` error" at `configuration.md:443`. No link uses the old anchor. +- **EZ-25.40** Broken anchors, checked against a local VitePress build of `docs/`. `docs/guide/capabilities.md:106` links `/guide/adapters/axum#bounded-http1-ingress`, which should be `#bounded-http-1-ingress` for the heading at `axum.md:83`. `docs/guide/cli-reference.md:408` links `#config-gc`, which should be `#edgezero-config-gc`. `docs/guide/deploy-action-adoption.md:138-139` links `./adapters/fastly.md#migrating-a-custom-entrypoint`, a heading this PR removed, which should link `./adapters/fastly.md#entrypoint` (heading at `fastly.md:56`). +- **EZ-25.41** `docs/guide/cli-reference.md:588`. Now "(KV in-memory; config in `.edgezero/local-config-.json`)". Local KV is a redb file (`edgezero:crates/edgezero-adapter-axum/src/dev_server.rs:231-243`). Suggested replacement is "(KV in `.edgezero/kv--.redb`; config in `.edgezero/local-config-.json`)". + +Checks that keep the guides honest: + +- **EZ-25.42** Every complete Rust item in `docs/guide/` SHOULD compile in EdgeZero's CI against the current workspace. A fragment SHOULD either compile inside a documented wrapper or be marked `ignore` with a one-line reason. EZ-25.2, EZ-25.14, EZ-25.16, and EZ-25.17 are compile errors that this would have caught. +- **EZ-25.43** Drift of the kinds in EZ-25.6, EZ-25.27, and EZ-25.30 SHOULD be caught automatically. The check MUST NOT flag the old-version column of a migration table, the audit records EZ-25.29 keeps, or the explanation at `fastly.md:347-349`. +- **EZ-25.44** The fixes MUST leave EdgeZero's existing documentation checks passing. The docs contract check MUST accept EZ-25.6's corrected phrase, MUST add EZ-25.10's row to its expected Limits rows, and MUST keep its `axum.md` build-string assertion satisfied by EZ-25.25's sample. + +#### Acceptance criteria + +1. Environment is EdgeZero CI on the recorded commit. Given the fixes, when `docs/guide` is searched for `stream_to_client`, then only `adapters/fastly.md:347` matches. +2. Environment is EdgeZero CI on the recorded commit. Given the fixes, when `CHANGELOG.md` and `docs/guide/adapters/axum.md` are searched for `1.95.0`, then only the audit statements EZ-25.29 keeps match. +3. Environment is EdgeZero CI on the recorded commit. Given EZ-25.42, when CI runs, then every changed snippet that is a complete item compiles, including the forwarding example, and every changed fragment either compiles inside its documented wrapper or is marked `ignore` with a reason. +4. Environment is EdgeZero CI on the recorded commit. Given a manifest using EZ-25.8's declaration, when the capability gate runs for each of the four adapters, then it passes. +5. Environment is EdgeZero CI on the recorded commit. Given the fixes, when EdgeZero's documentation checks run (`scripts/check_outbound_docs_contract.mjs`, `scripts/check_outbound_legacy_api.sh`, and the docs formatter), then they pass. +6. Environment is Axum, Trusted Server. TS's publisher forwarding, written from the corrected example once EZ-1 lands, forwards a bodiless GET whose inbound body is an unread stream to a local origin with no request body. TS's Fastly custom lifecycle compiles against the five-argument helper the README now shows. + +#### Out of scope + +- Edits to `docs/superpowers/specs/`, which EdgeZero treats as immutable (`edgezero:.github/workflows/test.yml:72-73`). Where a spec is stale, the guides carry the current text. +- New capability cells. EZ-25.9 adds footnote text to an existing BestEffort cell, and EZ-25.8 changes only the example declaration. + +#### Open questions + +1. Should guide snippets compile in CI (EZ-25.42)? TS prefers yes, starting with `proxying.md`, `handlers.md`, and `adapters/fastly.md`, where every compile error in this list lives. +2. Should the axum allocation audits be re-run on Rust 1.98.1 (EZ-25.29)? TS prefers yes, because the audit text promises the numbers hold for the pinned toolchain. + +#### Interim for Trusted Server + +TS ports from the code and from the corrected texts in this section, not from the guides as they stand at F. No transport code is involved. + +## Qualification gate + +### EZ-Q1. Recorded passing qualification run of the canonical Fastly composition + +**Priority:** Gate +**Adapters:** all four, with hosted Fastly for the criteria that name it +**Related:** EZ-1 to EZ-9 (their acceptance criteria are what this record covers). EZ-2, EZ-6, and EZ-15 (their reuse-mode parts gate later). The Trusted Server cutover spec holds the run plan that produces the record. + +#### Need + +Fastly is TS's production target, and the cut moves TS onto `run_custom` with `send_request_with_registries_and_hooks`, a composition TS has never run. EdgeZero's capability guide says "the production `run_app` composition has not yet been exercised end to end under Viceroy" (`edgezero:docs/guide/capabilities.md:284-288`). The fixture README says its cases "describe fixture coverage, not a recorded pass for the current pinned SDK/runtime graph" (`edgezero:tests/fixtures/reusable-app/README.md:285-287`). TS will not ship the cut to production without a record, for the exact commit it pins, showing that the blocking requirements hold in the environments their criteria name. + +#### Gap at F + +CI run 38934086784 passed EdgeZero's Fastly smoke at F with Viceroy 0.21.0, Rust 1.98.1, and Fastly SDK 0.13.1 (`edgezero-ci:run 38934086784`). It does not meet the need. Its artifact expires after 14 days (`edgezero:.github/workflows/test.yml:405-411`), it records no source revision (`edgezero:tests/fixtures/reusable-app/README.md:315-317`), its runner label does not pin an image, its cases do not cover the blocking requirements in this document, and no hosted Fastly run exists. The hosted egress tool contacts no outbound backend (`edgezero:tests/fixtures/reusable-app/README.md:69-71`). + +#### Requirements + +- **EZ-Q1.1** For the exact EdgeZero commit TS pins, there MUST be a recorded result for every acceptance criterion of the blocking parts of EZ-1 to EZ-9 and of any requirement promoted to blocking, each in the environment that criterion names. +- **EZ-Q1.2** Every such result MUST pass. Each of those criteria is pass-or-fail, and a measurement it names is judged against the bound it states. A leg or value that a criterion says only to record passes when the record contains it. A blocking criterion without a pass condition is a defect in this document, to be fixed before the record is produced. +- **EZ-Q1.3** The record MUST name the EdgeZero commit, the toolchain (Rust 1.98.1, Viceroy 0.21.0, Fastly SDK 0.13.1, Spin 4.1, and the workerd version, where used), the runner image, and the TS baseline and candidate builds behind any comparison, and for each criterion its environment, result, and any measurements with sample counts. A record covers that one commit only. +- **EZ-Q1.4** EdgeZero produces the results for criteria in unit, Viceroy 0.21.0, workerd, Spin 4.1, and Axum that do not need Trusted Server, and assembles and keeps the record. For hosted Fastly criteria whose settings TS's configuration cannot express, EdgeZero MUST provide a qualification build that TS can deploy to a TS staging service, serving single-request sandboxes (`run_custom`) unless a criterion says otherwise. These are EZ-1 criterion 2 and the hosted leg of criterion 6, EZ-2 criteria 11, 12, 13b, and 14, EZ-4 criterion 8, the hosted legs of EZ-6 scenarios 1, 2, 6, 7, 8, and 11, and EZ-8's hosted criterion. TS produces its own end-to-end checks and the remaining hosted Fastly results with its own build, in single-request mode unless a criterion says otherwise. +- **EZ-Q1.5** Criteria of reuse-mode follow-ups (EZ-2's reuse-mode part and the reuse-mode legs of EZ-6 and EZ-15) are not part of the gate for single-request production. They MUST pass in a record before TS enables reuse mode. +- **EZ-Q1.6** A follow-up present at the pinned commit, as its CHANGELOG states and TS confirms, adds its acceptance criteria to those EZ-Q1.1 requires, and EZ-Q1.2 applies to them. The reuse-mode criteria of EZ-Q1.5 stay out of the gate even when present. An absent follow-up's interim applies instead. +- **EZ-Q1.7** The record MUST stay retrievable for as long as TS pins that commit. + +#### Acceptance criteria + +1. Trusted Server review. A record that names a commit other than the pin is rejected. +2. Trusted Server review. A record that lacks a result for any criterion EZ-Q1.1 or EZ-Q1.6 requires, records a result in an environment other than the one the criterion names, or records a failing result is rejected. +3. Trusted Server review. A record whose toolchain differs from EZ-Q1.3, that names no runner image, or that omits measurement sample counts or the builds behind a comparison is rejected. +4. Trusted Server review. A record stored only where it expires within 90 days, such as a CI artifact, is rejected. + +#### Out of scope + +The run plan. Which party runs which leg, the case list, scripts, and evidence formats belong in the Trusted Server cutover spec, which consumes this record and does not hold them yet. EdgeZero is asked for no runs beyond the acceptance criteria in this document. + +#### Open questions + +1. Where EdgeZero keeps the record. Trusted Server prefers a release asset attached to the tag TS pins, because CI artifacts expire. + +## Review log + +- **Round 1 (Claude Code, five fresh reviewers by section plus one whole-document).** Thirteen + blocking findings, almost all contradictions between sections (synchronous versus async begin, + two owners for in-flight request state, an unbounded background drain, drop and fire-and-forget + semantics, stale references to Fastly internals, tests that depended on a follow-up, and an + unreachable qualification case). All were resolved, then the TS maintainer asked for requirements + rather than implementation, and every section was rewritten to observable behavior and acceptance + criteria. +- **Round 2 (Claude Code, three fresh reviewers on a different model).** Thirteen blocking findings. + Resolved by defining when a response arrives and when a request's lifetime ends, rejecting only + options marked Unsupported (BestEffort options are accepted), correcting the Viceroy 0.21.0 timer + facts, making EZ-Q1 gate every blocking criterion with hosted evidence from TS's own composition + and a pre-cut TS baseline, splitting EZ-2 into a blocking part and a reuse-mode follow-up because + production runs single-request sandboxes, and removing the remaining mechanism and harness text. +- **Round 3 (Claude Code, two fresh reviewers).** Every round 2 blocking finding was confirmed + resolved in the text. New blocking findings were all in how EZ-Q1 assigned runs. Resolved by + separating EdgeZero-side Viceroy evidence from Trusted Server end-to-end checks, running hosted + criteria that TS configuration cannot express on an EdgeZero qualification build that TS deploys, + gating reuse-mode criteria only before TS enables reuse mode, fitting Viceroy cases inside its + six-request guest limit, and defining fire-and-forget lifetime as the one exception to request + lifetime. +- **Round 4 (Claude Code, one fresh reviewer on the round 3 changes).** Every round 3 finding was + resolved. One blocking finding, that two hosted measurements had no pass condition. Resolved by + giving every blocking measurement a pass condition. EZ-Q1 had grown into a full run plan, which is + Trusted Server's rollout work rather than an EdgeZero requirement, so it now states only what the + record must show, and the run plan moves to the Trusted Server cutover spec. +- **Round 5 (Claude Code, one fresh reviewer on the round 4 changes).** Round 4's blocking finding + was resolved. One new blocking finding, that EZ-Q1.5 and EZ-Q1.6 disagreed on a reuse-mode + follow-up present at the pinned commit, and that nothing rejected a record missing a present + follow-up's results. Fixed with the reviewer's replacement text and checked against the head by the + spec author, without a further agent round. Its should-fix items were applied (EZ-6 hosted timing, + the definition of "every adapter", four measurement bounds, conditional workerd and Spin + criteria, and the record's contents). +- **Declined findings that explain a requirement's meaning.** + - EZ-9. A followed 3xx body never fails a redirect chain, including its declared length, and is read only within the bound EZ-9.11 sets. + - EZ-6. A begun request does not accept streamed uploads. TS begins only its overlapped navigation + `GET`. A Fastly body read the host cannot interrupt is bounded by one slot body's transfer time, + not by the earliest deadline. + - EZ-14. Forcing every redirect hop to the default body policy would drop a streamed caller's idle + bound on the final hop. + - EZ-17. Promoting only part of EZ-17 would make TS migrate the same report twice. + - EZ-Q1. A record covers exactly one EdgeZero commit. There is no carry-forward to a later pin. diff --git a/edgezero.toml b/edgezero.toml index 64e6e0f1f0..c234856600 100644 --- a/edgezero.toml +++ b/edgezero.toml @@ -13,6 +13,13 @@ entry = "crates/trusted-server-core" kind = "http" version = "0.1.0" +# -- Capabilities ------------------------------------------------------------ +# Publisher origins and integrations may use HTTP or HTTPS. An absent list means +# HTTPS only, and `"*"` allows both, matching the Spin manifest's allowed hosts. + +[capabilities.outbound] +hosts = ["*"] + # -- Stores ------------------------------------------------------------------ # Logical store ids only. These are the portable Trusted Server names; the # physical store each adapter binds is overridable out of band (Fastly binds diff --git a/scripts/README.md b/scripts/README.md index a2e10e72a8..1b6f57f525 100644 --- a/scripts/README.md +++ b/scripts/README.md @@ -9,17 +9,19 @@ their documented orchestration handles a probe deliberately. | `benchmark.sh` | A running server; `curl`, `bc`, and optionally `hey` | May install `hey` through Homebrew. `--save` writes under `benchmark-results/`; it does not stop the server. | | `check-ci-node-pin.mjs` | Pinned Node; tests in `check-ci-node-pin.test.mjs` | Read-only. Fails when a CI job compiles Rust before an unconditional `actions/setup-node` step pinned to `.tool-versions`; `--root` checks another checkout. | | `profile.sh` | Fastly CLI, Rust WASM target, `curl`; endpoint and request options | Builds and starts the Fastly app, stops the owned process, and retains a profile under `benchmark-results/profiles/`. `--open` launches the local viewer. | -| `generate-integration-viceroy-configs.sh` | Rust toolchain; optional origin port and artifact directory | Builds the native generator and writes Viceroy config under `target/integration-test-artifacts/`; generated files persist. | +| `generate-integration-viceroy-configs.sh` | Rust toolchain; optional origin port, artifact directory, bidder origin | Builds the native generator and writes Viceroy config under `target/integration-test-artifacts/`; generated files persist. With `INTEGRATION_BIDDER_ORIGIN_URL` set, the generator replaces the scheme and authority of every `auction.providers.*.endpoint` with that origin. | | `integration-tests.sh` | Docker, Viceroy, Rust WASM target, pinned Node | Builds WASM/native artifacts and two Docker images, generates Viceroy config, and runs native integration tests serially. Build products and images persist. | | `integration-tests-browser.sh` | The integration prerequisites plus npm and Playwright | Installs package/browser dependencies, builds fixtures and images, runs both browser suites, and stops matching test containers on exit. Build and npm artifacts persist. | -| `smoke-axum.sh` | `cargo`, `curl`, Python | Uses an isolated temporary config/store and stub origin; stops owned processes and removes its workspace. | +| `smoke-axum.sh` | `cargo`, `curl`, Python | Uses an isolated temporary config/store and stub origin. Each negative case must stop Axum at startup with a non-zero exit and log its diagnostic. Stops owned processes and removes its workspace. | | `smoke-fastly.sh` | Fastly CLI, `cargo`, `curl`, Python | Uses an isolated Fastly project and application config; stops owned processes and removes its workspace. | | `smoke-cloudflare.sh` | Pinned Wrangler, `cargo`, `curl`, `jq`, and Python | Uses isolated Wrangler manifests, KV state, ports, and logs; stops owned processes and removes its workspace. | | `smoke-spin.sh` | Spin CLI, `cargo`, `curl`, and Python | Uses an isolated Spin manifest and SQLite KV store; stops owned processes and removes its workspace. | | `smoke-common.sh` | Sourced by the four smoke scripts | Defines bounded port, process, config, secret, and response assertions. Do not invoke it as a standalone smoke. | | `async-html-finalization-local-test.sh` | Rust, pinned Node and selected runtime; repository-pinned Viceroy | Builds feature-gated fixtures, preserves compiler settings, retains diagnostics, and stops owned process groups on failure, SIGTERM or SIGINT. | -| `template-cache-local-test.sh` | Viceroy, Node, OpenSSL, `curl`, `lsof`; optional ports and mode | Builds local artifacts, generates certificates and fixtures in a temporary directory, stops owned servers, and removes the directory. | +| `template-cache-local-test.sh` | Viceroy, Node, Python, `curl`, `lsof`; optional ports and mode | Builds local artifacts, generates fixtures in a temporary directory, including a stub bidder over plain HTTP on `127.0.0.1`, stops owned servers, and removes the directory. | +| `fastly-response-diff.sh` | `--base` and `--head` revisions; Viceroy, WASM target, pinned Node, `curl` | Builds each revision in a detached worktree under `$TMPDIR`, serves both under Viceroy with a loopback origin, and diffs them through `fastly-response-diff.mjs` against `fastly-response-diff.expected.json`. Stops its servers and removes the worktrees with `git worktree remove` on exit. | | `test-cli.sh` | Rustup and the host toolchain; optional host triple | May install the selected Rust target, then runs native CLI and browser-audit tests. Cargo artifacts persist. | +| `test-worker-auction-deadlines.mjs` | Adapter, its runtime CLI, pinned Node, prebuilt example; `wasm-bindgen` | Serves the example, which sends request batches through EdgeZero's Cloudflare or Spin client, against delayed local origins. Stops the runtime and removes its temporary directory. | | `docs-proposal/propose.sh` | Commit SHA on the branch, work dir, optional base SHA; Copilot CLI, Node, `npm` | Runs Copilot CLI to edit `docs/guide/**` or `docs/index.md`, formats docs, and writes a patch, rationale, evidence, and base into the work dir. Leaves the edits staged. | | `docs-proposal/validate.sh` | Merge SHA, work dir from `propose.sh`; Node, `npm`; no write credentials | Applies the patch, rejects paths outside the allowlist, and runs the docs gates. Leaves the edits staged. | | `docs-proposal/publish.sh` | Merge SHA, work dir from `propose.sh`; `gh` with write `GH_TOKEN` | Re-checks the allowlist without building, pushes `docs/auto/` (`docs/auto//` off `main`) unless a maintainer added to it, opens or edits its PR, and reviews each new commit per hunk. An empty rerun deletes the branch under a lease, then closes the PR; a failed branch lookup stops the run. Never writes the target branch. | diff --git a/scripts/fastly-response-diff.expected.json b/scripts/fastly-response-diff.expected.json new file mode 100644 index 0000000000..00bb963ef4 --- /dev/null +++ b/scripts/fastly-response-diff.expected.json @@ -0,0 +1,30 @@ +{ + "publisher-page": { + "header connection": { "item": 6, "head": [] }, + "header keep-alive": { "item": 6, "head": [] } + }, + "head-root": { + "header connection": { "item": 6, "head": [] }, + "header keep-alive": { "item": 6, "head": [] } + }, + "conditional-get-root": { + "header connection": { "item": 6, "head": [] }, + "header keep-alive": { "item": 6, "head": [] } + }, + "first-party-proxy": { + "header connection": { "item": 6, "head": [] }, + "header keep-alive": { "item": 6, "head": [] } + }, + "asset": { + "header connection": { "item": 6, "head": [] }, + "header keep-alive": { "item": 6, "head": [] }, + "header content-length": { "item": 25, "head": ["20"] }, + "header transfer-encoding": { "item": 25, "head": [] } + }, + "trace-method": { + "header allow": { + "item": 5, + "head": ["DELETE, GET, HEAD, OPTIONS, PATCH, POST, PUT"] + } + } +} diff --git a/scripts/fastly-response-diff.mjs b/scripts/fastly-response-diff.mjs new file mode 100755 index 0000000000..829d333992 --- /dev/null +++ b/scripts/fastly-response-diff.mjs @@ -0,0 +1,600 @@ +#!/usr/bin/env node +// Diff Trusted Server responses between a base and a head server. +// +// Usage: +// node scripts/fastly-response-diff.mjs [--staging] [--origin ] \ +// +// node scripts/fastly-response-diff.mjs --serve-origin +// +// The first form sends every request in requests.json to both servers, base +// first, and compares the status, the sorted response headers, and the +// SHA-256 of the response body. Before comparing, it replaces values that +// differ between two builds of the same revision with fixed placeholders: the +// x-ts-version, x-ts-sandbox-*, date, and age headers, and generated IDs +// (UUIDs and EC IDs) in any header or body. --staging also replaces the geo +// headers (x-geo-*), x-ts-fastly-version, and x-ts-env, which differ between +// two hosted services. +// +// expected.json maps a request id to the differences that request is meant +// to show. Each difference names the "Fastly differences" item of the +// EdgeZero PR 275 cutover spec that explains it: +// +// { "head-root": { "header content-length": 7 } } +// +// A difference is "status", "body", or "header ". An entry +// can also pin the value head must show, as { "item": , "head": }: +// a number for "status", the sorted list of normalized values for a header +// ([] when head drops it), and "sha256 ( bytes)" for "body": +// +// { "trace-method": { "status": { "item": 5, "head": 405 } } } +// +// The diff exits 1 when a difference is not listed, when head shows another +// value than a pinned one, and also when a listed difference does not occur. +// It exits 2 on bad input or when a server cannot be reached. +// +// Each entry in requests.json has an "id", a "method", optional "headers", +// and an optional "body" string or "json" value. It names either a "path" or +// a "sign" target. A sign target is signed once through base's +// /first-party/sign, and the returned /first-party/proxy URL is then +// requested from both servers. In any string, "{self}" stands for the URL of +// the server being asked and "{origin}" for --origin. An optional +// "base_status" must match base's status, so a fixture that stops producing +// the case it is named for, such as a 304, fails instead of diffing the +// wrong case. +// +// The second form serves the loopback publisher origin that the checked-in +// requests expect. scripts/fastly-response-diff.sh starts it. + +import { createHash } from 'node:crypto' +import { readFileSync } from 'node:fs' +import http from 'node:http' +import https from 'node:https' +import { parseArgs } from 'node:util' + +const USAGE = `Usage: + node scripts/fastly-response-diff.mjs [--staging] [--origin ] + node scripts/fastly-response-diff.mjs --serve-origin ` + +const REQUEST_TIMEOUT_MS = 30_000 +const NORMALIZED = '' + +// Headers whose values differ between two builds of the same revision. +const VOLATILE_HEADERS = new Set(['x-ts-version', 'date', 'age']) +const VOLATILE_HEADER_PREFIXES = ['x-ts-sandbox-'] + +// Headers whose values also differ between two hosted services. +const STAGING_HEADERS = new Set(['x-ts-fastly-version', 'x-ts-env']) +const STAGING_HEADER_PREFIXES = ['x-geo-'] + +const GENERATED_IDS = [ + [/[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}/gi, ''], + [/\b[0-9a-f]{64}\.[0-9A-Za-z]{6}\b/g, ''], +] + +const DIFFERENCE_KEY = /^(?:status|body|header [!#$%&'*+.^_`|~0-9a-z-]+)$/ + +// Pages the loopback origin serves. Each route can carry an ETag, which makes +// a matching If-None-Match answer 304. +const ORIGIN_ROUTES = new Map([ + [ + '/', + { + type: 'text/html; charset=utf-8', + etag: '"fastly-response-diff-page-1"', + body: + '\nExample publisher\n' + + '

Example publisher

Fixture page for the Fastly response diff.

\n', + }, + ], + [ + '/assets/site.css', + { + type: 'text/css; charset=utf-8', + etag: '"fastly-response-diff-css-1"', + body: 'body { margin: 0; }\n', + }, + ], + [ + '/proxied/widget.js', + { + type: 'application/javascript; charset=utf-8', + body: 'window.exampleWidget = true;\n', + }, + ], +]) + +export class InputError extends Error {} + +function fail(message) { + throw new InputError(message) +} + +function serveOrigin(port) { + const server = http.createServer((request, response) => { + // Drain any request body so keep-alive connections stay usable. + request.resume() + request.on('end', () => { + const path = new URL(request.url, 'http://127.0.0.1').pathname + const route = ORIGIN_ROUTES.get(path) + console.log(`${request.method} ${request.url}`) + if (!route) { + response.writeHead(404, { 'content-type': 'text/plain; charset=utf-8' }) + response.end('not found\n') + return + } + const headers = { + 'cache-control': 'max-age=60', + 'content-type': route.type, + } + if (route.etag) { + headers.etag = route.etag + if (request.headers['if-none-match'] === route.etag) { + response.writeHead(304, headers) + response.end() + return + } + } + headers['content-length'] = Buffer.byteLength(route.body) + response.writeHead(200, headers) + response.end(request.method === 'HEAD' ? undefined : route.body) + }) + }) + server.listen(port, '127.0.0.1', () => { + console.log(`origin listening on 127.0.0.1:${port}`) + }) + for (const signal of ['SIGINT', 'SIGTERM']) { + process.on(signal, () => server.close(() => process.exit(0))) + } +} + +function readJson(path) { + let text + try { + text = readFileSync(path, 'utf8') + } catch (error) { + fail(`cannot read ${path}: ${error.message}`) + } + try { + return JSON.parse(text) + } catch (error) { + fail(`${path} is not valid JSON: ${error.message}`) + } +} + +function isStringMap(value) { + return ( + value !== null && + typeof value === 'object' && + !Array.isArray(value) && + Object.values(value).every((entry) => typeof entry === 'string') + ) +} + +function loadRequests(path) { + const requests = readJson(path) + if (!Array.isArray(requests) || requests.length === 0) { + fail(`${path} must hold a non-empty array of requests`) + } + const ids = new Set() + for (const request of requests) { + const label = `${path} request ${JSON.stringify(request?.id)}` + if (typeof request?.id !== 'string' || request.id === '') { + fail(`${label} needs a string id`) + } + if (ids.has(request.id)) { + fail(`${label} repeats an id`) + } + ids.add(request.id) + if (typeof request.method !== 'string' || request.method === '') { + fail(`${label} needs a method`) + } + if ( + (typeof request.path === 'string') === + (typeof request.sign === 'string') + ) { + fail(`${label} needs exactly one of path and sign`) + } + if (request.headers !== undefined && !isStringMap(request.headers)) { + fail(`${label} headers must map names to strings`) + } + if (request.body !== undefined && typeof request.body !== 'string') { + fail(`${label} body must be a string`) + } + if (request.body !== undefined && request.json !== undefined) { + fail(`${label} cannot carry both body and json`) + } + if ( + request.base_status !== undefined && + !Number.isInteger(request.base_status) + ) { + fail(`${label} base_status must be an integer`) + } + } + return requests +} + +function isPlainObject(value) { + return value !== null && typeof value === 'object' && !Array.isArray(value) +} + +function loadExpected(path, requests) { + return parseExpected(readJson(path), requests, path) +} + +// Validates expected.json's `expected` against `requests` and returns it with +// every difference as { item, pinned, head }, where `pinned` says whether the +// entry fixed head's value. +export function parseExpected(expected, requests, label) { + if (!isPlainObject(expected)) { + fail(`${label} must hold an object`) + } + const ids = new Set(requests.map((request) => request.id)) + const parsed = {} + for (const [id, differences] of Object.entries(expected)) { + if (!ids.has(id)) { + fail(`${label} names unknown request ${JSON.stringify(id)}`) + } + if (!isPlainObject(differences)) { + fail(`${label} entry ${JSON.stringify(id)} must map differences to items`) + } + parsed[id] = {} + for (const [key, entry] of Object.entries(differences)) { + const where = `${label} entry ${JSON.stringify(id)} difference ${JSON.stringify(key)}` + if (!DIFFERENCE_KEY.test(key)) { + fail( + `${label} entry ${JSON.stringify(id)} has malformed difference ${JSON.stringify(key)}` + ) + } + const pinned = isPlainObject(entry) + const item = pinned ? entry.item : entry + if (!Number.isInteger(item) || item < 1) { + fail(`${where} must name a Fastly differences item number`) + } + if (pinned) { + const fields = Object.keys(entry).sort() + if (JSON.stringify(fields) !== JSON.stringify(['head', 'item'])) { + fail(`${where} must hold exactly "item" and "head"`) + } + } + parsed[id][key] = { item, pinned, head: pinned ? entry.head : undefined } + } + } + return parsed +} + +// Sorts one request's `differences` against its `listed` expectations. +// +// Returns one { kind, key, item, base, head } per difference and per listed +// difference that did not occur. `kind` is "expected", "unexpected", +// "mismatched" (listed, but head shows another value than the pinned one), +// or "missing". +export function classifyDifferences(differences, listed) { + const results = [] + for (const [key, { base, head }] of differences) { + const expectation = listed[key] + let kind = 'expected' + if (expectation === undefined) { + kind = 'unexpected' + } else if ( + expectation.pinned && + JSON.stringify(expectation.head) !== JSON.stringify(head) + ) { + kind = 'mismatched' + } + results.push({ kind, key, item: expectation?.item, base, head }) + } + for (const [key, expectation] of Object.entries(listed)) { + if (!differences.has(key)) { + results.push({ kind: 'missing', key, item: expectation.item }) + } + } + return results +} + +function substitute(text, placeholders) { + return text.replace(/\{(self|origin)\}/g, (match, name) => { + const value = placeholders[name] + if (value === undefined) { + fail(`${match} needs a value; pass --origin`) + } + return value + }) +} + +function send(serverUrl, method, path, headers, body) { + const url = new URL(path, serverUrl) + const transport = url.protocol === 'https:' ? https : http + const requestHeaders = { ...headers } + if (body !== undefined) { + requestHeaders['content-length'] = String(Buffer.byteLength(body)) + } + return new Promise((resolve, reject) => { + const request = transport.request( + url, + { method, headers: requestHeaders, timeout: REQUEST_TIMEOUT_MS }, + (response) => { + const chunks = [] + response.on('data', (chunk) => chunks.push(chunk)) + response.on('error', reject) + response.on('end', () => { + const pairs = [] + for (let index = 0; index < response.rawHeaders.length; index += 2) { + pairs.push([ + response.rawHeaders[index].toLowerCase(), + response.rawHeaders[index + 1], + ]) + } + resolve({ + status: response.statusCode, + headers: pairs, + body: Buffer.concat(chunks), + }) + }) + } + ) + request.on('timeout', () => + request.destroy(new Error(`timed out after ${REQUEST_TIMEOUT_MS} ms`)) + ) + request.on('error', reject) + request.end(body) + }) +} + +async function sendOrFail(serverUrl, method, path, headers, body) { + try { + return await send(serverUrl, method, path, headers, body) + } catch (error) { + fail(`${method} ${path} to ${serverUrl} failed: ${error.message}`) + } +} + +function placeholdersFor(serverUrl, placeholders) { + return { ...placeholders, self: new URL(serverUrl).origin } +} + +// Signs a request's target once, through base, so both servers receive the +// same /first-party/proxy URL. +async function signedPath(request, baseUrl, placeholders) { + const target = substitute( + request.sign, + placeholdersFor(baseUrl, placeholders) + ) + const signPath = `/first-party/sign?url=${encodeURIComponent(target)}` + const response = await sendOrFail(baseUrl, 'GET', signPath, {}, undefined) + if (response.status !== 200) { + fail( + `request ${request.id}: signing ${target} on base returned ${response.status}` + ) + } + let href + try { + href = JSON.parse(response.body.toString('utf8')).href + } catch { + fail(`request ${request.id}: base signing response is not JSON`) + } + if (typeof href !== 'string' || href === '') { + fail(`request ${request.id}: base signing response has no href`) + } + const signed = new URL(href, baseUrl) + return `${signed.pathname}${signed.search}` +} + +function scrubGeneratedIds(text) { + return GENERATED_IDS.reduce( + (scrubbed, [pattern, placeholder]) => + scrubbed.replace(pattern, placeholder), + text + ) +} + +function isNormalizedHeader(name, staging) { + if ( + VOLATILE_HEADERS.has(name) || + VOLATILE_HEADER_PREFIXES.some((prefix) => name.startsWith(prefix)) + ) { + return true + } + return ( + staging && + (STAGING_HEADERS.has(name) || + STAGING_HEADER_PREFIXES.some((prefix) => name.startsWith(prefix))) + ) +} + +function normalize(response, staging) { + const headers = new Map() + for (const [name, value] of response.headers) { + const normalized = isNormalizedHeader(name, staging) + ? NORMALIZED + : scrubGeneratedIds(value) + headers.set(name, [...(headers.get(name) ?? []), normalized]) + } + for (const values of headers.values()) { + values.sort() + } + // Latin-1 maps every byte to one code unit, so the scrub cannot alter + // bytes outside the IDs it replaces. + const body = Buffer.from( + scrubGeneratedIds(response.body.toString('latin1')), + 'latin1' + ) + return { + status: response.status, + headers, + bodyHash: createHash('sha256').update(body).digest('hex'), + bodyLength: body.length, + } +} + +function compare(base, head) { + const differences = new Map() + if (base.status !== head.status) { + differences.set('status', { base: base.status, head: head.status }) + } + const names = [...new Set([...base.headers.keys(), ...head.headers.keys()])] + for (const name of names.sort()) { + const baseValues = base.headers.get(name) ?? [] + const headValues = head.headers.get(name) ?? [] + if (JSON.stringify(baseValues) !== JSON.stringify(headValues)) { + differences.set(`header ${name}`, { base: baseValues, head: headValues }) + } + } + if (base.bodyHash !== head.bodyHash) { + differences.set('body', { + base: `sha256 ${base.bodyHash} (${base.bodyLength} bytes)`, + head: `sha256 ${head.bodyHash} (${head.bodyLength} bytes)`, + }) + } + return differences +} + +function describe(value) { + return Array.isArray(value) + ? value.length === 0 + ? '(absent)' + : value.map((entry) => JSON.stringify(entry)).join(', ') + : String(value) +} + +async function runDiff(values, positionals) { + if (positionals.length !== 4) { + fail(USAGE) + } + const [baseUrl, headUrl, requestsPath, expectedPath] = positionals + for (const url of [baseUrl, headUrl]) { + if (!URL.canParse(url)) { + fail(`${url} is not a URL`) + } + } + const requests = loadRequests(requestsPath) + const expected = loadExpected(expectedPath, requests) + const placeholders = + values.origin === undefined ? {} : { origin: values.origin } + const staging = values.staging === true + + let unexpected = 0 + let missing = 0 + for (const request of requests) { + const signed = + request.sign === undefined + ? undefined + : await signedPath(request, baseUrl, placeholders) + const results = [] + for (const serverUrl of [baseUrl, headUrl]) { + const local = placeholdersFor(serverUrl, placeholders) + const path = signed ?? substitute(request.path, local) + const headers = Object.fromEntries( + Object.entries(request.headers ?? {}).map(([name, value]) => [ + name, + substitute(value, local), + ]) + ) + const body = + request.json !== undefined + ? JSON.stringify(request.json) + : request.body === undefined + ? undefined + : substitute(request.body, local) + const response = await sendOrFail( + serverUrl, + request.method, + path, + headers, + body + ) + results.push(normalize(response, staging)) + } + const [base, head] = results + if ( + request.base_status !== undefined && + base.status !== request.base_status + ) { + fail( + `request ${request.id}: base answered ${base.status}, but the fixture expects ${request.base_status}` + ) + } + + const differences = compare(base, head) + const listed = expected[request.id] ?? {} + if (differences.size === 0 && Object.keys(listed).length === 0) { + console.log(`same ${request.id}`) + } + for (const result of classifyDifferences(differences, listed)) { + const { kind, key, item } = result + if (kind === 'missing') { + missing += 1 + console.log( + `missing ${request.id}: ${key} (item ${item}) did not differ` + ) + continue + } + if (kind === 'expected') { + console.log(`expected ${request.id}: ${key} (item ${item})`) + } else if (kind === 'mismatched') { + unexpected += 1 + console.log( + `mismatched ${request.id}: ${key} (item ${item}) head differs from the pinned ${describe(listed[key].head)}` + ) + } else { + unexpected += 1 + console.log(`unexpected ${request.id}: ${key}`) + } + console.log(` base: ${describe(result.base)}`) + console.log(` head: ${describe(result.head)}`) + } + } + + console.log( + `${requests.length} requests, ${unexpected} unexpected differences, ${missing} expected differences missing` + ) + return unexpected === 0 && missing === 0 ? 0 : 1 +} + +async function main() { + let parsed + try { + parsed = parseArgs({ + allowPositionals: true, + options: { + help: { type: 'boolean' }, + origin: { type: 'string' }, + 'serve-origin': { type: 'string' }, + staging: { type: 'boolean' }, + }, + }) + } catch (error) { + fail(`${error.message}\n${USAGE}`) + } + const { values, positionals } = parsed + if (values.help) { + console.log(USAGE) + return 0 + } + if (values['serve-origin'] !== undefined) { + const port = Number(values['serve-origin']) + if (!Number.isInteger(port) || port < 1 || port > 65_535) { + fail(`--serve-origin needs a port, got ${values['serve-origin']}`) + } + serveOrigin(port) + return undefined + } + return runDiff(values, positionals) +} + +// Run only as a command, so the test file can import the compare logic. +// `import.meta.main` holds through a symlinked path, where comparing the +// module URL with `process.argv[1]` would skip `main` and exit 0 silently. +if (import.meta.main) { + try { + const code = await main() + if (code !== undefined) { + process.exitCode = code + } + } catch (error) { + if (!(error instanceof InputError)) { + throw error + } + console.error(`fastly-response-diff: ${error.message}`) + process.exitCode = 2 + } +} diff --git a/scripts/fastly-response-diff.requests.json b/scripts/fastly-response-diff.requests.json new file mode 100644 index 0000000000..46a10f3cf8 --- /dev/null +++ b/scripts/fastly-response-diff.requests.json @@ -0,0 +1,96 @@ +[ + { + "id": "publisher-page", + "method": "GET", + "path": "/", + "headers": { + "accept": "text/html", + "sec-fetch-dest": "document", + "sec-fetch-mode": "navigate" + }, + "base_status": 200 + }, + { + "id": "head-root", + "method": "HEAD", + "path": "/", + "headers": { + "accept": "text/html" + }, + "base_status": 200 + }, + { + "id": "conditional-get-root", + "method": "GET", + "path": "/", + "headers": { + "accept": "text/html", + "if-none-match": "\"fastly-response-diff-page-1\"", + "sec-fetch-dest": "document", + "sec-fetch-mode": "navigate" + }, + "base_status": 304 + }, + { + "id": "post-auction", + "method": "POST", + "path": "/auction", + "headers": { + "content-type": "application/json" + }, + "json": { + "adUnits": [ + { + "code": "diff-slot", + "mediaTypes": { "banner": { "sizes": [[300, 250]] } } + } + ] + }, + "base_status": 200 + }, + { + "id": "first-party-proxy", + "method": "GET", + "sign": "{origin}/proxied/widget.js", + "base_status": 200 + }, + { + "id": "asset", + "method": "GET", + "path": "/assets/site.css", + "base_status": 200 + }, + { + "id": "tsjs-bundle", + "method": "GET", + "path": "/static/tsjs=tsjs-unified.min.js", + "base_status": 200 + }, + { + "id": "admin-eids", + "method": "GET", + "path": "/_ts/admin/eids", + "headers": { + "authorization": "Basic YWRtaW46aW50ZWdyYXRpb24tYWRtaW4tcGFzc3dvcmQtMzItYnl0ZXMtb2s=" + }, + "base_status": 200 + }, + { + "id": "trace-method", + "method": "TRACE", + "path": "/", + "base_status": 405 + }, + { + "id": "trace-enable-action", + "method": "POST", + "path": "/_ts/trace/enable", + "headers": { + "cookie": "__Host-ts-console=1", + "origin": "{self}", + "sec-fetch-site": "same-origin", + "x-ts-trace-action": "enable" + }, + "base_status": 200 + } +] diff --git a/scripts/fastly-response-diff.sh b/scripts/fastly-response-diff.sh new file mode 100755 index 0000000000..7eb3f0a80a --- /dev/null +++ b/scripts/fastly-response-diff.sh @@ -0,0 +1,237 @@ +#!/usr/bin/env bash +# +# Diff Fastly responses between two Trusted Server revisions under Viceroy. +# +# Usage: scripts/fastly-response-diff.sh --base --head +# +# Checks out each revision in a temporary detached worktree under $TMPDIR, +# builds its release Fastly WASM there, and serves it under Viceroy on its own +# port. Both use the revision's integration fixture config with two additions: +# the trace page is enabled, so the trace action reaches its handler, and one +# asset route points at the loopback origin. A Node origin serves the +# publisher pages for both. scripts/fastly-response-diff.mjs then sends +# scripts/fastly-response-diff.requests.json to both servers and checks the +# differences against scripts/fastly-response-diff.expected.json. +# +# Exits with the diff's status: 0 when every difference is expected, 1 when +# one is not or an expected one is missing, 2 on bad input. Build, config or +# startup failures exit 1 after printing the failing log. Worktrees, servers +# and logs are removed on exit. +# +# Environment overrides: FASTLY_RESPONSE_DIFF_BASE_PORT (default 17690), +# FASTLY_RESPONSE_DIFF_HEAD_PORT (17691), FASTLY_RESPONSE_DIFF_ORIGIN_PORT +# (17692). +set -euo pipefail + +REPO_ROOT="$(cd "$(dirname "$0")/.." && pwd)" +# Tool shims such as asdf's resolve versions from .tool-versions in the +# working directory, so run from the checkout. +cd "$REPO_ROOT" +BASE_PORT="${FASTLY_RESPONSE_DIFF_BASE_PORT:-17690}" +HEAD_PORT="${FASTLY_RESPONSE_DIFF_HEAD_PORT:-17691}" +ORIGIN_PORT="${FASTLY_RESPONSE_DIFF_ORIGIN_PORT:-17692}" +DIFF_SCRIPT="$REPO_ROOT/scripts/fastly-response-diff.mjs" +REQUESTS="$REPO_ROOT/scripts/fastly-response-diff.requests.json" +EXPECTED="$REPO_ROOT/scripts/fastly-response-diff.expected.json" +FIXTURE="crates/trusted-server-integration-tests/fixtures/configs/trusted-server.integration.toml" +WASM="target/wasm32-wasip1/release/trusted-server-adapter-fastly.wasm" + +usage() { + echo "Usage: $0 --base --head " >&2 + exit 2 +} + +die() { + echo "fastly-response-diff: $*" >&2 + exit 1 +} + +BASE_REV="" +HEAD_REV="" +while [ "$#" -gt 0 ]; do + case "$1" in + --base) + [ "$#" -ge 2 ] || usage + BASE_REV="$2" + shift 2 + ;; + --head) + [ "$#" -ge 2 ] || usage + HEAD_REV="$2" + shift 2 + ;; + -h | --help) + usage + ;; + *) + echo "fastly-response-diff: unknown argument: $1" >&2 + usage + ;; + esac +done +if [ -z "$BASE_REV" ] || [ -z "$HEAD_REV" ]; then + usage +fi + +for command in git cargo viceroy node curl awk; do + command -v "$command" >/dev/null 2>&1 || die "required command not found: $command" +done + +BASE_SHA="$(git -C "$REPO_ROOT" rev-parse --verify --quiet "$BASE_REV^{commit}")" || + die "cannot resolve --base $BASE_REV" +HEAD_SHA="$(git -C "$REPO_ROOT" rev-parse --verify --quiet "$HEAD_REV^{commit}")" || + die "cannot resolve --head $HEAD_REV" + +port_in_use() { + (exec 3<>"/dev/tcp/127.0.0.1/$1") 2>/dev/null +} + +for port in "$BASE_PORT" "$HEAD_PORT" "$ORIGIN_PORT"; do + if port_in_use "$port"; then + die "port $port is already in use; set the FASTLY_RESPONSE_DIFF_*_PORT overrides" + fi +done + +WORK="$(mktemp -d "${TMPDIR:-/tmp}/fastly-response-diff.XXXXXX")" +# Resolve symlinks such as macOS's /var -> /private/var, so the paths match +# what git records for the worktrees. +WORK="$(cd "$WORK" && pwd -P)" +ADDED_TREES=() +PIDS=() + +# The EXIT trap runs this. ShellCheck 0.10 reports a trap handler as +# unreachable when the script ends in `exit`. +# shellcheck disable=SC2317 +cleanup() { + local status=$? + local pid + for pid in "${PIDS[@]+"${PIDS[@]}"}"; do + kill "$pid" 2>/dev/null || true + wait "$pid" 2>/dev/null || true + done + local removed=true + local tree + for tree in "${ADDED_TREES[@]+"${ADDED_TREES[@]}"}"; do + if ! git -C "$REPO_ROOT" worktree remove "$tree"; then + removed=false + fi + done + if [ "$removed" = true ]; then + rm -rf -- "$WORK" + else + echo "fastly-response-diff: a worktree could not be removed; inspect $WORK and run git worktree remove on it" >&2 + fi + exit "$status" +} +trap cleanup EXIT +trap 'exit 130' INT +trap 'exit 143' TERM +mkdir "$WORK/logs" + +show_log() { + echo "---- last lines of $1 ----" >&2 + tail -n 40 "$1" >&2 || true +} + +# Writes the revision's integration fixture with the trace page enabled and +# an asset route to the loopback origin. +write_app_config() { + local tree="$1" + local output="$2" + [ -f "$tree/$FIXTURE" ] || die "$FIXTURE is missing at $(git -C "$tree" rev-parse --short HEAD)" + awk ' + { print } + $0 == "[integrations.gpt_diagnostics]" { + print "trace_page_enabled = true" + inserted++ + } + END { + if (inserted != 1) { + print "expected one [integrations.gpt_diagnostics] table; found " (inserted + 0) > "/dev/stderr" + exit 1 + } + } + ' "$tree/$FIXTURE" >"$output" + cat >>"$output" < Building $side ($(git -C "$tree" rev-parse --short HEAD))..." + # Each worktree keeps its own target directory, so the two builds never + # share artifacts. + if ! (cd "$tree" && CARGO_TARGET_DIR="$tree/target" cargo build \ + --package trusted-server-adapter-fastly --release --target wasm32-wasip1) >"$log" 2>&1; then + show_log "$log" + die "$side build failed" + fi + if ! (cd "$tree" && + INTEGRATION_ORIGIN_PORT="$ORIGIN_PORT" \ + ARTIFACTS_DIR="$WORK/$side-artifacts" \ + INTEGRATION_APP_CONFIG_PATH="$WORK/$side-app-config.toml" \ + ./scripts/generate-integration-viceroy-configs.sh) >>"$log" 2>&1; then + show_log "$log" + die "$side Viceroy config generation failed" + fi +} + +# Waits until `url` answers 200, or fails after about 60 seconds. +wait_for_ok() { + local url="$1" + local log="$2" + local attempt + for attempt in $(seq 1 120); do + if [ "$(curl --silent --output /dev/null --write-out '%{http_code}' "$url" || true)" = "200" ]; then + return 0 + fi + sleep 0.5 + done + show_log "$log" + die "$url did not answer 200 after $attempt attempts" +} + +prepare base "$BASE_SHA" +prepare head "$HEAD_SHA" + +echo "==> Starting the loopback origin on port $ORIGIN_PORT..." +node "$DIFF_SCRIPT" --serve-origin "$ORIGIN_PORT" >"$WORK/logs/origin.log" 2>&1 & +PIDS+=("$!") +wait_for_ok "http://127.0.0.1:$ORIGIN_PORT/" "$WORK/logs/origin.log" + +for side in base head; do + if [ "$side" = base ]; then port="$BASE_PORT"; else port="$HEAD_PORT"; fi + echo "==> Serving $side under Viceroy on port $port..." + viceroy "$WORK/$side/$WASM" \ + -C "$WORK/$side-artifacts/configs/viceroy.toml" \ + --addr "127.0.0.1:$port" >"$WORK/logs/$side-viceroy.log" 2>&1 & + PIDS+=("$!") + wait_for_ok "http://127.0.0.1:$port/health" "$WORK/logs/$side-viceroy.log" +done + +echo "==> Diffing $(git -C "$REPO_ROOT" rev-parse --short "$BASE_SHA") against $(git -C "$REPO_ROOT" rev-parse --short "$HEAD_SHA")..." +diff_status=0 +node "$DIFF_SCRIPT" \ + --origin "http://127.0.0.1:$ORIGIN_PORT" \ + "http://127.0.0.1:$BASE_PORT" \ + "http://127.0.0.1:$HEAD_PORT" \ + "$REQUESTS" \ + "$EXPECTED" || diff_status=$? +if [ "$diff_status" -eq 2 ]; then + show_log "$WORK/logs/base-viceroy.log" + show_log "$WORK/logs/head-viceroy.log" +fi +exit "$diff_status" diff --git a/scripts/fastly-response-diff.test.mjs b/scripts/fastly-response-diff.test.mjs new file mode 100644 index 0000000000..0415db5742 --- /dev/null +++ b/scripts/fastly-response-diff.test.mjs @@ -0,0 +1,182 @@ +import assert from 'node:assert/strict' +import { spawnSync } from 'node:child_process' +import { mkdtempSync, rmSync, symlinkSync } from 'node:fs' +import { tmpdir } from 'node:os' +import path from 'node:path' +import { after, test } from 'node:test' +import { fileURLToPath } from 'node:url' + +import { + InputError, + classifyDifferences, + parseExpected, +} from './fastly-response-diff.mjs' + +const SCRIPT = fileURLToPath( + new URL('./fastly-response-diff.mjs', import.meta.url) +) + +const scratch = mkdtempSync(path.join(tmpdir(), 'fastly-response-diff-')) +after(() => rmSync(scratch, { recursive: true, force: true })) + +let links = 0 + +/** Runs the script through a symlink to it, the way a symlinked checkout or macOS /tmp would. */ +function runThroughSymlink(args) { + links += 1 + const link = path.join(scratch, `linked-${links}.mjs`) + symlinkSync(SCRIPT, link) + return spawnSync(process.execPath, [link, ...args], { encoding: 'utf8' }) +} + +test('the script runs as a command through a symlinked path', () => { + const result = runThroughSymlink(['--help']) + + assert.equal(result.status, 0, result.stderr) + assert.match( + result.stdout, + /^Usage:/, + 'should print the usage instead of skipping main' + ) +}) + +test('the script rejects bad arguments through a symlinked path', () => { + const result = runThroughSymlink(['http://127.0.0.1:1']) + + assert.equal(result.status, 2, 'should exit 2 on bad input, not 0') + assert.match( + result.stderr, + /fastly-response-diff: Usage:/, + 'should report the bad input' + ) +}) + +const REQUESTS = [{ id: 'trace-method' }, { id: 'head-root' }] + +/** One request's differences, as `compare` builds them. */ +function differences(entries) { + return new Map(entries) +} + +test('parseExpected accepts a bare item and a pinned head value', () => { + const parsed = parseExpected( + { + 'trace-method': { status: { item: 5, head: 405 } }, + 'head-root': { 'header content-length': 7 }, + }, + REQUESTS, + 'expected.json' + ) + + assert.deepEqual(parsed, { + 'trace-method': { status: { item: 5, pinned: true, head: 405 } }, + 'head-root': { + 'header content-length': { item: 7, pinned: false, head: undefined }, + }, + }) +}) + +test('parseExpected rejects a pinned entry without exactly item and head', () => { + for (const entry of [{ item: 5 }, { item: 5, head: 405, note: 'x' }]) { + assert.throws( + () => + parseExpected( + { 'trace-method': { status: entry } }, + REQUESTS, + 'expected.json' + ), + (error) => + error instanceof InputError && + error.message.includes('must hold exactly "item" and "head"') + ) + } +}) + +test('parseExpected rejects a pinned entry with no item number', () => { + assert.throws( + () => + parseExpected( + { 'trace-method': { status: { item: 0, head: 405 } } }, + REQUESTS, + 'expected.json' + ), + (error) => + error instanceof InputError && + error.message.includes('must name a Fastly differences item number') + ) +}) + +test('classifyDifferences matches a key-only entry on the key alone', () => { + const listed = parseExpected( + { 'head-root': { 'header content-length': 7 } }, + REQUESTS, + 'expected.json' + )['head-root'] + + const results = classifyDifferences( + differences([['header content-length', { base: ['12'], head: [] }]]), + listed + ) + + assert.deepEqual( + results.map(({ kind, key }) => [kind, key]), + [['expected', 'header content-length']] + ) +}) + +test('classifyDifferences accepts a pinned head value that matches', () => { + const listed = parseExpected( + { 'trace-method': { status: { item: 5, head: 405 } } }, + REQUESTS, + 'expected.json' + )['trace-method'] + + const results = classifyDifferences( + differences([['status', { base: 501, head: 405 }]]), + listed + ) + + assert.deepEqual( + results.map(({ kind, item }) => [kind, item]), + [['expected', 5]] + ) +}) + +test('classifyDifferences flags a pinned head value that differs', () => { + const listed = parseExpected( + { 'trace-method': { status: { item: 5, head: 405 } } }, + REQUESTS, + 'expected.json' + )['trace-method'] + + const results = classifyDifferences( + differences([['status', { base: 405, head: 500 }]]), + listed + ) + + assert.deepEqual( + results.map(({ kind, head }) => [kind, head]), + [['mismatched', 500]] + ) +}) + +test('classifyDifferences reports unlisted and missing differences', () => { + const listed = parseExpected( + { 'head-root': { 'header content-length': 7 } }, + REQUESTS, + 'expected.json' + )['head-root'] + + const results = classifyDifferences( + differences([['header etag', { base: ['"a"'], head: ['"b"'] }]]), + listed + ) + + assert.deepEqual( + results.map(({ kind, key }) => [kind, key]), + [ + ['unexpected', 'header etag'], + ['missing', 'header content-length'], + ] + ) +}) diff --git a/scripts/generate-integration-viceroy-configs.sh b/scripts/generate-integration-viceroy-configs.sh index eef2a0e3c6..8ff18aac27 100755 --- a/scripts/generate-integration-viceroy-configs.sh +++ b/scripts/generate-integration-viceroy-configs.sh @@ -43,7 +43,7 @@ GENERATOR_ARGS=( --origin-url "$ORIGIN_URL" ) if [ -n "${INTEGRATION_BIDDER_ORIGIN_URL:-}" ]; then - GENERATOR_ARGS+=(--bidder-origin-url "$INTEGRATION_BIDDER_ORIGIN_URL") + GENERATOR_ARGS+=(--provider-endpoint-origin "$INTEGRATION_BIDDER_ORIGIN_URL") fi "$GENERATOR_BIN" "${GENERATOR_ARGS[@]}" diff --git a/scripts/integration-tests-browser.sh b/scripts/integration-tests-browser.sh index 12f152dce5..53849d058c 100755 --- a/scripts/integration-tests-browser.sh +++ b/scripts/integration-tests-browser.sh @@ -36,7 +36,6 @@ TRUSTED_SERVER__PUBLISHER__ORIGIN_URL="http://127.0.0.1:$ORIGIN_PORT" \ TRUSTED_SERVER__PUBLISHER__PROXY_SECRET="integration-test-proxy-secret" \ TRUSTED_SERVER__EC__PASSPHRASE="integration-test-ec-secret-padded-32" \ TRUSTED_SERVER__EC__PARTNERS='[{"name":"Integration Test Partner","source_domain":"inttest.example.com","bidstream_enabled":true,"api_token":"integration-test-token-alpha-32-bytes-ok"},{"name":"Integration Test Partner 2","source_domain":"inttest2.example.com","bidstream_enabled":true,"api_token":"integration-test-token-bravo-32-bytes-ok"}]' \ -TRUSTED_SERVER__PROXY__CERTIFICATE_CHECK=false \ cargo build --package trusted-server-adapter-fastly --release --target wasm32-wasip1 echo "==> Generating Viceroy configs..." diff --git a/scripts/integration-tests.sh b/scripts/integration-tests.sh index 621267acb4..476825d105 100755 --- a/scripts/integration-tests.sh +++ b/scripts/integration-tests.sh @@ -73,7 +73,6 @@ echo "==> Building Axum native binary (origin=http://127.0.0.1:$ORIGIN_PORT)..." TRUSTED_SERVER__PUBLISHER__ORIGIN_URL="http://127.0.0.1:$ORIGIN_PORT" \ TRUSTED_SERVER__PUBLISHER__PROXY_SECRET="integration-test-proxy-secret" \ TRUSTED_SERVER__EDGE_COOKIE__SECRET_KEY="integration-test-secret-key" \ -TRUSTED_SERVER__PROXY__CERTIFICATE_CHECK=false \ cargo build -p trusted-server-adapter-axum echo "==> Generating Viceroy configs..." diff --git a/scripts/smoke-axum.sh b/scripts/smoke-axum.sh index ae957e4ba6..648e65e355 100755 --- a/scripts/smoke-axum.sh +++ b/scripts/smoke-axum.sh @@ -56,6 +56,11 @@ ENVELOPE=$(jq --exit-status --raw-output \ '.trusted_server_config' \ "$WORKSPACE/.edgezero/local-config-trusted_server_config.json") +# Starts Axum with the given config and secret values, or fails at startup. +# +# Without `expected_diagnostic` the publisher page must answer 200 through +# the stub origin. With it, startup must fail: Axum exits non-zero before it +# listens and logs the diagnostic. run_case() { local case_name="$1" local port="$2" @@ -63,8 +68,7 @@ run_case() { local handler_value="$4" local proxy_value="$5" local ec_value="$6" - local expected_status="$7" - local expected_diagnostic="${8:-}" + local expected_diagnostic="${7:-}" local log_path="$WORKSPACE/$case_name.log" local body_path="$WORKSPACE/$case_name.body" local headers_path="$WORKSPACE/$case_name.headers" @@ -86,33 +90,58 @@ run_case() { PORT="$port" \ "$AXUM_BINARY" >"$log_path" 2>&1 & APP_PID=$! - smoke_wait_http "http://127.0.0.1:$port/" "$expected_status" \ - "$body_path" "$headers_path" + + if [ -n "$expected_diagnostic" ]; then + assert_startup_failure "$case_name" "$log_path" "$expected_diagnostic" + return + fi + + smoke_wait_http "http://127.0.0.1:$port/" 200 "$body_path" "$headers_path" smoke_assert_process_alive "$APP_PID" "Axum" "$log_path" smoke_assert_process_alive "$ORIGIN_PID" "stub origin" "$WORKSPACE/origin.log" smoke_stop_process "$APP_PID" APP_PID="" + smoke_assert_success "$body_path" "$ORIGIN_PORT" "$port" +} - if [ -n "$expected_diagnostic" ]; then - smoke_assert_failure "$expected_status" "$expected_diagnostic" "$log_path" - else - smoke_assert_success "$body_path" "$ORIGIN_PORT" "$port" +# Waits up to 15 seconds for Axum to exit, then requires a non-zero status and +# the diagnostic in its log. +assert_startup_failure() { + local case_name="$1" + local log_path="$2" + local expected_diagnostic="$3" + local attempt=0 + while kill -0 "$APP_PID" 2>/dev/null; do + if [ "$attempt" -ge 60 ]; then + sed -n '1,120p' "$log_path" >&2 + smoke_die "$case_name: Axum kept running after a startup failure" + fi + attempt=$((attempt + 1)) + sleep 0.25 + done + local status=0 + wait "$APP_PID" || status=$? + APP_PID="" + if [ "$status" -eq 0 ]; then + sed -n '1,120p' "$log_path" >&2 + smoke_die "$case_name: Axum exited 0 after a startup failure" fi + smoke_wait_log "$log_path" "$expected_diagnostic" } run_case missing-config "$BASE_PORT" "" \ - "$SMOKE_HANDLER_VALUE" "$SMOKE_PROXY_VALUE" "$SMOKE_EC_VALUE" 500 \ + "$SMOKE_HANDLER_VALUE" "$SMOKE_PROXY_VALUE" "$SMOKE_EC_VALUE" \ "env var '$SMOKE_CONFIG_ENV' not set" run_case missing-handler "$((BASE_PORT + 1))" "$ENVELOPE" \ - "" "$SMOKE_PROXY_VALUE" "$SMOKE_EC_VALUE" 500 \ + "" "$SMOKE_PROXY_VALUE" "$SMOKE_EC_VALUE" \ "failed to resolve secret reference at \`handlers[0].password\`" run_case missing-proxy "$((BASE_PORT + 2))" "$ENVELOPE" \ - "$SMOKE_HANDLER_VALUE" "" "$SMOKE_EC_VALUE" 500 \ + "$SMOKE_HANDLER_VALUE" "" "$SMOKE_EC_VALUE" \ "failed to resolve secret reference at \`publisher.proxy_secret\`" run_case missing-ec "$((BASE_PORT + 3))" "$ENVELOPE" \ - "$SMOKE_HANDLER_VALUE" "$SMOKE_PROXY_VALUE" "" 500 \ + "$SMOKE_HANDLER_VALUE" "$SMOKE_PROXY_VALUE" "" \ "failed to resolve secret reference at \`ec.passphrase\`" run_case positive "$POSITIVE_PORT" "$ENVELOPE" \ - "$SMOKE_HANDLER_VALUE" "$SMOKE_PROXY_VALUE" "$SMOKE_EC_VALUE" 200 + "$SMOKE_HANDLER_VALUE" "$SMOKE_PROXY_VALUE" "$SMOKE_EC_VALUE" echo "Axum first-success smoke passed" diff --git a/scripts/template-cache-local-test.sh b/scripts/template-cache-local-test.sh index bc683e410c..63da26bcd4 100755 --- a/scripts/template-cache-local-test.sh +++ b/scripts/template-cache-local-test.sh @@ -79,10 +79,6 @@ command -v node >/dev/null || { echo "node not found. The harness executes the real GPT bundle to verify slot setup." >&2 exit 1 } -command -v openssl >/dev/null || { - echo "openssl not found. The harness needs it for the local HTTPS bid endpoint." >&2 - exit 1 -} # A port already in use means requests would go to something else entirely — most # likely a leftover run, whose warm cache and stale config would read as a result. @@ -101,25 +97,9 @@ cargo build -p trusted-server-cli --target "$HOST_TRIPLE" >/dev/null WASM="$REPO_ROOT/target/wasm32-wasip1/debug/trusted-server-adapter-fastly.wasm" TS="$REPO_ROOT/target/$HOST_TRIPLE/debug/ts" -info "Generating a local CA and HTTPS bid certificate" -openssl req -x509 -newkey rsa:2048 -sha256 -days 1 -nodes \ - -subj "/CN=Trusted Server local harness CA" \ - -keyout "$WORK/ca-key.pem" -out "$WORK/ca-cert.pem" >/dev/null 2>&1 -openssl req -newkey rsa:2048 -sha256 -nodes -subj "/CN=localhost" \ - -keyout "$WORK/server-key.pem" -out "$WORK/server.csr" >/dev/null 2>&1 -cat > "$WORK/server.ext" <<'EOF' -basicConstraints=CA:FALSE -keyUsage=digitalSignature,keyEncipherment -extendedKeyUsage=serverAuth -subjectAltName=DNS:localhost,IP:127.0.0.1 -EOF -openssl x509 -req -sha256 -days 1 -in "$WORK/server.csr" \ - -CA "$WORK/ca-cert.pem" -CAkey "$WORK/ca-key.pem" -CAcreateserial \ - -extfile "$WORK/server.ext" -out "$WORK/server-cert.pem" >/dev/null 2>&1 - -info "Starting stub origin on :$ORIGIN_PORT and HTTPS bidder on :$BID_PORT" +info "Starting stub origin on :$ORIGIN_PORT and HTTP bidder on :$BID_PORT" cat > "$WORK/origin.py" < "$WORK/origin.log" 2>&1 & @@ -281,7 +258,7 @@ s = replace_once( 'cookie_domain = ".local-harness.example"', "publisher cookie domain", ) -# A real auction points at the slow HTTPS stub so the timings mean something. +# A real auction points at the slow loopback HTTP stub so the timings mean something. s = replace_once( s, '[integrations.prebid]\nenabled = false', @@ -292,7 +269,7 @@ s = replace_once( s = replace_once( s, 'endpoint = "https://prebid.example.com/openrtb2/auction"', - f'endpoint = "https://localhost:{bid_port}/bid"\ntimeout_ms = 5000', + f'endpoint = "http://127.0.0.1:{bid_port}/bid"\ntimeout_ms = 5000', "Prebid provider endpoint", ) s = replace_once( @@ -353,50 +330,6 @@ info "Seeding an isolated config store (tracked fastly.toml remains untouched)" cp "$REPO_ROOT/edgezero.toml" "$WORK/edgezero.toml" cp "$REPO_ROOT/fastly.toml" "$WORK/fastly.toml" -# The application registers provider backends dynamically. Pre-register the exact -# deterministic name so Viceroy reuses a local backend that trusts the temporary CA. -python3 - "$WORK/fastly.toml" "$WORK/ca-cert.pem" "$BID_PORT" <<'PYEOF' -import hashlib -import json -import sys - -manifest, ca_certificate, port = sys.argv[1:4] -provider_id = "pbs-main" -timeout_ms = "5000" - - -def field(value): - return f"{len(value)}:{value}" - - -canonical = "".join([ - field("https"), - field("localhost"), - field(port), - field("1"), - "n", - "s", - field(provider_id), - field(timeout_ms), - field(timeout_ms), -]) -digest = hashlib.sha256(canonical.encode()).hexdigest()[:32] -readable = f"https_localhost_{port}_p_{provider_id}_fb{timeout_ms}_bb{timeout_ms}" -backend_name = f"backend_{readable}_{digest}" -backend = f'''[local_server.backends.{backend_name}] -url = "https://localhost:{port}" -cert_host = "localhost" -ca_certificate.file = {json.dumps(ca_certificate)} -''' - -content = open(manifest).read() -marker = "[local_server.backends]\n\n" -if content.count(marker) != 1: - raise SystemExit("expected one local backend insertion target") -content = content.replace(marker, f"{marker}{backend}\n", 1) -open(manifest, "w").write(content) -PYEOF - python3 - "$WORK/fastly.toml" <<'PYEOF' import sys diff --git a/tinybird/README.md b/tinybird/README.md index c0a0e976ce..4351878753 100644 --- a/tinybird/README.md +++ b/tinybird/README.md @@ -47,11 +47,9 @@ Adding a field means changing three things together: the struct in Reports whether a request's origin response **would be** eligible to share between readers. **It is a predicate, not an outcome.** It records whether a request _would_ be eligible, -not whether anything was cached. A row with `1` still forced an origin fetch unless the -operator had set `creative_opportunities.origin_readthrough_enabled` (default `false`), and -even then the platform stores nothing if the origin's own `Cache-Control` refuses. So it -answers "how much traffic would the gate admit", and only in combination with that setting -does it bound "how much did it admit". +not whether anything was cached. Origin readthrough is unavailable, so a row with `1` still +forced an origin fetch. It answers "how much traffic would the gate admit", never "how much +did it admit". Three caveats, each of which silently produces wrong numbers if a query ignores it. @@ -64,9 +62,7 @@ bots, prefetches, consent-denied readers, and requests with auctions disabled ca traffic with creative opportunities disabled produces no publisher summary. The query below therefore measures shareability among matching-slot candidates, including -skipped summaries. It is neither an ad-serving-pageview rate nor a site-wide rate. The -readthrough gate also governs origin fetches outside this telemetry population; measure -those separately before estimating whole-site impact. +skipped summaries. It is neither an ad-serving-pageview rate nor a site-wide rate. ### `NULL` is "not measured", not "false" diff --git a/trusted-server.example.toml b/trusted-server.example.toml index 7b17751835..2f3ea16f0f 100644 --- a/trusted-server.example.toml +++ b/trusted-server.example.toml @@ -210,8 +210,7 @@ proxy = "enabled" # Proxy behavior and first-party asset routing. Kept active with defaults. [proxy] -# Verify TLS certs when proxying to HTTPS origins (default true; false only for -# local self-signed dev). +# Must be true. Origin TLS is always verified, and validation rejects false. # certificate_check = true # Allowlist for first-party proxy signing, initial fetch targets, and redirect # targets. Supports exact ("example.com") and wildcard ("*.example.com", also