diff --git a/.env.dev b/.env.dev index cdd6af510..fd7aa3ba4 100644 --- a/.env.dev +++ b/.env.dev @@ -1,3 +1,7 @@ +# Non-secret development overlays used while generating the Axum config blob. +# Sourcing this file alone does not configure the Axum server: also export the +# blob and referenced secret-store values as shown in docs/guide/getting-started.md. + # [publisher] TRUSTED_SERVER__PUBLISHER__ORIGIN_URL=http://localhost:9090 diff --git a/.env.example b/.env.example index c2ac88e3a..a7f5973cd 100644 --- a/.env.example +++ b/.env.example @@ -1,6 +1,14 @@ -# Trusted Server Environment Variables -# Copy this file to .env.dev, .env.staging, or .env.production and fill in values -# See docs/guide/configuration.md for details +# Trusted Server development environment variables +# Copy this file to .env.dev, .env.staging, or .env.production and fill in +# non-secret values. App-config secrets are key names in the pushed blob and +# their values belong in the platform secret store; see the configuration guide. +# For Axum runtime loading, export the config blob as: +# TRUSTED_SERVER_CONFIG_TRUSTED_SERVER_CONFIG_TRUSTED_SERVER_CONFIG= +# and export one secret per key name as: +# TRUSTED_SERVER_SECRET_TRUSTED_SERVER_SECRETS_= +# The commented examples below are CLI overlays for ordinary fields only. +# Fastly example: map logical app-config secrets to physical `ts_secrets`. +# EDGEZERO__STORES__SECRETS__TRUSTED_SERVER_SECRETS__NAME=ts_secrets # ============================================================================= # Publisher Settings @@ -8,14 +16,12 @@ TRUSTED_SERVER__PUBLISHER__DOMAIN=publisher.com TRUSTED_SERVER__PUBLISHER__COOKIE_DOMAIN=.publisher.com TRUSTED_SERVER__PUBLISHER__ORIGIN_URL=https://origin.publisher.com -TRUSTED_SERVER__PUBLISHER__PROXY_SECRET= # ============================================================================= # Synthetic ID Settings # ============================================================================= TRUSTED_SERVER__SYNTHETIC__COUNTER_STORE=counter_store TRUSTED_SERVER__SYNTHETIC__OPID_STORE=opid_store -TRUSTED_SERVER__SYNTHETIC__SECRET_KEY= # Template variables: client_ip, user_agent, first_party_id, auth_user_id, publisher_domain, accept_language TRUSTED_SERVER__SYNTHETIC__TEMPLATE={{ client_ip }}:{{ user_agent }}:{{ first_party_id }} diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index b7a145e02..1f1bbe27a 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -12,6 +12,7 @@ jobs: test-rust: name: cargo test runs-on: ubuntu-latest + timeout-minutes: 45 steps: - uses: actions/checkout@v4 @@ -230,9 +231,14 @@ jobs: run: | cargo clippy --manifest-path crates/trusted-server-cli/Cargo.toml --target "$(rustc -vV | sed -n 's/host: //p')" --all-targets -- -D warnings + - name: Set up Chrome for browser fixture tests + id: setup-chrome + uses: browser-actions/setup-chrome@v1 + - name: cargo test - run: | - cargo test --manifest-path crates/trusted-server-cli/Cargo.toml --target "$(rustc -vV | sed -n 's/host: //p')" + run: ./scripts/test-cli.sh + env: + CHROME: ${{ steps.setup-chrome.outputs.chrome-path }} test-typescript: name: vitest diff --git a/CHANGELOG.md b/CHANGELOG.md index b6c1aee9b..a4cc36f68 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,10 +9,13 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Changed +- **Breaking:** Auction providers and bidder routes now use the configuration-first `[auction.providers.]` and `[auction.bidders.]` maps. The removed `[auction].providers = [...]` list and removed server fields under `[integrations.prebid]` and `[integrations.aps]` are rejected even when those integrations are disabled, and `ts config push` rejects the old shape before publication. Move PBS `server_url` to provider `endpoint`, server timeout to provider `timeout_ms`, request controls and bidder-parameter overrides to the `prebid-server` `profile_config`, notification suppression to `notifications`, and each former server bidder to an `[auction.bidders.]` route. Move APS endpoint, timeout, account, inventory, debug, and creative controls to an `aps` provider and its `profile_config`. Browser Prebid settings remain under `[integrations.prebid]`; values such as timeout and debug that previously affected both browser and server behavior must now be configured for each owner. Provider endpoints must be absolute HTTPS URLs. Only bidder codes present in `[auction.bidders]` are folded into Trusted Server requests; unlisted publisher bids remain native browser demand. This schema has no mixed-version-safe deployment order: old binaries reject the maps and new binaries reject the retired fields, so activate the new binary and config blob together. Rollbacks must restore an old-schema blob together with the old binary. - **Breaking** — Admin Basic-auth coverage now includes `GET /_ts/admin/ec`, `GET /_ts/admin/ec/{id}`, and `GET /_ts/admin/eids`. Existing configurations whose `[[handlers]]` patterns protect only the key-management endpoints now fail startup; broaden coverage before deploying, preferably with a namespace-boundary pattern such as `^/_ts/admin(?:/|$)`. Coverage of the dynamic `/_ts/admin/ec/{id}` route is no longer inferred from ID-shaped samples: the router accepts any segment after `/_ts/admin/ec/` and Basic Auth runs on the raw path before routing, so patterns anchored to the EC ID grammar (for example `^/_ts/admin/ec/[a-f0-9]{64}[.][A-Za-z0-9]{6}$`) are rejected in favor of a prefix-level matcher. Placeholder and well-known weak handler passwords (`changeme`, `password`, `admin`, `replace-with-…`) now fail startup on every handler rather than only on handlers inferred to cover an admin endpoint, because first-match-wins handler selection lets a narrow handler shadow the admin namespace. +- Prebid Server provider endpoints now normalize origin-only legacy `server_url` values to `/openrtb2/auction`. Query parameters are preserved, the canonical path loses a trailing slash, and configured non-root custom paths remain exact. - Publisher HTML uses the browser-only `Cache-Control: private, max-age=60` policy for successful GET document responses and their `304 Not Modified` revalidations when server-side ad templates are structurally inactive, while preserving origin `private`/`no-store` policies and request-scoped bot, prefetch, or consent-denied responses. The `private` directive prevents shared caches that use `Cache-Control` from storing the document. Cookie-bearing responses using the generated inactive policy are finalized as `private, max-age=0`; CDN-specific cache headers remain unchanged and continue to control supporting CDNs independently. Set `[creative_opportunities].enabled = false` to disable publisher HTML and SPA template delivery without disabling direct `POST /auction` callers; an absent configuration, an unmatched slot, or a disabled auction also make the stack structurally inactive. An explicit `enabled = false` is not compatible with older binaries: restore the default, re-push and finalize the config before rolling back. - **Breaking** — Replaced the legacy APS contextual integration with APS OpenRTB at `/e/pb/bid`. APS configuration now uses canonical `account_id` (`pub_id` remains a compatibility alias), no longer requires APS-specific slot IDs, and defaults script creative eligibility off. Operators must update the endpoint, disable native APS demand for Trusted Server cohorts, and prepare GAM/Universal Creative targeting for `hb_bidder=aps` before rollout. `aps` entries in Prebid bidder lists are logged and stripped. APS renderer winners now preserve the upstream bid `id`, omit `crid` when APS omits it, and carry `ext.trusted_server.renderer` instead of `adm`; external `/auction` consumers must support this response shape. - **Breaking** — All auction paths now forward only a validated publisher-owned page URL as `site.page`, removing query and fragment data. APS OpenRTB omits `site.ref`; the existing Prebid Server path continues to forward the browser `Referer` as `site.ref`. Query-driven sites may lose contextual targeting and per-page reporting signals that previously came from query parameters. +- Publisher HTML now uses `Cache-Control: max-age=60` when server-side ad templates are inactive, while preserving origin `private`/`no-store` policies and CDN-specific cache headers. Set `[creative_opportunities].enabled = false` to disable publisher HTML and SPA template delivery without disabling direct `POST /auction` callers. - **Breaking** — `bid_param_zone_overrides` inner values must now be JSON objects; previously non-object or empty values (`"header" = "x"`, `"header" = {}`) were accepted and silently produced a dead rule at runtime. They now fail at startup with a configuration error. Operators upgrading should audit their `bid_param_zone_overrides` config for non-object zone entries. - **Breaking** — Integration configuration strings are no longer globally reinterpreted as JSON scalars. Operators upgrading should audit `[integrations.*]` settings and use native TOML/typed-config booleans and numbers (for example, `enabled = true`, not `enabled = "true"`); quoted numeric and boolean scalars now fail validation instead of silently converting. - **Breaking** — Sourcepoint browser module inclusion now requires explicit `[integrations.sourcepoint].enabled = true`; operators relying on the previous unconditional Sourcepoint module should enable the integration before upgrading. diff --git a/Cargo.lock b/Cargo.lock index e29380b77..ab29df9fa 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1427,7 +1427,7 @@ dependencies = [ [[package]] name = "edgezero-adapter" version = "0.1.0" -source = "git+https://github.com/stackpop/edgezero?tag=v0.0.4#9e661ae520a8130660f18fd10f42703d7f3e050b" +source = "git+https://github.com/stackpop/edgezero?rev=0d6ebf9b0250efa5f7031a93ec7b7f09f2c9bf34#0d6ebf9b0250efa5f7031a93ec7b7f09f2c9bf34" dependencies = [ "toml", ] @@ -1435,7 +1435,7 @@ dependencies = [ [[package]] name = "edgezero-adapter-axum" version = "0.1.0" -source = "git+https://github.com/stackpop/edgezero?tag=v0.0.4#9e661ae520a8130660f18fd10f42703d7f3e050b" +source = "git+https://github.com/stackpop/edgezero?rev=0d6ebf9b0250efa5f7031a93ec7b7f09f2c9bf34#0d6ebf9b0250efa5f7031a93ec7b7f09f2c9bf34" dependencies = [ "anyhow", "async-trait", @@ -1463,7 +1463,7 @@ dependencies = [ [[package]] name = "edgezero-adapter-cloudflare" version = "0.1.0" -source = "git+https://github.com/stackpop/edgezero?tag=v0.0.4#9e661ae520a8130660f18fd10f42703d7f3e050b" +source = "git+https://github.com/stackpop/edgezero?rev=0d6ebf9b0250efa5f7031a93ec7b7f09f2c9bf34#0d6ebf9b0250efa5f7031a93ec7b7f09f2c9bf34" dependencies = [ "anyhow", "async-trait", @@ -1478,7 +1478,7 @@ dependencies = [ "log", "serde_json", "tempfile", - "toml_edit", + "toml_edit 0.25.12+spec-1.1.0", "walkdir", "worker", ] @@ -1486,7 +1486,7 @@ dependencies = [ [[package]] name = "edgezero-adapter-fastly" version = "0.1.0" -source = "git+https://github.com/stackpop/edgezero?tag=v0.0.4#9e661ae520a8130660f18fd10f42703d7f3e050b" +source = "git+https://github.com/stackpop/edgezero?rev=0d6ebf9b0250efa5f7031a93ec7b7f09f2c9bf34#0d6ebf9b0250efa5f7031a93ec7b7f09f2c9bf34" dependencies = [ "anyhow", "async-stream", @@ -1508,14 +1508,14 @@ dependencies = [ "serde_json", "sha2 0.10.9", "thiserror 2.0.18", - "toml_edit", + "toml_edit 0.25.12+spec-1.1.0", "walkdir", ] [[package]] name = "edgezero-adapter-spin" version = "0.1.0" -source = "git+https://github.com/stackpop/edgezero?tag=v0.0.4#9e661ae520a8130660f18fd10f42703d7f3e050b" +source = "git+https://github.com/stackpop/edgezero?rev=0d6ebf9b0250efa5f7031a93ec7b7f09f2c9bf34#0d6ebf9b0250efa5f7031a93ec7b7f09f2c9bf34" dependencies = [ "anyhow", "async-trait", @@ -1535,14 +1535,14 @@ dependencies = [ "subtle", "thiserror 2.0.18", "toml", - "toml_edit", + "toml_edit 0.25.12+spec-1.1.0", "walkdir", ] [[package]] name = "edgezero-cli" version = "0.1.0" -source = "git+https://github.com/stackpop/edgezero?tag=v0.0.4#9e661ae520a8130660f18fd10f42703d7f3e050b" +source = "git+https://github.com/stackpop/edgezero?rev=0d6ebf9b0250efa5f7031a93ec7b7f09f2c9bf34#0d6ebf9b0250efa5f7031a93ec7b7f09f2c9bf34" dependencies = [ "chrono", "clap", @@ -1567,7 +1567,7 @@ dependencies = [ [[package]] name = "edgezero-core" version = "0.1.0" -source = "git+https://github.com/stackpop/edgezero?tag=v0.0.4#9e661ae520a8130660f18fd10f42703d7f3e050b" +source = "git+https://github.com/stackpop/edgezero?rev=0d6ebf9b0250efa5f7031a93ec7b7f09f2c9bf34#0d6ebf9b0250efa5f7031a93ec7b7f09f2c9bf34" dependencies = [ "anyhow", "async-compression", @@ -1598,14 +1598,14 @@ dependencies = [ [[package]] name = "edgezero-macros" version = "0.1.0" -source = "git+https://github.com/stackpop/edgezero?tag=v0.0.4#9e661ae520a8130660f18fd10f42703d7f3e050b" +source = "git+https://github.com/stackpop/edgezero?rev=0d6ebf9b0250efa5f7031a93ec7b7f09f2c9bf34#0d6ebf9b0250efa5f7031a93ec7b7f09f2c9bf34" dependencies = [ "log", "proc-macro2", "quote", "serde", "serde_json", - "syn 2.0.118", + "syn 3.0.3", "toml", "validator", ] @@ -4858,6 +4858,17 @@ dependencies = [ "unicode-ident", ] +[[package]] +name = "syn" +version = "3.0.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "53e9bae58849f64dfa4f5d5ae372c8341f7305f82a3868709269343628b659a3" +dependencies = [ + "proc-macro2", + "quote", + "unicode-ident", +] + [[package]] name = "sync_wrapper" version = "1.0.2" @@ -5163,6 +5174,19 @@ dependencies = [ "winnow 0.7.15", ] +[[package]] +name = "toml_edit" +version = "0.25.12+spec-1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d2153edc6955a6c354fad8f5efd38b6a8769bdccf9fe50f8e1329f81b0baa5d7" +dependencies = [ + "indexmap 2.14.0", + "toml_datetime 1.1.1+spec-1.1.0", + "toml_parser", + "toml_writer", + "winnow 1.0.3", +] + [[package]] name = "toml_parser" version = "1.1.2+spec-1.1.0" @@ -5374,7 +5398,7 @@ dependencies = [ "log-fastly", "serde", "serde_json", - "sha2 0.10.9", + "toml", "trusted-server-core", "url", "urlencoding", @@ -5413,8 +5437,11 @@ dependencies = [ "derive_more", "directories", "edgezero-cli", + "edgezero-core", "error-stack", "futures", + "glob", + "http", "http-body-util", "hyper", "hyper-util", @@ -5426,12 +5453,15 @@ dependencies = [ "scraper", "serde", "serde_json", + "similar", + "temp-env", "tempfile", "time", "tokio", "tokio-rustls", "toml", - "toml_edit", + "toml_edit 0.23.10+spec-1.0.0", + "tracing", "trusted-server-core", "url", "webpki-roots", diff --git a/Cargo.toml b/Cargo.toml index 25c367181..57ee2f41b 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -54,12 +54,12 @@ criterion = { version = "0.5", default-features = false, features = ["cargo_benc derive_more = { version = "2.0", features = ["display", "error"] } directories = "5" ed25519-dalek = { version = "2.2", features = ["rand_core"] } -edgezero-adapter-axum = { git = "https://github.com/stackpop/edgezero", tag = "v0.0.4", default-features = false } -edgezero-adapter-cloudflare = { git = "https://github.com/stackpop/edgezero", tag = "v0.0.4", default-features = false } -edgezero-adapter-fastly = { git = "https://github.com/stackpop/edgezero", tag = "v0.0.4", default-features = false } -edgezero-adapter-spin = { git = "https://github.com/stackpop/edgezero", tag = "v0.0.4", default-features = false } -edgezero-cli = { git = "https://github.com/stackpop/edgezero", tag = "v0.0.4" } -edgezero-core = { git = "https://github.com/stackpop/edgezero", tag = "v0.0.4", default-features = false } +edgezero-adapter-axum = { git = "https://github.com/stackpop/edgezero", rev = "0d6ebf9b0250efa5f7031a93ec7b7f09f2c9bf34", default-features = false } +edgezero-adapter-cloudflare = { git = "https://github.com/stackpop/edgezero", rev = "0d6ebf9b0250efa5f7031a93ec7b7f09f2c9bf34", default-features = false } +edgezero-adapter-fastly = { git = "https://github.com/stackpop/edgezero", rev = "0d6ebf9b0250efa5f7031a93ec7b7f09f2c9bf34", default-features = false } +edgezero-adapter-spin = { git = "https://github.com/stackpop/edgezero", rev = "0d6ebf9b0250efa5f7031a93ec7b7f09f2c9bf34", default-features = false } +edgezero-cli = { git = "https://github.com/stackpop/edgezero", rev = "0d6ebf9b0250efa5f7031a93ec7b7f09f2c9bf34" } +edgezero-core = { git = "https://github.com/stackpop/edgezero", rev = "0d6ebf9b0250efa5f7031a93ec7b7f09f2c9bf34", default-features = false } env_logger = "0.11" error-stack = "0.6" esi = "0.7.2" @@ -96,6 +96,7 @@ scraper = "0.24.0" serde = { version = "1.0", features = ["derive"] } serde_json = "1.0.149" sha2 = "0.10.9" +similar = "2.7" simple_logger = "5" spin-sdk = { version = "~6.0", default-features = false, features = ["http", "key-value", "variables"] } subtle = "2.6" @@ -108,6 +109,7 @@ tokio-rustls = "0.26" toml = "1.1" toml_edit = "0.23.10" tower = "0.4" +tracing = "0.1" trusted-server-core = { path = "crates/trusted-server-core" } trusted-server-js = { path = "crates/trusted-server-js" } trusted-server-openrtb = { path = "crates/trusted-server-openrtb" } diff --git a/README.md b/README.md index b87fe61ad..0ad5b0351 100644 --- a/README.md +++ b/README.md @@ -37,11 +37,12 @@ ts --help # Create local config, then edit placeholders before validation ts config init -# Edit trusted-server.toml +# Edit trusted-server.toml. Server auctions use map-shaped +# [auction.providers.] and [auction.bidders.] tables. ts config validate # Audit a public page with Chrome/Chromium to bootstrap a draft config -ts audit https://publisher.example +ts audit generate https://publisher.example # Run tests (Fastly/WASM crates — requires Viceroy) cargo test-fastly diff --git a/TESTING.md b/TESTING.md index e5ccba4cf..a68f33222 100644 --- a/TESTING.md +++ b/TESTING.md @@ -50,33 +50,39 @@ curl -X POST http://localhost:7676/auction \ - Optional mock-adserver mediation selecting winning bids - Final response with winning creatives -**With Orchestrator Disabled** (`auction.enabled = false`): -- Logs showing: `"Using legacy Prebid flow"` -- Direct Prebid Server call (backward compatible) +**With Auction Execution Disabled** (`auction.enabled = false`): +- Logs showing: `"/auction: auction is disabled; returning no-bid response"` +- Immediate no-bid response with no provider or mediator dispatch -##Configuration +## Configuration Edit `trusted-server.toml` to customize the auction: ```toml -# Enable/disable orchestrator [auction] enabled = true -providers = ["prebid", "aps"] -mediator = "adserver_mock" # If set: mediation, if omitted: highest bid wins timeout_ms = 2000 +mediator = "adserver_mock" -# APS OpenRTB provider. The built-in production endpoint is used when -# endpoint is omitted; use only an account authorized for test traffic. -[integrations.aps] -enabled = true -account_id = "example-account" -timeout_ms = 800 -debug = false +[auction.providers.pbs-main] +protocol = "openrtb-2.6" +profile = "prebid-server" +endpoint = "https://prebid.example.com/openrtb2/auction" +routing = "explicit" + +[auction.providers.aps-main] +protocol = "openrtb-2.6" +profile = "aps" +endpoint = "https://aps.example.com/e/pb/bid" +routing = "all_eligible" +profile_config = { account_id = "example-aps-account", debug = false } + +[auction.bidders.example-server] +provider = "pbs-main" [integrations.adserver_mock] enabled = true -endpoint = "http://localhost:6767/adserver/mediate" +endpoint = "https://mediator.example.com/mediate" timeout_ms = 500 ``` @@ -87,8 +93,7 @@ timeout_ms = 500 ```toml [auction] enabled = true -providers = ["prebid", "aps"] -mediator = "adserver_mock" # Mediator configured = parallel mediation strategy +mediator = "adserver_mock" # Providers come from [auction.providers.*] maps ``` **Expected Flow:** @@ -102,8 +107,7 @@ mediator = "adserver_mock" # Mediator configured = parallel mediation strategy ```toml [auction] enabled = true -providers = ["prebid", "aps"] -# No mediator = parallel only strategy +# Configured [auction.providers.*] run without a mediator ``` **Expected Flow:** @@ -111,16 +115,16 @@ providers = ["prebid", "aps"] 2. Highest bid wins automatically 3. No mediation -### Scenario 3: Legacy Mode (Backward Compatible) +### Scenario 3: Auction Disabled + **Config:** + ```toml [auction] enabled = false ``` -**Expected Flow:** -- Original Prebid-only behavior -- No orchestration overhead +**Expected Flow:** no auction provider dispatch. ## Debugging @@ -149,10 +153,10 @@ INFO: Registering auction provider: adserver_mock ### Common Issues **Issue:** `"Provider 'aps' not registered"` -**Fix:** Make sure `[integrations.aps]` is configured in `trusted-server.toml` +**Fix:** Make sure an `[auction.providers.]` entry selects `profile = "aps"` **Issue:** `"No providers configured"` -**Fix:** Make sure `providers = ["prebid", "aps"]` is set in `[auction]` +**Fix:** Make sure map-shaped `[auction.providers.]` entries are configured **Issue:** Tests fail with WASM errors **Explanation:** Async tests don't work in WASM test environment. Integration tests via HTTP work fine! diff --git a/crates/trusted-server-adapter-axum/src/app.rs b/crates/trusted-server-adapter-axum/src/app.rs index 9a371f805..1deddcab5 100644 --- a/crates/trusted-server-adapter-axum/src/app.rs +++ b/crates/trusted-server-adapter-axum/src/app.rs @@ -10,7 +10,9 @@ use edgezero_core::http::{ use edgezero_core::router::RouterService; use error_stack::Report; use trusted_server_core::auction::endpoints::handle_auction; -use trusted_server_core::auction::{AuctionOrchestrator, build_orchestrator}; +use trusted_server_core::auction::{ + AuctionOrchestrator, build_orchestrator_with_plan, compile_auction_plan, +}; use trusted_server_core::cache_policy::EdgeCacheHeader; use trusted_server_core::ec::EcContext; use trusted_server_core::ec::admin::{ @@ -38,7 +40,7 @@ use trusted_server_core::settings_data::{ use trusted_server_core::platform::RuntimeServices; use crate::middleware::{AuthMiddleware, FinalizeResponseMiddleware, SanitizeRequestMiddleware}; -use crate::platform::{AxumPlatformConfigStore, build_runtime_services}; +use crate::platform::{AxumPlatformConfigStore, AxumPlatformSecretStore, build_runtime_services}; // --------------------------------------------------------------------------- // AppState @@ -60,8 +62,13 @@ pub struct AppState { fn build_state() -> Result, Report> { let store_name = default_config_store_name(); let config_key = default_config_key(); - let settings = - get_settings_from_config_store(&AxumPlatformConfigStore, &store_name, &config_key)?; + let settings = get_settings_from_config_store( + &AxumPlatformConfigStore, + &AxumPlatformSecretStore, + &store_name, + &config_key, + &trusted_server_core::settings_data::default_secret_store_name(), + )?; build_state_with_settings(settings) } @@ -74,8 +81,10 @@ fn build_state() -> Result, Report> { fn build_state_with_settings( settings: Settings, ) -> Result, Report> { - let orchestrator = build_orchestrator(&settings)?; - let registry = IntegrationRegistry::new(&settings)?; + let plan = Arc::new(compile_auction_plan(&settings)?); + plan.validate_for_target(trusted_server_core::platform::AuctionTargetId::Axum)?; + let orchestrator = build_orchestrator_with_plan(Arc::clone(&plan), &settings)?; + let registry = IntegrationRegistry::with_plan(&settings, plan)?; Ok(Arc::new(AppState { settings: Arc::new(settings), diff --git a/crates/trusted-server-adapter-axum/src/middleware.rs b/crates/trusted-server-adapter-axum/src/middleware.rs index fd11d7728..974a23e46 100644 --- a/crates/trusted-server-adapter-axum/src/middleware.rs +++ b/crates/trusted-server-adapter-axum/src/middleware.rs @@ -105,6 +105,9 @@ impl AuthMiddleware { #[async_trait(?Send)] impl Middleware for AuthMiddleware { async fn handle(&self, mut ctx: RequestContext, next: Next<'_>) -> Result { + // Takes the request mutably because `enforce_basic_auth` marks requests + // whose credential it consumed itself; the shared template cache gate + // reads that marker later. match enforce_basic_auth(&self.settings, ctx.request_mut()) { Ok(Some(response)) => return Ok(response), Ok(None) => {} diff --git a/crates/trusted-server-adapter-axum/src/platform.rs b/crates/trusted-server-adapter-axum/src/platform.rs index a511daab2..7dcdd53d8 100644 --- a/crates/trusted-server-adapter-axum/src/platform.rs +++ b/crates/trusted-server-adapter-axum/src/platform.rs @@ -9,9 +9,10 @@ use async_trait::async_trait; use edgezero_core::http::{HeaderMap, HeaderName, HeaderValue, header}; use error_stack::{Report, ResultExt as _}; use trusted_server_core::platform::{ - ClientInfo, GeoInfo, PlatformBackend, PlatformBackendSpec, PlatformConfigStore, PlatformError, - PlatformGeo, PlatformHttpClient, PlatformHttpRequest, PlatformPendingRequest, PlatformResponse, - PlatformSecretStore, PlatformSelectResult, RuntimeServices, StoreId, StoreName, + BackendNamingPolicy, ClientInfo, GeoInfo, PlatformBackend, PlatformBackendSpec, + PlatformConfigStore, PlatformError, PlatformGeo, PlatformHttpClient, PlatformHttpRequest, + PlatformPendingRequest, PlatformResponse, PlatformSecretStore, PlatformSelectResult, + RuntimeServices, StoreId, StoreName, }; // --------------------------------------------------------------------------- @@ -154,24 +155,15 @@ impl PlatformSecretStore for AxumPlatformSecretStore { pub struct AxumPlatformBackend; impl PlatformBackend for AxumPlatformBackend { + fn naming_policy(&self) -> BackendNamingPolicy { + BackendNamingPolicy::Axum + } + fn predict_name(&self, spec: &PlatformBackendSpec) -> Result> { - let port = spec - .port - .unwrap_or(if spec.scheme == "https" { 443 } else { 80 }); - // Keep two providers that share an origin on distinct names so auction - // response correlation cannot cross providers. - let discriminator = spec - .discriminator - .as_deref() - .map(|d| format!("_p_{}", normalize_env_segment(d))) - .unwrap_or_default(); - Ok(format!( - "{}_{}_{}{}", - normalize_env_segment(&spec.scheme), - normalize_env_segment(&spec.host), - port, - discriminator, - )) + self.naming_policy() + .predict(spec) + .map(|prediction| prediction.name) + .change_context(PlatformError::Backend) } fn ensure(&self, spec: &PlatformBackendSpec) -> Result> { @@ -601,6 +593,21 @@ mod tests { use std::time::Duration; use tokio::io::{AsyncReadExt as _, AsyncWriteExt as _}; + #[test] + fn auction_http_capabilities_are_explicit() { + let client = AxumPlatformHttpClient::new(); + let capabilities = trusted_server_core::platform::AuctionTargetId::Axum + .descriptor() + .capabilities(); + assert!(client.supports_concurrent_fanout()); + assert!(capabilities.supports_concurrent_provider_fanout()); + assert!(!client.has_enforceable_total_request_deadline()); + assert!( + !capabilities.has_enforceable_total_request_deadline(), + "reqwest's transport timeout is not an adapter-enforced auction deadline" + ); + } + #[test] fn config_store_reads_from_env_var() { temp_env::with_var( @@ -693,6 +700,33 @@ mod tests { assert!(with_ip.is_none(), "should return None for any IP"); } + #[tokio::test(flavor = "multi_thread", worker_threads = 2)] + async fn http_client_surfaces_redirect_without_following() { + let url = serve_raw_response( + b"HTTP/1.1 302 Found\r\nLocation: https://redirect.example/next\r\nContent-Length: 0\r\n\r\n", + ) + .await; + let request = edgezero_core::http::request_builder() + .uri(url) + .body(EdgeBody::empty()) + .expect("should build outbound request"); + + let response = AxumPlatformHttpClient::new() + .send(PlatformHttpRequest::new(request, "test_backend")) + .await + .expect("should surface redirect") + .response; + + assert_eq!(response.status().as_u16(), 302); + assert_eq!( + response + .headers() + .get(edgezero_core::http::header::LOCATION) + .and_then(|value| value.to_str().ok()), + Some("https://redirect.example/next") + ); + } + #[tokio::test(flavor = "multi_thread", worker_threads = 2)] async fn http_client_strips_hop_by_hop_response_headers() { let url = serve_raw_response( diff --git a/crates/trusted-server-adapter-axum/tests/routes.rs b/crates/trusted-server-adapter-axum/tests/routes.rs index ed199e6bf..6812b7421 100644 --- a/crates/trusted-server-adapter-axum/tests/routes.rs +++ b/crates/trusted-server-adapter-axum/tests/routes.rs @@ -18,8 +18,8 @@ const LEGACY_ADMIN_DENY_METHODS: &[&str] = /// The settings baked into the binary contain placeholder secrets that /// `get_settings()` rejects by design, which would turn every route into a /// startup error page (and its route table into the fallback-only set). -fn test_router() -> edgezero_core::router::RouterService { - let settings = trusted_server_core::settings::Settings::from_toml( +fn test_settings() -> trusted_server_core::settings::Settings { + trusted_server_core::settings::Settings::from_toml( r#" [[handlers]] path = "^/_ts/admin" @@ -36,9 +36,11 @@ fn test_router() -> edgezero_core::router::RouterService { passphrase = "test-secret-key-32-bytes-minimum" "#, ) - .expect("should parse route test settings"); + .expect("should parse route test settings") +} - TrustedServerApp::routes_with_settings(settings) +fn test_router() -> edgezero_core::router::RouterService { + TrustedServerApp::routes_with_settings(test_settings()) .expect("should build router from test settings") } @@ -62,6 +64,42 @@ fn assert_route_registered(method: &str, path: &str) { ); } +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn aps_profile_serves_renderer_through_adapter_fallback() { + let mut settings = test_settings(); + settings.auction.providers.insert( + "aps-main".parse().expect("should parse APS provider ID"), + trusted_server_core::auction::ProviderConfig { + protocol: "openrtb-2.6".to_string(), + profile: "aps".to_string(), + endpoint: "https://aps.example/e/pb/bid".to_string(), + timeout_ms: None, + routing: trusted_server_core::auction::RoutingMode::AllEligible, + notifications: trusted_server_core::auction::NotificationConfig::default(), + profile_config: "{\"account_id\":\"example-account\"}" + .parse() + .expect("should parse APS profile config"), + }, + ); + let router = TrustedServerApp::routes_with_settings(settings) + .expect("should build router with APS profile"); + let mut service = EdgeZeroAxumService::new(router); + let request = Request::builder() + .method("GET") + .uri("/integrations/aps/renderer") + .body(AxumBody::empty()) + .expect("should build APS renderer request"); + + let response = service + .ready() + .await + .expect("should be ready") + .call(request) + .await + .expect("should serve APS renderer"); + assert_eq!(response.status().as_u16(), 200); +} + /// Verify that every expected explicit route is registered in the route table. /// /// Uses [`RouterService::routes()`] for introspection rather than checking diff --git a/crates/trusted-server-adapter-cloudflare/src/app.rs b/crates/trusted-server-adapter-cloudflare/src/app.rs index 6ce0a5ee3..b57c4a6b8 100644 --- a/crates/trusted-server-adapter-cloudflare/src/app.rs +++ b/crates/trusted-server-adapter-cloudflare/src/app.rs @@ -9,10 +9,12 @@ use edgezero_core::http::{HeaderValue, Method, Request, Response, StatusCode, he use edgezero_core::router::RouterService; use error_stack::Report; use trusted_server_core::auction::endpoints::handle_auction; -use trusted_server_core::auction::{AuctionOrchestrator, build_orchestrator}; +use trusted_server_core::auction::{ + AuctionOrchestrator, build_orchestrator_with_plan, compile_auction_plan, +}; use trusted_server_core::cache_policy::EdgeCacheHeader; #[cfg(target_arch = "wasm32")] -use trusted_server_core::config_payload::settings_from_config_blob; +use trusted_server_core::config_payload::{DEFAULT_SECRET_STORE_ID, settings_from_config_blob}; use trusted_server_core::ec::EcContext; use trusted_server_core::ec::admin::{ admin_ec_lookup_not_supported as core_admin_ec_lookup_not_supported, @@ -22,6 +24,8 @@ use trusted_server_core::ec::registry::PartnerRegistry; use trusted_server_core::error::{IntoHttpResponse as _, TrustedServerError}; use trusted_server_core::integrations::{IntegrationRegistry, ProxyDispatchInput}; use trusted_server_core::platform::RuntimeServices; +#[cfg(target_arch = "wasm32")] +use trusted_server_core::platform::StoreName; use trusted_server_core::proxy::{ handle_first_party_click, handle_first_party_proxy, handle_first_party_proxy_rebuild, handle_first_party_proxy_sign, @@ -44,11 +48,23 @@ use crate::platform::build_runtime_services; // --------------------------------------------------------------------------- #[cfg(target_arch = "wasm32")] -static CLOUDFLARE_CONFIG_JSON: std::sync::OnceLock = std::sync::OnceLock::new(); +thread_local! { + static CLOUDFLARE_CONFIG_JSON: std::cell::OnceCell = const { std::cell::OnceCell::new() }; + static CLOUDFLARE_ENV: std::cell::OnceCell = const { std::cell::OnceCell::new() }; +} #[cfg(target_arch = "wasm32")] pub fn set_cloudflare_config_json(value: String) { - let _ = CLOUDFLARE_CONFIG_JSON.set(value); + CLOUDFLARE_CONFIG_JSON.with(|slot| { + let _ = slot.set(value); + }); +} + +#[cfg(target_arch = "wasm32")] +pub fn set_cloudflare_env(env: worker::Env) { + CLOUDFLARE_ENV.with(|slot| { + let _ = slot.set(env); + }); } /// Application state built once at startup and shared across all requests. @@ -76,18 +92,22 @@ fn load_startup_settings() -> Result> { #[cfg(not(target_arch = "wasm32"))] fn load_startup_settings() -> Result> { - Settings::from_toml(include_str!("../../../trusted-server.example.toml")) + Err(Report::new(TrustedServerError::Configuration { + message: "Cloudflare startup settings require a Worker config binding".to_string(), + }) + .attach("use TrustedServerApp::routes_with_settings for host tests")) } #[cfg(target_arch = "wasm32")] fn settings_from_cloudflare_config_json() -> Result> { - let raw_config = CLOUDFLARE_CONFIG_JSON.get().ok_or_else(|| { + let raw_config = CLOUDFLARE_CONFIG_JSON.with(|slot| slot.get().cloned()); + let raw_config = raw_config.ok_or_else(|| { Report::new(TrustedServerError::Configuration { message: "Cloudflare TRUSTED_SERVER_CONFIG is required".to_string(), }) .attach("set TRUSTED_SERVER_CONFIG to JSON containing the app_config blob envelope") })?; - let value: serde_json::Value = serde_json::from_str(raw_config).map_err(|error| { + let value: serde_json::Value = serde_json::from_str(&raw_config).map_err(|error| { Report::new(TrustedServerError::Configuration { message: "invalid Cloudflare TRUSTED_SERVER_CONFIG JSON".to_string(), }) @@ -101,7 +121,16 @@ fn settings_from_cloudflare_config_json() -> Result Result Result, Report> { - let orchestrator = build_orchestrator(&settings)?; - let registry = IntegrationRegistry::new(&settings)?; + let plan = Arc::new(compile_auction_plan(&settings)?); + plan.validate_for_target(trusted_server_core::platform::AuctionTargetId::Cloudflare)?; + let orchestrator = build_orchestrator_with_plan(Arc::clone(&plan), &settings)?; + let registry = IntegrationRegistry::with_plan(&settings, plan)?; Ok(Arc::new(AppState { settings: Arc::new(settings), @@ -625,3 +656,149 @@ fn build_router(state: &Arc) -> RouterService { router.build() } } + +#[cfg(test)] +mod tests { + use super::*; + + fn aps_profile_settings() -> Settings { + let mut settings = Settings::from_toml( + r#" + [[handlers]] + path = "^/_ts/admin" + username = "admin" + password = "admin-password" + + [publisher] + domain = "publisher.example" + cookie_domain = ".publisher.example" + origin_url = "https://origin.publisher.example" + proxy_secret = "fictional-proxy-secret" + + [ec] + passphrase = "fictional-secret-key-32-bytes-minimum" + "#, + ) + .expect("should parse startup test settings"); + settings.auction.providers.insert( + "aps-main".parse().expect("should parse APS provider ID"), + trusted_server_core::auction::ProviderConfig { + protocol: "openrtb-2.6".to_string(), + profile: "aps".to_string(), + endpoint: "https://aps.example/e/pb/bid".to_string(), + timeout_ms: None, + routing: trusted_server_core::auction::RoutingMode::AllEligible, + notifications: trusted_server_core::auction::NotificationConfig::default(), + profile_config: serde_json::json!({"account_id":"example-account"}), + }, + ); + settings + } + + #[test] + fn startup_registers_aps_renderer_route() { + let state = build_state_with_settings(aps_profile_settings()) + .expect("Cloudflare startup should register APS renderer"); + assert!( + state.registry.has_route( + &edgezero_core::http::Method::GET, + "/integrations/aps/renderer" + ), + "Cloudflare startup registry should expose the APS renderer" + ); + } + + #[test] + fn disabled_startup_accepts_dormant_multi_provider_auction_plan() { + let mut settings = Settings::from_toml( + r#" + [[handlers]] + path = "^/_ts/admin" + username = "admin" + password = "admin-password" + + [publisher] + domain = "publisher.example" + cookie_domain = ".publisher.example" + origin_url = "https://origin.publisher.example" + proxy_secret = "fictional-proxy-secret" + + [ec] + passphrase = "fictional-secret-key-32-bytes-minimum" + "#, + ) + .expect("should parse startup test settings"); + settings.auction.enabled = false; + settings.auction.providers = + std::iter::IntoIterator::into_iter(["provider-a", "provider-b"]) + .map(|id| { + ( + id.parse().expect("should parse provider ID"), + trusted_server_core::auction::ProviderConfig { + protocol: "openrtb-2.6".to_string(), + profile: "standard".to_string(), + endpoint: format!("https://{id}.example/openrtb"), + timeout_ms: None, + routing: trusted_server_core::auction::RoutingMode::AllEligible, + notifications: + trusted_server_core::auction::NotificationConfig::default(), + profile_config: serde_json::json!({}), + }, + ) + }) + .collect(); + + build_state_with_settings(settings) + .expect("disabled Cloudflare auction should accept dormant fanout"); + } + + #[test] + fn startup_rejects_multi_provider_auction_plan() { + let mut settings = Settings::from_toml( + r#" + [[handlers]] + path = "^/_ts/admin" + username = "admin" + password = "admin-password" + + [publisher] + domain = "publisher.example" + cookie_domain = ".publisher.example" + origin_url = "https://origin.publisher.example" + proxy_secret = "fictional-proxy-secret" + + [ec] + passphrase = "fictional-secret-key-32-bytes-minimum" + "#, + ) + .expect("should parse startup test settings"); + settings.auction.enabled = true; + settings.auction.providers = + std::iter::IntoIterator::into_iter(["provider-a", "provider-b"]) + .map(|id| { + ( + id.parse().expect("should parse provider ID"), + trusted_server_core::auction::ProviderConfig { + protocol: "openrtb-2.6".to_string(), + profile: "standard".to_string(), + endpoint: format!("https://{id}.example/openrtb"), + timeout_ms: None, + routing: trusted_server_core::auction::RoutingMode::AllEligible, + notifications: + trusted_server_core::auction::NotificationConfig::default(), + profile_config: serde_json::json!({}), + }, + ) + }) + .collect(); + + let error = match build_state_with_settings(settings) { + Ok(_) => panic!("Cloudflare startup should reject multi-provider fanout"), + Err(error) => error, + }; + assert!( + format!("{error:?}").contains("concurrent provider fanout"), + "should identify unsupported fanout: {error:?}" + ); + } +} diff --git a/crates/trusted-server-adapter-cloudflare/src/lib.rs b/crates/trusted-server-adapter-cloudflare/src/lib.rs index 2ce435b17..980a9b209 100644 --- a/crates/trusted-server-adapter-cloudflare/src/lib.rs +++ b/crates/trusted-server-adapter-cloudflare/src/lib.rs @@ -27,6 +27,7 @@ pub async fn main(req: Request, env: Env, ctx: Context) -> Result { if let Ok(config) = env.var("TRUSTED_SERVER_CONFIG") { app::set_cloudflare_config_json(config.to_string()); } + app::set_cloudflare_env(env.clone()); match edgezero_adapter_cloudflare::run_app::(req, env, ctx).await { Ok(resp) => Ok(resp), diff --git a/crates/trusted-server-adapter-cloudflare/src/middleware.rs b/crates/trusted-server-adapter-cloudflare/src/middleware.rs index 14efed56a..1e5f79ef3 100644 --- a/crates/trusted-server-adapter-cloudflare/src/middleware.rs +++ b/crates/trusted-server-adapter-cloudflare/src/middleware.rs @@ -113,6 +113,9 @@ impl AuthMiddleware { #[async_trait(?Send)] impl Middleware for AuthMiddleware { async fn handle(&self, mut ctx: RequestContext, next: Next<'_>) -> Result { + // Takes the request mutably because `enforce_basic_auth` marks requests + // whose credential it consumed itself; the shared template cache gate + // reads that marker later. match enforce_basic_auth(&self.settings, ctx.request_mut()) { Ok(Some(response)) => return Ok(response), Ok(None) => {} diff --git a/crates/trusted-server-adapter-cloudflare/src/platform.rs b/crates/trusted-server-adapter-cloudflare/src/platform.rs index fff0bfed1..cded42a0d 100644 --- a/crates/trusted-server-adapter-cloudflare/src/platform.rs +++ b/crates/trusted-server-adapter-cloudflare/src/platform.rs @@ -5,18 +5,16 @@ use std::time::Duration; use bytes::Bytes; use edgezero_core::config_store::ConfigStoreHandle; use edgezero_core::key_value_store::{KvHandle, KvPage, KvStore}; -use error_stack::Report; +use error_stack::{Report, ResultExt as _}; use trusted_server_core::platform::{ - ClientInfo, GeoInfo, KvError, PlatformBackend, PlatformBackendSpec, PlatformConfigStore, - PlatformError, PlatformGeo, PlatformHttpClient, PlatformKvStore, PlatformSecretStore, - RuntimeServices, StoreId, StoreName, UnavailableKvStore, + BackendNamingPolicy, ClientInfo, GeoInfo, KvError, PlatformBackend, PlatformBackendSpec, + PlatformConfigStore, PlatformError, PlatformGeo, PlatformHttpClient, PlatformKvStore, + PlatformSecretStore, RuntimeServices, StoreId, StoreName, UnavailableKvStore, }; #[cfg(not(target_arch = "wasm32"))] use trusted_server_core::platform::UnavailableHttpClient; -#[cfg(target_arch = "wasm32")] -use error_stack::ResultExt as _; #[cfg(target_arch = "wasm32")] use trusted_server_core::platform::{ PlatformHttpRequest, PlatformPendingRequest, PlatformResponse, PlatformSelectResult, @@ -61,27 +59,15 @@ impl PlatformSecretStore for NoopSecretStore { struct NoopBackend; impl PlatformBackend for NoopBackend { + fn naming_policy(&self) -> BackendNamingPolicy { + BackendNamingPolicy::Cloudflare + } + fn predict_name(&self, spec: &PlatformBackendSpec) -> Result> { - let port = spec - .port - .unwrap_or(if spec.scheme == "https" { 443 } else { 80 }); - let timeout_ms = spec.first_byte_timeout.as_millis(); - let cert_suffix = if spec.certificate_check { - "" - } else { - "_nocert" - }; - // Keep two providers that share an origin on distinct names so auction - // response correlation cannot cross providers. - let discriminator = spec - .discriminator - .as_deref() - .map(|d| format!("_p_{d}")) - .unwrap_or_default(); - Ok(format!( - "{}_{}_{}_{timeout_ms}ms{cert_suffix}{discriminator}", - spec.scheme, spec.host, port - )) + self.naming_policy() + .predict(spec) + .map(|prediction| prediction.name) + .change_context(PlatformError::Backend) } fn ensure(&self, spec: &PlatformBackendSpec) -> Result> { @@ -284,13 +270,22 @@ fn outbound_cache_mode(bypass_cache: bool) -> OutboundCacheMode { } } +#[cfg(target_arch = "wasm32")] +fn outbound_request_init(method: worker::Method, headers: worker::Headers) -> worker::RequestInit { + let mut init = worker::RequestInit::new(); + init.with_method(method) + .with_headers(headers) + .with_redirect(worker::RequestRedirect::Manual); + init +} + #[cfg(target_arch = "wasm32")] impl CloudflareHttpClient { async fn execute( &self, request: PlatformHttpRequest, ) -> Result> { - use worker::{CacheMode, Fetch, Headers, Method, Request, RequestInit, RequestRedirect}; + use worker::{CacheMode, Fetch, Headers, Method, Request}; // The Cloudflare fetch path cannot honor Fastly-style Image Optimizer // metadata, and it always buffers the response body (see below). The @@ -340,7 +335,6 @@ impl CloudflareHttpClient { } }; - let mut init = RequestInit::new(); // Force manual redirect handling: the Workers runtime otherwise defaults // to `RequestRedirect::Follow` and transparently chases 3xx responses to // any host inside `Fetch::send()`. Core's `proxy_with_redirects` does its @@ -348,9 +342,7 @@ impl CloudflareHttpClient { // `allowed_domains`; auto-following here would bypass that allowlist // (SSRF). `Manual` surfaces the 3xx + Location back to core unfollowed, // matching the Axum adapter's `redirect::Policy::none()`. - init.with_method(method) - .with_headers(headers) - .with_redirect(RequestRedirect::Manual); + let mut init = outbound_request_init(method, headers); // Setting the `cache` field requires the `cache_option_enabled` // compatibility flag, which is only on by default from compatibility // date 2024-11-11. `wrangler.toml`/`wrangler.ci.toml` pin an earlier @@ -547,8 +539,8 @@ impl PlatformHttpClient for CloudflareHttpClient { /// Bridges [`worker::Env`] secrets to [`PlatformSecretStore`] by calling /// `env.secret(key)` synchronously. Writes and deletes return errors. #[cfg(target_arch = "wasm32")] -struct CloudflareSecretStoreAdapter { - env: worker::Env, +pub(crate) struct CloudflareSecretStoreAdapter { + pub(crate) env: worker::Env, } #[cfg(target_arch = "wasm32")] @@ -762,6 +754,25 @@ fn reject_multi_provider_fanout(len: usize) -> Result<(), Report> mod tests { use super::*; use edgezero_core::context::RequestContext; + + #[cfg(target_arch = "wasm32")] + #[test] + fn outbound_request_creation_sets_manual_redirect_mode() { + let init = outbound_request_init(worker::Method::Get, worker::Headers::new()); + assert!(matches!(init.redirect, worker::RequestRedirect::Manual)); + } + + #[test] + fn auction_http_capabilities_are_explicit() { + let capabilities = trusted_server_core::platform::AuctionTargetId::Cloudflare + .descriptor() + .capabilities(); + assert!(!capabilities.supports_concurrent_provider_fanout()); + assert!( + !capabilities.has_enforceable_total_request_deadline(), + "Workers fetch does not expose an enforceable hard total request deadline" + ); + } use edgezero_core::http::{HeaderValue, request_builder}; use edgezero_core::params::PathParams; diff --git a/crates/trusted-server-adapter-cloudflare/wrangler.ci.toml b/crates/trusted-server-adapter-cloudflare/wrangler.ci.toml index e6891eb79..9992db712 100644 --- a/crates/trusted-server-adapter-cloudflare/wrangler.ci.toml +++ b/crates/trusted-server-adapter-cloudflare/wrangler.ci.toml @@ -14,3 +14,12 @@ id = "ci-local-kv" # Placeholder replaced by the integration test harness with a JSON object that # contains the runtime Trusted Server app-config blob envelope. TRUSTED_SERVER_CONFIG = "{}" + +# Fictitious integration-only secret values. `worker::Env::secret` reads these +# string bindings in local Wrangler runs; production values are provisioned with +# `wrangler secret put` instead of being committed to a manifest. +integration_admin_password = "integration-admin-password-32-bytes-ok" +integration_proxy_secret = "integration-test-proxy-secret-32-bytes-ok" +integration_ec_passphrase = "integration-test-ec-secret-padded-32" +integration_partner_token_alpha = "integration-test-token-alpha-32-bytes-ok" +integration_partner_token_bravo = "integration-test-token-bravo-32-bytes-ok" diff --git a/crates/trusted-server-adapter-cloudflare/wrangler.toml b/crates/trusted-server-adapter-cloudflare/wrangler.toml index 7c91173fc..48eb2db8d 100644 --- a/crates/trusted-server-adapter-cloudflare/wrangler.toml +++ b/crates/trusted-server-adapter-cloudflare/wrangler.toml @@ -26,3 +26,7 @@ id = "REPLACE_WITH_YOUR_KV_NAMESPACE_ID" # invalid placeholder with JSON containing an `app_config` blob envelope before # deploying or running `wrangler dev` against real traffic. TRUSTED_SERVER_CONFIG = '{"app_config":""}' + +# App-config secret values are provisioned as Worker secrets with +# `wrangler secret put `. The pushed blob contains only those key +# names; never add secret values to this file. diff --git a/crates/trusted-server-adapter-fastly/Cargo.toml b/crates/trusted-server-adapter-fastly/Cargo.toml index 47cc609b2..65320faa6 100644 --- a/crates/trusted-server-adapter-fastly/Cargo.toml +++ b/crates/trusted-server-adapter-fastly/Cargo.toml @@ -27,7 +27,6 @@ log = { workspace = true } log-fastly = { workspace = true } serde = { workspace = true } serde_json = { workspace = true } -sha2 = { workspace = true } trusted-server-core = { workspace = true } url = { workspace = true } urlencoding = { workspace = true } @@ -35,4 +34,5 @@ urlencoding = { workspace = true } [dev-dependencies] bytes = { workspace = true } edgezero-core = { workspace = true, features = ["test-utils"] } +toml = { workspace = true } trusted-server-core = { workspace = true, features = ["test-utils"] } diff --git a/crates/trusted-server-adapter-fastly/src/app.rs b/crates/trusted-server-adapter-fastly/src/app.rs index 06a0a155f..c1ba5ed6b 100644 --- a/crates/trusted-server-adapter-fastly/src/app.rs +++ b/crates/trusted-server-adapter-fastly/src/app.rs @@ -90,8 +90,10 @@ use std::sync::Arc; use crate::rate_limiter::{FastlyRateLimiter, RATE_COUNTER_NAME}; use edgezero_adapter_fastly::context::FastlyRequestContext; -use edgezero_core::app::{App, Hooks}; +use edgezero_adapter_fastly::env_config_from_runtime_dictionary; +use edgezero_core::app::{App, Hooks, StoreMetadata, StoresMetadata}; use edgezero_core::context::RequestContext; +use edgezero_core::env_config::EnvConfig; use edgezero_core::error::EdgeError; use edgezero_core::http::{ HandlerFuture, HeaderValue, Method, Request, Response, StatusCode, header, @@ -100,8 +102,11 @@ use edgezero_core::router::RouterService; use error_stack::Report; use trusted_server_core::auction::AuctionTelemetrySink; use trusted_server_core::auction::endpoints::handle_auction; -use trusted_server_core::auction::{AuctionOrchestrator, build_orchestrator}; +use trusted_server_core::auction::{ + AuctionOrchestrator, build_orchestrator_with_plan, compile_auction_plan, +}; use trusted_server_core::cache_policy::EdgeCacheHeader; +use trusted_server_core::config_payload::DEFAULT_SECRET_STORE_ID; use trusted_server_core::constants::{COOKIE_SHAREDID, COOKIE_TS_EIDS}; use trusted_server_core::ec::EcContext; use trusted_server_core::ec::admin::{ @@ -119,7 +124,9 @@ use trusted_server_core::integrations::{ IntegrationRegistry, ProxyDispatchInput, RequestFilterEffects, RequestFilterRegistryInput, RequestFilterRegistryOutcome, }; -use trusted_server_core::platform::{ClientInfo, GeoInfo, PlatformKvStore, RuntimeServices}; +use trusted_server_core::platform::{ + ClientInfo, GeoInfo, PlatformKvStore, RuntimeServices, StoreName, +}; use trusted_server_core::proxy::{ AssetProxyCachePolicy, handle_asset_proxy_request, handle_first_party_click, handle_first_party_proxy, handle_first_party_proxy_rebuild, handle_first_party_proxy_sign, @@ -134,9 +141,7 @@ use trusted_server_core::request_signing::{ handle_verify_signature, }; use trusted_server_core::settings::{ProxyAssetRoute, Settings}; -use trusted_server_core::settings_data::{ - default_config_key, default_config_store_name, get_settings_from_config_store, -}; +use trusted_server_core::settings_data::{DEFAULT_CONFIG_STORE_ID, get_settings_from_config_store}; use trusted_server_core::tester_cookie::{handle_clear_tester, handle_set_tester}; use crate::middleware::{AuthMiddleware, FinalizeResponseMiddleware}; @@ -149,6 +154,23 @@ use crate::platform::{ // AppState // --------------------------------------------------------------------------- +#[derive(Debug, Clone, PartialEq, Eq)] +pub(crate) struct RuntimeStoreConfig { + pub(crate) config_store_name: StoreName, + pub(crate) config_key: String, + pub(crate) secret_store_name: StoreName, +} + +impl RuntimeStoreConfig { + pub(crate) fn from_env(env: &EnvConfig) -> Self { + Self { + config_store_name: StoreName::from(env.store_name("config", DEFAULT_CONFIG_STORE_ID)), + config_key: env.store_key("config", DEFAULT_CONFIG_STORE_ID), + secret_store_name: StoreName::from(env.store_name("secrets", DEFAULT_SECRET_STORE_ID)), + } + } +} + /// Application state built once per Wasm instance and shared for its lifetime. /// /// In Fastly Compute each request spawns a new Wasm instance, so this struct is @@ -167,14 +189,22 @@ pub(crate) struct AppState { /// /// Returns an error when settings, the auction orchestrator, or the integration /// registry fail to initialise. -pub(crate) fn build_state() -> Result, Report> { - build_state_from_settings(load_settings_from_config_store()?) +pub(crate) fn build_state( + stores: &RuntimeStoreConfig, +) -> Result, Report> { + build_state_from_settings(load_settings_from_config_store(stores)?) } -pub(crate) fn load_settings_from_config_store() -> Result> { - let store_name = default_config_store_name(); - let config_key = default_config_key(); - get_settings_from_config_store(&FastlyPlatformConfigStore, &store_name, &config_key) +pub(crate) fn load_settings_from_config_store( + stores: &RuntimeStoreConfig, +) -> Result> { + get_settings_from_config_store( + &FastlyPlatformConfigStore, + &FastlyPlatformSecretStore, + &stores.config_store_name, + &stores.config_key, + &stores.secret_store_name, + ) } pub(crate) fn build_state_from_settings( @@ -182,8 +212,10 @@ pub(crate) fn build_state_from_settings( ) -> Result, Report> { warn_if_certificate_check_disabled(&settings); - let orchestrator = build_orchestrator(&settings)?; - let registry = IntegrationRegistry::new(&settings)?; + let plan = Arc::new(compile_auction_plan(&settings)?); + plan.validate_for_target(trusted_server_core::platform::AuctionTargetId::Fastly)?; + let orchestrator = build_orchestrator_with_plan(Arc::clone(&plan), &settings)?; + let registry = IntegrationRegistry::with_plan(&settings, plan)?; let auction_telemetry_sink = crate::tinybird::auction_sink_from_settings(&settings); let default_kv_store = Arc::new(UnavailableKvStore) as Arc; @@ -1228,15 +1260,17 @@ fn fallback_route_handler( pub struct TrustedServerApp; impl TrustedServerApp { - pub(crate) fn build_app_with_state() -> (App, Option>) { - let (router, state) = Self::router_with_state(); + pub(crate) fn build_app_with_state( + stores: &RuntimeStoreConfig, + ) -> (App, Option>) { + let (router, state) = Self::router_with_state(stores); let mut app = App::with_name(router, Self::name()); Self::configure(&mut app); (app, state) } - fn router_with_state() -> (RouterService, Option>) { - let state = match build_state() { + fn router_with_state(stores: &RuntimeStoreConfig) -> (RouterService, Option>) { + let state = match build_state(stores) { Ok(state) => state, Err(ref e) => { log::error!("failed to build application state: {:?}", e); @@ -1294,7 +1328,26 @@ impl Hooks for TrustedServerApp { } fn routes() -> RouterService { - Self::router_with_state().0 + let runtime_env = env_config_from_runtime_dictionary(Self::stores()); + let stores = RuntimeStoreConfig::from_env(&runtime_env); + Self::router_with_state(&stores).0 + } + + fn stores() -> StoresMetadata { + StoresMetadata { + config: Some(StoreMetadata { + default: DEFAULT_CONFIG_STORE_ID, + ids: &[DEFAULT_CONFIG_STORE_ID], + }), + kv: Some(StoreMetadata { + default: "trusted_server_kv", + ids: &["trusted_server_kv"], + }), + secrets: Some(StoreMetadata { + default: DEFAULT_SECRET_STORE_ID, + ids: &[DEFAULT_SECRET_STORE_ID], + }), + } } } @@ -1307,14 +1360,17 @@ mod tests { use super::{ AppState, AuctionDispatch, EcContext, EdgeCacheHeader, HandlerFuture, NAMED_ROUTES, - NamedRouteHandler, PAGE_BIDS_LEGACY_PATH, PAGE_BIDS_PATH, TrustedServerApp, - build_per_request_services, build_state_from_settings, handle_publisher_request, + NamedRouteHandler, PAGE_BIDS_LEGACY_PATH, PAGE_BIDS_PATH, RuntimeStoreConfig, + TrustedServerApp, build_orchestrator_with_plan, build_per_request_services, + build_state_from_settings, compile_auction_plan, handle_publisher_request, publisher_response_into_streaming_response, startup_error_router, }; use base64::Engine as _; use bytes::Bytes; + use edgezero_core::app::Hooks as _; use edgezero_core::body::Body; use edgezero_core::context::RequestContext; + use edgezero_core::env_config::EnvConfig; use edgezero_core::http::{Method, Response, StatusCode, header, request_builder}; use edgezero_core::key_value_store::NoopKvStore; use edgezero_core::params::PathParams; @@ -1340,6 +1396,82 @@ mod tests { }; use trusted_server_core::settings::Settings; + #[test] + fn hooks_store_metadata_matches_edgezero_manifest() { + let manifest: toml::Value = toml::from_str(include_str!("../../../edgezero.toml")) + .expect("should parse edgezero manifest"); + let manifest_stores = manifest + .get("stores") + .and_then(toml::Value::as_table) + .expect("manifest should declare stores"); + let metadata = TrustedServerApp::stores(); + + for (kind, runtime_store) in [ + ( + "config", + metadata.config.expect("should declare config stores"), + ), + ("kv", metadata.kv.expect("should declare KV stores")), + ( + "secrets", + metadata.secrets.expect("should declare secret stores"), + ), + ] { + let manifest_store = manifest_stores + .get(kind) + .and_then(toml::Value::as_table) + .unwrap_or_else(|| panic!("manifest should declare {kind} stores")); + let manifest_default = manifest_store + .get("default") + .and_then(toml::Value::as_str) + .unwrap_or_else(|| panic!("manifest {kind} stores should declare a default")); + let manifest_ids = manifest_store + .get("ids") + .and_then(toml::Value::as_array) + .unwrap_or_else(|| panic!("manifest {kind} stores should declare ids")) + .iter() + .map(toml::Value::as_str) + .collect::>>() + .unwrap_or_else(|| panic!("manifest {kind} store ids should be strings")); + + assert_eq!(runtime_store.default, manifest_default); + assert_eq!(runtime_store.ids, manifest_ids); + } + } + + #[test] + fn runtime_store_config_maps_logical_store_names_and_config_key() { + let env = EnvConfig::from_vars([ + ( + "EDGEZERO__STORES__CONFIG__TRUSTED_SERVER_CONFIG__NAME", + "physical_config", + ), + ( + "EDGEZERO__STORES__CONFIG__TRUSTED_SERVER_CONFIG__KEY", + "active_config", + ), + ( + "EDGEZERO__STORES__SECRETS__TRUSTED_SERVER_SECRETS__NAME", + "ts_secrets", + ), + ]); + + let stores = RuntimeStoreConfig::from_env(&env); + + assert_eq!(stores.config_store_name.as_ref(), "physical_config"); + assert_eq!(stores.config_key, "active_config"); + assert_eq!(stores.secret_store_name.as_ref(), "ts_secrets"); + } + + #[test] + fn runtime_store_config_uses_logical_defaults_without_overrides() { + let stores = RuntimeStoreConfig::from_env(&EnvConfig::default()); + + assert_eq!(stores.config_store_name.as_ref(), "trusted_server_config"); + assert_eq!(stores.config_key, "trusted_server_config"); + assert_eq!(stores.secret_store_name.as_ref(), "trusted_server_secrets"); + } + fn settings_with_missing_consent_store() -> Settings { Settings::from_toml( r#" @@ -1370,7 +1502,6 @@ mod tests { [integrations.prebid] enabled = true - server_url = "https://test-prebid.com/openrtb2/auction" external_bundle_url = "https://assets.example/prebid/trusted-prebid.js" [integrations.datadome] @@ -1378,7 +1509,10 @@ mod tests { [auction] enabled = true - providers = ["prebid"] + [auction.providers.prebid] + protocol = "openrtb-2.6" + profile = "prebid-server" + endpoint = "https://test-prebid.com/openrtb2/auction" timeout_ms = 2000 "#, ) @@ -1436,12 +1570,14 @@ mod tests { [integrations.prebid] enabled = true - server_url = "https://test-prebid.com/openrtb2/auction" external_bundle_url = "https://assets.example/prebid/trusted-prebid.js" [auction] enabled = true - providers = ["prebid"] + [auction.providers.prebid] + protocol = "openrtb-2.6" + profile = "prebid-server" + endpoint = "https://test-prebid.com/openrtb2/auction" timeout_ms = 2000 "#, ) @@ -1490,8 +1626,13 @@ mod tests { filters: Vec>, ) -> RouterService { let settings = test_settings(); - let orchestrator = trusted_server_core::auction::build_orchestrator(&settings) - .expect("should build orchestrator"); + let plan = Arc::new( + trusted_server_core::auction::compile_auction_plan(&settings) + .expect("should compile auction plan"), + ); + let orchestrator = + trusted_server_core::auction::build_orchestrator_with_plan(plan, &settings) + .expect("should build orchestrator"); let registry = IntegrationRegistry::from_request_filters(filters); let default_kv_store = Arc::new(crate::platform::UnavailableKvStore) as Arc; @@ -1577,6 +1718,34 @@ mod tests { } } + #[test] + fn startup_registers_aps_renderer_route() { + let mut settings = test_settings(); + settings.auction.providers.clear(); + settings.auction.providers.insert( + "aps-main".parse().expect("should parse APS provider ID"), + trusted_server_core::auction::ProviderConfig { + protocol: "openrtb-2.6".to_string(), + profile: "aps".to_string(), + endpoint: "https://aps.example/e/pb/bid".to_string(), + timeout_ms: None, + routing: trusted_server_core::auction::RoutingMode::AllEligible, + notifications: trusted_server_core::auction::NotificationConfig::default(), + profile_config: serde_json::json!({"account_id":"example-account"}), + }, + ); + + let state = build_state_from_settings(settings) + .expect("Fastly startup should register APS renderer"); + assert!( + state.registry.has_route( + &edgezero_core::http::Method::GET, + "/integrations/aps/renderer" + ), + "Fastly startup registry should expose the APS renderer" + ); + } + #[test] fn startup_error_router_handles_head_and_options() { let report = Report::new(TrustedServerError::BadRequest { @@ -2551,6 +2720,10 @@ mod tests { struct FixedBackend; impl PlatformBackend for FixedBackend { + fn naming_policy(&self) -> trusted_server_core::platform::BackendNamingPolicy { + trusted_server_core::platform::BackendNamingPolicy::Fastly + } + fn predict_name( &self, spec: &PlatformBackendSpec, @@ -2775,7 +2948,6 @@ mod tests { [auction] enabled = true - providers = [] [creative_opportunities] gam_network_id = "99999" @@ -2802,11 +2974,13 @@ mod tests { .geo(Arc::new(crate::platform::FastlyPlatformGeo)) .client_info(ClientInfo::default()) .build(); + let plan = Arc::new(compile_auction_plan(&settings).expect("should compile auction plan")); let registry = Arc::new( - IntegrationRegistry::new(&settings).expect("should build integration registry"), + IntegrationRegistry::with_plan(&settings, Arc::clone(&plan)) + .expect("should build integration registry"), ); let orchestrator = Arc::new( - trusted_server_core::auction::build_orchestrator(&settings) + build_orchestrator_with_plan(plan, &settings) .expect("should build auction orchestrator"), ); diff --git a/crates/trusted-server-adapter-fastly/src/backend.rs b/crates/trusted-server-adapter-fastly/src/backend.rs index f2ff5d9e5..39e3d7169 100644 --- a/crates/trusted-server-adapter-fastly/src/backend.rs +++ b/crates/trusted-server-adapter-fastly/src/backend.rs @@ -1,13 +1,16 @@ -use core::fmt::Write as _; use std::time::Duration; use error_stack::{Report, ResultExt as _}; use fastly::backend::Backend; -use sha2::{Digest as _, Sha256}; use url::Url; use trusted_server_core::error::TrustedServerError; -use trusted_server_core::host_header::validate_host_header_override_value; +use trusted_server_core::platform::{BackendNamingPolicy, PlatformBackendSpec, PredictedBackend}; + +#[cfg(test)] +const MAX_BACKEND_NAME_LEN: usize = 255; +#[cfg(test)] +const SPEC_DIGEST_HEX_LEN: usize = 32; /// Returns the default port for the given scheme (443 for HTTPS, 80 for HTTP). #[inline] @@ -19,6 +22,44 @@ fn default_port_for_scheme(scheme: &str) -> u16 { } } +#[derive(Debug, Clone, Eq, PartialEq)] +struct NormalizedBackendHost { + identity: String, + authority: String, + is_ip_literal: bool, +} + +/// Normalize URL-derived and direct hosts for transport and TLS use. +/// +/// `url::Url::host_str()` preserves brackets around IPv6 literals. Fastly's +/// backend target and HTTP authority require those brackets, while certificate +/// identity matching requires the bare address and SNI must not be sent for IP +/// literals. +#[inline] +fn normalize_backend_host(host: &str) -> NormalizedBackendHost { + let unbracketed = host + .strip_prefix('[') + .and_then(|value| value.strip_suffix(']')) + .unwrap_or(host); + match unbracketed.parse::() { + Ok(std::net::IpAddr::V6(_)) => NormalizedBackendHost { + identity: unbracketed.to_owned(), + authority: format!("[{unbracketed}]"), + is_ip_literal: true, + }, + Ok(std::net::IpAddr::V4(_)) => NormalizedBackendHost { + identity: unbracketed.to_owned(), + authority: unbracketed.to_owned(), + is_ip_literal: true, + }, + Err(_) => NormalizedBackendHost { + identity: host.to_owned(), + authority: host.to_owned(), + is_ip_literal: false, + }, + } +} + /// Compute the Host header value for a backend request. /// /// For standard ports (443 for HTTPS, 80 for HTTP), returns just the hostname. @@ -29,54 +70,14 @@ fn default_port_for_scheme(scheme: &str) -> u16 { /// would generate URLs without the port when the Host header didn't include it. #[inline] fn compute_host_header(scheme: &str, host: &str, port: u16) -> String { + let host = normalize_backend_host(host).authority; if port == default_port_for_scheme(scheme) { - host.to_owned() + host } else { format!("{host}:{port}") } } -fn sanitize_backend_name_component(value: &str) -> String { - value - .chars() - .map(|ch| { - if ch.is_ascii_alphanumeric() || matches!(ch, '-' | '_') { - ch - } else { - '_' - } - }) - .collect() -} - -/// Fastly's documented maximum length for a dynamic backend name. -const MAX_BACKEND_NAME_LEN: usize = 255; -/// Maximum length of the human-readable prefix folded into a backend name. -/// -/// Bounds the name so that `backend__` can never exceed -/// [`MAX_BACKEND_NAME_LEN`]: 8 (`backend_`) + 200 + 1 (`_`) + -/// [`SPEC_DIGEST_HEX_LEN`] = 241 ≤ 255. -const MAX_READABLE_PREFIX_LEN: usize = 200; -/// Width of the hex digest suffix — the first 128 bits of a SHA-256 over the -/// full backend spec, which is collision-resistant at the handful-of-hundreds -/// scale of a service's dynamic backends. -const SPEC_DIGEST_HEX_LEN: usize = 32; - -/// Hex-encode the first 128 bits of a SHA-256 digest of `canonical`. -/// -/// Used to make a backend name a collision-resistant function of the complete -/// backend spec (see [`BackendConfig::canonical_spec_string`]). -fn spec_digest_hex(canonical: &str) -> String { - let mut hasher = Sha256::new(); - hasher.update(canonical.as_bytes()); - let digest = hasher.finalize(); - let mut hex = String::with_capacity(SPEC_DIGEST_HEX_LEN); - for byte in digest.iter().take(SPEC_DIGEST_HEX_LEN / 2) { - write!(hex, "{byte:02x}").expect("should write hex digit to string"); - } - hex -} - /// Default first-byte timeout for backends (15 seconds). pub(crate) const DEFAULT_FIRST_BYTE_TIMEOUT: Duration = Duration::from_secs(15); /// Default timeout between response body bytes for backends (10 seconds). @@ -173,163 +174,47 @@ impl<'a> BackendConfig<'a> { self } - /// Build an unambiguous, length-prefixed encoding of the complete backend - /// spec for digesting. - /// - /// Every field is prefixed with its byte length so that no two distinct - /// specs can encode to the same string (a lossy substitution like - /// `sanitize_backend_name_component` cannot guarantee this). `Option` fields - /// are presence-tagged so a `None` never aliases a `Some("")`. The result is - /// fed to [`spec_digest_hex`]; it is never parsed, only hashed. - fn canonical_spec_string(&self, target_port: u16) -> String { - fn push_field(buf: &mut String, field: &str) { - buf.push_str(&field.len().to_string()); - buf.push(':'); - buf.push_str(field); - } - - let mut buf = String::new(); - push_field(&mut buf, self.scheme); - push_field(&mut buf, self.host); - push_field(&mut buf, &target_port.to_string()); - push_field(&mut buf, if self.certificate_check { "1" } else { "0" }); - match self.host_header_override { - Some(value) => { - buf.push('s'); - push_field(&mut buf, value); - } - None => buf.push('n'), + fn platform_spec(&self) -> PlatformBackendSpec { + PlatformBackendSpec { + scheme: self.scheme.to_owned(), + host: normalize_backend_host(self.host).identity, + port: self.port, + host_header_override: self.host_header_override.map(str::to_owned), + certificate_check: self.certificate_check, + first_byte_timeout: self.first_byte_timeout, + between_bytes_timeout: self.between_bytes_timeout, + discriminator: self.discriminator.map(str::to_owned), } - match self.discriminator { - Some(value) => { - buf.push('s'); - push_field(&mut buf, value); - } - None => buf.push('n'), - } - push_field(&mut buf, &self.first_byte_timeout.as_millis().to_string()); - push_field( - &mut buf, - &self.between_bytes_timeout.as_millis().to_string(), - ); - buf } /// Compute the deterministic backend name and resolved port without /// registering anything. - /// - /// The name is `backend__`, where `` is a - /// collision-resistant SHA-256 over an unambiguous encoding of the - /// *complete* backend spec — scheme, host, port, certificate setting, Host - /// override, provider discriminator, and the first-byte/between-bytes - /// timeouts (see [`canonical_spec_string`](Self::canonical_spec_string)). - /// Because distinct specs yield distinct digests, name equality implies spec - /// equality: that is what makes reusing a `NameInUse` backend provably safe, - /// and it prevents "first-registration-wins" poisoning where a later request - /// with a tighter timeout would inherit an earlier registration's value. The - /// `` half is a lossy, bounded slug carried only for logs — any - /// collision there is harmless because uniqueness comes from the digest. The - /// whole name is bounded to [`MAX_BACKEND_NAME_LEN`] so a long host or - /// discriminator can never produce a name Fastly rejects at registration. - fn compute_name(&self) -> Result<(String, u16), Report> { - if self.host.is_empty() { - return Err(Report::new(TrustedServerError::Proxy { - message: "missing host".to_owned(), - })); - } - if self.host.chars().any(char::is_control) { - return Err(Report::new(TrustedServerError::Proxy { - message: "host contains control characters".to_owned(), - })); - } - if self.scheme.chars().any(char::is_control) { - return Err(Report::new(TrustedServerError::Proxy { - message: "scheme contains control characters".to_owned(), - })); - } - if let Some(host_header_override) = self.host_header_override { - validate_host_header_override_value(host_header_override).map_err(|reason| { - Report::new(TrustedServerError::Proxy { - message: format!("host header override {reason}"), - }) - })?; - } - - let target_port = self - .port - .unwrap_or_else(|| default_port_for_scheme(self.scheme)); - - let name_base = format!("{}_{}_{}", self.scheme, self.host, target_port); - let host_override_suffix = self - .host_header_override - .map(|host| format!("_oh_{}", sanitize_backend_name_component(host))) - .unwrap_or_default(); - let cert_suffix = if self.certificate_check { - "" - } else { - "_nocert" - }; - let discriminator_suffix = self - .discriminator - .map(|d| format!("_p_{}", sanitize_backend_name_component(d))) - .unwrap_or_default(); - let first_byte_timeout_ms = self.first_byte_timeout.as_millis(); - let between_bytes_timeout_ms = self.between_bytes_timeout.as_millis(); - - // Lossy, human-readable slug for logs. Correctness does not depend on - // it — uniqueness comes from the digest below — so it is bounded to a - // fixed length. Sanitization only emits ASCII, so a char-boundary take - // is byte-exact. - let readable_full = format!( - "{}{}{}{}_fb{}_bb{}", - sanitize_backend_name_component(&name_base), - host_override_suffix, - cert_suffix, - discriminator_suffix, - first_byte_timeout_ms, - between_bytes_timeout_ms - ); - let readable: String = readable_full - .chars() - .take(MAX_READABLE_PREFIX_LEN) - .collect(); - - // Collision-resistant over the *complete* spec, so name equality implies - // spec equality and `NameInUse` reuse is safe. - let digest = spec_digest_hex(&self.canonical_spec_string(target_port)); - let backend_name = format!("backend_{readable}_{digest}"); - - // Bounded by construction; assert it so any future format change fails - // attributably during prediction rather than at Fastly registration. - if backend_name.len() > MAX_BACKEND_NAME_LEN { - return Err(Report::new(TrustedServerError::Proxy { - message: format!( - "backend name exceeds {MAX_BACKEND_NAME_LEN}-char limit ({} chars)", - backend_name.len() - ), - })); - } - - Ok((backend_name, target_port)) + fn predict_backend(&self) -> Result> { + BackendNamingPolicy::Fastly + .predict(&self.platform_spec()) + .change_context(TrustedServerError::Proxy { + message: "backend name prediction failed".to_owned(), + }) } /// Return the deterministic backend name without registering anything. /// - /// Convenience wrapper over `Self::compute_name` that discards the + /// Convenience wrapper over `Self::predict_backend` that discards the /// resolved port, used by [`crate::platform::PlatformBackend`] /// implementations that only need the name for correlation. /// /// # Errors /// /// Returns an error if the host is empty. + #[allow(dead_code, reason = "retained for backend-name parity tests")] pub fn predict_name(self) -> Result> { - self.compute_name().map(|(name, _)| name) + self.predict_backend().map(|prediction| prediction.name) } /// Ensure a dynamic backend exists for this configuration and return its name. /// /// The name is a collision-resistant function of the complete backend spec - /// (see `Self::compute_name`), so different specs — for example, different + /// (see `Self::predict_backend`), so different specs — for example, different /// timeout values — always produce different backend registrations and a /// tight deadline cannot be silently widened by an earlier registration. /// @@ -338,12 +223,15 @@ impl<'a> BackendConfig<'a> { /// Returns an error if the host is empty or if backend creation fails /// (except for `NameInUse` which reuses the existing backend). pub fn ensure(self) -> Result> { - let (backend_name, target_port) = self.compute_name()?; + let prediction = self.predict_backend()?; + let backend_name = prediction.name; + let target_port = prediction.port; + let host = normalize_backend_host(self.host); - let host_with_port = format!("{}:{}", self.host, target_port); + let host_with_port = format!("{}:{target_port}", host.authority); let host_header = self.host_header_override.map_or_else( - || compute_host_header(self.scheme, self.host, target_port), + || compute_host_header(self.scheme, &host.identity, target_port), str::to_owned, ); @@ -354,9 +242,12 @@ impl<'a> BackendConfig<'a> { .first_byte_timeout(self.first_byte_timeout) .between_bytes_timeout(self.between_bytes_timeout); if self.scheme.eq_ignore_ascii_case("https") { - builder = builder.enable_ssl().sni_hostname(self.host); + builder = builder.enable_ssl(); + if !host.is_ip_literal { + builder = builder.sni_hostname(&host.identity); + } if self.certificate_check { - builder = builder.check_certificate(self.host); + builder = builder.check_certificate(&host.identity); } else { log::warn!("INSECURE: certificate check disabled for backend: {backend_name}"); } @@ -474,7 +365,12 @@ impl<'a> BackendConfig<'a> { #[cfg(test)] mod tests { - use super::{BackendConfig, MAX_BACKEND_NAME_LEN, SPEC_DIGEST_HEX_LEN, compute_host_header}; + use trusted_server_core::platform::BackendNamingError; + + use super::{ + BackendConfig, MAX_BACKEND_NAME_LEN, SPEC_DIGEST_HEX_LEN, compute_host_header, + normalize_backend_host, + }; /// Assert a computed name is `backend__` and stays within /// Fastly's length limit. The digest is what makes the name injective, so @@ -503,6 +399,63 @@ mod tests { } // Tests for compute_host_header - the fix for port preservation in Host header + #[test] + fn ipv6_hosts_are_bracketed_only_for_authority_values() { + let bare = normalize_backend_host("2001:db8::1"); + let bracketed = normalize_backend_host("[2001:db8::1]"); + assert_eq!(bare, bracketed); + assert_eq!(bare.identity, "2001:db8::1"); + assert_eq!(bare.authority, "[2001:db8::1]"); + assert!(bare.is_ip_literal, "IPv6 must not be sent as TLS SNI"); + assert_eq!( + normalize_backend_host("cdn.example.com"), + super::NormalizedBackendHost { + identity: "cdn.example.com".to_string(), + authority: "cdn.example.com".to_string(), + is_ip_literal: false, + } + ); + assert_eq!( + compute_host_header("https", "[2001:db8::1]", 443), + "[2001:db8::1]" + ); + assert_eq!( + compute_host_header("https", "[2001:db8::1]", 8443), + "[2001:db8::1]:8443" + ); + } + + #[test] + fn url_derived_ipv6_host_uses_bare_tls_identity_without_sni() { + let (scheme, url_host, port) = + BackendConfig::parse_origin("https://[2001:db8::7]:8443/openrtb") + .expect("should parse IPv6 provider URL"); + assert_eq!(scheme, "https"); + assert_eq!(url_host, "[2001:db8::7]"); + assert_eq!(port, Some(8443)); + + let normalized = normalize_backend_host(&url_host); + assert_eq!(normalized.identity, "2001:db8::7"); + assert_eq!(normalized.authority, "[2001:db8::7]"); + assert!( + normalized.is_ip_literal, + "IP literals must omit TLS SNI while retaining a bare certificate identity" + ); + + let from_url_name = BackendConfig::new(&scheme, &url_host) + .port(port) + .predict_name() + .expect("should predict URL-derived IPv6 backend name"); + let from_bare_name = BackendConfig::new(&scheme, "2001:db8::7") + .port(port) + .predict_name() + .expect("should predict bare IPv6 backend name"); + assert_eq!( + from_url_name, from_bare_name, + "URL and direct IPv6 paths must preserve backend naming parity" + ); + } + #[test] fn host_header_includes_port_for_non_standard_https() { assert_eq!( @@ -584,8 +537,8 @@ mod tests { .predict_name() .expect_err("should reject host containing newline"); assert!( - err.to_string().contains("control characters"), - "should report control characters in error message" + err.contains::(), + "should preserve the backend naming error report context" ); } @@ -594,10 +547,9 @@ mod tests { let err = BackendConfig::new("https", "") .ensure() .expect_err("should reject empty host"); - let msg = err.to_string(); assert!( - msg.contains("missing host"), - "should report missing host in error message" + err.contains::(), + "should preserve the original backend naming error report context" ); } @@ -617,13 +569,13 @@ mod tests { #[test] fn host_header_overrides_produce_different_names() { - let (name_a, _) = BackendConfig::new("https", "origin.example.com") + let name_a = BackendConfig::new("https", "origin.example.com") .host_header_override(Some("www.example.com")) - .compute_name() + .predict_name() .expect("should compute name with host header override"); - let (name_b, _) = BackendConfig::new("https", "origin.example.com") + let name_b = BackendConfig::new("https", "origin.example.com") .host_header_override(Some("m.example.com")) - .compute_name() + .predict_name() .expect("should compute name with different host header override"); assert_ne!( @@ -648,8 +600,8 @@ mod tests { .expect_err("should reject host header override containing newline"); assert!( - err.to_string().contains("control characters"), - "should report control characters in error message" + err.contains::(), + "should preserve the backend naming error report context" ); } @@ -668,8 +620,8 @@ mod tests { .expect_err("should reject invalid host header override"); assert!( - err.to_string().contains("host header override"), - "should report host header override error for {host_header_override:?}" + err.contains::(), + "should preserve the backend naming error report context for {host_header_override:?}" ); } } @@ -678,13 +630,13 @@ mod tests { fn different_timeouts_produce_different_names() { use std::time::Duration; - let (name_a, _) = BackendConfig::new("https", "origin.example.com") + let name_a = BackendConfig::new("https", "origin.example.com") .first_byte_timeout(Duration::from_secs(2)) - .compute_name() + .predict_name() .expect("should compute name with 2000ms timeout"); - let (name_b, _) = BackendConfig::new("https", "origin.example.com") + let name_b = BackendConfig::new("https", "origin.example.com") .first_byte_timeout(Duration::from_millis(500)) - .compute_name() + .predict_name() .expect("should compute name with 500ms timeout"); assert_ne!( name_a, name_b, @@ -704,13 +656,13 @@ mod tests { fn different_between_bytes_timeouts_produce_different_names() { use std::time::Duration; - let (name_a, _) = BackendConfig::new("https", "origin.example.com") + let name_a = BackendConfig::new("https", "origin.example.com") .between_bytes_timeout(Duration::from_secs(2)) - .compute_name() + .predict_name() .expect("should compute name with 2000ms between-bytes timeout"); - let (name_b, _) = BackendConfig::new("https", "origin.example.com") + let name_b = BackendConfig::new("https", "origin.example.com") .between_bytes_timeout(Duration::from_millis(500)) - .compute_name() + .predict_name() .expect("should compute name with 500ms between-bytes timeout"); assert_ne!( diff --git a/crates/trusted-server-adapter-fastly/src/main.rs b/crates/trusted-server-adapter-fastly/src/main.rs index 90879c3ba..b086467e3 100644 --- a/crates/trusted-server-adapter-fastly/src/main.rs +++ b/crates/trusted-server-adapter-fastly/src/main.rs @@ -1,7 +1,9 @@ use std::sync::Arc; use edgezero_adapter_fastly::config_store::FastlyConfigStore as EdgeZeroFastlyConfigStore; +use edgezero_adapter_fastly::env_config_from_runtime_dictionary; use edgezero_adapter_fastly::request::into_core_request; +use edgezero_core::app::Hooks as _; use edgezero_core::body::Body as EdgeBody; use edgezero_core::config_store::ConfigStoreHandle; use edgezero_core::error::EdgeError; @@ -40,24 +42,22 @@ mod rate_limiter; mod template_cache; mod tinybird; -use crate::app::{EcFinalizeState, TrustedServerApp, load_settings_from_config_store}; +use crate::app::{ + EcFinalizeState, RuntimeStoreConfig, TrustedServerApp, load_settings_from_config_store, +}; use crate::ec_kv::FastlyEcKvStore; use crate::middleware::{HEADER_X_TS_FINALIZED, apply_finalize_headers, resolve_geo_for_response}; use crate::platform::{FastlyPlatformGeo, client_info_from_request}; use crate::rate_limiter::{FastlyRateLimiter, RATE_COUNTER_NAME}; -const TRUSTED_SERVER_CONFIG_STORE: &str = "trusted_server_config"; - /// Opens the Fastly Config Store used by the `EdgeZero` dispatcher. /// /// # Errors /// /// Returns [`fastly::Error`] if the config store cannot be opened. -fn open_trusted_server_config_store() -> Result { - let store = EdgeZeroFastlyConfigStore::try_open(TRUSTED_SERVER_CONFIG_STORE).map_err(|e| { - fastly::Error::msg(format!( - "failed to open config store `{TRUSTED_SERVER_CONFIG_STORE}`: {e}" - )) +fn open_trusted_server_config_store(store_name: &str) -> Result { + let store = EdgeZeroFastlyConfigStore::try_open(store_name).map_err(|e| { + fastly::Error::msg(format!("failed to open config store `{store_name}`: {e}")) })?; Ok(ConfigStoreHandle::new(Arc::new(store))) } @@ -90,11 +90,14 @@ fn main() { /// Handles a request through the `EdgeZero` router path. fn edgezero_main(mut req: FastlyRequest) { + let runtime_env = env_config_from_runtime_dictionary(TrustedServerApp::stores()); + let runtime_stores = RuntimeStoreConfig::from_env(&runtime_env); + // Short-circuit the JA4 debug probe before app construction. Must run here // because TLS/JA4 accessors are only available on FastlyRequest before // conversion to edgezero types. if req.get_method() == FastlyMethod::GET && req.get_path() == "/_ts/debug/ja4" { - match load_settings_from_config_store() { + match load_settings_from_config_store(&runtime_stores) { Ok(settings) if settings.debug.ja4_endpoint_enabled => { build_ja4_debug_response(&req).send_to_client(); } @@ -111,18 +114,19 @@ fn edgezero_main(mut req: FastlyRequest) { return; } - let config_store = match open_trusted_server_config_store() { - Ok(cs) => cs, - Err(e) => { - log::error!("failed to open config store: {e}"); - FastlyResponse::from_status(fastly::http::StatusCode::INTERNAL_SERVER_ERROR) - .with_body_text_plain("Internal Server Error") - .send_to_client(); - return; - } - }; + let config_store = + match open_trusted_server_config_store(runtime_stores.config_store_name.as_ref()) { + Ok(cs) => cs, + Err(e) => { + log::error!("failed to open config store: {e}"); + FastlyResponse::from_status(fastly::http::StatusCode::INTERNAL_SERVER_ERROR) + .with_body_text_plain("Internal Server Error") + .send_to_client(); + return; + } + }; - let (app, app_state) = TrustedServerApp::build_app_with_state(); + let (app, app_state) = TrustedServerApp::build_app_with_state(&runtime_stores); let settings_snapshot = app_state.as_ref().map(|state| Arc::clone(&state.settings)); let trusted_client_ip = settings_snapshot .as_deref() @@ -197,7 +201,7 @@ fn edgezero_main(mut req: FastlyRequest) { if let Some(settings) = settings_snapshot.as_deref() { apply_entry_point_finalize_headers(settings, &mut response, client_ip); } else { - match load_settings_from_config_store() { + match load_settings_from_config_store(&runtime_stores) { Ok(settings) => { apply_entry_point_finalize_headers(&settings, &mut response, client_ip); } @@ -227,7 +231,7 @@ fn edgezero_main(mut req: FastlyRequest) { } } } else { - match load_settings_from_config_store() { + match load_settings_from_config_store(&runtime_stores) { Ok(settings) => { match apply_edgezero_ec_finalize(&settings, &ec_state, &mut response) { Ok(partner_registry) => { @@ -605,7 +609,7 @@ mod tests { .headers() .get("cache-control") .and_then(|value| value.to_str().ok()), - Some("no-store, private") + Some("private, no-store") ); assert!(response.headers().get("surrogate-control").is_none()); assert!(response.headers().get("cdn-cache-control").is_none()); @@ -631,7 +635,7 @@ mod tests { .headers() .get("cache-control") .and_then(|value| value.to_str().ok()), - Some("no-store, private") + Some("private, no-store") ); assert!(response.headers().get("surrogate-control").is_none()); assert!(response.headers().get("cdn-cache-control").is_none()); @@ -716,7 +720,7 @@ mod tests { .headers() .get("cache-control") .and_then(|value| value.to_str().ok()), - Some("no-store, private"), + Some("private, no-store"), "request-scoped diagnostics HTML must never become shared-cacheable" ); assert!( diff --git a/crates/trusted-server-adapter-fastly/src/middleware.rs b/crates/trusted-server-adapter-fastly/src/middleware.rs index 283f16255..7b2bb636e 100644 --- a/crates/trusted-server-adapter-fastly/src/middleware.rs +++ b/crates/trusted-server-adapter-fastly/src/middleware.rs @@ -128,6 +128,9 @@ impl AuthMiddleware { #[async_trait(?Send)] impl Middleware for AuthMiddleware { async fn handle(&self, mut ctx: RequestContext, next: Next<'_>) -> Result { + // Takes the request mutably because `enforce_basic_auth` marks requests + // whose credential it consumed itself; the shared template cache gate + // reads that marker later. match enforce_basic_auth(&self.settings, ctx.request_mut()) { Ok(Some(response)) => return Ok(response), Ok(None) => {} diff --git a/crates/trusted-server-adapter-fastly/src/platform.rs b/crates/trusted-server-adapter-fastly/src/platform.rs index 638aed82b..e612830b9 100644 --- a/crates/trusted-server-adapter-fastly/src/platform.rs +++ b/crates/trusted-server-adapter-fastly/src/platform.rs @@ -15,11 +15,12 @@ use fastly::{ConfigStore, Request, SecretStore}; use crate::backend::BackendConfig; pub(crate) use trusted_server_core::platform::UnavailableKvStore; use trusted_server_core::platform::{ - ClientInfo, GeoInfo, PlatformBackend, PlatformBackendSpec, PlatformConfigStore, PlatformError, - PlatformGeo, PlatformHttpClient, PlatformHttpRequest, PlatformImageOptimizerCrop, - PlatformImageOptimizerCropMode, PlatformImageOptimizerOptions, PlatformImageOptimizerParams, - PlatformImageOptimizerRegion, PlatformKvStore, PlatformPendingRequest, PlatformResponse, - PlatformSecretStore, PlatformSelectResult, StoreId, StoreName, + BackendNamingPolicy, ClientInfo, GeoInfo, PlatformBackend, PlatformBackendSpec, + PlatformConfigStore, PlatformError, PlatformGeo, PlatformHttpClient, PlatformHttpRequest, + PlatformImageOptimizerCrop, PlatformImageOptimizerCropMode, PlatformImageOptimizerOptions, + PlatformImageOptimizerParams, PlatformImageOptimizerRegion, PlatformKvStore, + PlatformPendingRequest, PlatformResponse, PlatformSecretStore, PlatformSelectResult, StoreId, + StoreName, }; use trusted_server_core::settings::TrustedClientIpConfig; @@ -150,6 +151,11 @@ impl PlatformSecretStore for FastlyPlatformSecretStore { /// timeout → unique name). pub struct FastlyPlatformBackend; +#[cfg(test)] +const TRANSPORT_TIMEOUT_QUANTUM_MS: u32 = 250; +#[cfg(test)] +const SUB_QUANTUM_LADDER_MS: [u32; 4] = [200, 150, 100, 50]; + fn backend_config_from_spec(spec: &PlatformBackendSpec) -> BackendConfig<'_> { BackendConfig::new(&spec.scheme, &spec.host) .port(spec.port) @@ -160,83 +166,15 @@ fn backend_config_from_spec(spec: &PlatformBackendSpec) -> BackendConfig<'_> { .discriminator(spec.discriminator.as_deref()) } -/// Transport-timeout quantum for auction backends (see -/// [`FastlyPlatformBackend::canonicalize_transport_timeout_ms`]). -const TRANSPORT_TIMEOUT_QUANTUM_MS: u32 = 250; - -/// Upper bound of the fine-grained quantum range. -/// -/// Budget-bound values below this ceiling are floored to a -/// [`TRANSPORT_TIMEOUT_QUANTUM_MS`] multiple (the issue #847 behavior for the -/// default 2000 ms auction). At or above it, values snap to the coarse -/// [`TRANSPORT_TIMEOUT_COARSE_LADDER_MS`] instead so the total number of -/// distinct budget-derived buckets stays globally bounded regardless of how -/// large the configured ceiling is. -const TRANSPORT_TIMEOUT_QUANTUM_CEILING_MS: u32 = 2000; - -/// Coarse rungs for budget-bound transport timeouts below one quantum, -/// ordered high to low. -/// -/// Below one quantum, passing the exact wall-clock remainder through would mint -/// a distinct backend name for every millisecond in `1..250`, so the -/// near-exhausted tail alone could exceed Fastly's per-service dynamic backend -/// limit. Snapping to this finite ladder instead bounds the number of -/// budget-derived names an origin can produce. Budgets below the smallest rung -/// round to zero, which callers treat as "budget exhausted — skip the launch". -const SUB_QUANTUM_LADDER_MS: [u32; 4] = [200, 150, 100, 50]; - -/// Coarse rungs for budget-bound transport timeouts at or above the quantum -/// ceiling, ascending. Every rung is a [`TRANSPORT_TIMEOUT_QUANTUM_MS`] -/// multiple. -/// -/// Above [`TRANSPORT_TIMEOUT_QUANTUM_CEILING_MS`], flooring to a 250 ms multiple -/// would let a large configured ceiling (e.g. 60,000 ms) mint hundreds of -/// distinct backend names — recreating the per-service dynamic backend -/// exhaustion this quantization exists to prevent. This fixed, globally finite -/// ladder caps the number of high-budget buckets instead: values are floored to -/// the greatest rung no larger than the remaining budget, and anything above -/// the top rung clamps to it. Rounding down never extends a transport cap past -/// the remaining budget. -/// -/// The rung spacing trades transport window for cardinality: just below a rung -/// the haircut approaches the gap to the rung beneath (worst case ~50%, e.g. a -/// remaining budget of 9,999 ms snaps to 5,000 ms). This is accepted — on the -/// mediator path this value is the effective bound, but a denser ladder would -/// buy back at most half a bucket of transport time at the cost of -/// proportionally more backend names. -const TRANSPORT_TIMEOUT_COARSE_LADDER_MS: [u32; 8] = - [2000, 3000, 5000, 10000, 20000, 30000, 45000, 60000]; - -/// Round a budget-bound transport timeout down to a stable, globally bounded -/// bucket. -/// -/// - At or above [`TRANSPORT_TIMEOUT_QUANTUM_CEILING_MS`], floors to the -/// greatest [`TRANSPORT_TIMEOUT_COARSE_LADDER_MS`] rung no larger than -/// `remaining_ms` (clamping to the top rung above it). -/// - Within the quantum range, floors to a [`TRANSPORT_TIMEOUT_QUANTUM_MS`] -/// multiple. -/// - Below one quantum, snaps down to the greatest [`SUB_QUANTUM_LADDER_MS`] -/// rung no larger than `remaining_ms` (or zero). -fn quantize_transport_timeout_ms(remaining_ms: u32) -> u32 { - if remaining_ms >= TRANSPORT_TIMEOUT_QUANTUM_CEILING_MS { - return TRANSPORT_TIMEOUT_COARSE_LADDER_MS - .into_iter() - .rev() - .find(|&rung| rung <= remaining_ms) - .unwrap_or(TRANSPORT_TIMEOUT_QUANTUM_CEILING_MS); - } - let floored = (remaining_ms / TRANSPORT_TIMEOUT_QUANTUM_MS) * TRANSPORT_TIMEOUT_QUANTUM_MS; - if floored > 0 { - return floored; - } - SUB_QUANTUM_LADDER_MS - .into_iter() - .find(|&rung| rung <= remaining_ms) - .unwrap_or(0) -} - impl PlatformBackend for FastlyPlatformBackend { + fn naming_policy(&self) -> BackendNamingPolicy { + BackendNamingPolicy::Fastly + } + fn predict_name(&self, spec: &PlatformBackendSpec) -> Result> { + // Use the same host normalization as registration. In particular, + // URL-derived IPv6 hosts arrive bracketed, but both forms must predict + // the backend that `ensure` actually registers. backend_config_from_spec(spec) .predict_name() .change_context(PlatformError::Backend) @@ -247,28 +185,6 @@ impl PlatformBackend for FastlyPlatformBackend { .ensure() .change_context(PlatformError::Backend) } - - /// Quantize the transport timeout so budget-derived values do not mint a - /// new dynamic backend name on every request. - /// - /// Fastly embeds the first-byte and between-bytes timeouts in the dynamic - /// backend name (see [`BackendConfig`]) and pools connections per backend - /// name. A per-request wall-clock budget would otherwise defeat that - /// pooling and accumulate registrations toward the per-service dynamic - /// backend limit. - /// - /// A provider's own configured timeout is a constant, so when it is the - /// binding constraint it is returned verbatim — including sub-quantum - /// configured values, which must not be rounded away or the provider could - /// never launch. Only the budget-bound value is snapped to a stable bucket - /// via [`quantize_transport_timeout_ms`]. Rounding down never extends a - /// transport cap past the remaining budget. - fn canonicalize_transport_timeout_ms(&self, remaining_ms: u32, configured_ms: u32) -> u32 { - if remaining_ms >= configured_ms { - return configured_ms; - } - quantize_transport_timeout_ms(remaining_ms) - } } // --------------------------------------------------------------------------- @@ -544,6 +460,14 @@ fn apply_fastly_cache_bypass(request: &mut fastly::Request, bypass_cache: bool) /// - [`select`](PlatformHttpClient::select) downcasts each /// [`PlatformPendingRequest`] back to `fastly::PendingRequest` and calls /// `fastly::http::request::select()`. +/// +/// Fastly's Compute HTTP API sends one request to the named backend and returns +/// the origin response; it has no client-side redirect-follow mode. Consequently +/// each trait call below performs exactly one underlying `.send()` or +/// `.send_async()`, and an original 3xx remains visible to core. The host test +/// environment cannot register a real Fastly backend, so the common +/// `StubHttpClient` driver test records the one-send 3xx behavior while adapter +/// tests cover request conversion and the single-send boundary. pub struct FastlyPlatformHttpClient; #[async_trait::async_trait(?Send)] @@ -1136,8 +1060,72 @@ mod tests { ); } + #[test] + fn bracketed_ipv6_predict_name_matches_bare_and_ensured_backend_name() { + let backend = FastlyPlatformBackend; + let bracketed = PlatformBackendSpec { + scheme: "https".to_string(), + host: "[2001:db8::9]".to_string(), + port: Some(8443), + host_header_override: None, + certificate_check: true, + first_byte_timeout: Duration::from_millis(750), + between_bytes_timeout: Duration::from_millis(750), + discriminator: Some("ipv6-provider".to_string()), + }; + let mut bare = bracketed.clone(); + bare.host = "2001:db8::9".to_string(); + + let predicted = backend + .predict_name(&bracketed) + .expect("should predict bracketed IPv6 backend name"); + let bare_predicted = backend + .predict_name(&bare) + .expect("should predict bare IPv6 backend name"); + let ensured = backend + .ensure(&bracketed) + .expect("should register bracketed IPv6 backend"); + + assert_eq!(predicted, bare_predicted); + assert_eq!(predicted, ensured); + } + // --- FastlyPlatformHttpClient ------------------------------------------- + #[test] + fn auction_http_capabilities_are_explicit() { + let client = FastlyPlatformHttpClient; + let capabilities = trusted_server_core::platform::AuctionTargetId::Fastly + .descriptor() + .capabilities(); + assert!(client.supports_concurrent_fanout()); + assert!(capabilities.supports_concurrent_provider_fanout()); + assert!(!client.has_enforceable_total_request_deadline()); + assert!( + !capabilities.has_enforceable_total_request_deadline(), + "first-byte and between-byte timers are not a hard total request deadline" + ); + } + + #[test] + fn response_conversion_preserves_original_redirect_at_single_send_boundary() { + let mut response = fastly::Response::from_status(fastly::http::StatusCode::FOUND); + response.set_header("location", "https://redirect.example/next"); + + let platform = fastly_response_to_platform(response, "origin", false, false) + .expect("should convert redirect response"); + + assert_eq!(platform.response.status().as_u16(), 302); + assert_eq!( + platform + .response + .headers() + .get("location") + .and_then(|value| value.to_str().ok()), + Some("https://redirect.example/next") + ); + } + #[test] fn apply_fastly_cache_bypass_sets_pass_when_enabled() { let mut request = fastly::Request::get("https://example.com/"); diff --git a/crates/trusted-server-adapter-fastly/src/tinybird.rs b/crates/trusted-server-adapter-fastly/src/tinybird.rs index f2df61744..a4d5c023c 100644 --- a/crates/trusted-server-adapter-fastly/src/tinybird.rs +++ b/crates/trusted-server-adapter-fastly/src/tinybird.rs @@ -10,9 +10,8 @@ use trusted_server_core::auction::telemetry::{ AuctionEventBatch, AuctionTelemetrySink, NoopAuctionTelemetrySink, }; use trusted_server_core::error::TrustedServerError; -use trusted_server_core::platform::{ - PlatformBackendSpec, PlatformHttpRequest, RuntimeServices, StoreName, -}; +use trusted_server_core::platform::{PlatformBackendSpec, PlatformHttpRequest, RuntimeServices}; +use trusted_server_core::redacted::Redacted; use trusted_server_core::settings::{Settings, TinybirdSettings}; const TINYBIRD_EVENTS_PATH: &str = "/v0/events"; @@ -43,8 +42,7 @@ struct FastlyTinybirdAuctionTelemetrySink { struct TinybirdEventsTarget { api_host: String, dataset: String, - secret_store: StoreName, - token_secret: String, + append_token: Redacted, uri: String, backend_spec: PlatformBackendSpec, max_body_bytes: usize, @@ -57,8 +55,9 @@ impl TinybirdEventsTarget { Self { api_host: config.api_host, dataset: config.auction_dataset, - secret_store: StoreName::from(config.secret_store), - token_secret: config.auction_token_secret, + append_token: config + .auction_token_secret + .expect("should contain a resolved Tinybird auction token when enabled"), uri, backend_spec, max_body_bytes: config.max_body_bytes, @@ -95,25 +94,6 @@ impl FastlyTinybirdAuctionTelemetrySink { batch.to_ndjson(self.target.max_body_bytes) } - fn load_append_token( - &self, - services: &RuntimeServices, - ) -> Result> { - let token = services - .secret_store() - .get_string(&self.target.secret_store, &self.target.token_secret) - .change_context(TrustedServerError::Proxy { - message: "Tinybird auction append token unavailable".to_owned(), - })?; - let token = token.trim().to_owned(); - if token.is_empty() { - return Err(Report::new(TrustedServerError::Proxy { - message: "Tinybird auction append token is empty".to_owned(), - })); - } - Ok(token) - } - fn ensure_backend( &self, services: &RuntimeServices, @@ -185,8 +165,7 @@ impl AuctionTelemetrySink for FastlyTinybirdAuctionTelemetrySink { Self::validate_batch(&batch)?; let body = self.serialize_batch(&batch)?; let body_len = body.len(); - let token = self.load_append_token(services)?; - let auth_header = Self::authorization_header(&token)?; + let auth_header = Self::authorization_header(self.target.append_token.expose())?; let backend_name = self.ensure_backend(services)?; let request = self.build_events_request(body, auth_header)?; @@ -233,7 +212,7 @@ mod tests { use trusted_server_core::platform::{ ClientInfo, PlatformBackend, PlatformConfigStore, PlatformError, PlatformGeo, PlatformHttpClient, PlatformPendingRequest, PlatformResponse, PlatformSecretStore, - PlatformSelectResult, RuntimeServices, StoreId, + PlatformSelectResult, RuntimeServices, StoreId, StoreName, }; use super::*; @@ -297,6 +276,10 @@ mod tests { } impl PlatformBackend for RecordingBackend { + fn naming_policy(&self) -> trusted_server_core::platform::BackendNamingPolicy { + trusted_server_core::platform::BackendNamingPolicy::Fastly + } + fn predict_name( &self, _spec: &PlatformBackendSpec, @@ -399,6 +382,7 @@ mod tests { region: None, is_mobile: 0, is_known_browser: 1, + browser_family: Some("chrome".to_owned()), gdpr_applies: 0, consent_present: 0, terminal_status: Some("completed".to_owned()), @@ -444,12 +428,12 @@ mod tests { TinybirdSettings { enabled: true, api_host: "api.us-east.aws.tinybird.co".to_owned(), - secret_store: "ts_secrets".to_owned(), + secret_store: None, auction_dataset: "auction_events_raw".to_owned(), - auction_token_secret: "tinybird_auction_append_token".to_owned(), + auction_token_secret: Some(Redacted::new("append-token".to_owned())), access_enabled: false, access_dataset: "access_logs_raw".to_owned(), - access_token_secret: "tinybird_access_append_token".to_owned(), + access_token_secret: None, access_sample_rate: 0.0, max_body_bytes: 1024 * 1024, } @@ -481,16 +465,13 @@ mod tests { } #[test] - fn sink_posts_ndjson_with_secret_token_and_does_not_wait() { + fn sink_posts_ndjson_with_resolved_token_and_does_not_wait() { let backend = Arc::new(RecordingBackend::default()); let http_client = Arc::new(RecordingHttpClient::default()); let services = services( Arc::clone(&backend), Arc::clone(&http_client), - HashMap::from([( - "tinybird_auction_append_token".to_owned(), - b" append-token\n".to_vec(), - )]), + HashMap::new(), ); let sink = FastlyTinybirdAuctionTelemetrySink::new(enabled_config()); @@ -601,31 +582,6 @@ mod tests { ); } - #[test] - fn sink_drops_missing_secret_as_setup_error() { - let backend = Arc::new(RecordingBackend::default()); - let http_client = Arc::new(RecordingHttpClient::default()); - let services = services(backend, Arc::clone(&http_client), HashMap::new()); - let sink = FastlyTinybirdAuctionTelemetrySink::new(enabled_config()); - - let result = futures::executor::block_on( - sink.emit_auction_events(&services, AuctionEventBatch::new(vec![test_row()])), - ); - - assert!( - result.is_err(), - "best-effort caller will suppress this error" - ); - assert!( - http_client - .requests - .lock() - .expect("should lock recorded requests") - .is_empty(), - "should not send without a token" - ); - } - #[test] fn sink_drops_row_count_oversize_before_sending() { let backend = Arc::new(RecordingBackend::default()); diff --git a/crates/trusted-server-adapter-spin/spin.toml b/crates/trusted-server-adapter-spin/spin.toml index 9bc3634d8..684c171b0 100644 --- a/crates/trusted-server-adapter-spin/spin.toml +++ b/crates/trusted-server-adapter-spin/spin.toml @@ -25,6 +25,15 @@ version = "0.1.0" [variables] v_current_x2dkid = { default = "" } v_active_x2dkids = { default = "" } +# These declared variables match the example config's secret key names. Regenerate +# or extend them for deployment-specific keys, including handler key names such as +# `admin_password` or `api_handler_password`. Replace the empty defaults with values +# supplied by the deployment's secret provider; never commit values here. +v_trusted_x5fserver_x5fsecrets_v_publisher_x5fproxy_x5fsecret = { default = "", secret = true } +v_trusted_x5fserver_x5fsecrets_v_ec_x5fpassphrase = { default = "", secret = true } +v_trusted_x5fserver_x5fsecrets_v_partner_x5fapi_x5ftoken = { default = "", secret = true } +v_trusted_x5fserver_x5fsecrets_v_partner_x5fts_x5fpull_x5ftoken = { default = "", secret = true } +v_trusted_x5fserver_x5fsecrets_v_handler_x5fpassword = { default = "", secret = true } [[trigger.http]] route = "/..." @@ -43,6 +52,11 @@ key_value_stores = ["default"] [component.trusted-server.variables] v_current_x2dkid = "{{ v_current_x2dkid }}" v_active_x2dkids = "{{ v_active_x2dkids }}" +v_trusted_x5fserver_x5fsecrets_v_publisher_x5fproxy_x5fsecret = "{{ v_trusted_x5fserver_x5fsecrets_v_publisher_x5fproxy_x5fsecret }}" +v_trusted_x5fserver_x5fsecrets_v_ec_x5fpassphrase = "{{ v_trusted_x5fserver_x5fsecrets_v_ec_x5fpassphrase }}" +v_trusted_x5fserver_x5fsecrets_v_partner_x5fapi_x5ftoken = "{{ v_trusted_x5fserver_x5fsecrets_v_partner_x5fapi_x5ftoken }}" +v_trusted_x5fserver_x5fsecrets_v_partner_x5fts_x5fpull_x5ftoken = "{{ v_trusted_x5fserver_x5fsecrets_v_partner_x5fts_x5fpull_x5ftoken }}" +v_trusted_x5fserver_x5fsecrets_v_handler_x5fpassword = "{{ v_trusted_x5fserver_x5fsecrets_v_handler_x5fpassword }}" [component.trusted-server.build] command = "cargo build --target wasm32-wasip1 --release -p trusted-server-adapter-spin --features spin" diff --git a/crates/trusted-server-adapter-spin/src/app.rs b/crates/trusted-server-adapter-spin/src/app.rs index f24b5b717..e6abe75ea 100644 --- a/crates/trusted-server-adapter-spin/src/app.rs +++ b/crates/trusted-server-adapter-spin/src/app.rs @@ -1,16 +1,24 @@ use std::net::{IpAddr, SocketAddr}; use std::sync::Arc; +#[cfg(all(feature = "spin", target_arch = "wasm32"))] +use edgezero_adapter_spin::config_store::SpinConfigStore; use edgezero_adapter_spin::context::SpinRequestContext; use edgezero_core::app::Hooks; +#[cfg(all(feature = "spin", target_arch = "wasm32"))] +use edgezero_core::config_store::ConfigStoreHandle; use edgezero_core::context::RequestContext; use edgezero_core::error::EdgeError; use edgezero_core::http::{HeaderValue, Method, Request, Response, StatusCode, header}; use edgezero_core::router::RouterService; use error_stack::Report; use trusted_server_core::auction::endpoints::handle_auction; -use trusted_server_core::auction::{AuctionOrchestrator, build_orchestrator}; +use trusted_server_core::auction::{ + AuctionOrchestrator, build_orchestrator_with_plan, compile_auction_plan, +}; use trusted_server_core::cache_policy::EdgeCacheHeader; +#[cfg(all(feature = "spin", target_arch = "wasm32"))] +use trusted_server_core::config_payload::settings_from_config_blob; use trusted_server_core::ec::EcContext; use trusted_server_core::ec::admin::{ admin_ec_lookup_not_supported as core_admin_ec_lookup_not_supported, @@ -21,6 +29,8 @@ use trusted_server_core::error::{IntoHttpResponse as _, TrustedServerError}; use trusted_server_core::http_util::sanitize_forwarded_headers; use trusted_server_core::integrations::{IntegrationRegistry, ProxyDispatchInput}; use trusted_server_core::platform::RuntimeServices; +#[cfg(all(feature = "spin", target_arch = "wasm32"))] +use trusted_server_core::platform::{PlatformConfigStore, StoreName}; use trusted_server_core::proxy::{ handle_first_party_click, handle_first_party_proxy, handle_first_party_proxy_rebuild, handle_first_party_proxy_sign, @@ -34,16 +44,24 @@ use trusted_server_core::request_signing::{ handle_trusted_server_discovery, handle_verify_signature, }; use trusted_server_core::settings::Settings; +#[cfg(all(feature = "spin", target_arch = "wasm32"))] +use trusted_server_core::settings_data::{default_config_key, default_secret_store_name}; use crate::middleware::{ AuthMiddleware, FinalizeResponseMiddleware, NormalizeMiddleware, SanitizeRequestMiddleware, }; use crate::platform::build_runtime_services; +#[cfg(all(feature = "spin", target_arch = "wasm32"))] +use crate::platform::{ConfigStoreHandleAdapter, SpinSecretStoreAdapter}; // --------------------------------------------------------------------------- // AppState // --------------------------------------------------------------------------- +/// Spin auto-provides this key-value store label without runtime configuration. +#[cfg(all(feature = "spin", target_arch = "wasm32"))] +const SPIN_DEFAULT_CONFIG_STORE: &str = "default"; + /// Application state built once at startup and shared across all requests. pub struct AppState { settings: Arc, @@ -58,10 +76,44 @@ pub struct AppState { /// Returns an error when settings, the auction orchestrator, or the integration /// registry fail to initialise. fn build_state() -> Result, Report> { - let settings = Settings::from_toml(include_str!("../../../trusted-server.example.toml"))?; + let settings = load_startup_settings()?; build_state_with_settings(settings) } +#[cfg(all(feature = "spin", target_arch = "wasm32"))] +fn load_startup_settings() -> Result> { + let config_store_name = StoreName::from(SPIN_DEFAULT_CONFIG_STORE); + let config_key = default_config_key(); + let config_store = + futures::executor::block_on(SpinConfigStore::open(config_store_name.as_ref().to_owned())) + .map_err(|error| { + Report::new(TrustedServerError::Configuration { + message: "failed to open Spin Trusted Server config store".to_string(), + }) + .attach(error.to_string()) + })?; + let config_handle = ConfigStoreHandle::new(Arc::new(config_store)); + let config_adapter = ConfigStoreHandleAdapter(config_handle); + let raw_envelope = config_adapter + .get(&config_store_name, &config_key) + .map_err(|error| { + Report::new(TrustedServerError::Configuration { + message: "failed to read Spin Trusted Server app-config blob".to_string(), + }) + .attach(error.to_string()) + })?; + let secret_store = SpinSecretStoreAdapter; + settings_from_config_blob(&raw_envelope, &secret_store, &default_secret_store_name()) +} + +#[cfg(not(all(feature = "spin", target_arch = "wasm32")))] +fn load_startup_settings() -> Result> { + Err(Report::new(TrustedServerError::Configuration { + message: "Spin startup settings require the production config store".to_string(), + }) + .attach("use TrustedServerApp::routes_with_settings for host tests")) +} + /// Build the application state from explicit settings. /// /// # Errors @@ -71,8 +123,10 @@ fn build_state() -> Result, Report> { fn build_state_with_settings( settings: Settings, ) -> Result, Report> { - let orchestrator = build_orchestrator(&settings)?; - let registry = IntegrationRegistry::new(&settings)?; + let plan = Arc::new(compile_auction_plan(&settings)?); + plan.validate_for_target(trusted_server_core::platform::AuctionTargetId::Spin)?; + let orchestrator = build_orchestrator_with_plan(Arc::clone(&plan), &settings)?; + let registry = IntegrationRegistry::with_plan(&settings, plan)?; Ok(Arc::new(AppState { settings: Arc::new(settings), @@ -856,6 +910,100 @@ fn build_router(state: &Arc) -> RouterService { mod tests { use super::*; + fn multi_provider_settings() -> Settings { + let mut settings = Settings::from_toml( + r#" + [[handlers]] + path = "^/_ts/admin" + username = "admin" + password = "admin-password" + + [publisher] + domain = "publisher.example" + cookie_domain = ".publisher.example" + origin_url = "https://origin.publisher.example" + proxy_secret = "fictional-proxy-secret" + + [ec] + passphrase = "fictional-secret-key-32-bytes-minimum" + "#, + ) + .expect("should parse startup test settings"); + settings.auction.enabled = true; + settings.auction.providers = + std::iter::IntoIterator::into_iter(["provider-a", "provider-b"]) + .map(|id| { + ( + id.parse().expect("should parse provider ID"), + trusted_server_core::auction::ProviderConfig { + protocol: "openrtb-2.6".to_string(), + profile: "standard".to_string(), + endpoint: format!("https://{id}.example/openrtb"), + timeout_ms: None, + routing: trusted_server_core::auction::RoutingMode::AllEligible, + notifications: + trusted_server_core::auction::NotificationConfig::default(), + profile_config: "{}" + .parse() + .expect("should parse empty profile config object"), + }, + ) + }) + .collect(); + settings + } + + #[test] + fn startup_registers_aps_renderer_route() { + let mut settings = multi_provider_settings(); + settings.auction.providers.clear(); + settings.auction.providers.insert( + "aps-main".parse().expect("should parse APS provider ID"), + trusted_server_core::auction::ProviderConfig { + protocol: "openrtb-2.6".to_string(), + profile: "aps".to_string(), + endpoint: "https://aps.example/e/pb/bid".to_string(), + timeout_ms: None, + routing: trusted_server_core::auction::RoutingMode::AllEligible, + notifications: trusted_server_core::auction::NotificationConfig::default(), + profile_config: "{\"account_id\":\"example-account\"}" + .parse() + .expect("should parse APS profile config"), + }, + ); + + let state = + build_state_with_settings(settings).expect("Spin startup should register APS renderer"); + assert!( + state.registry.has_route( + &edgezero_core::http::Method::GET, + "/integrations/aps/renderer" + ), + "Spin startup registry should expose the APS renderer" + ); + } + + #[test] + fn disabled_startup_accepts_dormant_multi_provider_auction_plan() { + let mut settings = multi_provider_settings(); + settings.auction.enabled = false; + + build_state_with_settings(settings) + .expect("disabled Spin auction should accept dormant fanout"); + } + + #[test] + fn startup_rejects_multi_provider_auction_plan() { + let error = match build_state_with_settings(multi_provider_settings()) { + Ok(_) => panic!("Spin startup should reject multi-provider fanout"), + Err(error) => error, + }; + assert!( + format!("{error:?}").contains("concurrent provider fanout"), + "should identify unsupported fanout: {error:?}" + ); + } + #[test] fn scheme_host_from_spin_url_extracts_localhost_with_port() { assert_eq!( diff --git a/crates/trusted-server-adapter-spin/src/middleware.rs b/crates/trusted-server-adapter-spin/src/middleware.rs index d7a09987a..9ee6d2b50 100644 --- a/crates/trusted-server-adapter-spin/src/middleware.rs +++ b/crates/trusted-server-adapter-spin/src/middleware.rs @@ -106,6 +106,9 @@ impl AuthMiddleware { #[async_trait(?Send)] impl Middleware for AuthMiddleware { async fn handle(&self, mut ctx: RequestContext, next: Next<'_>) -> Result { + // Takes the request mutably because `enforce_basic_auth` marks requests + // whose credential it consumed itself; the shared template cache gate + // reads that marker later. match enforce_basic_auth(&self.settings, ctx.request_mut()) { Ok(Some(response)) => return Ok(response), Ok(None) => {} diff --git a/crates/trusted-server-adapter-spin/src/platform.rs b/crates/trusted-server-adapter-spin/src/platform.rs index 492f1a518..8e7cb4bf3 100644 --- a/crates/trusted-server-adapter-spin/src/platform.rs +++ b/crates/trusted-server-adapter-spin/src/platform.rs @@ -5,20 +5,18 @@ use std::time::Duration; use bytes::Bytes; use edgezero_core::config_store::ConfigStoreHandle; use edgezero_core::key_value_store::{KvHandle, KvPage, KvStore}; -use error_stack::Report; +use error_stack::{Report, ResultExt as _}; #[cfg(all(feature = "spin", target_arch = "wasm32"))] use http_body_util::BodyExt as _; use trusted_server_core::platform::{ - ClientInfo, GeoInfo, KvError, PlatformBackend, PlatformBackendSpec, PlatformConfigStore, - PlatformError, PlatformGeo, PlatformHttpClient, PlatformKvStore, PlatformSecretStore, - RuntimeServices, StoreId, StoreName, UnavailableKvStore, + BackendNamingPolicy, ClientInfo, GeoInfo, KvError, PlatformBackend, PlatformBackendSpec, + PlatformConfigStore, PlatformError, PlatformGeo, PlatformHttpClient, PlatformKvStore, + PlatformSecretStore, RuntimeServices, StoreId, StoreName, UnavailableKvStore, }; #[cfg(not(all(feature = "spin", target_arch = "wasm32")))] use trusted_server_core::platform::UnavailableHttpClient; -#[cfg(all(feature = "spin", target_arch = "wasm32"))] -use error_stack::ResultExt as _; #[cfg(any(test, all(feature = "spin", target_arch = "wasm32")))] use std::io::Read as _; #[cfg(any(test, all(feature = "spin", target_arch = "wasm32")))] @@ -39,6 +37,7 @@ type HeaderPairs = Vec<(String, Vec)>; #[cfg(any(test, all(feature = "spin", target_arch = "wasm32")))] type BufferedResponseParts = (HeaderPairs, Vec); +#[cfg(any(test, all(feature = "spin", target_arch = "wasm32")))] const SPIN_VARIABLE_HEX: &[u8; 16] = b"0123456789abcdef"; // --------------------------------------------------------------------------- @@ -82,27 +81,15 @@ impl PlatformSecretStore for NoopSecretStore { struct NoopBackend; impl PlatformBackend for NoopBackend { + fn naming_policy(&self) -> BackendNamingPolicy { + BackendNamingPolicy::Spin + } + fn predict_name(&self, spec: &PlatformBackendSpec) -> Result> { - let port = spec - .port - .unwrap_or(if spec.scheme == "https" { 443 } else { 80 }); - let timeout_ms = spec.first_byte_timeout.as_millis(); - let cert_suffix = if spec.certificate_check { - "" - } else { - "_nocert" - }; - // Keep two providers that share an origin on distinct names so auction - // response correlation cannot cross providers. - let discriminator = spec - .discriminator - .as_deref() - .map(|d| format!("_p_{d}")) - .unwrap_or_default(); - Ok(format!( - "{}_{}_{}_{timeout_ms}ms{cert_suffix}{discriminator}", - spec.scheme, spec.host, port - )) + self.naming_policy() + .predict(spec) + .map(|prediction| prediction.name) + .change_context(PlatformError::Backend) } fn ensure(&self, spec: &PlatformBackendSpec) -> Result> { @@ -116,25 +103,22 @@ impl PlatformBackend for NoopBackend { /// Bridges edgezero's [`ConfigStoreHandle`] to [`PlatformConfigStore`]. /// -/// Reads delegate through the handle after mapping Trusted Server keys to Spin -/// variable names. Writes are unsupported on current Spin runtime config and -/// return typed errors. -struct ConfigStoreHandleAdapter(ConfigStoreHandle); +/// Spin config stores are KV-backed, so reads preserve the requested key +/// verbatim. Writes are unsupported on current Spin runtime config and return +/// typed errors. +pub(crate) struct ConfigStoreHandleAdapter(pub(crate) ConfigStoreHandle); impl PlatformConfigStore for ConfigStoreHandleAdapter { fn get(&self, _store_name: &StoreName, key: &str) -> Result> { - let variable_name = spin_variable_name(key, PlatformError::ConfigStore)?; - futures::executor::block_on(self.0.get(&variable_name)) - .map_err(|e| { - Report::new(PlatformError::ConfigStore) - .attach(format!( - "config store lookup failed for key `{key}` as Spin variable `{variable_name}`: {e}" - )) - })? - .ok_or_else(|| { + futures::executor::block_on(self.0.get(key)) + .map_err(|error| { Report::new(PlatformError::ConfigStore).attach(format!( - "key `{key}` not found as Spin variable `{variable_name}`" + "config store lookup failed for key `{key}`: {error}" )) + })? + .ok_or_else(|| { + Report::new(PlatformError::ConfigStore) + .attach(format!("key `{key}` not found in Spin config store")) }) } @@ -149,6 +133,7 @@ impl PlatformConfigStore for ConfigStoreHandleAdapter { } } +#[cfg(any(test, all(feature = "spin", target_arch = "wasm32")))] fn spin_variable_name( key: &str, error_context: PlatformError, @@ -187,6 +172,7 @@ fn spin_variable_name( Ok(out) } +#[cfg(any(test, all(feature = "spin", target_arch = "wasm32")))] fn push_spin_variable_escape(out: &mut String, byte: u8) { out.push('_'); out.push('x'); @@ -462,6 +448,13 @@ struct SpinPendingResponse { /// request launches. `select` keeps a defense-in-depth rejection for more /// than one pending request, matching the Cloudflare adapter behavior. /// +/// Spin's WASI HTTP API sends one request and returns the original response; it +/// exposes no redirect-follow policy. Each trait call therefore reaches exactly +/// one `spin_sdk::http::send` boundary and returns an original 3xx to core. Host +/// tests cannot instantiate Spin's WASI transport, so the common +/// `StubHttpClient` driver records the one-send 3xx behavior while adapter tests +/// cover request/response policy around that single boundary. +/// /// # Known MVP limits /// /// **No configurable outbound timeout.** `spin_sdk::http::send` does not @@ -676,7 +669,7 @@ fn into_spin_method(method: &edgezero_core::http::Method) -> spin_sdk::http::Met /// with a real secret-provider source (e.g. Vault, Azure Key Vault) to avoid /// storing signing keys in plaintext on disk. #[cfg(all(feature = "spin", target_arch = "wasm32"))] -struct SpinSecretStoreAdapter; +pub(crate) struct SpinSecretStoreAdapter; #[cfg(all(feature = "spin", target_arch = "wasm32"))] impl PlatformSecretStore for SpinSecretStoreAdapter { @@ -794,12 +787,33 @@ mod tests { use super::*; use edgezero_core::body::Body; + use edgezero_core::config_store::{ConfigStore, ConfigStoreError}; use edgezero_core::context::RequestContext; use edgezero_core::http::request_builder; use edgezero_core::params::PathParams; use flate2::Compression; use flate2::write::GzEncoder; use std::io::Write as _; + use trusted_server_core::platform::AuctionTargetId; + + #[test] + fn auction_http_capabilities_are_explicit() { + let capabilities = AuctionTargetId::Spin.descriptor().capabilities(); + assert!(!capabilities.supports_concurrent_provider_fanout()); + assert!( + !capabilities.has_enforceable_total_request_deadline(), + "Spin outbound HTTP does not expose an enforceable hard total request deadline" + ); + } + + struct InMemoryConfigStore(std::collections::BTreeMap); + + #[async_trait::async_trait(?Send)] + impl ConfigStore for InMemoryConfigStore { + async fn get(&self, key: &str) -> Result, ConfigStoreError> { + Ok(self.0.get(key).cloned()) + } + } fn make_ctx_without_spin_context() -> RequestContext { let req = request_builder() @@ -845,6 +859,29 @@ mod tests { apply_spin_response_policy(&edgezero_core::http::Method::GET, 200, headers, body) } + #[test] + fn response_policy_preserves_original_redirect_at_single_send_boundary() { + let (headers, body) = apply_spin_response_policy( + &edgezero_core::http::Method::GET, + 302, + vec![( + "location".to_string(), + b"https://redirect.example/next".to_vec(), + )], + Vec::new(), + ) + .expect("should preserve redirect response"); + + assert_eq!( + headers, + vec![( + "location".to_string(), + b"https://redirect.example/next".to_vec(), + )] + ); + assert!(body.is_empty()); + } + #[test] fn extract_client_ip_reads_spin_request_context() { let mut req = request_builder() @@ -894,6 +931,29 @@ mod tests { ); } + #[test] + fn config_store_handle_adapter_reads_verbatim_kv_key() { + let handle = ConfigStoreHandle::new(Arc::new(InMemoryConfigStore( + std::collections::BTreeMap::from([( + "trusted_server_config".to_owned(), + "blob-envelope".to_owned(), + )]), + ))); + let adapter = ConfigStoreHandleAdapter(handle); + + let value = adapter + .get( + &StoreName::from("trusted_server_config"), + "trusted_server_config", + ) + .expect("should read the verbatim config-store key"); + + assert_eq!( + value, "blob-envelope", + "should not translate a KV-backed config key into a Spin variable name" + ); + } + #[test] fn spin_variable_name_encodes_trusted_server_keys() { assert_eq!( diff --git a/crates/trusted-server-cli/Cargo.toml b/crates/trusted-server-cli/Cargo.toml index fe9c3664b..8cce8fd5a 100644 --- a/crates/trusted-server-cli/Cargo.toml +++ b/crates/trusted-server-cli/Cargo.toml @@ -17,17 +17,23 @@ workspace = true [target.'cfg(not(target_arch = "wasm32"))'.dependencies] chromiumoxide = { workspace = true } clap = { workspace = true } +derive_more = { workspace = true } edgezero-cli = { workspace = true } +edgezero-core = { workspace = true } futures = { workspace = true } +glob = { workspace = true } +http = { workspace = true } log = { workspace = true } regex = { workspace = true } scraper = { workspace = true } serde = { workspace = true } serde_json = { workspace = true } +similar = { workspace = true } tempfile = { workspace = true } tokio = { workspace = true } toml = { workspace = true } toml_edit = { workspace = true } +tracing = { workspace = true } trusted-server-core = { workspace = true } url = { workspace = true } which = { workspace = true } @@ -41,7 +47,6 @@ which = { workspace = true } [target.'cfg(target_os = "macos")'.dependencies] base64 = { workspace = true } bytes = { workspace = true } -derive_more = { workspace = true } directories = { workspace = true } error-stack = { workspace = true } http-body-util = { workspace = true } @@ -62,4 +67,5 @@ tokio = { workspace = true, features = ["test-util"] } x509-parser = { workspace = true } [target.'cfg(not(target_arch = "wasm32"))'.dev-dependencies] +temp-env = { workspace = true } tempfile = { workspace = true } diff --git a/crates/trusted-server-cli/src/ad_templates/compare.rs b/crates/trusted-server-cli/src/ad_templates/compare.rs new file mode 100644 index 000000000..48ab71f3e --- /dev/null +++ b/crates/trusted-server-cli/src/ad_templates/compare.rs @@ -0,0 +1,830 @@ +//! Pure comparison of configured expected slots against browser ad evidence. +//! +//! This module is collector-independent and Chrome-free: it takes decoded +//! [`BrowserAdEvidence`] plus the [`ExpectedSlot`] set and produces a +//! [`PageVerificationResult`] with per-slot statuses, warnings, and unmatched +//! extra evidence, mirroring spec §5.3–§5.6. +//! +use serde::Deserialize; + +use trusted_server_core::auction::types::MediaType; +use trusted_server_core::creative_opportunities::RuntimeAdStackExpected; + +use crate::ad_templates::expected::ExpectedSlot; +use crate::ad_templates::output::Warning; + +/// The phase in which a piece of evidence was observed. +#[derive(Debug, Clone, Copy, Eq, PartialEq, Deserialize)] +#[serde(rename_all = "snake_case")] +pub enum EvidencePhase { + /// Observed during the initial load and settle. + InitialLoad, + /// Observed only after the deterministic scroll pass. + Scroll, +} + +/// A DOM element ID observed on the page. +#[derive(Debug, Clone, Deserialize)] +pub struct DomEvidence { + /// The element ID. + pub dom_id: String, + /// The phase it was first observed in. + pub phase: EvidencePhase, +} + +/// A GPT slot observed on the page. +#[derive(Debug, Clone, Deserialize)] +pub struct GptSlotEvidence { + /// The observed GAM ad unit path. + pub gam_unit_path: String, + /// The observed GPT slot element ID. + pub div_id: String, + /// Observed numeric sizes as `(width, height)` pairs (non-numeric dropped upstream). + pub sizes: Vec<(u32, u32)>, + /// The phase it was first observed in. + pub phase: EvidencePhase, +} + +/// An `apstag.fetchBids` call the page made, if any were recorded. +/// +/// The collector no longer hooks `apstag`: server-side APS configuration is +/// metadata rather than a client assertion, so a missing client call is not a +/// finding. The field and this shape stay for the evidence payload's schema, and +/// the list arrives empty. +#[derive(Debug, Clone, Deserialize)] +#[allow( + dead_code, + reason = "decoded for schema stability; the collector records no APS calls" +)] +pub struct ApsFetchBidsEvidence { + /// The APS slot ID requested. + pub slot_id: String, + /// Sizes requested for the slot. + pub sizes: Vec<(u32, u32)>, + /// The phase it was observed in. + pub phase: EvidencePhase, +} + +/// A `/__ts/page-bids` observation for SPA routes (spec §5.2). +/// +/// DEFERRED in Phase 1: kept as forward scaffolding so the decoded evidence shape +/// stays forward-compatible. Not populated by the collector or surfaced in JSON. +#[derive(Debug, Clone, Deserialize)] +#[allow( + dead_code, + reason = "reserved decoded shape for the optional bids phase" +)] +pub struct PageBidsEvidence { + /// The slot ID present in the page-bids response. + pub slot_id: String, + /// The phase it was observed in. + pub phase: EvidencePhase, +} + +/// All read-only ad evidence decoded from a single browser page. +#[derive(Debug, Clone, Deserialize)] +pub struct BrowserAdEvidence { + /// DOM element IDs matching configured prefixes. + pub dom_ids: Vec, + /// GPT slots observed via `defineSlot` and `getSlots()`. + pub gpt_slots: Vec, + /// `apstag.fetchBids` calls observed. + pub aps_calls: Vec, + /// `/__ts/page-bids` observations (deferred; default empty). + #[serde(default)] + #[allow(dead_code, reason = "reserved for the optional bids phase")] + pub page_bids: Vec, + /// Collector-level warnings (no page HTML/cookies/storage). + #[serde(default)] + pub warnings: Vec, +} + +/// Summary of the runtime ad-stack gate for a page. +#[derive(Debug, Clone, Copy)] +pub struct RuntimeGateSummary { + /// The three-state ad-stack expectation. + pub expected: RuntimeAdStackExpected, +} + +impl RuntimeGateSummary { + /// Builds a summary from a computed runtime expectation. + #[must_use] + pub fn from_expected(expected: RuntimeAdStackExpected) -> Self { + Self { expected } + } + + #[cfg(test)] + fn unknown_allowed() -> Self { + Self::from_expected(RuntimeAdStackExpected::Unknown) + } + + #[cfg(test)] + fn auction_disabled() -> Self { + Self::from_expected(RuntimeAdStackExpected::No) + } +} + +/// Confirmation status for a single configured slot (compare-side mirror of the +/// output `SlotStatus`). +#[derive(Debug, Clone, Copy, Eq, PartialEq)] +pub enum SlotStatus { + /// GPT evidence matches GAM path, div, and a compatible size. + Confirmed, + /// Some evidence, but not enough to confirm. + Partial, + /// No DOM or GPT evidence confirms the slot. + Missing, + /// The checker cannot confirm this slot type; this is not page drift. + Unconfirmable, +} + +/// The verification result for one audited page. +#[derive(Debug, Clone)] +pub struct PageVerificationResult { + /// Whether the runtime ad stack was expected to run for this page. + pub runtime_ad_stack_expected: RuntimeAdStackExpected, + /// Per-slot results, in expected-slot order. + pub slots: Vec, + /// Live evidence that matched no configured slot. + pub extra_evidence: Vec, +} + +impl PageVerificationResult { + /// Whether `--strict` should fail for this page. + /// + /// False when the runtime ad stack is not expected to run (a known gate + /// suppressed it); otherwise true if any slot is missing or partial. Provider + /// warnings and extra evidence alone never fail strict. + #[must_use] + pub fn strict_failed(&self) -> bool { + if self.runtime_ad_stack_expected == RuntimeAdStackExpected::No { + return false; + } + self.slots + .iter() + .any(|slot| matches!(slot.status, SlotStatus::Missing | SlotStatus::Partial)) + } +} + +/// Per-slot verification result. +#[derive(Debug, Clone)] +pub struct SlotResult { + /// The configured slot id. + pub id: String, + /// The confirmation status. + pub status: SlotStatus, + /// The phase the confirming evidence was observed in. + pub phase: Option, + /// The live evidence observed for this slot. + pub evidence: SlotEvidence, + /// Slot-level warnings (size, provider, etc.). + pub warnings: Vec, +} + +/// Live evidence observed for a configured slot. +#[derive(Debug, Clone)] +pub struct SlotEvidence { + /// The resolved DOM element ID, if any. + pub dom_id: Option, + /// The matched GPT slot, if any. + pub gpt: Option, +} + +/// Live ad-slot evidence with no matching configured slot. +#[derive(Debug, Clone)] +pub struct ExtraEvidence { + /// Evidence kind. Only `gpt` is produced today; the field is a string so a + /// later evidence source can be added without changing the JSON schema. + pub kind: String, + /// The phase it was observed in. + pub phase: EvidencePhase, + /// The DOM element ID, if any. + pub dom_id: Option, + /// The GAM unit path, if any. + pub gam_unit_path: Option, + /// Observed numeric sizes. + pub sizes: Vec<(u32, u32)>, + /// Why this evidence is reported as extra. + pub reason: String, +} + +fn warning(code: &str, message: String) -> Warning { + Warning { + code: code.to_string(), + message, + } +} + +/// Resolves the slot root DOM element per spec §5.3. +/// +/// Exact `div_id` match first, then the first element whose ID starts with +/// `div_id`, ignoring `-container` wrappers. +fn resolve_dom<'a>(dom_ids: &'a [DomEvidence], div_id: &str) -> Option<&'a DomEvidence> { + if let Some(exact) = dom_ids.iter().find(|dom| dom.dom_id == div_id) { + return Some(exact); + } + dom_ids + .iter() + .find(|dom| dom.dom_id.starts_with(div_id) && !dom.dom_id.ends_with("-container")) +} + +/// Returns true when a GPT slot's element ID matches the resolved DOM id (or its +/// `-container`), per spec §5.4. +fn gpt_div_matches(gpt_div: &str, expected: &ExpectedSlot, resolved_dom_id: Option<&str>) -> bool { + match resolved_dom_id { + Some(dom_id) => gpt_div == dom_id || gpt_div == format!("{dom_id}-container"), + None => { + gpt_div == expected.div_id + || (gpt_div.starts_with(&expected.div_id) && !gpt_div.ends_with("-container")) + } + } +} + +fn banner_sizes(expected: &ExpectedSlot) -> Vec<(u32, u32)> { + expected + .formats + .iter() + .filter(|format| format.media_type == MediaType::Banner) + .map(|format| (format.width, format.height)) + .collect() +} + +/// Compares configured expected slots against decoded browser evidence. +#[must_use] +pub fn compare_page_evidence( + expected: &[ExpectedSlot], + evidence: &BrowserAdEvidence, + gate: RuntimeGateSummary, +) -> PageVerificationResult { + let mut consumed_gpt = vec![false; evidence.gpt_slots.len()]; + let mut slots = Vec::with_capacity(expected.len()); + + for slot in expected { + let resolved = resolve_dom(&evidence.dom_ids, &slot.div_id); + let resolved_id = resolved.map(|dom| dom.dom_id.clone()); + // An unrenderable (`None`) configured path can never match live GPT + // evidence; matching on anything else would confirm the wrong unit. + let gpt_idx = slot.gam_unit_path.as_deref().and_then(|unit_path| { + evidence.gpt_slots.iter().position(|gpt| { + gpt.gam_unit_path == unit_path + && gpt_div_matches(&gpt.div_id, slot, resolved_id.as_deref()) + }) + }); + + let banner = banner_sizes(slot); + let mut warnings = Vec::new(); + // `expected_slots_for_path` drops a slot whose template does not render, + // so on the verify path this arm is unreachable; it exists for callers + // that build expected slots directly, and as a guard if that filter ever + // changes. + if slot.gam_unit_path.is_none() { + warnings.push(warning( + "gam_unit_path_unrenderable", + format!( + "slot `{}` gam_unit_path template renders past GAM's unit-path byte limit \ + for this page's section; the runtime omits this slot on this path", + slot.id + ), + )); + } + + let (status, dom_for_evidence, gpt_for_evidence, phase) = if let Some(idx) = gpt_idx { + consumed_gpt[idx] = true; + let gpt = &evidence.gpt_slots[idx]; + let dom_id = resolved_id.clone().or_else(|| Some(gpt.div_id.clone())); + if banner.is_empty() { + warnings.push(warning( + "unsupported_format", + format!( + "slot `{}` has only non-banner formats; not confirmable in Phase 1", + slot.id + ), + )); + ( + SlotStatus::Unconfirmable, + dom_id, + Some(gpt.clone()), + Some(gpt.phase), + ) + } else if gpt.sizes.is_empty() { + warnings.push(warning( + "out_of_page_slot", + format!( + "slot `{}` matched an out-of-page GPT slot with no sizes", + slot.id + ), + )); + ( + SlotStatus::Partial, + dom_id, + Some(gpt.clone()), + Some(gpt.phase), + ) + } else if banner.iter().any(|size| gpt.sizes.contains(size)) { + let extra: Vec<(u32, u32)> = gpt + .sizes + .iter() + .copied() + .filter(|size| !banner.contains(size)) + .collect(); + if !extra.is_empty() { + warnings.push(warning( + "extra_observed_size", + format!("slot `{}` observed extra GPT sizes {extra:?}", slot.id), + )); + } + let missing: Vec<(u32, u32)> = banner + .iter() + .copied() + .filter(|size| !gpt.sizes.contains(size)) + .collect(); + if !missing.is_empty() { + warnings.push(warning( + "configured_size_not_observed", + format!( + "slot `{}` configured sizes {missing:?} were not observed", + slot.id + ), + )); + } + ( + SlotStatus::Confirmed, + dom_id, + Some(gpt.clone()), + Some(gpt.phase), + ) + } else { + warnings.push(warning( + "incompatible_sizes", + format!( + "slot `{}` GPT path and div matched but no configured size overlapped", + slot.id + ), + )); + ( + SlotStatus::Partial, + dom_id, + Some(gpt.clone()), + Some(gpt.phase), + ) + } + } else if let Some(dom) = resolved { + warnings.push(warning( + "dom_without_gpt", + "DOM element matched, but no GPT slot evidence was observed".to_string(), + )); + ( + SlotStatus::Partial, + Some(dom.dom_id.clone()), + None, + Some(dom.phase), + ) + } else { + (SlotStatus::Missing, None, None, None) + }; + + slots.push(SlotResult { + id: slot.id.clone(), + status, + phase, + evidence: SlotEvidence { + dom_id: dom_for_evidence, + gpt: gpt_for_evidence, + }, + warnings, + }); + } + + let extra_evidence = evidence + .gpt_slots + .iter() + .enumerate() + .filter(|(idx, _)| !consumed_gpt[*idx]) + .map(|(_, gpt)| ExtraEvidence { + kind: "gpt".to_string(), + phase: gpt.phase, + dom_id: Some(gpt.div_id.clone()), + gam_unit_path: Some(gpt.gam_unit_path.clone()), + sizes: gpt.sizes.clone(), + reason: "no_configured_slot_matched".to_string(), + }) + .collect(); + + PageVerificationResult { + runtime_ad_stack_expected: gate.expected, + slots, + extra_evidence, + } +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::ad_templates::expected::ExpectedFormat; + + fn dom(id: &str) -> DomEvidence { + DomEvidence { + dom_id: id.to_string(), + phase: EvidencePhase::InitialLoad, + } + } + + fn gpt_slot(gam_unit_path: &str, div_id: &str, sizes: &[(u32, u32)]) -> GptSlotEvidence { + GptSlotEvidence { + gam_unit_path: gam_unit_path.to_string(), + div_id: div_id.to_string(), + sizes: sizes.to_vec(), + phase: EvidencePhase::InitialLoad, + } + } + + fn aps(slot_id: &str, sizes: &[(u32, u32)]) -> ApsFetchBidsEvidence { + ApsFetchBidsEvidence { + slot_id: slot_id.to_string(), + sizes: sizes.to_vec(), + phase: EvidencePhase::InitialLoad, + } + } + + fn evidence( + doms: Vec, + gpts: Vec, + aps: Vec, + ) -> BrowserAdEvidence { + BrowserAdEvidence { + dom_ids: doms, + gpt_slots: gpts, + aps_calls: aps, + page_bids: Vec::new(), + warnings: Vec::new(), + } + } + + fn expected_slot( + id: &str, + div_id: &str, + gam_unit_path: &str, + sizes: &[(u32, u32)], + providers: &[&str], + ) -> ExpectedSlot { + ExpectedSlot { + id: id.to_string(), + div_id: div_id.to_string(), + gam_unit_path: Some(gam_unit_path.to_string()), + formats: sizes + .iter() + .map(|&(width, height)| ExpectedFormat { + width, + height, + media_type: MediaType::Banner, + }) + .collect(), + providers: providers.iter().copied().map(String::from).collect(), + page_patterns: Vec::new(), + } + } + + fn expected_slot_video(id: &str, div_id: &str, gam_unit_path: &str) -> ExpectedSlot { + ExpectedSlot { + id: id.to_string(), + div_id: div_id.to_string(), + gam_unit_path: Some(gam_unit_path.to_string()), + formats: vec![ExpectedFormat { + width: 0, + height: 0, + media_type: MediaType::Video, + }], + providers: Vec::new(), + page_patterns: Vec::new(), + } + } + + #[test] + fn gpt_path_div_and_size_overlap_confirms_slot() { + let expected = expected_slot("atf", "ad-atf-", "/123/news/atf", &[(300, 250)], &[]); + let evidence = evidence( + vec![dom("ad-atf-0")], + vec![gpt_slot("/123/news/atf", "ad-atf-0", &[(300, 250)])], + Vec::new(), + ); + + let result = compare_page_evidence( + &[expected], + &evidence, + RuntimeGateSummary::unknown_allowed(), + ); + + assert_eq!(result.slots[0].status, SlotStatus::Confirmed); + assert!( + result.slots[0].warnings.is_empty(), + "confirmed slot should carry no warnings" + ); + } + + #[test] + fn unrenderable_gam_unit_path_never_confirms() { + let mut expected = expected_slot("atf", "ad-atf-", "/123/news/atf", &[(300, 250)], &[]); + expected.gam_unit_path = None; + let evidence = evidence( + vec![dom("ad-atf-0")], + vec![gpt_slot("/123/news/atf", "ad-atf-0", &[(300, 250)])], + Vec::new(), + ); + + let result = compare_page_evidence( + &[expected], + &evidence, + RuntimeGateSummary::unknown_allowed(), + ); + + assert_eq!( + result.slots[0].status, + SlotStatus::Partial, + "an unrenderable configured path must not confirm against GPT evidence" + ); + assert!( + result.slots[0] + .warnings + .iter() + .any(|w| w.code == "gam_unit_path_unrenderable"), + "should explain why the slot cannot be confirmed" + ); + } + + #[test] + fn dom_only_is_partial() { + let expected = expected_slot("atf", "ad-atf-", "/123/news/atf", &[(300, 250)], &[]); + let evidence = evidence(vec![dom("ad-atf-0")], Vec::new(), Vec::new()); + + let result = compare_page_evidence( + &[expected], + &evidence, + RuntimeGateSummary::unknown_allowed(), + ); + + assert_eq!(result.slots[0].status, SlotStatus::Partial); + assert!( + result.slots[0] + .warnings + .iter() + .any(|w| w.code == "dom_without_gpt") + ); + } + + #[test] + fn no_dom_or_gpt_is_missing() { + let expected = expected_slot("atf", "ad-atf-", "/123/news/atf", &[(300, 250)], &[]); + let evidence = evidence(Vec::new(), Vec::new(), Vec::new()); + + let result = compare_page_evidence( + &[expected], + &evidence, + RuntimeGateSummary::unknown_allowed(), + ); + + assert_eq!(result.slots[0].status, SlotStatus::Missing); + } + + #[test] + fn prefix_dom_resolution_ignores_container_suffix() { + let expected = expected_slot( + "header", + "ad-header-0-", + "/123/homepage/header", + &[(728, 90)], + &[], + ); + let evidence = evidence( + vec![dom("ad-header-0--container"), dom("ad-header-0-_R_abc123")], + Vec::new(), + Vec::new(), + ); + + let result = compare_page_evidence( + &[expected], + &evidence, + RuntimeGateSummary::unknown_allowed(), + ); + + assert_eq!( + result.slots[0].evidence.dom_id.as_deref(), + Some("ad-header-0-_R_abc123"), + "prefix match should skip -container" + ); + assert_eq!(result.slots[0].status, SlotStatus::Partial); + } + + #[test] + fn unmatched_gpt_slot_becomes_extra_evidence() { + let expected = expected_slot("atf", "ad-atf-", "/123/news/atf", &[(300, 250)], &[]); + let evidence = evidence( + vec![dom("ad-atf-0")], + vec![ + gpt_slot("/123/news/atf", "ad-atf-0", &[(300, 250)]), + gpt_slot( + "/123/publisher/right-rail", + "ad-right-rail-0", + &[(300, 250)], + ), + ], + Vec::new(), + ); + + let result = compare_page_evidence( + &[expected], + &evidence, + RuntimeGateSummary::unknown_allowed(), + ); + + assert_eq!(result.slots[0].status, SlotStatus::Confirmed); + assert_eq!(result.extra_evidence.len(), 1); + assert_eq!(result.extra_evidence[0].kind, "gpt"); + assert!( + !result.strict_failed(), + "extra evidence alone must not fail strict" + ); + } + + #[test] + fn auction_disabled_skips_strict_missing_failure() { + let expected = expected_slot("atf", "ad-atf-", "/123/news/atf", &[(300, 250)], &[]); + let evidence = evidence(Vec::new(), Vec::new(), Vec::new()); + + let result = compare_page_evidence( + &[expected], + &evidence, + RuntimeGateSummary::auction_disabled(), + ); + + assert_eq!(result.runtime_ad_stack_expected, RuntimeAdStackExpected::No); + assert_eq!(result.slots[0].status, SlotStatus::Missing); + assert!( + !result.strict_failed(), + "missing slot must not fail strict when ad stack is No" + ); + } + + #[test] + fn gpt_incompatible_sizes_is_partial() { + let expected = expected_slot("atf", "ad-atf-", "/123/news/atf", &[(300, 250)], &[]); + let evidence = evidence( + vec![dom("ad-atf-0")], + vec![gpt_slot("/123/news/atf", "ad-atf-0", &[(728, 90)])], + Vec::new(), + ); + + let result = compare_page_evidence( + &[expected], + &evidence, + RuntimeGateSummary::unknown_allowed(), + ); + + assert_eq!(result.slots[0].status, SlotStatus::Partial); + assert!( + result.slots[0] + .warnings + .iter() + .any(|w| w.code == "incompatible_sizes") + ); + } + + #[test] + fn non_banner_only_slot_is_unconfirmable_and_does_not_fail_strict() { + let expected = expected_slot_video("video", "ad-video-", "/123/news/video"); + let evidence = evidence( + vec![dom("ad-video-0")], + vec![gpt_slot("/123/news/video", "ad-video-0", &[(640, 480)])], + Vec::new(), + ); + + let result = compare_page_evidence( + &[expected], + &evidence, + RuntimeGateSummary::unknown_allowed(), + ); + + assert_eq!(result.slots[0].status, SlotStatus::Unconfirmable); + assert!( + result.slots[0] + .warnings + .iter() + .any(|w| w.code == "unsupported_format") + ); + assert!( + !result.strict_failed(), + "checker limitations should not fail strict" + ); + } + + #[test] + fn gpt_container_element_id_confirms() { + let expected = expected_slot("atf", "ad-atf-0", "/123/news/atf", &[(300, 250)], &[]); + let evidence = evidence( + vec![dom("ad-atf-0"), dom("ad-atf-0-container")], + vec![gpt_slot( + "/123/news/atf", + "ad-atf-0-container", + &[(300, 250)], + )], + Vec::new(), + ); + + let result = compare_page_evidence( + &[expected], + &evidence, + RuntimeGateSummary::unknown_allowed(), + ); + + assert_eq!( + result.slots[0].status, + SlotStatus::Confirmed, + "container element id is a valid GPT div match" + ); + } + + #[test] + fn sizeless_live_slot_is_partial_when_config_declares_banner_sizes() { + let expected = expected_slot( + "interstitial", + "ad-oop-", + "/123/news/oop", + &[(300, 250)], + &[], + ); + let evidence = evidence( + vec![dom("ad-oop-0")], + vec![gpt_slot("/123/news/oop", "ad-oop-0", &[])], + Vec::new(), + ); + + let result = compare_page_evidence( + &[expected], + &evidence, + RuntimeGateSummary::unknown_allowed(), + ); + + assert_eq!(result.slots[0].status, SlotStatus::Partial); + assert!( + result.slots[0] + .warnings + .iter() + .any(|w| w.code == "out_of_page_slot") + ); + assert!( + result.strict_failed(), + "a live sizeless slot drifting from configured banner sizes must fail strict" + ); + } + + #[test] + fn aps_match_adds_no_warning() { + let expected = expected_slot("atf", "ad-atf-", "/123/news/atf", &[(300, 250)], &["aps"]); + let evidence = evidence( + vec![dom("ad-atf-0")], + vec![gpt_slot("/123/news/atf", "ad-atf-0", &[(300, 250)])], + vec![aps("atf", &[(300, 250)])], + ); + + let result = compare_page_evidence( + &[expected], + &evidence, + RuntimeGateSummary::unknown_allowed(), + ); + + assert_eq!(result.slots[0].status, SlotStatus::Confirmed); + assert!( + !result.slots[0] + .warnings + .iter() + .any(|w| w.code.starts_with("aps_")), + "matching APS should not warn" + ); + } + + #[test] + fn server_side_aps_config_does_not_require_client_fetch_bids_evidence() { + let expected = expected_slot("atf", "ad-atf-", "/123/news/atf", &[(300, 250)], &["aps"]); + let evidence = evidence( + vec![dom("ad-atf-0")], + vec![gpt_slot("/123/news/atf", "ad-atf-0", &[(300, 250)])], + Vec::new(), + ); + + let result = compare_page_evidence( + &[expected], + &evidence, + RuntimeGateSummary::unknown_allowed(), + ); + + assert_eq!( + result.slots[0].status, + SlotStatus::Confirmed, + "missing APS does not flip status" + ); + assert!(result.slots[0].warnings.is_empty()); + assert!( + !result.strict_failed(), + "provider warning alone must not fail strict" + ); + } +} diff --git a/crates/trusted-server-cli/src/ad_templates/expected.rs b/crates/trusted-server-cli/src/ad_templates/expected.rs new file mode 100644 index 000000000..9392963ff --- /dev/null +++ b/crates/trusted-server-cli/src/ad_templates/expected.rs @@ -0,0 +1,336 @@ +//! Pure expected-slot projection from the runtime creative-opportunity matcher. +//! +//! This module owns path/URL normalization and converts the slots matched by +//! [`match_slots`] into stable, owned [`ExpectedSlot`] records for output and +//! browser-evidence comparison. It must not duplicate glob-matching semantics. + +use trusted_server_core::auction::types::MediaType; +use trusted_server_core::creative_opportunities::{CreativeOpportunitiesConfig, match_slots}; +use url::Url; + +/// The expected slots for a single page path, in configured slot order. +#[derive(Debug, Clone, PartialEq)] +pub struct ExpectedSlots { + /// The page path the slots were matched against. + pub path: String, + /// Matched slots projected into stable records, in configured order. + pub slots: Vec, +} + +/// A single configured slot expected to appear for a page path. +#[derive(Debug, Clone, PartialEq)] +pub struct ExpectedSlot { + /// The slot identifier. + pub id: String, + /// Resolved HTML `div` element ID (override or the slot id). + pub div_id: String, + /// Resolved GAM unit path: the rendered `gam_unit_path` template (or + /// `//` when the slot has none). + /// + /// `None` only for manually constructed comparison fixtures. Projection + /// omits a slot when the runtime cannot render it for this path. + pub gam_unit_path: Option, + /// Configured ad formats. + pub formats: Vec, + /// Configured provider names, in `aps`, `prebid` order. + pub providers: Vec, + /// Glob patterns configured for this slot. + pub page_patterns: Vec, +} + +/// A configured ad format as a stable width/height/media-type record. +#[derive(Debug, Clone, PartialEq)] +pub struct ExpectedFormat { + /// Creative width in pixels. + pub width: u32, + /// Creative height in pixels. + pub height: u32, + /// Configured media type. + pub media_type: MediaType, +} + +/// Projects the slots matching `path` into stable expected-slot records. +/// +/// Uses [`match_slots`] so glob semantics stay identical to the runtime, and +/// preserves configured slot order. `path` is assumed already normalized via +/// [`normalize_path_or_url`]. +/// +/// `gam_unit_path` templates are rendered against the section the runtime would +/// derive from `path` (per the config's `section_root`/`section_segment` +/// policy), so `{section}`-bearing configs project the same unit path the live +/// page requests. +// Shared projection used by the audit verifier; the static commands match slots +// directly against the runtime matcher. +#[must_use] +pub fn expected_slots_for_path(path: &str, config: &CreativeOpportunitiesConfig) -> ExpectedSlots { + let section = config.section_for_path(path); + let slots = match_slots(&config.slot, path) + .into_iter() + .filter_map(|slot| { + let gam_unit_path = slot.render_gam_unit_path(&config.gam_network_id, §ion)?; + Some(ExpectedSlot { + id: slot.id.clone(), + div_id: slot.resolved_div_id().to_string(), + gam_unit_path: Some(gam_unit_path), + formats: slot + .formats + .iter() + .map(|format| ExpectedFormat { + width: format.width, + height: format.height, + media_type: format.media_type.clone(), + }) + .collect(), + providers: provider_names(slot), + page_patterns: slot.page_patterns.clone(), + }) + }) + .collect(); + + ExpectedSlots { + path: path.to_string(), + slots, + } +} + +fn provider_names( + slot: &trusted_server_core::creative_opportunities::CreativeOpportunitySlot, +) -> Vec { + let mut providers = Vec::new(); + if slot.providers.aps.is_some() { + providers.push("aps".to_string()); + } + if slot.providers.prebid.is_some() { + providers.push("prebid".to_string()); + } + providers +} + +/// Normalizes a page path or full URL into a request path. +/// +/// Full `scheme://` inputs are parsed and reduced to their path; bare inputs have +/// query and fragment stripped and a leading `/` ensured. Empty paths become `/`. +/// +/// # Errors +/// +/// Returns a user-facing string when a `scheme://` input cannot be parsed as a URL. +pub fn normalize_path_or_url(input: &str) -> Result { + let path_input = input.split(['?', '#']).next().unwrap_or(input); + let scheme_prefix = path_input.split_once("://").map(|(scheme, _)| scheme); + let has_url_scheme = scheme_prefix.is_some_and(|scheme| { + let mut chars = scheme.chars(); + chars.next().is_some_and(|ch| ch.is_ascii_alphabetic()) + && chars.all(|ch| ch.is_ascii_alphanumeric() || matches!(ch, '+' | '-' | '.')) + }); + if has_url_scheme { + let url = Url::parse(input).map_err(|err| format!("invalid URL `{input}`: {err}"))?; + let path = url.path(); + return Ok(if path.is_empty() { + "/".to_string() + } else { + path.to_string() + }); + } + + let base = Url::parse("https://path-normalizer.example/") + .expect("should parse static path normalization base"); + let relative = input.trim_start_matches('/'); + let normalized = base + .join(&format!("./{relative}")) + .map_err(|error| format!("invalid path `{input}`: {error}"))?; + Ok(normalized.path().to_string()) +} + +#[cfg(test)] +mod tests { + use super::*; + + fn creative_config_with_slots(patterns: &[&str]) -> CreativeOpportunitiesConfig { + let page_patterns = patterns + .iter() + .map(|pattern| format!("\"{pattern}\"")) + .collect::>() + .join(", "); + let toml = format!( + "gam_network_id = \"123\"\n\ + \n\ + [[slot]]\n\ + id = \"atf\"\n\ + gam_unit_path = \"/123/news/atf\"\n\ + div_id = \"ad-atf-\"\n\ + page_patterns = [{page_patterns}]\n\ + formats = [{{ width = 300, height = 250 }}]\n\ + \n\ + [slot.providers.prebid]\n\ + bidders = {{}}\n" + ); + let mut config = toml::from_str::(&toml) + .expect("should deserialize creative opportunities config"); + config.compile_slots(); + config + } + + #[test] + fn expected_slots_use_runtime_matcher_and_config_order() { + let config = creative_config_with_slots(&["/news/*", "/"]); + let expected = expected_slots_for_path("/news/story", &config); + + assert_eq!(expected.path, "/news/story"); + assert_eq!( + expected + .slots + .iter() + .map(|slot| slot.id.as_str()) + .collect::>(), + ["atf"] + ); + assert_eq!(expected.slots[0].div_id, "ad-atf-"); + assert_eq!( + expected.slots[0].gam_unit_path.as_deref(), + Some("/123/news/atf") + ); + assert_eq!(expected.slots[0].providers, ["prebid"]); + assert_eq!( + expected.slots[0].formats, + vec![ExpectedFormat { + width: 300, + height: 250, + media_type: MediaType::Banner, + }] + ); + } + + #[test] + fn expected_slots_default_resolution_without_overrides() { + let toml = "gam_network_id = \"42\"\n\ + \n\ + [[slot]]\n\ + id = \"footer\"\n\ + page_patterns = [\"/\"]\n\ + formats = [{ width = 728, height = 90 }]\n"; + let mut config = + toml::from_str::(toml).expect("should deserialize"); + config.compile_slots(); + + let expected = expected_slots_for_path("/", &config); + assert_eq!(expected.slots[0].div_id, "footer"); + assert_eq!( + expected.slots[0].gam_unit_path.as_deref(), + Some("/42/footer") + ); + assert!(expected.slots[0].providers.is_empty()); + } + + #[test] + fn expected_slots_render_section_templates_per_path() { + let toml = "gam_network_id = \"99999\"\n\ + section_root = \"homepage\"\n\ + \n\ + [[slot]]\n\ + id = \"ad-header-0\"\n\ + gam_unit_path = \"/{network_id}/example/{section}\"\n\ + page_patterns = [\"/\", \"/news\", \"/news/*\"]\n\ + formats = [{ width = 728, height = 90 }]\n"; + let mut config = + toml::from_str::(toml).expect("should deserialize"); + config.compile_slots(); + + // A path with a section segment renders that segment. + assert_eq!( + expected_slots_for_path("/news/story", &config).slots[0] + .gam_unit_path + .as_deref(), + Some("/99999/example/news"), + "a section template should render the path's section" + ); + // The site root falls back to the configured section_root. + assert_eq!( + expected_slots_for_path("/", &config).slots[0] + .gam_unit_path + .as_deref(), + Some("/99999/example/homepage"), + "the root path should render section_root" + ); + } + + #[test] + fn expected_slots_omit_dynamic_template_the_runtime_cannot_render() { + // A `{section}` template that renders past GAM's 100-byte unit-path + // limit. The runtime omits this slot for the request path, so diagnostics + // must not match it against a truncated or otherwise different path. + let toml = "gam_network_id = \"99999\"\n\ + section_root = \"homepage\"\n\ + \n\ + [[slot]]\n\ + id = \"ad-header-0\"\n\ + gam_unit_path = \"/{section}/{section}\"\n\ + page_patterns = [\"/*\"]\n\ + formats = [{ width = 728, height = 90 }]\n"; + let mut config = + toml::from_str::(toml).expect("should deserialize"); + config.compile_slots(); + + let long_path = format!("/{}", "a".repeat(60)); + let expected = expected_slots_for_path(&long_path, &config); + + assert!( + expected.slots.is_empty(), + "the runtime omits an over-limit dynamic slot on this path" + ); + } + + #[test] + fn normalize_path_or_url_strips_query_and_fragment() { + assert_eq!( + normalize_path_or_url("https://www.example.com/news/story?x=1#top") + .expect("should normalize"), + "/news/story" + ); + assert_eq!( + normalize_path_or_url("news/story?x=1").expect("should normalize"), + "/news/story" + ); + } + + #[test] + fn normalize_path_or_url_roots_empty_input() { + assert_eq!( + normalize_path_or_url("https://www.example.com").expect("should normalize"), + "/" + ); + assert_eq!(normalize_path_or_url("").expect("should normalize"), "/"); + } + + #[test] + fn normalize_path_or_url_uses_identical_url_rules_for_bare_paths() { + assert_eq!( + normalize_path_or_url("/a/../b").expect("should normalize bare dot segment"), + "/b" + ); + assert_eq!( + normalize_path_or_url("https://example.com/a/../b") + .expect("should normalize URL dot segment"), + "/b" + ); + assert_eq!( + normalize_path_or_url("/a b").expect("should encode bare path"), + "/a%20b" + ); + assert_eq!( + normalize_path_or_url("/r?to=https://example.com") + .expect("query URL should not change input classification"), + "/r" + ); + assert_eq!( + normalize_path_or_url("/news:latest").expect("colon should stay in bare path"), + "/news:latest", + "a colon in the first segment must not be parsed as a URL scheme" + ); + assert_eq!( + normalize_path_or_url("https://example.com/news:latest") + .expect("colon should stay in URL path"), + "/news:latest", + "bare and absolute forms should normalize identically" + ); + } +} diff --git a/crates/trusted-server-cli/src/ad_templates/mod.rs b/crates/trusted-server-cli/src/ad_templates/mod.rs new file mode 100644 index 000000000..3c26bf121 --- /dev/null +++ b/crates/trusted-server-cli/src/ad_templates/mod.rs @@ -0,0 +1,7 @@ +//! Pure, host-only ad-template CLI logic shared by the static `ts config +//! ad-templates ...` commands and the browser-backed `ts audit ad-templates +//! verify` command. + +pub mod compare; +pub mod expected; +pub mod output; diff --git a/crates/trusted-server-cli/src/ad_templates/output.rs b/crates/trusted-server-cli/src/ad_templates/output.rs new file mode 100644 index 000000000..afcf78ed0 --- /dev/null +++ b/crates/trusted-server-cli/src/ad_templates/output.rs @@ -0,0 +1,483 @@ +//! Stable, serializable output model for ad-template diagnostics. +//! +//! These types mirror the `--json` contract in +//! `docs/superpowers/specs/2026-06-26-server-side-ad-template-cli-design.md` §8. +//! Field names and declaration order are load-bearing: `serde` serializes struct +//! fields in declaration order, so the order here must match the spec examples. +//! +//! The model is consumed by the `ts audit ad-templates verify` orchestrator, +//! which assembles these wire types from the URL/gate context and comparison result. + +use std::borrow::Cow; + +use serde::{Deserialize, Serialize}; + +use trusted_server_core::creative_opportunities::RuntimeAdStackExpected; + +/// Escapes control characters in page-controlled text bound for a terminal. +/// +/// Page titles and collector warning messages are attacker-controlled: an +/// audited page can put ANSI/OSC escape sequences in `document.title` and drive +/// the operator's terminal (cursor movement, clipboard writes, forged output) +/// when the value is printed verbatim. Every C0 control (including ESC), DEL, +/// and the C1 range are rendered as `\u{XXXX}` so the text stays inert. JSON +/// output is unaffected — `serde_json` escapes these already. +/// +/// Returns a borrowed `Cow` when the input needs no escaping. +#[must_use] +pub fn escape_terminal_text(value: &str) -> Cow<'_, str> { + if !value.chars().any(is_terminal_control) { + return Cow::Borrowed(value); + } + let mut escaped = String::with_capacity(value.len()); + for ch in value.chars() { + if is_terminal_control(ch) { + escaped.push_str(&format!("\\u{{{:04X}}}", ch as u32)); + } else { + escaped.push(ch); + } + } + Cow::Owned(escaped) +} + +/// Whether `ch` can act as a terminal control code (C0, DEL, or C1). +fn is_terminal_control(ch: char) -> bool { + let code = ch as u32; + code < 0x20 + || (0x7f..=0x9f).contains(&code) + || (0x202a..=0x202e).contains(&code) + || (0x2066..=0x2069).contains(&code) +} + +/// Confirmation status for a single configured slot. +#[derive(Debug, Clone, Copy, Eq, PartialEq, Serialize)] +#[serde(rename_all = "snake_case")] +pub enum SlotStatus { + /// GPT evidence matches GAM path, div, and a compatible size. + Confirmed, + /// Some evidence, but not enough to confirm. + Partial, + /// No DOM or GPT evidence confirms the slot. + Missing, + /// The checker does not support confirming this slot type. + Unconfirmable, +} + +/// JSON rendering of the runtime ad-stack expectation. +#[derive(Debug, Clone, Copy, Eq, PartialEq, Serialize)] +#[serde(rename_all = "snake_case")] +pub enum RuntimeAdStackExpectedJson { + /// The server-side ad stack is expected to run. + Yes, + /// A known gate blocks the server-side ad stack. + No, + /// Consent or another gate is unprovable. + Unknown, +} + +impl From for RuntimeAdStackExpectedJson { + fn from(value: RuntimeAdStackExpected) -> Self { + match value { + RuntimeAdStackExpected::Yes => Self::Yes, + RuntimeAdStackExpected::No => Self::No, + RuntimeAdStackExpected::Unknown => Self::Unknown, + } + } +} + +/// State of a single runtime gate. +#[derive(Debug, Clone, Copy, Eq, PartialEq, Serialize)] +#[serde(rename_all = "snake_case")] +pub enum GateState { + /// The gate passed. + Pass, + /// The gate blocked the ad stack. + Fail, + /// The gate state could not be proven. + Unknown, +} + +/// Evidence-collection phase, rendered for JSON output. +#[derive(Debug, Clone, Copy, Eq, PartialEq, Serialize)] +#[serde(rename_all = "snake_case")] +pub enum EvidencePhaseJson { + /// Observed during the initial page load and settle. + InitialLoad, + /// Observed only after the deterministic scroll pass. + Scroll, +} + +/// A structured warning with a stable machine code and human message. +/// +/// `Serialize` for output; `Deserialize` because the browser collector payload +/// carries warning objects decoded into the comparison input. +#[derive(Debug, Clone, Eq, PartialEq, Serialize, Deserialize)] +pub struct Warning { + /// Stable machine-readable code (e.g. `dom_without_gpt`). + pub code: String, + /// Human-readable message; JSON consumers must not parse this. + pub message: String, +} + +/// Top-level `--json` document for `ts audit ad-templates verify`. +#[derive(Debug, Clone, PartialEq, Serialize)] +pub struct VerificationReport { + /// True when no strict failure and no page-level error occurred. + pub ok: bool, + /// Whether `--strict` was set. + pub strict: bool, + /// One entry per requested URL, in input order. + pub pages: Vec, + /// Run-level warnings not attributable to a single page. + /// + /// Always empty today — every warning the verifier raises belongs to a page + /// or a slot. Kept because the JSON schema declares it, so a consumer can + /// read it unconditionally. + pub warnings: Vec, +} + +/// A single audited page result. +/// +/// `error` is declared immediately after `path` so the serialized key order +/// matches the spec §8 `navigation_failed` shape; on normal pages it is `None` +/// and skipped, leaving the runtime/gates fields in §8 order. +#[derive(Debug, Clone, PartialEq, Serialize)] +pub struct PageJson { + /// The requested URL. + pub url: String, + /// The final URL after redirects, or `null` on navigation failure. + pub final_url: Option, + /// The requested URL's path. + pub requested_path: String, + /// The final path used for matching, or `null` on navigation failure. + pub path: Option, + /// Present only on a page-level collection failure. + #[serde(skip_serializing_if = "Option::is_none")] + pub error: Option, + /// Three-state runtime ad-stack expectation; absent on error pages. + #[serde(skip_serializing_if = "Option::is_none")] + pub runtime_ad_stack_expected: Option, + /// Per-gate evidence; absent on error pages. + #[serde(skip_serializing_if = "Option::is_none")] + pub gates: Option, + /// Number of configured slots matched for the final path; absent on error pages. + #[serde(skip_serializing_if = "Option::is_none")] + pub matched_slot_count: Option, + /// Per-slot verification results. + pub slots: Vec, + /// Live ad-slot evidence with no matching configured slot. + pub extra_evidence: Vec, + /// Page-level warnings. + pub warnings: Vec, +} + +/// Runtime gate states for a page, one field per spec §5.2 gate. +#[derive(Debug, Clone, PartialEq, Serialize)] +pub struct Gates { + /// Request method is `GET`. + pub method_get: GateState, + /// Request is a top-level navigation. + pub navigation: GateState, + /// Request is not a prefetch. + pub not_prefetch: GateState, + /// Request is not from a known bot. + pub not_bot: GateState, + /// At least one configured slot matched the final path. + pub matched_slots: GateState, + /// The `[auction].enabled` kill switch is on. + pub auction_enabled: GateState, + /// The `[creative_opportunities].enabled` template switch is on. + pub ad_templates_enabled: GateState, + /// Consent allows the auction (often `unknown` for live requests). + pub consent_allows_auction: GateState, +} + +/// A single configured slot's verification result. +#[derive(Debug, Clone, PartialEq, Serialize)] +pub struct SlotJson { + /// The configured slot id. + pub id: String, + /// The slot's confirmation status. + pub status: SlotStatus, + /// The phase the confirming evidence was observed in. + #[serde(skip_serializing_if = "Option::is_none")] + pub phase: Option, + /// The configured shape of the slot (no `id`/`page_patterns` per §8). + pub configured: ConfiguredJson, + /// The live evidence observed for this slot. + pub evidence: SlotEvidenceJson, + /// Slot-level warnings (e.g. provider or size warnings). + pub warnings: Vec, +} + +/// The configured shape of a slot, as rendered in §8 `configured`. +#[derive(Debug, Clone, PartialEq, Serialize)] +pub struct ConfiguredJson { + /// Resolved div element ID. + pub div_id: String, + /// Resolved GAM unit path, or `null` when a dynamic template renders past + /// GAM's unit-path byte limit for this page's section. + pub gam_unit_path: Option, + /// Configured formats. + pub formats: Vec, + /// Configured provider names. + pub providers: Vec, +} + +/// A configured format, as rendered in §8. +#[derive(Debug, Clone, PartialEq, Serialize)] +pub struct FormatJson { + /// Creative width in pixels. + pub width: u32, + /// Creative height in pixels. + pub height: u32, + /// Media type string (`banner`, `video`, `native`). + pub media_type: String, +} + +/// Live evidence observed for a configured slot. +#[derive(Debug, Clone, PartialEq, Serialize)] +pub struct SlotEvidenceJson { + /// The resolved DOM element ID observed, if any. + pub dom_id: Option, + /// GPT slot evidence, if any (no `phase` key per §8). + pub gpt: Option, +} + +/// GPT slot evidence, as rendered in §8 `evidence.gpt`. +#[derive(Debug, Clone, PartialEq, Serialize)] +pub struct GptEvidenceJson { + /// The observed GAM ad unit path. + pub gam_unit_path: String, + /// The observed GPT slot element ID. + pub div_id: String, + /// Observed numeric sizes as `[width, height]` pairs. + pub sizes: Vec<[u32; 2]>, +} + +/// Live ad-slot evidence with no matching configured slot. +#[derive(Debug, Clone, PartialEq, Serialize)] +pub struct ExtraEvidenceJson { + /// Evidence kind: `dom`, `gpt`, or `aps`. + pub kind: String, + /// The phase the evidence was observed in. + pub phase: EvidencePhaseJson, + /// The DOM element ID, if any. + pub dom_id: Option, + /// The GAM unit path, if any. + pub gam_unit_path: Option, + /// Observed numeric sizes as `[width, height]` pairs. + pub sizes: Vec<[u32; 2]>, + /// Why this evidence is reported as extra. + pub reason: String, +} + +#[cfg(test)] +impl VerificationReport { + fn example_confirmed_with_extra_evidence() -> Self { + VerificationReport { + ok: true, + strict: false, + pages: vec![PageJson { + url: "https://www.example.com/news/story".to_string(), + final_url: Some("https://www.example.com/news/story".to_string()), + requested_path: "/news/story".to_string(), + path: Some("/news/story".to_string()), + error: None, + runtime_ad_stack_expected: Some(RuntimeAdStackExpectedJson::Unknown), + gates: Some(Gates { + method_get: GateState::Pass, + navigation: GateState::Pass, + not_prefetch: GateState::Pass, + not_bot: GateState::Pass, + matched_slots: GateState::Pass, + auction_enabled: GateState::Pass, + ad_templates_enabled: GateState::Pass, + consent_allows_auction: GateState::Unknown, + }), + matched_slot_count: Some(1), + slots: vec![SlotJson { + id: "atf".to_string(), + status: SlotStatus::Confirmed, + phase: Some(EvidencePhaseJson::InitialLoad), + configured: ConfiguredJson { + div_id: "ad-atf-".to_string(), + gam_unit_path: Some("/123/news/atf".to_string()), + formats: vec![FormatJson { + width: 300, + height: 250, + media_type: "banner".to_string(), + }], + providers: vec!["aps".to_string()], + }, + evidence: SlotEvidenceJson { + dom_id: Some("ad-atf-0".to_string()), + gpt: Some(GptEvidenceJson { + gam_unit_path: "/123/news/atf".to_string(), + div_id: "ad-atf-0".to_string(), + sizes: vec![[300, 250]], + }), + }, + warnings: Vec::new(), + }], + extra_evidence: vec![ExtraEvidenceJson { + kind: "gpt".to_string(), + phase: EvidencePhaseJson::InitialLoad, + dom_id: Some("ad-right-rail-0".to_string()), + gam_unit_path: Some("/123/publisher/right-rail".to_string()), + sizes: vec![[300, 250]], + reason: "no_configured_slot_matched".to_string(), + }], + warnings: vec![Warning { + code: "redirected".to_string(), + message: "navigation redirected to the final path".to_string(), + }], + }], + warnings: Vec::new(), + } + } + + fn example_navigation_failed() -> Self { + VerificationReport { + ok: false, + strict: false, + pages: vec![PageJson { + url: "https://www.example.com/broken".to_string(), + final_url: None, + requested_path: "/broken".to_string(), + path: None, + error: Some(Warning { + code: "navigation_failed".to_string(), + message: "failed to read main document navigation response".to_string(), + }), + runtime_ad_stack_expected: None, + gates: None, + matched_slot_count: None, + slots: Vec::new(), + extra_evidence: Vec::new(), + warnings: Vec::new(), + }], + warnings: Vec::new(), + } + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn escape_terminal_text_passes_through_ordinary_titles() { + assert!( + matches!( + escape_terminal_text("Example News — Story"), + Cow::Borrowed(_) + ), + "text with no control characters should not allocate" + ); + assert_eq!( + escape_terminal_text("Example News — Story"), + "Example News — Story" + ); + } + + #[test] + fn escape_terminal_text_neutralizes_control_sequences() { + // ESC-based CSI/OSC sequences and a raw newline are the terminal-driving + // primitives a hostile page would put in `document.title`. + assert_eq!( + escape_terminal_text("a\u{1b}]0;pwned\u{7}b"), + "a\\u{001B}]0;pwned\\u{0007}b", + "ESC and BEL should be rendered inert" + ); + assert_eq!( + escape_terminal_text("line\nforged: ok"), + "line\\u{000A}forged: ok", + "a newline should not let a title forge an output line" + ); + assert_eq!( + escape_terminal_text("del\u{7f}c1\u{9b}"), + "del\\u{007F}c1\\u{009B}", + "DEL and the C1 range should be escaped too" + ); + assert_eq!( + escape_terminal_text("safe\u{202E}forged\u{2066}tail"), + "safe\\u{202E}forged\\u{2066}tail", + "Unicode bidi controls should be rendered inert" + ); + } + + #[test] + fn verification_json_contains_gate_state_and_extra_evidence() { + let result = VerificationReport::example_confirmed_with_extra_evidence(); + let value = serde_json::to_value(&result).expect("should serialize"); + + assert_eq!(value["ok"], true); + assert_eq!(value["pages"][0]["requested_path"], "/news/story"); + assert_eq!(value["pages"][0]["runtime_ad_stack_expected"], "unknown"); + assert_eq!( + value["pages"][0]["gates"]["consent_allows_auction"], + "unknown" + ); + assert_eq!(value["pages"][0]["slots"][0]["status"], "confirmed"); + assert_eq!( + value["pages"][0]["slots"][0]["evidence"]["gpt"]["sizes"][0][0], + 300 + ); + assert_eq!(value["pages"][0]["extra_evidence"][0]["kind"], "gpt"); + assert_eq!(value["pages"][0]["warnings"][0]["code"], "redirected"); + // `configured` excludes id/page_patterns per §8. + assert!(value["pages"][0]["slots"][0]["configured"]["id"].is_null()); + assert!(value["pages"][0]["slots"][0]["configured"]["page_patterns"].is_null()); + // `evidence.gpt` has no `phase` key per §8. + assert!(value["pages"][0]["slots"][0]["evidence"]["gpt"]["phase"].is_null()); + } + + #[test] + fn page_error_json_matches_navigation_failed_shape() { + let result = VerificationReport::example_navigation_failed(); + let value = serde_json::to_value(&result).expect("should serialize"); + let page = &value["pages"][0]; + + assert_eq!(page["error"]["code"], "navigation_failed"); + assert!(page["final_url"].is_null(), "final_url should be null"); + assert!(page["path"].is_null(), "path should be null"); + assert!( + page.get("runtime_ad_stack_expected").is_none(), + "runtime field absent on error page" + ); + assert!(page.get("gates").is_none(), "gates absent on error page"); + assert!( + page.get("matched_slot_count").is_none(), + "matched_slot_count absent on error page" + ); + assert_eq!(value["ok"], false); + } + + #[test] + fn missing_slot_json_omits_evidence_phase() { + let slot = SlotJson { + id: "missing".to_string(), + status: SlotStatus::Missing, + phase: None, + configured: ConfiguredJson { + div_id: "ad-missing-".to_string(), + gam_unit_path: Some("/123/publisher/missing".to_string()), + formats: Vec::new(), + providers: Vec::new(), + }, + evidence: SlotEvidenceJson { + dom_id: None, + gpt: None, + }, + warnings: Vec::new(), + }; + + let value = serde_json::to_value(slot).expect("should serialize missing slot"); + + assert!( + value.get("phase").is_none(), + "missing evidence should not claim an initial-load phase" + ); + } +} diff --git a/crates/trusted-server-cli/src/app_config.rs b/crates/trusted-server-cli/src/app_config.rs new file mode 100644 index 000000000..bee536146 --- /dev/null +++ b/crates/trusted-server-cli/src/app_config.rs @@ -0,0 +1,171 @@ +//! Shared effective Trusted Server app-config loading for the `ts` CLI. +//! +//! Both the static `ts config ad-templates ...` commands and the browser-backed +//! `ts audit ad-templates verify` command load the same effective app config +//! through [`load_settings`], so config-path resolution and the `EdgeZero` +//! environment overlay stay consistent across command families. + +use std::path::{Path, PathBuf}; + +use clap::Args; +use edgezero_core::app_config::{self, AppConfigLoadOptions}; +use edgezero_core::manifest::ManifestLoader; +use trusted_server_core::config::TrustedServerAppConfig; +use trusted_server_core::settings::Settings; + +/// Shared local app-config flags accepted by every config/audit ad-template command. +#[derive(Clone, Debug, Args)] +pub struct AppConfigArgs { + /// Path to `trusted-server.toml`. Defaults to `.toml` beside `edgezero.toml`. + #[arg(long)] + pub app_config: Option, + /// Path to `edgezero.toml`. + #[arg(long, default_value = "edgezero.toml")] + pub manifest: PathBuf, + /// Skip app-config environment overlay. + #[arg(long)] + pub no_env: bool, +} + +/// Effective settings plus the resolved app-config path they were loaded from. +#[derive(Debug)] +pub struct LoadedSettings { + /// The `trusted-server.toml` path the settings were loaded from. + pub app_config_path: PathBuf, + /// The deserialized effective settings. + pub settings: Settings, +} + +/// Loads the effective Trusted Server settings described by `args`. +/// +/// Resolves the app-config path from `args` (or the manifest's `.toml` +/// default), applies the `EdgeZero` environment overlay unless `no_env` is set, and +/// returns the deserialized [`Settings`]. +/// +/// # Errors +/// +/// Returns a user-facing string when the manifest cannot be loaded, has no +/// `[app].name`, or the resolved app-config file cannot be read or parsed. When an +/// explicit `--app-config` path is given and is missing, the error names that +/// exact path rather than silently falling back. +pub fn load_settings(args: &AppConfigArgs) -> Result { + load_settings_with_env_overlay(args, !args.no_env) +} + +/// Loads Trusted Server settings from the resolved app-config file without +/// applying environment overlays. +/// +/// Mutating commands use this path so environment-only values are never +/// persisted into the operator-owned TOML file. +/// +/// # Errors +/// +/// Returns the same path-resolution, read, and parse errors as +/// [`load_settings`]. +#[cfg(test)] +pub(crate) fn load_file_settings(args: &AppConfigArgs) -> Result { + load_settings_with_env_overlay(args, false) +} + +/// Resolves the operator-owned app-config path without deserializing settings. +/// +/// Mutating recovery commands use this when the existing config may already be +/// invalid but still needs a narrowly scoped structural repair. +/// +/// # Errors +/// +/// Returns a user-facing string when the manifest cannot be loaded or has no +/// `[app].name` and no explicit config path was supplied. +pub fn resolve_app_config_file(args: &AppConfigArgs) -> Result { + if let Some(path) = &args.app_config { + return Ok(path.clone()); + } + let manifest_loader = ManifestLoader::from_path(&args.manifest) + .map_err(|err| format!("failed to load {}: {err}", args.manifest.display()))?; + let app_name = manifest_loader.manifest().app.name.clone().ok_or_else(|| { + format!( + "{} has no [app].name; cannot resolve trusted-server.toml", + args.manifest.display() + ) + })?; + Ok(resolve_app_config_path(None, &args.manifest, &app_name)) +} + +fn load_settings_with_env_overlay( + args: &AppConfigArgs, + env_overlay: bool, +) -> Result { + let manifest_loader = ManifestLoader::from_path(&args.manifest) + .map_err(|err| format!("failed to load {}: {err}", args.manifest.display()))?; + let app_name = manifest_loader.manifest().app.name.clone().ok_or_else(|| { + format!( + "{} has no [app].name; cannot resolve trusted-server.toml", + args.manifest.display() + ) + })?; + let app_config_path = + resolve_app_config_path(args.app_config.as_deref(), &args.manifest, &app_name); + + let mut opts = AppConfigLoadOptions::default(); + opts.env_overlay = env_overlay; + let app_config = app_config::deserialize_app_config_with_options::( + &app_config_path, + &app_name, + &opts, + ) + .map_err(|err| format!("failed to load {}: {err}", app_config_path.display()))?; + + Ok(LoadedSettings { + app_config_path, + settings: app_config.into_settings(), + }) +} + +fn resolve_app_config_path( + explicit: Option<&Path>, + manifest_path: &Path, + app_name: &str, +) -> PathBuf { + if let Some(path) = explicit { + return path.to_path_buf(); + } + let file_name = format!("{app_name}.toml"); + if let Some(parent) = manifest_path + .parent() + .filter(|parent| !parent.as_os_str().is_empty()) + { + parent.join(file_name) + } else { + PathBuf::from(file_name) + } +} + +#[cfg(test)] +mod tests { + use std::fs; + + use tempfile::TempDir; + + use super::*; + + #[test] + fn explicit_missing_app_config_does_not_fall_back() { + let temp = TempDir::new().expect("should create temp dir"); + let manifest_path = temp.path().join("edgezero.toml"); + fs::write(&manifest_path, "[app]\nname = \"trusted-server\"\n") + .expect("should write manifest"); + let missing_path = temp.path().join("missing.toml"); + + let args = AppConfigArgs { + app_config: Some(missing_path.clone()), + manifest: manifest_path, + no_env: true, + }; + + let err = load_settings(&args).expect_err("should reject missing explicit config"); + assert!( + err.contains(missing_path.to_string_lossy().as_ref()), + "error should mention the explicit missing path" + ); + } +} diff --git a/crates/trusted-server-cli/src/commands/audit/ad_template_collector.js b/crates/trusted-server-cli/src/commands/audit/ad_template_collector.js new file mode 100644 index 000000000..6938808f5 --- /dev/null +++ b/crates/trusted-server-cli/src/commands/audit/ad_template_collector.js @@ -0,0 +1,241 @@ +// Bounded ad-template evidence collector, injected before publisher scripts run. +// +// This body runs inside an IIFE that defines `__TS_CONFIG` (the configured div +// prefixes). It records evidence into `window.__tsAdTemplateEvidence` +// and never captures page HTML, cookies, storage, request bodies, or arbitrary DOM. +// It always calls original page functions with unchanged arguments and never +// spoofs the browser automation flag. + +const __ts_config = typeof __TS_CONFIG === "object" && __TS_CONFIG ? __TS_CONFIG : {} +const __ts_prefixes = Array.isArray(__ts_config.div_prefixes) ? __ts_config.div_prefixes : [] + +const __ts_ev = (window.__tsAdTemplateEvidence = window.__tsAdTemplateEvidence || { + dom_ids: [], + gpt_slots: [], + aps_calls: [], + warnings: [] +}) + +const __ts_phase = () => (window.__tsScrollPhase ? "scroll" : "initial_load") + +// Hard cap per evidence list so a hostile page cannot grow the store without +// bound; the page controls how many slots/elements/warnings it produces. +const __ts_max_entries = 128 +const __ts_max_string_length = 512 +const __ts_wrapped_googletags = new WeakSet() + +function __ts_text(value) { + return String(value).slice(0, __ts_max_string_length) +} + +// Truncation has to be visible: surplus configured slots classify Missing, and +// `--strict` counts that, so a silent drop is indistinguishable from real drift. +let __ts_truncated = false +function __ts_push(list, entry) { + if (list.length < __ts_max_entries) { + list.push(entry) + return + } + if (__ts_truncated) return + __ts_truncated = true + if (__ts_ev.warnings.length < __ts_max_entries) { + __ts_ev.warnings.push({ + code: "evidence_truncated", + message: "an evidence list hit the " + __ts_max_entries + "-entry cap; results are incomplete" + }) + } +} + +function __ts_warn(code, error) { + __ts_push(__ts_ev.warnings, { code, message: __ts_text(error) }) +} + +// GPT sizes reach Rust as u32 pairs, so anything non-integral (fluid slots, +// NaN, negative or fractional dimensions) must be dropped here — a single bad +// pair would fail deserialization of the whole evidence payload and discard +// every other slot's otherwise valid evidence. +function __ts_size_pair(width, height) { + if (!Number.isInteger(width) || !Number.isInteger(height)) return null + if (width < 0 || height < 0 || width > 4294967295 || height > 4294967295) return null + return [width, height] +} + +function __ts_warn_ignored_size(width, height) { + const numeric = Number.isInteger(width) && Number.isInteger(height) + const outOfRange = + numeric && (width < 0 || height < 0 || width > 4294967295 || height > 4294967295) + __ts_push(__ts_ev.warnings, { + code: outOfRange ? "size_out_of_range" : "fluid_size_ignored", + message: outOfRange ? "GPT size outside u32 range ignored" : "non-integer GPT size ignored" + }) +} + +function __ts_normalize_sizes(sizes) { + const out = [] + if (!Array.isArray(sizes)) return out + // Accept [w, h] or [[w, h], ...]; treat numeric-leading arrays as a single pair. + const pairs = typeof sizes[0] === "number" ? [sizes] : sizes + for (const size of pairs) { + if (out.length >= __ts_max_entries) break + const pair = Array.isArray(size) ? __ts_size_pair(size[0], size[1]) : null + if (pair) { + out.push(pair) + } else { + __ts_warn_ignored_size( + Array.isArray(size) ? size[0] : undefined, + Array.isArray(size) ? size[1] : undefined + ) + } + } + return out +} + +function __ts_record_define_slot(adUnitPath, sizes, divId) { + __ts_push(__ts_ev.gpt_slots, { + gam_unit_path: __ts_text(adUnitPath), + div_id: __ts_text(divId), + sizes: __ts_normalize_sizes(sizes), + phase: __ts_phase() + }) +} + +function __ts_wrap_googletag(googletag) { + if (!googletag || (typeof googletag !== "object" && typeof googletag !== "function")) { + return googletag + } + if (__ts_wrapped_googletags.has(googletag)) return googletag + __ts_wrapped_googletags.add(googletag) + // Wrap defineSlot so both direct calls and calls dispatched from the cmd queue + // are recorded (queued callbacks call this same wrapped function). + const originalDefineSlot = googletag.defineSlot + if (typeof originalDefineSlot === "function") { + try { + const descriptor = Object.getOwnPropertyDescriptor(googletag, "defineSlot") + Object.defineProperty(googletag, "defineSlot", { + configurable: true, + enumerable: descriptor ? descriptor.enumerable : true, + writable: true, + value: function (adUnitPath, sizes, divId) { + const slot = originalDefineSlot.apply(this, arguments) + try { + __ts_record_define_slot(adUnitPath, sizes, divId) + } catch (error) { + __ts_warn("define_slot_capture_failed", error) + } + return slot + } + }) + } catch (error) { + __ts_warn("define_slot_wrap_failed", error) + } + } + return googletag +} + +// Wrap an existing global or intercept a later assignment of it. +function __ts_install(name, wrap) { + if (window[name]) { + try { + wrap(window[name]) + } catch (error) { + __ts_warn(name + "_wrap_failed", error) + } + return + } + let internal + Object.defineProperty(window, name, { + configurable: true, + // A real `window.googletag` is an ordinary enumerable global; matching that + // keeps `Object.keys(window)` identical with and without the collector. + enumerable: true, + get() { + return internal + }, + set(value) { + internal = value + try { + internal = wrap(value) + } catch (error) { + __ts_warn(name + "_wrap_failed", error) + } + } + }) +} + +__ts_install("googletag", __ts_wrap_googletag) + +// On-demand DOM + getSlots scrape, invoked by the collector after settle/scroll. +window.__tsCollectAdTemplateEvidence = function () { + try { + const seen = new Set(__ts_ev.dom_ids.map((entry) => entry.dom_id)) + for (const element of document.querySelectorAll("[id]")) { + const id = __ts_text(element.id) + if (id.endsWith("-container")) continue + if (__ts_prefixes.some((prefix) => id.startsWith(prefix)) && !seen.has(id)) { + __ts_push(__ts_ev.dom_ids, { dom_id: id, phase: __ts_phase() }) + seen.add(id) + } + } + const googletag = window.googletag + if (googletag && typeof googletag.pubads === "function") { + const pubads = googletag.pubads() + const slots = typeof pubads.getSlots === "function" ? pubads.getSlots() : [] + for (const slot of slots) { + try { + const path = typeof slot.getAdUnitPath === "function" ? slot.getAdUnitPath() : "" + const divId = typeof slot.getSlotElementId === "function" ? slot.getSlotElementId() : "" + const rawSizes = typeof slot.getSizes === "function" ? slot.getSizes() : [] + const sizes = [] + for (const size of rawSizes) { + if (sizes.length >= __ts_max_entries) break + let pair = null + if ( + size && + typeof size.getWidth === "function" && + typeof size.getHeight === "function" + ) { + // A fluid GPT size answers getWidth()/getHeight() with a + // non-numeric value rather than throwing. + pair = __ts_size_pair(size.getWidth(), size.getHeight()) + } else if (Array.isArray(size)) { + pair = __ts_size_pair(size[0], size[1]) + } + if (pair) { + sizes.push(pair) + } else { + const width = + size && typeof size.getWidth === "function" + ? size.getWidth() + : Array.isArray(size) + ? size[0] + : undefined + const height = + size && typeof size.getHeight === "function" + ? size.getHeight() + : Array.isArray(size) + ? size[1] + : undefined + __ts_warn_ignored_size(width, height) + } + } + const exists = __ts_ev.gpt_slots.some( + (entry) => entry.gam_unit_path === __ts_text(path) && entry.div_id === __ts_text(divId) + ) + if (!exists) { + __ts_push(__ts_ev.gpt_slots, { + gam_unit_path: __ts_text(path), + div_id: __ts_text(divId), + sizes, + phase: __ts_phase() + }) + } + } catch (error) { + __ts_warn("gpt_scrape_failed", error) + } + } + } + } catch (error) { + __ts_warn("collect_failed", error) + } + return __ts_ev +} diff --git a/crates/trusted-server-cli/src/commands/audit/ad_templates.rs b/crates/trusted-server-cli/src/commands/audit/ad_templates.rs new file mode 100644 index 000000000..0e2b51371 --- /dev/null +++ b/crates/trusted-server-cli/src/commands/audit/ad_templates.rs @@ -0,0 +1,1014 @@ +//! Browser-backed `ts audit ad-templates verify` orchestration. +//! +//! For each URL: collect live evidence through an [`AuditCollector`], match +//! configured slots against the **final** (post-redirect) path, evaluate the +//! runtime gate, compare evidence, and assemble the stable §8 wire result. The +//! orchestration is collector-agnostic so it is fully tested with an in-memory +//! fake collector, with no Chrome dependency. + +use std::io::{self, Write}; + +use trusted_server_core::auction::types::MediaType; +use trusted_server_core::creative_opportunities::{ + AdStackGateInput, CreativeOpportunitiesConfig, evaluate_ad_stack_gate, +}; + +use crate::ad_templates::compare::{ + BrowserAdEvidence, EvidencePhase, ExtraEvidence, RuntimeGateSummary, SlotEvidence, SlotResult, + SlotStatus as CompareStatus, compare_page_evidence, +}; +use crate::ad_templates::expected::{ExpectedSlot, expected_slots_for_path, normalize_path_or_url}; +use crate::ad_templates::output::{ + ConfiguredJson, EvidencePhaseJson, ExtraEvidenceJson, FormatJson, GateState, Gates, + GptEvidenceJson, PageJson, RuntimeAdStackExpectedJson, SlotEvidenceJson, SlotJson, SlotStatus, + VerificationReport, Warning, escape_terminal_text, +}; +use crate::commands::audit::AuditAdTemplatesVerifyArgs; +use crate::commands::audit::collector::{ + AdTemplateCollectorConfig, AuditCollector, BrowserCollectRequest, build_ad_template_init_script, +}; +use crate::run::RunOutcome; + +/// Verifies configured ad-template slots against live page evidence. +/// +/// # Errors +/// +/// Returns a user-facing string when config loading fails, or when verification +/// surfaces a page-level error or a `--strict` failure (after writing output). +pub(crate) fn run_verify(args: &AuditAdTemplatesVerifyArgs) -> Result { + args.browser.validate()?; + validate_cookie_scope(&args.urls, &args.cookies)?; + let loaded = crate::app_config::load_settings(&args.config)?; + let collector = crate::commands::audit::browser::BrowserCollector::from_opts(&args.browser); + let report = build_report( + &collector, + loaded.settings.creative_opportunities.as_ref(), + loaded.settings.auction.enabled, + &args.urls, + VerifyOptions { + strict: args.strict, + scroll: args.scroll, + allow_cross_origin_redirect: args.allow_cross_origin_redirect, + }, + &args.cookies, + )?; + + let stdout = io::stdout(); + let mut out = stdout.lock(); + if args.json { + write_json(&mut out, &report)?; + } else { + write_human(&mut out, &report)?; + } + + if report.pages.iter().any(|page| page.error.is_some()) { + Err("ad-template verification reported problems".to_string()) + } else if report.ok { + Ok(RunOutcome::Success) + } else { + Ok(RunOutcome::AssertionFailed) + } +} + +fn validate_cookie_scope(urls: &[url::Url], cookies: &[(String, String)]) -> Result<(), String> { + if cookies.is_empty() { + return Ok(()); + } + let origins: std::collections::BTreeSet = urls + .iter() + .map(|url| url.origin().ascii_serialization()) + .collect(); + if origins.len() > 1 { + return Err( + "--cookie may be used only when every verification URL has one origin; split this run so credentials are never copied to another origin" + .to_string(), + ); + } + Ok(()) +} + +/// Run-level verification switches. +#[derive(Debug, Clone, Copy)] +struct VerifyOptions { + /// Exit non-zero when a matched slot is missing or only partially confirmed. + strict: bool, + /// Perform a deterministic scroll pass after the initial settle. + scroll: bool, + /// Accept evidence from a page that redirected to a different origin. + allow_cross_origin_redirect: bool, +} + +/// Builds the verification report for `urls` using `collector`. +/// +/// `creative` is the effective `[creative_opportunities]` config (if any) and +/// `auction_enabled` is the `[auction].enabled` kill switch. +fn build_report( + collector: &dyn AuditCollector, + creative: Option<&CreativeOpportunitiesConfig>, + auction_enabled: bool, + urls: &[url::Url], + options: VerifyOptions, + cookies: &[(String, String)], +) -> Result { + let init_script = build_init_script(creative)?; + + let requests: Vec<_> = urls + .iter() + .map(|url| BrowserCollectRequest { + url: url.clone(), + init_scripts: vec![init_script.clone()], + scroll: options.scroll, + collect_ad_evidence: true, + cookies: cookies.to_vec(), + }) + .collect(); + let collected_pages = collector.collect_pages(&requests); + + let mut pages = Vec::with_capacity(urls.len()); + let mut any_error = false; + let mut any_strict_fail = false; + + for (url, collected) in urls.iter().zip(collected_pages) { + match collected { + Err(message) => { + any_error = true; + pages.push(error_page(url, &message)); + } + // Slots are matched on the *final* path, so a redirect to a + // different origin would let an unrelated site's evidence satisfy + // `--strict` — and the path-equality redirect warning would not even + // fire when the paths happen to agree. Reject unless opted in. + Ok(collected) + if !options.allow_cross_origin_redirect + && origin_changed(url, &collected.final_url) => + { + any_error = true; + pages.push(cross_origin_page(url, &collected.final_url)); + } + Ok(collected) => { + let (page, strict_failed) = build_page(url, &collected, creative, auction_enabled); + if options.strict && strict_failed { + any_strict_fail = true; + } + pages.push(page); + } + } + } + + let ok = !(any_error || (options.strict && any_strict_fail)); + Ok(VerificationReport { + ok, + strict: options.strict, + pages, + warnings: Vec::new(), + }) +} + +/// The URL without its fragment, for comparisons the server can observe. +pub(super) fn without_fragment(url: &url::Url) -> url::Url { + let mut url = url.clone(); + url.set_fragment(None); + url +} + +/// Whether navigation left the requested URL's origin (scheme, host, or port). +/// +/// A same-host default-port `http:80` to `https:443` redirect is *not* a change: +/// the host is the cookie boundary, and that upgrade is the ordinary canonical +/// redirect. Host changes, port changes, and HTTPS downgrades all are. +pub(super) fn origin_changed(requested: &url::Url, final_url: &url::Url) -> bool { + if requested.host_str() != final_url.host_str() { + return true; + } + + match (requested.scheme(), final_url.scheme()) { + ("http", "https") => { + requested.port_or_known_default() != Some(80) + || final_url.port_or_known_default() != Some(443) + } + (requested_scheme @ ("http" | "https"), final_scheme) + if requested_scheme == final_scheme => + { + requested.port_or_known_default() != final_url.port_or_known_default() + } + // Refuse HTTPS downgrades and any unexpected scheme transition. + _ => true, + } +} + +/// Builds the read-only collector init script from the configured slots. +fn build_init_script(creative: Option<&CreativeOpportunitiesConfig>) -> Result { + let config = AdTemplateCollectorConfig { + div_prefixes: creative + .map(|creative| { + creative + .slot + .iter() + .map(|slot| slot.resolved_div_id().to_string()) + .collect() + }) + .unwrap_or_default(), + }; + build_ad_template_init_script(&config) +} + +/// Assembles a successful page result, returning the wire `PageJson` and whether +/// the page would fail `--strict`. +fn build_page( + requested: &url::Url, + collected: &crate::commands::audit::collector::CollectedPage, + creative: Option<&CreativeOpportunitiesConfig>, + auction_enabled: bool, +) -> (PageJson, bool) { + let requested_path = normalize_path_or_url(requested.as_str()).unwrap_or_else(|_| "/".into()); + let final_url = &collected.final_url; + let final_path = normalize_path_or_url(final_url.as_str()).unwrap_or_else(|_| "/".into()); + + let expected = creative + .map(|creative| expected_slots_for_path(&final_path, creative).slots) + .unwrap_or_default(); + let matched = !expected.is_empty(); + + let gate = evaluate_ad_stack_gate(AdStackGateInput { + method_get: true, + navigation: true, + prefetch: false, + bot: false, + matched_slots: matched, + consent_allows_auction: None, + auction_enabled, + // Absent creative opportunities block here as they do at runtime. + ad_templates_enabled: creative.is_some_and(|creative| creative.enabled), + }); + + let evidence = collected.ad_evidence.clone().unwrap_or_else(empty_evidence); + let result = compare_page_evidence( + &expected, + &evidence, + RuntimeGateSummary::from_expected(gate.expected), + ); + let strict_failed = result.strict_failed(); + + let mut warnings: Vec = collected.warnings.to_vec(); + warnings.extend(evidence.warnings.iter().map(|warning| Warning { + code: format!("page_{}", warning.code), + message: warning.message.clone(), + })); + // Fragments never reach the server, so a fragment-only difference is not a + // redirect and slots match on the path either way. + if without_fragment(requested) != without_fragment(final_url) { + warnings.push(Warning { + code: "redirected".to_string(), + message: format!("navigation redirected from {requested} to {final_url}"), + }); + } + + let slots = expected + .iter() + .zip(result.slots.iter()) + .map(|(expected_slot, slot_result)| to_slot_json(expected_slot, slot_result)) + .collect(); + let extra_evidence = result.extra_evidence.iter().map(to_extra_json).collect(); + + let page = PageJson { + url: requested.to_string(), + final_url: Some(final_url.to_string()), + requested_path, + path: Some(final_path), + error: None, + runtime_ad_stack_expected: Some(RuntimeAdStackExpectedJson::from( + result.runtime_ad_stack_expected, + )), + gates: Some(to_gates( + matched, + auction_enabled, + creative.is_some_and(|creative| creative.enabled), + )), + matched_slot_count: Some(expected.len()), + slots, + extra_evidence, + warnings, + }; + (page, strict_failed) +} + +/// Builds a page-level navigation-failure result (spec §8 `navigation_failed`). +fn error_page(requested: &url::Url, message: &str) -> PageJson { + let requested_path = normalize_path_or_url(requested.as_str()).unwrap_or_else(|_| "/".into()); + PageJson { + url: requested.to_string(), + final_url: None, + requested_path, + path: None, + error: Some(Warning { + code: "navigation_failed".to_string(), + message: message.to_string(), + }), + runtime_ad_stack_expected: None, + gates: None, + matched_slot_count: None, + slots: Vec::new(), + extra_evidence: Vec::new(), + warnings: Vec::new(), + } +} + +/// Builds a page-level cross-origin-redirect refusal. +/// +/// The final URL is reported so the operator can re-run against it explicitly +/// (or pass `--allow-cross-origin-redirect`) once they have confirmed it is +/// their own property. +fn cross_origin_page(requested: &url::Url, final_url: &url::Url) -> PageJson { + let requested_path = normalize_path_or_url(requested.as_str()).unwrap_or_else(|_| "/".into()); + PageJson { + url: requested.to_string(), + final_url: Some(final_url.to_string()), + requested_path, + path: None, + error: Some(Warning { + code: "cross_origin_redirect".to_string(), + message: format!( + "navigation left the requested origin ({} -> {}); \ + evidence from another origin is not accepted as verification. \ + Re-run against the final URL, or pass --allow-cross-origin-redirect", + requested.origin().ascii_serialization(), + final_url.origin().ascii_serialization(), + ), + }), + runtime_ad_stack_expected: None, + gates: None, + matched_slot_count: None, + slots: Vec::new(), + extra_evidence: Vec::new(), + warnings: Vec::new(), + } +} + +fn empty_evidence() -> BrowserAdEvidence { + BrowserAdEvidence { + dom_ids: Vec::new(), + gpt_slots: Vec::new(), + aps_calls: Vec::new(), + page_bids: Vec::new(), + warnings: Vec::new(), + } +} + +fn to_gates(matched: bool, auction_enabled: bool, ad_templates_enabled: bool) -> Gates { + let pass_if = |cond: bool| { + if cond { + GateState::Pass + } else { + GateState::Fail + } + }; + Gates { + method_get: GateState::Pass, + navigation: GateState::Pass, + not_prefetch: GateState::Pass, + not_bot: GateState::Pass, + matched_slots: pass_if(matched), + auction_enabled: pass_if(auction_enabled), + ad_templates_enabled: pass_if(ad_templates_enabled), + // Live consent is not provable from a browser navigation in Phase 1. + consent_allows_auction: GateState::Unknown, + } +} + +fn to_slot_json(expected: &ExpectedSlot, result: &SlotResult) -> SlotJson { + SlotJson { + id: result.id.clone(), + status: to_status(result.status), + phase: result.phase.map(to_phase), + configured: ConfiguredJson { + div_id: expected.div_id.clone(), + gam_unit_path: expected.gam_unit_path.clone(), + formats: expected + .formats + .iter() + .map(|format| FormatJson { + width: format.width, + height: format.height, + media_type: media_type_label(&format.media_type).to_string(), + }) + .collect(), + providers: expected.providers.clone(), + }, + evidence: to_slot_evidence(&result.evidence), + warnings: result.warnings.clone(), + } +} + +fn to_slot_evidence(evidence: &SlotEvidence) -> SlotEvidenceJson { + SlotEvidenceJson { + dom_id: evidence.dom_id.clone(), + gpt: evidence.gpt.as_ref().map(|gpt| GptEvidenceJson { + gam_unit_path: gpt.gam_unit_path.clone(), + div_id: gpt.div_id.clone(), + sizes: gpt.sizes.iter().map(|&(w, h)| [w, h]).collect(), + }), + } +} + +fn to_extra_json(extra: &ExtraEvidence) -> ExtraEvidenceJson { + ExtraEvidenceJson { + kind: extra.kind.clone(), + phase: to_phase(extra.phase), + dom_id: extra.dom_id.clone(), + gam_unit_path: extra.gam_unit_path.clone(), + sizes: extra.sizes.iter().map(|&(w, h)| [w, h]).collect(), + reason: extra.reason.clone(), + } +} + +fn to_status(status: CompareStatus) -> SlotStatus { + match status { + CompareStatus::Confirmed => SlotStatus::Confirmed, + CompareStatus::Partial => SlotStatus::Partial, + CompareStatus::Missing => SlotStatus::Missing, + CompareStatus::Unconfirmable => SlotStatus::Unconfirmable, + } +} + +fn to_phase(phase: EvidencePhase) -> EvidencePhaseJson { + match phase { + EvidencePhase::InitialLoad => EvidencePhaseJson::InitialLoad, + EvidencePhase::Scroll => EvidencePhaseJson::Scroll, + } +} + +fn media_type_label(media_type: &MediaType) -> &'static str { + match media_type { + MediaType::Banner => "banner", + MediaType::Video => "video", + MediaType::Native => "native", + } +} + +fn write_json(out: &mut dyn Write, report: &VerificationReport) -> Result<(), String> { + let json = serde_json::to_string_pretty(report) + .map_err(|error| format!("failed to serialize verification report: {error}"))?; + writeln!(out, "{json}").map_err(write_err) +} + +fn write_human(out: &mut dyn Write, report: &VerificationReport) -> Result<(), String> { + // Warning codes and messages can originate in the audited page (the + // collector forwards `String(error)` from page scripts), so escape control + // characters before writing them to the operator's terminal. + let write_warning = |out: &mut dyn Write, indent: &str, warning: &Warning| { + writeln!( + out, + "{indent}warning [{}]: {}", + escape_terminal_text(&warning.code), + escape_terminal_text(&warning.message) + ) + .map_err(write_err) + }; + + for warning in &report.warnings { + write_warning(out, "", warning)?; + } + for page in &report.pages { + writeln!(out, "url: {}", escape_terminal_text(&page.url)).map_err(write_err)?; + if let Some(error) = &page.error { + writeln!( + out, + " error [{}]: {}", + escape_terminal_text(&error.code), + escape_terminal_text(&error.message) + ) + .map_err(write_err)?; + continue; + } + if let Some(path) = &page.path { + writeln!(out, " path: {}", escape_terminal_text(path)).map_err(write_err)?; + } + if let Some(expected) = page.runtime_ad_stack_expected { + writeln!(out, " runtime ad stack: {}", runtime_label(expected)).map_err(write_err)?; + } + if let Some(count) = page.matched_slot_count { + writeln!(out, " matched slots: {count}").map_err(write_err)?; + } + if let Some(gates) = &page.gates { + writeln!(out, " gates: {}", gates_label(gates)).map_err(write_err)?; + } + for slot in &page.slots { + writeln!( + out, + " slot {}: {}", + escape_terminal_text(&slot.id), + status_label(slot.status) + ) + .map_err(write_err)?; + for warning in &slot.warnings { + write_warning(out, " ", warning)?; + } + } + for extra in &page.extra_evidence { + writeln!( + out, + " extra {} evidence: div={} gam={} sizes={:?} ({})", + escape_terminal_text(&extra.kind), + escape_terminal_text(extra.dom_id.as_deref().unwrap_or("-")), + escape_terminal_text(extra.gam_unit_path.as_deref().unwrap_or("-")), + extra.sizes, + escape_terminal_text(&extra.reason), + ) + .map_err(write_err)?; + } + for warning in &page.warnings { + write_warning(out, " ", warning)?; + } + } + writeln!(out, "ok: {}", report.ok).map_err(write_err) +} + +fn status_label(status: SlotStatus) -> &'static str { + match status { + SlotStatus::Confirmed => "confirmed", + SlotStatus::Partial => "partial", + SlotStatus::Missing => "missing", + SlotStatus::Unconfirmable => "unconfirmable", + } +} + +fn runtime_label(expected: RuntimeAdStackExpectedJson) -> &'static str { + match expected { + RuntimeAdStackExpectedJson::Yes => "yes", + RuntimeAdStackExpectedJson::No => "no", + RuntimeAdStackExpectedJson::Unknown => "unknown", + } +} + +fn gate_label(gate: GateState) -> &'static str { + match gate { + GateState::Pass => "pass", + GateState::Fail => "fail", + GateState::Unknown => "unknown", + } +} + +fn gates_label(gates: &Gates) -> String { + format!( + "method_get={} navigation={} not_prefetch={} not_bot={} matched_slots={} auction_enabled={} consent={}", + gate_label(gates.method_get), + gate_label(gates.navigation), + gate_label(gates.not_prefetch), + gate_label(gates.not_bot), + gate_label(gates.matched_slots), + gate_label(gates.auction_enabled), + gate_label(gates.consent_allows_auction), + ) +} + +#[allow( + clippy::needless_pass_by_value, + reason = "used as a map_err fn that receives io::Error by value" +)] +fn write_err(error: io::Error) -> String { + format!("failed to write command output: {error}") +} + +#[cfg(test)] +mod tests { + use std::cell::Cell; + use std::collections::HashMap; + + use super::*; + use crate::ad_templates::compare::{DomEvidence, GptSlotEvidence}; + use crate::commands::audit::collector::CollectedPage; + + struct FakeCollector { + pages: HashMap>, + batch_calls: Cell, + } + + impl FakeCollector { + fn page(requested: &str, final_url: &str, evidence: BrowserAdEvidence) -> Self { + let mut pages = HashMap::new(); + pages.insert( + requested.to_string(), + Ok(CollectedPage { + final_url: url::Url::parse(final_url).expect("should parse final URL"), + title: String::new(), + script_count: 0, + resource_count: 0, + warnings: Vec::new(), + ad_evidence: Some(evidence), + }), + ); + Self { + pages, + batch_calls: Cell::new(0), + } + } + + fn with_error(mut self, requested: &str, message: &str) -> Self { + self.pages + .insert(requested.to_string(), Err(message.to_string())); + self + } + } + + impl AuditCollector for FakeCollector { + fn collect_page(&self, request: BrowserCollectRequest) -> Result { + self.pages + .get(request.url.as_str()) + .cloned() + .unwrap_or_else(|| Err(format!("no fake page for {}", request.url))) + } + + fn collect_pages( + &self, + requests: &[BrowserCollectRequest], + ) -> Vec> { + self.batch_calls.set(self.batch_calls.get() + 1); + requests + .iter() + .cloned() + .map(|request| self.collect_page(request)) + .collect() + } + } + + fn news_config() -> CreativeOpportunitiesConfig { + let toml = "gam_network_id = \"123\"\n\ + \n\ + [[slot]]\n\ + id = \"atf\"\n\ + gam_unit_path = \"/123/news/atf\"\n\ + div_id = \"ad-atf-\"\n\ + page_patterns = [\"/news/*\"]\n\ + formats = [{ width = 300, height = 250 }]\n"; + let mut config = + toml::from_str::(toml).expect("should deserialize"); + config.compile_slots(); + config + } + + fn confirmed_news_evidence() -> BrowserAdEvidence { + BrowserAdEvidence { + dom_ids: vec![DomEvidence { + dom_id: "ad-atf-0".to_string(), + phase: EvidencePhase::InitialLoad, + }], + gpt_slots: vec![GptSlotEvidence { + gam_unit_path: "/123/news/atf".to_string(), + div_id: "ad-atf-0".to_string(), + sizes: vec![(300, 250)], + phase: EvidencePhase::InitialLoad, + }], + aps_calls: Vec::new(), + page_bids: Vec::new(), + warnings: Vec::new(), + } + } + + fn report_for( + collector: &dyn AuditCollector, + auction_enabled: bool, + strict: bool, + urls: &[&str], + ) -> VerificationReport { + report_for_with_options( + collector, + auction_enabled, + urls, + VerifyOptions { + strict, + scroll: false, + allow_cross_origin_redirect: false, + }, + ) + } + + fn report_for_with_options( + collector: &dyn AuditCollector, + auction_enabled: bool, + urls: &[&str], + options: VerifyOptions, + ) -> VerificationReport { + let config = news_config(); + let parsed: Vec = urls + .iter() + .map(|url| url::Url::parse(url).expect("should parse URL")) + .collect(); + build_report( + collector, + Some(&config), + auction_enabled, + &parsed, + options, + &[], + ) + .expect("typed collector configuration should serialize") + } + + #[test] + fn verify_uses_final_url_for_matching_after_redirect() { + let collector = FakeCollector::page( + "https://www.example.com/", + "https://www.example.com/news/story", + confirmed_news_evidence(), + ); + let report = report_for(&collector, true, false, &["https://www.example.com/"]); + let json = serde_json::to_value(&report).expect("should serialize"); + + assert_eq!(json["pages"][0]["path"], "/news/story"); + assert_eq!(json["pages"][0]["slots"][0]["status"], "confirmed"); + let warnings = json["pages"][0]["warnings"] + .as_array() + .expect("should have warnings array"); + assert!( + warnings.iter().any(|w| w["code"] == "redirected"), + "redirect should emit a `redirected` warning" + ); + } + + #[test] + fn cross_origin_redirect_is_rejected_even_when_paths_match() { + // Same path on a different origin: the redirect warning would not fire, + // so without the origin check this unrelated page's evidence would + // satisfy --strict. + let collector = FakeCollector::page( + "https://www.example.com/news/story", + "https://impostor.example.net/news/story", + confirmed_news_evidence(), + ); + let report = report_for( + &collector, + true, + true, + &["https://www.example.com/news/story"], + ); + + assert!(!report.ok, "a cross-origin redirect must not report ok"); + let json = serde_json::to_value(&report).expect("should serialize"); + assert_eq!(json["pages"][0]["error"]["code"], "cross_origin_redirect"); + assert!( + json["pages"][0]["slots"] + .as_array() + .expect("should have slots array") + .is_empty(), + "off-origin evidence must not be reported as slot verification" + ); + } + + #[test] + fn cross_origin_redirect_is_accepted_with_explicit_opt_in() { + let collector = FakeCollector::page( + "https://example.com/news/story", + "https://www.example.com/news/story", + confirmed_news_evidence(), + ); + let report = report_for_with_options( + &collector, + true, + &["https://example.com/news/story"], + VerifyOptions { + strict: true, + scroll: false, + allow_cross_origin_redirect: true, + }, + ); + + assert!( + report.ok, + "an opted-in apex -> www redirect should verify normally" + ); + assert_eq!(report.pages[0].matched_slot_count, Some(1)); + } + + #[test] + fn same_origin_path_redirect_still_verifies() { + let collector = FakeCollector::page( + "https://www.example.com/", + "https://www.example.com/news/story", + confirmed_news_evidence(), + ); + let report = report_for(&collector, true, true, &["https://www.example.com/"]); + + assert!( + report.ok, + "a same-origin redirect should still be verified, not refused" + ); + } + + #[test] + fn same_host_http_to_https_upgrade_is_accepted() { + let collector = FakeCollector::page( + "http://www.example.com/news/story", + "https://www.example.com/news/story", + confirmed_news_evidence(), + ); + let report = report_for( + &collector, + true, + true, + &["http://www.example.com/news/story"], + ); + + assert!(report.ok, "a default-port HTTPS upgrade should be accepted"); + } + + #[test] + fn downgrade_and_port_changes_are_rejected() { + for (requested, final_url) in [ + ( + "https://www.example.com/news/story", + "http://www.example.com/news/story", + ), + ( + "https://www.example.com:8443/news/story", + "https://www.example.com:9443/news/story", + ), + ( + "http://www.example.com:8080/news/story", + "https://www.example.com:8443/news/story", + ), + ] { + let collector = FakeCollector::page(requested, final_url, confirmed_news_evidence()); + let report = report_for(&collector, true, true, &[requested]); + assert!(!report.ok, "redirect {requested} -> {final_url} must fail"); + } + } + + #[test] + fn confirmed_page_is_ok_in_default_mode() { + let collector = FakeCollector::page( + "https://www.example.com/news/story", + "https://www.example.com/news/story", + confirmed_news_evidence(), + ); + let report = report_for( + &collector, + true, + false, + &["https://www.example.com/news/story"], + ); + + assert!(report.ok, "confirmed page should be ok"); + assert_eq!(report.pages[0].matched_slot_count, Some(1)); + } + + #[test] + fn verifier_surfaces_injected_collector_warnings() { + let mut evidence = confirmed_news_evidence(); + evidence.warnings.push(Warning { + code: "fluid_size_ignored".to_string(), + message: "a fluid size could not be compared".to_string(), + }); + let collector = FakeCollector::page( + "https://www.example.com/news/story", + "https://www.example.com/news/story", + evidence, + ); + + let report = report_for( + &collector, + true, + false, + &["https://www.example.com/news/story"], + ); + + assert!( + report.pages[0] + .warnings + .iter() + .any(|warning| warning.code == "page_fluid_size_ignored"), + "collector warning should be visible in the page report" + ); + } + + #[test] + fn human_output_includes_runtime_and_extra_evidence_diagnostics() { + let mut evidence = confirmed_news_evidence(); + evidence.gpt_slots.push(GptSlotEvidence { + gam_unit_path: "/123/publisher/extra".to_string(), + div_id: "ad-extra-0".to_string(), + sizes: vec![(728, 90)], + phase: EvidencePhase::InitialLoad, + }); + let collector = FakeCollector::page( + "https://www.example.com/news/story", + "https://www.example.com/news/story", + evidence, + ); + let report = report_for( + &collector, + true, + false, + &["https://www.example.com/news/story"], + ); + let mut output = Vec::new(); + + write_human(&mut output, &report).expect("should write human report"); + let output = String::from_utf8(output).expect("should be UTF-8 output"); + + assert!(output.contains("runtime ad stack: unknown")); + assert!(output.contains("matched slots: 1")); + assert!(output.contains("gates: method_get=pass")); + assert!(output.contains("extra gpt evidence")); + } + + #[test] + fn strict_missing_slot_fails() { + let collector = FakeCollector::page( + "https://www.example.com/news/story", + "https://www.example.com/news/story", + empty_evidence(), + ); + let report = report_for( + &collector, + true, + true, + &["https://www.example.com/news/story"], + ); + + assert!( + !report.ok, + "strict mode with a missing slot should not be ok" + ); + } + + #[test] + fn auction_disabled_skips_strict_missing_failure() { + let collector = FakeCollector::page( + "https://www.example.com/news/story", + "https://www.example.com/news/story", + empty_evidence(), + ); + // auction disabled -> runtime expected No -> strict does not fail on missing. + let report = report_for( + &collector, + false, + true, + &["https://www.example.com/news/story"], + ); + + assert!( + report.ok, + "missing slot must not fail strict when auction is disabled" + ); + assert_eq!( + report.pages[0].runtime_ad_stack_expected, + Some(RuntimeAdStackExpectedJson::No) + ); + } + + #[test] + fn multi_url_page_error_sets_ok_false() { + let collector = FakeCollector::page( + "https://www.example.com/news/story", + "https://www.example.com/news/story", + confirmed_news_evidence(), + ) + .with_error("https://www.example.com/broken", "navigation failed"); + let report = report_for( + &collector, + true, + false, + &[ + "https://www.example.com/news/story", + "https://www.example.com/broken", + ], + ); + + assert!(!report.ok, "a page-level error sets ok=false"); + assert_eq!( + collector.batch_calls.get(), + 1, + "all verifier URLs should use one collector batch" + ); + let json = serde_json::to_value(&report).expect("should serialize"); + assert_eq!(json["pages"][1]["error"]["code"], "navigation_failed"); + assert!(json["pages"][1]["final_url"].is_null()); + } + + #[test] + fn supplied_cookies_are_rejected_for_multiple_origins() { + let urls = [ + url::Url::parse("https://a.example/x").expect("should parse first URL"), + url::Url::parse("https://b.example/y").expect("should parse second URL"), + ]; + + let error = validate_cookie_scope(&urls, &[("session".to_string(), "secret".to_string())]) + .expect_err("should not replicate one cookie across origins"); + + assert!( + error.contains("one origin"), + "the refusal should explain cookie scope, got {error}" + ); + } + + #[test] + fn supplied_cookies_are_allowed_for_same_origin_urls() { + let urls = [ + url::Url::parse("https://a.example/x").expect("should parse first URL"), + url::Url::parse("https://a.example/y").expect("should parse second URL"), + ]; + + validate_cookie_scope(&urls, &[("session".to_string(), "secret".to_string())]) + .expect("same-origin URLs share the intended cookie scope"); + } +} diff --git a/crates/trusted-server-cli/src/commands/audit/browser.rs b/crates/trusted-server-cli/src/commands/audit/browser.rs new file mode 100644 index 000000000..a4c0514a6 --- /dev/null +++ b/crates/trusted-server-cli/src/commands/audit/browser.rs @@ -0,0 +1,1196 @@ +//! Chrome/Chromium-backed implementation of [`AuditCollector`] using +//! `chromiumoxide` (CDP). +//! +//! The collector installs optional pre-navigation init scripts, sets any +//! operator-supplied cookies, navigates, waits for the page to settle, optionally +//! scrolls, and reads back a bounded set of evidence. It never *captures* page +//! HTML, cookies, or storage; supplied cookies are only *sent* to carry an +//! existing session past origin gates. + +use std::time::Duration; + +use chromiumoxide::browser::{Browser, BrowserConfig}; +use chromiumoxide::cdp::browser_protocol::network::CookieParam; +use chromiumoxide::handler::viewport::Viewport; +use chromiumoxide::page::Page; +use futures::StreamExt as _; + +use crate::ad_templates::compare::BrowserAdEvidence; +use crate::ad_templates::output::Warning; +use crate::commands::audit::browser_scroll; +use crate::commands::audit::collector::{ + AuditCollector, BrowserCollectRequest, BrowserOpts, BrowserProfile, CollectedPage, + PAGE_SETTLE_MAX_MS, PAGE_SETTLE_QUIET_MS, +}; + +/// Candidate Chrome/Chromium executable names searched on `PATH`. +pub(crate) const CHROME_NAMES: &[&str] = &[ + "google-chrome", + "google-chrome-stable", + "chromium", + "chromium-browser", + "chrome", + "Google Chrome", + "Google Chrome for Testing", +]; + +/// Poll interval while waiting for the page network to settle, in milliseconds. +const SETTLE_POLL_MS: u64 = 250; +/// Hard cap on page navigation so a stalled load cannot hang the audit. +const NAVIGATION_TIMEOUT: Duration = Duration::from_secs(30); +/// Bound for each CDP operation after navigation. +const CDP_OPERATION_TIMEOUT: Duration = Duration::from_secs(5); +/// Hard cap per decoded evidence list, so a hostile page cannot inflate CLI +/// memory. +/// +/// Must equal `__ts_max_entries` in `ad_template_collector.js`. The collector +/// already caps each list, but the evidence object lives on `window`, so a page +/// that appends to it directly is bounded here instead. Anything the collector +/// itself dropped is reported as an `evidence_truncated` warning. +const MAX_EVIDENCE_ENTRIES: usize = 128; +/// Hard cap on the UTF-8 JSON payload before CDP transfers it back to Rust. +const MAX_EVIDENCE_PAYLOAD_BYTES: usize = 1024 * 1024; +/// Hard cap on browser teardown so a wedged Chrome cannot hang the audit. +const BROWSER_CLOSE_TIMEOUT: Duration = Duration::from_secs(5); + +/// Page-settle timing thresholds. +#[derive(Debug, Clone, Copy)] +struct SettleConfig { + /// Quiet window with no new resources marking the page settled. + quiet: Duration, + /// Hard cap on total settle time. + max: Duration, +} + +/// Immutable browser/session settings shared by every URL in one audit batch. +struct BrowserSessionOptions<'a> { + chrome: &'a std::path::Path, + profile_dir: &'a std::path::Path, + settle: SettleConfig, + accept_invalid_certs: bool, + headful: bool, + assume_consent: bool, + proxy: Option<&'a str>, + profile: BrowserProfile, +} + +/// A `chromiumoxide`-backed page collector launching a local Chrome/Chromium. +#[derive(Debug, Clone)] +pub struct BrowserCollector { + /// Explicit Chrome/Chromium executable override (else `$CHROME`, else auto-detect). + chrome: Option, + /// Quiet window marking the page settled. + settle_quiet: Duration, + /// Hard cap on settling. + settle_max: Duration, + /// Navigate to origins with invalid TLS certificates (dangerous opt-in). + accept_invalid_certs: bool, + /// Run visible Chrome rather than new headless Chrome. + headful: bool, + /// Install the standard consent API stub before publisher scripts. + assume_consent: bool, + /// Optional browser proxy endpoint. + proxy: Option, + /// Device viewport/user-agent profile. + profile: BrowserProfile, +} + +impl Default for BrowserCollector { + fn default() -> Self { + Self::new() + } +} + +impl BrowserCollector { + /// Creates a collector with default tuning and auto-detected Chrome. + #[must_use] + pub fn new() -> Self { + Self { + chrome: None, + settle_quiet: Duration::from_millis(PAGE_SETTLE_QUIET_MS), + settle_max: Duration::from_millis(PAGE_SETTLE_MAX_MS), + accept_invalid_certs: false, + headful: false, + assume_consent: true, + proxy: None, + profile: BrowserProfile::Desktop, + } + } + + /// Creates a collector from operator-supplied browser options. + #[must_use] + pub fn from_opts(opts: &BrowserOpts) -> Self { + Self { + chrome: opts.chrome.clone(), + settle_quiet: Duration::from_millis(opts.settle_quiet_ms), + settle_max: Duration::from_millis(opts.settle_max_ms), + accept_invalid_certs: opts.danger_accept_invalid_certs, + headful: opts.headful, + assume_consent: !opts.no_assume_consent, + proxy: opts.browser_proxy.clone(), + profile: opts.profile, + } + } +} + +/// Pre-document consent behavior shared with the generation crawler. +pub(crate) const CONSENT_STUB_SCRIPT: &str = include_str!("consent_stub.js"); + +/// Shared browser launch inputs used by both audit collectors. +pub(crate) struct BrowserLaunchOptions<'a> { + pub(crate) chrome: &'a std::path::Path, + pub(crate) profile_dir: &'a std::path::Path, + pub(crate) headful: bool, + pub(crate) proxy: Option<&'a str>, + pub(crate) accept_invalid_certs: bool, + pub(crate) viewport: Viewport, + pub(crate) user_agent: Option<&'a str>, +} + +/// Builds the common Chrome configuration for all browser-backed audits. +pub(crate) fn build_browser_config( + options: BrowserLaunchOptions<'_>, +) -> Result { + let mut builder = BrowserConfig::builder() + .chrome_executable(options.chrome) + .user_data_dir(options.profile_dir); + if !options.accept_invalid_certs { + builder = builder.respect_https_errors(); + } + if let Some(proxy) = options.proxy { + let endpoint = if proxy.contains("://") { + proxy.to_string() + } else { + format!("http://{proxy}") + }; + builder = builder + .arg(("proxy-server", endpoint.as_str())) + .arg(("proxy-bypass-list", "<-loopback>")); + } + builder = if options.headful { + builder.with_head() + } else { + builder.new_headless_mode() + }; + builder = builder + .window_size(options.viewport.width, options.viewport.height) + .viewport(options.viewport); + if let Some(user_agent) = options.user_agent { + builder = builder.arg(("user-agent", user_agent)); + } + builder + .build() + .map_err(|error| format!("failed to build browser config: {error}")) +} + +fn browser_profile(profile: BrowserProfile) -> (Viewport, Option<&'static str>) { + match profile { + BrowserProfile::Desktop => ( + Viewport { + width: 1280, + height: 800, + device_scale_factor: Some(1.0), + emulating_mobile: false, + is_landscape: true, + has_touch: false, + }, + None, + ), + BrowserProfile::Mobile => ( + Viewport { + width: 390, + height: 844, + device_scale_factor: Some(3.0), + emulating_mobile: true, + is_landscape: false, + has_touch: true, + }, + Some( + "Mozilla/5.0 (iPhone; CPU iPhone OS 17_0 like Mac OS X) \ + AppleWebKit/605.1.15 (KHTML, like Gecko) Version/17.0 Mobile/15E148 Safari/604.1", + ), + ), + } +} + +/// Resolves the Chrome/Chromium executable to launch. +/// +/// Precedence: explicit `--chrome` override, then the `CHROME` environment +/// variable, then auto-detection on `PATH` and standard install locations. +pub(crate) fn resolve_chrome( + override_path: Option<&std::path::Path>, +) -> Result { + if let Some(path) = override_path { + return if path.is_file() { + Ok(path.to_path_buf()) + } else { + Err(format!( + "--chrome path does not point to a file: {}", + path.display() + )) + }; + } + if let Ok(env_path) = std::env::var("CHROME") { + let path = std::path::PathBuf::from(&env_path); + return if path.is_file() { + Ok(path) + } else { + Err(format!("CHROME={env_path} does not point to a file")) + }; + } + find_chrome() +} + +/// Builds a host-only cookie that applies to every path on `url`'s host. +/// +/// Scoped by origin rather than by the full URL: only the origin is load-bearing +/// for a host-only cookie, and a full URL would carry the path, query, and any +/// `user:password@` into CDP and into this function's error message. +pub(crate) fn host_cookie(name: &str, value: &str, url: &url::Url) -> Result { + let origin = url.origin(); + if !origin.is_tuple() { + return Err(format!( + "cannot scope cookie `{name}` because the audited URL has no host" + )); + } + let mut cookie = CookieParam::new(name.to_string(), value.to_string()); + cookie.url = Some(origin.ascii_serialization()); + cookie.path = Some("/".to_string()); + cookie.secure = Some(url.scheme() == "https"); + Ok(cookie) +} + +fn format_cookie_install_error(name: &str, _error: impl std::fmt::Display) -> String { + // Do not forward the CDP error: a browser implementation may include the + // rejected cookie value in its diagnostic. + format!("failed to set cookie `{name}`") +} + +/// Installs host-only, root-scoped cookies before a page has an origin. +pub(crate) async fn set_browser_cookies( + browser: &Browser, + cookies: &[(String, String)], + url: &url::Url, +) -> Result<(), String> { + for (name, value) in cookies { + let cookie = host_cookie(name, value, url)?; + browser + .set_cookies(vec![cookie]) + .await + .map_err(|error| format_cookie_install_error(name, error))?; + } + Ok(()) +} + +/// Auto-detects a Chrome/Chromium executable. +/// +/// Searches `PATH` by common names first, then well-known per-OS install +/// locations (e.g. the macOS `.app` bundle, which is not on `PATH`). +fn find_chrome() -> Result { + if let Some(path) = CHROME_NAMES.iter().find_map(|name| which::which(name).ok()) { + return Ok(path); + } + if let Some(path) = well_known_chrome_paths() + .into_iter() + .find(|path| path.is_file()) + { + return Ok(path); + } + Err(format!( + "could not find Chrome/Chromium on PATH or in standard install locations (looked for: {})", + CHROME_NAMES.join(", ") + )) +} + +/// Well-known absolute Chrome/Chromium install locations for the host OS. +fn well_known_chrome_paths() -> Vec { + let mut paths = Vec::new(); + + #[cfg(target_os = "macos")] + { + const APPS: &[&str] = &[ + "Google Chrome.app/Contents/MacOS/Google Chrome", + "Google Chrome Canary.app/Contents/MacOS/Google Chrome Canary", + "Chromium.app/Contents/MacOS/Chromium", + ]; + for app in APPS { + paths.push(std::path::PathBuf::from(format!("/Applications/{app}"))); + if let Ok(home) = std::env::var("HOME") { + paths.push(std::path::PathBuf::from(format!( + "{home}/Applications/{app}" + ))); + } + } + } + + #[cfg(target_os = "linux")] + { + for path in [ + "/usr/bin/google-chrome", + "/usr/bin/google-chrome-stable", + "/usr/bin/chromium", + "/usr/bin/chromium-browser", + "/snap/bin/chromium", + ] { + paths.push(std::path::PathBuf::from(path)); + } + } + + #[cfg(target_os = "windows")] + { + for path in [ + r"C:\Program Files\Google\Chrome\Application\chrome.exe", + r"C:\Program Files (x86)\Google\Chrome\Application\chrome.exe", + ] { + paths.push(std::path::PathBuf::from(path)); + } + } + + paths +} + +impl AuditCollector for BrowserCollector { + fn collect_page(&self, request: BrowserCollectRequest) -> Result { + self.collect_pages(std::slice::from_ref(&request)) + .into_iter() + .next() + .expect("should return one result for one browser request") + } + + fn collect_pages( + &self, + requests: &[BrowserCollectRequest], + ) -> Vec> { + if requests.is_empty() { + return Vec::new(); + } + // HTTP(S) scheme is enforced by the CLI value parser before we get here. + let chrome = match resolve_chrome(self.chrome.as_deref()) { + Ok(chrome) => chrome, + Err(error) => return vec![Err(error); requests.len()], + }; + let profile = match tempfile::tempdir() { + Ok(profile) => profile, + Err(error) => { + let error = format!("failed to create browser profile dir: {error}"); + return vec![Err(error); requests.len()]; + } + }; + + let runtime = match tokio::runtime::Builder::new_current_thread() + .enable_all() + .build() + { + Ok(runtime) => runtime, + Err(error) => { + let error = format!("failed to build browser runtime: {error}"); + return vec![Err(error); requests.len()]; + } + }; + + let settle = SettleConfig { + quiet: self.settle_quiet, + max: self.settle_max, + }; + + let accept_invalid_certs = self.accept_invalid_certs; + let headful = self.headful; + let assume_consent = self.assume_consent; + let proxy = self.proxy.clone(); + let browser_profile = self.profile; + let request_count = requests.len(); + let requests = requests.to_vec(); + let result = runtime.block_on(async move { + let options = BrowserSessionOptions { + chrome: &chrome, + profile_dir: profile.path(), + settle, + accept_invalid_certs, + headful, + assume_consent, + proxy: proxy.as_deref(), + profile: browser_profile, + }; + collect(requests, &options).await + }); + match result { + Ok(results) => results, + Err(error) => vec![Err(error); request_count], + } + } +} + +/// Drives a single page collection on the current-thread runtime. +async fn collect( + requests: Vec, + options: &BrowserSessionOptions<'_>, +) -> Result>, String> { + // chromiumoxide defaults to ignoring TLS errors. The audit sends + // operator-supplied session cookies and treats what it reads back as + // verification evidence, so a certificate-invalid impersonator could both + // harvest the session and fabricate the evidence. Validate certificates + // unless the operator explicitly opts out. + let (viewport, user_agent) = browser_profile(options.profile); + let config = build_browser_config(BrowserLaunchOptions { + chrome: options.chrome, + profile_dir: options.profile_dir, + headful: options.headful, + proxy: options.proxy, + accept_invalid_certs: options.accept_invalid_certs, + viewport, + user_agent, + })?; + + let (mut browser, mut handler) = Browser::launch(config) + .await + .map_err(|error| format!("failed to launch browser: {error}"))?; + + // Drive the CDP event loop for the duration of the session. + let handler_task = tokio::spawn(async move { while handler.next().await.is_some() {} }); + + let mut results = Vec::with_capacity(requests.len()); + for request in requests { + results.push( + collect_with_browser(&browser, request, options.settle, options.assume_consent).await, + ); + } + + // Best-effort teardown; ignore errors since we already have a result, but + // bound it so a Chrome that ignores `close` cannot hang the command. + let _ = tokio::time::timeout(BROWSER_CLOSE_TIMEOUT, browser.close()).await; + let _ = tokio::time::timeout(BROWSER_CLOSE_TIMEOUT, browser.wait()).await; + handler_task.abort(); + + Ok(results) +} + +async fn collect_with_browser( + browser: &Browser, + request: BrowserCollectRequest, + settle_config: SettleConfig, + assume_consent: bool, +) -> Result { + set_browser_cookies(browser, &request.cookies, &request.url).await?; + + // Open a blank page first so init scripts are installed before the real + // document loads (evaluate-on-new-document applies to subsequent navigations). + let page = browser + .new_page("about:blank") + .await + .map_err(|error| format!("failed to open browser page: {error}"))?; + + let result = collect_open_page(&page, &request, settle_config, assume_consent).await; + let close_result = tokio::time::timeout(BROWSER_CLOSE_TIMEOUT, page.close()).await; + + match (result, close_result) { + (Err(error), _) => Err(error), + (Ok(mut collected), Err(_)) => { + collected.warnings.push(Warning { + code: "page_close_timeout".to_string(), + message: "timed out closing the browser tab after collection".to_string(), + }); + Ok(collected) + } + (Ok(mut collected), Ok(Err(error))) => { + collected.warnings.push(Warning { + code: "page_close_failed".to_string(), + message: format!("failed to close the browser tab after collection: {error}"), + }); + Ok(collected) + } + (Ok(collected), Ok(Ok(_))) => Ok(collected), + } +} + +/// Collects from an open tab. The caller owns tab teardown so every return path, +/// including an error from this function, closes the page before continuing. +async fn collect_open_page( + page: &Page, + request: &BrowserCollectRequest, + settle_config: SettleConfig, + assume_consent: bool, +) -> Result { + let mut warnings = Vec::new(); + + if assume_consent { + page.evaluate_on_new_document(CONSENT_STUB_SCRIPT) + .await + .map_err(|error| format!("failed to install consent init script: {error}"))?; + warnings.push(Warning { + code: "consent_stub_active".to_string(), + message: "audit consent APIs were stubbed; re-run with --no-assume-consent to observe the publisher CMP without substitution".to_string(), + }); + } + page.evaluate_on_new_document("performance.setResourceTimingBufferSize(100000)") + .await + .map_err(|error| format!("failed to increase resource timing buffer: {error}"))?; + + for script in &request.init_scripts { + page.evaluate_on_new_document(script.clone()) + .await + .map_err(|error| format!("failed to install init script: {error}"))?; + } + + tokio::time::timeout(NAVIGATION_TIMEOUT, page.goto(request.url.as_str())) + .await + .map_err(|_| format!("navigation to {} timed out", request.url))? + .map_err(|error| format!("failed to navigate to {}: {error}", request.url))?; + match tokio::time::timeout(NAVIGATION_TIMEOUT, page.wait_for_navigation()).await { + Ok(Ok(_)) => {} + Ok(Err(error)) => warnings.push(Warning { + code: "navigation_wait_failed".to_string(), + message: format!( + "navigation load event could not be read ({error}); continuing with settled page evidence" + ), + }), + Err(_) => warnings.push(Warning { + code: "navigation_wait_timeout".to_string(), + message: format!( + "navigation did not fire its load event within {} seconds; continuing with settled page evidence", + NAVIGATION_TIMEOUT.as_secs() + ), + }), + } + + settle(page, settle_config, &mut warnings).await; + + if request.scroll { + if request.collect_ad_evidence { + // Snapshot evidence before scrolling so entries already present at + // initial load keep phase "load"; the store dedups first-seen, so + // the post-scroll scrape only adds genuinely scroll-phase entries. + if tokio::time::timeout( + CDP_OPERATION_TIMEOUT, + page.evaluate( + "(typeof window.__tsCollectAdTemplateEvidence === 'function' \ + && window.__tsCollectAdTemplateEvidence(), null)", + ), + ) + .await + .is_err() + { + warnings.push(Warning { + code: "ad_evidence_snapshot_timeout".to_string(), + message: "timed out snapshotting ad evidence before scroll".to_string(), + }); + } + } + // Mark subsequent observations as scroll-phase for the verifier's + // injected evidence collector before shared scrolling begins. + eval_discard(page, "window.__tsScrollPhase = true", &mut warnings).await; + warnings.extend( + browser_scroll::scroll_page(page) + .await + .into_iter() + .map(|failure| Warning { + code: failure.code().to_string(), + message: failure.to_string(), + }), + ); + settle(page, settle_config, &mut warnings).await; + } + + let final_url_text = tokio::time::timeout(CDP_OPERATION_TIMEOUT, page.url()) + .await + .map_err(|_| "timed out reading final page URL".to_string())? + .map_err(|error| format!("failed to read final page URL: {error}"))? + .ok_or_else(|| "browser page URL was empty after navigation".to_string())?; + let final_url = url::Url::parse(&final_url_text).map_err(|error| { + format!("browser returned invalid final URL `{final_url_text}`: {error}") + })?; + let title = match tokio::time::timeout(CDP_OPERATION_TIMEOUT, page.get_title()).await { + Ok(Ok(title)) => title.unwrap_or_default(), + Ok(Err(error)) => { + warnings.push(Warning { + code: "page_title_failed".to_string(), + message: format!("failed to read page title: {error}"), + }); + String::new() + } + Err(_) => { + warnings.push(Warning { + code: "page_title_timeout".to_string(), + message: "timed out reading page title".to_string(), + }); + String::new() + } + }; + let script_count = eval_usize(page, "document.querySelectorAll('script').length") + .await + .unwrap_or_else(|message| { + warnings.push(Warning { + code: "script_count_failed".to_string(), + message, + }); + 0 + }); + let resource_count = resource_count(page).await.unwrap_or_else(|message| { + warnings.push(Warning { + code: "resource_count_failed".to_string(), + message, + }); + 0 + }); + + if resource_count >= 250 { + warnings.push(Warning { + code: "resource_timing_heavy".to_string(), + message: format!("page recorded {resource_count} network resources"), + }); + } + + if let Ok(Ok(frames)) = tokio::time::timeout(CDP_OPERATION_TIMEOUT, page.frames()).await + && frames.len() > 1 + { + warnings.push(Warning { + code: "child_frames_not_inspected".to_string(), + message: format!( + "ad-template evidence inspected only the main frame; {} child frame(s) were present", + frames.len() - 1 + ), + }); + } + + let ad_evidence = if request.collect_ad_evidence { + extract_ad_evidence(page, &mut warnings).await + } else { + None + }; + + Ok(CollectedPage { + final_url, + title, + script_count, + resource_count, + warnings, + ad_evidence, + }) +} + +/// Waits for the page network to go quiet after navigation or scroll. +/// +/// Polls the resource-entry count and returns once it stays unchanged for a +/// quiet window, or when the hard cap elapses — so ad-heavy pages finish loading +/// before evidence is read, without hanging on pages that never go idle. +async fn settle(page: &Page, config: SettleConfig, warnings: &mut Vec) { + let start = std::time::Instant::now(); + let mut last = None; + let mut quiet_since = None; + + loop { + if start.elapsed() >= config.max { + warnings.push(Warning { + code: "settle_timeout".to_string(), + message: "page did not settle before the configured maximum wait".to_string(), + }); + return; + } + + let ready_state = match eval_string(page, "document.readyState").await { + Ok(state) => state, + Err(message) => { + warnings.push(Warning { + code: "settle_read_failed".to_string(), + message, + }); + return; + } + }; + let current = match resource_count(page).await { + Ok(count) => count, + Err(message) => { + warnings.push(Warning { + code: "settle_read_failed".to_string(), + message, + }); + return; + } + }; + let ready = matches!(ready_state.as_str(), "interactive" | "complete"); + if ready && last == Some(current) { + let quiet_start = quiet_since.get_or_insert_with(std::time::Instant::now); + if quiet_start.elapsed() >= config.quiet { + return; + } + } else { + quiet_since = None; + } + last = Some(current); + + let remaining_max = config.max.saturating_sub(start.elapsed()); + let remaining_quiet = quiet_since + .map(|quiet_start| config.quiet.saturating_sub(quiet_start.elapsed())) + .unwrap_or(config.quiet); + let sleep_for = Duration::from_millis(SETTLE_POLL_MS) + .min(remaining_max) + .min(remaining_quiet.max(Duration::from_millis(1))); + tokio::time::sleep(sleep_for).await; + } +} + +/// Reads the number of resource timing entries observed so far. +async fn resource_count(page: &Page) -> Result { + eval_usize(page, "performance.getEntriesByType('resource').length").await +} + +async fn eval_discard(page: &Page, expression: impl Into, warnings: &mut Vec) { + if let Err(failure) = browser_scroll::evaluate(page, expression).await { + warnings.push(Warning { + code: failure.code().to_string(), + message: failure.to_string(), + }); + } +} + +async fn eval_usize(page: &Page, expression: &str) -> Result { + tokio::time::timeout(CDP_OPERATION_TIMEOUT, page.evaluate(expression)) + .await + .map_err(|_| format!("timed out evaluating `{expression}`"))? + .map_err(|error| format!("failed to evaluate `{expression}`: {error}"))? + .into_value::() + .map_err(|error| format!("failed to decode `{expression}`: {error}")) +} + +async fn eval_string(page: &Page, expression: &str) -> Result { + tokio::time::timeout(CDP_OPERATION_TIMEOUT, page.evaluate(expression)) + .await + .map_err(|_| format!("timed out evaluating `{expression}`"))? + .map_err(|error| format!("failed to evaluate `{expression}`: {error}"))? + .into_value::() + .map_err(|error| format!("failed to decode `{expression}`: {error}")) +} + +/// Reads and decodes `window.__tsAdTemplateEvidence`, warning (not failing) on a +/// decode error. +async fn extract_ad_evidence( + page: &Page, + warnings: &mut Vec, +) -> Option { + // Serialize and size-check in the page so a hostile publisher-controlled + // evidence object cannot force an unbounded CDP response and Rust decode. + let evaluation = tokio::time::timeout( + CDP_OPERATION_TIMEOUT, + page.evaluate(format!( + r#"(() => {{ + const evidence = typeof window.__tsCollectAdTemplateEvidence === 'function' + ? window.__tsCollectAdTemplateEvidence() + : (window.__tsAdTemplateEvidence || null) + if (evidence === null) return {{ kind: 'absent' }} + try {{ + const json = JSON.stringify(evidence) + const bytes = new TextEncoder().encode(json).byteLength + if (bytes > {MAX_EVIDENCE_PAYLOAD_BYTES}) return {{ kind: 'too_large' }} + return {{ kind: 'evidence', json }} + }} catch (error) {{ + return {{ + kind: 'serialization_failed', + message: String(error).slice(0, 512), + }} + }} + }})()"# + )), + ) + .await; + + let envelope = match evaluation { + Ok(Ok(result)) => match result.into_value::() { + Ok(envelope) => Some(envelope), + Err(error) => { + warnings.push(Warning { + code: "ad_evidence_decode_failed".to_string(), + message: format!("failed to decode ad-template evidence envelope: {error}"), + }); + return None; + } + }, + Ok(Err(error)) => { + warnings.push(Warning { + code: "ad_evidence_read_failed".to_string(), + message: format!("failed to read ad-template evidence: {error}"), + }); + return None; + } + Err(_) => { + warnings.push(Warning { + code: "ad_evidence_read_timeout".to_string(), + message: "timed out reading ad-template evidence".to_string(), + }); + return None; + } + }; + + match envelope { + Some(envelope) => decode_ad_evidence_envelope(envelope, warnings), + None => { + warnings.push(Warning { + code: "ad_evidence_absent".to_string(), + message: "no ad-template evidence was collected from the page".to_string(), + }); + None + } + } +} + +#[derive(Debug, serde::Deserialize)] +#[serde(tag = "kind", rename_all = "snake_case")] +enum EvidenceEnvelope { + Absent, + TooLarge, + Evidence { json: String }, + SerializationFailed { message: String }, +} + +fn decode_ad_evidence_envelope( + envelope: EvidenceEnvelope, + warnings: &mut Vec, +) -> Option { + match envelope { + EvidenceEnvelope::Absent => { + warnings.push(Warning { + code: "ad_evidence_absent".to_string(), + message: "no ad-template evidence was collected from the page".to_string(), + }); + None + } + EvidenceEnvelope::TooLarge => { + warnings.push(Warning { + code: "ad_evidence_too_large".to_string(), + message: format!( + "ad-template evidence exceeded the {MAX_EVIDENCE_PAYLOAD_BYTES}-byte limit" + ), + }); + None + } + EvidenceEnvelope::SerializationFailed { message } => { + warnings.push(Warning { + code: "ad_evidence_encode_failed".to_string(), + message: format!("failed to serialize ad-template evidence in the page: {message}"), + }); + None + } + EvidenceEnvelope::Evidence { json } => { + match serde_json::from_str::(&json) { + Ok(mut evidence) => { + // Defense in depth: the injected script caps these lists, but the + // page owns that store, so re-cap after decode. + evidence.dom_ids.truncate(MAX_EVIDENCE_ENTRIES); + evidence.gpt_slots.truncate(MAX_EVIDENCE_ENTRIES); + evidence.aps_calls.truncate(MAX_EVIDENCE_ENTRIES); + evidence.warnings.truncate(MAX_EVIDENCE_ENTRIES); + Some(evidence) + } + Err(error) => { + warnings.push(Warning { + code: "ad_evidence_decode_failed".to_string(), + message: format!("failed to decode ad-template evidence: {error}"), + }); + None + } + } + } + } +} + +/// Whether a Chrome/Chromium fixture is available for browser-backed tests. +/// +/// Skips optional local runs, but makes the scripted/CI contract fail loudly. +/// Shared with the generation collector's tests so the contract has one +/// definition. +#[cfg(test)] +pub(crate) fn browser_fixture_available() -> bool { + if resolve_chrome(None).is_ok() { + return true; + } + assert!( + std::env::var_os("TS_AUDIT_BROWSER_TESTS").is_none(), + "TS_AUDIT_BROWSER_TESTS requires Chrome/Chromium; set CHROME to its executable" + ); + false +} + +#[cfg(test)] +mod tests { + use std::io::{Read as _, Write as _}; + use std::net::TcpListener; + use std::sync::mpsc; + + use super::*; + use crate::commands::audit::collector::{ + AdTemplateCollectorConfig, build_ad_template_init_script, + }; + + const AD_TEMPLATE_COLLECTOR_JS: &str = include_str!("ad_template_collector.js"); + + #[test] + fn rust_and_javascript_evidence_entry_caps_match() { + // Parse the declared value rather than matching the whole line, so JS + // punctuation or spacing cannot false-alarm on a still-correct cap. + let declared = AD_TEMPLATE_COLLECTOR_JS + .lines() + .find_map(|line| line.trim().strip_prefix("const __ts_max_entries =")) + .and_then(|value| value.trim().trim_end_matches(';').parse::().ok()) + .expect("should declare __ts_max_entries in the collector script"); + + assert_eq!( + declared, MAX_EVIDENCE_ENTRIES, + "should keep the JS cap equal to MAX_EVIDENCE_ENTRIES" + ); + } + + #[test] + fn well_known_chrome_paths_are_known_for_this_os() { + // macOS/Linux/Windows each have candidate paths; guards the cfg branches. + assert!( + !well_known_chrome_paths().is_empty(), + "supported OSes should list candidate Chrome install paths" + ); + } + + #[test] + fn oversized_ad_evidence_is_an_explicit_warning() { + let mut warnings = Vec::new(); + let evidence = decode_ad_evidence_envelope(EvidenceEnvelope::TooLarge, &mut warnings); + + assert!(evidence.is_none()); + assert_eq!(warnings.len(), 1); + assert_eq!(warnings[0].code, "ad_evidence_too_large"); + } + + #[test] + fn supplied_cookie_is_host_only_and_root_scoped() { + let url = + url::Url::parse("https://publisher.example/news/story").expect("should parse test URL"); + let cookie = host_cookie("clearance", "token", &url).expect("should build cookie"); + + assert!(cookie.domain.is_none(), "host-only cookies omit Domain"); + assert_eq!(cookie.path.as_deref(), Some("/")); + assert_eq!( + cookie.url.as_deref(), + Some("https://publisher.example"), + "the origin scopes a host-only cookie before first navigation" + ); + assert_eq!(cookie.secure, Some(true), "HTTPS cookies must be Secure"); + } + + #[test] + fn cookie_install_error_identifies_name_without_a_value() { + let error = format_cookie_install_error( + "datadome", + "invalid cookie value operator-secret-cookie-value", + ); + + assert_eq!(error, "failed to set cookie `datadome`"); + assert!(!error.contains("operator-secret-cookie-value")); + } + + #[test] + #[ignore = "requires local Chrome/Chromium; run through scripts/test-cli.sh"] + fn supplied_cookie_reaches_first_navigation() { + if !browser_fixture_available() { + return; + } + + let listener = TcpListener::bind("127.0.0.1:0").expect("should bind fixture server"); + let address = listener.local_addr().expect("should read fixture address"); + let (request_tx, request_rx) = mpsc::channel(); + std::thread::spawn(move || { + let (mut stream, _) = listener.accept().expect("should accept browser request"); + stream + .set_read_timeout(Some(Duration::from_secs(10))) + .expect("should set fixture read timeout"); + let mut request = Vec::new(); + while !request.ends_with(b"\r\n\r\n") { + let mut chunk = [0_u8; 1024]; + let chunk_len = stream.read(&mut chunk).expect("should read HTTP request"); + assert!(chunk_len > 0, "request should contain complete headers"); + request.extend_from_slice(&chunk[..chunk_len]); + assert!( + request.len() <= 16 * 1024, + "request headers should be bounded" + ); + } + request_tx + .send(String::from_utf8_lossy(&request).into_owned()) + .expect("should send captured request"); + + let body = b"cookie fixture"; + write!( + stream, + "HTTP/1.1 200 OK\r\nContent-Type: text/html\r\nContent-Length: {}\r\nConnection: close\r\n\r\n", + body.len() + ) + .expect("should write fixture headers"); + stream.write_all(body).expect("should write fixture body"); + }); + + let collector = BrowserCollector { + settle_quiet: Duration::from_millis(100), + settle_max: Duration::from_secs(1), + ..BrowserCollector::new() + }; + collector + .collect_page(BrowserCollectRequest { + url: url::Url::parse(&format!("http://{address}/")) + .expect("should parse fixture URL"), + init_scripts: Vec::new(), + scroll: false, + collect_ad_evidence: false, + cookies: vec![("clearance".to_string(), "token".to_string())], + }) + .expect("cookie should be installed before first navigation"); + + let request = request_rx + .recv_timeout(Duration::from_secs(5)) + .expect("fixture should receive the first navigation"); + assert!( + request.lines().any(|line| { + line.split_once(':').is_some_and(|(name, value)| { + name.eq_ignore_ascii_case("cookie") + && value + .trim() + .split(';') + .any(|cookie| cookie.trim() == "clearance=token") + }) + }), + "first navigation should carry the supplied cookie; request was {request:?}" + ); + } + + /// A self-contained page that stubs just enough of GPT (no network) for the + /// collector to observe a defined slot via the wrapped `defineSlot` and the + /// `getSlots()` scrape. + const GPT_FIXTURE: &str = r#" + + + +
+ + + +"#; + + #[test] + #[ignore = "requires local Chrome/Chromium; run through scripts/test-cli.sh"] + fn collects_gpt_slot_from_local_fixture() { + if !browser_fixture_available() { + // Browser fixture test requires a local Chrome/Chromium; skipping. + return; + } + let mut fixture = tempfile::Builder::new() + .suffix(".html") + .tempfile() + .expect("should create fixture file"); + fixture + .write_all(GPT_FIXTURE.as_bytes()) + .expect("should write fixture"); + let url = url::Url::from_file_path(fixture.path()).expect("should build file url"); + + let script = build_ad_template_init_script(&AdTemplateCollectorConfig { + div_prefixes: vec!["ad-atf-".to_string()], + }) + .expect("should build init script"); + + let collector = BrowserCollector::new(); + let page = collector + .collect_page(BrowserCollectRequest { + url, + init_scripts: vec![script], + scroll: false, + collect_ad_evidence: true, + cookies: Vec::new(), + }) + .expect("should collect fixture page"); + + let evidence = page.ad_evidence.expect("fixture should yield ad evidence"); + assert!( + evidence + .gpt_slots + .iter() + .any(|slot| slot.gam_unit_path == "/123/news/atf"), + "should capture the defined GPT slot" + ); + assert!( + evidence.dom_ids.iter().any(|dom| dom.dom_id == "ad-atf-0"), + "should capture the configured-prefix DOM id" + ); + } + + #[test] + #[ignore = "requires local Chrome/Chromium; run through scripts/test-cli.sh"] + fn scroll_pass_keeps_initial_load_phase_for_load_time_evidence() { + if !browser_fixture_available() { + // Browser fixture test requires a local Chrome/Chromium; skipping. + return; + } + let mut fixture = tempfile::Builder::new() + .suffix(".html") + .tempfile() + .expect("should create fixture file"); + fixture + .write_all(GPT_FIXTURE.as_bytes()) + .expect("should write fixture"); + let url = url::Url::from_file_path(fixture.path()).expect("should build file url"); + + let script = build_ad_template_init_script(&AdTemplateCollectorConfig { + div_prefixes: vec!["ad-atf-".to_string()], + }) + .expect("should build init script"); + + let collector = BrowserCollector::new(); + let page = collector + .collect_page(BrowserCollectRequest { + url, + init_scripts: vec![script], + scroll: true, + collect_ad_evidence: true, + cookies: Vec::new(), + }) + .expect("should collect fixture page"); + + // The slot and DOM id exist at load time, so the pre-scroll snapshot + // must record them as initial-load even though a scroll pass ran. + let evidence = page.ad_evidence.expect("fixture should yield ad evidence"); + assert!( + evidence.dom_ids.iter().any(|dom| dom.dom_id == "ad-atf-0" + && dom.phase == crate::ad_templates::compare::EvidencePhase::InitialLoad), + "load-time DOM id should keep phase initial_load under --scroll" + ); + assert!( + evidence.gpt_slots.iter().any(|slot| { + slot.gam_unit_path == "/123/news/atf" + && slot.phase == crate::ad_templates::compare::EvidencePhase::InitialLoad + }), + "load-time GPT slot should keep phase initial_load under --scroll" + ); + } +} diff --git a/crates/trusted-server-cli/src/commands/audit/browser_collector.rs b/crates/trusted-server-cli/src/commands/audit/browser_collector.rs deleted file mode 100644 index 87a2ccc2c..000000000 --- a/crates/trusted-server-cli/src/commands/audit/browser_collector.rs +++ /dev/null @@ -1,435 +0,0 @@ -use std::path::{Path, PathBuf}; -use std::time::Duration; - -use chromiumoxide::ArcHttpRequest; -use chromiumoxide::browser::{Browser, BrowserConfig}; -use futures::StreamExt as _; -use serde::Deserialize; -use tempfile::TempDir; -use tokio::runtime::Builder; -use tokio::time::{sleep, timeout}; -use url::Url; -use which::which; - -use crate::commands::audit::collector::{ - AuditCollector, CollectedPage, CollectedRequest, CollectedScriptTag, -}; -use crate::error::{CliResult, report_error}; - -const SETTLE_QUIET_PERIOD: Duration = Duration::from_millis(750); -const SETTLE_POLL_INTERVAL: Duration = Duration::from_millis(250); -const SETTLE_MAX_WAIT: Duration = Duration::from_secs(6); -const NAVIGATION_TIMEOUT: Duration = Duration::from_secs(30); -const BROWSER_CLOSE_TIMEOUT: Duration = Duration::from_secs(5); -const RESOURCE_TIMING_BUFFER_WARNING_THRESHOLD: usize = 250; -const RESOURCE_TIMING_BUFFER_WARNING: &str = - "browser resource timing buffer reached its default size; some network assets may be missing"; - -#[derive(Default)] -pub(crate) struct BrowserAuditCollector; - -impl AuditCollector for BrowserAuditCollector { - fn collect_page(&self, target_url: &Url) -> CliResult { - let runtime = Builder::new_current_thread() - .enable_all() - .build() - .map_err(|error| { - report_error(format!( - "failed to build Tokio runtime for browser audit: {error}" - )) - })?; - - runtime.block_on(collect_page_via_browser_async(target_url)) - } -} - -async fn collect_page_via_browser_async(target_url: &Url) -> CliResult { - let chrome_executable = find_browser_executable()?; - let user_data_dir = TempDir::new().map_err(|error| { - report_error(format!( - "failed to create temporary browser profile for audit: {error}" - )) - })?; - let config = BrowserConfig::builder() - .chrome_executable(chrome_executable) - .user_data_dir(user_data_dir.path()) - .new_headless_mode() - .build() - .map_err(|error| { - report_error(format!( - "failed to build Chromium configuration for audit: {error}" - )) - })?; - - let (mut browser, mut handler) = Browser::launch(config).await.map_err(|error| { - report_error(format!( - "failed to launch Chrome/Chromium for audit: {error}" - )) - })?; - - let handler_task = tokio::spawn(async move { - while let Some(event) = handler.next().await { - if event.is_err() { - break; - } - } - }); - - let result = collect_page_from_browser(&mut browser, target_url).await; - - let close_result = timeout(BROWSER_CLOSE_TIMEOUT, browser.close()) - .await - .map_err(|_| report_error("timed out closing browser after audit")) - .and_then(|result| { - result.map_err(|error| { - report_error(format!("failed to close browser after audit: {error}")) - }) - }); - if close_result.is_err() { - handler_task.abort(); - } - let _ = handler_task.await; - - match (result, close_result) { - (Ok(collected), Ok(_)) => Ok(collected), - (Ok(_), Err(error)) | (Err(error), _) => Err(error), - } -} - -async fn collect_page_from_browser( - browser: &mut Browser, - target_url: &Url, -) -> CliResult { - let page = browser.new_page("about:blank").await.map_err(|error| { - report_error(format!("failed to create browser page for audit: {error}")) - })?; - - timeout(NAVIGATION_TIMEOUT, page.goto(target_url.as_str())) - .await - .map_err(|_| report_error(format!("timed out navigating to `{target_url}`")))? - .map_err(|error| report_error(format!("failed to navigate to `{target_url}`: {error}")))?; - - let navigation_response = timeout(NAVIGATION_TIMEOUT, page.wait_for_navigation_response()) - .await - .map_err(|_| { - report_error(format!( - "timed out waiting for main document navigation response from `{target_url}`" - )) - })? - .map_err(|error| { - report_error(format!( - "failed to read main document navigation response: {error}" - )) - })?; - - let mut warnings = Vec::new(); - if let Some(warning) = validate_navigation_response(navigation_response)? { - warnings.push(warning); - } - if !wait_for_page_settle(&page).await? { - warnings.push( - "browser audit timed out while waiting for the page to settle; results may be partial" - .to_string(), - ); - } - - let final_url = page - .url() - .await - .map_err(|error| report_error(format!("failed to read final page URL: {error}")))? - .ok_or_else(|| report_error("browser page URL was empty after navigation"))?; - let page_title = page - .get_title() - .await - .map_err(|error| report_error(format!("failed to read page title: {error}")))?; - let html = page - .content() - .await - .map_err(|error| report_error(format!("failed to read rendered page HTML: {error}")))?; - - let script_tags: Vec = page - .evaluate( - r#"() => Array.from(document.scripts).map((script) => ({ - src: script.src || null, - inline_text: script.src ? null : (script.textContent || null), - }))"#, - ) - .await - .map_err(|error| report_error(format!("failed to read rendered script tags: {error}")))? - .into_value() - .map_err(|error| { - report_error(format!( - "failed to decode rendered script tag data: {error}" - )) - })?; - - let network_requests: Vec = page - .evaluate( - r#"() => performance.getEntriesByType('resource').map((entry) => ({ - url: entry.name, - initiator_type: entry.initiatorType || null, - }))"#, - ) - .await - .map_err(|error| { - report_error(format!( - "failed to read browser performance resource entries: {error}" - )) - })? - .into_value() - .map_err(|error| { - report_error(format!( - "failed to decode browser performance resource data: {error}" - )) - })?; - - if let Some(warning) = resource_timing_buffer_warning(network_requests.len()) { - warnings.push(warning.to_string()); - } - - Ok(CollectedPage { - requested_url: target_url.to_string(), - final_url, - page_title: page_title.filter(|title| !title.trim().is_empty()), - html, - script_tags: script_tags - .into_iter() - .map(|script| CollectedScriptTag { - src: script.src, - inline_text: script.inline_text.filter(|text| !text.trim().is_empty()), - }) - .collect(), - network_requests: network_requests - .into_iter() - .map(|entry| CollectedRequest { - url: entry.url, - resource_type: entry.initiator_type, - }) - .collect(), - warnings, - }) -} - -async fn wait_for_page_settle(page: &chromiumoxide::Page) -> CliResult { - let mut elapsed = Duration::ZERO; - let mut previous_count = None; - let mut stable_for = Duration::ZERO; - - while elapsed < SETTLE_MAX_WAIT { - let ready_state: String = page - .evaluate("document.readyState") - .await - .map_err(|error| report_error(format!("failed to read document ready state: {error}")))? - .into_value() - .map_err(|error| { - report_error(format!("failed to decode document ready state: {error}")) - })?; - let resource_count: usize = page - .evaluate("performance.getEntriesByType('resource').length") - .await - .map_err(|error| report_error(format!("failed to read resource count: {error}")))? - .into_value() - .map_err(|error| report_error(format!("failed to decode resource count: {error}")))?; - - if ready_state == "complete" { - if previous_count == Some(resource_count) { - stable_for += SETTLE_POLL_INTERVAL; - } else { - stable_for = Duration::ZERO; - } - - if stable_for >= SETTLE_QUIET_PERIOD { - return Ok(true); - } - } - - previous_count = Some(resource_count); - sleep(SETTLE_POLL_INTERVAL).await; - elapsed += SETTLE_POLL_INTERVAL; - } - - Ok(false) -} - -fn validate_navigation_response(navigation_response: ArcHttpRequest) -> CliResult> { - let request = navigation_response - .ok_or_else(|| report_error("browser audit did not capture the main document response"))?; - - if let Some(failure_text) = &request.failure_text { - return Err(report_error(format!( - "main document request failed: {failure_text}" - ))); - } - - let response = request.response.as_ref().ok_or_else(|| { - report_error("browser audit did not capture the main document HTTP response") - })?; - - if is_successful_navigation_status(response.status) { - return Ok(None); - } - - Ok(Some(format!( - "audit request returned HTTP {} {} for `{}`; results may be partial", - response.status, response.status_text, response.url - ))) -} - -fn is_successful_navigation_status(status: i64) -> bool { - (200..400).contains(&status) -} - -fn resource_timing_buffer_warning(resource_count: usize) -> Option<&'static str> { - (resource_count >= RESOURCE_TIMING_BUFFER_WARNING_THRESHOLD) - .then_some(RESOURCE_TIMING_BUFFER_WARNING) -} - -fn find_browser_executable() -> CliResult { - for candidate in browser_executable_path_candidates() { - if let Ok(path) = which(candidate) { - return Ok(path); - } - } - - for candidate in browser_executable_fallbacks() { - let candidate_path = Path::new(candidate); - if candidate_path.is_file() { - return Ok(candidate_path.to_path_buf()); - } - } - - Err(report_error( - "Chrome/Chromium was not found on PATH or in the standard local install locations checked by `ts audit`. Install a local Chrome or Chromium binary before running `ts audit`.", - )) -} - -fn browser_executable_path_candidates() -> &'static [&'static str] { - &[ - "google-chrome", - "google-chrome-stable", - "chromium", - "chromium-browser", - "chrome", - "Google Chrome", - "Google Chrome for Testing", - ] -} - -fn browser_executable_fallbacks() -> &'static [&'static str] { - #[cfg(target_os = "macos")] - { - &[ - "/Applications/Google Chrome.app/Contents/MacOS/Google Chrome", - "/Applications/Chromium.app/Contents/MacOS/Chromium", - "/Applications/Google Chrome for Testing.app/Contents/MacOS/Google Chrome for Testing", - ] - } - - #[cfg(target_os = "linux")] - { - &[ - "/usr/bin/google-chrome", - "/usr/bin/google-chrome-stable", - "/usr/bin/chromium", - "/usr/bin/chromium-browser", - "/snap/bin/chromium", - ] - } - - #[cfg(not(any(target_os = "macos", target_os = "linux")))] - { - &[] - } -} - -#[derive(Debug, Deserialize)] -struct BrowserScriptTag { - src: Option, - inline_text: Option, -} - -#[derive(Debug, Deserialize)] -struct BrowserPerformanceEntry { - url: String, - initiator_type: Option, -} - -#[cfg(test)] -mod tests { - use std::sync::Arc; - - use chromiumoxide::cdp::browser_protocol::network::{Headers, RequestId, Response}; - use chromiumoxide::cdp::browser_protocol::security::SecurityState; - use chromiumoxide::handler::http::HttpRequest; - - use super::*; - - #[test] - fn successful_navigation_status_allows_redirects_but_rejects_errors() { - assert!(is_successful_navigation_status(200)); - assert!(is_successful_navigation_status(302)); - assert!(is_successful_navigation_status(399)); - assert!(!is_successful_navigation_status(199)); - assert!(!is_successful_navigation_status(400)); - assert!(!is_successful_navigation_status(500)); - } - - #[test] - fn navigation_response_returns_warning_for_http_error_status() { - let warning = - validate_navigation_response(navigation_response_with_status(403, "Forbidden")) - .expect("should validate navigation response") - .expect("should return warning for HTTP error status"); - - assert_eq!( - warning, - "audit request returned HTTP 403 Forbidden for `https://example.com/`; results may be partial", - "should warn and continue when the main document returns an HTTP error" - ); - } - - #[test] - fn resource_timing_buffer_warning_starts_at_threshold() { - assert_eq!( - resource_timing_buffer_warning(RESOURCE_TIMING_BUFFER_WARNING_THRESHOLD - 1), - None, - "should not warn before the resource timing buffer threshold" - ); - assert_eq!( - resource_timing_buffer_warning(RESOURCE_TIMING_BUFFER_WARNING_THRESHOLD), - Some(RESOURCE_TIMING_BUFFER_WARNING), - "should warn when the resource timing buffer reaches the threshold" - ); - } - - #[test] - fn browser_path_candidates_include_common_names() { - let candidates = browser_executable_path_candidates(); - - assert!(candidates.contains(&"google-chrome")); - assert!(candidates.contains(&"chromium")); - assert!(candidates.contains(&"Google Chrome for Testing")); - } - - fn navigation_response_with_status(status: i64, status_text: &str) -> ArcHttpRequest { - let mut request = - HttpRequest::new(RequestId::new("request-1"), None, None, false, Vec::new()); - request.response = Some( - Response::builder() - .url("https://example.com/") - .status(status) - .status_text(status_text) - .headers(Headers::default()) - .mime_type("text/html") - .charset("utf-8") - .connection_reused(false) - .connection_id(1.0) - .encoded_data_length(0.0) - .security_state(SecurityState::Secure) - .build() - .expect("should build navigation response"), - ); - - Some(Arc::new(request)) - } -} diff --git a/crates/trusted-server-cli/src/commands/audit/browser_scroll.rs b/crates/trusted-server-cli/src/commands/audit/browser_scroll.rs new file mode 100644 index 000000000..0663b158b --- /dev/null +++ b/crates/trusted-server-cli/src/commands/audit/browser_scroll.rs @@ -0,0 +1,82 @@ +//! Shared deterministic browser scrolling for audit commands. + +use std::time::Duration; + +use chromiumoxide::Page; + +const SCROLL_STEP_DELAY: Duration = Duration::from_millis(250); +const SCROLL_OPERATION_TIMEOUT: Duration = Duration::from_secs(5); + +/// A best-effort browser scroll operation that could not be completed. +#[derive(Debug, derive_more::Display)] +pub(crate) enum ScrollFailure { + /// Chrome rejected the page evaluation. + #[display("browser page evaluation failed: {_0}")] + Evaluation(String), + /// Chrome did not complete the page evaluation within the operation bound. + #[display("browser page evaluation timed out")] + Timeout, +} + +impl core::error::Error for ScrollFailure {} + +impl ScrollFailure { + /// Stable warning code used by structured audit output. + pub(crate) const fn code(&self) -> &'static str { + match self { + Self::Evaluation(_) => "page_evaluation_failed", + Self::Timeout => "page_evaluation_timeout", + } + } +} + +/// Scrolls a page through deterministic fractions to trigger lazy content. +pub(crate) async fn scroll_page(page: &chromiumoxide::Page) -> Vec { + let mut failures = Vec::new(); + for fraction in ["0.33", "0.66", "1"] { + let script = format!( + "window.scrollTo(0, Math.floor(Math.max(document.body.scrollHeight, \ + document.documentElement.scrollHeight) * {fraction}))" + ); + if let Err(failure) = evaluate(page, script).await { + failures.push(failure); + } + tokio::time::sleep(SCROLL_STEP_DELAY).await; + } + if let Err(failure) = evaluate(page, "window.scrollTo(0, 0)").await { + failures.push(failure); + } + failures +} + +/// Evaluates a browser expression with the shared operation bound and errors. +pub(crate) async fn evaluate( + page: &Page, + expression: impl Into, +) -> Result<(), ScrollFailure> { + tokio::time::timeout(SCROLL_OPERATION_TIMEOUT, page.evaluate(expression.into())) + .await + .map_err(|_| ScrollFailure::Timeout)? + .map(|_| ()) + .map_err(|error| ScrollFailure::Evaluation(error.to_string())) +} + +#[cfg(test)] +mod tests { + use super::ScrollFailure; + + #[test] + fn scroll_failures_have_stable_messages() { + assert_eq!( + ScrollFailure::Evaluation("execution context was destroyed".to_string()).to_string(), + "browser page evaluation failed: execution context was destroyed" + ); + assert_eq!( + ScrollFailure::Timeout.to_string(), + "browser page evaluation timed out" + ); + + fn assert_error() {} + assert_error::(); + } +} diff --git a/crates/trusted-server-cli/src/commands/audit/collector.rs b/crates/trusted-server-cli/src/commands/audit/collector.rs index 314ae54fc..25aa9236f 100644 --- a/crates/trusted-server-cli/src/commands/audit/collector.rs +++ b/crates/trusted-server-cli/src/commands/audit/collector.rs @@ -1,41 +1,350 @@ -use serde::{Deserialize, Serialize}; -use url::Url; +//! Collector abstraction shared by the generic page audit and the ad-template +//! verifier. +//! +//! Decoupling collection behind [`AuditCollector`] lets the verifier orchestration +//! (Task 9) be tested with an in-memory fake collector, with no Chrome dependency. -use crate::error::CliResult; +use std::path::PathBuf; -pub(crate) trait AuditCollector { - fn collect_page(&self, target_url: &Url) -> CliResult; +use clap::{Args, ValueEnum}; + +use crate::ad_templates::compare::BrowserAdEvidence; + +/// Default quiet window for generation's browser collector. +pub(crate) const GENERATE_SETTLE_QUIET_MS: u64 = 750; +/// Default maximum settle wait for generation's browser collector. +pub(crate) const GENERATE_SETTLE_MAX_MS: u64 = 12_000; +/// Default quiet window for `ts audit page` and `ts audit ad-templates verify`. +/// +/// [`BrowserOpts`] and `BrowserCollector::new` must agree, or a collector built +/// in code drifts from the parsed flags without anything failing. +pub(crate) const PAGE_SETTLE_QUIET_MS: u64 = 750; +/// Default maximum settle wait for `ts audit page` and +/// `ts audit ad-templates verify`. +/// +/// See [`PAGE_SETTLE_QUIET_MS`] for why this is shared rather than duplicated. +pub(crate) const PAGE_SETTLE_MAX_MS: u64 = 10_000; + +/// Operator-tunable browser options shared by `ts audit page` and +/// `ts audit ad-templates verify`. +/// +/// These are audit-tool knobs, not publisher runtime config, so they live on the +/// CLI (flags / `CHROME` env) rather than in `trusted-server.toml`. +#[derive(Debug, Clone, Args)] +pub struct BrowserOpts { + /// Path to the Chrome/Chromium executable. Falls back to `$CHROME`, then + /// auto-detection on `PATH` and standard install locations. + #[arg(long)] + pub chrome: Option, + /// Browser device profile used for viewport and user-agent emulation. + #[arg(long = "browser-profile", value_enum, default_value_t = BrowserProfile::Desktop)] + pub profile: BrowserProfile, + /// Run a visible browser instead of Chrome's new headless mode. + #[arg(long)] + pub headful: bool, + /// Do not answer the standard IAB consent APIs for the fresh audit profile. + #[arg(long)] + pub no_assume_consent: bool, + /// Route the browser through this proxy, as `host:port` or a full URL. + #[arg(long, value_name = "HOST:PORT")] + pub browser_proxy: Option, + /// Quiet window in milliseconds (no new network resources) that marks the + /// page settled. + #[arg(long, default_value_t = PAGE_SETTLE_QUIET_MS)] + pub settle_quiet_ms: u64, + /// Hard cap in milliseconds on waiting for the page to settle. + #[arg(long, default_value_t = PAGE_SETTLE_MAX_MS)] + pub settle_max_ms: u64, + /// Navigate to origins whose TLS certificate does not validate. + /// + /// DANGEROUS: the audit sends any `--cookie` session to the origin and + /// treats what it reads back as verification evidence, so an invalid + /// certificate could mean an impersonator is harvesting the session and + /// fabricating the evidence. Use only against a host you control with a + /// known self-signed certificate. + #[arg(long)] + pub danger_accept_invalid_certs: bool, +} + +/// Browser options for generation, whose device selection is controlled by +/// `--profiles` rather than the verifier's singular `--browser-profile`. +#[derive(Debug, Clone, Args)] +pub struct GenerateBrowserOpts { + /// Path to the Chrome/Chromium executable. Falls back to `$CHROME`, then auto-detection. + #[arg(long)] + pub chrome: Option, + /// Run a visible browser instead of Chrome's new headless mode. + #[arg(long)] + pub headful: bool, + /// Do not answer the standard IAB consent APIs for the fresh audit profile. + #[arg(long)] + pub no_assume_consent: bool, + /// Route the browser through this proxy, as `host:port` or a full URL. + #[arg(long, value_name = "HOST:PORT")] + pub browser_proxy: Option, + /// Quiet window in milliseconds that marks the page settled. + #[arg(long, default_value_t = GENERATE_SETTLE_QUIET_MS)] + pub settle_quiet_ms: u64, + /// Hard cap in milliseconds on waiting for the page to settle. + #[arg(long, default_value_t = GENERATE_SETTLE_MAX_MS)] + pub settle_max_ms: u64, + /// Navigate to origins whose TLS certificate does not validate. + /// + /// DANGEROUS: the audit sends any `--cookie` session to the origin and + /// treats what it reads back as the evidence it writes config from, so an + /// invalid certificate could mean an impersonator is harvesting the session + /// and fabricating the evidence. Use only against a host you control with a + /// known self-signed certificate. + #[arg(long)] + pub danger_accept_invalid_certs: bool, +} + +/// Defaults mirroring the `#[arg(default_value_t)]` values above, so a path that +/// builds these options in code (the legacy `ts audit ` form) behaves like +/// the parsed command. +impl Default for GenerateBrowserOpts { + fn default() -> Self { + Self { + chrome: None, + headful: false, + no_assume_consent: false, + browser_proxy: None, + settle_quiet_ms: GENERATE_SETTLE_QUIET_MS, + settle_max_ms: GENERATE_SETTLE_MAX_MS, + danger_accept_invalid_certs: false, + } + } +} + +impl GenerateBrowserOpts { + /// Validates relationships between independently parsed browser flags. + pub fn validate(&self) -> Result<(), String> { + validate_settle_window(self.settle_quiet_ms, self.settle_max_ms) + } +} + +/// Browser device profile shared by page audits and ad-template verification. +#[derive(Debug, Clone, Copy, Default, PartialEq, Eq, ValueEnum)] +pub enum BrowserProfile { + /// Desktop Chrome at 1280×800. + #[default] + Desktop, + /// Mobile-sized viewport with a mobile user agent. + Mobile, +} + +impl BrowserOpts { + /// Validates relationships between independently parsed browser flags. + pub fn validate(&self) -> Result<(), String> { + validate_settle_window(self.settle_quiet_ms, self.settle_max_ms) + } } -#[derive(Debug, Clone, Deserialize, Serialize, PartialEq, Eq)] -pub(crate) struct CollectedPage { - pub(crate) requested_url: String, - pub(crate) final_url: String, - pub(crate) page_title: Option, - pub(crate) html: String, - pub(crate) script_tags: Vec, - pub(crate) network_requests: Vec, - pub(crate) warnings: Vec, +fn validate_settle_window(quiet_ms: u64, max_ms: u64) -> Result<(), String> { + if quiet_ms > max_ms { + return Err(format!( + "--settle-quiet-ms ({quiet_ms}) cannot exceed --settle-max-ms ({max_ms})" + )); + } + Ok(()) +} + +/// A request to collect a single page. +#[derive(Debug, Clone)] +pub struct BrowserCollectRequest { + /// The URL to navigate to. + pub url: url::Url, + /// Pre-navigation init scripts (evaluate-on-new-document). Empty for a plain + /// page audit; the ad-template verifier supplies the read-only collector here. + pub init_scripts: Vec, + /// Whether to perform the deterministic scroll pass after settle. + pub scroll: bool, + /// Whether to extract `window.__tsAdTemplateEvidence` after settle/scroll. + pub collect_ad_evidence: bool, + /// Operator-supplied `(name, value)` cookies set on the browser context + /// before navigation, scoped to the request URL. Used to carry an existing + /// authenticated session (e.g. a valid bot-protection clearance cookie) so + /// the origin serves the real page instead of a challenge. The collector + /// only sends these; it never reads cookies back. + pub cookies: Vec<(String, String)>, +} + +/// The result of collecting a single page. +#[derive(Debug, Clone)] +pub struct CollectedPage { + /// The final URL after redirects. + pub final_url: url::Url, + /// The page title. + pub title: String, + /// Number of ` + +"#; + + fn lazy_gpt_fixture_url() -> Url { + let listener = TcpListener::bind("127.0.0.1:0").expect("should bind fixture server"); + let address = listener.local_addr().expect("should read fixture address"); + std::thread::spawn(move || { + let (mut stream, _) = listener.accept().expect("should accept browser request"); + stream + .set_read_timeout(Some(Duration::from_secs(10))) + .expect("should set fixture read timeout"); + let mut request = Vec::new(); + while !request.ends_with(b"\r\n\r\n") { + let mut chunk = [0_u8; 1024]; + let chunk_len = stream.read(&mut chunk).expect("should read HTTP request"); + assert!(chunk_len > 0, "request should contain complete headers"); + request.extend_from_slice(&chunk[..chunk_len]); + assert!( + request.len() <= 16 * 1024, + "request headers should be bounded" + ); + } + write!( + stream, + "HTTP/1.1 200 OK\r\nContent-Type: text/html\r\nContent-Length: {}\r\nConnection: close\r\n\r\n{}", + LAZY_GPT_FIXTURE.len(), + LAZY_GPT_FIXTURE, + ) + .expect("should write fixture response"); + }); + Url::parse(&format!("http://{address}/")).expect("should parse fixture URL") + } + + #[test] + fn successful_navigation_status_allows_redirects_but_rejects_errors() { + assert!(is_successful_navigation_status(200)); + assert!(is_successful_navigation_status(302)); + assert!(is_successful_navigation_status(399)); + assert!(!is_successful_navigation_status(199)); + assert!(!is_successful_navigation_status(400)); + assert!(!is_successful_navigation_status(500)); + } + + #[test] + fn navigation_response_returns_warning_for_http_error_status() { + let warning = + validate_navigation_response(navigation_response_with_status(403, "Forbidden")) + .expect("should validate navigation response") + .expect("should return warning for HTTP error status"); + + assert_eq!( + warning, + "audit request returned HTTP 403 Forbidden for `https://example.com/`; results may be partial", + "should warn and continue when the main document returns an HTTP error" + ); + } + + #[test] + fn navigation_response_reports_chromium_request_failure() { + let mut request = + HttpRequest::new(RequestId::new("request-1"), None, None, false, Vec::new()); + request.failure_text = Some("net::ERR_BLOCKED_BY_ORB".to_string()); + + let error = validate_navigation_response(Some(Arc::new(request))) + .expect_err("should reject Chromium request failures"); + + assert_eq!( + error, "main document request failed: net::ERR_BLOCKED_BY_ORB", + "the crawl should retain the browser failure for its final skipped-page note" + ); + } + + #[test] + fn resource_timing_buffer_warning_starts_at_threshold() { + assert_eq!( + resource_timing_buffer_warning(RESOURCE_TIMING_BUFFER_SIZE - 1), + None, + "should not warn before the resource timing buffer threshold" + ); + assert_eq!( + resource_timing_buffer_warning(RESOURCE_TIMING_BUFFER_SIZE), + Some(RESOURCE_TIMING_BUFFER_WARNING), + "should warn when the resource timing buffer reaches the threshold" + ); + } + + #[test] + fn browser_path_candidates_include_common_names() { + let candidates = crate::commands::audit::browser::CHROME_NAMES; + + assert!(candidates.contains(&"google-chrome")); + assert!(candidates.contains(&"chromium")); + assert!(candidates.contains(&"Google Chrome for Testing")); + } + + #[test] + fn browser_run_reports_close_error_before_wait_error() { + let result = combine_browser_run_results( + Ok(()), + Ok(()), + Err("close failed".to_string()), + Err("wait failed".to_string()), + ); + + assert_eq!( + result.expect_err("should preserve teardown error"), + "close failed", + "the close failure is the first teardown failure" + ); + } + + #[test] + fn browser_run_reports_wait_error_when_close_succeeds() { + let result = + combine_browser_run_results(Ok(()), Ok(()), Ok(()), Err("wait failed".to_string())); + + assert_eq!( + result.expect_err("should preserve wait error"), + "wait failed", + "a wait failure must not be mislabeled as a close failure" + ); + } + + #[test] + fn browser_run_preserves_collection_error_over_later_failures() { + let result = combine_browser_run_results( + Err("collection failed".to_string()), + Err("finalization progress failed".to_string()), + Err("close failed".to_string()), + Err("wait failed".to_string()), + ); + + assert_eq!( + result.expect_err("should preserve first browser run error"), + "collection failed" + ); + } + + #[test] + fn browser_run_reports_finalization_progress_before_teardown_errors() { + let result = combine_browser_run_results( + Ok(()), + Err("finalization progress failed".to_string()), + Err("close failed".to_string()), + Err("wait failed".to_string()), + ); + + assert_eq!( + result.expect_err("should preserve finalization progress error"), + "finalization progress failed" + ); + } + + #[test] + #[ignore = "requires local Chrome/Chromium; run through scripts/test-cli.sh"] + fn progress_failure_still_finalizes_browser_session() { + if !browser_fixture_available() { + return; + } + + let collector = BrowserAuditCollector::default(); + let target = Url::parse("http://127.0.0.1:9/").expect("should parse fixture URL"); + let mut phases = Vec::new(); + let error = collector + .collect_pages( + &[target], + &[], + &mut |progress| match progress { + CollectionProgress::Launching => { + phases.push("launching"); + Ok(()) + } + CollectionProgress::Loading { .. } => { + phases.push("loading"); + Err(report_error("simulated progress failure")) + } + CollectionProgress::Planning => { + phases.push("planning"); + Ok(()) + } + CollectionProgress::Finalizing => { + phases.push("finalizing"); + Ok(()) + } + }, + &mut |_, _| panic!("page sink should not run after progress failure"), + ) + .expect_err("should return progress failure after browser teardown"); + + let rendered_error = format!("{error:?}"); + assert!( + rendered_error.contains("simulated progress failure"), + "should preserve progress failure, got {rendered_error}" + ); + assert_eq!(phases, ["launching", "loading", "finalizing"]); + } + + #[test] + #[ignore = "requires local Chrome/Chromium; run through scripts/test-cli.sh"] + fn collects_lazy_gpt_slot_only_when_scroll_is_enabled() { + if !browser_fixture_available() { + return; + } + + let without_scroll = BrowserAuditCollector::default() + .collect_page(&lazy_gpt_fixture_url(), &[]) + .expect("should collect without scrolling"); + let with_scroll = BrowserAuditCollector::default() + .with_scroll(true) + .collect_page(&lazy_gpt_fixture_url(), &[]) + .expect("should collect with scrolling"); + + assert!( + without_scroll.gpt_slots.is_empty(), + "lazy GPT slot should not exist before scrolling" + ); + assert!( + with_scroll + .gpt_slots + .iter() + .any(|slot| { slot.gam_unit_path == "/123/lazy" && slot.div_id == "ad-lazy-0" }), + "scrolling should trigger and collect the lazy GPT slot" + ); + } + + fn navigation_response_with_status(status: i64, status_text: &str) -> ArcHttpRequest { + let mut request = + HttpRequest::new(RequestId::new("request-1"), None, None, false, Vec::new()); + request.response = Some( + Response::builder() + .url("https://example.com/") + .status(status) + .status_text(status_text) + .headers(Headers::default()) + .mime_type("text/html") + .charset("utf-8") + .connection_reused(false) + .connection_id(1.0) + .encoded_data_length(0.0) + .security_state(SecurityState::Secure) + .build() + .expect("should build navigation response"), + ); + + Some(Arc::new(request)) + } +} diff --git a/crates/trusted-server-cli/src/commands/audit/generate/collector.rs b/crates/trusted-server-cli/src/commands/audit/generate/collector.rs new file mode 100644 index 000000000..dc23af09c --- /dev/null +++ b/crates/trusted-server-cli/src/commands/audit/generate/collector.rs @@ -0,0 +1,366 @@ +use serde::{Deserialize, Serialize}; +use url::Url; + +use crate::error::CliResult; + +/// Warning recorded on a page collected with the audit consent stub installed. +/// +/// A whole-run fact rather than a property of one page, so consumers report it +/// once and unscoped instead of once per page and per profile. +pub(crate) const CONSENT_STUB_WARNING: &str = "consent_stub_active: audit consent APIs were stubbed; re-run with --no-assume-consent to observe the publisher CMP without substitution"; + +/// A user-visible phase reached while collecting browser audit evidence. +#[derive(Debug, Clone, Copy)] +pub(crate) enum CollectionProgress<'a> { + /// The browser process is about to launch. + Launching, + /// A page navigation is about to begin. + Loading { + /// One-based position of this attempted page in the crawl. + current: usize, + /// Total pages when planning has completed, or `None` for the root. + total: Option, + /// Target page; renderers must omit credentials, query, and fragment. + url: &'a Url, + }, + /// Follow-up pages are being selected from the collected root page. + Planning, + /// The browser session is being closed and its process reaped. + Finalizing, +} + +/// Sink invoked synchronously when browser collection reaches a visible phase. +/// +/// Returning an error stops new collection work. An already-launched browser +/// must still be finalized, closed, and waited on before that error is returned. +pub(crate) type ProgressSink<'a> = + &'a mut dyn for<'event> FnMut(CollectionProgress<'event>) -> CliResult<()>; + +/// Sink invoked once per collected page during a batch crawl. +/// +/// Receives the per-page outcome so a failed page can be folded into the run as +/// a warning rather than aborting it; returning `Err` stops the crawl. +pub(crate) type PageSink<'a> = + &'a mut dyn FnMut(&Url, CliResult) -> CliResult; + +/// Plans follow-up URLs from the successfully collected root page. +pub(crate) type RootPlanner<'a> = &'a mut dyn FnMut(&Url, &CollectedPage) -> CliResult>; + +/// Whether a batch crawl should keep going after a page. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub(crate) enum ControlFlow { + /// Collect the next target. + Continue, + /// Stop the crawl without an error (budget reached, challenge rate exceeded). + /// + /// What this can prevent depends on the collector: a sequential one loads no + /// further pages, while the browser collector has already finished + /// navigating by the time it folds, so there it only stops the fold. + Stop, +} + +pub(crate) trait AuditCollector { + /// Collects a live page. `cookies` are `(name, value)` pairs set on the + /// browser context before navigation (scoped to `target_url`) so an existing + /// session — e.g. a valid bot-protection clearance cookie — can carry the + /// audit past an origin challenge. + fn collect_page( + &self, + target_url: &Url, + cookies: &[(String, String)], + ) -> CliResult; + + /// Collects several pages in one session, handing each result to `on_page`. + /// + /// The default implementation loops over [`collect_page`](Self::collect_page), + /// which keeps every existing implementor working unchanged. The browser + /// collector overrides it to reuse one Chrome instance and profile across the + /// crawl — a fresh launch per page dominates the cost of a multi-page run, + /// and a shared profile carries bot-protection clearance cookies site-wide. + /// + /// Collectors may buffer results until the browser session closes so CPU-heavy + /// HTML analysis cannot starve a single-threaded CDP event pump. The sink API + /// keeps that buffering policy private and lets simple collectors stream. + /// + /// # Errors + /// + /// Returns an error when `on_page` does, or when the session itself cannot + /// be established. Individual page failures are delivered to `on_page`. + fn collect_pages( + &self, + targets: &[Url], + cookies: &[(String, String)], + on_progress: ProgressSink<'_>, + on_page: PageSink<'_>, + ) -> CliResult<()> { + for (index, target) in targets.iter().enumerate() { + on_progress(CollectionProgress::Loading { + current: index + 1, + total: Some(targets.len()), + url: target, + })?; + let collected = self.collect_page(target, cookies); + if on_page(target, collected)? == ControlFlow::Stop { + break; + } + } + Ok(()) + } + + /// Collects a root and follow-up URLs planned from it in one logical crawl. + /// + /// The browser implementation overrides this so planning happens while the + /// root's browser/profile remains open. Simple collectors retain equivalent + /// behavior through the default implementation. + fn collect_site( + &self, + root: &Url, + cookies: &[(String, String)], + on_progress: ProgressSink<'_>, + planner: RootPlanner<'_>, + on_page: PageSink<'_>, + ) -> CliResult<()> { + on_progress(CollectionProgress::Loading { + current: 1, + total: None, + url: root, + })?; + // A root failure is reported through `on_page` rather than returned, so + // the caller sees the reason as a per-page note exactly as it does from + // the browser collector. With no root page there is nothing to plan + // from, so the crawl ends here. + let root_page = match self.collect_page(root, cookies) { + Ok(page) => page, + Err(error) => { + on_page(root, Err(error))?; + return Ok(()); + } + }; + on_progress(CollectionProgress::Planning)?; + let targets = planner(root, &root_page)?; + if on_page(root, Ok(root_page))? == ControlFlow::Stop { + return Ok(()); + } + let total = targets.len() + 1; + for (index, target) in targets.iter().enumerate() { + on_progress(CollectionProgress::Loading { + current: index + 2, + total: Some(total), + url: target, + })?; + let collected = self.collect_page(target, cookies); + if on_page(target, collected)? == ControlFlow::Stop { + break; + } + } + Ok(()) + } +} + +#[derive(Debug, Clone, Deserialize, Serialize, PartialEq, Eq)] +pub(crate) struct CollectedPage { + pub(crate) requested_url: String, + pub(crate) final_url: String, + pub(crate) page_title: Option, + pub(crate) html: String, + pub(crate) script_tags: Vec, + pub(crate) network_requests: Vec, + /// Slots read from the live GPT registry (`googletag.pubads().getSlots()`). + /// + /// Populated at `defineSlot` time, so this captures configured slots even + /// when the ad request never fires (consent-gated or iframe-issued). + #[serde(default)] + pub(crate) gpt_slots: Vec, + /// Same-origin `a[href]` targets read from the hydrated DOM, absolutized. + /// + /// Read from the live DOM rather than the served HTML on purpose: an + /// app-router page keeps its link graph in the framework payload, so parsing + /// the raw markup finds only a fraction of the site's sections. + #[serde(default)] + pub(crate) links: Vec, + /// Sitemap `` entries discovered from `robots.txt`, when fetched. + /// + /// Empty unless sitemap discovery ran (root page only). + #[serde(default)] + pub(crate) sitemap_locs: Vec, + pub(crate) warnings: Vec, +} + +/// A same-origin link observed in the hydrated DOM. +#[derive(Debug, Clone, Deserialize, Serialize, PartialEq, Eq)] +pub(crate) struct CollectedLink { + /// Absolute URL of the link target. + pub(crate) url: String, + /// Whether the anchor sits inside site navigation (`nav`, `header`, + /// `[role="navigation"]`). Nav links are the publisher's own declaration of + /// its taxonomy, so they rank above body links when choosing sections. + pub(crate) in_nav: bool, +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::error::{cli_error, report_error}; + + struct ProgressCollector; + + impl AuditCollector for ProgressCollector { + fn collect_page( + &self, + target_url: &Url, + _cookies: &[(String, String)], + ) -> CliResult { + if target_url.path() == "/broken" { + return cli_error("simulated page failure"); + } + Ok(CollectedPage { + requested_url: target_url.to_string(), + final_url: target_url.to_string(), + page_title: None, + html: String::new(), + script_tags: Vec::new(), + network_requests: Vec::new(), + gpt_slots: Vec::new(), + links: Vec::new(), + sitemap_locs: Vec::new(), + warnings: Vec::new(), + }) + } + } + + fn record_progress(event: CollectionProgress<'_>) -> String { + match event { + CollectionProgress::Launching => "launching".to_string(), + CollectionProgress::Loading { + current, + total, + url, + } => format!( + "loading:{current}/{}:{}", + total.map_or_else(|| "?".to_string(), |total| total.to_string()), + url.path() + ), + CollectionProgress::Planning => "planning".to_string(), + CollectionProgress::Finalizing => "finalizing".to_string(), + } + } + + #[test] + fn default_collect_site_reports_root_planning_and_offset_followups() { + let collector = ProgressCollector; + let root = Url::parse("https://publisher.example/").expect("should parse root URL"); + let news = Url::parse("https://publisher.example/news").expect("should parse news URL"); + let broken = + Url::parse("https://publisher.example/broken").expect("should parse broken URL"); + let mut events = Vec::new(); + let mut outcomes = Vec::new(); + + collector + .collect_site( + &root, + &[], + &mut |event| { + events.push(record_progress(event)); + Ok(()) + }, + &mut |_, _| Ok(vec![news.clone(), broken.clone()]), + &mut |url, result| { + outcomes.push((url.path().to_string(), result.is_ok())); + Ok(ControlFlow::Continue) + }, + ) + .expect("should collect site despite one page outcome failing"); + + assert_eq!( + events, + [ + "loading:1/?:/", + "planning", + "loading:2/3:/news", + "loading:3/3:/broken", + ] + ); + assert_eq!( + outcomes, + [ + ("/".to_string(), true), + ("/news".to_string(), true), + ("/broken".to_string(), false) + ] + ); + } + + #[test] + fn default_collect_pages_reports_a_fixed_total() { + let collector = ProgressCollector; + let targets = [ + Url::parse("https://publisher.example/").expect("should parse root URL"), + Url::parse("https://publisher.example/broken").expect("should parse broken URL"), + ]; + let mut events = Vec::new(); + + collector + .collect_pages( + &targets, + &[], + &mut |event| { + events.push(record_progress(event)); + Ok(()) + }, + &mut |_, _| Ok(ControlFlow::Continue), + ) + .expect("should deliver failed page as an outcome"); + + assert_eq!(events, ["loading:1/2:/", "loading:2/2:/broken"]); + } + + #[test] + fn default_collection_stops_when_progress_fails() { + let collector = ProgressCollector; + let targets = [Url::parse("https://publisher.example/").expect("should parse root URL")]; + + let error = collector + .collect_pages( + &targets, + &[], + &mut |_| Err(report_error("simulated progress failure")), + &mut |_, _| panic!("page sink should not run after progress failure"), + ) + .expect_err("should return progress failure"); + + assert!(format!("{error:?}").contains("simulated progress failure")); + } +} + +/// A single slot read from the page's live GPT registry. +#[derive(Debug, Clone, Deserialize, Serialize, PartialEq, Eq)] +pub(crate) struct CollectedGptSlot { + /// The GAM ad-unit path (`slot.getAdUnitPath()`). + pub(crate) gam_unit_path: String, + /// The slot's div element id (`slot.getSlotElementId()`). + pub(crate) div_id: String, + /// Numeric `[width, height]` sizes (`slot.getSizes()`, fluid entries dropped). + pub(crate) sizes: Vec<(u32, u32)>, +} + +#[derive(Debug, Clone, Deserialize, Serialize, PartialEq, Eq)] +pub(crate) struct CollectedScriptTag { + pub(crate) src: Option, + pub(crate) inline_text: Option, +} + +#[derive(Debug, Clone, Deserialize, Serialize, PartialEq, Eq)] +pub(crate) struct CollectedRequest { + pub(crate) url: String, + pub(crate) resource_type: Option, +} + +impl CollectedPage { + pub(crate) fn requested_url(&self) -> Result { + Url::parse(&self.requested_url) + } + + pub(crate) fn final_url(&self) -> Result { + Url::parse(&self.final_url) + } +} diff --git a/crates/trusted-server-cli/src/commands/audit/generate/crawl_plan.rs b/crates/trusted-server-cli/src/commands/audit/generate/crawl_plan.rs new file mode 100644 index 000000000..76d57cf9e --- /dev/null +++ b/crates/trusted-server-cli/src/commands/audit/generate/crawl_plan.rs @@ -0,0 +1,864 @@ +//! Pure crawl planning: turn discovered links and sitemap entries into the +//! bounded set of pages worth loading in a browser. +//! +//! The goal is deliberately *not* site coverage. Ad slots repeat per site +//! section, and the generated config needs one glob pair per section +//! (`/news` and `/news/*`), so one representative page per section is enough. +//! That keeps the crawl proportional to the publisher's taxonomy (a dozen +//! sections) rather than its catalog (tens of thousands of articles). +//! +//! Two sources feed the plan and each supplies a half the other cannot: +//! +//! - **Navigation links** give section *landing* paths (`/news`), which +//! sitemaps routinely omit, and are the publisher's own taxonomy declaration. +//! - **Sitemap entries** give a real *article* per section (`/news/story-abc`), +//! which is where in-content slots live, and reveal sections hidden behind a +//! navigation overflow menu. + +use std::collections::BTreeMap; + +use url::Url; + +use super::collector::CollectedLink; + +/// Path segments that are never a content section worth sampling. +/// +/// These carry either no ad stack at all or an unrepresentative one, and +/// crawling them spends budget that a real section needs. +const NOISE_SEGMENTS: &[&str] = &[ + "about", + "about-us", + "account", + "author", + "cart", + "contact", + "editorial-policy", + "login", + "logout", + "newsletter", + "page", + "press", + "privacy", + "register", + "search", + "sitemap", + "subscribe", + "terms", +]; + +/// File extensions that are assets rather than pages. +const NON_PAGE_EXTENSIONS: &[&str] = &[ + ".jpg", ".jpeg", ".png", ".gif", ".webp", ".avif", ".svg", ".ico", ".css", ".js", ".json", + ".xml", ".pdf", ".zip", ".mp4", ".mp3", ".rss", +]; + +/// ISO 639-1 alpha-2 language codes, sorted for binary search. +/// +/// Country codes are deliberately absent: `/us` and `/tv` are section roots on +/// plenty of publishers, and only the language form appears as a URL locale +/// prefix on its own. +const ISO_639_1_CODES: &[&str] = &[ + "aa", "ab", "ae", "af", "ak", "am", "an", "ar", "as", "av", "ay", "az", "ba", "be", "bg", "bh", + "bi", "bm", "bn", "bo", "br", "bs", "ca", "ce", "ch", "co", "cr", "cs", "cu", "cv", "cy", "da", + "de", "dv", "dz", "ee", "el", "en", "eo", "es", "et", "eu", "fa", "ff", "fi", "fj", "fo", "fr", + "fy", "ga", "gd", "gl", "gn", "gu", "gv", "ha", "he", "hi", "ho", "hr", "ht", "hu", "hy", "hz", + "ia", "id", "ie", "ig", "ii", "ik", "io", "is", "it", "iu", "ja", "jv", "ka", "kg", "ki", "kj", + "kk", "kl", "km", "kn", "ko", "kr", "ks", "ku", "kv", "kw", "ky", "la", "lb", "lg", "li", "ln", + "lo", "lt", "lu", "lv", "mg", "mh", "mi", "mk", "ml", "mn", "mr", "ms", "mt", "my", "na", "nb", + "nd", "ne", "ng", "nl", "nn", "no", "nr", "nv", "ny", "oc", "oj", "om", "or", "os", "pa", "pi", + "pl", "ps", "pt", "qu", "rm", "rn", "ro", "ru", "rw", "sa", "sc", "sd", "se", "sg", "si", "sk", + "sl", "sm", "sn", "so", "sq", "sr", "ss", "st", "su", "sv", "sw", "ta", "te", "tg", "th", "ti", + "tk", "tl", "tn", "to", "tr", "ts", "tt", "tw", "ty", "ug", "uk", "ur", "uz", "ve", "vi", "vo", + "wa", "wo", "xh", "yi", "yo", "za", "zh", "zu", +]; + +/// Filenames that name a directory's index document rather than a page of their +/// own, so a link to one is treated as a link to the parent directory. +const DIRECTORY_INDEX_NAMES: &[&str] = &[ + "index.html", + "index.htm", + "index.php", + "default.html", + "default.htm", + "default.php", + "home.html", + "home.htm", + "home.php", +]; + +/// Bounds on how much of a site a single run will load. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub(crate) struct CrawlBudget { + /// Maximum number of sections to sample. + pub(crate) max_sections: usize, + /// Maximum number of pages to load in total, including the root. + pub(crate) max_pages: usize, +} + +impl Default for CrawlBudget { + fn default() -> Self { + Self { + max_sections: 8, + max_pages: 17, + } + } +} + +/// One section selected for sampling. +#[derive(Debug, Clone, PartialEq, Eq)] +pub(super) struct PlannedSection { + /// The path segment at [`CrawlPlan::section_segment`] identifying the section. + pub(super) segment: String, + /// The section landing page, when one was observed. + pub(super) landing: Option, + /// A representative content page inside the section, when one was observed. + pub(super) article: Option, +} + +impl PlannedSection { + /// The pages to load for this section, landing first. + fn targets(&self) -> impl Iterator { + self.landing.iter().chain(self.article.iter()) + } +} + +/// The bounded outcome of planning. +#[derive(Debug, Clone, PartialEq, Eq)] +pub(super) struct CrawlPlan { + /// Sections selected for sampling, highest confidence first. + pub(super) sections: Vec, + /// Sections found but dropped because the budget was already spent. + pub(super) dropped_sections: Vec, + /// Human-readable notes about how the plan was reached. + pub(super) notes: Vec, + /// Path segment used to distinguish sections in this crawl. + pub(super) section_segment: usize, +} + +impl CrawlPlan { + /// Page URLs to load, in crawl order. The root is *not* included — the + /// caller has already collected it in order to plan at all. + pub(super) fn targets(&self) -> Vec { + self.sections + .iter() + .flat_map(PlannedSection::targets) + .cloned() + .collect() + } +} + +/// Evidence gathered about one candidate section before ranking. +#[derive(Debug, Default)] +struct SectionCandidate { + landing: Option, + article: Option, + in_nav: bool, + in_sitemap: bool, + link_count: usize, +} + +impl SectionCandidate { + /// Confidence ordering: corroborated by both sources beats either alone, + /// and navigation beats a sitemap-only hit because navigation is the + /// publisher's own statement of what its sections are. + fn rank(&self) -> u8 { + match (self.in_nav, self.in_sitemap) { + (true, true) => 3, + (true, false) => 2, + (false, true) => 1, + (false, false) => 0, + } + } +} + +/// Plans the crawl from the root page's links and any sitemap entries. +/// +/// `root` bounds the crawl: every candidate must share its origin, which also +/// stops a hostile or misconfigured `robots.txt` from redirecting the crawl (and +/// the operator's cookies) at an unrelated host. +pub(super) fn plan_crawl( + root: &Url, + links: &[CollectedLink], + sitemap_locs: &[String], + budget: CrawlBudget, +) -> CrawlPlan { + let section_segment = usize::from(root_is_locale_prefix(root)); + let mut candidates: BTreeMap = BTreeMap::new(); + let mut notes = Vec::new(); + + for link in links { + let Some(url) = same_origin_page_url(root, &link.url, section_segment) else { + continue; + }; + let Some(segment) = section_at(&url, section_segment) else { + continue; + }; + let entry = candidates.entry(segment).or_default(); + entry.in_nav |= link.in_nav; + entry.link_count += 1; + record_url(entry, &url, section_segment); + } + + let mut sitemap_pages = 0_usize; + for loc in sitemap_locs { + let Some(url) = same_origin_page_url(root, loc, section_segment) else { + continue; + }; + let Some(segment) = section_at(&url, section_segment) else { + continue; + }; + sitemap_pages += 1; + let entry = candidates.entry(segment).or_default(); + entry.in_sitemap = true; + record_url(entry, &url, section_segment); + } + + if !sitemap_locs.is_empty() { + notes.push(format!( + "sitemap contributed {sitemap_pages} same-origin page(s) across {} section(s)", + candidates.values().filter(|c| c.in_sitemap).count() + )); + } + if links.iter().all(|link| !link.in_nav) && !links.is_empty() { + notes.push( + "no navigation links were found; sections were inferred from body links only" + .to_string(), + ); + } + + // Rank before truncating: confidence first, then how heavily the section is + // linked, then the segment name so runs are reproducible. + let mut ranked: Vec<(String, SectionCandidate)> = candidates.into_iter().collect(); + ranked.sort_by(|(left_segment, left), (right_segment, right)| { + right + .rank() + .cmp(&left.rank()) + .then(right.link_count.cmp(&left.link_count)) + .then(left_segment.cmp(right_segment)) + }); + + let mut sections = Vec::new(); + let mut dropped_sections = Vec::new(); + // The root page is already collected and counts against the page budget. + let mut pages_used = 1_usize; + for (segment, candidate) in ranked { + let planned = PlannedSection { + segment: segment.clone(), + landing: candidate.landing, + article: candidate.article, + }; + let cost = planned.targets().count(); + if cost == 0 { + continue; + } + if sections.len() >= budget.max_sections || pages_used + cost > budget.max_pages { + dropped_sections.push(segment); + continue; + } + pages_used += cost; + sections.push(planned); + } + + if !dropped_sections.is_empty() { + let shown = dropped_sections + .iter() + .take(10) + .cloned() + .collect::>() + .join(", "); + let remainder = dropped_sections.len().saturating_sub(10); + let suffix = if remainder == 0 { + String::new() + } else { + format!(", and {remainder} more") + }; + notes.push(format!( + "budget reached: {} section(s) not sampled ({shown}{suffix}); raise --max-sections/--max-pages to include them", + dropped_sections.len(), + )); + } + + CrawlPlan { + sections, + dropped_sections, + notes, + section_segment, + } +} + +/// Files a URL as the section's landing page or its representative article. +/// +/// The first candidate of each kind wins, so a run is stable given stable input. +fn record_url(entry: &mut SectionCandidate, url: &Url, section_segment: usize) { + if segment_count(url) == section_segment + 1 { + if entry.landing.is_none() { + entry.landing = Some(url.clone()); + } + } else if entry.article.is_none() { + entry.article = Some(url.clone()); + } +} + +/// Parses `raw` against `root` and keeps it only if it is a same-origin page. +/// +/// Rejects other origins, non-HTTP schemes, asset extensions, and paginated or +/// utility paths. Query and fragment are dropped so `/news?page=2` and +/// `/news#top` collapse onto `/news`. +fn same_origin_page_url(root: &Url, raw: &str, section_segment: usize) -> Option { + let mut url = root.join(raw).ok()?; + if !matches!(url.scheme(), "http" | "https") || url.origin() != root.origin() { + return None; + } + url.set_query(None); + url.set_fragment(None); + + let path = percent_decode_for_filtering(url.path()).to_ascii_lowercase(); + if NON_PAGE_EXTENSIONS + .iter() + .any(|extension| path.ends_with(extension)) + { + return None; + } + // A section reachable only through its index document is still that section: + // `/news/index.html` is `/news`. Rejecting the URL outright loses the + // section; dropping the filename keeps it. + if path + .split('/') + .rfind(|part| !part.is_empty()) + .is_some_and(|last| DIRECTORY_INDEX_NAMES.contains(&last)) + { + url.path_segments_mut().ok()?.pop(); + } + let path = percent_decode_for_filtering(url.path()).to_ascii_lowercase(); + let segments: Vec<&str> = path.split('/').filter(|part| !part.is_empty()).collect(); + if segments.is_empty() { + return None; + } + if NOISE_SEGMENTS.contains(&segments.get(section_segment).copied().unwrap_or_default()) { + return None; + } + if section_segment > 0 { + let root_path = percent_decode_for_filtering(root.path()).to_ascii_lowercase(); + let root_segments: Vec<&str> = root_path + .split('/') + .filter(|part| !part.is_empty()) + .collect(); + if !segments.starts_with(&root_segments) { + return None; + } + } + // `/news/page/2` is the same inventory as `/news`, so it is not a second + // sample worth spending a page load on. + if segments.contains(&"page") { + return None; + } + Some(url) +} + +/// The non-empty path segment at `index`, percent-decoded and lowercased. +fn section_at(url: &Url, index: usize) -> Option { + percent_decode_for_filtering(url.path()) + .split('/') + .filter(|part| !part.is_empty()) + .nth(index) + .map(str::to_ascii_lowercase) +} + +/// Whether the requested root is nothing but a locale prefix, which puts +/// sections one segment deeper than usual. +fn root_is_locale_prefix(root: &Url) -> bool { + let segments: Vec<&str> = root + .path() + .split('/') + .filter(|part| !part.is_empty()) + .collect(); + matches!(segments.as_slice(), [locale] if is_locale_segment(locale)) +} + +/// Whether a root's single path segment is a locale prefix (`/en`, `/en-gb`) +/// rather than a content section. +/// +/// The language half must be a real ISO 639-1 code. Accepting any two letters +/// read ordinary section roots — `/tv`, `/ai`, `/us` — as locales, which shifts +/// `section_segment` by one: article slugs then become "sections" and the +/// containment check below discards the root's real siblings. +fn is_locale_segment(segment: &str) -> bool { + let segment = segment.to_ascii_lowercase(); + match segment.as_bytes() { + [_, _] => is_language_code(&segment), + [_, _, b'-', c, d] => { + is_language_code(&segment[..2]) && c.is_ascii_alphabetic() && d.is_ascii_alphabetic() + } + _ => false, + } +} + +/// Whether `segment` is an ISO 639-1 alpha-2 language code. +fn is_language_code(segment: &str) -> bool { + ISO_639_1_CODES.binary_search(&segment).is_ok() +} + +/// Decodes percent escapes solely for normalized path classification. +fn percent_decode_for_filtering(path: &str) -> String { + let bytes = path.as_bytes(); + let mut decoded = Vec::with_capacity(bytes.len()); + let mut index = 0; + while index < bytes.len() { + if bytes[index] == b'%' + && index + 2 < bytes.len() + && let (Some(high), Some(low)) = + (hex_value(bytes[index + 1]), hex_value(bytes[index + 2])) + { + decoded.push((high << 4) | low); + index += 3; + } else { + decoded.push(bytes[index]); + index += 1; + } + } + String::from_utf8_lossy(&decoded).into_owned() +} + +/// Converts one ASCII hexadecimal digit to its numeric value. +fn hex_value(byte: u8) -> Option { + match byte { + b'0'..=b'9' => Some(byte - b'0'), + b'a'..=b'f' => Some(byte - b'a' + 10), + b'A'..=b'F' => Some(byte - b'A' + 10), + _ => None, + } +} + +/// Count of non-empty path segments. +fn segment_count(url: &Url) -> usize { + url.path() + .split('/') + .filter(|part| !part.is_empty()) + .count() +} + +#[cfg(test)] +mod tests { + use super::*; + + fn root() -> Url { + Url::parse("https://publisher.example/").expect("valid root") + } + + fn nav(path: &str) -> CollectedLink { + CollectedLink { + url: format!("https://publisher.example{path}"), + in_nav: true, + } + } + + fn body(path: &str) -> CollectedLink { + CollectedLink { + url: format!("https://publisher.example{path}"), + in_nav: false, + } + } + + fn segments(plan: &CrawlPlan) -> Vec<&str> { + plan.sections + .iter() + .map(|section| section.segment.as_str()) + .collect() + } + + #[test] + fn pairs_a_landing_page_with_an_article_from_the_sitemap() { + let plan = plan_crawl( + &root(), + &[nav("/news")], + &["https://publisher.example/news/story-abc".to_string()], + CrawlBudget::default(), + ); + + assert_eq!( + segments(&plan), + ["news"], + "only the witnessed section should be planned" + ); + let section = &plan.sections[0]; + assert_eq!( + section.landing.as_ref().map(Url::as_str), + Some("https://publisher.example/news") + ); + assert_eq!( + section.article.as_ref().map(Url::as_str), + Some("https://publisher.example/news/story-abc") + ); + assert_eq!(plan.targets().len(), 2, "should load landing then article"); + } + + #[test] + fn cross_origin_candidates_are_dropped() { + // Guards both the sitemap (a `Sitemap:` directive can point anywhere) + // and links: the crawl carries operator cookies, so it must not leave + // the requested origin. + let plan = plan_crawl( + &root(), + &[CollectedLink { + url: "https://tracker.example/news".to_string(), + in_nav: true, + }], + &["https://other.example/deals/x".to_string()], + CrawlBudget::default(), + ); + + assert!( + plan.sections.is_empty(), + "no off-origin section should survive, got {:?}", + segments(&plan) + ); + } + + #[test] + fn utility_paths_and_assets_are_filtered() { + let plan = plan_crawl( + &root(), + &[ + nav("/about-us"), + nav("/search"), + nav("/editorial-policy"), + nav("/logo.png"), + nav("/feed.xml"), + nav("/news/page/2"), + nav("/news"), + ], + &[], + CrawlBudget::default(), + ); + + assert_eq!( + segments(&plan), + ["news"], + "only the real content section should remain" + ); + } + + #[test] + fn query_and_fragment_collapse_onto_one_landing_page() { + let plan = plan_crawl( + &root(), + &[nav("/news?utm_source=x"), nav("/news#top"), nav("/news")], + &[], + CrawlBudget::default(), + ); + + assert_eq!( + segments(&plan), + ["news"], + "only the witnessed section should be planned" + ); + assert_eq!( + plan.sections[0].landing.as_ref().map(Url::as_str), + Some("https://publisher.example/news"), + "tracking query and fragment should be stripped" + ); + } + + #[test] + fn nav_and_sitemap_corroboration_outranks_either_alone() { + let plan = plan_crawl( + &root(), + &[nav("/features"), body("/reviews")], + &[ + "https://publisher.example/features/story".to_string(), + "https://publisher.example/deals/x".to_string(), + ], + CrawlBudget::default(), + ); + + assert_eq!( + segments(&plan)[0], + "features", + "nav + sitemap should rank first, got {:?}", + segments(&plan) + ); + } + + #[test] + fn budget_truncates_and_reports_what_was_dropped() { + let links: Vec = ["a", "b", "c", "d"] + .iter() + .map(|segment| nav(&format!("/{segment}"))) + .collect(); + + let plan = plan_crawl( + &root(), + &links, + &[], + CrawlBudget { + max_sections: 2, + max_pages: 17, + }, + ); + + assert_eq!(plan.sections.len(), 2, "section cap should be honoured"); + assert_eq!( + plan.dropped_sections.len(), + 2, + "sections past the budget should be reported as dropped" + ); + assert!( + plan.notes + .iter() + .any(|note| note.contains("budget reached")), + "dropping sections must be reported, not silent: {:?}", + plan.notes + ); + } + + #[test] + fn page_budget_counts_the_already_collected_root() { + // max_pages = 3 leaves room for exactly one landing+article pair on top + // of the root page the caller already loaded. + let plan = plan_crawl( + &root(), + &[nav("/news"), nav("/deals")], + &[ + "https://publisher.example/news/a".to_string(), + "https://publisher.example/deals/b".to_string(), + ], + CrawlBudget { + max_sections: 8, + max_pages: 3, + }, + ); + + assert_eq!( + plan.targets().len(), + 2, + "root + 2 pages fills max_pages = 3" + ); + assert_eq!( + plan.dropped_sections.len(), + 1, + "the section past the budget should be reported as dropped" + ); + } + + #[test] + fn body_only_links_still_yield_sections_with_a_note() { + let plan = plan_crawl( + &root(), + &[body("/news"), body("/deals")], + &[], + CrawlBudget::default(), + ); + + assert_eq!(segments(&plan), ["deals", "news"]); + assert!( + plan.notes + .iter() + .any(|note| note.contains("no navigation links")), + "a nav-less page should say so: {:?}", + plan.notes + ); + } + + #[test] + fn empty_input_plans_nothing_rather_than_panicking() { + let plan = plan_crawl(&root(), &[], &[], CrawlBudget::default()); + + assert!( + plan.sections.is_empty(), + "no input means no sections to sample" + ); + assert!( + plan.targets().is_empty(), + "no sections means nothing to load" + ); + } + + #[test] + fn locale_root_plans_sections_from_the_second_segment() { + let locale_root = Url::parse("https://publisher.example/en").expect("should parse root"); + let plan = plan_crawl( + &locale_root, + &[nav("/en/news"), nav("/en/deals")], + &[ + "https://publisher.example/en/news/story".to_string(), + "https://publisher.example/en/deals/item".to_string(), + ], + CrawlBudget::default(), + ); + + assert_eq!( + plan.section_segment, 1, + "a locale root puts sections one segment deeper" + ); + assert_eq!(segments(&plan), ["deals", "news"]); + assert_eq!( + plan.targets().len(), + 4, + "each section contributes a landing page and an article" + ); + } + + #[test] + fn encoded_noise_and_page_extensions_are_filtered() { + let plan = plan_crawl( + &root(), + &[ + nav("/%70rivacy"), + nav("/index.html"), + nav("/archive.htm"), + nav("/story.php"), + nav("/news"), + ], + &[], + CrawlBudget::default(), + ); + + assert_eq!( + segments(&plan), + ["archive.htm", "news", "story.php"], + "only directory-index documents should be excluded by extension" + ); + } + + #[test] + fn a_two_letter_section_root_is_not_read_as_a_locale() { + // `/tv`, `/ai` and `/us` are section roots, not locales. Reading them as + // locales moves the section segment to 1, so article slugs become + // "sections" and the root's real siblings are discarded. + for root_path in ["/tv", "/ai", "/us"] { + let section_root = Url::parse(&format!("https://publisher.example{root_path}")) + .expect("should parse root"); + let plan = plan_crawl( + §ion_root, + &[ + nav(&format!("{root_path}/story-one")), + nav(&format!("{root_path}/story-two")), + ], + &[], + CrawlBudget::default(), + ); + + assert_eq!( + plan.section_segment, 0, + "`{root_path}` should be a section root, not a locale prefix" + ); + assert_eq!( + segments(&plan), + [root_path.trim_start_matches('/')], + "articles below `{root_path}` should stay one section" + ); + } + } + + #[test] + fn a_real_language_prefix_is_still_read_as_a_locale() { + for root_path in ["/en", "/fr", "/pt-br"] { + let locale_root = Url::parse(&format!("https://publisher.example{root_path}")) + .expect("should parse root"); + let plan = plan_crawl( + &locale_root, + &[nav(&format!("{root_path}/news"))], + &[], + CrawlBudget::default(), + ); + + assert_eq!( + plan.section_segment, 1, + "`{root_path}` is a locale prefix, so sections start one segment in" + ); + assert_eq!(segments(&plan), ["news"]); + } + } + + #[test] + fn a_section_reachable_only_by_its_index_document_collapses_to_the_parent() { + let plan = plan_crawl( + &root(), + &[ + nav("/news/index.html"), + nav("/deals/index.php"), + nav("/sport/home.htm"), + ], + &[], + CrawlBudget::default(), + ); + + assert_eq!( + segments(&plan), + ["deals", "news", "sport"], + "an index document names its section rather than disqualifying it" + ); + let targets: Vec = plan + .targets() + .iter() + .map(|url| url.path().to_string()) + .collect(); + assert_eq!( + targets, + ["/deals", "/news", "/sport"], + "the parent directory is what gets loaded" + ); + } + + #[test] + fn locale_root_rejects_candidates_outside_its_path_prefix() { + let locale_root = Url::parse("https://publisher.example/en").expect("should parse root"); + let plan = plan_crawl( + &locale_root, + &[nav("/en/news"), nav("/fr/deals")], + &[], + CrawlBudget::default(), + ); + + assert_eq!( + segments(&plan), + ["news"], + "a locale-root crawl must not mix another locale on the same origin" + ); + } + + #[test] + fn a_section_root_does_not_treat_article_slugs_as_sections() { + let section_root = Url::parse("https://publisher.example/news").expect("should parse root"); + let plan = plan_crawl( + §ion_root, + &[nav("/news/story-one"), nav("/news/story-two")], + &[], + CrawlBudget::default(), + ); + + assert_eq!( + plan.section_segment, 0, + "a generic one-segment root is a section, not necessarily a locale" + ); + assert_eq!( + segments(&plan), + ["news"], + "articles below a section root should remain one section" + ); + } + + #[test] + fn dropped_section_note_is_capped() { + let links: Vec<_> = (0..15) + .map(|index| nav(&format!("/section-{index:02}"))) + .collect(); + let plan = plan_crawl( + &root(), + &links, + &[], + CrawlBudget { + max_sections: 0, + max_pages: 1, + }, + ); + let note = plan + .notes + .iter() + .find(|note| note.contains("budget reached")) + .expect("should report dropped sections"); + + assert!(note.contains("and 5 more")); + assert!(!note.contains("section-14")); + } +} diff --git a/crates/trusted-server-cli/src/commands/audit/generate/evidence.rs b/crates/trusted-server-cli/src/commands/audit/generate/evidence.rs new file mode 100644 index 000000000..0e50b9a83 --- /dev/null +++ b/crates/trusted-server-cli/src/commands/audit/generate/evidence.rs @@ -0,0 +1,733 @@ +//! Cross-page slot evidence: what each slot looked like on every page it was +//! observed on. +//! +//! A single page cannot distinguish a literal ad-unit path from a templated one, +//! so inference needs the *set* of observations per slot rather than one +//! snapshot. This module accumulates that set and is deliberately the only place +//! that reconciles a slot seen more than once: +//! +//! - **Formats union.** A size that appears only on article pages (a 300x600 +//! rail, say) must survive alongside the homepage's sizes. Taking the first +//! page's formats would silently narrow the slot. +//! - **Unit paths are kept, not collapsed.** Divergence across pages is the +//! signal inference reads; discarding it is what makes templating impossible. +//! - **Network ids must agree.** Two different GAM networks in one crawl means +//! the pages are not one property, and writing either one would be a guess. +//! +//! Slots are keyed on the *normalized div stem* produced by +//! [`discover_gpt_slots`](super::gpt_slots::discover_gpt_slots), because raw GPT +//! div ids carry per-render framework hashes and would otherwise look like a new +//! slot on every page. + +use std::collections::{BTreeMap, BTreeSet}; + +use super::gpt_slots::DiscoveredSlots; +use crate::error::{CliResult, cli_error}; + +/// One observation of a slot on one page. +#[derive(Debug, Clone, PartialEq, Eq, PartialOrd, Ord)] +pub(super) struct EvidenceRow { + /// The page path the slot was observed on, normalized (leading `/`, no + /// query or fragment). + pub(super) path: String, + /// The literal GAM ad-unit path the live page used for this slot. + pub(super) unit_path: String, +} + +/// Everything observed about one slot across the crawl. +#[derive(Debug, Clone, PartialEq, Eq)] +pub(super) struct SlotEvidence { + /// Config slot id derived from the div stem. + pub(super) id: String, + /// Normalized div stem, used as the runtime `div_id` prefix. + pub(super) div_id: String, + /// Union of every pixel size observed for this slot, smallest first. + pub(super) formats: BTreeSet<(u32, u32)>, + /// Whether any page carrying this slot showed header-bidding signals. + pub(super) has_prebid: bool, + /// Distinct `(path, unit_path)` observations, in a stable order. + pub(super) rows: BTreeSet, +} + +impl SlotEvidence { + /// The distinct literal unit paths observed for this slot. + pub(super) fn unit_paths(&self) -> BTreeSet<&str> { + self.rows.iter().map(|row| row.unit_path.as_str()).collect() + } + + /// The distinct page paths this slot was observed on. + pub(super) fn paths(&self) -> BTreeSet<&str> { + self.rows.iter().map(|row| row.path.as_str()).collect() + } +} + +/// Slots grouped by the shape that would make them one placement: an identical +/// ad-unit path and an identical format set. +type SlotsByShape<'a> = BTreeMap<(String, Vec<(u32, u32)>), Vec<&'a SlotEvidence>>; + +/// Several observed slots that are really one placement under volatile div ids. +#[derive(Debug, Clone, PartialEq, Eq)] +pub(super) struct FragmentGroup { + /// The volatile div ids observed, in evidence order. + pub(super) div_ids: Vec, + /// The ad-unit path every fragment shared. + pub(super) unit_path: String, + /// The stable prefix the ids share, when they share a useful one. + /// + /// Offered to the operator as a starting point only. It is deliberately not + /// written as a `div_id`: the shared prefix reaches only as far as the + /// *observed* tokens happen to agree, so it would keep matching this crawl's + /// ids and stop matching the next render's. + pub(super) suggested_prefix: Option, +} + +/// Whether no two slots were ever seen on the same page. +fn pages_are_disjoint(slots: &[&SlotEvidence]) -> bool { + for (index, slot) in slots.iter().enumerate() { + let pages = slot.paths(); + if slots[index + 1..] + .iter() + .any(|other| other.paths().intersection(&pages).next().is_some()) + { + return false; + } + } + true +} + +/// The longest prefix the div ids share, trimmed back to a separator. +/// +/// Trimming matters: the raw common prefix usually ends mid-token (the leading +/// digits of a timestamp two fragments happen to share), which is worse than +/// useless as a suggestion. Cutting at the last `-` or `_` yields the part a +/// human would recognise as the placement's name. +fn shared_div_prefix(slots: &[&SlotEvidence]) -> Option { + let mut prefix: &str = slots.first()?.div_id.as_str(); + for slot in &slots[1..] { + let mut shared_end = 0; + for ((byte_index, left), right) in prefix.char_indices().zip(slot.div_id.chars()) { + if left != right { + break; + } + shared_end = byte_index + left.len_utf8(); + } + prefix = &prefix[..shared_end]; + } + let trimmed = prefix.trim_end_matches(|ch: char| ch != '-' && ch != '_'); + let candidate = trimmed.trim_end_matches(['-', '_']); + (!candidate.is_empty()).then(|| candidate.to_string()) +} + +/// Slot evidence accumulated across every collected page. +#[derive(Debug, Clone, Default)] +pub(super) struct EvidenceTable { + slots: BTreeMap, + /// Div stems in first-seen order, so generated config keeps crawl order + /// rather than alphabetical order. + order: Vec, + network_ids: BTreeSet, + /// Every page path folded in, including those that yielded no slots. + pages: BTreeSet, + /// Page paths that produced no slot evidence at all. + empty_pages: BTreeSet, + /// Page paths that produced slot evidence on at least one selected profile. + non_empty_pages: BTreeSet, + /// Div stems any page refused as ambiguous, unioned across the crawl. + /// + /// The verdict has to outlive the page that reached it. Article pages carry + /// several in-content units and refuse the shared prefix; a landing page + /// carries one and would otherwise contribute it as a usable slot, so the + /// written config would depend on which pages the crawl happened to sample. + ambiguous_stems: BTreeSet, + /// Normalized div IDs refused from generation but observed live. + refused_div_ids: BTreeSet, +} + +impl EvidenceTable { + /// Folds one page's discovered slots into the table. + /// + /// `path` is the page's normalized request path; it is what page patterns + /// and `{section}` derivation are computed from later, so it must be the + /// post-redirect path actually audited. + pub(super) fn fold_page(&mut self, path: &str, discovered: &DiscoveredSlots) { + self.pages.insert(path.to_string()); + if let Some(network_id) = &discovered.gam_network_id { + self.network_ids.insert(network_id.clone()); + } + if !discovered.had_slot_evidence { + if !self.non_empty_pages.contains(path) { + self.empty_pages.insert(path.to_string()); + } + return; + } + self.non_empty_pages.insert(path.to_string()); + self.empty_pages.remove(path); + self.ambiguous_stems + .extend(discovered.ambiguous_stems.iter().cloned()); + self.refused_div_ids + .extend(discovered.refused_div_ids.iter().cloned()); + + for slot in &discovered.slots { + let entry = self.slots.entry(slot.div_id.clone()).or_insert_with(|| { + self.order.push(slot.div_id.clone()); + SlotEvidence { + id: slot.id.clone(), + div_id: slot.div_id.clone(), + formats: BTreeSet::new(), + has_prebid: false, + rows: BTreeSet::new(), + } + }); + // Union rather than replace: a size seen only on one page type is + // still a size this slot serves. + entry.formats.extend(slot.formats.iter().copied()); + entry.has_prebid |= slot.has_prebid; + entry.rows.insert(EvidenceRow { + path: path.to_string(), + unit_path: slot.gam_unit_path.clone(), + }); + } + } + + /// Slots in first-seen order, excluding stems any page refused as ambiguous. + pub(super) fn slots(&self) -> impl Iterator { + self.order + .iter() + .filter(|div_id| !self.ambiguous_stems.contains(*div_id)) + .filter_map(|div_id| self.slots.get(div_id)) + } + + /// Every normalized div ID observed, including all refused evidence. + pub(super) fn observed_div_ids(&self) -> impl Iterator { + self.order + .iter() + .map(String::as_str) + .chain(self.ambiguous_stems.iter().map(String::as_str)) + .chain(self.refused_div_ids.iter().map(String::as_str)) + } + + /// Number of usable distinct slots observed. + pub(super) fn slot_count(&self) -> usize { + self.slots().count() + } + + /// Every page path folded in, whether or not it yielded slots. + pub(super) fn pages(&self) -> &BTreeSet { + &self.pages + } + + /// Page paths that produced no slot evidence. + /// + /// A high proportion of these is the signature of a bot challenge serving + /// interstitials instead of the real site, which is worth refusing to write + /// from rather than persisting a half-empty config. + pub(super) fn empty_pages(&self) -> &BTreeSet { + &self.empty_pages + } + + /// Whether any slot was observed at all, ambiguous ones included. + /// + /// Deliberately not `slot_count() == 0`: a crawl that saw only ambiguous + /// placements did observe an ad stack, and the caller distinguishes "this + /// page has no slots" from "every slot found was refused". + pub(super) fn is_empty(&self) -> bool { + self.slots.is_empty() + } + + /// Groups of slots that are one slot wearing a different div id per page. + /// + /// Some ad stacks build div ids from a per-render token — a timestamp, a + /// framework id — so the same placement arrives under a new key on every + /// page. Written verbatim those ids never match at runtime, and the + /// fragmentation also starves template inference, which needs to see one + /// slot more than once. + /// + /// Detection is by evidence rather than by guessing at token shapes, because + /// each stack invents its own. Candidates share an identical ad-unit path and + /// identical formats; what separates a fragmented slot from two legitimate + /// siblings on the same unit is **co-occurrence**. Real siblings appear + /// together on a page; fragments of one slot never do, because each page + /// produces exactly one of them. + pub(super) fn fragmented_slots(&self) -> Vec { + let mut by_shape: SlotsByShape<'_> = BTreeMap::new(); + for slot in self.slots() { + // Only slots pinned to exactly one unit path can be compared this + // way; a slot whose unit varies is inference's problem, not this one. + let units = slot.unit_paths(); + if units.len() != 1 { + continue; + } + let unit = (*units.iter().next().expect("should have one unit path")).to_string(); + let formats: Vec<(u32, u32)> = slot.formats.iter().copied().collect(); + by_shape.entry((unit, formats)).or_default().push(slot); + } + + by_shape + .into_iter() + .filter(|(_, slots)| slots.len() > 1) + .filter(|(_, slots)| pages_are_disjoint(slots)) + .filter_map(|((unit_path, _), slots)| { + let suggested_prefix = shared_div_prefix(&slots); + (suggested_prefix.is_some() || slots.len() >= 3).then(|| FragmentGroup { + div_ids: slots.iter().map(|slot| slot.div_id.clone()).collect(), + unit_path, + suggested_prefix, + }) + }) + .collect() + } + + /// The single GAM network id observed across the crawl. + /// + /// # Errors + /// + /// Returns an error when pages disagreed. Two networks in one crawl means + /// the pages are not one property (a syndicated subdomain, a child network, + /// an off-origin redirect that slipped through), and picking either would be + /// a guess that silently bids against the wrong inventory. + pub(super) fn network_id(&self) -> CliResult> { + let mut found = self.network_ids.iter(); + let Some(first) = found.next() else { + return Ok(None); + }; + if self.network_ids.len() > 1 { + let all: Vec<&str> = self.network_ids.iter().map(String::as_str).collect(); + return cli_error(format!( + "the crawled pages reported more than one GAM network id ({}); \ + they do not appear to be one property, so no network id can be \ + chosen safely. Audit a single property, or pass explicit URLs", + all.join(", ") + )); + } + Ok(Some(first.clone())) + } +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::commands::audit::generate::collector::CollectedGptSlot; + use crate::commands::audit::generate::gpt_slots::discover_gpt_slots; + + /// One live slot as `(unit path, div id, sizes)`. + type SlotFixture<'a> = (&'a str, &'a str, &'a [(u32, u32)]); + + fn page(slots: &[SlotFixture<'_>], has_prebid: bool) -> DiscoveredSlots { + let registry: Vec = slots + .iter() + .map(|(unit_path, div_id, sizes)| CollectedGptSlot { + gam_unit_path: (*unit_path).to_string(), + div_id: (*div_id).to_string(), + sizes: sizes.to_vec(), + }) + .collect(); + discover_gpt_slots(®istry, &[], has_prebid) + } + + #[test] + fn formats_union_across_pages_instead_of_first_seen_winning() { + // The 300x600 rail only ever renders on article pages. Keeping the + // homepage's format list alone would silently narrow the slot. + let mut table = EvidenceTable::default(); + table.fold_page( + "/", + &page(&[("/123/site/home", "ad-rail", &[(300, 250)])], false), + ); + table.fold_page( + "/news/story", + &page(&[("/123/site/news", "ad-rail", &[(300, 600)])], false), + ); + + let slot = table.slots().next().expect("should have one slot"); + assert_eq!( + slot.formats.iter().copied().collect::>(), + [(300, 250), (300, 600)], + "both pages' sizes should survive" + ); + assert_eq!(table.slot_count(), 1, "one div stem is one slot"); + } + + #[test] + fn divergent_unit_paths_are_preserved_as_separate_rows() { + // This divergence is the entire signal template inference reads. + let mut table = EvidenceTable::default(); + table.fold_page( + "/", + &page(&[("/123/site/home", "ad-header", &[(728, 90)])], false), + ); + table.fold_page( + "/news/story", + &page(&[("/123/site/news", "ad-header", &[(728, 90)])], false), + ); + + let slot = table.slots().next().expect("should have one slot"); + assert_eq!( + slot.unit_paths().into_iter().collect::>(), + ["/123/site/home", "/123/site/news"], + "both observed unit paths must be retained" + ); + assert_eq!( + slot.paths().into_iter().collect::>(), + ["/", "/news/story"] + ); + } + + #[test] + fn repeated_identical_observations_collapse() { + let mut table = EvidenceTable::default(); + let observed = page(&[("/123/site/home", "ad-header", &[(728, 90)])], false); + table.fold_page("/", &observed); + table.fold_page("/", &observed); + + let slot = table.slots().next().expect("should have one slot"); + assert_eq!(slot.rows.len(), 1, "the same page twice is one observation"); + } + + #[test] + fn prebid_is_sticky_once_any_page_shows_it() { + let mut table = EvidenceTable::default(); + table.fold_page( + "/", + &page(&[("/123/site/home", "ad-header", &[(728, 90)])], false), + ); + table.fold_page( + "/news/story", + &page(&[("/123/site/news", "ad-header", &[(728, 90)])], true), + ); + + let slot = table.slots().next().expect("should have one slot"); + assert!( + slot.has_prebid, + "a slot proven to run prebid on any page runs prebid" + ); + } + + #[test] + fn slots_keep_first_seen_order_not_alphabetical_order() { + let mut table = EvidenceTable::default(); + table.fold_page( + "/", + &page( + &[ + ("/123/site/home", "zeta-slot", &[(728, 90)]), + ("/123/site/home", "alpha-slot", &[(300, 250)]), + ], + false, + ), + ); + + let ids: Vec<&str> = table.slots().map(|slot| slot.div_id.as_str()).collect(); + assert_eq!( + ids, + ["zeta-slot", "alpha-slot"], + "generated config should follow crawl order" + ); + } + + #[test] + fn one_placement_under_per_render_div_ids_is_detected() { + // Each page yields a new key for the same placement: same unit, same + // formats, never co-occurring. The tokens here deliberately do *not* + // match the digit-led shape `discover_gpt_slots` refuses on sight, so + // this exercises the evidence-based detector that catches the stacks + // whose token shape cannot be recognized from one observation. + let mut table = EvidenceTable::default(); + for (path, div) in [ + ("/features/a", "ex_slot_ce6Bj0uc8sL0aa_overlay_1"), + ("/news/b", "ex_slot_aoYmv4RQyN3nbb_overlay_1"), + ("/deals/c", "ex_slot_mYPDB3tz8cpBcc_overlay_1"), + ] { + table.fold_page( + path, + &page(&[("/99/site_Overlay", div, &[(300, 250)])], false), + ); + } + + let groups = table.fragmented_slots(); + + assert_eq!(groups.len(), 1, "the three fragments should form one group"); + assert_eq!(groups[0].div_ids.len(), 3); + assert_eq!(groups[0].unit_path, "/99/site_Overlay"); + assert_eq!( + groups[0].suggested_prefix.as_deref(), + Some("ex_slot"), + "the suggestion should be trimmed back off the volatile token" + ); + } + + #[test] + fn an_ambiguous_stem_stays_refused_on_every_page() { + // The article page carries two in-content units and refuses the shared + // prefix; the landing page carries one. Folding the landing page must + // not resurrect a prefix that cannot resolve to one element site-wide. + let mut table = EvidenceTable::default(); + table.fold_page( + "/news/story", + &page( + &[ + ( + "/123/site/news", + "ad-in_content-de669245b2ea4b05826dc96f07a36272-in_content-0", + &[(300, 250)], + ), + ( + "/123/site/news", + "ad-in_content-8aec8129a83d4e5abc197423120cb19e-in_content-1", + &[(300, 250)], + ), + ], + false, + ), + ); + table.fold_page( + "/", + &page( + &[( + "/123/site/home", + "ad-in_content-1c0de08e5a2f4d6f9b3a7e5c8d1f2a4b-in_content-0", + &[(300, 250)], + )], + false, + ), + ); + + assert_eq!( + table.slots().count(), + 0, + "a stem refused on one page must stay refused, got {:?}", + table.slots().map(|slot| &slot.div_id).collect::>() + ); + assert_eq!( + table.slot_count(), + 0, + "the count should match what is written" + ); + assert!( + !table.is_empty(), + "the crawl did observe an ad stack, so this is not an empty result" + ); + } + + #[test] + fn genuine_siblings_on_one_unit_are_not_treated_as_fragments() { + // Two real in-content positions can share a unit path and formats. What + // distinguishes them from fragments is that they appear *together* on a + // page, so refusing to write them would lose real inventory. + let mut table = EvidenceTable::default(); + table.fold_page( + "/news/story", + &page( + &[ + ("/99/site/news", "ad-in_content-1", &[(300, 250)]), + ("/99/site/news", "ad-in_content-2", &[(300, 250)]), + ], + false, + ), + ); + + assert!( + table.fragmented_slots().is_empty(), + "co-occurring slots are siblings, not fragments" + ); + } + + #[test] + fn slots_differing_in_formats_are_not_fragments() { + let mut table = EvidenceTable::default(); + table.fold_page( + "/a", + &page(&[("/99/site/x", "slot-aaaa", &[(300, 250)])], false), + ); + table.fold_page( + "/b", + &page(&[("/99/site/x", "slot-bbbb", &[(728, 90)])], false), + ); + + assert!( + table.fragmented_slots().is_empty(), + "a differing format set means these are different placements" + ); + } + + #[test] + fn a_slot_seen_alone_is_never_a_fragment() { + let mut table = EvidenceTable::default(); + table.fold_page( + "/a", + &page(&[("/99/site/x", "only-slot", &[(300, 250)])], false), + ); + + assert!(table.fragmented_slots().is_empty()); + } + + #[test] + fn fragments_with_no_shared_prefix_report_none() { + let mut table = EvidenceTable::default(); + table.fold_page( + "/a", + &page(&[("/99/site/x", "alpha-1111", &[(300, 250)])], false), + ); + table.fold_page( + "/b", + &page(&[("/99/site/x", "beta-2222", &[(300, 250)])], false), + ); + + let groups = table.fragmented_slots(); + + assert!( + groups.is_empty(), + "two unrelated placements are too ambiguous to classify as fragments" + ); + } + + #[test] + fn three_disjoint_same_shape_ids_are_fragment_evidence_without_a_prefix() { + let mut table = EvidenceTable::default(); + for (path, div_id) in [("/a", "alpha"), ("/b", "bravo"), ("/c", "charlie")] { + table.fold_page(path, &page(&[("/99/site/x", div_id, &[(300, 250)])], false)); + } + + let groups = table.fragmented_slots(); + assert_eq!(groups.len(), 1); + assert_eq!(groups[0].suggested_prefix, None); + } + + #[test] + fn unicode_shared_prefix_uses_a_utf8_boundary() { + let mut table = EvidenceTable::default(); + table.fold_page( + "/a", + &page(&[("/99/site/x", "ünicode-ad-a", &[(300, 250)])], false), + ); + table.fold_page( + "/b", + &page(&[("/99/site/x", "ünicode-ad-b", &[(300, 250)])], false), + ); + + let groups = table.fragmented_slots(); + assert_eq!(groups.len(), 1); + assert_eq!(groups[0].suggested_prefix.as_deref(), Some("ünicode-ad")); + } + + #[test] + fn a_later_non_empty_profile_clears_the_empty_page_marker() { + let mut table = EvidenceTable::default(); + table.fold_page("/news", &page(&[], false)); + table.fold_page( + "/news", + &page(&[("/99/site/news", "ad-atf", &[(300, 250)])], false), + ); + + assert!(table.empty_pages().is_empty()); + } + + #[test] + fn a_later_empty_profile_does_not_re_mark_a_non_empty_page() { + let mut table = EvidenceTable::default(); + table.fold_page( + "/news", + &page(&[("/99/site/news", "ad-atf", &[(300, 250)])], false), + ); + table.fold_page("/news", &page(&[], false)); + + assert!( + table.empty_pages().is_empty(), + "emptiness is a page-level fact across all selected profiles" + ); + } + + #[test] + fn conflicting_network_ids_are_a_hard_error() { + let mut table = EvidenceTable::default(); + table.fold_page( + "/", + &page(&[("/111/site/home", "ad-header", &[(728, 90)])], false), + ); + table.fold_page( + "/news/story", + &page(&[("/222/site/news", "ad-header", &[(728, 90)])], false), + ); + + let error = table + .network_id() + .expect_err("two networks in one crawl should not resolve"); + + let rendered = format!("{error:?}"); + assert!( + rendered.contains("111") && rendered.contains("222"), + "the error should name both observed ids, got {rendered}" + ); + } + + #[test] + fn agreeing_network_ids_resolve_to_one_value() { + let mut table = EvidenceTable::default(); + table.fold_page( + "/", + &page(&[("/123/site/home", "ad-header", &[(728, 90)])], false), + ); + table.fold_page( + "/news/story", + &page(&[("/123/site/news", "ad-header", &[(728, 90)])], false), + ); + + assert_eq!( + table.network_id().expect("agreeing ids should resolve"), + Some("123".to_string()) + ); + } + + #[test] + fn pages_without_slots_are_recorded_for_challenge_detection() { + let mut table = EvidenceTable::default(); + table.fold_page( + "/", + &page(&[("/123/site/home", "ad-header", &[(728, 90)])], false), + ); + table.fold_page("/blocked", &page(&[], false)); + + assert_eq!( + table + .empty_pages() + .iter() + .map(String::as_str) + .collect::>(), + ["/blocked"], + "a slot-less page must be visible to the caller, not silently dropped" + ); + assert_eq!( + table.pages().len(), + 2, + "every folded page should be counted" + ); + } + + #[test] + fn collision_only_page_is_not_classified_as_empty() { + let discovered = page( + &[ + ("/123/site/home", "ad-x-aaaaaaaaaaaaaaaa-0", &[(300, 250)]), + ("/123/site/home", "ad-x-bbbbbbbbbbbbbbbb-1", &[(300, 250)]), + ], + false, + ); + let mut table = EvidenceTable::default(); + + table.fold_page("/collision-only", &discovered); + + assert!(discovered.had_slot_evidence); + assert!(discovered.slots.is_empty()); + assert!( + table.empty_pages().is_empty(), + "intentionally omitted GPT evidence must not look like a bot challenge" + ); + } + + #[test] + fn empty_table_resolves_no_network_id_rather_than_erroring() { + let table = EvidenceTable::default(); + + assert!(table.is_empty()); + assert_eq!(table.network_id().expect("empty is not a conflict"), None); + } +} diff --git a/crates/trusted-server-cli/src/commands/audit/generate/gpt_slots.rs b/crates/trusted-server-cli/src/commands/audit/generate/gpt_slots.rs new file mode 100644 index 000000000..c05e3efbd --- /dev/null +++ b/crates/trusted-server-cli/src/commands/audit/generate/gpt_slots.rs @@ -0,0 +1,1483 @@ +//! Reconstructs `[creative_opportunities]` slots from a live page's GPT state. +//! +//! Two complementary sources feed the reconstruction: +//! +//! 1. The **live GPT registry** (`googletag.pubads().getSlots()`) is the primary +//! source. It exposes each defined slot's ad-unit path, div id, and sizes +//! directly, and is populated at `defineSlot` time — so it captures slots even +//! when the ad request never fires (consent-gated stacks, iframe-issued +//! requests). It carries no per-slot header-bidding signal, so Prebid is +//! inferred from page-level detection. +//! 2. Captured **`gampad/ads` requests** are a fallback for any div the registry +//! did not report. Each request URL encodes the ad-unit path (`iu_parts`), div +//! id (`dids`), sizes (`prev_iu_szs`), and targeting (`prev_scp`, which does +//! carry a per-slot Prebid signal). +//! +//! Neither source executes the page's ad-stack logic ourselves; both read state +//! the page's own GPT/Prebid setup produced. + +use std::collections::{BTreeMap, BTreeSet}; +use std::sync::LazyLock; + +use regex::Regex; +use trusted_server_core::creative_opportunities::validate_slot_id; +use url::Url; + +use crate::commands::audit::generate::collector::{CollectedGptSlot, CollectedRequest}; + +/// A hyphen-delimited hex hash *segment* (16+ hex chars bounded by `-` or end), +/// e.g. the UUID GPT embeds in `ad-in_content--in_content-0`. Marks the +/// start of ephemeral div-id noise, like the React `_R_` hash. The trailing +/// boundary avoids truncating a legit token that merely starts with hex-like +/// characters (only `start()` of the match is used). +static HEX_HASH_SEGMENT: LazyLock = + LazyLock::new(|| Regex::new(r"-[0-9a-f]{16,}(?:-|$)").expect("should compile hex hash regex")); +static UUID_SEGMENT: LazyLock = LazyLock::new(|| { + Regex::new(r"-[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}(?:-|$)") + .expect("should compile UUID regex") +}); + +/// Matches a React `useId` token, which changes on every render. +/// +/// React emits these in both cases — `_R_3f_` from a server render and `_r_0_` +/// from a client one — so matching only the uppercase form leaves the lowercase +/// variant in the stem. That is not merely untidy: the suffix differs per +/// render, so one logical slot fragments into a new key on every page, which +/// both breaks runtime div matching and starves template inference of the +/// repeated observations it needs. +/// +/// The uppercase form is distinctive enough to match bare, and its hash is +/// included so the match spans the whole ephemeral token — [`normalize_div_stem`] +/// only reads the match *start*, but [`ephemeral_marker_residue`] excises the +/// match, and a residue that still carried the hash would make two renders of one +/// element look like two elements. The lowercase form is anchored (`_r_`, a short +/// alphanumeric run, `_`) so an ordinary id that merely contains `_r_` keeps its +/// full stem. +static REACT_USE_ID: LazyLock = LazyLock::new(|| { + Regex::new(r"_R_[0-9a-z]*_?|_r_[0-9a-z]{1,8}_").expect("should compile react id regex") +}); + +/// Hosts that serve GPT `gampad/ads` requests. +const GAMPAD_HOSTS: &[&str] = &["securepubads.g.doubleclick.net", "pubads.g.doubleclick.net"]; + +/// Common GPT div-id prefix stripped when deriving a slot id. +const GPT_DIV_PREFIX: &str = "div-gpt-ad-"; + +/// Minimum width/height for a format to be treated as a real creative size. +/// +/// GPT encodes fluid/native aspect-ratio markers (e.g. `4x1`, `8x1`) alongside +/// pixel sizes in `prev_iu_szs`; those are not banner dimensions, so they are +/// dropped from the drafted `formats`. +const MIN_FORMAT_DIMENSION: u32 = 50; + +/// A slot reconstructed from a single GPT ad request. +#[derive(Debug, Clone, PartialEq, Eq)] +pub(crate) struct DiscoveredSlot { + /// Slot id derived from the div id (GPT prefix stripped). + pub(crate) id: String, + /// The HTML div id that holds the creative. + pub(crate) div_id: String, + /// The full GAM ad-unit path (e.g. `/123/desktop/homepage/leaderboard`). + pub(crate) gam_unit_path: String, + /// Candidate creative sizes as `(width, height)` pixel pairs. + pub(crate) formats: Vec<(u32, u32)>, + /// Whether the slot's targeting shows Prebid/header-bidding signals. + pub(crate) has_prebid: bool, +} + +/// The result of scanning captured requests for GPT slots. +#[derive(Debug, Clone, Default, PartialEq, Eq)] +pub(crate) struct DiscoveredSlots { + /// GAM network id shared by the discovered slots, if any were found. + pub(crate) gam_network_id: Option, + /// Whether the page exposed any otherwise usable slot evidence, including + /// ambiguous placements that were intentionally omitted from `slots`. + pub(crate) had_slot_evidence: bool, + /// The reconstructed slots, deduplicated by div id in first-seen order. + pub(crate) slots: Vec, + /// Div stems refused because several live elements normalized onto them. + /// + /// Carried separately from `slots` because the verdict is a property of the + /// *site*, not of this page: another page that happens to render only one + /// member of the group must not resurrect the ambiguous prefix. + pub(crate) ambiguous_stems: BTreeSet, + /// Normalized div IDs refused from generation but still observed live. + pub(crate) refused_div_ids: BTreeSet, + /// Diagnostics for placements whose normalized stable stems collided. + pub(crate) warnings: Vec, +} + +/// Reconstructs GPT slots from the page's live registry and ad requests. +/// +/// The live registry (`googletag.pubads().getSlots()`) is the primary source: it +/// carries the authoritative path/div/size for every defined slot and is present +/// even when the ad request never fires. Captured `gampad/ads` requests are a +/// fallback for any div the registry did not report, and also supply per-slot +/// Prebid signals. Slots are deduplicated by div id in first-seen order. +/// +/// `page_has_prebid` marks registry slots as Prebid-enabled when the page as a +/// whole was detected running Prebid (the registry alone carries no such signal). +pub(crate) fn discover_gpt_slots( + registry: &[CollectedGptSlot], + requests: &[CollectedRequest], + page_has_prebid: bool, +) -> DiscoveredSlots { + let mut slots = Vec::new(); + let mut warnings = Vec::new(); + let mut ambiguous_stems = BTreeSet::new(); + let mut refused_div_ids = BTreeSet::new(); + let mut gam_network_id = None; + let mut had_slot_evidence = false; + let mut registry_residues: BTreeMap> = BTreeMap::new(); + // Stems refused outright, so the request fallback cannot re-add them. Kept + // apart from `registry_residues` so a later registry entry cannot read a + // refused stem as a one-member collision group. + let mut refused_stems: BTreeSet = BTreeSet::new(); + + for entry in registry { + let Some(slot) = slot_from_registry(entry, page_has_prebid) else { + continue; + }; + had_slot_evidence = true; + if gam_network_id.is_none() { + gam_network_id = network_id_from_unit_path(&entry.gam_unit_path); + } + if let Some(prefix) = volatile_prefix_before_placement(&entry.div_id) { + refused_stems.insert(slot.div_id.clone()); + refused_div_ids.insert(slot.div_id); + push_unique_warning(&mut warnings, volatile_prefix_warning(&prefix)); + continue; + } + if let Some(prefix) = + push_slot_refusing_collisions(&mut slots, &mut registry_residues, slot, &entry.div_id) + { + warnings.push(ambiguous_collision_warning(&prefix)); + ambiguous_stems.insert(prefix); + } + } + + let registry_stems: BTreeSet = registry_residues + .keys() + .cloned() + .chain(refused_stems) + .collect(); + let mut request_residues: BTreeMap> = BTreeMap::new(); + for request in requests { + let Some((network_id, slot, raw_div)) = parse_gampad_request(&request.url) else { + continue; + }; + had_slot_evidence = true; + if gam_network_id.is_none() { + gam_network_id = Some(network_id); + } + if registry_stems.contains(&slot.div_id) { + continue; + } + if let Some(prefix) = volatile_prefix_before_placement(&raw_div) { + refused_div_ids.insert(slot.div_id); + push_unique_warning(&mut warnings, volatile_prefix_warning(&prefix)); + continue; + } + if let Some(prefix) = + push_slot_refusing_collisions(&mut slots, &mut request_residues, slot, &raw_div) + { + warnings.push(ambiguous_collision_warning(&prefix)); + ambiguous_stems.insert(prefix); + } + } + make_slot_ids_unique(&mut slots); + + DiscoveredSlots { + gam_network_id, + had_slot_evidence, + slots, + ambiguous_stems, + refused_div_ids, + warnings, + } +} + +/// Adds one source-local slot unless two distinct *elements* share its stem. +/// +/// Sharing a stem is not by itself ambiguity: one element re-rendered under a +/// fresh framework token is exactly what normalization exists to absorb, and it +/// produces two raw ids that collapse onto one stem. Ambiguity is two elements, +/// which [`ephemeral_marker_residue`] separates from two renders of one. +/// +/// The first distinct residue removes the tentatively accepted slot and returns +/// its stem for one diagnostic. Repeats and later collision members stay +/// suppressed and return `None`. +fn push_slot_refusing_collisions( + slots: &mut Vec, + seen_residues: &mut BTreeMap>, + slot: DiscoveredSlot, + raw_div: &str, +) -> Option { + let normalized = slot.div_id.clone(); + let residue = ephemeral_marker_residue(raw_div); + match seen_residues.get_mut(&normalized) { + None => { + seen_residues.insert(normalized, BTreeSet::from([residue])); + slots.push(slot); + None + } + Some(residues) if residues.contains(&residue) => None, + Some(residues) => { + let became_ambiguous = residues.len() == 1; + residues.insert(residue); + if became_ambiguous { + slots.retain(|entry| entry.div_id != normalized); + Some(normalized) + } else { + None + } + } + } +} + +/// Operator-facing text for a stem several live elements normalized onto. +fn ambiguous_collision_warning(prefix: &str) -> String { + format!( + "skipped ambiguous div-id prefix `{prefix}`: multiple active elements normalized to it, \ + but the runtime can resolve a prefix to only one active element and exact div ids change \ + across renders; expose distinct stable div ids in publisher markup before configuring \ + these placements" + ) +} + +/// The stable prefix of a div id whose per-render token precedes more of the id. +/// +/// Some ad stacks build ids as `__` — a +/// millisecond timestamp plus a random suffix sitting *before* the part that +/// distinguishes one placement from the next. Such an id can be written neither +/// literally (the token changes on the next render) nor as a prefix: the only +/// stable prefix stops at the token, and that prefix reaches every placement in +/// the family, while the runtime resolves a prefix to a single element. So the +/// slot is refused from a single observation, without waiting for a second +/// placement to prove the collision. +/// +/// The shape decides, not the vendor: any segment that is a long digit run +/// followed by more alphanumerics counts, so a new stack with the same layout +/// needs no code change. A token in *trailing* position is deliberately not this +/// case — everything before it still identifies the element — and is left to +/// normalization and the same-page collision check. +fn volatile_prefix_before_placement(div_id: &str) -> Option { + let div_id = div_id.strip_suffix("-container").unwrap_or(div_id); + let mut start = 0_usize; + for (index, character) in div_id.char_indices() { + if character != '_' && character != '-' { + continue; + } + if is_per_render_token(&div_id[start..index]) { + let prefix = div_id[..start].trim_end_matches(['_', '-']); + // A delimiter is one byte, so the remainder starts just past it. + return (!prefix.is_empty() && !div_id[index + 1..].is_empty()) + .then(|| prefix.to_string()); + } + start = index + character.len_utf8(); + } + None +} + +/// Whether one div-id segment is a per-render token: a long leading digit run +/// followed by alphanumerics, or a shorter counter paired with a long random +/// suffix. +/// +/// Both halves are required. Eight-digit values need at least eight suffix +/// characters and a mixed-case random-looking suffix; this avoids treating +/// calendar labels followed by stable words as generated ids. A bare digit run +/// is how publishers write stable placement indices, and a token with a +/// non-alphanumeric character is some other structure than a generated id. +fn is_per_render_token(segment: &str) -> bool { + let leading_digits = segment.bytes().take_while(u8::is_ascii_digit).count(); + let suffix_length = segment.len().saturating_sub(leading_digits); + let suffix = &segment[leading_digits..]; + ((leading_digits >= 10 && suffix_length >= 1) + || (leading_digits >= 8 && suffix_length >= 8 && has_random_case_alternation(suffix))) + && segment.bytes().all(|byte| byte.is_ascii_alphanumeric()) +} + +/// Whether letter case alternates densely enough to resemble a random token. +fn has_random_case_alternation(value: &str) -> bool { + let mut previous = None; + let mut comparisons = 0_usize; + let mut transitions = 0_usize; + for uppercase in value.bytes().filter_map(|byte| { + byte.is_ascii_lowercase() + .then_some(false) + .or_else(|| byte.is_ascii_uppercase().then_some(true)) + }) { + if let Some(previous) = previous { + comparisons += 1; + transitions += usize::from(previous != uppercase); + } + previous = Some(uppercase); + } + transitions >= 3 && transitions.saturating_mul(3) >= comparisons.saturating_mul(2) +} + +/// Operator-facing text for a div-id family carrying a per-render token. +fn volatile_prefix_warning(prefix: &str) -> String { + format!( + "skipped volatile div-id family `{prefix}`: a per-render token sits before the placement \ + suffix, so exact div ids change across renders and no distinct stable element prefix is \ + available; expose distinct stable div ids in publisher markup before configuring these \ + placements" + ) +} + +/// Records `warning` unless the same text was already recorded for this page. +fn push_unique_warning(warnings: &mut Vec, warning: String) { + if !warnings.contains(&warning) { + warnings.push(warning); + } +} + +/// Converts a live-registry slot into a [`DiscoveredSlot`]. +/// +/// Returns `None` when the slot has no usable pixel size or its div id is a +/// multi-slot (SRA) concatenation rather than a single element. +fn slot_from_registry(entry: &CollectedGptSlot, page_has_prebid: bool) -> Option { + if is_multi_slot_div(&entry.div_id) { + return None; + } + if !is_usable_unit_path(&entry.gam_unit_path) { + return None; + } + let formats: Vec<(u32, u32)> = entry + .sizes + .iter() + .copied() + .filter(|(width, height)| *width >= MIN_FORMAT_DIMENSION && *height >= MIN_FORMAT_DIMENSION) + .collect(); + if formats.is_empty() { + return None; + } + let div_stem = normalize_div_stem(&entry.div_id); + // Normalization truncates at the first ephemeral marker, so a div id that is + // *entirely* ephemeral (`_R_9sl…`, or exactly `-container`) reduces to the + // empty string. An empty `div_id` override fails config load outright, and + // an empty prefix would bind the slot to the first id-bearing element on the + // page, so such a slot is unusable rather than merely imprecise. + if div_stem.is_empty() { + return None; + } + Some(DiscoveredSlot { + id: slot_id_from_div(&div_stem), + div_id: div_stem, + gam_unit_path: entry.gam_unit_path.clone(), + formats, + has_prebid: page_has_prebid, + }) +} + +/// Whether a div id is a GPT single-request (SRA) concatenation of multiple +/// slots (joined with `~`) rather than one element. +fn is_multi_slot_div(div_id: &str) -> bool { + div_id.contains('~') +} + +/// Whether a scraped GAM ad-unit path can be represented in config. +/// +/// `gam_unit_path` is a template: `{` and `}` delimit placeholders and +/// [`parse_unit_template`](trusted_server_core::creative_opportunities) offers no +/// escape syntax. A live path containing a brace would either fail config load +/// or, worse, be silently reinterpreted as a placeholder-bearing template. A +/// blank path is rejected for the same reason config load rejects it. +fn is_usable_unit_path(path: &str) -> bool { + !path.trim().is_empty() && !path.contains(['{', '}']) +} + +/// Strips ephemeral GPT div-id noise so the stored id is stable across renders. +/// +/// Removes a trailing `-container` wrapper, then truncates at the first ephemeral +/// marker — a React SSR hash (`_R_`) or a hex-UUID segment — since both +/// change on every page load. Truncating (rather than excising) keeps the result +/// a valid **prefix** of the live div id, which is how verify matches slots. +/// +/// `div-gpt-ad-leaderboard-1` (stable) is unchanged; `ad-header-0-_R_9sl…-container` +/// and `ad-header-0-_r_8_` → `ad-header-0`; `ad-in_content-de66…f272-in_content-0` +/// → `ad-in_content`. +fn normalize_div_stem(div_id: &str) -> String { + let stem = div_id.strip_suffix("-container").unwrap_or(div_id); + let cut = ephemeral_marker_ranges(stem) + .first() + .map_or(stem.len(), |range| range.start); + stem[..cut].trim_end_matches('-').to_string() +} + +/// Byte ranges of every ephemeral per-render marker in `stem`, in order and +/// without overlaps. +/// +/// A hex-hash candidate must contain at least one `a`-`f`; a run of 16+ digits +/// is how publishers write stable ids, not a hash. +fn ephemeral_marker_ranges(stem: &str) -> Vec> { + let mut ranges: Vec> = REACT_USE_ID + .find_iter(stem) + .chain(UUID_SEGMENT.find_iter(stem)) + .chain(HEX_HASH_SEGMENT.find_iter(stem).filter(|matched| { + matched + .as_str() + .bytes() + .any(|byte| matches!(byte, b'a'..=b'f')) + })) + .map(|matched| matched.range()) + .collect(); + ranges.sort_by_key(|range| range.start); + let mut merged: Vec> = Vec::with_capacity(ranges.len()); + for range in ranges { + match merged.last_mut() { + Some(last) if range.start < last.end => last.end = last.end.max(range.end), + _ => merged.push(range), + } + } + merged +} + +/// The parts of a raw div id that no ephemeral marker covered, NUL-joined. +/// +/// [`normalize_div_stem`] truncates at the first marker, so two ids differing +/// only *inside* a marker collapse onto one stem — the signature of one element +/// re-rendered. What the markers did not cover separates that from two elements: +/// `ad-header-0-_R_3f_` and `ad-header-0-_r_0_` leave the same residue (one +/// element, two renders), while `…-in_content-0` and `…-in_content-1` do not +/// (two siblings). A live div id cannot contain NUL, so joining on it cannot +/// make two different residues compare equal. +fn ephemeral_marker_residue(div_id: &str) -> String { + let stem = div_id.strip_suffix("-container").unwrap_or(div_id); + let mut residue = String::with_capacity(stem.len()); + let mut previous = 0_usize; + for range in ephemeral_marker_ranges(stem) { + residue.push_str(&stem[previous..range.start]); + residue.push('\0'); + previous = range.end; + } + residue.push_str(&stem[previous..]); + residue +} + +/// Extracts the leading network id from a GAM ad-unit path (`//...`). +fn network_id_from_unit_path(path: &str) -> Option { + let segment = path.trim_start_matches('/').split('/').next()?; + (!segment.is_empty() && segment.bytes().all(|byte| byte.is_ascii_digit())) + .then(|| segment.to_string()) +} + +/// Parses a single `gampad/ads` request URL into `(network_id, slot)`. +/// +/// Returns `None` when the URL is not a GPT ad request or is missing the fields +/// needed to describe a slot (ad-unit path, div id, and at least one size). +fn parse_gampad_request(raw_url: &str) -> Option<(String, DiscoveredSlot, String)> { + let url = Url::parse(raw_url).ok()?; + let host = url.host_str()?; + if !GAMPAD_HOSTS.contains(&host) || !url.path().ends_with("/gampad/ads") { + return None; + } + + let mut iu_parts = None; + let mut dids = None; + let mut sizes_raw = None; + let mut fallback_sizes_raw = None; + let mut scp = None; + for (key, value) in url.query_pairs() { + match key.as_ref() { + "iu_parts" => iu_parts = Some(value.into_owned()), + "dids" => dids = Some(value.into_owned()), + "prev_iu_szs" => sizes_raw = Some(value.into_owned()), + "pb_szs" => fallback_sizes_raw = Some(value.into_owned()), + "prev_scp" => scp = Some(value.into_owned()), + _ => {} + } + } + + let iu_parts = iu_parts?; + let mut parts = iu_parts.split(',').filter(|part| !part.is_empty()); + // Mirror the registry path's validation: a GAM network id is digits only. + // The percent-decoded query value is page-controlled and gets spliced into + // generated TOML, so reject anything else. + let network_id = parts + .next() + .filter(|segment| segment.bytes().all(|byte| byte.is_ascii_digit()))? + .to_string(); + let gam_unit_path = format!("/{}", iu_parts.replace(',', "/")); + if !is_usable_unit_path(&gam_unit_path) { + return None; + } + // A usable unit path needs the network id plus at least one path segment. + parts.next()?; + + let raw_div = dids?; + if raw_div.contains(',') { + return None; + } + let raw_div = raw_div.trim().to_string(); + if raw_div.is_empty() { + return None; + } + if is_multi_slot_div(&raw_div) { + return None; + } + let div_id = normalize_div_stem(&raw_div); + // See `slot_from_registry`: a fully ephemeral div id normalizes to nothing, + // which is neither a valid config value nor a usable runtime prefix. + if div_id.is_empty() { + return None; + } + + let formats = parse_sizes(sizes_raw.as_deref().or(fallback_sizes_raw.as_deref())?); + if formats.is_empty() { + return None; + } + + let id = slot_id_from_div(&div_id); + let has_prebid = scp.as_deref().is_some_and(scp_shows_prebid); + + Some(( + network_id, + DiscoveredSlot { + id, + div_id, + gam_unit_path, + formats, + has_prebid, + }, + raw_div, + )) +} + +/// Parses a GPT size list (e.g. `970x250|4x1|620x366`) into pixel pairs. +/// +/// Accepts `|` or `,` separators, ignores non-`WxH` tokens, and drops +/// fluid/native ratio markers below [`MIN_FORMAT_DIMENSION`]. +fn parse_sizes(raw: &str) -> Vec<(u32, u32)> { + let mut sizes = Vec::new(); + for token in raw.split(['|', ',']) { + let Some((width, height)) = token.trim().split_once('x') else { + continue; + }; + let (Ok(width), Ok(height)) = (width.parse::(), height.parse::()) else { + continue; + }; + if width < MIN_FORMAT_DIMENSION || height < MIN_FORMAT_DIMENSION { + continue; + } + if !sizes.contains(&(width, height)) { + sizes.push((width, height)); + } + } + sizes +} + +/// Derives a runtime-safe slot id from a div id. +/// +/// The common GPT prefix is stripped, invalid character runs become one +/// hyphen, and an all-invalid value falls back to `slot`. +fn slot_id_from_div(div_id: &str) -> String { + let candidate = div_id.strip_prefix(GPT_DIV_PREFIX).unwrap_or(div_id); + let mut id = String::with_capacity(candidate.len()); + let mut previous_was_hyphen = false; + for character in candidate.chars() { + if character.is_ascii_alphanumeric() || character == '_' { + id.push(character); + previous_was_hyphen = false; + } else if !id.is_empty() && !previous_was_hyphen { + id.push('-'); + previous_was_hyphen = true; + } + } + while id.ends_with('-') { + id.pop(); + } + if id.is_empty() { + id.push_str("slot"); + } + + if validate_slot_id(&id).is_ok() { + id + } else { + "slot".to_string() + } +} + +/// Adds deterministic numeric suffixes when sanitization produces duplicate ids. +fn make_slot_ids_unique(slots: &mut [DiscoveredSlot]) { + let mut used = BTreeSet::new(); + for slot in slots { + if used.insert(slot.id.clone()) { + continue; + } + + let base = slot.id.clone(); + let mut suffix = 2_usize; + loop { + let candidate = format!("{base}-{suffix}"); + if used.insert(candidate.clone()) { + slot.id = candidate; + break; + } + suffix += 1; + } + } +} + +/// Detects Prebid/header-bidding signals in a slot's `prev_scp` targeting. +fn scp_shows_prebid(scp: &str) -> bool { + url::form_urlencoded::parse(scp.as_bytes()).any(|(key, value)| { + let key = key.to_ascii_lowercase(); + let value = value.to_ascii_lowercase(); + (key == "test" && value == "prebid") + || (key == "tude" && value == "true") + || key.starts_with("prebid") + }) +} + +#[cfg(test)] +mod tests { + use super::*; + + /// A sample GPT leaderboard ad request (truncated to the fields the + /// parser reads; values are otherwise unmodified live output). + const SAMPLE_LEADERBOARD: &str = "https://securepubads.g.doubleclick.net/gampad/ads?\ + gdfp_req=1&iu_parts=123456789%2Cdesktop%2Chomepage%2Cleaderboard1\ + &prev_iu_szs=970x250%7C4x1%7C8x1%7C620x366%7C325x508%7C325x204\ + &dids=div-gpt-ad-leaderboard-1\ + &prev_scp=ad-loc%3Dleaderboard-1%26baseDivId%3Ddiv-gpt-ad-leaderboard-1%26test%3Dprebid%26tude%3Dtrue\ + &pb_szs=970x250%7C620x366"; + const SHORT_VOLATILE_DIV: &str = "vendor-tag_12345678AbCdEfGhIjKl_slot_overlay_1"; + + fn request(url: &str) -> CollectedRequest { + CollectedRequest { + url: url.to_string(), + resource_type: Some("fetch".to_string()), + } + } + + /// Discovers slots from ad requests only (no live registry). + fn from_requests(requests: &[CollectedRequest]) -> DiscoveredSlots { + discover_gpt_slots(&[], requests, false) + } + + #[test] + fn parses_leaderboard_slot() { + let discovered = from_requests(&[request(SAMPLE_LEADERBOARD)]); + + assert_eq!(discovered.gam_network_id.as_deref(), Some("123456789")); + assert_eq!(discovered.slots.len(), 1, "should find one slot"); + let slot = &discovered.slots[0]; + assert_eq!(slot.id, "leaderboard-1", "should strip the GPT div prefix"); + assert_eq!(slot.div_id, "div-gpt-ad-leaderboard-1"); + assert_eq!( + slot.gam_unit_path, + "/123456789/desktop/homepage/leaderboard1" + ); + assert_eq!( + slot.formats, + vec![(970, 250), (620, 366), (325, 508), (325, 204)], + "should keep pixel sizes and drop 4x1/8x1 fluid markers" + ); + assert!(slot.has_prebid, "prev_scp test=prebid should flag prebid"); + } + + #[test] + fn prebid_detection_requires_a_targeting_key_not_a_substring() { + assert!(scp_shows_prebid("test=prebid")); + assert!(!scp_shows_prebid("noprebid=true")); + } + + #[test] + fn deduplicates_refreshed_slot_requests() { + // GPT refreshes the same slot; a second identical request must not + // produce a duplicate slot. + let discovered = from_requests(&[request(SAMPLE_LEADERBOARD), request(SAMPLE_LEADERBOARD)]); + + assert_eq!( + discovered.slots.len(), + 1, + "repeat requests for the same div should collapse" + ); + } + + #[test] + fn ignores_non_gampad_requests() { + let discovered = from_requests(&[ + request("https://securepubads.g.doubleclick.net/tag/js/gpt.js"), + request("https://cdn.example.com/app.js"), + request("https://analytics.example.com/collect?iu_parts=1%2Cfoo&dids=x"), + ]); + + assert!( + discovered.slots.is_empty(), + "only doubleclick gampad/ads requests should yield slots" + ); + assert_eq!(discovered.gam_network_id, None); + } + + #[test] + fn skips_requests_missing_sizes() { + let discovered = from_requests(&[request( + "https://securepubads.g.doubleclick.net/gampad/ads?iu_parts=123%2Cslot&dids=div-gpt-ad-x", + )]); + + assert!( + discovered.slots.is_empty(), + "a slot with no usable size should be skipped" + ); + } + + #[test] + fn skips_requests_with_only_network_id() { + // iu_parts with just the network id yields no unit path segment. + let discovered = from_requests(&[request( + "https://securepubads.g.doubleclick.net/gampad/ads?iu_parts=123&dids=div-gpt-ad-x&prev_iu_szs=300x250", + )]); + + assert!( + discovered.slots.is_empty(), + "a bare network id is not a usable ad-unit path" + ); + } + + #[test] + fn skips_requests_with_non_numeric_network_id() { + // A page-controlled iu_parts value must not smuggle a non-numeric + // network id (it gets spliced into generated TOML). + let discovered = from_requests(&[request( + "https://securepubads.g.doubleclick.net/gampad/ads?iu_parts=123%22evil%2Cslot&dids=div-gpt-ad-x&prev_iu_szs=300x250", + )]); + + assert!( + discovered.slots.is_empty(), + "a non-numeric network id should be rejected" + ); + assert_eq!(discovered.gam_network_id, None); + } + + #[test] + fn falls_back_to_pb_szs_when_prev_iu_szs_absent() { + let discovered = from_requests(&[request( + "https://securepubads.g.doubleclick.net/gampad/ads?iu_parts=123%2Cslot&dids=div-gpt-ad-x&pb_szs=300x250%7C728x90", + )]); + + assert_eq!(discovered.slots.len(), 1); + assert_eq!(discovered.slots[0].formats, vec![(300, 250), (728, 90)]); + } + + fn registry_slot(path: &str, div: &str, sizes: &[(u32, u32)]) -> CollectedGptSlot { + CollectedGptSlot { + gam_unit_path: path.to_string(), + div_id: div.to_string(), + sizes: sizes.to_vec(), + } + } + + #[test] + fn lowercase_react_use_id_suffixes_collapse_to_one_slot() { + // React emits `_r_0_` client-side and `_R_3f_` server-side, and the + // token changes per render. Leaving it in the stem fragments one slot + // into a new key on every page, which starves template inference. + for volatile in [ + "ad-header-0-_r_0_", + "ad-header-0-_r_8_", + "ad-header-0-_r_a_", + "ad-header-0-_R_3f_", + ] { + let registry = vec![registry_slot("/123/site/news", volatile, &[(728, 90)])]; + let discovered = discover_gpt_slots(®istry, &[], false); + assert_eq!( + discovered.slots[0].div_id, "ad-header-0", + "`{volatile}` should normalize to a stable stem" + ); + } + } + + #[test] + fn an_ordinary_id_containing_r_is_left_alone() { + // The React shape is anchored, so a legitimate id keeps its full stem. + let registry = vec![registry_slot("/123/site/news", "ad_r_rail", &[(300, 250)])]; + + let discovered = discover_gpt_slots(®istry, &[], false); + + assert_eq!(discovered.slots[0].div_id, "ad_r_rail"); + } + + #[test] + fn registry_slot_with_brace_in_unit_path_is_skipped() { + // `gam_unit_path` is a template and there is no escape syntax, so a + // literal brace either fails config load or is silently reinterpreted as + // a placeholder. Neither is acceptable to persist. + let registry = vec![ + registry_slot("/123/home/{section}", "div-gpt-ad-a", &[(300, 250)]), + registry_slot("/123/home/ok", "div-gpt-ad-b", &[(300, 250)]), + ]; + + let discovered = discover_gpt_slots(®istry, &[], false); + + assert_eq!( + discovered.slots.len(), + 1, + "the brace-bearing slot should be dropped, the clean one kept" + ); + assert_eq!(discovered.slots[0].gam_unit_path, "/123/home/ok"); + } + + #[test] + fn registry_slot_whose_div_id_is_entirely_ephemeral_is_skipped() { + // `_R_…` is a React SSR marker; normalizing truncates at it, leaving an + // empty stem. An empty div_id fails config load, and as a runtime prefix + // it would match the first id-bearing element on the page. + let registry = vec![registry_slot( + "/123/home/header", + "_R_9slkta7pd6", + &[(728, 90)], + )]; + + let discovered = discover_gpt_slots(®istry, &[], false); + + assert!( + discovered.slots.is_empty(), + "a slot with no stable div stem should be dropped, got {:?}", + discovered.slots + ); + } + + #[test] + fn volatile_guid_div_id_still_normalizes_to_a_usable_prefix() { + // A GUID between two copies of the placement name must still yield a + // usable stable stem; only an entirely ephemeral id is dropped. + let registry = vec![registry_slot( + "/123456789/publisher/homepage", + "ad-in_content-0949b6c5726343bf8bbec2ac47b494b4-in_content-0", + &[(300, 250)], + )]; + + let discovered = discover_gpt_slots(®istry, &[], false); + + assert_eq!(discovered.slots.len(), 1); + assert_eq!( + discovered.slots[0].div_id, "ad-in_content", + "the GUID and trailing index should be truncated to a stable prefix" + ); + } + + #[test] + fn reads_slots_from_live_registry() { + let registry = vec![registry_slot( + "/123456789/desktop/homepage/leaderboard1", + "div-gpt-ad-leaderboard-1", + &[(970, 250), (1, 1), (620, 366)], + )]; + + let discovered = discover_gpt_slots(®istry, &[], true); + + assert_eq!( + discovered.gam_network_id.as_deref(), + Some("123456789"), + "network id should come from the unit path" + ); + assert_eq!(discovered.slots.len(), 1); + let slot = &discovered.slots[0]; + assert_eq!(slot.id, "leaderboard-1"); + assert_eq!( + slot.formats, + vec![(970, 250), (620, 366)], + "should drop the 1x1 out-of-page marker" + ); + assert!( + slot.has_prebid, + "page-level prebid should mark registry slots" + ); + } + + #[test] + fn registry_wins_and_requests_fill_gaps() { + // The registry reports the leaderboard; a gampad request reports a + // different div that the registry missed. Both should appear once. + let registry = vec![registry_slot( + "/123456789/desktop/homepage/leaderboard1", + "div-gpt-ad-leaderboard-1", + &[(970, 250)], + )]; + let requests = vec![ + // Same div as the registry — must not duplicate. + request(SAMPLE_LEADERBOARD), + // A div the registry did not report — must be added. + request( + "https://securepubads.g.doubleclick.net/gampad/ads?iu_parts=123456789%2Cdesktop%2Chomepage%2Csidebar1&dids=div-gpt-ad-sidebar-1&prev_iu_szs=300x600", + ), + ]; + + let discovered = discover_gpt_slots(®istry, &requests, false); + + let ids: Vec<&str> = discovered + .slots + .iter() + .map(|slot| slot.id.as_str()) + .collect(); + assert_eq!( + ids, + vec!["leaderboard-1", "sidebar-1"], + "registry slot kept, request fills the missing div, no duplicate" + ); + } + + #[test] + fn registry_slot_without_pixel_sizes_is_skipped() { + let registry = vec![registry_slot("/123/fluid", "div-gpt-ad-fluid", &[(1, 1)])]; + + let discovered = discover_gpt_slots(®istry, &[], false); + + assert!( + discovered.slots.is_empty(), + "a registry slot with only fluid markers is not usable" + ); + } + + #[test] + fn normalizes_ephemeral_hash_and_container_and_dedups() { + // A framework-hashed div: the same placement appears as a hashed inner div, + // a `-container` wrapper, and re-rendered with a different hash. All must + // collapse to one stable stem. + let registry = vec![ + registry_slot( + "/987654321/homepage/header-0", + "ad-header-0-_R_9slinpflik6lb_", + &[(728, 90)], + ), + registry_slot( + "/987654321/homepage/header-0", + "ad-header-0-_R_9slinpflik6lb_-container", + &[(728, 90)], + ), + ]; + + let discovered = discover_gpt_slots(®istry, &[], false); + + assert_eq!( + discovered.slots.len(), + 1, + "hash + container variants collapse" + ); + assert_eq!( + discovered.slots[0].div_id, "ad-header-0", + "ephemeral React hash and -container are stripped to a stable stem" + ); + assert_eq!(discovered.slots[0].id, "ad-header-0"); + } + + #[test] + fn drops_sra_multi_slot_concatenations() { + let registry = vec![registry_slot( + "/987654321/homepage/header-0/fixed_bottom-0", + "ad-header-0-_R_9slin~ad-fixed_bottom-0-_R_ainp", + &[(728, 90)], + )]; + + let discovered = discover_gpt_slots(®istry, &[], false); + + assert!( + discovered.slots.is_empty(), + "tilde-joined SRA multi-slot divs are not real single elements" + ); + } + + #[test] + fn leaves_clean_div_ids_unchanged() { + assert_eq!( + normalize_div_stem("div-gpt-ad-leaderboard-1"), + "div-gpt-ad-leaderboard-1" + ); + } + + #[test] + fn sanitizes_page_controlled_div_ids_for_runtime_slot_ids() { + let registry = vec![registry_slot( + "/123456789/homepage/header", + "div-gpt-ad-header.main: 1", + &[(728, 90)], + )]; + + let discovered = discover_gpt_slots(®istry, &[], false); + + assert_eq!(discovered.slots[0].id, "header-main-1"); + assert_eq!( + discovered.slots[0].div_id, "div-gpt-ad-header.main: 1", + "matching should retain the original normalized div stem" + ); + trusted_server_core::creative_opportunities::validate_slot_id(&discovered.slots[0].id) + .expect("generated id should pass runtime validation"); + } + + #[test] + fn uses_fallback_for_div_id_without_safe_slot_id_characters() { + let registry = vec![registry_slot( + "/123456789/homepage/fallback", + "div-gpt-ad-...", + &[(300, 250)], + )]; + + let discovered = discover_gpt_slots(®istry, &[], false); + + assert_eq!(discovered.slots[0].id, "slot"); + } + + #[test] + fn makes_colliding_sanitized_slot_ids_unique() { + let registry = vec![ + registry_slot( + "/123456789/homepage/dotted", + "div-gpt-ad-header.main", + &[(728, 90)], + ), + registry_slot( + "/123456789/homepage/colon", + "div-gpt-ad-header:main", + &[(300, 250)], + ), + ]; + + let discovered = discover_gpt_slots(®istry, &[], false); + let ids = discovered + .slots + .iter() + .map(|slot| slot.id.as_str()) + .collect::>(); + + assert_eq!(ids, ["header-main", "header-main-2"]); + } + + #[test] + fn normalizes_react_and_hex_hashes_to_stable_prefixes() { + assert_eq!( + normalize_div_stem("ad-header-0-_R_9slinpflik6lb_-container"), + "ad-header-0" + ); + let stem = + normalize_div_stem("ad-in_content-de669245b2ea4b05826dc96f07a36272-in_content-0"); + assert_eq!(stem, "ad-in_content"); + assert!( + "ad-in_content-de669245b2ea4b05826dc96f07a36272-in_content-0".starts_with(&stem), + "stem must prefix-match any re-rendered hex variant" + ); + } + + #[test] + fn hex_hash_truncation_requires_a_segment_boundary() { + // Hex UUID bounded by `-` → truncated to the stem. + assert_eq!( + normalize_div_stem("ad-x-de669245b2ea4b05826dc96f07a36272-y"), + "ad-x" + ); + // A token that merely starts with 16 hex chars (no boundary) is left intact. + assert_eq!( + normalize_div_stem("ad-de669245b2ea4b05z"), + "ad-de669245b2ea4b05z" + ); + } + + #[test] + fn long_numeric_segments_are_stable_ids_not_hex_hashes() { + assert_eq!( + normalize_div_stem("ad-slot-1234567890123456-tail"), + "ad-slot-1234567890123456-tail" + ); + } + + #[test] + fn comma_separated_sra_dids_are_ignored() { + let discovered = from_requests(&[request( + "https://securepubads.g.doubleclick.net/gampad/ads?iu_parts=123%2Cnews%2Catf&dids=ad-a%2Cad-b&prev_iu_szs=300x250", + )]); + + assert!( + discovered.slots.is_empty(), + "a comma-joined SRA did list is not one element" + ); + } + + #[test] + fn one_element_under_two_render_tokens_is_not_a_collision() { + // Both ids describe in-content placement 0; only the hash between the + // two copies of the placement name differs, which is what one element + // re-rendered looks like. Refusing here would refuse the very shape + // normalization exists to absorb. + let registry = vec![ + registry_slot( + "/987654321/site/homepage", + "ad-in_content-de669245b2ea4b05826dc96f07a36272-in_content-0", + &[(300, 250)], + ), + registry_slot( + "/987654321/site/homepage", + "ad-in_content-8aec8129a83d4e5abc197423120cb19e-in_content-0", + &[(300, 250)], + ), + ]; + + let discovered = discover_gpt_slots(®istry, &[], false); + + assert_eq!( + discovered.slots.len(), + 1, + "two renders of one element are one slot, got {:?}", + discovered.slots + ); + assert_eq!(discovered.slots[0].div_id, "ad-in_content"); + assert!( + discovered.warnings.is_empty(), + "a re-render is not an ambiguity to report, got {:?}", + discovered.warnings + ); + assert!(discovered.ambiguous_stems.is_empty()); + } + + #[test] + fn sibling_placements_sharing_one_stem_are_refused() { + // Same shape as above, but the trailing placement index differs: these + // are two live elements, and one prefix cannot resolve to both. + let registry = vec![ + registry_slot( + "/987654321/site/homepage", + "ad-in_content-de669245b2ea4b05826dc96f07a36272-in_content-0", + &[(300, 250)], + ), + registry_slot( + "/987654321/site/homepage", + "ad-in_content-8aec8129a83d4e5abc197423120cb19e-in_content-1", + &[(300, 250)], + ), + ]; + + let discovered = discover_gpt_slots(®istry, &[], false); + + assert!( + discovered.had_slot_evidence, + "a refused placement is still evidence of an ad stack" + ); + assert!( + discovered.slots.is_empty(), + "neither a broad prefix nor per-render exact IDs are safe" + ); + assert_ambiguous_collision_warning(&discovered, "ad-in_content"); + assert!( + discovered.ambiguous_stems.contains("ad-in_content"), + "the verdict must travel with the evidence, got {:?}", + discovered.ambiguous_stems + ); + } + + #[test] + fn react_server_and_client_render_tokens_are_one_slot() { + // A hydrating publisher reports the SSR id and the client id for the + // same element. Both must collapse rather than refuse each other. + let registry = vec![ + registry_slot("/123456789/site/news", "ad-header-0-_R_3f_", &[(728, 90)]), + registry_slot("/123456789/site/news", "ad-header-0-_r_0_", &[(728, 90)]), + ]; + + let discovered = discover_gpt_slots(®istry, &[], false); + + assert_eq!( + discovered.slots.len(), + 1, + "SSR and client renders of one element are one slot, got {:?}", + discovered.slots + ); + assert_eq!(discovered.slots[0].div_id, "ad-header-0"); + assert!(discovered.warnings.is_empty()); + } + + #[test] + fn repeated_raw_div_after_a_normalization_collision_is_deduplicated() { + let first = "ad-x-aaaaaaaaaaaaaaaa-0"; + let second = "ad-x-bbbbbbbbbbbbbbbb-1"; + let third = "ad-x-cccccccccccccccc-2"; + let registry = vec![ + registry_slot("/123456789/site/home", first, &[(300, 250)]), + registry_slot("/123456789/site/home", second, &[(300, 250)]), + registry_slot("/123456789/site/home", first, &[(300, 250)]), + registry_slot("/123456789/site/home", second, &[(300, 250)]), + registry_slot("/123456789/site/home", third, &[(300, 250)]), + ]; + + let discovered = discover_gpt_slots(®istry, &[], false); + + assert!( + discovered.had_slot_evidence, + "a refused placement is still evidence of an ad stack" + ); + assert!( + discovered.slots.is_empty(), + "no repeat or later collision member may resurrect the group" + ); + assert_ambiguous_collision_warning(&discovered, "ad-x"); + } + + #[test] + fn request_normalization_collision_is_refused() { + let discovered = from_requests(&[ + request( + "https://securepubads.g.doubleclick.net/gampad/ads?iu_parts=123456789%2Csite%2Chome&dids=ad-x-aaaaaaaaaaaaaaaa-0&prev_iu_szs=300x250", + ), + request( + "https://securepubads.g.doubleclick.net/gampad/ads?iu_parts=123456789%2Csite%2Chome&dids=ad-x-bbbbbbbbbbbbbbbb-1&prev_iu_szs=300x250", + ), + ]); + + assert!( + discovered.had_slot_evidence, + "a refused placement is still evidence of an ad stack" + ); + assert!( + discovered.slots.is_empty(), + "a refused placement must not be written, got {:?}", + discovered.slots + ); + assert_eq!( + discovered.gam_network_id.as_deref(), + Some("123456789"), + "refusing a slot must not discard the network id" + ); + assert_ambiguous_collision_warning(&discovered, "ad-x"); + } + + #[test] + fn single_volatile_family_registry_slot_is_refused() { + let discovered = discover_gpt_slots( + &[registry_slot( + "/123456789/site_in-article_desktop_1", + "vendor-tag_1724112345678AbCdEfGh_slot_inarticle_1", + &[(300, 250)], + )], + &[], + false, + ); + + assert!( + discovered.had_slot_evidence, + "a refused placement is still evidence of an ad stack" + ); + assert!( + discovered.slots.is_empty(), + "one observation of a per-render family must not be written literally" + ); + assert_eq!(discovered.gam_network_id.as_deref(), Some("123456789")); + assert!( + discovered + .refused_div_ids + .contains("vendor-tag_1724112345678AbCdEfGh_slot_inarticle_1"), + "registry refusal should retain its normalized div as observed evidence" + ); + assert_volatile_prefix_warning(&discovered, "vendor-tag"); + } + + #[test] + fn single_volatile_family_request_slot_is_refused() { + let discovered = from_requests(&[request( + "https://securepubads.g.doubleclick.net/gampad/ads?iu_parts=123456789%2Csite_in-article_desktop_1&dids=vendor-tag_1724112345678AbCdEfGh_slot_inarticle_1&prev_iu_szs=300x250", + )]); + + assert!( + discovered.had_slot_evidence, + "a refused placement is still evidence of an ad stack" + ); + assert!( + discovered.slots.is_empty(), + "a refused placement must not be written, got {:?}", + discovered.slots + ); + assert_eq!( + discovered.gam_network_id.as_deref(), + Some("123456789"), + "refusing a slot must not discard the network id" + ); + assert!( + discovered + .refused_div_ids + .contains("vendor-tag_1724112345678AbCdEfGh_slot_inarticle_1"), + "request refusal should retain its normalized div as observed evidence" + ); + assert_volatile_prefix_warning(&discovered, "vendor-tag"); + } + + #[test] + fn shorter_high_entropy_singleton_registry_slot_is_refused() { + let discovered = discover_gpt_slots( + &[registry_slot( + "/123456789/publisher.example_overlay_mobile", + SHORT_VOLATILE_DIV, + &[(300, 250)], + )], + &[], + false, + ); + + assert!(discovered.had_slot_evidence); + assert!( + discovered.slots.is_empty(), + "a singleton per-render ID must not be written literally" + ); + assert_volatile_prefix_warning(&discovered, "vendor-tag"); + } + + #[test] + fn shorter_high_entropy_singleton_request_slot_is_refused() { + let discovered = from_requests(&[request(&format!( + "https://securepubads.g.doubleclick.net/gampad/ads?\ + iu_parts=123456789%2Cpublisher.example_overlay_mobile\ + &dids={SHORT_VOLATILE_DIV}&prev_iu_szs=300x250" + ))]); + + assert!(discovered.had_slot_evidence); + assert!( + discovered.slots.is_empty(), + "request fallback must not write a singleton per-render ID" + ); + assert_volatile_prefix_warning(&discovered, "vendor-tag"); + } + + #[test] + fn volatile_prefix_covers_every_placement_after_the_token() { + // The token's position is what makes the id unusable, so the placement + // that follows it is irrelevant: every one of these leaves `vendor-tag` + // as the only stable prefix, and that prefix reaches all of them. + for volatile in [ + "vendor-tag_1724112345678AbCdEfGh_slot_inarticle_1", + "vendor-tag_12345678AbCdEfGh_slot_inarticle_1", + "vendor-tag_20260820AbCdEfGh_slot_inarticle_1", + "vendor-tag_1724112345678AbCdEfGh_slot_overlay_1-container", + "vendor-tag_1724112345678AbCdEfGh_slot_sidebar_1", + "vendor-tag_1724112345678AbCdEfGh_slot_overlay_stable", + "vendor-tag_1724112345678AbCdEfGh_slot_overlay_1_extra", + ] { + assert_eq!( + volatile_prefix_before_placement(volatile).as_deref(), + Some("vendor-tag"), + "`{volatile}` should be refused as a volatile family" + ); + } + } + + #[test] + fn volatile_prefix_does_not_claim_stable_div_ids() { + for stable in [ + // No per-render token at all. + "vendor-tag_stable_slot_inarticle_1", + // A bare digit run is how stable placement indices are written. + "vendor-tag_12345678_slot_inarticle_1", + "ad-slot-1234567890123456-tail", + // Shorter counter/suffix combinations do not carry enough entropy. + "vendor-tag_1234567AbCdEfGh_slot_inarticle_1", + "vendor-tag_12345678AbCdEfG_slot_inarticle_1", + // An eight-digit calendar date plus a stable suffix is not a + // timestamp-like per-render token. + "promo-20260820a-sidebar", + "promo-20260820Football-sidebar", + "promo-20260820football-sidebar", + "promo-20260820TopStories-sidebar", + "ad-19700101Thumbnail-rail", + "ad-00000001AAAAAAAA-rail", + // The token is trailing, so the prefix before it still identifies + // this element and normalization/collision handling own the case. + "vendor-tag_slot_inarticle_1724112345678AbCdEfGh", + "vendor-tag-header", + ] { + assert_eq!( + volatile_prefix_before_placement(stable), + None, + "`{stable}` should stay eligible" + ); + } + } + + #[test] + fn ambiguous_registry_stem_still_suppresses_request_fallback() { + let registry = vec![ + registry_slot( + "/123456789/site/home", + "ad-x-aaaaaaaaaaaaaaaa-0", + &[(300, 250)], + ), + registry_slot( + "/123456789/site/home", + "ad-x-bbbbbbbbbbbbbbbb-1", + &[(300, 250)], + ), + ]; + let requests = vec![request( + "https://securepubads.g.doubleclick.net/gampad/ads?iu_parts=123456789%2Csite%2Chome&dids=ad-x-cccccccccccccccc-2&prev_iu_szs=300x250", + )]; + + let discovered = discover_gpt_slots(®istry, &requests, false); + + assert!( + discovered.had_slot_evidence, + "a refused placement is still evidence of an ad stack" + ); + assert!( + discovered.slots.is_empty(), + "request fallback must not resurrect an ambiguous registry stem" + ); + assert_eq!(discovered.gam_network_id.as_deref(), Some("123456789")); + assert_ambiguous_collision_warning(&discovered, "ad-x"); + } + + #[test] + fn request_rerender_does_not_rewrite_a_stable_registry_slot() { + let registry = vec![registry_slot( + "/123456789/site/home", + "ad-x-aaaaaaaaaaaaaaaa-0", + &[(300, 250)], + )]; + let requests = vec![request( + "https://securepubads.g.doubleclick.net/gampad/ads?iu_parts=123456789%2Csite%2Chome&dids=ad-x-bbbbbbbbbbbbbbbb-1&prev_iu_szs=300x250", + )]; + + let discovered = discover_gpt_slots(®istry, &requests, false); + + assert_eq!(discovered.slots.len(), 1, "registry evidence should win"); + assert_eq!( + discovered.slots[0].div_id, "ad-x", + "request fallback must not destabilize a registry-derived prefix" + ); + } + + fn assert_ambiguous_collision_warning(discovered: &DiscoveredSlots, prefix: &str) { + assert_eq!(discovered.warnings.len(), 1); + let warning = &discovered.warnings[0]; + assert!(warning.contains(prefix), "warning should name the prefix"); + assert!( + warning.contains("one active element"), + "warning should explain why the broad prefix is unsafe" + ); + assert!( + warning.contains("change across renders"), + "warning should explain why raw IDs are unsafe" + ); + assert!( + warning.contains("distinct stable div ids"), + "warning should tell the operator how to make the placements configurable" + ); + } + + fn assert_volatile_prefix_warning(discovered: &DiscoveredSlots, prefix: &str) { + assert_eq!( + discovered.warnings.len(), + 1, + "should report the family once, got {:?}", + discovered.warnings + ); + let warning = &discovered.warnings[0]; + assert!( + warning.contains(prefix), + "warning should name the family prefix, got {warning}" + ); + assert!( + warning.contains("change across renders"), + "warning should explain why the exact ids are unsafe, got {warning}" + ); + assert!( + warning.contains("distinct stable div ids"), + "warning should tell the operator how to make the placements configurable, got {warning}" + ); + } +} diff --git a/crates/trusted-server-cli/src/commands/audit/generate/mod.rs b/crates/trusted-server-cli/src/commands/audit/generate/mod.rs new file mode 100644 index 000000000..490ccba80 --- /dev/null +++ b/crates/trusted-server-cli/src/commands/audit/generate/mod.rs @@ -0,0 +1,3618 @@ +mod analyzer; +pub(crate) mod browser_collector; +pub(crate) mod collector; +mod crawl_plan; +mod evidence; +mod gpt_slots; +mod page_patterns; +mod slot_toml; +mod unit_template; +mod validate; + +use std::collections::BTreeSet; +use std::fs; +use std::io::Write; +use std::path::{Path, PathBuf}; + +use serde::Serialize; +use trusted_server_core::creative_opportunities::{ + CreativeOpportunitiesConfig, validate_page_pattern, +}; +use url::Url; + +use crate::commands::audit::ad_templates::{origin_changed, without_fragment}; +use crate::commands::audit::collector::GenerateBrowserOpts; +use crate::commands::audit::generate::collector::AuditCollector; +use crate::commands::audit::generate::slot_toml::{ + render_slots, replace_key_in_section, resolve_network_id, splice_creative_slots, toml_string, +}; +use crate::commands::config::init::EXAMPLE_CONFIG; +use crate::error::{CliResult, cli_error, report_error}; + +use analyzer::{analyze_collected_page, extract_gtm_container_id}; + +pub(crate) use browser_collector::DeviceProfile; +pub(crate) use crawl_plan::CrawlBudget; + +/// Writes `contents` to `path` atomically: a same-directory temp file is +/// written and fsynced, then renamed over the target, then the directory entry +/// is fsynced. +/// +/// A plain `fs::write` truncates the destination before writing, so a full disk +/// or an interrupted run would leave an operator's `trusted-server.toml` empty +/// or half-written. `rename` within a directory is atomic, so a reader sees +/// either the old file or the complete new one. +/// +/// The target's existing permissions are carried onto the replacement, since +/// the temp file is created 0600 and the config may intentionally be broader. +/// +/// # Errors +/// +/// Returns the underlying I/O error when the temp file cannot be created, +/// written, synced, or renamed over `path`. +fn write_file_atomically(path: &Path, contents: &str) -> std::io::Result<()> { + let directory = path + .parent() + .filter(|parent| !parent.as_os_str().is_empty()) + .unwrap_or_else(|| Path::new(".")); + + let mut temp = tempfile::Builder::new() + .prefix(".ts-audit-") + .tempfile_in(directory)?; + temp.write_all(contents.as_bytes())?; + temp.as_file().sync_all()?; + if let Ok(metadata) = fs::metadata(path) { + temp.as_file().set_permissions(metadata.permissions())?; + } + temp.persist(path).map_err(|error| error.error)?; + + // Best-effort durability for the rename itself. Opening a directory handle + // is not portable (Windows rejects it), and the content is already safely + // on disk either way, so a failure here is not worth failing the command. + let _ = fs::File::open(directory).and_then(|handle| handle.sync_all()); + Ok(()) +} + +/// Arguments for `ts audit generate ` — bootstraps draft Trusted Server +/// config and JavaScript asset audit files from a live page (issue #800). +#[derive(Debug, clap::Args)] +pub(crate) struct GenerateArgs { + /// Public HTTP(S) URL to audit. + pub(crate) url: String, + /// JavaScript asset audit output path. + #[arg(long)] + pub(crate) js_assets: Option, + /// Draft Trusted Server config output path. + #[arg(long)] + pub(crate) config: Option, + /// Do not write the JavaScript asset audit file. + #[arg(long)] + pub(crate) no_js_assets: bool, + /// Do not write the draft Trusted Server config file. + #[arg(long)] + pub(crate) no_config: bool, + /// Overwrite existing output files. + #[arg(long)] + pub(crate) force: bool, + /// Cookie to send with the page request, as `name=value`. Repeatable. + /// Use to carry an existing session (e.g. a valid bot-protection clearance + /// cookie) so the origin serves the real page instead of a challenge. + #[arg(long = "cookie", value_name = "NAME=VALUE", value_parser = crate::commands::audit::parse_cookie)] + pub(crate) cookies: Vec<(String, String)>, + /// Browser and consent options shared with `ts audit ad-templates generate`. + #[command(flatten)] + pub(crate) browser: GenerateBrowserOpts, +} + +const DEFAULT_JS_ASSETS_PATH: &str = "js-assets.toml"; +const DEFAULT_CONFIG_PATH: &str = "trusted-server.toml"; + +#[derive(Debug, Clone, Serialize, PartialEq, Eq)] +#[serde(rename_all = "kebab-case")] +pub(crate) enum AssetParty { + FirstParty, + ThirdParty, +} + +#[derive(Debug, Clone, Serialize, PartialEq, Eq)] +pub(crate) struct AuditedAsset { + pub(crate) kind: String, + pub(crate) url: String, + pub(crate) host: String, + pub(crate) party: AssetParty, + #[serde(skip_serializing_if = "Option::is_none")] + pub(crate) integration: Option, +} + +#[derive(Debug, Clone, Serialize, PartialEq, Eq)] +pub(crate) struct DetectedIntegration { + pub(crate) id: String, + pub(crate) evidence: String, +} + +#[derive(Debug, Clone, Serialize, PartialEq, Eq)] +pub(crate) struct AuditArtifact { + pub(crate) audited_url: String, + #[serde(skip_serializing_if = "Option::is_none")] + pub(crate) page_title: Option, + pub(crate) js_asset_count: usize, + pub(crate) third_party_asset_count: usize, + pub(crate) detected_integrations: Vec, + pub(crate) assets: Vec, + pub(crate) warnings: Vec, +} + +#[derive(Debug, Clone)] +pub(crate) struct AuditOutputs { + pub(crate) artifact: AuditArtifact, + pub(crate) js_assets_toml: String, + pub(crate) draft_config_toml: String, + pub(crate) ad_slot_count: usize, +} + +#[derive(Debug, Clone, PartialEq, Eq)] +struct AuditOutputPlan { + js_assets_path: Option, + config_path: Option, +} + +pub(crate) fn run_generate( + args: &GenerateArgs, + collector: &dyn AuditCollector, + out: &mut dyn Write, +) -> CliResult<()> { + let target_url = parse_audit_url(&args.url)?; + let plan = resolve_output_plan(args)?; + let collected = collector.collect_page(&target_url, &args.cookies)?; + let outputs = build_audit_outputs(&collected)?; + let wrote_config = plan.config_path.is_some(); + let written = write_audit_outputs(&outputs, &plan)?; + write_success_summary(&outputs, &written, wrote_config, out) +} + +fn parse_audit_url(value: &str) -> CliResult { + let url = Url::parse(value) + .map_err(|error| report_error(format!("invalid audit URL `{value}`: {error}")))?; + if !matches!(url.scheme(), "http" | "https") { + return cli_error(format!( + "`ts audit` only supports http/https URLs, got `{}`", + url.scheme() + )); + } + Ok(url) +} + +fn resolve_output_plan(args: &GenerateArgs) -> CliResult { + if args.no_js_assets && args.no_config { + return cli_error("nothing to do: both --no-js-assets and --no-config were set"); + } + + let js_assets_path = if args.no_js_assets { + None + } else { + Some(resolve_output_path( + args.js_assets.as_deref(), + DEFAULT_JS_ASSETS_PATH, + )?) + }; + let config_path = if args.no_config { + None + } else { + Some(resolve_output_path( + args.config.as_deref(), + DEFAULT_CONFIG_PATH, + )?) + }; + + if js_assets_path.is_some() && js_assets_path == config_path { + return cli_error("audit output paths must be distinct"); + } + + for path in [&js_assets_path, &config_path].into_iter().flatten() { + if path.exists() && !args.force { + return cli_error(format!( + "refusing to overwrite existing file `{}`; re-run with --force", + path.display() + )); + } + } + + Ok(AuditOutputPlan { + js_assets_path, + config_path, + }) +} + +fn resolve_output_path(path: Option<&Path>, default: &str) -> CliResult { + let candidate = path.unwrap_or_else(|| Path::new(default)); + if candidate.is_absolute() { + Ok(candidate.to_path_buf()) + } else { + Ok(std::env::current_dir() + .map_err(|error| report_error(format!("failed to read current directory: {error}")))? + .join(candidate)) + } +} + +fn build_audit_outputs(collected: &collector::CollectedPage) -> CliResult { + let artifact = analyze_collected_page(collected)?; + let final_url = collected + .final_url() + .map_err(|error| report_error(format!("invalid final URL: {error}")))?; + let js_assets_toml = toml::to_string_pretty(&artifact) + .map_err(|error| report_error(format!("failed to serialize audit artifact: {error}")))?; + let page_has_prebid = artifact + .detected_integrations + .iter() + .any(|integration| integration.id == "prebid"); + let slots = gpt_slots::discover_gpt_slots( + &collected.gpt_slots, + &collected.network_requests, + page_has_prebid, + ); + let ad_slot_count = slots.slots.len(); + let draft_config_toml = build_draft_config(&final_url, &artifact, &slots)?; + + Ok(AuditOutputs { + artifact, + js_assets_toml, + draft_config_toml, + ad_slot_count, + }) +} + +fn write_audit_outputs(outputs: &AuditOutputs, plan: &AuditOutputPlan) -> CliResult> { + let selected_paths = [&plan.js_assets_path, &plan.config_path] + .into_iter() + .flatten() + .collect::>(); + for path in &selected_paths { + if let Some(parent) = path + .parent() + .filter(|parent| !parent.as_os_str().is_empty()) + { + fs::create_dir_all(parent).map_err(|error| { + report_error(format!( + "failed to create parent directory {}: {error}", + parent.display() + )) + })?; + } + } + + let mut written_paths = Vec::new(); + if let Some(path) = &plan.js_assets_path { + write_file_atomically(path, &outputs.js_assets_toml).map_err(|error| { + report_error(format!( + "failed to write JS asset audit {}: {error}", + path.display() + )) + })?; + written_paths.push(path.display().to_string()); + } + if let Some(path) = &plan.config_path { + write_file_atomically(path, &outputs.draft_config_toml).map_err(|error| { + report_error(format!( + "failed to write draft config {}: {error}", + path.display() + )) + })?; + written_paths.push(path.display().to_string()); + } + + Ok(written_paths) +} + +fn write_success_summary( + outputs: &AuditOutputs, + written: &[String], + wrote_config: bool, + out: &mut dyn Write, +) -> CliResult<()> { + let integrations = outputs + .artifact + .detected_integrations + .iter() + .map(|integration| integration.id.as_str()) + .collect::>(); + let draft_note = if wrote_config { + "\nDraft config: review before validation and push" + } else { + "" + }; + writeln!( + out, + "Audited {}\nTitle: {}\nJS assets: {}\nThird-party assets: {}\nAd slots: {}\nDetected integrations: {}\nWrote: {}{}", + outputs.artifact.audited_url, + outputs + .artifact + .page_title + .as_deref() + .unwrap_or(""), + outputs.artifact.js_asset_count, + outputs.artifact.third_party_asset_count, + outputs.ad_slot_count, + if integrations.is_empty() { + "none".to_string() + } else { + integrations.join(", ") + }, + if written.is_empty() { + "none".to_string() + } else { + written.join(", ") + }, + draft_note + ) + .map_err(|error| report_error(format!("failed to write command output: {error}"))) +} + +fn build_draft_config( + target_url: &Url, + artifact: &AuditArtifact, + slots: &gpt_slots::DiscoveredSlots, +) -> CliResult { + let host = target_url + .host_str() + .ok_or_else(|| report_error("audited URL is missing a host"))?; + let origin = target_url.origin().ascii_serialization(); + let mut draft = EXAMPLE_CONFIG.to_string(); + + draft = replace_key_in_section( + &draft, + "publisher", + "domain", + &format!("domain = \"{host}\""), + )?; + draft = replace_key_in_section( + &draft, + "publisher", + "cookie_domain", + &format!("cookie_domain = \".{host}\""), + )?; + draft = replace_key_in_section( + &draft, + "publisher", + "origin_url", + &format!("origin_url = \"{origin}\""), + )?; + + let detected = artifact + .detected_integrations + .iter() + .map(|integration| integration.id.as_str()) + .collect::>(); + + if detected.contains("gpt") { + draft = replace_key_in_section(&draft, "integrations.gpt", "enabled", "enabled = true")?; + } + if detected.contains("didomi") { + draft = replace_key_in_section(&draft, "integrations.didomi", "enabled", "enabled = true")?; + } + if detected.contains("datadome") { + draft = + replace_key_in_section(&draft, "integrations.datadome", "enabled", "enabled = true")?; + } + + let mut manual_review = Vec::new(); + if detected.contains("google_tag_manager") { + if let Some(gtm_id) = extract_gtm_container_id(artifact) { + draft = replace_key_in_section( + &draft, + "integrations.google_tag_manager", + "enabled", + "enabled = true", + )?; + draft = replace_key_in_section( + &draft, + "integrations.google_tag_manager", + "container_id", + &format!("container_id = \"{gtm_id}\""), + )?; + } else { + manual_review.push("google_tag_manager"); + } + } + + for integration in detected { + if !matches!( + integration, + "gpt" | "didomi" | "datadome" | "google_tag_manager" + ) { + manual_review.push(integration); + } + } + + if !manual_review.is_empty() { + if !draft.ends_with('\n') { + draft.push('\n'); + } + draft.push_str("\n# Audit findings requiring manual review\n"); + for integration in manual_review { + draft.push_str(&format!( + "# - Detected {integration}; review the corresponding [integrations.{integration}] section before enabling it.\n" + )); + } + } + + if !slots.slots.is_empty() { + if let Some(network_id) = &slots.gam_network_id { + draft = replace_key_in_section( + &draft, + "creative_opportunities", + "gam_network_id", + &format!("gam_network_id = {}", toml_string(network_id)), + )?; + } + draft.push_str(&render_discovered_slots(target_url, slots)); + } + + Ok(draft) +} + +/// Renders discovered GPT slots as appended `[[creative_opportunities.slot]]` +/// tables. Page patterns default to the audited path and are flagged for review. +fn render_discovered_slots(target_url: &Url, slots: &gpt_slots::DiscoveredSlots) -> String { + let path = target_url.path(); + let page_pattern = if path.is_empty() { "/" } else { path }; + + let mut out = String::from( + "\n# Slots discovered from live GPT ad requests during the audit.\n\ + # Review page_patterns and formats before validating/pushing.\n", + ); + for slot in &slots.slots { + let formats = slot + .formats + .iter() + .map(|(width, height)| format!("{{ width = {width}, height = {height} }}")) + .collect::>() + .join(", "); + out.push_str(&format!( + "\n[[creative_opportunities.slot]]\n\ + id = {id}\n\ + div_id = {div_id}\n\ + gam_unit_path = {gam_unit_path}\n\ + page_patterns = [{page_pattern}]\n\ + formats = [{formats}]\n", + id = toml_string(&slot.id), + div_id = toml_string(&slot.div_id), + gam_unit_path = toml_string(&slot.gam_unit_path), + page_pattern = toml_string(page_pattern), + )); + if slot.has_prebid { + out.push_str("[creative_opportunities.slot.providers.prebid]\nbidders = {}\n"); + } + } + out +} + +/// Everything one `ts audit ad-templates generate` invocation needs. +pub(crate) struct UpdateSlotsRequest<'a> { + /// Page URL to start from; also bounds the crawl to its origin. + pub(crate) url: &'a str, + /// Operator config to rewrite in place. + pub(crate) config_path: &'a Path, + /// The config's current `[creative_opportunities]`, when it has one. + pub(crate) existing_creative: Option<&'a CreativeOpportunitiesConfig>, + /// Explicit `--page-pattern` values. When non-empty these apply to every + /// slot and pattern inference is skipped entirely. + pub(crate) page_patterns: &'a [String], + /// Replace existing slots rather than merging into them. + pub(crate) replace: bool, + /// Cookies to carry into the crawl. + pub(crate) cookies: &'a [(String, String)], + /// Print the candidate instead of writing it. + pub(crate) dry_run: bool, + /// Whether the crawl used the deterministic scroll pass. + pub(crate) scroll: bool, + /// Crawl bounds. + pub(crate) budget: crawl_plan::CrawlBudget, +} + +/// Share of crawled pages that may yield no slots before the run is refused. +/// +/// A bot-protection challenge serves an interstitial that loads fine and +/// contains no ad stack, so it looks like a page with no slots. Writing a config +/// from a crawl that was mostly challenges would silently narrow the operator's +/// slot set; refusing is the safer failure. +const MAX_EMPTY_PAGE_SHARE: f64 = 0.25; + +/// Runs `ts audit ad-templates generate`: crawl the site's sections, reconcile +/// what each slot looked like across them, infer a `{section}` ad-unit template +/// where the evidence proves one, and rewrite the config's slot array in place. +/// +/// # Errors +/// +/// Returns an error when the config cannot be read, the root page cannot be +/// collected, no slots are discovered, too many pages came back empty, the +/// pages disagree about the GAM network id, or the resulting config would not +/// load. +pub(crate) fn run_update_slots( + request: &UpdateSlotsRequest<'_>, + collectors: &[(&str, &dyn AuditCollector)], + out: &mut dyn Write, + err: &mut dyn Write, +) -> CliResult<()> { + let Some((first_label, first_collector)) = collectors.first() else { + return cli_error("no device profile was selected to audit with"); + }; + let target_url = parse_audit_url(request.url)?; + let existing = fs::read_to_string(request.config_path).map_err(|error| { + report_error(format!( + "failed to read config {}: {error}", + request.config_path.display() + )) + })?; + + let mut table = evidence::EvidenceTable::default(); + let mut notes = Vec::new(); + let mut root_url = target_url.clone(); + let mut planned = None; + let mut fold_error = None; + + { + let mut progress_writer = CollectionProgressWriter { + out: err, + profile_label: first_label, + }; + let mut report_progress = + |progress: collector::CollectionProgress<'_>| progress_writer.write(progress); + first_collector.collect_site( + &target_url, + request.cookies, + &mut report_progress, + &mut |_, root| { + root_url = root.final_url().unwrap_or_else(|_| target_url.clone()); + if origin_changed(&target_url, &root_url) { + // Origins only: the origin is what the refusal is about, and + // a full URL would echo any `user:password@` the operator + // passed into stderr. + return cli_error(format!( + "refusing cross-origin root redirect from {} to {}; the requested origin is the audit and cookie trust boundary", + target_url.origin().ascii_serialization(), + root_url.origin().ascii_serialization() + )); + } + let plan = crawl_plan::plan_crawl( + &root_url, + &root.links, + &root.sitemap_locs, + request.budget, + ); + let targets = plan.targets(); + planned = Some(plan); + Ok(targets) + }, + &mut |url, collected| { + match collected { + Ok(page) => { + let final_url = page.final_url().unwrap_or_else(|_| url.clone()); + // The requested origin is the trust boundary for every + // page, not just the root: a section page that redirects + // away would otherwise contribute foreign slots, formats + // and ad-unit paths to the generated config. + if origin_changed(&target_url, &final_url) { + notes.push(format!( + "skipped `{}` on {first_label}: it left the audited origin for {}", + url.path(), + final_url.origin().ascii_serialization() + )); + return Ok(collector::ControlFlow::Continue); + } + if let Err(error) = + fold_collected( + &mut table, + &final_url, + &page, + first_label, + &mut notes, + ) + { + fold_error = Some(error); + return Ok(collector::ControlFlow::Stop); + } + } + Err(error) => { + // Path only, like the progress lines: a planned target + // still carries the origin and any userinfo. + notes.push(format!( + "skipped `{}` on {first_label}: {error}", + url.path() + )); + } + } + Ok(collector::ControlFlow::Continue) + }, + )?; + } + if let Some(error) = fold_error { + return Err(error); + } + let plan = planned.ok_or_else(|| { + report_error(format!( + "the {first_label} browser session did not produce a root page" + )) + })?; + notes.extend(plan.notes.iter().cloned()); + // Fragments never reach the server, so only a difference the origin acted on + // counts as a redirect worth reporting. + if without_fragment(&root_url) != without_fragment(&target_url) { + notes.push(format!( + "followed a root redirect from `{}{}` to `{}{}`; slots and page patterns are derived from the final URL", + target_url.origin().ascii_serialization(), + target_url.path(), + root_url.origin().ascii_serialization(), + root_url.path() + )); + } + + // Every profile walks the same pages into the same table. When two profiles + // disagree about a slot's ad-unit path, that shows up as two observations of + // one page, which inference already refuses to represent. + for (label, collector) in collectors.iter().skip(1) { + let mut progress_writer = CollectionProgressWriter { + out: err, + profile_label: label, + }; + let successful_pages = crawl_sections( + *collector, + &root_url, + &plan, + request.cookies, + &mut table, + &mut notes, + &mut progress_writer, + )?; + if successful_pages == 0 { + return cli_error(format!( + "the selected {label} device profile did not collect any required page; refusing to generate from incomplete profile coverage" + )); + } + } + if collectors.len() > 1 { + notes.push(format!( + "audited {} device profile(s): {}", + collectors.len(), + collectors + .iter() + .map(|(label, _)| *label) + .collect::>() + .join(", ") + )); + } + + // Emit what the crawl learned before any refusal below can return early. + // The guards exist precisely for runs that went wrong, so that is when the + // per-page reasons matter most. + emit_notes(err, &mut notes)?; + + if table.is_empty() { + return cli_error(format!( + "no ad-template slots were discovered on any of the {} crawled page(s); \ + see the notes above for what each page reported", + table.pages().len() + )); + } + guard_challenge_rate(&table)?; + + let discovered_network_id = table.network_id()?; + let network_id = resolve_network_id( + request.existing_creative, + discovered_network_id.as_deref(), + request.replace, + ); + + // Templating needs a network id to bind `{network_id}` against; without one + // every path stays literal. + let inference = network_id + .as_deref() + .map(|id| unit_template::infer_unit_templates(&table, id)); + if let Some(outcome) = &inference { + notes.extend(outcome.diagnostics.iter().cloned()); + } + let policy = inference + .as_ref() + .and_then(|outcome| outcome.policy.clone()); + validate_merge_policy(request.existing_creative, policy.as_ref(), request.replace)?; + + // Slots that are one placement wearing a per-render div id cannot be + // written: the ids never match at runtime. Report them so the operator can + // add the placement once with a prefix they know is stable. + let fragmented = table.fragmented_slots(); + for group in &fragmented { + let suggestion = group.suggested_prefix.as_deref().map_or_else( + || "no stable prefix was shared".to_string(), + |prefix| format!("they share the prefix `{prefix}`"), + ); + notes.push(format!( + "skipped {} slot(s) that look like one placement under a per-render div id on \ + `{}` ({}); {suggestion}. Add it once by hand with a div_id prefix that is \ + stable across renders", + group.div_ids.len(), + group.unit_path, + group.div_ids.join(", "), + )); + } + + let slots = build_render_slots( + &table, + inference.as_ref(), + policy.as_ref(), + request, + plan.section_segment, + &fragmented, + &mut notes, + )?; + let observed_div_ids = table + .observed_div_ids() + .map(str::to_string) + .collect::>(); + let (merged, merge_diagnostics) = slot_toml::merge_render_slots_with_observed_diagnostics( + request.existing_creative, + slots, + &observed_div_ids, + request.replace, + ); + notes.extend(merge_diagnostics.notes); + if !merge_diagnostics.unobserved_existing_slot_ids.is_empty() { + let slot_ids = merge_diagnostics.unobserved_existing_slot_ids.join(", "); + let follow_up = if request.scroll { + "Re-run with broader page/profile coverage; `--replace` prunes them but also discards every hand-written field on the slots the run did rediscover." + } else { + "Re-run with broader coverage or --scroll; `--replace` prunes them but also discards every hand-written field on the slots the run did rediscover." + }; + notes.push(format!( + "preserved {} configured slot(s) not observed during this crawl: {slot_ids}. {follow_up}", + merge_diagnostics.unobserved_existing_slot_ids.len(), + )); + } + if merged.is_empty() { + emit_notes(err, &mut notes)?; + return cli_error( + "refusing to write zero generated slots after the crawl discovered slot evidence; review the refused-slot notes and keep the existing configuration", + ); + } + let rendered_slots = render_slots(&merged); + let updated = splice_creative_slots( + &existing, + &slot_toml::CreativeSectionKeys { + network_id: network_id.as_deref(), + section_root: policy.as_ref().map(|policy| policy.section_root.as_str()), + section_segment: policy.as_ref().map(|policy| policy.section_segment), + }, + &rendered_slots, + )?; + + // Everything above is derived from a live, page-controlled ad stack, so the + // candidate has to clear the runtime's own load path before it can replace + // the operator's file. This runs on the dry-run path too — otherwise "the + // preview looked fine" would not be evidence that the config loads. + notes.extend(validate::check_candidate(&updated, &existing)?); + + emit_notes(err, &mut notes)?; + if policy.is_some() { + writeln!( + err, + "note: this config now uses a {{section}} ad-unit template. Deploy a \ + template-aware binary BEFORE pushing it, and do not roll that binary \ + back while this config is live — an older binary rejects the whole \ + config and serves an error on every route." + ) + .map_err(|error| report_error(format!("failed to write command output: {error}")))?; + } + + if request.dry_run { + let old_managed = managed_creative_projection(&existing)?; + let new_managed = managed_creative_projection(&updated)?; + if old_managed == new_managed { + // Stdout is the diff surface, so an English sentence there would + // break a redirected `--dry-run`; an empty diff is the stdout answer. + writeln!(err, "No managed creative-opportunity changes.").map_err(|error| { + report_error(format!("failed to write preview output: {error}")) + })?; + return Ok(()); + } + let diff = similar::TextDiff::from_lines(&old_managed, &new_managed); + writeln!( + out, + "{}", + diff.unified_diff().context_radius(0).header( + "configured creative opportunities", + "generated creative opportunities" + ) + ) + .map_err(|error| report_error(format!("failed to write preview diff: {error}")))?; + return Ok(()); + } + let current = fs::read_to_string(request.config_path).map_err(|error| { + report_error(format!( + "failed to re-read config {} before writing: {error}", + request.config_path.display() + )) + })?; + if current != existing { + return cli_error(format!( + "refusing to overwrite {} because it changed during the browser audit; re-run against the current file", + request.config_path.display() + )); + } + // A writer could still land between this check and the rename below. That + // window is microseconds against a browser crawl's minutes, and the rename + // is atomic, so the loser of the race loses a whole write rather than half + // of one. Closing it properly would need file locking the operator's editor + // does not take part in. + write_file_atomically(request.config_path, &updated).map_err(|error| { + report_error(format!( + "failed to write config {}: {error}", + request.config_path.display() + )) + })?; + writeln!( + out, + "Wrote {} slot(s) to {} ({} slot(s) seen across {} page(s))", + merged.len(), + request.config_path.display(), + table.slot_count(), + table.pages().len(), + ) + .map_err(|error| report_error(format!("failed to write command output: {error}"))) +} + +/// Renders only fields managed by ad-template generation, excluding secrets and +/// unrelated operator configuration from dry-run output. +fn managed_creative_projection(document: &str) -> CliResult { + let value = toml::from_str::(document).map_err(|error| { + report_error(format!("failed to parse config for dry-run diff: {error}")) + })?; + let creative = value + .get("creative_opportunities") + .and_then(toml::Value::as_table); + let mut managed = toml::map::Map::new(); + if let Some(creative) = creative { + for key in ["gam_network_id", "section_root", "section_segment", "slot"] { + if let Some(value) = creative.get(key) { + managed.insert(key.to_string(), value.clone()); + } + } + } + let mut root = toml::map::Map::new(); + root.insert( + "creative_opportunities".to_string(), + toml::Value::Table(managed), + ); + toml::to_string_pretty(&toml::Value::Table(root)) + .map_err(|error| report_error(format!("failed to render dry-run projection: {error}"))) +} + +/// A page carrying fewer scripts than this is not a real publisher page. +/// +/// A production page runs dozens: the ad stack, analytics, consent, and the +/// site's own bundles. A bot-protection interstitial runs its own challenge +/// script and little else. +const INTERSTITIAL_SCRIPT_CEILING: usize = 3; + +/// Whether a page that loaded successfully is nonetheless not the real page. +/// +/// Bot protection commonly answers with **200** and a challenge document rather +/// than a 4xx, so status-code checks pass and the page simply appears to have no +/// ad stack. Left unexplained, that is indistinguishable from a publisher who +/// genuinely runs no ads on that page — and the operator's next move is entirely +/// different in each case. +fn looks_like_an_interstitial(artifact: &AuditArtifact) -> Option { + if artifact.js_asset_count > INTERSTITIAL_SCRIPT_CEILING + || !artifact.detected_integrations.is_empty() + { + return None; + } + Some(format!( + "the page returned successfully but carried only {} script(s) and no recognised \ + integrations, which is the shape of a bot-protection challenge rather than the \ + real page. Supply a current --cookie for the origin", + artifact.js_asset_count + )) +} + +/// Writes and clears the pending notes, so each is reported exactly once. +fn emit_notes(out: &mut dyn Write, notes: &mut Vec) -> CliResult<()> { + for note in notes.drain(..) { + writeln!( + out, + "note: {}", + crate::ad_templates::output::escape_terminal_text(¬e) + ) + .map_err(|error| report_error(format!("failed to write command output: {error}")))?; + } + Ok(()) +} + +/// Writes one immediately visible, profile-aware crawl progress line. +fn write_collection_progress( + out: &mut dyn Write, + profile_label: &str, + progress: collector::CollectionProgress<'_>, +) -> CliResult<()> { + let line = match progress { + collector::CollectionProgress::Launching => { + format!("Auditing {profile_label}: launching browser") + } + collector::CollectionProgress::Loading { + current, + total, + url, + } => { + let path = if url.path().is_empty() { + "/" + } else { + url.path() + }; + let path = crate::ad_templates::output::escape_terminal_text(path); + let total = total.map_or_else(|| "?".to_string(), |total| total.to_string()); + format!("Auditing {profile_label} [{current}/{total}]: {path}") + } + collector::CollectionProgress::Planning => { + format!("Auditing {profile_label}: planning site crawl") + } + collector::CollectionProgress::Finalizing => { + format!("Auditing {profile_label}: finalizing browser session") + } + }; + writeln!(out, "{line}") + .map_err(|error| report_error(format!("failed to write audit progress: {error}")))?; + out.flush() + .map_err(|error| report_error(format!("failed to flush audit progress: {error}"))) +} + +struct CollectionProgressWriter<'a> { + out: &'a mut dyn Write, + profile_label: &'a str, +} + +impl CollectionProgressWriter<'_> { + fn write(&mut self, progress: collector::CollectionProgress<'_>) -> CliResult<()> { + write_collection_progress(self.out, self.profile_label, progress) + } +} + +/// Discovers a collected page's slots and folds them into `table`. +/// +/// Per-page collector warnings are appended to `notes`. They carry the reason a +/// page came back without slots — a non-2xx main document, a navigation that +/// never settled — which is the difference between "this publisher has no ad +/// stack here" and "bot protection served a challenge". Dropping them leaves +/// the operator with a refusal and no way to act on it. +fn fold_collected( + table: &mut evidence::EvidenceTable, + url: &Url, + collected: &collector::CollectedPage, + profile_label: &str, + notes: &mut Vec, +) -> CliResult<()> { + // `analyze_collected_page` already carries the collector's warnings forward, + // so this is the complete set, not a second copy. + let artifact = analyze_collected_page(collected)?; + for warning in &artifact.warnings { + // The consent stub is a property of the run, not of this page. Scoping it + // to a path and repeating it per page and profile buries the per-page + // diagnostics an operator is reading these notes for. + let note = if warning == collector::CONSENT_STUB_WARNING { + warning.clone() + } else { + format!("`{}` on {profile_label}: {warning}", url.path()) + }; + if !notes.contains(¬e) { + notes.push(note); + } + } + if let Some(reason) = looks_like_an_interstitial(&artifact) { + notes.push(format!("`{}` on {profile_label}: {reason}", url.path())); + } + let page_has_prebid = artifact + .detected_integrations + .iter() + .any(|integration| integration.id == "prebid"); + let discovered = gpt_slots::discover_gpt_slots( + &collected.gpt_slots, + &collected.network_requests, + page_has_prebid, + ); + for warning in &discovered.warnings { + if !notes.contains(warning) { + notes.push(warning.clone()); + } + } + table.fold_page(url.path(), &discovered); + Ok(()) +} + +/// Walks the planned section pages, folding each into `table`. +/// +/// A page that fails to collect is recorded as a note rather than aborting: on a +/// multi-section crawl one blocked or slow page should not discard the sections +/// that did work. The empty-page guard afterwards catches the case where enough +/// of them failed that the result is untrustworthy. +fn crawl_sections( + collector: &dyn AuditCollector, + root_url: &Url, + plan: &crawl_plan::CrawlPlan, + cookies: &[(String, String)], + table: &mut evidence::EvidenceTable, + notes: &mut Vec, + progress_writer: &mut CollectionProgressWriter<'_>, +) -> CliResult { + let additional_targets = plan.targets(); + if additional_targets.is_empty() { + notes.push( + "no additional site sections were discovered, so only the requested page was \ + audited; pass explicit --page-pattern values or more URLs to widen coverage" + .to_string(), + ); + } + // The root is deliberately part of every profile's shared batch: browser + // clearance/session state established there then carries into section pages. + let mut targets = Vec::with_capacity(additional_targets.len() + 1); + targets.push(root_url.clone()); + targets.extend(additional_targets); + + let mut fold_error = None; + let mut successful_pages = 0_usize; + { + let profile_label = progress_writer.profile_label; + let mut report_progress = + |progress: collector::CollectionProgress<'_>| progress_writer.write(progress); + collector.collect_pages( + &targets, + cookies, + &mut report_progress, + &mut |url, collected| { + match collected { + Ok(page) => { + let final_url = page.final_url().unwrap_or_else(|_| url.clone()); + // Same boundary as the first profile, and it covers this + // profile's root page too: a cross-origin redirect is not + // a page this run may learn inventory from, so it must + // not count towards profile coverage either. + if origin_changed(root_url, &final_url) { + notes.push(format!( + "skipped `{}` on {profile_label}: it left the audited origin for {}", + url.path(), + final_url.origin().ascii_serialization() + )); + return Ok(collector::ControlFlow::Continue); + } + successful_pages += 1; + if let Err(error) = + fold_collected(table, &final_url, &page, profile_label, notes) + { + fold_error = Some(error); + return Ok(collector::ControlFlow::Stop); + } + } + Err(error) => { + notes.push(format!( + "skipped `{}` on {profile_label}: {error}", + url.path() + )); + } + } + Ok(collector::ControlFlow::Continue) + }, + )?; + } + match fold_error { + Some(error) => Err(error), + None => Ok(successful_pages), + } +} + +/// Refuses a crawl where too many pages produced no slots. +fn guard_challenge_rate(table: &evidence::EvidenceTable) -> CliResult<()> { + let total = table.pages().len(); + let empty = table.empty_pages().len(); + if total == 0 || (empty as f64) <= (total as f64) * MAX_EMPTY_PAGE_SHARE { + return Ok(()); + } + let blocked: Vec<&str> = table.empty_pages().iter().map(String::as_str).collect(); + cli_error(format!( + "{empty} of {total} crawled page(s) produced no ad slots ({}), which usually means \ + bot protection served a challenge instead of the real page. Refusing to write a \ + config from partial evidence; re-run with a valid --cookie for the origin", + blocked.join(", ") + )) +} + +/// Refuses a merge that would reinterpret templated slots the config already has. +/// +/// # Errors +/// +/// Returns an error when preserved `{section}` slots were written against a +/// different section policy than this run inferred, since the merge would leave +/// them pointing at ad units nobody configured. +fn validate_merge_policy( + existing: Option<&CreativeOpportunitiesConfig>, + inferred: Option<&unit_template::SectionPolicy>, + replace: bool, +) -> CliResult<()> { + if replace { + return Ok(()); + } + let Some(existing) = existing else { + return Ok(()); + }; + let preserves_template = existing.slot.iter().any(|slot| { + slot.gam_unit_path + .as_deref() + .is_some_and(|path| path.contains("{section}")) + }); + let Some(inferred) = inferred.filter(|_| preserves_template) else { + return Ok(()); + }; + if let Some(configured_segment) = existing.section_segment + && configured_segment != inferred.section_segment + { + return cli_error(format!( + "refusing to change the section_segment used by preserved templated slots during merge: configured section_segment={configured_segment}; inferred section_segment={}. Re-run with --replace only for an intentional migration", + inferred.section_segment + )); + } + // A `{section}` slot with no `section_root` cannot load at all — + // `validate_runtime` requires one — so there is no root value to preserve. + // Adopting the inferred root makes such a config loadable, provided the + // independently configured section segment above still agrees. + let Some(configured_root) = existing + .section_root + .as_deref() + .filter(|root| !root.is_empty()) + else { + return Ok(()); + }; + let configured_segment = existing.section_segment.unwrap_or(0); + if configured_root != inferred.section_root || configured_segment != inferred.section_segment { + return cli_error(format!( + "refusing to change the section policy used by preserved templated slots during merge: configured section_root={configured_root:?}, section_segment={configured_segment}; inferred section_root={:?}, section_segment={}. Re-run with --replace only for an intentional migration", + inferred.section_root, inferred.section_segment + )); + } + Ok(()) +} + +/// Turns the evidence table into slots ready to render. +fn build_render_slots( + table: &evidence::EvidenceTable, + inference: Option<&unit_template::InferenceOutcome>, + policy: Option<&unit_template::SectionPolicy>, + request: &UpdateSlotsRequest<'_>, + fallback_section_segment: usize, + fragmented: &[evidence::FragmentGroup], + notes: &mut Vec, +) -> CliResult> { + let skip: std::collections::BTreeSet<&str> = fragmented + .iter() + .flat_map(|group| group.div_ids.iter().map(String::as_str)) + .collect(); + // Explicit `--page-pattern` values are an operator override: they apply to + // every slot and disable inference from observed paths entirely. + let explicit = !request.page_patterns.is_empty(); + if explicit { + validate_page_patterns(request.page_patterns)?; + // Not filtered against `skip`: a borrowed root implies the slot's + // ad-unit path varied across pages, and `fragmented_slots` only groups + // slots pinned to exactly one unit path, so the two sets are disjoint. + if let Some(outcome) = inference + && !outcome.borrowed_section_root.is_empty() + { + let affected = outcome + .borrowed_section_root + .iter() + .map(|stem| format!("`{stem}`")) + .collect::>() + .join(", "); + return cli_error(format!( + "cannot apply --page-pattern to slot(s) with div id(s) {affected} because their \ + {{section}} templates borrow section_root; remove --page-pattern so patterns \ + can be derived from the paths where each slot was observed" + )); + } + } + let section_segment = policy.map_or(fallback_section_segment, |policy| policy.section_segment); + + let mut slots = Vec::with_capacity(table.slot_count()); + for slot in table.slots() { + if skip.contains(slot.div_id.as_str()) { + continue; + } + let patterns = if explicit { + request.page_patterns.to_vec() + } else { + let derived = page_patterns::patterns_for_paths(slot.paths(), section_segment); + validate_page_patterns(&derived)?; + derived + }; + let unit_path = match inference.and_then(|outcome| outcome.decision(&slot.div_id)) { + Some(unit_template::SlotDecision::Template(template)) => Some(template.clone()), + Some(unit_template::SlotDecision::Literal(path)) => Some(path.clone()), + Some(unit_template::SlotDecision::Refuse { reasons }) => { + notes.push(format!( + "skipped refused slot `{}` (`{}`): {}", + slot.id, + slot.div_id, + reasons.join("; ") + )); + continue; + } + None => None, + }; + slots.push(slot_toml::RenderSlot::from_evidence( + &slot.id, + &slot.div_id, + unit_path, + slot.formats.iter().copied(), + patterns, + slot.has_prebid, + )); + } + Ok(slots) +} +/// Rejects any page pattern the runtime's glob compiler would not accept. +/// +/// Uses [`validate_page_pattern`] so the accepted set is exactly what +/// `CreativeOpportunitySlot::compile_patterns` accepts at startup, including the +/// `**`→`*` normalisation. All patterns are reported at once so an operator +/// passing several `--page-pattern` values fixes them in one pass. +/// +/// # Errors +/// +/// Returns a user-facing error listing every pattern that does not compile. +fn validate_page_patterns(patterns: &[String]) -> CliResult<()> { + let invalid: Vec = patterns + .iter() + .filter_map(|pattern| validate_page_pattern(pattern).err()) + .collect(); + if invalid.is_empty() { + return Ok(()); + } + cli_error(format!( + "refusing to write invalid page pattern(s): {}", + invalid.join("; ") + )) +} + +#[cfg(test)] +mod tests { + use std::cell::{Cell, RefCell}; + use std::io; + use std::rc::Rc; + + use tempfile::TempDir; + + use super::*; + use crate::app_config::AppConfigArgs; + use crate::commands::audit::generate::collector::{ + CollectedPage, CollectedRequest, CollectedScriptTag, + }; + use crate::commands::config::init::EXAMPLE_CONFIG; + + struct FakeCollector { + collected: CollectedPage, + calls: Cell, + } + + struct MutatingCollector { + collected: CollectedPage, + config_path: std::path::PathBuf, + replacement: String, + } + + impl AuditCollector for MutatingCollector { + fn collect_page( + &self, + _target_url: &Url, + _cookies: &[(String, String)], + ) -> CliResult { + fs::write(&self.config_path, &self.replacement) + .map_err(|error| report_error(format!("failed to mutate test config: {error}")))?; + Ok(self.collected.clone()) + } + } + + impl FakeCollector { + fn new(collected: CollectedPage) -> Self { + Self { + collected, + calls: Cell::new(0), + } + } + } + + impl AuditCollector for FakeCollector { + fn collect_page( + &self, + _target_url: &Url, + _cookies: &[(String, String)], + ) -> CliResult { + self.calls.set(self.calls.get() + 1); + Ok(self.collected.clone()) + } + } + + /// A collector serving a distinct page per URL, recording the crawl order. + struct SiteCollector { + pages: std::collections::HashMap, + visited: std::cell::RefCell>, + } + + struct FailingCollector; + + #[derive(Clone, Default)] + struct SharedProgressState { + bytes: Rc>>, + flushes: Rc>, + } + + struct SharedProgressWriter { + state: SharedProgressState, + } + + impl Write for SharedProgressWriter { + fn write(&mut self, buffer: &[u8]) -> io::Result { + self.state.bytes.borrow_mut().extend_from_slice(buffer); + Ok(buffer.len()) + } + + fn flush(&mut self) -> io::Result<()> { + self.state.flushes.set(self.state.flushes.get() + 1); + Ok(()) + } + } + + struct ObservingProgressCollector { + collected: CollectedPage, + state: SharedProgressState, + saw_flushed_progress: Cell, + } + + impl AuditCollector for ObservingProgressCollector { + fn collect_page( + &self, + _target_url: &Url, + _cookies: &[(String, String)], + ) -> CliResult { + Ok(self.collected.clone()) + } + + fn collect_site( + &self, + root: &Url, + _cookies: &[(String, String)], + on_progress: collector::ProgressSink<'_>, + planner: collector::RootPlanner<'_>, + on_page: collector::PageSink<'_>, + ) -> CliResult<()> { + on_progress(collector::CollectionProgress::Loading { + current: 1, + total: None, + url: root, + })?; + self.saw_flushed_progress + .set(!self.state.bytes.borrow().is_empty() && self.state.flushes.get() > 0); + on_progress(collector::CollectionProgress::Planning)?; + let _ = planner(root, &self.collected)?; + let _ = on_page(root, Ok(self.collected.clone()))?; + Ok(()) + } + } + + #[derive(Default)] + struct ProgressWriter { + bytes: Vec, + flushes: usize, + fail_write: bool, + fail_flush: bool, + } + + impl Write for ProgressWriter { + fn write(&mut self, buffer: &[u8]) -> io::Result { + if self.fail_write { + return Err(io::Error::other("simulated progress write failure")); + } + self.bytes.extend_from_slice(buffer); + Ok(buffer.len()) + } + + fn flush(&mut self) -> io::Result<()> { + self.flushes += 1; + if self.fail_flush { + return Err(io::Error::other("simulated progress flush failure")); + } + Ok(()) + } + } + + #[test] + fn progress_lines_are_profile_aware_and_flush_immediately() { + let url = + Url::parse("https://user:pass@publisher.example/news\u{1b}[31m?token=secret#fragment") + .expect("should parse progress URL"); + let mut writer = ProgressWriter::default(); + + for progress in [ + collector::CollectionProgress::Launching, + collector::CollectionProgress::Loading { + current: 1, + total: None, + url: &url, + }, + collector::CollectionProgress::Planning, + collector::CollectionProgress::Loading { + current: 2, + total: Some(17), + url: &url, + }, + collector::CollectionProgress::Finalizing, + ] { + write_collection_progress(&mut writer, "desktop", progress) + .expect("should write progress"); + } + + let rendered = String::from_utf8(writer.bytes).expect("should render UTF-8 progress"); + assert_eq!( + rendered, + "Auditing desktop: launching browser\n\ + Auditing desktop [1/?]: /news%1B[31m\n\ + Auditing desktop: planning site crawl\n\ + Auditing desktop [2/17]: /news%1B[31m\n\ + Auditing desktop: finalizing browser session\n" + ); + assert_eq!(writer.flushes, 5, "should flush every progress line"); + assert!(!rendered.contains("user"), "should omit URL userinfo"); + assert!(!rendered.contains("secret"), "should omit URL query values"); + assert!(!rendered.contains("fragment"), "should omit URL fragments"); + assert!( + !rendered.contains('\u{1b}'), + "should not emit terminal escapes" + ); + } + + #[test] + fn progress_write_and_flush_failures_are_reported() { + let mut write_failure = ProgressWriter { + fail_write: true, + ..ProgressWriter::default() + }; + let write_error = write_collection_progress( + &mut write_failure, + "desktop", + collector::CollectionProgress::Launching, + ) + .expect_err("should report progress write failure"); + assert!(format!("{write_error:?}").contains("failed to write audit progress")); + + let mut flush_failure = ProgressWriter { + fail_flush: true, + ..ProgressWriter::default() + }; + let flush_error = write_collection_progress( + &mut flush_failure, + "desktop", + collector::CollectionProgress::Finalizing, + ) + .expect_err("should report progress flush failure"); + assert!(format!("{flush_error:?}").contains("failed to flush audit progress")); + } + + #[test] + fn update_slots_flushes_progress_before_collection_returns() { + let temp = TempDir::new().expect("should create temp dir"); + let config_path = temp.path().join("trusted-server.toml"); + fs::write( + &config_path, + "[creative_opportunities]\ngam_network_id = \"123456789\"\n", + ) + .expect("should write config"); + let state = SharedProgressState::default(); + let collector = ObservingProgressCollector { + collected: collected_page_with_header_slot(), + state: state.clone(), + saw_flushed_progress: Cell::new(false), + }; + let mut progress_writer = SharedProgressWriter { state }; + let mut out = Vec::new(); + + run_update_slots( + &UpdateSlotsRequest { + url: "https://publisher.example/", + config_path: &config_path, + existing_creative: None, + page_patterns: &[], + replace: false, + cookies: &[], + dry_run: false, + scroll: false, + budget: CrawlBudget::default(), + }, + &[("desktop", &collector)], + &mut out, + &mut progress_writer, + ) + .expect("should generate slots"); + + assert!( + collector.saw_flushed_progress.get(), + "collector should observe flushed progress before returning" + ); + assert!( + !String::from_utf8(out) + .expect("should write UTF-8 output") + .contains("Auditing "), + "stdout should not contain progress" + ); + } + + impl AuditCollector for FailingCollector { + fn collect_page( + &self, + target_url: &Url, + _cookies: &[(String, String)], + ) -> CliResult { + cli_error(format!("simulated navigation failure for {target_url}")) + } + } + + impl SiteCollector { + fn new(pages: Vec<(&str, CollectedPage)>) -> Self { + Self { + pages: pages + .into_iter() + .map(|(url, page)| (url.to_string(), page)) + .collect(), + visited: std::cell::RefCell::new(Vec::new()), + } + } + } + + impl AuditCollector for SiteCollector { + fn collect_page( + &self, + target_url: &Url, + _cookies: &[(String, String)], + ) -> CliResult { + self.visited.borrow_mut().push(target_url.to_string()); + self.pages + .get(target_url.as_str()) + .cloned() + .ok_or_else(|| report_error(format!("no fake page for {target_url}"))) + } + } + + /// Builds a page carrying one GPT slot plus same-origin nav links. + fn site_page(url: &str, unit_path: &str, nav_paths: &[&str]) -> CollectedPage { + let mut page = collected_page(); + page.requested_url = url.to_string(); + page.final_url = url.to_string(); + page.gpt_slots = vec![collector::CollectedGptSlot { + gam_unit_path: unit_path.to_string(), + div_id: "ad-header-0".to_string(), + sizes: vec![(728, 90)], + }]; + page.links = nav_paths + .iter() + .map(|path| collector::CollectedLink { + url: format!("https://publisher.example{path}"), + in_nav: true, + }) + .collect(); + page + } + + fn collected_page() -> CollectedPage { + CollectedPage { + requested_url: "https://publisher.example/page".to_string(), + final_url: "https://publisher.example/page".to_string(), + page_title: Some("Example Publisher".to_string()), + html: r#"Example Publisher"#.to_string(), + script_tags: vec![ + CollectedScriptTag { + src: Some("https://www.googletagmanager.com/gtm.js?id=GTM-ABC123".to_string()), + inline_text: None, + }, + CollectedScriptTag { + src: Some("https://securepubads.g.doubleclick.net/tag/js/gpt.js".to_string()), + inline_text: None, + }, + ], + network_requests: vec![CollectedRequest { + url: "https://cdn.publisher.example/app.js".to_string(), + resource_type: Some("script".to_string()), + }], + gpt_slots: Vec::new(), + links: Vec::new(), + sitemap_locs: Vec::new(), + warnings: Vec::new(), + } + } + + /// A collected page carrying one discoverable GPT slot, for `run_update_slots`. + fn collected_page_with_header_slot() -> CollectedPage { + let mut collected = collected_page(); + collected.requested_url = "https://publisher.example/".to_string(); + collected.final_url = "https://publisher.example/".to_string(); + collected.gpt_slots = vec![collector::CollectedGptSlot { + gam_unit_path: "/222/homepage/header".to_string(), + div_id: "div-gpt-ad-header".to_string(), + sizes: vec![(728, 90)], + }]; + collected + } + + fn collected_page_with_ambiguous_slots(url: &str) -> CollectedPage { + let mut collected = collected_page(); + collected.requested_url = url.to_string(); + collected.final_url = url.to_string(); + collected.gpt_slots = vec![ + collector::CollectedGptSlot { + gam_unit_path: "/222/homepage/in-content".to_string(), + div_id: "ad-x-aaaaaaaaaaaaaaaa-0".to_string(), + sizes: vec![(300, 250)], + }, + collector::CollectedGptSlot { + gam_unit_path: "/222/homepage/in-content".to_string(), + div_id: "ad-x-bbbbbbbbbbbbbbbb-1".to_string(), + sizes: vec![(300, 250)], + }, + ]; + collected + } + + fn audit_args(url: &str) -> GenerateArgs { + GenerateArgs { + url: url.to_string(), + js_assets: None, + config: None, + no_js_assets: false, + no_config: false, + force: false, + cookies: Vec::new(), + browser: GenerateBrowserOpts::default(), + } + } + + #[test] + fn parse_audit_url_accepts_http_and_https() { + assert!(parse_audit_url("http://publisher.example").is_ok()); + assert!(parse_audit_url("https://publisher.example").is_ok()); + } + + #[test] + fn parse_audit_url_rejects_non_http_schemes() { + for url in [ + "file:///etc/passwd", + "data:text/html,hello", + "chrome://version", + ] { + let error = parse_audit_url(url).expect_err("should reject non-http URL"); + assert!( + format!("{error:?}").contains("only supports http/https"), + "should explain scheme restriction" + ); + } + } + + #[test] + fn repeated_ambiguous_collision_note_is_emitted_once() { + let mut table = evidence::EvidenceTable::default(); + let mut notes = Vec::new(); + for url in [ + "https://publisher.example/", + "https://publisher.example/news", + ] { + fold_collected( + &mut table, + &Url::parse(url).expect("should parse fixture URL"), + &collected_page_with_ambiguous_slots(url), + "desktop", + &mut notes, + ) + .expect("should fold ambiguous page evidence"); + } + + assert_eq!( + notes.len(), + 1, + "the same site-wide collision guidance should not repeat per page" + ); + } + + #[test] + fn merge_refuses_to_change_policy_used_by_preserved_templates() { + let existing: CreativeOpportunitiesConfig = toml::from_str( + "gam_network_id = \"123\"\nsection_root = \"home\"\nsection_segment = 0\n\ + [[slot]]\nid = \"header\"\ndiv_id = \"ad-header\"\n\ + gam_unit_path = \"/{network_id}/site/{section}\"\npage_patterns = [\"/\"]\n\ + formats = [{ width = 728, height = 90 }]\n", + ) + .expect("should parse creative config"); + let inferred = unit_template::SectionPolicy { + section_root: "homepage".to_string(), + section_segment: 1, + }; + + let error = validate_merge_policy(Some(&existing), Some(&inferred), false) + .expect_err("merge must preserve the existing template policy"); + + assert!(format!("{error:?}").contains("--replace")); + validate_merge_policy(Some(&existing), Some(&inferred), true) + .expect("replace is an explicit policy migration"); + } + + #[test] + fn the_consent_stub_note_is_reported_once_and_unscoped() { + let mut table = evidence::EvidenceTable::default(); + let mut notes = Vec::new(); + for url in [ + "https://publisher.example/", + "https://publisher.example/news", + ] { + let mut page = collected_page(); + page.requested_url = url.to_string(); + page.final_url = url.to_string(); + page.warnings + .push(collector::CONSENT_STUB_WARNING.to_string()); + fold_collected( + &mut table, + &Url::parse(url).expect("should parse fixture URL"), + &page, + "desktop", + &mut notes, + ) + .expect("should fold page evidence"); + } + + assert_eq!( + notes, + [collector::CONSENT_STUB_WARNING.to_string()], + "a run-wide fact should appear once, without a page path" + ); + } + + #[test] + fn page_warnings_remain_distinct_across_profiles() { + let mut table = evidence::EvidenceTable::default(); + let mut notes = Vec::new(); + let mut page = collected_page(); + page.requested_url = "https://publisher.example/news".to_string(); + page.final_url = page.requested_url.clone(); + page.warnings.push("navigation did not settle".to_string()); + let url = Url::parse(&page.final_url).expect("should parse fixture URL"); + + fold_collected(&mut table, &url, &page, "desktop", &mut notes) + .expect("should fold desktop evidence"); + fold_collected(&mut table, &url, &page, "mobile", &mut notes) + .expect("should fold mobile evidence"); + + assert_eq!( + notes.len(), + 2, + "profile-specific warnings must not collapse" + ); + assert!(notes.iter().any(|note| note.contains("on desktop"))); + assert!(notes.iter().any(|note| note.contains("on mobile"))); + } + + #[test] + fn merge_adopts_the_inferred_policy_when_none_is_configured() { + // A hand-written `{section}` slot with no `section_root` describes a + // config the runtime refuses to load, so the first merge should repair it + // rather than demand `--replace` (which would discard the hand-tuned + // slots it is preserving). + let existing: CreativeOpportunitiesConfig = toml::from_str( + "gam_network_id = \"123\"\n\ + [[slot]]\nid = \"header\"\ndiv_id = \"ad-header\"\n\ + gam_unit_path = \"/{network_id}/site/{section}\"\npage_patterns = [\"/\"]\n\ + formats = [{ width = 728, height = 90 }]\n", + ) + .expect("should parse creative config"); + let inferred = unit_template::SectionPolicy { + section_root: "homepage".to_string(), + section_segment: 1, + }; + + validate_merge_policy(Some(&existing), Some(&inferred), false) + .expect("should have no policy to preserve when section_root is unset"); + } + + #[test] + fn merge_preserves_an_explicit_segment_when_section_root_is_unset() { + let existing: CreativeOpportunitiesConfig = toml::from_str( + "gam_network_id = \"123\"\nsection_segment = 1\n\ + [[slot]]\nid = \"header\"\ndiv_id = \"ad-header\"\n\ + gam_unit_path = \"/{network_id}/site/{section}\"\npage_patterns = [\"/\"]\n\ + formats = [{ width = 728, height = 90 }]\n", + ) + .expect("should parse creative config"); + let mismatched = unit_template::SectionPolicy { + section_root: "homepage".to_string(), + section_segment: 0, + }; + + let error = validate_merge_policy(Some(&existing), Some(&mismatched), false) + .expect_err("should preserve an explicitly configured segment"); + + assert!(format!("{error:?}").contains("section_segment=1")); + + let matching = unit_template::SectionPolicy { + section_root: "homepage".to_string(), + section_segment: 1, + }; + validate_merge_policy(Some(&existing), Some(&matching), false) + .expect("should adopt a root without changing the configured segment"); + } + + #[test] + fn resolve_output_plan_rejects_no_outputs() { + let mut args = audit_args("https://publisher.example"); + args.no_js_assets = true; + args.no_config = true; + + let error = resolve_output_plan(&args).expect_err("should reject empty output set"); + + assert!( + format!("{error:?}").contains("nothing to do"), + "should explain no-output error" + ); + } + + #[test] + fn resolve_output_plan_rejects_existing_files_without_force() { + let temp = TempDir::new().expect("should create temp dir"); + let path = temp.path().join("js-assets.toml"); + fs::write(&path, "existing").expect("should write existing file"); + let mut args = audit_args("https://publisher.example"); + args.js_assets = Some(path); + args.no_config = true; + + let error = resolve_output_plan(&args).expect_err("should reject overwrite"); + + assert!( + format!("{error:?}").contains("refusing to overwrite"), + "should explain overwrite refusal" + ); + } + + #[test] + fn resolve_output_plan_allows_existing_files_with_force() { + let temp = TempDir::new().expect("should create temp dir"); + let path = temp.path().join("js-assets.toml"); + fs::write(&path, "existing").expect("should write existing file"); + let mut args = audit_args("https://publisher.example"); + args.js_assets = Some(path.clone()); + args.no_config = true; + args.force = true; + + let plan = resolve_output_plan(&args).expect("should allow forced overwrite"); + + assert_eq!(plan.js_assets_path.as_deref(), Some(path.as_path())); + } + + #[test] + fn run_generate_writes_selected_outputs_and_summary() { + let temp = TempDir::new().expect("should create temp dir"); + let js_assets = temp.path().join("audit/js-assets.toml"); + let config = temp.path().join("audit/trusted-server.toml"); + let args = GenerateArgs { + url: "https://publisher.example/page".to_string(), + js_assets: Some(js_assets.clone()), + config: Some(config.clone()), + no_js_assets: false, + no_config: false, + force: false, + cookies: Vec::new(), + browser: GenerateBrowserOpts::default(), + }; + let collector = FakeCollector::new(collected_page()); + let mut out = Vec::new(); + + run_generate(&args, &collector, &mut out).expect("should run audit"); + + assert_eq!(collector.calls.get(), 1, "should collect page once"); + assert!(js_assets.exists(), "should write JS assets"); + assert!(config.exists(), "should write draft config"); + let summary = String::from_utf8(out).expect("summary should be UTF-8"); + assert!(summary.contains("Audited https://publisher.example/page")); + assert!(summary.contains("Detected integrations: google_tag_manager, gpt")); + assert!(summary.contains("Draft config: review before validation and push")); + } + + #[test] + fn run_generate_respects_no_config() { + let temp = TempDir::new().expect("should create temp dir"); + let js_assets = temp.path().join("js-assets.toml"); + let mut args = audit_args("https://publisher.example/page"); + args.js_assets = Some(js_assets.clone()); + args.no_config = true; + let collector = FakeCollector::new(collected_page()); + + run_generate(&args, &collector, &mut Vec::new()).expect("should run audit"); + + assert!(js_assets.exists(), "should write assets"); + assert!( + !temp.path().join("trusted-server.toml").exists(), + "should not write config" + ); + } + + #[test] + fn run_generate_respects_no_js_assets() { + let temp = TempDir::new().expect("should create temp dir"); + let config = temp.path().join("trusted-server.toml"); + let mut args = audit_args("https://publisher.example/page"); + args.config = Some(config.clone()); + args.no_js_assets = true; + let collector = FakeCollector::new(collected_page()); + let mut out = Vec::new(); + + run_generate(&args, &collector, &mut out).expect("should run audit"); + + assert!(config.exists(), "should write config"); + assert!( + !temp.path().join("js-assets.toml").exists(), + "should not write JS assets" + ); + let summary = String::from_utf8(out).expect("summary should be UTF-8"); + assert!(summary.contains("Draft config: review before validation and push")); + } + + #[test] + fn run_generate_writes_collector_warnings_to_asset_artifact() { + let temp = TempDir::new().expect("should create temp dir"); + let js_assets = temp.path().join("js-assets.toml"); + let mut args = audit_args("https://publisher.example/page"); + args.js_assets = Some(js_assets.clone()); + args.no_config = true; + let mut collected = collected_page(); + collected.warnings.push( + "browser audit timed out while waiting for the page to settle; results may be partial" + .to_string(), + ); + let collector = FakeCollector::new(collected); + + run_generate(&args, &collector, &mut Vec::new()).expect("should run audit"); + + let artifact = fs::read_to_string(js_assets).expect("should read artifact"); + assert!( + artifact.contains("results may be partial"), + "should persist collector warning" + ); + } + + #[test] + fn run_generate_conflict_prevents_collection() { + let temp = TempDir::new().expect("should create temp dir"); + let js_assets = temp.path().join("js-assets.toml"); + fs::write(&js_assets, "existing").expect("should write existing file"); + let mut args = audit_args("https://publisher.example/page"); + args.js_assets = Some(js_assets); + args.no_config = true; + let collector = FakeCollector::new(collected_page()); + + let error = run_generate(&args, &collector, &mut Vec::new()) + .expect_err("should reject existing output"); + + assert_eq!(collector.calls.get(), 0, "should not collect page"); + assert!( + format!("{error:?}").contains("refusing to overwrite"), + "should report overwrite conflict" + ); + } + + #[test] + fn build_draft_config_uses_final_url_and_detected_integrations() { + let url = Url::parse("https://www.publisher.example:8443/path").expect("should parse URL"); + let artifact = AuditArtifact { + audited_url: url.to_string(), + page_title: Some("Example".to_string()), + js_asset_count: 2, + third_party_asset_count: 2, + detected_integrations: vec![ + DetectedIntegration { + id: "google_tag_manager".to_string(), + evidence: "GTM-ABC123".to_string(), + }, + DetectedIntegration { + id: "gpt".to_string(), + evidence: "https://securepubads.g.doubleclick.net/tag/js/gpt.js".to_string(), + }, + DetectedIntegration { + id: "prebid".to_string(), + evidence: "inline script matched `prebid`".to_string(), + }, + ], + assets: Vec::new(), + warnings: Vec::new(), + }; + + let draft = build_draft_config(&url, &artifact, &gpt_slots::DiscoveredSlots::default()) + .expect("should build draft config"); + + assert!(draft.contains("domain = \"www.publisher.example\"")); + assert!(draft.contains("cookie_domain = \".www.publisher.example\"")); + assert!(draft.contains("origin_url = \"https://www.publisher.example:8443\"")); + assert!(draft.contains("[integrations.gpt]\nenabled = true")); + assert!(draft.contains("[integrations.google_tag_manager]\nenabled = true")); + assert!(draft.contains("container_id = \"GTM-ABC123\"")); + assert!(draft.contains("Detected prebid")); + toml::from_str::(&draft).expect("draft should parse as TOML"); + } + + #[test] + fn build_draft_config_does_not_enable_gtm_without_container_id() { + let url = Url::parse("https://publisher.example/path").expect("should parse URL"); + let artifact = AuditArtifact { + audited_url: url.to_string(), + page_title: None, + js_asset_count: 1, + third_party_asset_count: 1, + detected_integrations: vec![DetectedIntegration { + id: "google_tag_manager".to_string(), + evidence: "https://www.googletagmanager.com/gtm.js".to_string(), + }], + assets: Vec::new(), + warnings: Vec::new(), + }; + + let draft = build_draft_config(&url, &artifact, &gpt_slots::DiscoveredSlots::default()) + .expect("should build draft config"); + + assert!(draft.contains("[integrations.google_tag_manager]\nenabled = false")); + assert!(draft.contains("Detected google_tag_manager")); + } + + #[test] + fn build_audit_outputs_reconstructs_creative_opportunity_slots() { + let collected = CollectedPage { + requested_url: "https://example.com/".to_string(), + final_url: "https://example.com/".to_string(), + page_title: Some("Example Publisher".to_string()), + html: "".to_string(), + script_tags: Vec::new(), + network_requests: vec![CollectedRequest { + url: "https://securepubads.g.doubleclick.net/gampad/ads?\ + iu_parts=123456789%2Cdesktop%2Chomepage%2Cleaderboard1\ + &prev_iu_szs=970x250%7C4x1%7C620x366\ + &dids=div-gpt-ad-leaderboard-1\ + &prev_scp=baseDivId%3Ddiv-gpt-ad-leaderboard-1%26test%3Dprebid" + .to_string(), + resource_type: Some("fetch".to_string()), + }], + gpt_slots: Vec::new(), + links: Vec::new(), + sitemap_locs: Vec::new(), + warnings: Vec::new(), + }; + + let outputs = build_audit_outputs(&collected).expect("should build outputs"); + assert_eq!(outputs.ad_slot_count, 1, "should discover one slot"); + + // The drafted config must be valid TOML with the reconstructed slot. + let value = toml::from_str::(&outputs.draft_config_toml) + .expect("should parse draft config"); + let creative = &value["creative_opportunities"]; + assert_eq!(creative["gam_network_id"].as_str(), Some("123456789")); + let slot = &creative["slot"][0]; + assert_eq!(slot["id"].as_str(), Some("leaderboard-1")); + assert_eq!( + slot["gam_unit_path"].as_str(), + Some("/123456789/desktop/homepage/leaderboard1") + ); + assert_eq!( + slot["formats"][0]["width"].as_integer(), + Some(970), + "should keep the 970x250 pixel size" + ); + assert!( + slot["providers"]["prebid"].is_table(), + "prev_scp test=prebid should emit a prebid provider" + ); + } + + #[test] + fn render_discovered_slots_escapes_page_controlled_strings() { + // Slot fields scraped from the live page must be escaped so a quote + // cannot inject TOML into the drafted config. + let registry = vec![collector::CollectedGptSlot { + gam_unit_path: "/222/homepage/head\"er".to_string(), + div_id: "div-gpt-ad-head\"er".to_string(), + sizes: vec![(728, 90)], + }]; + let slots = gpt_slots::discover_gpt_slots(®istry, &[], false); + let url = Url::parse("https://publisher.example/").expect("should parse URL"); + + let rendered = render_discovered_slots(&url, &slots); + + let value = toml::from_str::(&rendered) + .expect("should render valid TOML despite embedded quotes"); + let slot = &value["creative_opportunities"]["slot"][0]; + assert_eq!( + slot["div_id"].as_str(), + Some("div-gpt-ad-head\"er"), + "should keep the quote as data, not TOML syntax" + ); + } + + #[test] + fn update_slots_defaults_pattern_to_final_url_after_redirect() { + let temp = TempDir::new().expect("should create temp dir"); + let config_path = temp.path().join("trusted-server.toml"); + fs::write( + &config_path, + "[creative_opportunities]\ngam_network_id = \"111\"\n", + ) + .expect("should write config"); + // The requested URL redirects; slots are scraped from the final page. + let mut collected = collected_page(); + collected.requested_url = "https://publisher.example/".to_string(); + collected.final_url = "https://publisher.example/news/story".to_string(); + collected.gpt_slots = vec![collector::CollectedGptSlot { + gam_unit_path: "/222/homepage/header".to_string(), + div_id: "div-gpt-ad-header".to_string(), + sizes: vec![(728, 90)], + }]; + let collector = FakeCollector::new(collected); + let mut out = Vec::new(); + + run_update_slots( + &UpdateSlotsRequest { + url: "https://publisher.example/", + config_path: &config_path, + existing_creative: None, + page_patterns: &[], + replace: false, + cookies: &[], + dry_run: false, + scroll: false, + budget: CrawlBudget::default(), + }, + &[("desktop", &collector)], + &mut out, + &mut std::io::sink(), + ) + .expect("should update slots"); + + let written = fs::read_to_string(&config_path).expect("should read config"); + let value = toml::from_str::(&written).expect("should parse valid TOML"); + let patterns: Vec<&str> = value["creative_opportunities"]["slot"][0]["page_patterns"] + .as_array() + .expect("should have page_patterns array") + .iter() + .map(|entry| entry.as_str().expect("should have pattern string")) + .collect(); + // Patterns come from the post-redirect path: had the requested `/` been + // used, this would be `["/"]`. They now cover the whole section rather + // than only the one article that happened to be scraped. + assert_eq!( + patterns, + ["/news", "/news/*"], + "should derive section patterns from the post-redirect path" + ); + } + + #[test] + fn update_slots_reports_preserved_unobserved_slots_contextually() { + for (scroll, expected_follow_up, unexpected_follow_up) in [ + (false, "or --scroll", "page/profile coverage"), + (true, "page/profile coverage", "or --scroll"), + ] { + let temp = TempDir::new().expect("should create temp dir"); + let config_path = temp.path().join("trusted-server.toml"); + let mut original = loadable_config() + .replace("gam_network_id = \"123456789\"", "gam_network_id = \"222\""); + original.push_str( + "\n[[creative_opportunities.slot]]\n\ + id = \"header\"\n\ + div_id = \"div-gpt-ad-header\"\n\ + gam_unit_path = \"/222/homepage/header\"\n\ + page_patterns = [\"/\"]\n\ + formats = [{ width = 728, height = 90 }]\n\n\ + [[creative_opportunities.slot]]\n\ + id = \"sidebar\"\n\ + div_id = \"ad-sidebar\"\n\ + gam_unit_path = \"/222/sidebar\"\n\ + page_patterns = [\"/news/*\"]\n\ + formats = [{ width = 300, height = 250 }]\n", + ); + fs::write(&config_path, &original).expect("should write config"); + let existing = crate::commands::audit::creative_config(&original, &config_path) + .expect("should parse config") + .expect("should have creative opportunities"); + let collector = FakeCollector::new(collected_page_with_header_slot()); + let mut out = Vec::new(); + let mut notes = Vec::new(); + + run_update_slots( + &UpdateSlotsRequest { + url: "https://publisher.example/", + config_path: &config_path, + existing_creative: Some(&existing), + page_patterns: &[], + replace: false, + cookies: &[], + dry_run: true, + scroll, + budget: CrawlBudget::default(), + }, + &[("desktop", &collector)], + &mut out, + &mut notes, + ) + .expect("should preserve unobserved slot"); + + let notes = String::from_utf8(notes).expect("notes should be UTF-8"); + assert!( + notes.contains( + "preserved 1 configured slot(s) not observed during this crawl: sidebar" + ), + "should name the preserved slot, got {notes:?}" + ); + assert!( + notes.contains(expected_follow_up), + "should suggest the follow-up matching the scroll setting, got {notes:?}" + ); + assert!( + !notes.contains(unexpected_follow_up), + "should omit the follow-up that does not apply, got {notes:?}" + ); + assert!( + notes.contains("discards every hand-written field"), + "should explain the full cost of --replace, got {notes:?}" + ); + assert!(out.is_empty(), "unchanged dry-run stdout should stay empty"); + assert_eq!( + fs::read_to_string(&config_path).expect("should read config"), + original, + "dry-run should preserve the original config" + ); + } + } + + #[test] + fn observed_but_refused_slot_is_not_reported_as_unobserved() { + let temp = TempDir::new().expect("should create temp dir"); + let config_path = temp.path().join("trusted-server.toml"); + let mut original = loadable_config(); + original.push_str( + "\n[[creative_opportunities.slot]]\n\ + id = \"stable\"\n\ + div_id = \"ad-stable\"\n\ + gam_unit_path = \"/123456789/site/header\"\n\ + page_patterns = [\"/\"]\n\ + formats = [{ width = 728, height = 90 }]\n\n\ + [[creative_opportunities.slot]]\n\ + id = \"ad-refused\"\n\ + gam_unit_path = \"/123456789/desktop/homepage\"\n\ + page_patterns = [\"/\"]\n\ + formats = [{ width = 300, height = 250 }]\n", + ); + fs::write(&config_path, &original).expect("should write config"); + let existing = crate::commands::audit::creative_config(&original, &config_path) + .expect("should parse config") + .expect("should have creative opportunities"); + + let page = |profile: &str| { + let mut page = collected_page(); + page.requested_url = "https://publisher.example/".to_string(); + page.final_url = page.requested_url.clone(); + page.gpt_slots = vec![ + collector::CollectedGptSlot { + gam_unit_path: "/123456789/site/header".to_string(), + div_id: "ad-stable".to_string(), + sizes: vec![(728, 90)], + }, + collector::CollectedGptSlot { + gam_unit_path: format!("/123456789/{profile}/homepage"), + div_id: "ad-refused".to_string(), + sizes: vec![(300, 250)], + }, + ]; + page + }; + let desktop = FakeCollector::new(page("desktop")); + let mobile = FakeCollector::new(page("mobile")); + let mut out = Vec::new(); + let mut notes = Vec::new(); + + run_update_slots( + &UpdateSlotsRequest { + url: "https://publisher.example/", + config_path: &config_path, + existing_creative: Some(&existing), + page_patterns: &[], + replace: false, + cookies: &[], + dry_run: true, + scroll: false, + budget: CrawlBudget::default(), + }, + &[("desktop", &desktop), ("mobile", &mobile)], + &mut out, + &mut notes, + ) + .expect("the accepted slot should let generation complete"); + + let notes = String::from_utf8(notes).expect("notes should be UTF-8"); + assert!( + notes.contains("skipped refused slot `ad-refused` (`ad-refused`)"), + "should retain the refusal diagnostic, got {notes:?}" + ); + assert!( + !notes.contains("not observed during this crawl: ad-refused"), + "a crawl-observed refused slot must not be labeled unobserved, got {notes:?}" + ); + } + + #[test] + fn ambiguous_configured_stem_is_not_reported_as_unobserved() { + let temp = TempDir::new().expect("should create temp dir"); + let config_path = temp.path().join("trusted-server.toml"); + let mut original = + loadable_config().replace("gam_network_id = \"123456789\"", "gam_network_id = \"222\""); + original.push_str( + "\n[[creative_opportunities.slot]]\n\ + id = \"in-content\"\n\ + div_id = \"ad-x\"\n\ + gam_unit_path = \"/222/homepage/in-content\"\n\ + page_patterns = [\"/\"]\n\ + formats = [{ width = 300, height = 250 }]\n", + ); + fs::write(&config_path, &original).expect("should write config"); + let existing = crate::commands::audit::creative_config(&original, &config_path) + .expect("should parse config") + .expect("should have creative opportunities"); + let mut page = collected_page_with_ambiguous_slots("https://publisher.example/"); + page.gpt_slots.push(collector::CollectedGptSlot { + gam_unit_path: "/222/site/header".to_string(), + div_id: "ad-stable".to_string(), + sizes: vec![(728, 90)], + }); + let collector = FakeCollector::new(page); + let mut notes = Vec::new(); + + run_update_slots( + &UpdateSlotsRequest { + url: "https://publisher.example/", + config_path: &config_path, + existing_creative: Some(&existing), + page_patterns: &[], + replace: false, + cookies: &[], + dry_run: true, + scroll: true, + budget: CrawlBudget::default(), + }, + &[("desktop", &collector)], + &mut std::io::sink(), + &mut notes, + ) + .expect("should preserve the configured ambiguous placement"); + + let notes = String::from_utf8(notes).expect("notes should be UTF-8"); + assert!( + notes.contains("skipped ambiguous div-id prefix `ad-x`"), + "should retain the ambiguity diagnostic, got {notes:?}" + ); + assert!( + !notes.contains("not observed during this crawl: in-content"), + "an ambiguity-refused placement must not be labeled unobserved, got {notes:?}" + ); + } + + #[test] + fn volatile_refusals_from_registry_and_requests_keep_prefix_observed() { + for source in ["registry", "request"] { + let temp = TempDir::new().expect("should create temp dir"); + let config_path = temp.path().join("trusted-server.toml"); + let mut original = loadable_config(); + original.push_str( + "\n[[creative_opportunities.slot]]\n\ + id = \"stable\"\n\ + div_id = \"ad-stable\"\n\ + gam_unit_path = \"/123456789/site/header\"\n\ + page_patterns = [\"/\"]\n\ + formats = [{ width = 728, height = 90 }]\n\n\ + [[creative_opportunities.slot]]\n\ + id = \"volatile-family\"\n\ + div_id = \"vendor-tag\"\n\ + gam_unit_path = \"/123456789/site/overlay\"\n\ + page_patterns = [\"/\"]\n\ + formats = [{ width = 300, height = 250 }]\n", + ); + fs::write(&config_path, &original).expect("should write config"); + let existing = crate::commands::audit::creative_config(&original, &config_path) + .expect("should parse config") + .expect("should have creative opportunities"); + let mut page = collected_page(); + page.requested_url = "https://publisher.example/".to_string(); + page.final_url = page.requested_url.clone(); + page.gpt_slots.push(collector::CollectedGptSlot { + gam_unit_path: "/123456789/site/header".to_string(), + div_id: "ad-stable".to_string(), + sizes: vec![(728, 90)], + }); + let volatile_div = "vendor-tag_1724112345678AbCdEfGh_slot_overlay_1"; + if source == "registry" { + page.gpt_slots.push(collector::CollectedGptSlot { + gam_unit_path: "/123456789/site/overlay".to_string(), + div_id: volatile_div.to_string(), + sizes: vec![(300, 250)], + }); + } else { + page.network_requests.push(CollectedRequest { + url: format!( + "https://securepubads.g.doubleclick.net/gampad/ads?\ + iu_parts=123456789%2Csite%2Coverlay&dids={volatile_div}\ + &prev_iu_szs=300x250" + ), + resource_type: Some("fetch".to_string()), + }); + } + let collector = FakeCollector::new(page); + let mut notes = Vec::new(); + + run_update_slots( + &UpdateSlotsRequest { + url: "https://publisher.example/", + config_path: &config_path, + existing_creative: Some(&existing), + page_patterns: &[], + replace: false, + cookies: &[], + dry_run: true, + scroll: true, + budget: CrawlBudget::default(), + }, + &[("desktop", &collector)], + &mut std::io::sink(), + &mut notes, + ) + .expect("the stable slot should let generation complete"); + + let notes = String::from_utf8(notes).expect("notes should be UTF-8"); + assert!( + notes.contains("skipped volatile div-id family `vendor-tag`"), + "should retain the {source} volatile refusal, got {notes:?}" + ); + assert!( + !notes.contains("not observed during this crawl: volatile-family"), + "a live configured prefix refused from {source} evidence must stay observed, got {notes:?}" + ); + } + } + + #[test] + fn static_locale_root_slot_uses_the_planned_section_depth_for_patterns() { + let temp = TempDir::new().expect("should create temp dir"); + let config_path = temp.path().join("trusted-server.toml"); + fs::write( + &config_path, + "[creative_opportunities]\ngam_network_id = \"123456789\"\n", + ) + .expect("should write config"); + let nav = ["/en/news"]; + let mut root_page = site_page("https://publisher.example/en", "/123456789/site/root", &nav); + root_page.gpt_slots[0].div_id = "ad-root-only".to_string(); + let collector = SiteCollector::new(vec![ + ("https://publisher.example/en", root_page), + ( + "https://publisher.example/en/news", + site_page( + "https://publisher.example/en/news", + "/123456789/site/static", + &nav, + ), + ), + ]); + + run_update_slots( + &UpdateSlotsRequest { + url: "https://publisher.example/en", + config_path: &config_path, + existing_creative: None, + page_patterns: &[], + replace: false, + cookies: &[], + dry_run: false, + scroll: false, + budget: CrawlBudget::default(), + }, + &[("desktop", &collector)], + &mut std::io::sink(), + &mut std::io::sink(), + ) + .expect("should write static locale-root slot"); + + let written = fs::read_to_string(&config_path).expect("should read config"); + let value = toml::from_str::(&written).expect("should parse config"); + let slots = value["creative_opportunities"]["slot"] + .as_array() + .expect("should have slots"); + let target = slots + .iter() + .find(|slot| slot["div_id"].as_str() == Some("ad-header-0")) + .expect("should have the section slot"); + let patterns = target["page_patterns"] + .as_array() + .expect("should have patterns") + .iter() + .map(|pattern| pattern.as_str().expect("should be string")) + .collect::>(); + assert_eq!(patterns, ["/en/news", "/en/news/*"]); + } + + #[test] + fn update_slots_rejects_a_cross_origin_root_redirect() { + let temp = TempDir::new().expect("should create temp dir"); + let config_path = temp.path().join("trusted-server.toml"); + let original = "[creative_opportunities]\ngam_network_id = \"111\"\n"; + fs::write(&config_path, original).expect("should write config"); + let mut collected = collected_page_with_header_slot(); + collected.final_url = "https://foreign.example/news".to_string(); + let collector = FakeCollector::new(collected); + + let error = run_update_slots( + &UpdateSlotsRequest { + url: "https://publisher.example/", + config_path: &config_path, + existing_creative: None, + page_patterns: &[], + replace: false, + cookies: &[("session".to_string(), "secret".to_string())], + dry_run: false, + scroll: false, + budget: CrawlBudget::default(), + }, + &[("desktop", &collector)], + &mut std::io::sink(), + &mut std::io::sink(), + ) + .expect_err("cross-origin redirect must leave the requested trust boundary"); + + assert!(format!("{error:?}").contains("cross-origin")); + assert_eq!( + fs::read_to_string(&config_path).expect("should read config"), + original, + "foreign evidence must not rewrite the config" + ); + } + + #[test] + fn update_slots_skips_a_section_page_that_redirects_off_origin() { + // Only the root navigation was origin-checked before planning. A section + // page that redirects away must not contribute its slots, unit paths or + // page patterns to the generated config either. + let temp = TempDir::new().expect("should create temp dir"); + let config_path = temp.path().join("trusted-server.toml"); + fs::write( + &config_path, + "[creative_opportunities]\ngam_network_id = \"123456789\"\n", + ) + .expect("should write config"); + let nav = ["/news"]; + let mut root_page = site_page( + "https://publisher.example/", + "/123456789/site/homepage", + &nav, + ); + root_page.gpt_slots[0].div_id = "ad-root".to_string(); + let mut redirected = site_page( + "https://publisher.example/news", + "/999888777/foreign/news", + &nav, + ); + redirected.final_url = "https://foreign.example/news".to_string(); + redirected.gpt_slots[0].div_id = "ad-foreign".to_string(); + let collector = SiteCollector::new(vec![ + ("https://publisher.example/", root_page), + ("https://publisher.example/news", redirected), + ]); + let mut err = Vec::new(); + + run_update_slots( + &UpdateSlotsRequest { + url: "https://publisher.example/", + config_path: &config_path, + existing_creative: None, + page_patterns: &[], + replace: false, + cookies: &[], + dry_run: false, + budget: CrawlBudget::default(), + scroll: false, + }, + &[("desktop", &collector)], + &mut std::io::sink(), + &mut err, + ) + .expect("should generate from the same-origin evidence alone"); + + let written = fs::read_to_string(&config_path).expect("should read config"); + assert!( + written.contains("ad-root"), + "same-origin evidence should still be written, got:\n{written}" + ); + assert!( + !written.contains("ad-foreign") && !written.contains("999888777"), + "the redirect destination must not reach the config, got:\n{written}" + ); + let progress = String::from_utf8_lossy(&err); + assert!( + progress.contains( + "skipped `/news` on desktop: it left the audited origin for https://foreign.example" + ), + "the skipped section page should be reported, got:\n{progress}" + ); + } + + #[test] + fn update_slots_skips_a_later_profile_root_that_redirects_off_origin() { + // The later profiles re-walk the plan without a fresh root origin check. + // A mobile root that redirects away carries a foreign ad unit for the + // same div the desktop profile saw; folding it would both write foreign + // inventory and fake a device disagreement on the real slot. + let temp = TempDir::new().expect("should create temp dir"); + let config_path = temp.path().join("trusted-server.toml"); + fs::write( + &config_path, + "[creative_opportunities]\ngam_network_id = \"123456789\"\n", + ) + .expect("should write config"); + let nav = ["/news"]; + let section_page = |unit_path: &str| { + let mut page = site_page("https://publisher.example/news", unit_path, &nav); + page.gpt_slots[0].div_id = "ad-news".to_string(); + page + }; + let mut desktop_root = site_page( + "https://publisher.example/", + "/123456789/site/homepage", + &nav, + ); + desktop_root.gpt_slots[0].div_id = "ad-root".to_string(); + let mut mobile_root = site_page( + "https://publisher.example/", + "/999888777/foreign/homepage", + &nav, + ); + mobile_root.gpt_slots[0].div_id = "ad-root".to_string(); + mobile_root.final_url = "https://foreign.example/".to_string(); + let desktop = SiteCollector::new(vec![ + ("https://publisher.example/", desktop_root), + ( + "https://publisher.example/news", + section_page("/123456789/site/news"), + ), + ]); + let mobile = SiteCollector::new(vec![ + ("https://publisher.example/", mobile_root), + ( + "https://publisher.example/news", + section_page("/123456789/site/news"), + ), + ]); + let mut err = Vec::new(); + + run_update_slots( + &UpdateSlotsRequest { + url: "https://publisher.example/", + config_path: &config_path, + existing_creative: None, + page_patterns: &[], + replace: false, + cookies: &[], + dry_run: false, + budget: CrawlBudget::default(), + scroll: false, + }, + &[("desktop", &desktop), ("mobile", &mobile)], + &mut std::io::sink(), + &mut err, + ) + .expect("the same-origin pages of both profiles agree"); + + let written = fs::read_to_string(&config_path).expect("should read config"); + assert!( + written.contains("/123456789/site/homepage"), + "the same-origin root unit path should be written, got:\n{written}" + ); + assert!( + !written.contains("999888777"), + "the redirect destination must not reach the config, got:\n{written}" + ); + let progress = String::from_utf8_lossy(&err); + assert!( + progress.contains( + "skipped `/` on mobile: it left the audited origin for https://foreign.example" + ), + "the skipped profile root should be reported, got:\n{progress}" + ); + } + + #[test] + fn update_slots_accepts_a_same_host_https_upgrade() { + // The ordinary canonical redirect: an operator types the bare http URL + // and the site upgrades it. The host is unchanged, so the cookie and + // audit trust boundary is unchanged, and generation must not stall on it. + let temp = TempDir::new().expect("should create temp dir"); + let config_path = temp.path().join("trusted-server.toml"); + fs::write( + &config_path, + "[creative_opportunities]\ngam_network_id = \"111\"\n", + ) + .expect("should write config"); + let mut collected = collected_page_with_header_slot(); + collected.requested_url = "http://publisher.example/".to_string(); + collected.final_url = "https://publisher.example/".to_string(); + let collector = FakeCollector::new(collected); + let mut notes = Vec::new(); + + run_update_slots( + &UpdateSlotsRequest { + url: "http://publisher.example/", + config_path: &config_path, + existing_creative: None, + page_patterns: &[], + replace: false, + cookies: &[("session".to_string(), "secret".to_string())], + dry_run: false, + scroll: false, + budget: CrawlBudget::default(), + }, + &[("desktop", &collector)], + &mut std::io::sink(), + &mut notes, + ) + .expect("a same-host HTTPS upgrade should not be treated as cross-origin"); + + let written = fs::read_to_string(&config_path).expect("should read config"); + let value = toml::from_str::(&written).expect("should parse config"); + assert_eq!( + value["creative_opportunities"]["slot"][0]["div_id"].as_str(), + Some("div-gpt-ad-header"), + "evidence from the upgraded root should be written" + ); + let notes = String::from_utf8(notes).expect("notes should be UTF-8"); + assert!( + notes.contains( + "followed a root redirect from `http://publisher.example/` to \ + `https://publisher.example/`" + ), + "an accepted redirect should say the run switched URLs, got {notes:?}" + ); + } + + #[test] + fn update_slots_rejects_an_https_downgrade_root_redirect() { + // The mirror image of the accepted upgrade: same host, but dropping TLS + // leaves the requested trust boundary and must still be refused. + let temp = TempDir::new().expect("should create temp dir"); + let config_path = temp.path().join("trusted-server.toml"); + let original = "[creative_opportunities]\ngam_network_id = \"111\"\n"; + fs::write(&config_path, original).expect("should write config"); + let mut collected = collected_page_with_header_slot(); + collected.final_url = "http://publisher.example/".to_string(); + let collector = FakeCollector::new(collected); + + let error = run_update_slots( + &UpdateSlotsRequest { + url: "https://publisher.example/", + config_path: &config_path, + existing_creative: None, + page_patterns: &[], + replace: false, + cookies: &[("session".to_string(), "secret".to_string())], + dry_run: false, + scroll: false, + budget: CrawlBudget::default(), + }, + &[("desktop", &collector)], + &mut std::io::sink(), + &mut std::io::sink(), + ) + .expect_err("an HTTPS downgrade must leave the requested trust boundary"); + + assert!(format!("{error:?}").contains("cross-origin")); + assert_eq!( + fs::read_to_string(&config_path).expect("should read config"), + original, + "downgraded evidence must not rewrite the config" + ); + } + + #[test] + fn update_slots_requires_evidence_from_every_selected_profile() { + let temp = TempDir::new().expect("should create temp dir"); + let config_path = temp.path().join("trusted-server.toml"); + let original = loadable_config(); + fs::write(&config_path, &original).expect("should write config"); + let desktop = FakeCollector::new(collected_page_with_header_slot()); + + let error = run_update_slots( + &UpdateSlotsRequest { + url: "https://publisher.example/", + config_path: &config_path, + existing_creative: None, + page_patterns: &[], + replace: false, + cookies: &[], + dry_run: false, + scroll: false, + budget: CrawlBudget::default(), + }, + &[("desktop", &desktop), ("mobile", &FailingCollector)], + &mut std::io::sink(), + &mut std::io::sink(), + ) + .expect_err("a selected profile with no usable page must refuse generation"); + + assert!(format!("{error:?}").contains("mobile")); + assert_eq!( + fs::read_to_string(&config_path).expect("should read config"), + original, + "incomplete profile coverage must not rewrite the config" + ); + } + + #[test] + fn update_slots_rejects_invalid_page_pattern_without_touching_config() { + let temp = TempDir::new().expect("should create temp dir"); + let config_path = temp.path().join("trusted-server.toml"); + let original = "[creative_opportunities]\ngam_network_id = \"111\"\n"; + fs::write(&config_path, original).expect("should write config"); + let collector = FakeCollector::new(collected_page_with_header_slot()); + let mut out = Vec::new(); + + let error = run_update_slots( + &UpdateSlotsRequest { + url: "https://publisher.example/", + config_path: &config_path, + existing_creative: None, + page_patterns: &["[".to_string()], + replace: false, + cookies: &[], + dry_run: false, + scroll: false, + budget: CrawlBudget::default(), + }, + &[("desktop", &collector)], + &mut out, + &mut std::io::sink(), + ) + .expect_err("should reject an invalid glob"); + + assert!( + format!("{error:?}").contains("page pattern '['"), + "error should name the offending pattern, got {error:?}" + ); + assert_eq!( + fs::read_to_string(&config_path).expect("should read config"), + original, + "a rejected pattern must leave the operator config untouched" + ); + } + + #[test] + fn explicit_page_patterns_refuse_a_template_that_borrows_section_root() { + let temp = TempDir::new().expect("should create temp dir"); + let config_path = temp.path().join("trusted-server.toml"); + let original = loadable_config(); + fs::write(&config_path, &original).expect("should write config"); + + let nav = ["/news", "/deals"]; + let root = site_page( + "https://publisher.example/", + "/123456789/site/homepage", + &nav, + ); + let mut news = site_page( + "https://publisher.example/news", + "/123456789/site/news", + &nav, + ); + news.gpt_slots.push(collector::CollectedGptSlot { + gam_unit_path: "/123456789/site/news".to_string(), + div_id: "ad-sidebar".to_string(), + sizes: vec![(300, 250)], + }); + let mut deals = site_page( + "https://publisher.example/deals", + "/123456789/site/deals", + &nav, + ); + deals.gpt_slots.push(collector::CollectedGptSlot { + gam_unit_path: "/123456789/site/deals".to_string(), + div_id: "ad-sidebar".to_string(), + sizes: vec![(300, 250)], + }); + let collector = SiteCollector::new(vec![ + ("https://publisher.example/", root), + ("https://publisher.example/news", news), + ("https://publisher.example/deals", deals), + ]); + + let error = run_update_slots( + &UpdateSlotsRequest { + url: "https://publisher.example/", + config_path: &config_path, + existing_creative: None, + page_patterns: &["/".to_string(), "/*".to_string()], + replace: false, + cookies: &[], + dry_run: false, + scroll: false, + budget: CrawlBudget::default(), + }, + &[("desktop", &collector)], + &mut std::io::sink(), + &mut std::io::sink(), + ) + .expect_err("explicit patterns cannot preserve borrowed-root safety"); + + let message = format!("{error:?}"); + assert!(message.contains("--page-pattern"), "got {message}"); + assert!(message.contains("ad-sidebar"), "got {message}"); + assert_eq!( + fs::read_to_string(&config_path).expect("should read config"), + original, + "a refused override must leave the config unchanged" + ); + } + + #[test] + fn update_slots_accepts_double_star_pattern_like_the_runtime() { + // `/20**` does not compile directly but the runtime normalises it to + // `/20*`; validation must accept exactly what the runtime accepts. + let temp = TempDir::new().expect("should create temp dir"); + let config_path = temp.path().join("trusted-server.toml"); + fs::write( + &config_path, + "[creative_opportunities]\ngam_network_id = \"111\"\n", + ) + .expect("should write config"); + let collector = FakeCollector::new(collected_page_with_header_slot()); + let mut out = Vec::new(); + + run_update_slots( + &UpdateSlotsRequest { + url: "https://publisher.example/", + config_path: &config_path, + existing_creative: None, + page_patterns: &["/20**".to_string()], + replace: false, + cookies: &[], + dry_run: false, + scroll: false, + budget: CrawlBudget::default(), + }, + &[("desktop", &collector)], + &mut out, + &mut std::io::sink(), + ) + .expect("should accept a runtime-normalisable pattern"); + + let written = fs::read_to_string(&config_path).expect("should read config"); + let value = toml::from_str::(&written).expect("valid TOML"); + assert_eq!( + value["creative_opportunities"]["slot"][0]["page_patterns"][0].as_str(), + Some("/20**") + ); + } + + #[test] + fn update_slots_write_replaces_the_config_without_leaving_temp_files() { + let temp = TempDir::new().expect("should create temp dir"); + let config_path = temp.path().join("trusted-server.toml"); + fs::write( + &config_path, + "[creative_opportunities]\ngam_network_id = \"111\"\n", + ) + .expect("should write config"); + let collector = FakeCollector::new(collected_page_with_header_slot()); + let mut out = Vec::new(); + + run_update_slots( + &UpdateSlotsRequest { + url: "https://publisher.example/", + config_path: &config_path, + existing_creative: None, + page_patterns: &[], + replace: false, + cookies: &[], + dry_run: false, + scroll: false, + budget: CrawlBudget::default(), + }, + &[("desktop", &collector)], + &mut out, + &mut std::io::sink(), + ) + .expect("should update slots"); + + let entries: Vec = fs::read_dir(temp.path()) + .expect("should read temp dir") + .map(|entry| { + entry + .expect("should read entry") + .file_name() + .to_string_lossy() + .into_owned() + }) + .collect(); + assert_eq!( + entries, + ["trusted-server.toml"], + "the atomic write should leave no stray temp file behind" + ); + let written = fs::read_to_string(&config_path).expect("should read config"); + toml::from_str::(&written).expect("rewritten config is valid TOML"); + } + + /// A full, loadable config with real secrets substituted, so the write-side + /// validation gate is live rather than downgraded by a broken baseline. + fn loadable_config() -> String { + EXAMPLE_CONFIG + .replace("handler_password", "test-admin-password-32-bytes-minimum") + .replace("ec_passphrase", "test-ec-passphrase-32-bytes-minimum") + .replace( + "publisher_proxy_secret", + "test-proxy-secret-32-bytes-minimum", + ) + } + + #[test] + fn a_crawl_writes_a_section_template_and_per_section_patterns() { + // The end-to-end payoff: crawl sections, reconcile the slot across them, + // infer `{section}`, and write a config the runtime loads. + let temp = TempDir::new().expect("should create temp dir"); + let config_path = temp.path().join("trusted-server.toml"); + let original = loadable_config(); + fs::write(&config_path, &original).expect("should write config"); + + let nav = ["/news", "/deals"]; + let collector = SiteCollector::new(vec![ + ( + "https://publisher.example/", + site_page( + "https://publisher.example/", + "/123456789/site/homepage", + &nav, + ), + ), + ( + "https://publisher.example/news", + site_page( + "https://publisher.example/news", + "/123456789/site/news", + &nav, + ), + ), + ( + "https://publisher.example/deals", + site_page( + "https://publisher.example/deals", + "/123456789/site/deals", + &nav, + ), + ), + ]); + let mut out = Vec::new(); + let mut err = Vec::new(); + + run_update_slots( + &UpdateSlotsRequest { + url: "https://publisher.example/", + config_path: &config_path, + existing_creative: None, + page_patterns: &[], + replace: false, + cookies: &[], + dry_run: false, + scroll: false, + budget: CrawlBudget::default(), + }, + &[("desktop", &collector)], + &mut out, + &mut err, + ) + .expect("should crawl and update slots"); + + let written = fs::read_to_string(&config_path).expect("should read config"); + let value = toml::from_str::(&written).expect("valid TOML"); + let creative = &value["creative_opportunities"]; + + assert_eq!( + creative["section_root"].as_str(), + Some("homepage"), + "the unvisited-section fallback should come from the root page" + ); + assert_eq!(creative["section_segment"].as_integer(), Some(0)); + let slot = &creative["slot"][0]; + assert_eq!( + slot["gam_unit_path"].as_str(), + Some("/{network_id}/site/{section}"), + "the varying segment should become a template" + ); + let patterns: Vec<&str> = slot["page_patterns"] + .as_array() + .expect("patterns array") + .iter() + .map(|entry| entry.as_str().expect("pattern")) + .collect(); + assert_eq!( + patterns, + ["/", "/deals", "/deals/*", "/news", "/news/*"], + "each witnessed section should contribute both halves of its pair" + ); + + // The whole point of the gate: what was written must actually load. + trusted_server_core::settings::Settings::from_toml(&written) + .expect("generated config must load through the runtime path"); + + let report = String::from_utf8(err).expect("should produce UTF-8 output"); + assert!( + report.contains("Deploy a template-aware binary BEFORE pushing"), + "a templated config must warn about the rollback contract, got:\n{report}" + ); + } + + #[test] + fn disagreeing_device_profiles_refuse_to_write_a_unit_path() { + // Two profiles serving different ad units for the same page is exactly + // the failure a single-profile crawl cannot see. Writing either path + // would be correct for one device and silently wrong for the other. + let temp = TempDir::new().expect("should create temp dir"); + let config_path = temp.path().join("trusted-server.toml"); + let original = loadable_config(); + fs::write(&config_path, &original).expect("should write config"); + + let nav = ["/news"]; + let desktop = SiteCollector::new(vec![ + ( + "https://publisher.example/", + site_page( + "https://publisher.example/", + "/123456789/desktop/homepage", + &nav, + ), + ), + ( + "https://publisher.example/news", + site_page( + "https://publisher.example/news", + "/123456789/desktop/news", + &nav, + ), + ), + ]); + let mobile = SiteCollector::new(vec![ + ( + "https://publisher.example/", + site_page( + "https://publisher.example/", + "/123456789/mobile/homepage", + &nav, + ), + ), + ( + "https://publisher.example/news", + site_page( + "https://publisher.example/news", + "/123456789/mobile/news", + &nav, + ), + ), + ]); + let mut out = Vec::new(); + let mut err = Vec::new(); + + let error = run_update_slots( + &UpdateSlotsRequest { + url: "https://publisher.example/", + config_path: &config_path, + existing_creative: None, + page_patterns: &[], + replace: false, + cookies: &[], + dry_run: false, + scroll: false, + budget: CrawlBudget::default(), + }, + &[("desktop", &desktop), ("mobile", &mobile)], + &mut out, + &mut err, + ) + .expect_err("an all-refused crawl must not write an empty slot array"); + + assert!(format!("{error:?}").contains("zero generated slots")); + let progress = String::from_utf8_lossy(&err); + for expected in [ + "Auditing desktop [1/?]: /", + "Auditing desktop: planning site crawl", + "Auditing desktop [2/2]: /news", + "Auditing mobile [1/2]: /", + "Auditing mobile [2/2]: /news", + ] { + assert!( + progress.contains(expected), + "should report `{expected}` while crawling, got:\n{progress}" + ); + } + assert!( + !String::from_utf8_lossy(&out).contains("Auditing "), + "progress must remain on stderr" + ); + assert!( + progress.contains("skipped refused slot"), + "the refusal reason should be reported" + ); + assert_eq!( + fs::read_to_string(&config_path).expect("read config"), + original, + "a refused crawl must preserve the operator config" + ); + } + + #[test] + fn a_root_only_site_is_still_collected_on_every_device_profile() { + // A site whose root offers no crawl targets is audited on the root page + // alone. If the later profiles never load it, a device split there is + // invisible and the first profile's literal path gets written as if + // every device agreed with it. + let temp = TempDir::new().expect("should create temp dir"); + let config_path = temp.path().join("trusted-server.toml"); + let original = loadable_config(); + fs::write(&config_path, &original).expect("should write config"); + + let desktop = SiteCollector::new(vec![( + "https://publisher.example/", + site_page( + "https://publisher.example/", + "/123456789/desktop/homepage", + &[], + ), + )]); + let mobile = SiteCollector::new(vec![( + "https://publisher.example/", + site_page( + "https://publisher.example/", + "/123456789/mobile/homepage", + &[], + ), + )]); + let mut out = Vec::new(); + + let error = run_update_slots( + &UpdateSlotsRequest { + url: "https://publisher.example/", + config_path: &config_path, + existing_creative: None, + page_patterns: &[], + replace: false, + cookies: &[], + dry_run: false, + scroll: false, + budget: CrawlBudget::default(), + }, + &[("desktop", &desktop), ("mobile", &mobile)], + &mut out, + &mut std::io::sink(), + ) + .expect_err("an all-refused crawl must not write an empty slot array"); + + assert_eq!( + mobile.visited.borrow().as_slice(), + ["https://publisher.example/"], + "the mobile profile must load the root even when there is nothing else to crawl" + ); + assert!(format!("{error:?}").contains("zero generated slots")); + assert_eq!( + fs::read_to_string(&config_path).expect("read config"), + original, + "a root-only refusal must preserve the operator config" + ); + } + + #[test] + fn a_crawl_refuses_when_most_pages_are_challenged() { + // Bot protection serves an interstitial that loads fine and has no ad + // stack, so it looks like a page with no slots. Writing from that would + // silently narrow the operator's slot set. + let temp = TempDir::new().expect("should create temp dir"); + let config_path = temp.path().join("trusted-server.toml"); + let original = loadable_config(); + fs::write(&config_path, &original).expect("should write config"); + + let nav = ["/news", "/deals"]; + let mut blocked_news = site_page("https://publisher.example/news", "/123456789/x", &nav); + blocked_news.gpt_slots.clear(); + let mut blocked_deals = site_page("https://publisher.example/deals", "/123456789/x", &nav); + blocked_deals.gpt_slots.clear(); + let collector = SiteCollector::new(vec![ + ( + "https://publisher.example/", + site_page( + "https://publisher.example/", + "/123456789/site/homepage", + &nav, + ), + ), + ("https://publisher.example/news", blocked_news), + ("https://publisher.example/deals", blocked_deals), + ]); + let mut out = Vec::new(); + + let error = run_update_slots( + &UpdateSlotsRequest { + url: "https://publisher.example/", + config_path: &config_path, + existing_creative: None, + page_patterns: &[], + replace: false, + cookies: &[], + dry_run: false, + scroll: false, + budget: CrawlBudget::default(), + }, + &[("desktop", &collector)], + &mut out, + &mut std::io::sink(), + ) + .expect_err("a mostly-challenged crawl should refuse"); + + assert!( + format!("{error:?}").contains("bot protection"), + "the error should name the likely cause, got {error:?}" + ); + assert_eq!( + fs::read_to_string(&config_path).expect("read config"), + original, + "a refused run must leave the config untouched" + ); + } + + #[test] + fn max_pages_one_restores_single_page_behavior() { + let temp = TempDir::new().expect("should create temp dir"); + let config_path = temp.path().join("trusted-server.toml"); + fs::write(&config_path, loadable_config()).expect("should write config"); + + let nav = ["/news", "/deals"]; + let collector = SiteCollector::new(vec![( + "https://publisher.example/", + site_page( + "https://publisher.example/", + "/123456789/site/homepage", + &nav, + ), + )]); + let mut out = Vec::new(); + + run_update_slots( + &UpdateSlotsRequest { + url: "https://publisher.example/", + config_path: &config_path, + existing_creative: None, + page_patterns: &[], + replace: false, + cookies: &[], + dry_run: false, + scroll: false, + budget: CrawlBudget { + max_sections: 8, + max_pages: 1, + }, + }, + &[("desktop", &collector)], + &mut out, + &mut std::io::sink(), + ) + .expect("should update from the single page"); + + assert_eq!( + collector.visited.borrow().len(), + 1, + "max_pages = 1 must not crawl beyond the requested page" + ); + let written = fs::read_to_string(&config_path).expect("read config"); + let value = toml::from_str::(&written).expect("valid TOML"); + assert!( + value["creative_opportunities"] + .get("section_root") + .is_none(), + "one page cannot witness a section, so no rollback-fatal key may be written" + ); + assert_eq!( + value["creative_opportunities"]["slot"][0]["gam_unit_path"].as_str(), + Some("/123456789/site/homepage"), + "a single page keeps the literal path" + ); + } + + #[test] + fn generated_config_loads_through_the_runtime_settings_path() { + // The end-to-end contract: whatever `generate` writes must survive the + // same load path the adapter runs at startup. An unloadable config is a + // full-site outage once pushed, not a degraded ad stack. + let temp = TempDir::new().expect("should create temp dir"); + let config_path = temp.path().join("trusted-server.toml"); + let baseline = loadable_config(); + trusted_server_core::settings::Settings::from_toml(&baseline) + .expect("test baseline must itself be loadable or the gate is not exercised"); + fs::write(&config_path, &baseline).expect("should write config"); + let collector = FakeCollector::new(collected_page_with_header_slot()); + let mut out = Vec::new(); + + run_update_slots( + &UpdateSlotsRequest { + url: "https://publisher.example/", + config_path: &config_path, + existing_creative: None, + page_patterns: &[], + replace: false, + cookies: &[], + dry_run: false, + scroll: false, + budget: CrawlBudget::default(), + }, + &[("desktop", &collector)], + &mut out, + &mut std::io::sink(), + ) + .expect("should update slots"); + + let written = fs::read_to_string(&config_path).expect("should read config"); + let settings = trusted_server_core::settings::Settings::from_toml(&written) + .expect("generated config must load through the runtime path"); + let creative = settings + .creative_opportunities + .expect("generated config should carry creative opportunities"); + assert_eq!( + creative.slot.len(), + 1, + "the discovered slot should be present after a real load" + ); + assert_eq!( + creative.slot[0].div_id.as_deref(), + Some("div-gpt-ad-header") + ); + } + + #[test] + fn update_slots_dry_run_does_not_persist_environment_overlay_config() { + let temp = TempDir::new().expect("should create temp dir"); + let manifest_path = temp.path().join("edgezero.toml"); + let config_path = temp.path().join("trusted-server.toml"); + fs::write(&manifest_path, "[app]\nname = \"trusted-server\"\n") + .expect("should write manifest"); + let config = EXAMPLE_CONFIG + .replace("handler_password", "test-admin-password-32-bytes-minimum") + .replace("ec_passphrase", "test-ec-passphrase-32-bytes-minimum") + .replace( + "publisher_proxy_secret", + "test-proxy-secret-32-bytes-minimum", + ); + let config = format!( + "{config}\n\ + [[creative_opportunities.slot]]\n\ + id = \"file-only\"\n\ + div_id = \"div-gpt-ad-file\"\n\ + gam_unit_path = \"/123456789/homepage/file\"\n\ + page_patterns = [\"/\"]\n\ + formats = [{{ width = 728, height = 90 }}]\n" + ); + fs::write(&config_path, &config).expect("should write config"); + let args = AppConfigArgs { + app_config: Some(config_path.clone()), + manifest: manifest_path, + no_env: false, + }; + + temp_env::with_var( + "TRUSTED_SERVER__CREATIVE_OPPORTUNITIES__GAM_NETWORK_ID", + Some("987654321"), + || { + let effective = crate::app_config::load_settings(&args) + .expect("should load effective settings"); + assert_eq!( + effective + .settings + .creative_opportunities + .as_ref() + .expect("should have creative config") + .gam_network_id, + "987654321", + "test environment should override the network id" + ); + let loaded = crate::app_config::load_file_settings(&args) + .expect("should load file-only settings"); + let mut collected = collected_page(); + collected.gpt_slots = vec![collector::CollectedGptSlot { + gam_unit_path: "/123456789/homepage/file".to_string(), + div_id: "div-gpt-ad-file".to_string(), + sizes: vec![(728, 90)], + }]; + let collector = FakeCollector::new(collected); + let mut out = Vec::new(); + + run_update_slots( + &UpdateSlotsRequest { + url: "https://publisher.example/", + config_path: &loaded.app_config_path, + existing_creative: loaded.settings.creative_opportunities.as_ref(), + page_patterns: &[], + replace: false, + cookies: &[], + dry_run: true, + scroll: false, + budget: CrawlBudget::default(), + }, + &[("desktop", &collector)], + &mut out, + &mut std::io::sink(), + ) + .expect("should render dry-run update"); + + let output = String::from_utf8(out).expect("output should be UTF-8"); + assert!(output.starts_with("--- configured creative opportunities\n")); + assert!(output.contains("+++ generated creative opportunities\n")); + assert!( + !output.contains("test-admin-password-32-bytes-minimum"), + "dry run must not expose unrelated secrets" + ); + assert!( + !output.contains("987654321"), + "dry run must not persist environment-only config" + ); + assert_eq!( + fs::read_to_string(&config_path).expect("should re-read config"), + config, + "dry run must not modify the config file" + ); + }, + ); + } + + #[test] + fn update_slots_refuses_to_overwrite_a_config_changed_during_collection() { + let temp = TempDir::new().expect("should create temp dir"); + let config_path = temp.path().join("trusted-server.toml"); + let original = loadable_config(); + let replacement = format!("{original}\n# edited while the browser was running\n"); + fs::write(&config_path, &original).expect("should write config"); + let collector = MutatingCollector { + collected: collected_page_with_header_slot(), + config_path: config_path.clone(), + replacement: replacement.clone(), + }; + + let error = run_update_slots( + &UpdateSlotsRequest { + url: "https://publisher.example/", + config_path: &config_path, + existing_creative: None, + page_patterns: &[], + replace: false, + cookies: &[], + dry_run: false, + scroll: false, + budget: CrawlBudget::default(), + }, + &[("desktop", &collector)], + &mut std::io::sink(), + &mut std::io::sink(), + ) + .expect_err("a stale update should be refused"); + + assert!(format!("{error:?}").contains("changed during the browser audit")); + assert_eq!( + fs::read_to_string(&config_path).expect("should re-read config"), + replacement, + "the concurrent edit must not be overwritten" + ); + } +} diff --git a/crates/trusted-server-cli/src/commands/audit/generate/page_patterns.rs b/crates/trusted-server-cli/src/commands/audit/generate/page_patterns.rs new file mode 100644 index 000000000..740acc86f --- /dev/null +++ b/crates/trusted-server-cli/src/commands/audit/generate/page_patterns.rs @@ -0,0 +1,150 @@ +//! Derives `page_patterns` globs from the paths a slot was actually observed on. +//! +//! A slot seen on `/news/story-abc` should serve every article in that section, +//! not just that one URL — but nothing here extrapolates beyond a *witnessed* +//! section. Each observed path contributes the section prefix it belongs to and +//! nothing else, so a crawl that never visited `/reviews` never claims it. +//! +//! Each section yields a pair, because one glob cannot cover both halves: +//! `*` crosses `/` in this glob dialect, so `/news/*` matches `/news/a/b` but +//! **not** the bare `/news` landing page. Emitting only the star form silently +//! drops the landing page from the slot. + +use std::collections::BTreeSet; + +/// The root pattern, matching only the site root. +const ROOT_PATTERN: &str = "/"; + +/// Expands observed page paths into the glob set a slot should carry. +/// +/// `section_segment` is the index the section is taken from, matching the +/// config key of the same name: a path is reduced to its first +/// `section_segment + 1` segments, which is the prefix every page of that +/// section shares. A shorter observed landing path is emitted literally; only +/// the actual site root contributes `/`. +/// +/// Results are deduplicated and ordered with `/` first, then alphabetically, so +/// re-running against unchanged evidence produces an unchanged file. +pub(super) fn patterns_for_paths<'a>( + paths: impl IntoIterator, + section_segment: usize, +) -> Vec { + let mut patterns: BTreeSet = BTreeSet::new(); + let mut has_root = false; + + for path in paths { + let segments: Vec<&str> = path.split('/').filter(|part| !part.is_empty()).collect(); + if segments.len() <= section_segment { + if segments.is_empty() { + has_root = true; + } else { + patterns.insert(glob::Pattern::escape(path)); + } + continue; + } + let prefix = glob::Pattern::escape(&format!("/{}", segments[..=section_segment].join("/"))); + // The landing page and everything beneath it. + patterns.insert(prefix.clone()); + patterns.insert(format!("{prefix}/*")); + } + + let mut out = Vec::with_capacity(patterns.len() + usize::from(has_root)); + if has_root { + out.push(ROOT_PATTERN.to_string()); + } + out.extend(patterns); + out +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn a_section_article_yields_both_halves_of_the_pair() { + // `/news/*` alone would not match the bare `/news` landing page, because + // `*` crosses `/` but does not match the empty remainder. + let patterns = patterns_for_paths(["/news/story-abc"], 0); + + assert_eq!(patterns, ["/news", "/news/*"]); + } + + #[test] + fn the_root_path_contributes_the_root_pattern_first() { + let patterns = patterns_for_paths(["/deals/x", "/", "/news/y"], 0); + + assert_eq!( + patterns, + ["/", "/deals", "/deals/*", "/news", "/news/*"], + "root first, then sections alphabetically" + ); + } + + #[test] + fn a_landing_page_and_its_article_collapse_to_one_pair() { + let patterns = patterns_for_paths(["/news", "/news/story-abc"], 0); + + assert_eq!(patterns, ["/news", "/news/*"], "no duplicate entries"); + } + + #[test] + fn a_locale_prefixed_site_keeps_the_locale_in_the_prefix() { + // section_segment = 1 means the section is the second segment, so the + // shared prefix every page of that section carries includes the locale. + let patterns = patterns_for_paths(["/en/news/story", "/en/deals/x", "/en"], 1); + + assert_eq!( + patterns, + ["/en", "/en/deals", "/en/deals/*", "/en/news", "/en/news/*"] + ); + } + + #[test] + fn literal_glob_metacharacters_are_escaped_and_match_the_source() { + let source = "/news[local]/story"; + let patterns = patterns_for_paths([source], 0); + + assert_eq!(patterns, ["/news[[]local[]]", "/news[[]local[]]/*"]); + assert!(patterns.iter().any(|pattern| { + glob::Pattern::new(pattern) + .expect("should compile emitted glob") + .matches(source) + })); + } + + #[test] + fn unwitnessed_sections_are_never_invented() { + let patterns = patterns_for_paths(["/news/story"], 0); + + assert_eq!( + patterns, + ["/news", "/news/*"], + "only the crawled section may appear" + ); + } + + #[test] + fn output_is_stable_regardless_of_input_order() { + let one = patterns_for_paths(["/news/a", "/deals/b", "/"], 0); + let two = patterns_for_paths(["/", "/deals/b", "/news/a"], 0); + + assert_eq!(one, two, "re-running should not reorder the written file"); + } + + #[test] + fn every_emitted_pattern_compiles_as_a_runtime_glob() { + let patterns = patterns_for_paths(["/", "/news/story", "/site-news/x"], 0); + + for pattern in &patterns { + trusted_server_core::creative_opportunities::validate_page_pattern(pattern) + .unwrap_or_else(|error| { + panic!("emitted pattern `{pattern}` must compile: {error}") + }); + } + } + + #[test] + fn no_paths_yield_no_patterns() { + assert!(patterns_for_paths([], 0).is_empty()); + } +} diff --git a/crates/trusted-server-cli/src/commands/audit/generate/slot_toml.rs b/crates/trusted-server-cli/src/commands/audit/generate/slot_toml.rs new file mode 100644 index 000000000..f795ea41b --- /dev/null +++ b/crates/trusted-server-cli/src/commands/audit/generate/slot_toml.rs @@ -0,0 +1,2251 @@ +//! TOML-side slot config: the [`RenderSlot`] model, run merging, rendering, +//! and in-place `[creative_opportunities]` splicing for `ts audit ad-templates +//! generate`. + +use std::collections::{BTreeMap, BTreeSet}; + +use toml_edit::{DocumentMut, Item, Table}; +use trusted_server_core::auction::types::MediaType; +use trusted_server_core::creative_opportunities::{ + CreativeOpportunitiesConfig, CreativeOpportunitySlot, +}; + +#[cfg(test)] +use crate::commands::audit::generate::gpt_slots; +use crate::error::{CliResult, cli_error, report_error}; + +/// A slot ready to render — the union of discovered and existing fields, without +/// the core type's `pub(crate)` compiled-pattern cache. +#[derive(Debug, Clone)] +pub(super) struct RenderSlot { + id: String, + div_id: Option, + gam_unit_path: Option, + page_patterns: Vec, + /// `(width, height, non-banner media type)`. + formats: Vec<(u32, u32, Option<&'static str>)>, + floor_price: Option, + targeting: BTreeMap, + aps_slot_id: Option, + /// `Some` when the slot runs Prebid; the map is per-bidder params (often empty). + prebid_bidders: Option>, +} + +impl RenderSlot { + /// The stable exact identity fallback used when no configured div prefix + /// matches a discovered slot. + fn key(&self) -> String { + self.div_id + .as_deref() + .unwrap_or(&self.id) + .trim_end_matches('-') + .to_string() + } + + /// Whether this configured slot carries fields that discovery cannot infer. + fn has_tuned_fields(&self) -> bool { + self.floor_price.is_some() + || !self.targeting.is_empty() + || self.aps_slot_id.is_some() + || self.prebid_bidders.is_some() + } + + /// Builds a slot from one page's discovery. + /// + /// Superseded in production by [`RenderSlot::from_evidence`], which reads + /// cross-page evidence; retained as test scaffolding for the merge cases. + #[cfg(test)] + fn from_discovered(slot: &gpt_slots::DiscoveredSlot, patterns: &[String]) -> Self { + Self { + id: slot.id.clone(), + div_id: Some(slot.div_id.clone()), + gam_unit_path: Some(slot.gam_unit_path.clone()), + page_patterns: patterns.to_vec(), + formats: slot + .formats + .iter() + .map(|&(width, height)| (width, height, None)) + .collect(), + floor_price: None, + targeting: BTreeMap::new(), + aps_slot_id: None, + prebid_bidders: slot.has_prebid.then(BTreeMap::new), + } + } + + /// Builds a slot from cross-page evidence and the inferred unit path. + /// + /// Refused inference decisions are filtered before this constructor. A + /// `None` path therefore means inference was unavailable and deliberately + /// leaves the runtime's configured default-path behavior in effect. + pub(super) fn from_evidence( + id: &str, + div_id: &str, + gam_unit_path: Option, + formats: impl IntoIterator, + page_patterns: Vec, + has_prebid: bool, + ) -> Self { + Self { + id: id.to_string(), + div_id: Some(div_id.to_string()), + gam_unit_path, + page_patterns, + formats: formats + .into_iter() + .map(|(width, height)| (width, height, None)) + .collect(), + floor_price: None, + targeting: BTreeMap::new(), + aps_slot_id: None, + prebid_bidders: has_prebid.then(BTreeMap::new), + } + } + + fn from_existing(slot: &CreativeOpportunitySlot) -> Self { + Self { + id: slot.id.clone(), + div_id: slot.div_id.clone(), + gam_unit_path: slot.gam_unit_path.clone(), + page_patterns: slot.page_patterns.clone(), + formats: slot + .formats + .iter() + .map(|format| { + ( + format.width, + format.height, + media_type_label(&format.media_type), + ) + }) + .collect(), + floor_price: slot.floor_price, + targeting: slot + .targeting + .iter() + .map(|(key, value)| (key.clone(), value.clone())) + .collect(), + aps_slot_id: slot.providers.aps.as_ref().map(|aps| aps.slot_id.clone()), + prebid_bidders: slot.providers.prebid.as_ref().map(|prebid| { + prebid + .bidders + .iter() + .map(|(name, params)| (name.clone(), params.clone())) + .collect() + }), + } + } +} + +/// The non-default (non-banner) media-type label to emit, or `None` for banner. +fn media_type_label(media_type: &MediaType) -> Option<&'static str> { + match media_type { + MediaType::Banner => None, + MediaType::Video => Some("video"), + MediaType::Native => Some("native"), + } +} + +/// Merges discovered slots into the existing slot set, keyed by [`RenderSlot::key`]. +/// +/// - `--replace` (or no existing slots): the result is exactly the discovered set. +/// - Otherwise existing slots are preserved (covering other pages / hand-tuned +/// fields); a slot re-seen this run has its page patterns and formats unioned; +/// slots seen only this run are appended. +/// - Format identity includes media type, so equal dimensions observed for two +/// media types remain two intentional entries. +#[cfg(test)] +pub(super) fn merge_slots( + existing: Option<&CreativeOpportunitiesConfig>, + discovered: &gpt_slots::DiscoveredSlots, + run_patterns: &[String], + replace: bool, +) -> Vec { + let discovered_slots: Vec = discovered + .slots + .iter() + .map(|slot| RenderSlot::from_discovered(slot, run_patterns)) + .collect(); + merge_render_slots(existing, discovered_slots, replace) +} + +/// Merges already-built slots into the existing set. +/// +/// Same reconciliation as the single-page test helper, but the caller supplies the slots — +/// the crawl path builds them from cross-page evidence rather than from one +/// page's discoveries. A slot re-seen this run keeps its configured fields and +/// gains this run's patterns; a genuinely new slot is appended with a +/// non-colliding id. +#[cfg(test)] +pub(super) fn merge_render_slots( + existing: Option<&CreativeOpportunitiesConfig>, + discovered_slots: Vec, + replace: bool, +) -> Vec { + merge_render_slots_with_diagnostics(existing, discovered_slots, replace).0 +} + +/// Diagnostics produced while merging discovered and configured slots. +#[derive(Debug, Default, PartialEq, Eq)] +pub(super) struct MergeDiagnostics { + /// Operator-facing reconciliation notes. + pub(super) notes: Vec, + /// Configured slots preserved without matching any normalized evidence div. + pub(super) unobserved_existing_slot_ids: Vec, +} + +/// Merges slots and reports prefix collisions and unobserved preserved slots. +#[cfg(test)] +pub(super) fn merge_render_slots_with_diagnostics( + existing: Option<&CreativeOpportunitiesConfig>, + discovered_slots: Vec, + replace: bool, +) -> (Vec, MergeDiagnostics) { + let observed_div_ids = discovered_slots + .iter() + .filter_map(|slot| slot.div_id.clone()) + .collect::>(); + merge_render_slots_with_observed_diagnostics( + existing, + discovered_slots, + &observed_div_ids, + replace, + ) +} + +/// Merges renderable slots using normalized evidence div IDs for observation. +/// +/// `observed_div_ids` must be the full normalized evidence set, including divs +/// refused by template inference, skipped as fragments, or refused as +/// ambiguous. Passing only the rendered subset can make a configured literal +/// act as a prefix again or falsely report a live configured slot as unobserved. +pub(super) fn merge_render_slots_with_observed_diagnostics( + existing: Option<&CreativeOpportunitiesConfig>, + discovered_slots: Vec, + observed_div_ids: &[String], + replace: bool, +) -> (Vec, MergeDiagnostics) { + let existing_slots = existing.map(|config| config.slot.as_slice()).unwrap_or(&[]); + if replace || existing_slots.is_empty() { + return (discovered_slots, MergeDiagnostics::default()); + } + + let observed_literals = observed_div_ids + .iter() + .map(String::as_str) + .collect::>(); + + let mut merged: Vec = existing_slots + .iter() + .map(RenderSlot::from_existing) + .collect(); + let existing_count = merged.len(); + let mut prefix_claims: BTreeMap> = BTreeMap::new(); + let mut split_warnings = BTreeSet::new(); + let mut observed_existing = observed_div_ids + .iter() + .flat_map(|div_id| matching_observed_div_indexes(&merged, div_id, &observed_literals)) + .collect::>(); + for mut slot in discovered_slots { + // Prefix reconciliation is a property of the operator's config, so only + // the slots that were already configured may claim a discovered div. + // Slots this run appended match by exact identity instead, otherwise + // discovery order decides whether `ad-top` swallows a later + // `ad-top-sidebar` and discards its unit path and provider state. + let matched = matching_slot_index(&merged[..existing_count], &slot, &observed_literals) + .or_else(|| { + let key = slot.key(); + merged[existing_count..] + .iter() + .position(|added| added.key() == key) + .map(|offset| offset + existing_count) + }); + if let Some(index) = matched { + if index < existing_count { + observed_existing.insert(index); + } + if index < existing_count + && let (Some(prefix), Some(discovered_div)) = + (merged[index].div_id.as_deref(), slot.div_id.as_deref()) + && discovered_div.starts_with(prefix) + { + prefix_claims + .entry(index) + .or_default() + .insert(discovered_div.to_string()); + } + let present = &mut merged[index]; + for pattern in &slot.page_patterns { + if !present.page_patterns.contains(pattern) { + present.page_patterns.push(pattern.clone()); + } + } + for format in &slot.formats { + if !present.formats.contains(format) { + present.formats.push(*format); + } + } + } else { + if let Some(discovered_div) = slot.div_id.as_deref() + && let Some(parent) = merged[..existing_count] + .iter() + .filter(|configured| { + configured.div_id.as_deref().is_some_and(|prefix| { + !prefix.is_empty() + && observed_literals.contains(prefix) + && discovered_div != prefix + && discovered_div.starts_with(prefix) + }) && configured.has_tuned_fields() + }) + .max_by_key(|configured| configured.div_id.as_deref().map_or(0, str::len)) + { + split_warnings.insert(format!( + "discovered div `{discovered_div}` was split from configured literal prefix \ + `{}`; the new slot does not inherit that configured slot's floor price, \ + targeting, or provider settings", + parent.div_id.as_deref().unwrap_or_default(), + )); + } + slot.id = unique_slot_id(&slot.id, &merged); + merged.push(slot); + } + } + let notes = split_warnings + .into_iter() + .chain( + prefix_claims + .into_iter() + .filter(|(_, divs)| divs.len() > 1) + .map(|(index, divs)| { + let slot = &merged[index]; + let sample = divs.iter().take(5).cloned().collect::>().join(", "); + let remainder = divs.len().saturating_sub(5); + let suffix = if remainder == 0 { + String::new() + } else { + format!(", and {remainder} more") + }; + format!( + "configured slot `{}` with div_id prefix `{}` matched {} discovered divs \ + ({sample}{suffix}); runtime can resolve this configured slot to at most one \ + active element, so review whether they are distinct placements", + slot.id, + slot.div_id.as_deref().unwrap_or_default(), + divs.len(), + ) + }), + ) + .collect(); + let unobserved_existing_slot_ids = existing_slots + .iter() + .enumerate() + .filter(|(index, _)| !observed_existing.contains(index)) + .map(|(_, slot)| slot.id.clone()) + .collect(); + ( + merged, + MergeDiagnostics { + notes, + unobserved_existing_slot_ids, + }, + ) +} + +fn unique_slot_id(candidate: &str, existing: &[RenderSlot]) -> String { + if existing.iter().all(|slot| slot.id != candidate) { + return candidate.to_string(); + } + + let mut suffix = 2_usize; + loop { + let unique = format!("{candidate}-{suffix}"); + if existing.iter().all(|slot| slot.id != unique) { + return unique; + } + suffix += 1; + } +} + +/// Finds the configured slot matching a discovered normalized slot. +/// +/// Stable-key equality wins first. Otherwise, configured `div_id` values are +/// eligible runtime prefixes unless that value was itself observed as a +/// distinct literal. Equal-length prefix ties retain configuration order. +fn matching_slot_index( + existing: &[RenderSlot], + discovered: &RenderSlot, + observed_literals: &BTreeSet<&str>, +) -> Option { + let key = discovered.key(); + if let Some(index) = existing.iter().position(|slot| slot.key() == key) { + return Some(index); + } + + discovered + .div_id + .as_deref() + .and_then(|div_id| matching_div_id_index(existing, div_id, observed_literals)) +} + +fn matching_div_id_index( + existing: &[RenderSlot], + discovered_div: &str, + observed_literals: &BTreeSet<&str>, +) -> Option { + let mut best = None; + let mut best_length = 0; + for (index, slot) in existing.iter().enumerate() { + let Some(prefix) = slot.div_id.as_deref().filter(|prefix| !prefix.is_empty()) else { + continue; + }; + if observed_literals.contains(prefix) { + continue; + } + if discovered_div.starts_with(prefix) && prefix.len() > best_length { + best = Some(index); + best_length = prefix.len(); + } + } + best +} + +/// Finds every configured slot that can resolve to one normalized evidence div. +/// +/// Merge routing remains exact-then-longest-prefix through +/// [`matching_slot_index`], but observation is deliberately multi-match: an +/// exact configured slot and every eligible broad prefix are all live when the +/// element exists. +fn matching_observed_div_indexes( + existing: &[RenderSlot], + discovered_div: &str, + observed_literals: &BTreeSet<&str>, +) -> Vec { + let discovered_key = discovered_div.trim_end_matches('-'); + existing + .iter() + .enumerate() + .filter_map(|(index, slot)| { + if slot.key() == discovered_key { + return Some(index); + } + let prefix = slot.div_id.as_deref().filter(|prefix| !prefix.is_empty())?; + (!observed_literals.contains(prefix) && discovered_div.starts_with(prefix)) + .then_some(index) + }) + .collect() +} + +/// Header comment emitted above the structurally replaced managed slot array. +const MANAGED_SLOTS_COMMENT: &str = "# Slots managed by `ts audit ad-templates generate`."; +/// Second line of the managed-slot header comment. +const MANAGED_SLOTS_REVIEW_COMMENT: &str = + "# Review page_patterns and formats before validating/pushing."; + +/// Renders merged slots as compact `[[creative_opportunities.slot]]` TOML blocks. +pub(super) fn render_slots(slots: &[RenderSlot]) -> String { + let mut out = format!("\n{MANAGED_SLOTS_COMMENT}\n{MANAGED_SLOTS_REVIEW_COMMENT}\n"); + for slot in slots { + out.push_str("\n[[creative_opportunities.slot]]\n"); + out.push_str(&format!("id = {}\n", toml_string(&slot.id))); + if let Some(div_id) = &slot.div_id { + out.push_str(&format!("div_id = {}\n", toml_string(div_id))); + } + if let Some(path) = &slot.gam_unit_path { + out.push_str(&format!("gam_unit_path = {}\n", toml_string(path))); + } + out.push_str("page_patterns = [\n"); + for pattern in &slot.page_patterns { + out.push_str(&format!(" {},\n", toml_string(pattern))); + } + out.push_str("]\n"); + out.push_str("formats = [\n"); + for (width, height, media_type) in &slot.formats { + let rendered = match media_type { + Some(kind) => { + format!("{{ width = {width}, height = {height}, media_type = \"{kind}\" }}") + } + None => format!("{{ width = {width}, height = {height} }}"), + }; + out.push_str(&format!(" {rendered},\n")); + } + out.push_str("]\n"); + if let Some(floor) = slot.floor_price { + // `f64` Display prints `NaN`, which is not valid TOML (`nan` is); + // normalize non-finite values so the spliced config stays parseable. + if floor.is_finite() { + out.push_str(&format!("floor_price = {floor}\n")); + } else if floor.is_nan() { + out.push_str("floor_price = nan\n"); + } else if floor.is_sign_positive() { + out.push_str("floor_price = inf\n"); + } else { + out.push_str("floor_price = -inf\n"); + } + } + if !slot.targeting.is_empty() { + let pairs = slot + .targeting + .iter() + .map(|(key, value)| format!("{} = {}", toml_key(key), toml_string(value))) + .collect::>() + .join(", "); + out.push_str(&format!("targeting = {{ {pairs} }}\n")); + } + if let Some(slot_id) = &slot.aps_slot_id { + out.push_str("[creative_opportunities.slot.providers.aps]\n"); + out.push_str(&format!("slot_id = {}\n", toml_string(slot_id))); + } + if let Some(bidders) = &slot.prebid_bidders { + out.push_str("[creative_opportunities.slot.providers.prebid]\n"); + let rendered = bidders + .iter() + .map(|(name, params)| format!("{} = {}", toml_key(name), toml_inline_value(params))) + .collect::>() + .join(", "); + if rendered.is_empty() { + out.push_str("bidders = {}\n"); + } else { + out.push_str(&format!("bidders = {{ {rendered} }}\n")); + } + } + } + out +} + +/// Quotes and escapes a string as a TOML basic string, including control chars. +pub(super) fn toml_string(value: &str) -> String { + let mut out = String::with_capacity(value.len() + 2); + out.push('"'); + for ch in value.chars() { + match ch { + '"' => out.push_str("\\\""), + '\\' => out.push_str("\\\\"), + '\n' => out.push_str("\\n"), + '\r' => out.push_str("\\r"), + '\t' => out.push_str("\\t"), + // TOML basic strings reject U+0000..U+001F and DEL (U+007F). + control if (control as u32) < 0x20 || control == '\u{7f}' => { + out.push_str(&format!("\\u{:04X}", control as u32)); + } + other => out.push(other), + } + } + out.push('"'); + out +} + +/// Renders a TOML table key: bare when it is a valid bare key, else a quoted key. +fn toml_key(key: &str) -> String { + let is_bare = !key.is_empty() + && key + .chars() + .all(|ch| ch.is_ascii_alphanumeric() || ch == '_' || ch == '-'); + if is_bare { + key.to_string() + } else { + toml_string(key) + } +} + +/// Renders a JSON value as a compact inline TOML value (for prebid bidder params). +fn toml_inline_value(value: &serde_json::Value) -> String { + match value { + serde_json::Value::Null => "{}".to_string(), + serde_json::Value::Bool(bool) => bool.to_string(), + serde_json::Value::Number(number) => number.to_string(), + serde_json::Value::String(string) => toml_string(string), + serde_json::Value::Array(items) => { + let rendered = items + .iter() + .map(toml_inline_value) + .collect::>() + .join(", "); + format!("[{rendered}]") + } + serde_json::Value::Object(map) => { + let rendered = map + .iter() + .map(|(key, value)| format!("{} = {}", toml_key(key), toml_inline_value(value))) + .collect::>() + .join(", "); + format!("{{ {rendered} }}") + } + } +} + +/// The config-level values a splice writes alongside the slot array. +#[derive(Debug, Clone, Default)] +pub(super) struct CreativeSectionKeys<'a> { + /// GAM network id, when one was resolved. + pub(super) network_id: Option<&'a str>, + /// `section_root`, written only when a slot uses a `{section}` template. + pub(super) section_root: Option<&'a str>, + /// `section_segment`, written only alongside `section_root`. + pub(super) section_segment: Option, +} + +fn max_table_position(table: &Table) -> Option { + table.iter().fold(table.position(), |maximum, (_, item)| { + let child_maximum = match item { + Item::Table(child) => max_table_position(child), + Item::ArrayOfTables(array) => array.iter().filter_map(max_table_position).max(), + Item::None | Item::Value(_) => None, + }; + maximum.max(child_maximum) + }) +} + +fn set_table_position_recursive(table: &mut Table, position: isize) { + table.set_position(position); + for (_, item) in table.iter_mut() { + match item { + Item::Table(child) => set_table_position_recursive(child, position), + Item::ArrayOfTables(array) => { + for child in array.iter_mut() { + set_table_position_recursive(child, position); + } + } + Item::None | Item::Value(_) => {} + } + } +} + +/// Structurally replaces the generator-managed creative-opportunities fields. +/// +/// All unrelated TOML items and their decorations remain in the parsed +/// document. Missing inferred scalar values preserve their existing values; a +/// fresh section is created only when a network id is available. +pub(super) fn splice_creative_slots( + existing: &str, + keys: &CreativeSectionKeys<'_>, + rendered_slots: &str, +) -> CliResult { + let mut document = existing.parse::().map_err(|error| { + report_error(format!( + "failed to parse target config before updating slots: {error}" + )) + })?; + let had_section = document.get("creative_opportunities").is_some(); + let existing_section_position = document + .get("creative_opportunities") + .and_then(Item::as_table) + .and_then(Table::position); + let section_position = existing_section_position + .unwrap_or_else(|| max_table_position(document.as_table()).unwrap_or(0) + 1); + if !had_section && keys.network_id.is_none() { + return cli_error( + "refusing to create a `[creative_opportunities]` section without a \ + GAM network id: none could be determined from the audited page, and \ + the key is required. Add `[creative_opportunities]` with a \ + `gam_network_id` to the config and re-run", + ); + } + + let generated = format!( + "[creative_opportunities]\n{}\n", + rendered_slots.trim_matches('\n') + ); + let mut generated = generated + .parse::() + .map_err(|error| report_error(format!("failed to parse generated slot tables: {error}")))?; + let mut generated_slots = generated["creative_opportunities"] + .as_table_mut() + .and_then(|table| table.remove("slot")) + .unwrap_or_else(|| Item::ArrayOfTables(toml_edit::ArrayOfTables::new())); + if let Item::ArrayOfTables(array) = &mut generated_slots { + for table in array.iter_mut() { + set_table_position_recursive(table, section_position); + } + } + + if !had_section { + document["creative_opportunities"] = Item::Table(toml_edit::Table::new()); + } + let creative = document["creative_opportunities"] + .as_table_mut() + .ok_or_else(|| { + report_error( + "target config's `creative_opportunities` value is not an editable table; \ + rewrite it as a `[creative_opportunities]` table and re-run", + ) + })?; + // `toml_edit` stably sorts tables by document position. Imported tables + // retain positions from their source document, so anchor the whole subtree + // here to keep the parent, slots, and provider tables together. + creative.set_position(section_position); + if let Some(network_id) = keys.network_id { + creative["gam_network_id"] = toml_edit::value(network_id); + } + if let Some(section_root) = keys.section_root { + creative["section_root"] = toml_edit::value(section_root); + if let Some(section_segment) = keys.section_segment { + creative["section_segment"] = toml_edit::value(section_segment as i64); + } + } + creative.insert("slot", generated_slots); + + let mut result = document.to_string(); + if uses_crlf(existing) { + result = convert_document_lf_to_crlf(&result); + } + ensure_only_managed_fields_changed(existing, &result)?; + Ok(result) +} + +/// Verifies that the structural update changed only generator-managed fields. +fn ensure_only_managed_fields_changed(before: &str, after: &str) -> CliResult<()> { + fn unmanaged(document: &str) -> CliResult { + let mut value = toml::from_str::(document) + .map_err(|error| report_error(format!("failed to validate updated config: {error}")))?; + if let Some(root) = value.as_table_mut() { + let remove_empty = if let Some(creative) = root + .get_mut("creative_opportunities") + .and_then(toml::Value::as_table_mut) + { + for key in ["slot", "gam_network_id", "section_root", "section_segment"] { + creative.remove(key); + } + creative.is_empty() + } else { + false + }; + if remove_empty { + root.remove("creative_opportunities"); + } + } + Ok(value) + } + + if unmanaged(before)? != unmanaged(after)? { + return cli_error( + "refusing to update config because fields outside the managed \ + creative-opportunities keys would change", + ); + } + Ok(()) +} + +/// Byte offsets of the `\n` bytes that terminate a document line. +/// +/// Only newlines outside comments and string values delimit lines, so the scan +/// skips a `#` comment to end of line, skips single-line basic and literal +/// strings, and tracks multiline `"""` / `'''` bodies. Without the comment and +/// single-line-string cases a stray triple quote desynchronizes the scan and the +/// document's line endings are flipped or left mixed — a rewrite +/// [`ensure_only_managed_fields_changed`] cannot catch, because it compares +/// parsed values. +fn document_newlines(document: &str) -> Vec { + let bytes = document.as_bytes(); + let mut newlines = Vec::new(); + let mut index = 0_usize; + while index < bytes.len() { + match bytes[index] { + b'#' => { + while index < bytes.len() && bytes[index] != b'\n' { + index += 1; + } + } + b'\n' => { + newlines.push(index); + index += 1; + } + quote @ (b'"' | b'\'') => { + if bytes[index..].starts_with(&[quote, quote, quote]) { + index += 3; + while index < bytes.len() && !bytes[index..].starts_with(&[quote, quote, quote]) + { + index += 1; + } + index = index.saturating_add(3).min(bytes.len()); + } else { + index += 1; + while index < bytes.len() && bytes[index] != quote && bytes[index] != b'\n' { + index += if quote == b'"' && bytes[index] == b'\\' { + 2 + } else { + 1 + }; + } + if index < bytes.len() && bytes[index] == quote { + index += 1; + } + } + } + _ => index += 1, + } + } + newlines +} + +/// Whether `document` uses CRLF line endings (so edits preserve them). +fn uses_crlf(document: &str) -> bool { + let bytes = document.as_bytes(); + document_newlines(document) + .first() + .is_some_and(|&index| index > 0 && bytes[index - 1] == b'\r') +} + +/// Converts document line terminators while leaving string content intact. +fn convert_document_lf_to_crlf(document: &str) -> String { + let bytes = document.as_bytes(); + let mut output = String::with_capacity(document.len()); + let mut previous = 0_usize; + for index in document_newlines(document) { + output.push_str(&document[previous..index]); + if index == 0 || bytes[index - 1] != b'\r' { + output.push('\r'); + } + output.push('\n'); + previous = index + 1; + } + output.push_str(&document[previous..]); + output +} + +/// Strips a trailing inline `# comment` from a candidate table-header line. +/// +/// Only valid on header candidates: header lines cannot contain `#` before the +/// closing bracket unless it is inside a quoted key, which the configs this +/// updater manages never use. +fn strip_inline_comment(line: &str) -> &str { + match line.find('#') { + Some(position) => line[..position].trim_end(), + None => line, + } +} + +pub(super) fn replace_key_in_section( + document: &str, + section: &str, + key: &str, + replacement_line: &str, +) -> CliResult { + let section_header = format!("[{section}]"); + let mut in_section = false; + let mut replaced = false; + let mut saw_section = false; + let mut lines = Vec::new(); + + for line in document.lines() { + let trimmed = line.trim(); + let header_candidate = strip_inline_comment(trimmed); + if header_candidate.starts_with('[') && header_candidate.ends_with(']') { + in_section = header_candidate == section_header; + saw_section |= in_section; + } + + if in_section && !replaced && is_key_line(trimmed, key) { + lines.push(replacement_line.to_string()); + replaced = true; + } else { + lines.push(line.to_string()); + } + } + + if !saw_section { + return cli_error(format!( + "failed to update starter config because section `{section_header}` was not found" + )); + } + if !replaced { + return cli_error(format!( + "failed to update starter config because key `{key}` was not found in `{section_header}`" + )); + } + + let mut output = lines.join("\n"); + if document.ends_with('\n') { + output.push('\n'); + } + if uses_crlf(document) { + // `lines()` stripped the `\r`s; restore the document's CRLF endings. + output = output.replace("\r\n", "\n").replace('\n', "\r\n"); + } + Ok(output) +} + +fn is_key_line(trimmed_line: &str, key: &str) -> bool { + trimmed_line + .strip_prefix(key) + .and_then(|remaining| remaining.trim_start().strip_prefix('=')) + .is_some() +} + +/// Chooses the `gam_network_id` to write. +/// +/// The existing id is kept only when a real merge preserves existing slots. +/// On `--replace`, or when the config had no slots (e.g. a placeholder +/// `[creative_opportunities]` section), the discovered id wins — mirroring +/// the slot merge, which returns discovered-only in those cases. +pub(super) fn resolve_network_id( + existing: Option<&CreativeOpportunitiesConfig>, + discovered_network_id: Option<&str>, + replace: bool, +) -> Option { + let existing_network_id = existing.map(|config| config.gam_network_id.clone()); + let preserving_existing = !replace && existing.is_some_and(|config| !config.slot.is_empty()); + if preserving_existing { + existing_network_id.or_else(|| discovered_network_id.map(str::to_string)) + } else { + discovered_network_id + .map(str::to_string) + .or(existing_network_id) + } +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::commands::audit::generate::collector; + + fn discovered_header_slot() -> gpt_slots::DiscoveredSlots { + let registry = vec![collector::CollectedGptSlot { + gam_unit_path: "/222/homepage/header".to_string(), + div_id: "div-gpt-ad-header".to_string(), + sizes: vec![(728, 90)], + }]; + gpt_slots::discover_gpt_slots(®istry, &[], false) + } + + /// Rendered slot text for the discovered header slot, patterns = `/`. + fn header_rendered() -> String { + let merged = merge_slots(None, &discovered_header_slot(), &["/".to_string()], true); + render_slots(&merged) + } + + fn two_provider_slots_rendered() -> &'static str { + r#" +# Slots managed by `ts audit ad-templates generate`. +# Review page_patterns and formats before validating/pushing. + +[[creative_opportunities.slot]] +id = "header" +div_id = "header" +gam_unit_path = "/222/{section}/header" +page_patterns = ["/"] +formats = [{ width = 728, height = 90 }] +[creative_opportunities.slot.providers.prebid] +bidders = {} + +[[creative_opportunities.slot]] +id = "sidebar" +div_id = "sidebar" +gam_unit_path = "/222/{section}/sidebar" +page_patterns = ["/"] +formats = [{ width = 300, height = 250 }] +[creative_opportunities.slot.providers.aps] +slot_id = "sidebar" +"# + } + + fn table_headers(document: &str) -> Vec<&str> { + document + .lines() + .map(str::trim) + .filter(|line| line.starts_with('[')) + .collect() + } + + /// Section keys carrying only a network id, the common test case. + fn network_keys(network_id: &str) -> CreativeSectionKeys<'_> { + CreativeSectionKeys { + network_id: Some(network_id), + ..CreativeSectionKeys::default() + } + } + + fn existing_config(toml_str: &str) -> CreativeOpportunitiesConfig { + toml::from_str::(toml_str).expect("valid creative config") + } + + #[test] + fn splice_replaces_slots_and_preserves_other_sections() { + let existing = "[publisher]\ndomain = \"x\"\n\n\ + [creative_opportunities]\ngam_network_id = \"111\"\nprice_granularity = \"dense\"\n\n\ + [[creative_opportunities.slot]]\nid = \"old\"\ndiv_id = \"old\"\n\ + gam_unit_path = \"/111/old\"\npage_patterns = [\"/\"]\n\ + formats = [{ width = 300, height = 250 }]\n\n\ + [auction]\nenabled = true\n"; + + let out = splice_creative_slots(existing, &network_keys("222"), &header_rendered()) + .expect("should splice"); + + assert!( + out.contains("gam_network_id = \"222\""), + "network id updated" + ); + assert!(!out.contains("id = \"old\""), "old slot removed"); + assert!( + out.contains("gam_unit_path = \"/222/homepage/header\""), + "new slot written" + ); + assert!( + out.contains("[publisher]") && out.contains("domain = \"x\""), + "publisher section preserved" + ); + assert!( + out.contains("[auction]") && out.contains("enabled = true"), + "trailing auction section preserved" + ); + toml::from_str::(&out).expect("spliced config is valid TOML"); + } + + #[test] + fn splice_updates_a_quoted_section_header_structurally() { + let existing = "[\"creative_opportunities\"]\ngam_network_id = \"111\"\n"; + + let updated = splice_creative_slots(existing, &network_keys("222"), &header_rendered()) + .expect("should update quoted table structurally"); + + assert_eq!(updated.matches("creative_opportunities").count(), 2); + assert!(updated.contains("gam_network_id = \"222\"")); + toml::from_str::(&updated).expect("should remain valid TOML"); + } + + #[test] + fn splice_preserves_multiline_values_comments_and_noncontiguous_tables() { + let existing = "title = \"publisher\" # keep this comment\n\ + description = \"\"\"a line that looks like [creative_opportunities]\n\ + and another [[creative_opportunities.slot]] line\"\"\"\n\ + dimensions = [\n 300,\n 250,\n]\n\n\ + [creative_opportunities] # managed section\n\ + gam_network_id = \"111\" # old network\n\n\ + [[creative_opportunities.slot]]\nid = \"old-a\"\ndiv_id = \"old-a\"\n\ + gam_unit_path = \"/111/a\"\npage_patterns = [\"/\"]\n\ + formats = [{ width = 300, height = 250 }]\n\n\ + [auction]\nenabled = true # keep auction comment\n\n\ + [[creative_opportunities.slot]]\nid = \"old-b\"\ndiv_id = \"old-b\"\n\ + gam_unit_path = \"/111/b\"\npage_patterns = [\"/b\"]\n\ + formats = [{ width = 320, height = 50 }]\n"; + + let updated = splice_creative_slots(existing, &network_keys("222"), &header_rendered()) + .expect("should update structurally"); + + assert!(updated.contains("looks like [creative_opportunities]")); + assert!(updated.contains("dimensions = [\n 300,\n 250,\n]")); + assert!(updated.contains("enabled = true # keep auction comment")); + assert!(!updated.contains("id = \"old-a\"")); + assert!(!updated.contains("id = \"old-b\"")); + let value = toml::from_str::(&updated).expect("should remain valid TOML"); + assert_eq!( + value["creative_opportunities"]["slot"] + .as_array() + .map(Vec::len), + Some(1) + ); + } + + #[test] + fn splice_keeps_generated_slots_and_providers_contiguous() { + let existing = "[publisher]\ndomain = \"example.com\"\n\n\ + [tester_cookie]\nenabled = true\n\n\ + [creative_opportunities]\ngam_network_id = \"111\"\n\n\ + [debug]\nauction_html_comment = true\n"; + + let updated = splice_creative_slots( + existing, + &network_keys("222"), + two_provider_slots_rendered(), + ) + .expect("should splice slots"); + + assert_eq!( + table_headers(&updated), + vec![ + "[publisher]", + "[tester_cookie]", + "[creative_opportunities]", + "[[creative_opportunities.slot]]", + "[creative_opportunities.slot.providers.prebid]", + "[[creative_opportunities.slot]]", + "[creative_opportunities.slot.providers.aps]", + "[debug]", + ] + ); + } + + #[test] + fn splice_groups_a_new_creative_section_with_its_slots() { + let existing = "[publisher]\ndomain = \"example.com\"\n\n\ + [debug]\nauction_html_comment = true\n\n\ + [auction]\nenabled = true\n"; + + let updated = splice_creative_slots( + existing, + &network_keys("222"), + two_provider_slots_rendered(), + ) + .expect("should create creative section and splice slots"); + + assert_eq!( + table_headers(&updated), + vec![ + "[publisher]", + "[debug]", + "[auction]", + "[creative_opportunities]", + "[[creative_opportunities.slot]]", + "[creative_opportunities.slot.providers.prebid]", + "[[creative_opportunities.slot]]", + "[creative_opportunities.slot.providers.aps]", + ] + ); + } + + #[test] + fn splice_rejects_top_level_inline_creative_opportunities_table() { + let existing = "creative_opportunities = { gam_network_id = \"111\" }\n"; + + let error = splice_creative_slots(existing, &network_keys("222"), &header_rendered()) + .expect_err("should refuse a top-level inline table"); + + assert!( + format!("{error:?}").contains("rewrite it as"), + "error should tell the operator to rewrite the section, got {error:?}" + ); + } + + /// Section keys for a templated run: network id plus the section policy. + fn template_keys<'a>( + network_id: &'a str, + root: &'a str, + segment: usize, + ) -> CreativeSectionKeys<'a> { + CreativeSectionKeys { + network_id: Some(network_id), + section_root: Some(root), + section_segment: Some(segment), + } + } + + #[test] + fn splice_inserts_section_policy_keys_a_config_does_not_have_yet() { + // The whole point of `upsert`: every config predating templating lacks + // these keys, so a replace-only writer could never add them. + let existing = "[creative_opportunities]\ngam_network_id = \"111\"\n\n\ + [auction]\nenabled = true\n"; + + let out = splice_creative_slots( + existing, + &template_keys("222", "homepage", 0), + &header_rendered(), + ) + .expect("should splice"); + + let value = toml::from_str::(&out).expect("spliced config is valid TOML"); + let creative = &value["creative_opportunities"]; + assert_eq!(creative["gam_network_id"].as_str(), Some("222")); + assert_eq!(creative["section_root"].as_str(), Some("homepage")); + assert_eq!(creative["section_segment"].as_integer(), Some(0)); + assert_eq!( + value["auction"]["enabled"].as_bool(), + Some(true), + "inserting must not disturb later sections" + ); + } + + #[test] + fn splice_replaces_section_policy_keys_that_are_already_present() { + let existing = "[creative_opportunities]\ngam_network_id = \"111\"\n\ + section_root = \"old\"\nsection_segment = 2\n"; + + let out = splice_creative_slots( + existing, + &template_keys("111", "homepage", 1), + &header_rendered(), + ) + .expect("should splice"); + + let value = toml::from_str::(&out).expect("valid TOML"); + let creative = &value["creative_opportunities"]; + assert_eq!(creative["section_root"].as_str(), Some("homepage")); + assert_eq!(creative["section_segment"].as_integer(), Some(1)); + assert_eq!( + out.matches("section_root").count(), + 1, + "the key must be replaced, not duplicated" + ); + } + + #[test] + fn splice_omits_section_policy_when_no_slot_needs_it() { + // `section_root`/`section_segment` are `deny_unknown_fields` additions: + // writing them into a config that does not need them would make it + // unloadable by an older binary for no benefit. + let existing = "[creative_opportunities]\ngam_network_id = \"111\"\n"; + + let out = splice_creative_slots(existing, &network_keys("222"), &header_rendered()) + .expect("should splice"); + + assert!( + !out.contains("section_root") && !out.contains("section_segment"), + "an untemplated run must not add rollback-fatal keys, got:\n{out}" + ); + } + + #[test] + fn splice_writes_section_policy_into_a_freshly_created_section() { + let existing = "[publisher]\ndomain = \"x\"\n"; + + let out = splice_creative_slots( + existing, + &template_keys("222", "homepage", 0), + &header_rendered(), + ) + .expect("should append a fresh section"); + + let value = toml::from_str::(&out).expect("valid TOML"); + let creative = &value["creative_opportunities"]; + assert_eq!(creative["gam_network_id"].as_str(), Some("222")); + assert_eq!(creative["section_root"].as_str(), Some("homepage")); + assert_eq!(creative["section_segment"].as_integer(), Some(0)); + } + + #[test] + fn splice_refuses_fresh_section_without_a_network_id() { + // Reachable whenever the scraped unit path has no all-digit leading + // segment (MCM/child-network paths). Writing the section anyway produces + // a config missing a required field, which fails load and takes every + // route to the startup error router once pushed. + let existing = "[publisher]\ndomain = \"x\"\n"; + + let error = splice_creative_slots( + existing, + &CreativeSectionKeys::default(), + &header_rendered(), + ) + .expect_err("should refuse to create a section with no network id"); + + assert!( + format!("{error:?}").contains("without a GAM network id"), + "error should name the missing network id, got {error:?}" + ); + } + + #[test] + fn splice_appends_section_when_config_has_none() { + let existing = "[publisher]\ndomain = \"x\"\n"; + + let out = splice_creative_slots(existing, &network_keys("222"), &header_rendered()) + .expect("should append a fresh section"); + + let value = toml::from_str::(&out).expect("appended config is valid TOML"); + assert_eq!( + value["creative_opportunities"]["gam_network_id"].as_str(), + Some("222") + ); + } + + #[test] + fn splice_preserves_section_scalars_and_provider_subtables() { + // Mirrors the templated operator shape: section policy scalars in the + // head block and a per-slot prebid provider subtable. + let existing = "[creative_opportunities]\n\ + gam_network_id = \"111\"\n\ + auction_timeout_ms = 2000\n\ + section_root = \"homepage\"\n\n\ + [[creative_opportunities.slot]]\n\ + id = \"ad-header-0\"\n\ + div_id = \"ad-header-0\"\n\ + gam_unit_path = \"/{network_id}/example/{section}\"\n\ + page_patterns = [\"/\"]\n\ + formats = [{ width = 728, height = 90 }]\n\ + [creative_opportunities.slot.providers.prebid]\n\ + bidders = {}\n\n\ + [auction]\nenabled = true\n"; + let existing_config = existing_config( + &existing + .replace("[creative_opportunities]\n", "") + .replace("[[creative_opportunities.slot]]", "[[slot]]") + .replace("[creative_opportunities.slot.", "[slot.") + .replace("\n[auction]\nenabled = true\n", ""), + ); + let discovered = discovered_header_slot(); + let merged = merge_slots( + Some(&existing_config), + &discovered, + &["/news/*".to_string()], + false, + ); + + let out = splice_creative_slots(existing, &network_keys("111"), &render_slots(&merged)) + .expect("should splice"); + + let value = toml::from_str::(&out).expect("spliced config is valid TOML"); + let creative = &value["creative_opportunities"]; + assert_eq!( + creative["section_root"].as_str(), + Some("homepage"), + "section policy scalars must survive the splice" + ); + assert_eq!(creative["auction_timeout_ms"].as_integer(), Some(2000)); + assert_eq!( + creative["slot"][0]["gam_unit_path"].as_str(), + Some("/{network_id}/example/{section}"), + "an existing templated unit path must not be rewritten to a literal" + ); + assert!( + creative["slot"][0]["providers"]["prebid"]["bidders"].is_table(), + "the prebid provider subtable must be re-emitted" + ); + assert_eq!( + value["auction"]["enabled"].as_bool(), + Some(true), + "trailing sections must be preserved" + ); + } + + #[test] + fn splice_preserves_crlf_line_endings() { + let existing = "[creative_opportunities]\r\ngam_network_id = \"111\"\r\n\r\n\ + [auction]\r\nenabled = true\r\n"; + + let out = splice_creative_slots(existing, &network_keys("222"), &header_rendered()) + .expect("should splice"); + + assert!( + !out.replace("\r\n", "").contains('\n'), + "every line ending should stay CRLF" + ); + let value = toml::from_str::(&out).expect("spliced CRLF config is valid TOML"); + assert_eq!( + value["creative_opportunities"]["gam_network_id"].as_str(), + Some("222"), + "network id updated in CRLF config" + ); + } + + #[test] + fn splice_does_not_infer_document_endings_from_multiline_string_content() { + let existing = "[publisher]\nother = \"\"\"a\r\nb\"\"\"\n\n\ + [creative_opportunities]\ngam_network_id = \"111\"\n"; + + let out = splice_creative_slots(existing, &network_keys("222"), &header_rendered()) + .expect("should splice LF document"); + + assert!( + out.contains("[publisher]\nother"), + "an embedded CRLF must not convert document line endings" + ); + assert!( + out.contains("a\r\nb"), + "an unrelated multiline string value must remain byte-identical" + ); + } + + #[test] + fn a_triple_quote_in_a_comment_does_not_desynchronize_the_line_scan() { + // A `"""` inside a comment is not a multiline string. Treating it as one + // makes the rest of the document read as string content, so a CRLF file + // is detected as LF and gets rewritten wholesale. + let existing = "# see \"\"\" docs\r\n[creative_opportunities]\r\n\ + gam_network_id = \"111\"\r\n"; + + let out = splice_creative_slots(existing, &network_keys("222"), &header_rendered()) + .expect("should splice CRLF document"); + + assert!( + !out.replace("\r\n", "").contains('\n'), + "the document's CRLF endings must survive a triple quote in a comment, got {out:?}" + ); + } + + #[test] + fn a_triple_quote_in_a_single_line_string_does_not_desynchronize_the_line_scan() { + let existing = "[publisher]\r\nlabel = 'a \"\"\" b'\r\n\r\n\ + [creative_opportunities]\r\ngam_network_id = \"111\"\r\n"; + + let out = splice_creative_slots(existing, &network_keys("222"), &header_rendered()) + .expect("should splice CRLF document"); + + assert!( + !out.replace("\r\n", "").contains('\n'), + "the document's CRLF endings must survive a triple quote in a value, got {out:?}" + ); + } + + #[test] + fn splice_does_not_rewrite_bare_lf_inside_crlf_multiline_string() { + let existing = "[publisher]\r\nother = \"\"\"a\nb\"\"\"\r\n\r\n\ + [creative_opportunities]\r\ngam_network_id = \"111\"\r\n"; + + let out = splice_creative_slots(existing, &network_keys("222"), &header_rendered()) + .expect("should splice CRLF document"); + + assert!( + out.contains("a\nb"), + "a bare LF inside an unrelated multiline value must remain unchanged" + ); + } + + #[test] + fn render_slots_writes_non_finite_floor_price_as_valid_toml() { + let slot = RenderSlot { + id: "header".to_string(), + div_id: Some("div-gpt-ad-header".to_string()), + gam_unit_path: Some("/222/homepage/header".to_string()), + page_patterns: vec!["/".to_string()], + formats: vec![(728, 90, None)], + floor_price: Some(f64::NAN), + targeting: BTreeMap::new(), + aps_slot_id: None, + prebid_bidders: None, + }; + + let rendered = render_slots(&[slot]); + + assert!( + rendered.contains("floor_price = nan"), + "NaN should render as TOML `nan`, not Rust `NaN`" + ); + toml::from_str::(&rendered).expect("rendered slots are valid TOML"); + } + + #[test] + fn render_slots_formats_long_arrays_across_indented_lines() { + let slot = RenderSlot { + id: "header".to_string(), + div_id: Some("div-gpt-ad-header".to_string()), + gam_unit_path: Some("/222/homepage/header".to_string()), + page_patterns: vec!["/".to_string(), "/news".to_string(), "/news/*".to_string()], + formats: vec![(728, 90, None), (970, 250, None), (300, 250, None)], + floor_price: None, + targeting: BTreeMap::new(), + aps_slot_id: None, + prebid_bidders: None, + }; + + let rendered = render_slots(&[slot]); + + assert!( + rendered.contains("page_patterns = [\n \"/\",\n \"/news\",\n \"/news/*\",\n]\n"), + "page patterns should be readable one-per-line" + ); + assert!( + rendered.contains( + "formats = [\n { width = 728, height = 90 },\n \ + { width = 970, height = 250 },\n \ + { width = 300, height = 250 },\n]\n" + ), + "formats should be readable one-per-line" + ); + toml::from_str::(&rendered).expect("formatted slots are valid TOML"); + } + + #[test] + fn splice_creates_section_when_absent() { + // Config with no [creative_opportunities] at all — generate should append it. + let existing = "[publisher]\ndomain = \"x\"\n\n[auction]\nenabled = true\n"; + + let out = splice_creative_slots(existing, &network_keys("222"), &header_rendered()) + .expect("should splice"); + + let value = toml::from_str::(&out).expect("valid TOML"); + assert_eq!( + value["creative_opportunities"]["gam_network_id"].as_str(), + Some("222"), + "appended section carries the discovered network id" + ); + assert_eq!( + value["creative_opportunities"]["slot"][0]["id"].as_str(), + Some("header") + ); + assert!( + value["publisher"]["domain"].as_str() == Some("x") + && value["auction"]["enabled"].as_bool() == Some(true), + "existing sections preserved when appending" + ); + } + + #[test] + fn resplice_does_not_accumulate_managed_comment() { + // A re-run splices into a config that already carries the managed + // header comment; it must keep exactly one copy, not append another. + let first = splice_creative_slots( + "[publisher]\ndomain = \"x\"\n\n[auction]\nenabled = true\n", + &network_keys("222"), + &header_rendered(), + ) + .expect("first splice"); + let second = splice_creative_slots(&first, &network_keys("222"), &header_rendered()) + .expect("second splice"); + let third = splice_creative_slots(&second, &network_keys("222"), &header_rendered()) + .expect("third splice"); + + assert_eq!( + third + .lines() + .filter(|line| line.trim() == MANAGED_SLOTS_COMMENT) + .count(), + 1, + "managed header comment must not accumulate across re-splices" + ); + toml::from_str::(&third).expect("re-spliced config stays valid TOML"); + } + + #[test] + fn splice_recognizes_inline_commented_section_header() { + // `[creative_opportunities] # comment` is valid TOML; the splice must + // update it in place instead of appending a duplicate section. + let existing = "[creative_opportunities] # ad templates\ngam_network_id = \"111\"\n\n\ + [auction] # flags\nenabled = true\n"; + + let out = splice_creative_slots(existing, &network_keys("222"), &header_rendered()) + .expect("should splice"); + + assert_eq!( + out.lines() + .filter(|line| { strip_inline_comment(line.trim()) == "[creative_opportunities]" }) + .count(), + 1, + "commented header must not be duplicated" + ); + let value = toml::from_str::(&out).expect("spliced config is valid TOML"); + assert_eq!( + value["creative_opportunities"]["gam_network_id"].as_str(), + Some("222"), + "network id updated under a commented header" + ); + assert_eq!( + value["creative_opportunities"]["slot"][0]["id"].as_str(), + Some("header") + ); + assert_eq!( + value["auction"]["enabled"].as_bool(), + Some(true), + "commented trailing section preserved" + ); + } + + #[test] + fn splice_inserts_when_no_existing_slots() { + let existing = + "[creative_opportunities]\ngam_network_id = \"111\"\n\n[auction]\nenabled = true\n"; + + let out = splice_creative_slots(existing, &network_keys("222"), &header_rendered()) + .expect("should splice"); + + let value = toml::from_str::(&out).expect("valid TOML"); + assert_eq!( + value["creative_opportunities"]["slot"][0]["id"].as_str(), + Some("header"), + "inserted slot id strips the div-gpt-ad- prefix" + ); + assert_eq!( + value["creative_opportunities"]["slot"][0]["div_id"].as_str(), + Some("div-gpt-ad-header"), + "div_id keeps the stable stem" + ); + assert!( + value["auction"]["enabled"].as_bool() == Some(true), + "auction section preserved after inserted slots" + ); + } + + #[test] + fn splice_replaces_inline_slot_array() { + let existing = "[creative_opportunities]\n\ + gam_network_id = \"111\"\n\ + slot = [{ id = \"old\", div_id = \"old\", gam_unit_path = \"/111/old\", page_patterns = [\"/\"], formats = [{ width = 300, height = 250 }] }]\n\n\ + [auction]\nenabled = true\n"; + + let out = splice_creative_slots(existing, &network_keys("222"), &header_rendered()) + .expect("should replace inline slot array"); + + let value = toml::from_str::(&out).expect("spliced config should be valid"); + let slots = value["creative_opportunities"]["slot"] + .as_array() + .expect("slots should be an array"); + assert_eq!(slots.len(), 1, "old inline slot should be removed"); + assert_eq!(slots[0]["id"].as_str(), Some("header")); + assert_eq!( + value["auction"]["enabled"].as_bool(), + Some(true), + "unrelated tables should be preserved" + ); + } + + #[test] + fn splice_replaces_inline_slot_map() { + let existing = "[creative_opportunities]\n\ + gam_network_id = \"111\"\n\ + slot = { \"0\" = { id = \"old\", div_id = \"old\", gam_unit_path = \"/111/old\", page_patterns = [\"/\"], formats = [{ width = 300, height = 250 }] } }\n"; + + let out = splice_creative_slots(existing, &network_keys("222"), &header_rendered()) + .expect("should replace inline slot map"); + + let value = toml::from_str::(&out).expect("spliced config should be valid"); + let slots = value["creative_opportunities"]["slot"] + .as_array() + .expect("slots should be an array"); + assert_eq!(slots.len(), 1, "old inline slot should be removed"); + assert_eq!(slots[0]["id"].as_str(), Some("header")); + } + + #[test] + fn merge_second_run_unions_page_patterns() { + // Existing slot on "/"; re-discovered this run with "/news/*". + let existing = existing_config( + "gam_network_id = \"222\"\n\n\ + [[slot]]\nid = \"header\"\ndiv_id = \"div-gpt-ad-header\"\n\ + gam_unit_path = \"/222/homepage/header\"\npage_patterns = [\"/\"]\n\ + formats = [{ width = 728, height = 90 }]\n", + ); + + let merged = merge_slots( + Some(&existing), + &discovered_header_slot(), + &["/news/*".to_string()], + false, + ); + + assert_eq!(merged.len(), 1, "same slot is not duplicated"); + assert_eq!( + merged[0].page_patterns, + vec!["/".to_string(), "/news/*".to_string()], + "this run's pattern is unioned into the existing slot" + ); + } + + #[test] + fn merge_second_run_unions_formats() { + let existing = existing_config( + "gam_network_id = \"222\"\n\n\ + [[slot]]\nid = \"header\"\ndiv_id = \"div-gpt-ad-header\"\n\ + gam_unit_path = \"/222/homepage/header\"\npage_patterns = [\"/\"]\n\ + formats = [{ width = 728, height = 90 }]\n", + ); + let registry = vec![collector::CollectedGptSlot { + gam_unit_path: "/222/homepage/header".to_string(), + div_id: "div-gpt-ad-header".to_string(), + sizes: vec![(728, 90), (970, 250)], + }]; + let discovered = gpt_slots::discover_gpt_slots(®istry, &[], false); + + let merged = merge_slots(Some(&existing), &discovered, &["/".to_string()], false); + + assert_eq!( + merged[0].formats, + [(728, 90, None), (970, 250, None)], + "a later audit must retain newly observed formats" + ); + } + + #[test] + fn merge_uses_longest_existing_div_prefix() { + let existing = existing_config( + "gam_network_id = \"222\"\n\n\ + [[slot]]\nid = \"broad\"\ndiv_id = \"ad-\"\n\ + gam_unit_path = \"/222/broad\"\npage_patterns = [\"/broad/*\"]\n\ + formats = [{ width = 300, height = 250 }]\n\n\ + [[slot]]\nid = \"atf\"\ndiv_id = \"ad-atf-\"\n\ + gam_unit_path = \"/222/atf\"\npage_patterns = [\"/\"]\n\ + formats = [{ width = 728, height = 90 }]\n", + ); + let registry = vec![collector::CollectedGptSlot { + gam_unit_path: "/222/atf".to_string(), + div_id: "ad-atf-0".to_string(), + sizes: vec![(728, 90)], + }]; + let discovered = gpt_slots::discover_gpt_slots(®istry, &[], false); + + let merged = merge_slots( + Some(&existing), + &discovered, + &["/news/*".to_string()], + false, + ); + + assert_eq!( + merged.len(), + 2, + "prefix match should not append a duplicate" + ); + let broad = merged + .iter() + .find(|slot| slot.id == "broad") + .expect("should keep broad slot"); + assert_eq!( + broad.page_patterns, + ["/broad/*"], + "shorter prefix should not claim the discovered div" + ); + let atf = merged + .iter() + .find(|slot| slot.id == "atf") + .expect("should keep specific slot"); + assert_eq!( + atf.page_patterns, + ["/", "/news/*"], + "longest matching prefix should receive this run's pattern" + ); + } + + #[test] + fn observed_literal_does_not_claim_numeric_siblings() { + let existing = existing_config( + "gam_network_id = \"222\"\n\n\ + [[slot]]\nid = \"ad-sidebar-1\"\ndiv_id = \"ad-sidebar-1\"\n\ + gam_unit_path = \"/222/sidebar\"\npage_patterns = [\"/\"]\n\ + formats = [{ width = 300, height = 250 }]\n", + ); + let discovered = ["ad-sidebar-1", "ad-sidebar-10", "ad-sidebar-11"] + .into_iter() + .map(|div_id| { + RenderSlot::from_evidence( + div_id, + div_id, + Some("/222/sidebar".to_string()), + [(300, 250)], + vec!["/news/*".to_string()], + false, + ) + }) + .collect(); + + let (merged, diagnostics) = + merge_render_slots_with_diagnostics(Some(&existing), discovered, false); + + assert_eq!(merged.len(), 3); + assert!(merged.iter().any(|slot| slot.id == "ad-sidebar-10")); + assert!(merged.iter().any(|slot| slot.id == "ad-sidebar-11")); + assert!(diagnostics.notes.is_empty()); + assert!(diagnostics.unobserved_existing_slot_ids.is_empty()); + } + + #[test] + fn split_sibling_warns_when_tuned_parent_fields_are_not_inherited() { + let existing = existing_config( + "gam_network_id = \"222\"\n\n\ + [[slot]]\nid = \"ad-sidebar-1\"\ndiv_id = \"ad-sidebar-1\"\n\ + gam_unit_path = \"/222/sidebar\"\npage_patterns = [\"/\"]\n\ + formats = [{ width = 300, height = 250 }]\nfloor_price = 1.5\n", + ); + let discovered = ["ad-sidebar-1", "ad-sidebar-10"] + .into_iter() + .map(|div_id| { + RenderSlot::from_evidence( + div_id, + div_id, + Some("/222/sidebar".to_string()), + [(300, 250)], + vec!["/news/*".to_string()], + false, + ) + }) + .collect(); + + let (merged, diagnostics) = + merge_render_slots_with_diagnostics(Some(&existing), discovered, false); + + let sibling = merged + .iter() + .find(|slot| slot.id == "ad-sidebar-10") + .expect("should append the distinct sibling"); + assert_eq!( + sibling.floor_price, None, + "a distinct placement must not inherit the configured parent's floor" + ); + assert_eq!(diagnostics.notes.len(), 1, "should emit one split warning"); + assert!( + diagnostics.notes[0].contains("discovered div `ad-sidebar-10`"), + "should name the split sibling, got {:?}", + diagnostics.notes + ); + assert!( + diagnostics.notes[0].contains("configured literal prefix `ad-sidebar-1`"), + "should name the disqualified parent prefix, got {:?}", + diagnostics.notes + ); + assert!( + diagnostics.notes[0].contains("does not inherit"), + "should explain the tuned-field consequence, got {:?}", + diagnostics.notes + ); + } + + #[test] + fn newly_appended_literal_does_not_claim_numeric_sibling() { + let existing = existing_config( + "gam_network_id = \"222\"\n\n\ + [[slot]]\nid = \"legacy\"\ndiv_id = \"legacy-slot\"\n\ + gam_unit_path = \"/222/legacy\"\npage_patterns = [\"/\"]\n\ + formats = [{ width = 300, height = 250 }]\n", + ); + let discovered = ["ad-sidebar-1", "ad-sidebar-10"] + .into_iter() + .map(|div_id| { + RenderSlot::from_evidence( + div_id, + div_id, + Some("/222/sidebar".to_string()), + [(300, 250)], + vec!["/news/*".to_string()], + false, + ) + }) + .collect(); + + let (merged, diagnostics) = + merge_render_slots_with_diagnostics(Some(&existing), discovered, false); + + assert_eq!(merged.len(), 3); + assert!(merged.iter().any(|slot| slot.id == "ad-sidebar-1")); + assert!(merged.iter().any(|slot| slot.id == "ad-sidebar-10")); + assert!(diagnostics.notes.is_empty()); + } + + #[test] + fn normalized_stem_is_the_literal_merge_boundary() { + let existing = existing_config( + "gam_network_id = \"222\"\n\n\ + [[slot]]\nid = \"ad-header-0\"\ndiv_id = \"ad-header-0\"\n\ + gam_unit_path = \"/222/header\"\npage_patterns = [\"/\"]\n\ + formats = [{ width = 728, height = 90 }]\n", + ); + let registry = vec![ + collector::CollectedGptSlot { + gam_unit_path: "/222/header".to_string(), + div_id: "ad-header-0-_R_3f_".to_string(), + sizes: vec![(728, 90)], + }, + collector::CollectedGptSlot { + gam_unit_path: "/222/header".to_string(), + div_id: "ad-header-01".to_string(), + sizes: vec![(728, 90)], + }, + ]; + let discovered = gpt_slots::discover_gpt_slots(®istry, &[], false); + + let merged = merge_slots(Some(&existing), &discovered, &["/".to_string()], false); + + assert_eq!(merged.len(), 2); + assert!(merged.iter().any(|slot| slot.id == "ad-header-0")); + assert!(merged.iter().any(|slot| slot.id == "ad-header-01")); + } + + #[test] + fn merge_reports_when_a_broad_prefix_claims_multiple_discovered_divs() { + let existing = existing_config( + "gam_network_id = \"222\"\n\n\ + [[slot]]\nid = \"broad\"\ndiv_id = \"ad-\"\n\ + gam_unit_path = \"/222/broad\"\npage_patterns = [\"/\"]\n\ + formats = [{ width = 300, height = 250 }]\n", + ); + let discovered = vec![ + RenderSlot::from_evidence( + "header", + "ad-header", + Some("/222/header".to_string()), + [(728, 90)], + vec!["/".to_string()], + false, + ), + RenderSlot::from_evidence( + "footer", + "ad-footer", + Some("/222/footer".to_string()), + [(300, 250)], + vec!["/".to_string()], + false, + ), + ]; + + let (merged, diagnostics) = + merge_render_slots_with_diagnostics(Some(&existing), discovered, false); + + assert_eq!( + merged.len(), + 1, + "the configured prefix still controls merging" + ); + assert_eq!(diagnostics.notes.len(), 1); + assert!(diagnostics.notes[0].contains("matched 2 discovered divs")); + assert!(diagnostics.notes[0].contains("ad-footer")); + assert!( + diagnostics.notes[0] + .contains("runtime can resolve this configured slot to at most one"), + "diagnostic should explain the runtime consequence" + ); + assert!(diagnostics.notes[0].contains("ad-header")); + } + + #[test] + fn a_slot_appended_this_run_never_absorbs_a_later_discovery() { + // Prefix reconciliation belongs to the operator's config. If a slot + // appended during this run could act as a prefix, `ad-top` would swallow + // `ad-top-sidebar` whenever discovery happened to see it first, dropping + // the absorbed slot's unit path and provider state, and no broad-prefix + // diagnostic would report it. + let existing = existing_config( + "gam_network_id = \"222\"\n\n\ + [[slot]]\nid = \"sidebar\"\ndiv_id = \"sidebar-ad\"\n\ + gam_unit_path = \"/222/sidebar\"\npage_patterns = [\"/\"]\n\ + formats = [{ width = 300, height = 600 }]\n", + ); + let candidates = [ + RenderSlot::from_evidence( + "ad-top", + "ad-top", + Some("/222/top".to_string()), + [(728, 90)], + vec!["/".to_string()], + false, + ), + RenderSlot::from_evidence( + "ad-top-sidebar", + "ad-top-sidebar", + Some("/222/top-sidebar".to_string()), + [(300, 250)], + vec!["/news/*".to_string()], + true, + ), + ]; + + for order in [[0_usize, 1], [1, 0]] { + let discovered: Vec = order + .iter() + .map(|index| candidates[*index].clone()) + .collect(); + + let (merged, diagnostics) = + merge_render_slots_with_diagnostics(Some(&existing), discovered, false); + + assert!( + diagnostics.notes.is_empty(), + "no configured prefix claimed a discovered div in order {order:?}, got {diagnostics:?}" + ); + assert_eq!( + merged.len(), + 3, + "both discovered slots must survive in order {order:?}" + ); + let sidebar_ad = merged + .iter() + .find(|slot| slot.div_id.as_deref() == Some("ad-top-sidebar")) + .unwrap_or_else(|| { + panic!("the longer div must stay its own slot in order {order:?}") + }); + assert_eq!( + sidebar_ad.gam_unit_path.as_deref(), + Some("/222/top-sidebar"), + "the absorbed slot's unit path must survive in order {order:?}" + ); + assert_eq!( + sidebar_ad.page_patterns, + ["/news/*"], + "patterns must not be pooled in order {order:?}" + ); + assert!( + sidebar_ad.prebid_bidders.is_some(), + "provider state must survive in order {order:?}" + ); + let top = merged + .iter() + .find(|slot| slot.div_id.as_deref() == Some("ad-top")) + .unwrap_or_else(|| panic!("the shorter div must stay in order {order:?}")); + assert_eq!( + top.page_patterns, + ["/"], + "the longer slot's pattern must not leak into the shorter one in order {order:?}" + ); + } + } + + #[test] + fn merge_renames_new_slot_id_that_collides_with_existing_config() { + let existing = existing_config( + "gam_network_id = \"222\"\n\n\ + [[slot]]\nid = \"header-main\"\ndiv_id = \"legacy-header\"\n\ + gam_unit_path = \"/222/legacy\"\npage_patterns = [\"/\"]\n\ + formats = [{ width = 300, height = 250 }]\n", + ); + let registry = vec![collector::CollectedGptSlot { + gam_unit_path: "/222/header".to_string(), + div_id: "div-gpt-ad-header.main".to_string(), + sizes: vec![(728, 90)], + }]; + let discovered = gpt_slots::discover_gpt_slots(®istry, &[], false); + + let merged = merge_slots( + Some(&existing), + &discovered, + &["/news/*".to_string()], + false, + ); + let ids = merged + .iter() + .map(|slot| slot.id.as_str()) + .collect::>(); + + assert_eq!(ids, ["header-main", "header-main-2"]); + } + + #[test] + fn merge_keeps_existing_only_slots() { + // Existing has header + sidebar; this run re-sees only header. + let existing = existing_config( + "gam_network_id = \"222\"\n\n\ + [[slot]]\nid = \"header\"\ndiv_id = \"div-gpt-ad-header\"\n\ + gam_unit_path = \"/222/homepage/header\"\npage_patterns = [\"/\"]\n\ + formats = [{ width = 728, height = 90 }]\n\n\ + [[slot]]\nid = \"sidebar\"\ndiv_id = \"ad-sidebar\"\n\ + gam_unit_path = \"/222/sidebar\"\npage_patterns = [\"/news/*\"]\n\ + formats = [{ width = 300, height = 250 }]\nfloor_price = 0.5\n", + ); + + let merged = merge_slots( + Some(&existing), + &discovered_header_slot(), + &["/".to_string()], + false, + ); + + let ids: Vec<&str> = merged.iter().map(|slot| slot.id.as_str()).collect(); + assert_eq!(ids, vec!["header", "sidebar"], "sidebar preserved"); + let sidebar = merged + .iter() + .find(|slot| slot.id == "sidebar") + .expect("sidebar"); + assert_eq!( + sidebar.floor_price, + Some(0.5), + "hand-tuned fields preserved" + ); + } + + #[test] + fn merge_reports_preserved_unobserved_slots_in_config_order() { + let existing = existing_config( + "gam_network_id = \"222\"\n\n\ + [[slot]]\nid = \"header\"\ndiv_id = \"div-gpt-ad-header\"\n\ + gam_unit_path = \"/222/header\"\npage_patterns = [\"/\"]\n\ + formats = [{ width = 728, height = 90 }]\n\n\ + [[slot]]\nid = \"sidebar\"\ndiv_id = \"ad-sidebar\"\n\ + gam_unit_path = \"/222/sidebar\"\npage_patterns = [\"/news/*\"]\n\ + formats = [{ width = 300, height = 250 }]\n\n\ + [[slot]]\nid = \"footer\"\ndiv_id = \"ad-footer\"\n\ + gam_unit_path = \"/222/footer\"\npage_patterns = [\"/\"]\n\ + formats = [{ width = 728, height = 90 }]\n", + ); + let discovered = vec![RenderSlot::from_evidence( + "header", + "div-gpt-ad-header", + Some("/222/header".to_string()), + [(728, 90)], + vec!["/".to_string()], + false, + )]; + + let (_, diagnostics) = + merge_render_slots_with_diagnostics(Some(&existing), discovered, false); + + assert_eq!( + diagnostics.unobserved_existing_slot_ids, + ["sidebar", "footer"], + "unobserved slots should retain configuration order" + ); + + let all_discovered = vec![ + RenderSlot::from_evidence( + "header", + "div-gpt-ad-header", + Some("/222/header".to_string()), + [(728, 90)], + vec!["/".to_string()], + false, + ), + RenderSlot::from_evidence( + "sidebar", + "ad-sidebar", + Some("/222/sidebar".to_string()), + [(300, 250)], + vec!["/news/*".to_string()], + false, + ), + RenderSlot::from_evidence( + "footer", + "ad-footer", + Some("/222/footer".to_string()), + [(728, 90)], + vec!["/".to_string()], + false, + ), + ]; + let (_, fully_observed) = + merge_render_slots_with_diagnostics(Some(&existing), all_discovered.clone(), false); + let (_, replaced) = + merge_render_slots_with_diagnostics(Some(&existing), all_discovered.clone(), true); + let (_, no_existing) = merge_render_slots_with_diagnostics(None, all_discovered, false); + + assert!( + fully_observed.unobserved_existing_slot_ids.is_empty(), + "fully observed slots should not be reported as stale" + ); + assert!( + replaced.unobserved_existing_slot_ids.is_empty(), + "--replace should not report discarded existing slots as stale" + ); + assert!( + no_existing.unobserved_existing_slot_ids.is_empty(), + "a config without existing slots should not report stale slots" + ); + } + + #[test] + fn observed_div_marks_exact_slot_and_live_broad_prefix() { + let existing = existing_config( + "gam_network_id = \"222\"\n\n\ + [[slot]]\nid = \"broad\"\ndiv_id = \"ad-\"\n\ + gam_unit_path = \"/222/broad\"\npage_patterns = [\"/\"]\n\ + formats = [{ width = 300, height = 250 }]\n\n\ + [[slot]]\nid = \"header\"\ndiv_id = \"ad-header\"\n\ + gam_unit_path = \"/222/header\"\npage_patterns = [\"/\"]\n\ + formats = [{ width = 728, height = 90 }]\n", + ); + let discovered = vec![RenderSlot::from_evidence( + "header", + "ad-header", + Some("/222/header".to_string()), + [(728, 90)], + vec!["/news/*".to_string()], + false, + )]; + + let (_, diagnostics) = merge_render_slots_with_observed_diagnostics( + Some(&existing), + discovered, + &["ad-header".to_string()], + false, + ); + + assert!( + diagnostics.unobserved_existing_slot_ids.is_empty(), + "the exact slot and every live configured prefix should be observed, got {diagnostics:?}" + ); + } + + #[test] + fn merge_replace_wipes_existing() { + let existing = existing_config( + "gam_network_id = \"222\"\n\n\ + [[slot]]\nid = \"sidebar\"\ndiv_id = \"ad-sidebar\"\n\ + gam_unit_path = \"/222/sidebar\"\npage_patterns = [\"/\"]\n\ + formats = [{ width = 300, height = 250 }]\n", + ); + + let merged = merge_slots( + Some(&existing), + &discovered_header_slot(), + &["/".to_string()], + true, + ); + + let ids: Vec<&str> = merged.iter().map(|slot| slot.id.as_str()).collect(); + assert_eq!(ids, vec!["header"], "--replace keeps only discovered slots"); + } + + #[test] + fn resolve_network_id_prefers_discovered_unless_preserving_existing() { + let with_slots = existing_config( + "gam_network_id = \"111\"\n\n[[slot]]\nid = \"s\"\ndiv_id = \"ad-s\"\n\ + gam_unit_path = \"/111/s\"\npage_patterns = [\"/\"]\n\ + formats = [{ width = 300, height = 250 }]\n", + ); + let empty = existing_config("gam_network_id = \"111\"\n"); + + // Real merge → keep existing. + assert_eq!( + resolve_network_id(Some(&with_slots), Some("222"), false).as_deref(), + Some("111") + ); + // Placeholder section with no slots → discovered wins. + assert_eq!( + resolve_network_id(Some(&empty), Some("222"), false).as_deref(), + Some("222") + ); + // --replace → discovered wins. + assert_eq!( + resolve_network_id(Some(&with_slots), Some("222"), true).as_deref(), + Some("222") + ); + // No existing config → discovered. + assert_eq!( + resolve_network_id(None, Some("222"), false).as_deref(), + Some("222") + ); + } + + #[test] + fn toml_key_quotes_only_non_bare_keys() { + assert_eq!(toml_key("zone"), "zone"); + assert_eq!(toml_key("ad-loc"), "ad-loc"); + assert_eq!(toml_key("a.b"), "\"a.b\""); + assert_eq!(toml_key("with space"), "\"with space\""); + assert_eq!(toml_key(""), "\"\""); + } + + #[test] + fn toml_string_escapes_quotes_backslashes_and_controls() { + assert_eq!(toml_string("a\"b\\c"), "\"a\\\"b\\\\c\""); + assert_eq!(toml_string("line\nbreak\t!"), "\"line\\nbreak\\t!\""); + } + + #[test] + fn toml_string_escapes_del_control_char() { + assert_eq!(toml_string("a\u{7f}b"), "\"a\\u007Fb\""); + let doc = format!("value = {}", toml_string("a\u{7f}b")); + let value = toml::from_str::(&doc).expect("DEL escapes to valid TOML"); + assert_eq!( + value["value"].as_str(), + Some("a\u{7f}b"), + "escaped DEL round-trips as data" + ); + } + + #[test] + fn replace_key_handles_inline_commented_headers() { + let document = "[creative_opportunities] # managed\ngam_network_id = \"111\"\n\n\ + [auction] # flags\nenabled = true\n"; + + let updated = replace_key_in_section( + document, + "creative_opportunities", + "gam_network_id", + "gam_network_id = \"222\"", + ) + .expect("should find the commented section header"); + + assert!( + updated.contains("gam_network_id = \"222\""), + "key replaced under a commented header" + ); + assert!( + updated.contains("enabled = true"), + "later commented section left untouched" + ); + } + + #[test] + fn render_quotes_exotic_targeting_keys_to_valid_toml() { + let existing = existing_config( + "gam_network_id = \"1\"\n\n\ + [[slot]]\nid = \"s\"\ndiv_id = \"ad-s\"\ngam_unit_path = \"/1/s\"\n\ + page_patterns = [\"/\"]\nformats = [{ width = 300, height = 250 }]\n\ + targeting = { \"a.b\" = \"x\" }\n", + ); + + let merged = merge_slots( + Some(&existing), + &discovered_header_slot(), + &["/".to_string()], + false, + ); + let doc = format!( + "[creative_opportunities]\ngam_network_id = \"1\"\n{}", + render_slots(&merged) + ); + + toml::from_str::(&doc).expect("exotic targeting key renders as valid TOML"); + } +} diff --git a/crates/trusted-server-cli/src/commands/audit/generate/unit_template.rs b/crates/trusted-server-cli/src/commands/audit/generate/unit_template.rs new file mode 100644 index 000000000..06b7c955d --- /dev/null +++ b/crates/trusted-server-cli/src/commands/audit/generate/unit_template.rs @@ -0,0 +1,1028 @@ +//! Infers a `{network_id}`/`{section}` ad-unit template from observed evidence. +//! +//! The generator otherwise writes the literal path each page happened to +//! request, which pins a slot to the one section it was scraped from. A template +//! generalizes across sections — but a *wrong* template makes the publisher bid +//! against inventory that does not exist, which is worse than a narrow literal. +//! So this module is built to refuse rather than guess. +//! +//! The inference applies three evidence rules: +//! +//! 1. **Positional binding.** `{network_id}` is bound to unit segment 0 and only +//! if that segment is the resolved network id. Substring replacement would +//! corrupt `/123/sports123/home` into `/{network_id}/sports{network_id}/home`. +//! 2. **Exactly one varying segment.** Zero means nothing was proven and the +//! path stays literal; two means the unit varies along a dimension the +//! request path cannot supply (device, geo, experiment), so it is refused. +//! 3. **Cross-page variation.** Two pages must show *different* derived sections +//! and different unit segments. A single-page crawl is +//! indistinguishable from a static path — literal, `{network_id}`-only and +//! `{section}` all reproduce one observation equally well, and round-trip +//! verification cannot tell them apart. Only variation can. +//! +//! Every accepted template is then replayed through the runtime's own +//! [`render_gam_unit_path`](CreativeOpportunitySlot::render_gam_unit_path) and +//! [`derive_section`] against every observation. A template that does not +//! reproduce what the live page actually requested is downgraded, not written. + +use std::collections::{BTreeMap, BTreeSet}; + +use trusted_server_core::creative_opportunities::{CreativeOpportunitySlot, derive_section}; + +use super::evidence::{EvidenceTable, SlotEvidence}; +use super::slot_toml::toml_string; + +/// Candidate `section_segment` values considered, `0..=MAX_SECTION_SEGMENT`. +/// +/// A locale-prefixed site (`/en/news/story`) needs 1. Beyond 2 the "section" is +/// no longer a taxonomy the operator would recognise, and every extra candidate +/// is another chance for two indices to both fit and force a refusal. +const MAX_SECTION_SEGMENT: usize = 2; + +/// The config-level section policy an inferred template depends on. +#[derive(Debug, Clone, PartialEq, Eq)] +pub(super) struct SectionPolicy { + /// Value substituted for `{section}` on paths with no section segment. + pub(super) section_root: String, + /// Index of the path segment `{section}` is taken from. + pub(super) section_segment: usize, +} + +/// What to write for one slot's `gam_unit_path`. +#[derive(Debug, Clone, PartialEq, Eq)] +pub(super) enum SlotDecision { + /// Write this templated path; it reproduced every observation. + Template(String), + /// Write this literal path; nothing generalizable was proven. + Literal(String), + /// Write no path at all — the observations cannot be represented. + Refuse { + /// Operator-facing explanations, one per reason. + reasons: Vec, + }, +} + +/// The outcome of inference across the whole evidence table. +#[derive(Debug, Clone, PartialEq, Eq)] +pub(super) struct InferenceOutcome { + /// Section policy to write, present only when some slot templated. + pub(super) policy: Option, + /// Per-slot decision, keyed by div stem, in evidence order. + pub(super) decisions: Vec<(String, SlotDecision)>, + /// Operator-facing notes about why inference went the way it did. + pub(super) diagnostics: Vec, + /// Div stems whose templates rely on a root witnessed by another slot. + pub(super) borrowed_section_root: Vec, +} + +impl InferenceOutcome { + /// The decision for a slot, by div stem. + pub(super) fn decision(&self, div_id: &str) -> Option<&SlotDecision> { + self.decisions + .iter() + .find(|(key, _)| key == div_id) + .map(|(_, decision)| decision) + } +} + +/// Per-slot analysis under one candidate `section_segment`. +#[derive(Debug, Clone, PartialEq, Eq)] +enum SlotAnalysis { + /// Templatable: unit segment `varying` tracks the derived section, and root + /// pages agreed on `section_root`. + Templatable { + varying: usize, + section_root: String, + }, + /// The unit path never varied, so nothing about `{section}` was proven. + Static, + /// Cannot be represented; carries the operator-facing reason. + Refuse(String), + /// Unit segment `varying` tracks the derived section on every page this slot + /// was seen on, but none of those pages lacked the section segment, so the + /// slot witnessed no `section_root` of its own. + /// + /// Carries `varying` because such a slot is still templatable *when another + /// slot witnessed the config-level `section_root`*: a placement that only + /// exists on section pages (a sidebar, an in-article unit) never renders on a + /// path where `{section}` would fall back to the root. + RootUnwitnessed { varying: usize }, +} + +/// Infers unit-path templates for every slot in `table`. +/// +/// `network_id` is the resolved GAM network id; `{network_id}` is only ever +/// bound to a unit segment that already equals it. +pub(super) fn infer_unit_templates(table: &EvidenceTable, network_id: &str) -> InferenceOutcome { + let slots: Vec<&SlotEvidence> = table.slots().collect(); + let mut diagnostics = Vec::new(); + + // Evaluate every candidate index independently; ambiguity between two that + // both fit is a refusal, not a preference for the smaller one. + let mut qualifying: Vec<(usize, String, BTreeMap)> = Vec::new(); + let mut root_witness_missing = false; + let mut root_unwitnessed_stems = BTreeSet::new(); + for segment in 0..=MAX_SECTION_SEGMENT { + let analyses: BTreeMap = slots + .iter() + .map(|slot| (slot.div_id.clone(), analyse_slot(slot, network_id, segment))) + .collect(); + + let roots: BTreeSet<&str> = analyses + .values() + .filter_map(|analysis| match analysis { + SlotAnalysis::Templatable { section_root, .. } => Some(section_root.as_str()), + _ => None, + }) + .collect(); + // Slots must agree: `section_root` is one config-level value, so two + // slots claiming different roots means this index is not the real one. + let Some(root) = roots.iter().next().copied() else { + // Distinguish "nothing tracks the section" from "everything does but + // no crawled page lacked the section segment": the second is a crawl + // gap the operator can close, and the generic literal-path refusal + // below does not say so. + root_witness_missing |= analyses + .values() + .any(|analysis| matches!(analysis, SlotAnalysis::RootUnwitnessed { .. })); + root_unwitnessed_stems.extend( + analyses + .iter() + .filter(|(_, analysis)| { + matches!(analysis, SlotAnalysis::RootUnwitnessed { .. }) + }) + .map(|(stem, _)| stem.clone()), + ); + continue; + }; + if roots.len() > 1 { + continue; + } + qualifying.push((segment, root.to_string(), analyses)); + } + + let chosen = match qualifying.len() { + 0 => None, + 1 => qualifying.into_iter().next(), + _ => { + let indices: Vec = qualifying + .iter() + .map(|(segment, _, _)| segment.to_string()) + .collect(); + diagnostics.push(format!( + "more than one section_segment ({}) explains the observed ad-unit paths \ + equally well, so no template can be chosen safely; slots without one safe literal path are omitted", + indices.join(", ") + )); + None + } + }; + + let Some((section_segment, section_root, analyses)) = chosen else { + // Only a crawl gap justifies rewriting the per-slot reasons. When + // inference stopped on segment ambiguity instead, that pushed its own + // diagnostic, and blaming the crawl here would send the operator to + // widen it when the remedy is pinning `section_segment`. + let root_gap = diagnostics.is_empty() && root_witness_missing; + if diagnostics.is_empty() { + diagnostics.push(if root_witness_missing { + "the ad-unit paths do track the page section, but no crawled page lacked a \ + section segment, so `section_root` could not be witnessed and no {section} \ + template can be written; include the site root in the crawl (or set \ + section_root by hand) to template these slots" + .to_string() + } else { + "no ad-unit path varied by page section across the crawl, so paths were kept \ + literal; crawl more sections to enable a {section} template" + .to_string() + }); + } + let mut decisions = literal_decisions(&slots); + if root_gap { + for (stem, decision) in &mut decisions { + if root_unwitnessed_stems.contains(stem) + && let SlotDecision::Refuse { reasons } = decision + { + *reasons = vec![ + "the paths tracked the page section, but no crawled page lacked a \ + section segment, so `section_root` could not be witnessed" + .to_string(), + ]; + } + } + } + return InferenceOutcome { + policy: None, + decisions, + diagnostics, + borrowed_section_root: Vec::new(), + }; + }; + + let mut decisions = Vec::with_capacity(slots.len()); + let mut borrowed_section_root = Vec::new(); + let mut templated = 0_usize; + for slot in &slots { + let analysis = analyses + .get(&slot.div_id) + .cloned() + .unwrap_or(SlotAnalysis::Static); + let templatable = match analysis { + SlotAnalysis::Templatable { varying, .. } => Some((varying, true)), + // The config-level `section_root` is witnessed by another slot on the + // same property, and this slot's page patterns are derived from the + // paths it was seen on — all of which carry a section segment — so + // `{section}` never falls back to the root for it. Refusing here cost + // real inventory: a sidebar or in-article unit that simply does not + // exist on the site root was omitted from the config entirely. + SlotAnalysis::RootUnwitnessed { varying } => Some((varying, false)), + SlotAnalysis::Static | SlotAnalysis::Refuse(_) => None, + }; + let decision = match (templatable, analysis) { + (Some((varying, witnessed_root)), _) => { + let template = build_template(slot, varying); + match verify_round_trip(&template, slot, network_id, §ion_root, section_segment) + { + Ok(()) => { + templated += 1; + if !witnessed_root { + borrowed_section_root.push(slot.div_id.clone()); + diagnostics.push(format!( + "slot `{}` was never observed on a page without a section \ + segment, so its `{{section}}` template relies on the \ + config-level section_root `{section_root}` witnessed by other \ + slots; it is only rendered for the paths this slot was seen on", + slot.id + )); + } + SlotDecision::Template(template) + } + Err(reason) => { + diagnostics.push(format!( + "slot `{}` template `{template}` did not reproduce the observed \ + ad-unit paths ({reason}); refusing any unsafe fallback", + slot.id + )); + literal_decision(slot) + } + } + } + (None, SlotAnalysis::Refuse(reason)) => SlotDecision::Refuse { + reasons: vec![reason], + }, + (None, _) => literal_decision(slot), + }; + decisions.push((slot.div_id.clone(), decision)); + } + + if templated == 0 { + return InferenceOutcome { + policy: None, + decisions, + diagnostics, + borrowed_section_root: Vec::new(), + }; + } + + diagnostics.push(format!( + "inferred section_segment = {section_segment} and section_root = \"{section_root}\" \ + from {} page(s); {templated} slot(s) templated", + table.pages().len() + )); + InferenceOutcome { + policy: Some(SectionPolicy { + section_root, + section_segment, + }), + decisions, + diagnostics, + borrowed_section_root, + } +} + +/// Checks the properties of a slot's observations that do not depend on which +/// `section_segment` is being considered. +/// +/// Kept separate because these refusals are final: no candidate index can +/// rescue a slot whose observations are not one template with a single hole in +/// them, and the operator needs the specific reason rather than a generic one. +/// +/// Returns the single varying unit segment, `None` when nothing varied, or the +/// reason the observations cannot be represented at all. +fn structural_check(slot: &SlotEvidence) -> Result, String> { + // One page reporting two different ad-unit paths for the same slot means the + // unit varies along something the request path cannot express — a device or + // geo split, or two profiles disagreeing. Nothing here can represent that. + let mut per_path: BTreeMap<&str, BTreeSet<&str>> = BTreeMap::new(); + for row in &slot.rows { + per_path + .entry(row.path.as_str()) + .or_default() + .insert(row.unit_path.as_str()); + } + if let Some((path, units)) = per_path.iter().find(|(_, units)| units.len() > 1) { + let observed: Vec<&str> = units.iter().copied().collect(); + return Err(format!( + "page `{path}` requested more than one ad-unit path for this slot ({}); \ + the unit varies by something the request path cannot derive", + observed.join(", ") + )); + } + + let split: Vec> = slot + .rows + .iter() + .map(|row| segments(&row.unit_path)) + .collect(); + let Some(first) = split.first() else { + return Ok(None); + }; + // Differing shapes are not one template with a hole in it. + if split.iter().any(|parts| parts.len() != first.len()) { + return Err( + "the observed ad-unit paths have different segment counts, so they are not \ + one template" + .to_string(), + ); + } + + let varying: Vec = (0..first.len()) + .filter(|index| { + split + .iter() + .map(|parts| parts[*index]) + .collect::>() + .len() + > 1 + }) + .collect(); + match varying.len() { + 0 => Ok(None), + 1 if varying[0] == 0 => { + Err("the network-id segment of the ad-unit path varied across pages".to_string()) + } + 1 => Ok(Some(varying[0])), + count => Err(format!( + "{count} ad-unit segments vary across pages, so the path does not track the \ + page section alone" + )), + } +} + +/// Analyses one slot under a candidate `section_segment`. +/// +/// [`structural_check`] has already established that a templatable candidate +/// contains more than one observed unit path. Therefore a successful derived +/// section match here is itself the required variation witness; a second +/// witness predicate would only restate that invariant. +fn analyse_slot(slot: &SlotEvidence, network_id: &str, section_segment: usize) -> SlotAnalysis { + let varying = match structural_check(slot) { + Err(reason) => return SlotAnalysis::Refuse(reason), + Ok(None) => return SlotAnalysis::Static, + Ok(Some(varying)) => varying, + }; + + let split: Vec> = slot + .rows + .iter() + .map(|row| segments(&row.unit_path)) + .collect(); + // `{network_id}` binds positionally and only to the resolved id. Substring + // replacement would rewrite an unrelated segment that merely contains it. + if split.first().and_then(|parts| parts.first()) != Some(&network_id) { + return SlotAnalysis::Static; + } + + // Partition observations into pages that have a section segment and pages + // that do not; the latter are what determine `section_root`. + let mut root_values = BTreeSet::new(); + for (row, parts) in slot.rows.iter().zip(split.iter()) { + let observed = parts[varying]; + if path_segments(&row.path).len() > section_segment { + // The empty root is unused here: the path has this segment. + if derive_section(&row.path, "", section_segment) != observed { + return SlotAnalysis::Static; + } + } else { + root_values.insert(observed); + } + } + + let mut roots = root_values.into_iter(); + let Some(section_root) = roots.next() else { + // Without a root observation, `section_root` would be a guess that + // silently mis-renders every short path. + return SlotAnalysis::RootUnwitnessed { varying }; + }; + if roots.next().is_some() { + return SlotAnalysis::Static; + } + // A root that is not `[A-Za-z0-9_-]+` makes any `{section}` template fail + // config load; catch it here rather than at push time. + if section_root.is_empty() + || !section_root + .chars() + .all(|ch| ch.is_ascii_alphanumeric() || ch == '_' || ch == '-') + { + return SlotAnalysis::Static; + } + + SlotAnalysis::Templatable { + varying, + section_root: section_root.to_string(), + } +} + +/// Builds the template text by substituting the two proven placeholders. +fn build_template(slot: &SlotEvidence, varying: usize) -> String { + let first = slot + .rows + .iter() + .next() + .map(|row| row.unit_path.as_str()) + .unwrap_or_default(); + let rendered: Vec = segments(first) + .into_iter() + .enumerate() + .map(|(index, value)| { + if index == 0 { + "{network_id}".to_string() + } else if index == varying { + "{section}".to_string() + } else { + value.to_string() + } + }) + .collect(); + format!("/{}", rendered.join("/")) +} + +/// Replays `template` through the runtime renderer against every observation. +/// +/// Defense in depth rather than the primary gate: [`analyse_slot`] already +/// refuses to call a slot templatable when the derived section and the observed +/// segment disagree — a publisher whose `/site-news` pages request +/// `.../sitenews`, say — so a mismatch reaching here would mean inference and +/// the runtime renderer disagree. The template is then dropped instead of +/// written, and the diagnostic names the paths that did not reproduce. +fn verify_round_trip( + template: &str, + slot: &SlotEvidence, + network_id: &str, + section_root: &str, + section_segment: usize, +) -> Result<(), String> { + let probe = probe_slot(template)?; + for row in &slot.rows { + let section = derive_section(&row.path, section_root, section_segment); + match probe.render_gam_unit_path(network_id, §ion) { + Some(rendered) if rendered == row.unit_path => {} + Some(rendered) => { + return Err(format!( + "on `{}` it renders `{rendered}` but the page requested `{}`", + row.path, row.unit_path + )); + } + None => { + return Err(format!( + "on `{}` it renders past the GAM ad-unit path byte limit", + row.path + )); + } + } + } + Ok(()) +} + +/// Builds a throwaway slot carrying `template`, for rendering only. +/// +/// Deserializing is how the runtime itself builds slots, so this exercises the +/// same template parsing rather than a parallel implementation. +fn probe_slot(template: &str) -> Result { + let document = format!( + "id = \"probe\"\ngam_unit_path = {}\npage_patterns = [\"/\"]\n\ + formats = [{{ width = 1, height = 1 }}]\n", + toml_string(template) + ); + toml::from_str::(&document) + .map_err(|error| format!("template is not representable in config: {error}")) +} + +/// The decision for a slot no template was proven for. +/// +/// A structural refusal wins over the generic "several paths" message, so the +/// operator sees *why* the slot could not be represented (a device split, an +/// extra varying dimension) rather than only that it could not. +fn literal_decision(slot: &SlotEvidence) -> SlotDecision { + if let Err(reason) = structural_check(slot) { + return SlotDecision::Refuse { + reasons: vec![reason], + }; + } + let units = slot.unit_paths(); + let mut found = units.iter(); + match (found.next(), found.next()) { + (Some(only), None) => SlotDecision::Literal((*only).to_string()), + (Some(_), Some(_)) => SlotDecision::Refuse { + reasons: vec![format!( + "the slot used several ad-unit paths ({}) and none generalized, so no \ + single literal path is correct", + units.into_iter().collect::>().join(", ") + )], + }, + _ => SlotDecision::Refuse { + reasons: vec!["no ad-unit path was observed for this slot".to_string()], + }, + } +} + +fn literal_decisions(slots: &[&SlotEvidence]) -> Vec<(String, SlotDecision)> { + slots + .iter() + .map(|slot| (slot.div_id.clone(), literal_decision(slot))) + .collect() +} + +/// Non-empty path segments of an ad-unit path. +fn segments(unit_path: &str) -> Vec<&str> { + unit_path + .split('/') + .filter(|part| !part.is_empty()) + .collect() +} + +/// Non-empty path segments of a request path. +fn path_segments(path: &str) -> Vec<&str> { + path.split('/').filter(|part| !part.is_empty()).collect() +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::commands::audit::generate::collector::CollectedGptSlot; + use crate::commands::audit::generate::gpt_slots::discover_gpt_slots; + + /// Folds `(path, unit_path)` observations for one div into a table. + fn table_for(div_id: &str, observations: &[(&str, &str)]) -> EvidenceTable { + let mut table = EvidenceTable::default(); + for (path, unit_path) in observations { + let registry = vec![CollectedGptSlot { + gam_unit_path: (*unit_path).to_string(), + div_id: div_id.to_string(), + sizes: vec![(728, 90)], + }]; + table.fold_page(path, &discover_gpt_slots(®istry, &[], false)); + } + table + } + + /// Folds pages carrying different slot sets into one table. + /// + /// Each entry is `(request path, [(div id, ad-unit path)])`. + fn table_for_pages(pages: &[(&str, &[(&str, &str)])]) -> EvidenceTable { + let mut table = EvidenceTable::default(); + for (path, slots) in pages { + let registry: Vec = slots + .iter() + .map(|(div_id, unit_path)| CollectedGptSlot { + gam_unit_path: (*unit_path).to_string(), + div_id: (*div_id).to_string(), + sizes: vec![(728, 90)], + }) + .collect(); + table.fold_page(path, &discover_gpt_slots(®istry, &[], false)); + } + table + } + + fn only_decision(outcome: &InferenceOutcome) -> &SlotDecision { + assert_eq!(outcome.decisions.len(), 1, "fixture should have one slot"); + &outcome.decisions[0].1 + } + + #[test] + fn templates_a_section_varying_unit_path() { + // The shape the operator writes by hand today. + let table = table_for( + "ad-header", + &[ + ("/", "/123456789/publisher/homepage"), + ("/news/story-abc", "/123456789/publisher/news"), + ("/deals/thing", "/123456789/publisher/deals"), + ], + ); + + let outcome = infer_unit_templates(&table, "123456789"); + + assert_eq!( + outcome.policy, + Some(SectionPolicy { + section_root: "homepage".to_string(), + section_segment: 0, + }) + ); + assert_eq!( + only_decision(&outcome), + &SlotDecision::Template("/{network_id}/publisher/{section}".to_string()) + ); + } + + #[test] + fn a_single_page_never_templates() { + // Literal, {network_id}-only and {section} all reproduce one observation, + // so only variation can distinguish them. This is the witness rule. + let table = table_for("ad-header", &[("/news/story", "/123/site/news")]); + + let outcome = infer_unit_templates(&table, "123"); + + assert_eq!(outcome.policy, None); + assert_eq!( + only_decision(&outcome), + &SlotDecision::Literal("/123/site/news".to_string()) + ); + } + + #[test] + fn a_static_unit_path_across_sections_stays_literal() { + let table = table_for( + "ad-header", + &[ + ("/", "/123/site/fixed"), + ("/news/story", "/123/site/fixed"), + ("/deals/x", "/123/site/fixed"), + ], + ); + + let outcome = infer_unit_templates(&table, "123"); + + assert_eq!(outcome.policy, None, "nothing varied, so nothing is proven"); + assert_eq!( + only_decision(&outcome), + &SlotDecision::Literal("/123/site/fixed".to_string()) + ); + } + + #[test] + fn a_device_split_is_refused_rather_than_guessed() { + // Two units for the SAME path: the desktop/mobile cross-check surfaces + // here, and the request path cannot express the difference. + let table = table_for( + "ad-header", + &[ + ("/news/story", "/123/desktop/news"), + ("/news/story", "/123/mobile/news"), + ], + ); + + let outcome = infer_unit_templates(&table, "123"); + + let SlotDecision::Refuse { reasons } = only_decision(&outcome) else { + panic!( + "a device split must refuse, got {:?}", + only_decision(&outcome) + ); + }; + assert!( + reasons[0].contains("more than one ad-unit path"), + "reason should name the conflict, got {reasons:?}" + ); + } + + #[test] + fn two_varying_segments_are_refused() { + let table = table_for( + "ad-header", + &[ + ("/news/story", "/123/desktop/news"), + ("/deals/x", "/123/mobile/deals"), + ], + ); + + let outcome = infer_unit_templates(&table, "123"); + + let SlotDecision::Refuse { reasons } = only_decision(&outcome) else { + panic!("two varying dimensions must refuse"); + }; + assert!( + reasons[0].contains("segments vary"), + "reason should name the extra dimension, got {reasons:?}" + ); + } + + #[test] + fn a_slug_the_path_cannot_reproduce_is_refused() { + // `/site-news` requests `.../sitenews`: the derived section and + // the observed segment differ, so the template would render the wrong + // unit. Candidate analysis rejects the inconsistent section mapping. + let table = table_for( + "ad-header", + &[ + ("/", "/123/site/homepage"), + ("/news/story", "/123/site/news"), + ("/site-news/x", "/123/site/sitenews"), + ], + ); + + let outcome = infer_unit_templates(&table, "123"); + + assert_eq!( + outcome.policy, None, + "a section whose slug is not derivable must not template" + ); + assert!(matches!( + only_decision(&outcome), + SlotDecision::Refuse { .. } + )); + } + + #[test] + fn an_unwitnessed_root_is_refused() { + // Every crawled page had a section, so `section_root` would be a guess + // that silently mis-renders the homepage. + let table = table_for( + "ad-header", + &[ + ("/news/story", "/123/site/news"), + ("/deals/x", "/123/site/deals"), + ], + ); + + let outcome = infer_unit_templates(&table, "123"); + + assert_eq!(outcome.policy, None); + let SlotDecision::Refuse { reasons } = only_decision(&outcome) else { + panic!("two literal paths and no template is not representable as one literal"); + }; + assert!( + reasons + .iter() + .any(|reason| reason.contains("section_root") && reason.contains("witnessed")), + "the per-slot reason should name the crawl gap; got {reasons:?}" + ); + assert!( + outcome + .diagnostics + .iter() + .any(|note| note.contains("section_root` could not be witnessed")), + "the crawl gap, not \"nothing generalized\", is the reason; got {:?}", + outcome.diagnostics + ); + } + + #[test] + fn a_slot_absent_from_the_root_templates_from_the_witnessed_policy() { + // The live shape behind the `ad-atf_sidebar-0` refusal: a header on the + // root and every section witnesses `section_root`, while a sidebar exists + // only on section pages. The sidebar's unit path tracks the section just + // as well, and its page patterns never cover the root, so refusing it + // dropped real inventory from the config. + let mut table = EvidenceTable::default(); + let pages: &[(&str, &[(&str, &str)])] = &[ + ("/", &[("ad-header", "/123/site/homepage")]), + ( + "/news/story", + &[ + ("ad-header", "/123/site/news"), + ("ad-sidebar", "/123/site/news"), + ], + ), + ( + "/deals/x", + &[ + ("ad-header", "/123/site/deals"), + ("ad-sidebar", "/123/site/deals"), + ], + ), + ]; + for (path, slots) in pages { + let registry: Vec = slots + .iter() + .map(|(div_id, unit_path)| CollectedGptSlot { + gam_unit_path: (*unit_path).to_string(), + div_id: (*div_id).to_string(), + sizes: vec![(728, 90)], + }) + .collect(); + table.fold_page(path, &discover_gpt_slots(®istry, &[], false)); + } + + let outcome = infer_unit_templates(&table, "123"); + + assert_eq!( + outcome.policy, + Some(SectionPolicy { + section_root: "homepage".to_string(), + section_segment: 0, + }), + "the header witnesses the config-level policy" + ); + assert_eq!( + outcome.decision("ad-sidebar"), + Some(&SlotDecision::Template( + "/{network_id}/site/{section}".to_string() + )), + "a slot that only exists on section pages is still templatable" + ); + assert_eq!( + outcome.decision("ad-header"), + Some(&SlotDecision::Template( + "/{network_id}/site/{section}".to_string() + )) + ); + assert_eq!( + outcome.borrowed_section_root, + ["ad-sidebar".to_string()], + "the outcome should identify templates whose safety depends on derived patterns" + ); + assert!( + outcome.diagnostics.iter().any(|note| note + .contains("`ad-sidebar` was never observed on a page without a section segment")), + "the borrowed section_root should be stated; got {:?}", + outcome.diagnostics + ); + } + + #[test] + fn segment_ambiguity_does_not_blame_the_crawl_for_an_unwitnessed_root() { + // `ad-header` fits section_segment 0 and `ad-locale` fits 1, so + // inference stops on ambiguity. `ad-deep` is separately + // `RootUnwitnessed` at segment 2. Its refusal must not tell the + // operator to widen the crawl when the remedy is pinning + // `section_segment`. + let table = table_for_pages(&[ + ("/", &[("ad-header", "/99/site/home")]), + ("/news", &[("ad-header", "/99/site/news")]), + ("/en", &[("ad-locale", "/99/site/en-root")]), + ("/en/news", &[("ad-locale", "/99/site/news")]), + ("/a/b/news", &[("ad-deep", "/99/site/news")]), + ("/a/b/deals", &[("ad-deep", "/99/site/deals")]), + ]); + + let outcome = infer_unit_templates(&table, "99"); + + assert!( + outcome + .diagnostics + .iter() + .any(|note| note.contains("more than one section_segment")), + "the fixture should stop on ambiguity; got {:?}", + outcome.diagnostics + ); + assert!( + !outcome + .diagnostics + .iter() + .any(|note| note.contains("include the site root in the crawl")), + "an ambiguous run must not also blame the crawl; got {:?}", + outcome.diagnostics + ); + let Some(SlotDecision::Refuse { reasons }) = outcome.decision("ad-deep") else { + panic!("expected a refusal, got {:?}", outcome.decision("ad-deep")); + }; + assert!( + reasons + .iter() + .all(|reason| !reason.contains("no crawled page lacked a section segment")), + "the crawl-gap reason belongs only to a run that stopped on the crawl gap; got {reasons:?}" + ); + } + + #[test] + fn a_locale_prefixed_site_infers_the_deeper_segment() { + let table = table_for( + "ad-header", + &[ + ("/en", "/123/site/homepage"), + ("/en/news/story", "/123/site/news"), + ("/en/deals/x", "/123/site/deals"), + ], + ); + + let outcome = infer_unit_templates(&table, "123"); + + assert_eq!( + outcome.policy, + Some(SectionPolicy { + section_root: "homepage".to_string(), + section_segment: 1, + }), + "the locale prefix should push the section one segment deeper" + ); + } + + #[test] + fn network_id_is_bound_positionally_not_by_substring() { + // `sports123` merely contains the network id; substring replacement + // would corrupt it into `sports{network_id}`. + let table = table_for( + "ad-header", + &[ + ("/", "/123/sports123/homepage"), + ("/news/story", "/123/sports123/news"), + ("/deals/x", "/123/sports123/deals"), + ], + ); + + let outcome = infer_unit_templates(&table, "123"); + + assert_eq!( + only_decision(&outcome), + &SlotDecision::Template("/{network_id}/sports123/{section}".to_string()), + "only segment 0 may become {{network_id}}" + ); + } + + #[test] + fn a_unit_path_not_starting_with_the_network_id_stays_literal() { + let table = table_for( + "ad-header", + &[ + ("/", "/999/site/homepage"), + ("/news/story", "/999/site/news"), + ], + ); + + let outcome = infer_unit_templates(&table, "123"); + + assert_eq!( + outcome.policy, None, + "segment 0 must equal the resolved network id" + ); + } + + #[test] + fn differing_segment_counts_are_refused() { + let table = table_for( + "ad-header", + &[ + ("/", "/123/site/homepage"), + ("/news/story", "/123/site/news/extra"), + ], + ); + + let outcome = infer_unit_templates(&table, "123"); + + let SlotDecision::Refuse { reasons } = only_decision(&outcome) else { + panic!("differing shapes are not one template"); + }; + assert!( + reasons[0].contains("segment counts"), + "reason should name the shape mismatch, got {reasons:?}" + ); + } + + #[test] + fn a_static_slot_stays_literal_alongside_a_templated_one() { + let mut table = EvidenceTable::default(); + for (path, section_unit) in [ + ("/", "homepage"), + ("/news/story", "news"), + ("/deals/x", "deals"), + ] { + let registry = vec![ + CollectedGptSlot { + gam_unit_path: format!("/123/site/{section_unit}"), + div_id: "ad-header".to_string(), + sizes: vec![(728, 90)], + }, + CollectedGptSlot { + gam_unit_path: "/123/site/sticky".to_string(), + div_id: "ad-sticky".to_string(), + sizes: vec![(300, 250)], + }, + ]; + table.fold_page(path, &discover_gpt_slots(®istry, &[], false)); + } + + let outcome = infer_unit_templates(&table, "123"); + + assert!(outcome.policy.is_some(), "the varying slot should template"); + assert_eq!( + outcome.decision("ad-header"), + Some(&SlotDecision::Template( + "/{network_id}/site/{section}".to_string() + )) + ); + assert_eq!( + outcome.decision("ad-sticky"), + Some(&SlotDecision::Literal("/123/site/sticky".to_string())), + "a genuinely static slot must not be dragged into the template" + ); + } + + #[test] + fn diagnostics_explain_why_nothing_templated() { + let table = table_for("ad-header", &[("/news/story", "/123/site/news")]); + + let outcome = infer_unit_templates(&table, "123"); + + assert!( + outcome + .diagnostics + .iter() + .any(|note| note.contains("crawl more sections")), + "the operator should learn why, got {:?}", + outcome.diagnostics + ); + } +} diff --git a/crates/trusted-server-cli/src/commands/audit/generate/validate.rs b/crates/trusted-server-cli/src/commands/audit/generate/validate.rs new file mode 100644 index 000000000..36664f273 --- /dev/null +++ b/crates/trusted-server-cli/src/commands/audit/generate/validate.rs @@ -0,0 +1,113 @@ +//! Write-side validation for generated ad-template config. +//! +//! Everything the generator writes is derived from a live, page-controlled ad +//! stack, so the candidate document has to clear the same bar the runtime +//! applies at startup *before* it replaces the operator's file. A config the +//! runtime rejects is not a degraded ad stack — `build_state` fails and the +//! adapter answers every route from the startup error router, so an unloadable +//! `trusted-server.toml` is a full-site outage once pushed. + +use trusted_server_core::settings::Settings; + +use crate::error::{CliResult, cli_error}; + +/// Validates the candidate config text the generator is about to persist. +/// +/// Runs [`Settings::from_toml`], which drives the identical +/// `finalize_deserialized` chain the runtime uses — serde (`deny_unknown_fields` +/// plus required fields), then `compile_slots` → `compile_unit_templates` → +/// `validate_runtime`, then the validator pass — with no I/O. +/// +/// `baseline` is the config as it was read from disk. When the baseline is +/// *already* unloadable, this run cannot be blamed for it: the candidate is +/// accepted and the pre-existing error is returned as a warning instead. Without +/// that escape hatch a freshly bootstrapped config carrying placeholder secrets +/// could never be updated by `generate`. +/// +/// # Errors +/// +/// Returns a user-facing error when the candidate fails to load and the baseline +/// loaded cleanly — that is, when this run introduced the failure. +pub(super) fn check_candidate(candidate: &str, baseline: &str) -> CliResult> { + let Err(candidate_error) = Settings::from_toml(candidate) else { + return Ok(Vec::new()); + }; + + if let Err(baseline_error) = Settings::from_toml(baseline) { + return Ok(vec![format!( + "target config was already invalid before this run, so the generated \ + result could not be verified: {baseline_error}" + )]); + } + + cli_error(format!( + "refusing to write: the generated config would fail to load, which would \ + take the service down once pushed: {candidate_error}" + )) +} + +#[cfg(test)] +mod tests { + use super::*; + + /// A minimal config that loads cleanly, used as the valid baseline. + fn baseline() -> String { + crate::commands::config::init::EXAMPLE_CONFIG + .replace("handler_password", "test-admin-password-32-bytes-minimum") + .replace("ec_passphrase", "test-ec-passphrase-32-bytes-minimum") + .replace( + "publisher_proxy_secret", + "test-proxy-secret-32-bytes-minimum", + ) + } + + #[test] + fn valid_candidate_passes_without_warnings() { + let config = baseline(); + + let warnings = check_candidate(&config, &config).expect("should accept valid candidate"); + + assert!( + warnings.is_empty(), + "a clean candidate should not warn, got {warnings:?}" + ); + } + + #[test] + fn candidate_this_run_broke_is_refused() { + let good = baseline(); + // An empty div_id override is exactly what a div id normalized down to + // nothing would produce, and `validate_runtime` rejects it. + let broken = format!( + "{good}\n[[creative_opportunities.slot]]\n\ + id = \"broken\"\ndiv_id = \"\"\n\ + page_patterns = [\"/\"]\n\ + formats = [{{ width = 300, height = 250 }}]\n" + ); + + let error = check_candidate(&broken, &good).expect_err("should refuse a broken candidate"); + + assert!( + format!("{error:?}").contains("refusing to write"), + "error should name the refusal, got {error:?}" + ); + } + + #[test] + fn pre_existing_breakage_downgrades_to_a_warning() { + // The operator's file was already unloadable; `generate` must still be + // able to update it rather than blaming this run for the old error. + let broken_baseline = "[creative_opportunities]\n"; + let broken_candidate = "[creative_opportunities]\n"; + + let warnings = check_candidate(broken_candidate, broken_baseline) + .expect("a pre-existing failure should not block the write"); + + assert_eq!(warnings.len(), 1, "should surface exactly one warning"); + assert!( + warnings[0].contains("already invalid"), + "warning should name the pre-existing failure, got {:?}", + warnings[0] + ); + } +} diff --git a/crates/trusted-server-cli/src/commands/audit/mod.rs b/crates/trusted-server-cli/src/commands/audit/mod.rs index 2f473defd..211060662 100644 --- a/crates/trusted-server-cli/src/commands/audit/mod.rs +++ b/crates/trusted-server-cli/src/commands/audit/mod.rs @@ -1,710 +1,611 @@ -mod analyzer; -pub(crate) mod browser_collector; -pub(crate) mod collector; - -use std::collections::BTreeSet; -use std::fs; -use std::io::Write; -use std::path::{Path, PathBuf}; - -use serde::Serialize; -use url::Url; - -use crate::commands::audit::collector::AuditCollector; -use crate::commands::config::init::EXAMPLE_CONFIG; -use crate::error::{CliResult, cli_error, report_error}; +//! Browser-backed `ts audit` command namespace. +//! +//! `ts audit page ` is the generic page audit; `ts audit ad-templates verify +//! ...` is the ad-template verifier; `ts audit generate ` bootstraps a +//! draft config from a live page (issue #800). `ts audit ` is a hidden +//! compatibility alias for `ts audit generate `. + +pub mod ad_templates; +pub mod browser; +mod browser_scroll; +pub mod collector; +pub mod generate; +pub mod page; + +use clap::{Args, Subcommand}; + +use crate::app_config::AppConfigArgs; +use crate::commands::audit::collector::{BrowserOpts, GenerateBrowserOpts}; +use crate::commands::audit::page::PageAuditArgs; +use crate::error::{CliResult, cli_error}; +use crate::run::RunOutcome; + +/// Parses and validates an `http`/`https` URL, rejecting all other schemes. +/// +/// # Errors +/// +/// Returns a user-facing string when the input is not a valid `http`/`https` URL. +pub(crate) fn parse_http_url(raw: &str) -> Result { + let url = url::Url::parse(raw).map_err(|error| format!("invalid URL `{raw}`: {error}"))?; + match url.scheme() { + "http" | "https" => Ok(url), + other => Err(format!( + "unsupported URL scheme `{other}` (expected http or https)" + )), + } +} -use analyzer::{analyze_collected_page, extract_gtm_container_id}; +/// Parses a `name=value` cookie argument into its `(name, value)` parts. +/// +/// Splits on the first `=` so cookie values may themselves contain `=`. The name +/// must be non-empty; the value may be empty. +/// +/// # Errors +/// +/// Returns a user-facing string when the input has no `=` or an empty name. +pub(crate) fn parse_cookie(raw: &str) -> Result<(String, String), String> { + let (name, value) = raw + .split_once('=') + .ok_or_else(|| format!("invalid cookie `{raw}` (expected NAME=VALUE)"))?; + if name.is_empty() { + return Err(format!("invalid cookie `{raw}` (empty name)")); + } + Ok((name.to_string(), value.to_string())) +} -/// Arguments for the `ts audit` command. -#[derive(Debug, clap::Args)] +/// `ts audit` arguments: an optional subcommand plus a hidden legacy URL positional. +#[derive(Debug, Args)] +#[command(arg_required_else_help = true)] pub(crate) struct AuditArgs { - /// Public HTTP(S) URL to audit. - pub(crate) url: String, + #[command(subcommand)] + pub(crate) command: Option, + /// Hidden compatibility alias: `ts audit ` behaves like `ts audit generate `. + /// + /// The hidden flags below all `requires` this positional, so putting one + /// before a subcommand (`ts audit --chrome X generate `) is rejected + /// rather than silently dropped. `value_name` keeps that rejection from + /// naming the field: an operator told to supply `` cannot find + /// it in `--help`, because the alias is deliberately undocumented. + #[arg(value_parser = parse_http_url, hide = true, value_name = "URL")] + pub(crate) legacy_url: Option, + #[command(flatten)] + pub(crate) legacy_generate: LegacyGenerateArgs, +} + +/// Hidden generation flags retained for the legacy `ts audit ` form. +#[derive(Debug, Default, Args)] +pub(crate) struct LegacyGenerateArgs { /// JavaScript asset audit output path. - #[arg(long)] + #[arg(long, hide = true, requires = "legacy_url")] pub(crate) js_assets: Option, /// Draft Trusted Server config output path. - #[arg(long)] + #[arg(long, hide = true, requires = "legacy_url")] pub(crate) config: Option, /// Do not write the JavaScript asset audit file. - #[arg(long)] + #[arg(long, hide = true, requires = "legacy_url")] pub(crate) no_js_assets: bool, /// Do not write the draft Trusted Server config file. - #[arg(long)] + #[arg(long, hide = true, requires = "legacy_url")] pub(crate) no_config: bool, /// Overwrite existing output files. - #[arg(long)] + #[arg(long, hide = true, requires = "legacy_url")] pub(crate) force: bool, + /// Cookie to send with the page request, as `name=value`. Repeatable. + #[arg( + long = "cookie", + value_name = "NAME=VALUE", + value_parser = parse_cookie, + hide = true, + requires = "legacy_url" + )] + pub(crate) cookies: Vec<(String, String)>, + #[command(flatten)] + pub(crate) browser: LegacyBrowserOpts, } -const DEFAULT_JS_ASSETS_PATH: &str = "js-assets.toml"; -const DEFAULT_CONFIG_PATH: &str = "trusted-server.toml"; - -#[derive(Debug, Clone, Serialize, PartialEq, Eq)] -#[serde(rename_all = "kebab-case")] -pub(crate) enum AssetParty { - FirstParty, - ThirdParty, -} - -#[derive(Debug, Clone, Serialize, PartialEq, Eq)] -pub(crate) struct AuditedAsset { - pub(crate) kind: String, - pub(crate) url: String, - pub(crate) host: String, - pub(crate) party: AssetParty, - #[serde(skip_serializing_if = "Option::is_none")] - pub(crate) integration: Option, +/// Hidden browser flags retained for the legacy `ts audit ` form. +#[derive(Debug, Args)] +pub(crate) struct LegacyBrowserOpts { + /// Path to the Chrome/Chromium executable. + #[arg(long, hide = true, requires = "legacy_url")] + pub(crate) chrome: Option, + /// Run a visible browser instead of Chrome's new headless mode. + #[arg(long, hide = true, requires = "legacy_url")] + pub(crate) headful: bool, + /// Do not answer the standard IAB consent APIs for the fresh audit profile. + #[arg(long, hide = true, requires = "legacy_url")] + pub(crate) no_assume_consent: bool, + /// Route the browser through this proxy. + #[arg(long, value_name = "HOST:PORT", hide = true, requires = "legacy_url")] + pub(crate) browser_proxy: Option, + /// Quiet window in milliseconds that marks the page settled. + #[arg( + long, + default_value_t = crate::commands::audit::collector::GENERATE_SETTLE_QUIET_MS, + hide = true, + requires = "legacy_url" + )] + pub(crate) settle_quiet_ms: u64, + /// Hard cap in milliseconds on waiting for the page to settle. + #[arg( + long, + default_value_t = crate::commands::audit::collector::GENERATE_SETTLE_MAX_MS, + hide = true, + requires = "legacy_url" + )] + pub(crate) settle_max_ms: u64, + /// Navigate to origins whose TLS certificate does not validate. + #[arg(long, hide = true, requires = "legacy_url")] + pub(crate) danger_accept_invalid_certs: bool, } -#[derive(Debug, Clone, Serialize, PartialEq, Eq)] -pub(crate) struct DetectedIntegration { - pub(crate) id: String, - pub(crate) evidence: String, -} - -#[derive(Debug, Clone, Serialize, PartialEq, Eq)] -pub(crate) struct AuditArtifact { - pub(crate) audited_url: String, - #[serde(skip_serializing_if = "Option::is_none")] - pub(crate) page_title: Option, - pub(crate) js_asset_count: usize, - pub(crate) third_party_asset_count: usize, - pub(crate) detected_integrations: Vec, - pub(crate) assets: Vec, - pub(crate) warnings: Vec, -} - -#[derive(Debug, Clone)] -pub(crate) struct AuditOutputs { - pub(crate) artifact: AuditArtifact, - pub(crate) js_assets_toml: String, - pub(crate) draft_config_toml: String, -} - -#[derive(Debug, Clone, PartialEq, Eq)] -struct AuditOutputPlan { - js_assets_path: Option, - config_path: Option, -} - -pub(crate) fn run_audit( - args: &AuditArgs, - collector: &dyn AuditCollector, - out: &mut dyn Write, -) -> CliResult<()> { - let target_url = parse_audit_url(&args.url)?; - let plan = resolve_output_plan(args)?; - let collected = collector.collect_page(&target_url)?; - let outputs = build_audit_outputs(&collected)?; - let wrote_config = plan.config_path.is_some(); - let written = write_audit_outputs(&outputs, &plan)?; - write_success_summary(&outputs, &written, wrote_config, out) -} - -fn parse_audit_url(value: &str) -> CliResult { - let url = Url::parse(value) - .map_err(|error| report_error(format!("invalid audit URL `{value}`: {error}")))?; - if !matches!(url.scheme(), "http" | "https") { - return cli_error(format!( - "`ts audit` only supports http/https URLs, got `{}`", - url.scheme() - )); +impl Default for LegacyBrowserOpts { + fn default() -> Self { + Self { + chrome: None, + headful: false, + no_assume_consent: false, + browser_proxy: None, + settle_quiet_ms: crate::commands::audit::collector::GENERATE_SETTLE_QUIET_MS, + settle_max_ms: crate::commands::audit::collector::GENERATE_SETTLE_MAX_MS, + danger_accept_invalid_certs: false, + } } - Ok(url) } -fn resolve_output_plan(args: &AuditArgs) -> CliResult { - if args.no_js_assets && args.no_config { - return cli_error("nothing to do: both --no-js-assets and --no-config were set"); - } - - let js_assets_path = if args.no_js_assets { - None - } else { - Some(resolve_output_path( - args.js_assets.as_deref(), - DEFAULT_JS_ASSETS_PATH, - )?) - }; - let config_path = if args.no_config { - None - } else { - Some(resolve_output_path( - args.config.as_deref(), - DEFAULT_CONFIG_PATH, - )?) - }; - - if js_assets_path.is_some() && js_assets_path == config_path { - return cli_error("audit output paths must be distinct"); - } - - for path in [&js_assets_path, &config_path].into_iter().flatten() { - if path.exists() && !args.force { - return cli_error(format!( - "refusing to overwrite existing file `{}`; re-run with --force", - path.display() - )); +impl From<&LegacyBrowserOpts> for GenerateBrowserOpts { + fn from(options: &LegacyBrowserOpts) -> Self { + Self { + chrome: options.chrome.clone(), + headful: options.headful, + no_assume_consent: options.no_assume_consent, + browser_proxy: options.browser_proxy.clone(), + settle_quiet_ms: options.settle_quiet_ms, + settle_max_ms: options.settle_max_ms, + danger_accept_invalid_certs: options.danger_accept_invalid_certs, } } +} - Ok(AuditOutputPlan { - js_assets_path, - config_path, - }) +/// `ts audit` subcommands. +#[derive(Debug, Subcommand)] +pub(crate) enum AuditSubcommand { + /// Audit a single page and print a read-only summary. + Page(PageAuditArgs), + /// Verify configured ad-template slots against live page evidence. + #[command(name = "ad-templates", subcommand)] + AdTemplates(AuditAdTemplatesCommand), + /// Bootstrap a draft Trusted Server config + JS asset audit from a live page. + Generate(generate::GenerateArgs), } -fn resolve_output_path(path: Option<&Path>, default: &str) -> CliResult { - let candidate = path.unwrap_or_else(|| Path::new(default)); - if candidate.is_absolute() { - Ok(candidate.to_path_buf()) - } else { - Ok(std::env::current_dir() - .map_err(|error| report_error(format!("failed to read current directory: {error}")))? - .join(candidate)) - } +/// `ts audit ad-templates` subcommands. +#[derive(Debug, Subcommand)] +pub(crate) enum AuditAdTemplatesCommand { + /// Scrape a live page's GPT slots and update the config's + /// `[creative_opportunities]` slots in place. + Generate(AuditAdTemplatesGenerateArgs), + /// Verify ad-template slots for one or more live URLs. + Verify(AuditAdTemplatesVerifyArgs), } -fn build_audit_outputs(collected: &collector::CollectedPage) -> CliResult { - let artifact = analyze_collected_page(collected)?; - let final_url = collected - .final_url() - .map_err(|error| report_error(format!("invalid final URL: {error}")))?; - let js_assets_toml = toml::to_string_pretty(&artifact) - .map_err(|error| report_error(format!("failed to serialize audit artifact: {error}")))?; - let draft_config_toml = build_draft_config(&final_url, &artifact)?; - - Ok(AuditOutputs { - artifact, - js_assets_toml, - draft_config_toml, - }) +/// Arguments for `ts audit ad-templates generate `. +#[derive(Debug, Args)] +pub(crate) struct AuditAdTemplatesGenerateArgs { + #[command(flatten)] + pub config: AppConfigArgs, + /// Page URL to scrape for GPT slots (http or https). + #[arg(value_parser = parse_http_url)] + pub url: url::Url, + /// Glob applied to every slot discovered this run (e.g. `/`, `/news/*`). + /// Repeatable. Defaults to the scraped URL's path. Re-running with a + /// different pattern unions it into slots already in the config. + #[arg(long = "page-pattern", value_name = "GLOB")] + pub page_patterns: Vec, + /// Replace all existing slots instead of merging this run into them. + #[arg(long)] + pub replace: bool, + /// Preview the updated config on stdout instead of writing it. + #[arg(long)] + pub dry_run: bool, + /// Perform a deterministic scroll pass after each page initially settles. + #[arg(long)] + pub scroll: bool, + /// Cookie to send with the page request, as `name=value`. Repeatable. + /// Use to carry an existing session (e.g. a valid bot-protection clearance + /// cookie) so the origin serves the real page instead of a challenge. + #[arg(long = "cookie", value_name = "NAME=VALUE", value_parser = parse_cookie)] + pub cookies: Vec<(String, String)>, + /// Maximum site sections to sample. Each contributes a landing page and an + /// article, so this bounds how much of the publisher's taxonomy is covered. + #[arg(long, default_value_t = 8)] + pub max_sections: usize, + /// Maximum pages to load in total, including the requested page. + /// + /// Set to 1 to restore single-page behavior: no crawl, no section + /// discovery, and the audited path as the only page pattern. + #[arg(long, default_value_t = 17)] + pub max_pages: usize, + /// Device profiles to audit, comma-separated: `desktop`, `mobile`. + /// + /// Defaults to `desktop`. Publishers often serve different GAM ad units per + /// device, which a single-profile crawl cannot see — it would infer a + /// template correct for the profile it used and silently wrong elsewhere. + /// Passing both crawls each page twice and refuses to write an ad-unit path + /// for any slot where the profiles disagree. + #[arg(long, value_delimiter = ',', default_value = "desktop")] + pub profiles: Vec, + /// Pause in milliseconds between page loads during the crawl. + /// + /// A crawl issues a dozen navigations in a row. Firing them back to back is + /// discourteous to the origin, and request pacing is one of the signals bot + /// protection scores, so an unpaced crawl can trigger the challenge that + /// empties the rest of the run. + #[arg(long, default_value_t = 750)] + pub page_delay_ms: u64, + /// Browser and consent options shared with `ts audit generate`. + #[command(flatten)] + pub browser: GenerateBrowserOpts, } -fn write_audit_outputs(outputs: &AuditOutputs, plan: &AuditOutputPlan) -> CliResult> { - let selected_paths = [&plan.js_assets_path, &plan.config_path] - .into_iter() - .flatten() - .collect::>(); - for path in &selected_paths { - if let Some(parent) = path - .parent() - .filter(|parent| !parent.as_os_str().is_empty()) - { - fs::create_dir_all(parent).map_err(|error| { - report_error(format!( - "failed to create parent directory {}: {error}", - parent.display() - )) - })?; +impl AuditAdTemplatesGenerateArgs { + /// The crawl bounds these arguments describe. + pub(crate) fn budget(&self) -> generate::CrawlBudget { + generate::CrawlBudget { + max_sections: self.max_sections, + max_pages: self.max_pages, } } - let mut written_paths = Vec::new(); - if let Some(path) = &plan.js_assets_path { - fs::write(path, &outputs.js_assets_toml).map_err(|error| { - report_error(format!( - "failed to write JS asset audit {}: {error}", - path.display() - )) - })?; - written_paths.push(path.display().to_string()); - } - if let Some(path) = &plan.config_path { - fs::write(path, &outputs.draft_config_toml).map_err(|error| { - report_error(format!( - "failed to write draft config {}: {error}", - path.display() - )) - })?; - written_paths.push(path.display().to_string()); + /// The device profiles to audit, deduplicated in the order given. + /// + /// # Errors + /// + /// Returns an error when a name is not a known profile, or when none were + /// given. + pub(crate) fn profiles(&self) -> Result, String> { + let mut profiles: Vec = Vec::new(); + for raw in &self.profiles { + let profile = generate::DeviceProfile::parse(raw)?; + if !profiles.contains(&profile) { + profiles.push(profile); + } + } + if profiles.is_empty() { + return Err("--profiles needs at least one of: desktop, mobile".to_string()); + } + Ok(profiles) } - - Ok(written_paths) } -fn write_success_summary( - outputs: &AuditOutputs, - written: &[String], - wrote_config: bool, - out: &mut dyn Write, -) -> CliResult<()> { - let integrations = outputs - .artifact - .detected_integrations - .iter() - .map(|integration| integration.id.as_str()) - .collect::>(); - let draft_note = if wrote_config { - "\nDraft config: review before validation and push" - } else { - "" - }; - writeln!( - out, - "Audited {}\nTitle: {}\nJS assets: {}\nThird-party assets: {}\nDetected integrations: {}\nWrote: {}{}", - outputs.artifact.audited_url, - outputs - .artifact - .page_title - .as_deref() - .unwrap_or(""), - outputs.artifact.js_asset_count, - outputs.artifact.third_party_asset_count, - if integrations.is_empty() { - "none".to_string() - } else { - integrations.join(", ") - }, - if written.is_empty() { - "none".to_string() - } else { - written.join(", ") - }, - draft_note - ) - .map_err(|error| report_error(format!("failed to write command output: {error}"))) +/// Arguments for `ts audit ad-templates verify ...`. +#[derive(Debug, Args)] +pub(crate) struct AuditAdTemplatesVerifyArgs { + #[command(flatten)] + pub config: AppConfigArgs, + /// One or more page URLs to verify (http or https). + #[arg(required = true, value_parser = parse_http_url)] + pub urls: Vec, + /// Exit non-zero when a matched slot is missing or only partially confirmed. + #[arg(long)] + pub strict: bool, + /// Emit machine-readable JSON instead of human output. + #[arg(long)] + pub json: bool, + /// Perform a deterministic scroll pass after the initial settle. + #[arg(long)] + pub scroll: bool, + /// Accept evidence from a page that redirected to a different origin. + /// + /// Off by default: slots are matched on the post-redirect path, so an + /// off-origin page could otherwise satisfy `--strict`. Enable only for a + /// known redirect between your own properties (e.g. apex to `www`). + #[arg(long)] + pub allow_cross_origin_redirect: bool, + /// Cookie to send with each page request, as `name=value`. Repeatable. + /// Use to carry an existing session (e.g. a valid bot-protection clearance + /// cookie) so the origin serves the real page instead of a challenge. + #[arg(long = "cookie", value_name = "NAME=VALUE", value_parser = parse_cookie)] + pub cookies: Vec<(String, String)>, + #[command(flatten)] + pub browser: BrowserOpts, } -fn build_draft_config(target_url: &Url, artifact: &AuditArtifact) -> CliResult { - let host = target_url - .host_str() - .ok_or_else(|| report_error("audited URL is missing a host"))?; - let origin = target_url.origin().ascii_serialization(); - let mut draft = EXAMPLE_CONFIG.to_string(); - - draft = replace_key_in_section( - &draft, - "publisher", - "domain", - &format!("domain = \"{host}\""), - )?; - draft = replace_key_in_section( - &draft, - "publisher", - "cookie_domain", - &format!("cookie_domain = \".{host}\""), - )?; - draft = replace_key_in_section( - &draft, - "publisher", - "origin_url", - &format!("origin_url = \"{origin}\""), - )?; - - let detected = artifact - .detected_integrations - .iter() - .map(|integration| integration.id.as_str()) - .collect::>(); - - if detected.contains("gpt") { - draft = replace_key_in_section(&draft, "integrations.gpt", "enabled", "enabled = true")?; - } - if detected.contains("didomi") { - draft = replace_key_in_section(&draft, "integrations.didomi", "enabled", "enabled = true")?; - } - if detected.contains("datadome") { - draft = - replace_key_in_section(&draft, "integrations.datadome", "enabled", "enabled = true")?; - } - - let mut manual_review = Vec::new(); - if detected.contains("google_tag_manager") { - if let Some(gtm_id) = extract_gtm_container_id(artifact) { - draft = replace_key_in_section( - &draft, - "integrations.google_tag_manager", - "enabled", - "enabled = true", - )?; - draft = replace_key_in_section( - &draft, - "integrations.google_tag_manager", - "container_id", - &format!("container_id = \"{gtm_id}\""), - )?; - } else { - manual_review.push("google_tag_manager"); +/// Dispatches a `ts audit` invocation. +/// +/// `legacy_url` (if present) routes to artifact generation, while the `page` +/// subcommand routes to the generic read-only page audit. +/// +/// # Errors +/// +/// Returns a user-facing string when no URL or subcommand is provided, or when +/// the underlying command fails. +pub(crate) fn run_audit(args: &AuditArgs) -> Result { + match &args.command { + Some(AuditSubcommand::Page(page_args)) => { + page::run_page(page_args).map(|()| RunOutcome::Success) } - } - - for integration in detected { - if !matches!( - integration, - "gpt" | "didomi" | "datadome" | "google_tag_manager" - ) { - manual_review.push(integration); + Some(AuditSubcommand::AdTemplates(AuditAdTemplatesCommand::Generate(gen_args))) => { + gen_args.browser.validate()?; + let app_config_path = crate::app_config::resolve_app_config_file(&gen_args.config)?; + let raw_config = std::fs::read_to_string(&app_config_path).map_err(|error| { + format!("failed to read {}: {error}", app_config_path.display()) + })?; + let existing_creative = creative_config(&raw_config, &app_config_path)?; + let profiles = gen_args.profiles()?; + let collectors: Vec = profiles + .iter() + .map(|profile| { + generate::browser_collector::BrowserAuditCollector::with_profile(*profile) + .with_page_delay(std::time::Duration::from_millis(gen_args.page_delay_ms)) + .with_browser_options(&gen_args.browser) + .with_scroll(gen_args.scroll) + }) + .collect(); + let selected: Vec<(&str, &dyn generate::collector::AuditCollector)> = profiles + .iter() + .zip(collectors.iter()) + .map(|(profile, collector)| { + ( + profile.label(), + collector as &dyn generate::collector::AuditCollector, + ) + }) + .collect(); + let stdout = std::io::stdout(); + let mut out = stdout.lock(); + let stderr = std::io::stderr(); + let mut err = stderr.lock(); + generate::run_update_slots( + &generate::UpdateSlotsRequest { + url: gen_args.url.as_str(), + config_path: &app_config_path, + existing_creative: existing_creative.as_ref(), + page_patterns: &gen_args.page_patterns, + replace: gen_args.replace, + cookies: &gen_args.cookies, + dry_run: gen_args.dry_run, + scroll: gen_args.scroll, + budget: gen_args.budget(), + }, + &selected, + &mut out, + &mut err, + ) + .map(|()| RunOutcome::Success) } - } - - if !manual_review.is_empty() { - if !draft.ends_with('\n') { - draft.push('\n'); + Some(AuditSubcommand::AdTemplates(AuditAdTemplatesCommand::Verify(verify_args))) => { + ad_templates::run_verify(verify_args) } - draft.push_str("\n# Audit findings requiring manual review\n"); - for integration in manual_review { - draft.push_str(&format!( - "# - Detected {integration}; review the corresponding [integrations.{integration}] section before enabling it.\n" - )); + Some(AuditSubcommand::Generate(generate_args)) => { + generate_args.browser.validate()?; + let stdout = std::io::stdout(); + let mut out = stdout.lock(); + let collector = generate::browser_collector::BrowserAuditCollector::default() + .with_browser_options(&generate_args.browser); + generate::run_generate(generate_args, &collector, &mut out) + .map(|()| RunOutcome::Success) } + None => match args.legacy_url.as_ref() { + Some(url) => { + let generate_args = legacy_generate_args(args, url); + generate_args.browser.validate()?; + let stdout = std::io::stdout(); + let mut out = stdout.lock(); + let collector = generate::browser_collector::BrowserAuditCollector::default() + .with_browser_options(&generate_args.browser); + generate::run_generate(&generate_args, &collector, &mut out) + .map(|()| RunOutcome::Success) + } + None => Err( + "provide a URL or a subcommand (`generate`, `page`, `ad-templates`)".to_string(), + ), + }, } - - Ok(draft) } -fn replace_key_in_section( +/// Reads the config's `[creative_opportunities]` section, when it has one. +/// +/// An unrelated invalid setting elsewhere in the document must not hide the +/// section — the runtime rejects such a file, but the operator still has to be +/// able to update slots in it — so the document is read as plain TOML rather +/// than through [`Settings`](trusted_server_core::settings::Settings). +/// +/// A section that is present but unreadable is *not* treated as absent. +/// `CreativeOpportunitiesConfig` uses `deny_unknown_fields`, so one mistyped key +/// would otherwise leave the merge with nothing to merge into and replace the +/// operator's entire slot array. +/// +/// # Errors +/// +/// Returns a user-facing error when the document is malformed or the section is +/// present but cannot be deserialized. +fn creative_config( document: &str, - section: &str, - key: &str, - replacement_line: &str, -) -> CliResult { - let section_header = format!("[{section}]"); - let mut in_section = false; - let mut replaced = false; - let mut saw_section = false; - let mut lines = Vec::new(); - - for line in document.lines() { - let trimmed = line.trim(); - if trimmed.starts_with('[') && trimmed.ends_with(']') { - in_section = trimmed == section_header; - saw_section |= in_section; - } - - if in_section && !replaced && is_key_line(trimmed, key) { - lines.push(replacement_line.to_string()); - replaced = true; - } else { - lines.push(line.to_string()); - } - } - - if !saw_section { - return cli_error(format!( - "failed to update starter config because section `{section_header}` was not found" - )); - } - if !replaced { - return cli_error(format!( - "failed to update starter config because key `{key}` was not found in `{section_header}`" - )); - } - - let mut output = lines.join("\n"); - if document.ends_with('\n') { - output.push('\n'); + path: &std::path::Path, +) -> CliResult> { + // Plain `format!`, not `report_error`: the top-level `[ts]` printer already + // logs whatever is returned here, and this message embeds a multi-line + // `toml::de::Error`, so logging it here too would print the whole block + // twice. The guidance leads so the parse error can trail unbroken. + let value = toml::from_str::(document).map_err(|error| { + format!( + "failed to parse {} before generating slots; fix the TOML syntax and re-run:\n{error}", + path.display() + ) + })?; + let Some(section) = value.get("creative_opportunities").cloned() else { + return Ok(None); + }; + match section.try_into() { + Ok(config) => Ok(Some(config)), + Err(error) => cli_error(format!( + "failed to read the existing `[creative_opportunities]` section, so generating \ + slots would discard the configured ones: {error}. Fix the section (or delete it) \ + and re-run" + )), } - Ok(output) } -fn is_key_line(trimmed_line: &str, key: &str) -> bool { - trimmed_line - .strip_prefix(key) - .and_then(|remaining| remaining.trim_start().strip_prefix('=')) - .is_some() +fn legacy_generate_args(args: &AuditArgs, url: &url::Url) -> generate::GenerateArgs { + generate::GenerateArgs { + url: url.to_string(), + js_assets: args.legacy_generate.js_assets.clone(), + config: args.legacy_generate.config.clone(), + no_js_assets: args.legacy_generate.no_js_assets, + no_config: args.legacy_generate.no_config, + force: args.legacy_generate.force, + cookies: args.legacy_generate.cookies.clone(), + browser: GenerateBrowserOpts::from(&args.legacy_generate.browser), + } } #[cfg(test)] mod tests { - use std::cell::Cell; - - use tempfile::TempDir; - use super::*; - use crate::commands::audit::collector::{CollectedPage, CollectedRequest, CollectedScriptTag}; - - struct FakeCollector { - collected: CollectedPage, - calls: Cell, - } - - impl FakeCollector { - fn new(collected: CollectedPage) -> Self { - Self { - collected, - calls: Cell::new(0), - } - } - } - - impl AuditCollector for FakeCollector { - fn collect_page(&self, _target_url: &Url) -> CliResult { - self.calls.set(self.calls.get() + 1); - Ok(self.collected.clone()) - } - } - - fn collected_page() -> CollectedPage { - CollectedPage { - requested_url: "https://publisher.example/page".to_string(), - final_url: "https://publisher.example/page".to_string(), - page_title: Some("Example Publisher".to_string()), - html: r#"Example Publisher"#.to_string(), - script_tags: vec![ - CollectedScriptTag { - src: Some("https://www.googletagmanager.com/gtm.js?id=GTM-ABC123".to_string()), - inline_text: None, - }, - CollectedScriptTag { - src: Some("https://securepubads.g.doubleclick.net/tag/js/gpt.js".to_string()), - inline_text: None, - }, - ], - network_requests: vec![CollectedRequest { - url: "https://cdn.publisher.example/app.js".to_string(), - resource_type: Some("script".to_string()), - }], - warnings: Vec::new(), - } - } - - fn audit_args(url: &str) -> AuditArgs { - AuditArgs { - url: url.to_string(), - js_assets: None, - config: None, - no_js_assets: false, - no_config: false, - force: false, - } - } #[test] - fn parse_audit_url_accepts_http_and_https() { - assert!(parse_audit_url("http://publisher.example").is_ok()); - assert!(parse_audit_url("https://publisher.example").is_ok()); + fn parse_cookie_splits_on_first_equals() { + let (name, value) = parse_cookie("datadome=abc=def~ghi").expect("should parse cookie"); + assert_eq!(name, "datadome", "name should be the pre-`=` portion"); + assert_eq!( + value, "abc=def~ghi", + "value should keep later `=` characters" + ); } #[test] - fn parse_audit_url_rejects_non_http_schemes() { - for url in [ - "file:///etc/passwd", - "data:text/html,hello", - "chrome://version", - ] { - let error = parse_audit_url(url).expect_err("should reject non-http URL"); - assert!( - format!("{error:?}").contains("only supports http/https"), - "should explain scheme restriction" - ); - } + fn parse_cookie_allows_empty_value() { + let (name, value) = parse_cookie("session=").expect("should parse empty value"); + assert_eq!(name, "session"); + assert!(value.is_empty(), "empty value should be allowed"); } #[test] - fn resolve_output_plan_rejects_no_outputs() { - let mut args = audit_args("https://publisher.example"); - args.no_js_assets = true; - args.no_config = true; + fn invalid_setting_outside_the_section_still_yields_creative_config() { + let document = "unknown_runtime_key = true\n\ + [creative_opportunities]\ngam_network_id = \"123\"\n"; - let error = resolve_output_plan(&args).expect_err("should reject empty output set"); + let creative = creative_config(document, std::path::Path::new("trusted-server.toml")) + .expect("an unrelated invalid setting must not hide creative config") + .expect("the section is present"); - assert!( - format!("{error:?}").contains("nothing to do"), - "should explain no-output error" - ); + assert_eq!(creative.gam_network_id, "123"); } #[test] - fn resolve_output_plan_rejects_existing_files_without_force() { - let temp = TempDir::new().expect("should create temp dir"); - let path = temp.path().join("js-assets.toml"); - fs::write(&path, "existing").expect("should write existing file"); - let mut args = audit_args("https://publisher.example"); - args.js_assets = Some(path); - args.no_config = true; - - let error = resolve_output_plan(&args).expect_err("should reject overwrite"); + fn absent_section_reads_as_absent() { + let creative = creative_config( + "[auction]\nenabled = true\n", + std::path::Path::new("trusted-server.toml"), + ) + .expect("should read the document"); assert!( - format!("{error:?}").contains("refusing to overwrite"), - "should explain overwrite refusal" + creative.is_none(), + "a document with no `[creative_opportunities]` has no configured slots" ); } #[test] - fn resolve_output_plan_allows_existing_files_with_force() { - let temp = TempDir::new().expect("should create temp dir"); - let path = temp.path().join("js-assets.toml"); - fs::write(&path, "existing").expect("should write existing file"); - let mut args = audit_args("https://publisher.example"); - args.js_assets = Some(path.clone()); - args.no_config = true; - args.force = true; - - let plan = resolve_output_plan(&args).expect("should allow forced overwrite"); - - assert_eq!(plan.js_assets_path.as_deref(), Some(path.as_path())); - } - - #[test] - fn run_audit_writes_selected_outputs_and_summary() { - let temp = TempDir::new().expect("should create temp dir"); - let js_assets = temp.path().join("audit/js-assets.toml"); - let config = temp.path().join("audit/trusted-server.toml"); - let args = AuditArgs { - url: "https://publisher.example/page".to_string(), - js_assets: Some(js_assets.clone()), - config: Some(config.clone()), - no_js_assets: false, - no_config: false, - force: false, - }; - let collector = FakeCollector::new(collected_page()); - let mut out = Vec::new(); - - run_audit(&args, &collector, &mut out).expect("should run audit"); - - assert_eq!(collector.calls.get(), 1, "should collect page once"); - assert!(js_assets.exists(), "should write JS assets"); - assert!(config.exists(), "should write draft config"); - let summary = String::from_utf8(out).expect("summary should be UTF-8"); - assert!(summary.contains("Audited https://publisher.example/page")); - assert!(summary.contains("Detected integrations: google_tag_manager, gpt")); - assert!(summary.contains("Draft config: review before validation and push")); - } - - #[test] - fn run_audit_respects_no_config() { - let temp = TempDir::new().expect("should create temp dir"); - let js_assets = temp.path().join("js-assets.toml"); - let mut args = audit_args("https://publisher.example/page"); - args.js_assets = Some(js_assets.clone()); - args.no_config = true; - let collector = FakeCollector::new(collected_page()); - - run_audit(&args, &collector, &mut Vec::new()).expect("should run audit"); + fn malformed_document_is_rejected_before_creative_config_extraction() { + let error = creative_config( + "[creative_opportunities\ngam_network_id = \"123\"\n", + std::path::Path::new("/tmp/example/trusted-server.toml"), + ) + .expect_err("should reject malformed TOML"); - assert!(js_assets.exists(), "should write assets"); assert!( - !temp.path().join("trusted-server.toml").exists(), - "should not write config" + error.contains("failed to parse /tmp/example/trusted-server.toml"), + "error should name the config file it could not parse, got {error}" ); - } - - #[test] - fn run_audit_respects_no_js_assets() { - let temp = TempDir::new().expect("should create temp dir"); - let config = temp.path().join("trusted-server.toml"); - let mut args = audit_args("https://publisher.example/page"); - args.config = Some(config.clone()); - args.no_js_assets = true; - let collector = FakeCollector::new(collected_page()); - let mut out = Vec::new(); - - run_audit(&args, &collector, &mut out).expect("should run audit"); - - assert!(config.exists(), "should write config"); assert!( - !temp.path().join("js-assets.toml").exists(), - "should not write JS assets" + error.contains("fix the TOML syntax and re-run:\n"), + "the guidance should lead so the multi-line parse error trails it, got {error}" ); - let summary = String::from_utf8(out).expect("summary should be UTF-8"); - assert!(summary.contains("Draft config: review before validation and push")); } #[test] - fn run_audit_writes_collector_warnings_to_asset_artifact() { - let temp = TempDir::new().expect("should create temp dir"); - let js_assets = temp.path().join("js-assets.toml"); - let mut args = audit_args("https://publisher.example/page"); - args.js_assets = Some(js_assets.clone()); - args.no_config = true; - let mut collected = collected_page(); - collected.warnings.push( - "browser audit timed out while waiting for the page to settle; results may be partial" - .to_string(), - ); - let collector = FakeCollector::new(collected); + fn unreadable_section_is_refused_rather_than_read_as_absent() { + // `deny_unknown_fields` makes one mistyped key inside the section fail + // to deserialize. Reading that as "no slots configured" would let a + // merge replace the operator's entire slot array. + let document = "[creative_opportunities]\n\ + gam_network_id = \"123\"\n\ + gam_netwrok_id = \"123\"\n\ + [[creative_opportunities.slot]]\n\ + id = \"header\"\n\ + div_id = \"ad-header\"\n\ + page_patterns = [\"/\"]\n\ + formats = [{ width = 728, height = 90 }]\n"; + + let error = creative_config(document, std::path::Path::new("trusted-server.toml")) + .expect_err("should refuse an unreadable section"); - run_audit(&args, &collector, &mut Vec::new()).expect("should run audit"); - - let artifact = fs::read_to_string(js_assets).expect("should read artifact"); assert!( - artifact.contains("results may be partial"), - "should persist collector warning" + error.contains("would discard the configured ones"), + "error should say what merging would cost, got {error}" ); } #[test] - fn run_audit_conflict_prevents_collection() { - let temp = TempDir::new().expect("should create temp dir"); - let js_assets = temp.path().join("js-assets.toml"); - fs::write(&js_assets, "existing").expect("should write existing file"); - let mut args = audit_args("https://publisher.example/page"); - args.js_assets = Some(js_assets); - args.no_config = true; - let collector = FakeCollector::new(collected_page()); - - let error = run_audit(&args, &collector, &mut Vec::new()) - .expect_err("should reject existing output"); - - assert_eq!(collector.calls.get(), 0, "should not collect page"); + fn parse_cookie_rejects_missing_equals() { + let err = parse_cookie("datadome").expect_err("should reject missing `=`"); assert!( - format!("{error:?}").contains("refusing to overwrite"), - "should report overwrite conflict" + err.contains("NAME=VALUE"), + "error should show expected form" ); } #[test] - fn build_draft_config_uses_final_url_and_detected_integrations() { - let url = Url::parse("https://www.publisher.example:8443/path").expect("should parse URL"); - let artifact = AuditArtifact { - audited_url: url.to_string(), - page_title: Some("Example".to_string()), - js_asset_count: 2, - third_party_asset_count: 2, - detected_integrations: vec![ - DetectedIntegration { - id: "google_tag_manager".to_string(), - evidence: "GTM-ABC123".to_string(), - }, - DetectedIntegration { - id: "gpt".to_string(), - evidence: "https://securepubads.g.doubleclick.net/tag/js/gpt.js".to_string(), - }, - DetectedIntegration { - id: "prebid".to_string(), - evidence: "inline script matched `prebid`".to_string(), - }, - ], - assets: Vec::new(), - warnings: Vec::new(), - }; - - let draft = build_draft_config(&url, &artifact).expect("should build draft config"); - - assert!(draft.contains("domain = \"www.publisher.example\"")); - assert!(draft.contains("cookie_domain = \".www.publisher.example\"")); - assert!(draft.contains("origin_url = \"https://www.publisher.example:8443\"")); - assert!(draft.contains("[integrations.gpt]\nenabled = true")); - assert!(draft.contains("[integrations.google_tag_manager]\nenabled = true")); - assert!(draft.contains("container_id = \"GTM-ABC123\"")); - assert!(draft.contains("Detected prebid")); - toml::from_str::(&draft).expect("draft should parse as TOML"); + fn parse_cookie_rejects_empty_name() { + let err = parse_cookie("=value").expect_err("should reject empty name"); + assert!(err.contains("empty name"), "error should name the problem"); } #[test] - fn build_draft_config_does_not_enable_gtm_without_container_id() { - let url = Url::parse("https://publisher.example/path").expect("should parse URL"); - let artifact = AuditArtifact { - audited_url: url.to_string(), - page_title: None, - js_asset_count: 1, - third_party_asset_count: 1, - detected_integrations: vec![DetectedIntegration { - id: "google_tag_manager".to_string(), - evidence: "https://www.googletagmanager.com/gtm.js".to_string(), - }], - assets: Vec::new(), - warnings: Vec::new(), + fn legacy_url_builds_artifact_generation_args() { + let args = AuditArgs { + command: None, + legacy_url: Some( + url::Url::parse("https://www.example.com/").expect("should parse URL"), + ), + legacy_generate: LegacyGenerateArgs { + js_assets: Some("audit/assets.toml".into()), + config: Some("audit/config.toml".into()), + no_js_assets: false, + no_config: false, + force: true, + cookies: vec![("session".to_string(), "example".to_string())], + browser: LegacyBrowserOpts { + headful: true, + ..LegacyBrowserOpts::default() + }, + }, }; - let draft = build_draft_config(&url, &artifact).expect("should build draft config"); + let generate = legacy_generate_args( + &args, + args.legacy_url.as_ref().expect("should have legacy URL"), + ); - assert!(draft.contains("[integrations.google_tag_manager]\nenabled = false")); - assert!(draft.contains("Detected google_tag_manager")); + assert_eq!(generate.url, "https://www.example.com/"); + assert_eq!( + generate.js_assets.as_deref(), + Some(std::path::Path::new("audit/assets.toml")) + ); + assert_eq!( + generate.config.as_deref(), + Some(std::path::Path::new("audit/config.toml")) + ); + assert!(generate.force); + assert_eq!( + generate.cookies, + [("session".to_string(), "example".to_string())] + ); + assert!( + generate.browser.headful, + "browser flags passed to the legacy form should reach generation" + ); } } diff --git a/crates/trusted-server-cli/src/commands/audit/page.rs b/crates/trusted-server-cli/src/commands/audit/page.rs new file mode 100644 index 000000000..31cbf4b1b --- /dev/null +++ b/crates/trusted-server-cli/src/commands/audit/page.rs @@ -0,0 +1,157 @@ +//! Generic `ts audit page ` command: a read-only page summary. + +use std::io::{self, Write}; + +use clap::Args; + +use crate::ad_templates::output::escape_terminal_text; +use crate::commands::audit::browser::BrowserCollector; +use crate::commands::audit::collector::{ + AuditCollector, BrowserCollectRequest, BrowserOpts, CollectedPage, +}; + +/// Arguments for `ts audit page `. +#[derive(Debug, Args)] +pub(crate) struct PageAuditArgs { + /// The page URL to audit (http or https). + #[arg(value_parser = crate::commands::audit::parse_http_url)] + pub url: url::Url, + /// Perform a deterministic scroll pass after the initial settle. + #[arg(long)] + pub scroll: bool, + #[command(flatten)] + pub browser: BrowserOpts, +} + +/// Runs the generic page audit for the `page` subcommand. +/// +/// # Errors +/// +/// Returns a user-facing string when the browser cannot collect the page. +pub(crate) fn run_page(args: &PageAuditArgs) -> Result<(), String> { + args.browser.validate()?; + run_with_collector( + &BrowserCollector::from_opts(&args.browser), + &args.url, + args.scroll, + ) +} + +fn run_with_collector( + collector: &dyn AuditCollector, + url: &url::Url, + scroll: bool, +) -> Result<(), String> { + let page = collector.collect_page(BrowserCollectRequest { + url: url.clone(), + init_scripts: Vec::new(), + scroll, + collect_ad_evidence: false, + cookies: Vec::new(), + })?; + + let stdout = io::stdout(); + let mut out = stdout.lock(); + write_summary(&mut out, url, &page) +} + +fn write_summary(out: &mut dyn Write, url: &url::Url, page: &CollectedPage) -> Result<(), String> { + let to_err = |error: io::Error| format!("failed to write command output: {error}"); + writeln!(out, "url: {url}").map_err(to_err)?; + // The final URL, title, and collector warning messages are page-controlled, + // so escape control characters before they reach the operator's terminal. + writeln!( + out, + "final url: {}", + escape_terminal_text(page.final_url.as_str()) + ) + .map_err(to_err)?; + writeln!(out, "title: {}", escape_terminal_text(&page.title)).map_err(to_err)?; + writeln!(out, "scripts: {}", page.script_count).map_err(to_err)?; + writeln!(out, "resources: {}", page.resource_count).map_err(to_err)?; + for warning in &page.warnings { + writeln!( + out, + "warning [{}]: {}", + escape_terminal_text(&warning.code), + escape_terminal_text(&warning.message) + ) + .map_err(to_err)?; + } + Ok(()) +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::ad_templates::output::Warning; + + fn collected(final_url: &str, title: &str, warnings: Vec) -> CollectedPage { + CollectedPage { + final_url: url::Url::parse(final_url).expect("should parse fixture URL"), + title: title.to_string(), + script_count: 3, + resource_count: 42, + warnings, + ad_evidence: None, + } + } + + fn summary(page: &CollectedPage, requested: &str) -> String { + let url = url::Url::parse(requested).expect("should parse requested URL"); + let mut out = Vec::new(); + write_summary(&mut out, &url, page).expect("should write summary"); + String::from_utf8(out).expect("summary should be UTF-8") + } + + #[test] + fn summary_reports_the_requested_and_final_urls_with_counts() { + let page = collected( + "https://publisher.example/news/story", + "Example Publisher", + Vec::new(), + ); + + let out = summary(&page, "https://publisher.example/news"); + + assert!( + out.contains("url: https://publisher.example/news\n"), + "should echo the requested URL, got {out:?}" + ); + assert!( + out.contains("final url: https://publisher.example/news/story\n"), + "should report the post-redirect URL, got {out:?}" + ); + assert!(out.contains("scripts: 3"), "got {out:?}"); + assert!(out.contains("resources: 42"), "got {out:?}"); + } + + #[test] + fn page_controlled_text_is_escaped_before_it_reaches_the_terminal() { + // Title and warning text are page-controlled and can contain raw + // terminal controls. URL percent-encoding is asserted separately. + let page = collected( + "https://publisher.example/a%1B%5B2Jb", + "Example\u{1b}[2J", + vec![Warning { + code: "page_\u{1b}[31m".to_string(), + message: "message\u{1b}[0m".to_string(), + }], + ); + + let out = summary(&page, "https://publisher.example/"); + + assert!( + !out.contains('\u{1b}'), + "no escape sequence may reach the terminal, got {out:?}" + ); + assert!( + out.contains("final url: https://publisher.example/a%1B%5B2Jb\n"), + "the final URL should retain URL's percent encoding, got {out:?}" + ); + assert!( + out.contains("warning [page_"), + "warnings should still be reported, got {out:?}" + ); + } +} diff --git a/crates/trusted-server-cli/src/commands/config/ad_templates.rs b/crates/trusted-server-cli/src/commands/config/ad_templates.rs new file mode 100644 index 000000000..59ec2a174 --- /dev/null +++ b/crates/trusted-server-cli/src/commands/config/ad_templates.rs @@ -0,0 +1,823 @@ +use std::collections::BTreeSet; +use std::io::{self, Write}; + +use crate::ad_templates::expected::normalize_path_or_url; +use crate::ad_templates::output::escape_terminal_text; +use crate::app_config::{AppConfigArgs, load_settings}; +use clap::{ArgGroup, Args, Subcommand}; +use http::Method; +use trusted_server_core::auction::types::MediaType; +use trusted_server_core::creative_opportunities::{ + AdStackGateInput, AdStackGateName, CreativeOpportunityFormat, CreativeOpportunitySlot, + RuntimeAdStackExpected, evaluate_ad_stack_gate, match_slots, validate_page_pattern, +}; + +use crate::run::RunOutcome; + +enum CheckFailure { + Tool(String), + Assertion(String), +} + +#[derive(Debug, Subcommand)] +pub enum AdTemplatesCommand { + /// Validate ad-template config and summarize deploy-time implications. + Lint(AdTemplatesLintArgs), + /// Show creative opportunity slots matching a page path or URL. + Match(AdTemplatesMatchArgs), + /// Assert that a page path or URL matches the expected slot set. + Check(AdTemplatesCheckArgs), + /// Explain why a page path or URL would or would not run the ad stack. + Explain(AdTemplatesExplainArgs), +} + +#[derive(Debug, Args)] +pub struct AdTemplatesLintArgs { + #[command(flatten)] + pub config: AppConfigArgs, +} + +#[derive(Debug, Args)] +pub struct AdTemplatesMatchArgs { + #[command(flatten)] + pub config: AppConfigArgs, + /// Page path or full URL to evaluate. + pub path_or_url: String, + /// Include slot div, GAM path, formats, and providers. + #[arg(long)] + pub details: bool, +} + +#[derive(Debug, Args)] +#[command(group( + ArgGroup::new("expectation") + .required(true) + .args(["expected_slots", "expect_no_slots"]) +))] +pub struct AdTemplatesCheckArgs { + #[command(flatten)] + pub config: AppConfigArgs, + /// Page path or full URL to evaluate. + pub path_or_url: String, + /// Expected slot id. Repeat for multiple slots. + #[arg(long = "expected-slot", value_name = "ID")] + pub expected_slots: Vec, + /// Assert that no slots match the page path or URL. + #[arg(long)] + pub expect_no_slots: bool, + /// Allow additional matched slots beyond --expected-slot values. + #[arg(long, conflicts_with = "expect_no_slots")] + pub allow_extra_slots: bool, +} + +#[derive(Debug, Args)] +pub struct AdTemplatesExplainArgs { + #[command(flatten)] + pub config: AppConfigArgs, + /// Page path or full URL to evaluate. + pub path_or_url: String, + /// HTTP method to model. + #[arg(long, default_value = "GET", value_parser = parse_http_method)] + pub method: Method, + /// Model a non-navigation request. + #[arg(long)] + pub non_navigation: bool, + /// Model a prefetch request. + #[arg(long)] + pub prefetch: bool, + /// Model a known crawler user agent. + #[arg(long)] + pub bot: bool, + /// Model consent denying server-side auction. + #[arg(long)] + pub consent_denied: bool, +} + +fn parse_http_method(raw: &str) -> Result { + let normalized = raw.to_ascii_uppercase(); + Method::from_bytes(normalized.as_bytes()) + .map_err(|error| format!("invalid HTTP method `{raw}`: {error}")) +} + +/// Run an ad-template CLI command. +/// +/// # Errors +/// +/// Returns a user-facing string when config loading, matching, or assertion +/// checks fail. +pub fn run_ad_templates(args: &AdTemplatesCommand) -> Result { + let stdout = io::stdout(); + let mut out = stdout.lock(); + if let AdTemplatesCommand::Check(args) = args { + return match run_check_classified(args, &mut out) { + Ok(()) => Ok(RunOutcome::Success), + Err(CheckFailure::Tool(error)) => Err(error), + Err(CheckFailure::Assertion(message)) => { + let stderr = io::stderr(); + let mut err = stderr.lock(); + writeln!(err, "{message}").map_err(output_error)?; + Ok(RunOutcome::AssertionFailed) + } + }; + } + run_ad_templates_with_writer(args, &mut out).map(|()| RunOutcome::Success) +} + +fn run_ad_templates_with_writer( + args: &AdTemplatesCommand, + out: &mut dyn Write, +) -> Result<(), String> { + match args { + AdTemplatesCommand::Lint(args) => run_lint(args, out), + AdTemplatesCommand::Match(args) => run_match(args, out), + AdTemplatesCommand::Check(args) => run_check(args, out), + AdTemplatesCommand::Explain(args) => run_explain(args, out), + } +} + +fn run_lint(args: &AdTemplatesLintArgs, out: &mut dyn Write) -> Result<(), String> { + let loaded = load_settings(&args.config)?; + writeln!(out, "app config: {}", loaded.app_config_path.display()).map_err(output_error)?; + + let Some(config) = &loaded.settings.creative_opportunities else { + writeln!(out, "server-side ad templates: not configured").map_err(output_error)?; + return Ok(()); + }; + + writeln!( + out, + "server-side ad templates: configured ({} slot{})", + config.slot.len(), + plural(config.slot.len()) + ) + .map_err(output_error)?; + writeln!( + out, + "gam_network_id: {}", + escape_terminal_text(&config.gam_network_id) + ) + .map_err(output_error)?; + writeln!( + out, + "auction_timeout_ms: {}", + config + .auction_timeout_ms + .unwrap_or(loaded.settings.auction.timeout_ms) + ) + .map_err(output_error)?; + writeln!( + out, + "creative_opportunities.enabled: {}", + if config.enabled { "true" } else { "false" } + ) + .map_err(output_error)?; + writeln!( + out, + "auction.enabled: {}", + if loaded.settings.auction.enabled { + "true" + } else { + "false" + } + ) + .map_err(output_error)?; + writeln!( + out, + "auction.providers: {}", + if loaded.settings.auction.providers.is_empty() { + "(none)".to_string() + } else { + escape_terminal_text( + &loaded + .settings + .auction + .providers + .keys() + .map(ToString::to_string) + .collect::>() + .join(", "), + ) + .into_owned() + } + ) + .map_err(output_error)?; + + if config.slot.is_empty() { + writeln!(out, "status: disabled because no slots are configured").map_err(output_error)?; + } else if !config.enabled { + writeln!( + out, + "status: slots are configured, but [creative_opportunities].enabled is false" + ) + .map_err(output_error)?; + } else if !loaded.settings.auction.enabled { + writeln!( + out, + "status: slots are configured, but [auction].enabled is false" + ) + .map_err(output_error)?; + } else if loaded.settings.auction.providers.is_empty() { + writeln!( + out, + "status: slots are configured, but [auction].providers is empty" + ) + .map_err(output_error)?; + } else { + writeln!(out, "status: eligible for legacy-path server-side auctions") + .map_err(output_error)?; + } + + for slot in &config.slot { + for pattern in &slot.page_patterns { + if let Err(error) = validate_page_pattern(pattern) { + writeln!( + out, + "invalid page pattern for slot `{}`: {}", + escape_terminal_text(&slot.id), + escape_terminal_text(&error), + ) + .map_err(output_error)?; + } + } + } + + Ok(()) +} + +fn run_match(args: &AdTemplatesMatchArgs, out: &mut dyn Write) -> Result<(), String> { + let loaded = load_settings(&args.config)?; + let path = normalize_path_or_url(&args.path_or_url)?; + let Some(config) = &loaded.settings.creative_opportunities else { + writeln!( + out, + "{path}: no slots matched (creative_opportunities not configured)" + ) + .map_err(output_error)?; + return Ok(()); + }; + let matched = match_slots(&config.slot, &path); + + write_match_result( + out, + &path, + &matched, + &config.gam_network_id, + &config.section_for_path(&path), + args.details, + ) +} + +fn run_check(args: &AdTemplatesCheckArgs, out: &mut dyn Write) -> Result<(), String> { + run_check_classified(args, out).map_err(|failure| match failure { + CheckFailure::Tool(error) | CheckFailure::Assertion(error) => error, + }) +} + +fn run_check_classified( + args: &AdTemplatesCheckArgs, + out: &mut dyn Write, +) -> Result<(), CheckFailure> { + let loaded = load_settings(&args.config).map_err(CheckFailure::Tool)?; + let path = normalize_path_or_url(&args.path_or_url).map_err(CheckFailure::Tool)?; + let matched = loaded + .settings + .creative_opportunities + .as_ref() + .map(|config| match_slots(&config.slot, &path)) + .unwrap_or_default(); + let actual: BTreeSet<&str> = matched.iter().map(|slot| slot.id.as_str()).collect(); + + if args.expect_no_slots { + if actual.is_empty() { + writeln!(out, "{path}: OK, no slots matched") + .map_err(output_error) + .map_err(CheckFailure::Tool)?; + return Ok(()); + } + return Err(CheckFailure::Assertion(format!( + "{path}: expected no slots, matched {}", + join_set(&actual) + ))); + } + + let expected: BTreeSet<&str> = args.expected_slots.iter().map(String::as_str).collect(); + let missing: BTreeSet<&str> = expected.difference(&actual).copied().collect(); + let extra: BTreeSet<&str> = actual.difference(&expected).copied().collect(); + + if missing.is_empty() && (args.allow_extra_slots || extra.is_empty()) { + writeln!(out, "{path}: OK, matched {}", join_set(&actual)) + .map_err(output_error) + .map_err(CheckFailure::Tool)?; + return Ok(()); + } + + let mut problems = Vec::new(); + if !missing.is_empty() { + problems.push(format!("missing {}", join_set(&missing))); + } + if !args.allow_extra_slots && !extra.is_empty() { + problems.push(format!("unexpected {}", join_set(&extra))); + } + Err(CheckFailure::Assertion(format!( + "{path}: {}", + problems.join("; ") + ))) +} + +fn run_explain(args: &AdTemplatesExplainArgs, out: &mut dyn Write) -> Result<(), String> { + let loaded = load_settings(&args.config)?; + let path = normalize_path_or_url(&args.path_or_url)?; + writeln!(out, "path: {path}").map_err(output_error)?; + + let has_matches = if let Some(config) = &loaded.settings.creative_opportunities { + let matched = match_slots(&config.slot, &path); + write_match_result( + out, + &path, + &matched, + &config.gam_network_id, + &config.section_for_path(&path), + true, + )?; + !matched.is_empty() + } else { + writeln!(out, "creative_opportunities: not configured").map_err(output_error)?; + false + }; + + let method_pass = args.method == Method::GET; + let navigation_pass = !args.non_navigation; + let consent_pass = !args.consent_denied; + let auction_enabled = loaded.settings.auction.enabled; + let ad_templates_enabled = loaded + .settings + .creative_opportunities + .as_ref() + .is_some_and(|config| config.enabled); + let providers_configured = !loaded.settings.auction.providers.is_empty(); + + let gate = evaluate_ad_stack_gate(AdStackGateInput { + method_get: method_pass, + navigation: navigation_pass, + prefetch: args.prefetch, + bot: args.bot, + matched_slots: has_matches, + consent_allows_auction: Some(consent_pass), + auction_enabled, + ad_templates_enabled, + }); + let blocked: Vec = gate.blocking_gates().collect(); + write_gate( + out, + "method GET", + !blocked.contains(&AdStackGateName::MethodGet), + )?; + write_gate( + out, + "navigation", + !blocked.contains(&AdStackGateName::Navigation), + )?; + write_gate( + out, + "not prefetch", + !blocked.contains(&AdStackGateName::NotPrefetch), + )?; + write_gate(out, "not bot", !blocked.contains(&AdStackGateName::NotBot))?; + write_gate( + out, + "consent allows auction", + !blocked.contains(&AdStackGateName::ConsentAllowsAuction), + )?; + write_gate( + out, + "auction.enabled", + !blocked.contains(&AdStackGateName::AuctionEnabled), + )?; + write_gate( + out, + "creative_opportunities.enabled", + !blocked.contains(&AdStackGateName::AdTemplatesEnabled), + )?; + write_gate( + out, + "matched slots", + !blocked.contains(&AdStackGateName::MatchedSlots), + )?; + writeln!( + out, + "advisory auction providers configured: {}", + if providers_configured { "yes" } else { "no" } + ) + .map_err(output_error)?; + writeln!( + out, + "server-side ad stack: {}", + match gate.expected { + RuntimeAdStackExpected::Yes => "yes", + RuntimeAdStackExpected::No => "no", + // `explain` always supplies a consent decision, which is the only + // input that yields `Unknown`; the arm is here for exhaustiveness. + RuntimeAdStackExpected::Unknown => "unknown", + } + ) + .map_err(output_error)?; + + Ok(()) +} + +fn write_match_result( + out: &mut dyn Write, + path: &str, + matched: &[&CreativeOpportunitySlot], + gam_network_id: &str, + section: &str, + details: bool, +) -> Result<(), String> { + if matched.is_empty() { + writeln!(out, "{}: no slots matched", escape_terminal_text(path)).map_err(output_error)?; + return Ok(()); + } + + let ids = matched + .iter() + .map(|slot| escape_terminal_text(&slot.id).into_owned()) + .collect::>() + .join(", "); + writeln!(out, "{}: matched {ids}", escape_terminal_text(path)).map_err(output_error)?; + + if details { + for slot in matched { + writeln!(out, "- {}", format_slot(slot, gam_network_id, section)) + .map_err(output_error)?; + } + } + + Ok(()) +} + +fn write_gate(out: &mut dyn Write, label: &str, pass: bool) -> Result<(), String> { + writeln!(out, "gate {label}: {}", if pass { "pass" } else { "block" }).map_err(output_error) +} + +/// Formats one matched slot for `--details` output. +/// +/// `section` is the value the runtime derives from the evaluated path, so a +/// `{section}` template renders the same unit path the live request would use. +fn format_slot(slot: &CreativeOpportunitySlot, gam_network_id: &str, section: &str) -> String { + let formats = slot + .formats + .iter() + .map(format_format) + .collect::>() + .join(", "); + let providers = format_providers(slot); + // `None` means a dynamic template renders past GAM's unit-path byte limit — + // a config the runtime rejects, so surface it rather than printing a path. + let gam_unit_path = slot + .render_gam_unit_path(gam_network_id, section) + .unwrap_or_else(|| "".to_string()); + format!( + "{} div={} gam={} patterns=[{}] formats=[{}] providers=[{}]", + escape_terminal_text(&slot.id), + escape_terminal_text(slot.resolved_div_id()), + escape_terminal_text(&gam_unit_path), + escape_terminal_text(&slot.page_patterns.join(", ")), + formats, + providers, + ) +} + +fn format_format(format: &CreativeOpportunityFormat) -> String { + let media_type = match format.media_type { + MediaType::Banner => "banner", + MediaType::Video => "video", + MediaType::Native => "native", + }; + format!("{}x{} {media_type}", format.width, format.height) +} + +fn format_providers(slot: &CreativeOpportunitySlot) -> String { + let mut providers = Vec::new(); + if slot.providers.aps.is_some() { + providers.push("aps"); + } + if slot.providers.prebid.is_some() { + providers.push("prebid"); + } + if providers.is_empty() { + return "none".to_string(); + } + providers.join(", ") +} + +/// Renders a set of config-derived slot ids for the terminal. +/// +/// Config can arrive from a pushed blob or the env overlay, not only from a file +/// the operator read, so the ids are escaped before they reach a terminal — the +/// assertion-failure path prints them too. +fn join_set(set: &BTreeSet<&str>) -> String { + if set.is_empty() { + return "(none)".to_string(); + } + set.iter() + .map(|id| escape_terminal_text(id).into_owned()) + .collect::>() + .join(", ") +} + +fn plural(count: usize) -> &'static str { + if count == 1 { "" } else { "s" } +} + +#[allow( + clippy::needless_pass_by_value, + reason = "used as a map_err fn that receives io::Error by value" +)] +fn output_error(err: io::Error) -> String { + format!("failed to write command output: {err}") +} + +#[cfg(test)] +mod tests { + use std::fs; + + use tempfile::TempDir; + + use super::*; + + const EXAMPLE_CONFIG: &str = include_str!(concat!( + env!("CARGO_MANIFEST_DIR"), + "/../../trusted-server.example.toml" + )); + + fn project_with_config(config: &str) -> (TempDir, AppConfigArgs) { + let temp = TempDir::new().expect("should create temp dir"); + let manifest_path = temp.path().join("edgezero.toml"); + let config_path = temp.path().join("trusted-server.toml"); + fs::write(&manifest_path, "[app]\nname = \"trusted-server\"\n") + .expect("should write manifest"); + fs::write(&config_path, config).expect("should write app config"); + ( + temp, + AppConfigArgs { + app_config: Some(config_path), + manifest: manifest_path, + no_env: true, + }, + ) + } + + fn config_with_slots() -> String { + let base_config = EXAMPLE_CONFIG + .replace("handler_password", "test-admin-password-32-bytes-minimum") + .replace("ec_passphrase", "test-ec-passphrase-32-bytes-minimum") + .replace( + "publisher_proxy_secret", + "test-proxy-secret-32-bytes-minimum", + ); + format!( + "{base_config}\n\ + [[creative_opportunities.slot]]\n\ + id = \"atf\"\n\ + page_patterns = [\"/news/*\", \"/\"]\n\ + formats = [{{ width = 300, height = 250 }}]\n\ + targeting = {{ zone = \"atf\" }}\n\ + [creative_opportunities.slot.providers.prebid]\n\ + bidders = {{}}\n\ + \n\ + [[creative_opportunities.slot]]\n\ + id = \"sports-sidebar\"\n\ + div_id = \"sports-ad\"\n\ + page_patterns = [\"/sports/*\"]\n\ + formats = [{{ width = 300, height = 600 }}]\n" + ) + } + + #[test] + fn match_reports_slots_for_path() { + let (_temp, config) = project_with_config(&config_with_slots()); + let mut out = Vec::new(); + + run_ad_templates_with_writer( + &AdTemplatesCommand::Match(AdTemplatesMatchArgs { + config, + path_or_url: "https://example.com/news/story?utm=1".to_string(), + details: true, + }), + &mut out, + ) + .expect("should match slots"); + + let output = String::from_utf8(out).expect("should be utf8"); + assert!( + output.contains("/news/story: matched atf"), + "should report matched slot" + ); + assert!( + output.contains("formats=[300x250 banner]"), + "should include details" + ); + } + + #[test] + fn check_rejects_unexpected_extra_slots_by_default() { + let (_temp, config) = project_with_config(&config_with_slots()); + + let err = run_ad_templates_with_writer( + &AdTemplatesCommand::Check(AdTemplatesCheckArgs { + config, + path_or_url: "/sports/game".to_string(), + expected_slots: vec!["atf".to_string()], + expect_no_slots: false, + allow_extra_slots: false, + }), + &mut Vec::new(), + ) + .expect_err("should reject mismatch"); + + assert!( + err.contains("missing atf") && err.contains("unexpected sports-sidebar"), + "should describe missing and unexpected slots" + ); + } + + #[test] + fn check_accepts_no_slots() { + let (_temp, config) = project_with_config(&config_with_slots()); + let mut out = Vec::new(); + + run_ad_templates_with_writer( + &AdTemplatesCommand::Check(AdTemplatesCheckArgs { + config, + path_or_url: "/weather/today".to_string(), + expected_slots: Vec::new(), + expect_no_slots: true, + allow_extra_slots: false, + }), + &mut out, + ) + .expect("should accept no slots"); + + let output = String::from_utf8(out).expect("should be utf8"); + assert!( + output.contains("/weather/today: OK, no slots matched"), + "should report no-slot assertion" + ); + } + + #[test] + fn explain_keeps_provider_state_separate_from_runtime_verdict() { + let (_temp, config) = project_with_config(&config_with_slots()); + let mut out = Vec::new(); + + run_ad_templates_with_writer( + &AdTemplatesCommand::Explain(AdTemplatesExplainArgs { + config, + path_or_url: "/news/story".to_string(), + method: Method::GET, + non_navigation: false, + prefetch: false, + bot: false, + consent_denied: true, + }), + &mut out, + ) + .expect("should explain path"); + + let output = String::from_utf8(out).expect("should be utf8"); + assert!( + output.contains("server-side ad stack: no"), + "runtime verdict should not include provider configuration" + ); + assert!( + output.contains("advisory auction providers configured: yes"), + "provider state should be a separate advisory" + ); + } + + #[test] + fn lint_reports_configured_slot_count_and_auction_state() { + let (_temp, config) = project_with_config(&config_with_slots()); + let mut out = Vec::new(); + + run_ad_templates_with_writer( + &AdTemplatesCommand::Lint(AdTemplatesLintArgs { config }), + &mut out, + ) + .expect("should lint configured slots"); + + let output = String::from_utf8(out).expect("should be utf8"); + assert!( + output.contains("server-side ad templates: configured (2 slots)"), + "should report the configured slot count" + ); + assert!( + output.contains("auction.enabled:"), + "should report the auction kill-switch state" + ); + assert!(!output.contains("legacy fallback")); + } + + #[test] + fn lint_and_explain_report_the_disabled_template_switch() { + // `[creative_opportunities].enabled = false` is a runtime kill switch: + // the publisher path matches no slots at all while it is off, so the + // diagnostics must not claim the ad stack would run. + let config_text = config_with_slots().replace("enabled = true", "enabled = false"); + let (_temp, config) = project_with_config(&config_text); + let mut out = Vec::new(); + + run_ad_templates_with_writer( + &AdTemplatesCommand::Lint(AdTemplatesLintArgs { + config: config.clone(), + }), + &mut out, + ) + .expect("should lint a disabled template switch"); + let lint_output = String::from_utf8(out).expect("should be utf8"); + + assert!( + lint_output.contains("creative_opportunities.enabled: false"), + "lint should report the template switch state: {lint_output}" + ); + assert!( + lint_output.contains( + "status: slots are configured, but [creative_opportunities].enabled is false" + ), + "lint status should name the template switch: {lint_output}" + ); + + let mut out = Vec::new(); + run_ad_templates_with_writer( + &AdTemplatesCommand::Explain(AdTemplatesExplainArgs { + config, + path_or_url: "/news/story".to_string(), + method: Method::GET, + non_navigation: false, + prefetch: false, + bot: false, + consent_denied: false, + }), + &mut out, + ) + .expect("should explain a disabled template switch"); + let explain_output = String::from_utf8(out).expect("should be utf8"); + + assert!( + explain_output.contains("gate creative_opportunities.enabled: block"), + "explain should fail the template-switch gate: {explain_output}" + ); + assert!( + explain_output.contains("server-side ad stack: no"), + "explain verdict should follow the switch: {explain_output}" + ); + } + + #[test] + fn lint_reports_page_patterns_the_runtime_drops() { + let config_text = config_with_slots().replace( + "page_patterns = [\"/news/*\", \"/\"]", + "page_patterns = [\"/news/*\", \"[\"]", + ); + let (_temp, config) = project_with_config(&config_text); + let mut out = Vec::new(); + + run_ad_templates_with_writer( + &AdTemplatesCommand::Lint(AdTemplatesLintArgs { config }), + &mut out, + ) + .expect("should lint mixed valid and invalid patterns"); + let output = String::from_utf8(out).expect("should be utf8"); + + assert!( + output.contains("invalid page pattern for slot `atf`") + && output.contains("page pattern '[' is not a valid glob"), + "lint should surface the runtime-dropped pattern: {output}" + ); + } + + #[test] + fn public_check_reports_drift_as_assertion_outcome() { + let (_temp, config) = project_with_config(&config_with_slots()); + + let outcome = run_ad_templates(&AdTemplatesCommand::Check(AdTemplatesCheckArgs { + config, + path_or_url: "/sports/game".to_string(), + expected_slots: vec!["atf".to_string()], + expect_no_slots: false, + allow_extra_slots: false, + })) + .expect("assertion drift should not be a tool error"); + + assert_eq!(outcome, RunOutcome::AssertionFailed); + } + + #[test] + fn http_method_parser_normalizes_standard_methods() { + assert_eq!( + parse_http_method("get").expect("should parse lowercase GET"), + Method::GET, + "lowercase GET must evaluate the same runtime gate as uppercase GET" + ); + } +} diff --git a/crates/trusted-server-cli/src/commands/config/mod.rs b/crates/trusted-server-cli/src/commands/config/mod.rs index 43763f10a..77af7de85 100644 --- a/crates/trusted-server-cli/src/commands/config/mod.rs +++ b/crates/trusted-server-cli/src/commands/config/mod.rs @@ -1 +1,2 @@ +pub mod ad_templates; pub mod init; diff --git a/crates/trusted-server-cli/src/lib.rs b/crates/trusted-server-cli/src/lib.rs index 405bc6187..a3352ba93 100644 --- a/crates/trusted-server-cli/src/lib.rs +++ b/crates/trusted-server-cli/src/lib.rs @@ -1,4 +1,8 @@ #[cfg(not(target_arch = "wasm32"))] +mod ad_templates; +#[cfg(not(target_arch = "wasm32"))] +mod app_config; +#[cfg(not(target_arch = "wasm32"))] mod error; #[cfg(not(target_arch = "wasm32"))] mod prebid_bundle; @@ -6,7 +10,7 @@ mod prebid_bundle; mod run; #[cfg(not(target_arch = "wasm32"))] -pub use run::run_from_env; +pub use run::{RunOutcome, run_from_env}; // Every `ts` subcommand's implementation lives under `commands/`. The // `ts dev` group is available on every host target; its only subcommand, diff --git a/crates/trusted-server-cli/src/main.rs b/crates/trusted-server-cli/src/main.rs index 7cee5b1ca..0a325bd55 100644 --- a/crates/trusted-server-cli/src/main.rs +++ b/crates/trusted-server-cli/src/main.rs @@ -2,10 +2,21 @@ fn main() { use std::process; + // Dependencies such as chromiumoxide instrument their internals with + // `tracing`. Without a subscriber, tracing's log-compatibility fallback + // forwards tolerated CDP decode warnings into the CLI's user-facing logger. + // Trusted Server uses `log` for intentional operator output, so install a + // no-op tracing subscriber to keep dependency diagnostics out of stdout and + // stderr without changing the process-wide `log` level. + let _ = tracing::subscriber::set_global_default(tracing::subscriber::NoSubscriber::default()); edgezero_cli::init_cli_logger(); - if let Err(err) = trusted_server_cli::run_from_env() { - log::error!("[ts] {err}"); - process::exit(2); + match trusted_server_cli::run_from_env() { + Ok(outcome) if outcome.exit_code() != 0 => process::exit(outcome.exit_code()), + Ok(_) => {} + Err(err) => { + log::error!("[ts] {err}"); + process::exit(2); + } } } diff --git a/crates/trusted-server-cli/src/prebid_bundle.rs b/crates/trusted-server-cli/src/prebid_bundle.rs index 802d854ce..abc545926 100644 --- a/crates/trusted-server-cli/src/prebid_bundle.rs +++ b/crates/trusted-server-cli/src/prebid_bundle.rs @@ -559,7 +559,6 @@ mod tests { r#" [integrations.prebid] enabled = true -server_url = "https://prebid.example.com/openrtb2/auction" external_bundle_url = "https://assets.example.com/prebid/trusted-prebid-old.js" [integrations.prebid.bundle] @@ -595,7 +594,6 @@ user_id_modules = ["sharedIdSystem", "uid2IdSystem"] r#" [integrations.prebid] enabled = true -server_url = "https://prebid.example.com/openrtb2/auction" [integrations.prebid.bundle] adapters = ["rubicon"] @@ -626,7 +624,6 @@ adapters = ["rubicon"] r#" [integrations.prebid] enabled = true -server_url = "https://prebid.example.com/openrtb2/auction" "#, ); @@ -646,7 +643,6 @@ server_url = "https://prebid.example.com/openrtb2/auction" r#" [integrations.prebid] enabled = true -server_url = "https://prebid.example.com/openrtb2/auction" [integrations.prebid.bundle] adapters = [] @@ -667,7 +663,6 @@ adapters = [] r#" [integrations.prebid] enabled = true -server_url = "https://prebid.example.com/openrtb2/auction" [integrations.prebid.bundle] adapters = ["rubicon", 123] diff --git a/crates/trusted-server-cli/src/run.rs b/crates/trusted-server-cli/src/run.rs index 1b0bdfa29..4d73a2ab5 100644 --- a/crates/trusted-server-cli/src/run.rs +++ b/crates/trusted-server-cli/src/run.rs @@ -2,18 +2,18 @@ use std::process; use clap::{Parser, Subcommand}; use edgezero_cli::args::{ - AuthArgs, BuildArgs, ConfigDiffArgs, ConfigPushArgs, ConfigValidateArgs, DeployArgs, - ProvisionArgs, ServeArgs, + ActiveVersionArgs, AuthArgs, BuildArgs, ConfigDiffArgs, ConfigGcArgs, ConfigPushArgs, + ConfigValidateArgs, DeployArgs, HealthcheckArgs, ProvisionArgs, RollbackArgs, ServeArgs, }; use trusted_server_core::config::TrustedServerAppConfig; -use crate::commands::audit::AuditArgs; -use crate::commands::audit::browser_collector::BrowserAuditCollector; +use crate::commands::audit::{AuditArgs, run_audit}; +use crate::commands::config::ad_templates::{AdTemplatesCommand, run_ad_templates}; use crate::commands::config::init::{ConfigInitArgs, run_config_init}; use crate::prebid_bundle::{NpmPrebidBundleGenerator, PrebidBundleArgs, run_bundle}; #[derive(Debug, Parser)] -#[command(name = "ts", about = "Trusted Server CLI")] +#[command(name = "ts", version, about = "Trusted Server CLI")] struct Args { #[command(subcommand)] command: Command, @@ -21,8 +21,10 @@ struct Args { #[derive(Debug, Subcommand)] enum Command { - /// Audit a public page and write draft Trusted Server artifacts. - Audit(AuditArgs), + /// Print the currently active deployment version for a target adapter. + ActiveVersion(ActiveVersionArgs), + /// Browser-backed page and ad-template audits. + Audit(Box), /// Sign in / out / status against an `EdgeZero` adapter. Auth(AuthArgs), /// Build the project for a target adapter. @@ -32,10 +34,14 @@ enum Command { Config(ConfigCommand), /// Deploy the project through a target adapter. Deploy(DeployArgs), + /// Probe a deployed version until it reports healthy. + Healthcheck(HealthcheckArgs), /// Trusted Server Prebid commands. Prebid(PrebidArgs), /// Provision platform resources through a target adapter. Provision(ProvisionArgs), + /// Roll a service back to a previously active deployment version. + Rollback(RollbackArgs), /// Serve the project locally through a target adapter. Serve(ServeArgs), /// Local developer tools (e.g. the macOS-only production-hostname proxy). @@ -45,10 +51,15 @@ enum Command { #[derive(Debug, Subcommand)] enum ConfigCommand { + /// Diagnose server-side ad-template configuration and path matching. + #[command(name = "ad-templates", subcommand)] + AdTemplates(AdTemplatesCommand), /// Initialize a Trusted Server config file from the example template. Init(ConfigInitArgs), /// Diff `trusted-server.toml` against the live `EdgeZero` config. Diff(ConfigDiffArgs), + /// Reclaim orphaned chunk entries leaked from prior oversized pushes. + Gc(ConfigGcArgs), /// Push `trusted-server.toml` as a blob envelope through `EdgeZero`. Push(ConfigPushArgs), /// Validate `edgezero.toml` and the typed Trusted Server config. @@ -67,41 +78,71 @@ enum PrebidCommand { Bundle(PrebidBundleArgs), } +/// Process-level outcome for commands that distinguish drift from tool errors. +#[derive(Debug, Clone, Copy, Eq, PartialEq)] +pub enum RunOutcome { + /// Command completed without drift. + Success, + /// Command ran successfully and found assertion drift. + AssertionFailed, +} + +impl RunOutcome { + /// Stable process exit code for this outcome. + #[must_use] + pub const fn exit_code(self) -> i32 { + match self { + Self::Success => 0, + Self::AssertionFailed => 1, + } + } +} + /// Run the CLI using process arguments. /// /// # Errors /// /// Returns an error when command parsing, config validation, `EdgeZero` /// delegation, audit collection, config initialization, or Prebid bundle generation fails. -pub fn run_from_env() -> Result<(), String> { +pub fn run_from_env() -> Result { dispatch(Args::parse()) } -fn dispatch(args: Args) -> Result<(), String> { +fn dispatch(args: Args) -> Result { match args.command { - Command::Audit(args) => { - let stdout = std::io::stdout(); - let mut out = stdout.lock(); - let collector = BrowserAuditCollector; - crate::commands::audit::run_audit(&args, &collector, &mut out) + Command::ActiveVersion(args) => { + edgezero_cli::run_active_version(&args).map(|()| RunOutcome::Success) + } + Command::Auth(args) => edgezero_cli::run_auth(&args).map(|()| RunOutcome::Success), + Command::Audit(args) => run_audit(&args), + Command::Build(args) => edgezero_cli::run_build(&args).map(|()| RunOutcome::Success), + Command::Config(ConfigCommand::AdTemplates(args)) => run_ad_templates(&args), + Command::Config(ConfigCommand::Init(args)) => { + run_config_init(&args).map(|()| RunOutcome::Success) } - Command::Auth(args) => edgezero_cli::run_auth(&args), - Command::Build(args) => edgezero_cli::run_build(&args), - Command::Config(ConfigCommand::Init(args)) => run_config_init(&args), Command::Config(ConfigCommand::Diff(args)) => { match edgezero_cli::run_config_diff_typed::(&args) { - Ok(edgezero_cli::DiffExit { code: 0 }) => Ok(()), + Ok(edgezero_cli::DiffExit { code: 0 }) => Ok(RunOutcome::Success), + Ok(edgezero_cli::DiffExit { code: 1 }) => Ok(RunOutcome::AssertionFailed), Ok(edgezero_cli::DiffExit { code }) => process::exit(code), Err(err) => Err(err), } } + Command::Config(ConfigCommand::Gc(args)) => { + edgezero_cli::run_config_gc(&args).map(|()| RunOutcome::Success) + } Command::Config(ConfigCommand::Push(args)) => { edgezero_cli::run_config_push_typed::(&args) + .map(|()| RunOutcome::Success) } Command::Config(ConfigCommand::Validate(args)) => { edgezero_cli::run_config_validate_typed::(&args) + .map(|()| RunOutcome::Success) + } + Command::Deploy(args) => edgezero_cli::run_deploy(&args).map(|()| RunOutcome::Success), + Command::Healthcheck(args) => { + edgezero_cli::run_healthcheck(&args).map(|()| RunOutcome::Success) } - Command::Deploy(args) => edgezero_cli::run_deploy(&args), Command::Prebid(prebid) => { let mut generator = NpmPrebidBundleGenerator; let mut stdout = std::io::stdout(); @@ -109,12 +150,16 @@ fn dispatch(args: Args) -> Result<(), String> { match prebid.command { PrebidCommand::Bundle(args) => { run_bundle(&args, &mut generator, &mut stdout, &mut stderr) + .map(|()| RunOutcome::Success) } } } - Command::Provision(args) => edgezero_cli::run_provision(&args), - Command::Serve(args) => edgezero_cli::run_serve(&args), - Command::Dev(command) => crate::commands::dev::run(command), + Command::Provision(args) => { + edgezero_cli::run_provision(&args).map(|()| RunOutcome::Success) + } + Command::Rollback(args) => edgezero_cli::run_rollback(&args).map(|()| RunOutcome::Success), + Command::Serve(args) => edgezero_cli::run_serve(&args).map(|()| RunOutcome::Success), + Command::Dev(command) => crate::commands::dev::run(command).map(|()| RunOutcome::Success), } } @@ -132,44 +177,210 @@ mod tests { } #[test] - fn parses_audit_with_default_outputs() { - let args = parse(&["ts", "audit", "https://publisher.example"]); - let Command::Audit(audit) = args.command else { - panic!("expected audit command"); + fn top_level_version_flag_is_available() { + let err = Args::try_parse_from(["ts", "--version"]) + .expect_err("should short-circuit parsing on --version"); + assert_eq!( + err.kind(), + clap::error::ErrorKind::DisplayVersion, + "should print the version rather than fail to parse" + ); + } + + #[test] + fn parses_active_version() { + let args = parse(&[ + "ts", + "active-version", + "--adapter", + "fastly", + "--service-id", + "service-123", + ]); + let Command::ActiveVersion(active_version) = args.command else { + panic!("expected active-version command"); }; - assert_eq!(audit.url, "https://publisher.example"); - assert_eq!(audit.js_assets, None); - assert_eq!(audit.config, None); - assert!(!audit.no_js_assets); - assert!(!audit.no_config); - assert!(!audit.force); + assert_eq!(active_version.adapter, "fastly"); + assert_eq!(active_version.service_id, "service-123"); } #[test] - fn parses_audit_with_custom_outputs() { + fn parses_healthcheck_with_retry_defaults() { let args = parse(&[ "ts", - "audit", - "https://publisher.example", - "--js-assets", - "audit/js-assets.toml", - "--config", - "audit/trusted-server.toml", - "--no-js-assets", - "--no-config", - "--force", + "healthcheck", + "--adapter", + "fastly", + "--service-id", + "service-123", + "--version", + "7", + "--domain", + "edge.example", ]); - let Command::Audit(audit) = args.command else { - panic!("expected audit command"); + let Command::Healthcheck(healthcheck) = args.command else { + panic!("expected healthcheck command"); + }; + assert_eq!(healthcheck.domain, "edge.example"); + assert_eq!(healthcheck.version, "7"); + assert_eq!(healthcheck.path, "/", "should default to probing `/`"); + assert_eq!( + healthcheck.retry, 3, + "should default to 3 total attempts, not 3 retries after a first try" + ); + assert_eq!( + healthcheck.retry_delay, 5, + "should default to a 5s retry delay" + ); + assert_eq!(healthcheck.timeout, 10, "should default to a 10s timeout"); + assert!(!healthcheck.staging, "should probe production by default"); + } + + #[test] + fn parses_healthcheck_with_staging_overrides() { + let args = parse(&[ + "ts", + "healthcheck", + "--adapter", + "fastly", + "--service-id", + "service-123", + "--version", + "7", + "--domain", + "edge.example", + "--staging", + "--retry", + "9", + "--retry-delay", + "2", + "--timeout", + "30", + ]); + let Command::Healthcheck(healthcheck) = args.command else { + panic!("expected healthcheck command"); + }; + assert!(healthcheck.staging); + assert_eq!(healthcheck.retry, 9); + assert_eq!(healthcheck.retry_delay, 2); + assert_eq!(healthcheck.timeout, 30); + } + + #[test] + fn healthcheck_requires_domain() { + Args::try_parse_from([ + "ts", + "healthcheck", + "--adapter", + "fastly", + "--service-id", + "service-123", + "--version", + "7", + ]) + .expect_err("should reject healthcheck without a domain"); + } + + #[test] + fn parses_rollback_with_explicit_target() { + let args = parse(&[ + "ts", + "rollback", + "--adapter", + "fastly", + "--service-id", + "service-123", + "--version", + "8", + "--rollback-to", + "7", + ]); + let Command::Rollback(rollback) = args.command else { + panic!("expected rollback command"); + }; + assert_eq!(rollback.version, "8"); + assert_eq!(rollback.rollback_to, Some("7".to_owned())); + assert!(!rollback.staging); + } + + #[test] + fn parses_staging_rollback_without_target() { + let args = parse(&[ + "ts", + "rollback", + "--adapter", + "fastly", + "--service-id", + "service-123", + "--version", + "8", + "--staging", + ]); + let Command::Rollback(rollback) = args.command else { + panic!("expected rollback command"); + }; + assert!(rollback.staging); + assert_eq!( + rollback.rollback_to, None, + "staging rollback should not need an explicit target" + ); + } + + #[test] + fn rollback_requires_service_id() { + Args::try_parse_from(["ts", "rollback", "--adapter", "fastly", "--version", "8"]) + .expect_err("should reject rollback without a service id"); + } + + #[test] + fn parses_deploy_with_staging_flags() { + let args = parse(&[ + "ts", + "deploy", + "--adapter", + "fastly", + "--service-id", + "service-123", + "--staging", + ]); + let Command::Deploy(deploy) = args.command else { + panic!("expected deploy command"); + }; + assert_eq!(deploy.service_id, Some("service-123".to_owned())); + assert!(deploy.staging); + } + + #[test] + fn deploy_rejects_renamed_stage_flag_before_separator() { + // `--stage` was renamed to `--staging`, and adapter passthrough is + // `last = true` (only captured after `--`). A stray `--stage` before the + // separator must fail closed at parse time rather than being swallowed as + // passthrough, which would leave `staging` false and route a + // staging-intended deploy to production. + Args::try_parse_from(["ts", "deploy", "--adapter", "fastly", "--stage"]) + .expect_err("should reject the renamed-away --stage flag, not route it to production"); + } + + #[test] + fn deploy_captures_adapter_passthrough_after_separator() { + let args = parse(&[ + "ts", + "deploy", + "--adapter", + "fastly", + "--", + "--comment", + "ci", + ]); + let Command::Deploy(deploy) = args.command else { + panic!("expected deploy command"); }; - assert_eq!(audit.js_assets, Some(PathBuf::from("audit/js-assets.toml"))); + assert!(!deploy.staging, "should default to a production deploy"); assert_eq!( - audit.config, - Some(PathBuf::from("audit/trusted-server.toml")) + deploy.adapter_args, + vec!["--comment", "ci"], + "should capture args after -- as adapter passthrough" ); - assert!(audit.no_js_assets); - assert!(audit.no_config); - assert!(audit.force); } #[test] @@ -189,6 +400,12 @@ mod tests { ); } + #[test] + fn run_outcomes_use_documented_exit_codes() { + assert_eq!(RunOutcome::Success.exit_code(), 0); + assert_eq!(RunOutcome::AssertionFailed.exit_code(), 1); + } + #[test] fn parses_build_with_adapter_args() { let args = parse(&[ @@ -270,6 +487,128 @@ mod tests { assert!(!diff.no_env); } + #[test] + fn config_push_parses_staging_and_rejects_explicit_key() { + let args = parse(&["ts", "config", "push", "--adapter", "fastly", "--staging"]); + let Command::Config(ConfigCommand::Push(push)) = args.command else { + panic!("expected config push command"); + }; + assert!(push.staging, "should target the derived staging key"); + + Args::try_parse_from([ + "ts", + "config", + "push", + "--adapter", + "fastly", + "--staging", + "--key", + "custom", + ]) + .expect_err("should reject --key with --staging; the staging key is derived"); + } + + #[test] + fn config_diff_parses_staging_and_rejects_explicit_key() { + let args = parse(&["ts", "config", "diff", "--adapter", "fastly", "--staging"]); + let Command::Config(ConfigCommand::Diff(diff)) = args.command else { + panic!("expected config diff command"); + }; + assert!( + diff.staging, + "should compare against the derived staging key" + ); + + Args::try_parse_from([ + "ts", + "config", + "diff", + "--adapter", + "fastly", + "--staging", + "--key", + "custom", + ]) + .expect_err("should reject --key with --staging; the staging key is derived"); + } + + #[test] + fn config_gc_previews_by_default() { + let args = parse(&["ts", "config", "gc", "--adapter", "fastly"]); + let Command::Config(ConfigCommand::Gc(gc)) = args.command else { + panic!("expected config gc command"); + }; + assert_eq!(gc.adapter, "fastly"); + assert_eq!( + gc.older_than, None, + "should not require an older-than window to preview" + ); + assert!(!gc.dry_run); + assert!(!gc.no_env); + assert_eq!( + gc.store, None, + "should default to the manifest's config-store id" + ); + assert!( + !gc.yes, + "should not delete without an explicit --yes; --yes is the only destructive gate" + ); + } + + #[test] + fn config_gc_parses_store_override() { + let args = parse(&[ + "ts", + "config", + "gc", + "--adapter", + "fastly", + "--store", + "other_config_store", + ]); + let Command::Config(ConfigCommand::Gc(gc)) = args.command else { + panic!("expected config gc command"); + }; + assert_eq!( + gc.store, + Some("other_config_store".to_owned()), + "should retarget which store a sweep inspects" + ); + } + + #[test] + fn config_gc_parses_destructive_sweep() { + let args = parse(&[ + "ts", + "config", + "gc", + "--adapter", + "fastly", + "--yes", + "--older-than", + "7d", + ]); + let Command::Config(ConfigCommand::Gc(gc)) = args.command else { + panic!("expected config gc command"); + }; + assert!(gc.yes); + assert_eq!(gc.older_than, Some("7d".to_owned())); + } + + #[test] + fn config_gc_rejects_dry_run_with_yes() { + Args::try_parse_from([ + "ts", + "config", + "gc", + "--adapter", + "fastly", + "--dry-run", + "--yes", + ]) + .expect_err("should reject conflicting --dry-run and --yes"); + } + #[test] fn config_validate_uses_edgezero_app_config_flag() { let args = parse(&[ @@ -292,6 +631,406 @@ mod tests { assert_eq!(validate.manifest, default_validate.manifest); } + #[test] + fn config_ad_templates_match_parses_app_config_flags() { + let args = parse(&[ + "ts", + "config", + "ad-templates", + "match", + "--app-config", + "publisher-a.toml", + "--no-env", + "--details", + "/news/story", + ]); + let Command::Config(ConfigCommand::AdTemplates(AdTemplatesCommand::Match(match_args))) = + args.command + else { + panic!("expected ad-templates match command"); + }; + assert_eq!( + match_args.config.app_config, + Some(PathBuf::from("publisher-a.toml")) + ); + assert!(match_args.config.no_env); + assert!(match_args.details); + assert_eq!(match_args.path_or_url, "/news/story"); + } + + #[test] + fn config_ad_templates_check_parses_expected_slots() { + let args = parse(&[ + "ts", + "config", + "ad-templates", + "check", + "/sports/game", + "--expected-slot", + "atf", + "--expected-slot", + "sports-sidebar", + "--allow-extra-slots", + ]); + let Command::Config(ConfigCommand::AdTemplates(AdTemplatesCommand::Check(check_args))) = + args.command + else { + panic!("expected ad-templates check command"); + }; + assert_eq!(check_args.path_or_url, "/sports/game"); + assert_eq!(check_args.expected_slots, ["atf", "sports-sidebar"]); + assert!(check_args.allow_extra_slots); + assert!(!check_args.expect_no_slots); + } + + #[test] + fn config_ad_templates_check_requires_an_expectation_mode() { + assert!(Args::try_parse_from(["ts", "config", "ad-templates", "check", "/news"]).is_err()); + } + + #[test] + fn config_ad_templates_check_rejects_extra_slots_with_no_slots_mode() { + assert!( + Args::try_parse_from([ + "ts", + "config", + "ad-templates", + "check", + "/news", + "--expect-no-slots", + "--allow-extra-slots", + ]) + .is_err() + ); + } + + #[test] + fn config_ad_templates_explain_rejects_removed_edgezero_model() { + assert!( + Args::try_parse_from([ + "ts", + "config", + "ad-templates", + "explain", + "/news", + "--edgezero-enabled", + ]) + .is_err() + ); + } + + #[test] + fn cli_definition_is_valid() { + // clap validates `requires` / `conflicts_with` argument-id references + // only from an explicit `debug_assert`. Without this, renaming or + // typoing an id compiles and ships. + ::command().debug_assert(); + } + + #[test] + fn bare_audit_namespace_displays_help_as_an_error() { + let error = Args::try_parse_from(["ts", "audit"]).expect_err("should require audit mode"); + + assert_eq!( + error.kind(), + clap::error::ErrorKind::DisplayHelpOnMissingArgumentOrSubcommand + ); + } + + #[test] + fn audit_legacy_url_parses_with_artifact_generation_flags() { + let args = parse(&[ + "ts", + "audit", + "https://www.example.com/", + "--js-assets", + "audit/assets.toml", + "--config", + "audit/config.toml", + "--force", + "--cookie", + "session=example", + "--chrome", + "/tmp/test-chrome", + "--headful", + "--no-assume-consent", + "--browser-proxy", + "127.0.0.1:8080", + "--settle-quiet-ms", + "900", + "--settle-max-ms", + "13000", + "--danger-accept-invalid-certs", + ]); + let Command::Audit(audit) = args.command else { + panic!("expected audit command"); + }; + assert_eq!( + audit.legacy_generate.js_assets, + Some(PathBuf::from("audit/assets.toml")) + ); + assert_eq!( + audit.legacy_generate.config, + Some(PathBuf::from("audit/config.toml")) + ); + assert!(audit.legacy_generate.force); + assert_eq!( + audit.legacy_generate.cookies, + [("session".to_string(), "example".to_string())] + ); + assert_eq!( + audit.legacy_generate.browser.chrome, + Some(PathBuf::from("/tmp/test-chrome")) + ); + assert!(audit.legacy_generate.browser.headful); + assert!(audit.legacy_generate.browser.no_assume_consent); + assert_eq!( + audit.legacy_generate.browser.browser_proxy.as_deref(), + Some("127.0.0.1:8080") + ); + assert_eq!(audit.legacy_generate.browser.settle_quiet_ms, 900); + assert_eq!(audit.legacy_generate.browser.settle_max_ms, 13_000); + assert!(audit.legacy_generate.browser.danger_accept_invalid_certs); + } + + #[test] + fn audit_help_does_not_advertise_hidden_legacy_browser_flags() { + let error = + Args::try_parse_from(["ts", "audit", "--help"]).expect_err("should render audit help"); + let help = error.to_string(); + + for flag in [ + "--chrome", + "--headful", + "--no-assume-consent", + "--browser-proxy", + "--settle-quiet-ms", + "--settle-max-ms", + "--danger-accept-invalid-certs", + ] { + assert!( + !help.contains(flag), + "`{flag}` is a legacy-only alias flag and must stay hidden; got {help}" + ); + } + } + + #[test] + fn audit_rejects_parent_browser_flags_before_a_subcommand() { + // `is_err()` alone would also pass if `--chrome` were deleted from + // `LegacyBrowserOpts` (an `UnknownArgument`), which is the opposite of + // the invariant this pins: the flag exists but requires the legacy URL. + let error = Args::try_parse_from([ + "ts", + "audit", + "--chrome", + "/tmp/test-chrome", + "generate", + "https://www.example.com/", + ]) + .expect_err("a parent-level browser flag must not be silently ignored"); + + assert_eq!( + error.kind(), + clap::error::ErrorKind::MissingRequiredArgument, + "should reject the flag for lacking the legacy URL it requires" + ); + } + + #[test] + fn audit_page_subcommand_parses_with_page_settle_defaults() { + let args = parse(&["ts", "audit", "page", "https://www.example.com/"]); + let Command::Audit(audit) = args.command else { + panic!("expected audit command"); + }; + let Some(crate::commands::audit::AuditSubcommand::Page(page)) = audit.command else { + panic!("expected audit page command"); + }; + assert_eq!(page.browser.settle_quiet_ms, 750); + assert_eq!(page.browser.settle_max_ms, 10_000); + } + + #[test] + fn audit_generate_subcommands_use_generation_settle_defaults() { + let args = parse(&["ts", "audit", "generate", "https://www.example.com/"]); + let Command::Audit(audit) = args.command else { + panic!("expected audit command"); + }; + let Some(crate::commands::audit::AuditSubcommand::Generate(generate)) = audit.command + else { + panic!("expected audit generate command"); + }; + assert_eq!(generate.browser.settle_quiet_ms, 750); + assert_eq!(generate.browser.settle_max_ms, 12_000); + + let args = parse(&[ + "ts", + "audit", + "ad-templates", + "generate", + "https://www.example.com/", + ]); + let Command::Audit(audit) = args.command else { + panic!("expected audit command"); + }; + let Some(crate::commands::audit::AuditSubcommand::AdTemplates( + crate::commands::audit::AuditAdTemplatesCommand::Generate(generate), + )) = audit.command + else { + panic!("expected audit ad-templates generate command"); + }; + assert_eq!(generate.browser.settle_quiet_ms, 750); + assert_eq!(generate.browser.settle_max_ms, 12_000); + assert!(!generate.scroll, "generation should not scroll by default"); + } + + #[test] + fn audit_ad_templates_generate_parses_scroll() { + let args = parse(&[ + "ts", + "audit", + "ad-templates", + "generate", + "https://www.example.com/", + "--scroll", + ]); + let Command::Audit(audit) = args.command else { + panic!("expected audit command"); + }; + let Some(crate::commands::audit::AuditSubcommand::AdTemplates( + crate::commands::audit::AuditAdTemplatesCommand::Generate(generate), + )) = audit.command + else { + panic!("expected audit ad-templates generate command"); + }; + + assert!( + generate.scroll, + "--scroll should enable generation scrolling" + ); + } + + #[test] + fn audit_ad_templates_verify_parses() { + let args = parse(&[ + "ts", + "audit", + "ad-templates", + "verify", + "https://www.example.com/", + ]); + assert!(matches!(args.command, Command::Audit(_))); + } + + #[test] + fn audit_browser_options_are_shared_by_generate_and_verify() { + for mode in ["generate", "verify"] { + let args = parse(&[ + "ts", + "audit", + "ad-templates", + mode, + "https://www.example.com/", + "--chrome", + "/tmp/test-chrome", + "--headful", + "--browser-proxy", + "127.0.0.1:8080", + "--no-assume-consent", + "--settle-quiet-ms", + "100", + "--settle-max-ms", + "200", + ]); + let Command::Audit(audit) = args.command else { + panic!("expected audit command"); + }; + let (chrome, headful, no_assume_consent, browser_proxy, validation) = + match audit.command.expect("should parse audit subcommand") { + crate::commands::audit::AuditSubcommand::AdTemplates( + crate::commands::audit::AuditAdTemplatesCommand::Generate(args), + ) => { + let validation = args.browser.validate(); + ( + args.browser.chrome, + args.browser.headful, + args.browser.no_assume_consent, + args.browser.browser_proxy, + validation, + ) + } + crate::commands::audit::AuditSubcommand::AdTemplates( + crate::commands::audit::AuditAdTemplatesCommand::Verify(args), + ) => { + let validation = args.browser.validate(); + ( + args.browser.chrome, + args.browser.headful, + args.browser.no_assume_consent, + args.browser.browser_proxy, + validation, + ) + } + _ => panic!("expected ad-template mode"), + }; + assert_eq!(chrome, Some(PathBuf::from("/tmp/test-chrome"))); + assert!(headful); + assert!(no_assume_consent); + assert_eq!(browser_proxy.as_deref(), Some("127.0.0.1:8080")); + validation.expect("should validate settle bounds"); + } + } + + #[test] + fn audit_generate_does_not_expose_the_ignored_browser_profile_flag() { + assert!( + Args::try_parse_from([ + "ts", + "audit", + "ad-templates", + "generate", + "https://www.example.com/", + "--browser-profile", + "mobile", + ]) + .is_err(), + "generation device selection must use --profiles" + ); + } + + #[test] + fn browser_settle_quiet_cannot_exceed_maximum() { + let args = parse(&[ + "ts", + "audit", + "page", + "https://www.example.com/", + "--settle-quiet-ms", + "201", + "--settle-max-ms", + "200", + ]); + let Command::Audit(audit) = args.command else { + panic!("expected audit command"); + }; + let crate::commands::audit::AuditSubcommand::Page(page) = + audit.command.expect("should parse page subcommand") + else { + panic!("expected page audit"); + }; + assert!(page.browser.validate().is_err()); + } + + #[test] + fn audit_ad_templates_without_verify_is_error() { + assert!(Args::try_parse_from(["ts", "audit", "ad-templates"]).is_err()); + } + + #[test] + fn audit_rejects_non_http_url() { + assert!(Args::try_parse_from(["ts", "audit", "ftp://www.example.com/"]).is_err()); + } + #[test] fn prebid_bundle_defaults_match_spec() { let args = parse(&["ts", "prebid", "bundle"]); diff --git a/crates/trusted-server-cli/tests/config_env_overlay.rs b/crates/trusted-server-cli/tests/config_env_overlay.rs index 5caab273b..45eab5cc9 100644 --- a/crates/trusted-server-cli/tests/config_env_overlay.rs +++ b/crates/trusted-server-cli/tests/config_env_overlay.rs @@ -31,6 +31,8 @@ const REWRITE_ENV: &str = "TRUSTED_SERVER__AUCTION__REWRITE_CREATIVES"; const SANITIZE_ENV: &str = "TRUSTED_SERVER__AUCTION__SANITIZE_CREATIVES"; const GAM_ATTRIBUTION_ENV: &str = "TRUSTED_SERVER__INTEGRATIONS__GPT__GAM_ATTRIBUTION_ENABLED"; const AD_TEMPLATES_ENABLED_ENV: &str = "TRUSTED_SERVER__CREATIVE_OPPORTUNITIES__ENABLED"; +const PROVIDER_ENDPOINT_ENV: &str = "TRUSTED_SERVER__AUCTION__PROVIDERS__PBS-MAIN__ENDPOINT"; +const BIDDER_PROVIDER_ENV: &str = "TRUSTED_SERVER__AUCTION__BIDDERS__EXAMPLE-BIDDER__PROVIDER"; struct MigratedProject { directory: TempDir, @@ -38,20 +40,27 @@ struct MigratedProject { manifest_path: std::path::PathBuf, } -fn migrated_legacy_project() -> MigratedProject { +fn migrated_project() -> MigratedProject { let directory = tempfile::tempdir().expect("should create temporary config directory"); let config_path = directory.path().join("trusted-server.toml"); let manifest_path = directory.path().join("edgezero.toml"); let mut document = LEGACY_CONFIG .parse::() .expect("should parse legacy integration config"); - // EdgeZero v0.0.4 environment overlays cannot create missing TOML leaves, + // EdgeZero environment overlays cannot create missing TOML leaves, // so a migrated config must carry every leaf whose environment override is // expected to take effect. document["auction"]["rewrite_creatives"] = value(true); document["auction"]["sanitize_creatives"] = value(false); document["creative_opportunities"]["enabled"] = value(true); document["creative_opportunities"]["gam_network_id"] = value("123456789"); + document["auction"]["providers"]["pbs-main"] = toml_edit::table(); + document["auction"]["providers"]["pbs-main"]["protocol"] = value("openrtb-2.6"); + document["auction"]["providers"]["pbs-main"]["profile"] = value("standard"); + document["auction"]["providers"]["pbs-main"]["endpoint"] = + value("https://original.example/openrtb2/auction"); + document["auction"]["bidders"]["example-bidder"] = toml_edit::table(); + document["auction"]["bidders"]["example-bidder"]["provider"] = value("pbs-main"); fs::write(&config_path, document.to_string()).expect("should write migrated config"); fs::write(&manifest_path, MANIFEST).expect("should write test manifest"); MigratedProject { @@ -74,8 +83,8 @@ fn validate_with_overlay(project: &MigratedProject, raw_value: &str) -> Output { } #[test] -fn migrated_legacy_config_applies_rewrite_creatives_environment_override() { - let project = migrated_legacy_project(); +fn map_config_applies_rewrite_creatives_environment_override() { + let project = migrated_project(); let output = Command::new(env!("CARGO_BIN_EXE_ts")) .args(["config", "push", "--adapter", "axum", "--manifest"]) .arg(&project.manifest_path) @@ -117,8 +126,8 @@ fn migrated_legacy_config_applies_rewrite_creatives_environment_override() { } #[test] -fn migrated_legacy_config_applies_boolean_environment_overrides() { - let project = migrated_legacy_project(); +fn migrated_config_applies_boolean_environment_overrides() { + let project = migrated_project(); let output = Command::new(env!("CARGO_BIN_EXE_ts")) .args(["config", "push", "--adapter", "axum", "--manifest"]) .arg(&project.manifest_path) @@ -166,8 +175,8 @@ fn migrated_legacy_config_applies_boolean_environment_overrides() { } #[test] -fn migrated_legacy_config_applies_sanitize_creatives_environment_override() { - let project = migrated_legacy_project(); +fn migrated_config_applies_sanitize_creatives_environment_override() { + let project = migrated_project(); let output = Command::new(env!("CARGO_BIN_EXE_ts")) .args(["config", "push", "--adapter", "axum", "--manifest"]) .arg(&project.manifest_path) @@ -209,8 +218,8 @@ fn migrated_legacy_config_applies_sanitize_creatives_environment_override() { } #[test] -fn migrated_legacy_config_default_rewrite_creatives_has_no_local_diff() { - let project = migrated_legacy_project(); +fn map_config_default_rewrite_creatives_has_no_local_diff() { + let project = migrated_project(); let push = Command::new(env!("CARGO_BIN_EXE_ts")) .args(["config", "push", "--adapter", "axum", "--manifest"]) .arg(&project.manifest_path) @@ -279,8 +288,8 @@ fn migrated_legacy_config_default_rewrite_creatives_has_no_local_diff() { } #[test] -fn migrated_legacy_config_rejects_invalid_rewrite_creatives_environment_override() { - let project = migrated_legacy_project(); +fn map_config_rejects_invalid_rewrite_creatives_environment_override() { + let project = migrated_project(); let output = validate_with_overlay(&project, "not-a-boolean"); let stderr = String::from_utf8_lossy(&output.stderr); @@ -293,3 +302,52 @@ fn migrated_legacy_config_rejects_invalid_rewrite_creatives_environment_override "error should identify the invalid boolean overlay: {stderr}" ); } + +#[test] +fn map_shaped_provider_and_bidder_environment_overlays_apply() { + let project = migrated_project(); + let output = Command::new(env!("CARGO_BIN_EXE_ts")) + .args(["config", "push", "--adapter", "axum", "--manifest"]) + .arg(&project.manifest_path) + .arg("--app-config") + .arg(&project.config_path) + .args(["--yes", "--no-diff"]) + .current_dir(project.directory.path()) + .env( + PROVIDER_ENDPOINT_ENV, + "https://overlay.example/openrtb2/auction", + ) + .env(BIDDER_PROVIDER_ENV, "pbs-main") + .output() + .expect("should run ts config push with map overlays"); + + assert!( + output.status.success(), + "map-shaped overlays should push successfully: {}", + String::from_utf8_lossy(&output.stderr) + ); + let local_store_path = project + .directory + .path() + .join(".edgezero/local-config-trusted_server_config.json"); + let local_store: serde_json::Value = serde_json::from_str( + &fs::read_to_string(local_store_path).expect("should read pushed local config"), + ) + .expect("should parse local config store"); + let envelope_json = local_store + .as_object() + .and_then(|entries| entries.values().next()) + .and_then(serde_json::Value::as_str) + .expect("should contain a blob envelope"); + let envelope: serde_json::Value = + serde_json::from_str(envelope_json).expect("should parse blob envelope"); + + assert_eq!( + envelope["data"]["auction"]["providers"]["pbs-main"]["endpoint"], + "https://overlay.example/openrtb2/auction" + ); + assert_eq!( + envelope["data"]["auction"]["bidders"]["example-bidder"]["provider"], + "pbs-main" + ); +} diff --git a/crates/trusted-server-core/src/auction/README.md b/crates/trusted-server-core/src/auction/README.md index 69c19475b..3599624f6 100644 --- a/crates/trusted-server-core/src/auction/README.md +++ b/crates/trusted-server-core/src/auction/README.md @@ -412,171 +412,116 @@ Manages the execution of an auction flow, coordinates providers, and collects re ## Auction Strategies -### 1. Parallel + Mediation (Recommended) -**Use case:** Header bidding with ad server mediation +### 1. Parallel + Mediation ```toml [auction] enabled = true -providers = ["prebid", "aps"] -mediator = "adserver_mock" # Setting mediator enables parallel mediation strategy timeout_ms = 2000 +mediator = "adserver_mock" + +[auction.providers.pbs-main] +protocol = "openrtb-2.6" +profile = "prebid-server" +endpoint = "https://prebid.example.com/openrtb2/auction" +routing = "explicit" + +[auction.providers.aps-main] +protocol = "openrtb-2.6" +profile = "aps" +endpoint = "https://aps.example.com/e/pb/bid" +routing = "all_eligible" +profile_config = { account_id = "example-aps-account" } ``` -**Flow:** -1. Prebid and APS run in parallel -2. Both return their bids simultaneously -3. Bids are sent to the mediator for final decision -4. Mediator competes house inventory and returns winning creative +Providers run in parallel, then the separately registered mediator chooses from +decoded-price bids. ### 2. Parallel Only -**Use case:** Client-side auction, no mediation -```toml -[auction] -enabled = true -providers = ["prebid", "aps"] -# No mediator = parallel only strategy (highest CPM wins) -timeout_ms = 2000 -``` - -**Flow:** -1. All providers run in parallel -2. Highest bid wins -3. No mediation server involved +Omit `mediator` from the same map-shaped configuration. The orchestrator selects +the highest decoded CPM per slot and applies floors locally. ## Configuration -### Configuration - -All auction settings are configured directly under `[auction]`: +`[auction.providers.]` is the only bidder-provider inventory. +`[auction.bidders.]` maps a client-visible bidder to exactly one +provider. The mediator is selected separately by `[auction].mediator`. ```toml [auction] -enabled = true # Enable/disable auction orchestration -providers = ["prebid", "aps"] # List of bidder providers -mediator = "adserver_mock" # Optional: if set, uses mediation; if omitted, highest bid wins -timeout_ms = 2000 # Overall auction timeout -``` - -**Strategy Auto-Detection:** -- When `mediator` is configured → Runs **parallel mediation** (providers in parallel, mediator decides winner) -- When `mediator` is omitted → Runs **parallel only** (providers in parallel, highest CPM wins) - -### Provider Configuration - -Each provider has its own configuration section: - -```toml -[integrations.prebid] -enabled = true -server_url = "https://prebid-server.example.com" -timeout_ms = 1000 - -[integrations.aps] enabled = true -mock = true # Set to false for real integration -timeout_ms = 800 - -[integrations.adserver_mock] -enabled = true -endpoint = "http://localhost:6767/adserver/mediate" -timeout_ms = 500 -``` - -## Adding a New Provider - -1. Create a new file in `src/auction/providers/your_provider.rs` - -```rust -use async_trait::async_trait; -use crate::auction::provider::{AuctionProvider, ProviderRequestOutcome}; -use crate::auction::types::{AuctionContext, AuctionRequest, AuctionResponse}; -use crate::platform::PlatformResponse; - -pub struct YourAuctionProvider { - config: YourConfig, -} - -#[async_trait(?Send)] -impl AuctionProvider for YourAuctionProvider { - fn provider_name(&self) -> &'static str { - "your_provider" - } +timeout_ms = 2000 - async fn request_bids( - &self, - request: &AuctionRequest, - _context: &AuctionContext<'_>, - ) -> Result> { - // 1. Transform AuctionRequest to your provider's format - // 2. Launch through services.http_client().send_async(...) - // 3. Wrap the handle with ProviderRequestOutcome::pending(...) - todo!() - } +[auction.providers.pbs-main] +protocol = "openrtb-2.6" +profile = "prebid-server" +endpoint = "https://prebid.example.com/openrtb2/auction" +timeout_ms = 900 +routing = "explicit" - async fn parse_response( - &self, - response: PlatformResponse, - response_time_ms: u64, - ) -> Result> { - // 4. Parse PlatformResponse into AuctionResponse - todo!() - } +[auction.providers.pbs-main.profile_config] +debug = false +test_mode = false +consent_forwarding = "both" - fn timeout_ms(&self) -> u32 { - self.config.timeout_ms - } +[auction.providers.pbs-main.notifications] +suppress_all = false +suppress_seats = ["example-seat"] - fn is_enabled(&self) -> bool { - self.config.enabled - } -} +[auction.bidders.example-server] +provider = "pbs-main" ``` -2. Register the provider in `src/auction/providers/mod.rs` - -3. Configure it in `trusted-server.toml` - -## Testing - -### Mock Providers - -APS and adserver_mock providers are used for testing the orchestration pattern: +Provider IDs own backend correlation and response identity. The configured +profile supplies typed OpenRTB behavior. Common endpoint, timeout, routing, and +notification policy do not belong to browser integration configuration. -- **APS Mock**: Returns mock bids with Amazon branding -- **AdServer Mock**: Acts as mediator by calling mocktioneer's mediation endpoint, selects winning bids based on highest CPM +## Adding a Provider -Set `mock = false` in APS config when real APS integration is ready. +A standards-compatible OpenRTB 2.6 endpoint does not require a Rust provider +implementation. Add an `[auction.providers.]` table, select the `standard` +profile, and route bidder codes through `[auction.bidders.]`. Endpoint, +timeout, routing, and notification behavior are compiled into the shared +`AuctionPlan` at startup. -### Example Test Flow +Add Rust code only when an endpoint needs behavior that the existing +`standard`, `prebid-server`, or `aps` profiles cannot express. New profile work +belongs in `profile.rs` and `openrtb.rs`: define and validate typed profile +configuration, register the profile with the central profile registry, and add +request/response golden tests. Production provider registration is plan-backed; +`AuctionOrchestrator::register_provider` exists only in the legacy test parity +harness and is not an application extension API. -```rust -let orchestrator = AuctionOrchestrator::new(config); -orchestrator.register_provider(Arc::new(PrebidAuctionProvider::try_new(prebid_config)?)); -orchestrator.register_provider(Arc::new(ApsAuctionProvider::new(aps_config))); +See the maintained [auction orchestration guide](../../../../docs/guide/auction-orchestration.md) +and [integration guide](../../../../docs/guide/integration-guide.md) for complete +configuration and validation examples. -let result = orchestrator.run_auction(&request, &context, &services).await?; +## Testing -// Check results -assert_eq!(result.winning_bids.len(), 2); -assert!(result.total_time_ms < 2000); -``` +Compile test settings with `compile_auction_plan`, construct the orchestrator and +integration registry from the same `Arc`, and exercise requests +through the normal adapter or auction endpoint. Profile tests should cover typed +configuration validation, exact OpenRTB request output, response admission, +provider-local failures, routing, and target capability validation. Legacy +provider constructors and manual registration are retained only for parity tests. ## Performance Considerations - **Parallel Execution**: Providers are launched concurrently via `select()` over `PendingRequest`s; responses are processed as they become ready within the auction deadline - **Timeouts**: Each provider has independent timeout; global timeout enforced at flow level -- **Error Handling**: Provider failures don't fail entire auction; partial results returned +- **Error Handling**: Provider failures don't fail the entire auction; partial results are returned ## Related Files -- `src/auction/mod.rs` - Module exports +- `src/auction/mod.rs` - Plan compilation and module exports +- `src/auction/plan.rs` - Typed provider plan and target validation +- `src/auction/profile.rs` - Typed OpenRTB profile registry +- `src/auction/routing.rs` - Central bidder-to-provider routing +- `src/auction/openrtb.rs` - Shared request construction and response parsing +- `src/auction/provider.rs` - Plan-backed provider execution +- `src/auction/orchestrator.rs` - Fan-out, deadline, and mediation flow - `src/auction/types.rs` - Core auction types -- `src/auction/provider.rs` - Provider trait definition -- `src/auction/orchestrator.rs` - Orchestration logic -- `src/auction/config.rs` - Configuration types -- `src/auction/providers/` - Provider implementations ## Questions? diff --git a/crates/trusted-server-core/src/auction/endpoints.rs b/crates/trusted-server-core/src/auction/endpoints.rs index fdf387e93..de05dfdd8 100644 --- a/crates/trusted-server-core/src/auction/endpoints.rs +++ b/crates/trusted-server-core/src/auction/endpoints.rs @@ -179,6 +179,32 @@ pub async fn handle_auction( }; let consent_context = ec_context.consent().clone(); + if !orchestrator.is_enabled() { + log::info!("/auction: auction is disabled; returning no-bid response"); + let auction_request = convert_tsjs_to_auction_request( + &body, + settings, + services, + &http_req, + consent_context, + ec_id, + None, + )?; + let empty_result = OrchestrationResult { + provider_responses: Vec::new(), + mediator_response: None, + winning_bids: HashMap::new(), + total_time_ms: 0, + metadata: HashMap::new(), + }; + return convert_to_openrtb_response( + &empty_result, + settings, + &auction_request, + ec_context.ec_allowed(), + ); + } + // Server-side auction consent gate. The publisher-navigation and // `/_ts/page-bids` paths fail closed for GDPR/unknown jurisdictions that // lack effective TCF Purpose 1. `/auction` is the programmatic entry point @@ -293,6 +319,7 @@ pub async fn handle_auction( settings, request: &http_req, timeout_ms: settings.auction.timeout_ms, + transport_timeout_ms: settings.auction.timeout_ms, provider_responses: None, services, }; @@ -632,7 +659,7 @@ mod tests { #[async_trait::async_trait(?Send)] impl AuctionProvider for PanicOnBidProvider { - fn provider_name(&self) -> &'static str { + fn provider_name(&self) -> &str { "panic_provider" } @@ -670,7 +697,7 @@ mod tests { #[async_trait::async_trait(?Send)] impl AuctionProvider for TemplateSwitchProbeProvider { fn provider_name(&self) -> &'static str { - "template_switch_probe" + "template-switch-probe" } async fn request_bids( @@ -722,7 +749,7 @@ mod tests { #[tokio::test] async fn direct_auction_remains_available_when_templates_are_disabled() { let settings_toml = format!( - "{}\n[auction]\nenabled = true\nproviders = [\"template_switch_probe\"]\n\n[creative_opportunities]\nenabled = false\ngam_network_id = \"12345\"\n", + "{}\n[auction]\nenabled = true\n\n[auction.providers.template-switch-probe]\nprotocol = \"openrtb-2.6\"\nendpoint = \"https://bidder.example/auction\"\nrouting = \"all_eligible\"\n\n[creative_opportunities]\nenabled = false\ngam_network_id = \"12345\"\n", crate_test_settings_str() ); let settings = Settings::from_toml(&settings_toml) @@ -788,7 +815,7 @@ mod tests { let settings = create_test_settings(); let config = AuctionConfig { enabled: true, - providers: vec!["panic_provider".to_string()], + providers: AuctionConfig::legacy_provider_map(&["panic_provider"]), timeout_ms: 2000, mediator: None, ..Default::default() @@ -872,7 +899,7 @@ mod tests { #[async_trait::async_trait(?Send)] impl AuctionProvider for EidCapturingProvider { - fn provider_name(&self) -> &'static str { + fn provider_name(&self) -> &str { "eid_capturing_provider" } @@ -916,7 +943,7 @@ mod tests { let settings = create_test_settings(); let config = AuctionConfig { enabled: true, - providers: vec!["eid_capturing_provider".to_string()], + providers: AuctionConfig::legacy_provider_map(&["eid_capturing_provider"]), timeout_ms: 2000, mediator: None, ..Default::default() diff --git a/crates/trusted-server-core/src/auction/formats.rs b/crates/trusted-server-core/src/auction/formats.rs index e09912aab..571d9d484 100644 --- a/crates/trusted-server-core/src/auction/formats.rs +++ b/crates/trusted-server-core/src/auction/formats.rs @@ -552,6 +552,10 @@ pub(crate) fn convert_to_openrtb_response_with_report( #[cfg(test)] mod tests { use super::*; + use crate::auction::plan::{ + AuctionPlan, AuctionPlanConfig, NotificationConfig, ProviderConfig, ProviderId, RoutingMode, + }; + use crate::auction::routing::route_auction; use crate::auction::types::{ ApsRendererV1, ApsTagType, AuctionResponse, Bid, BidRenderer, BidStatus, }; @@ -560,7 +564,8 @@ mod tests { use crate::test_support::tests::create_test_settings; use http::Method; use serde_json::json; - use std::collections::HashSet; + use std::collections::{BTreeMap, HashSet}; + use std::str::FromStr as _; fn make_request() -> Request { Request::builder() @@ -575,6 +580,28 @@ mod tests { create_test_settings() } + fn single_prebid_plan() -> AuctionPlan { + AuctionPlan::compile(AuctionPlanConfig { + timeout_ms: 900, + providers: BTreeMap::from([( + ProviderId::from_str("pbs-primary").expect("should parse provider ID"), + ProviderConfig { + protocol: "openrtb-2.6".to_string(), + profile: "prebid-server".to_string(), + endpoint: "https://pbs.example.test/openrtb".to_string(), + timeout_ms: None, + routing: RoutingMode::Explicit, + notifications: NotificationConfig::default(), + profile_config: json!({}), + }, + )]), + bidders: BTreeMap::new(), + mediator: None, + request_signing: None, + }) + .expect("should compile plan") + } + fn make_auction_request() -> AuctionRequest { AuctionRequest { id: "auction-1".to_string(), @@ -622,6 +649,7 @@ mod tests { creative: Some("
Ad
".to_string()), adomain: Some(vec!["advertiser.example.com".to_string()]), bidder: bidder.to_string(), + returned_seat: None, width: 300, height: 250, nurl: None, @@ -715,6 +743,31 @@ mod tests { .expect("should convert banner request") } + #[test] + fn canonical_tsjs_request_without_bids_feeds_stored_request_router() { + let body = AdRequest { + ad_units: vec![AdUnit { + code: "stored-slot".to_string(), + media_types: Some(MediaTypes { + banner: Some(BannerUnit { + sizes: vec![vec![300, 250]], + }), + }), + bids: None, + }], + config: None, + eids: None, + }; + let request = convert_body_to_auction_request(&body, &make_settings()); + let routed = route_auction(request, &make_request(), &single_prebid_plan(), None); + + assert_eq!(routed.inputs().len(), 1); + assert!( + routed.inputs()[0].slots()[0].has_trusted_stored_request(), + "canonical empty bidder map should preserve stored-request intent" + ); + } + #[test] fn response_serializes_prebid_immediate_no_bid_without_error_metadata() { let request = make_auction_request(); @@ -1630,6 +1683,7 @@ mod tests { "should omit adm for renderer bids" ); assert_eq!(bid["id"], json!("fictional-bid")); + assert_eq!(json["seatbid"][0]["seat"], json!("aps")); assert_eq!(bid["adid"], json!("fictional-ad")); assert_eq!(bid["crid"], json!("fictional-creative")); assert_eq!( diff --git a/crates/trusted-server-core/src/auction/mod.rs b/crates/trusted-server-core/src/auction/mod.rs index 986beb984..432303928 100644 --- a/crates/trusted-server-core/src/auction/mod.rs +++ b/crates/trusted-server-core/src/auction/mod.rs @@ -17,8 +17,12 @@ pub mod config; pub mod context; pub mod endpoints; pub mod formats; +pub(crate) mod openrtb; pub mod orchestrator; +pub mod plan; +pub(crate) mod profile; pub mod provider; +pub(crate) mod routing; pub mod telemetry; #[cfg(test)] pub(crate) mod test_support; @@ -27,6 +31,10 @@ pub mod types; pub use config::AuctionConfig; pub use context::{ContextQueryParams, ContextValue, build_url_with_context_params}; pub use orchestrator::AuctionOrchestrator; +pub use plan::{ + AuctionPlan, BidderId, BidderRouteConfig, NotificationConfig, ProviderConfig, ProviderId, + RoutingMode, +}; pub use provider::AuctionProvider; pub use telemetry::{ AbandonedProviderCall, AuctionEventBatch, AuctionEventRow, AuctionObservationContext, @@ -37,132 +45,189 @@ pub use types::{ AdFormat, AuctionContext, AuctionRequest, AuctionResponse, Bid, BidStatus, MediaType, }; -/// Type alias for provider builder functions. -type ProviderBuilder = - fn(&Settings) -> Result>, Report>; - -/// Returns the list of all available provider builder functions. +/// Compile the canonical target-independent auction plan for [`Settings`]. /// -/// This list is used to auto-discover and register auction providers from settings. -/// Each builder function checks the settings for its specific provider configuration -/// and returns any enabled providers. -fn provider_builders() -> &'static [ProviderBuilder] { - &[ - crate::integrations::prebid::register_auction_provider, - crate::integrations::aps::register_providers, - crate::integrations::adserver_mock::register_providers, - ] +/// This is the single settings-to-plan boundary used by deploy validation, +/// adapter startup, and operator tooling. Global request signing remains owned +/// by [`Settings`] and is copied into compiler input only at this boundary. +/// +/// # Errors +/// +/// Returns an error when auction provider, bidder route, signing, or mediator +/// configuration is invalid. +pub fn compile_auction_plan( + settings: &Settings, +) -> Result> { + AuctionPlan::compile(plan::AuctionPlanConfig { + timeout_ms: settings.auction.timeout_ms, + providers: settings.auction.providers.clone(), + bidders: settings.auction.bidders.clone(), + mediator: settings.auction.mediator.clone(), + request_signing: settings.request_signing.clone(), + }) + .map(|plan| plan.with_enabled(settings.auction.enabled)) } -/// Build a new auction orchestrator for the current settings. +/// Build a new auction orchestrator from one shared compiled plan. /// /// This constructor registers all auction providers discovered from the provided settings. /// Callers can reuse the returned [`AuctionOrchestrator`] across requests. /// /// # Arguments -/// * `settings` - Application settings used to configure the orchestrator and providers +/// * `plan` - Shared immutable compiled plan +/// * `settings` - Application settings used only for the separately registered mediator /// /// # Errors /// /// Returns an error when an enabled auction provider has invalid configuration. -pub fn build_orchestrator( +pub fn build_orchestrator_with_plan( + plan: Arc, settings: &Settings, ) -> Result> { - log::info!("Building auction orchestrator"); - - let mut orchestrator = AuctionOrchestrator::new(settings.auction.clone()); - - // Auto-discover and register all auction providers from settings - for builder in provider_builders() { - for provider in builder(settings)? { - orchestrator.register_provider(provider); - } - } - - orchestrator.validate_configured_provider_names()?; + log::info!("Building plan-backed auction orchestrator"); + + let mediator = if let Some(expected_id) = plan.mediator() { + let provider = crate::integrations::adserver_mock::register_providers(settings)? + .into_iter() + .find(|provider| provider.provider_name() == expected_id) + .ok_or_else(|| { + Report::new(TrustedServerError::Configuration { + message: format!( + "auction mediator `{expected_id}` must reference a separately registered enabled integration with the exact same ID" + ), + }) + })?; + Some(provider) + } else { + None + }; + let orchestrator = AuctionOrchestrator::from_plan(plan, mediator); log::info!( - "Auction orchestrator built with {} providers", + "Auction orchestrator built with {} bidder providers", orchestrator.provider_count() ); Ok(orchestrator) } +/// Test convenience constructor that compiles a plan before construction. +/// +/// # Errors +/// +/// Returns an error when plan compilation or mediator construction fails. #[cfg(test)] -mod tests { - use crate::settings::Settings; - use crate::test_support::tests::crate_test_settings_str; +pub fn build_orchestrator( + settings: &Settings, +) -> Result> { + let plan = Arc::new(compile_auction_plan(settings)?); + build_orchestrator_with_plan(plan, settings) +} - use super::build_orchestrator; +#[cfg(test)] +mod plan_sharing_tests { + use super::*; + use crate::integrations::IntegrationRegistry; + use crate::test_support::tests::create_test_settings; - fn settings_with_auction_config(auction_config: &str) -> Settings { - let settings_str = format!("{}\n{auction_config}", crate_test_settings_str()); - let mut settings = Settings::from_toml(&settings_str) - .expect("should parse auction provider validation test settings"); - settings.proxy.allowed_domains = vec!["*.example".to_string(), "*.example.com".to_string()]; - settings + #[test] + fn orchestrator_and_registry_share_the_compiled_plan_allocation() { + let settings = create_test_settings(); + let plan = Arc::new(compile_auction_plan(&settings).expect("should compile auction plan")); + let orchestrator = build_orchestrator_with_plan(Arc::clone(&plan), &settings) + .expect("should build orchestrator"); + let registry = IntegrationRegistry::with_plan(&settings, Arc::clone(&plan)) + .expect("should build integration registry"); + + assert!(orchestrator.shares_plan(&plan)); + assert!(registry.shares_plan(&plan)); } - fn assert_orchestrator_error_contains(settings: &Settings, expected: &str) { - let Err(err) = build_orchestrator(settings) else { - panic!("build_orchestrator should reject invalid auction providers"); - }; - assert!( - err.to_string().contains(expected), - "should include expected validation message: {expected}" - ); + #[test] + fn configured_mediator_requires_enabled_exact_registration() { + for mediator_config in [None, Some(serde_json::json!({"enabled": false}))] { + let mut settings = create_test_settings(); + settings.auction.mediator = Some("adserver_mock".to_string()); + if let Some(config) = mediator_config { + settings + .integrations + .insert_config("adserver_mock", &config) + .expect("should insert mediator config"); + } else { + settings.integrations.remove("adserver_mock"); + } + let plan = Arc::new(compile_auction_plan(&settings).expect("should compile plan")); + + let error = match build_orchestrator_with_plan(plan, &settings) { + Ok(_) => panic!("should require enabled mediator registration"), + Err(error) => error, + }; + assert!(error.to_string().contains("adserver_mock")); + } } #[test] - fn configured_unregistered_provider_fails_startup() { - let settings = settings_with_auction_config( - r#" - [auction] - enabled = true - providers = ["missing-provider"] - timeout_ms = 2000 - "#, - ); - - assert_orchestrator_error_contains( - &settings, - "Auction provider `missing-provider` is listed in [auction] but no enabled integration provides it", - ); + fn configured_mediator_builds_when_exact_registration_is_enabled() { + let mut settings = create_test_settings(); + settings.auction.mediator = Some("adserver_mock".to_string()); + settings + .integrations + .insert_config( + "adserver_mock", + &serde_json::json!({ + "enabled": true, + "endpoint": "https://mediator.example/mediate" + }), + ) + .expect("should insert mediator config"); + let plan = Arc::new(compile_auction_plan(&settings).expect("should compile plan")); + + build_orchestrator_with_plan(plan, &settings) + .expect("should build with enabled exact mediator registration"); } #[test] - fn mixed_registered_and_unregistered_providers_fail_startup() { - let settings = settings_with_auction_config( - r#" - [auction] - enabled = true - providers = ["prebid", "missing-provider"] - timeout_ms = 2000 - "#, - ); - - assert_orchestrator_error_contains( - &settings, - "Auction provider `missing-provider` is listed in [auction] but no enabled integration provides it", - ); + fn cloudflare_and_spin_reject_multi_provider_plans_before_runtime_construction() { + let mut settings = create_test_settings(); + settings.auction.enabled = true; + settings.auction.providers = + AuctionConfig::legacy_provider_map(&["provider-a", "provider-b"]); + let plan = compile_auction_plan(&settings).expect("should compile target-independent plan"); + + for target in [ + crate::platform::AuctionTargetId::Cloudflare, + crate::platform::AuctionTargetId::Spin, + ] { + let error = plan + .validate_for_target(target) + .expect_err("should reject unsupported multi-provider fanout"); + assert!( + error + .to_string() + .contains("does not support concurrent provider fanout") + ); + } } #[test] - fn configured_unregistered_mediator_fails_startup() { - let settings = settings_with_auction_config( - r#" - [auction] - enabled = true - providers = ["prebid"] - mediator = "missing-mediator" - timeout_ms = 2000 - "#, - ); - - assert_orchestrator_error_contains( - &settings, - "Auction provider `missing-mediator` is listed in [auction] but no enabled integration provides it", - ); + fn aps_profile_registers_renderer_without_browser_aps_config() { + let mut settings = create_test_settings(); + settings.auction.providers = std::collections::BTreeMap::from([( + "aps-main".parse().expect("should parse APS provider ID"), + ProviderConfig { + protocol: "openrtb-2.6".to_string(), + profile: "aps".to_string(), + endpoint: "https://aps.example/e/pb/bid".to_string(), + timeout_ms: None, + routing: RoutingMode::AllEligible, + notifications: NotificationConfig::default(), + profile_config: serde_json::json!({"account_id":"example-account"}), + }, + )]); + let plan = Arc::new(compile_auction_plan(&settings).expect("should compile APS plan")); + let registry = IntegrationRegistry::with_plan(&settings, plan) + .expect("should build APS renderer registry"); + + assert!(registry.has_route(&http::Method::GET, "/integrations/aps/renderer")); } } diff --git a/crates/trusted-server-core/src/auction/openrtb.rs b/crates/trusted-server-core/src/auction/openrtb.rs new file mode 100644 index 000000000..d9f20c42e --- /dev/null +++ b/crates/trusted-server-core/src/auction/openrtb.rs @@ -0,0 +1,779 @@ +//! Shared `OpenRTB` 2.6 request/response support for config-first providers. +//! +//! Profiles receive only routed, privacy-approved facts and never the raw +//! downstream request or unrestricted runtime services. + +use std::collections::{BTreeMap, HashMap, HashSet}; + +use error_stack::Report; +use serde_json::{Map, Value, json}; +use url::Url; + +use super::plan::{NotificationPolicy, ProviderPlan}; +use super::profile::{ + ApsProfilePlan, CompiledOpenRtbProfile, PrebidProfilePlan, StandardProfilePlan, +}; +use super::routing::{ + PrebidTransportHeaders, ProviderAuctionInput, ProviderSlotInput, RoutedAuction, +}; +use super::types::{AuctionResponse, Bid}; +use crate::consent::ConsentSource; +use crate::error::TrustedServerError; +use crate::openrtb::{ + Banner, ConsentedProvidersSettings, Device, Format, Geo, Imp, OpenRtbRequest, Publisher, Regs, + RegsExt, Site, ToExt as _, TrustedServerExt, User, UserExt, to_openrtb_i32, +}; +use crate::request_signing::{RequestSigner, SIGNING_VERSION, SigningParams}; + +const DEFAULT_CURRENCY: &str = "USD"; +const APS_SDK_SOURCE: &str = "prebid"; +const APS_SDK_VERSION: &str = "2.2.0"; +const MAX_CONSERVATIVE_LANGUAGE_BYTES: usize = 8; + +/// Result of request construction before transport. +#[derive(Debug)] +#[allow(clippy::large_enum_variant)] +pub(crate) enum OpenRtbBuildOutcome { + Ready(OpenRtbRequest), + NoImpressions, +} + +/// Explicit, deterministic signing input. No signer is loaded by this driver. +pub(crate) struct RequestFinalization<'a> { + pub(crate) signer: Option<&'a RequestSigner>, + pub(crate) signing_params: SigningParams, +} + +/// Build one provider request from its immutable routed input. +/// +/// # Errors +/// +/// Returns an auction error when static/profile extensions cannot be merged or +/// the supplied signing input does not bind the already-fixed request ID. +pub(crate) fn build_request( + input: &ProviderAuctionInput, + routed: &RoutedAuction, + provider: &ProviderPlan, + effective_timeout_ms: u32, + finalization: &RequestFinalization<'_>, +) -> Result> { + let policy = ProfilePolicy::from(&provider.profile); + let mut request = build_common_request(input, routed, policy, effective_timeout_ms); + if request.imp.is_empty() { + return Ok(OpenRtbBuildOutcome::NoImpressions); + } + policy.augment_request(&mut request, input, routed)?; + finalize_request(&mut request, policy, finalization)?; + Ok(OpenRtbBuildOutcome::Ready(request)) +} + +#[derive(Clone, Copy)] +enum ProfilePolicy<'a> { + Standard(&'a StandardProfilePlan), + Prebid(&'a PrebidProfilePlan), + Aps(&'a ApsProfilePlan), +} + +impl<'a> From<&'a CompiledOpenRtbProfile> for ProfilePolicy<'a> { + fn from(profile: &'a CompiledOpenRtbProfile) -> Self { + match profile { + CompiledOpenRtbProfile::Standard(plan) => Self::Standard(plan), + CompiledOpenRtbProfile::PrebidServer(plan) => Self::Prebid(plan), + CompiledOpenRtbProfile::Aps(plan) => Self::Aps(plan), + } + } +} + +impl ProfilePolicy<'_> { + fn augment_request( + self, + request: &mut OpenRtbRequest, + input: &ProviderAuctionInput, + routed: &RoutedAuction, + ) -> Result<(), Report> { + match self { + Self::Standard(plan) => apply_standard(request, plan), + Self::Prebid(plan) => apply_prebid(request, input, routed, plan), + Self::Aps(plan) => apply_aps(request, plan), + } + } + + fn keeps_pbs_identity_when_unsigned(self) -> bool { + matches!(self, Self::Prebid(_)) + } +} + +fn build_common_request( + input: &ProviderAuctionInput, + routed: &RoutedAuction, + policy: ProfilePolicy<'_>, + effective_timeout_ms: u32, +) -> OpenRtbRequest { + let common = input.common_request(); + let imps = input + .slots() + .iter() + .filter_map(|slot| build_imp(slot, policy)) + .collect(); + let site_domain = match policy { + ProfilePolicy::Aps(plan) => plan + .inventory_domain + .clone() + .unwrap_or_else(|| common.publisher.domain.clone()), + _ => common.publisher.domain.clone(), + }; + let page = match policy { + ProfilePolicy::Aps(plan) => { + aps_inventory_page(plan, common.publisher.page_url.as_deref(), &site_domain) + } + ProfilePolicy::Prebid(plan) => common.publisher.page_url.as_deref().map(|page| { + plan.debug_query_params.as_deref().map_or_else( + || page.to_string(), + |query| append_query_fragment(page, query), + ) + }), + ProfilePolicy::Standard(_) => common.publisher.page_url.clone(), + }; + let consent = common.user.consent.as_ref(); + let body_consent = match policy { + ProfilePolicy::Prebid(plan) => consent.filter(|value| { + plan.consent_forwarding.includes_body_consent() + || !matches!(value.source, ConsentSource::Cookie) + }), + _ => consent, + }; + let raw_tc = body_consent.and_then(|value| value.raw_tc_string.clone()); + let user = Some(User { + id: common.user.id.clone(), + consent: raw_tc.clone(), + ext: UserExt { + consent: raw_tc, + consented_providers_settings: matches!(policy, ProfilePolicy::Prebid(_)) + .then(|| { + body_consent + .and_then(|value| value.raw_ac_string.clone()) + .map(|consented_providers| ConsentedProvidersSettings { + consented_providers: Some(consented_providers), + }) + }) + .flatten(), + eids: common.user.eids.clone(), + } + .to_ext(), + ..Default::default() + }); + let language = normalized_language(routed.prebid_transport_headers(), policy); + let device = common + .device + .as_ref() + .map(|device| Device { + ua: device.user_agent.clone(), + ip: device.ip.clone(), + geo: device.geo.as_ref().map(|geo| Geo { + country: Some(geo.country.clone()), + region: geo.region.clone(), + city: Some(geo.city.clone()), + lat: matches!(policy, ProfilePolicy::Prebid(_)).then_some(geo.latitude), + lon: matches!(policy, ProfilePolicy::Prebid(_)).then_some(geo.longitude), + metro: (geo.metro_code > 0).then(|| geo.metro_code.to_string()), + r#type: Some(2), + ..Default::default() + }), + dnt: routed.dnt(), + language: language.clone(), + ..Default::default() + }) + .or_else(|| { + (routed.dnt().is_some() || language.is_some()).then_some(Device { + dnt: routed.dnt(), + language, + ..Default::default() + }) + }); + + OpenRtbRequest { + id: Some(common.id.clone()), + imp: imps, + site: Some(Site { + domain: Some(site_domain.clone()), + page, + r#ref: matches!(policy, ProfilePolicy::Prebid(_)) + .then(|| header_string(routed.prebid_transport_headers().referer())) + .flatten(), + publisher: Some(Publisher { + domain: Some(site_domain), + ..Default::default() + }), + ..Default::default() + }), + user, + device, + regs: build_regs(body_consent, policy), + test: match policy { + ProfilePolicy::Prebid(plan) => plan.test_mode.then_some(true), + _ => None, + }, + tmax: to_openrtb_i32( + effective_timeout_ms, + "tmax", + "config-first provider request", + ), + cur: vec![DEFAULT_CURRENCY.to_string()], + ..Default::default() + } +} + +fn build_imp(slot: &ProviderSlotInput, policy: ProfilePolicy<'_>) -> Option { + let formats = slot + .slot() + .formats + .iter() + .filter_map(|format| { + Some(Format { + w: to_openrtb_i32(format.width, "format.w", "routed slot"), + h: to_openrtb_i32(format.height, "format.h", "routed slot"), + ..Default::default() + }) + .filter(|value| value.w.is_some() && value.h.is_some()) + }) + .collect::>(); + let first_width = formats.first()?.w; + let first_height = formats.first()?.h; + let aps_banner = matches!(policy, ProfilePolicy::Aps(_)); + Some(Imp { + id: Some(slot.slot().id.clone()), + banner: Some(Banner { + format: formats, + w: aps_banner.then_some(first_width).flatten(), + h: aps_banner.then_some(first_height).flatten(), + topframe: aps_banner.then_some(false), + ..Default::default() + }), + tagid: matches!(policy, ProfilePolicy::Prebid(_)).then(|| slot.slot().id.clone()), + bidfloor: slot.slot().floor_price, + bidfloorcur: slot + .slot() + .floor_price + .map(|_| DEFAULT_CURRENCY.to_string()), + secure: Some(true), + ..Default::default() + }) +} + +fn apply_standard( + request: &mut OpenRtbRequest, + plan: &StandardProfilePlan, +) -> Result<(), Report> { + request.ext = nonempty_map(plan.request_ext.as_object().clone()); + for imp in &mut request.imp { + imp.ext = nonempty_map(plan.imp_ext.as_object().clone()); + } + Ok(()) +} + +fn apply_prebid( + request: &mut OpenRtbRequest, + input: &ProviderAuctionInput, + _routed: &RoutedAuction, + plan: &PrebidProfilePlan, +) -> Result<(), Report> { + debug_assert_eq!( + request.imp.len(), + input.slots().len(), + "should keep one impression per routed slot" + ); + for (imp, slot) in request.imp.iter_mut().zip(input.slots()) { + let bidder = slot + .bidder_params() + .iter() + .filter_map(|(bidder, params)| { + let mut params = params.clone(); + plan.override_engine + .apply_routed(bidder.as_str(), slot.prebid_zone(), &mut params); + params + .as_object() + .is_some_and(|params| !params.is_empty()) + .then(|| (bidder.as_str().to_string(), params)) + }) + .collect::>(); + let mut prebid = Map::new(); + if !bidder.is_empty() { + prebid.insert("bidder".to_string(), Value::Object(bidder)); + } else if slot.has_trusted_stored_request() || !slot.bidder_params().is_empty() { + prebid.insert("storedrequest".to_string(), json!({"id": slot.slot().id})); + } + imp.ext = Some(Map::from_iter([( + "prebid".to_string(), + Value::Object(prebid), + )])); + } + let mut prebid_request = Map::new(); + if plan.debug { + prebid_request.insert("debug".to_string(), Value::Bool(true)); + prebid_request.insert("returnallbidstatus".to_string(), Value::Bool(true)); + } + request.ext = Some(Map::from_iter([( + "prebid".to_string(), + Value::Object(prebid_request), + )])); + Ok(()) +} + +fn apply_aps( + request: &mut OpenRtbRequest, + plan: &ApsProfilePlan, +) -> Result<(), Report> { + request.ext = Some(Map::from_iter([ + ( + "account".to_string(), + Value::String(plan.account_id.clone()), + ), + ( + "sdk".to_string(), + json!({"source": APS_SDK_SOURCE, "version": APS_SDK_VERSION}), + ), + ])); + Ok(()) +} + +fn finalize_request( + request: &mut OpenRtbRequest, + policy: ProfilePolicy<'_>, + finalization: &RequestFinalization<'_>, +) -> Result<(), Report> { + let request_id = request.id.as_deref().ok_or_else(|| { + Report::new(TrustedServerError::Auction { + message: "OpenRTB request ID must be fixed before signing".to_string(), + }) + })?; + if request_id != finalization.signing_params.request_id { + return Err(Report::new(TrustedServerError::Auction { + message: "OpenRTB signing params do not bind the fixed request ID".to_string(), + })); + } + let trusted_server = if let Some(signer) = finalization.signer { + let signature = signer.sign_request(&finalization.signing_params)?; + Some(TrustedServerExt { + version: Some(SIGNING_VERSION.to_string()), + signature: Some(signature), + kid: Some(signer.kid.clone()), + request_host: Some(finalization.signing_params.request_host.clone()), + request_scheme: Some(finalization.signing_params.request_scheme.clone()), + ts: Some(finalization.signing_params.timestamp), + }) + } else if policy.keeps_pbs_identity_when_unsigned() { + Some(TrustedServerExt { + version: None, + signature: None, + kid: None, + request_host: Some(finalization.signing_params.request_host.clone()), + request_scheme: Some(finalization.signing_params.request_scheme.clone()), + ts: None, + }) + } else { + None + }; + if let Some(trusted_server) = trusted_server { + let ext = request.ext.get_or_insert_with(Map::new); + let serialized = serde_json::to_value(trusted_server).map_err(|error| { + Report::new(TrustedServerError::Auction { + message: format!("Failed to serialize Trusted Server extension: {error}"), + }) + })?; + ext.insert("trusted_server".to_string(), serialized); + } + Ok(()) +} + +fn build_regs( + consent: Option<&crate::consent::ConsentContext>, + policy: ProfilePolicy<'_>, +) -> Option { + let consent = consent?; + if matches!(policy, ProfilePolicy::Aps(_)) { + // Preserve APS exactly: any admitted context produces regs and GDPR is + // derived only from the applicability bit, without jurisdiction rules. + let ext = RegsExt { + gdpr: Some(u8::from(consent.gdpr_applies)), + us_privacy: consent.raw_us_privacy.clone(), + gpp: consent.raw_gpp_string.clone(), + gpp_sid: consent.gpp_section_ids.clone(), + }; + return Some(Regs { + coppa: None, + gdpr: Some(consent.gdpr_applies), + us_privacy: ext.us_privacy.clone(), + gpp: ext.gpp.clone(), + gpp_sid: ext + .gpp_sid + .as_ref() + .map(|ids| ids.iter().copied().map(i32::from).collect()) + .unwrap_or_default(), + ext: ext.to_ext(), + }); + } + + // Standard deliberately shares PBS's conservative consent baseline. Keep + // the legacy PBS empty-context and jurisdiction behavior byte-for-byte. + let has_data = consent.gdpr_applies + || consent.raw_us_privacy.is_some() + || consent.raw_gpp_string.is_some() + || consent.gpp_section_ids.is_some() + || consent.gpc; + if !has_data { + return None; + } + let gdpr = if consent.gdpr_applies + || matches!( + consent.jurisdiction, + crate::consent::jurisdiction::Jurisdiction::Gdpr + ) { + Some(true) + } else if matches!( + consent.jurisdiction, + crate::consent::jurisdiction::Jurisdiction::Unknown + ) { + None + } else { + Some(false) + }; + let us_privacy = consent.raw_us_privacy.clone(); + let gpp = consent.raw_gpp_string.clone(); + let gpp_sid = consent.gpp_section_ids.clone(); + let ext = RegsExt { + gdpr: gdpr.map(u8::from), + us_privacy: us_privacy.clone(), + gpp: gpp.clone(), + gpp_sid: gpp_sid.clone(), + }; + Some(Regs { + coppa: None, + gdpr, + us_privacy, + gpp, + gpp_sid: gpp_sid + .map(|ids| ids.into_iter().map(i32::from).collect()) + .unwrap_or_default(), + ext: ext.to_ext(), + }) +} + +fn normalized_language( + headers: &PrebidTransportHeaders, + policy: ProfilePolicy<'_>, +) -> Option { + let value = header_string(headers.accept_language()) + .and_then(|value| value.split(',').next().map(str::to_string)) + .and_then(|value| value.split(';').next().map(str::to_string)) + .and_then(|value| value.split('-').next().map(str::to_string)) + .map(|value| value.trim().to_string()) + .filter(|value| !value.is_empty())?; + match policy { + ProfilePolicy::Prebid(_) => Some(value), + ProfilePolicy::Aps(_) | ProfilePolicy::Standard(_) => { + (value.len() <= MAX_CONSERVATIVE_LANGUAGE_BYTES).then_some(value) + } + } +} + +fn header_string(value: Option<&http::HeaderValue>) -> Option { + value + .and_then(|value| value.to_str().ok()) + .map(str::to_string) +} + +fn aps_inventory_page( + plan: &ApsProfilePlan, + publisher_page: Option<&str>, + domain: &str, +) -> Option { + let fallback = publisher_page + .and_then(valid_aps_page_url) + .unwrap_or_else(|| format!("https://{domain}")); + let Some(origin) = plan.inventory_page_origin.as_deref() else { + return Some(fallback); + }; + let (Ok(mut canonical), Ok(current)) = (Url::parse(origin), Url::parse(&fallback)) else { + return Some(fallback); + }; + canonical.set_path(current.path()); + canonical.set_query(current.query()); + canonical.set_fragment(None); + Some(canonical.to_string()) +} + +fn append_query_fragment(url: &str, query: &str) -> String { + if query.is_empty() || url.contains(query) { + return url.to_string(); + } + let separator = if url.contains('?') { '&' } else { '?' }; + format!("{url}{separator}{query}") +} + +fn valid_aps_page_url(value: &str) -> Option { + const MAX_APS_PAGE_URL_BYTES: usize = 8192; + + if value.len() > MAX_APS_PAGE_URL_BYTES { + return None; + } + let parsed = Url::parse(value).ok()?; + (matches!(parsed.scheme(), "http" | "https") + && parsed.host_str().is_some() + && parsed.username().is_empty() + && parsed.password().is_none()) + .then(|| parsed.to_string()) +} + +fn nonempty_map(value: Map) -> Option> { + (!value.is_empty()).then_some(value) +} + +/// Suppress notification URLs using exact returned-seat identity. +pub(crate) fn apply_notification_policy(bids: &mut [Bid], policy: &NotificationPolicy) { + for bid in bids { + let suppress = policy.suppress_all + || bid + .returned_seat + .as_ref() + .is_some_and(|seat| policy.suppress_seats.contains(seat)); + if suppress { + bid.nurl = None; + bid.burl = None; + } + } +} + +/// Parse ordinary `OpenRTB` bids independently. Response ID is informational. +pub(crate) fn extract_standard_response( + provider_id: &str, + input: &ProviderAuctionInput, + value: &Value, + response_time_ms: u64, +) -> AuctionResponse { + let Some(response) = value.as_object() else { + return AuctionResponse::error(provider_id, response_time_ms) + .with_metadata("error_type", json!("parse_response")); + }; + match response.get("cur") { + None => {} + Some(Value::String(currency)) if currency.eq_ignore_ascii_case(DEFAULT_CURRENCY) => {} + Some(Value::String(currency)) => { + return AuctionResponse::no_bid(provider_id, response_time_ms) + .with_metadata("unsupported_currency", json!(currency)); + } + Some(_) => { + return AuctionResponse::error(provider_id, response_time_ms) + .with_metadata("error_type", json!("parse_response")); + } + } + let allowed_impressions = input + .slots() + .iter() + .map(|slot| { + let dimensions = slot + .slot() + .formats + .iter() + .map(|format| (format.width, format.height)) + .collect::>(); + (slot.slot().id.as_str(), dimensions) + }) + .collect::>(); + let mut bids = Vec::new(); + for seatbid in response + .get("seatbid") + .and_then(Value::as_array) + .into_iter() + .flatten() + { + let returned_seat = seatbid + .get("seat") + .and_then(Value::as_str) + .filter(|seat| !seat.is_empty()); + let Some(entries) = seatbid.get("bid").and_then(Value::as_array) else { + continue; + }; + for value in entries { + if let Some(bid) = extract_standard_bid(value, returned_seat) + && allowed_impressions + .get(bid.slot_id.as_str()) + .is_some_and(|dimensions| dimensions.contains(&(bid.width, bid.height))) + { + bids.push(bid); + } + } + } + if bids.is_empty() { + AuctionResponse::no_bid(provider_id, response_time_ms) + } else { + AuctionResponse::success(provider_id, bids, response_time_ms) + } +} + +fn extract_standard_bid(value: &Value, returned_seat: Option<&str>) -> Option { + let slot_id = value.get("impid")?.as_str()?.to_string(); + let price = value + .get("price")? + .as_f64() + .filter(|price| price.is_finite() && *price >= 0.0)?; + let width = u32::try_from(value.get("w")?.as_u64()?) + .ok() + .filter(|value| *value > 0)?; + let height = u32::try_from(value.get("h")?.as_u64()?) + .ok() + .filter(|value| *value > 0)?; + let creative = value + .get("adm") + .and_then(Value::as_str) + .filter(|creative| !creative.is_empty()) + .map(str::to_string)?; + Some(Bid { + slot_id, + price: Some(price), + currency: DEFAULT_CURRENCY.to_string(), + creative: Some(creative), + adomain: value + .get("adomain") + .and_then(Value::as_array) + .map(|domains| { + domains + .iter() + .filter_map(Value::as_str) + .map(str::to_string) + .collect() + }), + bidder: returned_seat.unwrap_or("unknown").to_string(), + returned_seat: returned_seat.map(str::to_string), + width, + height, + nurl: value + .get("nurl") + .and_then(Value::as_str) + .map(str::to_string), + burl: value + .get("burl") + .and_then(Value::as_str) + .map(str::to_string), + bid_id: value + .get("id") + .and_then(Value::as_str) + .filter(|id| !id.is_empty()) + .map(str::to_string), + ad_id: value + .get("adid") + .and_then(Value::as_str) + .map(str::to_string), + creative_id: value + .get("crid") + .and_then(Value::as_str) + .map(str::to_string), + renderer: None, + cache_id: None, + cache_host: None, + cache_path: None, + metadata: HashMap::new(), + }) +} + +/// Count bidder parameter objects a profile did not consume. +#[must_use] +pub(crate) fn unused_bidder_params_count( + profile: &CompiledOpenRtbProfile, + input: &ProviderAuctionInput, +) -> u32 { + if profile.is_prebid_server() { + return 0; + } + ignored_bidder_params_count(input) +} + +/// Count routed bidder params for a profile known to ignore them. +#[must_use] +pub(crate) fn ignored_bidder_params_count(input: &ProviderAuctionInput) -> u32 { + saturating_bidder_param_counts(input.slots().iter().map(|slot| slot.bidder_params().len())) +} + +fn saturating_bidder_param_counts(counts: impl IntoIterator) -> u32 { + counts.into_iter().fold(0_u32, |count, slot_count| { + count.saturating_add(u32::try_from(slot_count).unwrap_or(u32::MAX)) + }) +} + +#[cfg(test)] +mod routing_metadata_tests { + use std::collections::BTreeMap; + use std::str::FromStr as _; + + use serde_json::json; + + use super::{saturating_bidder_param_counts, unused_bidder_params_count}; + use crate::auction::plan::{ + AuctionPlan, AuctionPlanConfig, BidderId, BidderRouteConfig, NotificationConfig, + ProviderConfig, ProviderId, RoutingMode, + }; + use crate::auction::routing::route_auction; + use crate::auction::test_support::canonical_parity_auction_request; + + #[test] + fn unused_bidder_param_count_saturates_across_slots_and_large_values() { + assert_eq!(saturating_bidder_param_counts([1, 2, 3]), 6); + assert_eq!( + saturating_bidder_param_counts([usize::try_from(u32::MAX).unwrap_or(usize::MAX), 1]), + u32::MAX + ); + assert_eq!(saturating_bidder_param_counts([usize::MAX]), u32::MAX); + } + + #[test] + fn unused_bidder_param_count_is_profile_aware() { + for (profile, profile_config, expected) in [ + ("prebid-server", json!({}), 0), + ("standard", json!({}), 1), + ("aps", json!({"account_id":"example-account"}), 1), + ] { + let provider_id = + ProviderId::from_str("fictional-provider").expect("should parse provider ID"); + let plan = AuctionPlan::compile(AuctionPlanConfig { + timeout_ms: 1_000, + providers: BTreeMap::from([( + provider_id.clone(), + ProviderConfig { + protocol: "openrtb-2.6".to_string(), + profile: profile.to_string(), + endpoint: if profile == "aps" { + "https://aps.example/e/pb/bid".to_string() + } else { + "https://provider.example/openrtb".to_string() + }, + timeout_ms: None, + routing: RoutingMode::Explicit, + notifications: NotificationConfig::default(), + profile_config, + }, + )]), + bidders: BTreeMap::from([( + BidderId::from_str("exampleBidder").expect("should parse bidder ID"), + BidderRouteConfig { + provider: provider_id, + }, + )]), + mediator: None, + request_signing: None, + }) + .expect("should compile profile plan"); + let inbound = http::Request::new(edgezero_core::body::Body::empty()); + let routed = route_auction(canonical_parity_auction_request(), &inbound, &plan, None); + + assert_eq!( + unused_bidder_params_count(&plan.providers()[0].profile, &routed.inputs()[0]), + expected, + "{profile} should report only bidder params it ignores" + ); + } + } +} + +#[cfg(test)] +mod test_executor; +#[cfg(test)] +mod tests; diff --git a/crates/trusted-server-core/src/auction/openrtb/test_executor.rs b/crates/trusted-server-core/src/auction/openrtb/test_executor.rs new file mode 100644 index 000000000..b8ac835c4 --- /dev/null +++ b/crates/trusted-server-core/src/auction/openrtb/test_executor.rs @@ -0,0 +1,107 @@ +//! Fictional standard-profile executor compiled only for automated tests. + +use edgezero_core::body::Body as EdgeBody; +use error_stack::{Report, ResultExt as _}; +use http::{Method, Request, StatusCode, header}; +use serde_json::{Value, json}; + +use super::{apply_notification_policy, extract_standard_response, unused_bidder_params_count}; +use crate::auction::plan::ProviderPlan; +use crate::auction::routing::ProviderAuctionInput; +use crate::auction::types::AuctionResponse; +use crate::error::TrustedServerError; +use crate::platform::{PlatformBackend, PlatformHttpClient, PlatformHttpRequest}; + +const MAX_STANDARD_RESPONSE_BYTES: usize = 1024 * 1024; + +/// Execute one fictional standard-profile request through a supplied test client. +/// +/// The HTTP client receives exactly one request. Redirect statuses are +/// classified as the original provider error and never followed. +pub(super) async fn execute_standard_fixture( + provider: &ProviderPlan, + input: &ProviderAuctionInput, + request: &trusted_server_openrtb::BidRequest, + backend: &dyn PlatformBackend, + http_client: &dyn PlatformHttpClient, +) -> Result> { + let spec = provider.backend_spec(); + let predicted_name = + backend + .predict_name(&spec) + .change_context(TrustedServerError::Auction { + message: "Failed to predict fictional standard backend".to_string(), + })?; + let backend_name = backend + .ensure(&spec) + .change_context(TrustedServerError::Auction { + message: "Failed to ensure fictional standard backend".to_string(), + })?; + if backend_name != predicted_name { + return Err(Report::new(TrustedServerError::Auction { + message: "Fictional standard backend ensure did not match prediction".to_string(), + })); + }; + let body = serde_json::to_vec(request).change_context(TrustedServerError::Auction { + message: "Failed to serialize fictional standard request".to_string(), + })?; + let outbound = Request::builder() + .method(Method::POST) + .uri(provider.endpoint.as_str()) + .header(header::CONTENT_TYPE, "application/json") + .header(header::ACCEPT, "application/json") + .body(EdgeBody::from(body)) + .change_context(TrustedServerError::Auction { + message: "Failed to build fictional standard request".to_string(), + })?; + let response = http_client + .send(PlatformHttpRequest::new(outbound, backend_name)) + .await + .change_context(TrustedServerError::Auction { + message: "Fictional standard transport failed".to_string(), + })? + .response; + let status = response.status(); + if status == StatusCode::NO_CONTENT { + return Ok( + AuctionResponse::no_bid(provider.id.as_str(), 0).with_metadata( + "routing", + json!({"unused_bidder_params_count": unused_bidder_params_count(&provider.profile, input)}), + ), + ); + } + if !status.is_success() { + return Ok(AuctionResponse::error(provider.id.as_str(), 0) + .with_metadata("error_type", json!("http_status")) + .with_metadata("http_status", json!(status.as_u16())) + .with_metadata( + "routing", + json!({"unused_bidder_params_count": unused_bidder_params_count(&provider.profile, input)}), + )); + } + let body = response + .into_body() + .into_bytes_bounded(MAX_STANDARD_RESPONSE_BYTES) + .await + .change_context(TrustedServerError::Auction { + message: "Fictional standard response exceeded its limit".to_string(), + })?; + let value: Value = match serde_json::from_slice(&body) { + Ok(value) => value, + Err(_) => { + return Ok(AuctionResponse::error(provider.id.as_str(), 0) + .with_metadata("error_type", json!("parse_response")) + .with_metadata( + "routing", + json!({"unused_bidder_params_count": unused_bidder_params_count(&provider.profile, input)}), + )); + } + }; + let mut parsed = extract_standard_response(provider.id.as_str(), input, &value, 0); + apply_notification_policy(&mut parsed.bids, &provider.notifications); + parsed.metadata.insert( + "routing".to_string(), + json!({"unused_bidder_params_count": unused_bidder_params_count(&provider.profile, input)}), + ); + Ok(parsed) +} diff --git a/crates/trusted-server-core/src/auction/openrtb/tests.rs b/crates/trusted-server-core/src/auction/openrtb/tests.rs new file mode 100644 index 000000000..41c4e59d1 --- /dev/null +++ b/crates/trusted-server-core/src/auction/openrtb/tests.rs @@ -0,0 +1,1152 @@ +use std::collections::{BTreeMap, BTreeSet, HashMap, HashSet}; +use std::str::FromStr as _; +use std::sync::Arc; + +use base64::Engine as _; +use edgezero_core::body::Body as EdgeBody; +use http::{Request, header}; +use serde_json::{Value, json}; + +use super::test_executor::execute_standard_fixture; +use super::*; +use crate::auction::plan::{ + AuctionPlan, AuctionPlanConfig, BidderRouteConfig, NotificationConfig, ProviderConfig, + ProviderId, RoutingMode, +}; +use crate::auction::provider::{GenericOpenRtbProvider, ProviderRequestOutcome}; +use crate::auction::routing::route_auction; +use crate::auction::test_support::canonical_parity_auction_request; +use crate::auction::types::{AdFormat, AdSlot, BidStatus, MediaType}; +use crate::consent::jurisdiction::Jurisdiction; +use crate::consent::{ConsentContext, ConsentSource}; +use crate::platform::PlatformHttpClient; +use crate::platform::test_support::{ + HashMapConfigStore, HashMapSecretStore, NoopHttpClient, StubBackend, StubHttpClient, + build_services_with_backend_and_http_client, build_services_with_config_secret_and_http_client, +}; +use crate::request_signing::RequestSigner; + +fn config(profile: &str, profile_config: Value) -> AuctionPlanConfig { + config_with_endpoint( + profile, + profile_config, + "https://exchange.example.test/openrtb", + ) +} + +fn config_with_endpoint(profile: &str, profile_config: Value, endpoint: &str) -> AuctionPlanConfig { + AuctionPlanConfig { + timeout_ms: 321, + providers: BTreeMap::from([( + ProviderId::from_str("fictional-provider").expect("should parse provider"), + ProviderConfig { + protocol: "openrtb-2.6".to_string(), + profile: profile.to_string(), + endpoint: endpoint.to_string(), + timeout_ms: Some(321), + routing: RoutingMode::AllEligible, + notifications: NotificationConfig::default(), + profile_config, + }, + )]), + bidders: BTreeMap::new(), + mediator: None, + request_signing: None, + } +} + +fn routed(profile: &str, profile_config: Value) -> (AuctionPlan, RoutedAuction) { + let plan = AuctionPlan::compile(config(profile, profile_config)).expect("should compile plan"); + let inbound = Request::builder() + .uri("https://publisher.example/auction") + .header( + header::REFERER, + "https://referrer.example/story?fictional=1", + ) + .header(header::ACCEPT_LANGUAGE, "en-US,en;q=0.9") + .header("dnt", "1") + .body(EdgeBody::empty()) + .expect("should build inbound request"); + let routed = route_auction(canonical_parity_auction_request(), &inbound, &plan, None); + (plan, routed) +} + +fn routed_with_request( + profile: &str, + profile_config: Value, + request: crate::auction::types::AuctionRequest, + accept_language: Option<&str>, +) -> (AuctionPlan, RoutedAuction) { + let plan = AuctionPlan::compile(config(profile, profile_config)).expect("should compile plan"); + let mut builder = Request::builder().uri("https://publisher.example/auction"); + if let Some(language) = accept_language { + builder = builder.header(header::ACCEPT_LANGUAGE, language); + } + let inbound = builder + .body(EdgeBody::empty()) + .expect("should build inbound request"); + let routed = route_auction(request, &inbound, &plan, None); + (plan, routed) +} + +fn build_with_request( + profile: &str, + profile_config: Value, + request: crate::auction::types::AuctionRequest, + accept_language: Option<&str>, +) -> OpenRtbRequest { + let (plan, routed) = routed_with_request(profile, profile_config, request, accept_language); + match build_request( + &routed.inputs()[0], + &routed, + &plan.providers()[0], + 321, + &finalization(None), + ) + .expect("should build request") + { + OpenRtbBuildOutcome::Ready(request) => request, + OpenRtbBuildOutcome::NoImpressions => panic!("should retain impression"), + } +} + +fn finalization<'a>(signer: Option<&'a RequestSigner>) -> RequestFinalization<'a> { + RequestFinalization { + signer, + signing_params: SigningParams { + request_id: "fictional-auction".to_string(), + request_host: "publisher.example".to_string(), + request_scheme: "https".to_string(), + timestamp: 1_706_900_000, + }, + } +} + +fn build(profile: &str, profile_config: Value, signer: Option<&RequestSigner>) -> OpenRtbRequest { + let (plan, routed) = routed(profile, profile_config); + match build_request( + &routed.inputs()[0], + &routed, + &plan.providers()[0], + 321, + &finalization(signer), + ) + .expect("should build request") + { + OpenRtbBuildOutcome::Ready(request) => request, + OpenRtbBuildOutcome::NoImpressions => panic!("should retain impression"), + } +} + +fn deterministic_signer() -> RequestSigner { + let mut config_data = HashMap::new(); + config_data.insert("current-kid".to_string(), "fictional-kid".to_string()); + let mut secret_data = HashMap::new(); + secret_data.insert( + "fictional-kid".to_string(), + base64::engine::general_purpose::STANDARD + .encode([7_u8; 32]) + .into_bytes(), + ); + let services = build_services_with_config_secret_and_http_client( + HashMapConfigStore::new(config_data), + HashMapSecretStore::new(secret_data), + Arc::new(NoopHttpClient), + ); + RequestSigner::from_services(&services).expect("should load deterministic signer") +} + +#[test] +fn consent_matrix_preserves_pbs_standard_and_aps_policies() { + let cases = [ + ("empty", ConsentContext::default()), + ( + "gdpr", + ConsentContext { + gdpr_applies: true, + raw_tc_string: Some("tc-string".to_string()), + jurisdiction: Jurisdiction::Gdpr, + ..Default::default() + }, + ), + ( + "unknown-gpc", + ConsentContext { + gpc: true, + jurisdiction: Jurisdiction::Unknown, + ..Default::default() + }, + ), + ( + "nonregulated-gpc", + ConsentContext { + gpc: true, + jurisdiction: Jurisdiction::NonRegulated, + ..Default::default() + }, + ), + ( + "usp-gpp", + ConsentContext { + raw_us_privacy: Some("1YNN".to_string()), + raw_gpp_string: Some("gpp-string".to_string()), + gpp_section_ids: Some(vec![7, 8]), + jurisdiction: Jurisdiction::NonRegulated, + ..Default::default() + }, + ), + ]; + for (name, consent) in cases { + for profile in ["standard", "prebid-server", "aps"] { + let mut canonical = canonical_parity_auction_request(); + canonical.user.consent = Some(consent.clone()); + let config = if profile == "aps" { + json!({"account_id": "example-account-id"}) + } else { + json!({}) + }; + let value = serde_json::to_value(build_with_request(profile, config, canonical, None)) + .expect("should serialize request"); + let regs = value.get("regs"); + if profile == "aps" { + let regs = regs.expect("APS should preserve empty admitted context"); + assert_eq!( + regs["gdpr"], + json!(u8::from(consent.gdpr_applies)), + "{name}" + ); + } else if name == "empty" { + assert!(regs.is_none(), "{profile} should omit empty regs"); + } else { + let regs = regs.expect("should emit actionable regs"); + let expected_gdpr = match consent.jurisdiction { + Jurisdiction::Gdpr => Some(true), + Jurisdiction::Unknown if !consent.gdpr_applies => None, + _ => Some(consent.gdpr_applies), + }; + assert_eq!( + regs.get("gdpr"), + expected_gdpr.map(|value| json!(u8::from(value))).as_ref(), + "{name} {profile}" + ); + } + let serialized = value.to_string(); + assert!( + !serialized.contains("1YYY"), + "must never synthesize USP from GPC" + ); + if name == "usp-gpp" { + let regs = regs.expect("should have explicit fields"); + assert_eq!(regs["us_privacy"], "1YNN"); + assert_eq!(regs["gpp"], "gpp-string"); + assert_eq!(regs["gpp_sid"], json!([7, 8])); + assert_eq!(regs["ext"]["us_privacy"], "1YNN"); + assert_eq!(regs["ext"]["gpp"], "gpp-string"); + assert_eq!(regs["ext"]["gpp_sid"], json!([7, 8])); + } + } + } +} + +#[test] +fn pbs_body_consent_respects_source_and_forwarding_mode() { + for (mode, source, expected) in [ + ("cookies_only", ConsentSource::Cookie, false), + ("cookies_only", ConsentSource::KvStore, true), + ("cookies_only", ConsentSource::PolicyDefault, true), + ("openrtb_only", ConsentSource::Cookie, true), + ("both", ConsentSource::Cookie, true), + ] { + let mut canonical = canonical_parity_auction_request(); + canonical.user.consent.as_mut().expect("consent").source = source; + let value = serde_json::to_value(build_with_request( + "prebid-server", + json!({"consent_forwarding": mode}), + canonical, + None, + )) + .expect("should serialize request"); + assert_eq!( + value["user"].get("consent").is_some(), + expected, + "{mode:?} {source:?}" + ); + assert_eq!(value.get("regs").is_some(), expected, "{mode:?} {source:?}"); + } +} + +#[test] +fn language_limits_are_profile_specific() { + let language = "abcdefghijk"; + for (profile, expected) in [ + ("prebid-server", Some(language)), + ("aps", None), + ("standard", None), + ] { + let config = if profile == "aps" { + json!({"account_id": "example-account-id"}) + } else { + json!({}) + }; + let request = build_with_request( + profile, + config, + canonical_parity_auction_request(), + Some(language), + ); + assert_eq!( + request.device.and_then(|device| device.language).as_deref(), + expected + ); + } + for profile in ["prebid-server", "aps", "standard"] { + let config = if profile == "aps" { + json!({"account_id": "example-account-id"}) + } else { + json!({}) + }; + let request = build_with_request( + profile, + config, + canonical_parity_auction_request(), + Some("en-US,en;q=0.9"), + ); + assert_eq!( + request.device.and_then(|device| device.language).as_deref(), + Some("en") + ); + } +} + +#[test] +fn pbs_debug_query_fragment_preserves_exact_legacy_configured_semantics() { + for (page, fragment, expected) in [ + ( + "https://publisher.example/article", + "pbjs_debug=true", + "https://publisher.example/article?pbjs_debug=true", + ), + ( + "https://publisher.example/article?existing=1", + "pbjs_debug=true", + "https://publisher.example/article?existing=1&pbjs_debug=true", + ), + ( + "https://publisher.example/article", + "?pbjs_debug=true", + "https://publisher.example/article??pbjs_debug=true", + ), + ( + "https://publisher.example/article?pbjs_debug=true", + "pbjs_debug=true", + "https://publisher.example/article?pbjs_debug=true", + ), + ( + "https://publisher.example/article", + "", + "https://publisher.example/article", + ), + ] { + let mut request = canonical_parity_auction_request(); + request.publisher.page_url = Some(page.to_string()); + let built = build_with_request( + "prebid-server", + json!({"debug_query_params": fragment}), + request, + None, + ); + assert_eq!( + built.site.and_then(|site| site.page).as_deref(), + Some(expected), + "should preserve exact legacy query fragment semantics" + ); + } +} + +#[test] +fn pbs_routed_overrides_are_ordered_and_stored_request_is_trusted_fallback() { + let mut raw = config( + "prebid-server", + json!({ + "debug": true, + "test_mode": true, + "bid_param_overrides": {"exampleBidder": {"generic": 1, "shared": "generic"}}, + "bid_param_zone_overrides": {"exampleBidder": {"zone-a": {"zone": 2, "shared": "zone"}}}, + "bid_param_override_rules": [ + {"when":{"bidder":"exampleBidder"},"set":{"ordered":1,"shared":"rule-one"}}, + {"when":{"bidder":"exampleBidder","zone":"zone-a"},"set":{"ordered":2,"shared":"rule-two"}} + ] + }), + ); + raw.providers + .get_mut(&ProviderId::from_str("fictional-provider").expect("should parse provider")) + .expect("should find provider") + .routing = RoutingMode::Explicit; + raw.bidders.insert( + crate::auction::plan::BidderId::from_str("exampleBidder").expect("should parse bidder"), + BidderRouteConfig { + provider: ProviderId::from_str("fictional-provider").expect("should parse provider"), + }, + ); + let plan = AuctionPlan::compile(raw).expect("should compile PBS override plan"); + let mut request = canonical_parity_auction_request(); + request.slots[0].bidders = HashMap::from([( + "trustedServer".to_string(), + json!({"zone":"zone-a","bidderParams":{"exampleBidder":{"original":true,"shared":"original"}}}), + )]); + let inbound = Request::builder() + .uri("https://publisher.example/auction") + .body(EdgeBody::empty()) + .expect("should build inbound request"); + let routed = route_auction(request, &inbound, &plan, None); + let built = match build_request( + &routed.inputs()[0], + &routed, + &plan.providers()[0], + 321, + &finalization(None), + ) + .expect("should build request") + { + OpenRtbBuildOutcome::Ready(request) => request, + OpenRtbBuildOutcome::NoImpressions => panic!("should retain impression"), + }; + let value = serde_json::to_value(built).expect("should serialize request"); + assert_eq!( + value["imp"][0]["ext"]["prebid"]["bidder"]["exampleBidder"], + json!({"generic":1,"ordered":2,"original":true,"shared":"rule-two","zone":2}) + ); + assert_eq!(value["ext"]["prebid"]["debug"], true); + assert_eq!(value["ext"]["prebid"]["returnallbidstatus"], true); + assert_eq!(value["test"], 1); + + let mut empty_overridden = canonical_parity_auction_request(); + empty_overridden.slots[0].bidders = HashMap::from([( + "trustedServer".to_string(), + json!({"zone":"zone-a","bidderParams":{"exampleBidder":{}}}), + )]); + let routed = route_auction(empty_overridden, &inbound, &plan, None); + let built = match build_request( + &routed.inputs()[0], + &routed, + &plan.providers()[0], + 321, + &finalization(None), + ) + .expect("should build request after populating empty params") + { + OpenRtbBuildOutcome::Ready(request) => request, + OpenRtbBuildOutcome::NoImpressions => panic!("should retain overridden impression"), + }; + let value = serde_json::to_value(built).expect("should serialize overridden request"); + assert_eq!( + value["imp"][0]["ext"]["prebid"]["bidder"]["exampleBidder"], + json!({"generic":1,"ordered":2,"shared":"rule-two","zone":2}), + "should allow profile overrides to populate empty browser params" + ); + + let mut stored = canonical_parity_auction_request(); + stored.slots[0].bidders.clear(); + let routed = route_auction(stored, &inbound, &plan, None); + let built = match build_request( + &routed.inputs()[0], + &routed, + &plan.providers()[0], + 321, + &finalization(None), + ) + .expect("should build stored request") + { + OpenRtbBuildOutcome::Ready(request) => request, + OpenRtbBuildOutcome::NoImpressions => panic!("should retain impression"), + }; + let value = serde_json::to_value(built).expect("should serialize stored request"); + assert_eq!( + value["imp"][0]["ext"]["prebid"]["storedrequest"]["id"], + "fictional-slot" + ); +} + +#[test] +fn pbs_pairs_each_impression_with_its_routed_slot_params() { + let mut raw = config("prebid-server", json!({})); + raw.providers + .get_mut(&ProviderId::from_str("fictional-provider").expect("should parse provider")) + .expect("should find provider") + .routing = RoutingMode::Explicit; + raw.bidders.insert( + crate::auction::plan::BidderId::from_str("exampleBidder").expect("should parse bidder"), + BidderRouteConfig { + provider: ProviderId::from_str("fictional-provider").expect("should parse provider"), + }, + ); + let plan = AuctionPlan::compile(raw).expect("should compile PBS plan"); + let mut request = canonical_parity_auction_request(); + request.slots[0].id = "first-slot".to_string(); + request.slots[0].bidders = HashMap::from([( + "trustedServer".to_string(), + json!({"bidderParams":{"exampleBidder":{"placement":"first"}}}), + )]); + let mut second_slot = request.slots[0].clone(); + second_slot.id = "second-slot".to_string(); + second_slot.bidders = HashMap::from([( + "trustedServer".to_string(), + json!({"bidderParams":{"exampleBidder":{"placement":"second"}}}), + )]); + request.slots.push(second_slot); + let inbound = Request::builder() + .uri("https://publisher.example/auction") + .body(EdgeBody::empty()) + .expect("should build inbound request"); + let routed = route_auction(request, &inbound, &plan, None); + + let built = match build_request( + &routed.inputs()[0], + &routed, + &plan.providers()[0], + 321, + &finalization(None), + ) + .expect("should build request") + { + OpenRtbBuildOutcome::Ready(request) => request, + OpenRtbBuildOutcome::NoImpressions => panic!("should retain impressions"), + }; + let value = serde_json::to_value(built).expect("should serialize request"); + + assert_eq!(value["imp"][0]["id"], "first-slot"); + assert_eq!( + value["imp"][0]["ext"]["prebid"]["bidder"]["exampleBidder"], + json!({"placement":"first"}) + ); + assert_eq!(value["imp"][1]["id"], "second-slot"); + assert_eq!( + value["imp"][1]["ext"]["prebid"]["bidder"]["exampleBidder"], + json!({"placement":"second"}) + ); +} + +#[test] +fn pbs_empty_params_without_matching_override_fall_back_to_stored_request() { + let mut raw = config("prebid-server", json!({})); + raw.providers + .get_mut(&ProviderId::from_str("fictional-provider").expect("should parse provider")) + .expect("should find provider") + .routing = RoutingMode::Explicit; + raw.bidders.insert( + crate::auction::plan::BidderId::from_str("exampleBidder").expect("should parse bidder"), + BidderRouteConfig { + provider: ProviderId::from_str("fictional-provider").expect("should parse provider"), + }, + ); + let plan = AuctionPlan::compile(raw).expect("should compile PBS plan"); + let mut request = canonical_parity_auction_request(); + request.slots[0].bidders = HashMap::from([( + "trustedServer".to_string(), + json!({"bidderParams":{"exampleBidder":{}}}), + )]); + let inbound = Request::builder() + .uri("https://publisher.example/auction") + .body(EdgeBody::empty()) + .expect("should build inbound request"); + let routed = route_auction(request, &inbound, &plan, None); + let built = match build_request( + &routed.inputs()[0], + &routed, + &plan.providers()[0], + 321, + &finalization(None), + ) + .expect("should build stored request") + { + OpenRtbBuildOutcome::Ready(request) => request, + OpenRtbBuildOutcome::NoImpressions => panic!("should retain stored impression"), + }; + let value = serde_json::to_value(built).expect("should serialize stored request"); + assert_eq!( + value["imp"][0]["ext"]["prebid"]["storedrequest"]["id"], + "fictional-slot" + ); + assert!(value["imp"][0]["ext"]["prebid"].get("bidder").is_none()); +} + +#[test] +fn pbs_driver_exact_golden_preserves_profile_policy() { + let mut raw = config("prebid-server", json!({"consent_forwarding": "both"})); + raw.providers + .get_mut(&ProviderId::from_str("fictional-provider").expect("should parse provider")) + .expect("should find provider") + .routing = RoutingMode::Explicit; + raw.bidders.insert( + crate::auction::plan::BidderId::from_str("exampleBidder").expect("should parse bidder"), + BidderRouteConfig { + provider: ProviderId::from_str("fictional-provider").expect("should parse provider"), + }, + ); + let plan = AuctionPlan::compile(raw).expect("should compile PBS plan"); + let mut common = canonical_parity_auction_request(); + common.slots[0].bidders = HashMap::from([( + "exampleBidder".to_string(), + json!({"placement": "fictional-placement"}), + )]); + let inbound = Request::builder() + .uri("https://publisher.example/auction") + .header( + header::REFERER, + "https://referrer.example/story?fictional=1", + ) + .header(header::ACCEPT_LANGUAGE, "en-US,en;q=0.9") + .header("dnt", "1") + .body(EdgeBody::empty()) + .expect("should build inbound request"); + let routed = route_auction(common, &inbound, &plan, None); + let request = match build_request( + &routed.inputs()[0], + &routed, + &plan.providers()[0], + 321, + &finalization(None), + ) + .expect("should build PBS request") + { + OpenRtbBuildOutcome::Ready(request) => request, + OpenRtbBuildOutcome::NoImpressions => panic!("should retain impression"), + }; + assert_eq!( + serde_json::to_string(&request).expect("should serialize PBS driver request"), + r#"{"id":"fictional-auction","imp":[{"id":"fictional-slot","banner":{"format":[{"w":300,"h":250},{"w":728,"h":90}]},"tagid":"fictional-slot","bidfloor":1.0,"bidfloorcur":"USD","secure":1,"ext":{"prebid":{"bidder":{"exampleBidder":{"placement":"fictional-placement"}}}}}],"site":{"domain":"publisher.example","page":"https://publisher.example/article","ref":"https://referrer.example/story?fictional=1","publisher":{"domain":"publisher.example"}},"device":{"geo":{"lat":12.34,"lon":56.78,"type":2,"country":"US","region":"CA","metro":"501","city":"Example City"},"dnt":1,"ua":"Fictional Browser","ip":"192.0.2.10","language":"en"},"user":{"id":"fictional-user","consent":"fictional-tcf","ext":{"ConsentedProvidersSettings":{"consented_providers":"fictional-ac"},"consent":"fictional-tcf","eids":[{"source":"identity.example","uids":[{"atype":1,"id":"fictional-uid"}]}]}},"tmax":321,"cur":["USD"],"regs":{"gdpr":1,"us_privacy":"1YNN","gpp":"fictional-gpp","gpp_sid":[2,6],"ext":{"gdpr":1,"gpp":"fictional-gpp","gpp_sid":[2,6],"us_privacy":"1YNN"}},"ext":{"prebid":{},"trusted_server":{"request_host":"publisher.example","request_scheme":"https"}}}"#, + "should preserve PBS parity differences" + ); +} + +#[test] +fn aps_inventory_identity_and_page_fallback_preserve_legacy_policy() { + let mut request = canonical_parity_auction_request(); + request.publisher.domain = "deployment.example".to_string(); + request.publisher.page_url = + Some("https://deployment.example/news/story?edition=fictional#section".to_string()); + let built = build_with_request( + "aps", + json!({ + "account_id": "example-account-id", + "inventory_domain": "publisher.example", + "inventory_page_origin": "https://www.publisher.example" + }), + request, + None, + ); + let site = built.site.expect("should include APS site"); + assert_eq!(site.domain.as_deref(), Some("publisher.example")); + assert_eq!( + site.page.as_deref(), + Some("https://www.publisher.example/news/story?edition=fictional") + ); + assert_eq!( + site.publisher + .and_then(|publisher| publisher.domain) + .as_deref(), + Some("publisher.example") + ); + + for unsafe_page in [ + "https://user:password@publisher.example/private", + "data:text/html,fictional", + ] { + let mut request = canonical_parity_auction_request(); + request.publisher.page_url = Some(unsafe_page.to_string()); + let built = build_with_request( + "aps", + json!({"account_id":"example-account-id"}), + request, + None, + ); + assert_eq!( + built.site.and_then(|site| site.page).as_deref(), + Some("https://publisher.example"), + "unsafe page should fall back to publisher domain" + ); + } +} + +#[test] +fn aps_driver_exact_golden_preserves_profile_policy() { + let request = build("aps", json!({"account_id": "example-account-id"}), None); + assert_eq!( + serde_json::to_string(&request).expect("should serialize APS driver request"), + r#"{"id":"fictional-auction","imp":[{"id":"fictional-slot","banner":{"format":[{"w":300,"h":250},{"w":728,"h":90}],"w":300,"h":250,"topframe":0},"bidfloor":1.0,"bidfloorcur":"USD","secure":1}],"site":{"domain":"publisher.example","page":"https://publisher.example/article","publisher":{"domain":"publisher.example"}},"device":{"geo":{"type":2,"country":"US","region":"CA","metro":"501","city":"Example City"},"dnt":1,"ua":"Fictional Browser","ip":"192.0.2.10","language":"en"},"user":{"id":"fictional-user","consent":"fictional-tcf","ext":{"consent":"fictional-tcf","eids":[{"source":"identity.example","uids":[{"atype":1,"id":"fictional-uid"}]}]}},"tmax":321,"cur":["USD"],"regs":{"gdpr":1,"us_privacy":"1YNN","gpp":"fictional-gpp","gpp_sid":[2,6],"ext":{"gdpr":1,"gpp":"fictional-gpp","gpp_sid":[2,6],"us_privacy":"1YNN"}},"ext":{"account":"example-account-id","sdk":{"source":"prebid","version":"2.2.0"}}}"#, + "should preserve APS parity differences" + ); +} + +#[test] +fn signing_finalization_is_after_profiles_and_asserts_every_owned_key() { + let signer = deterministic_signer(); + for (profile, config) in [ + ("standard", json!({"request_ext": {"fictional": true}})), + ("prebid-server", json!({})), + ("aps", json!({"account_id": "example-account-id"})), + ] { + let unsigned = serde_json::to_value(build(profile, config.clone(), None)) + .expect("should serialize unsigned request"); + let unsigned_ts = unsigned["ext"].get("trusted_server"); + if profile == "prebid-server" { + assert_eq!( + unsigned_ts, + Some(&json!({"request_host": "publisher.example", "request_scheme": "https"})), + "should retain only PBS host and scheme when unsigned" + ); + } else { + assert!( + unsigned_ts.is_none(), + "should omit unsigned non-PBS extension" + ); + } + + let signed = serde_json::to_value(build(profile, config, Some(&signer))) + .expect("should serialize signed request"); + let extension = &signed["ext"]["trusted_server"]; + assert_eq!(extension["version"], "1.1", "should set signing version"); + assert_eq!(extension["kid"], "fictional-kid", "should set key ID"); + assert_eq!( + extension["request_host"], "publisher.example", + "should set host" + ); + assert_eq!(extension["request_scheme"], "https", "should set scheme"); + assert_eq!( + extension["ts"], 1_706_900_000_u64, + "should set explicit time" + ); + assert!( + extension["signature"] + .as_str() + .is_some_and(|value| !value.is_empty()), + "should set signature" + ); + } +} + +#[test] +fn signed_profiles_and_unsigned_standard_have_exact_full_goldens() { + let signer = deterministic_signer(); + let cases = [ + ( + "standard", + json!({"request_ext": {"fictional": true}}), + r#"{"id":"fictional-auction","imp":[{"id":"fictional-slot","banner":{"format":[{"w":300,"h":250},{"w":728,"h":90}]},"bidfloor":1.0,"bidfloorcur":"USD","secure":1}],"site":{"domain":"publisher.example","page":"https://publisher.example/article","publisher":{"domain":"publisher.example"}},"device":{"geo":{"type":2,"country":"US","region":"CA","metro":"501","city":"Example City"},"dnt":1,"ua":"Fictional Browser","ip":"192.0.2.10","language":"en"},"user":{"id":"fictional-user","consent":"fictional-tcf","ext":{"consent":"fictional-tcf","eids":[{"source":"identity.example","uids":[{"atype":1,"id":"fictional-uid"}]}]}},"tmax":321,"cur":["USD"],"regs":{"gdpr":1,"us_privacy":"1YNN","gpp":"fictional-gpp","gpp_sid":[2,6],"ext":{"gdpr":1,"gpp":"fictional-gpp","gpp_sid":[2,6],"us_privacy":"1YNN"}},"ext":{"fictional":true,"trusted_server":{"kid":"fictional-kid","request_host":"publisher.example","request_scheme":"https","signature":"LU_JUIA1BT80ShZNjSa4PIF5T-uMjEeodwKrV_6bXgh0hi1SYVtCKn9g_DTW62krmjCOFgoFYPHsu6L0nAcuDg","ts":1706900000,"version":"1.1"}}}"#, + ), + ( + "prebid-server", + json!({}), + r#"{"id":"fictional-auction","imp":[{"id":"fictional-slot","banner":{"format":[{"w":300,"h":250},{"w":728,"h":90}]},"tagid":"fictional-slot","bidfloor":1.0,"bidfloorcur":"USD","secure":1,"ext":{"prebid":{}}}],"site":{"domain":"publisher.example","page":"https://publisher.example/article","ref":"https://referrer.example/story?fictional=1","publisher":{"domain":"publisher.example"}},"device":{"geo":{"lat":12.34,"lon":56.78,"type":2,"country":"US","region":"CA","metro":"501","city":"Example City"},"dnt":1,"ua":"Fictional Browser","ip":"192.0.2.10","language":"en"},"user":{"id":"fictional-user","consent":"fictional-tcf","ext":{"ConsentedProvidersSettings":{"consented_providers":"fictional-ac"},"consent":"fictional-tcf","eids":[{"source":"identity.example","uids":[{"atype":1,"id":"fictional-uid"}]}]}},"tmax":321,"cur":["USD"],"regs":{"gdpr":1,"us_privacy":"1YNN","gpp":"fictional-gpp","gpp_sid":[2,6],"ext":{"gdpr":1,"gpp":"fictional-gpp","gpp_sid":[2,6],"us_privacy":"1YNN"}},"ext":{"prebid":{},"trusted_server":{"kid":"fictional-kid","request_host":"publisher.example","request_scheme":"https","signature":"LU_JUIA1BT80ShZNjSa4PIF5T-uMjEeodwKrV_6bXgh0hi1SYVtCKn9g_DTW62krmjCOFgoFYPHsu6L0nAcuDg","ts":1706900000,"version":"1.1"}}}"#, + ), + ( + "aps", + json!({"account_id": "example-account-id"}), + r#"{"id":"fictional-auction","imp":[{"id":"fictional-slot","banner":{"format":[{"w":300,"h":250},{"w":728,"h":90}],"w":300,"h":250,"topframe":0},"bidfloor":1.0,"bidfloorcur":"USD","secure":1}],"site":{"domain":"publisher.example","page":"https://publisher.example/article","publisher":{"domain":"publisher.example"}},"device":{"geo":{"type":2,"country":"US","region":"CA","metro":"501","city":"Example City"},"dnt":1,"ua":"Fictional Browser","ip":"192.0.2.10","language":"en"},"user":{"id":"fictional-user","consent":"fictional-tcf","ext":{"consent":"fictional-tcf","eids":[{"source":"identity.example","uids":[{"atype":1,"id":"fictional-uid"}]}]}},"tmax":321,"cur":["USD"],"regs":{"gdpr":1,"us_privacy":"1YNN","gpp":"fictional-gpp","gpp_sid":[2,6],"ext":{"gdpr":1,"gpp":"fictional-gpp","gpp_sid":[2,6],"us_privacy":"1YNN"}},"ext":{"account":"example-account-id","sdk":{"source":"prebid","version":"2.2.0"},"trusted_server":{"kid":"fictional-kid","request_host":"publisher.example","request_scheme":"https","signature":"LU_JUIA1BT80ShZNjSa4PIF5T-uMjEeodwKrV_6bXgh0hi1SYVtCKn9g_DTW62krmjCOFgoFYPHsu6L0nAcuDg","ts":1706900000,"version":"1.1"}}}"#, + ), + ]; + for (profile, config, expected) in cases { + assert_eq!( + serde_json::to_string(&build(profile, config, Some(&signer))) + .expect("should serialize signed request"), + expected, + "{profile} signed wire fixture should stay exact" + ); + } + + assert_eq!( + serde_json::to_string(&build( + "standard", + json!({"request_ext": {"fictional": true}}), + None, + )) + .expect("should serialize unsigned standard request"), + r#"{"id":"fictional-auction","imp":[{"id":"fictional-slot","banner":{"format":[{"w":300,"h":250},{"w":728,"h":90}]},"bidfloor":1.0,"bidfloorcur":"USD","secure":1}],"site":{"domain":"publisher.example","page":"https://publisher.example/article","publisher":{"domain":"publisher.example"}},"device":{"geo":{"type":2,"country":"US","region":"CA","metro":"501","city":"Example City"},"dnt":1,"ua":"Fictional Browser","ip":"192.0.2.10","language":"en"},"user":{"id":"fictional-user","consent":"fictional-tcf","ext":{"consent":"fictional-tcf","eids":[{"source":"identity.example","uids":[{"atype":1,"id":"fictional-uid"}]}]}},"tmax":321,"cur":["USD"],"regs":{"gdpr":1,"us_privacy":"1YNN","gpp":"fictional-gpp","gpp_sid":[2,6],"ext":{"gdpr":1,"gpp":"fictional-gpp","gpp_sid":[2,6],"us_privacy":"1YNN"}},"ext":{"fictional":true}}"#, + "unsigned standard wire fixture should stay exact" + ); +} + +#[test] +fn standard_static_extensions_have_no_invented_bidder_param_location() { + let request = build( + "standard", + json!({ + "request_ext": {"fictional_request": {"enabled": true}}, + "imp_ext": {"fictional_imp": "value"} + }), + None, + ); + let value = serde_json::to_value(request).expect("should serialize request"); + assert_eq!(value["ext"]["fictional_request"]["enabled"], true); + assert_eq!(value["imp"][0]["ext"]["fictional_imp"], "value"); + assert!( + !value.to_string().contains("exampleBidder"), + "standard profile must not invent bidder params placement" + ); +} + +#[test] +fn defensive_no_impression_outcome_does_not_build_transportable_request() { + let (plan, mut routed) = routed("standard", json!({})); + let mut common = routed.inputs()[0].common_request().clone(); + common.slots = vec![AdSlot { + id: "video-only".to_string(), + formats: vec![AdFormat { + media_type: MediaType::Video, + width: 640, + height: 480, + }], + floor_price: None, + targeting: HashMap::new(), + bidders: HashMap::new(), + }]; + let inbound = Request::builder() + .uri("https://publisher.example/auction") + .body(EdgeBody::empty()) + .expect("should build inbound request"); + routed = route_auction(common, &inbound, &plan, None); + assert!( + routed.inputs().is_empty(), + "should omit provider input before build" + ); +} + +fn standard_fixture() -> (AuctionPlan, RoutedAuction, OpenRtbRequest) { + let mut raw = config( + "standard", + json!({"request_ext": {"fixture": true}, "imp_ext": {"slot_fixture": true}}), + ); + raw.providers + .get_mut(&ProviderId::from_str("fictional-provider").expect("should parse provider")) + .expect("should find provider") + .routing = RoutingMode::Explicit; + raw.bidders.insert( + crate::auction::plan::BidderId::from_str("exampleBidder").expect("should parse bidder"), + BidderRouteConfig { + provider: ProviderId::from_str("fictional-provider").expect("should parse provider"), + }, + ); + let plan = AuctionPlan::compile(raw).expect("should compile standard fixture plan"); + let inbound = Request::builder() + .uri("https://publisher.example/auction") + .body(EdgeBody::empty()) + .expect("should build inbound request"); + let routed = route_auction(canonical_parity_auction_request(), &inbound, &plan, None); + let request = match build_request( + &routed.inputs()[0], + &routed, + &plan.providers()[0], + 321, + &finalization(None), + ) + .expect("should build standard fixture request") + { + OpenRtbBuildOutcome::Ready(request) => request, + OpenRtbBuildOutcome::NoImpressions => panic!("should retain impression"), + }; + (plan, routed, request) +} + +#[test] +fn standard_response_extraction_isolates_malformed_siblings_and_ignores_response_id() { + let (_plan, routed, _request) = standard_fixture(); + let response = extract_standard_response( + "fictional-provider", + &routed.inputs()[0], + &json!({ + "id": "informational-mismatch", + "seatbid": [{"seat": "fictional-seat", "bid": [ + {"id": "good", "impid": "fictional-slot", "price": 1.5, "adm": "
ok
", "w": 300, "h": 250}, + {"id": "bad", "impid": "fictional-slot", "price": "bad", "adm": "
bad
", "w": 300, "h": 250} + ]}] + }), + 9, + ); + assert_eq!(response.status, BidStatus::Success); + assert_eq!(response.bids.len(), 1, "should isolate malformed sibling"); + assert_eq!( + response.bids[0].returned_seat.as_deref(), + Some("fictional-seat") + ); +} + +#[test] +fn standard_response_currency_accepts_omitted_and_usd_but_rejects_other_or_malformed_values() { + let (_plan, routed, _request) = standard_fixture(); + let bid = json!({"seatbid": [{"seat": "seat", "bid": [ + {"id":"good","impid":"fictional-slot","price":1.0,"adm":"ok","w":300,"h":250} + ]}]}); + + for currency in [None, Some(json!("USD")), Some(json!("usd"))] { + let mut value = bid.clone(); + if let Some(currency) = currency { + value["cur"] = currency; + } + let response = + extract_standard_response("fictional-provider", &routed.inputs()[0], &value, 0); + assert_eq!( + response.status, + BidStatus::Success, + "should accept omitted or USD currency" + ); + assert_eq!(response.bids[0].currency, "USD"); + } + + let mut eur = bid.clone(); + eur["cur"] = json!("EUR"); + let response = extract_standard_response("fictional-provider", &routed.inputs()[0], &eur, 0); + assert_eq!(response.status, BidStatus::NoBid); + assert_eq!(response.metadata["unsupported_currency"], "EUR"); + + let mut malformed = bid; + malformed["cur"] = json!(["USD"]); + let response = + extract_standard_response("fictional-provider", &routed.inputs()[0], &malformed, 0); + assert_eq!(response.status, BidStatus::Error); + assert_eq!(response.metadata["error_type"], "parse_response"); +} + +#[test] +fn standard_response_rejects_unknown_impressions_and_dimensions_but_keeps_siblings() { + let (_plan, routed, _request) = standard_fixture(); + let response = extract_standard_response( + "fictional-provider", + &routed.inputs()[0], + &json!({"seatbid": [{"seat": "seat", "bid": [ + {"id":"good","impid":"fictional-slot","price":1.0,"adm":"ok","w":300,"h":250}, + {"id":"unknown","impid":"unknown-slot","price":2.0,"adm":"bad","w":300,"h":250}, + {"id":"dimension","impid":"fictional-slot","price":3.0,"adm":"bad","w":320,"h":50} + ]}]}), + 0, + ); + assert_eq!(response.status, BidStatus::Success); + assert_eq!(response.bids.len(), 1); + assert_eq!(response.bids[0].bid_id.as_deref(), Some("good")); +} + +#[test] +fn notification_suppression_matrix_uses_only_exact_valid_returned_seat() { + let (_plan, routed, _request) = standard_fixture(); + let response = extract_standard_response( + "fictional-provider", + &routed.inputs()[0], + &json!({"seatbid": [ + {"seat": "exact", "bid": [{"id":"exact","impid":"fictional-slot","price":1.0,"adm":"ok","w":300,"h":250,"nurl":"https://n.example","burl":"https://b.example"}]}, + {"seat": "Exact", "bid": [{"id":"case","impid":"fictional-slot","price":1.0,"adm":"ok","w":300,"h":250,"nurl":"https://n.example","burl":"https://b.example"}]}, + {"bid": [{"id":"missing","impid":"fictional-slot","price":1.0,"adm":"ok","w":300,"h":250,"nurl":"https://n.example","burl":"https://b.example"}]}, + {"seat": 7, "bid": [{"id":"nonstring","impid":"fictional-slot","price":1.0,"adm":"ok","w":300,"h":250,"nurl":"https://n.example","burl":"https://b.example"}]} + ]}), + 0, + ); + let mut exact = response.bids.clone(); + apply_notification_policy( + &mut exact, + &NotificationPolicy { + suppress_all: false, + suppress_seats: BTreeSet::from(["exact".to_string(), "unknown".to_string()]), + }, + ); + assert!(exact[0].nurl.is_none(), "should suppress exact seat"); + assert!(exact[1].nurl.is_some(), "matching should be case-sensitive"); + assert!(exact[2].nurl.is_some(), "missing seat must not match"); + assert!(exact[3].nurl.is_some(), "non-string seat must not match"); + assert_eq!(exact[2].bidder, "unknown"); + assert_eq!(exact[3].bidder, "unknown"); + + let mut all = response.bids; + apply_notification_policy( + &mut all, + &NotificationPolicy { + suppress_all: true, + suppress_seats: BTreeSet::new(), + }, + ); + assert!( + all.iter() + .all(|bid| bid.nurl.is_none() && bid.burl.is_none()), + "suppress_all should remove every notification" + ); +} + +#[test] +fn fictional_standard_executor_covers_bid_no_bid_malformed_unused_and_redirect() { + futures::executor::block_on(async { + let (plan, routed, request) = standard_fixture(); + let provider = &plan.providers()[0]; + let client = Arc::new(StubHttpClient::new()); + client.push_response( + 200, + serde_json::to_vec(&json!({"id":"mismatch","seatbid":[{"seat":"fictional-seat","bid":[ + {"id":"good","impid":"fictional-slot","price":1.25,"adm":"
fictional
","w":300,"h":250}, + {"id":"bad","impid":"fictional-slot","price":null,"adm":"bad","w":300,"h":250} + ]}]})).expect("should serialize response"), + ); + let response = execute_standard_fixture( + provider, + &routed.inputs()[0], + &request, + &StubBackend, + client.as_ref(), + ) + .await + .expect("should execute ordinary fixture"); + assert_eq!(response.status, BidStatus::Success); + assert_eq!(response.bids.len(), 1, "should isolate malformed sibling"); + assert_eq!( + response.metadata["routing"]["unused_bidder_params_count"], + 1 + ); + assert_eq!( + client.recorded_request_uris(), + vec![provider.endpoint.as_str()] + ); + let headers = &client.recorded_request_headers()[0]; + assert!( + !headers.iter().any(|(name, _)| name == "authorization"), + "fixture must add no authentication" + ); + let body: Value = serde_json::from_slice(&client.recorded_request_bodies()[0]) + .expect("should parse recorded body"); + assert_eq!(body["ext"]["fixture"], true, "should send static extension"); + + let no_bid = Arc::new(StubHttpClient::new()); + no_bid.push_response(204, Vec::new()); + let response = execute_standard_fixture( + provider, + &routed.inputs()[0], + &request, + &StubBackend, + no_bid.as_ref(), + ) + .await + .expect("should execute no-bid fixture"); + assert_eq!(response.status, BidStatus::NoBid); + + let malformed = Arc::new(StubHttpClient::new()); + malformed.push_response(200, b"not-json".to_vec()); + let response = execute_standard_fixture( + provider, + &routed.inputs()[0], + &request, + &StubBackend, + malformed.as_ref(), + ) + .await + .expect("should classify malformed response"); + assert_eq!(response.status, BidStatus::Error); + assert_eq!(response.metadata["error_type"], "parse_response"); + + let redirect = Arc::new(StubHttpClient::new()); + redirect.push_response_with_headers( + 302, + Vec::new(), + vec![("location", "https://redirect.example.test/openrtb")], + ); + let response = execute_standard_fixture( + provider, + &routed.inputs()[0], + &request, + &StubBackend, + redirect.as_ref(), + ) + .await + .expect("should classify redirect"); + assert_eq!(response.status, BidStatus::Error); + assert_eq!(response.metadata["error_type"], "http_status"); + assert_eq!(response.metadata["http_status"], 302); + assert_eq!( + redirect.recorded_request_uris(), + vec![provider.endpoint.as_str()], + "a 3xx Location must not trigger a second HTTP request" + ); + assert_eq!( + redirect.recorded_backend_names(), + vec!["stub-backend"], + "the common driver must perform exactly one underlying send for a 3xx" + ); + let spec = provider.backend_spec(); + assert_eq!(spec.host, "exchange.example.test"); + assert_eq!(spec.discriminator.as_deref(), Some("fictional-provider")); + }); +} + +#[test] +fn prebid_endpoint_normalization_reaches_generic_execution_and_preserves_custom_paths() { + futures::executor::block_on(async { + for (configured_endpoint, expected_endpoint) in [ + ( + "https://pbs.example", + "https://pbs.example/openrtb2/auction", + ), + ("https://pbs.example/bid", "https://pbs.example/bid"), + ] { + let plan = AuctionPlan::compile(config_with_endpoint( + "prebid-server", + json!({}), + configured_endpoint, + )) + .expect("should compile Prebid Server endpoint"); + let inbound = Request::builder() + .uri("https://publisher.example/auction") + .body(EdgeBody::empty()) + .expect("should build inbound request"); + let routed = route_auction(canonical_parity_auction_request(), &inbound, &plan, None); + let provider_plan = plan.providers()[0].clone(); + let provider = GenericOpenRtbProvider::new(provider_plan.clone()); + let client = Arc::new(StubHttpClient::new()); + client.push_response(204, Vec::new()); + let services = build_services_with_backend_and_http_client( + Arc::new(StubBackend), + Arc::clone(&client) as Arc, + ); + let mut reserved_backend_names = HashSet::new(); + + let outcome = provider + .request_bids_routed( + &routed.inputs()[0], + &routed, + 321, + 321, + None, + &services, + &mut reserved_backend_names, + ) + .await + .expect("should launch one Prebid Server request"); + let ProviderRequestOutcome::Pending { request, .. } = outcome else { + panic!("should launch a pending Prebid Server request"); + }; + let selected = services + .http_client() + .select(vec![request]) + .await + .expect("should select one Prebid Server response"); + let response = selected + .ready + .expect("should receive the Prebid Server response"); + + assert_eq!(response.response.status(), http::StatusCode::NO_CONTENT); + assert_eq!(client.recorded_backend_names(), vec!["stub-backend"]); + assert_eq!(client.recorded_request_methods(), vec!["POST"]); + assert_eq!(client.recorded_request_uris(), vec![expected_endpoint]); + assert_eq!(provider_plan.backend_spec().host, "pbs.example"); + let body: Value = serde_json::from_slice(&client.recorded_request_bodies()[0]) + .expect("should parse recorded Prebid Server body"); + assert!( + !body + .as_object() + .expect("should serialize an object") + .is_empty() + ); + } + }); +} + +#[test] +fn malformed_top_level_standard_response_is_error() { + let (_plan, routed, _request) = standard_fixture(); + let response = + extract_standard_response("fictional-provider", &routed.inputs()[0], &json!([]), 0); + assert_eq!(response.status, BidStatus::Error); + assert_eq!(response.metadata["error_type"], "parse_response"); +} diff --git a/crates/trusted-server-core/src/auction/orchestrator.rs b/crates/trusted-server-core/src/auction/orchestrator.rs index 728cc1efe..763c3bef5 100644 --- a/crates/trusted-server-core/src/auction/orchestrator.rs +++ b/crates/trusted-server-core/src/auction/orchestrator.rs @@ -3,18 +3,26 @@ use edgezero_core::body::Body as EdgeBody; use error_stack::{Report, ResultExt}; use http::Request; -use std::collections::{HashMap, HashSet}; +use std::collections::{HashMap, HashSet, hash_map::Entry}; use std::sync::Arc; -use std::time::Duration; use web_time::Instant; use crate::error::TrustedServerError; use crate::platform::{PlatformPendingRequest, RuntimeServices}; +#[cfg(test)] use super::config::AuctionConfig; -use super::provider::{AuctionProvider, ProviderParseState, ProviderRequestOutcome}; +use super::openrtb::unused_bidder_params_count; +use super::plan::AuctionPlan; +use super::provider::{ + AuctionProvider, GenericOpenRtbProvider, ProviderParseState, ProviderRequestOutcome, +}; +#[cfg(test)] +use super::routing::RoutedAuction; +use super::routing::route_auction; use super::telemetry::AbandonedProviderCall; use super::types::{AuctionContext, AuctionRequest, AuctionResponse, Bid, BidStatus}; +use crate::request_signing::RequestSigner; /// In-flight auction requests dispatched to SSP backends. /// @@ -26,6 +34,7 @@ use super::types::{AuctionContext, AuctionRequest, AuctionResponse, Bid, BidStat pub struct DispatchedAuction { pending_requests: Vec, backend_to_provider: HashMap, + planned_backend_to_provider: HashMap, completed_responses: Vec, auction_start: Instant, timeout_ms: u32, @@ -33,6 +42,9 @@ pub struct DispatchedAuction { provider_request_context: Box>, /// Carried so the mediator call in collect can pass it as the auction request. request: AuctionRequest, + planned_unused_bidder_params: HashMap, + planned_unroutable_bidder_count: u32, + planned_provider_order: HashMap, } struct ProviderLaunchState { @@ -44,6 +56,7 @@ struct ProviderLaunchState { } /// Outcome of attempting to dispatch split-phase auction provider requests. +#[allow(clippy::large_enum_variant)] pub enum DispatchAuctionOutcome { /// No provider request was started and no provider failure was observed. NotStarted, @@ -53,6 +66,13 @@ pub enum DispatchAuctionOutcome { request: AuctionRequest, /// Provider launch-failure responses. provider_responses: Vec, + /// Fatal admission error that synchronous execution must propagate. + /// + /// Split publisher dispatch records the failure and continues without + /// attempting provider network I/O. + fatal_admission_error: Option>, + /// Auction-level metadata materialized before the failure. + metadata: HashMap, /// Elapsed dispatch time. elapsed_ms: u64, }, @@ -75,10 +95,16 @@ impl DispatchedAuction { let abandoned = self .backend_to_provider .into_values() - .map(|state| { + .map(|state| (state.provider_name, state.started_at)) + .chain( + self.planned_backend_to_provider + .into_values() + .map(|state| (state.provider.provider_name().to_string(), state.started_at)), + ) + .map(|(provider_name, started_at)| { AbandonedProviderCall::bidder( - state.provider_name, - Some(u32::try_from(state.started_at.elapsed().as_millis()).unwrap_or(u32::MAX)), + provider_name, + Some(u32::try_from(started_at.elapsed().as_millis()).unwrap_or(u32::MAX)), ) }) .collect(); @@ -97,12 +123,16 @@ impl DispatchedAuction { Self { pending_requests: Vec::new(), backend_to_provider: HashMap::new(), + planned_backend_to_provider: HashMap::new(), completed_responses: Vec::new(), auction_start: Instant::now(), timeout_ms, floor_prices: HashMap::new(), provider_request_context: Box::new(Request::new(EdgeBody::empty())), request, + planned_unused_bidder_params: HashMap::new(), + planned_unroutable_bidder_count: 0, + planned_provider_order: HashMap::new(), } } } @@ -182,6 +212,13 @@ fn provider_timeout_response(provider_name: &str, response_time_ms: u64) -> Auct .with_metadata("message", serde_json::json!("Provider request timed out")) } +fn provider_skipped_response(provider_name: &str) -> AuctionResponse { + AuctionResponse::no_bid(provider_name, 0).with_metadata( + "routing", + serde_json::json!({"skipped_no_eligible_slots": true}), + ) +} + /// Compute the remaining time budget from a deadline. /// /// Returns the number of milliseconds left before `timeout_ms` is exceeded, @@ -192,6 +229,68 @@ fn remaining_budget_ms(start: Instant, timeout_ms: u32) -> u32 { timeout_ms.saturating_sub(elapsed) } +/// Runtime policy for classifying responses that complete after the logical auction budget. +/// +/// Current adapters do not expose an enforceable total-request deadline. They +/// therefore drain already-launched work and accept completed late responses. +#[derive(Debug, Clone, Copy, Default)] +struct AuctionDeadlinePolicy { + enforceable_total_request_deadline: bool, +} + +impl AuctionDeadlinePolicy { + fn rejects_late_completion(self, start: Instant, timeout_ms: u32) -> bool { + self.enforceable_total_request_deadline && remaining_budget_ms(start, timeout_ms) == 0 + } + + fn for_runtime(services: &RuntimeServices) -> Self { + Self { + enforceable_total_request_deadline: services + .http_client() + .has_enforceable_total_request_deadline(), + } + } +} + +fn routing_metadata(unroutable_bidder_count: u32) -> HashMap { + HashMap::from([( + "routing".to_string(), + serde_json::json!({"unroutable_bidder_count": unroutable_bidder_count}), + )]) +} + +/// Attach only the count derived from the routed provider input at dispatch. +/// +/// This is intentionally applied after every provider outcome is materialized, +/// including failures produced before or during parsing. Skipped providers are +/// routed separately and retain their exclusive skipped diagnostic. +fn materialize_planned_response( + mut response: AuctionResponse, + unused_bidder_params_count: u32, +) -> AuctionResponse { + let routing = response + .metadata + .entry("routing".to_string()) + .or_insert_with(|| serde_json::json!({})); + if routing + .get("skipped_no_eligible_slots") + .is_some_and(|value| value == &serde_json::json!(true)) + { + return response; + } + if !routing.is_object() { + *routing = serde_json::json!({}); + } + routing + .as_object_mut() + .expect("should normalize planned routing metadata to an object") + .insert( + "unused_bidder_params_count".to_string(), + serde_json::json!(unused_bidder_params_count), + ); + response +} + fn snapshot_context_request(request: &Request) -> Request { let mut snapshot = Request::new(EdgeBody::empty()); *snapshot.method_mut() = request.method().clone(); @@ -203,22 +302,525 @@ fn snapshot_context_request(request: &Request) -> Request { /// Manages auction execution across multiple providers. pub struct AuctionOrchestrator { + enabled: bool, + plan_backed: bool, + plan: Arc, + planned_providers: Vec>, + mediator: Option>, + #[cfg(test)] config: AuctionConfig, + #[cfg(test)] providers: HashMap>, } +/// Test harness for the live plan-backed orchestrator semantics. +#[cfg(test)] +pub(crate) struct AuctionOrchestratorHarness { + plan: Arc, + providers: Vec>, + mediator: Option>, +} + +struct PlannedLaunchState { + provider: Arc, + started_at: Instant, + parse_state: Option, +} + +#[cfg(test)] +#[allow( + dead_code, + reason = "test harness exercises plan-backed runtime behavior" +)] +impl AuctionOrchestratorHarness { + pub(crate) fn new( + plan: impl Into>, + mediator: Option>, + ) -> Self { + let plan = plan.into(); + let providers = plan + .providers() + .iter() + .cloned() + .map(GenericOpenRtbProvider::new) + .map(Arc::new) + .collect(); + Self { + plan, + providers, + mediator, + } + } + + pub(crate) fn provider_count(&self) -> usize { + self.providers.len() + } + + pub(crate) fn mediator(&self) -> Option<&Arc> { + self.mediator.as_ref() + } + + /// Route and execute config-first bidder providers in deterministic order. + pub(crate) async fn run_auction( + &self, + request: &AuctionRequest, + context: &AuctionContext<'_>, + ) -> Result> { + // Admission, including signer-store reads and routing, consumes the same + // request-local deadline as provider transport and response collection. + let auction_start = Instant::now(); + let routed = route_auction( + request.clone(), + context.request, + &self.plan, + context.services.client_info().client_ip, + ); + if context.timeout_ms == 0 { + return self + .run_routed(request, &routed, context, None, auction_start) + .await; + } + if self.providers.len() > 1 && !context.services.http_client().supports_concurrent_fanout() + { + return Err(Report::new(TrustedServerError::Auction { + message: format!( + "{} auction providers configured, but this platform's HTTP client does not support concurrent fanout", + self.providers.len() + ), + })); + } + + // Signing admission deliberately precedes every backend call. + let signer = self + .plan + .signing_enabled() + .then(|| RequestSigner::from_services(context.services)) + .transpose()?; + self.run_routed(request, &routed, context, signer.as_ref(), auction_start) + .await + } + + async fn run_routed( + &self, + original_request: &AuctionRequest, + routed: &RoutedAuction, + context: &AuctionContext<'_>, + signer: Option<&RequestSigner>, + auction_start: Instant, + ) -> Result> { + let mut responses = routed + .skipped_no_eligible_provider_ids() + .iter() + .map(|id| provider_skipped_response(id.as_str())) + .collect::>(); + let planned_unused_bidder_params = routed + .inputs() + .iter() + .map(|input| { + ( + input.provider_id().as_str().to_string(), + unused_bidder_params_count( + &self + .plan + .provider(input.provider_id()) + .expect("should find routed provider in compiled plan") + .profile, + input, + ), + ) + }) + .collect::>(); + let mut pending = Vec::new(); + let mut launches = HashMap::new(); + let mut reserved_backend_names = HashSet::new(); + + for input in routed.inputs() { + let Some(provider) = self + .providers + .iter() + .find(|provider| provider.provider_name() == input.provider_id().as_str()) + .cloned() + else { + responses.push(provider_launch_failed_response( + input.provider_id().as_str(), + 0, + )); + continue; + }; + let remaining_ms = remaining_budget_ms(auction_start, context.timeout_ms); + let logical_budget_ms = remaining_ms.min(provider.timeout_ms()); + if logical_budget_ms == 0 { + responses.push(provider_timeout_response(provider.provider_name(), 0)); + continue; + } + let transport_timeout_ms = context + .services + .backend() + .canonicalize_transport_timeout_ms(logical_budget_ms, provider.timeout_ms()); + if transport_timeout_ms == 0 { + responses.push(provider_timeout_response(provider.provider_name(), 0)); + continue; + } + let started_at = Instant::now(); + match provider + .request_bids_routed( + input, + routed, + logical_budget_ms, + transport_timeout_ms, + signer, + context.services, + &mut reserved_backend_names, + ) + .await + { + Ok(ProviderRequestOutcome::Pending { + request: launched, + parse_state, + }) => { + let Some(backend_name) = launched.backend_name().map(str::to_string) else { + log::warn!( + "Planned provider '{}' pending request had no backend name", + provider.provider_name() + ); + responses.push(provider_launch_failed_response( + provider.provider_name(), + started_at.elapsed().as_millis() as u64, + )); + continue; + }; + match launches.entry(backend_name) { + Entry::Vacant(entry) => { + entry.insert(PlannedLaunchState { + provider, + started_at, + parse_state, + }); + pending.push(launched); + } + Entry::Occupied(entry) => { + log::warn!( + "Planned provider '{}' pending backend '{}' already belongs to another provider", + provider.provider_name(), + entry.key(), + ); + responses.push(provider_launch_failed_response( + provider.provider_name(), + started_at.elapsed().as_millis() as u64, + )); + } + } + } + Ok(ProviderRequestOutcome::Immediate(response)) => responses.push(response), + Err(error) => { + log::warn!( + "Planned provider '{}' failed to launch: {:?}", + provider.provider_name(), + error + ); + responses.push(provider_launch_failed_response( + provider.provider_name(), + started_at.elapsed().as_millis() as u64, + )); + } + } + } + + while !pending.is_empty() { + let select_result = match context.services.http_client().select(pending).await { + Ok(result) => result, + Err(error) => { + log::warn!("Planned provider select failed: {:?}", error); + break; + } + }; + pending = select_result.remaining; + match select_result.ready { + Ok(platform_response) => { + let backend_name = platform_response + .backend_name + .as_deref() + .unwrap_or_default() + .to_string(); + if let Some(state) = launches.remove(&backend_name) { + let elapsed_ms = state.started_at.elapsed().as_millis() as u64; + let deadline_policy = AuctionDeadlinePolicy::for_runtime(context.services); + if deadline_policy + .rejects_late_completion(auction_start, context.timeout_ms) + { + responses.push(provider_timeout_response( + state.provider.provider_name(), + elapsed_ms, + )); + continue; + } + match state + .provider + .parse_response_with_state( + platform_response, + elapsed_ms, + state.parse_state.as_deref(), + ) + .await + { + Ok(response) => responses.push(response), + Err(error) => responses.push(provider_error_response( + state.provider.provider_name(), + elapsed_ms, + ERROR_TYPE_PARSE_RESPONSE, + &error, + )), + } + } + } + Err(error) => { + if let Some(backend_name) = select_result.failed_backend_name + && let Some(state) = launches.remove(&backend_name) + { + let elapsed_ms = state.started_at.elapsed().as_millis() as u64; + log::warn!( + "Planned provider '{}' transport failed: {:?}", + state.provider.provider_name(), + error + ); + responses.push(provider_transport_failed_response( + state.provider.provider_name(), + elapsed_ms, + )); + } + } + } + } + for state in launches.into_values() { + responses.push(provider_timeout_response( + state.provider.provider_name(), + state.started_at.elapsed().as_millis() as u64, + )); + } + + for response in &mut responses { + if let Some(&unused_bidder_params_count) = + planned_unused_bidder_params.get(response.provider.as_str()) + { + *response = + materialize_planned_response(response.clone(), unused_bidder_params_count); + } + } + + let provider_order = self + .plan + .providers() + .iter() + .enumerate() + .map(|(index, provider)| (provider.id.as_str(), index)) + .collect::>(); + responses.sort_by_key(|response| { + provider_order + .get(response.provider.as_str()) + .copied() + .unwrap_or(usize::MAX) + }); + + let floor_prices = original_request + .slots + .iter() + .filter_map(|slot| slot.floor_price.map(|floor| (slot.id.clone(), floor))) + .collect::>(); + let helper = AuctionOrchestrator::new(AuctionConfig::default()); + let local_winners = || helper.select_winning_bids(&responses, &floor_prices); + let (mediator_response, winning_bids) = if let Some(mediator) = &self.mediator { + let remaining_ms = remaining_budget_ms(auction_start, context.timeout_ms); + let logical_budget_ms = remaining_ms.min(mediator.timeout_ms()); + if logical_budget_ms == 0 { + log::warn!( + "Auction deadline exhausted before planned mediator; using local ranking" + ); + (None, local_winners()) + } else { + let transport_timeout_ms = context + .services + .backend() + .canonicalize_transport_timeout_ms(logical_budget_ms, mediator.timeout_ms()); + if transport_timeout_ms == 0 { + log::warn!( + "Planned mediator transport budget canonicalized to zero; using local ranking" + ); + let winning_bids = local_winners(); + return Ok(OrchestrationResult { + provider_responses: responses, + mediator_response: None, + winning_bids, + total_time_ms: auction_start.elapsed().as_millis() as u64, + metadata: routing_metadata(routed.diagnostics().unroutable_bidder_count()), + }); + } + let mediator_context = AuctionContext { + settings: context.settings, + request: context.request, + timeout_ms: logical_budget_ms, + transport_timeout_ms, + provider_responses: Some(&responses), + services: context.services, + }; + let mediator_start = Instant::now(); + let mediated = match mediator + .request_bids(original_request, &mediator_context) + .await + { + Ok(ProviderRequestOutcome::Immediate(response)) => Some(response), + Ok(ProviderRequestOutcome::Pending { + request: pending, + parse_state, + }) => match context.services.http_client().wait(pending).await { + Ok(platform_response) => { + let response_time_ms = mediator_start.elapsed().as_millis() as u64; + if AuctionDeadlinePolicy::for_runtime(context.services) + .rejects_late_completion(auction_start, context.timeout_ms) + { + log::warn!( + "Planned mediator '{}' completed after the hard auction deadline; using local ranking ({}ms)", + mediator.provider_name(), + response_time_ms + ); + None + } else { + mediator + .parse_response_with_context_and_state( + platform_response, + response_time_ms, + original_request, + &mediator_context, + parse_state.as_deref(), + ) + .await + .map_err(|error| { + log::warn!( + "Planned mediator '{}' parse failed: {:?}", + mediator.provider_name(), + error + ); + }) + .ok() + } + } + Err(error) => { + log::warn!( + "Planned mediator '{}' request failed: {:?}", + mediator.provider_name(), + error + ); + None + } + }, + Err(error) => { + log::warn!( + "Planned mediator '{}' failed to launch: {:?}", + mediator.provider_name(), + error + ); + None + } + }; + if let Some(mediated) = mediated { + let winners = mediated + .bids + .iter() + .filter_map(|bid| { + if bid.price.is_none() { + log::warn!( + "Planned mediator returned a bid without a decoded price" + ); + None + } else { + Some((bid.slot_id.clone(), bid.clone())) + } + }) + .collect(); + ( + Some(mediated), + helper.apply_floor_prices(winners, &floor_prices), + ) + } else { + (None, local_winners()) + } + } + } else { + (None, local_winners()) + }; + let unroutable_bidder_count = routed.diagnostics().unroutable_bidder_count(); + log::info!( + "Auction routing diagnostics: unroutable_bidder_count={}", + unroutable_bidder_count + ); + Ok(OrchestrationResult { + provider_responses: responses, + mediator_response, + winning_bids, + total_time_ms: auction_start.elapsed().as_millis() as u64, + metadata: routing_metadata(unroutable_bidder_count), + }) + } +} + impl AuctionOrchestrator { - /// Create a new orchestrator with the given configuration. + /// Create a legacy orchestrator for parity tests. + #[cfg(test)] #[must_use] - pub fn new(config: AuctionConfig) -> Self { + pub(crate) fn new(config: AuctionConfig) -> Self { + let plan = Arc::new( + AuctionPlan::compile(super::plan::AuctionPlanConfig { + timeout_ms: config.timeout_ms, + providers: std::collections::BTreeMap::new(), + bidders: std::collections::BTreeMap::new(), + mediator: None, + request_signing: None, + }) + .expect("should compile empty legacy test plan") + .with_enabled(config.enabled), + ); Self { + enabled: config.enabled, + plan_backed: false, config, + plan, + planned_providers: Vec::new(), + mediator: None, + providers: HashMap::new(), + } + } + + /// Create the live orchestrator from one shared compiled auction plan. + #[must_use] + pub fn from_plan(plan: Arc, mediator: Option>) -> Self { + let planned_providers = plan + .providers() + .iter() + .cloned() + .map(GenericOpenRtbProvider::new) + .map(Arc::new) + .collect(); + Self { + enabled: plan.enabled(), + plan_backed: true, + plan, + planned_providers, + mediator, + #[cfg(test)] + config: AuctionConfig::default(), + #[cfg(test)] providers: HashMap::new(), } } - /// Register an auction provider. - pub fn register_provider(&mut self, provider: Arc) { + /// Return whether this orchestrator and another plan consumer share the same plan allocation. + #[must_use] + pub fn shares_plan(&self, plan: &Arc) -> bool { + Arc::ptr_eq(&self.plan, plan) + } + + /// Register an auction provider in the legacy parity harness. + #[cfg(test)] + pub(crate) fn register_provider(&mut self, provider: Arc) { let name = provider.provider_name().to_string(); log::info!("Registering auction provider: {}", name); self.providers.insert(name, provider); @@ -227,61 +829,51 @@ impl AuctionOrchestrator { /// Get the number of registered providers. #[must_use] pub fn provider_count(&self) -> usize { - self.providers.len() + self.planned_providers.len() } - /// Validate that every configured provider name has an enabled provider integration. - pub(crate) fn validate_configured_provider_names( + async fn run_planned_auction( &self, - ) -> Result<(), Report> { - if !self.config.enabled { - return Ok(()); - } - - let mut configured_providers = HashSet::new(); - for provider_name in &self.config.providers { - if !configured_providers.insert(provider_name.as_str()) { - return Err(Report::new(TrustedServerError::Configuration { - message: format!( - "Auction provider `{provider_name}` is listed more than once in [auction].providers; each provider may appear at most once" - ), - })); + request: &AuctionRequest, + context: &AuctionContext<'_>, + ) -> Result> { + match self.dispatch_auction(request, context).await { + DispatchAuctionOutcome::Dispatched(dispatched) => Ok(self + .collect_dispatched_auction(dispatched, context.services, context) + .await), + DispatchAuctionOutcome::DispatchFailed { + provider_responses, + fatal_admission_error, + metadata, + elapsed_ms, + .. + } => { + if let Some(error) = fatal_admission_error { + return Err(error.change_context(TrustedServerError::Auction { + message: "Planned auction admission failed".to_string(), + })); + } + Ok(OrchestrationResult { + provider_responses, + mediator_response: None, + winning_bids: HashMap::new(), + total_time_ms: elapsed_ms, + metadata, + }) } - } - - if let Some(mediator_name) = &self.config.mediator - && configured_providers.contains(mediator_name.as_str()) - { - return Err(Report::new(TrustedServerError::Configuration { - message: format!( - "Auction mediator `{mediator_name}` is also listed in [auction].providers; a provider may not mediate its own auction" - ), - })); - } - - for provider_name in self - .config - .providers - .iter() - .chain(self.config.mediator.iter()) - { - if !self.providers.contains_key(provider_name) { - return Err(Report::new(TrustedServerError::Configuration { - message: format!( - "Auction provider `{provider_name}` is listed in [auction] but no enabled integration provides it" - ), - })); + DispatchAuctionOutcome::NotStarted => { + if self.planned_providers.is_empty() { + Ok(OrchestrationResult::no_bid()) + } else { + Err(Report::new(TrustedServerError::Auction { + message: "No planned provider request was started".to_string(), + })) + } } } - - Ok(()) } - /// Execute an auction using the auto-detected strategy. - /// - /// Strategy is determined by mediator configuration: - /// - If mediator is configured: runs parallel mediation (bidders → mediator decides) - /// - If no mediator: runs parallel only (bidders → highest CPM wins) + /// Execute an auction through the compiled plan. /// /// # Errors /// @@ -292,9 +884,20 @@ impl AuctionOrchestrator { request: &AuctionRequest, context: &AuctionContext<'_>, ) -> Result> { + if !self.enabled { + return Ok(OrchestrationResult::no_bid()); + } + #[cfg(not(test))] + return self.run_planned_auction(request, context).await; + #[cfg(test)] + if self.plan_backed { + return self.run_planned_auction(request, context).await; + } + #[cfg(test)] let start_time = Instant::now(); - // Auto-detect strategy based on mediator configuration + // Auto-detect strategy based on mediator configuration. + #[cfg(test)] let (strategy_name, result) = if self.config.has_mediator() { ( "parallel_mediation", @@ -307,11 +910,13 @@ impl AuctionOrchestrator { ) }; + #[cfg(test)] log::info!( "Running auction with strategy: {} (auto-detected from mediator config)", strategy_name ); + #[cfg(test)] Ok(OrchestrationResult { total_time_ms: start_time.elapsed().as_millis() as u64, ..result @@ -319,6 +924,7 @@ impl AuctionOrchestrator { } /// Run auction with parallel bidding + mediation. + #[cfg(test)] /// /// Flow: /// 1. Run all bidders in parallel @@ -368,6 +974,7 @@ impl AuctionOrchestrator { settings: context.settings, request: context.request, timeout_ms: mediator_timeout, + transport_timeout_ms: mediator_timeout, provider_responses: Some(&provider_responses), services: context.services, }; @@ -395,11 +1002,29 @@ impl AuctionOrchestrator { mediator.provider_name() ), })?; - - mediator - .parse_response_with_context_and_state( - platform_resp, - start_time.elapsed().as_millis() as u64, + let response_time_ms = start_time.elapsed().as_millis() as u64; + if AuctionDeadlinePolicy::for_runtime(context.services) + .rejects_late_completion(mediation_start, context.timeout_ms) + { + log::warn!( + "Mediator '{}' completed after the hard auction deadline; using local ranking ({}ms)", + mediator.provider_name(), + response_time_ms + ); + let winning = self.select_winning_bids(&provider_responses, &floor_prices); + return Ok(OrchestrationResult { + provider_responses, + mediator_response: None, + winning_bids: winning, + total_time_ms: 0, + metadata: HashMap::new(), + }); + } + + mediator + .parse_response_with_context_and_state( + platform_resp, + response_time_ms, request, &mediator_context, parse_state.as_deref(), @@ -449,6 +1074,7 @@ impl AuctionOrchestrator { } /// Run auction with only parallel bidding (no mediation). + #[cfg(test)] async fn run_parallel_only( &self, request: &AuctionRequest, @@ -468,6 +1094,7 @@ impl AuctionOrchestrator { } /// Run all providers in parallel and collect responses. + #[cfg(test)] /// /// Uses `PlatformHttpClient::select()` to process responses as they /// become ready, rather than waiting for each response sequentially. @@ -476,7 +1103,12 @@ impl AuctionOrchestrator { request: &AuctionRequest, context: &AuctionContext<'_>, ) -> Result, Report> { - let provider_names = self.config.provider_names(); + let provider_names = self + .config + .providers + .keys() + .map(super::plan::ProviderId::as_str) + .collect::>(); if provider_names.is_empty() { return Err(Report::new(TrustedServerError::Auction { @@ -515,8 +1147,8 @@ impl AuctionOrchestrator { let mut responses = Vec::new(); let mut immediate_response_count = 0usize; - for provider_name in provider_names { - let provider = match self.providers.get(provider_name) { + for provider_name in &provider_names { + let provider = match self.providers.get(*provider_name) { Some(p) => p, None => { log::warn!("Provider '{}' not registered, skipping", provider_name); @@ -568,6 +1200,7 @@ impl AuctionOrchestrator { settings: context.settings, request: context.request, timeout_ms: effective_timeout, + transport_timeout_ms: effective_timeout, provider_responses: context.provider_responses, services: context.services, }; @@ -671,7 +1304,7 @@ impl AuctionOrchestrator { })); } - let deadline = Duration::from_millis(u64::from(context.timeout_ms)); + let deadline_policy = AuctionDeadlinePolicy::for_runtime(context.services); log::info!( "Launched {} concurrent provider request(s); waiting for responses", pending_requests.len() @@ -715,10 +1348,20 @@ impl AuctionOrchestrator { if let Some(state) = backend_to_provider.remove(&backend_name) { let response_time_ms = state.started_at.elapsed().as_millis() as u64; + if deadline_policy + .rejects_late_completion(auction_start, context.timeout_ms) + { + responses.push(provider_timeout_response( + &state.provider_name, + response_time_ms, + )); + continue; + } let provider_context = AuctionContext { settings: context.settings, request: context.request, timeout_ms: state.effective_timeout_ms, + transport_timeout_ms: state.effective_timeout_ms, provider_responses: context.provider_responses, services: context.services, }; @@ -796,16 +1439,10 @@ impl AuctionOrchestrator { } } - // Check auction deadline after processing each response. - // Remaining PendingRequests are dropped, which abandons the - // in-flight HTTP calls on the Fastly host. - if auction_start.elapsed() >= deadline && !remaining.is_empty() { - log::warn!( - "Auction timeout reached; dropping {} remaining request(s)", - remaining.len() - ); - break; - } + // Current adapters cannot enforce a hard total-request deadline, so + // drain already-launched handles and retain completed late responses. + // A future adapter that explicitly claims the capability classifies + // each late completion as a timeout instead. } for state in backend_to_provider.into_values() { @@ -920,6 +1557,7 @@ impl AuctionOrchestrator { } /// Get a provider by name. + #[cfg(test)] fn get_provider( &self, name: &str, @@ -936,6 +1574,225 @@ impl AuctionOrchestrator { }) } + async fn dispatch_planned_auction( + &self, + request: &AuctionRequest, + context: &AuctionContext<'_>, + ) -> DispatchAuctionOutcome { + let plan = &self.plan; + if self.planned_providers.is_empty() { + return DispatchAuctionOutcome::NotStarted; + } + if self.planned_providers.len() > 1 + && !context.services.http_client().supports_concurrent_fanout() + { + log::warn!( + "{} planned auction providers configured on a runtime without concurrent fanout", + self.planned_providers.len() + ); + return DispatchAuctionOutcome::NotStarted; + } + + let auction_start = Instant::now(); + let routed = route_auction( + request.clone(), + context.request, + plan, + context.services.client_info().client_ip, + ); + let planned_unused_bidder_params = routed + .inputs() + .iter() + .map(|input| { + ( + input.provider_id().as_str().to_string(), + unused_bidder_params_count( + &plan + .provider(input.provider_id()) + .expect("should find routed provider in compiled plan") + .profile, + input, + ), + ) + }) + .collect::>(); + let planned_unroutable_bidder_count = routed.diagnostics().unroutable_bidder_count(); + // A zero request budget is terminal before signing admission. In the + // split path, signer initialization would otherwise read config and + // secret stores even though every provider is materialized as timeout. + let signer_result = if context.timeout_ms == 0 { + Ok(None) + } else { + plan.signing_enabled() + .then(|| RequestSigner::from_services(context.services)) + .transpose() + }; + let signer = match signer_result { + Ok(signer) => signer, + Err(error) => { + log::warn!("Planned auction signer initialization failed: {error:?}"); + let mut provider_responses = routed + .skipped_no_eligible_provider_ids() + .iter() + .map(|id| provider_skipped_response(id.as_str())) + .chain(routed.inputs().iter().map(|input| { + materialize_planned_response( + provider_launch_failed_response(input.provider_id().as_str(), 0), + unused_bidder_params_count( + &plan + .provider(input.provider_id()) + .expect("should find routed provider in compiled plan") + .profile, + input, + ), + ) + })) + .collect::>(); + let provider_order = plan + .providers() + .iter() + .enumerate() + .map(|(index, provider)| (provider.id.as_str(), index)) + .collect::>(); + provider_responses.sort_by_key(|response| { + provider_order + .get(response.provider.as_str()) + .copied() + .unwrap_or(usize::MAX) + }); + return DispatchAuctionOutcome::DispatchFailed { + request: request.clone(), + provider_responses, + fatal_admission_error: Some(error), + metadata: routing_metadata(planned_unroutable_bidder_count), + elapsed_ms: auction_start.elapsed().as_millis() as u64, + }; + } + }; + let mut completed_responses = routed + .skipped_no_eligible_provider_ids() + .iter() + .map(|id| provider_skipped_response(id.as_str())) + .collect::>(); + let planned_provider_order = plan + .providers() + .iter() + .enumerate() + .map(|(index, provider)| (provider.id.as_str().to_string(), index)) + .collect::>(); + let mut pending_requests = Vec::new(); + let mut planned_backend_to_provider = HashMap::new(); + let mut reserved_backend_names = HashSet::new(); + let mut immediate_response_count = 0usize; + + for input in routed.inputs() { + let Some(provider) = self + .planned_providers + .iter() + .find(|provider| provider.provider_name() == input.provider_id().as_str()) + .cloned() + else { + completed_responses.push(provider_launch_failed_response( + input.provider_id().as_str(), + 0, + )); + continue; + }; + let logical_budget_ms = + remaining_budget_ms(auction_start, context.timeout_ms).min(provider.timeout_ms()); + if logical_budget_ms == 0 { + completed_responses.push(provider_timeout_response(provider.provider_name(), 0)); + continue; + } + let transport_timeout_ms = context + .services + .backend() + .canonicalize_transport_timeout_ms(logical_budget_ms, provider.timeout_ms()); + if transport_timeout_ms == 0 { + completed_responses.push(provider_timeout_response(provider.provider_name(), 0)); + continue; + } + let started_at = Instant::now(); + match provider + .request_bids_routed( + input, + &routed, + logical_budget_ms, + transport_timeout_ms, + signer.as_ref(), + context.services, + &mut reserved_backend_names, + ) + .await + { + Ok(ProviderRequestOutcome::Pending { + request: pending, + parse_state, + }) => { + let Some(backend_name) = pending.backend_name().map(str::to_string) else { + completed_responses.push(provider_launch_failed_response( + provider.provider_name(), + started_at.elapsed().as_millis() as u64, + )); + continue; + }; + match planned_backend_to_provider.entry(backend_name.clone()) { + Entry::Vacant(entry) => { + entry.insert(PlannedLaunchState { + provider, + started_at, + parse_state, + }); + pending_requests.push(pending.with_backend_name(backend_name)); + } + Entry::Occupied(_) => { + completed_responses.push(provider_launch_failed_response( + provider.provider_name(), + started_at.elapsed().as_millis() as u64, + )) + } + } + } + Ok(ProviderRequestOutcome::Immediate(response)) => { + immediate_response_count += 1; + completed_responses.push(response); + } + Err(error) => { + log::warn!( + "Planned provider '{}' failed to dispatch: {error:?}", + provider.provider_name() + ); + completed_responses.push(provider_launch_failed_response( + provider.provider_name(), + started_at.elapsed().as_millis() as u64, + )); + } + } + } + + if pending_requests.is_empty() + && immediate_response_count == 0 + && completed_responses.is_empty() + { + return DispatchAuctionOutcome::NotStarted; + } + + DispatchAuctionOutcome::Dispatched(DispatchedAuction { + pending_requests, + backend_to_provider: HashMap::new(), + planned_backend_to_provider, + completed_responses, + auction_start, + timeout_ms: context.timeout_ms, + floor_prices: self.floor_prices_by_slot(request), + provider_request_context: Box::new(snapshot_context_request(context.request)), + request: request.clone(), + planned_unused_bidder_params, + planned_unroutable_bidder_count, + planned_provider_order, + }) + } + /// Dispatch SSP bid requests without blocking WASM. /// /// Calls each enabled provider's [`AuctionProvider::request_bids`] (which @@ -953,7 +1810,20 @@ impl AuctionOrchestrator { request: &AuctionRequest, context: &AuctionContext<'_>, ) -> DispatchAuctionOutcome { - let provider_names = self.config.provider_names(); + if !self.enabled { + return DispatchAuctionOutcome::NotStarted; + } + if !cfg!(test) || self.plan_backed { + return self.dispatch_planned_auction(request, context).await; + } + #[cfg(test)] + let provider_names = self + .config + .providers + .keys() + .map(super::plan::ProviderId::as_str) + .collect::>(); + #[cfg(test)] if provider_names.is_empty() { return DispatchAuctionOutcome::NotStarted; } @@ -963,6 +1833,7 @@ impl AuctionOrchestrator { // (e.g. Cloudflare Workers, Spin). Sequential execution would accrue // the sum of provider latencies before the origin fetch and then fail // collection with empty bids. + #[cfg(test)] if provider_names.len() > 1 && !context.services.http_client().supports_concurrent_fanout() { log::warn!( @@ -976,13 +1847,26 @@ impl AuctionOrchestrator { } let auction_start = Instant::now(); + #[cfg(test)] let mut backend_to_provider: HashMap = HashMap::new(); + #[cfg(not(test))] + let backend_to_provider: HashMap = HashMap::new(); + #[cfg(test)] let mut pending_requests: Vec = Vec::new(); + #[cfg(not(test))] + let pending_requests: Vec = Vec::new(); + #[cfg(test)] let mut completed_responses: Vec = Vec::new(); + #[cfg(not(test))] + let completed_responses: Vec = Vec::new(); + #[cfg(test)] let mut immediate_response_count = 0usize; + #[cfg(not(test))] + let immediate_response_count = 0usize; - for provider_name in provider_names { - let provider = match self.providers.get(provider_name) { + #[cfg(test)] + for provider_name in &provider_names { + let provider = match self.providers.get(*provider_name) { Some(p) => p, None => { // lgtm[rust/cleartext-logging] @@ -1038,6 +1922,7 @@ impl AuctionOrchestrator { settings: context.settings, request: context.request, timeout_ms: effective_timeout, + transport_timeout_ms: effective_timeout, provider_responses: context.provider_responses, services: context.services, }; @@ -1125,6 +2010,8 @@ impl AuctionOrchestrator { DispatchAuctionOutcome::DispatchFailed { request: request.clone(), provider_responses: completed_responses, + fatal_admission_error: None, + metadata: HashMap::new(), elapsed_ms: auction_start.elapsed().as_millis() as u64, } }; @@ -1140,12 +2027,16 @@ impl AuctionOrchestrator { DispatchAuctionOutcome::Dispatched(DispatchedAuction { pending_requests, backend_to_provider, + planned_backend_to_provider: HashMap::new(), completed_responses, auction_start, timeout_ms: context.timeout_ms, floor_prices: self.floor_prices_by_slot(request), provider_request_context: Box::new(snapshot_context_request(context.request)), request: request.clone(), + planned_unused_bidder_params: HashMap::new(), + planned_unroutable_bidder_count: 0, + planned_provider_order: HashMap::new(), }) } @@ -1168,12 +2059,16 @@ impl AuctionOrchestrator { let DispatchedAuction { pending_requests, mut backend_to_provider, + mut planned_backend_to_provider, completed_responses, auction_start, timeout_ms, floor_prices, provider_request_context, request, + planned_unused_bidder_params, + planned_unroutable_bidder_count, + planned_provider_order, } = dispatched; log::info!( @@ -1185,6 +2080,7 @@ impl AuctionOrchestrator { let mut responses: Vec = completed_responses; let mut remaining = pending_requests; + let deadline_policy = AuctionDeadlinePolicy::for_runtime(services); while !remaining.is_empty() { let select_result = match services @@ -1197,6 +2093,30 @@ impl AuctionOrchestrator { Ok(r) => r, Err(e) => { log::warn!("select() failed during auction collection: {:?}", e); + // An outer select failure means the platform could not poll + // any outstanding handle. Attribute every tracked launch as + // a transport failure rather than later relabeling it as a + // timeout. Drain through a sorted buffer because HashMap + // iteration order is intentionally nondeterministic. + let mut transport_failures = backend_to_provider + .drain() + .map(|(_, state)| { + let response_time_ms = state.started_at.elapsed().as_millis() as u64; + provider_transport_failed_response( + &state.provider_name, + response_time_ms, + ) + }) + .chain(planned_backend_to_provider.drain().map(|(_, state)| { + let response_time_ms = state.started_at.elapsed().as_millis() as u64; + provider_transport_failed_response( + state.provider.provider_name(), + response_time_ms, + ) + })) + .collect::>(); + transport_failures.sort_by(|left, right| left.provider.cmp(&right.provider)); + responses.extend(transport_failures); break; } }; @@ -1214,10 +2134,18 @@ impl AuctionOrchestrator { let backend_name = platform_response.backend_name.clone().unwrap_or_default(); if let Some(state) = backend_to_provider.remove(&backend_name) { let response_time_ms = state.started_at.elapsed().as_millis() as u64; + if deadline_policy.rejects_late_completion(auction_start, timeout_ms) { + responses.push(provider_timeout_response( + &state.provider_name, + response_time_ms, + )); + continue; + } let provider_context = AuctionContext { settings: context.settings, request: &provider_request_context, timeout_ms: state.effective_timeout_ms, + transport_timeout_ms: state.effective_timeout_ms, provider_responses: context.provider_responses, services: context.services, }; @@ -1232,28 +2160,39 @@ impl AuctionOrchestrator { ) .await { - Ok(auction_response) => { - log::info!( - "Provider '{}' returned {} bids ({}ms)", - auction_response.provider, - auction_response.bids.len(), - auction_response.response_time_ms - ); - responses.push(auction_response); - } - Err(e) => { - log::warn!( - "Provider '{}' parse failed: {:?}", - state.provider_name, - e - ); - responses.push(provider_error_response( - &state.provider_name, - response_time_ms, - ERROR_TYPE_PARSE_RESPONSE, - &e, - )); - } + Ok(auction_response) => responses.push(auction_response), + Err(error) => responses.push(provider_error_response( + &state.provider_name, + response_time_ms, + ERROR_TYPE_PARSE_RESPONSE, + &error, + )), + } + } else if let Some(state) = planned_backend_to_provider.remove(&backend_name) { + let response_time_ms = state.started_at.elapsed().as_millis() as u64; + if deadline_policy.rejects_late_completion(auction_start, timeout_ms) { + responses.push(provider_timeout_response( + state.provider.provider_name(), + response_time_ms, + )); + continue; + } + match state + .provider + .parse_response_with_state( + platform_response, + response_time_ms, + state.parse_state.as_deref(), + ) + .await + { + Ok(response) => responses.push(response), + Err(error) => responses.push(provider_error_response( + state.provider.provider_name(), + response_time_ms, + ERROR_TYPE_PARSE_RESPONSE, + &error, + )), } } else { log::warn!( @@ -1278,6 +2217,18 @@ impl AuctionOrchestrator { &state.provider_name, response_time_ms, )); + } else if let Some(state) = planned_backend_to_provider.remove(backend_name) + { + let response_time_ms = state.started_at.elapsed().as_millis() as u64; + log::warn!( + "Planned provider '{}' request failed: {:?}", + state.provider.provider_name(), + e + ); + responses.push(provider_transport_failed_response( + state.provider.provider_name(), + response_time_ms, + )); } else { log::warn!( "A provider request failed (backend '{}' not tracked): {:?}", @@ -1315,79 +2266,136 @@ impl AuctionOrchestrator { )); } backend_to_provider.clear(); + for state in planned_backend_to_provider.into_values() { + responses.push(provider_timeout_response( + state.provider.provider_name(), + state.started_at.elapsed().as_millis() as u64, + )); + } + for response in &mut responses { + if let Some(&count) = planned_unused_bidder_params.get(response.provider.as_str()) { + *response = materialize_planned_response(response.clone(), count); + } + } + if !planned_provider_order.is_empty() { + responses.sort_by_key(|response| { + planned_provider_order + .get(response.provider.as_str()) + .copied() + .unwrap_or(usize::MAX) + }); + } - let (mediator_response, winning_bids) = if let Some(mediator_name) = &self.config.mediator { - match self.providers.get(mediator_name.as_str()) { - Some(mediator) => { - // Cap the mediator at whichever is tighter: its own configured - // timeout or the remaining auction budget (A_deadline). Backend - // first-byte and between-bytes timeouts bound normal collection, but - // they are transport timers rather than absolute wall-clock limits: - // connection setup and byte-trickling can still consume more of the - // auction budget. Recomputing the remaining budget here prevents the - // mediator from extending that bounded response hold. - let remaining = remaining_budget_ms(auction_start, timeout_ms); - let mediator_timeout = services - .backend() - .canonicalize_transport_timeout_ms(remaining, mediator.timeout_ms()); - if mediator_timeout == 0 { - log::warn!( - "A_deadline exhausted before mediator '{}' — returning {} SSP bids without mediation", - mediator.provider_name(), - responses.len(), - ); - let winning = self.select_winning_bids(&responses, &floor_prices); - return OrchestrationResult { - provider_responses: responses, - mediator_response: None, - winning_bids: winning, - total_time_ms: auction_start.elapsed().as_millis() as u64, - metadata: HashMap::new(), - }; - } - let mediator_start = Instant::now(); - log::info!( - "Running mediator '{}' with {}ms budget (A_deadline remaining: {}ms, configured: {}ms)", + #[cfg(not(test))] + let mediator = self.mediator.as_ref(); + #[cfg(test)] + let mediator = self.mediator.as_ref().or_else(|| { + self.config + .mediator + .as_ref() + .and_then(|name| self.providers.get(name)) + }); + let (mediator_response, winning_bids) = if let Some(mediator) = mediator { + { + // Cap the mediator at whichever is tighter: its own configured + // timeout or the remaining auction budget (A_deadline). Backend + // first-byte and between-bytes timeouts bound normal collection, but + // they are transport timers rather than absolute wall-clock limits: + // connection setup and byte-trickling can still consume more of the + // auction budget. Recomputing the remaining budget here prevents the + // mediator from extending that bounded response hold. + let remaining = remaining_budget_ms(auction_start, timeout_ms); + let logical_budget_ms = remaining.min(mediator.timeout_ms()); + if logical_budget_ms == 0 { + log::warn!( + "A_deadline exhausted before mediator '{}' — returning {} SSP bids without mediation", mediator.provider_name(), - mediator_timeout, - remaining, - mediator.timeout_ms(), + responses.len(), ); - // The mediator runs on the collect path. See the doc-comment on - // `AuctionContext::request`: the real client request was already - // consumed by `send_async` during dispatch, so we substitute a - // canonical placeholder URL. Any future mediator that needs real - // client headers must snapshot them at dispatch time onto - // `DispatchedAuction` rather than reading `context.request` here. - let placeholder = http::Request::builder() - .uri(crate::auction::types::MEDIATOR_PLACEHOLDER_URL) - .body(edgezero_core::body::Body::empty()) - .unwrap_or_else(|_| http::Request::new(edgezero_core::body::Body::empty())); - let mediator_context = AuctionContext { - settings: context.settings, - request: &placeholder, - timeout_ms: mediator_timeout, - provider_responses: Some(&responses), - services: context.services, + let winning = self.select_winning_bids(&responses, &floor_prices); + return OrchestrationResult { + provider_responses: responses, + mediator_response: None, + winning_bids: winning, + total_time_ms: auction_start.elapsed().as_millis() as u64, + metadata: routing_metadata(planned_unroutable_bidder_count), }; - let mediator_response = - match mediator.request_bids(&request, &mediator_context).await { - Ok(ProviderRequestOutcome::Immediate(response)) => Some(response), - Ok(ProviderRequestOutcome::Pending { - request: pending, - parse_state, - }) => match services.http_client().wait(pending).await.change_context( - TrustedServerError::Auction { - message: format!( - "Mediator {} request failed", - mediator.provider_name() - ), - }, - ) { - Ok(platform_resp) => match mediator + } + let transport_timeout_ms = services + .backend() + .canonicalize_transport_timeout_ms(logical_budget_ms, mediator.timeout_ms()); + if transport_timeout_ms == 0 { + log::warn!( + "Mediator '{}' transport budget canonicalized to zero — returning {} SSP bids without mediation", + mediator.provider_name(), + responses.len(), + ); + let winning = self.select_winning_bids(&responses, &floor_prices); + return OrchestrationResult { + provider_responses: responses, + mediator_response: None, + winning_bids: winning, + total_time_ms: auction_start.elapsed().as_millis() as u64, + metadata: routing_metadata(planned_unroutable_bidder_count), + }; + } + let mediator_start = Instant::now(); + log::info!( + "Running mediator '{}' with {}ms logical budget and {}ms transport timeout (A_deadline remaining: {}ms, configured: {}ms)", + mediator.provider_name(), + logical_budget_ms, + transport_timeout_ms, + remaining, + mediator.timeout_ms(), + ); + // The mediator runs on the collect path. See the doc-comment on + // `AuctionContext::request`: the real client request was already + // consumed by `send_async` during dispatch, so we substitute a + // canonical placeholder URL. Any future mediator that needs real + // client headers must snapshot them at dispatch time onto + // `DispatchedAuction` rather than reading `context.request` here. + let placeholder = http::Request::builder() + .uri(crate::auction::types::MEDIATOR_PLACEHOLDER_URL) + .body(edgezero_core::body::Body::empty()) + .unwrap_or_else(|_| http::Request::new(edgezero_core::body::Body::empty())); + let mediator_context = AuctionContext { + settings: context.settings, + request: &placeholder, + timeout_ms: logical_budget_ms, + transport_timeout_ms, + provider_responses: Some(&responses), + services: context.services, + }; + let mediator_response = match mediator + .request_bids(&request, &mediator_context) + .await + { + Ok(ProviderRequestOutcome::Immediate(response)) => Some(response), + Ok(ProviderRequestOutcome::Pending { + request: pending, + parse_state, + }) => match services.http_client().wait(pending).await.change_context( + TrustedServerError::Auction { + message: format!( + "Mediator {} request failed", + mediator.provider_name() + ), + }, + ) { + Ok(platform_resp) => { + let response_time_ms = mediator_start.elapsed().as_millis() as u64; + if deadline_policy.rejects_late_completion(auction_start, timeout_ms) { + log::warn!( + "Mediator '{}' completed after the hard auction deadline; using local ranking ({}ms)", + mediator.provider_name(), + response_time_ms + ); + None + } else { + match mediator .parse_response_with_context_and_state( platform_resp, - mediator_start.elapsed().as_millis() as u64, + response_time_ms, &request, &mediator_context, parse_state.as_deref(), @@ -1403,24 +2411,26 @@ impl AuctionOrchestrator { ); None } - }, - Err(error) => { - log::warn!("Mediator request failed: {:?}", error); - None } - }, - Err(error) => { - log::warn!( - "Mediator '{}' failed to dispatch: {:?}", - mediator.provider_name(), - error - ); - None } - }; + } + Err(error) => { + log::warn!("Mediator request failed: {:?}", error); + None + } + }, + Err(error) => { + log::warn!( + "Mediator '{}' failed to dispatch: {:?}", + mediator.provider_name(), + error + ); + None + } + }; - if let Some(mediator_response) = mediator_response { - let winning = mediator_response + if let Some(mediator_response) = mediator_response { + let winning = mediator_response .bids .iter() .filter_map(|bid| { @@ -1436,16 +2446,9 @@ impl AuctionOrchestrator { } }) .collect(); - let winning = self.apply_floor_prices(winning, &floor_prices); - (Some(mediator_response), winning) - } else { - (None, self.select_winning_bids(&responses, &floor_prices)) - } - } - None => { - // lgtm[rust/cleartext-logging] - // The mediator name is a static config identifier, not a secret. - log::warn!("Mediator '{}' not registered", mediator_name); + let winning = self.apply_floor_prices(winning, &floor_prices); + (Some(mediator_response), winning) + } else { (None, self.select_winning_bids(&responses, &floor_prices)) } } @@ -1458,14 +2461,14 @@ impl AuctionOrchestrator { mediator_response, winning_bids, total_time_ms: auction_start.elapsed().as_millis() as u64, - metadata: HashMap::new(), + metadata: routing_metadata(planned_unroutable_bidder_count), } } /// Check if orchestrator is enabled. #[must_use] pub fn is_enabled(&self) -> bool { - self.config.enabled + self.enabled } } @@ -1485,6 +2488,16 @@ pub struct OrchestrationResult { } impl OrchestrationResult { + fn no_bid() -> Self { + Self { + provider_responses: Vec::new(), + mediator_response: None, + winning_bids: HashMap::new(), + total_time_ms: 0, + metadata: HashMap::new(), + } + } + /// Get the winning bid for a specific slot. #[must_use] pub fn get_winning_bid(&self, slot_id: &str) -> Option<&Bid> { @@ -1510,45 +2523,585 @@ impl OrchestrationResult { #[cfg(test)] mod tests { + use std::str::FromStr as _; use std::time::Duration; + + use base64::Engine as _; use web_time::Instant; use crate::auction::config::AuctionConfig; use crate::auction::orchestrator::DispatchAuctionOutcome; - use crate::auction::provider::{AuctionProvider, ProviderRequestOutcome}; + use crate::auction::plan::{ + AuctionPlan, AuctionPlanConfig, NotificationConfig, ProviderConfig, ProviderId, RoutingMode, + }; + use crate::auction::provider::{ + AuctionProvider, GenericOpenRtbProvider, ProviderRequestOutcome, + }; + use crate::auction::routing::{RoutingDiagnostics, route_auction}; use crate::auction::test_support::create_test_auction_context; use crate::auction::types::{ AdFormat, AdSlot, ApsRendererV1, ApsTagType, AuctionContext, AuctionRequest, AuctionResponse, Bid, BidRenderer, BidStatus, MediaType, PublisherInfo, UserInfo, }; use crate::error::TrustedServerError; + use crate::integrations::adserver_mock::{AdServerMockConfig, AdServerMockProvider}; use crate::platform::test_support::{ StubHttpClient, build_services_with_backend_and_http_client, build_services_with_http_client, noop_services, }; use crate::platform::{ - PlatformBackend, PlatformBackendSpec, PlatformError, PlatformHttpRequest, PlatformResponse, - RuntimeServices, + BackendNamingPolicy, PlatformBackend, PlatformBackendSpec, PlatformConfigStore, + PlatformError, PlatformHttpClient, PlatformHttpRequest, PlatformPendingRequest, + PlatformResponse, PlatformSecretStore, PlatformSelectResult, RuntimeServices, StoreId, + StoreName, }; use crate::test_support::tests::crate_test_settings_str; use error_stack::{Report, ResultExt}; - use std::collections::{HashMap, HashSet}; + use std::collections::{BTreeMap, HashMap, HashSet}; + use std::sync::atomic::{AtomicUsize, Ordering}; use std::sync::{Arc, Mutex}; - use super::AuctionOrchestrator; + use super::{ + AuctionOrchestrator, AuctionOrchestratorHarness, DispatchedAuction, ERROR_TYPE_TIMEOUT, + OrchestrationResult, + }; - // --------------------------------------------------------------------------- - // Minimal test double for AuctionProvider - // --------------------------------------------------------------------------- + fn planned_config(providers: &[(&str, RoutingMode)], signing: bool) -> AuctionPlanConfig { + AuctionPlanConfig { + timeout_ms: 777, + providers: providers + .iter() + .map(|(id, routing)| { + ( + ProviderId::from_str(id).expect("should parse fictional provider ID"), + ProviderConfig { + protocol: "openrtb-2.6".to_string(), + profile: "standard".to_string(), + endpoint: "https://example.test/openrtb".to_string(), + timeout_ms: Some(1_000), + routing: *routing, + notifications: Default::default(), + profile_config: serde_json::json!({}), + }, + ) + }) + .collect(), + bidders: BTreeMap::new(), + mediator: None, + request_signing: signing.then(|| crate::settings::RequestSigning { + enabled: true, + config_store_id: "fictional-config-store".to_string(), + secret_store_id: "fictional-secret-store".to_string(), + }), + } + } - struct StubAuctionProvider { + fn planned_prebid_config( + providers: &[(&str, serde_json::Value, NotificationConfig)], + ) -> AuctionPlanConfig { + AuctionPlanConfig { + timeout_ms: 777, + providers: providers + .iter() + .map(|(id, profile_config, notifications)| { + ( + ProviderId::from_str(id).expect("should parse fictional provider ID"), + ProviderConfig { + protocol: "openrtb-2.6".to_string(), + profile: "prebid-server".to_string(), + endpoint: format!("https://{id}.example.test/openrtb"), + timeout_ms: Some(1_000), + routing: RoutingMode::AllEligible, + notifications: notifications.clone(), + profile_config: profile_config.clone(), + }, + ) + }) + .collect(), + bidders: BTreeMap::new(), + mediator: None, + request_signing: None, + } + } + + fn planned_aps_config() -> AuctionPlanConfig { + planned_aps_instances_config(&[( + "aps-instance", + serde_json::json!({"account_id": "example-account"}), + NotificationConfig::default(), + )]) + } + + fn planned_aps_instances_config( + providers: &[(&str, serde_json::Value, NotificationConfig)], + ) -> AuctionPlanConfig { + AuctionPlanConfig { + timeout_ms: 777, + providers: providers + .iter() + .map(|(id, profile_config, notifications)| { + ( + ProviderId::from_str(id).expect("should parse fictional provider ID"), + ProviderConfig { + protocol: "openrtb-2.6".to_string(), + profile: "aps".to_string(), + endpoint: "https://aps.example/e/pb/bid".to_string(), + timeout_ms: Some(1_000), + routing: RoutingMode::AllEligible, + notifications: notifications.clone(), + profile_config: profile_config.clone(), + }, + ) + }) + .collect(), + bidders: BTreeMap::new(), + mediator: None, + request_signing: None, + } + } + + fn planned_request() -> AuctionRequest { + AuctionRequest { + id: "fictional-auction".to_string(), + slots: vec![AdSlot { + id: "fictional-slot".to_string(), + formats: vec![AdFormat { + media_type: MediaType::Banner, + width: 300, + height: 250, + }], + floor_price: Some(1.0), + targeting: HashMap::new(), + bidders: HashMap::new(), + }], + publisher: PublisherInfo { + domain: "publisher.example".to_string(), + page_url: Some("https://publisher.example/article".to_string()), + }, + user: UserInfo { + id: None, + consent: None, + eids: None, + }, + device: None, + site: None, + context: HashMap::new(), + } + } + + #[tokio::test] + async fn disabled_from_plan_is_a_no_work_kill_switch_for_sync_and_split_paths() { + let plan = Arc::new( + AuctionPlan::compile(planned_config( + &[("provider-a", RoutingMode::AllEligible)], + false, + )) + .expect("should compile plan") + .with_enabled(false), + ); + let orchestrator = AuctionOrchestrator::from_plan(plan, None); + let http = Arc::new(StubHttpClient::new()); + let services = build_services_with_http_client(Arc::clone(&http) as Arc<_>); + let settings = create_test_settings(); + let inbound = http::Request::new(edgezero_core::body::Body::empty()); + let context = AuctionContext { + settings: &settings, + request: &inbound, + timeout_ms: 777, + transport_timeout_ms: 777, + provider_responses: None, + services: &services, + }; + let request = planned_request(); + + let result = orchestrator + .run_auction(&request, &context) + .await + .expect("disabled auction should complete as no-bid"); + assert!(result.provider_responses.is_empty()); + assert!(result.winning_bids.is_empty()); + assert!(matches!( + orchestrator.dispatch_auction(&request, &context).await, + DispatchAuctionOutcome::NotStarted + )); + assert!(http.recorded_backend_names().is_empty()); + } + + #[tokio::test] + async fn enabled_empty_plan_is_successful_no_bid_without_dispatch() { + let plan = Arc::new( + AuctionPlan::compile(planned_config(&[], false)).expect("should compile empty plan"), + ); + let orchestrator = AuctionOrchestrator::from_plan(plan, None); + let http = Arc::new(StubHttpClient::new()); + let services = build_services_with_http_client(Arc::clone(&http) as Arc<_>); + let settings = create_test_settings(); + let inbound = http::Request::new(edgezero_core::body::Body::empty()); + let context = AuctionContext { + settings: &settings, + request: &inbound, + timeout_ms: 777, + transport_timeout_ms: 777, + provider_responses: None, + services: &services, + }; + let request = planned_request(); + + let result = orchestrator + .run_auction(&request, &context) + .await + .expect("empty auction should complete as no-bid"); + assert!(result.provider_responses.is_empty()); + assert!(result.winning_bids.is_empty()); + assert!(matches!( + orchestrator.dispatch_auction(&request, &context).await, + DispatchAuctionOutcome::NotStarted + )); + assert!(http.recorded_backend_names().is_empty()); + } + + #[tokio::test] + async fn all_skipped_from_plan_completes_with_routing_metadata_in_sync_and_split_paths() { + for split in [false, true] { + let plan = Arc::new( + AuctionPlan::compile(planned_config(&[("skipped", RoutingMode::Explicit)], false)) + .expect("should compile all-skipped plan"), + ); + let orchestrator = AuctionOrchestrator::from_plan(plan, None); + let http = Arc::new(StubHttpClient::new()); + let services = build_services_with_http_client(Arc::clone(&http) as Arc<_>); + let settings = create_test_settings(); + let inbound = http::Request::new(edgezero_core::body::Body::empty()); + let context = AuctionContext { + settings: &settings, + request: &inbound, + timeout_ms: 777, + transport_timeout_ms: 777, + provider_responses: None, + services: &services, + }; + let request = planned_request(); + + let result = if split { + let DispatchAuctionOutcome::Dispatched(dispatched) = + orchestrator.dispatch_auction(&request, &context).await + else { + panic!("all-skipped auction should produce a completed dispatch token"); + }; + orchestrator + .collect_dispatched_auction(dispatched, &services, &context) + .await + } else { + orchestrator + .run_auction(&request, &context) + .await + .expect("all-skipped auction should complete") + }; + + assert_eq!(result.provider_responses.len(), 1); + assert_eq!( + result.provider_responses[0].metadata["routing"]["skipped_no_eligible_slots"], + true + ); + assert!(result.metadata.contains_key("routing")); + assert!(http.recorded_backend_names().is_empty()); + } + } + + struct NamingBackend { + policy: BackendNamingPolicy, + predicted: AtomicUsize, + ensured: AtomicUsize, + specs: Mutex>, + fail_ensure_for: Mutex>, + } + + impl NamingBackend { + fn new(policy: BackendNamingPolicy) -> Self { + Self { + policy, + predicted: AtomicUsize::new(0), + ensured: AtomicUsize::new(0), + specs: Mutex::new(Vec::new()), + fail_ensure_for: Mutex::new(HashSet::new()), + } + } + + fn fail_ensure_for(&self, provider_id: &str) { + self.fail_ensure_for + .lock() + .expect("should lock failing provider IDs") + .insert(provider_id.to_string()); + } + + fn name(&self, spec: &PlatformBackendSpec) -> Result> { + self.policy + .predict(spec) + .map(|prediction| prediction.name) + .change_context(PlatformError::Backend) + } + } + + impl PlatformBackend for NamingBackend { + fn naming_policy(&self) -> BackendNamingPolicy { + self.policy + } + + fn predict_name( + &self, + spec: &PlatformBackendSpec, + ) -> Result> { + self.predicted.fetch_add(1, Ordering::Relaxed); + self.name(spec) + } + + fn ensure(&self, spec: &PlatformBackendSpec) -> Result> { + self.ensured.fetch_add(1, Ordering::Relaxed); + if spec.discriminator.as_deref().is_some_and(|provider_id| { + self.fail_ensure_for + .lock() + .expect("should lock failing provider IDs") + .contains(provider_id) + }) { + return Err(Report::new(PlatformError::Backend)); + } + self.specs + .lock() + .expect("should lock planned backend specs") + .push(spec.clone()); + self.name(spec) + } + } + + struct ZeroCanonicalBackend { + predicted: AtomicUsize, + ensured: AtomicUsize, + } + + impl ZeroCanonicalBackend { + fn new() -> Self { + Self { + predicted: AtomicUsize::new(0), + ensured: AtomicUsize::new(0), + } + } + } + + impl PlatformBackend for ZeroCanonicalBackend { + fn naming_policy(&self) -> BackendNamingPolicy { + BackendNamingPolicy::Fastly + } + + fn predict_name( + &self, + _spec: &PlatformBackendSpec, + ) -> Result> { + self.predicted.fetch_add(1, Ordering::Relaxed); + Ok("zero-canonical-backend".to_string()) + } + + fn ensure(&self, _spec: &PlatformBackendSpec) -> Result> { + self.ensured.fetch_add(1, Ordering::Relaxed); + Ok("zero-canonical-backend".to_string()) + } + + fn canonicalize_transport_timeout_ms( + &self, + _remaining_ms: u32, + _configured_ms: u32, + ) -> u32 { + 0 + } + } + + struct CollidingBackend; + + impl PlatformBackend for CollidingBackend { + fn naming_policy(&self) -> BackendNamingPolicy { + BackendNamingPolicy::Axum + } + + fn predict_name( + &self, + _spec: &PlatformBackendSpec, + ) -> Result> { + Ok("colliding-backend".to_string()) + } + + fn ensure(&self, _spec: &PlatformBackendSpec) -> Result> { + Ok("colliding-backend".to_string()) + } + } + + struct FailingCountingConfigStore { + reads: AtomicUsize, + } + + impl PlatformConfigStore for FailingCountingConfigStore { + fn get( + &self, + _store_name: &StoreName, + _key: &str, + ) -> Result> { + self.reads.fetch_add(1, Ordering::Relaxed); + Err(Report::new(PlatformError::ConfigStore)) + } + + fn put( + &self, + _store_id: &StoreId, + _key: &str, + _value: &str, + ) -> Result<(), Report> { + Err(Report::new(PlatformError::Unsupported)) + } + + fn delete(&self, _store_id: &StoreId, _key: &str) -> Result<(), Report> { + Err(Report::new(PlatformError::Unsupported)) + } + } + + struct CountingConfigStore { + reads: AtomicUsize, + current_kid: String, + delay: Duration, + } + + impl PlatformConfigStore for CountingConfigStore { + fn get(&self, _store_name: &StoreName, key: &str) -> Result> { + self.reads.fetch_add(1, Ordering::Relaxed); + if !self.delay.is_zero() { + std::thread::sleep(self.delay); + } + (key == "current-kid") + .then(|| self.current_kid.clone()) + .ok_or_else(|| Report::new(PlatformError::ConfigStore)) + } + + fn put( + &self, + _store_id: &StoreId, + _key: &str, + _value: &str, + ) -> Result<(), Report> { + Err(Report::new(PlatformError::Unsupported)) + } + + fn delete(&self, _store_id: &StoreId, _key: &str) -> Result<(), Report> { + Err(Report::new(PlatformError::Unsupported)) + } + } + + struct CountingSecretStore { + reads: AtomicUsize, + key: Vec, + } + + impl PlatformSecretStore for CountingSecretStore { + fn get_bytes( + &self, + _store_name: &StoreName, + _key: &str, + ) -> Result, Report> { + self.reads.fetch_add(1, Ordering::Relaxed); + Ok(self.key.clone()) + } + + fn create( + &self, + _store_id: &StoreId, + _name: &str, + _value: &str, + ) -> Result<(), Report> { + Err(Report::new(PlatformError::Unsupported)) + } + + fn delete(&self, _store_id: &StoreId, _name: &str) -> Result<(), Report> { + Err(Report::new(PlatformError::Unsupported)) + } + } + + struct OuterSelectErrorHttpClient { + inner: StubHttpClient, + selected_pending: AtomicUsize, + } + + impl OuterSelectErrorHttpClient { + fn new() -> Self { + Self { + inner: StubHttpClient::new(), + selected_pending: AtomicUsize::new(0), + } + } + + fn push_response(&self, status: u16, body: Vec) { + self.inner.push_response(status, body); + } + } + + #[async_trait::async_trait(?Send)] + impl PlatformHttpClient for OuterSelectErrorHttpClient { + async fn send( + &self, + request: PlatformHttpRequest, + ) -> Result> { + self.inner.send(request).await + } + + async fn send_async( + &self, + request: PlatformHttpRequest, + ) -> Result> { + self.inner.send_async(request).await + } + + async fn select( + &self, + pending_requests: Vec, + ) -> Result> { + self.selected_pending + .store(pending_requests.len(), Ordering::Relaxed); + Err(Report::new(PlatformError::HttpClient)) + } + } + + struct UnusedSecretStore; + + impl PlatformSecretStore for UnusedSecretStore { + fn get_bytes( + &self, + _store_name: &StoreName, + _key: &str, + ) -> Result, Report> { + panic!("signing key should not be read after current-kid failure") + } + + fn create( + &self, + _store_id: &StoreId, + _name: &str, + _value: &str, + ) -> Result<(), Report> { + Err(Report::new(PlatformError::Unsupported)) + } + + fn delete(&self, _store_id: &StoreId, _name: &str) -> Result<(), Report> { + Err(Report::new(PlatformError::Unsupported)) + } + } + + // --------------------------------------------------------------------------- + // Minimal test double for AuctionProvider + // --------------------------------------------------------------------------- + + struct StubAuctionProvider { name: &'static str, backend: &'static str, } #[async_trait::async_trait(?Send)] impl AuctionProvider for StubAuctionProvider { - fn provider_name(&self) -> &'static str { + fn provider_name(&self) -> &str { self.name } @@ -1616,17 +3169,14 @@ mod tests { } } - struct RecordingTimeoutProvider { + struct DeadlineBidProvider { name: &'static str, backend: &'static str, - configured_timeout_ms: u32, - predicted: Arc>>, - requested: Arc>>, } #[async_trait::async_trait(?Send)] - impl AuctionProvider for RecordingTimeoutProvider { - fn provider_name(&self) -> &'static str { + impl AuctionProvider for DeadlineBidProvider { + fn provider_name(&self) -> &str { self.name } @@ -1635,16 +3185,12 @@ mod tests { _request: &AuctionRequest, context: &AuctionContext<'_>, ) -> Result> { - self.requested - .lock() - .expect("should lock requested timeouts") - .push(context.timeout_ms); let request = PlatformHttpRequest::new( http::Request::builder() .method("POST") .uri("https://example.com/bid") .body(edgezero_core::body::Body::empty()) - .expect("should build recording request"), + .expect("should build deadline test request"), self.backend, ); context @@ -1653,7 +3199,7 @@ mod tests { .send_async(request) .await .change_context(TrustedServerError::Auction { - message: "recording launch failed".to_string(), + message: "deadline test provider launch failed".to_string(), }) .map(ProviderRequestOutcome::pending) } @@ -1665,34 +3211,33 @@ mod tests { ) -> Result> { Ok(AuctionResponse::success( self.name, - vec![], + vec![auction_bid(self.name, 3.0)], response_time_ms, )) } fn timeout_ms(&self) -> u32 { - self.configured_timeout_ms + 1_000 } - fn backend_name(&self, _services: &RuntimeServices, timeout_ms: u32) -> Option { - self.predicted - .lock() - .expect("should lock predicted timeouts") - .push(timeout_ms); + fn backend_name(&self, _services: &RuntimeServices, _timeout_ms: u32) -> Option { Some(self.backend.to_string()) } } - struct DivergentBackendProvider { - name: &'static str, - predicted: &'static str, - resolved: &'static str, + type RecordedMediatorBudgets = Arc>>; + + struct DeadlineRecordingMediator { + launches: Arc, + budgets: Option, } + struct PendingDeadlineMediator; + #[async_trait::async_trait(?Send)] - impl AuctionProvider for DivergentBackendProvider { - fn provider_name(&self) -> &'static str { - self.name + impl AuctionProvider for PendingDeadlineMediator { + fn provider_name(&self) -> &str { + "pending-deadline-mediator" } async fn request_bids( @@ -1703,10 +3248,10 @@ mod tests { let request = PlatformHttpRequest::new( http::Request::builder() .method("POST") - .uri("https://example.com/bid") + .uri("https://example.com/mediate") .body(edgezero_core::body::Body::empty()) - .expect("should build divergent request"), - self.resolved, + .expect("should build pending mediator request"), + "pending-mediator-backend", ); context .services @@ -1714,7 +3259,7 @@ mod tests { .send_async(request) .await .change_context(TrustedServerError::Auction { - message: "divergent launch failed".to_string(), + message: "pending mediator launch failed".to_string(), }) .map(ProviderRequestOutcome::pending) } @@ -1725,28 +3270,193 @@ mod tests { response_time_ms: u64, ) -> Result> { Ok(AuctionResponse::success( - self.name, - vec![], + self.provider_name(), + vec![auction_bid("mediated", 9.0)], response_time_ms, )) } fn timeout_ms(&self) -> u32 { - 2000 + 1_000 } fn backend_name(&self, _services: &RuntimeServices, _timeout_ms: u32) -> Option { - Some(self.predicted.to_string()) + Some("pending-mediator-backend".to_string()) } } - struct CanonicalTimeoutBackend { - canonical_ms: u32, - calls: Arc>>, - } + #[async_trait::async_trait(?Send)] + impl AuctionProvider for DeadlineRecordingMediator { + fn provider_name(&self) -> &str { + "deadline-mediator" + } - impl PlatformBackend for CanonicalTimeoutBackend { - fn predict_name( + async fn request_bids( + &self, + _request: &AuctionRequest, + context: &AuctionContext<'_>, + ) -> Result> { + self.launches.fetch_add(1, Ordering::Relaxed); + if let Some(budgets) = &self.budgets { + budgets + .lock() + .expect("should lock mediator budgets") + .push((context.timeout_ms, context.transport_timeout_ms)); + } + Ok(ProviderRequestOutcome::Immediate(AuctionResponse::no_bid( + self.provider_name(), + 0, + ))) + } + + async fn parse_response( + &self, + _response: PlatformResponse, + _response_time_ms: u64, + ) -> Result> { + panic!("immediate mediator response should not be parsed"); + } + + fn timeout_ms(&self) -> u32 { + 1_000 + } + } + + struct RecordingTimeoutProvider { + name: &'static str, + backend: &'static str, + configured_timeout_ms: u32, + predicted: Arc>>, + requested: Arc>>, + } + + #[async_trait::async_trait(?Send)] + impl AuctionProvider for RecordingTimeoutProvider { + fn provider_name(&self) -> &str { + self.name + } + + async fn request_bids( + &self, + _request: &AuctionRequest, + context: &AuctionContext<'_>, + ) -> Result> { + self.requested + .lock() + .expect("should lock requested timeouts") + .push(context.transport_timeout_ms); + let request = PlatformHttpRequest::new( + http::Request::builder() + .method("POST") + .uri("https://example.com/bid") + .body(edgezero_core::body::Body::empty()) + .expect("should build recording request"), + self.backend, + ); + context + .services + .http_client() + .send_async(request) + .await + .change_context(TrustedServerError::Auction { + message: "recording launch failed".to_string(), + }) + .map(ProviderRequestOutcome::pending) + } + + async fn parse_response( + &self, + _response: PlatformResponse, + response_time_ms: u64, + ) -> Result> { + Ok(AuctionResponse::success( + self.name, + vec![], + response_time_ms, + )) + } + + fn timeout_ms(&self) -> u32 { + self.configured_timeout_ms + } + + fn backend_name(&self, _services: &RuntimeServices, timeout_ms: u32) -> Option { + self.predicted + .lock() + .expect("should lock predicted timeouts") + .push(timeout_ms); + Some(self.backend.to_string()) + } + } + + struct DivergentBackendProvider { + name: &'static str, + predicted: &'static str, + resolved: &'static str, + } + + #[async_trait::async_trait(?Send)] + impl AuctionProvider for DivergentBackendProvider { + fn provider_name(&self) -> &str { + self.name + } + + async fn request_bids( + &self, + _request: &AuctionRequest, + context: &AuctionContext<'_>, + ) -> Result> { + let request = PlatformHttpRequest::new( + http::Request::builder() + .method("POST") + .uri("https://example.com/bid") + .body(edgezero_core::body::Body::empty()) + .expect("should build divergent request"), + self.resolved, + ); + context + .services + .http_client() + .send_async(request) + .await + .change_context(TrustedServerError::Auction { + message: "divergent launch failed".to_string(), + }) + .map(ProviderRequestOutcome::pending) + } + + async fn parse_response( + &self, + _response: PlatformResponse, + response_time_ms: u64, + ) -> Result> { + Ok(AuctionResponse::success( + self.name, + vec![], + response_time_ms, + )) + } + + fn timeout_ms(&self) -> u32 { + 2000 + } + + fn backend_name(&self, _services: &RuntimeServices, _timeout_ms: u32) -> Option { + Some(self.predicted.to_string()) + } + } + + struct CanonicalTimeoutBackend { + canonical_ms: u32, + calls: Arc>>, + } + + impl PlatformBackend for CanonicalTimeoutBackend { + fn naming_policy(&self) -> crate::platform::BackendNamingPolicy { + crate::platform::BackendNamingPolicy::Axum + } + + fn predict_name( &self, _spec: &PlatformBackendSpec, ) -> Result> { @@ -1810,6 +3520,7 @@ mod tests { .then(|| "
ordinary
".to_string()), adomain: None, bidder: bidder.to_string(), + returned_seat: None, width: 300, height: 250, nurl: None, @@ -1833,6 +3544,7 @@ mod tests { creative: Some("
ad
".to_string()), adomain: None, bidder: "mediator".to_string(), + returned_seat: None, width: 728, height: 90, nurl: nurl.clone(), @@ -1850,7 +3562,7 @@ mod tests { #[async_trait::async_trait(?Send)] impl AuctionProvider for CacheRestoringMediator { - fn provider_name(&self) -> &'static str { + fn provider_name(&self) -> &str { "mediator" } @@ -1919,7 +3631,7 @@ mod tests { #[async_trait::async_trait(?Send)] impl AuctionProvider for ImmediateMediator { - fn provider_name(&self) -> &'static str { + fn provider_name(&self) -> &str { "immediate-mediator" } @@ -1965,7 +3677,7 @@ mod tests { let config = AuctionConfig { enabled: true, - providers: vec!["bidder".to_string()], + providers: AuctionConfig::legacy_provider_map(&["bidder"]), mediator: Some("mediator".to_string()), timeout_ms: 2000, ..Default::default() @@ -1988,6 +3700,7 @@ mod tests { settings: &settings, request: &req, timeout_ms: 2000, + transport_timeout_ms: 2000, provider_responses: None, services, }; @@ -2021,7 +3734,7 @@ mod tests { let services = build_services_with_http_client(stub); let config = AuctionConfig { enabled: true, - providers: vec!["bidder".to_string()], + providers: AuctionConfig::legacy_provider_map(&["bidder"]), mediator: Some("immediate-mediator".to_string()), timeout_ms: 2000, ..Default::default() @@ -2039,6 +3752,7 @@ mod tests { settings: &settings, request: &downstream, timeout_ms: 2000, + transport_timeout_ms: 2000, provider_responses: None, services: &services, }; @@ -2076,74 +3790,334 @@ mod tests { } } - fn create_test_auction_request() -> AuctionRequest { - AuctionRequest { - id: "test-auction-123".to_string(), - slots: vec![ - AdSlot { - id: "header-banner".to_string(), - formats: vec![AdFormat { - media_type: MediaType::Banner, - width: 728, - height: 90, - }], - floor_price: Some(1.50), - targeting: HashMap::new(), - bidders: HashMap::new(), - }, - AdSlot { - id: "sidebar".to_string(), - formats: vec![AdFormat { - media_type: MediaType::Banner, - width: 300, - height: 250, - }], - floor_price: Some(1.00), - targeting: HashMap::new(), - bidders: HashMap::new(), - }, - ], - publisher: PublisherInfo { - domain: "test.com".to_string(), - page_url: Some("https://test.com/article".to_string()), - }, - user: UserInfo { - id: Some("user-123".to_string()), - consent: None, - eids: None, - }, - device: None, - site: None, - context: HashMap::new(), - } - } + async fn collect_deadline_test_result( + split: bool, + enforceable_total_request_deadline: bool, + ) -> OrchestrationResult { + let stub = Arc::new(StubHttpClient::new()); + stub.set_enforceable_total_request_deadline(enforceable_total_request_deadline); + stub.push_response(200, b"{}".to_vec()); + stub.push_response(200, b"{}".to_vec()); + stub.push_select_delay(Duration::from_millis(50)); + let services = build_services_with_http_client(Arc::clone(&stub) as Arc<_>); + let config = AuctionConfig { + enabled: true, + providers: AuctionConfig::legacy_provider_map(&["late-one", "late-two"]), + timeout_ms: 10, + ..Default::default() + }; + let mut orchestrator = AuctionOrchestrator::new(config); + orchestrator.register_provider(Arc::new(DeadlineBidProvider { + name: "late-one", + backend: "late-one-backend", + })); + orchestrator.register_provider(Arc::new(DeadlineBidProvider { + name: "late-two", + backend: "late-two-backend", + })); + let request = create_test_auction_request(); + let settings = create_test_settings(); + let downstream = http::Request::new(edgezero_core::body::Body::empty()); + let context = AuctionContext { + settings: &settings, + request: &downstream, + timeout_ms: 10, + transport_timeout_ms: 10, + provider_responses: None, + services: &services, + }; - fn create_test_settings() -> crate::settings::Settings { - let settings_str = crate_test_settings_str(); - crate::settings::Settings::from_toml(&settings_str).expect("should parse test settings") + if split { + let DispatchAuctionOutcome::Dispatched(dispatched) = + orchestrator.dispatch_auction(&request, &context).await + else { + panic!("deadline test providers should dispatch"); + }; + orchestrator + .collect_dispatched_auction(dispatched, &services, &context) + .await + } else { + orchestrator + .run_auction(&request, &context) + .await + .expect("deadline test auction should complete") + } } - struct ImmediateNoBidProvider; - - #[async_trait::async_trait(?Send)] - impl AuctionProvider for ImmediateNoBidProvider { - fn provider_name(&self) -> &'static str { - "immediate" + #[tokio::test] + async fn current_adapter_deadline_drains_late_responses_in_both_paths() { + for split in [false, true] { + let result = collect_deadline_test_result(split, false).await; + assert_eq!(result.provider_responses.len(), 2); + assert_eq!(result.provider_responses[0].provider, "late-one"); + assert_eq!(result.provider_responses[0].status, BidStatus::Success); + assert_eq!(result.provider_responses[1].provider, "late-two"); + assert_eq!(result.provider_responses[1].status, BidStatus::Success); + assert!( + result + .provider_responses + .iter() + .all(|response| response.response_time_ms >= 50), + "late response times should retain actual elapsed duration" + ); + assert_eq!( + result.winning_bids["slot-1"].bidder, "late-one", + "a completed response remains eligible after the logical deadline" + ); } + } - async fn request_bids( - &self, - _request: &AuctionRequest, - _context: &AuctionContext<'_>, - ) -> Result> { - Ok(ProviderRequestOutcome::Immediate(AuctionResponse::no_bid( - "immediate", - 0, - ))) + #[tokio::test] + async fn synthetic_hard_deadline_classifies_late_responses_in_both_paths() { + for split in [false, true] { + let result = collect_deadline_test_result(split, true).await; + assert_eq!(result.provider_responses.len(), 2); + assert!(result.provider_responses.iter().all(|response| { + response.status == BidStatus::Error + && response.metadata["error_type"] == ERROR_TYPE_TIMEOUT + && response.response_time_ms >= 50 + })); + assert!(result.winning_bids.is_empty()); } + } - async fn parse_response( - &self, + async fn pending_mediator_deadline_test_result( + split: bool, + enforceable_total_request_deadline: bool, + ) -> OrchestrationResult { + let stub = Arc::new(StubHttpClient::new()); + stub.set_enforceable_total_request_deadline(enforceable_total_request_deadline); + stub.push_response(200, b"{}".to_vec()); + stub.push_response(200, b"{}".to_vec()); + stub.push_select_delay(Duration::ZERO); + stub.push_select_delay(Duration::from_millis(50)); + let services = build_services_with_http_client(Arc::clone(&stub) as Arc<_>); + let config = AuctionConfig { + enabled: true, + providers: AuctionConfig::legacy_provider_map(&["local"]), + mediator: Some("pending-deadline-mediator".to_string()), + timeout_ms: 20, + ..Default::default() + }; + let mut orchestrator = AuctionOrchestrator::new(config); + orchestrator.register_provider(Arc::new(DeadlineBidProvider { + name: "local", + backend: "local-backend", + })); + orchestrator.register_provider(Arc::new(PendingDeadlineMediator)); + let request = create_test_auction_request(); + let settings = create_test_settings(); + let downstream = http::Request::new(edgezero_core::body::Body::empty()); + let context = AuctionContext { + settings: &settings, + request: &downstream, + timeout_ms: 20, + transport_timeout_ms: 20, + provider_responses: None, + services: &services, + }; + + if split { + let DispatchAuctionOutcome::Dispatched(dispatched) = + orchestrator.dispatch_auction(&request, &context).await + else { + panic!("deadline test provider should dispatch"); + }; + orchestrator + .collect_dispatched_auction(dispatched, &services, &context) + .await + } else { + orchestrator + .run_auction(&request, &context) + .await + .expect("deadline test auction should complete") + } + } + + #[tokio::test] + async fn pending_mediator_late_completion_policy_is_equivalent_in_sync_and_split_paths() { + for split in [false, true] { + let current = pending_mediator_deadline_test_result(split, false).await; + let current_mediator = current + .mediator_response + .as_ref() + .expect("current adapters should accept completed late mediator responses"); + assert!( + current_mediator.response_time_ms >= 50, + "mediator timing should preserve actual elapsed duration" + ); + assert_eq!(current.winning_bids["slot-1"].bidder, "mediated"); + + let hard = pending_mediator_deadline_test_result(split, true).await; + assert!(hard.mediator_response.is_none()); + assert_eq!(hard.winning_bids["slot-1"].bidder, "local"); + assert!( + hard.total_time_ms >= 50, + "discarding a late mediator must retain actual total elapsed time" + ); + } + } + + #[tokio::test] + async fn split_deadline_skips_mediator_and_falls_back_to_provider_winner() { + let stub = Arc::new(StubHttpClient::new()); + stub.push_response(200, b"{}".to_vec()); + stub.push_select_delay(Duration::from_millis(50)); + let services = build_services_with_http_client(Arc::clone(&stub) as Arc<_>); + let launches = Arc::new(AtomicUsize::new(0)); + let config = AuctionConfig { + enabled: true, + providers: AuctionConfig::legacy_provider_map(&["late-one"]), + mediator: Some("deadline-mediator".to_string()), + timeout_ms: 10, + ..Default::default() + }; + let mut orchestrator = AuctionOrchestrator::new(config); + orchestrator.register_provider(Arc::new(DeadlineBidProvider { + name: "late-one", + backend: "late-one-backend", + })); + orchestrator.register_provider(Arc::new(DeadlineRecordingMediator { + launches: Arc::clone(&launches), + budgets: None, + })); + let request = create_test_auction_request(); + let settings = create_test_settings(); + let downstream = http::Request::new(edgezero_core::body::Body::empty()); + let context = AuctionContext { + settings: &settings, + request: &downstream, + timeout_ms: 10, + transport_timeout_ms: 10, + provider_responses: None, + services: &services, + }; + let DispatchAuctionOutcome::Dispatched(dispatched) = + orchestrator.dispatch_auction(&request, &context).await + else { + panic!("deadline test provider should dispatch"); + }; + let result = orchestrator + .collect_dispatched_auction(dispatched, &services, &context) + .await; + + assert_eq!(launches.load(Ordering::Relaxed), 0); + assert!(result.mediator_response.is_none()); + assert_eq!(result.winning_bids["slot-1"].bidder, "late-one"); + } + + #[tokio::test] + async fn synchronous_deadline_skips_mediator_and_falls_back_to_provider_winner() { + let stub = Arc::new(StubHttpClient::new()); + stub.push_response(200, b"{}".to_vec()); + stub.push_select_delay(Duration::from_millis(50)); + let services = build_services_with_http_client(Arc::clone(&stub) as Arc<_>); + let launches = Arc::new(AtomicUsize::new(0)); + let config = AuctionConfig { + enabled: true, + providers: AuctionConfig::legacy_provider_map(&["late-one"]), + mediator: Some("deadline-mediator".to_string()), + timeout_ms: 10, + ..Default::default() + }; + let mut orchestrator = AuctionOrchestrator::new(config); + orchestrator.register_provider(Arc::new(DeadlineBidProvider { + name: "late-one", + backend: "late-one-backend", + })); + orchestrator.register_provider(Arc::new(DeadlineRecordingMediator { + launches: Arc::clone(&launches), + budgets: None, + })); + let request = create_test_auction_request(); + let settings = create_test_settings(); + let downstream = http::Request::new(edgezero_core::body::Body::empty()); + let context = AuctionContext { + settings: &settings, + request: &downstream, + timeout_ms: 10, + transport_timeout_ms: 10, + provider_responses: None, + services: &services, + }; + let result = orchestrator + .run_auction(&request, &context) + .await + .expect("synchronous deadline test should complete"); + + assert_eq!(launches.load(Ordering::Relaxed), 0); + assert!(result.mediator_response.is_none()); + assert_eq!(result.winning_bids["slot-1"].bidder, "late-one"); + } + + fn create_test_auction_request() -> AuctionRequest { + AuctionRequest { + id: "test-auction-123".to_string(), + slots: vec![ + AdSlot { + id: "header-banner".to_string(), + formats: vec![AdFormat { + media_type: MediaType::Banner, + width: 728, + height: 90, + }], + floor_price: Some(1.50), + targeting: HashMap::new(), + bidders: HashMap::new(), + }, + AdSlot { + id: "sidebar".to_string(), + formats: vec![AdFormat { + media_type: MediaType::Banner, + width: 300, + height: 250, + }], + floor_price: Some(1.00), + targeting: HashMap::new(), + bidders: HashMap::new(), + }, + ], + publisher: PublisherInfo { + domain: "test.com".to_string(), + page_url: Some("https://test.com/article".to_string()), + }, + user: UserInfo { + id: Some("user-123".to_string()), + consent: None, + eids: None, + }, + device: None, + site: None, + context: HashMap::new(), + } + } + + fn create_test_settings() -> crate::settings::Settings { + let settings_str = crate_test_settings_str(); + crate::settings::Settings::from_toml(&settings_str).expect("should parse test settings") + } + + struct ImmediateNoBidProvider; + + #[async_trait::async_trait(?Send)] + impl AuctionProvider for ImmediateNoBidProvider { + fn provider_name(&self) -> &str { + "immediate" + } + + async fn request_bids( + &self, + _request: &AuctionRequest, + _context: &AuctionContext<'_>, + ) -> Result> { + Ok(ProviderRequestOutcome::Immediate(AuctionResponse::no_bid( + "immediate", + 0, + ))) + } + + async fn parse_response( + &self, _response: PlatformResponse, _response_time_ms: u64, ) -> Result> { @@ -2159,7 +4133,7 @@ mod tests { #[async_trait::async_trait(?Send)] impl AuctionProvider for LaunchFailingProvider { - fn provider_name(&self) -> &'static str { + fn provider_name(&self) -> &str { "launch-failing" } @@ -2201,6 +4175,7 @@ mod tests { settings, request, timeout_ms: 2000, + transport_timeout_ms: 2000, provider_responses: None, services, } @@ -2210,7 +4185,7 @@ mod tests { async fn synchronous_auction_accepts_an_all_immediate_no_bid_result() { let config = AuctionConfig { enabled: true, - providers: vec!["immediate".to_string()], + providers: AuctionConfig::legacy_provider_map(&["immediate"]), timeout_ms: 2000, ..Default::default() }; @@ -2235,7 +4210,7 @@ mod tests { async fn split_auction_accepts_an_all_immediate_no_bid_result() { let config = AuctionConfig { enabled: true, - providers: vec!["immediate".to_string()], + providers: AuctionConfig::legacy_provider_map(&["immediate"]), timeout_ms: 2000, ..Default::default() }; @@ -2266,7 +4241,7 @@ mod tests { for split in [false, true] { let config = AuctionConfig { enabled: true, - providers: vec!["immediate".to_string(), "pending".to_string()], + providers: AuctionConfig::legacy_provider_map(&["immediate", "pending"]), timeout_ms: 2000, ..Default::default() }; @@ -2424,6 +4399,7 @@ mod tests { creative: Some("
Ad
".to_string()), adomain: None, bidder: "test-bidder".to_string(), + returned_seat: None, width: 300, height: 250, nurl: None, @@ -2447,6 +4423,7 @@ mod tests { creative: Some("
Ad
".to_string()), adomain: None, bidder: "test-bidder".to_string(), + returned_seat: None, width: 300, height: 250, nurl: None, @@ -2492,7 +4469,8 @@ mod tests { enabled: true, sanitize_creatives: true, rewrite_creatives: true, - providers: vec![], + providers: AuctionConfig::legacy_provider_map(&[]), + bidders: Default::default(), mediator: None, timeout_ms: 2000, creative_store: "creative_store".to_string(), @@ -2523,7 +4501,7 @@ mod tests { futures::executor::block_on(async { let config = AuctionConfig { enabled: true, - providers: vec!["launch-failing".to_string()], + providers: AuctionConfig::legacy_provider_map(&["launch-failing"]), timeout_ms: 2000, ..Default::default() }; @@ -2553,41 +4531,12 @@ mod tests { }); } - #[test] - fn rejects_duplicate_configured_providers() { - let config = AuctionConfig { - enabled: true, - providers: vec!["prebid".to_string(), "prebid".to_string()], - timeout_ms: 2000, - ..Default::default() - }; - let err = AuctionOrchestrator::new(config) - .validate_configured_provider_names() - .expect_err("should reject a provider listed more than once"); - assert!(err.to_string().contains("listed more than once")); - } - - #[test] - fn rejects_mediator_also_listed_as_provider() { - let config = AuctionConfig { - enabled: true, - providers: vec!["prebid".to_string()], - mediator: Some("prebid".to_string()), - timeout_ms: 2000, - ..Default::default() - }; - let err = AuctionOrchestrator::new(config) - .validate_configured_provider_names() - .expect_err("should reject a mediator also configured as a provider"); - assert!(err.to_string().contains("may not mediate its own auction")); - } - #[tokio::test] async fn duplicate_backend_name_fails_second_provider_attributably_in_both_paths() { for split in [false, true] { let config = AuctionConfig { enabled: true, - providers: vec!["provider-a".to_string(), "provider-b".to_string()], + providers: AuctionConfig::legacy_provider_map(&["provider-a", "provider-b"]), timeout_ms: 2000, ..Default::default() }; @@ -2702,7 +4651,7 @@ mod tests { let requested = Arc::new(Mutex::new(Vec::new())); let mut orchestrator = AuctionOrchestrator::new(AuctionConfig { enabled: true, - providers: vec!["bidder".to_string()], + providers: AuctionConfig::legacy_provider_map(&["bidder"]), timeout_ms: 2000, ..Default::default() }); @@ -2746,7 +4695,7 @@ mod tests { let requested = Arc::new(Mutex::new(Vec::new())); let mut orchestrator = AuctionOrchestrator::new(AuctionConfig { enabled: true, - providers: vec!["bidder".to_string()], + providers: AuctionConfig::legacy_provider_map(&["bidder"]), timeout_ms: 2000, ..Default::default() }); @@ -2789,7 +4738,7 @@ mod tests { let requested = Arc::new(Mutex::new(Vec::new())); let mut orchestrator = AuctionOrchestrator::new(AuctionConfig { enabled: true, - providers: vec!["bidder".to_string()], + providers: AuctionConfig::legacy_provider_map(&["bidder"]), mediator: Some("mediator".to_string()), timeout_ms: 2000, ..Default::default() @@ -2839,7 +4788,7 @@ mod tests { let mediator_requested = Arc::new(Mutex::new(Vec::new())); let mut orchestrator = AuctionOrchestrator::new(AuctionConfig { enabled: true, - providers: vec!["bidder".to_string()], + providers: AuctionConfig::legacy_provider_map(&["bidder"]), mediator: Some("mediator".to_string()), timeout_ms: 2000, ..Default::default() @@ -2893,6 +4842,64 @@ mod tests { }); } + #[test] + fn planned_collect_skips_mediator_with_zero_canonical_transport_budget() { + futures::executor::block_on(async { + let calls = Arc::new(Mutex::new(Vec::new())); + let services = build_services_with_backend_and_http_client( + Arc::new(CanonicalTimeoutBackend { + canonical_ms: 0, + calls: Arc::clone(&calls), + }), + Arc::new(StubHttpClient::new()), + ); + let launches = Arc::new(AtomicUsize::new(0)); + let budgets = Arc::new(Mutex::new(Vec::new())); + let plan = AuctionPlan::compile(AuctionPlanConfig { + timeout_ms: 49, + providers: BTreeMap::new(), + bidders: BTreeMap::new(), + mediator: Some("adserver_mock".to_string()), + request_signing: None, + }) + .expect("should compile mediator-only plan") + .with_enabled(true); + let orchestrator = AuctionOrchestrator::from_plan( + Arc::new(plan), + Some(Arc::new(DeadlineRecordingMediator { + launches: Arc::clone(&launches), + budgets: Some(Arc::clone(&budgets)), + })), + ); + let request = create_test_auction_request(); + let settings = create_test_settings(); + let downstream = http::Request::new(edgezero_core::body::Body::empty()); + let context = AuctionContext { + settings: &settings, + request: &downstream, + timeout_ms: 49, + transport_timeout_ms: 49, + provider_responses: None, + services: &services, + }; + let dispatched = DispatchedAuction::empty_for_test(request, 49); + + let result = orchestrator + .collect_dispatched_auction(dispatched, &services, &context) + .await; + + assert_eq!(launches.load(Ordering::Relaxed), 0); + assert!( + budgets + .lock() + .expect("should lock mediator budgets") + .is_empty() + ); + assert_eq!(calls.lock().expect("should lock calls").len(), 1); + assert!(result.mediator_response.is_none()); + }); + } + #[test] fn dispatched_resolved_backend_name_diverging_from_prediction_still_correlates() { futures::executor::block_on(async { @@ -2901,7 +4908,7 @@ mod tests { let services = build_services_with_http_client(stub); let mut orchestrator = AuctionOrchestrator::new(AuctionConfig { enabled: true, - providers: vec!["provider-a".to_string()], + providers: AuctionConfig::legacy_provider_map(&["provider-a"]), timeout_ms: 2000, ..Default::default() }); @@ -2939,7 +4946,7 @@ mod tests { let services = build_services_with_http_client(stub); let mut orchestrator = AuctionOrchestrator::new(AuctionConfig { enabled: true, - providers: vec!["provider-a".to_string(), "provider-b".to_string()], + providers: AuctionConfig::legacy_provider_map(&["provider-a", "provider-b"]), timeout_ms: 2000, ..Default::default() }); @@ -2994,7 +5001,7 @@ mod tests { let config = AuctionConfig { enabled: true, - providers: vec!["provider-a".to_string(), "provider-b".to_string()], + providers: AuctionConfig::legacy_provider_map(&["provider-a", "provider-b"]), timeout_ms: 2000, mediator: None, ..Default::default() @@ -3020,6 +5027,7 @@ mod tests { settings: &settings, request: &req, timeout_ms: 2000, + transport_timeout_ms: 2000, provider_responses: None, services, }; @@ -3061,6 +5069,69 @@ mod tests { }); } + #[tokio::test] + async fn outer_select_error_materializes_all_planned_launches_as_transport_failures() { + let http = Arc::new(OuterSelectErrorHttpClient::new()); + http.push_response(204, Vec::new()); + http.push_response(204, Vec::new()); + let backend = Arc::new(NamingBackend::new(BackendNamingPolicy::Axum)); + let services = build_services_with_backend_and_http_client( + Arc::clone(&backend) as Arc<_>, + Arc::clone(&http) as Arc<_>, + ); + let plan = Arc::new( + AuctionPlan::compile(planned_config( + &[ + ("provider-b", RoutingMode::AllEligible), + ("provider-a", RoutingMode::AllEligible), + ], + false, + )) + .expect("should compile planned auction"), + ); + let orchestrator = AuctionOrchestrator::from_plan(plan, None); + let request = planned_request(); + let settings = create_test_settings(); + let inbound = http::Request::new(edgezero_core::body::Body::empty()); + let context = AuctionContext { + settings: &settings, + request: &inbound, + timeout_ms: 777, + transport_timeout_ms: 777, + provider_responses: None, + services: &services, + }; + + let DispatchAuctionOutcome::Dispatched(dispatched) = + orchestrator.dispatch_auction(&request, &context).await + else { + panic!("should dispatch both planned providers"); + }; + tokio::time::sleep(Duration::from_millis(5)).await; + let result = orchestrator + .collect_dispatched_auction(dispatched, &services, &context) + .await; + + assert_eq!(http.selected_pending.load(Ordering::Relaxed), 2); + assert_eq!( + result + .provider_responses + .iter() + .map(|response| response.provider.as_str()) + .collect::>(), + vec!["provider-a", "provider-b"], + "outer select errors should retain deterministic plan order" + ); + for response in &result.provider_responses { + assert_eq!(response.status, BidStatus::Error); + assert_eq!(response.metadata["error_type"], "transport"); + assert!( + response.response_time_ms >= 5, + "transport failure should preserve launch elapsed time" + ); + } + } + #[test] fn dispatched_collection_reuses_provider_launch_context() { futures::executor::block_on(async { @@ -3069,7 +5140,7 @@ mod tests { let services = build_services_with_http_client(stub); let config = AuctionConfig { enabled: true, - providers: vec!["provider-a".to_string()], + providers: AuctionConfig::legacy_provider_map(&["provider-a"]), timeout_ms: 750, mediator: None, ..Default::default() @@ -3090,6 +5161,7 @@ mod tests { settings: &settings, request: &downstream, timeout_ms: 750, + transport_timeout_ms: 750, provider_responses: None, services: &services, }; @@ -3108,6 +5180,7 @@ mod tests { settings: &settings, request: &placeholder, timeout_ms: 750, + transport_timeout_ms: 750, provider_responses: None, services: &services, }; @@ -3147,7 +5220,7 @@ mod tests { let config = AuctionConfig { enabled: true, - providers: vec!["provider-a".to_string(), "provider-b".to_string()], + providers: AuctionConfig::legacy_provider_map(&["provider-a", "provider-b"]), timeout_ms: 2000, mediator: None, ..Default::default() @@ -3173,6 +5246,7 @@ mod tests { settings: &settings, request: &req, timeout_ms: 2000, + transport_timeout_ms: 2000, provider_responses: None, services, }; @@ -3212,7 +5286,7 @@ mod tests { let config = AuctionConfig { enabled: true, - providers: vec!["provider-a".to_string(), "provider-b".to_string()], + providers: AuctionConfig::legacy_provider_map(&["provider-a", "provider-b"]), timeout_ms: 2000, mediator: None, ..Default::default() @@ -3238,6 +5312,7 @@ mod tests { settings: &settings, request: &req, timeout_ms: 2000, + transport_timeout_ms: 2000, provider_responses: None, services, }; @@ -3257,6 +5332,2118 @@ mod tests { }); } + #[tokio::test] + async fn from_plan_standard_provider_runs_direct_and_split_with_correlation_and_metadata() { + for split in [false, true] { + let http = Arc::new(StubHttpClient::new()); + http.push_response( + 200, + serde_json::to_vec(&serde_json::json!({ + "seatbid": [{"seat": "provider-seat", "bid": [{ + "id": "provider-bid", "impid": "fictional-slot", "price": 2.0, + "adm": "
provider
", "w": 300, "h": 250 + }]}] + })) + .expect("should serialize provider response"), + ); + let backend = Arc::new(NamingBackend::new(BackendNamingPolicy::Axum)); + let services = build_services_with_backend_and_http_client( + Arc::clone(&backend) as Arc<_>, + Arc::clone(&http) as Arc<_>, + ); + let mut config = planned_config(&[("provider-a", RoutingMode::AllEligible)], false); + config.bidders.insert( + "routed-bidder" + .parse() + .expect("should parse fictional bidder ID"), + crate::auction::plan::BidderRouteConfig { + provider: "provider-a" + .parse() + .expect("should parse fictional provider ID"), + }, + ); + let plan = Arc::new(AuctionPlan::compile(config).expect("should compile plan")); + let orchestrator = AuctionOrchestrator::from_plan(plan, None); + let mut request = planned_request(); + request.slots[0].bidders.insert( + "routed-bidder".to_string(), + serde_json::json!({"placement": 7}), + ); + request.slots[0].bidders.insert( + "unknown-private-id".to_string(), + serde_json::json!({"secret": 9}), + ); + let settings = create_test_settings(); + let inbound = http::Request::new(edgezero_core::body::Body::empty()); + let context = AuctionContext { + settings: &settings, + request: &inbound, + timeout_ms: 777, + transport_timeout_ms: 777, + provider_responses: None, + services: &services, + }; + + let result = if split { + let DispatchAuctionOutcome::Dispatched(dispatched) = + orchestrator.dispatch_auction(&request, &context).await + else { + panic!("standard provider should dispatch"); + }; + orchestrator + .collect_dispatched_auction(dispatched, &services, &context) + .await + } else { + orchestrator + .run_auction(&request, &context) + .await + .expect("standard provider should run") + }; + + assert_eq!(http.recorded_backend_names().len(), 1); + assert_eq!(backend.ensured.load(Ordering::Relaxed), 1); + assert_eq!(result.provider_responses.len(), 1); + assert_eq!(result.provider_responses[0].provider, "provider-a"); + assert_eq!(result.provider_responses[0].status, BidStatus::Success); + assert_eq!( + result.provider_responses[0].metadata["routing"]["unused_bidder_params_count"], + 1 + ); + assert_eq!(result.metadata["routing"]["unroutable_bidder_count"], 1); + assert_eq!( + result.winning_bids["fictional-slot"].bid_id.as_deref(), + Some("provider-bid") + ); + let metadata = + serde_json::to_string(&result.metadata).expect("should serialize auction metadata"); + assert!(!metadata.contains("unknown-private-id") && !metadata.contains("secret")); + } + } + + #[tokio::test] + async fn planned_executor_invokes_immediate_mediator_and_applies_floor() { + let http = Arc::new(StubHttpClient::new()); + http.push_response( + 200, + serde_json::to_vec(&serde_json::json!({ + "seatbid": [{"seat": "provider-seat", "bid": [{ + "id": "provider", "impid": "fictional-slot", "price": 2.0, + "adm": "
provider
", "w": 300, "h": 250 + }]}] + })) + .expect("should serialize provider response"), + ); + let backend = Arc::new(NamingBackend::new(BackendNamingPolicy::Axum)); + let services = build_services_with_backend_and_http_client( + Arc::clone(&backend) as Arc<_>, + Arc::clone(&http) as Arc<_>, + ); + let plan = AuctionPlan::compile(planned_config( + &[("provider-a", RoutingMode::AllEligible)], + false, + )) + .expect("should compile planned auction"); + let orchestrator = AuctionOrchestratorHarness::new(plan, Some(Arc::new(ImmediateMediator))); + let request = planned_request(); + let settings = create_test_settings(); + let inbound = http::Request::new(edgezero_core::body::Body::empty()); + let context = AuctionContext { + settings: &settings, + request: &inbound, + timeout_ms: 777, + transport_timeout_ms: 777, + provider_responses: None, + services: &services, + }; + + let result = orchestrator + .run_auction(&request, &context) + .await + .expect("should execute planned mediation"); + + assert_eq!( + result + .mediator_response + .as_ref() + .map(|response| response.provider.as_str()), + Some("immediate-mediator") + ); + assert_eq!( + result.winning_bids["header-banner"].nurl.as_deref(), + Some("https://nurl.example/immediate") + ); + assert!( + !result.winning_bids.contains_key("fictional-slot"), + "mediator output owns final selection" + ); + } + + async fn planned_pending_mediator_deadline_result( + enforceable_total_request_deadline: bool, + ) -> OrchestrationResult { + let http = Arc::new(StubHttpClient::new()); + http.set_enforceable_total_request_deadline(enforceable_total_request_deadline); + http.push_response( + 200, + serde_json::to_vec(&serde_json::json!({ + "seatbid": [{"seat": "provider-seat", "bid": [{ + "id": "provider", "impid": "fictional-slot", "price": 2.0, + "adm": "
provider
", "w": 300, "h": 250 + }]}] + })) + .expect("should serialize provider response"), + ); + http.push_response(200, b"{}".to_vec()); + http.push_select_delay(Duration::ZERO); + http.push_select_delay(Duration::from_millis(50)); + let backend = Arc::new(NamingBackend::new(BackendNamingPolicy::Axum)); + let services = build_services_with_backend_and_http_client( + Arc::clone(&backend) as Arc<_>, + Arc::clone(&http) as Arc<_>, + ); + let plan = AuctionPlan::compile(planned_config( + &[("provider-a", RoutingMode::AllEligible)], + false, + )) + .expect("should compile planned auction"); + let orchestrator = + AuctionOrchestratorHarness::new(plan, Some(Arc::new(PendingDeadlineMediator))); + let request = planned_request(); + let settings = create_test_settings(); + let inbound = http::Request::new(edgezero_core::body::Body::empty()); + let context = AuctionContext { + settings: &settings, + request: &inbound, + timeout_ms: 20, + transport_timeout_ms: 20, + provider_responses: None, + services: &services, + }; + + orchestrator + .run_auction(&request, &context) + .await + .expect("should execute planned pending mediator") + } + + #[tokio::test] + async fn planned_pending_mediator_applies_explicit_hard_deadline_policy() { + let current = planned_pending_mediator_deadline_result(false).await; + let current_mediator = current + .mediator_response + .as_ref() + .expect("current adapters should accept completed late mediator responses"); + assert!(current_mediator.response_time_ms >= 50); + assert_eq!(current.winning_bids["slot-1"].bidder, "mediated"); + + let hard = planned_pending_mediator_deadline_result(true).await; + assert!(hard.mediator_response.is_none()); + assert_eq!( + hard.winning_bids["fictional-slot"].bid_id.as_deref(), + Some("provider") + ); + assert!(hard.total_time_ms >= 50); + } + + #[tokio::test] + async fn planned_executor_mediator_transport_failure_falls_back_locally() { + let http = Arc::new(StubHttpClient::new()); + http.push_response( + 200, + serde_json::to_vec(&serde_json::json!({ + "seatbid": [{"seat": "provider-seat", "bid": [{ + "id": "provider", "impid": "fictional-slot", "price": 2.0, + "adm": "
provider
", "w": 300, "h": 250 + }]}] + })) + .expect("should serialize provider response"), + ); + http.push_response(200, b"{}".to_vec()); + http.push_select_success(); + http.push_select_error(); + let backend = Arc::new(NamingBackend::new(BackendNamingPolicy::Axum)); + let services = build_services_with_backend_and_http_client( + Arc::clone(&backend) as Arc<_>, + Arc::clone(&http) as Arc<_>, + ); + let plan = AuctionPlan::compile(planned_config( + &[("provider-a", RoutingMode::AllEligible)], + false, + )) + .expect("should compile planned auction"); + let orchestrator = + AuctionOrchestratorHarness::new(plan, Some(Arc::new(CacheRestoringMediator))); + let request = planned_request(); + let settings = create_test_settings(); + let inbound = http::Request::new(edgezero_core::body::Body::empty()); + let context = AuctionContext { + settings: &settings, + request: &inbound, + timeout_ms: 777, + transport_timeout_ms: 777, + provider_responses: None, + services: &services, + }; + + let result = orchestrator + .run_auction(&request, &context) + .await + .expect("should fall back from mediator transport failure"); + + assert!(result.mediator_response.is_none()); + assert_eq!( + result.winning_bids["fictional-slot"].bid_id.as_deref(), + Some("provider") + ); + } + + #[tokio::test] + async fn planned_prebid_instances_preserve_headers_metadata_suppression_and_identity() { + let http = Arc::new(StubHttpClient::new()); + http.push_response( + 200, + serde_json::to_vec(&serde_json::json!({ + "seatbid": [{"seat": "suppress-exact", "bid": [ + {"id":"good-a","impid":"fictional-slot","price":1.25,"adm":"
a
","w":300,"h":250,"nurl":"https://notify.example/win","burl":"https://notify.example/bill","ext":{"prebid":{"cache":{"bids":{"cacheId":"cache-a","url":"https://cache-a.example/cache/path"}}}}}, + {"id":"bad-a","price":2.0} + ]}], + "ext": {"responsetimemillis":{"suppress-exact":4},"errors":{"other":["fictional"]},"warnings":{"other":["warning"]},"debug":{"httpcalls":[]},"prebid":{"bidstatus":{"suppress-exact":[{"bidid":"good-a"}]}}} + })) + .expect("should serialize PBS response a"), + ); + http.push_response( + 200, + serde_json::to_vec(&serde_json::json!({ + "seatbid": [{"seat": "keep-seat", "bid": [{ + "id":"good-b","impid":"fictional-slot","price":2.5,"adm":"
b
","w":300,"h":250,"nurl":"https://notify.example/win","burl":"https://notify.example/bill" + }]}] + })) + .expect("should serialize PBS response b"), + ); + let backend = Arc::new(NamingBackend::new(BackendNamingPolicy::Fastly)); + let services = build_services_with_backend_and_http_client( + Arc::clone(&backend) as Arc<_>, + Arc::clone(&http) as Arc<_>, + ); + let notifications = NotificationConfig { + suppress_all: false, + suppress_seats: vec!["suppress-exact".to_string()], + }; + let plan = AuctionPlan::compile(planned_prebid_config(&[ + ( + "pbs-a", + serde_json::json!({"debug":true,"test_mode":true,"consent_forwarding":"openrtb_only"}), + notifications, + ), + ("pbs-b", serde_json::json!({}), NotificationConfig::default()), + ])) + .expect("should compile planned PBS auction"); + let orchestrator = AuctionOrchestratorHarness::new(plan, None); + let request = planned_request(); + let settings = create_test_settings(); + let inbound = http::Request::builder() + .uri("https://publisher.example/auction") + .header( + http::header::COOKIE, + "consent=keep; euconsent-v2=drop; other=value", + ) + .header(http::header::USER_AGENT, "Fictional Browser/7") + .header(http::header::REFERER, "https://referrer.example/story") + .header(http::header::ACCEPT_LANGUAGE, "en-US,en;q=0.9") + .header("x-forwarded-for", "203.0.113.250") + .body(edgezero_core::body::Body::empty()) + .expect("should build inbound request"); + let context = AuctionContext { + settings: &settings, + request: &inbound, + timeout_ms: 777, + transport_timeout_ms: 777, + provider_responses: None, + services: &services, + }; + + let result = orchestrator + .run_auction(&request, &context) + .await + .expect("should execute planned PBS auction"); + + assert_eq!(result.provider_responses.len(), 2); + let first = &result.provider_responses[0]; + assert_eq!(first.provider, "pbs-a"); + assert_eq!(first.bids.len(), 1, "should isolate malformed sibling"); + assert_eq!( + first.bids[0].returned_seat.as_deref(), + Some("suppress-exact") + ); + assert_eq!(first.bids[0].bidder, "suppress-exact"); + assert!( + first.bids[0].nurl.is_none(), + "should suppress after normalization" + ); + assert!( + first.bids[0].burl.is_none(), + "should suppress billing notification" + ); + assert_eq!(first.bids[0].cache_id.as_deref(), Some("cache-a")); + assert_eq!(first.bids[0].cache_host.as_deref(), Some("cache-a.example")); + assert_eq!(first.bids[0].cache_path.as_deref(), Some("/cache/path")); + assert_eq!(first.metadata["responsetimemillis"]["suppress-exact"], 4); + assert!(first.metadata.contains_key("errors")); + assert!(first.metadata.contains_key("warnings")); + assert!(first.metadata.contains_key("debug")); + assert!(first.metadata.contains_key("bidstatus")); + let second = &result.provider_responses[1]; + assert_eq!(second.provider, "pbs-b"); + assert_eq!(second.bids[0].returned_seat.as_deref(), Some("keep-seat")); + assert!(second.bids[0].nurl.is_some()); + assert!(!second.metadata.contains_key("debug")); + assert!(!second.metadata.contains_key("bidstatus")); + + let headers = http.recorded_request_headers(); + assert_eq!(headers.len(), 2); + for request_headers in &headers { + assert!( + request_headers + .iter() + .any(|(name, value)| name == "user-agent" && value == "Fictional Browser/7") + ); + assert!(request_headers.iter().any( + |(name, value)| name == "referer" && value == "https://referrer.example/story" + )); + assert!( + request_headers + .iter() + .any(|(name, value)| name == "accept-language" && value == "en-US,en;q=0.9") + ); + assert!( + request_headers + .iter() + .all(|(name, _)| name != "x-forwarded-for"), + "must ignore inbound XFF without attestation" + ); + assert!( + request_headers.iter().all(|(name, _)| name != "accept"), + "planned PBS transport must not add Accept beyond legacy headers" + ); + } + let first_cookie = headers[0] + .iter() + .find(|(name, _)| name == "cookie") + .map(|(_, value)| value.as_str()); + assert_eq!(first_cookie, Some("consent=keep; other=value")); + let second_cookie = headers[1] + .iter() + .find(|(name, _)| name == "cookie") + .map(|(_, value)| value.as_str()); + assert_eq!( + second_cookie, + Some("consent=keep; euconsent-v2=drop; other=value") + ); + } + + #[tokio::test] + async fn planned_aps_mock_mediation_preserves_three_identities_and_renderer() { + let http = Arc::new(StubHttpClient::new()); + http.push_response( + 200, + serde_json::to_vec(&serde_json::json!({ + "seatbid": [{"seat": "upstream-seat", "bid": [{ + "id": "aps-bid", "impid": "fictional-slot", "price": 2.0, + "w": 300, "h": 250, + "ext": {"creativeurl": "https://creative.example/render", "tagtype": "iframe"} + }]}] + })) + .expect("should serialize APS response"), + ); + http.push_response( + 200, + serde_json::to_vec(&serde_json::json!({ + "seatbid": [{"seat": "aps-instance", "bid": [{ + "id": "mediated-aps", "impid": "fictional-slot", "price": 2.0, + "adm": "ignored", "w": 300, "h": 250, "crid": "aps-creative" + }]}] + })) + .expect("should serialize mediator response"), + ); + let backend = Arc::new(NamingBackend::new(BackendNamingPolicy::Axum)); + let services = build_services_with_backend_and_http_client( + Arc::clone(&backend) as Arc<_>, + Arc::clone(&http) as Arc<_>, + ); + let plan = + AuctionPlan::compile(planned_aps_config()).expect("should compile planned APS auction"); + let mediator = AdServerMockProvider::new(AdServerMockConfig { + enabled: true, + endpoint: "https://mediator.example/mediate".to_string(), + timeout_ms: 500, + ..AdServerMockConfig::default() + }); + let orchestrator = AuctionOrchestratorHarness::new(plan, Some(Arc::new(mediator))); + let request = planned_request(); + let settings = create_test_settings(); + let inbound = http::Request::new(edgezero_core::body::Body::empty()); + let context = AuctionContext { + settings: &settings, + request: &inbound, + timeout_ms: 777, + transport_timeout_ms: 777, + provider_responses: None, + services: &services, + }; + + let result = orchestrator + .run_auction(&request, &context) + .await + .expect("should mediate planned APS bid"); + + let provider_bid = &result.provider_responses[0].bids[0]; + assert_eq!(result.provider_responses[0].provider, "aps-instance"); + assert_eq!(provider_bid.returned_seat.as_deref(), Some("upstream-seat")); + assert_eq!(provider_bid.bidder, "aps"); + let winner = &result.winning_bids["fictional-slot"]; + assert_eq!(winner.returned_seat.as_deref(), Some("upstream-seat")); + assert_eq!(winner.bidder, "aps"); + assert!(winner.renderer.is_some()); + assert!(winner.creative.is_none()); + assert_eq!( + result + .mediator_response + .as_ref() + .map(|response| response.provider.as_str()), + Some("adserver_mock") + ); + } + + #[tokio::test] + async fn planned_aps_transport_omits_accept_header() { + let http = Arc::new(StubHttpClient::new()); + http.push_response(400, Vec::new()); + let backend = Arc::new(NamingBackend::new(BackendNamingPolicy::Fastly)); + let services = build_services_with_backend_and_http_client( + Arc::clone(&backend) as Arc<_>, + Arc::clone(&http) as Arc<_>, + ); + let plan = + AuctionPlan::compile(planned_aps_config()).expect("should compile planned APS auction"); + let orchestrator = AuctionOrchestratorHarness::new(plan, None); + let request = planned_request(); + let settings = create_test_settings(); + let inbound = http::Request::builder() + .uri("https://publisher.example/auction") + .body(edgezero_core::body::Body::empty()) + .expect("should build inbound request"); + let context = AuctionContext { + settings: &settings, + request: &inbound, + timeout_ms: 777, + transport_timeout_ms: 777, + provider_responses: None, + services: &services, + }; + + orchestrator + .run_auction(&request, &context) + .await + .expect("should execute planned APS auction"); + + let headers = http.recorded_request_headers(); + assert_eq!(headers.len(), 1); + assert!( + headers[0].iter().all(|(name, _)| name != "accept"), + "planned APS transport must not add Accept beyond legacy headers" + ); + } + + #[tokio::test] + async fn planned_aps_profile_normalizes_renderer_reduction_and_metadata() { + let http = Arc::new(StubHttpClient::new()); + http.push_response_with_headers( + 200, + serde_json::to_vec(&serde_json::json!({ + "cur": "USD", + "seatbid": [ + {"seat": "returned-seat", "bid": [ + {"id": "z-high", "impid": "fictional-slot", "price": 2.0, "w": 300, "h": 250, + "nurl": "https://notice.example/win", "burl": "https://notice.example/bill", + "crid": "fictional-creative", "adomain": ["advertiser.example"], + "ext": {"creativeurl": "https://creative.example/render", "tagtype": "iframe"}}, + {"id": "a-high", "impid": "fictional-slot", "price": 2.0, "w": 300, "h": 250, + "ext": {"creativeurl": "https://creative.example/render", "tagtype": "iframe"}}, + {"id": "bad-script", "impid": "fictional-slot", "price": 9.0, "w": 300, "h": 250, + "ext": {"creativeurl": "https://creative.example/render", "tagtype": "script"}}, + {"id": "bad-domain", "impid": "fictional-slot", "price": 8.0, "w": 300, "h": 250, + "ext": {"creativeurl": "https://publisher.example/render", "tagtype": "iframe"}}, + {"id": "bad-credentials", "impid": "fictional-slot", "price": 8.0, "w": 300, "h": 250, + "ext": {"creativeurl": "https://user:password@creative.example/render", "tagtype": "iframe"}}, + {"id": "bad-imp", "impid": "unknown-slot", "price": 8.0, "w": 300, "h": 250, + "ext": {"creativeurl": "https://creative.example/render", "tagtype": "iframe"}}, + {"id": "bad-dimensions", "impid": "fictional-slot", "price": 8.0, "w": 320, "h": 50, + "ext": {"creativeurl": "https://creative.example/render", "tagtype": "iframe"}}, + {"id": "bad-price", "impid": "fictional-slot", "price": "high", "w": 300, "h": 250, + "ext": {"creativeurl": "https://creative.example/render", "tagtype": "iframe"}}, + {"id": "bad-mtype", "impid": "fictional-slot", "price": 8.0, "mtype": 2, "w": 300, "h": 250, + "ext": {"creativeurl": "https://creative.example/render", "tagtype": "iframe"}}, + {"id": "bad-tag", "impid": "fictional-slot", "price": 8.0, "w": 300, "h": 250, + "ext": {"creativeurl": "https://creative.example/render", "tagtype": "native"}}, + {"id": "bad-crid", "impid": "fictional-slot", "price": 8.0, "w": 300, "h": 250, + "crid": "x".repeat(1025), + "ext": {"creativeurl": "https://creative.example/render", "tagtype": "iframe"}}, + {"impid": "fictional-slot", "price": 8.0, "w": 300, "h": 250, + "ext": {"creativeurl": "https://creative.example/render", "tagtype": "iframe"}} + ]}, + {"seat": 7, "bid": "bad-shape"} + ] + })) + .expect("should serialize APS profile response"), + vec![ + ("content-type", "application/json"), + ("authorization", "fictional-secret"), + ], + ); + let backend = Arc::new(NamingBackend::new(BackendNamingPolicy::Fastly)); + let services = build_services_with_backend_and_http_client( + Arc::clone(&backend) as Arc<_>, + Arc::clone(&http) as Arc<_>, + ); + let plan = AuctionPlan::compile(planned_aps_instances_config(&[( + "aps-instance", + serde_json::json!({"account_id": "example-account", "debug": true}), + NotificationConfig { + suppress_all: false, + suppress_seats: vec!["returned-seat".to_string()], + }, + )])) + .expect("should compile planned APS profile"); + let orchestrator = AuctionOrchestratorHarness::new(plan, None); + let request = planned_request(); + let settings = create_test_settings(); + let inbound = http::Request::new(edgezero_core::body::Body::empty()); + let context = AuctionContext { + settings: &settings, + request: &inbound, + timeout_ms: 777, + transport_timeout_ms: 777, + provider_responses: None, + services: &services, + }; + + let result = orchestrator + .run_auction(&request, &context) + .await + .expect("should execute planned APS profile"); + + let response = &result.provider_responses[0]; + assert_eq!(response.provider, "aps-instance"); + assert_eq!(response.status, BidStatus::Success); + assert_eq!( + response.bids.len(), + 1, + "should retain one bid per impression" + ); + let bid = &response.bids[0]; + assert_eq!(bid.bidder, "aps"); + assert_eq!(bid.returned_seat.as_deref(), Some("returned-seat")); + assert_eq!( + bid.bid_id.as_deref(), + Some("a-high"), + "lexical ID should break equal-price tie" + ); + assert!(bid.creative.is_none()); + assert!( + bid.nurl.is_none() && bid.burl.is_none(), + "APS must discard notification URLs" + ); + let renderer = bid + .renderer + .as_ref() + .and_then(BidRenderer::as_aps) + .expect("should construct typed APS renderer"); + assert_eq!(renderer.account_id, "example-account"); + let decoded = base64::engine::general_purpose::STANDARD + .decode(&renderer.aax_response) + .expect("should decode minimized APS response"); + assert_eq!( + serde_json::from_slice::(&decoded) + .expect("should parse minimized APS response"), + serde_json::json!({"seatbid":[{"bid":[{ + "id":"a-high","price":2.0,"w":300,"h":250, + "ext":{"creativeurl":"https://creative.example/render","tagtype":"iframe"} + }]}]}) + ); + assert_eq!(response.metadata["seatbid_count"], 2); + assert_eq!(response.metadata["accepted_bid_count"], 1); + assert_eq!(response.metadata["dropped_bid_count"], 12); + for reason in [ + "lost_to_higher_bid", + "script_rendering_disabled", + "unknown_impid", + "invalid_dimensions", + "invalid_price", + "unsupported_media_type", + "unsupported_tagtype", + "creative_id_too_large", + "missing_render_source", + "empty_seatbid_bids", + ] { + assert_eq!(response.metadata["drop_reasons"][reason], 1, "{reason}"); + } + assert_eq!( + response.metadata["drop_reasons"]["invalid_creative_url"], 2, + "same-publisher and credentialed URLs should both be rejected" + ); + assert_eq!( + response.metadata["routing"]["unused_bidder_params_count"], + 0 + ); + let debug = &response.metadata["debug"]["httpcalls"]["aps"][0]; + assert_eq!(debug["uri"], "https://aps.example/e/pb/bid"); + assert_eq!( + debug["responseheaders"], + serde_json::json!({}), + "async stub does not preserve queued response headers" + ); + assert!( + debug["requestbody"] + .as_str() + .is_some_and(|body| body.contains("example-account")) + ); + assert!(debug["requestheaders"].get("authorization").is_none()); + assert!(debug["responseheaders"].get("authorization").is_none()); + } + + #[tokio::test] + async fn two_planned_aps_instances_correlate_independently() { + let http = Arc::new(StubHttpClient::new()); + for (seat, id, price) in [("seat-a", "bid-a", 1.0), ("seat-b", "bid-b", 2.0)] { + http.push_response( + 200, + serde_json::to_vec(&serde_json::json!({"seatbid":[{"seat":seat,"bid":[{ + "id":id,"impid":"fictional-slot","price":price,"w":300,"h":250, + "ext":{"creativeurl":"https://creative.example/render","tagtype":"iframe"} + }]}]})) + .expect("should serialize APS instance response"), + ); + } + let backend = Arc::new(NamingBackend::new(BackendNamingPolicy::Fastly)); + let services = build_services_with_backend_and_http_client( + Arc::clone(&backend) as Arc<_>, + Arc::clone(&http) as Arc<_>, + ); + let plan = AuctionPlan::compile(planned_aps_instances_config(&[ + ( + "aps-a", + serde_json::json!({"account_id":"account-a"}), + NotificationConfig::default(), + ), + ( + "aps-b", + serde_json::json!({"account_id":"account-b"}), + NotificationConfig::default(), + ), + ])) + .expect("should compile two APS instances"); + let orchestrator = AuctionOrchestratorHarness::new(plan, None); + let request = planned_request(); + let settings = create_test_settings(); + let inbound = http::Request::new(edgezero_core::body::Body::empty()); + let context = AuctionContext { + settings: &settings, + request: &inbound, + timeout_ms: 777, + transport_timeout_ms: 777, + provider_responses: None, + services: &services, + }; + + let result = orchestrator + .run_auction(&request, &context) + .await + .expect("should execute two APS instances"); + + assert_eq!(result.provider_responses.len(), 2); + assert_eq!(result.provider_responses[0].provider, "aps-a"); + assert_eq!( + result.provider_responses[0].bids[0].bid_id.as_deref(), + Some("bid-a") + ); + assert_eq!(result.provider_responses[1].provider, "aps-b"); + assert_eq!( + result.provider_responses[1].bids[0].bid_id.as_deref(), + Some("bid-b") + ); + assert_eq!(http.recorded_request_bodies().len(), 2); + assert_eq!( + result.winning_bids["fictional-slot"].bid_id.as_deref(), + Some("bid-b"), + "global ranking should remain orchestrator-owned" + ); + let specs = backend.specs.lock().expect("should lock specs"); + assert_eq!(specs.len(), 2); + assert_ne!(specs[0].discriminator, specs[1].discriminator); + } + + #[tokio::test] + async fn planned_aps_returned_seat_accepts_only_valid_nonempty_strings() { + let plan = AuctionPlan::compile(planned_aps_config()).expect("should compile APS plan"); + let routed = route_auction( + planned_request(), + &http::Request::new(edgezero_core::body::Body::empty()), + &plan, + None, + ); + let provider = GenericOpenRtbProvider::new(plan.providers()[0].clone()); + for (seat, expected) in [ + (serde_json::Value::Null, None), + (serde_json::json!(7), None), + (serde_json::json!(""), None), + (serde_json::json!("exact-seat"), Some("exact-seat")), + ] { + let state = provider.parse_state_for_test(routed.inputs()[0].clone()); + let response = PlatformResponse::new( + edgezero_core::http::response_builder() + .status(200) + .body(edgezero_core::body::Body::from( + serde_json::to_vec(&serde_json::json!({"seatbid":[{"seat":seat,"bid":[{ + "id":"bid","impid":"fictional-slot","price":1.0,"w":300,"h":250, + "nurl":"https://notice.example/win","burl":"https://notice.example/bill", + "ext":{"creativeurl":"https://creative.example/render","tagtype":"iframe"} + }]}]})) + .expect("should serialize seat identity response"), + )) + .expect("should build seat identity response"), + ); + let parsed = provider + .parse_response_with_state(response, 4, Some(state.as_ref())) + .await + .expect("should parse seat identity response"); + assert_eq!(parsed.bids[0].returned_seat.as_deref(), expected); + assert!(parsed.bids[0].nurl.is_none() && parsed.bids[0].burl.is_none()); + } + } + + #[tokio::test] + async fn planned_aps_response_status_shape_and_currency_matrix() { + let plan = AuctionPlan::compile(planned_aps_config()).expect("should compile APS plan"); + let routed = route_auction( + planned_request(), + &http::Request::new(edgezero_core::body::Body::empty()), + &plan, + None, + ); + let provider = GenericOpenRtbProvider::new(plan.providers()[0].clone()); + let cases = [ + (204, Vec::new(), BidStatus::NoBid, None, None), + (400, Vec::new(), BidStatus::Error, None, Some("http_status")), + ( + 200, + b"not-json".to_vec(), + BidStatus::Error, + Some("unexpected_response_shape"), + Some("parse_response"), + ), + ( + 200, + b"[]".to_vec(), + BidStatus::Error, + Some("unexpected_response_shape"), + Some("parse_response"), + ), + ( + 200, + br#"{"contextual":true}"#.to_vec(), + BidStatus::Error, + Some("unexpected_response_shape"), + Some("parse_response"), + ), + ( + 200, + br#"{"cur":"EUR","seatbid":[]}"#.to_vec(), + BidStatus::NoBid, + Some("unsupported_currency"), + None, + ), + ]; + for (status, body, expected, reason, error_type) in cases { + let state = provider.parse_state_for_test(routed.inputs()[0].clone()); + let response = PlatformResponse::new( + edgezero_core::http::response_builder() + .status(status) + .body(edgezero_core::body::Body::from(body)) + .expect("should build APS matrix response"), + ); + let parsed = provider + .parse_response_with_state(response, 4, Some(state.as_ref())) + .await + .expect("should classify APS matrix response"); + assert_eq!(parsed.status, expected, "status {status}"); + if let Some(reason) = reason { + assert_eq!( + parsed.metadata["drop_reasons"][reason], 1, + "status {status}" + ); + } + if let Some(error_type) = error_type { + assert_eq!(parsed.metadata["error_type"], error_type, "status {status}"); + } + } + } + + #[tokio::test] + async fn planned_provider_outcome_matrix_has_fixed_count_only_routing_metadata() { + let standard_plan = AuctionPlan::compile(planned_config( + &[("standard", RoutingMode::AllEligible)], + false, + )) + .expect("should compile standard plan"); + let prebid_plan = AuctionPlan::compile(planned_prebid_config(&[( + "pbs", + serde_json::json!({}), + NotificationConfig::default(), + )])) + .expect("should compile PBS plan"); + let cases = [ + (&standard_plan, 204, Vec::new(), BidStatus::NoBid), + (&standard_plan, 502, Vec::new(), BidStatus::Error), + (&standard_plan, 200, b"not-json".to_vec(), BidStatus::Error), + ( + &standard_plan, + 200, + br#"{"seatbid":[]}"#.to_vec(), + BidStatus::NoBid, + ), + (&prebid_plan, 204, b"{}".to_vec(), BidStatus::NoBid), + (&prebid_plan, 502, Vec::new(), BidStatus::Error), + (&prebid_plan, 200, b"not-json".to_vec(), BidStatus::Error), + ( + &prebid_plan, + 200, + br#"{"seatbid":[]}"#.to_vec(), + BidStatus::NoBid, + ), + ]; + + for (plan, status, body, expected) in cases { + let routed = route_auction( + planned_request(), + &http::Request::new(edgezero_core::body::Body::empty()), + plan, + None, + ); + let provider = GenericOpenRtbProvider::new(plan.providers()[0].clone()); + let state = provider.parse_state_for_test(routed.inputs()[0].clone()); + let response = PlatformResponse::new( + edgezero_core::http::response_builder() + .status(status) + .body(edgezero_core::body::Body::from(body)) + .expect("should build provider matrix response"), + ); + let parsed = provider + .parse_response_with_state(response, 4, Some(state.as_ref())) + .await + .expect("should classify provider matrix response"); + assert_eq!(parsed.status, expected, "status {status}"); + if expected == BidStatus::Error { + let expected_error_type = if (200..300).contains(&status) { + "parse_response" + } else { + "http_status" + }; + assert_eq!(parsed.metadata["error_type"], expected_error_type); + } + assert_eq!( + parsed.metadata["routing"], + serde_json::json!({"unused_bidder_params_count": 0}) + ); + let serialized = serde_json::to_string(&parsed.metadata["routing"]) + .expect("should serialize routing metadata"); + assert!(!serialized.contains("fictional-provider")); + assert!(!serialized.contains("fictional-slot")); + } + } + + #[tokio::test] + async fn planned_aps_script_opt_in_matches_shared_renderer_fixture() { + let plan = AuctionPlan::compile(planned_aps_instances_config(&[( + "aps-instance", + serde_json::json!({ + "account_id":"example-account-id", + "allow_script_creatives":true + }), + NotificationConfig::default(), + )])) + .expect("should compile script-enabled APS plan"); + let routed = route_auction( + planned_request(), + &http::Request::new(edgezero_core::body::Body::empty()), + &plan, + None, + ); + let provider = GenericOpenRtbProvider::new(plan.providers()[0].clone()); + let state = provider.parse_state_for_test(routed.inputs()[0].clone()); + let response = PlatformResponse::new( + edgezero_core::http::response_builder() + .status(200) + .body(edgezero_core::body::Body::from( + serde_json::to_vec(&serde_json::json!({"seatbid":[{"bid":[{ + "id":"fictional-selected-bid-id","impid":"fictional-slot","price":1.23, + "w":300,"h":250,"crid":"fictional-creative", + "ext":{"creativeurl":"https://creative.example/render","tagtype":"iframe"} + },{ + "id":"script-bid","impid":"fictional-slot","price":1.0, + "w":300,"h":250, + "ext":{"creativeurl":"https://creative.example/script","tagtype":"script"} + }]}]})) + .expect("should serialize APS renderer fixture response"), + )) + .expect("should build APS renderer fixture response"), + ); + + let parsed = provider + .parse_response_with_state(response, 3, Some(state.as_ref())) + .await + .expect("should parse APS renderer fixture response"); + + assert_eq!(parsed.status, BidStatus::Success); + assert_eq!( + parsed.metadata["drop_reasons"]["lost_to_higher_bid"], 1, + "enabled script creative should be eligible before reduction" + ); + let renderer = parsed.bids[0] + .renderer + .as_ref() + .and_then(BidRenderer::as_aps) + .expect("should construct APS renderer"); + let decoded = base64::engine::general_purpose::STANDARD + .decode(&renderer.aax_response) + .expect("should decode APS fixture envelope"); + let fixture: serde_json::Value = serde_json::from_str(include_str!( + "../../../trusted-server-js/lib/test/fixtures/aps-renderer-v1.json" + )) + .expect("should parse shared APS renderer fixture"); + assert_eq!( + serde_json::from_slice::(&decoded) + .expect("should parse decoded APS renderer"), + fixture + ); + } + + #[tokio::test] + async fn planned_aps_debug_response_headers_are_allowlisted() { + let plan = AuctionPlan::compile(planned_aps_instances_config(&[( + "aps-instance", + serde_json::json!({"account_id":"example-account","debug":true}), + NotificationConfig::default(), + )])) + .expect("should compile debug APS plan"); + let routed = route_auction( + planned_request(), + &http::Request::new(edgezero_core::body::Body::empty()), + &plan, + None, + ); + let provider = GenericOpenRtbProvider::new(plan.providers()[0].clone()); + let state = provider.parse_state_for_test(routed.inputs()[0].clone()); + let response = PlatformResponse::new( + edgezero_core::http::response_builder() + .status(200) + .header("content-type", "application/json") + .header("authorization", "fictional-secret") + .body(edgezero_core::body::Body::from("{}")) + .expect("should build debug APS response"), + ); + + let parsed = provider + .parse_response_with_state(response, 3, Some(state.as_ref())) + .await + .expect("should parse debug APS response"); + + let headers = &parsed.metadata["debug"]["httpcalls"]["aps"][0]["responseheaders"]; + assert_eq!( + headers, + &serde_json::json!({"content-type":["application/json"]}) + ); + assert!(headers.get("authorization").is_none()); + } + + #[tokio::test] + async fn planned_standard_instances_have_distinct_backends_and_independent_results() { + let http = Arc::new(StubHttpClient::new()); + http.push_response( + 200, + serde_json::to_vec(&serde_json::json!({ + "seatbid": [{"seat": "fictional-seat-a", "bid": [{ + "id": "bid-a", "impid": "fictional-slot", "price": 1.25, + "adm": "
a
", "w": 300, "h": 250 + }]}] + })) + .expect("should serialize response a"), + ); + http.push_response( + 200, + serde_json::to_vec(&serde_json::json!({ + "seatbid": [{"seat": "fictional-seat-b", "bid": [{ + "id": "bid-b", "impid": "fictional-slot", "price": 2.5, + "adm": "
b
", "w": 300, "h": 250 + }]}] + })) + .expect("should serialize response b"), + ); + let backend = Arc::new(NamingBackend::new(BackendNamingPolicy::Fastly)); + let services = build_services_with_backend_and_http_client( + Arc::clone(&backend) as Arc<_>, + Arc::clone(&http) as Arc<_>, + ); + let plan = AuctionPlan::compile(planned_config( + &[ + ("provider-a", RoutingMode::AllEligible), + ("provider-b", RoutingMode::AllEligible), + ], + false, + )) + .expect("should compile planned auction"); + let orchestrator = AuctionOrchestratorHarness::new(plan, None); + let request = planned_request(); + let settings = create_test_settings(); + let inbound = http::Request::builder() + .uri("https://publisher.example/auction") + .body(edgezero_core::body::Body::empty()) + .expect("should build inbound request"); + let context = AuctionContext { + settings: &settings, + request: &inbound, + timeout_ms: 777, + transport_timeout_ms: 777, + provider_responses: None, + services: &services, + }; + + let result = orchestrator + .run_auction(&request, &context) + .await + .expect("should execute planned auction"); + + assert_eq!(orchestrator.provider_count(), 2); + assert!(orchestrator.mediator().is_none()); + assert_eq!(result.provider_responses.len(), 2); + assert_eq!(result.provider_responses[0].provider, "provider-a"); + assert_eq!( + result.provider_responses[0].bids[0].bid_id.as_deref(), + Some("bid-a") + ); + assert_eq!( + result.provider_responses[0].bids[0] + .returned_seat + .as_deref(), + Some("fictional-seat-a") + ); + assert_eq!( + result.provider_responses[0].metadata["routing"]["unused_bidder_params_count"], + 0 + ); + assert_eq!(result.provider_responses[1].provider, "provider-b"); + assert_eq!( + result.provider_responses[1].bids[0].bid_id.as_deref(), + Some("bid-b") + ); + assert_eq!( + result.provider_responses[1].bids[0] + .returned_seat + .as_deref(), + Some("fictional-seat-b") + ); + assert_eq!( + result.provider_responses[1].metadata["routing"]["unused_bidder_params_count"], + 0 + ); + assert_eq!( + result.winning_bids["fictional-slot"].bid_id.as_deref(), + Some("bid-b") + ); + let request_headers = http.recorded_request_headers(); + assert_eq!(request_headers.len(), 2); + for headers in request_headers { + assert!( + headers + .iter() + .any(|(name, value)| name == "accept" && value == "application/json"), + "standard planned transport should retain its JSON Accept header" + ); + } + let backend_names = http.recorded_backend_names(); + assert_eq!(backend_names.len(), 2); + assert_ne!(backend_names[0], backend_names[1]); + let request_bodies = http.recorded_request_bodies(); + assert_eq!(request_bodies.len(), 2); + for body in request_bodies { + let value: serde_json::Value = + serde_json::from_slice(&body).expect("should parse planned request"); + let tmax = value["tmax"].as_u64().expect("should include logical tmax"); + assert!( + (750..=777).contains(&tmax), + "logical budget should remain near the auction budget, got {tmax}" + ); + assert_eq!(value["imp"].as_array().map(Vec::len), Some(1)); + } + let specs = backend.specs.lock().expect("should lock specs"); + assert_eq!(specs.len(), 2); + assert_eq!(specs[0].first_byte_timeout, Duration::from_millis(750)); + assert_eq!(specs[1].first_byte_timeout, Duration::from_millis(750)); + assert_ne!(specs[0].discriminator, specs[1].discriminator); + } + + #[tokio::test] + async fn planned_backend_collision_does_not_overwrite_first_launch_state() { + let http = Arc::new(StubHttpClient::new()); + http.push_response( + 200, + serde_json::to_vec(&serde_json::json!({ + "seatbid": [{"seat": "first", "bid": [{ + "id": "first-bid", "impid": "fictional-slot", "price": 2.0, + "adm": "
first
", "w": 300, "h": 250 + }]}] + })) + .expect("should serialize first response"), + ); + let backend = Arc::new(CollidingBackend); + let services = build_services_with_backend_and_http_client( + Arc::clone(&backend) as Arc<_>, + Arc::clone(&http) as Arc<_>, + ); + let plan = AuctionPlan::compile(planned_config( + &[ + ("provider-a", RoutingMode::AllEligible), + ("provider-b", RoutingMode::AllEligible), + ], + false, + )) + .expect("should compile planned auction"); + let orchestrator = AuctionOrchestratorHarness::new(plan, None); + let request = planned_request(); + let settings = create_test_settings(); + let inbound = http::Request::new(edgezero_core::body::Body::empty()); + let context = AuctionContext { + settings: &settings, + request: &inbound, + timeout_ms: 777, + transport_timeout_ms: 777, + provider_responses: None, + services: &services, + }; + + let result = orchestrator + .run_auction(&request, &context) + .await + .expect("should isolate backend collision"); + + assert_eq!(http.recorded_backend_names().len(), 1); + assert_eq!(result.provider_responses[0].provider, "provider-a"); + assert_eq!( + result.provider_responses[0].bids[0].bid_id.as_deref(), + Some("first-bid") + ); + assert_eq!(result.provider_responses[1].provider, "provider-b"); + assert_eq!( + result.provider_responses[1].metadata["error_type"], + "launch_failed" + ); + } + + #[tokio::test] + async fn planned_pending_backend_divergence_isolated_from_valid_provider() { + let http = Arc::new(StubHttpClient::new()); + http.push_response(204, Vec::new()); + http.push_response(204, Vec::new()); + http.push_pending_backend_name_override(Some("divergent-backend")); + let backend = Arc::new(NamingBackend::new(BackendNamingPolicy::Axum)); + let services = build_services_with_backend_and_http_client( + Arc::clone(&backend) as Arc<_>, + Arc::clone(&http) as Arc<_>, + ); + let plan = AuctionPlan::compile(planned_config( + &[ + ("provider-a", RoutingMode::AllEligible), + ("provider-b", RoutingMode::AllEligible), + ], + false, + )) + .expect("should compile planned auction"); + let orchestrator = AuctionOrchestratorHarness::new(plan, None); + let request = planned_request(); + let settings = create_test_settings(); + let inbound = http::Request::new(edgezero_core::body::Body::empty()); + let context = AuctionContext { + settings: &settings, + request: &inbound, + timeout_ms: 777, + transport_timeout_ms: 777, + provider_responses: None, + services: &services, + }; + + let result = orchestrator + .run_auction(&request, &context) + .await + .expect("should isolate divergent pending backend"); + + assert_eq!(result.provider_responses.len(), 2); + assert_eq!(result.provider_responses[0].provider, "provider-a"); + assert_eq!( + result.provider_responses[0].metadata["error_type"], + "launch_failed" + ); + assert_eq!(result.provider_responses[1].provider, "provider-b"); + assert_eq!(result.provider_responses[1].status, BidStatus::NoBid); + } + + #[tokio::test] + async fn planned_pending_backend_missing_isolated_from_valid_provider() { + let http = Arc::new(StubHttpClient::new()); + http.push_response(204, Vec::new()); + http.push_response(204, Vec::new()); + http.push_pending_backend_name_override(None); + let backend = Arc::new(NamingBackend::new(BackendNamingPolicy::Axum)); + let services = build_services_with_backend_and_http_client( + Arc::clone(&backend) as Arc<_>, + Arc::clone(&http) as Arc<_>, + ); + let plan = AuctionPlan::compile(planned_config( + &[ + ("provider-a", RoutingMode::AllEligible), + ("provider-b", RoutingMode::AllEligible), + ], + false, + )) + .expect("should compile planned auction"); + let orchestrator = AuctionOrchestratorHarness::new(plan, None); + let request = planned_request(); + let settings = create_test_settings(); + let inbound = http::Request::new(edgezero_core::body::Body::empty()); + let context = AuctionContext { + settings: &settings, + request: &inbound, + timeout_ms: 777, + transport_timeout_ms: 777, + provider_responses: None, + services: &services, + }; + + let result = orchestrator + .run_auction(&request, &context) + .await + .expect("should isolate missing pending backend"); + + assert_eq!(result.provider_responses.len(), 2); + assert_eq!(result.provider_responses[0].provider, "provider-a"); + assert_eq!( + result.provider_responses[0].metadata["error_type"], + "launch_failed" + ); + assert_eq!(result.provider_responses[1].provider, "provider-b"); + assert_eq!(result.provider_responses[1].status, BidStatus::NoBid); + } + + #[tokio::test] + async fn planned_same_profile_rejects_cross_provider_parse_state() { + let plan = AuctionPlan::compile(planned_config( + &[ + ("provider-a", RoutingMode::AllEligible), + ("provider-b", RoutingMode::AllEligible), + ], + false, + )) + .expect("should compile planned auction"); + let routed = route_auction( + planned_request(), + &http::Request::new(edgezero_core::body::Body::empty()), + &plan, + None, + ); + let provider_a = GenericOpenRtbProvider::new(plan.providers()[0].clone()); + let provider_b = GenericOpenRtbProvider::new(plan.providers()[1].clone()); + let parse_state = provider_a.parse_state_for_test(routed.inputs()[0].clone()); + let response = PlatformResponse::new( + edgezero_core::http::response_builder() + .status(204) + .body(edgezero_core::body::Body::empty()) + .expect("should build no-content response"), + ); + + let error = provider_b + .parse_response_with_state(response, 1, Some(parse_state.as_ref())) + .await + .expect_err("should reject another provider's parse state"); + + assert!( + error.to_string().contains("owned by provider provider-a"), + "should identify cross-provider state ownership" + ); + } + + #[tokio::test] + async fn planned_prebid_rejects_cross_provider_parse_state() { + let plan = AuctionPlan::compile(planned_prebid_config(&[ + ( + "pbs-a", + serde_json::json!({}), + NotificationConfig::default(), + ), + ( + "pbs-b", + serde_json::json!({}), + NotificationConfig::default(), + ), + ])) + .expect("should compile planned PBS auction"); + let routed = route_auction( + planned_request(), + &http::Request::new(edgezero_core::body::Body::empty()), + &plan, + None, + ); + let provider_a = GenericOpenRtbProvider::new(plan.providers()[0].clone()); + let provider_b = GenericOpenRtbProvider::new(plan.providers()[1].clone()); + let parse_state = provider_a.parse_state_for_test(routed.inputs()[0].clone()); + let response = PlatformResponse::new( + edgezero_core::http::response_builder() + .status(200) + .body(edgezero_core::body::Body::from_bytes(b"{}".as_slice())) + .expect("should build PBS response"), + ); + + let error = provider_b + .parse_response_with_state(response, 1, Some(parse_state.as_ref())) + .await + .expect_err("should reject another PBS provider's parse state"); + + assert!( + error.to_string().contains("owned by provider pbs-a"), + "should identify cross-provider PBS state ownership" + ); + } + + #[tokio::test] + async fn planned_skipped_provider_has_no_io_and_no_zero_impression_launch() { + let http = Arc::new(StubHttpClient::new()); + http.push_response(204, Vec::new()); + let backend = Arc::new(NamingBackend::new(BackendNamingPolicy::Axum)); + let services = build_services_with_backend_and_http_client( + Arc::clone(&backend) as Arc<_>, + Arc::clone(&http) as Arc<_>, + ); + let plan = AuctionPlan::compile(planned_config( + &[ + ("eligible", RoutingMode::AllEligible), + ("skipped", RoutingMode::Explicit), + ], + false, + )) + .expect("should compile planned auction"); + let orchestrator = AuctionOrchestratorHarness::new(plan, None); + let request = planned_request(); + let settings = create_test_settings(); + let inbound = http::Request::new(edgezero_core::body::Body::empty()); + let context = AuctionContext { + settings: &settings, + request: &inbound, + timeout_ms: 777, + transport_timeout_ms: 777, + provider_responses: None, + services: &services, + }; + + let result = orchestrator + .run_auction(&request, &context) + .await + .expect("should execute eligible provider only"); + + assert_eq!(http.recorded_backend_names().len(), 1); + assert_eq!(backend.ensured.load(Ordering::Relaxed), 1); + let skipped = result + .provider_responses + .iter() + .find(|response| response.provider == "skipped") + .expect("should materialize skipped provider"); + assert_eq!(skipped.status, BidStatus::NoBid); + assert_eq!( + skipped.metadata["routing"]["skipped_no_eligible_slots"], + true + ); + let request_body = &http.recorded_request_bodies()[0]; + let request_value: serde_json::Value = + serde_json::from_slice(request_body).expect("should parse request body"); + assert_eq!(request_value["imp"].as_array().map(Vec::len), Some(1)); + } + + #[tokio::test] + async fn zero_canonical_timeout_skips_plan_backed_direct_and_split_launches() { + for split in [false, true] { + let http = Arc::new(StubHttpClient::new()); + let backend = Arc::new(ZeroCanonicalBackend::new()); + let services = build_services_with_backend_and_http_client( + Arc::clone(&backend) as Arc<_>, + Arc::clone(&http) as Arc<_>, + ); + let mut config = planned_config(&[("provider", RoutingMode::AllEligible)], false); + config.mediator = Some("adserver_mock".to_string()); + let plan = AuctionPlan::compile(config).expect("should compile planned auction"); + let mediator_predicted = Arc::new(Mutex::new(Vec::new())); + let mediator_requested = Arc::new(Mutex::new(Vec::new())); + let mediator = Arc::new(recording_provider( + "adserver_mock", + "mediator-backend", + 777, + &mediator_predicted, + &mediator_requested, + )); + let orchestrator = AuctionOrchestrator::from_plan(Arc::new(plan), Some(mediator)); + let request = planned_request(); + let settings = create_test_settings(); + let inbound = http::Request::new(edgezero_core::body::Body::empty()); + let context = AuctionContext { + settings: &settings, + request: &inbound, + timeout_ms: 777, + transport_timeout_ms: 777, + provider_responses: None, + services: &services, + }; + + let result = if split { + let DispatchAuctionOutcome::Dispatched(dispatched) = + orchestrator.dispatch_auction(&request, &context).await + else { + panic!("zero canonical timeout should materialize a split response"); + }; + orchestrator + .collect_dispatched_auction(dispatched, &services, &context) + .await + } else { + orchestrator + .run_auction(&request, &context) + .await + .expect("should materialize direct timeout response") + }; + + assert_eq!(result.provider_responses.len(), 1); + assert_eq!( + result.provider_responses[0].metadata["error_type"], + "timeout" + ); + assert_eq!(backend.predicted.load(Ordering::Relaxed), 0); + assert_eq!(backend.ensured.load(Ordering::Relaxed), 0); + assert!(http.recorded_backend_names().is_empty()); + assert!( + mediator_predicted + .lock() + .expect("should lock mediator predictions") + .is_empty() + ); + assert!( + mediator_requested + .lock() + .expect("should lock mediator requests") + .is_empty() + ); + } + } + + #[tokio::test] + async fn planned_launch_transport_parse_failures_are_isolated_from_valid_winner_and_floor() { + let http = Arc::new(StubHttpClient::new()); + // BTreeMap plan order is alphabetical: below-floor, parse-fail, + // transport-fail, valid-winner. Queue responses in that exact order. + http.push_response( + 200, + serde_json::to_vec(&serde_json::json!({ + "seatbid": [{"seat": "below-floor", "bid": [{ + "id": "below", "impid": "fictional-slot", "price": 0.5, + "adm": "
below
", "w": 300, "h": 250 + }, { + "id": "below-only", "impid": "below-only-slot", "price": 0.5, + "adm": "
below only
", "w": 300, "h": 250 + }]}] + })) + .expect("should serialize below-floor response"), + ); + http.push_response(200, b"not-json".to_vec()); + http.push_response(200, b"{}".to_vec()); + http.push_response( + 200, + serde_json::to_vec(&serde_json::json!({ + "seatbid": [{"seat": "winner", "bid": [{ + "id": "winner", "impid": "fictional-slot", "price": 2.0, + "adm": "
winner
", "w": 300, "h": 250 + }]}] + })) + .expect("should serialize winner response"), + ); + http.push_select_success(); + http.push_select_success(); + http.push_select_error(); + let backend = Arc::new(NamingBackend::new(BackendNamingPolicy::Axum)); + backend.fail_ensure_for("launch-fail"); + let services = build_services_with_backend_and_http_client( + Arc::clone(&backend) as Arc<_>, + Arc::clone(&http) as Arc<_>, + ); + let plan = AuctionPlan::compile(planned_config( + &[ + ("launch-fail", RoutingMode::AllEligible), + ("transport-fail", RoutingMode::AllEligible), + ("parse-fail", RoutingMode::AllEligible), + ("below-floor", RoutingMode::AllEligible), + ("valid-winner", RoutingMode::AllEligible), + ], + false, + )) + .expect("should compile failure isolation plan"); + let orchestrator = AuctionOrchestratorHarness::new(plan, None); + let mut request = planned_request(); + let mut below_only_slot = request.slots[0].clone(); + below_only_slot.id = "below-only-slot".to_string(); + request.slots.push(below_only_slot); + let settings = create_test_settings(); + let inbound = http::Request::new(edgezero_core::body::Body::empty()); + let context = AuctionContext { + settings: &settings, + request: &inbound, + timeout_ms: 777, + transport_timeout_ms: 777, + provider_responses: None, + services: &services, + }; + + let result = orchestrator + .run_auction(&request, &context) + .await + .expect("should isolate planned provider failures"); + + let by_provider = result + .provider_responses + .iter() + .map(|response| (response.provider.as_str(), response)) + .collect::>(); + assert_eq!( + by_provider + .get("launch-fail") + .unwrap_or_else(|| panic!( + "should include launch-fail response; got {:?}", + by_provider.keys().collect::>() + )) + .metadata["error_type"], + "launch_failed" + ); + let below_floor = by_provider.get("below-floor").unwrap_or_else(|| { + panic!( + "should include below-floor response; got {:?}", + by_provider.keys().collect::>() + ) + }); + assert_eq!(below_floor.status, BidStatus::Success); + assert_eq!(below_floor.bids[0].bid_id.as_deref(), Some("below")); + assert_eq!(below_floor.bids[0].price, Some(0.5)); + assert_eq!(below_floor.bids[1].bid_id.as_deref(), Some("below-only")); + assert_eq!( + by_provider["transport-fail"].metadata["error_type"], + "transport" + ); + let parse_failure = by_provider.get("parse-fail").unwrap_or_else(|| { + panic!( + "should include parse-fail response; got {:?}", + by_provider.keys().collect::>() + ) + }); + assert_eq!(parse_failure.status, BidStatus::Error); + assert_eq!(parse_failure.metadata["error_type"], "parse_response"); + assert_eq!( + parse_failure.metadata["routing"]["unused_bidder_params_count"], + 0 + ); + for response in by_provider.values() { + assert_eq!( + response.metadata["routing"]["unused_bidder_params_count"], 0, + "every materialized planned provider response should have routing count" + ); + } + assert_eq!(by_provider["valid-winner"].status, BidStatus::Success); + assert_eq!( + result.winning_bids["fictional-slot"].bid_id.as_deref(), + Some("winner") + ); + assert!( + !result.winning_bids.contains_key("below-only-slot"), + "valid below-floor bid should be discarded when it is the only candidate" + ); + } + + #[tokio::test] + async fn planned_routing_count_survives_standard_and_aps_bounded_body_failures() { + for (profile, provider_id) in [("standard", "standard"), ("aps", "aps-instance")] { + let http = Arc::new(StubHttpClient::new()); + http.push_response(200, vec![b'x'; 1024 * 1024 + 1]); + let backend = Arc::new(NamingBackend::new(BackendNamingPolicy::Axum)); + let services = build_services_with_backend_and_http_client( + Arc::clone(&backend) as Arc<_>, + Arc::clone(&http) as Arc<_>, + ); + let mut config = if profile == "standard" { + planned_config(&[(provider_id, RoutingMode::AllEligible)], false) + } else { + planned_aps_config() + }; + config.bidders.insert( + "example-bidder" + .parse() + .expect("should parse fictional bidder ID"), + crate::auction::plan::BidderRouteConfig { + provider: provider_id + .parse() + .expect("should parse fictional provider ID"), + }, + ); + let plan = AuctionPlan::compile(config).expect("should compile bounded-body plan"); + let orchestrator = AuctionOrchestratorHarness::new(plan, None); + let mut request = planned_request(); + request.slots[0].bidders.insert( + "example-bidder".to_string(), + serde_json::json!({"private": "value"}), + ); + let settings = create_test_settings(); + let inbound = http::Request::new(edgezero_core::body::Body::empty()); + let context = AuctionContext { + settings: &settings, + request: &inbound, + timeout_ms: 777, + transport_timeout_ms: 777, + provider_responses: None, + services: &services, + }; + + let result = orchestrator + .run_auction(&request, &context) + .await + .expect("should materialize bounded-body failure"); + let response = &result.provider_responses[0]; + assert_eq!(response.status, BidStatus::Error, "{profile}"); + assert_eq!( + response.metadata["routing"]["unused_bidder_params_count"], 1, + "{profile} bounded-body failure should retain the input-derived count" + ); + let routing = serde_json::to_string(&response.metadata["routing"]) + .expect("should serialize routing metadata"); + assert!(!routing.contains("example-bidder") && !routing.contains("private")); + } + } + + #[tokio::test] + async fn planned_signer_admission_time_reduces_budget_and_total_time_includes_it() { + let config_store = Arc::new(CountingConfigStore { + reads: AtomicUsize::new(0), + current_kid: "test-kid".to_string(), + delay: Duration::from_millis(50), + }); + let secret_store = Arc::new(CountingSecretStore { + reads: AtomicUsize::new(0), + key: base64::Engine::encode(&base64::engine::general_purpose::STANDARD, [7_u8; 32]) + .into_bytes(), + }); + let backend = Arc::new(NamingBackend::new(BackendNamingPolicy::Axum)); + let http = Arc::new(StubHttpClient::new()); + http.push_response(204, Vec::new()); + let services = RuntimeServices::builder() + .config_store(Arc::clone(&config_store) as Arc<_>) + .secret_store(Arc::clone(&secret_store) as Arc<_>) + .kv_store(Arc::new(edgezero_core::key_value_store::NoopKvStore)) + .backend(Arc::clone(&backend) as Arc<_>) + .http_client(Arc::clone(&http) as Arc<_>) + .geo(Arc::new(crate::platform::test_support::NoopGeo)) + .auction_telemetry_sink(Arc::new( + crate::auction::telemetry::NoopAuctionTelemetrySink, + )) + .client_info(crate::platform::ClientInfo::default()) + .build(); + let plan = AuctionPlan::compile(planned_config( + &[("signed", RoutingMode::AllEligible)], + true, + )) + .expect("should compile signed plan"); + let orchestrator = AuctionOrchestratorHarness::new(plan, None); + let request = planned_request(); + let settings = create_test_settings(); + let inbound = http::Request::new(edgezero_core::body::Body::empty()); + let context = AuctionContext { + settings: &settings, + request: &inbound, + timeout_ms: 200, + transport_timeout_ms: 200, + provider_responses: None, + services: &services, + }; + + let result = orchestrator + .run_auction(&request, &context) + .await + .expect("should execute signed planned auction"); + + let body: serde_json::Value = serde_json::from_slice(&http.recorded_request_bodies()[0]) + .expect("should parse signed request"); + let tmax = body["tmax"].as_u64().expect("should include tmax"); + assert!( + (100..=175).contains(&tmax), + "signer delay should reduce logical budget, got {tmax}" + ); + assert!( + result.total_time_ms >= 50, + "total time should include signer admission" + ); + assert_eq!(config_store.reads.load(Ordering::Relaxed), 1); + assert_eq!(secret_store.reads.load(Ordering::Relaxed), 1); + } + + #[tokio::test] + async fn planned_signed_multi_provider_loads_signer_once_and_sends_twice() { + let config_store = Arc::new(CountingConfigStore { + reads: AtomicUsize::new(0), + current_kid: "test-kid".to_string(), + delay: Duration::ZERO, + }); + let secret_store = Arc::new(CountingSecretStore { + reads: AtomicUsize::new(0), + key: base64::Engine::encode(&base64::engine::general_purpose::STANDARD, [11_u8; 32]) + .into_bytes(), + }); + let backend = Arc::new(NamingBackend::new(BackendNamingPolicy::Axum)); + let http = Arc::new(StubHttpClient::new()); + http.push_response(204, Vec::new()); + http.push_response(204, Vec::new()); + let services = RuntimeServices::builder() + .config_store(Arc::clone(&config_store) as Arc<_>) + .secret_store(Arc::clone(&secret_store) as Arc<_>) + .kv_store(Arc::new(edgezero_core::key_value_store::NoopKvStore)) + .backend(Arc::clone(&backend) as Arc<_>) + .http_client(Arc::clone(&http) as Arc<_>) + .geo(Arc::new(crate::platform::test_support::NoopGeo)) + .auction_telemetry_sink(Arc::new( + crate::auction::telemetry::NoopAuctionTelemetrySink, + )) + .client_info(crate::platform::ClientInfo::default()) + .build(); + let plan = AuctionPlan::compile(planned_config( + &[ + ("provider-a", RoutingMode::AllEligible), + ("provider-b", RoutingMode::AllEligible), + ], + true, + )) + .expect("should compile signed plan"); + let orchestrator = AuctionOrchestratorHarness::new(plan, None); + let request = planned_request(); + let settings = create_test_settings(); + let inbound = http::Request::new(edgezero_core::body::Body::empty()); + let context = AuctionContext { + settings: &settings, + request: &inbound, + timeout_ms: 777, + transport_timeout_ms: 777, + provider_responses: None, + services: &services, + }; + + let result = orchestrator + .run_auction(&request, &context) + .await + .expect("should execute signed multi-provider auction"); + + assert_eq!(result.provider_responses.len(), 2); + assert_eq!(config_store.reads.load(Ordering::Relaxed), 1); + assert_eq!(secret_store.reads.load(Ordering::Relaxed), 1); + assert_eq!(http.recorded_backend_names().len(), 2); + for body in http.recorded_request_bodies() { + let value: serde_json::Value = + serde_json::from_slice(&body).expect("should parse signed provider request"); + assert!( + value["ext"]["trusted_server"]["signature"].is_string(), + "should sign every request" + ); + } + } + + #[tokio::test] + async fn from_plan_signing_failure_is_fatal_for_direct_and_safe_for_split() { + for split in [false, true] { + let config_store = Arc::new(FailingCountingConfigStore { + reads: AtomicUsize::new(0), + }); + let backend = Arc::new(NamingBackend::new(BackendNamingPolicy::Axum)); + let http = Arc::new(StubHttpClient::new()); + let services = RuntimeServices::builder() + .config_store(Arc::clone(&config_store) as Arc<_>) + .secret_store(Arc::new(UnusedSecretStore)) + .kv_store(Arc::new(edgezero_core::key_value_store::NoopKvStore)) + .backend(Arc::clone(&backend) as Arc<_>) + .http_client(Arc::clone(&http) as Arc<_>) + .geo(Arc::new(crate::platform::test_support::NoopGeo)) + .auction_telemetry_sink(Arc::new( + crate::auction::telemetry::NoopAuctionTelemetrySink, + )) + .client_info(crate::platform::ClientInfo::default()) + .build(); + let mut config = planned_config(&[("signed", RoutingMode::AllEligible)], true); + config.bidders.insert( + "unknown-private-id" + .parse() + .expect("should parse fictional bidder ID"), + crate::auction::plan::BidderRouteConfig { + provider: "signed" + .parse() + .expect("should parse fictional provider ID"), + }, + ); + let plan = Arc::new(AuctionPlan::compile(config).expect("should compile signed plan")); + let orchestrator = AuctionOrchestrator::from_plan(plan, None); + let mut request = planned_request(); + request.slots[0].bidders.insert( + "unroutable-private-id".to_string(), + serde_json::json!({"secret": 9}), + ); + let settings = create_test_settings(); + let inbound = http::Request::new(edgezero_core::body::Body::empty()); + let context = AuctionContext { + settings: &settings, + request: &inbound, + timeout_ms: 777, + transport_timeout_ms: 777, + provider_responses: None, + services: &services, + }; + + if split { + let DispatchAuctionOutcome::DispatchFailed { + provider_responses, + fatal_admission_error, + metadata, + .. + } = orchestrator.dispatch_auction(&request, &context).await + else { + panic!("split signer failure should be explicit"); + }; + let error = fatal_admission_error.expect("should carry fatal admission error"); + assert!(format!("{error:?}").contains("current-kid")); + assert_eq!(provider_responses.len(), 1); + assert_eq!( + provider_responses[0].metadata["routing"]["unused_bidder_params_count"], + 0 + ); + assert_eq!(metadata["routing"]["unroutable_bidder_count"], 1); + let serialized = + serde_json::to_string(&metadata).expect("should serialize routing metadata"); + assert!( + !serialized.contains("unroutable-private-id") && !serialized.contains("secret") + ); + } else { + let error = orchestrator + .run_auction(&request, &context) + .await + .expect_err("direct signer failure should propagate"); + let report = format!("{error:?}"); + assert!(report.contains("Planned auction admission failed")); + assert!(report.contains("current-kid")); + } + + assert_eq!(config_store.reads.load(Ordering::Relaxed), 1); + assert_eq!(backend.predicted.load(Ordering::Relaxed), 0); + assert_eq!(backend.ensured.load(Ordering::Relaxed), 0); + assert!(http.recorded_backend_names().is_empty()); + } + } + + #[tokio::test] + async fn planned_signing_failure_reads_store_once_before_backend_or_send() { + let config_store = Arc::new(FailingCountingConfigStore { + reads: AtomicUsize::new(0), + }); + let backend = Arc::new(NamingBackend::new(BackendNamingPolicy::Axum)); + let http = Arc::new(StubHttpClient::new()); + let services = RuntimeServices::builder() + .config_store(Arc::clone(&config_store) as Arc<_>) + .secret_store(Arc::new(UnusedSecretStore)) + .kv_store(Arc::new(edgezero_core::key_value_store::NoopKvStore)) + .backend(Arc::clone(&backend) as Arc<_>) + .http_client(Arc::clone(&http) as Arc<_>) + .geo(Arc::new(crate::platform::test_support::NoopGeo)) + .auction_telemetry_sink(Arc::new( + crate::auction::telemetry::NoopAuctionTelemetrySink, + )) + .client_info(crate::platform::ClientInfo::default()) + .build(); + let plan = AuctionPlan::compile(planned_config( + &[("signed", RoutingMode::AllEligible)], + true, + )) + .expect("should compile signed plan"); + let orchestrator = AuctionOrchestratorHarness::new(plan, None); + let request = planned_request(); + let settings = create_test_settings(); + let inbound = http::Request::new(edgezero_core::body::Body::empty()); + let context = AuctionContext { + settings: &settings, + request: &inbound, + timeout_ms: 777, + transport_timeout_ms: 777, + provider_responses: None, + services: &services, + }; + + let _error = orchestrator + .run_auction(&request, &context) + .await + .expect_err("should fail signer admission"); + + assert_eq!(config_store.reads.load(Ordering::Relaxed), 1); + assert_eq!(backend.predicted.load(Ordering::Relaxed), 0); + assert_eq!(backend.ensured.load(Ordering::Relaxed), 0); + assert!(http.recorded_backend_names().is_empty()); + } + + #[tokio::test] + async fn from_plan_split_zero_budget_does_no_signer_backend_or_network_work() { + let config_store = Arc::new(CountingConfigStore { + reads: AtomicUsize::new(0), + current_kid: "unused-kid".to_string(), + delay: Duration::ZERO, + }); + let secret_store = Arc::new(CountingSecretStore { + reads: AtomicUsize::new(0), + key: base64::Engine::encode(&base64::engine::general_purpose::STANDARD, [13_u8; 32]) + .into_bytes(), + }); + let backend = Arc::new(NamingBackend::new(BackendNamingPolicy::Axum)); + let http = Arc::new(StubHttpClient::new()); + let services = RuntimeServices::builder() + .config_store(Arc::clone(&config_store) as Arc<_>) + .secret_store(Arc::clone(&secret_store) as Arc<_>) + .kv_store(Arc::new(edgezero_core::key_value_store::NoopKvStore)) + .backend(Arc::clone(&backend) as Arc<_>) + .http_client(Arc::clone(&http) as Arc<_>) + .geo(Arc::new(crate::platform::test_support::NoopGeo)) + .auction_telemetry_sink(Arc::new( + crate::auction::telemetry::NoopAuctionTelemetrySink, + )) + .client_info(crate::platform::ClientInfo::default()) + .build(); + let plan = Arc::new( + AuctionPlan::compile(planned_config( + &[("signed", RoutingMode::AllEligible)], + true, + )) + .expect("should compile signed plan"), + ); + let mediator_launches = Arc::new(AtomicUsize::new(0)); + let orchestrator = AuctionOrchestrator::from_plan( + plan, + Some(Arc::new(DeadlineRecordingMediator { + launches: Arc::clone(&mediator_launches), + budgets: None, + })), + ); + let request = planned_request(); + let settings = create_test_settings(); + let inbound = http::Request::new(edgezero_core::body::Body::empty()); + let context = AuctionContext { + settings: &settings, + request: &inbound, + timeout_ms: 0, + transport_timeout_ms: 0, + provider_responses: None, + services: &services, + }; + + let DispatchAuctionOutcome::Dispatched(dispatched) = + orchestrator.dispatch_auction(&request, &context).await + else { + panic!("zero budget should materialize a completed split dispatch"); + }; + let result = orchestrator + .collect_dispatched_auction(dispatched, &services, &context) + .await; + + assert_eq!(result.provider_responses.len(), 1); + assert_eq!( + result.provider_responses[0].metadata["error_type"], + "timeout" + ); + assert_eq!(config_store.reads.load(Ordering::Relaxed), 0); + assert_eq!(secret_store.reads.load(Ordering::Relaxed), 0); + assert_eq!(backend.predicted.load(Ordering::Relaxed), 0); + assert_eq!(backend.ensured.load(Ordering::Relaxed), 0); + assert!(http.recorded_backend_names().is_empty()); + assert_eq!( + mediator_launches.load(Ordering::Relaxed), + 0, + "zero budget must not invoke even an immediate mediator" + ); + } + + #[tokio::test] + async fn planned_fanout_rejection_happens_before_backend_or_send() { + let http = Arc::new(StubHttpClient::new()); + http.set_concurrent_fanout(false); + let backend = Arc::new(NamingBackend::new(BackendNamingPolicy::Cloudflare)); + let services = build_services_with_backend_and_http_client( + Arc::clone(&backend) as Arc<_>, + Arc::clone(&http) as Arc<_>, + ); + let plan = AuctionPlan::compile(planned_config( + &[ + ("provider-a", RoutingMode::AllEligible), + ("provider-b", RoutingMode::AllEligible), + ], + false, + )) + .expect("should compile planned auction"); + let orchestrator = AuctionOrchestratorHarness::new(plan, None); + let request = planned_request(); + let settings = create_test_settings(); + let inbound = http::Request::new(edgezero_core::body::Body::empty()); + let context = AuctionContext { + settings: &settings, + request: &inbound, + timeout_ms: 777, + transport_timeout_ms: 777, + provider_responses: None, + services: &services, + }; + + let _error = orchestrator + .run_auction(&request, &context) + .await + .expect_err("should reject unsupported concurrent fanout"); + + assert_eq!(backend.predicted.load(Ordering::Relaxed), 0); + assert_eq!(backend.ensured.load(Ordering::Relaxed), 0); + assert!(http.recorded_backend_names().is_empty()); + } + + #[test] + fn routing_metadata_is_fixed_count_only_and_saturating() { + let metadata = super::routing_metadata( + RoutingDiagnostics::saturated_for_test().unroutable_bidder_count(), + ); + assert_eq!( + metadata, + HashMap::from([( + "routing".to_string(), + serde_json::json!({"unroutable_bidder_count": u32::MAX}), + )]) + ); + assert!( + !serde_json::to_string(&metadata) + .expect("should serialize routing metadata") + .contains("bidder_id"), + "routing metadata must not expose bidder identifiers" + ); + } + #[test] fn decoded_aps_bid_competes_directly_by_cpm() { let orchestrator = AuctionOrchestrator::new(AuctionConfig::default()); @@ -3308,6 +7495,7 @@ mod tests { creative: Some("
Ad
".to_string()), adomain: None, bidder: "aps".to_string(), + returned_seat: None, width: 300, height: 250, nurl: None, @@ -3352,6 +7540,7 @@ mod tests { creative: Some("
APS Ad
".to_string()), adomain: None, bidder: "aps".to_string(), + returned_seat: None, width: 300, height: 250, nurl: None, @@ -3391,6 +7580,7 @@ mod tests { creative: Some("
APS Ad
".to_string()), adomain: None, bidder: "aps".to_string(), + returned_seat: None, width: 300, height: 250, nurl: None, diff --git a/crates/trusted-server-core/src/auction/plan.rs b/crates/trusted-server-core/src/auction/plan.rs new file mode 100644 index 000000000..621fbb687 --- /dev/null +++ b/crates/trusted-server-core/src/auction/plan.rs @@ -0,0 +1,1344 @@ +//! Target-independent config-first auction plan compiler. + +use std::collections::{BTreeMap, BTreeSet}; +use std::str::FromStr; +use std::time::Duration; + +use error_stack::{Report, ResultExt as _}; +use serde::{Deserialize, Serialize}; +use serde_json::Value; +use url::Url; + +use super::profile::{CompiledOpenRtbProfile, ProfileTimeoutDefault, find_profile}; +use crate::error::TrustedServerError; +use crate::platform::{AuctionTargetId, PlatformBackendSpec}; +use crate::settings::RequestSigning; + +const MAX_ID_BYTES: usize = 128; +const MAX_SUPPRESS_SEATS: usize = 128; +const MAX_SUPPRESS_SEAT_BYTES: usize = 128; +const MOCK_MEDIATOR_ID: &str = "adserver_mock"; +const RESERVED_BROWSER_ENVELOPE_BIDDER_ID: &str = "trustedServer"; + +/// Validated operator-defined provider identifier. +#[derive(Debug, Clone, Eq, Hash, Ord, PartialEq, PartialOrd, derive_more::Display)] +pub struct ProviderId(String); + +impl ProviderId { + /// Borrow the validated identifier. + #[must_use] + pub fn as_str(&self) -> &str { + &self.0 + } + + #[cfg(test)] + pub(crate) fn unchecked_for_legacy_test(value: &str) -> Self { + Self(value.to_string()) + } +} + +impl FromStr for ProviderId { + type Err = Report; + + fn from_str(value: &str) -> Result { + let valid = !value.is_empty() + && value.len() <= 63 + && value.as_bytes().first().is_some_and(u8::is_ascii_lowercase) + && value + .as_bytes() + .iter() + .all(|byte| byte.is_ascii_lowercase() || byte.is_ascii_digit() || *byte == b'-'); + if !valid { + return Err(configuration_error(format!( + "provider ID `{value}` must match ^[a-z][a-z0-9-]{{0,62}}$" + ))); + } + Ok(Self(value.to_string())) + } +} + +impl<'de> Deserialize<'de> for ProviderId { + fn deserialize(deserializer: D) -> Result + where + D: serde::Deserializer<'de>, + { + let value = String::deserialize(deserializer)?; + Self::from_str(&value).map_err(serde::de::Error::custom) + } +} + +impl Serialize for ProviderId { + fn serialize(&self, serializer: S) -> Result + where + S: serde::Serializer, + { + serializer.serialize_str(self.as_str()) + } +} + +/// Validated client-visible bidder identifier. +#[derive(Debug, Clone, Eq, Hash, Ord, PartialEq, PartialOrd, derive_more::Display)] +pub struct BidderId(String); + +impl BidderId { + /// Borrow the validated identifier. + #[must_use] + pub fn as_str(&self) -> &str { + &self.0 + } +} + +impl FromStr for BidderId { + type Err = Report; + + fn from_str(value: &str) -> Result { + if value.is_empty() + || value.len() > MAX_ID_BYTES + || value.chars().any(char::is_control) + || value.trim() != value + { + return Err(configuration_error(format!( + "bidder ID must be nonempty, at most {MAX_ID_BYTES} UTF-8 bytes, contain no control characters, and have no surrounding whitespace" + ))); + } + Ok(Self(value.to_string())) + } +} + +impl<'de> Deserialize<'de> for BidderId { + fn deserialize(deserializer: D) -> Result + where + D: serde::Deserializer<'de>, + { + let value = String::deserialize(deserializer)?; + Self::from_str(&value).map_err(serde::de::Error::custom) + } +} + +impl Serialize for BidderId { + fn serialize(&self, serializer: S) -> Result + where + S: serde::Serializer, + { + serializer.serialize_str(self.as_str()) + } +} + +/// Raw config-first provider declaration. +#[derive(Debug, Clone, Deserialize, Serialize)] +#[serde(deny_unknown_fields)] +pub struct ProviderConfig { + /// Protocol identifier. Version one accepts only `openrtb-2.6`. + pub protocol: String, + /// Registered profile identifier. + #[serde(default = "default_profile")] + pub profile: String, + /// Fixed provider endpoint. + pub endpoint: String, + /// Optional profile-default timeout override. + #[serde(default)] + pub timeout_ms: Option, + /// Slot routing mode. + #[serde(default)] + pub routing: RoutingMode, + /// Common `OpenRTB` notification policy. + #[serde(default)] + pub notifications: NotificationConfig, + /// Selected profile's typed configuration object. + #[serde(default = "empty_object")] + pub profile_config: Value, +} + +/// Raw central bidder route. +#[derive(Debug, Clone, Deserialize, Serialize)] +#[serde(deny_unknown_fields)] +pub struct BidderRouteConfig { + /// Referenced provider identifier. + pub provider: ProviderId, +} + +/// Provider slot routing behavior. +#[derive(Debug, Clone, Copy, Default, Deserialize, Eq, PartialEq, Serialize)] +#[serde(rename_all = "snake_case")] +pub enum RoutingMode { + /// Route only centrally assigned or trusted demand. + #[default] + Explicit, + /// Route every banner-compatible slot. + AllEligible, +} + +/// Common normalized-notification suppression configuration. +#[derive(Debug, Clone, Default, Deserialize, Serialize)] +#[serde(deny_unknown_fields)] +pub struct NotificationConfig { + /// Suppress notification URLs for every normalized bid. + #[serde(default)] + pub suppress_all: bool, + /// Suppress notification URLs for exact returned-seat matches. + #[serde(default)] + pub suppress_seats: Vec, +} + +/// Raw internal input for target-independent plan compilation. +#[derive(Debug, Clone, Default, Deserialize, Serialize)] +#[serde(deny_unknown_fields)] +pub struct AuctionPlanConfig { + /// Auction-wide logical timeout. + pub timeout_ms: u32, + /// Operator-defined provider instances. + #[serde(default)] + pub providers: BTreeMap, + /// Client bidder-to-provider routes. + #[serde(default)] + pub bidders: BTreeMap, + /// Existing separately registered mock mediator. + #[serde(default)] + pub mediator: Option, + /// Existing global Trusted Server signing configuration. + #[serde(default)] + pub request_signing: Option, +} + +/// Canonical absolute provider endpoint. +#[derive(Debug, Clone, Eq, PartialEq)] +pub struct CanonicalProviderEndpoint(Url); + +impl CanonicalProviderEndpoint { + /// Borrow the canonical endpoint string. + #[must_use] + pub fn as_str(&self) -> &str { + self.0.as_str() + } + + pub(crate) fn url(&self) -> &Url { + &self.0 + } +} + +/// Closed first-version protocol plan. +#[derive(Debug, Clone, Copy, Eq, PartialEq)] +pub enum ProtocolPlan { + /// `OpenRTB` version 2.6 subset. + OpenRtb26, +} + +/// Immutable common notification policy. +#[derive(Debug, Clone, Default, Eq, PartialEq)] +pub struct NotificationPolicy { + /// Suppress notification URLs for every bid. + pub suppress_all: bool, + /// Exact returned seats whose notification URLs are suppressed. + pub suppress_seats: BTreeSet, +} + +/// Immutable compiled provider instance. +#[derive(Debug, Clone)] +pub struct ProviderPlan { + /// Provider identity. + pub id: ProviderId, + /// Canonical endpoint. + pub endpoint: CanonicalProviderEndpoint, + /// Resolved profile-default or explicit timeout. + pub timeout_ms: u32, + /// Slot routing mode. + pub routing: RoutingMode, + /// Common notification policy. + pub notifications: NotificationPolicy, + /// Compiled protocol behavior. + pub protocol: ProtocolPlan, + /// Compiled typed profile behavior. + pub profile: CompiledOpenRtbProfile, +} + +/// Immutable target-independent auction plan. +#[derive(Debug, Clone)] +pub struct AuctionPlan { + enabled: bool, + timeout_ms: u32, + providers: Vec, + bidder_routes: BTreeMap, + signing_enabled: bool, + mediator: Option, +} + +impl AuctionPlan { + /// Return whether auction execution is enabled. + #[must_use] + pub fn enabled(&self) -> bool { + self.enabled + } + + pub(crate) fn with_enabled(mut self, enabled: bool) -> Self { + self.enabled = enabled; + self + } + + /// Compile a deterministic plan without adapter-specific validation. + /// + /// # Errors + /// + /// Returns a configuration error for invalid identifiers, protocol/profile + /// declarations, endpoints, profile configuration, routes, notifications, + /// signing structure, or mediator selection. + pub fn compile(config: AuctionPlanConfig) -> Result> { + if config.timeout_ms == 0 { + return Err(configuration_error( + "auction timeout_ms must be greater than zero", + )); + } + validate_mediator(config.mediator.as_deref())?; + let signing_enabled = compile_signing_enabled(config.request_signing.as_ref())?; + let mut providers = Vec::with_capacity(config.providers.len()); + let mut provider_indices = BTreeMap::new(); + for (id, raw) in config.providers { + if raw.protocol != "openrtb-2.6" { + return Err(configuration_error(format!( + "provider `{id}` uses unsupported protocol `{}`", + raw.protocol + ))); + } + let registration = find_profile(&raw.profile).ok_or_else(|| { + configuration_error(format!( + "provider `{id}` uses unknown OpenRTB profile `{}`", + raw.profile + )) + })?; + if !raw.profile_config.is_object() { + return Err(configuration_error(format!( + "provider `{id}` profile_config must be an object" + ))); + } + let endpoint = canonicalize_endpoint(&id, registration.id, &raw.endpoint)?; + let timeout_ms = raw + .timeout_ms + .unwrap_or(match registration.default_timeout { + ProfileTimeoutDefault::Auction => config.timeout_ms, + ProfileTimeoutDefault::Fixed(value) => value, + }); + if timeout_ms == 0 { + return Err(configuration_error(format!( + "provider `{id}` timeout_ms must be greater than zero" + ))); + } + let notifications = compile_notifications(&id, raw.notifications)?; + let profile = registration.compile(&raw.profile_config)?; + let index = providers.len(); + provider_indices.insert(id.clone(), index); + providers.push(ProviderPlan { + id, + endpoint, + timeout_ms, + routing: raw.routing, + notifications, + protocol: ProtocolPlan::OpenRtb26, + profile, + }); + } + let mut bidder_routes = BTreeMap::new(); + for (bidder, route) in config.bidders { + if bidder.as_str() == RESERVED_BROWSER_ENVELOPE_BIDDER_ID { + return Err(configuration_error(format!( + "bidder ID `{RESERVED_BROWSER_ENVELOPE_BIDDER_ID}` is reserved for browser admission" + ))); + } + let provider_index = + provider_indices + .get(&route.provider) + .copied() + .ok_or_else(|| { + configuration_error(format!( + "bidder `{bidder}` references unknown provider `{}`", + route.provider + )) + })?; + bidder_routes.insert(bidder, provider_index); + } + Ok(Self { + enabled: true, + timeout_ms: config.timeout_ms, + providers, + bidder_routes, + signing_enabled, + mediator: config.mediator, + }) + } +} + +impl ProviderPlan { + /// Build the canonical backend specification with the configured timeout. + #[must_use] + pub(crate) fn backend_spec(&self) -> PlatformBackendSpec { + self.backend_spec_with_transport_timeout(self.timeout_ms) + } + + /// Build the canonical backend specification with request-local transport timers. + #[must_use] + pub(crate) fn backend_spec_with_transport_timeout( + &self, + transport_timeout_ms: u32, + ) -> PlatformBackendSpec { + let endpoint = self.endpoint.url(); + let timeout = Duration::from_millis(u64::from(transport_timeout_ms)); + PlatformBackendSpec { + scheme: endpoint.scheme().to_owned(), + host: endpoint + .host_str() + .expect("should retain validated provider endpoint host") + .to_owned(), + port: endpoint.port(), + host_header_override: None, + certificate_check: true, + first_byte_timeout: timeout, + between_bytes_timeout: timeout, + discriminator: Some(self.id.as_str().to_owned()), + } + } +} + +impl AuctionPlan { + /// Validate adapter capabilities and backend-name correlation before I/O. + /// + /// Each provider is predicted from a canonical backend specification using + /// its exact configured provider timeout as both transport timers and its + /// provider ID as the stable discriminator. These transport timers do not + /// replace the auction-wide logical budget. + /// + /// # Errors + /// + /// Returns a configuration error when the target cannot fan out to every + /// configured provider, backend prediction fails, or two predicted names + /// collide. + pub fn validate_for_target( + &self, + target_id: AuctionTargetId, + ) -> Result<(), Report> { + if !self.enabled { + return Ok(()); + } + let target = target_id.descriptor(); + if self.providers.len() > 1 && !target.capabilities().supports_concurrent_provider_fanout() + { + return Err(configuration_error(format!( + "auction target `{}` does not support concurrent provider fanout; configured {} providers", + target_id.adapter_id(), + self.providers.len() + ))); + } + + let naming_policy = target.naming_policy(); + let backend_budget = naming_policy.auction_dynamic_backend_budget(); + let mut required_backend_names = 0_usize; + let mut predicted_names = BTreeMap::::new(); + for provider in &self.providers { + let reachable_timeout_ms = provider.timeout_ms.min(self.timeout_ms); + required_backend_names = required_backend_names + .saturating_add(naming_policy.transport_timeout_bucket_count(reachable_timeout_ms)); + if let Some(budget) = backend_budget + && required_backend_names > budget + { + return Err(configuration_error(format!( + "auction target `{}` requires up to {required_backend_names} dynamic provider backends, exceeding its auction budget of {budget}", + target_id.adapter_id(), + ))); + } + let spec = provider.backend_spec(); + let prediction = + naming_policy + .predict(&spec) + .change_context(TrustedServerError::Configuration { + message: format!( + "provider `{}` backend prediction failed for target `{}`", + provider.id, + target_id.adapter_id() + ), + })?; + if let Some(existing) = predicted_names.insert(prediction.name.clone(), &provider.id) { + return Err(configuration_error(format!( + "providers `{existing}` and `{}` predict the same backend name `{}` for target `{}`", + provider.id, + prediction.name, + target_id.adapter_id() + ))); + } + } + Ok(()) + } + + /// Borrow compiled providers in deterministic provider-ID order. + #[must_use] + pub fn providers(&self) -> &[ProviderPlan] { + &self.providers + } + + /// Borrow a compiled provider by its validated identity. + #[must_use] + pub(crate) fn provider(&self, id: &ProviderId) -> Option<&ProviderPlan> { + self.providers.iter().find(|provider| provider.id == *id) + } + + /// Return whether any compiled provider uses the named profile. + /// + /// This narrow query allows capability activation to follow the validated + /// plan without exposing profile configuration. + #[must_use] + pub fn has_profile(&self, profile_id: &str) -> bool { + self.providers + .iter() + .any(|provider| provider.profile.id() == profile_id) + } + + /// Borrow validated client-visible bidder route codes in deterministic order. + /// + /// This intentionally exposes route keys rather than provider identities or + /// profile configuration for the browser Prebid injection boundary. + pub(crate) fn browser_bidder_codes(&self) -> impl Iterator { + self.bidder_routes.keys().map(BidderId::as_str) + } + + /// Resolve a bidder route to a compiled provider. + #[must_use] + pub fn provider_for_bidder(&self, bidder: &BidderId) -> Option<&ProviderPlan> { + self.bidder_routes + .get(bidder) + .and_then(|index| self.providers.get(*index)) + } + + /// Return whether auction-wide signing is enabled. + #[must_use] + pub fn signing_enabled(&self) -> bool { + self.signing_enabled + } + + /// Borrow the separately validated static mediator identifier. + #[must_use] + pub fn mediator(&self) -> Option<&str> { + self.mediator.as_deref() + } +} + +fn default_profile() -> String { + "standard".to_string() +} + +fn empty_object() -> Value { + Value::Object(serde_json::Map::new()) +} + +fn configuration_error(message: impl Into) -> Report { + Report::new(TrustedServerError::Configuration { + message: message.into(), + }) +} + +fn compile_signing_enabled( + request_signing: Option<&RequestSigning>, +) -> Result> { + let Some(request_signing) = request_signing else { + return Ok(false); + }; + if request_signing.enabled + && (request_signing.config_store_id.trim().is_empty() + || request_signing.secret_store_id.trim().is_empty()) + { + return Err(configuration_error( + "enabled request_signing requires nonblank config_store_id and secret_store_id", + )); + } + Ok(request_signing.enabled) +} + +fn validate_mediator(mediator: Option<&str>) -> Result<(), Report> { + if mediator.is_some_and(|value| value != MOCK_MEDIATOR_ID) { + return Err(configuration_error(format!( + "auction mediator must be `{MOCK_MEDIATOR_ID}` when configured" + ))); + } + Ok(()) +} + +fn canonicalize_endpoint( + provider_id: &ProviderId, + profile_id: &str, + value: &str, +) -> Result> { + let mut endpoint = Url::parse(value).map_err(|error| { + configuration_error(format!( + "provider `{provider_id}` endpoint must be an absolute HTTPS URL: {error}" + )) + })?; + if endpoint.scheme() != "https" + || endpoint.host_str().is_none() + || !endpoint.username().is_empty() + || endpoint.password().is_some() + || endpoint.fragment().is_some() + { + return Err(configuration_error(format!( + "provider `{provider_id}` endpoint must be absolute HTTPS with a host and no credentials or fragment" + ))); + } + if profile_id == "aps" + && endpoint + .path() + .trim_end_matches('/') + .ends_with("/e/dtb/bid") + { + return Err(configuration_error(format!( + "provider `{provider_id}` uses unsupported legacy APS endpoint `/e/dtb/bid`" + ))); + } + if profile_id == "prebid-server" { + normalize_prebid_server_endpoint(&mut endpoint); + } + endpoint.set_fragment(None); + Ok(CanonicalProviderEndpoint(endpoint)) +} + +fn normalize_prebid_server_endpoint(endpoint: &mut Url) { + match endpoint.path() { + "" | "/" => endpoint.set_path("/openrtb2/auction"), + "/openrtb2/auction/" => endpoint.set_path("/openrtb2/auction"), + _ => {} + } +} + +fn compile_notifications( + provider_id: &ProviderId, + config: NotificationConfig, +) -> Result> { + if config.suppress_seats.len() > MAX_SUPPRESS_SEATS { + return Err(configuration_error(format!( + "provider `{provider_id}` notifications.suppress_seats exceeds {MAX_SUPPRESS_SEATS} entries" + ))); + } + let mut seats = BTreeSet::new(); + for seat in config.suppress_seats { + if seat.is_empty() + || seat.len() > MAX_SUPPRESS_SEAT_BYTES + || seat.chars().any(|character| character.is_ascii_control()) + { + return Err(configuration_error(format!( + "provider `{provider_id}` notification seat must be nonempty, at most {MAX_SUPPRESS_SEAT_BYTES} UTF-8 bytes, and contain no ASCII control characters" + ))); + } + if !seats.insert(seat.clone()) { + return Err(configuration_error(format!( + "provider `{provider_id}` notification seat `{seat}` is duplicated" + ))); + } + } + Ok(NotificationPolicy { + suppress_all: config.suppress_all, + suppress_seats: seats, + }) +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::auction::profile::CompiledOpenRtbProfile; + + fn provider(profile: &str) -> ProviderConfig { + ProviderConfig { + protocol: "openrtb-2.6".to_string(), + profile: profile.to_string(), + endpoint: "https://bid.example/openrtb2/auction".to_string(), + timeout_ms: None, + routing: RoutingMode::Explicit, + notifications: NotificationConfig::default(), + profile_config: empty_object(), + } + } + + fn config(providers: BTreeMap) -> AuctionPlanConfig { + AuctionPlanConfig { + timeout_ms: 1500, + providers, + ..AuctionPlanConfig::default() + } + } + + fn id(value: &str) -> ProviderId { + ProviderId::from_str(value).expect("should parse provider ID") + } + + fn bidder(value: &str) -> BidderId { + BidderId::from_str(value).expect("should parse bidder ID") + } + + fn nested_object(levels: usize) -> Value { + let mut value = Value::String("leaf".to_string()); + for level in 0..levels { + value = Value::Object(serde_json::Map::from_iter([( + format!("level-{level}"), + value, + )])); + } + value + } + + fn nested_array(levels: usize) -> Value { + let mut value = Value::String("leaf".to_string()); + for _ in 0..levels { + value = Value::Array(vec![value]); + } + value + } + + #[test] + fn target_validation_accepts_fanout_and_rejects_unsupported_targets() { + let providers = BTreeMap::from([ + (id("provider-one"), provider("standard")), + (id("provider-two"), provider("standard")), + ]); + let plan = AuctionPlan::compile(config(providers)).expect("should compile plan"); + + assert!( + plan.validate_for_target(crate::platform::AuctionTargetId::Fastly) + .is_ok(), + "Fastly should accept provider fanout" + ); + assert!( + plan.validate_for_target(crate::platform::AuctionTargetId::Axum) + .is_ok(), + "Axum should accept provider fanout" + ); + for target in [ + crate::platform::AuctionTargetId::Cloudflare, + crate::platform::AuctionTargetId::Spin, + ] { + let error = plan + .validate_for_target(target) + .expect_err("should reject unsupported provider fanout"); + assert!( + error.to_string().contains("fanout"), + "should explain fanout rejection: {error:?}" + ); + } + } + + #[test] + fn fastly_target_validation_canonicalizes_url_derived_ipv6_prediction() { + let mut ipv6_provider = provider("standard"); + ipv6_provider.endpoint = "https://[2001:db8::5]:8443/openrtb".to_string(); + ipv6_provider.timeout_ms = Some(750); + let plan = AuctionPlan::compile(config(BTreeMap::from([( + id("ipv6-provider"), + ipv6_provider, + )]))) + .expect("should compile IPv6 provider plan"); + + plan.validate_for_target(crate::platform::AuctionTargetId::Fastly) + .expect("Fastly target validation should accept canonical IPv6 prediction"); + let bracketed_spec = plan.providers()[0].backend_spec(); + assert_eq!(bracketed_spec.host, "[2001:db8::5]"); + let mut bare_spec = bracketed_spec.clone(); + bare_spec.host = "2001:db8::5".to_string(); + let policy = crate::platform::BackendNamingPolicy::Fastly; + assert_eq!( + policy + .predict(&bracketed_spec) + .expect("should predict URL-derived bracketed IPv6 backend"), + policy + .predict(&bare_spec) + .expect("should predict runtime-normalized bare IPv6 backend"), + "startup target validation and Fastly runtime must hash the same backend spec" + ); + } + + #[test] + fn fastly_target_validation_reserves_dynamic_backends_outside_auction() { + let plan_with = |count: usize| { + let providers = (0..count) + .map(|index| { + let mut provider = provider("standard"); + provider.timeout_ms = Some(1000); + (id(&format!("provider-{index}")), provider) + }) + .collect(); + AuctionPlan::compile(config(providers)).expect("should compile provider plan") + }; + + plan_with(19) + .validate_for_target(crate::platform::AuctionTargetId::Fastly) + .expect("below-budget provider plan should validate"); + plan_with(20) + .validate_for_target(crate::platform::AuctionTargetId::Fastly) + .expect("at-budget provider plan should validate"); + let error = plan_with(21) + .validate_for_target(crate::platform::AuctionTargetId::Fastly) + .expect_err("over-budget provider plan should fail"); + assert!( + error + .to_string() + .contains("exceeding its auction budget of 160") + ); + } + + #[test] + fn fastly_backend_quota_uses_auction_timeout_as_reachable_bucket_ceiling() { + let providers = (0..21) + .map(|index| { + let mut provider = provider("standard"); + provider.timeout_ms = Some(1000); + (id(&format!("provider-{index}")), provider) + }) + .collect(); + let mut bounded = config(providers); + bounded.timeout_ms = 100; + let plan = AuctionPlan::compile(bounded).expect("should compile bounded provider plan"); + + assert!( + plan.providers() + .iter() + .all(|provider| provider.timeout_ms == 1000), + "quota validation must not rewrite configured provider timeouts" + ); + plan.validate_for_target(crate::platform::AuctionTargetId::Fastly) + .expect("21 providers reach only the 50ms and 100ms Fastly buckets"); + } + + #[test] + fn disabled_target_validation_skips_fanout_and_collision_checks() { + let providers = BTreeMap::from([ + (id("provider-one"), provider("standard")), + (id("provider-two"), provider("standard")), + ]); + let disabled = AuctionPlan::compile(config(providers)) + .expect("should compile plan") + .with_enabled(false); + + for target in [ + crate::platform::AuctionTargetId::Cloudflare, + crate::platform::AuctionTargetId::Spin, + ] { + disabled + .validate_for_target(target) + .expect("disabled dormant providers should skip target validation"); + } + + let provider = disabled.providers[0].clone(); + let disabled_collision = AuctionPlan { + enabled: false, + timeout_ms: disabled.timeout_ms, + providers: vec![provider.clone(), provider], + bidder_routes: BTreeMap::new(), + signing_enabled: false, + mediator: None, + }; + disabled_collision + .validate_for_target(crate::platform::AuctionTargetId::Axum) + .expect("disabled dormant providers should skip collision validation"); + } + + #[test] + fn target_validation_keeps_same_origin_timeout_profile_instances_distinct() { + let shared = ProviderConfig { + timeout_ms: Some(777), + ..provider("standard") + }; + let plan = AuctionPlan::compile(config(BTreeMap::from([ + (id("provider-one"), shared.clone()), + (id("provider-two"), shared), + ]))) + .expect("should compile same-origin provider instances"); + + for target in [ + crate::platform::AuctionTargetId::Fastly, + crate::platform::AuctionTargetId::Axum, + ] { + plan.validate_for_target(target) + .expect("provider ID discriminators should prevent predicted collisions"); + } + } + + #[test] + fn target_validation_rejects_predicted_name_collisions() { + let compiled = AuctionPlan::compile(config(BTreeMap::from([( + id("provider-a"), + provider("standard"), + )]))) + .expect("should compile plan"); + let provider = compiled.providers[0].clone(); + // The compiler prevents duplicate provider IDs. Construct the otherwise + // impossible duplicate internally to pin validation's defense-in-depth + // collision rejection independently of compiler invariants. + let collision_plan = AuctionPlan { + enabled: true, + timeout_ms: compiled.timeout_ms, + providers: vec![provider.clone(), provider], + bidder_routes: BTreeMap::new(), + signing_enabled: false, + mediator: None, + }; + + let error = collision_plan + .validate_for_target(crate::platform::AuctionTargetId::Axum) + .expect_err("should reject predicted backend collision"); + assert!(error.to_string().contains("same backend name")); + } + + #[test] + fn provider_id_enforces_exact_grammar_and_bounds() { + for valid in ["a", "pbs-primary", &format!("a{}", "0".repeat(62))] { + assert!(ProviderId::from_str(valid).is_ok(), "should accept {valid}"); + } + for invalid in [ + "", + "A", + "1provider", + "provider_name", + "provider.name", + "provider/one", + &format!("a{}", "0".repeat(63)), + ] { + assert!( + ProviderId::from_str(invalid).is_err(), + "should reject {invalid}" + ); + } + } + + #[test] + fn bidder_id_enforces_admission_bounds() { + assert!(BidderId::from_str("exampleBidder").is_ok()); + for invalid in ["", " bidder", "bidder\n", &"a".repeat(129)] { + assert!(BidderId::from_str(invalid).is_err()); + } + } + + #[test] + fn compiler_rejects_exact_reserved_browser_envelope_bidder_id() { + let mut raw = config(BTreeMap::from([(id("one"), provider("standard"))])); + raw.bidders.insert( + bidder("trustedServer"), + BidderRouteConfig { + provider: id("one"), + }, + ); + assert!( + AuctionPlan::compile(raw).is_err(), + "exact reserved bidder ID should be rejected" + ); + + let mut case_distinct = config(BTreeMap::from([(id("one"), provider("standard"))])); + case_distinct.bidders.insert( + bidder("TrustedServer"), + BidderRouteConfig { + provider: id("one"), + }, + ); + assert!( + AuctionPlan::compile(case_distinct).is_ok(), + "reserved bidder comparison should remain case-sensitive" + ); + } + + #[test] + fn compiler_orders_providers_and_routes_deterministically() { + let mut providers = BTreeMap::new(); + providers.insert(id("z-provider"), provider("standard")); + providers.insert(id("a-provider"), provider("standard")); + let mut raw = config(providers); + raw.bidders.insert( + bidder("z-bidder"), + BidderRouteConfig { + provider: id("z-provider"), + }, + ); + raw.bidders.insert( + bidder("a-bidder"), + BidderRouteConfig { + provider: id("a-provider"), + }, + ); + let plan = AuctionPlan::compile(raw).expect("should compile deterministic plan"); + assert_eq!(plan.providers()[0].id.as_str(), "a-provider"); + assert_eq!(plan.providers()[1].id.as_str(), "z-provider"); + assert_eq!( + plan.browser_bidder_codes().collect::>(), + vec!["a-bidder", "z-bidder"], + "browser query should return deduplicated route codes in deterministic order" + ); + assert_eq!( + plan.provider_for_bidder(&bidder("a-bidder")) + .map(|provider| provider.id.as_str()), + Some("a-provider") + ); + } + + #[test] + fn compiler_supports_two_instances_of_the_same_profile() { + let mut providers = BTreeMap::new(); + providers.insert(id("pbs-a"), provider("prebid-server")); + providers.insert(id("pbs-b"), provider("prebid-server")); + let plan = AuctionPlan::compile(config(providers)).expect("should compile two PBS plans"); + assert_eq!(plan.providers().len(), 2); + assert!( + plan.providers().iter().all(|provider| matches!( + provider.profile, + CompiledOpenRtbProfile::PrebidServer(_) + )) + ); + } + + #[test] + fn profile_defaults_and_explicit_timeout_override_are_resolved() { + let mut providers = BTreeMap::new(); + providers.insert(id("standard-one"), provider("standard")); + providers.insert(id("pbs-one"), provider("prebid-server")); + providers.insert( + id("aps-one"), + ProviderConfig { + endpoint: "https://aps.example/e/pb/bid".to_string(), + profile_config: serde_json::json!({"account_id": "example-account"}), + ..provider("aps") + }, + ); + providers.insert( + id("pbs-override"), + ProviderConfig { + timeout_ms: Some(321), + ..provider("prebid-server") + }, + ); + let plan = AuctionPlan::compile(config(providers)).expect("should resolve timeouts"); + let timeouts = plan + .providers() + .iter() + .map(|provider| (provider.id.as_str(), provider.timeout_ms)) + .collect::>(); + assert_eq!(timeouts["standard-one"], 1500); + assert_eq!(timeouts["pbs-one"], 1000); + assert_eq!(timeouts["aps-one"], 800); + assert_eq!(timeouts["pbs-override"], 321); + } + + #[test] + fn profile_registry_is_independent_of_browser_configuration() { + let ids = crate::auction::profile::profile_registrations() + .iter() + .map(|registration| registration.id) + .collect::>(); + assert_eq!(ids, vec!["standard", "prebid-server", "aps"]); + let mut providers = BTreeMap::new(); + providers.insert(id("pbs"), provider("prebid-server")); + let plan = AuctionPlan::compile(config(providers)) + .expect("should compile without Settings or browser integration state"); + assert!(!plan.has_profile("aps")); + + let mut providers = BTreeMap::new(); + providers.insert( + id("aps-instance"), + ProviderConfig { + endpoint: "https://aps.example/e/pb/bid".to_string(), + profile_config: serde_json::json!({"account_id": "example-account"}), + ..provider("aps") + }, + ); + let plan = AuctionPlan::compile(config(providers)).expect("should compile APS plan"); + assert!( + plan.has_profile("aps"), + "validated plan should expose APS renderer capability" + ); + assert!(!plan.has_profile("prebid-server")); + } + + #[test] + fn compiler_rejects_unknown_protocol_profile_and_route() { + let mut unknown_protocol = provider("standard"); + unknown_protocol.protocol = "openrtb-2.5".to_string(); + assert!( + AuctionPlan::compile(config(BTreeMap::from([(id("one"), unknown_protocol)]))).is_err() + ); + assert!( + AuctionPlan::compile(config(BTreeMap::from([(id("one"), provider("unknown"))]))) + .is_err() + ); + let mut raw = config(BTreeMap::from([(id("one"), provider("standard"))])); + raw.bidders.insert( + bidder("example"), + BidderRouteConfig { + provider: id("missing"), + }, + ); + assert!(AuctionPlan::compile(raw).is_err()); + } + + #[test] + fn compiler_canonicalizes_https_endpoints_and_rejects_unsafe_forms() { + let mut canonical = provider("standard"); + canonical.endpoint = "https://BID.EXAMPLE:443/path".to_string(); + let plan = AuctionPlan::compile(config(BTreeMap::from([(id("one"), canonical)]))) + .expect("should canonicalize endpoint"); + assert_eq!( + plan.providers()[0].endpoint.as_str(), + "https://bid.example/path" + ); + for endpoint in [ + "http://bid.example/path", + "https://", + "https://user@bid.example/path", + "https://bid.example/path#fragment", + "/relative", + ] { + let mut raw_provider = provider("standard"); + raw_provider.endpoint = endpoint.to_string(); + assert!( + AuctionPlan::compile(config(BTreeMap::from([(id("one"), raw_provider)]))).is_err(), + "should reject {endpoint}" + ); + } + let mut aps = provider("aps"); + aps.endpoint = "https://aps.example/e/dtb/bid".to_string(); + aps.profile_config = serde_json::json!({"account_id": "example-account"}); + assert!(AuctionPlan::compile(config(BTreeMap::from([(id("aps"), aps)]))).is_err()); + } + + #[test] + fn compiler_normalizes_only_prebid_server_origin_and_canonical_paths() { + for (configured, expected) in [ + ( + "https://pbs.example", + "https://pbs.example/openrtb2/auction", + ), + ( + "https://pbs.example/", + "https://pbs.example/openrtb2/auction", + ), + ( + "https://pbs.example/openrtb2/auction", + "https://pbs.example/openrtb2/auction", + ), + ( + "https://pbs.example/openrtb2/auction/", + "https://pbs.example/openrtb2/auction", + ), + ( + "https://pbs.example?region=example", + "https://pbs.example/openrtb2/auction?region=example", + ), + ("https://pbs.example/bid", "https://pbs.example/bid"), + ( + "https://pbs.example/custom/pbs", + "https://pbs.example/custom/pbs", + ), + ] { + let mut pbs = provider("prebid-server"); + pbs.endpoint = configured.to_string(); + let plan = AuctionPlan::compile(config(BTreeMap::from([(id("pbs"), pbs)]))) + .expect("should compile Prebid Server endpoint"); + assert_eq!( + plan.providers()[0].endpoint.as_str(), + expected, + "{configured}" + ); + } + + let mut standard = provider("standard"); + standard.endpoint = "https://bid.example/".to_string(); + let plan = AuctionPlan::compile(config(BTreeMap::from([(id("standard"), standard)]))) + .expect("should compile standard root endpoint"); + assert_eq!( + plan.providers()[0].endpoint.as_str(), + "https://bid.example/" + ); + + let mut aps = provider("aps"); + aps.endpoint = "https://aps.example/e/pb/bid".to_string(); + aps.profile_config = serde_json::json!({"account_id": "example-account"}); + let plan = AuctionPlan::compile(config(BTreeMap::from([(id("aps"), aps)]))) + .expect("should compile APS endpoint"); + assert_eq!( + plan.providers()[0].endpoint.as_str(), + "https://aps.example/e/pb/bid" + ); + } + + #[test] + fn standard_extensions_are_typed_bounded_and_cannot_claim_reserved_fields() { + let mut valid = provider("standard"); + valid.profile_config = serde_json::json!({ + "request_ext": {"fictional_account": "example"}, + "imp_ext": {"placement_group": "display"} + }); + let plan = AuctionPlan::compile(config(BTreeMap::from([(id("one"), valid)]))) + .expect("should compile static extensions"); + let CompiledOpenRtbProfile::Standard(standard) = &plan.providers()[0].profile else { + panic!("should compile standard profile") + }; + assert_eq!( + standard.request_ext.as_object()["fictional_account"], + "example" + ); + + let mut standard_owned_fields = provider("standard"); + standard_owned_fields.profile_config = serde_json::json!({ + "request_ext": { + "account": "example-account", + "sdk": {"source": "example"}, + "prebid": {"example": true} + }, + "imp_ext": {"prebid": {"example": true}} + }); + AuctionPlan::compile(config(BTreeMap::from([( + id("standard-owned-fields"), + standard_owned_fields, + )]))) + .expect("should allow standard static extensions outside common-owned fields"); + + for profile_config in [ + serde_json::json!({"request_ext": "bad"}), + serde_json::json!({"request_ext": {"trusted_server": {}}}), + ] { + let mut invalid = provider("standard"); + invalid.profile_config = profile_config; + assert!(AuctionPlan::compile(config(BTreeMap::from([(id("one"), invalid)]))).is_err()); + } + let oversized = "x".repeat(16 * 1024); + let mut invalid = provider("standard"); + invalid.profile_config = serde_json::json!({"request_ext": {"value": oversized}}); + assert!(AuctionPlan::compile(config(BTreeMap::from([(id("one"), invalid)]))).is_err()); + + let too_many_keys = (0..257) + .map(|index| (format!("key-{index}"), Value::Bool(true))) + .collect::>(); + let mut invalid = provider("standard"); + invalid.profile_config = serde_json::json!({"imp_ext": too_many_keys}); + assert!(AuctionPlan::compile(config(BTreeMap::from([(id("one"), invalid)]))).is_err()); + } + + #[test] + fn standard_extension_depth_counts_container_levels() { + let mut object_valid = provider("standard"); + object_valid.profile_config = serde_json::json!({"request_ext": nested_object(8)}); + AuctionPlan::compile(config(BTreeMap::from([(id("object-valid"), object_valid)]))) + .expect("should accept eight nested object levels"); + + let mut object_invalid = provider("standard"); + object_invalid.profile_config = serde_json::json!({"request_ext": nested_object(9)}); + assert!( + AuctionPlan::compile(config(BTreeMap::from([( + id("object-invalid"), + object_invalid, + )]))) + .is_err(), + "should reject nine nested object levels" + ); + + let mut array_valid = provider("standard"); + array_valid.profile_config = serde_json::json!({"request_ext": {"value": nested_array(7)}}); + AuctionPlan::compile(config(BTreeMap::from([(id("array-valid"), array_valid)]))) + .expect("should accept one object plus seven nested array levels"); + + let mut array_invalid = provider("standard"); + array_invalid.profile_config = + serde_json::json!({"request_ext": {"value": nested_array(8)}}); + assert!( + AuctionPlan::compile(config(BTreeMap::from([( + id("array-invalid"), + array_invalid, + )]))) + .is_err(), + "should reject one object plus eight nested array levels" + ); + } + + #[test] + fn notification_policy_rejects_duplicates_and_bounds() { + let mut valid = provider("standard"); + valid.notifications = NotificationConfig { + suppress_all: true, + suppress_seats: vec!["seat-b".to_string(), "seat-a".to_string()], + }; + let plan = AuctionPlan::compile(config(BTreeMap::from([(id("one"), valid)]))) + .expect("should compile notifications"); + assert_eq!( + plan.providers()[0] + .notifications + .suppress_seats + .iter() + .map(String::as_str) + .collect::>(), + vec!["seat-a", "seat-b"] + ); + for seats in [ + vec!["same".to_string(), "same".to_string()], + vec![String::new()], + vec!["bad\nseat".to_string()], + vec!["x".repeat(129)], + (0..129).map(|index| format!("seat-{index}")).collect(), + ] { + let mut invalid = provider("standard"); + invalid.notifications.suppress_seats = seats; + assert!(AuctionPlan::compile(config(BTreeMap::from([(id("one"), invalid)]))).is_err()); + } + } + + #[test] + fn typed_profile_config_rejects_unknown_fields_and_validates_aps_pairing() { + let mut pbs = provider("prebid-server"); + pbs.profile_config = serde_json::json!({"browser_only": true}); + assert!(AuctionPlan::compile(config(BTreeMap::from([(id("pbs"), pbs)]))).is_err()); + + let mut non_object = provider("standard"); + non_object.profile_config = Value::Null; + assert!( + AuctionPlan::compile(config(BTreeMap::from([(id("standard"), non_object,)]))).is_err() + ); + + let mut aps = provider("aps"); + aps.endpoint = "https://aps.example/e/pb/bid".to_string(); + aps.profile_config = serde_json::json!({ + "account_id": "example-account", + "inventory_domain": "publisher.example" + }); + assert!(AuctionPlan::compile(config(BTreeMap::from([(id("aps"), aps)]))).is_err()); + } + + #[test] + fn enabled_signing_requires_nonblank_existing_global_store_ids() { + let mut raw = config(BTreeMap::from([(id("one"), provider("standard"))])); + raw.request_signing = Some(RequestSigning { + enabled: true, + config_store_id: " ".to_string(), + secret_store_id: "example-secret-store".to_string(), + }); + assert!( + AuctionPlan::compile(raw).is_err(), + "should reject enabled signing without a config store ID" + ); + + let mut raw = config(BTreeMap::from([(id("one"), provider("standard"))])); + raw.request_signing = Some(RequestSigning { + enabled: true, + config_store_id: "example-config-store".to_string(), + secret_store_id: "\t".to_string(), + }); + assert!( + AuctionPlan::compile(raw).is_err(), + "should reject enabled signing without a secret store ID" + ); + } + + #[test] + fn routing_signing_and_static_mediator_are_preserved_in_plan() { + let mut provider = provider("standard"); + provider.routing = RoutingMode::AllEligible; + let mut raw = config(BTreeMap::from([(id("one"), provider)])); + raw.request_signing = Some(RequestSigning { + enabled: true, + config_store_id: "example-config-store".to_string(), + secret_store_id: "example-secret-store".to_string(), + }); + raw.mediator = Some(MOCK_MEDIATOR_ID.to_string()); + let plan = AuctionPlan::compile(raw).expect("should compile common policies"); + assert_eq!(plan.providers()[0].routing, RoutingMode::AllEligible); + assert!(plan.signing_enabled()); + assert_eq!(plan.mediator(), Some(MOCK_MEDIATOR_ID)); + + let mut invalid = config(BTreeMap::new()); + invalid.mediator = Some("generic-mediator".to_string()); + assert!(AuctionPlan::compile(invalid).is_err()); + } +} diff --git a/crates/trusted-server-core/src/auction/profile.rs b/crates/trusted-server-core/src/auction/profile.rs new file mode 100644 index 000000000..8f5b25806 --- /dev/null +++ b/crates/trusted-server-core/src/auction/profile.rs @@ -0,0 +1,325 @@ +//! Compile-time `OpenRTB` profile registry and typed profile plans. + +use std::collections::BTreeMap; + +use error_stack::Report; +use serde::Deserialize; +use serde_json::{Map, Value}; + +use crate::consent_config::ConsentForwardingMode; +use crate::error::TrustedServerError; +use crate::integrations::aps::compile_profile_config as compile_aps_profile_config; +use crate::integrations::prebid::{ + BidParamOverrideEngine, BidParamOverrideRule, compile_profile_override_rules, +}; + +const STANDARD_PROFILE_ID: &str = "standard"; +const PREBID_PROFILE_ID: &str = "prebid-server"; +const APS_PROFILE_ID: &str = "aps"; +const STATIC_EXTENSION_MAX_BYTES: usize = 16 * 1024; +const STATIC_EXTENSION_MAX_DEPTH: usize = 8; +const STATIC_EXTENSION_MAX_KEYS: usize = 256; + +/// A registered profile's provider-timeout default. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum ProfileTimeoutDefault { + /// Inherit the configured auction timeout. + Auction, + /// Use this fixed profile timeout. + Fixed(u32), +} + +/// Compile-time profile registration. +#[derive(Clone, Copy)] +pub struct OpenRtbProfileRegistration { + /// Stable profile identifier used by configuration. + pub id: &'static str, + /// Profile timeout used when a provider has no explicit override. + pub default_timeout: ProfileTimeoutDefault, + compile: fn(&Value) -> Result>, +} + +impl core::fmt::Debug for OpenRtbProfileRegistration { + fn fmt(&self, formatter: &mut core::fmt::Formatter<'_>) -> core::fmt::Result { + formatter + .debug_struct("OpenRtbProfileRegistration") + .field("id", &self.id) + .field("default_timeout", &self.default_timeout) + .finish_non_exhaustive() + } +} + +impl OpenRtbProfileRegistration { + pub(crate) fn compile( + self, + config: &Value, + ) -> Result> { + (self.compile)(config) + } +} + +/// Immutable, typed profile behavior selected during plan compilation. +#[derive(Debug, Clone)] +pub enum CompiledOpenRtbProfile { + /// Generic `OpenRTB` 2.6 profile. + Standard(StandardProfilePlan), + /// Prebid Server compatibility profile. + PrebidServer(PrebidProfilePlan), + /// APS `OpenRTB` compatibility profile. + Aps(ApsProfilePlan), +} + +impl CompiledOpenRtbProfile { + /// Return the stable profile identifier. + #[must_use] + pub fn id(&self) -> &'static str { + match self { + Self::Standard(_) => STANDARD_PROFILE_ID, + Self::PrebidServer(_) => PREBID_PROFILE_ID, + Self::Aps(_) => APS_PROFILE_ID, + } + } + + /// Return whether this plan uses the Prebid Server profile. + #[must_use] + pub(crate) fn is_prebid_server(&self) -> bool { + matches!(self, Self::PrebidServer(_)) + } +} + +/// Validated static extension object. +#[derive(Debug, Clone, Default, PartialEq)] +pub struct StaticExtension(Map); + +impl StaticExtension { + /// Borrow the validated extension object. + #[must_use] + pub fn as_object(&self) -> &Map { + &self.0 + } +} + +/// Compiled generic `OpenRTB` profile configuration. +#[derive(Debug, Clone, Default)] +pub struct StandardProfilePlan { + /// Static request-level extension fields. + pub request_ext: StaticExtension, + /// Static impression-level extension fields. + pub imp_ext: StaticExtension, +} + +/// Compiled Prebid profile configuration. +#[derive(Debug, Clone)] +pub struct PrebidProfilePlan { + /// Include Prebid HTTP exchange diagnostics. + pub debug: bool, + /// Set `OpenRTB` test mode. + pub test_mode: bool, + /// Optional query fragment appended to the page URL under legacy rules. + pub debug_query_params: Option, + /// Compiled override matching and merge index. + pub(crate) override_engine: BidParamOverrideEngine, + /// Consent transport policy. + pub consent_forwarding: ConsentForwardingMode, +} + +/// Compiled APS profile configuration. +#[derive(Debug, Clone)] +pub struct ApsProfilePlan { + /// APS account identifier. + pub account_id: String, + /// Include APS request/response diagnostics. + pub debug: bool, + /// Permit APS script creatives. + pub allow_script_creatives: bool, + /// Optional authorized inventory domain. + pub inventory_domain: Option, + /// Optional canonical inventory page origin. + pub inventory_page_origin: Option, +} + +#[derive(Debug, Deserialize, Default)] +#[serde(deny_unknown_fields)] +struct StandardProfileConfig { + #[serde(default)] + request_ext: Option, + #[serde(default)] + imp_ext: Option, +} + +/// Typed operator configuration compiled into a [`PrebidProfilePlan`]. +#[derive(Debug, Deserialize, Default)] +#[serde(deny_unknown_fields)] +pub(crate) struct PrebidProfileConfig { + #[serde(default)] + debug: bool, + #[serde(default)] + test_mode: bool, + #[serde(default)] + debug_query_params: Option, + #[serde(default)] + bid_param_zone_overrides: BTreeMap>>, + #[serde(default)] + bid_param_overrides: BTreeMap>, + #[serde(default)] + bid_param_override_rules: Vec, + #[serde(default)] + consent_forwarding: ConsentForwardingMode, +} + +const PROFILE_REGISTRATIONS: [OpenRtbProfileRegistration; 3] = [ + OpenRtbProfileRegistration { + id: STANDARD_PROFILE_ID, + default_timeout: ProfileTimeoutDefault::Auction, + compile: compile_standard, + }, + OpenRtbProfileRegistration { + id: PREBID_PROFILE_ID, + default_timeout: ProfileTimeoutDefault::Fixed(1000), + compile: compile_prebid, + }, + OpenRtbProfileRegistration { + id: APS_PROFILE_ID, + default_timeout: ProfileTimeoutDefault::Fixed(800), + compile: compile_aps, + }, +]; + +/// Return the compile-time profile registry. +#[must_use] +pub fn profile_registrations() -> &'static [OpenRtbProfileRegistration] { + &PROFILE_REGISTRATIONS +} + +pub(crate) fn find_profile(id: &str) -> Option { + profile_registrations() + .iter() + .copied() + .find(|registration| registration.id == id) +} + +fn configuration_error(message: impl Into) -> Report { + Report::new(TrustedServerError::Configuration { + message: message.into(), + }) +} + +fn deserialize_profile(id: &str, value: &Value) -> Result> +where + T: for<'de> Deserialize<'de>, +{ + serde_json::from_value(value.clone()) + .map_err(|error| configuration_error(format!("invalid `{id}` profile_config: {error}"))) +} + +fn compile_standard(value: &Value) -> Result> { + let config: StandardProfileConfig = deserialize_profile(STANDARD_PROFILE_ID, value)?; + Ok(CompiledOpenRtbProfile::Standard(StandardProfilePlan { + request_ext: validate_static_extension("request_ext", config.request_ext)?, + imp_ext: validate_static_extension("imp_ext", config.imp_ext)?, + })) +} + +fn compile_prebid(value: &Value) -> Result> { + let config: PrebidProfileConfig = deserialize_profile(PREBID_PROFILE_ID, value)?; + let override_engine = compile_profile_override_rules( + &config.bid_param_zone_overrides, + &config.bid_param_overrides, + &config.bid_param_override_rules, + )?; + Ok(CompiledOpenRtbProfile::PrebidServer(PrebidProfilePlan { + debug: config.debug, + test_mode: config.test_mode, + debug_query_params: config.debug_query_params, + override_engine, + consent_forwarding: config.consent_forwarding, + })) +} + +fn compile_aps(value: &Value) -> Result> { + let config = compile_aps_profile_config(value.clone())?; + Ok(CompiledOpenRtbProfile::Aps(ApsProfilePlan { + account_id: config.account_id, + debug: config.debug, + allow_script_creatives: config.allow_script_creatives, + inventory_domain: config.inventory_domain, + inventory_page_origin: config.inventory_page_origin, + })) +} + +fn validate_static_extension( + field: &str, + value: Option, +) -> Result> { + let Some(value) = value else { + return Ok(StaticExtension::default()); + }; + let object = value.as_object().ok_or_else(|| { + configuration_error(format!("standard profile {field} must be an object")) + })?; + let size = serde_json::to_vec(&value) + .map_err(|error| configuration_error(format!("cannot serialize {field}: {error}")))? + .len(); + if size > STATIC_EXTENSION_MAX_BYTES { + return Err(configuration_error(format!( + "standard profile {field} exceeds {STATIC_EXTENSION_MAX_BYTES} bytes" + ))); + } + validate_extension_value(field, &value, 0)?; + reject_reserved_fields(field, object)?; + Ok(StaticExtension(object.clone())) +} + +fn validate_extension_value( + field: &str, + value: &Value, + container_depth: usize, +) -> Result<(), Report> { + match value { + Value::Object(object) => { + let container_depth = container_depth + 1; + if container_depth > STATIC_EXTENSION_MAX_DEPTH { + return Err(configuration_error(format!( + "standard profile {field} exceeds nesting depth {STATIC_EXTENSION_MAX_DEPTH}" + ))); + } + if object.len() > STATIC_EXTENSION_MAX_KEYS { + return Err(configuration_error(format!( + "standard profile {field} object exceeds {STATIC_EXTENSION_MAX_KEYS} keys" + ))); + } + for nested in object.values() { + validate_extension_value(field, nested, container_depth)?; + } + } + Value::Array(array) => { + let container_depth = container_depth + 1; + if container_depth > STATIC_EXTENSION_MAX_DEPTH { + return Err(configuration_error(format!( + "standard profile {field} exceeds nesting depth {STATIC_EXTENSION_MAX_DEPTH}" + ))); + } + for nested in array { + validate_extension_value(field, nested, container_depth)?; + } + } + _ => {} + } + Ok(()) +} + +fn reject_reserved_fields( + field: &str, + object: &Map, +) -> Result<(), Report> { + let reserved: &[&str] = match field { + "request_ext" => &["trusted_server"], + _ => &[], + }; + if let Some(key) = reserved.iter().find(|key| object.contains_key(**key)) { + return Err(configuration_error(format!( + "standard profile {field} cannot claim reserved field `{key}`" + ))); + } + Ok(()) +} diff --git a/crates/trusted-server-core/src/auction/provider.rs b/crates/trusted-server-core/src/auction/provider.rs index 766bd7a08..28f6dbd96 100644 --- a/crates/trusted-server-core/src/auction/provider.rs +++ b/crates/trusted-server-core/src/auction/provider.rs @@ -1,15 +1,45 @@ //! Trait definition for auction providers. use core::any::Any; +use std::collections::HashSet; use async_trait::async_trait; -use error_stack::Report; +use edgezero_core::body::Body as EdgeBody; +use error_stack::{Report, ResultExt as _}; +use http::{Method, Request, StatusCode, header}; +use serde_json::{Value, json}; + +use crate::integrations::aps::{ApsDebugRequest, parse_planned_aps_response}; +use crate::integrations::prebid::{apply_prebid_transport_headers, parse_planned_prebid_response}; use crate::error::TrustedServerError; -use crate::platform::{PlatformPendingRequest, PlatformResponse, RuntimeServices}; +use crate::platform::{ + PlatformHttpRequest, PlatformPendingRequest, PlatformResponse, RuntimeServices, +}; +use crate::request_signing::{RequestSigner, SigningParams}; +use super::openrtb::{ + OpenRtbBuildOutcome, RequestFinalization, apply_notification_policy, build_request, + extract_standard_response, unused_bidder_params_count, +}; +use super::plan::ProviderPlan; +use super::profile::CompiledOpenRtbProfile; +use super::routing::{ProviderAuctionInput, RoutedAuction}; use super::types::{AuctionContext, AuctionRequest, AuctionResponse}; +const MAX_PLANNED_RESPONSE_BYTES: usize = 1024 * 1024; + +fn attach_provider_routing_metadata( + response: &mut AuctionResponse, + profile: &CompiledOpenRtbProfile, + input: &ProviderAuctionInput, +) { + response.metadata.insert( + "routing".to_string(), + json!({"unused_bidder_params_count": unused_bidder_params_count(profile, input)}), + ); +} + /// Provider-local state carried from request dispatch to response parsing. pub type ProviderParseState = Box; @@ -52,8 +82,11 @@ impl ProviderRequestOutcome { /// Trait implemented by all auction providers (Prebid, APS, GAM, etc.). #[async_trait(?Send)] pub trait AuctionProvider: Send + Sync { - /// Unique identifier for this provider (e.g., "prebid", "aps", "gam"). - fn provider_name(&self) -> &'static str; + /// Borrow this provider instance's unique validated identifier. + /// + /// Legacy providers may return a string literal; config-first providers + /// return their owned operator-defined [`super::plan::ProviderId`]. + fn provider_name(&self) -> &str; /// Submit a bid request to this provider. /// @@ -155,3 +188,398 @@ pub trait AuctionProvider: Send + Sync { None } } + +/// One immutable config-first `OpenRTB` provider instance. +/// +/// Every instance owns its validated provider identity and carries only its own +/// typed response state across transport. +pub(crate) struct GenericOpenRtbProvider { + plan: ProviderPlan, +} + +/// Typed state created by and returned to one [`GenericOpenRtbProvider`]. +#[allow( + dead_code, + clippy::large_enum_variant, + reason = "typed Stage 6 state avoids provider-state confusion; Stage 7/8 replace profile variants" +)] +pub(crate) enum GenericOpenRtbParseState { + Standard { + provider_id: String, + input: ProviderAuctionInput, + }, + Prebid { + provider_id: String, + auction_id: String, + input: ProviderAuctionInput, + }, + Aps { + provider_id: String, + input: ProviderAuctionInput, + debug_request: Option, + }, +} + +impl GenericOpenRtbProvider { + pub(crate) fn new(plan: ProviderPlan) -> Self { + Self { plan } + } + + pub(crate) fn provider_name(&self) -> &str { + self.plan.id.as_str() + } + + pub(crate) fn timeout_ms(&self) -> u32 { + self.plan.timeout_ms + } + + #[cfg(test)] + pub(crate) fn parse_state_for_test(&self, input: ProviderAuctionInput) -> ProviderParseState { + let state = match &self.plan.profile { + CompiledOpenRtbProfile::Standard(_) => GenericOpenRtbParseState::Standard { + provider_id: self.provider_name().to_string(), + input, + }, + CompiledOpenRtbProfile::PrebidServer(_) => GenericOpenRtbParseState::Prebid { + provider_id: self.provider_name().to_string(), + auction_id: input.common_request().id.clone(), + input, + }, + CompiledOpenRtbProfile::Aps(_) => GenericOpenRtbParseState::Aps { + provider_id: self.provider_name().to_string(), + input, + debug_request: None, + }, + }; + Box::new(state) + } + + /// Build, register, and start exactly one routed provider request. + /// + /// The public [`AuctionProvider::request_bids`] seam cannot carry a + /// [`ProviderAuctionInput`], the complete [`RoutedAuction`], or an + /// auction-local [`RequestSigner`] without shared mutable provider state. + /// The plan-backed split dispatcher therefore supplies that explicit + /// execution context here, while this method reuses + /// [`ProviderRequestOutcome`] and [`ProviderParseState`] for the existing + /// request/parse token boundary. + #[allow( + clippy::too_many_arguments, + reason = "the internal driver keeps routed inputs, both budgets, signer, services, and collision state explicit" + )] + pub(crate) async fn request_bids_routed( + &self, + input: &ProviderAuctionInput, + routed: &RoutedAuction, + logical_budget_ms: u32, + transport_timeout_ms: u32, + signer: Option<&RequestSigner>, + services: &RuntimeServices, + reserved_backend_names: &mut HashSet, + ) -> Result> { + let signing_params = SigningParams::new( + input.common_request().id.clone(), + input.common_request().publisher.domain.clone(), + "https".to_string(), + ); + let request = match build_request( + input, + routed, + &self.plan, + logical_budget_ms, + &RequestFinalization { + signer, + signing_params, + }, + )? { + OpenRtbBuildOutcome::Ready(request) => request, + OpenRtbBuildOutcome::NoImpressions => { + return Ok(ProviderRequestOutcome::Immediate(AuctionResponse::no_bid( + self.provider_name(), + 0, + ))); + } + }; + + let spec = self + .plan + .backend_spec_with_transport_timeout(transport_timeout_ms); + let predicted_name = + services + .backend() + .predict_name(&spec) + .change_context(TrustedServerError::Auction { + message: format!( + "Provider {} backend prediction failed", + self.provider_name() + ), + })?; + let backend_name = + services + .backend() + .ensure(&spec) + .change_context(TrustedServerError::Auction { + message: format!( + "Provider {} backend registration failed", + self.provider_name() + ), + })?; + if backend_name != predicted_name { + return Err(Report::new(TrustedServerError::Auction { + message: format!( + "Provider {} backend registration did not match prediction", + self.provider_name() + ), + })); + } + if !reserved_backend_names.insert(backend_name.clone()) { + return Err(Report::new(TrustedServerError::Auction { + message: format!( + "Provider {} resolved an actual backend name already owned by another provider", + self.provider_name() + ), + })); + } + + let body = serde_json::to_vec(&request).change_context(TrustedServerError::Auction { + message: format!( + "Provider {} request serialization failed", + self.provider_name() + ), + })?; + let mut outbound = Request::builder() + .method(Method::POST) + .uri(self.plan.endpoint.as_str()) + .header(header::CONTENT_TYPE, "application/json"); + if matches!(&self.plan.profile, CompiledOpenRtbProfile::Standard(_)) { + outbound = outbound.header(header::ACCEPT, "application/json"); + } + let aps_debug_body = matches!( + &self.plan.profile, + CompiledOpenRtbProfile::Aps(profile) if profile.debug + ) + .then(|| body.clone()); + let mut outbound = + outbound + .body(EdgeBody::from(body)) + .change_context(TrustedServerError::Auction { + message: format!( + "Provider {} request construction failed", + self.provider_name() + ), + })?; + let aps_debug_request = aps_debug_body + .as_deref() + .map(|body| ApsDebugRequest::capture(body, outbound.headers())); + if let CompiledOpenRtbProfile::PrebidServer(profile) = &self.plan.profile { + apply_prebid_transport_headers( + routed.prebid_transport_headers(), + &mut outbound, + profile.consent_forwarding, + routed.attested_client_ip(), + ); + } + let pending = services + .http_client() + .send_async(PlatformHttpRequest::new(outbound, backend_name.clone())) + .await + .change_context(TrustedServerError::Auction { + message: format!("Provider {} request launch failed", self.provider_name()), + })?; + if pending.backend_name() != Some(backend_name.as_str()) { + return Err(Report::new(TrustedServerError::Auction { + message: format!( + "Provider {} pending request backend did not match registered backend", + self.provider_name() + ), + })); + } + let parse_state = match &self.plan.profile { + CompiledOpenRtbProfile::Standard(_) => GenericOpenRtbParseState::Standard { + provider_id: self.provider_name().to_string(), + input: input.clone(), + }, + CompiledOpenRtbProfile::PrebidServer(_) => GenericOpenRtbParseState::Prebid { + provider_id: self.provider_name().to_string(), + auction_id: input.common_request().id.clone(), + input: input.clone(), + }, + CompiledOpenRtbProfile::Aps(_) => GenericOpenRtbParseState::Aps { + provider_id: self.provider_name().to_string(), + input: input.clone(), + debug_request: aps_debug_request, + }, + }; + Ok(ProviderRequestOutcome::pending_with_state( + pending, + Box::new(parse_state), + )) + } + + /// Parse a response using state created by this exact provider instance. + pub(crate) async fn parse_response_with_state( + &self, + response: PlatformResponse, + response_time_ms: u64, + parse_state: Option<&(dyn Any + Send + Sync)>, + ) -> Result> { + let parse_state = parse_state + .and_then(|state| state.downcast_ref::()) + .ok_or_else(|| { + Report::new(TrustedServerError::Auction { + message: format!( + "Provider {} received missing or invalid response state", + self.provider_name() + ), + }) + })?; + let state_provider_id = match parse_state { + GenericOpenRtbParseState::Standard { provider_id, .. } + | GenericOpenRtbParseState::Prebid { provider_id, .. } + | GenericOpenRtbParseState::Aps { provider_id, .. } => provider_id, + }; + if state_provider_id != self.provider_name() { + return Err(Report::new(TrustedServerError::Auction { + message: format!( + "Provider {} received response state owned by provider {}", + self.provider_name(), + state_provider_id + ), + })); + } + + if let GenericOpenRtbParseState::Prebid { + auction_id, input, .. + } = parse_state + { + let CompiledOpenRtbProfile::PrebidServer(profile) = &self.plan.profile else { + return Err(Report::new(TrustedServerError::Auction { + message: format!( + "Provider {} received PBS response state for profile {}", + self.provider_name(), + self.plan.profile.id() + ), + })); + }; + let mut parsed = match parse_planned_prebid_response( + self.provider_name(), + profile, + input, + response, + response_time_ms, + auction_id, + ) + .await + { + Ok(parsed) => parsed, + Err(error) => { + log::warn!( + "Provider '{}' PBS response parse failed: {:?}", + self.provider_name(), + error + ); + AuctionResponse::error(self.provider_name(), response_time_ms) + .with_metadata("error_type", json!("parse_response")) + } + }; + apply_notification_policy(&mut parsed.bids, &self.plan.notifications); + attach_provider_routing_metadata(&mut parsed, &self.plan.profile, input); + return Ok(parsed); + } + + if let GenericOpenRtbParseState::Aps { + input, + debug_request, + .. + } = parse_state + { + let CompiledOpenRtbProfile::Aps(profile) = &self.plan.profile else { + return Err(Report::new(TrustedServerError::Auction { + message: format!( + "Provider {} received APS response state for profile {}", + self.provider_name(), + self.plan.profile.id() + ), + })); + }; + let mut parsed = match parse_planned_aps_response( + self.provider_name(), + profile, + self.plan.endpoint.as_str(), + input, + response, + response_time_ms, + debug_request.clone(), + ) + .await + { + Ok(parsed) => parsed, + Err(error) => { + log::warn!( + "Provider '{}' APS response parse failed: {:?}", + self.provider_name(), + error + ); + let mut parsed = AuctionResponse::error(self.provider_name(), response_time_ms) + .with_metadata("error_type", json!("parse_response")); + attach_provider_routing_metadata(&mut parsed, &self.plan.profile, input); + parsed + } + }; + apply_notification_policy(&mut parsed.bids, &self.plan.notifications); + return Ok(parsed); + } + + let response = response.response; + let status = response.status(); + let GenericOpenRtbParseState::Standard { input, .. } = parse_state else { + unreachable!("profile-specific states are handled before standard parsing"); + }; + if status == StatusCode::NO_CONTENT { + let mut parsed = AuctionResponse::no_bid(self.provider_name(), response_time_ms); + attach_provider_routing_metadata(&mut parsed, &self.plan.profile, input); + return Ok(parsed); + } + if !status.is_success() { + if status.is_redirection() { + log::warn!( + "Provider '{}' returned a redirect; generic OpenRTB redirects are refused", + self.provider_name() + ); + } + let mut parsed = AuctionResponse::error(self.provider_name(), response_time_ms) + .with_metadata("error_type", json!("http_status")) + .with_metadata("http_status", json!(status.as_u16())); + attach_provider_routing_metadata(&mut parsed, &self.plan.profile, input); + return Ok(parsed); + } + + let body = response + .into_body() + .into_bytes_bounded(MAX_PLANNED_RESPONSE_BYTES) + .await + .change_context(TrustedServerError::Auction { + message: format!("Provider {} response body failed", self.provider_name()), + })?; + let value: Value = match serde_json::from_slice(&body) { + Ok(value) => value, + Err(error) => { + log::warn!( + "Provider '{}' response JSON was invalid: {}", + self.provider_name(), + error + ); + let mut parsed = AuctionResponse::error(self.provider_name(), response_time_ms) + .with_metadata("error_type", json!("parse_response")); + attach_provider_routing_metadata(&mut parsed, &self.plan.profile, input); + return Ok(parsed); + } + }; + + let mut parsed = + extract_standard_response(self.provider_name(), input, &value, response_time_ms); + apply_notification_policy(&mut parsed.bids, &self.plan.notifications); + attach_provider_routing_metadata(&mut parsed, &self.plan.profile, input); + Ok(parsed) + } +} diff --git a/crates/trusted-server-core/src/auction/routing.rs b/crates/trusted-server-core/src/auction/routing.rs new file mode 100644 index 000000000..61b2302cc --- /dev/null +++ b/crates/trusted-server-core/src/auction/routing.rs @@ -0,0 +1,1200 @@ +//! Internal admission normalization and config-first provider routing. + +use std::collections::{BTreeMap, BTreeSet, HashMap}; +use std::net::IpAddr; + +use edgezero_core::body::Body as EdgeBody; +use http::{HeaderValue, Request, header}; +use serde_json::Value; + +use super::plan::{AuctionPlan, BidderId, ProviderId, RoutingMode}; +use super::types::{AdSlot, AuctionRequest, MediaType}; + +const TRUSTED_SERVER_ENVELOPE: &str = "trustedServer"; +const BIDDER_PARAMS_FIELD: &str = "bidderParams"; +const ZONE_FIELD: &str = "zone"; + +/// Maximum bidder entries admitted from one browser `bidderParams` envelope. +pub(crate) const MAX_BIDDER_ENTRIES: usize = 128; +/// Maximum UTF-8 byte length of the optional Prebid zone fact. +pub(crate) const MAX_PREBID_ZONE_BYTES: usize = 256; + +/// Immutable provider-local routing output in deterministic provider-ID order. +#[derive(Debug, Clone)] +pub(crate) struct RoutedAuction { + inputs: Vec, + skipped_no_eligible_provider_ids: Vec, + diagnostics: RoutingDiagnostics, + transport_headers: PrebidTransportHeaders, + attested_client_ip: Option, + dnt: Option, +} + +impl RoutedAuction { + pub(crate) fn inputs(&self) -> &[ProviderAuctionInput] { + &self.inputs + } + + /// Providers skipped because they had no eligible banner slots. + /// + /// IDs retain the compiled plan's deterministic provider-ID order. + pub(crate) fn skipped_no_eligible_provider_ids(&self) -> &[ProviderId] { + &self.skipped_no_eligible_provider_ids + } + + pub(crate) fn diagnostics(&self) -> RoutingDiagnostics { + self.diagnostics + } + + /// Request headers approved for later Prebid transport forwarding. + pub(crate) fn prebid_transport_headers(&self) -> &PrebidTransportHeaders { + &self.transport_headers + } + + /// Platform-attested client IP for transport forwarding. + pub(crate) fn attested_client_ip(&self) -> Option { + self.attested_client_ip + } + + /// Normalized Do Not Track fact; raw request headers are not exposed to profiles. + pub(crate) fn dnt(&self) -> Option { + self.dnt + } +} + +/// Saturating, count-only routing diagnostics. +#[derive(Debug, Clone, Copy, Default, Eq, PartialEq)] +pub(crate) struct RoutingDiagnostics { + unroutable_bidder_count: u32, + malformed_envelope_count: u32, + malformed_direct_demand_count: u32, + unroutable_trusted_provider_count: u32, +} + +impl RoutingDiagnostics { + pub(crate) fn unroutable_bidder_count(self) -> u32 { + self.unroutable_bidder_count + } + + #[cfg(test)] + pub(crate) fn malformed_envelope_count(self) -> u32 { + self.malformed_envelope_count + } + + #[cfg(test)] + pub(crate) fn malformed_direct_demand_count(self) -> u32 { + self.malformed_direct_demand_count + } + + #[cfg(test)] + pub(crate) fn unroutable_trusted_provider_count(self) -> u32 { + self.unroutable_trusted_provider_count + } + + fn record_unroutable_bidder(&mut self) { + self.unroutable_bidder_count = self.unroutable_bidder_count.saturating_add(1); + } + + fn record_malformed_envelope(&mut self) { + self.malformed_envelope_count = self.malformed_envelope_count.saturating_add(1); + } + + fn record_malformed_direct_demand(&mut self) { + self.malformed_direct_demand_count = self.malformed_direct_demand_count.saturating_add(1); + } + + fn record_unroutable_trusted_provider(&mut self) { + self.unroutable_trusted_provider_count = + self.unroutable_trusted_provider_count.saturating_add(1); + } + + #[cfg(test)] + pub(crate) fn saturated_for_test() -> Self { + let mut diagnostics = Self { + unroutable_bidder_count: u32::MAX, + ..Self::default() + }; + diagnostics.record_unroutable_bidder(); + diagnostics + } +} + +/// Provider-local immutable auction input. +#[derive(Debug, Clone)] +pub(crate) struct ProviderAuctionInput { + provider_id: ProviderId, + #[cfg_attr( + not(test), + allow( + dead_code, + reason = "retained in routed input to pin the provider budget invariant" + ) + )] + timeout_ms: u32, + common_request: AuctionRequest, + slots: Vec, +} + +impl ProviderAuctionInput { + pub(crate) fn provider_id(&self) -> &ProviderId { + &self.provider_id + } + + #[cfg(test)] + pub(crate) fn timeout_ms(&self) -> u32 { + self.timeout_ms + } + + /// Common privacy-approved request data. Its slot list is always empty. + pub(crate) fn common_request(&self) -> &AuctionRequest { + &self.common_request + } + + pub(crate) fn slots(&self) -> &[ProviderSlotInput] { + &self.slots + } +} + +/// One eligible slot with only the demand assigned to this provider. +#[derive(Debug, Clone)] +pub(crate) struct ProviderSlotInput { + slot: AdSlot, + bidder_params: BTreeMap, + prebid_zone: Option, + trusted_stored_request: bool, +} + +impl ProviderSlotInput { + /// Common slot facts. The legacy `bidders` map is always empty. + pub(crate) fn slot(&self) -> &AdSlot { + &self.slot + } + + pub(crate) fn bidder_params(&self) -> &BTreeMap { + &self.bidder_params + } + + pub(crate) fn prebid_zone(&self) -> Option<&str> { + self.prebid_zone.as_deref() + } + + pub(crate) fn has_trusted_stored_request(&self) -> bool { + self.trusted_stored_request + } +} + +/// Request headers approved for later Prebid transport forwarding. +/// +/// Values remain as raw [`HeaderValue`] instances so non-ASCII bytes retain +/// the same legacy handling. Client-supplied `X-Forwarded-For` is never read. +#[derive(Debug, Clone, Default)] +pub(crate) struct PrebidTransportHeaders { + cookie: Option, + user_agent: Option, + referer: Option, + accept_language: Option, +} + +impl PrebidTransportHeaders { + pub(crate) fn cookie(&self) -> Option<&HeaderValue> { + self.cookie.as_ref() + } + + pub(crate) fn user_agent(&self) -> Option<&HeaderValue> { + self.user_agent.as_ref() + } + + pub(crate) fn referer(&self) -> Option<&HeaderValue> { + self.referer.as_ref() + } + + pub(crate) fn accept_language(&self) -> Option<&HeaderValue> { + self.accept_language.as_ref() + } + + fn snapshot(request: &Request) -> Self { + Self { + cookie: request.headers().get(header::COOKIE).cloned(), + user_agent: request.headers().get(header::USER_AGENT).cloned(), + referer: request.headers().get(header::REFERER).cloned(), + accept_language: request.headers().get(header::ACCEPT_LANGUAGE).cloned(), + } + } +} + +/// Server-owned explicit provider routes aligned with canonical auction slots. +/// +/// This internal-only type has no deserializer, so browser input cannot select +/// a provider ID. The caller supplies one route list for each request slot. +#[derive(Debug, Default)] +pub(crate) struct TrustedProviderRoutes { + routes_by_slot: Vec>, +} + +impl TrustedProviderRoutes { + #[cfg(test)] + pub(crate) fn new(routes_by_slot: Vec>) -> Self { + Self { routes_by_slot } + } + + fn for_slot(&self, slot_index: usize) -> &[ProviderId] { + self.routes_by_slot + .get(slot_index) + .map_or(&[], Vec::as_slice) + } +} + +#[derive(Debug, Default)] +struct NormalizedSlotDemand { + bidder_params: BTreeMap, + stored_request: bool, + prebid_zone: Option, +} + +#[derive(Debug)] +struct ProviderInputBuilder { + provider_id: ProviderId, + timeout_ms: u32, + is_prebid: bool, + routing: RoutingMode, + slots: Vec, +} + +/// Normalize admitted demand and build deterministic provider-local inputs. +/// +/// This helper consumes the canonical request so the common request retained by +/// each provider can be scrubbed of slots. It performs no I/O. +pub(crate) fn route_auction( + request: AuctionRequest, + inbound_request: &Request, + plan: &AuctionPlan, + attested_client_ip: Option, +) -> RoutedAuction { + route_auction_with_trusted_routes( + request, + inbound_request, + plan, + attested_client_ip, + &TrustedProviderRoutes::default(), + ) +} + +/// Route an auction with server-owned explicit provider routes. +/// +/// Only server-generated entry points may construct [`TrustedProviderRoutes`]. +/// Browser/default admission must use [`route_auction`]. +pub(crate) fn route_auction_with_trusted_routes( + mut request: AuctionRequest, + inbound_request: &Request, + plan: &AuctionPlan, + attested_client_ip: Option, + trusted_routes: &TrustedProviderRoutes, +) -> RoutedAuction { + let slots = std::mem::take(&mut request.slots); + let common_request = request; + let transport_headers = PrebidTransportHeaders::snapshot(inbound_request); + let dnt = inbound_request + .headers() + .get("dnt") + .and_then(|value| value.to_str().ok()) + .is_some_and(|value| value.trim() == "1") + .then_some(true); + let mut diagnostics = RoutingDiagnostics::default(); + let mut builders = plan + .providers() + .iter() + .map(|provider| ProviderInputBuilder { + provider_id: provider.id.clone(), + timeout_ms: provider.timeout_ms, + is_prebid: provider.profile.is_prebid_server(), + routing: provider.routing, + slots: Vec::new(), + }) + .collect::>(); + let provider_indices = builders + .iter() + .enumerate() + .map(|(index, provider)| (provider.provider_id.clone(), index)) + .collect::>(); + + for (slot_index, slot) in slots.into_iter().enumerate() { + let Some(common_slot) = eligible_banner_slot(&slot) else { + continue; + }; + let demand = normalize_slot_demand(&slot.bidders, &mut diagnostics); + let mut routed_params = vec![BTreeMap::new(); builders.len()]; + for (bidder, params) in demand.bidder_params { + let Some(provider) = plan.provider_for_bidder(&bidder) else { + diagnostics.record_unroutable_bidder(); + continue; + }; + let provider_index = *provider_indices + .get(&provider.id) + .expect("should resolve compiled provider index"); + routed_params[provider_index].insert(bidder, params); + } + let mut trusted_provider_indices = BTreeSet::new(); + for provider_id in trusted_routes.for_slot(slot_index) { + let Some(provider_index) = provider_indices.get(provider_id).copied() else { + diagnostics.record_unroutable_trusted_provider(); + continue; + }; + trusted_provider_indices.insert(provider_index); + } + + for (provider_index, builder) in builders.iter_mut().enumerate() { + let bidder_params = std::mem::take(&mut routed_params[provider_index]); + let trusted_route = trusted_provider_indices.contains(&provider_index); + let trusted_stored_request = + builder.is_prebid && demand.stored_request && bidder_params.is_empty(); + let include = builder.routing == RoutingMode::AllEligible + || !bidder_params.is_empty() + || trusted_stored_request + || trusted_route; + if !include { + continue; + } + builder.slots.push(ProviderSlotInput { + slot: common_slot.clone(), + bidder_params, + prebid_zone: builder + .is_prebid + .then(|| demand.prebid_zone.clone()) + .flatten(), + trusted_stored_request, + }); + } + } + + let mut skipped_no_eligible_provider_ids = Vec::new(); + let inputs = builders + .into_iter() + .filter_map(|builder| { + if builder.slots.is_empty() { + skipped_no_eligible_provider_ids.push(builder.provider_id); + return None; + } + Some(ProviderAuctionInput { + provider_id: builder.provider_id, + timeout_ms: builder.timeout_ms, + common_request: common_request.clone(), + slots: builder.slots, + }) + }) + .collect(); + + RoutedAuction { + inputs, + skipped_no_eligible_provider_ids, + diagnostics, + transport_headers, + attested_client_ip, + dnt, + } +} + +fn eligible_banner_slot(slot: &AdSlot) -> Option { + let formats = slot + .formats + .iter() + .filter(|format| { + format.media_type == MediaType::Banner + && i32::try_from(format.width).is_ok_and(|width| width > 0) + && i32::try_from(format.height).is_ok_and(|height| height > 0) + }) + .cloned() + .collect::>(); + if formats.is_empty() { + return None; + } + Some(AdSlot { + id: slot.id.clone(), + formats, + floor_price: slot.floor_price, + targeting: slot.targeting.clone(), + bidders: HashMap::new(), + }) +} + +fn normalize_slot_demand( + bidders: &HashMap, + diagnostics: &mut RoutingDiagnostics, +) -> NormalizedSlotDemand { + if bidders.is_empty() { + return NormalizedSlotDemand { + stored_request: true, + ..Default::default() + }; + } + + let mut demand = NormalizedSlotDemand::default(); + if let Some(envelope) = bidders.get(TRUSTED_SERVER_ENVELOPE) { + match normalize_envelope(envelope) { + Some(normalized) => demand = normalized, + None => diagnostics.record_malformed_envelope(), + } + } + + let mut direct = bidders + .iter() + .filter(|(key, _)| key.as_str() != TRUSTED_SERVER_ENVELOPE) + .collect::>(); + direct.sort_by_key(|(left, _)| *left); + for (raw_bidder, params) in direct { + let Ok(bidder) = raw_bidder.parse::() else { + diagnostics.record_malformed_direct_demand(); + continue; + }; + if bidder.as_str() == TRUSTED_SERVER_ENVELOPE || !is_usable_params(params) { + diagnostics.record_malformed_direct_demand(); + continue; + } + demand.bidder_params.insert(bidder, params.clone()); + } + demand +} + +fn normalize_envelope(envelope: &Value) -> Option { + let object = envelope.as_object()?; + if object + .keys() + .any(|key| !matches!(key.as_str(), BIDDER_PARAMS_FIELD | ZONE_FIELD)) + { + return None; + } + let prebid_zone = match object.get(ZONE_FIELD) { + None => None, + Some(Value::String(zone)) if zone.len() <= MAX_PREBID_ZONE_BYTES => Some(zone.clone()), + Some(_) => return None, + }; + let Some(raw_params) = object.get(BIDDER_PARAMS_FIELD) else { + return Some(NormalizedSlotDemand { + stored_request: true, + prebid_zone, + ..Default::default() + }); + }; + if raw_params.is_null() { + return Some(NormalizedSlotDemand { + stored_request: true, + prebid_zone, + ..Default::default() + }); + } + let params = raw_params.as_object()?; + if params.is_empty() { + return Some(NormalizedSlotDemand { + stored_request: true, + prebid_zone, + ..Default::default() + }); + } + if params.len() > MAX_BIDDER_ENTRIES { + return None; + } + + let mut bidder_params = BTreeMap::new(); + for (raw_bidder, value) in params { + let bidder = raw_bidder.parse::().ok()?; + if bidder.as_str() == TRUSTED_SERVER_ENVELOPE || !value.is_object() { + return None; + } + bidder_params.insert(bidder, value.clone()); + } + Some(NormalizedSlotDemand { + bidder_params, + stored_request: false, + prebid_zone, + }) +} + +fn is_usable_params(value: &Value) -> bool { + value.as_object().is_some_and(|object| !object.is_empty()) +} + +#[cfg(test)] +mod tests { + use std::str::FromStr as _; + + use super::*; + use crate::auction::plan::{ + AuctionPlanConfig, BidderRouteConfig, NotificationConfig, ProviderConfig, + }; + use crate::auction::types::{AdFormat, DeviceInfo, PublisherInfo, SiteInfo, UserInfo}; + use http::HeaderName; + use serde_json::{Map, json}; + + fn provider(profile: &str, routing: RoutingMode) -> ProviderConfig { + ProviderConfig { + protocol: "openrtb-2.6".to_string(), + profile: profile.to_string(), + endpoint: format!("https://{profile}.example.test/openrtb"), + timeout_ms: None, + routing, + notifications: NotificationConfig::default(), + profile_config: if profile == "aps" { + json!({"account_id": "example-account"}) + } else { + json!({}) + }, + } + } + + fn plan() -> AuctionPlan { + AuctionPlan::compile(AuctionPlanConfig { + timeout_ms: 900, + providers: BTreeMap::from([ + ( + ProviderId::from_str("aps-primary").expect("should parse provider"), + provider("aps", RoutingMode::AllEligible), + ), + ( + ProviderId::from_str("pbs-a").expect("should parse provider"), + provider("prebid-server", RoutingMode::Explicit), + ), + ( + ProviderId::from_str("pbs-b").expect("should parse provider"), + provider("prebid-server", RoutingMode::Explicit), + ), + ( + ProviderId::from_str("standard-direct").expect("should parse provider"), + provider("standard", RoutingMode::Explicit), + ), + ]), + bidders: BTreeMap::from([ + ( + BidderId::from_str("alpha").expect("should parse bidder"), + BidderRouteConfig { + provider: ProviderId::from_str("pbs-a").expect("should parse provider"), + }, + ), + ( + BidderId::from_str("beta").expect("should parse bidder"), + BidderRouteConfig { + provider: ProviderId::from_str("standard-direct") + .expect("should parse provider"), + }, + ), + ]), + mediator: None, + request_signing: None, + }) + .expect("should compile plan") + } + + fn explicit_plan() -> AuctionPlan { + AuctionPlan::compile(AuctionPlanConfig { + timeout_ms: 900, + providers: BTreeMap::from([ + ( + ProviderId::from_str("aps-primary").expect("should parse provider"), + provider("aps", RoutingMode::Explicit), + ), + ( + ProviderId::from_str("pbs-a").expect("should parse provider"), + provider("prebid-server", RoutingMode::Explicit), + ), + ( + ProviderId::from_str("pbs-b").expect("should parse provider"), + provider("prebid-server", RoutingMode::Explicit), + ), + ( + ProviderId::from_str("standard-direct").expect("should parse provider"), + provider("standard", RoutingMode::Explicit), + ), + ]), + bidders: BTreeMap::new(), + mediator: None, + request_signing: None, + }) + .expect("should compile explicit plan") + } + + fn slot(bidders: HashMap) -> AdSlot { + AdSlot { + id: "slot-1".to_string(), + formats: vec![AdFormat { + media_type: MediaType::Banner, + width: 300, + height: 250, + }], + floor_price: Some(0.5), + targeting: HashMap::new(), + bidders, + } + } + + fn request(slots: Vec) -> AuctionRequest { + AuctionRequest { + id: "auction-1".to_string(), + slots, + publisher: PublisherInfo { + domain: "publisher.example.test".to_string(), + page_url: Some("https://publisher.example.test/article".to_string()), + }, + user: UserInfo { + id: None, + consent: None, + eids: None, + }, + device: Some(DeviceInfo { + user_agent: None, + ip: None, + geo: None, + }), + site: Some(SiteInfo { + domain: "publisher.example.test".to_string(), + page: "https://publisher.example.test/article".to_string(), + }), + context: HashMap::new(), + } + } + + fn inbound() -> Request { + Request::builder() + .uri("https://publisher.example.test/auction") + .body(EdgeBody::empty()) + .expect("should build request") + } + + fn envelope(bidder_params: Option) -> Value { + let mut value = Map::new(); + if let Some(params) = bidder_params { + value.insert(BIDDER_PARAMS_FIELD.to_string(), params); + } + Value::Object(value) + } + + fn input<'a>(routed: &'a RoutedAuction, provider_id: &str) -> &'a ProviderAuctionInput { + routed + .inputs() + .iter() + .find(|input| input.provider_id().as_str() == provider_id) + .expect("should find provider input") + } + + #[test] + fn missing_null_and_empty_envelope_params_fan_out_stored_routes() { + let cases = [ + ("missing", envelope(None)), + ("null", envelope(Some(Value::Null))), + ("empty", envelope(Some(json!({})))), + ]; + for (name, trusted_server) in cases { + let routed = route_auction( + request(vec![slot(HashMap::from([( + TRUSTED_SERVER_ENVELOPE.to_string(), + trusted_server, + )]))]), + &inbound(), + &plan(), + None, + ); + let ids = routed + .inputs() + .iter() + .map(|input| input.provider_id().as_str()) + .collect::>(); + assert_eq!( + ids, + vec!["aps-primary", "pbs-a", "pbs-b"], + "{name} should fan out to both PBS providers while APS remains all-eligible" + ); + assert!( + input(&routed, "pbs-a").slots()[0].has_trusted_stored_request(), + "{name} should create stored intent" + ); + assert!( + input(&routed, "pbs-b").slots()[0].has_trusted_stored_request(), + "{name} should create stored intent for every PBS provider" + ); + } + } + + #[test] + fn entirely_empty_bidder_map_is_trusted_stored_intent() { + let routed = route_auction( + request(vec![slot(HashMap::new())]), + &inbound(), + &plan(), + None, + ); + assert!( + input(&routed, "pbs-a").slots()[0].has_trusted_stored_request(), + "empty canonical demand should preserve stored-request behavior" + ); + assert!( + input(&routed, "pbs-b").slots()[0].has_trusted_stored_request(), + "empty canonical demand should fan out to same-profile PBS plans" + ); + } + + #[test] + fn malformed_envelopes_are_atomic_and_do_not_trigger_stored_routes() { + let too_many = Value::Object( + (0..=MAX_BIDDER_ENTRIES) + .map(|index| (format!("bidder-{index}"), json!({"placement": index}))) + .collect(), + ); + let cases = vec![ + ("nonobject envelope", json!("bad")), + ("nonobject params", envelope(Some(json!(true)))), + ("invalid key", envelope(Some(json!({" bad": {"x": 1}})))), + ( + "reserved key", + envelope(Some(json!({"trustedServer": {"x": 1}}))), + ), + ("nonobject value", envelope(Some(json!({"alpha": 1})))), + ( + "partial", + envelope(Some(json!({"alpha": {"x": 1}, "beta": null}))), + ), + ( + "unknown field", + json!({"bidderParams": {"alpha": {"x": 1}}, "endpoint": "https://bad.example"}), + ), + ("too many bidders", envelope(Some(too_many))), + ( + "oversized zone", + json!({"bidderParams": {}, "zone": "z".repeat(MAX_PREBID_ZONE_BYTES + 1)}), + ), + ("nonstring zone", json!({"bidderParams": {}, "zone": 1})), + ]; + for (name, malformed) in cases { + let bidders = HashMap::from([ + (TRUSTED_SERVER_ENVELOPE.to_string(), malformed), + ("beta".to_string(), json!({"placement": "direct"})), + ]); + let routed = route_auction(request(vec![slot(bidders)]), &inbound(), &plan(), None); + assert_eq!( + routed.diagnostics().malformed_envelope_count(), + 1, + "{name} should record one malformed envelope" + ); + assert!( + routed.inputs().iter().all(|provider| { + provider.provider_id().as_str() != "pbs-a" + && provider.provider_id().as_str() != "pbs-b" + }), + "{name} should not produce stored or inline PBS demand" + ); + assert_eq!( + input(&routed, "standard-direct").slots()[0] + .bidder_params() + .len(), + 1, + "{name} should preserve independent valid direct demand" + ); + assert!( + routed + .inputs() + .iter() + .any(|provider| provider.provider_id().as_str() == "aps-primary"), + "{name} should preserve independent all-eligible participation" + ); + } + } + + #[test] + fn envelope_preserves_empty_bidder_params_without_rejecting_valid_siblings() { + let normalized = normalize_envelope(&envelope(Some(json!({ + "alpha": {}, + "beta": {"placement": 42} + })))) + .expect("should preserve object-valued bidder params"); + + assert_eq!( + normalized + .bidder_params + .get(&BidderId::from_str("alpha").expect("should parse bidder")), + Some(&json!({})), + "should preserve empty params for profile overrides" + ); + assert_eq!( + normalized + .bidder_params + .get(&BidderId::from_str("beta").expect("should parse bidder")), + Some(&json!({"placement": 42})), + "should preserve valid sibling params" + ); + } + + #[test] + fn exact_envelope_bidder_entry_bound_is_accepted_and_next_entry_is_rejected() { + let accepted = Value::Object( + (0..MAX_BIDDER_ENTRIES) + .map(|index| (format!("bidder-{index}"), json!({"placement": index}))) + .collect(), + ); + let rejected = Value::Object( + (0..=MAX_BIDDER_ENTRIES) + .map(|index| (format!("bidder-{index}"), json!({"placement": index}))) + .collect(), + ); + assert!( + normalize_envelope(&envelope(Some(accepted))).is_some(), + "exactly 128 envelope bidder entries should be admitted" + ); + assert!( + normalize_envelope(&envelope(Some(rejected))).is_none(), + "129 envelope bidder entries should be rejected" + ); + } + + #[test] + fn unknown_bidder_is_counted_without_fallback() { + let routed = route_auction( + request(vec![slot(HashMap::from([( + TRUSTED_SERVER_ENVELOPE.to_string(), + envelope(Some(json!({"unknown": {"placement": 1}}))), + )]))]), + &inbound(), + &plan(), + None, + ); + assert_eq!( + routed.diagnostics().unroutable_bidder_count(), + 1, + "unknown bidder should increment bounded diagnostics" + ); + assert_eq!( + routed.inputs().len(), + 1, + "only all-eligible APS should remain" + ); + assert_eq!( + routed.inputs()[0].provider_id().as_str(), + "aps-primary", + "unknown demand should not cause PBS fallback" + ); + } + + #[test] + fn direct_usable_params_win_collision_and_unusable_direct_does_not_overwrite() { + let cases = [ + ("usable direct", json!({"source": "direct"}), "direct", 0), + ("empty direct", json!({}), "envelope", 1), + ("null direct", Value::Null, "envelope", 1), + ]; + for (name, direct, expected_source, malformed_count) in cases { + let routed = route_auction( + request(vec![slot(HashMap::from([ + ( + TRUSTED_SERVER_ENVELOPE.to_string(), + envelope(Some(json!({"alpha": {"source": "envelope"}}))), + ), + ("alpha".to_string(), direct), + ]))]), + &inbound(), + &plan(), + None, + ); + assert_eq!( + input(&routed, "pbs-a").slots()[0].bidder_params() + [&BidderId::from_str("alpha").expect("should parse bidder")]["source"], + expected_source, + "{name} should follow deterministic collision semantics" + ); + assert_eq!( + routed.diagnostics().malformed_direct_demand_count(), + malformed_count, + "{name} should record only unusable direct demand" + ); + } + } + + #[test] + fn hash_map_insertion_order_does_not_change_routing() { + let entries = [ + ("alpha".to_string(), json!({"a": 1})), + ("unknown".to_string(), json!({"u": 1})), + ( + TRUSTED_SERVER_ENVELOPE.to_string(), + envelope(Some(json!({"alpha": {"a": 0}}))), + ), + ]; + let forward = HashMap::from(entries.clone()); + let reverse = entries.into_iter().rev().collect::>(); + let first = route_auction(request(vec![slot(forward)]), &inbound(), &plan(), None); + let second = route_auction(request(vec![slot(reverse)]), &inbound(), &plan(), None); + let summarize = |routed: &RoutedAuction| { + routed + .inputs() + .iter() + .map(|provider| { + ( + provider.provider_id().as_str().to_string(), + provider.slots()[0] + .bidder_params() + .keys() + .map(|bidder| bidder.as_str().to_string()) + .collect::>(), + ) + }) + .collect::>() + }; + assert_eq!(summarize(&first), summarize(&second)); + assert_eq!(first.diagnostics(), second.diagnostics()); + } + + #[test] + fn mixed_routing_filters_params_per_provider_and_inline_wins_over_stored() { + let routed = route_auction( + request(vec![slot(HashMap::from([ + ( + TRUSTED_SERVER_ENVELOPE.to_string(), + json!({"zone": "home", "bidderParams": null}), + ), + ("alpha".to_string(), json!({"placement": "pbs"})), + ("beta".to_string(), json!({"placement": "direct"})), + ]))]), + &inbound(), + &plan(), + None, + ); + let ids = routed + .inputs() + .iter() + .map(|provider| provider.provider_id().as_str()) + .collect::>(); + assert_eq!( + ids, + vec!["aps-primary", "pbs-a", "pbs-b", "standard-direct"], + "inputs should follow deterministic provider-ID order" + ); + let aps = input(&routed, "aps-primary") + .slots() + .first() + .expect("should have slot"); + assert!( + aps.bidder_params().is_empty(), + "APS must receive no foreign params" + ); + assert_eq!(aps.prebid_zone(), None, "APS must receive no Prebid zone"); + let pbs_a = &input(&routed, "pbs-a").slots()[0]; + assert_eq!(pbs_a.bidder_params().len(), 1); + assert!( + !pbs_a.has_trusted_stored_request(), + "inline params should win for this PBS provider" + ); + assert_eq!(pbs_a.prebid_zone(), Some("home")); + let pbs_b = &input(&routed, "pbs-b").slots()[0]; + assert!(pbs_b.bidder_params().is_empty()); + assert!(pbs_b.has_trusted_stored_request()); + let direct = &input(&routed, "standard-direct").slots()[0]; + assert_eq!(direct.bidder_params().len(), 1); + assert!(direct.prebid_zone().is_none()); + for provider in routed.inputs() { + assert!(provider.common_request().slots.is_empty()); + assert!(provider.slots()[0].slot().bidders.is_empty()); + } + } + + #[test] + fn nonbanner_and_zero_sized_formats_are_removed_and_empty_slots_are_omitted() { + let mut mixed = slot(HashMap::new()); + mixed.formats = vec![ + AdFormat { + media_type: MediaType::Video, + width: 640, + height: 360, + }, + AdFormat { + media_type: MediaType::Banner, + width: 0, + height: 250, + }, + AdFormat { + media_type: MediaType::Banner, + width: u32::MAX, + height: 250, + }, + AdFormat { + media_type: MediaType::Banner, + width: 300, + height: 250, + }, + ]; + let mut invalid = slot(HashMap::new()); + invalid.id = "invalid".to_string(); + invalid.formats = vec![ + AdFormat { + media_type: MediaType::Native, + width: 1, + height: 1, + }, + AdFormat { + media_type: MediaType::Banner, + width: 300, + height: 0, + }, + ]; + let routed = route_auction(request(vec![mixed, invalid]), &inbound(), &plan(), None); + assert_eq!( + routed.inputs().len(), + 3, + "APS and two PBS providers should receive the eligible slot" + ); + for provider in routed.inputs() { + assert_eq!(provider.slots().len(), 1); + assert_eq!(provider.slots()[0].slot().id, "slot-1"); + assert_eq!(provider.slots()[0].slot().formats.len(), 1); + assert_eq!(provider.slots()[0].slot().formats[0].width, 300); + } + let none = route_auction( + request(vec![slot_with_formats(vec![AdFormat { + media_type: MediaType::Video, + width: 640, + height: 360, + }])]), + &inbound(), + &plan(), + None, + ); + assert!( + none.inputs().is_empty(), + "no eligible slots should omit every provider input" + ); + assert_eq!( + none.skipped_no_eligible_provider_ids() + .iter() + .map(ProviderId::as_str) + .collect::>(), + vec!["aps-primary", "pbs-a", "pbs-b", "standard-direct"], + "no-banner auction should retain every provider's deterministic skip outcome" + ); + } + + #[test] + fn explicit_no_demand_retains_deterministic_skip_outcomes() { + let routed = route_auction( + request(vec![slot(HashMap::from([( + "unknown".to_string(), + json!({"placement": "none"}), + )]))]), + &inbound(), + &plan(), + None, + ); + assert_eq!( + routed + .skipped_no_eligible_provider_ids() + .iter() + .map(ProviderId::as_str) + .collect::>(), + vec!["pbs-a", "pbs-b", "standard-direct"], + "explicit providers with no routed demand should be retained as skipped" + ); + } + + #[test] + fn trusted_routes_admit_explicit_aps_and_standard_and_ignore_unknown_provider() { + let trusted_routes = TrustedProviderRoutes::new(vec![vec![ + ProviderId::from_str("aps-primary").expect("should parse provider"), + ProviderId::from_str("standard-direct").expect("should parse provider"), + ProviderId::from_str("unknown-provider").expect("should parse provider"), + ]]); + let routed = route_auction_with_trusted_routes( + request(vec![slot(HashMap::from([( + "unknown".to_string(), + json!({"placement": "none"}), + )]))]), + &inbound(), + &explicit_plan(), + None, + &trusted_routes, + ); + assert_eq!( + routed + .inputs() + .iter() + .map(|input| input.provider_id().as_str()) + .collect::>(), + vec!["aps-primary", "standard-direct"], + "only known server-owned provider routes should admit explicit providers" + ); + assert_eq!( + routed.diagnostics().unroutable_trusted_provider_count(), + 1, + "unknown trusted provider should be ignored and counted" + ); + assert_eq!( + routed + .skipped_no_eligible_provider_ids() + .iter() + .map(ProviderId::as_str) + .collect::>(), + vec!["pbs-a", "pbs-b"], + "unrouted explicit providers should retain skip outcomes" + ); + } + + fn slot_with_formats(formats: Vec) -> AdSlot { + let mut value = slot(HashMap::new()); + value.formats = formats; + value + } + + #[test] + fn snapshots_first_headers_retains_raw_bytes_and_ignores_inbound_xff() { + let mut inbound = inbound(); + inbound + .headers_mut() + .append(header::COOKIE, HeaderValue::from_static("first=1")); + inbound + .headers_mut() + .append(header::COOKIE, HeaderValue::from_static("second=2")); + inbound.headers_mut().append( + header::USER_AGENT, + HeaderValue::from_bytes(b"agent-\x80").expect("should accept raw header"), + ); + inbound.headers_mut().append( + header::REFERER, + HeaderValue::from_static("https://publisher.example.test/article"), + ); + inbound + .headers_mut() + .append(header::ACCEPT_LANGUAGE, HeaderValue::from_static("en-US")); + inbound.headers_mut().append( + HeaderName::from_static("x-forwarded-for"), + HeaderValue::from_static("203.0.113.250"), + ); + inbound.headers_mut().append( + HeaderName::from_static("dnt"), + HeaderValue::from_static(" 1 "), + ); + let attested = IpAddr::from_str("192.0.2.10").expect("should parse IP"); + let routed = route_auction( + request(vec![slot(HashMap::new())]), + &inbound, + &plan(), + Some(attested), + ); + let headers = routed.prebid_transport_headers(); + assert_eq!(headers.cookie(), Some(&HeaderValue::from_static("first=1"))); + assert_eq!( + headers.user_agent().expect("should retain UA").as_bytes(), + b"agent-\x80" + ); + assert_eq!( + headers.referer(), + Some(&HeaderValue::from_static( + "https://publisher.example.test/article" + )) + ); + assert_eq!( + headers.accept_language(), + Some(&HeaderValue::from_static("en-US")) + ); + assert_eq!(routed.attested_client_ip(), Some(attested)); + assert_eq!(routed.dnt(), Some(true)); + for provider in routed.inputs() { + let expected_timeout = match provider.provider_id().as_str() { + "aps-primary" => 800, + id if id.starts_with("pbs-") => 1000, + _ => 900, + }; + assert_eq!(provider.timeout_ms(), expected_timeout); + } + } +} diff --git a/crates/trusted-server-core/src/auction/telemetry.rs b/crates/trusted-server-core/src/auction/telemetry.rs index b3e049eaf..2c7c9cc45 100644 --- a/crates/trusted-server-core/src/auction/telemetry.rs +++ b/crates/trusted-server-core/src/auction/telemetry.rs @@ -110,6 +110,8 @@ pub struct AuctionObservationContext { pub is_mobile: u8, /// `0` = bot, `1` = browser, `2` = unknown. pub is_known_browser: u8, + /// Coarse browser family from the UA, when recognized. + pub browser_family: Option, /// Whether GDPR applies. pub gdpr_applies: bool, /// Whether any consent signal was present. @@ -172,6 +174,7 @@ impl AuctionObservationContext { Some(false) => 0, None => 2, }, + browser_family: device.and_then(|signals| signals.browser_family.clone()), gdpr_applies: consent.gdpr_applies, consent_present: !consent.is_empty(), slot_count, @@ -196,6 +199,7 @@ impl AuctionObservationContext { region: Some("CA".to_owned()), is_mobile: 0, is_known_browser: 1, + browser_family: Some("chrome".to_owned()), gdpr_applies: false, consent_present: false, slot_count, @@ -282,6 +286,8 @@ pub struct AuctionEventRow { pub is_mobile: u8, /// `0` = bot, `1` = browser, `2` = unknown. pub is_known_browser: u8, + /// Coarse browser family from the UA, when recognized. + pub browser_family: Option, /// `0` or `1`. pub gdpr_applies: u8, /// `0` or `1`. @@ -341,6 +347,7 @@ impl AuctionEventRow { region: observation.region.clone(), is_mobile: observation.is_mobile, is_known_browser: observation.is_known_browser, + browser_family: observation.browser_family.clone(), gdpr_applies: u8::from(observation.gdpr_applies), consent_present: u8::from(observation.consent_present), terminal_status: None, @@ -770,7 +777,11 @@ fn bid_row( row.slot_w = Some(u16::try_from(bid.width).unwrap_or(u16::MAX)); row.slot_h = Some(u16::try_from(bid.height).unwrap_or(u16::MAX)); row.media_type = media_type_for_slot(request, &bid.slot_id).map(str::to_owned); - row.seat = Some(bid.bidder.clone()); + row.seat = Some( + bid.returned_seat + .clone() + .unwrap_or_else(|| bid.bidder.clone()), + ); row.price_cpm = price; row.currency = Some(bid.currency.clone()); row.is_win = Some(is_win); @@ -974,6 +985,7 @@ mod tests { creative: None, adomain: Some(vec!["advertiser.example".to_owned()]), bidder: bidder.to_owned(), + returned_seat: None, width: 300, height: 250, nurl: None, @@ -1100,6 +1112,112 @@ mod tests { ); } + #[test] + fn bid_rows_prefer_returned_seat_over_delivery_bidder() { + let request = test_request("ts-ec-derived-id"); + let mut aps_bid = bid("slot-1", "aps", Some("ad-1"), Some(1.25)); + aps_bid.returned_seat = Some("upstream-seat".to_string()); + let provider = AuctionResponse::success("aps-primary", vec![aps_bid.clone()], 12); + let result = OrchestrationResult { + provider_responses: vec![provider], + mediator_response: None, + winning_bids: HashMap::from([("slot-1".to_owned(), aps_bid.clone())]), + total_time_ms: 12, + metadata: HashMap::new(), + }; + let batch = build_auction_events( + AuctionObservationContext::for_test(AuctionSource::AuctionApi, "/auction", 1), + AuctionTerminalOutcome::Completed { + request: &request, + result: &result, + delivered_winner_slots: None, + }, + ); + + let provider_row = batch + .rows() + .iter() + .find(|row| row.event_kind == "provider_call") + .expect("should emit provider row"); + assert_eq!(provider_row.provider.as_deref(), Some("aps-primary")); + let bid_row = batch + .rows() + .iter() + .find(|row| row.event_kind == "bid") + .expect("should emit bid row"); + assert_eq!(bid_row.provider.as_deref(), Some("aps-primary")); + assert_eq!(bid_row.seat.as_deref(), Some("upstream-seat")); + + let mut fallback_bid = aps_bid; + fallback_bid.returned_seat = None; + let fallback = OrchestrationResult { + provider_responses: vec![AuctionResponse::success( + "aps-primary", + vec![fallback_bid.clone()], + 12, + )], + mediator_response: None, + winning_bids: HashMap::from([("slot-1".to_owned(), fallback_bid)]), + total_time_ms: 12, + metadata: HashMap::new(), + }; + let fallback_batch = build_auction_events( + AuctionObservationContext::for_test(AuctionSource::AuctionApi, "/auction", 1), + AuctionTerminalOutcome::Completed { + request: &request, + result: &fallback, + delivered_winner_slots: None, + }, + ); + assert_eq!( + fallback_batch + .rows() + .iter() + .find(|row| row.event_kind == "bid") + .and_then(|row| row.seat.as_deref()), + Some("aps") + ); + } + + #[test] + fn mediated_aps_telemetry_retains_provider_upstream_seat_and_delivery_identity() { + let request = test_request("ts-ec-derived-id"); + let mut aps_bid = bid("slot-1", "aps", Some("ad-1"), Some(1.25)); + aps_bid.returned_seat = Some("upstream-seat".to_string()); + let provider = AuctionResponse::success("aps-primary", vec![aps_bid.clone()], 12); + let mediator = AuctionResponse::success("adserver_mock", vec![aps_bid.clone()], 3); + let result = OrchestrationResult { + provider_responses: vec![provider], + mediator_response: Some(mediator), + winning_bids: HashMap::from([("slot-1".to_owned(), aps_bid.clone())]), + total_time_ms: 15, + metadata: HashMap::new(), + }; + let batch = build_auction_events( + AuctionObservationContext::for_test(AuctionSource::AuctionApi, "/auction", 1), + AuctionTerminalOutcome::Completed { + request: &request, + result: &result, + delivered_winner_slots: None, + }, + ); + + let provider_row = batch + .rows() + .iter() + .find(|row| row.event_kind == "provider_call") + .expect("should emit provider call"); + assert_eq!(provider_row.provider.as_deref(), Some("aps-primary")); + let bid_row = batch + .rows() + .iter() + .find(|row| row.event_kind == "bid") + .expect("should emit provider bid"); + assert_eq!(bid_row.provider.as_deref(), Some("aps-primary")); + assert_eq!(bid_row.seat.as_deref(), Some("upstream-seat")); + assert_eq!(aps_bid.bidder, "aps", "delivery identity remains distinct"); + } + #[test] fn completed_events_do_not_mark_dropped_winners_as_delivered() { let request = test_request("ts-ec-derived-id"); diff --git a/crates/trusted-server-core/src/auction/test_support.rs b/crates/trusted-server-core/src/auction/test_support.rs index e4b953e05..a83f8899a 100644 --- a/crates/trusted-server-core/src/auction/test_support.rs +++ b/crates/trusted-server-core/src/auction/test_support.rs @@ -1,9 +1,17 @@ +use std::collections::HashMap; use std::sync::LazyLock; use edgezero_core::body::Body as EdgeBody; use http::Request; +use serde_json::json; use super::AuctionContext; +use crate::auction::types::{ + AdFormat, AdSlot, AuctionRequest, DeviceInfo, MediaType, PublisherInfo, UserInfo, +}; +use crate::consent::ConsentContext; +use crate::geo::GeoInfo; +use crate::openrtb::{Eid, Uid}; use crate::platform::{RuntimeServices, test_support::noop_services}; use crate::settings::Settings; @@ -19,7 +27,95 @@ pub(crate) fn create_test_auction_context<'a>( settings, request, timeout_ms, + transport_timeout_ms: timeout_ms, provider_responses: None, services, } } + +/// Build canonical request facts shared by the PBS and APS Stage 1 wire goldens. +/// +/// The supported and unsupported formats deliberately exercise each profile's +/// existing filtering and field-ownership policy. `trustedServer` bidder +/// parameters are included to pin that PBS consumes them while APS ignores +/// them. +pub(crate) fn canonical_parity_auction_request() -> AuctionRequest { + AuctionRequest { + id: "fictional-auction".to_string(), + slots: vec![AdSlot { + id: "fictional-slot".to_string(), + formats: vec![ + AdFormat { + media_type: MediaType::Banner, + width: 300, + height: 250, + }, + AdFormat { + media_type: MediaType::Video, + width: 640, + height: 480, + }, + AdFormat { + media_type: MediaType::Banner, + width: u32::MAX, + height: 90, + }, + AdFormat { + media_type: MediaType::Banner, + width: 728, + height: 90, + }, + ], + floor_price: Some(1.0), + targeting: HashMap::new(), + bidders: HashMap::from([( + "trustedServer".to_string(), + json!({ + "bidderParams": { + "exampleBidder": { "placement": "fictional-placement" } + } + }), + )]), + }], + publisher: PublisherInfo { + domain: "publisher.example".to_string(), + page_url: Some("https://publisher.example/article".to_string()), + }, + user: UserInfo { + id: Some("fictional-user".to_string()), + consent: Some(ConsentContext { + gdpr_applies: true, + raw_tc_string: Some("fictional-tcf".to_string()), + raw_us_privacy: Some("1YNN".to_string()), + raw_gpp_string: Some("fictional-gpp".to_string()), + gpp_section_ids: Some(vec![2, 6]), + raw_ac_string: Some("fictional-ac".to_string()), + ..Default::default() + }), + eids: Some(vec![Eid { + source: "identity.example".to_string(), + uids: vec![Uid { + id: "fictional-uid".to_string(), + atype: Some(1), + ext: None, + }], + }]), + }, + device: Some(DeviceInfo { + user_agent: Some("Fictional Browser".to_string()), + ip: Some("192.0.2.10".to_string()), + geo: Some(GeoInfo { + city: "Example City".to_string(), + country: "US".to_string(), + continent: "NA".to_string(), + latitude: 12.34, + longitude: 56.78, + metro_code: 501, + region: Some("CA".to_string()), + asn: None, + }), + }), + site: None, + context: HashMap::new(), + } +} diff --git a/crates/trusted-server-core/src/auction/types.rs b/crates/trusted-server-core/src/auction/types.rs index f61334787..406915706 100644 --- a/crates/trusted-server-core/src/auction/types.rs +++ b/crates/trusted-server-core/src/auction/types.rs @@ -146,7 +146,14 @@ pub struct SiteInfo { pub struct AuctionContext<'a> { pub settings: &'a Settings, pub request: &'a Request, + /// Exact logical provider budget used by auction policy and payloads. pub timeout_ms: u32, + /// Canonical backend transport timeout used for provider registration. + /// + /// This can be lower than `timeout_ms` on runtimes whose backend names + /// encode timers. Providers that register a backend should use this value + /// for transport timers while retaining `timeout_ms` for logical policy. + pub transport_timeout_ms: u32, /// Provider responses from the bidding phase, used by mediators. /// This is `None` for regular bidders and `Some` when calling a mediator. pub provider_responses: Option<&'a [AuctionResponse]>, @@ -243,8 +250,11 @@ pub struct Bid { pub creative: Option, /// Advertiser domain pub adomain: Option>, - /// Bidder/seat identifier + /// Browser-facing delivery bidder code. pub bidder: String, + /// Exact valid upstream `seatbid.seat`, independent of delivery identity. + #[serde(skip)] + pub returned_seat: Option, /// Width of creative pub width: u32, /// Height of creative @@ -409,6 +419,7 @@ mod tests { creative: None, adomain: None, bidder: bidder.to_owned(), + returned_seat: None, width: 300, height: 250, nurl: None, @@ -532,6 +543,20 @@ mod tests { ); } + #[test] + fn returned_seat_is_internal_and_not_serialized() { + let mut bid = make_bid("aps"); + bid.returned_seat = Some("upstream-seat".to_string()); + + let serialized = serde_json::to_value(&bid).expect("should serialize bid"); + assert!( + serialized.get("returned_seat").is_none(), + "returned seat must not change client/debug wire shapes" + ); + let decoded: Bid = serde_json::from_value(serialized).expect("should deserialize bid"); + assert!(decoded.returned_seat.is_none()); + } + #[test] fn bid_with_cache_fields_round_trips_through_json() { let bid = Bid { @@ -541,6 +566,7 @@ mod tests { creative: None, adomain: None, bidder: "thetradedesk".to_string(), + returned_seat: None, width: 300, height: 250, nurl: None, @@ -647,6 +673,7 @@ mod tests { creative: None, adomain: None, bidder: "kargo".to_string(), + returned_seat: None, width: 300, height: 250, nurl: None, diff --git a/crates/trusted-server-core/src/auction_config_types.rs b/crates/trusted-server-core/src/auction_config_types.rs index af52b7ad2..7d918a7bd 100644 --- a/crates/trusted-server-core/src/auction_config_types.rs +++ b/crates/trusted-server-core/src/auction_config_types.rs @@ -1,9 +1,13 @@ //! Auction configuration types (separated to avoid circular deps in build.rs). use serde::{Deserialize, Serialize}; -use std::collections::HashSet; +use std::collections::{BTreeMap, HashSet}; use validator::Validate; +pub use crate::auction::plan::{ + BidderId, BidderRouteConfig, NotificationConfig, ProviderConfig, ProviderId, RoutingMode, +}; + /// Auction orchestration configuration. #[derive(Debug, Clone, Deserialize, Serialize, Validate)] #[serde(deny_unknown_fields)] @@ -41,12 +45,15 @@ pub struct AuctionConfig { )] pub rewrite_creatives: bool, - /// Provider names that participate in bidding - /// Simply list the provider names (e.g., ["prebid", "aps"]) - #[serde(default, deserialize_with = "crate::settings::vec_from_seq_or_map")] - pub providers: Vec, + /// Operator-defined bidder-provider instances, keyed by provider ID. + #[serde(default)] + pub providers: BTreeMap, + + /// Client-visible bidder routes, keyed by bidder code. + #[serde(default)] + pub bidders: BTreeMap, - /// Optional mediator provider name (e.g., "gam") + /// Optional separately registered mediator provider name. /// When set, runs parallel mediation strategy (bidders in parallel, then mediator decides) /// When omitted, runs parallel only strategy (bidders in parallel, highest CPM wins) pub mediator: Option, @@ -74,7 +81,8 @@ impl Default for AuctionConfig { enabled: false, sanitize_creatives: default_sanitize_creatives(), rewrite_creatives: default_rewrite_creatives(), - providers: Vec::new(), + providers: BTreeMap::new(), + bidders: BTreeMap::new(), mediator: None, timeout_ms: default_timeout(), creative_store: default_creative_store(), @@ -117,10 +125,26 @@ fn default_allowed_context_keys() -> HashSet { reason = "methods are used by the runtime crate but not by build.rs path inclusion" )] impl AuctionConfig { - /// Get all provider names. - #[must_use] - pub fn provider_names(&self) -> &[String] { - &self.providers + #[cfg(test)] + pub(crate) fn legacy_provider_map(names: &[&str]) -> BTreeMap { + names + .iter() + .map(|name| { + let id = ProviderId::unchecked_for_legacy_test(name); + ( + id, + ProviderConfig { + protocol: "openrtb-2.6".to_string(), + profile: "standard".to_string(), + endpoint: format!("https://{name}.example/openrtb2/auction"), + timeout_ms: None, + routing: RoutingMode::AllEligible, + notifications: NotificationConfig::default(), + profile_config: serde_json::json!({}), + }, + ) + }) + .collect() } /// Check if this config has a mediator configured. @@ -222,4 +246,37 @@ mod tests { "should preserve an explicit sanitize opt-in" ); } + + #[test] + fn provider_list_shape_is_rejected() { + let error = serde_json::from_value::(serde_json::json!({ + "providers": ["prebid"] + })) + .expect_err("should reject the removed provider-list schema"); + + assert!( + error.to_string().contains("map") || error.to_string().contains("object"), + "should require map-shaped providers: {error}" + ); + } + + #[test] + fn map_schema_round_trips_provider_and_bidder_routes() { + let config: AuctionConfig = serde_json::from_value(serde_json::json!({ + "providers": { + "pbs-main": { + "protocol": "openrtb-2.6", + "profile": "prebid-server", + "endpoint": "https://prebid.example/openrtb2/auction" + } + }, + "bidders": { + "example-bidder": { "provider": "pbs-main" } + } + })) + .expect("should parse map-shaped auction config"); + + assert_eq!(config.providers.len(), 1); + assert_eq!(config.bidders.len(), 1); + } } diff --git a/crates/trusted-server-core/src/config.rs b/crates/trusted-server-core/src/config.rs index f878489d6..fb88448a0 100644 --- a/crates/trusted-server-core/src/config.rs +++ b/crates/trusted-server-core/src/config.rs @@ -7,8 +7,8 @@ //! `EdgeZero`'s typed config push path. use std::borrow::Cow; -use std::collections::HashSet; +use edgezero_core::app_config::{SecretField, SecretKind, SecretPathSegment}; use error_stack::Report; use serde::{Deserialize, Deserializer, Serialize, Serializer}; use validator::{Validate, ValidationError, ValidationErrors}; @@ -22,7 +22,7 @@ use crate::integrations::{ osano::OsanoConfig, permutive::PermutiveConfig, prebid, sourcepoint::SourcepointConfig, testlight::TestlightConfig, }; -use crate::settings::{IntegrationConfig, Settings}; +use crate::settings::{AssetOriginAuth, IntegrationConfig, Settings}; const DEPLOY_VALIDATION_FIELD: &str = "trusted_server"; #[cfg(test)] @@ -54,15 +54,20 @@ pub struct TrustedServerAppConfig { } impl TrustedServerAppConfig { - /// Creates a validated app-config wrapper from [`Settings`]. + /// Creates a push-valid app-config wrapper from [`Settings`]. /// /// # Errors /// - /// Returns [`TrustedServerError::Configuration`] when deploy validation + /// Returns [`TrustedServerError::Configuration`] when push-safe validation /// fails. pub fn new(settings: Settings) -> Result> { - validate_settings_for_deploy(&settings)?; - Ok(Self { settings }) + let app_config = Self { settings }; + edgezero_core::app_config::validate_excluding_secrets(&app_config).map_err(|errors| { + Report::new(TrustedServerError::Configuration { + message: format!("Configuration validation failed: {errors}"), + }) + })?; + Ok(app_config) } /// Consumes the wrapper and returns the inner [`Settings`]. @@ -92,61 +97,174 @@ impl<'de> Deserialize<'de> for TrustedServerAppConfig { where D: Deserializer<'de>, { - let settings = Settings::deserialize(deserializer)?; - let settings = Settings::finalize_deserialized(settings, "Configuration") - .map_err(serde::de::Error::custom)?; + let mut settings = Settings::deserialize(deserializer)?; + settings.normalize_deserialized(); Ok(Self { settings }) } } impl Validate for TrustedServerAppConfig { fn validate(&self) -> Result<(), ValidationErrors> { - validate_settings_for_deploy(&self.settings) - .map_err(|report| report_to_validation_errors(&report)) + let mut errors = self.settings.validate().err().unwrap_or_default(); + if let Err(report) = validate_settings_for_deploy(&self.settings) { + errors.add( + DEPLOY_VALIDATION_FIELD, + report_to_validation_error(&report, "trusted_server_deploy_validation"), + ); + } + if errors.errors().is_empty() { + Ok(()) + } else { + Err(errors) + } } } impl edgezero_core::app_config::AppConfigMeta for TrustedServerAppConfig { - // Phase 1 intentionally preserves the existing inline-settings model: - // `ts config push` publishes the validated Trusted Server config as one - // app-config blob. Migrating app-level secrets to `EdgeZero` secret-store - // references needs nested/array extraction support and operator migration - // work tracked separately. - const SECRET_FIELDS: &'static [edgezero_core::app_config::SecretField] = &[]; + fn secret_fields() -> Vec { + let field = |path: Vec, optional| SecretField { + kind: SecretKind::KeyInDefault, + optional, + path, + }; + let object = |name: &'static str| SecretPathSegment::Field(Cow::Borrowed(name)); + let optional_object = + |name: &'static str| SecretPathSegment::OptionalField(Cow::Borrowed(name)); + + vec![ + field(vec![object("publisher"), object("proxy_secret")], false), + field(vec![object("ec"), object("passphrase")], false), + field( + vec![ + object("ec"), + optional_object("partners"), + SecretPathSegment::ArrayEach, + object("api_token"), + ], + true, + ), + field( + vec![ + object("ec"), + optional_object("partners"), + SecretPathSegment::ArrayEach, + object("ts_pull_token"), + ], + true, + ), + field( + vec![ + object("handlers"), + SecretPathSegment::ArrayEach, + object("password"), + ], + false, + ), + field( + vec![optional_object("tinybird"), object("auction_token_secret")], + true, + ), + field( + vec![ + optional_object("integrations"), + optional_object("datadome"), + object("server_side_key_secret_name"), + ], + true, + ), + field( + vec![ + optional_object("integrations"), + optional_object("datadome"), + optional_object("protection_test_bypass"), + object("credential_secret_name"), + ], + true, + ), + field( + vec![ + optional_object("proxy"), + optional_object("asset_routes"), + SecretPathSegment::ArrayEach, + optional_object("auth"), + object("access_key_id"), + ], + true, + ), + field( + vec![ + optional_object("proxy"), + optional_object("asset_routes"), + SecretPathSegment::ArrayEach, + optional_object("auth"), + object("secret_access_key"), + ], + true, + ), + field( + vec![ + optional_object("proxy"), + optional_object("asset_routes"), + SecretPathSegment::ArrayEach, + optional_object("auth"), + object("session_token"), + ], + true, + ), + ] + } } -/// Runs Trusted Server deploy-time validation for pushed app config. +/// Runs Trusted Server push-time validation for app config. /// -/// This supplements [`Settings`] structural validation with checks that should -/// fail before an operator publishes a config blob: placeholder secrets, -/// enabled integration startup checks, auction provider references, and EC -/// partner registry construction. +/// Secret fields contain secret-store key names at this stage, so this function +/// deliberately excludes checks that require resolved values. The `EdgeZero` CLI +/// additionally calls [`edgezero_core::app_config::validate_excluding_secrets`] +/// to remove validators attached to those leaves. /// /// # Errors /// -/// Returns [`TrustedServerError`] when the config should not be deployed. +/// Returns [`TrustedServerError`] when non-secret configuration or a secret key +/// reference is invalid. pub fn validate_settings_for_deploy(settings: &Settings) -> Result<(), Report> { settings.reject_placeholder_secrets()?; - let enabled_auction_providers = validate_enabled_integrations(settings)?; - validate_auction_provider_names(settings, &enabled_auction_providers)?; + validate_secret_key_references(settings)?; + + let mut structural_settings = settings.clone(); + structural_settings.prepare_runtime()?; + structural_settings.validate_admin_coverage()?; + + let plan = crate::auction::compile_auction_plan(settings)?; + validate_enabled_integrations(settings, &plan, false)?; + PartnerRegistry::validate_config_for_deploy(&settings.ec.partners)?; + Ok(()) +} + +/// Runs Trusted Server runtime validation after secret references are resolved. +/// +/// # Errors +/// +/// Returns [`TrustedServerError`] when resolved secrets or runtime-only +/// configuration checks are invalid. +pub fn validate_settings_for_runtime( + settings: &Settings, +) -> Result<(), Report> { + settings.reject_placeholder_secrets()?; + settings.validate_admin_handler_passwords()?; + let plan = crate::auction::compile_auction_plan(settings)?; + validate_enabled_integrations(settings, &plan, true)?; PartnerRegistry::from_config(&settings.ec.partners).map(|_| ())?; Ok(()) } fn validate_enabled_integrations( settings: &Settings, -) -> Result, Report> { - let mut enabled_auction_providers = HashSet::new(); - - if validate_prebid(settings)? { - enabled_auction_providers.insert("prebid"); - } - if validate_integration::(settings, "aps")? { - enabled_auction_providers.insert("aps"); - } - if validate_integration::(settings, "adserver_mock")? { - enabled_auction_providers.insert("adserver_mock"); - } + plan: &crate::auction::AuctionPlan, + resolved_secrets: bool, +) -> Result<(), Report> { + validate_prebid(settings, plan)?; + validate_integration::(settings, "aps")?; + validate_integration::(settings, "adserver_mock")?; validate_integration::(settings, "testlight")?; validate_integration::(settings, "nextjs")?; validate_integration::(settings, "permutive")?; @@ -156,16 +274,30 @@ fn validate_enabled_integrations( validate_integration::(settings, "osano")?; validate_integration::(settings, "google_tag_manager")?; if let Some(config) = settings.integration_config::("datadome")? { - crate::integrations::datadome::DataDomeIntegration::validate_config_for_startup(config)?; + if resolved_secrets { + crate::integrations::datadome::DataDomeIntegration::validate_config_for_startup( + config, + )?; + } else { + crate::integrations::datadome::DataDomeIntegration::validate_config_for_deploy(config)?; + } } validate_integration::(settings, "gpt")?; validate_integration::(settings, "gpt_diagnostics")?; - Ok(enabled_auction_providers) + Ok(()) } -fn validate_prebid(settings: &Settings) -> Result> { - prebid::validate_config_for_startup(settings).map(|config| config.is_some()) +fn validate_prebid( + settings: &Settings, + plan: &crate::auction::AuctionPlan, +) -> Result<(), Report> { + let Some(config) = settings.integration_config::("prebid")? + else { + return Ok(()); + }; + prebid::validate_browser_config_for_startup(&config, &settings.proxy.allowed_domains)?; + prebid::validate_browser_bidder_ownership(&config, plan) } fn validate_integration( @@ -180,45 +312,133 @@ where .map(|config| config.is_some()) } -fn validate_auction_provider_names( - settings: &Settings, - enabled_auction_providers: &HashSet<&'static str>, -) -> Result<(), Report> { - if !settings.auction.enabled { - return Ok(()); +fn validate_secret_key_references(settings: &Settings) -> Result<(), Report> { + validate_secret_key_reference( + "publisher.proxy_secret", + settings.publisher.proxy_secret.expose(), + )?; + validate_secret_key_reference("ec.passphrase", settings.ec.passphrase.expose())?; + + for (index, partner) in settings.ec.partners.iter().enumerate() { + if let Some(token) = &partner.api_token { + validate_secret_key_reference( + &format!("ec.partners[{index}].api_token"), + token.expose(), + )?; + } + if let Some(token) = &partner.ts_pull_token { + validate_secret_key_reference( + &format!("ec.partners[{index}].ts_pull_token"), + token.expose(), + )?; + } } - for provider_name in settings - .auction - .providers - .iter() - .chain(settings.auction.mediator.iter()) - { - if !enabled_auction_providers.contains(provider_name.as_str()) { - return Err(Report::new(TrustedServerError::Configuration { - message: format!( - "auction provider `{provider_name}` is listed in [auction] but no enabled integration provides it" - ), - })); + for (index, handler) in settings.handlers.iter().enumerate() { + validate_secret_key_reference( + &format!("handlers[{index}].password"), + handler.password.expose(), + )?; + } + + if settings.tinybird.enabled { + let token = settings + .tinybird + .auction_token_secret + .as_ref() + .ok_or_else(|| missing_secret_key_reference("tinybird.auction_token_secret"))?; + validate_secret_key_reference("tinybird.auction_token_secret", token.expose())?; + } + + if let Some(datadome) = settings.integration_config::("datadome")? { + if datadome.enable_protection { + let key = datadome + .server_side_key_secret_name + .as_ref() + .ok_or_else(|| { + missing_secret_key_reference( + "integrations.datadome.server_side_key_secret_name", + ) + })?; + validate_secret_key_reference( + "integrations.datadome.server_side_key_secret_name", + key.expose(), + )?; + } + if let Some(bypass) = datadome + .protection_test_bypass + .as_ref() + .filter(|bypass| bypass.enabled) + { + let credential = bypass.credential_secret_name.as_ref().ok_or_else(|| { + missing_secret_key_reference( + "integrations.datadome.protection_test_bypass.credential_secret_name", + ) + })?; + validate_secret_key_reference( + "integrations.datadome.protection_test_bypass.credential_secret_name", + credential.expose(), + )?; + } + } + + for (index, route) in settings.proxy.asset_routes.iter().enumerate() { + let Some(AssetOriginAuth::S3SigV4(auth)) = route.auth.as_ref() else { + continue; + }; + validate_secret_key_reference( + &format!("proxy.asset_routes[{index}].auth.access_key_id"), + auth.access_key_id.expose(), + )?; + validate_secret_key_reference( + &format!("proxy.asset_routes[{index}].auth.secret_access_key"), + auth.secret_access_key.expose(), + )?; + if let Some(token) = &auth.session_token { + validate_secret_key_reference( + &format!("proxy.asset_routes[{index}].auth.session_token"), + token.expose(), + )?; } } Ok(()) } -fn report_to_validation_errors(report: &Report) -> ValidationErrors { - let mut error = ValidationError::new("trusted_server_deploy_validation"); - error.message = Some(Cow::Owned(report.to_string())); +fn validate_secret_key_reference( + path: &str, + key_name: &str, +) -> Result<(), Report> { + if key_name.is_empty() { + return Err(missing_secret_key_reference(path)); + } + Ok(()) +} - let mut errors = ValidationErrors::new(); - errors.add(DEPLOY_VALIDATION_FIELD, error); - errors +fn missing_secret_key_reference(path: &str) -> Report { + Report::new(TrustedServerError::Configuration { + message: format!("secret key reference at `{path}` must not be empty"), + }) +} + +fn report_to_validation_error( + report: &Report, + code: &'static str, +) -> ValidationError { + let mut error = ValidationError::new(code); + error.message = Some(Cow::Owned(report.to_string())); + error } #[cfg(test)] mod tests { + use std::collections::HashSet; + use super::*; + use crate::redacted::Redacted; + use crate::settings::{ProxyAssetRoute, S3SigV4AuthConfig}; use crate::test_support::tests::crate_test_settings_str; + use edgezero_core::app_config::AppConfigMeta; #[derive(Debug, Deserialize)] #[serde(deny_unknown_fields)] @@ -233,7 +453,9 @@ mod tests { slot: Vec, } - fn serialized_creative_opportunities(gam_unit_path: Option<&str>) -> serde_json::Value { + fn app_config_with_creative_opportunities( + gam_unit_path: Option<&str>, + ) -> TrustedServerAppConfig { let mut toml = crate_test_settings_str(); toml.push_str( r#" @@ -251,9 +473,15 @@ formats = [{ width = 300, height = 250 }] toml.push_str(&format!("gam_unit_path = {gam_unit_path:?}\n")); } - let app_config: TrustedServerAppConfig = + let mut app_config: TrustedServerAppConfig = toml::from_str(&toml).expect("should deserialize app config wrapper"); - serde_json::to_value(app_config) + app_config.settings.proxy.allowed_domains = + vec!["*.example".to_owned(), "*.example.com".to_owned()]; + app_config + } + + fn serialized_creative_opportunities(gam_unit_path: Option<&str>) -> serde_json::Value { + serde_json::to_value(app_config_with_creative_opportunities(gam_unit_path)) .expect("should serialize app config wrapper") .get("creative_opportunities") .cloned() @@ -273,14 +501,20 @@ formats = [{ width = 300, height = 250 }] "/../../trusted-server.example.toml" )); - /// Returns the template with its deliberately-invalid placeholder admin - /// password swapped for a valid one, so parse-time validation succeeds and - /// the test can exercise the optional blocks it uncomments. + /// Returns the template with its secrets-store key references swapped for + /// literal strong values, so parse-time validation of resolved-secret + /// strength succeeds and the test can exercise the optional blocks it + /// uncomments. fn template_with_valid_admin_password() -> String { - EXAMPLE_TEMPLATE.replace( - "password = \"replace-with-admin-password-32-bytes\"", - "password = \"unit-test-admin-password-that-is-long-enough\"", - ) + EXAMPLE_TEMPLATE + .replace( + "password = \"handler_password\"", + "password = \"unit-test-admin-password-that-is-long-enough\"", + ) + .replace( + "passphrase = \"ec_passphrase\"", + "passphrase = \"unit-test-ec-passphrase-that-is-long-enough\"", + ) } /// Uncomments the contiguous `#`-prefixed block that begins at the line @@ -399,20 +633,184 @@ formats = [{ width = 300, height = 250 }] } #[test] - fn dynamic_gam_unit_templates_are_rejected_by_legacy_schema() { - for gam_unit_path in ["/{network_id}/example", "/example/{slot_id}"] { - let creative_opportunities = serialized_creative_opportunities(Some(gam_unit_path)); - let err = - serde_json::from_value::(creative_opportunities) - .expect_err("should reject dynamic GAM unit template"); + fn push_validation_accepts_secret_key_names() { + let mut settings = valid_settings(); + settings.publisher.proxy_secret = Redacted::new("publisher_proxy".to_owned()); + settings.ec.passphrase = Redacted::new("ec_key".to_owned()); + settings.handlers[0].password = Redacted::new("handler_password".to_owned()); + settings.handlers[1].password = Redacted::new("admin_password".to_owned()); + let app_config = TrustedServerAppConfig::new(settings) + .expect("should validate key names without values"); + + let serialized = + serde_json::to_string(&app_config).expect("should serialize key-name-only app config"); + assert!(serialized.contains("publisher_proxy")); + assert!(!serialized.contains("unit-test-proxy-secret")); + } + + #[test] + fn secret_metadata_lists_all_secret_paths_and_optionality() { + let fields = TrustedServerAppConfig::secret_fields(); + let paths = fields + .iter() + .map(|field| (field.dotted_path(), field.optional)) + .collect::>(); + + assert_eq!( + paths, + vec![ + ("publisher.proxy_secret".to_owned(), false), + ("ec.passphrase".to_owned(), false), + ("ec.partners[*].api_token".to_owned(), true), + ("ec.partners[*].ts_pull_token".to_owned(), true), + ("handlers[*].password".to_owned(), false), + ("tinybird.auction_token_secret".to_owned(), true), + ( + "integrations.datadome.server_side_key_secret_name".to_owned(), + true, + ), + ( + "integrations.datadome.protection_test_bypass.credential_secret_name" + .to_owned(), + true, + ), + ("proxy.asset_routes[*].auth.access_key_id".to_owned(), true), + ( + "proxy.asset_routes[*].auth.secret_access_key".to_owned(), + true, + ), + ("proxy.asset_routes[*].auth.session_token".to_owned(), true), + ], + "should expose the native EdgeZero secret metadata contract" + ); + assert!( + fields.iter().all(|field| matches!( + field.kind, + edgezero_core::app_config::SecretKind::KeyInDefault + )), + "all Trusted Server app secrets should use the default secret store" + ); + } + #[test] + fn partner_secret_metadata_makes_the_defaulted_array_optional() { + let fields = TrustedServerAppConfig::secret_fields(); + + for field in fields.iter().filter(|field| { + matches!( + field.dotted_path().as_str(), + "ec.partners[*].api_token" | "ec.partners[*].ts_pull_token" + ) + }) { + assert!(matches!( + &field.path[1], + SecretPathSegment::OptionalField(name) if name == "partners" + )); + } + } + + #[test] + fn omitted_s3_secret_references_materialize_as_defaults() { + let auth: S3SigV4AuthConfig = + toml::from_str("region = \"us-east-1\"").expect("should apply S3 secret defaults"); + + assert_eq!(auth.access_key_id.expose(), "access_key_id"); + assert_eq!(auth.secret_access_key.expose(), "secret_access_key"); + + let serialized = serde_json::to_value(auth).expect("should serialize S3 auth"); + assert_eq!(serialized["access_key_id"], "access_key_id"); + assert_eq!(serialized["secret_access_key"], "secret_access_key"); + } + + #[test] + fn legacy_static_secret_store_selectors_are_accepted_but_not_serialized() { + let mut settings = valid_settings(); + settings.tinybird.secret_store = Some("legacy-tinybird-store".to_string()); + settings + .integrations + .insert_config( + "datadome", + &serde_json::json!({ + "enabled": true, + "server_side_key_secret_store": "legacy-datadome-store", + "protection_test_bypass": { + "enabled": false, + "credential_secret_store": "legacy-bypass-store", + }, + }), + ) + .expect("should insert legacy DataDome selectors"); + let mut route = ProxyAssetRoute::new( + "/assets/", + "https://examplebucket.s3.us-east-1.amazonaws.com", + ); + route.auth = Some(AssetOriginAuth::S3SigV4(S3SigV4AuthConfig { + region: "us-east-1".to_string(), + secret_store: Some("legacy-s3-store".to_string()), + access_key_id: Redacted::new("s3-access-key".to_string()), + secret_access_key: Redacted::new("s3-secret-key".to_string()), + session_token: None, + origin_query: None, + })); + settings.proxy.asset_routes.push(route); + + settings.normalize_deserialized(); + let serialized = serde_json::to_string(&settings).expect("should serialize settings"); + + for legacy_store in [ + "legacy-tinybird-store", + "legacy-datadome-store", + "legacy-bypass-store", + "legacy-s3-store", + ] { assert!( - err.to_string().contains("section_segment"), - "legacy error should name section_segment: {err}" + !serialized.contains(legacy_store), + "serialized config should omit deprecated selector {legacy_store}" ); } } + #[test] + fn settings_debug_redacts_resolved_static_credentials() { + let mut settings = valid_settings(); + settings.tinybird.auction_token_secret = + Some(Redacted::new("resolved-tinybird-secret".to_string())); + settings + .integrations + .insert_config( + "datadome", + &serde_json::json!({ + "enabled": true, + "server_side_key_secret_name": "resolved-datadome-secret", + }), + ) + .expect("should insert resolved DataDome config"); + + let debug = format!("{settings:?}"); + + assert!(!debug.contains("resolved-tinybird-secret")); + assert!(!debug.contains("resolved-datadome-secret")); + assert!(debug.contains("datadome")); + } + + #[test] + fn app_config_deserialization_does_not_finalize_runtime_templates() { + let creative_opportunities = + serialized_creative_opportunities(Some("/{network_id}/example")); + let slot = creative_opportunities["slot"][0] + .as_object() + .expect("should serialize creative opportunity slot"); + + assert!( + slot.contains_key("gam_unit_path"), + "push deserialization should preserve the operator config field" + ); + assert!( + !slot.contains_key("section_segment"), + "push deserialization should not add runtime-only compiled fields" + ); + } + #[test] fn static_gam_unit_template_is_accepted_by_legacy_schema() { let creative_opportunities = serialized_creative_opportunities(Some("/99999/example/home")); @@ -457,7 +855,35 @@ gam_network_id = "99999" } #[test] - fn deploy_validation_rejects_placeholders() { + fn app_config_new_rejects_empty_secret_key_reference() { + let mut settings = valid_settings(); + settings.publisher.proxy_secret = Redacted::new(String::new()); + + let err = TrustedServerAppConfig::new(settings) + .expect_err("should reject an empty secret key reference"); + + assert!( + err.to_string().contains("publisher.proxy_secret"), + "error should identify the empty secret reference: {err:?}" + ); + } + + #[test] + fn app_config_new_rejects_invalid_non_secret_settings() { + let mut settings = valid_settings(); + settings.publisher.domain = "invalid/domain".to_owned(); + + let err = TrustedServerAppConfig::new(settings) + .expect_err("should reject invalid publisher domain before creating an app config"); + + assert!( + err.to_string().contains("invalid_publisher_domain"), + "error should identify the structural validation failure: {err:?}" + ); + } + + #[test] + fn runtime_validation_rejects_placeholders() { let settings = Settings::from_toml( r#" [publisher] @@ -475,10 +901,10 @@ username = "admin" password = "production-admin-password-32-bytes" "#, ) - .expect("should parse placeholder settings before deploy validation"); + .expect("should parse placeholder settings before runtime validation"); - let err = - validate_settings_for_deploy(&settings).expect_err("should reject placeholder secrets"); + let err = validate_settings_for_runtime(&settings) + .expect_err("should reject placeholder secrets at runtime"); assert!( err.to_string().contains("Insecure default"), @@ -569,55 +995,6 @@ password = "production-admin-password-32-bytes" ); } - #[test] - fn deploy_validation_rejects_blank_aps_account_id() { - // `deserialize_account_id` trims then rejects an empty result, so blank - // and whitespace-only ids fail at parse time. - for (label, account_id) in [("empty", ""), ("whitespace-only", " ")] { - let mut settings = valid_settings(); - settings - .integrations - .insert_config( - "aps", - &serde_json::json!({ - "enabled": true, - "account_id": account_id, - "endpoint": "https://aps.example.com/e/pb/bid" - }), - ) - .expect("should insert APS config"); - - let err = validate_settings_for_deploy(&settings) - .expect_err("should reject blank APS account_id when enabled"); - - assert!( - format!("{err:?}").contains("aps"), - "should mention the APS integration for {label} account_id: {err:?}" - ); - } - } - - #[test] - fn deploy_validation_normalizes_padded_aps_account_id() { - // Surrounding whitespace is normalized (trimmed) at deserialization, so - // a padded-but-otherwise-valid id deploys and reaches APS trimmed. - let mut settings = valid_settings(); - settings - .integrations - .insert_config( - "aps", - &serde_json::json!({ - "enabled": true, - "account_id": " example-account ", - "endpoint": "https://aps.example.com/e/pb/bid" - }), - ) - .expect("should insert APS config"); - - validate_settings_for_deploy(&settings) - .expect("should accept a padded-but-valid APS account_id (trimmed at deserialization)"); - } - #[test] fn deploy_validation_rejects_padded_request_signing_store_ids() { let mut settings = valid_settings(); @@ -638,22 +1015,13 @@ password = "production-admin-password-32-bytes" ); } - /// `enabled` defaults to `false` for APS, so a section that omits the flag - /// resolves to disabled and must not have its fields validated — otherwise - /// the documented template placeholder breaks existing configs on upgrade. + /// `enabled` defaults to `false` for `adserver_mock`, so a section that omits + /// the flag resolves to disabled and must not have its fields validated — + /// otherwise a documented template placeholder breaks existing configs on + /// upgrade. #[test] fn deploy_validation_skips_field_validation_for_integrations_with_omitted_enabled() { let mut settings = valid_settings(); - settings - .integrations - .insert_config( - "aps", - &serde_json::json!({ - "pub_id": "your-aps-publisher-id", - "endpoint": "https://aps.example.com/e/dtb/bid" - }), - ) - .expect("should insert APS config"); // `endpoint` parses as a plain string but would fail the `url` // validator, so this section only survives if validation is skipped for // integrations that resolve to disabled. @@ -684,6 +1052,93 @@ password = "production-admin-password-32-bytes" ); } + #[test] + fn deploy_validation_requires_external_bundle_url_for_enabled_prebid() { + let mut settings = valid_settings(); + settings + .integrations + .insert_config( + "prebid", + &serde_json::json!({ + "enabled": true, + "bundle": { "adapters": ["exampleBidder"] } + }), + ) + .expect("should insert enabled Prebid config"); + + let error = validate_settings_for_deploy(&settings) + .expect_err("should require enabled Prebid external bundle URL"); + assert!(error.to_string().contains("external_bundle_url")); + } + + #[test] + fn deploy_validation_rejects_conflicting_prebid_browser_bidder_ownership() { + let mut settings = valid_settings(); + settings.auction.enabled = true; + settings.auction.providers = crate::auction::AuctionConfig::legacy_provider_map(&["pbs"]); + settings.auction.bidders.insert( + "exampleBidder" + .parse() + .expect("should parse server-side bidder"), + crate::auction::BidderRouteConfig { + provider: "pbs".parse().expect("should parse provider"), + }, + ); + let mut prebid = settings + .integration_config::("prebid") + .expect("should parse Prebid config") + .expect("should have enabled Prebid config"); + prebid.client_side_bidders = vec!["exampleBidder".to_string()]; + settings + .integrations + .insert_config("prebid", &prebid) + .expect("should replace Prebid config"); + + let error = validate_settings_for_deploy(&settings) + .expect_err("should reject conflicting browser bidder ownership"); + assert!(error.to_string().contains("exampleBidder")); + assert!( + error + .to_string() + .contains("both client-side and server-side") + ); + } + + #[test] + fn deploy_validation_accepts_dormant_prebid_browser_bidder_overlap() { + let mut settings = valid_settings(); + settings.auction.enabled = false; + settings.auction.providers = crate::auction::AuctionConfig::legacy_provider_map(&["pbs"]); + settings.auction.bidders.insert( + "exampleBidder" + .parse() + .expect("should parse server-side bidder"), + crate::auction::BidderRouteConfig { + provider: "pbs".parse().expect("should parse provider"), + }, + ); + let mut prebid = settings + .integration_config::("prebid") + .expect("should parse Prebid config") + .expect("should have enabled Prebid config"); + prebid.client_side_bidders = vec!["exampleBidder".to_string()]; + settings + .integrations + .insert_config("prebid", &prebid) + .expect("should replace Prebid config"); + + validate_settings_for_deploy(&settings) + .expect("disabled plan overlap should remain deployable"); + } + + #[test] + fn deploy_validation_accepts_typed_prebid_bundle_build_table() { + let settings = valid_settings(); + + validate_settings_for_deploy(&settings) + .expect("test config with typed Prebid bundle build table should validate"); + } + #[test] fn deploy_validation_covers_registered_integration_builders() { let validated_ids: HashSet<&'static str> = @@ -721,15 +1176,9 @@ password = "production-admin-password-32-bytes" #[test] fn deploy_validation_rejects_invalid_datadome_test_bypass() { - for (enable_protection, store, name, expected_message) in [ - ( - false, - "ts_secrets", - "datadome_test_bypass", - "requires enable_protection", - ), - (true, "", "datadome_test_bypass", "credential_secret_store"), - (true, "ts_secrets", "", "credential_secret_name"), + for (enable_protection, name, expected_message) in [ + (false, "datadome_test_bypass", "requires enable_protection"), + (true, "", "credential_secret_name"), ] { let mut settings = valid_settings(); settings @@ -739,9 +1188,9 @@ password = "production-admin-password-32-bytes" &serde_json::json!({ "enabled": true, "enable_protection": enable_protection, + "server_side_key_secret_name": "datadome_server_side_key", "protection_test_bypass": { "enabled": true, - "credential_secret_store": store, "credential_secret_name": name, }, }), @@ -761,7 +1210,15 @@ password = "production-admin-password-32-bytes" fn validate_trait_reports_deploy_errors() { let mut settings = valid_settings(); settings.auction.enabled = true; - settings.auction.providers = vec!["missing-provider".to_string()]; + settings.auction.providers = + crate::auction::AuctionConfig::legacy_provider_map(&["missing-provider"]); + settings + .auction + .providers + .values_mut() + .next() + .expect("should have provider") + .protocol = "unsupported".to_string(); let app_config = TrustedServerAppConfig { settings }; let err = app_config diff --git a/crates/trusted-server-core/src/config_payload.rs b/crates/trusted-server-core/src/config_payload.rs index 6ede36e9c..c5afa28d9 100644 --- a/crates/trusted-server-core/src/config_payload.rs +++ b/crates/trusted-server-core/src/config_payload.rs @@ -8,20 +8,32 @@ use edgezero_core::blob_envelope::BlobEnvelope; use error_stack::Report; +use crate::config::TrustedServerAppConfig; use crate::error::TrustedServerError; +use crate::platform::{PlatformSecretStore, StoreName}; +use crate::secret_resolution::resolve_secret_references; use crate::settings::Settings; +/// Canonical logical secret store used by Trusted Server app-config secrets. +pub const DEFAULT_SECRET_STORE_ID: &str = "trusted_server_secrets"; + /// Default config-store key containing the Trusted Server app-config blob. pub const CONFIG_BLOB_KEY: &str = "trusted_server_config"; -/// Reconstruct validated [`Settings`] from a serialized config blob envelope. +/// Reconstruct runtime [`Settings`] from a serialized config blob envelope. +/// +/// Secret references are resolved after envelope verification and before +/// deserialization. The envelope data itself is never mutated or rewritten. /// /// # Errors /// /// Returns [`TrustedServerError::Configuration`] when the envelope cannot be -/// parsed, fails integrity verification, or contains invalid settings data. +/// parsed, fails integrity verification, secret resolution fails, or resolved +/// settings are invalid. pub fn settings_from_config_blob( envelope_json: &str, + secret_store: &dyn PlatformSecretStore, + default_secret_store_name: &StoreName, ) -> Result> { let envelope: BlobEnvelope = serde_json::from_str(envelope_json).map_err(|error| { Report::new(TrustedServerError::Configuration { @@ -36,40 +48,184 @@ pub fn settings_from_config_blob( .attach(error.to_string()) })?; - let settings = Settings::from_json_value(envelope.into_data())?; - settings.reject_placeholder_secrets()?; + let mut data = envelope.into_data(); + remove_inactive_secret_references(&mut data); + resolve_secret_references::( + &mut data, + secret_store, + default_secret_store_name, + )?; + let settings = Settings::from_json_value(data)?; + crate::config::validate_settings_for_runtime(&settings)?; Ok(settings) } +fn remove_inactive_secret_references(data: &mut serde_json::Value) { + if data + .pointer("/tinybird/enabled") + .and_then(serde_json::Value::as_bool) + != Some(true) + && let Some(tinybird) = data + .get_mut("tinybird") + .and_then(serde_json::Value::as_object_mut) + { + tinybird.remove("auction_token_secret"); + tinybird.remove("access_token_secret"); + } + + if let Some(partners) = data + .pointer_mut("/ec/partners") + .and_then(serde_json::Value::as_array_mut) + { + for partner in partners { + let Some(partner) = partner.as_object_mut() else { + continue; + }; + if partner + .get("pull_sync_enabled") + .and_then(serde_json::Value::as_bool) + != Some(true) + { + partner.remove("ts_pull_token"); + } + } + } + + let Some(datadome) = data + .pointer_mut("/integrations/datadome") + .and_then(serde_json::Value::as_object_mut) + else { + return; + }; + let integration_enabled = + datadome.get("enabled").and_then(serde_json::Value::as_bool) == Some(true); + let protection_enabled = integration_enabled + && datadome + .get("enable_protection") + .and_then(serde_json::Value::as_bool) + == Some(true); + if !protection_enabled { + datadome.remove("server_side_key_secret_name"); + } + + let bypass_enabled = protection_enabled + && datadome + .get("protection_test_bypass") + .and_then(serde_json::Value::as_object) + .and_then(|bypass| bypass.get("enabled")) + .and_then(serde_json::Value::as_bool) + == Some(true); + if !bypass_enabled + && let Some(bypass) = datadome + .get_mut("protection_test_bypass") + .and_then(serde_json::Value::as_object_mut) + { + bypass.remove("credential_secret_name"); + } +} + #[cfg(test)] mod tests { + use std::sync::Arc; + use super::*; + use crate::integrations::IntegrationRegistry; + use crate::platform::{PlatformError, StoreId}; use crate::redacted::Redacted; + use crate::settings::{AssetOriginAuth, EcPartner, ProxyAssetRoute, S3SigV4AuthConfig}; use crate::test_support::tests::crate_test_settings_str; - use serde::Deserialize; - - // Intentionally mirrors `AuctionConfig` before `rewrite_creatives` existed. - // Do not add fields introduced after that snapshot: this test proves a - // default payload remains readable by the previous binary schema. - #[derive(Deserialize)] - #[serde(deny_unknown_fields)] - struct LegacyAuctionConfig { - #[serde(rename = "enabled")] - _enabled: bool, - #[serde(rename = "providers")] - _providers: Vec, - #[serde(rename = "mediator")] - _mediator: Option, - #[serde(rename = "timeout_ms")] - _timeout_ms: u32, - #[serde(rename = "creative_store")] - _creative_store: String, - #[serde(rename = "allowed_context_keys")] - _allowed_context_keys: std::collections::HashSet, - } fn test_settings() -> Settings { - Settings::from_toml(&crate_test_settings_str()).expect("should parse test settings") + let mut settings = + Settings::from_toml(&crate_test_settings_str()).expect("should parse test settings"); + settings.proxy.allowed_domains = vec!["*.example".to_owned(), "*.example.com".to_owned()]; + settings + } + + struct EchoSecretStore; + + impl PlatformSecretStore for EchoSecretStore { + fn get_bytes( + &self, + _store_name: &StoreName, + key: &str, + ) -> Result, Report> { + let value = match key { + "placeholder_proxy" => "change-me-proxy-secret", + "unit-test-proxy-secret" => "unit-test-proxy-secret-32-bytes-ok", + _ => key, + }; + Ok(value.as_bytes().to_vec()) + } + + fn create( + &self, + _store_id: &StoreId, + _name: &str, + _value: &str, + ) -> Result<(), Report> { + Ok(()) + } + + fn delete(&self, _store_id: &StoreId, _name: &str) -> Result<(), Report> { + Ok(()) + } + } + + struct UnifiedSecretStore; + + impl PlatformSecretStore for UnifiedSecretStore { + fn get_bytes( + &self, + store_name: &StoreName, + key: &str, + ) -> Result, Report> { + if store_name.as_ref() != "ts_secrets" || key.starts_with("unused-") { + return Err(Report::new(PlatformError::SecretStore)); + } + let value = match key { + "unit-test-proxy-secret" => "unit-test-proxy-secret-32-bytes-ok", + "tinybird-token-key" => "resolved-tinybird-token", + "datadome-server-key" => "resolved-datadome-server-key", + "datadome-bypass-key" => "resolved-datadome-bypass-credential-32-bytes", + "access_key_id" | "s3-access-key" => "AKIAIOSFODNN7EXAMPLE", + "secret_access_key" | "s3-secret-key" => "wJalrXUtnFEMI/K7MDENG+bPxRfiCYEXAMPLEKEY", + "s3-session-key" => "resolved-session-token", + "partner-api-token-key" => "resolved-partner-api-token-32-bytes-ok", + "partner-pull-token-key" => "resolved-partner-pull-token-32-bytes-ok", + _ => key, + }; + Ok(value.as_bytes().to_vec()) + } + + fn create( + &self, + _store_id: &StoreId, + _name: &str, + _value: &str, + ) -> Result<(), Report> { + Ok(()) + } + + fn delete(&self, _store_id: &StoreId, _name: &str) -> Result<(), Report> { + Ok(()) + } + } + + fn partner_with_pull_sync(enabled: bool, token_key: &str) -> EcPartner { + let mut value = serde_json::json!({ + "name": "Example Partner", + "source_domain": "partner.example.com", + "api_token": "partner-api-token-key", + "pull_sync_enabled": enabled, + "ts_pull_token": token_key, + }); + if enabled { + value["pull_sync_url"] = + serde_json::Value::String("https://partner.example.com/sync".to_string()); + value["pull_sync_allowed_domains"] = serde_json::json!(["partner.example.com"]); + } + serde_json::from_value(value).expect("should build pull-sync partner") } fn envelope_json(settings: &Settings) -> String { @@ -78,11 +234,44 @@ mod tests { serde_json::to_string(&envelope).expect("should serialize envelope") } + fn load_settings(envelope_json: &str) -> Result> { + settings_from_config_blob( + envelope_json, + &EchoSecretStore, + &StoreName::from("trusted_server_secrets"), + ) + } + + fn settings_with_browser_bidder_overlap(auction_enabled: bool) -> Settings { + let mut settings = test_settings(); + settings.proxy.allowed_domains = vec!["*.example".to_string()]; + settings.auction.enabled = auction_enabled; + settings.auction.providers = crate::auction::AuctionConfig::legacy_provider_map(&["pbs"]); + settings.auction.bidders.insert( + "exampleBidder" + .parse() + .expect("should parse server-side bidder"), + crate::auction::BidderRouteConfig { + provider: "pbs".parse().expect("should parse provider"), + }, + ); + let mut prebid = settings + .integration_config::("prebid") + .expect("should parse Prebid config") + .expect("should have enabled Prebid config"); + prebid.client_side_bidders = vec!["exampleBidder".to_string()]; + settings + .integrations + .insert_config("prebid", &prebid) + .expect("should replace Prebid config"); + settings + } + #[test] fn payload_round_trips_through_blob_envelope() { let original = test_settings(); - let reconstructed = settings_from_config_blob(&envelope_json(&original)) - .expect("should reconstruct settings"); + let reconstructed = + load_settings(&envelope_json(&original)).expect("should reconstruct settings"); assert_eq!( reconstructed.publisher.domain, original.publisher.domain, @@ -99,6 +288,274 @@ mod tests { ); } + #[test] + fn resolves_all_static_credentials_from_the_mapped_default_store() { + let mut original = test_settings(); + original.tinybird.enabled = true; + original.tinybird.api_host = "api.example.com".to_string(); + original.tinybird.auction_token_secret = + Some(Redacted::new("tinybird-token-key".to_string())); + original + .integrations + .insert_config( + "datadome", + &serde_json::json!({ + "enabled": true, + "enable_protection": true, + "server_side_key_secret_name": "datadome-server-key", + "protection_test_bypass": { + "enabled": true, + "credential_secret_name": "datadome-bypass-key", + }, + }), + ) + .expect("should configure DataDome references"); + let mut route = ProxyAssetRoute::new( + "/assets/", + "https://examplebucket.s3.us-east-1.amazonaws.com", + ); + route.auth = Some(AssetOriginAuth::S3SigV4(S3SigV4AuthConfig { + region: "us-east-1".to_string(), + secret_store: Some("legacy-s3-store".to_string()), + access_key_id: Redacted::new("s3-access-key".to_string()), + secret_access_key: Redacted::new("s3-secret-key".to_string()), + session_token: Some(Redacted::new("s3-session-key".to_string())), + origin_query: None, + })); + original.proxy.asset_routes.push(route); + original + .ec + .partners + .push(partner_with_pull_sync(true, "partner-pull-token-key")); + + let reconstructed = settings_from_config_blob( + &envelope_json(&original), + &UnifiedSecretStore, + &StoreName::from("ts_secrets"), + ) + .expect("should resolve every static credential from the mapped store"); + + assert_eq!( + reconstructed + .tinybird + .auction_token_secret + .as_ref() + .map(Redacted::expose) + .map(String::as_str), + Some("resolved-tinybird-token") + ); + let datadome = reconstructed + .integration_config::("datadome") + .expect("should parse DataDome config") + .expect("should enable DataDome"); + assert_eq!( + datadome + .server_side_key_secret_name + .as_ref() + .map(Redacted::expose) + .map(String::as_str), + Some("resolved-datadome-server-key") + ); + let bypass = datadome + .protection_test_bypass + .as_ref() + .expect("should configure bypass"); + assert_eq!( + bypass + .credential_secret_name + .as_ref() + .map(Redacted::expose) + .map(String::as_str), + Some("resolved-datadome-bypass-credential-32-bytes") + ); + let auth = reconstructed.proxy.asset_routes[0] + .auth + .as_ref() + .expect("should preserve S3 auth"); + let AssetOriginAuth::S3SigV4(auth) = auth; + assert_eq!(auth.access_key_id.expose(), "AKIAIOSFODNN7EXAMPLE"); + assert_eq!( + auth.secret_access_key.expose(), + "wJalrXUtnFEMI/K7MDENG+bPxRfiCYEXAMPLEKEY" + ); + assert_eq!( + auth.session_token + .as_ref() + .map(Redacted::expose) + .map(String::as_str), + Some("resolved-session-token") + ); + assert!(auth.secret_store.is_none()); + assert_eq!( + reconstructed.ec.partners[0] + .api_token + .as_ref() + .map(Redacted::expose) + .map(String::as_str), + Some("resolved-partner-api-token-32-bytes-ok") + ); + assert_eq!( + reconstructed.ec.partners[0] + .ts_pull_token + .as_ref() + .map(Redacted::expose) + .map(String::as_str), + Some("resolved-partner-pull-token-32-bytes-ok") + ); + } + + #[test] + fn omitted_s3_secret_references_resolve_default_store_keys() { + let mut original = test_settings(); + let mut route = ProxyAssetRoute::new( + "/default-s3/", + "https://examplebucket.s3.us-east-1.amazonaws.com", + ); + route.auth = Some(AssetOriginAuth::S3SigV4( + toml::from_str("region = \"us-east-1\"").expect("should apply S3 secret defaults"), + )); + original.proxy.asset_routes.push(route); + + let reconstructed = settings_from_config_blob( + &envelope_json(&original), + &UnifiedSecretStore, + &StoreName::from("ts_secrets"), + ) + .expect("should resolve default S3 secret keys"); + + let AssetOriginAuth::S3SigV4(auth) = reconstructed.proxy.asset_routes[0] + .auth + .as_ref() + .expect("should preserve S3 auth"); + assert_eq!(auth.access_key_id.expose(), "AKIAIOSFODNN7EXAMPLE"); + assert_eq!( + auth.secret_access_key.expose(), + "wJalrXUtnFEMI/K7MDENG+bPxRfiCYEXAMPLEKEY" + ); + } + + #[test] + fn partner_without_api_token_loads_without_secret_resolution() { + let mut original = test_settings(); + let partner = serde_json::from_value(serde_json::json!({ + "name": "Example Partner", + "source_domain": "partner.example.com", + "bidstream_enabled": true, + })) + .expect("should build partner without API token"); + original.ec.partners.push(partner); + + let reconstructed = settings_from_config_blob( + &envelope_json(&original), + &UnifiedSecretStore, + &StoreName::from("ts_secrets"), + ) + .expect("should load partner without resolving an API token"); + + assert!( + reconstructed.ec.partners[0].api_token.is_none(), + "should preserve omitted API token" + ); + } + + #[test] + fn active_partner_pull_sync_fails_when_its_token_is_missing() { + let mut original = test_settings(); + original + .ec + .partners + .push(partner_with_pull_sync(true, "unused-partner-pull-token")); + + let error = settings_from_config_blob( + &envelope_json(&original), + &UnifiedSecretStore, + &StoreName::from("ts_secrets"), + ) + .expect_err("should reject a missing active pull-sync token"); + + assert!(error.to_string().contains("ec.partners[0].ts_pull_token")); + } + + #[test] + fn inactive_optional_features_do_not_resolve_stale_secret_references() { + let mut original = test_settings(); + original.tinybird.auction_token_secret = + Some(Redacted::new("unused-tinybird-key".to_string())); + original + .integrations + .insert_config( + "datadome", + &serde_json::json!({ + "enabled": true, + "enable_protection": false, + "server_side_key_secret_name": "unused-datadome-key", + "protection_test_bypass": { + "enabled": false, + "credential_secret_name": "unused-bypass-key", + }, + }), + ) + .expect("should configure inactive references"); + original + .ec + .partners + .push(partner_with_pull_sync(false, "unused-partner-pull-token")); + + let reconstructed = settings_from_config_blob( + &envelope_json(&original), + &UnifiedSecretStore, + &StoreName::from("ts_secrets"), + ) + .expect("should skip inactive optional feature references"); + + assert!(reconstructed.tinybird.auction_token_secret.is_none()); + assert!(reconstructed.ec.partners[0].ts_pull_token.is_none()); + let datadome = reconstructed + .integration_config::("datadome") + .expect("should parse inactive DataDome config") + .expect("client-side DataDome remains enabled"); + assert!(datadome.server_side_key_secret_name.is_none()); + assert!( + datadome + .protection_test_bypass + .as_ref() + .is_some_and(|bypass| bypass.credential_secret_name.is_none()) + ); + } + + #[test] + fn omitted_datadome_enabled_does_not_resolve_stale_protection_references() { + let mut original = test_settings(); + original + .integrations + .insert_config( + "datadome", + &serde_json::json!({ + "enable_protection": true, + "server_side_key_secret_name": "unused-datadome-key", + "protection_test_bypass": { + "enabled": true, + "credential_secret_name": "unused-bypass-key", + }, + }), + ) + .expect("should configure disabled DataDome references"); + + let reconstructed = settings_from_config_blob( + &envelope_json(&original), + &UnifiedSecretStore, + &StoreName::from("ts_secrets"), + ) + .expect("should skip stale DataDome protection references"); + + assert!( + reconstructed + .integration_config::("datadome") + .expect("should parse disabled DataDome config") + .is_none() + ); + } + #[test] fn legacy_blob_without_rewrite_creatives_preserves_rewriting() { let data = @@ -115,7 +572,7 @@ mod tests { let envelope_json = serde_json::to_string(&envelope).expect("should serialize envelope"); let reconstructed = - settings_from_config_blob(&envelope_json).expect("should reconstruct legacy settings"); + load_settings(&envelope_json).expect("should reconstruct legacy settings"); assert!( reconstructed.auction.rewrite_creatives, @@ -123,25 +580,12 @@ mod tests { ); } - #[test] - fn default_auction_payload_is_accepted_by_legacy_schema() { - let data = - serde_json::to_value(test_settings()).expect("should serialize settings to JSON"); - let auction = data - .get("auction") - .cloned() - .expect("should serialize auction settings"); - - serde_json::from_value::(auction) - .expect("should deserialize the default payload with the legacy schema"); - } - #[test] fn disabled_rewrite_creatives_survives_blob_round_trip() { let mut original = test_settings(); original.auction.rewrite_creatives = false; - let reconstructed = settings_from_config_blob(&envelope_json(&original)) + let reconstructed = load_settings(&envelope_json(&original)) .expect("should reconstruct disabled rewriting"); assert!( @@ -153,12 +597,13 @@ mod tests { #[test] fn strings_that_look_like_json_scalars_round_trip_as_strings() { let mut original = test_settings(); - original.publisher.proxy_secret = Redacted::new("1234567890".to_string()); + original.publisher.proxy_secret = + Redacted::new("12345678901234567890123456789012".to_string()); original.ec.passphrase = Redacted::new("12345678901234567890123456789012".to_string()); original.handlers[0].password = Redacted::new("true".to_string()); - let reconstructed = settings_from_config_blob(&envelope_json(&original)) - .expect("should reconstruct settings"); + let reconstructed = + load_settings(&envelope_json(&original)).expect("should reconstruct settings"); assert_eq!( reconstructed.publisher.proxy_secret.expose(), @@ -177,6 +622,85 @@ mod tests { ); } + #[test] + fn runtime_validation_accepts_short_resolved_proxy_secret() { + let mut settings = test_settings(); + settings.publisher.proxy_secret = Redacted::new("short_proxy".to_owned()); + + let reconstructed = load_settings(&envelope_json(&settings)) + .expect("should accept an existing short proxy secret"); + + assert_eq!( + reconstructed.publisher.proxy_secret.expose(), + "short_proxy", + "should preserve the resolved proxy secret" + ); + } + + #[test] + fn runtime_validation_rejects_short_resolved_passphrase() { + let mut settings = test_settings(); + settings.ec.passphrase = Redacted::new("short_key".to_owned()); + + let err = load_settings(&envelope_json(&settings)) + .expect_err("should reject a short resolved passphrase"); + + assert!( + err.to_string().contains("short_passphrase") || err.to_string().contains("validation"), + "error should indicate runtime validation: {err:?}" + ); + assert!( + !err.to_string().contains("short_key"), + "error should not expose the secret value" + ); + } + + #[test] + fn placeholder_rejection_happens_after_secret_resolution() { + let mut settings = test_settings(); + settings.publisher.proxy_secret = Redacted::new("placeholder_proxy".to_owned()); + + let err = load_settings(&envelope_json(&settings)) + .expect_err("should reject a placeholder resolved from the secret store"); + + assert!( + err.to_string().contains("Insecure default"), + "error should identify the insecure default: {err:?}" + ); + assert!( + !err.to_string().contains("change-me-proxy-secret"), + "error should not expose the resolved secret value" + ); + } + + #[test] + fn runtime_blob_rejects_enabled_browser_bidder_ownership_conflict() { + let original = settings_with_browser_bidder_overlap(true); + let error = load_settings(&envelope_json(&original)) + .expect_err("should reject enabled browser bidder ownership conflict"); + + assert!(error.to_string().contains("exampleBidder")); + assert!( + error + .to_string() + .contains("both client-side and server-side") + ); + } + + #[test] + fn runtime_blob_accepts_disabled_browser_bidder_ownership_overlap() { + let original = settings_with_browser_bidder_overlap(false); + let reconstructed = load_settings(&envelope_json(&original)) + .expect("should decode dormant conflicting runtime blob"); + let plan = Arc::new( + crate::auction::compile_auction_plan(&reconstructed) + .expect("should compile decoded disabled auction plan"), + ); + + IntegrationRegistry::with_plan(&reconstructed, plan) + .expect("runtime registry should accept disabled ownership overlap"); + } + #[test] fn tampered_blob_hash_is_rejected() { let mut envelope: BlobEnvelope = @@ -185,7 +709,7 @@ mod tests { let tampered = serde_json::to_string(&envelope).expect("should serialize tampered envelope"); - let err = settings_from_config_blob(&tampered).expect_err("should reject hash mismatch"); + let err = load_settings(&tampered).expect_err("should reject hash mismatch"); assert!( err.to_string().contains("integrity verification"), diff --git a/crates/trusted-server-core/src/creative_opportunities.rs b/crates/trusted-server-core/src/creative_opportunities.rs index ab272e4da..044d49ed3 100644 --- a/crates/trusted-server-core/src/creative_opportunities.rs +++ b/crates/trusted-server-core/src/creative_opportunities.rs @@ -173,8 +173,12 @@ fn sanitize_section(segment: &str) -> String { /// The path is used **raw** (not percent-decoded) so this stays consistent with /// how [`page_patterns`](CreativeOpportunitySlot::page_patterns) glob-match the /// same path — e.g. `/new%20s` yields `new_20s`, never the decoded `new_s`. +/// +/// Public so operator tooling that *infers* a `{section}` template from observed +/// ad-unit paths can check its inference against the exact derivation the +/// runtime will perform, rather than reimplementing the sanitization rules. #[must_use] -fn derive_section(path: &str, section_root: &str, section_segment: usize) -> String { +pub fn derive_section(path: &str, section_root: &str, section_segment: usize) -> String { match path .split('/') .filter(|segment| !segment.is_empty()) @@ -185,6 +189,14 @@ fn derive_section(path: &str, section_root: &str, section_segment: usize) -> Str } } +const fn default_enabled() -> bool { + true +} + +const fn is_default_enabled(value: &bool) -> bool { + *value == default_enabled() +} + /// How per-user ad state reaches the page. /// /// `Inline` is the shipped behaviour: the auction result is injected before @@ -215,14 +227,6 @@ pub enum AssemblyMode { Esi, } -const fn default_enabled() -> bool { - true -} - -const fn is_default_enabled(value: &bool) -> bool { - *value == default_enabled() -} - /// Top-level configuration for the creative opportunities system. #[derive(Debug, Clone, Deserialize, Serialize)] #[serde(deny_unknown_fields)] @@ -701,15 +705,7 @@ impl CreativeOpportunitySlot { // skip `compile_patterns`). Re-compiles on every call. self.page_patterns .iter() - .any(|pattern| match Pattern::new(pattern) { - Ok(p) => p.matches(path), - Err(_) => { - let normalised = pattern.replace("**", "*"); - Pattern::new(&normalised) - .map(|p| p.matches(path)) - .unwrap_or(false) - } - }) + .any(|pattern| compile_page_pattern(pattern).is_ok_and(|p| p.matches(path))) } /// Compile [`page_patterns`](Self::page_patterns) into the @@ -726,22 +722,20 @@ impl CreativeOpportunitySlot { self.compiled_patterns = self .page_patterns .iter() - .filter_map(|pattern| { - match Pattern::new(pattern).or_else(|_| Pattern::new(&pattern.replace("**", "*"))) { - Ok(compiled) => Some(compiled), - Err(_) => { - // Build-time validation only requires *one* valid pattern - // per slot, so a mixed valid/invalid set passes the build - // with the bad pattern silently dropped here. Warn so the - // operator can see the slot matches fewer pages than - // configured. - log::warn!( - "slot `{}`: dropping page pattern '{}' — it does not compile as a glob", - self.id, - pattern - ); - None - } + .filter_map(|pattern| match compile_page_pattern(pattern) { + Ok(compiled) => Some(compiled), + Err(error) => { + // Build-time validation only requires *one* valid pattern + // per slot, so a mixed valid/invalid set passes the build + // with the bad pattern silently dropped here. Warn so the + // operator can see the slot matches fewer pages than + // configured. + log::warn!( + "slot `{}`: dropping page pattern '{}': {error}", + self.id, + pattern + ); + None } }) .collect(); @@ -878,23 +872,16 @@ impl CreativeOpportunitySlot { /// Converts this slot into an [`AdSlot`] ready for use in an auction request. /// /// Prebid Server bidder params are wired into the `bidders` map keyed by - /// bidder name. Legacy APS slot params are accepted in configuration but - /// intentionally ignored by the APS `OpenRTB` provider. - /// - /// When [`PrebidSlotParams::bidders`] is empty, a `trustedServer` entry is - /// injected so [`PrebidAuctionProvider`] expands all `config.bidders` - /// automatically. The slot's `targeting.zone` value is forwarded as - /// `trustedServer.zone` so zone-aware bid-param override rules fire correctly. + /// bidder name. APS slot params are ignored by the generic APS profile. + /// When [`PrebidSlotParams::bidders`] is empty, a `trustedServer` marker is + /// inserted for stored-request compatibility and carries the optional zone. #[must_use] pub fn to_ad_slot(&self) -> AdSlot { let mut bidders: HashMap = HashMap::new(); if let Some(ref prebid) = self.providers.prebid { if prebid.bidders.is_empty() { - // No explicit per-bidder override: let the Prebid provider expand - // all config.bidders. The "trustedServer" key triggers - // expand_trusted_server_bidders in PrebidAuctionProvider, giving - // each bidder an empty params object that the override engine then - // fills with zone-aware rules. + // No explicit per-bidder params: preserve the stored-request + // marker and carry the zone for profile routing. let mut ts = serde_json::json!({ "bidderParams": {} }); if let Some(zone) = self.targeting.get("zone") { ts["zone"] = serde_json::Value::String(zone.clone()); @@ -984,17 +971,14 @@ pub struct ApsSlotParams { /// Inline Prebid Server bidder parameters for a slot. /// -/// When `bidders` is empty, `to_ad_slot` injects a `trustedServer` entry so -/// [`PrebidAuctionProvider`] expands all `config.bidders` automatically. -/// When `bidders` is non-empty the map is forwarded verbatim, bypassing -/// automatic expansion (useful for slots that need explicit per-bidder params). +/// When `bidders` is empty, `to_ad_slot` injects a `trustedServer` stored-request +/// marker. When non-empty, the bidder map is forwarded verbatim. #[derive(Debug, Clone, Deserialize, Serialize)] #[serde(deny_unknown_fields)] pub struct PrebidSlotParams { /// Per-bidder inline params map. Bidder name → params object. /// - /// Leave empty (or omit `bidders` in config) to auto-expand all - /// `config.bidders` with zone-aware param overrides. + /// Leave empty (or omit `bidders` in config) to use the stored-request path. /// /// Note: when this map is non-empty it is forwarded verbatim, so a slot's /// `targeting.zone` is **not** injected for these bidders (the `trustedServer` @@ -1004,6 +988,48 @@ pub struct PrebidSlotParams { pub bidders: HashMap, } +/// Compiles a [`page_patterns`](CreativeOpportunitySlot::page_patterns) entry +/// using the runtime's normalisation. +/// +/// This is the single definition of what the runtime accepts as a page glob: +/// a direct [`Pattern::new`], falling back to the `**`→`*` rewrite that +/// [`CreativeOpportunitySlot::compile_patterns`] and +/// [`matches_path`](CreativeOpportunitySlot::matches_path) apply. +/// +/// # Errors +/// +/// Returns an error string when the pattern compiles neither directly nor after +/// normalisation. +pub(crate) fn compile_page_pattern(pattern: &str) -> Result { + Pattern::new(pattern) + .or_else(|_| Pattern::new(&pattern.replace("**", "*"))) + .map_err(|error| format!("page pattern '{pattern}' is not a valid glob: {error}")) +} + +/// Validates a [`page_patterns`](CreativeOpportunitySlot::page_patterns) entry +/// using the runtime's normalisation. +/// +/// This exposes validation without leaking the runtime's `glob::Pattern` type +/// into the public API. +/// +/// # Errors +/// +/// Returns an error string when the pattern compiles neither directly nor after +/// the runtime's `**` to `*` normalisation. +/// +/// # Examples +/// +/// ``` +/// use trusted_server_core::creative_opportunities::validate_page_pattern; +/// +/// assert!(validate_page_pattern("/news/*").is_ok()); +/// assert!(validate_page_pattern("/20**").is_ok()); +/// assert!(validate_page_pattern("[").is_err()); +/// ``` +pub fn validate_page_pattern(pattern: &str) -> Result<(), String> { + compile_page_pattern(pattern).map(|_| ()) +} + /// Validates that a slot ID contains only safe characters. /// /// Allowed characters: ASCII alphanumerics, underscores (`_`), and hyphens (`-`). @@ -1036,9 +1062,318 @@ pub fn match_slots<'a>( slots.iter().filter(|s| s.matches_path(path)).collect() } +/// Three-state outcome of the server-side ad-stack gate. +/// +/// [`Yes`](RuntimeAdStackExpected::Yes) and [`No`](RuntimeAdStackExpected::No) +/// are decided purely from known inputs; [`Unknown`](RuntimeAdStackExpected::Unknown) +/// is reserved for callers (such as the operator CLI) that cannot prove the live +/// consent state and pass `None` for `consent_allows_auction`. +#[derive(Debug, Clone, Copy, Eq, PartialEq)] +pub enum RuntimeAdStackExpected { + /// All known gates pass and consent is known to allow the auction. + Yes, + /// At least one known gate blocks the server-side ad stack. + No, + /// All known gates pass but consent is unproven. + Unknown, +} + +/// Identifies a single gate evaluated by [`evaluate_ad_stack_gate`]. +#[derive(Debug, Clone, Copy, Eq, PartialEq)] +pub enum AdStackGateName { + /// Request method is `GET`. + MethodGet, + /// Request is a top-level navigation. + Navigation, + /// Request is not a prefetch. + NotPrefetch, + /// Request is not from a known bot. + NotBot, + /// At least one configured slot matches the request path. + MatchedSlots, + /// Consent is known to allow the auction. + ConsentAllowsAuction, + /// The global `[auction].enabled` kill switch is on. + AuctionEnabled, + /// The `[creative_opportunities].enabled` template switch is on. + AdTemplatesEnabled, +} + +impl AdStackGateName { + const ALL: [Self; 8] = [ + Self::MethodGet, + Self::Navigation, + Self::NotPrefetch, + Self::NotBot, + Self::MatchedSlots, + Self::ConsentAllowsAuction, + Self::AuctionEnabled, + Self::AdTemplatesEnabled, + ]; + + fn blocks(self, input: AdStackGateInput) -> bool { + match self { + Self::MethodGet => !input.method_get, + Self::Navigation => !input.navigation, + Self::NotPrefetch => input.prefetch, + Self::NotBot => input.bot, + Self::MatchedSlots => !input.matched_slots, + Self::ConsentAllowsAuction => input.consent_allows_auction == Some(false), + Self::AuctionEnabled => !input.auction_enabled, + Self::AdTemplatesEnabled => !input.ad_templates_enabled, + } + } +} + +/// Inputs to [`evaluate_ad_stack_gate`]. +/// +/// `consent_allows_auction` is tri-state: `Some(true)` allows, `Some(false)` +/// blocks, and `None` means the caller cannot prove the consent state. +#[derive(Debug, Clone, Copy, Eq, PartialEq)] +pub struct AdStackGateInput { + /// Request method is `GET`. + pub method_get: bool, + /// Request is a top-level navigation. + pub navigation: bool, + /// Request advertises itself as a prefetch. + pub prefetch: bool, + /// Request is from a known bot. + pub bot: bool, + /// At least one configured slot matches the request path. + pub matched_slots: bool, + /// Whether consent allows the auction. + /// + /// `Some(true)` allows the auction, `Some(false)` blocks it, and `None` + /// means the caller cannot prove either state. Unknown consent is not a + /// denial: it produces [`RuntimeAdStackExpected::Unknown`] when every known + /// boolean gate passes. + pub consent_allows_auction: Option, + /// The global `[auction].enabled` kill switch. + pub auction_enabled: bool, + /// The `[creative_opportunities].enabled` template switch. + /// + /// `false` whenever creative opportunities are absent from the + /// configuration, so an unconfigured publisher blocks here as well. + pub ad_templates_enabled: bool, +} + +/// Result of [`evaluate_ad_stack_gate`]: the three-state expectation plus the +/// original inputs used to derive per-gate diagnostics on demand. +#[derive(Debug, Clone, Eq, PartialEq)] +pub struct AdStackGateResult { + /// The three-state ad-stack expectation. + pub expected: RuntimeAdStackExpected, + input: AdStackGateInput, +} + +impl AdStackGateResult { + /// Returns the gates that blocked the server-side ad stack. + pub fn blocking_gates(&self) -> impl Iterator + '_ { + AdStackGateName::ALL + .into_iter() + .filter(|gate| gate.blocks(self.input)) + } +} + +/// Evaluates whether the server-side ad stack should run for a request. +/// +/// Any known gate that fails sets [`No`](RuntimeAdStackExpected::No) and is +/// recorded in [`AdStackGateResult::blocking_gates`]. When no known gate blocks, +/// the result is [`Yes`](RuntimeAdStackExpected::Yes) if consent is known to +/// allow the auction, or [`Unknown`](RuntimeAdStackExpected::Unknown) when +/// `consent_allows_auction` is `None`. +/// +/// Gate polarity mirrors the runtime publisher path: `method_get`, `navigation`, +/// `matched_slots`, `auction_enabled`, and `ad_templates_enabled` block when +/// `false`; `prefetch` and `bot` block when `true`. +#[must_use] +pub fn evaluate_ad_stack_gate(input: AdStackGateInput) -> AdStackGateResult { + let known_gate_blocks = !input.method_get + || !input.navigation + || input.prefetch + || input.bot + || !input.matched_slots + || input.consent_allows_auction == Some(false) + || !input.auction_enabled + || !input.ad_templates_enabled; + let expected = if known_gate_blocks { + RuntimeAdStackExpected::No + } else if input.consent_allows_auction.is_none() { + RuntimeAdStackExpected::Unknown + } else { + RuntimeAdStackExpected::Yes + }; + + AdStackGateResult { expected, input } +} + #[cfg(test)] mod tests { + use std::collections::BTreeMap; + use std::str::FromStr as _; + + use edgezero_core::body::Body as EdgeBody; + use http::Request; + use super::*; + use crate::auction::plan::{ + AuctionPlan, AuctionPlanConfig, NotificationConfig, ProviderConfig, ProviderId, RoutingMode, + }; + use crate::auction::routing::route_auction; + use crate::auction::types::{AuctionRequest, PublisherInfo, UserInfo}; + + #[test] + fn ad_stack_gate_passes_for_eligible_navigation() { + let result = evaluate_ad_stack_gate(AdStackGateInput { + method_get: true, + navigation: true, + prefetch: false, + bot: false, + matched_slots: true, + consent_allows_auction: Some(true), + auction_enabled: true, + ad_templates_enabled: true, + }); + + assert_eq!(result.expected, RuntimeAdStackExpected::Yes); + assert_eq!(result.blocking_gates().count(), 0); + } + + #[test] + fn ad_stack_gate_blocks_known_kill_switch() { + let result = evaluate_ad_stack_gate(AdStackGateInput { + method_get: true, + navigation: true, + prefetch: false, + bot: false, + matched_slots: true, + consent_allows_auction: Some(true), + auction_enabled: false, + ad_templates_enabled: true, + }); + + assert_eq!(result.expected, RuntimeAdStackExpected::No); + assert!( + result + .blocking_gates() + .any(|gate| gate == AdStackGateName::AuctionEnabled) + ); + } + + #[test] + fn ad_stack_gate_blocks_disabled_ad_templates() { + let result = evaluate_ad_stack_gate(AdStackGateInput { + method_get: true, + navigation: true, + prefetch: false, + bot: false, + matched_slots: true, + consent_allows_auction: Some(true), + auction_enabled: true, + ad_templates_enabled: false, + }); + + assert_eq!( + result.expected, + RuntimeAdStackExpected::No, + "a disabled [creative_opportunities].enabled switch should block the ad stack" + ); + assert!( + result + .blocking_gates() + .any(|gate| gate == AdStackGateName::AdTemplatesEnabled), + "the template switch should be named as the blocking gate" + ); + } + + #[test] + fn ad_stack_gate_is_unknown_when_consent_is_unknown() { + let result = evaluate_ad_stack_gate(AdStackGateInput { + method_get: true, + navigation: true, + prefetch: false, + bot: false, + matched_slots: true, + consent_allows_auction: None, + auction_enabled: true, + ad_templates_enabled: true, + }); + + assert_eq!(result.expected, RuntimeAdStackExpected::Unknown); + } + + // Locks the spec §5.2 mirror invariant: with Some(consent) supplied for every + // input combination, `expected == Yes` must equal the legacy all-AND boolean. + #[test] + fn ad_stack_gate_with_known_consent_matches_legacy_boolean() { + for bits in 0u16..256 { + let input = AdStackGateInput { + method_get: bits & 1 != 0, + navigation: bits & 2 != 0, + prefetch: bits & 4 != 0, + bot: bits & 8 != 0, + matched_slots: bits & 16 != 0, + consent_allows_auction: Some(bits & 32 != 0), + auction_enabled: bits & 64 != 0, + ad_templates_enabled: bits & 128 != 0, + }; + // Legacy semantics: all positive gates true, both negative gates false. + let legacy = input.method_get + && input.navigation + && !input.prefetch + && !input.bot + && input.matched_slots + && input.consent_allows_auction == Some(true) + && input.auction_enabled + && input.ad_templates_enabled; + let got = evaluate_ad_stack_gate(input).expected == RuntimeAdStackExpected::Yes; + assert_eq!(got, legacy, "gate mismatch for bits={bits}"); + } + } + + #[test] + fn ad_stack_gate_with_unknown_consent_matches_known_boolean_gates() { + for bits in 0u8..128 { + let input = AdStackGateInput { + method_get: bits & 1 != 0, + navigation: bits & 2 != 0, + prefetch: bits & 4 != 0, + bot: bits & 8 != 0, + matched_slots: bits & 16 != 0, + consent_allows_auction: None, + auction_enabled: bits & 32 != 0, + ad_templates_enabled: bits & 64 != 0, + }; + let known_gates_pass = input.method_get + && input.navigation + && !input.prefetch + && !input.bot + && input.matched_slots + && input.auction_enabled + && input.ad_templates_enabled; + let expected = if known_gates_pass { + RuntimeAdStackExpected::Unknown + } else { + RuntimeAdStackExpected::No + }; + + assert_eq!( + evaluate_ad_stack_gate(input).expected, + expected, + "should match unknown-consent gate semantics for bits={bits}" + ); + } + } + + #[test] + fn validate_page_pattern_preserves_specific_compile_error() { + let error = validate_page_pattern("[").expect_err("should reject invalid glob"); + + assert!( + error.contains("page pattern '[' is not a valid glob"), + "should retain the invalid pattern in the error: {error}" + ); + } fn make_slot(id: &str, patterns: Vec<&str>) -> CreativeOpportunitySlot { CreativeOpportunitySlot { @@ -1956,6 +2291,69 @@ mod tests { ); } + #[test] + fn creative_opportunity_canonical_slot_feeds_shared_stored_router_with_zone() { + let mut slot = make_slot("header", vec!["/"]); + slot.targeting + .insert("zone".to_string(), "header".to_string()); + slot.providers.prebid = Some(PrebidSlotParams { + bidders: HashMap::new(), + }); + let plan = AuctionPlan::compile(AuctionPlanConfig { + timeout_ms: 900, + providers: BTreeMap::from([( + ProviderId::from_str("pbs-primary").expect("should parse provider ID"), + ProviderConfig { + protocol: "openrtb-2.6".to_string(), + profile: "prebid-server".to_string(), + endpoint: "https://pbs.example.test/openrtb".to_string(), + timeout_ms: None, + routing: RoutingMode::Explicit, + notifications: NotificationConfig::default(), + profile_config: serde_json::json!({}), + }, + )]), + bidders: BTreeMap::new(), + mediator: None, + request_signing: None, + }) + .expect("should compile plan"); + let auction_request = AuctionRequest { + id: "auction-1".to_string(), + slots: vec![slot.to_ad_slot()], + publisher: PublisherInfo { + domain: "publisher.example.test".to_string(), + page_url: None, + }, + user: UserInfo { + id: None, + consent: None, + eids: None, + }, + device: None, + site: None, + context: HashMap::new(), + }; + let inbound = Request::builder() + .uri("https://publisher.example.test/") + .body(EdgeBody::empty()) + .expect("should build request"); + let routed = route_auction(auction_request, &inbound, &plan, None); + + assert_eq!(routed.inputs().len(), 1); + assert!(routed.inputs()[0].slots()[0].has_trusted_stored_request()); + assert_eq!(routed.inputs()[0].slots()[0].prebid_zone(), Some("header")); + assert!( + routed.inputs()[0].slots().iter().all(|slot| { + !slot + .bidder_params() + .keys() + .any(|bidder| bidder.as_str() == "pbs-primary") + }), + "provider ID should not reach client-controlled bidder input" + ); + } + #[test] fn to_ad_slot_injects_trusted_server_without_zone_when_targeting_absent() { let mut slot = make_slot("no-zone", vec!["/"]); diff --git a/crates/trusted-server-core/src/ec/admin.rs b/crates/trusted-server-core/src/ec/admin.rs index 6219af7a9..e44708742 100644 --- a/crates/trusted-server-core/src/ec/admin.rs +++ b/crates/trusted-server-core/src/ec/admin.rs @@ -694,7 +694,7 @@ mod tests { source_domain: source_domain.to_owned(), openrtb_atype: EcPartner::default_openrtb_atype(), bidstream_enabled, - api_token: Redacted::new(format!("test-token-{source_domain:-<32}")), + api_token: Some(Redacted::new(format!("test-token-{source_domain:-<32}"))), batch_rate_limit: EcPartner::default_batch_rate_limit(), pull_sync_enabled: false, pull_sync_url: None, diff --git a/crates/trusted-server-core/src/ec/auth.rs b/crates/trusted-server-core/src/ec/auth.rs index 0e609f9a7..b7db6f1fa 100644 --- a/crates/trusted-server-core/src/ec/auth.rs +++ b/crates/trusted-server-core/src/ec/auth.rs @@ -52,7 +52,7 @@ mod tests { source_domain: source_domain.to_owned(), openrtb_atype: EcPartner::default_openrtb_atype(), bidstream_enabled: true, - api_token: Redacted::new(api_token.to_owned()), + api_token: Some(Redacted::new(api_token.to_owned())), batch_rate_limit: EcPartner::default_batch_rate_limit(), pull_sync_enabled: false, pull_sync_url: None, @@ -125,4 +125,23 @@ mod tests { "should return the matching partner" ); } + + #[test] + fn authenticate_bearer_rejects_token_for_partner_without_api_access() { + let mut partner = make_test_partner("ssp.example.com", VALID_API_TOKEN); + partner.api_token = None; + let registry = + PartnerRegistry::from_config(&[partner]).expect("should build registry without token"); + let req = Request::builder() + .method("GET") + .uri("https://edge.example.com/_ts/api/v1/identify") + .header("authorization", format!("Bearer {VALID_API_TOKEN}")) + .body(EdgeBody::empty()) + .expect("should build test request"); + + assert!( + authenticate_bearer(®istry, &req).is_none(), + "should reject authentication when API access is not configured" + ); + } } diff --git a/crates/trusted-server-core/src/ec/batch_sync.rs b/crates/trusted-server-core/src/ec/batch_sync.rs index 0e0f3b900..248e1dd27 100644 --- a/crates/trusted-server-core/src/ec/batch_sync.rs +++ b/crates/trusted-server-core/src/ec/batch_sync.rs @@ -341,7 +341,7 @@ mod tests { source_domain: source_domain.to_owned(), openrtb_atype: EcPartner::default_openrtb_atype(), bidstream_enabled: true, - api_token: Redacted::new(api_token.to_owned()), + api_token: Some(Redacted::new(api_token.to_owned())), batch_rate_limit: EcPartner::default_batch_rate_limit(), pull_sync_enabled: false, pull_sync_url: None, diff --git a/crates/trusted-server-core/src/ec/device.rs b/crates/trusted-server-core/src/ec/device.rs index fbefa9586..352f3eddc 100644 --- a/crates/trusted-server-core/src/ec/device.rs +++ b/crates/trusted-server-core/src/ec/device.rs @@ -33,6 +33,9 @@ pub struct DeviceSignals { /// Coarse OS family: `"mac"`, `"windows"`, `"ios"`, `"android"`, /// `"linux"`. pub platform_class: Option, + /// Coarse browser family from the UA: `"chrome"`, `"safari"`, + /// `"firefox"`, `"edge"`, `"opera"`; `None` when unrecognized. + pub browser_family: Option, /// SHA256 prefix (12 hex chars) of the raw H2 SETTINGS string. pub h2_fp_hash: Option, /// `true` = known browser, `false` = known bot, `None` = unknown. @@ -50,6 +53,7 @@ impl DeviceSignals { let is_mobile = parse_is_mobile(ua); let ja4_class = ja4.and_then(extract_ja4_section1); let platform_class = parse_platform_class(ua); + let browser_family = parse_browser_family(ua); let h2_fp_hash = h2_fp.map(compute_h2_fp_hash); let known_browser = evaluate_known_browser(ja4_class.as_deref(), h2_fp_hash.as_deref()); @@ -57,6 +61,7 @@ impl DeviceSignals { is_mobile, ja4_class, platform_class, + browser_family, h2_fp_hash, known_browser, } @@ -146,6 +151,35 @@ fn parse_platform_class(ua: &str) -> Option { None } +/// Parses coarse browser family from the User-Agent string. +/// +/// Order matters: Edge, Opera, and the iOS Chrome/Firefox user agents embed +/// `Chrome` and/or `Safari` tokens, so the more specific families are matched +/// before the generic `Chrome` and `Safari` fallbacks. +/// +/// Returns `None` when no recognized browser pattern is found (bots, raw HTTP +/// clients, or empty UA). +#[must_use] +fn parse_browser_family(ua: &str) -> Option { + if ua.contains("Edg/") || ua.contains("EdgA/") || ua.contains("EdgiOS/") || ua.contains("Edge/") + { + return Some("edge".to_owned()); + } + if ua.contains("OPR/") || ua.contains("Opera") { + return Some("opera".to_owned()); + } + if ua.contains("Firefox/") || ua.contains("FxiOS/") { + return Some("firefox".to_owned()); + } + if ua.contains("Chrome/") || ua.contains("CriOS/") { + return Some("chrome".to_owned()); + } + if ua.contains("Safari/") { + return Some("safari".to_owned()); + } + None +} + /// Extracts Section 1 from a full JA4 string. /// /// JA4 format: `section1_section2_section3` separated by underscores. @@ -305,6 +339,67 @@ mod tests { ); } + #[test] + fn browser_family_recognized() { + assert_eq!( + parse_browser_family(CHROME_MAC_UA).as_deref(), + Some("chrome"), + "Chrome/Mac = chrome" + ); + assert_eq!( + parse_browser_family(CHROME_ANDROID_UA).as_deref(), + Some("chrome"), + "Chrome/Android = chrome" + ); + assert_eq!( + parse_browser_family(SAFARI_IOS_UA).as_deref(), + Some("safari"), + "Safari/iOS = safari" + ); + assert_eq!( + parse_browser_family(SAFARI_MAC_UA).as_deref(), + Some("safari"), + "Safari/Mac = safari" + ); + assert_eq!( + parse_browser_family(FIREFOX_MAC_UA).as_deref(), + Some("firefox"), + "Firefox/Mac = firefox" + ); + } + + #[test] + fn browser_family_edge_and_opera_win_over_chrome() { + let edge = "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 \ + (KHTML, like Gecko) Chrome/146.0.0.0 Safari/537.36 Edg/146.0.0.0"; + let opera = "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 \ + (KHTML, like Gecko) Chrome/146.0.0.0 Safari/537.36 OPR/120.0.0.0"; + assert_eq!( + parse_browser_family(edge).as_deref(), + Some("edge"), + "Edge UA embeds Chrome/Safari but should classify as edge" + ); + assert_eq!( + parse_browser_family(opera).as_deref(), + Some("opera"), + "Opera UA embeds Chrome/Safari but should classify as opera" + ); + } + + #[test] + fn browser_family_unknown() { + assert_eq!( + parse_browser_family(BOT_UA), + None, + "Googlebot = no browser family" + ); + assert_eq!( + parse_browser_family(""), + None, + "empty UA = no browser family" + ); + } + #[test] fn platform_class_linux() { let linux_ua = "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36"; diff --git a/crates/trusted-server-core/src/ec/eids.rs b/crates/trusted-server-core/src/ec/eids.rs index 5dd5f3b05..1359f7509 100644 --- a/crates/trusted-server-core/src/ec/eids.rs +++ b/crates/trusted-server-core/src/ec/eids.rs @@ -154,7 +154,9 @@ mod tests { source_domain: source_domain.to_owned(), openrtb_atype: EcPartner::default_openrtb_atype(), bidstream_enabled: true, - api_token: Redacted::new(format!("token-{source_domain}-32-bytes-minimum-value")), + api_token: Some(Redacted::new(format!( + "token-{source_domain}-32-bytes-minimum-value" + ))), batch_rate_limit: EcPartner::default_batch_rate_limit(), pull_sync_enabled: false, pull_sync_url: None, diff --git a/crates/trusted-server-core/src/ec/finalize.rs b/crates/trusted-server-core/src/ec/finalize.rs index a553bb7a7..15653ce6e 100644 --- a/crates/trusted-server-core/src/ec/finalize.rs +++ b/crates/trusted-server-core/src/ec/finalize.rs @@ -261,7 +261,9 @@ mod tests { source_domain: source_domain.to_owned(), openrtb_atype: EcPartner::default_openrtb_atype(), bidstream_enabled: true, - api_token: Redacted::new(format!("token-{source_domain}-32-bytes-minimum-value")), + api_token: Some(Redacted::new(format!( + "token-{source_domain}-32-bytes-minimum-value" + ))), batch_rate_limit: EcPartner::default_batch_rate_limit(), pull_sync_enabled: false, pull_sync_url: None, diff --git a/crates/trusted-server-core/src/ec/identify.rs b/crates/trusted-server-core/src/ec/identify.rs index 6ca251905..94464d1a5 100644 --- a/crates/trusted-server-core/src/ec/identify.rs +++ b/crates/trusted-server-core/src/ec/identify.rs @@ -367,7 +367,7 @@ mod tests { source_domain: source_domain.to_owned(), openrtb_atype: EcPartner::default_openrtb_atype(), bidstream_enabled: true, - api_token: Redacted::new(api_token.to_owned()), + api_token: Some(Redacted::new(api_token.to_owned())), batch_rate_limit: EcPartner::default_batch_rate_limit(), pull_sync_enabled: false, pull_sync_url: None, diff --git a/crates/trusted-server-core/src/ec/prebid_eids.rs b/crates/trusted-server-core/src/ec/prebid_eids.rs index 4a1a8d156..3c8712fa9 100644 --- a/crates/trusted-server-core/src/ec/prebid_eids.rs +++ b/crates/trusted-server-core/src/ec/prebid_eids.rs @@ -451,7 +451,9 @@ mod tests { source_domain: source_domain.to_owned(), openrtb_atype: EcPartner::default_openrtb_atype(), bidstream_enabled: true, - api_token: Redacted::new(format!("token-{source_domain}-32-bytes-minimum-value")), + api_token: Some(Redacted::new(format!( + "token-{source_domain}-32-bytes-minimum-value" + ))), batch_rate_limit: EcPartner::default_batch_rate_limit(), pull_sync_enabled: false, pull_sync_url: None, diff --git a/crates/trusted-server-core/src/ec/pull_sync.rs b/crates/trusted-server-core/src/ec/pull_sync.rs index 546605f8e..f683ee152 100644 --- a/crates/trusted-server-core/src/ec/pull_sync.rs +++ b/crates/trusted-server-core/src/ec/pull_sync.rs @@ -467,7 +467,7 @@ mod tests { fn pull_partner(ttl_sec: u64) -> PartnerConfig { PartnerConfig { name: "SSP X".to_owned(), - api_key_hash: "deadbeef".to_owned(), + api_key_hash: Some("deadbeef".to_owned()), bidstream_enabled: true, source_domain: "ssp.example.com".to_owned(), openrtb_atype: 3, diff --git a/crates/trusted-server-core/src/ec/registry.rs b/crates/trusted-server-core/src/ec/registry.rs index 8532de03b..c4637b431 100644 --- a/crates/trusted-server-core/src/ec/registry.rs +++ b/crates/trusted-server-core/src/ec/registry.rs @@ -4,7 +4,7 @@ //! in-memory registry. `HashMap` indexes provide O(1) //! lookup by source domain and API key hash. -use std::collections::HashMap; +use std::collections::{HashMap, HashSet}; use error_stack::{Report, ResultExt as _}; @@ -28,8 +28,8 @@ pub struct PartnerConfig { pub openrtb_atype: i32, /// Whether this partner's UIDs appear in auction `user.eids`. pub bidstream_enabled: bool, - /// SHA-256 hex of the partner's API token (precomputed at startup). - pub api_key_hash: String, + /// SHA-256 hex of the partner's API token, when inbound API access is enabled. + pub api_key_hash: Option, /// Max batch sync API requests per partner per minute. pub batch_rate_limit: u32, /// Whether server-to-server pull sync is enabled. @@ -61,6 +61,78 @@ pub struct PartnerRegistry { } impl PartnerRegistry { + /// Validates partner structure without inspecting secret values. + /// + /// This is the push-time half of partner validation. API-token length, + /// placeholder, and collision checks remain in [`Self::from_config`], + /// after secret references have been resolved. + /// + /// # Errors + /// + /// Returns [`TrustedServerError::Configuration`] when non-secret partner + /// structure is invalid. + pub fn validate_config_for_deploy( + partners: &[EcPartner], + ) -> Result<(), Report> { + let mut source_domains = HashSet::with_capacity(partners.len()); + let mut api_token_key_references = HashMap::with_capacity(partners.len()); + + for partner in partners { + let normalized_source = normalize_partner_source_domain(&partner.source_domain) + .map_err(|msg| { + Report::new(TrustedServerError::Configuration { + message: format!("ec.partners: {msg}"), + }) + })?; + + if !source_domains.insert(normalized_source.clone()) { + return Err(Report::new(TrustedServerError::Configuration { + message: format!("ec.partners: duplicate source_domain '{normalized_source}'"), + })); + } + + if let Some(api_token) = &partner.api_token + && let Some(previous_source) = + api_token_key_references.insert(api_token.expose(), normalized_source.clone()) + { + return Err(Report::new(TrustedServerError::Configuration { + message: format!( + "ec.partners: API token key reference is shared by source_domain \ + '{previous_source}' and '{normalized_source}'" + ), + })); + } + + validate_rate_limits_values(partner.batch_rate_limit, partner.pull_sync_rate_limit) + .map_err(|error| { + Report::new(TrustedServerError::Configuration { + message: format!( + "ec.partners: invalid rate limits for '{normalized_source}': {error}" + ), + }) + })?; + + if partner.pull_sync_enabled { + validate_pull_sync_fields( + partner.pull_sync_url.as_deref(), + &partner.pull_sync_allowed_domains, + partner + .ts_pull_token + .as_ref() + .map(|token| token.expose().as_str()), + false, + ) + .change_context(TrustedServerError::Configuration { + message: format!( + "ec.partners: pull sync config invalid for '{normalized_source}'" + ), + })?; + } + } + + Ok(()) + } + /// Builds a registry from the config-defined partner list. /// /// # Errors @@ -86,20 +158,24 @@ impl PartnerRegistry { })); } - validate_api_token(&normalized_source, partner.api_token.expose())?; - - let api_key_hash = hash_api_key(partner.api_token.expose()); - - if by_api_key_hash.contains_key(&api_key_hash) { - return Err(Report::new(TrustedServerError::Configuration { - message: format!( - "ec.partners: source_domain '{normalized_source}' has an API token that collides \ - with another partner's token hash" - ), - })); - } - - let config = build_partner_config(partner, &normalized_source, &api_key_hash); + let api_key_hash = if let Some(api_token) = &partner.api_token { + validate_api_token(&normalized_source, api_token.expose())?; + + let api_key_hash = hash_api_key(api_token.expose()); + if by_api_key_hash.contains_key(&api_key_hash) { + return Err(Report::new(TrustedServerError::Configuration { + message: format!( + "ec.partners: source_domain '{normalized_source}' has an API token that collides \ + with another partner's token hash" + ), + })); + } + Some(api_key_hash) + } else { + None + }; + + let config = build_partner_config(partner, &normalized_source, api_key_hash.as_deref()); validate_rate_limits(&config).change_context(TrustedServerError::Configuration { message: format!( @@ -117,7 +193,9 @@ impl PartnerRegistry { })?; } - by_api_key_hash.insert(api_key_hash, normalized_source.clone()); + if let Some(api_key_hash) = api_key_hash { + by_api_key_hash.insert(api_key_hash, normalized_source.clone()); + } by_source_domain.insert(normalized_source, config); } @@ -212,14 +290,14 @@ fn validate_api_token( fn build_partner_config( partner: &EcPartner, normalized_source: &str, - api_key_hash: &str, + api_key_hash: Option<&str>, ) -> PartnerConfig { PartnerConfig { name: partner.name.clone(), source_domain: normalized_source.to_owned(), openrtb_atype: partner.openrtb_atype, bidstream_enabled: partner.bidstream_enabled, - api_key_hash: api_key_hash.to_owned(), + api_key_hash: api_key_hash.map(ToOwned::to_owned), batch_rate_limit: partner.batch_rate_limit, pull_sync_enabled: partner.pull_sync_enabled, pull_sync_url: partner.pull_sync_url.clone(), @@ -231,34 +309,56 @@ fn build_partner_config( } fn validate_rate_limits(config: &PartnerConfig) -> Result<(), Report> { - if config.batch_rate_limit == 0 { - return Err(Report::new(TrustedServerError::Configuration { - message: "batch_rate_limit must be greater than 0".to_owned(), - })); + validate_rate_limits_values(config.batch_rate_limit, config.pull_sync_rate_limit).map_err( + |message| { + Report::new(TrustedServerError::Configuration { + message: message.to_owned(), + }) + }, + ) +} + +fn validate_rate_limits_values( + batch_rate_limit: u32, + pull_sync_rate_limit: u32, +) -> Result<(), &'static str> { + if batch_rate_limit == 0 { + return Err("batch_rate_limit must be greater than 0"); } - if config.pull_sync_rate_limit == 0 { - return Err(Report::new(TrustedServerError::Configuration { - message: "pull_sync_rate_limit must be greater than 0".to_owned(), - })); + if pull_sync_rate_limit == 0 { + return Err("pull_sync_rate_limit must be greater than 0"); } Ok(()) } fn validate_pull_sync(config: &PartnerConfig) -> Result<(), Report> { - let url_str = config.pull_sync_url.as_deref().unwrap_or(""); + validate_pull_sync_fields( + config.pull_sync_url.as_deref(), + &config.pull_sync_allowed_domains, + config + .ts_pull_token + .as_ref() + .map(|token| token.expose().as_str()), + true, + ) +} + +fn validate_pull_sync_fields( + url: Option<&str>, + allowed_domains: &[String], + token_value: Option<&str>, + require_nonempty_token: bool, +) -> Result<(), Report> { + let url_str = url.unwrap_or(""); if url_str.is_empty() { return Err(Report::new(TrustedServerError::Configuration { message: "pull_sync_url is required when pull_sync_enabled is true".to_owned(), })); } - if config - .ts_pull_token - .as_ref() - .is_none_or(|token| token.expose().trim().is_empty()) - { + if token_value.is_none() { return Err(Report::new(TrustedServerError::Configuration { message: "ts_pull_token is required when pull_sync_enabled is true".to_owned(), })); @@ -289,7 +389,7 @@ fn validate_pull_sync(config: &PartnerConfig) -> Result<(), Report Result<(), Report &'static str { + fn provider_name(&self) -> &str { "adserver_mock" } @@ -466,14 +470,14 @@ impl AuctionProvider for AdServerMockProvider { } } - // Uses context.timeout_ms (auction-scoped) rather than the 15 s fixed - // timeout in ensure_integration_backend, which is for proxy endpoints. - // Send async with auction-scoped timeout + // Uses the auction-scoped canonical transport timeout rather than the + // 15 s fixed timeout in ensure_integration_backend, which is for proxy + // endpoints. The exact logical budget remains in context.timeout_ms. let backend_name = ensure_integration_backend_with_timeout( context.services, &self.config.endpoint, "adserver_mock", - Duration::from_millis(u64::from(context.timeout_ms)), + Duration::from_millis(u64::from(context.transport_timeout_ms)), ) .change_context(TrustedServerError::Auction { message: format!( @@ -535,12 +539,16 @@ impl AuctionProvider for AdServerMockProvider { self.config.enabled } - fn backend_name(&self, services: &RuntimeServices, timeout_ms: u32) -> Option { + fn backend_name( + &self, + services: &RuntimeServices, + transport_timeout_ms: u32, + ) -> Option { predict_integration_backend_name( services, &self.config.endpoint, "adserver_mock", - Duration::from_millis(u64::from(timeout_ms)), + Duration::from_millis(u64::from(transport_timeout_ms)), ) .inspect_err(|e| { log::error!( @@ -638,6 +646,7 @@ mod tests { width: 728, height: 90, bidder: "aps".to_string(), + returned_seat: None, adomain: Some(vec!["advertiser.example".to_string()]), nurl: None, burl: None, @@ -687,6 +696,7 @@ mod tests { width: 728, height: 90, bidder: "aps".to_string(), + returned_seat: None, adomain: Some(vec!["advertiser.example".to_string()]), nurl: None, burl: None, @@ -713,6 +723,7 @@ mod tests { width: 728, height: 90, bidder: "test-bidder".to_string(), + returned_seat: None, adomain: None, nurl: Some("https://ssp.example/win?id=mock-bid-001".to_string()), burl: Some("https://ssp.example/bill?id=mock-bid-001".to_string()), @@ -834,6 +845,7 @@ mod tests { creative: Some("
Original Ad
".to_string()), adomain: Some(vec!["example.com".to_string()]), bidder: "mocktioneer".to_string(), + returned_seat: None, width: 728, height: 90, nurl: Some("https://ssp.example/win".to_string()), @@ -906,6 +918,10 @@ mod tests { Some("/cache"), "should restore PBS cache path" ); + assert!( + bid.returned_seat.is_none(), + "a matched original with no returned seat must not inherit mediator seat" + ); } #[test] @@ -942,6 +958,7 @@ mod tests { creative: Some("
Original Ad
".to_string()), adomain: None, bidder: "example-bidder".to_string(), + returned_seat: None, width: 728, height: 90, nurl: None, @@ -1103,6 +1120,7 @@ mod tests { width: 300, height: 250, bidder: "aps".to_string(), + returned_seat: None, adomain: Some(vec!["advertiser.example".to_string()]), nurl: None, burl: None, diff --git a/crates/trusted-server-core/src/integrations/aps.rs b/crates/trusted-server-core/src/integrations/aps.rs index 581e5c200..6bffddf16 100644 --- a/crates/trusted-server-core/src/integrations/aps.rs +++ b/crates/trusted-server-core/src/integrations/aps.rs @@ -1,7 +1,8 @@ //! Amazon Publisher Services (APS/TAM) `OpenRTB` integration. -use std::collections::{BTreeMap, HashMap}; +use std::collections::{BTreeMap, HashMap, HashSet}; use std::sync::Arc; +#[cfg(test)] use std::time::Duration; use async_trait::async_trait; @@ -16,34 +17,53 @@ use serde_json::{Value as Json, json}; use url::Url; use validator::{Validate, ValidationError}; +use crate::auction::openrtb::ignored_bidder_params_count; +#[cfg(test)] +use crate::auction::plan::{AuctionPlanConfig, NotificationConfig, ProviderConfig, RoutingMode}; +use crate::auction::profile::ApsProfilePlan; +#[cfg(test)] use crate::auction::provider::{AuctionProvider, ProviderRequestOutcome}; +use crate::auction::routing::ProviderAuctionInput; +#[cfg(test)] +use crate::auction::types::{AdSlot, AuctionContext, AuctionRequest}; use crate::auction::types::{ - AdSlot, ApsRendererV1, ApsTagType, AuctionContext, AuctionRequest, AuctionResponse, Bid, - BidRenderer, MediaType, + ApsRendererV1, ApsTagType, AuctionResponse, Bid, BidRenderer, MediaType, }; use crate::error::TrustedServerError; use crate::integrations::{ IntegrationEndpoint, IntegrationHeadInjector, IntegrationHtmlContext, IntegrationProxy, IntegrationRegistration, UPSTREAM_RTB_MAX_RESPONSE_BYTES, collect_response_bounded, +}; +#[cfg(test)] +use crate::integrations::{ ensure_integration_backend_with_timeout, predict_integration_backend_name, }; +#[cfg(test)] +use crate::openrtb::ToExt; +#[cfg(test)] use crate::openrtb::{ - Banner, Device, Format, Geo, Imp, OpenRtbRequest, Publisher, Regs, RegsExt, Site, ToExt, User, + Banner, Device, Format, Geo, Imp, OpenRtbRequest, Publisher, Regs, RegsExt, Site, User, UserExt, to_openrtb_i32, }; -use crate::platform::{PlatformHttpRequest, PlatformResponse, RuntimeServices}; +#[cfg(test)] +use crate::platform::PlatformHttpRequest; +use crate::platform::{PlatformResponse, RuntimeServices}; use crate::settings::{IntegrationConfig, Settings}; const APS_INTEGRATION_ID: &str = "aps"; const APS_RENDERER_ROUTE: &str = "/integrations/aps/renderer"; const DEFAULT_CURRENCY: &str = "USD"; +#[cfg(test)] const APS_SDK_SOURCE: &str = "prebid"; +#[cfg(test)] const APS_SDK_VERSION: &str = "2.2.0"; const MAX_ACCOUNT_ID_BYTES: usize = 1024; const MAX_CREATIVE_ID_BYTES: usize = 1024; const MAX_DEBUG_RESPONSE_PREVIEW_BYTES: usize = 512; const MAX_CREATIVE_URL_BYTES: usize = 4096; +#[cfg(test)] const MAX_LANGUAGE_BYTES: usize = 8; +#[cfg(test)] const MAX_PAGE_URL_BYTES: usize = 8192; const MAX_RENDER_ENVELOPE_BYTES: usize = 256 * 1024; const APS_RENDERER_CSP: &str = "default-src 'none'; sandbox allow-forms allow-pointer-lock allow-popups allow-popups-to-escape-sandbox allow-scripts allow-top-navigation-by-user-activation; script-src 'unsafe-inline' https:; connect-src https:; frame-src https:; img-src https: data:; media-src https: blob:; style-src 'unsafe-inline' https:; font-src https: data:;"; @@ -127,9 +147,10 @@ pub enum ApsRenderingMode { } /// Configuration for the APS `OpenRTB` integration. +#[cfg(test)] #[derive(Debug, Clone, Deserialize, Serialize, Validate)] #[validate(schema(function = "validate_inventory_identity_override"))] -pub struct ApsConfig { +pub struct LegacyApsProviderConfig { /// Whether APS integration is enabled. #[serde(default = "default_enabled")] pub enabled: bool, @@ -219,6 +240,7 @@ where deserializer.deserialize_any(AccountIdVisitor) } +#[cfg(test)] fn validate_aps_endpoint(value: &str) -> Result<(), ValidationError> { let parsed = Url::parse(value).map_err(|_| ValidationError::new("invalid_aps_endpoint"))?; if parsed.scheme() != "https" @@ -280,12 +302,12 @@ fn validate_inventory_page_origin(value: &str) -> Result<(), ValidationError> { Ok(()) } -fn validate_inventory_identity_override(config: &ApsConfig) -> Result<(), ValidationError> { - let (Some(domain), Some(origin)) = ( - config.inventory_domain.as_deref(), - config.inventory_page_origin.as_deref(), - ) else { - if config.inventory_domain.is_none() && config.inventory_page_origin.is_none() { +fn validate_inventory_identity_override_values( + inventory_domain: Option<&str>, + inventory_page_origin: Option<&str>, +) -> Result<(), ValidationError> { + let (Some(domain), Some(origin)) = (inventory_domain, inventory_page_origin) else { + if inventory_domain.is_none() && inventory_page_origin.is_none() { return Ok(()); } return Err(ValidationError::new( @@ -309,19 +331,33 @@ fn validate_inventory_identity_override(config: &ApsConfig) -> Result<(), Valida Ok(()) } +#[cfg(test)] +fn validate_inventory_identity_override( + config: &LegacyApsProviderConfig, +) -> Result<(), ValidationError> { + validate_inventory_identity_override_values( + config.inventory_domain.as_deref(), + config.inventory_page_origin.as_deref(), + ) +} + +#[cfg(test)] fn default_enabled() -> bool { false } +#[cfg(test)] fn default_endpoint() -> String { "https://web.ads.aps.amazon-adsystem.com/e/pb/bid".to_string() } +#[cfg(test)] fn default_timeout_ms() -> u32 { 800 } -impl Default for ApsConfig { +#[cfg(test)] +impl Default for LegacyApsProviderConfig { fn default() -> Self { Self { enabled: false, @@ -337,20 +373,93 @@ impl Default for ApsConfig { } } +/// Browser integration toggle retained independently from APS server providers. +#[derive(Debug, Clone, Default, Deserialize, Serialize, Validate)] +#[serde(deny_unknown_fields)] +pub struct ApsConfig { + /// Whether browser-side APS integration behavior is enabled. + #[serde(default)] + pub enabled: bool, + /// Rendering owner for selected APS bids. + #[serde(default)] + pub rendering_mode: ApsRenderingMode, +} + +#[cfg(test)] +impl IntegrationConfig for LegacyApsProviderConfig { + fn is_enabled(&self) -> bool { + self.enabled + } +} + impl IntegrationConfig for ApsConfig { fn is_enabled(&self) -> bool { self.enabled } } +/// Typed server-side APS profile configuration used by the auction compiler. +#[derive(Debug, Clone, Deserialize)] +#[serde(deny_unknown_fields)] +pub(crate) struct ApsProfileConfig { + #[serde(deserialize_with = "deserialize_account_id")] + pub(crate) account_id: String, + #[serde(default)] + pub(crate) debug: bool, + #[serde(default)] + pub(crate) allow_script_creatives: bool, + #[serde(default)] + pub(crate) inventory_domain: Option, + #[serde(default)] + pub(crate) inventory_page_origin: Option, +} + +/// Parse and validate server-owned APS profile fields without browser enablement. +pub(crate) fn compile_profile_config( + value: serde_json::Value, +) -> Result> { + let profile: ApsProfileConfig = serde_json::from_value(value).map_err(|error| { + Report::new(TrustedServerError::Configuration { + message: format!("invalid `aps` profile_config: {error}"), + }) + })?; + if let Some(domain) = profile.inventory_domain.as_deref() { + validate_inventory_domain(domain).map_err(|error| { + Report::new(TrustedServerError::Configuration { + message: format!("invalid `aps` profile_config inventory_domain: {error}"), + }) + })?; + } + if let Some(origin) = profile.inventory_page_origin.as_deref() { + validate_inventory_page_origin(origin).map_err(|error| { + Report::new(TrustedServerError::Configuration { + message: format!("invalid `aps` profile_config inventory_page_origin: {error}"), + }) + })?; + } + validate_inventory_identity_override_values( + profile.inventory_domain.as_deref(), + profile.inventory_page_origin.as_deref(), + ) + .map_err(|error| { + Report::new(TrustedServerError::Configuration { + message: format!("invalid `aps` profile_config inventory identity: {error}"), + }) + })?; + Ok(profile) +} + +#[cfg(test)] #[derive(Debug, Serialize)] struct ApsRequestExt<'a> { account: &'a str, sdk: ApsSdkExt, } +#[cfg(test)] impl ToExt for ApsRequestExt<'_> {} +#[cfg(test)] #[derive(Debug, Serialize)] struct ApsSdkExt { source: &'static str, @@ -367,21 +476,510 @@ struct ApsRendererInput<'a> { height: u32, } -#[derive(Clone)] -struct ApsDebugRequest { +#[derive(Debug, Clone)] +pub(crate) struct ApsDebugRequest { body: String, headers: BTreeMap>, } -/// APS `OpenRTB` auction provider. +impl ApsDebugRequest { + pub(crate) fn capture(body: &[u8], headers: &HeaderMap) -> Self { + Self { + body: String::from_utf8_lossy(body).into_owned(), + headers: aps_debug_headers(headers), + } + } +} + +struct PlannedApsResponsePolicy<'a> { + provider_id: &'a str, + endpoint: &'a str, + account_id: &'a str, + debug: bool, + allow_script_creatives: bool, + publisher_domain: &'a str, +} + +fn aps_debug_headers(headers: &HeaderMap) -> BTreeMap> { + // This metadata is client-visible. Keep the list fail-closed so upstream + // identity or authentication headers can never leak. + const ALLOWED_HEADERS: &[HeaderName] = &[header::CONTENT_TYPE]; + + let mut values = BTreeMap::>::new(); + for (name, value) in headers { + if !ALLOWED_HEADERS.contains(name) { + continue; + } + let Ok(value) = value.to_str() else { + continue; + }; + values + .entry(name.as_str().to_string()) + .or_default() + .push(value.to_string()); + } + values +} + +fn aps_debug_body_preview(body: &[u8]) -> String { + let preview_len = body.len().min(MAX_DEBUG_RESPONSE_PREVIEW_BYTES); + let mut preview = String::from_utf8_lossy(&body[..preview_len]).into_owned(); + if body.len() > preview_len { + preview.push_str(&format!("…(truncated {} bytes)", body.len() - preview_len)); + } + preview +} + +fn attach_planned_aps_metadata( + mut response: AuctionResponse, + policy: &PlannedApsResponsePolicy<'_>, + input: &ProviderAuctionInput, + debug_request: Option, + response_body: Option<&[u8]>, + response_headers: &BTreeMap>, + status: StatusCode, +) -> AuctionResponse { + response.metadata.insert( + "routing".to_string(), + json!({ + "unused_bidder_params_count": ignored_bidder_params_count(input) + }), + ); + if !policy.debug { + return response; + } + + let mut http_call = json!({ + "responseheaders": response_headers, + "status": status.as_u16(), + "uri": policy.endpoint, + }); + if let Some(http_call) = http_call.as_object_mut() { + if let Some(request) = debug_request { + http_call.insert("requestbody".to_string(), json!(request.body)); + http_call.insert("requestheaders".to_string(), json!(request.headers)); + } + if let Some(response_body) = response_body { + http_call.insert( + "responsebody".to_string(), + json!(aps_debug_body_preview(response_body)), + ); + } + } + response.with_metadata( + "debug", + json!({ + "httpcalls": { + (APS_INTEGRATION_ID): [http_call] + } + }), + ) +} + +fn planned_aps_renderer( + policy: &PlannedApsResponsePolicy<'_>, + input: ApsRendererInput<'_>, +) -> Option { + let tag_type_value = match input.tag_type { + ApsTagType::Iframe => "iframe", + ApsTagType::Script => "script", + }; + let envelope = json!({ + "seatbid": [{ + "bid": [{ + "id": input.bid_id, + "price": input.price, + "w": input.width, + "h": input.height, + "ext": { + "creativeurl": input.creative_url, + "tagtype": tag_type_value + } + }] + }] + }); + let serialized = serde_json::to_vec(&envelope).ok()?; + if serialized.len() > MAX_RENDER_ENVELOPE_BYTES { + return None; + } + Some(BidRenderer::Aps(ApsRendererV1 { + version: 1, + account_id: policy.account_id.to_string(), + bid_id: input.bid_id.to_string(), + creative_id: input.creative_id, + tag_type: input.tag_type, + creative_url: input.creative_url.to_string(), + aax_response: BASE64_STANDARD.encode(serialized), + width: input.width, + height: input.height, + })) +} + +fn planned_aps_valid_creative_url(value: &str, publisher_domain: &str) -> bool { + if value.len() > MAX_CREATIVE_URL_BYTES { + return false; + } + let Ok(parsed) = Url::parse(value) else { + return false; + }; + parsed.scheme() == "https" + && parsed + .host_str() + .is_some_and(|host| !host.eq_ignore_ascii_case(publisher_domain)) + && parsed.username().is_empty() + && parsed.password().is_none() +} + +fn planned_aps_parse_bid( + policy: &PlannedApsResponsePolicy<'_>, + value: &Json, + slots: &HashMap<&str, HashSet<(u32, u32)>>, + returned_seat: Option<&str>, +) -> Result { + let bid_id = value + .get("id") + .and_then(Json::as_str) + .filter(|value| !value.is_empty()) + .ok_or("missing_render_source")?; + let slot_id = value + .get("impid") + .and_then(Json::as_str) + .ok_or("unknown_impid")?; + let dimensions = slots.get(slot_id).ok_or("unknown_impid")?; + let price = value + .get("price") + .and_then(Json::as_f64) + .filter(|price| price.is_finite() && *price >= 0.0) + .ok_or("invalid_price")?; + if value + .get("mtype") + .is_some_and(|mtype| mtype.as_i64() != Some(1)) + { + return Err("unsupported_media_type"); + } + let width = value + .get("w") + .and_then(Json::as_u64) + .and_then(|value| u32::try_from(value).ok()) + .ok_or("invalid_dimensions")?; + let height = value + .get("h") + .and_then(Json::as_u64) + .and_then(|value| u32::try_from(value).ok()) + .ok_or("invalid_dimensions")?; + if width == 0 || height == 0 || !dimensions.contains(&(width, height)) { + return Err("invalid_dimensions"); + } + let ext = value + .get("ext") + .and_then(Json::as_object) + .ok_or("missing_render_source")?; + let creative_url = ext + .get("creativeurl") + .and_then(Json::as_str) + .ok_or("missing_render_source")?; + if !planned_aps_valid_creative_url(creative_url, policy.publisher_domain) { + return Err("invalid_creative_url"); + } + let tag_type = match ext.get("tagtype").and_then(Json::as_str) { + Some("iframe") => ApsTagType::Iframe, + Some("script") if policy.allow_script_creatives => ApsTagType::Script, + Some("script") => return Err("script_rendering_disabled"), + _ => return Err("unsupported_tagtype"), + }; + let creative_id = value + .get("crid") + .and_then(Json::as_str) + .filter(|creative_id| !creative_id.is_empty()) + .map(str::to_string); + if creative_id + .as_ref() + .is_some_and(|creative_id| creative_id.len() > MAX_CREATIVE_ID_BYTES) + { + return Err("creative_id_too_large"); + } + let renderer = planned_aps_renderer( + policy, + ApsRendererInput { + bid_id, + creative_id: creative_id.clone(), + tag_type, + creative_url, + price, + width, + height, + }, + ) + .ok_or("render_payload_too_large")?; + let adomain = value + .get("adomain") + .and_then(Json::as_array) + .map(|domains| { + domains + .iter() + .filter_map(Json::as_str) + .map(str::to_string) + .collect() + }); + + Ok(Bid { + slot_id: slot_id.to_string(), + price: Some(price), + currency: DEFAULT_CURRENCY.to_string(), + creative: None, + adomain, + bidder: APS_INTEGRATION_ID.to_string(), + returned_seat: returned_seat.map(str::to_string), + width, + height, + nurl: None, + burl: None, + bid_id: Some(bid_id.to_string()), + ad_id: value.get("adid").and_then(Json::as_str).map(str::to_string), + creative_id, + renderer: Some(renderer), + cache_id: None, + cache_host: None, + cache_path: None, + metadata: HashMap::new(), + }) +} + +fn increment_planned_aps_reason(reasons: &mut BTreeMap, reason: &'static str) { + *reasons.entry(reason.to_string()).or_default() += 1; +} + +fn parse_planned_aps_value( + value: &Json, + response_time_ms: u64, + input: &ProviderAuctionInput, + policy: &PlannedApsResponsePolicy<'_>, +) -> AuctionResponse { + if !value.is_object() + || value.get("contextual").is_some() + || value + .get("cur") + .is_some_and(|currency| !currency.is_string()) + || value + .get("seatbid") + .is_some_and(|seatbids| !seatbids.is_array()) + { + return AuctionResponse::error(policy.provider_id, response_time_ms) + .with_metadata("error_type", json!("parse_response")) + .with_metadata("drop_reasons", json!({"unexpected_response_shape": 1})); + } + if value + .get("cur") + .and_then(Json::as_str) + .is_some_and(|currency| !currency.eq_ignore_ascii_case(DEFAULT_CURRENCY)) + { + return AuctionResponse::no_bid(policy.provider_id, response_time_ms) + .with_metadata("drop_reasons", json!({"unsupported_currency": 1})); + } + + let slots = input + .slots() + .iter() + .map(|slot| { + let dimensions = slot + .slot() + .formats + .iter() + .filter(|format| format.media_type == MediaType::Banner) + .map(|format| (format.width, format.height)) + .collect::>(); + (slot.slot().id.as_str(), dimensions) + }) + .collect::>(); + let seatbids = value.get("seatbid").and_then(Json::as_array); + let seatbid_count = seatbids.map_or(0, Vec::len); + let mut reasons = BTreeMap::new(); + let mut selected: HashMap = HashMap::new(); + let mut dropped = 0_u64; + + for seatbid in seatbids.into_iter().flatten() { + let returned_seat = seatbid + .get("seat") + .and_then(Json::as_str) + .filter(|seat| !seat.is_empty()); + let Some(bids) = seatbid.get("bid").and_then(Json::as_array) else { + dropped += 1; + increment_planned_aps_reason(&mut reasons, "empty_seatbid_bids"); + continue; + }; + for value in bids { + match planned_aps_parse_bid(policy, value, &slots, returned_seat) { + Ok(candidate) => { + let replace = selected.get(&candidate.slot_id).is_none_or(|current| { + let candidate_price = candidate.price.unwrap_or_default(); + let current_price = current.price.unwrap_or_default(); + candidate_price > current_price + || (candidate_price == current_price + && candidate.bid_id.as_deref().unwrap_or_default() + < current.bid_id.as_deref().unwrap_or_default()) + }); + if replace { + if selected + .insert(candidate.slot_id.clone(), candidate) + .is_some() + { + dropped += 1; + increment_planned_aps_reason(&mut reasons, "lost_to_higher_bid"); + } + } else { + dropped += 1; + increment_planned_aps_reason(&mut reasons, "lost_to_higher_bid"); + } + } + Err(reason) => { + dropped += 1; + increment_planned_aps_reason(&mut reasons, reason); + } + } + } + } + + if seatbid_count == 0 { + increment_planned_aps_reason(&mut reasons, "empty_seatbid"); + } + let accepted = selected.len(); + let metadata = [ + ("seatbid_count".to_string(), json!(seatbid_count)), + ("accepted_bid_count".to_string(), json!(accepted)), + ("dropped_bid_count".to_string(), json!(dropped)), + ("drop_reasons".to_string(), json!(reasons)), + ]; + let mut response = if selected.is_empty() { + AuctionResponse::no_bid(policy.provider_id, response_time_ms) + } else { + AuctionResponse::success( + policy.provider_id, + selected.into_values().collect(), + response_time_ms, + ) + }; + response.metadata.extend(metadata); + response +} + +/// Parse one APS-profile response using only provider-local routed state. +pub(crate) async fn parse_planned_aps_response( + provider_id: &str, + profile: &ApsProfilePlan, + endpoint: &str, + input: &ProviderAuctionInput, + response: PlatformResponse, + response_time_ms: u64, + debug_request: Option, +) -> Result> { + let policy = PlannedApsResponsePolicy { + provider_id, + endpoint, + account_id: &profile.account_id, + debug: profile.debug, + allow_script_creatives: profile.allow_script_creatives, + publisher_domain: &input.common_request().publisher.domain, + }; + let response = response.response; + let status = response.status(); + let response_headers = if policy.debug { + aps_debug_headers(response.headers()) + } else { + BTreeMap::new() + }; + + if status == StatusCode::NO_CONTENT { + return Ok(attach_planned_aps_metadata( + AuctionResponse::no_bid(provider_id, response_time_ms), + &policy, + input, + debug_request, + Some(&[]), + &response_headers, + status, + )); + } + if !status.is_success() { + log::warn!("APS profile {provider_id} returns a non-success status"); + let body = if policy.debug { + match collect_response_bounded( + response.into_body(), + UPSTREAM_RTB_MAX_RESPONSE_BYTES, + APS_INTEGRATION_ID, + ) + .await + { + Ok(body) => Some(body), + Err(error) => { + log::warn!("Failed to read APS profile debug response body: {error:?}"); + None + } + } + } else { + None + }; + return Ok(attach_planned_aps_metadata( + AuctionResponse::error(provider_id, response_time_ms) + .with_metadata("error_type", json!("http_status")) + .with_metadata("http_status", json!(status.as_u16())), + &policy, + input, + debug_request, + body.as_deref(), + &response_headers, + status, + )); + } + let body = collect_response_bounded( + response.into_body(), + UPSTREAM_RTB_MAX_RESPONSE_BYTES, + APS_INTEGRATION_ID, + ) + .await + .change_context(TrustedServerError::Auction { + message: format!("Failed to read APS profile {provider_id} response body"), + })?; + let value: Json = match serde_json::from_slice(&body) { + Ok(value) => value, + Err(error) => { + log::warn!("Failed to parse APS profile {provider_id} response JSON: {error}"); + let parsed = AuctionResponse::error(provider_id, response_time_ms) + .with_metadata("error_type", json!("parse_response")) + .with_metadata("drop_reasons", json!({"unexpected_response_shape": 1})); + return Ok(attach_planned_aps_metadata( + parsed, + &policy, + input, + debug_request, + Some(&body), + &response_headers, + status, + )); + } + }; + let parsed = parse_planned_aps_value(&value, response_time_ms, input, &policy); + Ok(attach_planned_aps_metadata( + parsed, + &policy, + input, + debug_request, + Some(&body), + &response_headers, + status, + )) +} + +/// Legacy APS `OpenRTB` auction provider retained only for parity tests. +#[cfg(test)] pub struct ApsAuctionProvider { - config: ApsConfig, + config: LegacyApsProviderConfig, } +#[cfg(test)] impl ApsAuctionProvider { /// Create an APS provider from validated configuration. #[must_use] - pub fn new(config: ApsConfig) -> Self { + pub fn new(config: LegacyApsProviderConfig) -> Self { Self { config } } @@ -833,6 +1431,7 @@ impl ApsAuctionProvider { creative: None, adomain, bidder: APS_INTEGRATION_ID.to_string(), + returned_seat: None, width, height, nurl: None, @@ -1059,9 +1658,10 @@ impl ApsAuctionProvider { } } +#[cfg(test)] #[async_trait(?Send)] impl AuctionProvider for ApsAuctionProvider { - fn provider_name(&self) -> &'static str { + fn provider_name(&self) -> &str { APS_INTEGRATION_ID } @@ -1269,19 +1869,22 @@ impl IntegrationHeadInjector for ApsRendererIntegration { /// # Errors /// /// Returns an error when enabled APS configuration is invalid. -pub fn register( +pub fn register_for_plan( settings: &Settings, + plan: &crate::auction::AuctionPlan, ) -> Result, Report> { - let Some(config) = settings.integration_config::(APS_INTEGRATION_ID)? else { + if !plan.has_profile(APS_INTEGRATION_ID) { return Ok(None); - }; - let integration = Arc::new(ApsRendererIntegration { - rendering_mode: config.rendering_mode, - }); + } + let rendering_mode = settings + .integration_config::(APS_INTEGRATION_ID)? + .map(|config| config.rendering_mode) + .unwrap_or_default(); + let integration = Arc::new(ApsRendererIntegration { rendering_mode }); let registration = IntegrationRegistration::builder(APS_INTEGRATION_ID) .without_js() .with_head_injector(integration.clone()); - let registration = if config.rendering_mode == ApsRenderingMode::TrustedServer { + let registration = if rendering_mode == ApsRenderingMode::TrustedServer { registration.with_proxy(integration) } else { registration @@ -1294,10 +1897,54 @@ pub fn register( /// # Errors /// /// Returns an error when enabled APS configuration is invalid. +#[cfg(test)] +#[allow(clippy::missing_panics_doc)] +pub fn register( + settings: &Settings, +) -> Result, Report> { + let Some(config) = + settings.integration_config::(APS_INTEGRATION_ID)? + else { + return Ok(None); + }; + let mut browser_settings = settings.clone(); + browser_settings.integrations.insert_config( + APS_INTEGRATION_ID, + &ApsConfig { + enabled: true, + rendering_mode: config.rendering_mode, + }, + )?; + register_for_plan( + &browser_settings, + &crate::auction::AuctionPlan::compile(AuctionPlanConfig { + timeout_ms: 1000, + providers: BTreeMap::from([( + "aps".parse().expect("should parse APS provider ID"), + ProviderConfig { + protocol: "openrtb-2.6".to_string(), + profile: "aps".to_string(), + endpoint: default_endpoint(), + timeout_ms: None, + routing: RoutingMode::AllEligible, + notifications: NotificationConfig::default(), + profile_config: serde_json::json!({"account_id":"example-account"}), + }, + )]), + ..AuctionPlanConfig::default() + }) + .expect("should compile APS renderer test plan"), + ) +} + +#[cfg(test)] +#[allow(clippy::missing_errors_doc)] pub fn register_providers( settings: &Settings, ) -> Result>, Report> { - let Some(config) = settings.integration_config::(APS_INTEGRATION_ID)? else { + let Some(config) = + settings.integration_config::(APS_INTEGRATION_ID)? + else { return Ok(Vec::new()); }; log::info!("Registering APS OpenRTB provider"); @@ -1317,22 +1964,19 @@ pub fn register_providers( #[cfg(test)] mod tests { use super::*; + use crate::auction::test_support::canonical_parity_auction_request; use crate::auction::types::{ - AdFormat, AdSlot, AuctionContext, AuctionRequest, BidStatus, DeviceInfo, PublisherInfo, - UserInfo, + AdFormat, AdSlot, AuctionContext, AuctionRequest, BidStatus, PublisherInfo, UserInfo, }; - use crate::consent::ConsentContext; use crate::integrations::IntegrationDocumentState; - use crate::openrtb::{Eid, Uid}; - use crate::platform::GeoInfo; use crate::platform::test_support::{ StubHttpClient, build_services_with_http_client, noop_services, }; use crate::test_support::tests::create_test_settings; use serde_json::json; - fn config() -> ApsConfig { - ApsConfig { + fn config() -> LegacyApsProviderConfig { + LegacyApsProviderConfig { enabled: true, account_id: "example-account-id".to_string(), endpoint: default_endpoint(), @@ -1407,6 +2051,7 @@ mod tests { settings: &settings, request: &downstream, timeout_ms: 321, + transport_timeout_ms: 321, provider_responses: None, services: &services, }; @@ -1437,15 +2082,28 @@ mod tests { .expect("should parse APS response with context") } + #[test] + fn config_defaults_to_the_800ms_aps_budget() { + let parsed: LegacyApsProviderConfig = serde_json::from_value(json!({ + "account_id": "example-account" + })) + .expect("should parse APS defaults"); + + assert_eq!( + parsed.timeout_ms, 800, + "should preserve APS's 800ms default" + ); + } + #[test] fn config_accepts_canonical_alias_and_integer_ids() { - let canonical: ApsConfig = serde_json::from_value(json!({ + let canonical: LegacyApsProviderConfig = serde_json::from_value(json!({ "account_id": " example-account " })) .expect("should parse canonical account ID"); - let alias: ApsConfig = + let alias: LegacyApsProviderConfig = serde_json::from_value(json!({"pub_id": 1234})).expect("should parse legacy alias"); - let debug: ApsConfig = serde_json::from_value(json!({ + let debug: LegacyApsProviderConfig = serde_json::from_value(json!({ "account_id": "example-account", "debug": true })) @@ -1462,11 +2120,11 @@ mod tests { #[test] fn config_accepts_default_and_custom_openrtb_endpoints() { - let default = ApsConfig { + let default = LegacyApsProviderConfig { account_id: "example-account".to_string(), ..Default::default() }; - let custom: ApsConfig = serde_json::from_value(json!({ + let custom: LegacyApsProviderConfig = serde_json::from_value(json!({ "account_id": "example-account", "endpoint": "https://aps.example.com/custom/openrtb" })) @@ -1487,7 +2145,7 @@ mod tests { "https://aps.example.com/e/dtb/bid/", "https://aps.example.com/custom/e/dtb/bid", ] { - let parsed: ApsConfig = serde_json::from_value(json!({ + let parsed: LegacyApsProviderConfig = serde_json::from_value(json!({ "account_id": "example-account", "endpoint": endpoint })) @@ -1504,15 +2162,18 @@ mod tests { #[test] fn config_rejects_blank_duplicate_and_unsafe_endpoint() { - assert!(serde_json::from_value::(json!({"account_id": " "})).is_err()); assert!( - serde_json::from_value::( + serde_json::from_value::(json!({"account_id": " "})) + .is_err() + ); + assert!( + serde_json::from_value::( json!({"account_id": "x".repeat(MAX_ACCOUNT_ID_BYTES + 1)}) ) .is_err() ); assert!( - serde_json::from_value::(json!({ + serde_json::from_value::(json!({ "account_id": "one", "pub_id": "two" })) @@ -1520,7 +2181,7 @@ mod tests { ); assert!( serde_json::from_value::(json!({ - "account_id": "example-account", + "enabled": true, "rendering_mode": "unsupported" })) .is_err(), @@ -1531,7 +2192,7 @@ mod tests { "https://", "https://user:password@aps.example/e/pb/bid", ] { - let parsed: ApsConfig = serde_json::from_value(json!({ + let parsed: LegacyApsProviderConfig = serde_json::from_value(json!({ "account_id": "example-account", "endpoint": endpoint })) @@ -1572,7 +2233,7 @@ mod tests { "inventory_page_origin": "https://unrelated.example" }), ] { - let parsed: ApsConfig = + let parsed: LegacyApsProviderConfig = serde_json::from_value(value).expect("should deserialize before validation"); assert!( parsed.validate().is_err(), @@ -1583,7 +2244,7 @@ mod tests { #[test] fn inventory_identity_override_rewrites_site_and_preserves_page_path() { - let config: ApsConfig = serde_json::from_value(json!({ + let config: LegacyApsProviderConfig = serde_json::from_value(json!({ "enabled": true, "account_id": "example-account", "inventory_domain": "publisher.example", @@ -1612,6 +2273,7 @@ mod tests { settings: &settings, request: &downstream, timeout_ms: 321, + transport_timeout_ms: 321, provider_responses: None, services: &services, }; @@ -1641,54 +2303,7 @@ mod tests { #[test] fn builds_aps_openrtb_request_with_explicit_privacy_policy() { let provider = ApsAuctionProvider::new(config()); - let mut auction_request = request(); - auction_request.user.consent = Some(ConsentContext { - gdpr_applies: true, - raw_tc_string: Some("fictional-tcf".to_string()), - raw_us_privacy: Some("1YNN".to_string()), - raw_gpp_string: Some("fictional-gpp".to_string()), - gpp_section_ids: Some(vec![2, 6]), - ..Default::default() - }); - auction_request.user.eids = Some(vec![Eid { - source: "identity.example".to_string(), - uids: vec![Uid { - id: "fictional-uid".to_string(), - atype: Some(1), - ext: None, - }], - }]); - auction_request.slots[0].formats.extend([ - AdFormat { - media_type: MediaType::Video, - width: 640, - height: 480, - }, - AdFormat { - media_type: MediaType::Banner, - width: u32::MAX, - height: 90, - }, - AdFormat { - media_type: MediaType::Banner, - width: 728, - height: 90, - }, - ]); - auction_request.device = Some(DeviceInfo { - user_agent: Some("Fictional Browser".to_string()), - ip: Some("192.0.2.10".to_string()), - geo: Some(GeoInfo { - city: "Example City".to_string(), - country: "US".to_string(), - continent: "NA".to_string(), - latitude: 12.34, - longitude: 56.78, - metro_code: 501, - region: Some("CA".to_string()), - asn: None, - }), - }); + let auction_request = canonical_parity_auction_request(); let settings = create_test_settings(); let services = noop_services(); let downstream = http::Request::builder() @@ -1702,12 +2317,13 @@ mod tests { settings: &settings, request: &downstream, timeout_ms: 321, + transport_timeout_ms: 321, provider_responses: None, services: &services, }; let openrtb = provider.build_openrtb_request(&auction_request, &context); - let serialized = serde_json::to_value(openrtb).expect("should serialize request"); + let serialized = serde_json::to_value(&openrtb).expect("should serialize request"); assert_eq!(serialized["id"], "fictional-auction"); assert_eq!(serialized["tmax"], 321); @@ -1766,6 +2382,19 @@ mod tests { assert!(serialized["ext"].get("prebid").is_none()); assert!(serialized["ext"].get("trusted_server").is_none()); assert!(serialized["imp"][0].get("ext").is_none()); + assert!( + serialized["user"]["ext"].get("ConsentSettings").is_none(), + "should omit PBS-only Google Additional Consent placement" + ); + assert!( + serialized["imp"][0].get("tagid").is_none(), + "should ignore shared trustedServer bidder parameters" + ); + assert_eq!( + serde_json::to_string(&openrtb).expect("should serialize APS request"), + r#"{"id":"fictional-auction","imp":[{"id":"fictional-slot","banner":{"format":[{"w":300,"h":250},{"w":728,"h":90}],"w":300,"h":250,"topframe":0},"bidfloor":1.0,"bidfloorcur":"USD","secure":1}],"site":{"domain":"publisher.example","page":"https://publisher.example/article","publisher":{"domain":"publisher.example"}},"device":{"geo":{"type":2,"country":"US","region":"CA","metro":"501","city":"Example City"},"dnt":1,"ua":"Fictional Browser","ip":"192.0.2.10","language":"en"},"user":{"id":"fictional-user","consent":"fictional-tcf","ext":{"consent":"fictional-tcf","eids":[{"source":"identity.example","uids":[{"atype":1,"id":"fictional-uid"}]}]}},"tmax":321,"cur":["USD"],"regs":{"gdpr":1,"us_privacy":"1YNN","gpp":"fictional-gpp","gpp_sid":[2,6],"ext":{"gdpr":1,"gpp":"fictional-gpp","gpp_sid":[2,6],"us_privacy":"1YNN"}},"ext":{"account":"example-account-id","sdk":{"source":"prebid","version":"2.2.0"}}}"#, + "should preserve the complete APS wire shape without a signing extension" + ); } #[test] @@ -1786,6 +2415,7 @@ mod tests { settings: &settings, request: &downstream, timeout_ms: 321, + transport_timeout_ms: 321, provider_responses: None, services: &services, }; @@ -1802,13 +2432,17 @@ mod tests { fn parses_bid_and_builds_exact_minimized_envelope() { let provider = ApsAuctionProvider::new(config()); let response = provider.parse_aps_response( - &json!({"cur": "USD", "seatbid": [{"seat": 42, "bid": [bid("fictional-selected-bid-id", 1.23, "iframe")]}], "ext": {"userSyncs": []}}), + &json!({"cur": "USD", "seatbid": [{"seat": "fictional-upstream-seat", "bid": [bid("fictional-selected-bid-id", 1.23, "iframe")]}], "ext": {"userSyncs": []}}), 12, &request(), ); assert_eq!(response.bids.len(), 1); let parsed = &response.bids[0]; assert_eq!(parsed.bidder, "aps"); + assert!( + parsed.returned_seat.is_none(), + "legacy APS parsing must not attach planned telemetry identity" + ); assert_eq!(parsed.price, Some(1.23)); assert!(parsed.creative.is_none()); assert!(parsed.nurl.is_none()); @@ -1963,6 +2597,7 @@ mod tests { settings: &settings, request: &downstream, timeout_ms: 321, + transport_timeout_ms: 321, provider_responses: None, services: &services, }; diff --git a/crates/trusted-server-core/src/integrations/datadome.rs b/crates/trusted-server-core/src/integrations/datadome.rs index d95ee35ee..ba2a217c9 100644 --- a/crates/trusted-server-core/src/integrations/datadome.rs +++ b/crates/trusted-server-core/src/integrations/datadome.rs @@ -68,7 +68,6 @@ use serde_json::Value as JsonValue; use url::Url; use validator::Validate; -use crate::constants::ENV_FASTLY_IS_STAGING; use crate::error::TrustedServerError; use crate::integrations::{ AttributeRewriteAction, INTEGRATION_MAX_BODY_BYTES, IntegrationAttributeContext, @@ -78,6 +77,7 @@ use crate::integrations::{ collect_body_bounded, collect_response_bounded, ensure_integration_backend, }; use crate::platform::{PlatformHttpRequest, RuntimeServices}; +use crate::redacted::Redacted; use crate::settings::{IntegrationConfig, Settings}; mod protection; @@ -90,7 +90,8 @@ pub use protection_scope::{ use protection_scope::ProtectionScope; pub(crate) const DATADOME_INTEGRATION_ID: &str = "datadome"; -/// Fixed request header used by the staging-only protection test bypass. +pub(super) const MIN_TEST_BYPASS_CREDENTIAL_BYTES: usize = 32; +/// Fixed request header used by the configuration-gated protection test bypass. pub(crate) const HEADER_DATADOME_TEST_BYPASS: &str = "x-ts-datadome-bypass"; /// Request marker indicating that Trusted Server should omit its automatic @@ -122,7 +123,7 @@ static DATADOME_URL_PATTERN: LazyLock = LazyLock::new(|| { /// Temporary static-header bypass for server-side `DataDome` protection. /// -/// This is intended only for an access-controlled staging environment. A +/// This is intended only for an access-controlled test environment. A /// matching `x-ts-datadome-bypass` header bypasses the server-side Protection /// API and is removed before the publisher origin receives the request. The /// credential itself is loaded from the Secret Store at runtime. @@ -133,13 +134,16 @@ pub struct ProtectionTestBypassConfig { #[serde(default)] pub enabled: bool, - /// Secret Store containing the temporary bypass credential. - #[serde(default = "default_protection_test_bypass_secret_store")] - pub credential_secret_store: String, + /// Deprecated feature-specific store selector accepted for migration only. + #[serde(default)] + pub credential_secret_store: Option, - /// Secret name containing at least 32 bytes of high-entropy bypass material. - #[serde(default = "default_protection_test_bypass_secret_name")] - pub credential_secret_name: String, + /// Secret reference containing the bypass credential. + /// + /// Holds the store key name in app config and the resolved credential at + /// runtime. Treat it as secret material after settings are built. + #[serde(default)] + pub credential_secret_name: Option>, } /// Configuration for `DataDome` integration. @@ -175,13 +179,16 @@ pub struct DataDomeConfig { #[serde(default)] pub enable_protection: bool, - /// Runtime secret store containing the `DataDome` server-side key. - #[serde(default = "default_server_side_key_secret_store")] - pub server_side_key_secret_store: String, + /// Deprecated feature-specific store selector accepted for migration only. + #[serde(default)] + pub server_side_key_secret_store: Option, - /// Secret name containing the `DataDome` server-side key. - #[serde(default = "default_server_side_key_secret_name")] - pub server_side_key_secret_name: String, + /// Secret reference containing the `DataDome` server-side key. + /// + /// Holds the store key name in app config and the resolved key at runtime. + /// Treat it as secret material after settings are built. + #[serde(default)] + pub server_side_key_secret_name: Option>, /// Base URL for the `DataDome` Protection API. #[serde(default = "default_protection_api_origin")] @@ -224,7 +231,7 @@ pub struct DataDomeConfig { )] pub protection_exclusion_rules: Vec, - /// Temporary static-header bypass for access-controlled staging tests. + /// Temporary static-header bypass for access-controlled tests. #[serde(default)] pub protection_test_bypass: Option, @@ -273,22 +280,6 @@ fn default_protection_api_origin() -> String { "https://api-fastly.datadome.co".to_string() } -fn default_server_side_key_secret_store() -> String { - "ts_secrets".to_string() -} - -fn default_server_side_key_secret_name() -> String { - "datadome_server_side_key".to_string() -} - -fn default_protection_test_bypass_secret_store() -> String { - "ts_secrets".to_string() -} - -fn default_protection_test_bypass_secret_name() -> String { - "datadome_test_bypass".to_string() -} - fn default_timeout_ms() -> u32 { 1500 } @@ -356,8 +347,8 @@ impl Default for DataDomeConfig { cache_ttl_seconds: default_cache_ttl(), rewrite_sdk: default_rewrite_sdk(), enable_protection: false, - server_side_key_secret_store: default_server_side_key_secret_store(), - server_side_key_secret_name: default_server_side_key_secret_name(), + server_side_key_secret_store: None, + server_side_key_secret_name: None, protection_api_origin: default_protection_api_origin(), timeout_ms: default_timeout_ms(), protection_excluded_methods: default_protection_excluded_methods(), @@ -395,22 +386,35 @@ impl DataDomeIntegration { } fn try_new(mut config: DataDomeConfig) -> Result, Report> { - config.server_side_key_secret_store = - config.server_side_key_secret_store.trim().to_string(); - config.server_side_key_secret_name = config.server_side_key_secret_name.trim().to_string(); + if config.server_side_key_secret_store.take().is_some() { + log::warn!( + "DataDome server_side_key_secret_store is deprecated and ignored; static credentials resolve through the default app-config secret store" + ); + } + config.server_side_key_secret_name = + config.server_side_key_secret_name.take().and_then(|value| { + let value = value.expose().trim().to_string(); + (!value.is_empty()).then(|| Redacted::new(value)) + }); config.protection_api_origin = config.protection_api_origin.trim().to_string(); config.client_side_tag_url = config.client_side_tag_url.trim().to_string(); if let Some(bypass) = &mut config.protection_test_bypass { - bypass.credential_secret_store = bypass.credential_secret_store.trim().to_string(); - bypass.credential_secret_name = bypass.credential_secret_name.trim().to_string(); + if bypass.credential_secret_store.take().is_some() { + log::warn!( + "DataDome credential_secret_store is deprecated and ignored; static credentials resolve through the default app-config secret store" + ); + } + bypass.credential_secret_name = + bypass.credential_secret_name.take().and_then(|value| { + let value = value.expose().trim().to_string(); + (!value.is_empty()).then(|| Redacted::new(value)) + }); } if config.enable_protection { - if config.server_side_key_secret_store.is_empty() - || config.server_side_key_secret_name.is_empty() - { + if config.server_side_key_secret_name.is_none() { return Err(Report::new(Self::error( - "server_side_key_secret_store and server_side_key_secret_name are required when enable_protection is true", + "server_side_key_secret_name is required when enable_protection is true", ))); } Self::validate_protection_api_origin(&config.protection_api_origin)?; @@ -477,11 +481,13 @@ impl DataDomeIntegration { Self::try_new(config).map(|_| ()) } - fn active_protection_test_bypass(&self) -> Option<&ProtectionTestBypassConfig> { - if std::env::var(ENV_FASTLY_IS_STAGING).as_deref() != Ok("1") { - return None; - } + pub(crate) fn validate_config_for_deploy( + config: DataDomeConfig, + ) -> Result<(), Report> { + Self::try_new(config).map(|_| ()) + } + fn active_protection_test_bypass(&self) -> Option<&ProtectionTestBypassConfig> { self.config .protection_test_bypass .as_ref() @@ -504,9 +510,9 @@ impl DataDomeIntegration { "protection_test_bypass requires enable_protection to be true", ))); } - if bypass.credential_secret_store.is_empty() || bypass.credential_secret_name.is_empty() { + if bypass.credential_secret_name.is_none() { return Err(Report::new(Self::error( - "protection_test_bypass credential_secret_store and credential_secret_name must not be empty when enabled", + "protection_test_bypass credential_secret_name is required when enabled", ))); } @@ -945,17 +951,7 @@ fn build( }; let integration = DataDomeIntegration::try_new(config)?; - let protection_test_bypass_configured = integration - .config - .protection_test_bypass - .as_ref() - .is_some_and(|bypass| bypass.enabled); let protection_test_bypass_active = integration.active_protection_test_bypass().is_some(); - if protection_test_bypass_configured && !protection_test_bypass_active { - log::warn!( - "[datadome] DataDome test bypass is configured but inactive because FASTLY_IS_STAGING is not 1" - ); - } log::info!( "[datadome] Registering integration (sdk_origin: {}, rewrite_sdk: {}, enable_protection: {}, protection_test_bypass: {})", integration.config.sdk_origin, @@ -963,8 +959,6 @@ fn build( integration.config.enable_protection, if protection_test_bypass_active { "active" - } else if protection_test_bypass_configured { - "configured-inactive" } else { "disabled" }, @@ -1013,6 +1007,7 @@ mod tests { api_origin: "https://api-js.datadome.co".to_string(), cache_ttl_seconds: 3600, rewrite_sdk: true, + server_side_key_secret_name: Some(Redacted::new("server-side-key".to_string())), ..DataDomeConfig::default() } } @@ -1200,14 +1195,11 @@ mod tests { } #[test] - fn protection_secret_defaults_match_sample_config() { + fn protection_secrets_are_absent_by_default() { let config = DataDomeConfig::default(); - assert_eq!(config.server_side_key_secret_store, "ts_secrets"); - assert_eq!( - config.server_side_key_secret_name, - "datadome_server_side_key" - ); + assert!(config.server_side_key_secret_store.is_none()); + assert!(config.server_side_key_secret_name.is_none()); assert!( config.protection_test_bypass.is_none(), "the temporary test bypass should be disabled by default" @@ -1234,33 +1226,39 @@ mod tests { assert!(bypass.enabled, "should retain the enabled flag"); assert_eq!( - bypass.credential_secret_store, "ts_secrets", - "should retain the configured credential Secret Store" + bypass.credential_secret_store.as_deref(), + Some("ts_secrets"), + "should accept the deprecated credential Secret Store" ); assert_eq!( - bypass.credential_secret_name, "datadome_test_bypass", - "should retain the configured credential secret name" + bypass + .credential_secret_name + .as_ref() + .map(Redacted::expose) + .map(String::as_str), + Some("datadome_test_bypass"), + "should retain the configured credential secret reference" ); } #[test] - fn protection_test_bypass_requires_protection_and_secret_references() { - for (enable_protection, store, name, expected_message) in [ + fn protection_test_bypass_requires_protection_and_credential_reference() { + for (enable_protection, credential, expected_message) in [ ( false, - "ts_secrets", - "datadome_test_bypass", + Some("test-bypass-credential"), "requires enable_protection", ), - (true, "", "datadome_test_bypass", "credential_secret_store"), - (true, "ts_secrets", "", "credential_secret_name"), + (true, None, "credential_secret_name"), ] { let mut config = test_config(); config.enable_protection = enable_protection; + config.server_side_key_secret_name = + Some(Redacted::new("resolved-server-key".to_string())); config.protection_test_bypass = Some(ProtectionTestBypassConfig { enabled: true, - credential_secret_store: store.to_string(), - credential_secret_name: name.to_string(), + credential_secret_store: None, + credential_secret_name: credential.map(|value| Redacted::new(value.to_string())), }); let err = match DataDomeIntegration::try_new(config) { @@ -1275,26 +1273,25 @@ mod tests { } #[test] - fn protection_enabled_requires_server_side_key_secret_store() { + fn protection_test_bypass_accepts_short_resolved_credential() { let mut config = test_config(); config.enable_protection = true; - config.server_side_key_secret_store = " ".to_string(); + config.server_side_key_secret_name = Some(Redacted::new("resolved-server-key".to_string())); + config.protection_test_bypass = Some(ProtectionTestBypassConfig { + enabled: true, + credential_secret_store: None, + credential_secret_name: Some(Redacted::new("short".to_string())), + }); - let err = match DataDomeIntegration::try_new(config) { - Ok(_) => panic!("should reject empty store"), - Err(err) => err, - }; - assert!( - format!("{err:?}").contains("server_side_key_secret_store"), - "should mention secret store config" - ); + DataDomeIntegration::try_new(config) + .expect("should defer bypass credential strength enforcement to requests"); } #[test] fn protection_enabled_requires_server_side_key_secret_name() { let mut config = test_config(); config.enable_protection = true; - config.server_side_key_secret_name = " ".to_string(); + config.server_side_key_secret_name = Some(Redacted::new(" ".to_string())); let err = match DataDomeIntegration::try_new(config) { Ok(_) => panic!("should reject empty name"), diff --git a/crates/trusted-server-core/src/integrations/datadome/protection.rs b/crates/trusted-server-core/src/integrations/datadome/protection.rs index 75de88afb..fbbc1b70e 100644 --- a/crates/trusted-server-core/src/integrations/datadome/protection.rs +++ b/crates/trusted-server-core/src/integrations/datadome/protection.rs @@ -13,7 +13,7 @@ use crate::http_util::is_navigation_request; use crate::integrations::{ HeaderMutation, RequestFilterDecision, RequestFilterEffects, RequestFilterInput, }; -use crate::platform::{PlatformBackendSpec, PlatformHttpRequest, RuntimeServices, StoreName}; +use crate::platform::{PlatformBackendSpec, PlatformHttpRequest, RuntimeServices}; use crate::redacted::Redacted; use super::DataDomeIntegration; @@ -21,8 +21,6 @@ use super::protection_scope::{ ProtectionRequestFacts, ProtectionScopeDecision, ProtectionSkipReason, }; -const MIN_TEST_BYPASS_CREDENTIAL_BYTES: usize = 32; - const VALIDATE_REQUEST_PATH: &str = "/validate-request"; const REQUEST_MODULE_NAME: &str = "Trusted-Server-Rust"; const MODULE_VERSION: &str = env!("CARGO_PKG_VERSION"); @@ -43,8 +41,7 @@ impl DataDomeIntegration { &self, mut input: RequestFilterInput<'_>, ) -> RequestFilterDecision { - let test_bypass_matched = - self.take_protection_test_bypass_header(input.request, input.services); + let test_bypass_matched = self.take_protection_test_bypass_header(input.request); if test_bypass_matched { input .request @@ -87,9 +84,9 @@ impl DataDomeIntegration { .ensure_protection_backend(input.services, &api_url) .map_err(ProtectionRequestError::Setup)?; let server_side_key = self - .load_server_side_key(input.services) + .server_side_key() .map_err(ProtectionRequestError::Setup)?; - let payload = self.build_protection_payload(&input, &server_side_key); + let payload = self.build_protection_payload(&input, server_side_key); let encoded_body = form_encode(&payload.fields); let mut builder = request_builder() @@ -175,11 +172,7 @@ impl DataDomeIntegration { true } - fn take_protection_test_bypass_header( - &self, - req: &mut Request, - services: &RuntimeServices, - ) -> bool { + fn take_protection_test_bypass_header(&self, req: &mut Request) -> bool { let supplied_values = req .headers() .get_all(super::HEADER_DATADOME_TEST_BYPASS) @@ -200,28 +193,21 @@ impl DataDomeIntegration { return false; } - let store_name = StoreName::from(bypass.credential_secret_store.as_str()); - let credential = match services - .secret_store() - .get_string(&store_name, &bypass.credential_secret_name) - { - Ok(credential) if credential.len() >= MIN_TEST_BYPASS_CREDENTIAL_BYTES => credential, - Ok(_) => { - log::warn!( - "[datadome] DataDome test bypass credential does not meet security requirements; ignoring bypass header" - ); - return false; - } - Err(err) => { - log::warn!( - "[datadome] Failed to load DataDome test bypass credential; ignoring bypass header: {err:?}" - ); - return false; - } + let Some(credential) = bypass.credential_secret_name.as_ref() else { + log::warn!( + "[datadome] DataDome test bypass credential is unavailable; ignoring bypass header" + ); + return false; }; + if credential.expose().len() < super::MIN_TEST_BYPASS_CREDENTIAL_BYTES { + log::warn!( + "[datadome] DataDome test bypass credential does not meet security requirements; ignoring bypass header" + ); + return false; + } let actual = Sha256::digest(supplied_values[0].as_bytes()); - let expected = Sha256::digest(credential.as_bytes()); + let expected = Sha256::digest(credential.expose().as_bytes()); bool::from(actual.ct_eq(&expected)) } @@ -259,25 +245,15 @@ impl DataDomeIntegration { )) } - fn load_server_side_key( - &self, - services: &RuntimeServices, - ) -> Result, Report> { - let store_name = StoreName::from(self.config.server_side_key_secret_store.as_str()); - let key = services - .secret_store() - .get_string(&store_name, &self.config.server_side_key_secret_name) - .change_context(Self::error( - "Failed to read DataDome server-side key from secret store", - ))?; - let key = key.trim().to_string(); - if key.is_empty() { - return Err(Report::new(Self::error( - "DataDome server-side key secret must not be empty", - ))); - } - - Ok(Redacted::new(key)) + fn server_side_key(&self) -> Result<&Redacted, Report> { + self.config + .server_side_key_secret_name + .as_ref() + .ok_or_else(|| { + Report::new(Self::error( + "DataDome server-side key is unavailable after secret resolution", + )) + }) } fn build_protection_payload( @@ -836,7 +812,7 @@ fn truncate_utf8(value: &str, limit: i32) -> String { mod tests { use std::collections::HashMap; use std::net::{IpAddr, Ipv4Addr}; - use std::sync::{Arc, Mutex}; + use std::sync::Arc; use crate::integrations::datadome::{ DataDomeConfig, ProtectionExclusionRuleConfig, ProtectionMatcherConfig, @@ -852,15 +828,17 @@ mod tests { use super::*; - static FASTLY_IS_STAGING_ENV_LOCK: Mutex<()> = Mutex::new(()); - - fn protection_integration() -> Arc { - let config = DataDomeConfig { + fn protection_config() -> DataDomeConfig { + DataDomeConfig { enabled: true, enable_protection: true, + server_side_key_secret_name: Some(Redacted::new("server-side-key".to_string())), ..DataDomeConfig::default() - }; - DataDomeIntegration::try_new(config).expect("should create integration") + } + } + + fn protection_integration() -> Arc { + DataDomeIntegration::try_new(protection_config()).expect("should create integration") } fn request_for_filter() -> Request { @@ -871,24 +849,19 @@ mod tests { .expect("should build filter request") } - fn filter_with_staging( + fn filter_request( integration: &DataDomeIntegration, settings: &Settings, services: &RuntimeServices, request: &mut Request, ) -> RequestFilterDecision { - let _guard = FASTLY_IS_STAGING_ENV_LOCK - .lock() - .expect("should lock staging environment test guard"); - temp_env::with_var(crate::constants::ENV_FASTLY_IS_STAGING, Some("1"), || { - futures::executor::block_on(integration.filter_protection_request(RequestFilterInput { - settings, - services, - request, - geo_info: None, - is_integration_route: false, - })) - }) + futures::executor::block_on(integration.filter_protection_request(RequestFilterInput { + settings, + services, + request, + geo_info: None, + is_integration_route: false, + })) } fn filter_marks_request( @@ -950,10 +923,12 @@ mod tests { enable_protection: true, protection_test_bypass: Some(ProtectionTestBypassConfig { enabled: true, - credential_secret_store: "ts_secrets".to_string(), - credential_secret_name: "datadome_test_bypass".to_string(), + credential_secret_store: None, + credential_secret_name: Some(Redacted::new( + "temporary-test-credential-32-bytes!".to_string(), + )), }), - ..DataDomeConfig::default() + ..protection_config() }; let integration = DataDomeIntegration::try_new(config).expect("should create integration"); let mut secrets = HashMap::new(); @@ -973,7 +948,7 @@ mod tests { edgezero_core::http::HeaderValue::from_static("temporary-test-credential-32-bytes!"), ); - let decision = filter_with_staging(&integration, &settings, &services, &mut request); + let decision = filter_request(&integration, &settings, &services, &mut request); assert!( matches!(decision, RequestFilterDecision::Continue(_)), @@ -1002,15 +977,17 @@ mod tests { None, Some(ProtectionTestBypassConfig { enabled: false, - credential_secret_store: "ts_secrets".to_string(), - credential_secret_name: "datadome_test_bypass".to_string(), + credential_secret_store: None, + credential_secret_name: Some(Redacted::new( + "temporary-test-credential-32-bytes!".to_string(), + )), }), ] { let config = DataDomeConfig { enabled: true, enable_protection: true, protection_test_bypass, - ..DataDomeConfig::default() + ..protection_config() }; let integration = DataDomeIntegration::try_new(config).expect("should create integration"); @@ -1036,7 +1013,7 @@ mod tests { edgezero_core::http::HeaderValue::from_static("stale-test-credential"), ); - let decision = filter_with_staging(&integration, &settings, &services, &mut request); + let decision = filter_request(&integration, &settings, &services, &mut request); assert!( matches!(decision, RequestFilterDecision::Continue(_)), @@ -1062,33 +1039,26 @@ mod tests { } #[test] - fn protection_test_bypass_is_inactive_outside_staging() { + fn protection_test_bypass_works_without_staging_environment() { let config = DataDomeConfig { enabled: true, enable_protection: true, protection_test_bypass: Some(ProtectionTestBypassConfig { enabled: true, - credential_secret_store: "ts_secrets".to_string(), - credential_secret_name: "datadome_test_bypass".to_string(), + credential_secret_store: None, + credential_secret_name: Some(Redacted::new( + "temporary-test-credential-32-bytes!".to_string(), + )), }), - ..DataDomeConfig::default() + ..protection_config() }; let integration = DataDomeIntegration::try_new(config).expect("should create integration"); let mut secrets = HashMap::new(); - secrets.insert( - "datadome_server_side_key".to_string(), - b"server-side-key".to_vec(), - ); secrets.insert( "datadome_test_bypass".to_string(), b"temporary-test-credential-32-bytes!".to_vec(), ); let http_client = Arc::new(StubHttpClient::new()); - http_client.push_response_with_headers( - 200, - Vec::new(), - vec![(HEADER_DATADOME_RESPONSE, "200")], - ); let services = build_services_with_secret_and_http_client( HashMapSecretStore::new(secrets), http_client.clone(), @@ -1100,44 +1070,26 @@ mod tests { edgezero_core::http::HeaderValue::from_static("temporary-test-credential-32-bytes!"), ); - let _guard = FASTLY_IS_STAGING_ENV_LOCK - .lock() - .expect("should lock staging environment test guard"); - let decision = temp_env::with_var( - crate::constants::ENV_FASTLY_IS_STAGING, - None::<&str>, - || { - futures::executor::block_on(integration.filter_protection_request( - RequestFilterInput { - settings: &settings, - services: &services, - request: &mut request, - geo_info: None, - is_integration_route: false, - }, - )) - }, - ); + let decision = filter_request(&integration, &settings, &services, &mut request); assert!( matches!(decision, RequestFilterDecision::Continue(_)), - "an allowed Protection API response should continue" + "a matching test credential should continue without a staging environment" + ); + assert!( + has_client_tag_suppression_marker(&request), + "the bypass should suppress the automatic DataDome client tag" ); assert!( request .headers() .get(super::super::HEADER_DATADOME_TEST_BYPASS) .is_none(), - "the bypass credential must be stripped outside staging" + "the bypass credential must not reach the publisher origin" ); assert!( - !has_client_tag_suppression_marker(&request), - "the bypass must not suppress the DataDome client tag outside staging" - ); - assert_eq!( - http_client.recorded_backend_names().len(), - 1, - "the bypass must still call the Protection API outside staging" + http_client.recorded_backend_names().is_empty(), + "a matching test credential must not call the Protection API" ); } @@ -1156,10 +1108,12 @@ mod tests { }], protection_test_bypass: Some(ProtectionTestBypassConfig { enabled: true, - credential_secret_store: "ts_secrets".to_string(), - credential_secret_name: "datadome_test_bypass".to_string(), + credential_secret_store: None, + credential_secret_name: Some(Redacted::new( + "temporary-test-credential-32-bytes!".to_string(), + )), }), - ..DataDomeConfig::default() + ..protection_config() }; let integration = DataDomeIntegration::try_new(config).expect("should create integration"); let mut secrets = HashMap::new(); @@ -1179,7 +1133,7 @@ mod tests { edgezero_core::http::HeaderValue::from_static("temporary-test-credential-32-bytes!"), ); - let decision = filter_with_staging(&integration, &settings, &services, &mut request); + let decision = filter_request(&integration, &settings, &services, &mut request); assert!( matches!(decision, RequestFilterDecision::Continue(_)), @@ -1202,10 +1156,12 @@ mod tests { enable_protection: true, protection_test_bypass: Some(ProtectionTestBypassConfig { enabled: true, - credential_secret_store: "ts_secrets".to_string(), - credential_secret_name: "datadome_test_bypass".to_string(), + credential_secret_store: None, + credential_secret_name: Some(Redacted::new( + "temporary-test-credential-32-bytes!".to_string(), + )), }), - ..DataDomeConfig::default() + ..protection_config() }; let integration = DataDomeIntegration::try_new(config).expect("should create integration"); let mut secrets = HashMap::new(); @@ -1234,7 +1190,7 @@ mod tests { edgezero_core::http::HeaderValue::from_static("wrong-credential"), ); - let decision = filter_with_staging(&integration, &settings, &services, &mut request); + let decision = filter_request(&integration, &settings, &services, &mut request); assert!( matches!(decision, RequestFilterDecision::Continue(_)), @@ -1265,10 +1221,12 @@ mod tests { enable_protection: true, protection_test_bypass: Some(ProtectionTestBypassConfig { enabled: true, - credential_secret_store: "ts_secrets".to_string(), - credential_secret_name: "datadome_test_bypass".to_string(), + credential_secret_store: None, + credential_secret_name: Some(Redacted::new( + "temporary-test-credential-32-bytes!".to_string(), + )), }), - ..DataDomeConfig::default() + ..protection_config() }; let integration = DataDomeIntegration::try_new(config).expect("should create integration"); let mut secrets = HashMap::new(); @@ -1302,7 +1260,7 @@ mod tests { ); } - let decision = filter_with_staging(&integration, &settings, &services, &mut request); + let decision = filter_request(&integration, &settings, &services, &mut request); assert!(matches!(decision, RequestFilterDecision::Continue(_))); assert!( @@ -1317,67 +1275,48 @@ mod tests { } #[test] - fn test_bypass_credential_requires_at_least_32_bytes() { - for (credential, should_match) in [ - (Some("1234567890123456789012345678901"), false), - (Some("12345678901234567890123456789012"), true), - (Some(""), false), - (None, false), - ] { - let config = DataDomeConfig { + fn short_test_bypass_credential_is_ignored_without_failing_startup() { + let config = DataDomeConfig { + protection_test_bypass: Some(ProtectionTestBypassConfig { enabled: true, - enable_protection: true, - protection_test_bypass: Some(ProtectionTestBypassConfig { - enabled: true, - credential_secret_store: "ts_secrets".to_string(), - credential_secret_name: "datadome_test_bypass".to_string(), - }), - ..DataDomeConfig::default() - }; - let integration = - DataDomeIntegration::try_new(config).expect("should create integration"); - let mut secrets = HashMap::new(); - secrets.insert( - "datadome_server_side_key".to_string(), - b"server-side-key".to_vec(), - ); - if let Some(credential) = credential { - secrets.insert( - "datadome_test_bypass".to_string(), - credential.as_bytes().to_vec(), - ); - } - let http_client = Arc::new(StubHttpClient::new()); - if !should_match { - http_client.push_response_with_headers( - 200, - Vec::new(), - vec![(HEADER_DATADOME_RESPONSE, "200")], - ); - } - let services = build_services_with_secret_and_http_client( - HashMapSecretStore::new(secrets), - http_client.clone(), - ); - let settings = Settings::default(); - let mut request = request_for_filter(); - let supplied = credential.unwrap_or("12345678901234567890123456789012"); - request.headers_mut().insert( - super::super::HEADER_DATADOME_TEST_BYPASS, - edgezero_core::http::HeaderValue::from_str(supplied) - .expect("should build bypass header"), - ); + credential_secret_store: None, + credential_secret_name: Some(Redacted::new("short".to_string())), + }), + ..protection_config() + }; + let integration = + DataDomeIntegration::try_new(config).expect("should accept short bypass credential"); + let http_client = Arc::new(StubHttpClient::new()); + http_client.push_response_with_headers( + 200, + Vec::new(), + vec![(HEADER_DATADOME_RESPONSE, "200")], + ); + let services = + build_services_with_secret_and_http_client(NoopSecretStore, http_client.clone()); + let settings = Settings::default(); + let mut request = request_for_filter(); + request.headers_mut().insert( + super::super::HEADER_DATADOME_TEST_BYPASS, + edgezero_core::http::HeaderValue::from_static("short"), + ); - let decision = filter_with_staging(&integration, &settings, &services, &mut request); + let decision = filter_request(&integration, &settings, &services, &mut request); - assert!(matches!(decision, RequestFilterDecision::Continue(_))); - assert_eq!(has_client_tag_suppression_marker(&request), should_match); - assert_eq!( - http_client.recorded_backend_names().is_empty(), - should_match, - "only a credential meeting the minimum should skip the API" - ); - } + assert!(matches!(decision, RequestFilterDecision::Continue(_))); + assert!( + request + .headers() + .get(super::super::HEADER_DATADOME_TEST_BYPASS) + .is_none(), + "the invalid bypass credential should not reach the publisher origin" + ); + assert!(!has_client_tag_suppression_marker(&request)); + assert_eq!( + http_client.recorded_backend_names().len(), + 1, + "a short credential should not bypass the Protection API" + ); } #[test] @@ -1417,7 +1356,7 @@ mod tests { enabled: true, enable_protection: true, protection_excluded_ip_cidrs: vec!["192.0.2.0/24".to_string()], - ..DataDomeConfig::default() + ..protection_config() }; let inline_request = filter_marks_request(inline.clone(), &noop_services_with_client_ip(ip)); @@ -1456,7 +1395,7 @@ mod tests { cidrs: vec!["192.0.2.0/24".to_string()], }, }], - ..DataDomeConfig::default() + ..protection_config() }; let structured_request = filter_marks_request(structured_ip, &noop_services_with_client_ip(ip)); @@ -1477,7 +1416,7 @@ mod tests { key: "structured-source".to_string(), }, }], - ..DataDomeConfig::default() + ..protection_config() }; let mut structured_values = HashMap::new(); structured_values.insert("structured-source".to_string(), "192.0.2.0/24".to_string()); @@ -1534,7 +1473,7 @@ mod tests { methods: Vec::new(), matcher, }], - ..DataDomeConfig::default() + ..protection_config() }; let request = filter_marks_request_for_uri(config, &noop_services_with_client_ip(ip), None, uri); @@ -1569,7 +1508,7 @@ mod tests { }, }, ], - ..DataDomeConfig::default() + ..protection_config() }; let request = filter_marks_request(config, &noop_services_with_client_ip(ip)); @@ -1586,7 +1525,7 @@ mod tests { enabled: true, enable_protection: true, protection_excluded_asns: vec![64500], - ..DataDomeConfig::default() + ..protection_config() }; let geo_info = GeoInfo { city: String::new(), @@ -1615,7 +1554,7 @@ mod tests { enabled: true, enable_protection: true, protection_excluded_ip_cidrs: vec!["192.0.2.0/24".to_string()], - ..DataDomeConfig::default() + ..protection_config() }; let request = filter_marks_request( config, @@ -1628,39 +1567,27 @@ mod tests { } #[test] - fn load_server_side_key_reads_secret_store() { - let mut secrets = HashMap::new(); - secrets.insert( - "datadome_server_side_key".to_string(), - b"secret-from-store".to_vec(), - ); - let services = build_services_with_config_and_secret( - NoopConfigStore, - HashMapSecretStore::new(secrets), - ); + fn server_side_key_uses_resolved_config_value() { let integration = protection_integration(); let key = integration - .load_server_side_key(&services) - .expect("should load server-side key"); + .server_side_key() + .expect("should contain resolved server-side key"); - assert_eq!(key.expose(), "secret-from-store"); + assert_eq!(key.expose(), "server-side-key"); } #[test] - fn load_server_side_key_errors_when_secret_missing() { - let services = build_services_with_config_and_secret(NoopConfigStore, NoopSecretStore); + fn protection_startup_rejects_missing_resolved_server_side_key() { let config = DataDomeConfig { - enabled: true, - enable_protection: true, - server_side_key_secret_name: "missing_server_side_key".to_string(), - ..DataDomeConfig::default() + server_side_key_secret_name: None, + ..protection_config() }; - let integration = DataDomeIntegration::try_new(config).expect("should create integration"); - - let result = integration.load_server_side_key(&services); - assert!(result.is_err(), "should error when secret is missing"); + assert!( + DataDomeIntegration::try_new(config).is_err(), + "should reject a missing resolved server-side key" + ); } #[test] diff --git a/crates/trusted-server-core/src/integrations/didomi.rs b/crates/trusted-server-core/src/integrations/didomi.rs index f8472b796..4b0989183 100644 --- a/crates/trusted-server-core/src/integrations/didomi.rs +++ b/crates/trusted-server-core/src/integrations/didomi.rs @@ -412,7 +412,14 @@ mod tests { .insert_config(DIDOMI_INTEGRATION_ID, &config(true)) .expect("should insert config"); - let registry = IntegrationRegistry::new(&settings).expect("should create registry"); + let registry = IntegrationRegistry::with_plan( + &settings, + Arc::new( + crate::auction::compile_auction_plan(&settings) + .expect("should compile auction plan"), + ), + ) + .expect("should create registry"); assert!(registry.has_route(&Method::GET, "/integrations/didomi/consent/loader.js")); assert!(registry.has_route(&Method::POST, "/integrations/didomi/consent/api/events")); assert!(!registry.has_route(&Method::GET, "/other")); @@ -505,7 +512,14 @@ mod tests { .insert_config(DIDOMI_INTEGRATION_ID, &custom_config) .expect("should insert config"); - let registry = IntegrationRegistry::new(&settings).expect("should create registry"); + let registry = IntegrationRegistry::with_plan( + &settings, + Arc::new( + crate::auction::compile_auction_plan(&settings) + .expect("should compile auction plan"), + ), + ) + .expect("should create registry"); assert!(registry.has_route(&Method::GET, "/my-custom-consent/loader.js")); assert!(registry.has_route(&Method::POST, "/my-custom-consent/api/events")); assert!(!registry.has_route(&Method::GET, "/integrations/didomi/consent/loader.js")); diff --git a/crates/trusted-server-core/src/integrations/google_tag_manager.rs b/crates/trusted-server-core/src/integrations/google_tag_manager.rs index 162e9eb8c..43eb1d9a4 100644 --- a/crates/trusted-server-core/src/integrations/google_tag_manager.rs +++ b/crates/trusted-server-core/src/integrations/google_tag_manager.rs @@ -1638,7 +1638,14 @@ container_id = "GTM-DEFAULT" ) .expect("should update gtm config"); - let registry = IntegrationRegistry::new(&settings).expect("should create registry"); + let registry = IntegrationRegistry::with_plan( + &settings, + Arc::new( + crate::auction::compile_auction_plan(&settings) + .expect("should compile auction plan"), + ), + ) + .expect("should create registry"); let config = config_from_settings(&settings, ®istry); let processor = create_html_processor(config); let pipeline_config = PipelineConfig { @@ -1678,7 +1685,14 @@ container_id = "GTM-DEFAULT" .expect("should update gtm config"); // 2. Setup Pipeline - let registry = IntegrationRegistry::new(&settings).expect("should create registry"); + let registry = IntegrationRegistry::with_plan( + &settings, + Arc::new( + crate::auction::compile_auction_plan(&settings) + .expect("should compile auction plan"), + ), + ) + .expect("should create registry"); let config = config_from_settings(&settings, ®istry); let processor = create_html_processor(config); let pipeline_config = PipelineConfig { @@ -1744,7 +1758,14 @@ container_id = "GTM-DEFAULT" .expect("should update config"); // Inlined Pipeline Creation - let registry = IntegrationRegistry::new(&settings).expect("should create registry"); + let registry = IntegrationRegistry::with_plan( + &settings, + Arc::new( + crate::auction::compile_auction_plan(&settings) + .expect("should compile auction plan"), + ), + ) + .expect("should create registry"); let config = config_from_settings(&settings, ®istry); let processor = create_html_processor(config); let pipeline_config = PipelineConfig { @@ -2070,7 +2091,14 @@ container_id = "GTM-DEFAULT" ) .expect("should update config"); - let registry = IntegrationRegistry::new(&settings).expect("should create registry"); + let registry = IntegrationRegistry::with_plan( + &settings, + Arc::new( + crate::auction::compile_auction_plan(&settings) + .expect("should compile auction plan"), + ), + ) + .expect("should create registry"); let config = config_from_settings(&settings, ®istry); let processor = create_html_processor(config); @@ -2136,7 +2164,14 @@ container_id = "GTM-DEFAULT" ) .expect("should update nextjs config"); - let registry = IntegrationRegistry::new(&settings).expect("should create registry"); + let registry = IntegrationRegistry::with_plan( + &settings, + Arc::new( + crate::auction::compile_auction_plan(&settings) + .expect("should compile auction plan"), + ), + ) + .expect("should create registry"); let config = config_from_settings(&settings, ®istry); let processor = create_html_processor(config); @@ -2202,7 +2237,14 @@ container_id = "GTM-DEFAULT" ) .expect("should update nextjs config"); - let registry = IntegrationRegistry::new(&settings).expect("should create registry"); + let registry = IntegrationRegistry::with_plan( + &settings, + Arc::new( + crate::auction::compile_auction_plan(&settings) + .expect("should compile auction plan"), + ), + ) + .expect("should create registry"); let config = config_from_settings(&settings, ®istry); let processor = create_html_processor(config); diff --git a/crates/trusted-server-core/src/integrations/gpt.rs b/crates/trusted-server-core/src/integrations/gpt.rs index 84158c27e..9a0905455 100644 --- a/crates/trusted-server-core/src/integrations/gpt.rs +++ b/crates/trusted-server-core/src/integrations/gpt.rs @@ -1246,12 +1246,16 @@ mod tests { "should set ts_initial sentinel" ); assert!( - !combined.contains("addEventListener(\"slotRenderEnded\""), - "inline bootstrap cannot prove TS creative rendering from GPT slotRenderEnded" + combined.contains("addEventListener(\"slotRequested\""), + "should observe publisher GPT requests before delayed adInit" + ); + assert!( + combined.contains("addEventListener(\"slotRenderEnded\""), + "should observe publisher GPT renders before delayed adInit" ); assert!( !combined.contains("sendBeacon"), - "inline bootstrap must not fire win/billing beacons from GPT slotRenderEnded" + "inline bootstrap lifecycle ownership must not fire win/billing beacons" ); assert!( !combined.contains("getTargeting(\"hb_adid\")"), diff --git a/crates/trusted-server-core/src/integrations/gpt_bootstrap.js b/crates/trusted-server-core/src/integrations/gpt_bootstrap.js index 2475c5082..c7cceaa80 100644 --- a/crates/trusted-server-core/src/integrations/gpt_bootstrap.js +++ b/crates/trusted-server-core/src/integrations/gpt_bootstrap.js @@ -102,6 +102,179 @@ pubads.__tsInitialLoadHooked = true; }); + var FIRST_IMPRESSION_LEASE_MS = 5000; + var MAX_FIRST_IMPRESSION_SLOTS = 256; + + function firstImpressionState(now) { + var generation = ts.navGeneration || 0; + if ( + !ts.firstImpression || + ts.firstImpression.generation !== generation + ) { + ts.firstImpression = { + generation: generation, + nextToken: 0, + slots: {}, + fallbackSlots: {}, + }; + } + var state = ts.firstImpression; + state.slots = state.slots || {}; + state.fallbackSlots = state.fallbackSlots || {}; + Object.keys(state.slots).forEach(function (elementId) { + var claim = state.slots[elementId]; + if ( + claim.generation !== generation || + claim.slotElementId !== elementId || + claim.element.ownerDocument !== document || + claim.element !== document.getElementById(elementId) || + !claim.element.isConnected + ) { + delete state.slots[elementId]; + return; + } + var hasReservedFallback = + claim.owner === "publisher" && + (claim.phase === "auctioning" || claim.phase === "delivery_pending") && + state.fallbackSlots[elementId] === claim.element; + Object.keys(claim.publisherAuctions || {}).forEach(function (token) { + var auction = claim.publisherAuctions[token]; + if ( + auction.expiresAt <= now && + !hasReservedFallback && + !(claim.owner === "trusted_server" && auction.suppressDelivery) + ) { + delete claim.publisherAuctions[token]; + } + }); + if ( + claim.owner === "publisher" && + (claim.phase === "auctioning" || claim.phase === "delivery_pending") && + Object.keys(claim.publisherAuctions || {}).length === 0 && + claim.expiresAt <= now && + !hasReservedFallback + ) { + delete state.slots[elementId]; + } + }); + Object.keys(state.fallbackSlots).forEach(function (elementId) { + var element = state.fallbackSlots[elementId]; + if ( + !element.isConnected || + element.id !== elementId || + document.getElementById(elementId) !== element + ) { + delete state.fallbackSlots[elementId]; + } + }); + return state; + } + + function firstImpressionClaim(element) { + return firstImpressionState(Date.now()).slots[element.id]; + } + + function storeFirstImpressionClaim(state, claim) { + if ( + !state.slots[claim.slotElementId] && + Object.keys(state.slots).length >= MAX_FIRST_IMPRESSION_SLOTS + ) { + return false; + } + state.slots[claim.slotElementId] = claim; + return true; + } + + function claimFirstImpressionForTrustedServer(element) { + var now = Date.now(); + var state = firstImpressionState(now); + var existing = state.slots[element.id]; + if (existing) { + var canTransitionPublisherFallback = + existing.owner === "publisher" && + existing.phase !== "requested" && + existing.phase !== "rendered" && + existing.expiresAt <= now && + state.fallbackSlots[element.id] === element; + if (!canTransitionPublisherFallback) return null; + existing.owner = "trusted_server"; + existing.phase = "delivery_pending"; + existing.expiresAt = now + FIRST_IMPRESSION_LEASE_MS; + Object.keys(existing.publisherAuctions || {}).forEach(function (token) { + existing.publisherAuctions[token].suppressDelivery = true; + }); + return existing; + } + var claim = { + generation: state.generation, + slotElementId: element.id, + element: element, + owner: "trusted_server", + phase: "delivery_pending", + expiresAt: now + FIRST_IMPRESSION_LEASE_MS, + publisherAuctions: {}, + }; + return storeFirstImpressionClaim(state, claim) ? claim : null; + } + + function releaseTrustedServerFirstImpressionClaim(element, claim) { + var state = firstImpressionState(Date.now()); + if ( + state.slots[element.id] === claim && + claim.owner === "trusted_server" && + claim.phase === "delivery_pending" + ) { + delete state.slots[element.id]; + if (state.fallbackSlots[element.id] === element) { + delete state.fallbackSlots[element.id]; + } + } + } + + function installFirstImpressionListeners() { + if (ts.firstImpressionListenersInstalled) return; + tag.cmd.push(function () { + if (ts.firstImpressionListenersInstalled) return; + var pubads = window.googletag.pubads(); + if (!pubads || typeof pubads.addEventListener !== "function") return; + var observe = function (phase) { + return function (event) { + var elementId = + event.slot && event.slot.getSlotElementId + ? event.slot.getSlotElementId() + : ""; + var element = elementId && document.getElementById(elementId); + if (!element) return; + var state = firstImpressionState(Date.now()); + var claim = state.slots[elementId]; + if (!claim) { + storeFirstImpressionClaim(state, { + generation: state.generation, + slotElementId: elementId, + element: element, + owner: "publisher", + phase: phase, + expiresAt: Number.POSITIVE_INFINITY, + publisherAuctions: {}, + }); + return; + } + claim.phase = phase; + if (claim.owner === "publisher") { + claim.expiresAt = Number.POSITIVE_INFINITY; + } else { + claim.publisherRegistrationClosed = true; + } + }; + }; + pubads.addEventListener("slotRequested", observe("requested")); + pubads.addEventListener("slotRenderEnded", observe("rendered")); + ts.firstImpressionListenersInstalled = true; + }); + } + + installFirstImpressionListeners(); + // Minimal fallback for tsjs.scheduleInitialAdInit, mirroring the bundle's // hydration-safe scheduler in // crates/trusted-server-js/lib/src/integrations/gpt/index.ts: the @@ -412,10 +585,138 @@ installSlotHandoff(); + function bootstrapTargeting(slot, bid) { + var targeting = Object.assign({}, slot.targeting || {}); + ["hb_pb", "hb_bidder", "hb_adid", "hb_cache_host", "hb_cache_path"].forEach( + function (key) { + if (bid[key]) targeting[key] = String(bid[key]); + }, + ); + targeting.ts_initial = "1"; + return targeting; + } + + function scheduleFirstImpressionFallback(slot, bid, element, generation) { + var state = firstImpressionState(Date.now()); + if (state.fallbackSlots[element.id]) return; + state.fallbackSlots[element.id] = element; + + var retry = function () { + if ( + (ts.navGeneration || 0) !== generation || + !element.isConnected || + document.getElementById(element.id) !== element + ) { + return; + } + var claim = firstImpressionClaim(element); + if (claim) { + if ( + claim.owner !== "publisher" || + claim.phase === "requested" || + claim.phase === "rendered" + ) { + return; + } + var delay = Math.max(0, claim.expiresAt - Date.now()); + if (delay > 0) { + window.setTimeout(retry, delay + 1); + return; + } + } + + tag.cmd.push(function () { + if ( + (ts.navGeneration || 0) !== generation || + !element.isConnected || + document.getElementById(element.id) !== element + ) { + return; + } + var fallbackClaim = claimFirstImpressionForTrustedServer(element); + if (!fallbackClaim) return; + var pubads = window.googletag.pubads(); + var existingSlots = pubads.getSlots ? pubads.getSlots() : []; + var gptSlot = + existingSlots.find(function (candidate) { + return candidate.getSlotElementId() === element.id; + }) || null; + var tsOwned = false; + if (!gptSlot) { + gptSlot = runHandoffInternal(function () { + return window.googletag.defineSlot( + slot.gam_unit_path, + slot.formats, + element.id, + ); + }); + if (!gptSlot) { + releaseTrustedServerFirstImpressionClaim(element, fallbackClaim); + return; + } + gptSlot.addService(pubads); + tsOwned = true; + ts.gptSlotHandoffs = ts.gptSlotHandoffs || {}; + ts.gptSlotHandoffs[element.id] = { + gamUnitPath: slot.gam_unit_path, + formats: slot.formats, + divIdPrefix: slot.div_id, + slotElementId: element.id, + publisherClaimed: false, + suppressPublisherDisplay: false, + suppressPublisherRefresh: false, + }; + } + + var targeting = bootstrapTargeting(slot, bid); + Object.entries(targeting).forEach(function (entry) { + gptSlot.setTargeting(entry[0], entry[1]); + }); + fallbackClaim.targeting = targeting; + var slotElementId = gptSlot.getSlotElementId() || element.id; + ts.divToSlotId = ts.divToSlotId || {}; + ts.divToSlotId[element.id] = slot.id; + ts.divToSlotId[slotElementId] = slot.id; + ts.prevSlotTargetingKeys = ts.prevSlotTargetingKeys || {}; + var targetingKeys = Object.keys(slot.targeting || {}); + ts.prevSlotTargetingKeys[element.id] = targetingKeys; + ts.prevSlotTargetingKeys[slotElementId] = targetingKeys; + if (tsOwned) { + ts.prevGptSlots = ts.prevGptSlots || []; + ts.prevGptSlots.push(gptSlot); + } + if (!ts.servicesEnabled) { + pubads.enableSingleRequest(); + window.googletag.enableServices(); + ts.servicesEnabled = true; + } + if (tsOwned) { + runHandoffInternal(function () { + window.googletag.display(slotElementId); + }); + } + syncInitialLoadDisabled(window.googletag); + if (!tsOwned || ts.gptInitialLoadDisabled) { + ts.adInitRefreshInProgress = true; + try { + runHandoffInternal(function () { + pubads.refresh([gptSlot]); + }); + } finally { + ts.adInitRefreshInProgress = false; + } + } + }); + }; + + retry(); + } + ts.adInit = function () { var slots = ts.adSlots || []; var bids = ts.bids || {}; var divToSlotId = {}; + var nextSlotTargetingKeys = {}; // Generation this invocation belongs to. The slot work below is queued on // googletag.cmd, which drains only when GPT loads; recheck first inside // the queued callback so a navigation committed in the gap cancels the @@ -476,6 +777,14 @@ } var actualDivId = el.id; var b = bids[slot.id] || {}; + var tsClaim = claimFirstImpressionForTrustedServer(el); + if (!tsClaim) { + var currentClaim = firstImpressionClaim(el); + if (currentClaim && currentClaim.owner === "publisher") { + scheduleFirstImpressionFallback(slot, b, el, generation); + } + return; + } var existingSlots = googletag.pubads().getSlots(); var s = @@ -493,7 +802,10 @@ actualDivId, ); }); - if (!s) return; + if (!s) { + releaseTrustedServerFirstImpressionClaim(el, tsClaim); + return; + } s.addService(googletag.pubads()); tsOwned = true; ts.gptSlotHandoffs = ts.gptSlotHandoffs || {}; @@ -508,27 +820,21 @@ }; } - Object.entries(slot.targeting || {}).forEach(function (e) { - s.setTargeting(e[0], e[1]); - }); - [ - "hb_pb", - "hb_bidder", - "hb_adid", - "hb_cache_host", - "hb_cache_path", - ].forEach(function (k) { - if (b[k]) s.setTargeting(k, b[k]); + var targeting = bootstrapTargeting(slot, b); + Object.entries(targeting).forEach(function (entry) { + s.setTargeting(entry[0], entry[1]); }); - // Keep in sync with TS_INITIAL_TARGETING_KEY in index.ts - s.setTargeting("ts_initial", "1"); + tsClaim.targeting = targeting; // Map the resolved inner div to the slot ID. This bootstrap fires no // beacons and registers no slotRenderEnded listener; the map is consumed // by the bundle's render bridge (index.ts) once it loads. divToSlotId[actualDivId] = slot.id; var slotElementId = s.getSlotElementId(); + var targetingKeys = Object.keys(slot.targeting || {}); + nextSlotTargetingKeys[actualDivId] = targetingKeys; if (slotElementId && slotElementId !== actualDivId) { divToSlotId[slotElementId] = slot.id; + nextSlotTargetingKeys[slotElementId] = targetingKeys; } if (tsOwned) { newSlots.push(s); @@ -540,7 +846,10 @@ }); ts.prevGptSlots = newSlots; ts.divToSlotId = divToSlotId; - if (!ts.servicesEnabled) { + ts.prevSlotTargetingKeys = nextSlotTargetingKeys; + var hasRenderableWork = + slotsToDisplay.length > 0 || slotsToRefresh.length > 0; + if (!ts.servicesEnabled && hasRenderableWork) { googletag.pubads().enableSingleRequest(); googletag.enableServices(); ts.servicesEnabled = true; diff --git a/crates/trusted-server-core/src/integrations/gpt_diagnostics.rs b/crates/trusted-server-core/src/integrations/gpt_diagnostics.rs index b4a188f2a..01b444ef9 100644 --- a/crates/trusted-server-core/src/integrations/gpt_diagnostics.rs +++ b/crates/trusted-server-core/src/integrations/gpt_diagnostics.rs @@ -467,7 +467,12 @@ mod tests { #[test] fn register_excludes_diagnostics_from_unified_and_deferred_bundles() { - let registry = IntegrationRegistry::new(&settings(true)).expect("should build registry"); + let settings = settings(true); + let plan = std::sync::Arc::new( + crate::auction::compile_auction_plan(&settings).expect("should compile auction plan"), + ); + let registry = + IntegrationRegistry::with_plan(&settings, plan).expect("should build registry"); assert!(registry.integration_enabled(GPT_DIAGNOSTICS_INTEGRATION_ID)); assert!( @@ -589,7 +594,7 @@ mod tests { assert_eq!( response.headers()[header::CACHE_CONTROL], - "no-store, private", + "private, no-store", "should stamp diagnostics responses non-storable" ); assert_eq!(response.headers()[header::SET_COOKIE], SET_CONSOLE_COOKIE); diff --git a/crates/trusted-server-core/src/integrations/mod.rs b/crates/trusted-server-core/src/integrations/mod.rs index 90d688693..742960970 100644 --- a/crates/trusted-server-core/src/integrations/mod.rs +++ b/crates/trusted-server-core/src/integrations/mod.rs @@ -289,14 +289,6 @@ pub(crate) struct IntegrationBuilder { pub(crate) fn builders() -> &'static [IntegrationBuilder] { &[ - IntegrationBuilder { - id: "aps", - build: aps::register, - }, - IntegrationBuilder { - id: "prebid", - build: prebid::register, - }, IntegrationBuilder { id: "testlight", build: testlight::register, diff --git a/crates/trusted-server-core/src/integrations/nextjs/mod.rs b/crates/trusted-server-core/src/integrations/nextjs/mod.rs index 5452260e7..015ba2475 100644 --- a/crates/trusted-server-core/src/integrations/nextjs/mod.rs +++ b/crates/trusted-server-core/src/integrations/nextjs/mod.rs @@ -160,7 +160,14 @@ mod tests { }), ) .expect("should update nextjs config"); - let registry = IntegrationRegistry::new(&settings).expect("should create registry"); + let registry = IntegrationRegistry::with_plan( + &settings, + Arc::new( + crate::auction::compile_auction_plan(&settings) + .expect("should compile auction plan"), + ), + ) + .expect("should create registry"); let config = config_from_settings(&settings, ®istry); let processor = create_html_processor(config); let pipeline_config = PipelineConfig { @@ -246,7 +253,14 @@ mod tests { }), ) .expect("should update nextjs config"); - let registry = IntegrationRegistry::new(&settings).expect("should create registry"); + let registry = IntegrationRegistry::with_plan( + &settings, + Arc::new( + crate::auction::compile_auction_plan(&settings) + .expect("should compile auction plan"), + ), + ) + .expect("should create registry"); let config = config_from_settings(&settings, ®istry); let processor = create_html_processor(config); let pipeline_config = PipelineConfig { @@ -316,7 +330,14 @@ mod tests { }), ) .expect("should update nextjs config"); - let registry = IntegrationRegistry::new(&settings).expect("should create registry"); + let registry = IntegrationRegistry::with_plan( + &settings, + Arc::new( + crate::auction::compile_auction_plan(&settings) + .expect("should compile auction plan"), + ), + ) + .expect("should create registry"); let config = config_from_settings(&settings, ®istry); let processor = create_html_processor(config); let pipeline_config = PipelineConfig { @@ -362,7 +383,14 @@ mod tests { }), ) .expect("should update nextjs config"); - let registry = IntegrationRegistry::new(&settings).expect("should create registry"); + let registry = IntegrationRegistry::with_plan( + &settings, + Arc::new( + crate::auction::compile_auction_plan(&settings) + .expect("should compile auction plan"), + ), + ) + .expect("should create registry"); let config = config_from_settings(&settings, ®istry); let processor = create_html_processor(config); let pipeline_config = PipelineConfig { @@ -411,7 +439,14 @@ mod tests { }), ) .expect("should update nextjs config"); - let registry = IntegrationRegistry::new(&settings).expect("should create registry"); + let registry = IntegrationRegistry::with_plan( + &settings, + Arc::new( + crate::auction::compile_auction_plan(&settings) + .expect("should compile auction plan"), + ), + ) + .expect("should create registry"); let config = config_from_settings(&settings, ®istry); let processor = create_html_processor(config); let pipeline_config = PipelineConfig { @@ -474,7 +509,14 @@ mod tests { ) .expect("should update nextjs config"); - let registry = IntegrationRegistry::new(&settings).expect("should create registry"); + let registry = IntegrationRegistry::with_plan( + &settings, + Arc::new( + crate::auction::compile_auction_plan(&settings) + .expect("should compile auction plan"), + ), + ) + .expect("should create registry"); let config = config_from_settings(&settings, ®istry); let processor = create_html_processor(config); let pipeline_config = PipelineConfig { @@ -543,7 +585,14 @@ mod tests { }), ) .expect("should update nextjs config"); - let registry = IntegrationRegistry::new(&settings).expect("should create registry"); + let registry = IntegrationRegistry::with_plan( + &settings, + Arc::new( + crate::auction::compile_auction_plan(&settings) + .expect("should compile auction plan"), + ), + ) + .expect("should create registry"); let config = config_from_settings(&settings, ®istry); let processor = create_html_processor(config); // Use small chunk size to force fragmentation @@ -604,7 +653,14 @@ mod tests { }), ) .expect("should update nextjs config"); - let registry = IntegrationRegistry::new(&settings).expect("should create registry"); + let registry = IntegrationRegistry::with_plan( + &settings, + Arc::new( + crate::auction::compile_auction_plan(&settings) + .expect("should compile auction plan"), + ), + ) + .expect("should create registry"); let config = config_from_settings(&settings, ®istry); let processor = create_html_processor(config); @@ -666,7 +722,14 @@ mod tests { }), ) .expect("should update nextjs config"); - let registry = IntegrationRegistry::new(&settings).expect("should create registry"); + let registry = IntegrationRegistry::with_plan( + &settings, + Arc::new( + crate::auction::compile_auction_plan(&settings) + .expect("should compile auction plan"), + ), + ) + .expect("should create registry"); let config = config_from_settings(&settings, ®istry); let processor = create_html_processor(config); diff --git a/crates/trusted-server-core/src/integrations/prebid.rs b/crates/trusted-server-core/src/integrations/prebid.rs index d0cf37275..31b265cae 100644 --- a/crates/trusted-server-core/src/integrations/prebid.rs +++ b/crates/trusted-server-core/src/integrations/prebid.rs @@ -1,5 +1,8 @@ -use std::collections::{HashMap, HashSet}; +use std::collections::HashMap; +#[cfg(test)] +use std::collections::HashSet; use std::sync::{Arc, LazyLock}; +#[cfg(test)] use std::time::Duration; use async_trait::async_trait; @@ -20,28 +23,40 @@ use url::{Url, Url as ParsedUrl}; use validator::{Validate, ValidationError}; use crate::auction::orchestrator::ERROR_TYPE_HTTP_STATUS; +use crate::auction::plan::AuctionPlan; +use crate::auction::profile::PrebidProfilePlan; +#[cfg(test)] use crate::auction::provider::{AuctionProvider, ProviderRequestOutcome}; -use crate::auction::types::{ - AuctionContext, AuctionRequest, AuctionResponse, Bid as AuctionBid, MediaType, -}; +use crate::auction::routing::{PrebidTransportHeaders, ProviderAuctionInput}; +#[cfg(test)] +use crate::auction::types::{AuctionContext, AuctionRequest, MediaType}; +use crate::auction::types::{AuctionResponse, Bid as AuctionBid}; use crate::cache_policy::{CacheControlPolicy, EdgeCacheHeader}; use crate::consent_config::ConsentForwardingMode; use crate::cookies::{CONSENT_COOKIE_NAMES, strip_cookies}; use crate::error::TrustedServerError; +#[cfg(test)] use crate::http_util::RequestInfo; use crate::integrations::{ AttributeRewriteAction, IntegrationAttributeContext, IntegrationAttributeRewriter, IntegrationEndpoint, IntegrationHeadInjector, IntegrationHtmlContext, IntegrationProxy, IntegrationRegistration, UPSTREAM_RTB_MAX_RESPONSE_BYTES, collect_response_bounded, +}; +#[cfg(test)] +use crate::integrations::{ ensure_integration_backend_with_timeout, predict_integration_backend_name, }; +#[cfg(test)] use crate::openrtb::{ Banner, ConsentedProvidersSettings, Device, Format, Geo, Imp, ImpExt, ImpStoredRequest, OpenRtbRequest, PrebidExt, PrebidImpExt, Publisher, Regs, RegsExt, RequestExt, Site, ToExt, TrustedServerExt, User, UserExt, to_openrtb_i32, }; -use crate::platform::{PlatformHttpRequest, PlatformResponse, RuntimeServices}; +#[cfg(test)] +use crate::platform::PlatformHttpRequest; +use crate::platform::{PlatformResponse, RuntimeServices}; use crate::proxy::{ProxyRequestConfig, is_host_allowed, proxy_request}; +#[cfg(test)] use crate::request_signing::{RequestSigner, SIGNING_VERSION, SigningParams}; use crate::settings::{IntegrationConfig, Settings}; @@ -55,8 +70,11 @@ const PREBID_BUNDLE_ERROR_CACHE_CONTROL: &str = "no-store"; const PREBID_BUNDLE_ERROR_CONTENT_TYPE: &str = "text/plain; charset=utf-8"; const PREBID_BUNDLE_NOSNIFF_HEADER: &str = "x-content-type-options"; const PREBID_BUNDLE_NOSNIFF_VALUE: &str = "nosniff"; +#[cfg(test)] const TRUSTED_SERVER_BIDDER: &str = "trustedServer"; +#[cfg(test)] const BIDDER_PARAMS_KEY: &str = "bidderParams"; +#[cfg(test)] const ZONE_KEY: &str = "zone"; /// Default currency for `OpenRTB` bid floors and responses. @@ -200,8 +218,9 @@ fn extract_prebid_error_message( #[cfg(test)] const GPC_US_PRIVACY: &str = "1YYN"; +#[cfg(test)] #[derive(Debug, Clone, Deserialize, Serialize, Validate)] -pub struct PrebidIntegrationConfig { +pub struct LegacyPrebidServerConfig { #[serde(default = "default_enabled")] pub enabled: bool, #[validate(url)] @@ -343,13 +362,109 @@ pub struct PrebidIntegrationConfig { pub suppress_nurl_bidders: Vec, } +#[cfg(test)] +impl IntegrationConfig for LegacyPrebidServerConfig { + fn is_enabled(&self) -> bool { + self.enabled + } +} + +/// CLI build inputs retained in app config but ignored safely by the runtime. +#[derive(Debug, Clone, Default, Deserialize, Serialize)] +#[serde(deny_unknown_fields)] +pub struct PrebidBundleBuildConfig { + /// Prebid.js bidder adapters included by `ts prebid bundle`. + #[serde(default)] + pub adapters: Vec, + /// Optional Prebid.js user ID modules included by `ts prebid bundle`. + #[serde(default)] + pub user_id_modules: Option>, +} + +/// Browser-only Prebid integration settings. +#[derive(Debug, Clone, Deserialize, Serialize, Validate)] +#[serde(deny_unknown_fields)] +pub struct PrebidIntegrationConfig { + #[serde(default = "default_enabled")] + pub enabled: bool, + #[serde(default)] + pub account_id: Option, + #[serde(default = "default_timeout_ms")] + pub timeout_ms: u32, + #[serde(default)] + pub debug: bool, + #[serde( + default = "default_script_patterns", + deserialize_with = "crate::settings::vec_from_seq_or_map" + )] + pub script_patterns: Vec, + #[serde(default)] + #[validate(custom(function = "validate_external_bundle_url"))] + pub external_bundle_url: Option, + #[serde(default)] + #[validate(regex( + path = *EXTERNAL_BUNDLE_SHA256_PATTERN, + message = "external_bundle_sha256 must be a 64-character hex SHA-256" + ))] + pub external_bundle_sha256: Option, + #[serde(default)] + #[validate(custom(function = "validate_external_bundle_sri"))] + pub external_bundle_sri: Option, + #[serde(default, deserialize_with = "crate::settings::vec_from_seq_or_map")] + pub client_side_bidders: Vec, + #[serde(default, deserialize_with = "crate::settings::vec_from_seq_or_map")] + #[validate(custom(function = "validate_excluded_gam_ad_unit_path_suffixes"))] + pub excluded_gam_ad_unit_path_suffixes: Vec, + /// CLI-only external bundle build inputs; runtime registration ignores these fields. + #[serde(default)] + pub bundle: PrebidBundleBuildConfig, +} + +impl Default for PrebidIntegrationConfig { + fn default() -> Self { + Self { + enabled: default_enabled(), + account_id: None, + timeout_ms: default_timeout_ms(), + debug: false, + script_patterns: default_script_patterns(), + external_bundle_url: None, + external_bundle_sha256: None, + external_bundle_sri: None, + client_side_bidders: Vec::new(), + excluded_gam_ad_unit_path_suffixes: Vec::new(), + bundle: PrebidBundleBuildConfig::default(), + } + } +} + impl IntegrationConfig for PrebidIntegrationConfig { fn is_enabled(&self) -> bool { self.enabled } } -fn remove_aps_bidders(config: &mut PrebidIntegrationConfig) { +#[cfg(test)] +impl From<&LegacyPrebidServerConfig> for PrebidIntegrationConfig { + fn from(config: &LegacyPrebidServerConfig) -> Self { + Self { + enabled: config.enabled, + account_id: config.account_id.clone(), + timeout_ms: config.timeout_ms, + debug: config.debug, + script_patterns: config.script_patterns.clone(), + external_bundle_url: config.external_bundle_url.clone(), + external_bundle_sha256: config.external_bundle_sha256.clone(), + external_bundle_sri: config.external_bundle_sri.clone(), + client_side_bidders: config.client_side_bidders.clone(), + excluded_gam_ad_unit_path_suffixes: config.excluded_gam_ad_unit_path_suffixes.clone(), + bundle: PrebidBundleBuildConfig::default(), + } + } +} + +#[cfg(test)] +fn remove_aps_bidders(config: &mut LegacyPrebidServerConfig) { for (field, bidders) in [ ("bidders", &mut config.bidders), ("client_side_bidders", &mut config.client_side_bidders), @@ -406,7 +521,8 @@ fn validate_excluded_gam_ad_unit_path_suffixes(values: &[String]) -> Result<(), Ok(()) } -fn canonicalize_excluded_gam_ad_unit_path_suffixes(config: &mut PrebidIntegrationConfig) { +#[cfg(test)] +fn canonicalize_excluded_gam_ad_unit_path_suffixes(config: &mut LegacyPrebidServerConfig) { let mut canonical = Vec::with_capacity(config.excluded_gam_ad_unit_path_suffixes.len()); for suffix in std::mem::take(&mut config.excluded_gam_ad_unit_path_suffixes) { if !canonical.contains(&suffix) { @@ -416,11 +532,12 @@ fn canonicalize_excluded_gam_ad_unit_path_suffixes(config: &mut PrebidIntegratio config.excluded_gam_ad_unit_path_suffixes = canonical; } +#[cfg(test)] fn load_config( settings: &Settings, -) -> Result, Report> { +) -> Result, Report> { let Some(mut config) = - settings.integration_config::(PREBID_INTEGRATION_ID)? + settings.integration_config::(PREBID_INTEGRATION_ID)? else { return Ok(None); }; @@ -435,9 +552,10 @@ fn load_config( /// /// Returns a configuration error if enabled Prebid settings fail typed parsing, /// schema validation, or bidder-param override compilation. +#[cfg(test)] pub fn validate_config_for_startup( settings: &Settings, -) -> Result, Report> { +) -> Result, Report> { let Some(config) = load_config(settings)? else { return Ok(None); }; @@ -479,6 +597,7 @@ fn default_timeout_ms() -> u32 { 1000 } +#[cfg(test)] fn default_bidders() -> Vec { vec!["mocktioneer".to_string()] } @@ -609,11 +728,11 @@ fn validate_external_bundle_sri(value: &str) -> Result<(), ValidationError> { parse_external_bundle_sri(value) } -fn validate_external_bundle_config( - config: &PrebidIntegrationConfig, +fn validate_external_bundle_url_allowed( + external_bundle_url: Option<&str>, allowed_domains: &[String], ) -> Result<(), Report> { - let url = config.external_bundle_url.as_deref().ok_or_else(|| { + let url = external_bundle_url.ok_or_else(|| { Report::new(TrustedServerError::Configuration { message: "integrations.prebid.external_bundle_url is required when prebid is enabled" .to_string(), @@ -661,25 +780,96 @@ fn validate_external_bundle_config( Ok(()) } +pub(crate) fn validate_browser_config_for_startup( + config: &PrebidIntegrationConfig, + allowed_domains: &[String], +) -> Result<(), Report> { + validate_external_bundle_url_allowed(config.external_bundle_url.as_deref(), allowed_domains) +} + +pub(crate) fn validate_browser_bidder_ownership( + config: &PrebidIntegrationConfig, + plan: &AuctionPlan, +) -> Result<(), Report> { + if !plan.enabled() { + return Ok(()); + } + + let server_side = plan + .browser_bidder_codes() + .collect::>(); + let conflicts = config + .client_side_bidders + .iter() + .filter(|bidder| server_side.contains(bidder.as_str())) + .cloned() + .collect::>(); + if conflicts.is_empty() { + return Ok(()); + } + + Err(Report::new(TrustedServerError::Configuration { + message: format!( + "Prebid bidders must have exactly one browser owner; configured as both client-side and server-side: {}", + conflicts.into_iter().collect::>().join(", ") + ), + })) +} + +#[cfg(test)] +fn validate_external_bundle_config( + config: &LegacyPrebidServerConfig, + allowed_domains: &[String], +) -> Result<(), Report> { + validate_external_bundle_url_allowed(config.external_bundle_url.as_deref(), allowed_domains) +} + pub struct PrebidIntegration { config: PrebidIntegrationConfig, + planned_head_inserts: Option>, + #[cfg(test)] + legacy_config: Option, + #[cfg(test)] engine: Arc, } impl PrebidIntegration { - fn try_new(config: PrebidIntegrationConfig) -> Result, Report> { + #[cfg(test)] + fn try_new(config: LegacyPrebidServerConfig) -> Result, Report> { let engine = Arc::new(BidParamOverrideEngine::try_from_config(&config)?); - Ok(Arc::new(Self { config, engine })) + Ok(Arc::new(Self { + config: PrebidIntegrationConfig::from(&config), + planned_head_inserts: None, + legacy_config: Some(config), + engine, + })) } #[cfg(test)] - fn new(config: PrebidIntegrationConfig) -> Arc { + fn new(config: LegacyPrebidServerConfig) -> Arc { Self::try_new(config).expect("should compile prebid bid param overrides") } + fn for_browser_plan(config: &PrebidIntegrationConfig, plan: &AuctionPlan) -> Arc { + let mut integration = Self { + config: config.clone(), + planned_head_inserts: None, + #[cfg(test)] + legacy_config: None, + #[cfg(test)] + engine: Arc::new(BidParamOverrideEngine::default()), + }; + integration.planned_head_inserts = Some(integration.head_inserts_for_plan(config, plan)); + Arc::new(integration) + } + + #[cfg(test)] fn auction_provider(&self) -> PrebidAuctionProvider { PrebidAuctionProvider { - config: self.config.clone(), + config: self + .legacy_config + .clone() + .expect("should retain legacy config for legacy provider tests"), bid_param_override_engine: Arc::clone(&self.engine), } } @@ -767,23 +957,53 @@ impl PrebidIntegration { Ok(response) } - fn external_bundle_script_src(&self) -> String { - match self.config.external_bundle_sha256.as_deref() { - Some(sha256) => format!("{PREBID_BUNDLE_ROUTE}?v={sha256}"), - None => PREBID_BUNDLE_ROUTE.to_string(), - } + fn external_bundle_script_tag(&self) -> String { + external_bundle_script_tag( + self.config.external_bundle_sha256.as_deref(), + self.config.external_bundle_sri.as_deref(), + ) } - fn external_bundle_script_tag(&self) -> String { - let src = self.external_bundle_script_src(); - let integrity = self - .config - .external_bundle_sri - .as_deref() - .map(|value| format!(" integrity=\"{}\"", escape_html_attr(value))) - .unwrap_or_default(); + /// Build the prepared browser injection from browser settings and validated routes. + pub(crate) fn head_inserts_for_plan( + &self, + browser_config: &PrebidIntegrationConfig, + plan: &AuctionPlan, + ) -> Vec { + #[derive(Serialize)] + #[serde(rename_all = "camelCase")] + struct InjectedBrowserConfig<'a> { + account_id: &'a str, + timeout: u32, + debug: bool, + server_side_bidders: Vec<&'a str>, + #[serde(skip_serializing_if = "<[String]>::is_empty")] + client_side_bidders: &'a [String], + #[serde(skip_serializing_if = "<[String]>::is_empty")] + excluded_gam_ad_unit_path_suffixes: &'a [String], + } - format!("") + let payload = InjectedBrowserConfig { + account_id: browser_config.account_id.as_deref().unwrap_or_default(), + timeout: browser_config.timeout_ms, + debug: browser_config.debug, + server_side_bidders: if plan.enabled() { + plan.browser_bidder_codes().collect() + } else { + Vec::new() + }, + client_side_bidders: &browser_config.client_side_bidders, + excluded_gam_ad_unit_path_suffixes: &browser_config.excluded_gam_ad_unit_path_suffixes, + }; + let config_json = serialize_injected_prebid_config(&payload); + + vec![ + injected_prebid_config_script(&config_json), + external_bundle_script_tag( + browser_config.external_bundle_sha256.as_deref(), + browser_config.external_bundle_sri.as_deref(), + ), + ] } fn is_managed_external(&self) -> bool { @@ -952,6 +1172,23 @@ fn escape_html_attr(value: &str) -> String { .replace('>', ">") } +fn external_bundle_script_src(sha256: Option<&str>) -> String { + match sha256 { + Some(sha256) => format!("{PREBID_BUNDLE_ROUTE}?v={sha256}"), + None => PREBID_BUNDLE_ROUTE.to_string(), + } +} + +fn external_bundle_script_tag(sha256: Option<&str>, sri: Option<&str>) -> String { + let src = external_bundle_script_src(sha256); + let integrity = sri + .map(|value| format!(" integrity=\"{}\"", escape_html_attr(value))) + .unwrap_or_default(); + + format!("") +} + +#[cfg(test)] fn build( settings: &Settings, ) -> Result>, Report> { @@ -983,6 +1220,37 @@ fn build( /// /// Returns an error when the Prebid integration is enabled with invalid /// configuration. +pub fn register_for_plan( + settings: &Settings, + plan: &AuctionPlan, +) -> Result, Report> { + let Some(mut config) = + settings.integration_config::(PREBID_INTEGRATION_ID)? + else { + return Ok(None); + }; + let mut canonical = Vec::with_capacity(config.excluded_gam_ad_unit_path_suffixes.len()); + for suffix in std::mem::take(&mut config.excluded_gam_ad_unit_path_suffixes) { + if !canonical.contains(&suffix) { + canonical.push(suffix); + } + } + config.excluded_gam_ad_unit_path_suffixes = canonical; + validate_browser_config_for_startup(&config, &settings.proxy.allowed_domains)?; + validate_browser_bidder_ownership(&config, plan)?; + let integration = PrebidIntegration::for_browser_plan(&config, plan); + Ok(Some( + IntegrationRegistration::builder(PREBID_INTEGRATION_ID) + .with_proxy(integration.clone()) + .with_attribute_rewriter(integration.clone()) + .with_head_injector(integration) + .with_deferred_js() + .build(), + )) +} + +#[cfg(test)] +#[allow(clippy::missing_errors_doc)] pub fn register( settings: &Settings, ) -> Result, Report> { @@ -1072,12 +1340,31 @@ impl IntegrationAttributeRewriter for PrebidIntegration { } } +fn serialize_injected_prebid_config(payload: &impl Serialize) -> String { + // Escape ` String { + format!( + r#""# + ) +} + impl IntegrationHeadInjector for PrebidIntegration { fn integration_id(&self) -> &'static str { PREBID_INTEGRATION_ID } fn head_inserts(&self, _ctx: &IntegrationHtmlContext<'_>) -> Vec { + if let Some(inserts) = &self.planned_head_inserts { + return inserts.clone(); + } #[derive(Serialize)] #[serde(rename_all = "camelCase")] struct InjectedPrebidClientConfig<'a> { @@ -1095,22 +1382,24 @@ impl IntegrationHeadInjector for PrebidIntegration { account_id: self.config.account_id.as_deref().unwrap_or_default(), timeout: self.config.timeout_ms, debug: self.config.debug, - bidders: &self.config.bidders, + bidders: { + #[cfg(test)] + { + self.legacy_config + .as_ref() + .map_or(&[][..], |config| config.bidders.as_slice()) + } + #[cfg(not(test))] + { + &[] + } + }, client_side_bidders: &self.config.client_side_bidders, excluded_gam_ad_unit_path_suffixes: &self.config.excluded_gam_ad_unit_path_suffixes, }; - // Escape `window.pbjs=window.pbjs||{{}};window.pbjs.que=window.pbjs.que||[];window.pbjs.cmd=window.pbjs.cmd||[];window.__tsjs_prebid={config_json};"# - )]; + let config_json = serialize_injected_prebid_config(&payload); + let mut inserts = vec![injected_prebid_config_script(&config_json)]; inserts.push(self.external_bundle_script_tag()); @@ -1125,11 +1414,13 @@ impl IntegrationHeadInjector for PrebidIntegration { /// tell a fabricated empty from an explicitly supplied one — they are identical /// bytes on the wire. The merge uses this to stop an unusable value from /// clobbering real params, and the final pass uses it to drop whatever remains. +#[cfg(test)] fn is_unusable_bidder_params(params: &Json) -> bool { // `None` covers non-object values (e.g. `null`); an empty map covers `{}`. params.as_object().is_none_or(serde_json::Map::is_empty) } +#[cfg(test)] fn expand_trusted_server_bidders( configured_bidders: &[String], params: &Json, @@ -1187,7 +1478,8 @@ fn merge_bidder_param_object( // Generic bid-parameter override engine // ============================================================================ -fn warn_unconfigured_bidder(config: &PrebidIntegrationConfig, bidder: &str, field: &str) { +#[cfg(test)] +fn warn_unconfigured_bidder(config: &LegacyPrebidServerConfig, bidder: &str, field: &str) { if !config.bidders.iter().any(|b| b == bidder) { if config.client_side_bidders.iter().any(|b| b == bidder) { log::warn!( @@ -1204,7 +1496,7 @@ fn warn_unconfigured_bidder(config: &PrebidIntegrationConfig, bidder: &str, fiel } #[derive(Debug, Default, Clone)] -struct BidParamOverrideEngine { +pub(crate) struct BidParamOverrideEngine { rules: Vec, // Maps bidder name to the indices (into `rules`) of rules that constrain on that bidder. // Rules with no bidder constraint (zone-only or catch-all) are kept in `wildcard_indices`. @@ -1227,8 +1519,9 @@ struct BidParamOverrideFacts<'a> { } impl BidParamOverrideEngine { + #[cfg(test)] fn try_from_config( - config: &PrebidIntegrationConfig, + config: &LegacyPrebidServerConfig, ) -> Result> { let mut rules = Vec::new(); @@ -1284,6 +1577,45 @@ impl BidParamOverrideEngine { }) } + fn try_from_profile_config( + bid_param_zone_overrides: &std::collections::BTreeMap< + String, + std::collections::BTreeMap>, + >, + bid_param_overrides: &std::collections::BTreeMap>, + bid_param_override_rules: &[BidParamOverrideRule], + ) -> Result> { + let mut rules = Vec::new(); + for (bidder, set) in bid_param_overrides { + rules.push(CompiledBidParamOverrideRule::from_bidder_override( + bidder, set, + )?); + } + for (bidder, zone_override_sets) in bid_param_zone_overrides { + for (zone, set) in zone_override_sets { + rules.push(CompiledBidParamOverrideRule::from_zone_override( + bidder, zone, set, + )?); + } + } + for rule in bid_param_override_rules { + rules.push(CompiledBidParamOverrideRule::try_from(rule)?); + } + let mut bidder_index: HashMap> = HashMap::new(); + let mut wildcard_indices = Vec::new(); + for (index, rule) in rules.iter().enumerate() { + match &rule.bidder { + Some(bidder) => bidder_index.entry(bidder.clone()).or_default().push(index), + None => wildcard_indices.push(index), + } + } + Ok(Self { + rules, + bidder_index, + wildcard_indices, + }) + } + fn apply(&self, facts: BidParamOverrideFacts<'_>, params: &mut Json) { let bidder_indices = self.bidder_index.get(facts.bidder).map(Vec::as_slice); for idx in merged_rule_indices(&self.wildcard_indices, bidder_indices) { @@ -1306,6 +1638,30 @@ impl BidParamOverrideEngine { } } } + + /// Apply compiled profile rules to already centrally routed bidder params. + pub(crate) fn apply_routed(&self, bidder: &str, zone: Option<&str>, params: &mut Json) { + self.apply(BidParamOverrideFacts { bidder, zone }, params); + } +} + +/// Validate and compile server-side Prebid profile override fields. +/// +/// This narrow hook shares the existing override compiler without coupling +/// auction-profile availability to the browser integration's enablement. +pub(crate) fn compile_profile_override_rules( + bid_param_zone_overrides: &std::collections::BTreeMap< + String, + std::collections::BTreeMap>, + >, + bid_param_overrides: &std::collections::BTreeMap>, + bid_param_override_rules: &[BidParamOverrideRule], +) -> Result> { + BidParamOverrideEngine::try_from_profile_config( + bid_param_zone_overrides, + bid_param_overrides, + bid_param_override_rules, + ) } fn merged_rule_indices<'a>( @@ -1467,17 +1823,62 @@ fn non_empty_override_object( /// In [`ConsentForwardingMode::OpenrtbOnly`] mode, consent cookies are /// stripped from the `Cookie` header since consent travels exclusively /// through the `OpenRTB` body. +#[cfg(test)] fn copy_request_headers( from: &http::Request, to: &mut http::Request, consent_forwarding: ConsentForwardingMode, client_ip: Option, ) { - let headers_to_copy = [header::USER_AGENT, header::REFERER, header::ACCEPT_LANGUAGE]; + apply_prebid_header_values( + from.headers().get(header::COOKIE), + from.headers().get(header::USER_AGENT), + from.headers().get(header::REFERER), + from.headers().get(header::ACCEPT_LANGUAGE), + to, + consent_forwarding, + client_ip, + ); +} + +/// Apply the common raw-header transport policy for a planned PBS request. +pub(crate) fn apply_prebid_transport_headers( + from: &PrebidTransportHeaders, + to: &mut http::Request, + consent_forwarding: ConsentForwardingMode, + client_ip: Option, +) { + apply_prebid_header_values( + from.cookie(), + from.user_agent(), + from.referer(), + from.accept_language(), + to, + consent_forwarding, + client_ip, + ); +} - for header_name in &headers_to_copy { - if let Some(value) = from.headers().get(header_name) { - to.headers_mut().insert(header_name, value.clone()); +#[allow( + clippy::too_many_arguments, + reason = "the helper preserves four independently optional raw headers plus transport policy" +)] +fn apply_prebid_header_values( + cookie: Option<&HeaderValue>, + user_agent: Option<&HeaderValue>, + referer: Option<&HeaderValue>, + accept_language: Option<&HeaderValue>, + to: &mut http::Request, + consent_forwarding: ConsentForwardingMode, + client_ip: Option, +) { + for (name, value) in [ + (header::USER_AGENT, user_agent), + (header::REFERER, referer), + (header::ACCEPT_LANGUAGE, accept_language), + ] { + if let Some(value) = value { + to.headers_mut().insert(name, value.clone()); } } @@ -1488,24 +1889,20 @@ fn copy_request_headers( .insert(header::HeaderName::from_static("x-forwarded-for"), value); } - let Some(cookie_value) = from.headers().get(header::COOKIE) else { + let Some(cookie_value) = cookie else { return; }; - if !consent_forwarding.strips_consent_cookies() { to.headers_mut() .insert(header::COOKIE, cookie_value.clone()); return; } - match cookie_value.to_str() { Ok(value) => { let stripped = strip_cookies(value, CONSENT_COOKIE_NAMES); - if stripped.is_empty() { - return; - } - - if let Ok(cookie_header) = HeaderValue::from_str(&stripped) { + if !stripped.is_empty() + && let Ok(cookie_header) = HeaderValue::from_str(&stripped) + { to.headers_mut().insert(header::COOKIE, cookie_header); } } @@ -1516,43 +1913,340 @@ fn copy_request_headers( } } -/// Appends query parameters to a URL, handling both URLs with and without existing query strings. -/// Returns the original URL unchanged if params are empty or already present. -fn append_query_params(url: &str, params: &str) -> String { - if params.is_empty() || url.contains(params) { - return url.to_string(); - } - if url.contains('?') { - format!("{}&{}", url, params) - } else { - format!("{}?{}", url, params) +/// Appends query parameters to a URL, handling both URLs with and without existing query strings. +/// Returns the original URL unchanged if params are empty or already present. +#[cfg(test)] +fn append_query_params(url: &str, params: &str) -> String { + if params.is_empty() || url.contains(params) { + return url.to_string(); + } + if url.contains('?') { + format!("{}&{}", url, params) + } else { + format!("{}?{}", url, params) + } +} + +/// Parse a planned PBS response with the configured profile behavior. +/// +/// This preserves the legacy PBS status, body, bid, cache, and metadata +/// semantics while allowing each planned provider to retain its own identity. +pub(crate) async fn parse_planned_prebid_response( + provider_id: &str, + profile: &PrebidProfilePlan, + input: &ProviderAuctionInput, + response: PlatformResponse, + response_time_ms: u64, + auction_id: &str, +) -> Result> { + let response = response.response; + let status = response.status(); + let content_type = response + .headers() + .get(header::CONTENT_TYPE) + .and_then(|value| value.to_str().ok()) + .map(str::to_owned); + let body_bytes = collect_response_bounded( + response.into_body(), + UPSTREAM_RTB_MAX_RESPONSE_BYTES, + "prebid", + ) + .await + .change_context(TrustedServerError::Prebid { + message: "Failed to read Prebid response body".to_string(), + }); + + if !status.is_success() { + log::warn!("Prebid auction {auction_id:?} returned non-success status: {status}"); + let body_bytes = match body_bytes { + Ok(body_bytes) => Some(body_bytes), + Err(error) => { + log::warn!( + "Prebid auction {auction_id:?} failed to read non-success response body: {error:?}" + ); + None + } + }; + if profile.debug + && let Some(body_bytes) = body_bytes.as_deref() + { + match prebid_body_preview(body_bytes) { + Some(preview) => { + let truncation = if preview.truncated { + " (truncated)" + } else { + "" + }; + log::warn!( + "Prebid auction {auction_id:?} error response body preview{truncation}: {}", + preview.text + ); + } + None => log::warn!( + "Prebid auction {auction_id:?} returned an empty error response body" + ), + } + } + + let status_code = status.as_u16(); + let mut parsed = AuctionResponse::error(provider_id, response_time_ms) + .with_metadata("error_type", serde_json::json!(ERROR_TYPE_HTTP_STATUS)) + .with_metadata("http_status", serde_json::json!(status_code)) + .with_metadata( + "message", + serde_json::json!(format!("Prebid Server returned HTTP {status_code}")), + ); + if profile.debug + && let Some(message) = body_bytes + .as_deref() + .and_then(|body| extract_prebid_error_message(body, content_type.as_deref())) + { + parsed.metadata.insert( + "upstream_message".to_string(), + serde_json::json!(message.text), + ); + parsed.metadata.insert( + "upstream_message_truncated".to_string(), + serde_json::json!(message.truncated), + ); + } + return Ok(parsed); + } + + let body_bytes = body_bytes?; + let response_json: Json = + serde_json::from_slice(&body_bytes).change_context(TrustedServerError::Prebid { + message: "Failed to parse Prebid response".to_string(), + })?; + if profile.debug && log::log_enabled!(log::Level::Trace) { + match serde_json::to_string_pretty(&response_json) { + Ok(json) => log::trace!("Prebid OpenRTB response:\n{json}"), + Err(error) => log::warn!("Prebid: failed to serialize response for logging: {error}"), + } + } + + let mut parsed = + parse_planned_prebid_openrtb(provider_id, input, &response_json, response_time_ms); + enrich_planned_prebid_metadata(profile, &response_json, &mut parsed); + log::info!( + "Prebid provider {provider_id} returned {} bids in {}ms", + parsed.bids.len(), + response_time_ms + ); + Ok(parsed) +} + +fn parse_planned_prebid_openrtb( + provider_id: &str, + input: &ProviderAuctionInput, + response_json: &Json, + response_time_ms: u64, +) -> AuctionResponse { + let Some(response) = response_json.as_object() else { + return AuctionResponse::error(provider_id, response_time_ms) + .with_metadata("error_type", serde_json::json!("parse_response")); + }; + match response.get("cur") { + None => {} + Some(Json::String(currency)) if currency.eq_ignore_ascii_case(DEFAULT_CURRENCY) => {} + Some(Json::String(currency)) => { + return AuctionResponse::no_bid(provider_id, response_time_ms) + .with_metadata("unsupported_currency", serde_json::json!(currency)); + } + Some(_) => { + return AuctionResponse::error(provider_id, response_time_ms) + .with_metadata("error_type", serde_json::json!("parse_response")); + } + } + + let mut bids = Vec::new(); + if let Some(seatbids) = response_json.get("seatbid").and_then(Json::as_array) { + for seatbid in seatbids { + let returned_seat = seatbid + .get("seat") + .and_then(Json::as_str) + .filter(|seat| !seat.is_empty()); + let delivery_bidder = returned_seat.unwrap_or("unknown"); + if let Some(entries) = seatbid.get("bid").and_then(Json::as_array) { + for entry in entries { + match parse_planned_prebid_bid(entry, delivery_bidder, returned_seat) { + Ok(bid) if planned_prebid_bid_is_allowed(&bid, input) => bids.push(bid), + Ok(_) => {} + Err(()) => { + let impression = entry + .get("impid") + .and_then(Json::as_str) + .unwrap_or(""); + log::warn!( + "Prebid: failed to parse bid from seat '{delivery_bidder}' for imp '{impression}'" + ); + } + } + } + } + } + } + if bids.is_empty() { + AuctionResponse::no_bid(provider_id, response_time_ms) + } else { + AuctionResponse::success(provider_id, bids, response_time_ms) + } +} + +fn enrich_planned_prebid_metadata( + profile: &PrebidProfilePlan, + response_json: &Json, + parsed: &mut AuctionResponse, +) { + let ext = response_json.get("ext"); + for key in ["responsetimemillis", "errors", "warnings"] { + if let Some(value) = ext.and_then(|ext| ext.get(key)) { + parsed.metadata.insert(key.to_string(), value.clone()); + } + } + if profile.debug { + if let Some(value) = ext.and_then(|ext| ext.get("debug")) { + parsed.metadata.insert("debug".to_string(), value.clone()); + } + if let Some(value) = ext + .and_then(|ext| ext.get("prebid")) + .and_then(|prebid| prebid.get("bidstatus")) + { + parsed + .metadata + .insert("bidstatus".to_string(), value.clone()); + } + } +} + +fn planned_prebid_bid_is_allowed(bid: &AuctionBid, input: &ProviderAuctionInput) -> bool { + input.slots().iter().any(|slot| { + slot.slot().id == bid.slot_id + && slot + .slot() + .formats + .iter() + .any(|format| (format.width, format.height) == (bid.width, bid.height)) + }) +} + +fn parse_planned_prebid_bid( + bid: &Json, + delivery_bidder: &str, + returned_seat: Option<&str>, +) -> Result { + let slot_id = bid + .get("impid") + .and_then(Json::as_str) + .ok_or(())? + .to_string(); + let price = bid + .get("price") + .and_then(Json::as_f64) + .filter(|price| price.is_finite() && *price >= 0.0) + .ok_or(())?; + let creative = bid.get("adm").and_then(Json::as_str).map(String::from); + let width = bid + .get("w") + .and_then(Json::as_u64) + .and_then(|value| u32::try_from(value).ok()) + .unwrap_or(0); + let height = bid + .get("h") + .and_then(Json::as_u64) + .and_then(|value| u32::try_from(value).ok()) + .unwrap_or(0); + let cache_entry = bid + .get("ext") + .and_then(|ext| ext.get("prebid")) + .and_then(|prebid| prebid.get("cache")) + .and_then(|cache| cache.get("bids")); + let cache_id = cache_entry + .and_then(|cache| cache.get("cacheId")) + .and_then(Json::as_str) + .map(String::from); + let (cache_host, cache_path) = cache_entry + .and_then(|cache| cache.get("url")) + .and_then(Json::as_str) + .and_then(|value| { + ParsedUrl::parse(value) + .map_err(|error| log::debug!("PBS cache URL parse failed: {error}")) + .ok() + }) + .map(|url| { + let host = url.host_str().map(String::from); + let path = url.path().to_string(); + let path = (!path.is_empty() && path != "/").then_some(path); + (host, path) + }) + .unwrap_or((None, None)); + if cache_id.is_some() && cache_host.is_none() { + log::warn!( + "PBS bid has cache UUID but cache URL could not be parsed — creative will fail to render for slot '{slot_id}'" + ); } + + Ok(AuctionBid { + slot_id, + price: Some(price), + currency: DEFAULT_CURRENCY.to_string(), + creative, + adomain: bid.get("adomain").and_then(Json::as_array).map(|domains| { + domains + .iter() + .filter_map(Json::as_str) + .map(String::from) + .collect() + }), + bidder: delivery_bidder.to_string(), + returned_seat: returned_seat.map(str::to_string), + width, + height, + nurl: bid.get("nurl").and_then(Json::as_str).map(String::from), + burl: bid.get("burl").and_then(Json::as_str).map(String::from), + bid_id: bid + .get("id") + .and_then(Json::as_str) + .filter(|value| !value.is_empty()) + .map(String::from), + ad_id: bid.get("adid").and_then(Json::as_str).map(String::from), + creative_id: bid.get("crid").and_then(Json::as_str).map(String::from), + renderer: None, + cache_id, + cache_host, + cache_path, + metadata: HashMap::new(), + }) } // ============================================================================ // Prebid Auction Provider // ============================================================================ -/// Prebid Server auction provider. +/// Legacy Prebid Server auction provider retained only for parity tests. +#[cfg(test)] pub struct PrebidAuctionProvider { - config: PrebidIntegrationConfig, + config: LegacyPrebidServerConfig, bid_param_override_engine: Arc, } +#[cfg(test)] #[derive(Default)] struct PrebidImpressionDisposition { aps_only: usize, invalid: usize, } +#[cfg(test)] struct PrebidRequestBuild { request: OpenRtbRequest, disposition: PrebidImpressionDisposition, } +#[cfg(test)] impl PrebidAuctionProvider { #[cfg(test)] - fn new(config: PrebidIntegrationConfig) -> Self { + fn new(config: LegacyPrebidServerConfig) -> Self { Self::try_new(config).expect("should compile prebid bid param overrides") } @@ -1561,7 +2255,7 @@ impl PrebidAuctionProvider { /// # Errors /// /// Returns an error when the configured bidder-param override rules are invalid. - pub fn try_new(config: PrebidIntegrationConfig) -> Result> { + pub fn try_new(config: LegacyPrebidServerConfig) -> Result> { Ok(Self { bid_param_override_engine: Arc::new(BidParamOverrideEngine::try_from_config(&config)?), config, @@ -2438,6 +3132,7 @@ impl PrebidAuctionProvider { creative, adomain, bidder: seat.to_string(), + returned_seat: None, width, height, nurl, @@ -2454,9 +3149,10 @@ impl PrebidAuctionProvider { } } +#[cfg(test)] #[async_trait(?Send)] impl AuctionProvider for PrebidAuctionProvider { - fn provider_name(&self) -> &'static str { + fn provider_name(&self) -> &str { PREBID_INTEGRATION_ID } @@ -2664,6 +3360,7 @@ impl AuctionProvider for PrebidAuctionProvider { /// /// Returns an error when the Prebid provider is enabled with invalid /// configuration. +#[cfg(test)] pub fn register_auction_provider( settings: &Settings, ) -> Result>, Report> { @@ -2672,10 +3369,12 @@ pub fn register_auction_provider( return Ok(Vec::new()); }; - log::info!( - "Registering Prebid auction provider (server_url={})", - integration.config.server_url - ); + if let Some(config) = integration.legacy_config.as_ref() { + log::info!( + "Registering Prebid auction provider (server_url={})", + config.server_url + ); + } if integration.config.debug { log::warn!( "Prebid debug mode is ON — debug data (httpcalls, resolvedrequest, \ @@ -2693,7 +3392,14 @@ mod tests { use super::*; use crate::auction::formats::convert_to_openrtb_response; use crate::auction::orchestrator::OrchestrationResult; - use crate::auction::test_support::create_test_auction_context as shared_test_auction_context; + use crate::auction::plan::{ + AuctionPlanConfig, BidderId, BidderRouteConfig, NotificationConfig, ProviderConfig, + ProviderId, RoutingMode, + }; + use crate::auction::test_support::{ + canonical_parity_auction_request, + create_test_auction_context as shared_test_auction_context, + }; use crate::auction::types::{ AdFormat, AdSlot, AuctionContext, AuctionRequest, DeviceInfo, PublisherInfo, UserInfo, }; @@ -2705,8 +3411,9 @@ mod tests { AttributeRewriteAction, IntegrationDocumentState, IntegrationRegistry, }; use crate::platform::test_support::{ - NoopConfigStore, NoopGeo, NoopHttpClient, NoopSecretStore, StubHttpClient, - build_services_with_http_client, + HashMapConfigStore, HashMapSecretStore, NoopConfigStore, NoopGeo, NoopHttpClient, + NoopSecretStore, StubHttpClient, build_services_with_config_secret_and_http_client, + build_services_with_http_client, build_services_with_http_client_and_client_ip, }; use crate::platform::{ ClientInfo, PlatformBackend, PlatformBackendSpec, PlatformError, RuntimeServices, @@ -2718,8 +3425,9 @@ mod tests { use bytes::Bytes; use http::Method; use serde_json::json; - use std::collections::HashMap; + use std::collections::{BTreeMap, HashMap}; use std::io::Cursor; + use std::str::FromStr as _; #[test] fn external_bundle_sha256_validation_matches_hex_pattern() { @@ -2727,7 +3435,6 @@ mod tests { let config = |sha: &str| -> PrebidIntegrationConfig { serde_json::from_value(serde_json::json!({ - "server_url": "https://prebid.example.com/openrtb2/auction", "external_bundle_sha256": sha, })) .expect("should deserialize prebid config") @@ -2758,8 +3465,8 @@ mod tests { create_test_settings() } - fn base_config() -> PrebidIntegrationConfig { - PrebidIntegrationConfig { + fn base_config() -> LegacyPrebidServerConfig { + LegacyPrebidServerConfig { enabled: true, server_url: "https://prebid.example".to_string(), account_id: Some("test-account".to_string()), @@ -2788,6 +3495,10 @@ mod tests { struct PredictOnlyBackend; impl PlatformBackend for PredictOnlyBackend { + fn naming_policy(&self) -> crate::platform::BackendNamingPolicy { + crate::platform::BackendNamingPolicy::Axum + } + fn predict_name( &self, spec: &PlatformBackendSpec, @@ -2953,6 +3664,7 @@ mod tests { settings: &settings, request: &http_req, timeout_ms: 500, + transport_timeout_ms: 500, provider_responses: None, services: &services, }; @@ -2999,6 +3711,7 @@ mod tests { settings: &settings, request: &http_req, timeout_ms: 500, + transport_timeout_ms: 500, provider_responses: None, services: &services, }; @@ -3061,23 +3774,23 @@ passphrase = "test-secret-key-32-bytes-minimum" "#; /// Parse a TOML string containing only the `[integrations.prebid]` section - /// (plus any sub-tables) into a [`PrebidIntegrationConfig`]. - fn parse_prebid_toml(prebid_section: &str) -> PrebidIntegrationConfig { + /// (plus any sub-tables) into a [`LegacyPrebidServerConfig`]. + fn parse_prebid_toml(prebid_section: &str) -> LegacyPrebidServerConfig { let toml_str = format!("{}{}", TOML_BASE, prebid_section); let settings = Settings::from_toml(&toml_str).expect("should parse TOML"); settings - .integration_config::("prebid") + .integration_config::("prebid") .expect("should get config") .expect("should be enabled") } fn parse_prebid_toml_result( prebid_section: &str, - ) -> Result> { + ) -> Result> { let toml_str = format!("{}{}", TOML_BASE, prebid_section); let settings = Settings::from_toml(&toml_str)?; settings - .integration_config::("prebid")? + .integration_config::("prebid")? .ok_or_else(|| { Report::new(TrustedServerError::Configuration { message: "prebid integration config should be present and enabled".to_string(), @@ -3104,6 +3817,7 @@ server_url = "https://prebid.example/openrtb2/auction" ); } + /* Legacy mixed-config canonicalization test replaced by browser-only registration tests. #[test] fn startup_validation_and_runtime_build_canonicalize_excluded_gam_ad_unit_path_suffixes() { let mut settings = make_settings(); @@ -3113,7 +3827,6 @@ server_url = "https://prebid.example/openrtb2/auction" PREBID_INTEGRATION_ID, &json!({ "enabled": true, - "server_url": "https://prebid.example/openrtb2/auction", "external_bundle_url": "https://assets.example/prebid/trusted-prebid.js", "excluded_gam_ad_unit_path_suffixes": [ "/trackingonly", @@ -3155,6 +3868,7 @@ server_url = "https://prebid.example/openrtb2/auction" ); } + */ #[test] fn excluded_gam_ad_unit_path_suffixes_reject_invalid_values() { for (suffix, expected_message) in [ @@ -3225,16 +3939,21 @@ excluded_gam_ad_unit_path_suffixes = ["{suffix}"] "prebid", &json!({ "enabled": true, - "server_url": "https://test-prebid.com/openrtb2/auction", "external_bundle_url": "https://assets.example/prebid/trusted-prebid.js", "timeout_ms": 1000, - "bidders": ["mocktioneer"], "script_patterns": [], "debug": false }), ) .expect("should update prebid config"); - let registry = IntegrationRegistry::new(&settings).expect("should create registry"); + let registry = IntegrationRegistry::with_plan( + &settings, + Arc::new( + crate::auction::compile_auction_plan(&settings) + .expect("should compile auction plan"), + ), + ) + .expect("should create registry"); let config = config_from_settings(&settings, ®istry); let processor = create_html_processor(config); let pipeline_config = PipelineConfig { @@ -3276,16 +3995,21 @@ excluded_gam_ad_unit_path_suffixes = ["{suffix}"] "prebid", &json!({ "enabled": true, - "server_url": "https://test-prebid.com/openrtb2/auction", "external_bundle_url": "https://assets.example/prebid/trusted-prebid.js", "timeout_ms": 1000, - "bidders": ["mocktioneer"], "script_patterns": ["/prebid.js", "/prebid.min.js"], "debug": false }), ) .expect("should update prebid config"); - let registry = IntegrationRegistry::new(&settings).expect("should create registry"); + let registry = IntegrationRegistry::with_plan( + &settings, + Arc::new( + crate::auction::compile_auction_plan(&settings) + .expect("should compile auction plan"), + ), + ) + .expect("should create registry"); let config = config_from_settings(&settings, ®istry); let processor = create_html_processor(config); let pipeline_config = PipelineConfig { @@ -3479,15 +4203,20 @@ external_bundle_sri = "sha384-AAAA" "prebid", &json!({ "enabled": true, - "server_url": "https://prebid.example/openrtb2/auction", "external_bundle_url": "https://assets.example/prebid/trusted-prebid.js", "external_bundle_sha256": "0".repeat(64) }), ) .expect("should update prebid config"); - let registry = IntegrationRegistry::new(&settings) - .expect("should create registry with valid SHA-256 and no SRI"); + let registry = IntegrationRegistry::with_plan( + &settings, + Arc::new( + crate::auction::compile_auction_plan(&settings) + .expect("should compile auction plan"), + ), + ) + .expect("should create registry with valid SHA-256 and no SRI"); assert!( registry.has_route(&Method::GET, PREBID_BUNDLE_ROUTE), @@ -3504,7 +4233,6 @@ external_bundle_sri = "sha384-AAAA" "prebid", &json!({ "enabled": true, - "server_url": "https://prebid.example/openrtb2/auction", "external_bundle_url": "https://assets.example/prebid/trusted-prebid.js", "external_bundle_sha256": "0".repeat(64), "external_bundle_sri": test_sri("sha384", &[0; 48]) @@ -3512,8 +4240,14 @@ external_bundle_sri = "sha384-AAAA" ) .expect("should update prebid config"); - let registry = IntegrationRegistry::new(&settings) - .expect("should create registry with valid SHA-256 and SHA-384 SRI"); + let registry = IntegrationRegistry::with_plan( + &settings, + Arc::new( + crate::auction::compile_auction_plan(&settings) + .expect("should compile auction plan"), + ), + ) + .expect("should create registry with valid SHA-256 and SHA-384 SRI"); assert!( registry.has_route(&Method::GET, PREBID_BUNDLE_ROUTE), @@ -3521,6 +4255,7 @@ external_bundle_sri = "sha384-AAAA" ); } + /* Browser-only Prebid permits no managed external bundle URL. #[test] fn external_bundle_registration_requires_bundle_url() { let mut settings = make_settings(); @@ -3530,12 +4265,11 @@ external_bundle_sri = "sha384-AAAA" "prebid", &json!({ "enabled": true, - "server_url": "https://prebid.example/openrtb2/auction" }), ) .expect("should update prebid config"); - let err = match IntegrationRegistry::new(&settings) { + let err = match IntegrationRegistry::with_plan(&settings, Arc::new(crate::auction::compile_auction_plan(&settings).expect("should compile auction plan"))) { Ok(_) => panic!("should reject missing URL"), Err(err) => err, }; @@ -3545,6 +4279,7 @@ external_bundle_sri = "sha384-AAAA" ); } + */ #[test] fn external_bundle_registration_uses_proxy_allowed_domains() { let mut settings = make_settings(); @@ -3555,13 +4290,18 @@ external_bundle_sri = "sha384-AAAA" "prebid", &json!({ "enabled": true, - "server_url": "https://prebid.example/openrtb2/auction", "external_bundle_url": "https://blocked.example/prebid/trusted-prebid.js" }), ) .expect("should update prebid config"); - let err = match IntegrationRegistry::new(&settings) { + let err = match IntegrationRegistry::with_plan( + &settings, + Arc::new( + crate::auction::compile_auction_plan(&settings) + .expect("should compile auction plan"), + ), + ) { Ok(_) => panic!("should reject bundle host outside proxy.allowed_domains"), Err(err) => err, }; @@ -3876,6 +4616,7 @@ external_bundle_sri = "sha384-AAAA" ); } + /* Startup validation now uses the browser-only public config path. #[test] fn external_bundle_startup_validation_requires_proxy_allowed_domains() { let mut settings = make_settings(); @@ -3890,6 +4631,7 @@ external_bundle_sri = "sha384-AAAA" ); } + */ #[test] fn external_bundle_handler_fetches_and_sanitizes_with_platform_client() { futures::executor::block_on(async { @@ -4060,6 +4802,97 @@ external_bundle_sri = "sha384-AAAA" ); } + #[test] + #[allow(clippy::field_reassign_with_default)] + fn prepared_browser_injection_uses_only_plan_routes_and_browser_timeout_debug() { + let integration = PrebidIntegration::new(base_config()); + let mut browser_config = PrebidIntegrationConfig::default(); + browser_config.account_id = Some("browser-account".to_string()); + browser_config.timeout_ms = 1750; + browser_config.debug = false; + let plan = AuctionPlan::compile(AuctionPlanConfig { + timeout_ms: 2500, + providers: BTreeMap::from([ + ( + ProviderId::from_str("pbs-primary").expect("should parse provider ID"), + ProviderConfig { + protocol: "openrtb-2.6".to_string(), + profile: "prebid-server".to_string(), + endpoint: "https://primary.example.test/openrtb".to_string(), + timeout_ms: Some(3000), + routing: RoutingMode::Explicit, + notifications: NotificationConfig::default(), + profile_config: json!({"debug": true}), + }, + ), + ( + ProviderId::from_str("pbs-secondary").expect("should parse provider ID"), + ProviderConfig { + protocol: "openrtb-2.6".to_string(), + profile: "prebid-server".to_string(), + endpoint: "https://secondary.example.test/openrtb".to_string(), + timeout_ms: Some(4000), + routing: RoutingMode::Explicit, + notifications: NotificationConfig::default(), + profile_config: json!({"debug": true}), + }, + ), + ]), + bidders: BTreeMap::from([ + ( + BidderId::from_str("secondaryRoute").expect("should parse bidder ID"), + BidderRouteConfig { + provider: ProviderId::from_str("pbs-secondary") + .expect("should parse provider ID"), + }, + ), + ( + BidderId::from_str("primaryRoute").expect("should parse bidder ID"), + BidderRouteConfig { + provider: ProviderId::from_str("pbs-primary") + .expect("should parse provider ID"), + }, + ), + ]), + mediator: None, + request_signing: None, + }) + .expect("should compile plan while browser integration is not part of compilation"); + + let inserts = integration.head_inserts_for_plan(&browser_config, &plan); + let script = &inserts[0]; + + assert!(script.contains(r#""timeout":1750,"debug":false"#)); + assert!( + script.contains(r#""serverSideBidders":["primaryRoute","secondaryRoute"]"#), + "should inject deterministic browser route codes: {script}" + ); + assert!(!script.contains("pbs-primary")); + assert!(!script.contains("pbs-secondary")); + assert!(!script.contains("3000")); + assert!(!script.contains("4000")); + + let disabled_plan = plan.clone().with_enabled(false); + let disabled_inserts = integration.head_inserts_for_plan(&browser_config, &disabled_plan); + assert!( + disabled_inserts[0].contains(r#""serverSideBidders":[]"#), + "auction kill switch should suppress browser server-side bidders: {}", + disabled_inserts[0] + ); + } + + #[test] + fn browser_only_config_defaults_are_independent_and_can_be_disabled() { + let config = PrebidIntegrationConfig { + enabled: false, + ..PrebidIntegrationConfig::default() + }; + + assert!(!config.enabled); + assert_eq!(config.timeout_ms, 1000); + assert!(!config.debug); + } + #[test] fn head_injector_includes_excluded_gam_ad_unit_path_suffixes() { let mut config = base_config(); @@ -5902,7 +6735,7 @@ external_bundle_sri = "sha384-AAAA" } fn call_to_openrtb( - config: PrebidIntegrationConfig, + config: LegacyPrebidServerConfig, request: &AuctionRequest, ) -> OpenRtbRequest { use crate::platform::test_support::noop_services; @@ -5918,6 +6751,7 @@ external_bundle_sri = "sha384-AAAA" settings: &settings, request: &http_req, timeout_ms: 1000, + transport_timeout_ms: 1000, provider_responses: None, services: &services, }; @@ -6086,7 +6920,7 @@ set = {{ placementId = "13579" }} let runtime_settings = Settings::from_json_value(serialized).expect("should parse runtime JSON settings"); let config = runtime_settings - .integration_config::(PREBID_INTEGRATION_ID) + .integration_config::(PREBID_INTEGRATION_ID) .expect("should parse Prebid config") .expect("should enable Prebid config"); @@ -7404,6 +8238,398 @@ set = { networkId = 42 } ); } + fn planned_prebid_profile(debug: bool) -> PrebidProfilePlan { + PrebidProfilePlan { + debug, + test_mode: false, + debug_query_params: None, + override_engine: BidParamOverrideEngine::default(), + consent_forwarding: ConsentForwardingMode::default(), + } + } + + fn planned_prebid_input(slot_ids: &[&str]) -> ProviderAuctionInput { + let provider_id = ProviderId::from_str("pbs-instance").expect("should parse provider ID"); + let plan = crate::auction::plan::AuctionPlan::compile(AuctionPlanConfig { + timeout_ms: 1_000, + providers: BTreeMap::from([( + provider_id, + ProviderConfig { + protocol: "openrtb-2.6".to_string(), + profile: "prebid-server".to_string(), + endpoint: "https://pbs.example/openrtb2/auction".to_string(), + timeout_ms: None, + routing: RoutingMode::AllEligible, + notifications: NotificationConfig::default(), + profile_config: json!({}), + }, + )]), + bidders: BTreeMap::new(), + mediator: None, + request_signing: None, + }) + .expect("should compile planned PBS test plan"); + let inbound = http::Request::new(EdgeBody::empty()); + let request = make_auction_request( + slot_ids + .iter() + .map(|slot_id| make_slot(slot_id, HashMap::new())) + .collect(), + ); + crate::auction::routing::route_auction(request, &inbound, &plan, None) + .inputs() + .first() + .expect("should route planned PBS test slots") + .clone() + } + + #[test] + fn planned_parser_preserves_non_success_debug_and_204_json_parity() { + let profile = planned_prebid_profile(true); + let error = futures::executor::block_on(parse_planned_prebid_response( + "pbs-instance", + &profile, + &planned_prebid_input(&["fictional-slot"]), + prebid_platform_response( + StatusCode::BAD_GATEWAY, + Some("application/json"), + br#"{"errors":{"example":[{"message":" fictional rejection "}]}}"#.to_vec(), + ), + 42, + "auction-1", + )) + .expect("should classify non-success response"); + assert_eq!(error.provider, "pbs-instance"); + assert_eq!(error.status, crate::auction::types::BidStatus::Error); + assert_eq!(error.metadata["error_type"], ERROR_TYPE_HTTP_STATUS); + assert_eq!(error.metadata["http_status"], 502); + assert_eq!(error.metadata["upstream_message"], "fictional rejection"); + assert_eq!(error.metadata["upstream_message_truncated"], false); + + let no_content = futures::executor::block_on(parse_planned_prebid_response( + "pbs-instance", + &profile, + &planned_prebid_input(&["fictional-slot"]), + prebid_platform_response(StatusCode::NO_CONTENT, None, Vec::new()), + 7, + "auction-2", + )); + assert!( + no_content.is_err(), + "PBS must attempt JSON parsing for every successful status including 204" + ); + } + + #[test] + fn planned_parser_validates_top_level_currency_and_preserves_debug_metadata() { + let profile = planned_prebid_profile(true); + let input = planned_prebid_input(&["fictional-slot"]); + let cases = [ + ( + "omitted", + None, + crate::auction::types::BidStatus::Success, + None, + None, + ), + ( + "usd", + Some(json!("USD")), + crate::auction::types::BidStatus::Success, + None, + None, + ), + ( + "lowercase-usd", + Some(json!("usd")), + crate::auction::types::BidStatus::Success, + None, + None, + ), + ( + "eur", + Some(json!("EUR")), + crate::auction::types::BidStatus::NoBid, + Some("EUR"), + None, + ), + ( + "malformed", + Some(json!(["USD"])), + crate::auction::types::BidStatus::Error, + None, + Some("parse_response"), + ), + ]; + + for (name, currency, expected_status, unsupported_currency, error_type) in cases { + let mut body = json!({ + "seatbid": [{ + "seat": "exampleBidder", + "bid": [{ + "impid": "fictional-slot", + "price": 1.25, + "w": 300, + "h": 250 + }] + }], + "ext": { + "responsetimemillis": {"exampleBidder": 12}, + "errors": {"fictional": []}, + "warnings": {"fictional": ["warning"]}, + "debug": {"httpcalls": {"exampleBidder": []}}, + "prebid": {"bidstatus": [{"bidder": "exampleBidder"}]} + } + }); + if let Some(currency) = currency { + body.as_object_mut() + .expect("should build response object") + .insert("cur".to_string(), currency); + } + let parsed = futures::executor::block_on(parse_planned_prebid_response( + "pbs-instance", + &profile, + &input, + prebid_platform_response( + StatusCode::OK, + Some("application/json"), + serde_json::to_vec(&body).expect("should serialize planned PBS response"), + ), + 9, + name, + )) + .expect("currency classification should return a materialized provider response"); + + assert_eq!(parsed.status, expected_status, "{name}"); + assert_eq!( + parsed + .metadata + .get("unsupported_currency") + .and_then(Json::as_str), + unsupported_currency, + "{name}" + ); + assert_eq!( + parsed.metadata.get("error_type").and_then(Json::as_str), + error_type, + "{name}" + ); + assert_eq!(parsed.metadata["responsetimemillis"]["exampleBidder"], 12); + assert!(parsed.metadata.contains_key("errors"), "{name}"); + assert!(parsed.metadata.contains_key("warnings"), "{name}"); + assert_eq!( + parsed.metadata["debug"]["httpcalls"]["exampleBidder"], + json!([]), + "{name}" + ); + assert_eq!( + parsed.metadata["bidstatus"][0]["bidder"], "exampleBidder", + "{name}" + ); + + if expected_status == crate::auction::types::BidStatus::Success { + assert_eq!(parsed.bids.len(), 1, "{name}"); + assert_eq!(parsed.bids[0].currency, DEFAULT_CURRENCY, "{name}"); + } else { + assert!(parsed.bids.is_empty(), "{name}"); + } + } + } + + #[test] + fn planned_parser_preserves_seat_identity_and_suppression() { + let profile = planned_prebid_profile(false); + let mut parsed = futures::executor::block_on(parse_planned_prebid_response( + "pbs-instance", + &profile, + &planned_prebid_input(&["literal", "missing", "non-string"]), + prebid_platform_response( + StatusCode::OK, + Some("application/json"), + br#"{"seatbid":[{"seat":"unknown","bid":[{"impid":"literal","price":1.0,"w":300,"h":250,"nurl":"https://notify.example/literal","burl":"https://notify.example/literal"}]},{"bid":[{"impid":"missing","price":2.0,"w":300,"h":250,"nurl":"https://notify.example/missing","burl":"https://notify.example/missing"}]},{"seat":4,"bid":[{"impid":"non-string","price":3.0,"w":300,"h":250,"nurl":"https://notify.example/non-string","burl":"https://notify.example/non-string"}]}]}"#.to_vec(), + ), + 8, + "auction-3", + )) + .expect("should parse seats and fallback delivery bidders"); + + assert_eq!(parsed.bids[0].bidder, "unknown"); + assert_eq!(parsed.bids[0].returned_seat.as_deref(), Some("unknown")); + for bid in &parsed.bids[1..] { + assert_eq!(bid.bidder, "unknown"); + assert!(bid.returned_seat.is_none()); + } + + crate::auction::openrtb::apply_notification_policy( + &mut parsed.bids, + &crate::auction::plan::NotificationPolicy { + suppress_all: false, + suppress_seats: std::collections::BTreeSet::from(["unknown".to_string()]), + }, + ); + assert!(parsed.bids[0].nurl.is_none()); + assert!(parsed.bids[0].burl.is_none()); + for bid in &parsed.bids[1..] { + assert!( + bid.nurl.is_some(), + "fallback bidder must not match seat suppression" + ); + assert!( + bid.burl.is_some(), + "fallback bidder must not match seat suppression" + ); + } + } + + #[test] + fn planned_parser_rejects_unrequested_mismatched_and_negative_bids() { + let profile = planned_prebid_profile(false); + let input = planned_prebid_input(&["requested"]); + let parsed = futures::executor::block_on(parse_planned_prebid_response( + "pbs-instance", + &profile, + &input, + prebid_platform_response( + StatusCode::OK, + Some("application/json"), + br#"{"seatbid":[{"bid":[{"impid":"requested","price":1.0,"w":300,"h":250},{"impid":"unrequested","price":2.0,"w":300,"h":250},{"impid":"requested","price":3.0,"w":1,"h":1},{"impid":"requested","price":-1.0,"w":300,"h":250}]}]}"#.to_vec(), + ), + 42, + "auction-validation", + )) + .expect("should parse planned PBS response"); + + assert_eq!(parsed.bids.len(), 1, "should retain only the admitted bid"); + assert_eq!(parsed.bids[0].slot_id, "requested"); + assert_eq!(parsed.bids[0].price, Some(1.0)); + } + + #[test] + fn planned_parser_matches_legacy_error_content_type_and_debug_metadata() { + let debug_profile = planned_prebid_profile(true); + for (content_type, body, expected_message) in [ + ( + Some("application/json"), + br#"{"message":" JSON rejection "}"#.as_slice(), + Some("JSON rejection"), + ), + ( + Some("text/plain; charset=utf-8"), + b" plain text rejection ".as_slice(), + Some("plain text rejection"), + ), + ( + Some("text/html"), + b"proxy failure".as_slice(), + None, + ), + ( + Some("application/octet-stream"), + b"unsupported plain text".as_slice(), + None, + ), + ] { + let parsed = futures::executor::block_on(parse_planned_prebid_response( + "pbs-instance", + &debug_profile, + &planned_prebid_input(&["fictional-slot"]), + prebid_platform_response(StatusCode::BAD_REQUEST, content_type, body.to_vec()), + 42, + "auction-content-type", + )) + .expect("should classify non-success response"); + assert_eq!( + parsed + .metadata + .get("upstream_message") + .and_then(Json::as_str), + expected_message, + "should match legacy error content-type handling" + ); + } + + let no_debug = planned_prebid_profile(false); + let parsed = futures::executor::block_on(parse_planned_prebid_response( + "pbs-instance", + &no_debug, + &planned_prebid_input(&["fictional-slot"]), + prebid_platform_response( + StatusCode::BAD_REQUEST, + Some("application/json"), + br#"{"message":"must stay hidden"}"#.to_vec(), + ), + 42, + "auction-no-debug", + )) + .expect("should classify non-success response without debug metadata"); + assert!( + !parsed.metadata.contains_key("upstream_message") + && !parsed.metadata.contains_key("upstream_message_truncated"), + "debug-disabled planned PBS must not expose upstream error metadata" + ); + } + + #[test] + fn planned_parser_bounds_oversized_success_and_error_bodies() { + let profile = planned_prebid_profile(true); + for response in [ + prebid_platform_response( + StatusCode::OK, + Some("application/json"), + vec![b'x'; UPSTREAM_RTB_MAX_RESPONSE_BYTES + 1], + ), + prebid_platform_response_with_body( + StatusCode::OK, + Some("application/json"), + EdgeBody::stream(futures::stream::iter([ + Bytes::from(vec![b'x'; UPSTREAM_RTB_MAX_RESPONSE_BYTES]), + Bytes::from_static(b"x"), + ])), + ), + ] { + assert!( + futures::executor::block_on(parse_planned_prebid_response( + "pbs-instance", + &profile, + &planned_prebid_input(&["fictional-slot"]), + response, + 42, + "auction-oversized-success", + )) + .is_err(), + "oversized successful planned PBS responses must fail bounded collection" + ); + } + + for response in [ + prebid_platform_response( + StatusCode::BAD_GATEWAY, + Some("text/plain"), + vec![b'x'; UPSTREAM_RTB_MAX_RESPONSE_BYTES + 1], + ), + prebid_platform_response_with_body( + StatusCode::SERVICE_UNAVAILABLE, + Some("text/plain"), + EdgeBody::stream(futures::stream::iter([ + Bytes::from(vec![b'x'; UPSTREAM_RTB_MAX_RESPONSE_BYTES]), + Bytes::from_static(b"x"), + ])), + ), + ] { + let expected_status = response.response.status().as_u16(); + let parsed = futures::executor::block_on(parse_planned_prebid_response( + "pbs-instance", + &profile, + &planned_prebid_input(&["fictional-slot"]), + response, + 42, + "auction-oversized-error", + )) + .expect("should preserve HTTP classification for oversized planned error body"); + assert_oversized_http_error_is_classified(&parsed, expected_status); + } + } + #[test] fn parse_bid_extracts_cache_id_from_ext_prebid_cache_bids() { let bid_json = serde_json::json!({ @@ -7537,7 +8763,7 @@ set = { networkId = 42 } "nurl": "https://ssp.example/win?id=abc123", "burl": "https://ssp.example/bill?id=abc123" }); - let config = PrebidIntegrationConfig { + let config = LegacyPrebidServerConfig { suppress_nurl: true, ..base_config() }; @@ -7565,7 +8791,7 @@ set = { networkId = 42 } "nurl": "https://ssp.example/win?id=abc123", "burl": "https://ssp.example/bill?id=abc123" }); - let config = PrebidIntegrationConfig { + let config = LegacyPrebidServerConfig { suppress_nurl_bidders: vec!["appnexus".to_string()], ..base_config() }; @@ -7655,6 +8881,10 @@ set = { networkId = 42 } let bid = provider .parse_bid(&bid_json, "example-bidder") .expect("should parse bid"); + assert!( + bid.returned_seat.is_none(), + "legacy PBS parsing must not attach planned telemetry identity" + ); assert_eq!( bid.bid_id.as_deref(), Some("019f7e2a-b45b-70b0-a2d1-b651c430700b"), @@ -7688,6 +8918,64 @@ set = { networkId = 42 } ); } + #[test] + fn planned_transport_preserves_malformed_cookie_bytes_and_attested_xff() { + let inbound = http::Request::builder() + .uri("https://publisher.example/auction") + .header( + header::COOKIE, + HeaderValue::from_bytes(b"session=fictional;\xffbroken") + .expect("should build malformed cookie header"), + ) + .header(header::USER_AGENT, "Fictional Browser/2") + .header("x-forwarded-for", "198.51.100.8") + .body(EdgeBody::empty()) + .expect("should build inbound request"); + let plan = + crate::auction::plan::AuctionPlan::compile(crate::auction::plan::AuctionPlanConfig { + timeout_ms: 321, + providers: std::collections::BTreeMap::new(), + bidders: std::collections::BTreeMap::new(), + mediator: None, + request_signing: None, + }) + .expect("should compile empty plan"); + let routed = crate::auction::routing::route_auction( + canonical_parity_auction_request(), + &inbound, + &plan, + Some(std::net::IpAddr::from([203, 0, 113, 9])), + ); + let mut outbound = http::Request::builder() + .uri("https://pbs.example.test/openrtb2/auction") + .body(EdgeBody::empty()) + .expect("should build outbound request"); + + apply_prebid_transport_headers( + routed.prebid_transport_headers(), + &mut outbound, + ConsentForwardingMode::OpenrtbOnly, + routed.attested_client_ip(), + ); + + assert_eq!( + outbound + .headers() + .get(header::COOKIE) + .map(HeaderValue::as_bytes), + Some(b"session=fictional;\xffbroken".as_slice()), + "should forward malformed cookie bytes unchanged" + ); + assert_eq!( + outbound + .headers() + .get("x-forwarded-for") + .and_then(|value| value.to_str().ok()), + Some("203.0.113.9"), + "should use only attested XFF" + ); + } + #[test] fn copy_request_headers_replaces_client_supplied_xff_with_attested_ip() { let from = http::Request::builder() @@ -7743,4 +9031,194 @@ set = { networkId = 42 } "should not forward the client-supplied XFF when no attested IP exists" ); } + + fn header_value<'a>(headers: &'a [(String, Vec)], name: &str) -> Option<&'a [u8]> { + headers + .iter() + .find(|(header_name, _)| header_name.eq_ignore_ascii_case(name)) + .map(|(_, value)| value.as_slice()) + } + + #[test] + fn dispatch_preserves_pbs_transport_header_and_cookie_matrix() { + for (mode, cookie, expected_cookie) in [ + ( + ConsentForwardingMode::Both, + HeaderValue::from_static("session=fictional; euconsent-v2=fictional-tcf"), + Some(b"session=fictional; euconsent-v2=fictional-tcf".as_slice()), + ), + ( + ConsentForwardingMode::CookiesOnly, + HeaderValue::from_static("session=fictional; euconsent-v2=fictional-tcf"), + Some(b"session=fictional; euconsent-v2=fictional-tcf".as_slice()), + ), + ( + ConsentForwardingMode::OpenrtbOnly, + HeaderValue::from_static("session=fictional; euconsent-v2=fictional-tcf"), + Some(b"session=fictional".as_slice()), + ), + ( + ConsentForwardingMode::OpenrtbOnly, + HeaderValue::from_static("euconsent-v2=fictional-tcf; us_privacy=1YNN"), + None, + ), + ( + ConsentForwardingMode::OpenrtbOnly, + HeaderValue::from_bytes(b"session=fictional;\xffbroken") + .expect("should build malformed cookie header"), + Some(b"session=fictional;\xffbroken".as_slice()), + ), + ] { + let stub = Arc::new(StubHttpClient::new()); + stub.push_response(204, Vec::new()); + let services = build_services_with_http_client_and_client_ip( + Arc::clone(&stub) as Arc, + std::net::IpAddr::from([203, 0, 113, 9]), + ); + let settings = make_settings(); + let mut config = base_config(); + config.consent_forwarding = mode; + let provider = PrebidAuctionProvider::new(config); + let auction_request = create_test_auction_request(); + let inbound = http::Request::builder() + .uri("https://pub.example/auction") + .header( + header::REFERER, + "https://referrer.example/story?fictional=1", + ) + .header(header::USER_AGENT, "Fictional Browser/1.0") + .header(header::ACCEPT_LANGUAGE, "en-US,en;q=0.9") + .header("x-forwarded-for", "198.51.100.8") + .header(header::COOKIE, cookie) + .body(EdgeBody::empty()) + .expect("should build inbound request"); + let context = AuctionContext { + settings: &settings, + request: &inbound, + timeout_ms: 321, + transport_timeout_ms: 321, + provider_responses: None, + services: &services, + }; + + futures::executor::block_on(provider.request_bids(&auction_request, &context)) + .expect("should dispatch PBS request"); + + let headers = stub.recorded_request_header_bytes(); + assert_eq!(headers.len(), 1, "should dispatch one request"); + let headers = &headers[0]; + assert_eq!( + header_value(headers, "referer"), + Some(b"https://referrer.example/story?fictional=1".as_slice()) + ); + assert_eq!( + header_value(headers, "user-agent"), + Some(b"Fictional Browser/1.0".as_slice()) + ); + assert_eq!( + header_value(headers, "accept-language"), + Some(b"en-US,en;q=0.9".as_slice()) + ); + assert_eq!( + header_value(headers, "x-forwarded-for"), + Some(b"203.0.113.9".as_slice()), + "should replace spoofable XFF with platform-attested IP" + ); + assert_eq!(header_value(headers, "cookie"), expected_cookie); + + let body = stub + .recorded_request_bodies() + .into_iter() + .next() + .expect("should capture PBS request body"); + let body: Json = serde_json::from_slice(&body).expect("should parse PBS request body"); + assert_eq!( + body["site"]["ref"], "https://referrer.example/story?fictional=1", + "should place the raw Referer in PBS site.ref as well as forwarding it" + ); + } + } + + #[test] + fn pbs_request_serialization_goldens_cover_disabled_and_deterministic_enabled_signing() { + let provider = PrebidAuctionProvider::new(base_config()); + let auction_request = canonical_parity_auction_request(); + let settings = make_settings(); + let inbound = http::Request::builder() + .uri("https://edge.example/auction") + .header( + header::REFERER, + "https://referrer.example/story?fictional=1", + ) + .header(header::ACCEPT_LANGUAGE, "en-US,en;q=0.9") + .body(EdgeBody::empty()) + .expect("should build inbound request"); + let services = crate::platform::test_support::noop_services(); + let context = AuctionContext { + settings: &settings, + request: &inbound, + timeout_ms: 321, + transport_timeout_ms: 321, + provider_responses: None, + services: &services, + }; + let request_info = RequestInfo { + host: "publisher.example".to_string(), + scheme: "https".to_string(), + }; + let disabled = provider.to_openrtb(&auction_request, &context, None, request_info.clone()); + let disabled = + serde_json::to_string(&disabled).expect("should serialize disabled PBS request"); + let disabled_value: Json = + serde_json::from_str(&disabled).expect("should parse disabled PBS golden request"); + assert_eq!( + disabled_value["user"]["ext"]["ConsentedProvidersSettings"]["consented_providers"], + json!("fictional-ac"), + "should retain Google Additional Consent only in PBS's existing extension" + ); + assert_eq!( + disabled, + r#"{"id":"fictional-auction","imp":[{"id":"fictional-slot","banner":{"format":[{"w":300,"h":250},{"w":728,"h":90}]},"tagid":"fictional-slot","bidfloor":1.0,"bidfloorcur":"USD","secure":1,"ext":{"prebid":{"bidder":{"exampleBidder":{"placement":"fictional-placement"}}}}}],"site":{"domain":"publisher.example","page":"https://publisher.example/article","ref":"https://referrer.example/story?fictional=1","publisher":{"domain":"publisher.example"}},"device":{"geo":{"lat":12.34,"lon":56.78,"type":2,"country":"US","region":"CA","metro":"501","city":"Example City"},"ua":"Fictional Browser","ip":"192.0.2.10","language":"en"},"user":{"id":"fictional-user","consent":"fictional-tcf","ext":{"ConsentedProvidersSettings":{"consented_providers":"fictional-ac"},"consent":"fictional-tcf","eids":[{"source":"identity.example","uids":[{"atype":1,"id":"fictional-uid"}]}]}},"tmax":321,"cur":["USD"],"regs":{"gdpr":1,"us_privacy":"1YNN","gpp":"fictional-gpp","gpp_sid":[2,6],"ext":{"gdpr":1,"gpp":"fictional-gpp","gpp_sid":[2,6],"us_privacy":"1YNN"}},"ext":{"prebid":{},"trusted_server":{"request_host":"publisher.example","request_scheme":"https"}}}"#, + "should preserve the complete disabled PBS wire shape" + ); + + let mut config_data = HashMap::new(); + config_data.insert("current-kid".to_string(), "fictional-kid".to_string()); + let mut secret_data = HashMap::new(); + secret_data.insert( + "fictional-kid".to_string(), + base64::engine::general_purpose::STANDARD + .encode([7_u8; 32]) + .into_bytes(), + ); + let signing_services = build_services_with_config_secret_and_http_client( + HashMapConfigStore::new(config_data), + HashMapSecretStore::new(secret_data), + Arc::new(NoopHttpClient), + ); + let signer = RequestSigner::from_services(&signing_services) + .expect("should load deterministic test signer"); + let signing = SigningParams { + request_id: "fictional-auction".to_string(), + request_host: "publisher.example".to_string(), + request_scheme: "https".to_string(), + timestamp: 1_706_900_000, + }; + let signature = signer + .sign_request(&signing) + .expect("should sign deterministic PBS request"); + let enabled = provider.to_openrtb( + &auction_request, + &context, + Some((&signer, signature, &signing)), + request_info, + ); + let enabled = + serde_json::to_string(&enabled).expect("should serialize enabled PBS request"); + assert_eq!( + enabled, + r#"{"id":"fictional-auction","imp":[{"id":"fictional-slot","banner":{"format":[{"w":300,"h":250},{"w":728,"h":90}]},"tagid":"fictional-slot","bidfloor":1.0,"bidfloorcur":"USD","secure":1,"ext":{"prebid":{"bidder":{"exampleBidder":{"placement":"fictional-placement"}}}}}],"site":{"domain":"publisher.example","page":"https://publisher.example/article","ref":"https://referrer.example/story?fictional=1","publisher":{"domain":"publisher.example"}},"device":{"geo":{"lat":12.34,"lon":56.78,"type":2,"country":"US","region":"CA","metro":"501","city":"Example City"},"ua":"Fictional Browser","ip":"192.0.2.10","language":"en"},"user":{"id":"fictional-user","consent":"fictional-tcf","ext":{"ConsentedProvidersSettings":{"consented_providers":"fictional-ac"},"consent":"fictional-tcf","eids":[{"source":"identity.example","uids":[{"atype":1,"id":"fictional-uid"}]}]}},"tmax":321,"cur":["USD"],"regs":{"gdpr":1,"us_privacy":"1YNN","gpp":"fictional-gpp","gpp_sid":[2,6],"ext":{"gdpr":1,"gpp":"fictional-gpp","gpp_sid":[2,6],"us_privacy":"1YNN"}},"ext":{"prebid":{},"trusted_server":{"kid":"fictional-kid","request_host":"publisher.example","request_scheme":"https","signature":"LU_JUIA1BT80ShZNjSa4PIF5T-uMjEeodwKrV_6bXgh0hi1SYVtCKn9g_DTW62krmjCOFgoFYPHsu6L0nAcuDg","ts":1706900000,"version":"1.1"}}}"#, + "should preserve the complete enabled PBS wire shape" + ); + } } diff --git a/crates/trusted-server-core/src/integrations/registry.rs b/crates/trusted-server-core/src/integrations/registry.rs index 280eae847..5b11b105e 100644 --- a/crates/trusted-server-core/src/integrations/registry.rs +++ b/crates/trusted-server-core/src/integrations/registry.rs @@ -8,6 +8,7 @@ use error_stack::Report; use http::{Method, Request, Response}; use matchit::Router; +use crate::auction::AuctionPlan; use crate::constants::HEADER_X_TS_EC; use crate::ec::EcContext; use crate::ec::kv::KvIdentityGraph; @@ -777,9 +778,21 @@ pub struct ProxyDispatchInput<'a> { #[derive(Clone, Default)] pub struct IntegrationRegistry { inner: Arc, + plan: Option>, } impl IntegrationRegistry { + /// Build a test registry from settings and a newly compiled auction plan. + /// + /// # Errors + /// + /// Returns an error when plan compilation or integration registration fails. + #[cfg(test)] + pub fn new(settings: &Settings) -> Result> { + let plan = Arc::new(crate::auction::compile_auction_plan(settings)?); + Self::with_plan(settings, plan) + } + /// Build a registry from the provided settings. /// /// # Errors @@ -789,89 +802,109 @@ impl IntegrationRegistry { /// # Panics /// /// Panics if a route path ends with `/*` but `strip_suffix` unexpectedly fails (invariant violation). - pub fn new(settings: &Settings) -> Result> { + pub fn with_plan( + settings: &Settings, + plan: Arc, + ) -> Result> { let mut inner = IntegrationRegistryInner::default(); - + let mut registrations = Vec::new(); + if let Some(registration) = crate::integrations::prebid::register_for_plan(settings, &plan)? + { + registrations.push(registration); + } + if let Some(registration) = crate::integrations::aps::register_for_plan(settings, &plan)? { + registrations.push(registration); + } for builder in crate::integrations::builders() { if let Some(registration) = (builder.build)(settings)? { - debug_assert_eq!( - registration.integration_id, builder.id, - "integration builder ID should match registration ID" - ); - inner - .enabled_integration_ids - .push(registration.integration_id); - - for proxy in registration.proxies { - for route in proxy.routes() { - let value = (proxy.clone(), registration.integration_id); - - // Convert /* wildcard to matchit's {*rest} syntax - let matchit_path = if route.path.ends_with("/*") { - format!( - "{}/{{*rest}}", - route - .path - .strip_suffix("/*") - .expect("path should end with '/*'") - ) - } else { - route.path.clone() - }; - - // Select appropriate router and insert - let router = match route.method { - Method::GET => &mut inner.get_router, - Method::POST => &mut inner.post_router, - Method::PUT => &mut inner.put_router, - Method::DELETE => &mut inner.delete_router, - Method::PATCH => &mut inner.patch_router, - Method::HEAD => &mut inner.head_router, - Method::OPTIONS => &mut inner.options_router, - _ => { - log::warn!( - "Unsupported HTTP method {} for route {}", - route.method, - route.path - ); - continue; - } - }; - - if let Err(e) = router.insert(&matchit_path, value) { - return Err(Report::new(TrustedServerError::Configuration { - message: format!( - "Integration route registration failed for {} {}: {:?}", - route.method, route.path, e - ), - })); - } + debug_assert_eq!(registration.integration_id, builder.id); + registrations.push(registration); + } + } - inner.routes.push((route, registration.integration_id)); + for registration in registrations { + inner + .enabled_integration_ids + .push(registration.integration_id); + + for proxy in registration.proxies { + for route in proxy.routes() { + let value = (proxy.clone(), registration.integration_id); + + // Convert /* wildcard to matchit's {*rest} syntax + let matchit_path = if route.path.ends_with("/*") { + format!( + "{}/{{*rest}}", + route + .path + .strip_suffix("/*") + .expect("path should end with '/*'") + ) + } else { + route.path.clone() + }; + + // Select appropriate router and insert + let router = match route.method { + Method::GET => &mut inner.get_router, + Method::POST => &mut inner.post_router, + Method::PUT => &mut inner.put_router, + Method::DELETE => &mut inner.delete_router, + Method::PATCH => &mut inner.patch_router, + Method::HEAD => &mut inner.head_router, + Method::OPTIONS => &mut inner.options_router, + _ => { + log::warn!( + "Unsupported HTTP method {} for route {}", + route.method, + route.path + ); + continue; + } + }; + + if let Err(e) = router.insert(&matchit_path, value) { + return Err(Report::new(TrustedServerError::Configuration { + message: format!( + "Integration route registration failed for {} {}: {:?}", + route.method, route.path, e + ), + })); } - } - inner - .html_rewriters - .extend(registration.attribute_rewriters); - inner.script_rewriters.extend(registration.script_rewriters); - inner - .html_post_processors - .extend(registration.html_post_processors); - inner.head_injectors.extend(registration.head_injectors); - inner.request_filters.extend(registration.request_filters); - if registration.js_disabled { - inner.disabled_js_ids.push(registration.integration_id); - } else if registration.js_deferred { - inner.deferred_js_ids.push(registration.integration_id); + + inner.routes.push((route, registration.integration_id)); } } + inner + .html_rewriters + .extend(registration.attribute_rewriters); + inner.script_rewriters.extend(registration.script_rewriters); + inner + .html_post_processors + .extend(registration.html_post_processors); + inner.head_injectors.extend(registration.head_injectors); + inner.request_filters.extend(registration.request_filters); + if registration.js_disabled { + inner.disabled_js_ids.push(registration.integration_id); + } else if registration.js_deferred { + inner.deferred_js_ids.push(registration.integration_id); + } } Ok(Self { inner: Arc::new(inner), + plan: Some(plan), }) } + /// Return whether this registry and another consumer share the same plan allocation. + #[must_use] + pub fn shares_plan(&self, plan: &Arc) -> bool { + self.plan + .as_ref() + .is_some_and(|owned| Arc::ptr_eq(owned, plan)) + } + fn find_route(&self, method: &Method, path: &str) -> Option<&RouteValue> { let router = match *method { Method::GET => &self.inner.get_router, @@ -1192,6 +1225,7 @@ impl IntegrationRegistry { pub fn empty_for_tests() -> Self { Self { inner: Arc::new(IntegrationRegistryInner::default()), + plan: None, } } @@ -1220,6 +1254,7 @@ impl IntegrationRegistry { deferred_js_ids: Vec::new(), disabled_js_ids: Vec::new(), }), + plan: None, } } @@ -1249,6 +1284,7 @@ impl IntegrationRegistry { deferred_js_ids: Vec::new(), disabled_js_ids: Vec::new(), }), + plan: None, } } @@ -1274,6 +1310,7 @@ impl IntegrationRegistry { deferred_js_ids: Vec::new(), disabled_js_ids: Vec::new(), }), + plan: None, } } @@ -1339,6 +1376,7 @@ impl IntegrationRegistry { deferred_js_ids: Vec::new(), disabled_js_ids: Vec::new(), }), + plan: None, } } } @@ -2104,17 +2142,21 @@ mod tests { "prebid", &serde_json::json!({ "enabled": true, - "server_url": "https://test-prebid.com/openrtb2/auction", "external_bundle_url": "https://assets.example/prebid/trusted-prebid.js", "timeout_ms": 1000, - "bidders": ["mocktioneer"], "debug": false }), ) .expect("should insert prebid config"); - let registry = - IntegrationRegistry::new(&settings_with_prebid).expect("should create registry"); + let registry = IntegrationRegistry::with_plan( + &settings_with_prebid, + Arc::new( + crate::auction::compile_auction_plan(&settings_with_prebid) + .expect("should compile auction plan"), + ), + ) + .expect("should create registry"); let all = registry.js_module_ids(); let immediate = registry.js_module_ids_immediate(); @@ -2150,7 +2192,14 @@ mod tests { .insert_config("nextjs", &serde_json::json!({ "enabled": true })) .expect("should insert nextjs config"); - let registry = IntegrationRegistry::new(&settings).expect("should create registry"); + let registry = IntegrationRegistry::with_plan( + &settings, + Arc::new( + crate::auction::compile_auction_plan(&settings) + .expect("should compile auction plan"), + ), + ) + .expect("should create registry"); let all = registry.js_module_ids(); assert!( @@ -2179,7 +2228,14 @@ mod tests { .insert_config("osano", &serde_json::json!({ "enabled": true })) .expect("should insert osano config"); - let registry = IntegrationRegistry::new(&settings).expect("should create registry"); + let registry = IntegrationRegistry::with_plan( + &settings, + Arc::new( + crate::auction::compile_auction_plan(&settings) + .expect("should compile auction plan"), + ), + ) + .expect("should create registry"); let immediate = registry.js_module_ids_immediate(); assert!( @@ -2207,13 +2263,19 @@ mod tests { "prebid", &serde_json::json!({ "enabled": false, - "server_url": "https://test-prebid.com/openrtb2/auction", "external_bundle_url": "https://assets.example/prebid/trusted-prebid.js", }), ) .expect("should update prebid config"); - let registry = IntegrationRegistry::new(&settings).expect("should create registry"); + let registry = IntegrationRegistry::with_plan( + &settings, + Arc::new( + crate::auction::compile_auction_plan(&settings) + .expect("should compile auction plan"), + ), + ) + .expect("should create registry"); let deferred = registry.js_module_ids_deferred(); assert!( @@ -2231,13 +2293,19 @@ mod tests { "prebid", &serde_json::json!({ "enabled": true, - "server_url": "https://test-prebid.com/openrtb2/auction", "external_bundle_url": "https://assets.example/prebid/trusted-prebid.js" }), ) .expect("should update prebid config"); - let registry = IntegrationRegistry::new(&settings).expect("should create registry"); + let registry = IntegrationRegistry::with_plan( + &settings, + Arc::new( + crate::auction::compile_auction_plan(&settings) + .expect("should compile auction plan"), + ), + ) + .expect("should create registry"); assert!( registry.js_module_ids().contains(&"prebid"), @@ -2267,17 +2335,21 @@ mod tests { "prebid", &serde_json::json!({ "enabled": true, - "server_url": "https://test-prebid.com/openrtb2/auction", "external_bundle_url": "https://assets.example/prebid/trusted-prebid.js", "timeout_ms": 1000, - "bidders": ["mocktioneer"], "debug": false }), ) .expect("should insert prebid config"); - let registry = - IntegrationRegistry::new(&settings_with_prebid).expect("should create registry"); + let registry = IntegrationRegistry::with_plan( + &settings_with_prebid, + Arc::new( + crate::auction::compile_auction_plan(&settings_with_prebid) + .expect("should compile auction plan"), + ), + ) + .expect("should create registry"); let all = registry.js_module_ids(); let mut recombined = registry.js_module_ids_immediate(); diff --git a/crates/trusted-server-core/src/integrations/sourcepoint.rs b/crates/trusted-server-core/src/integrations/sourcepoint.rs index 3caaadeef..7b2e49fb0 100644 --- a/crates/trusted-server-core/src/integrations/sourcepoint.rs +++ b/crates/trusted-server-core/src/integrations/sourcepoint.rs @@ -1396,7 +1396,14 @@ mod tests { .insert_config(SOURCEPOINT_INTEGRATION_ID, &json!({ "enabled": true })) .expect("should insert config"); - let registry = IntegrationRegistry::new(&settings).expect("should create registry"); + let registry = IntegrationRegistry::with_plan( + &settings, + Arc::new( + crate::auction::compile_auction_plan(&settings) + .expect("should compile auction plan"), + ), + ) + .expect("should create registry"); for method in [Method::GET, Method::POST, Method::HEAD, Method::OPTIONS] { assert!( registry.has_route(&method, "/integrations/sourcepoint/cdn/wrapper/v2/messages"), diff --git a/crates/trusted-server-core/src/lib.rs b/crates/trusted-server-core/src/lib.rs index 48e92faed..76621baf7 100644 --- a/crates/trusted-server-core/src/lib.rs +++ b/crates/trusted-server-core/src/lib.rs @@ -64,6 +64,7 @@ pub mod request_signing; pub mod response_privacy; pub mod rsc_flight; pub(crate) mod s3_sigv4; +pub mod secret_resolution; pub mod settings; pub mod settings_data; pub mod storage; diff --git a/crates/trusted-server-core/src/platform/backend_naming.rs b/crates/trusted-server-core/src/platform/backend_naming.rs new file mode 100644 index 000000000..54fc176b1 --- /dev/null +++ b/crates/trusted-server-core/src/platform/backend_naming.rs @@ -0,0 +1,652 @@ +//! Pure adapter backend naming and auction-target capability policies. +//! +//! These policies contain no platform SDK calls. Startup validation, CLI +//! validation, and runtime adapters can therefore predict the same backend +//! names before any backend registration occurs. + +use core::fmt::Write as _; + +use error_stack::Report; +use sha2::{Digest as _, Sha256}; + +use super::PlatformBackendSpec; +use crate::host_header::validate_host_header_override_value; + +const MAX_FASTLY_BACKEND_NAME_LEN: usize = 255; +const MAX_FASTLY_READABLE_PREFIX_LEN: usize = 200; +const FASTLY_SPEC_DIGEST_HEX_LEN: usize = 32; +const FASTLY_TRANSPORT_TIMEOUT_QUANTUM_MS: u32 = 250; +const FASTLY_TRANSPORT_TIMEOUT_QUANTUM_CEILING_MS: u32 = 2000; +const FASTLY_SUB_QUANTUM_LADDER_MS: [u32; 4] = [200, 150, 100, 50]; +const FASTLY_TRANSPORT_TIMEOUT_COARSE_LADDER_MS: [u32; 8] = + [2000, 3000, 5000, 10000, 20000, 30000, 45000, 60000]; +const FASTLY_DYNAMIC_BACKEND_LIMIT: usize = 200; +const FASTLY_NON_AUCTION_BACKEND_RESERVE: usize = 40; + +/// A pure backend-name and transport-timeout policy for one adapter. +/// +/// Cloudflare and Spin intentionally remain separate variants even though +/// their current no-registration name formats are identical. Keeping distinct +/// policies prevents a future adapter-specific change from silently affecting +/// the other target. +#[derive(Debug, Clone, Copy, Eq, PartialEq)] +pub enum BackendNamingPolicy { + /// Fastly dynamic backend naming and bounded timeout buckets. + Fastly, + /// Axum's environment-segment-compatible correlation name. + Axum, + /// Cloudflare's deterministic no-registration correlation name. + Cloudflare, + /// Spin's deterministic no-registration correlation name. + Spin, +} + +/// Pure backend prediction shared by validation and runtime registration. +#[derive(Debug, Clone, Eq, PartialEq)] +pub struct PredictedBackend { + /// Deterministic platform backend or correlation name. + pub name: String, + /// Explicit or scheme-derived target port. + pub port: u16, +} + +/// Detailed pure backend prediction error. +#[derive(Debug, Clone, Eq, PartialEq, derive_more::Display)] +pub enum BackendNamingError { + /// The backend host is empty. + #[display("missing host")] + MissingHost, + /// A field contains a control character. + #[display("{field} contains control characters")] + ControlCharacters { field: &'static str }, + /// An outbound Host override is invalid. + #[display("host header override {reason}")] + InvalidHostHeaderOverride { reason: &'static str }, + /// A generated Fastly backend name exceeded its documented limit. + #[display("backend name exceeds {limit}-char limit ({actual} chars)")] + NameTooLong { limit: usize, actual: usize }, +} + +impl core::error::Error for BackendNamingError {} + +impl BackendNamingPolicy { + /// Predict a backend name and resolved port without platform I/O. + /// + /// # Errors + /// + /// Returns a naming error when a Fastly backend specification contains an + /// invalid host, scheme, Host override, or cannot fit the platform limit. + pub fn predict( + self, + spec: &PlatformBackendSpec, + ) -> Result> { + match self { + Self::Fastly => predict_fastly(spec), + Self::Axum => Ok(predict_axum(spec)), + Self::Cloudflare => Ok(predict_cloudflare(spec)), + Self::Spin => Ok(predict_spin(spec)), + } + } + + /// Canonicalize a transport timer without changing the logical budget. + /// + /// The result configures adapter transport timers and backend-name + /// stability only. It is not an auction-wide deadline. Fastly bounds the + /// cardinality of budget-derived timers because timers are encoded in + /// dynamic backend names; other adapters preserve the exact bounded value. + #[must_use] + pub fn canonicalize_transport_timeout_ms(self, remaining_ms: u32, configured_ms: u32) -> u32 { + match self { + Self::Fastly => canonicalize_fastly_transport_timeout_ms(remaining_ms, configured_ms), + Self::Axum | Self::Cloudflare | Self::Spin => remaining_ms.min(configured_ms), + } + } + + /// Return the target-specific backend-name budget available to auction providers. + #[must_use] + pub(crate) fn auction_dynamic_backend_budget(self) -> Option { + matches!(self, Self::Fastly).then_some( + FASTLY_DYNAMIC_BACKEND_LIMIT.saturating_sub(FASTLY_NON_AUCTION_BACKEND_RESERVE), + ) + } + + /// Count every transport-timeout bucket one provider can reach. + #[must_use] + pub(crate) fn transport_timeout_bucket_count(self, configured_ms: u32) -> usize { + if !matches!(self, Self::Fastly) || configured_ms == 0 { + return 1; + } + let mut buckets = std::collections::BTreeSet::new(); + buckets.insert(configured_ms); + for remaining_ms in FASTLY_SUB_QUANTUM_LADDER_MS { + let timeout = self.canonicalize_transport_timeout_ms(remaining_ms, configured_ms); + if timeout > 0 { + buckets.insert(timeout); + } + } + for remaining_ms in (FASTLY_TRANSPORT_TIMEOUT_QUANTUM_MS + ..FASTLY_TRANSPORT_TIMEOUT_QUANTUM_CEILING_MS) + .step_by(FASTLY_TRANSPORT_TIMEOUT_QUANTUM_MS as usize) + { + let timeout = self.canonicalize_transport_timeout_ms(remaining_ms, configured_ms); + if timeout > 0 { + buckets.insert(timeout); + } + } + for remaining_ms in FASTLY_TRANSPORT_TIMEOUT_COARSE_LADDER_MS { + let timeout = self.canonicalize_transport_timeout_ms(remaining_ms, configured_ms); + if timeout > 0 { + buckets.insert(timeout); + } + } + buckets.len() + } +} + +/// Canonical adapter target identifier accepted by Trusted Server tooling. +#[derive(Debug, Clone, Copy, Eq, PartialEq)] +pub enum AuctionTargetId { + /// Fastly Compute. + Fastly, + /// Native Axum development server. + Axum, + /// Cloudflare Workers. + Cloudflare, + /// Fermyon Spin. + Spin, +} + +impl AuctionTargetId { + /// Map a canonical `EdgeZero` adapter registry ID to an auction target. + #[must_use] + pub fn from_adapter_id(adapter_id: &str) -> Option { + match adapter_id { + "fastly" => Some(Self::Fastly), + "axum" => Some(Self::Axum), + "cloudflare" => Some(Self::Cloudflare), + "spin" => Some(Self::Spin), + _ => None, + } + } + + /// Return the canonical `EdgeZero` adapter registry ID. + #[must_use] + pub const fn adapter_id(self) -> &'static str { + match self { + Self::Fastly => "fastly", + Self::Axum => "axum", + Self::Cloudflare => "cloudflare", + Self::Spin => "spin", + } + } + + /// Return the shared naming and capability descriptor for this target. + #[must_use] + pub const fn descriptor(self) -> AuctionTargetDescriptor { + match self { + Self::Fastly => AuctionTargetDescriptor { + id: self, + naming_policy: BackendNamingPolicy::Fastly, + capabilities: AuctionTargetCapabilities { + concurrent_provider_fanout: true, + enforceable_total_request_deadline: false, + }, + }, + Self::Axum => AuctionTargetDescriptor { + id: self, + naming_policy: BackendNamingPolicy::Axum, + capabilities: AuctionTargetCapabilities { + concurrent_provider_fanout: true, + enforceable_total_request_deadline: false, + }, + }, + Self::Cloudflare => AuctionTargetDescriptor { + id: self, + naming_policy: BackendNamingPolicy::Cloudflare, + capabilities: AuctionTargetCapabilities { + concurrent_provider_fanout: false, + enforceable_total_request_deadline: false, + }, + }, + Self::Spin => AuctionTargetDescriptor { + id: self, + naming_policy: BackendNamingPolicy::Spin, + capabilities: AuctionTargetCapabilities { + concurrent_provider_fanout: false, + enforceable_total_request_deadline: false, + }, + }, + } + } +} + +/// Adapter capabilities relevant to auction dispatch validation. +#[derive(Debug, Clone, Copy, Eq, PartialEq)] +pub struct AuctionTargetCapabilities { + concurrent_provider_fanout: bool, + enforceable_total_request_deadline: bool, +} + +impl AuctionTargetCapabilities { + /// Return whether multiple provider requests can be in flight concurrently. + #[must_use] + pub const fn supports_concurrent_provider_fanout(self) -> bool { + self.concurrent_provider_fanout + } + + /// Return whether the adapter enforces a total request deadline per provider. + /// + /// Transport first-byte or between-byte timers do not satisfy this + /// capability because they do not cap the complete request lifetime. + #[must_use] + pub const fn has_enforceable_total_request_deadline(self) -> bool { + self.enforceable_total_request_deadline + } +} + +/// Shared target descriptor used by plan validation. +#[derive(Debug, Clone, Copy, Eq, PartialEq)] +pub struct AuctionTargetDescriptor { + id: AuctionTargetId, + naming_policy: BackendNamingPolicy, + capabilities: AuctionTargetCapabilities, +} + +impl AuctionTargetDescriptor { + /// Return the canonical target identity. + #[must_use] + pub const fn id(self) -> AuctionTargetId { + self.id + } + + /// Return the pure backend naming and transport timer policy. + #[must_use] + pub const fn naming_policy(self) -> BackendNamingPolicy { + self.naming_policy + } + + /// Return this target's auction dispatch capabilities. + #[must_use] + pub const fn capabilities(self) -> AuctionTargetCapabilities { + self.capabilities + } +} + +fn default_port(scheme: &str, https_case_insensitive: bool) -> u16 { + let https = if https_case_insensitive { + scheme.eq_ignore_ascii_case("https") + } else { + scheme == "https" + }; + if https { 443 } else { 80 } +} + +fn sanitize_fastly_component(value: &str) -> String { + value + .chars() + .map(|ch| { + if ch.is_ascii_alphanumeric() || matches!(ch, '-' | '_') { + ch + } else { + '_' + } + }) + .collect() +} + +fn canonical_fastly_host(host: &str) -> &str { + host.strip_prefix('[') + .and_then(|value| value.strip_suffix(']')) + .filter(|value| value.parse::().is_ok()) + .unwrap_or(host) +} + +fn fastly_canonical_spec(spec: &PlatformBackendSpec, target_port: u16) -> String { + fn push_field(buffer: &mut String, field: &str) { + buffer.push_str(&field.len().to_string()); + buffer.push(':'); + buffer.push_str(field); + } + + let mut buffer = String::new(); + push_field(&mut buffer, &spec.scheme); + push_field(&mut buffer, canonical_fastly_host(&spec.host)); + push_field(&mut buffer, &target_port.to_string()); + push_field(&mut buffer, if spec.certificate_check { "1" } else { "0" }); + match spec.host_header_override.as_deref() { + Some(value) => { + buffer.push('s'); + push_field(&mut buffer, value); + } + None => buffer.push('n'), + } + match spec.discriminator.as_deref() { + Some(value) => { + buffer.push('s'); + push_field(&mut buffer, value); + } + None => buffer.push('n'), + } + push_field( + &mut buffer, + &spec.first_byte_timeout.as_millis().to_string(), + ); + push_field( + &mut buffer, + &spec.between_bytes_timeout.as_millis().to_string(), + ); + buffer +} + +fn fastly_spec_digest(canonical: &str) -> String { + let mut hasher = Sha256::new(); + hasher.update(canonical.as_bytes()); + let digest = hasher.finalize(); + let mut hex = String::with_capacity(FASTLY_SPEC_DIGEST_HEX_LEN); + for byte in digest.iter().take(FASTLY_SPEC_DIGEST_HEX_LEN / 2) { + write!(hex, "{byte:02x}").expect("should write hex digit to string"); + } + hex +} + +fn predict_fastly( + spec: &PlatformBackendSpec, +) -> Result> { + if spec.host.is_empty() { + return Err(Report::new(BackendNamingError::MissingHost)); + } + if spec.host.chars().any(char::is_control) { + return Err(Report::new(BackendNamingError::ControlCharacters { + field: "host", + })); + } + if spec.scheme.chars().any(char::is_control) { + return Err(Report::new(BackendNamingError::ControlCharacters { + field: "scheme", + })); + } + if let Some(host_header_override) = spec.host_header_override.as_deref() { + validate_host_header_override_value(host_header_override).map_err(|reason| { + Report::new(BackendNamingError::InvalidHostHeaderOverride { reason }) + })?; + } + + let port = spec + .port + .unwrap_or_else(|| default_port(&spec.scheme, true)); + let name_base = format!( + "{}_{}_{}", + spec.scheme, + canonical_fastly_host(&spec.host), + port + ); + let host_override_suffix = spec + .host_header_override + .as_deref() + .map(|host| format!("_oh_{}", sanitize_fastly_component(host))) + .unwrap_or_default(); + let cert_suffix = if spec.certificate_check { + "" + } else { + "_nocert" + }; + let discriminator_suffix = spec + .discriminator + .as_deref() + .map(|value| format!("_p_{}", sanitize_fastly_component(value))) + .unwrap_or_default(); + let readable_full = format!( + "{}{}{}{}_fb{}_bb{}", + sanitize_fastly_component(&name_base), + host_override_suffix, + cert_suffix, + discriminator_suffix, + spec.first_byte_timeout.as_millis(), + spec.between_bytes_timeout.as_millis() + ); + let readable = readable_full + .chars() + .take(MAX_FASTLY_READABLE_PREFIX_LEN) + .collect::(); + let digest = fastly_spec_digest(&fastly_canonical_spec(spec, port)); + let name = format!("backend_{readable}_{digest}"); + if name.len() > MAX_FASTLY_BACKEND_NAME_LEN { + return Err(Report::new(BackendNamingError::NameTooLong { + limit: MAX_FASTLY_BACKEND_NAME_LEN, + actual: name.len(), + })); + } + + Ok(PredictedBackend { name, port }) +} + +fn normalize_axum_segment(value: &str) -> String { + value.to_uppercase().replace(['-', '.', ' '], "_") +} + +fn predict_axum(spec: &PlatformBackendSpec) -> PredictedBackend { + let port = spec + .port + .unwrap_or_else(|| default_port(&spec.scheme, false)); + let discriminator = spec + .discriminator + .as_deref() + .map(|value| format!("_p_{}", normalize_axum_segment(value))) + .unwrap_or_default(); + PredictedBackend { + name: format!( + "{}_{}_{}{}", + normalize_axum_segment(&spec.scheme), + normalize_axum_segment(&spec.host), + port, + discriminator + ), + port, + } +} + +fn predict_no_registration(spec: &PlatformBackendSpec) -> PredictedBackend { + let port = spec + .port + .unwrap_or_else(|| default_port(&spec.scheme, false)); + let cert_suffix = if spec.certificate_check { + "" + } else { + "_nocert" + }; + let discriminator = spec + .discriminator + .as_deref() + .map(|value| format!("_p_{value}")) + .unwrap_or_default(); + PredictedBackend { + name: format!( + "{}_{}_{}_{}ms{cert_suffix}{discriminator}", + spec.scheme, + spec.host, + port, + spec.first_byte_timeout.as_millis() + ), + port, + } +} + +fn predict_cloudflare(spec: &PlatformBackendSpec) -> PredictedBackend { + predict_no_registration(spec) +} + +fn predict_spin(spec: &PlatformBackendSpec) -> PredictedBackend { + predict_no_registration(spec) +} + +fn canonicalize_fastly_transport_timeout_ms(remaining_ms: u32, configured_ms: u32) -> u32 { + if remaining_ms >= configured_ms { + return configured_ms; + } + if remaining_ms >= FASTLY_TRANSPORT_TIMEOUT_QUANTUM_CEILING_MS { + return FASTLY_TRANSPORT_TIMEOUT_COARSE_LADDER_MS + .into_iter() + .rev() + .find(|&rung| rung <= remaining_ms) + .unwrap_or(FASTLY_TRANSPORT_TIMEOUT_QUANTUM_CEILING_MS); + } + let floored = + (remaining_ms / FASTLY_TRANSPORT_TIMEOUT_QUANTUM_MS) * FASTLY_TRANSPORT_TIMEOUT_QUANTUM_MS; + if floored > 0 { + return floored; + } + FASTLY_SUB_QUANTUM_LADDER_MS + .into_iter() + .find(|&rung| rung <= remaining_ms) + .unwrap_or(0) +} + +#[cfg(test)] +mod tests { + use std::time::Duration; + + use super::*; + + fn spec() -> PlatformBackendSpec { + PlatformBackendSpec { + scheme: "https".to_owned(), + host: "origin.example.com".to_owned(), + port: None, + host_header_override: None, + certificate_check: true, + first_byte_timeout: Duration::from_millis(1500), + between_bytes_timeout: Duration::from_millis(1500), + discriminator: Some("provider-one".to_owned()), + } + } + + #[test] + fn adapter_prediction_outputs_are_pinned() { + let spec = spec(); + assert_eq!( + BackendNamingPolicy::Fastly + .predict(&spec) + .expect("should predict Fastly backend"), + PredictedBackend { + name: "backend_https_origin_example_com_443_p_provider-one_fb1500_bb1500_51fb8dba14db39e759e2c1c5d204a6eb".to_owned(), + port: 443, + } + ); + assert_eq!( + BackendNamingPolicy::Axum + .predict(&spec) + .expect("should predict Axum backend"), + PredictedBackend { + name: "HTTPS_ORIGIN_EXAMPLE_COM_443_p_PROVIDER_ONE".to_owned(), + port: 443, + } + ); + let no_registration = PredictedBackend { + name: "https_origin.example.com_443_1500ms_p_provider-one".to_owned(), + port: 443, + }; + assert_eq!( + BackendNamingPolicy::Cloudflare + .predict(&spec) + .expect("should predict Cloudflare backend"), + no_registration + ); + assert_eq!( + BackendNamingPolicy::Spin + .predict(&spec) + .expect("should predict Spin backend"), + no_registration + ); + } + + #[test] + fn fastly_prediction_canonicalizes_only_bracketed_ipv6_hosts() { + let mut bare = spec(); + bare.host = "2001:db8::1".to_string(); + bare.port = Some(8443); + let mut bracketed = bare.clone(); + bracketed.host = "[2001:db8::1]".to_string(); + + let bare_prediction = BackendNamingPolicy::Fastly + .predict(&bare) + .expect("should predict bare IPv6 backend"); + let bracketed_prediction = BackendNamingPolicy::Fastly + .predict(&bracketed) + .expect("should predict bracketed IPv6 backend"); + assert_eq!(bare_prediction, bracketed_prediction); + assert_eq!( + bare_prediction + .name + .rsplit_once('_') + .map(|(_, digest)| digest), + bracketed_prediction + .name + .rsplit_once('_') + .map(|(_, digest)| digest), + "canonical forms must hash the identical Fastly specification" + ); + + assert_eq!( + canonical_fastly_host("origin.example.com"), + "origin.example.com" + ); + assert_eq!(canonical_fastly_host("192.0.2.1"), "192.0.2.1"); + assert_eq!(canonical_fastly_host("[not-ipv6]"), "[not-ipv6]"); + } + + #[test] + fn target_descriptors_pin_all_capabilities_and_policies() { + let cases = [ + (AuctionTargetId::Fastly, true, BackendNamingPolicy::Fastly), + (AuctionTargetId::Axum, true, BackendNamingPolicy::Axum), + ( + AuctionTargetId::Cloudflare, + false, + BackendNamingPolicy::Cloudflare, + ), + (AuctionTargetId::Spin, false, BackendNamingPolicy::Spin), + ]; + for (id, fanout, naming_policy) in cases { + let descriptor = id.descriptor(); + assert_eq!(descriptor.id(), id); + assert_eq!(descriptor.naming_policy(), naming_policy); + assert_eq!( + descriptor + .capabilities() + .supports_concurrent_provider_fanout(), + fanout, + "should declare fanout accurately for {}", + id.adapter_id() + ); + assert!( + !descriptor + .capabilities() + .has_enforceable_total_request_deadline(), + "no current adapter should claim a total request deadline" + ); + assert_eq!(AuctionTargetId::from_adapter_id(id.adapter_id()), Some(id)); + } + assert_eq!(AuctionTargetId::from_adapter_id("FASTLY"), None); + assert_eq!(AuctionTargetId::from_adapter_id("unknown"), None); + } + + #[test] + fn timeout_policies_preserve_logical_transport_distinction() { + assert_eq!( + BackendNamingPolicy::Fastly.canonicalize_transport_timeout_ms(999, 2000), + 750 + ); + assert_eq!( + BackendNamingPolicy::Fastly.canonicalize_transport_timeout_ms(2000, 100), + 100 + ); + for policy in [ + BackendNamingPolicy::Axum, + BackendNamingPolicy::Cloudflare, + BackendNamingPolicy::Spin, + ] { + assert_eq!(policy.canonicalize_transport_timeout_ms(999, 2000), 999); + assert_eq!(policy.canonicalize_transport_timeout_ms(2000, 100), 100); + } + } +} diff --git a/crates/trusted-server-core/src/platform/http.rs b/crates/trusted-server-core/src/platform/http.rs index c93cd757d..039df014c 100644 --- a/crates/trusted-server-core/src/platform/http.rs +++ b/crates/trusted-server-core/src/platform/http.rs @@ -290,6 +290,16 @@ pub trait PlatformHttpClient: Send + Sync { true } + /// Whether the adapter enforces a hard total deadline for each request. + /// + /// First-byte and between-byte timers do not qualify: connection setup or a + /// byte-trickling response can still overrun the logical auction budget. The + /// auction collector uses this explicit capability to decide whether an + /// already-completed late response remains eligible. + fn has_enforceable_total_request_deadline(&self) -> bool { + false + } + /// Whether [`send`](Self::send) can preserve upstream response bodies as /// [`Body::Stream`](edgezero_core::body::Body::Stream) when requested via /// [`PlatformHttpRequest::with_stream_response`]. diff --git a/crates/trusted-server-core/src/platform/mod.rs b/crates/trusted-server-core/src/platform/mod.rs index 1c5bf4c2a..d09f9b549 100644 --- a/crates/trusted-server-core/src/platform/mod.rs +++ b/crates/trusted-server-core/src/platform/mod.rs @@ -34,6 +34,7 @@ use std::time::Duration; +mod backend_naming; mod error; mod http; mod image_optimizer; @@ -45,6 +46,10 @@ pub(crate) mod test_support; mod traits; mod types; +pub use backend_naming::{ + AuctionTargetCapabilities, AuctionTargetDescriptor, AuctionTargetId, BackendNamingError, + BackendNamingPolicy, PredictedBackend, +}; pub use edgezero_core::key_value_store::{KvError, KvHandle, KvStore as PlatformKvStore}; pub use error::PlatformError; pub use http::{ diff --git a/crates/trusted-server-core/src/platform/test_support.rs b/crates/trusted-server-core/src/platform/test_support.rs index 917f1bf50..856510c1d 100644 --- a/crates/trusted-server-core/src/platform/test_support.rs +++ b/crates/trusted-server-core/src/platform/test_support.rs @@ -1,6 +1,7 @@ use std::collections::{HashMap, VecDeque}; use std::net::IpAddr; use std::sync::{Arc, Mutex}; +use std::time::Duration; use base64::{Engine as _, engine::general_purpose}; use ed25519_dalek::SigningKey; @@ -132,6 +133,10 @@ impl PlatformSecretStore for HashMapSecretStore { pub(crate) struct NoopBackend; impl PlatformBackend for NoopBackend { + fn naming_policy(&self) -> super::BackendNamingPolicy { + super::BackendNamingPolicy::Axum + } + fn predict_name(&self, _spec: &PlatformBackendSpec) -> Result> { Err(Report::new(PlatformError::Unsupported)) } @@ -178,6 +183,10 @@ impl PlatformHttpClient for NoopHttpClient { pub(crate) struct StubBackend; impl PlatformBackend for StubBackend { + fn naming_policy(&self) -> super::BackendNamingPolicy { + super::BackendNamingPolicy::Axum + } + fn predict_name(&self, _spec: &PlatformBackendSpec) -> Result> { Ok("stub-backend".to_owned()) } @@ -213,17 +222,28 @@ struct StubPendingResponse { /// sites. /// Upper bound on the request body bytes captured per `send` call. const MAX_RECORDED_BODY_BYTES: usize = 64 * 1024 * 1024; +type RecordedHeaderBytes = Vec)>>; pub(crate) struct StubHttpClient { calls: Mutex>, responses: Mutex>, // Headers captured per send call, stored as (name, value) string pairs. request_headers: Mutex>>, - // Queued select() errors — each pop makes the next select() return ready: Err. - select_errors: Mutex>, + // Raw header values, including invalid UTF-8 values that cannot be represented + // by `recorded_request_headers`. + request_header_bytes: Mutex, + // Queued select() outcomes; true makes that select return ready: Err. + select_errors: Mutex>, + // Test-only overrides for backend metadata on returned pending handles. + pending_backend_name_overrides: Mutex>>, + // Test-only wall-clock delays applied before each select result is returned. + select_delays: Mutex>, // Reported by supports_concurrent_fanout(); set false to emulate // platforms whose send_async executes eagerly (e.g. Cloudflare Workers). concurrent_fanout: std::sync::atomic::AtomicBool, + // Reported by has_enforceable_total_request_deadline(); set true to emulate + // a future adapter with a hard total request deadline. + enforceable_total_request_deadline: std::sync::atomic::AtomicBool, // Reported by supports_streaming_responses(); set true to emulate Fastly's // streaming response support. streaming_responses_supported: std::sync::atomic::AtomicBool, @@ -248,8 +268,12 @@ impl StubHttpClient { calls: Mutex::new(Vec::new()), responses: Mutex::new(VecDeque::new()), request_headers: Mutex::new(Vec::new()), + request_header_bytes: Mutex::new(Vec::new()), select_errors: Mutex::new(VecDeque::new()), + pending_backend_name_overrides: Mutex::new(VecDeque::new()), + select_delays: Mutex::new(VecDeque::new()), concurrent_fanout: std::sync::atomic::AtomicBool::new(true), + enforceable_total_request_deadline: std::sync::atomic::AtomicBool::new(false), streaming_responses_supported: std::sync::atomic::AtomicBool::new(false), image_optimizer_options: Mutex::new(Vec::new()), cache_bypass_flags: Mutex::new(Vec::new()), @@ -266,6 +290,12 @@ impl StubHttpClient { .store(supported, std::sync::atomic::Ordering::Relaxed); } + /// Make `has_enforceable_total_request_deadline()` report the given value. + pub(crate) fn set_enforceable_total_request_deadline(&self, supported: bool) { + self.enforceable_total_request_deadline + .store(supported, std::sync::atomic::Ordering::Relaxed); + } + /// Make `supports_streaming_responses()` report the given value. pub fn set_streaming_responses_supported(&self, supported: bool) { self.streaming_responses_supported @@ -305,7 +335,36 @@ impl StubHttpClient { self.select_errors .lock() .expect("should lock select_errors") - .push_back(()); + .push_back(true); + } + + /// Make the next `select()` complete successfully before a later queued error. + pub(crate) fn push_select_success(&self) { + self.select_errors + .lock() + .expect("should lock select_errors") + .push_back(false); + } + + /// Override backend metadata on the next pending handle returned by + /// [`Self::send_async`]. `None` removes the metadata entirely. + pub(crate) fn push_pending_backend_name_override(&self, backend_name: Option<&str>) { + self.pending_backend_name_overrides + .lock() + .expect("should lock pending backend name overrides") + .push_back(backend_name.map(str::to_string)); + } + + /// Queue a wall-clock delay before the next [`Self::select`] result. + /// + /// This is test-only timing control for deadline behavior. It deliberately + /// uses a caller-selected, generous contrast with the tested budget rather + /// than relying on scheduler races. + pub(crate) fn push_select_delay(&self, delay: Duration) { + self.select_delays + .lock() + .expect("should lock select_delays") + .push_back(delay); } /// Return backend names recorded across all `send` calls, in order. @@ -323,6 +382,16 @@ impl StubHttpClient { .clone() } + /// Return raw request header values captured per request, in order. + /// + /// Unlike [`Self::recorded_request_headers`], this includes malformed bytes. + pub(crate) fn recorded_request_header_bytes(&self) -> Vec)>> { + self.request_header_bytes + .lock() + .expect("should lock request_header_bytes") + .clone() + } + /// Return Image Optimizer metadata captured per `send` call, in order. pub fn recorded_image_optimizer_options(&self) -> Vec> { self.image_optimizer_options @@ -383,6 +452,11 @@ impl PlatformHttpClient for StubHttpClient { .load(std::sync::atomic::Ordering::Relaxed) } + fn has_enforceable_total_request_deadline(&self) -> bool { + self.enforceable_total_request_deadline + .load(std::sync::atomic::Ordering::Relaxed) + } + fn supports_streaming_responses(&self) -> bool { self.streaming_responses_supported .load(std::sync::atomic::Ordering::Relaxed) @@ -433,6 +507,16 @@ impl PlatformHttpClient for StubHttpClient { .lock() .expect("should lock request_headers") .push(headers); + let header_bytes = request + .request + .headers() + .iter() + .map(|(name, value)| (name.as_str().to_owned(), value.as_bytes().to_vec())) + .collect(); + self.request_header_bytes + .lock() + .expect("should lock request_header_bytes") + .push(header_bytes); // Capture the outgoing request body so tests can assert it is forwarded. // Propagate collection failures instead of recording an empty body, so @@ -489,6 +573,14 @@ impl PlatformHttpClient for StubHttpClient { .lock() .expect("should lock cache bypass flags") .push(request.bypass_cache); + self.request_methods + .lock() + .expect("should lock request methods") + .push(request.request.method().to_string()); + self.request_uris + .lock() + .expect("should lock request URIs") + .push(request.request.uri().to_string()); let headers: Vec<(String, String)> = request .request @@ -505,6 +597,16 @@ impl PlatformHttpClient for StubHttpClient { .lock() .expect("should lock request_headers") .push(headers); + let header_bytes = request + .request + .headers() + .iter() + .map(|(name, value)| (name.as_str().to_owned(), value.as_bytes().to_vec())) + .collect(); + self.request_header_bytes + .lock() + .expect("should lock request_header_bytes") + .push(header_bytes); // Capture the outgoing request body, mirroring `send()`, so tests // exercising the async fan-out path (`request_bids` providers) can @@ -533,7 +635,17 @@ impl PlatformHttpClient for StubHttpClient { status: response.status, body: response.body, }; - Ok(PlatformPendingRequest::new(pending).with_backend_name(backend_name)) + let override_name = self + .pending_backend_name_overrides + .lock() + .expect("should lock pending backend name overrides") + .pop_front(); + let pending = PlatformPendingRequest::new(pending); + Ok(match override_name { + Some(Some(name)) => pending.with_backend_name(name), + Some(None) => pending, + None => pending.with_backend_name(backend_name), + }) } /// Always marks the first pending request in the input as ready (FIFO order). @@ -551,6 +663,15 @@ impl PlatformHttpClient for StubHttpClient { .attach("select called with empty pending_requests list")); } + let delay = self + .select_delays + .lock() + .expect("should lock select_delays") + .pop_front(); + if let Some(delay) = delay { + std::thread::sleep(delay); + } + let ready_platform = pending_requests.remove(0); let stub = ready_platform .downcast::() @@ -577,7 +698,7 @@ impl PlatformHttpClient for StubHttpClient { .lock() .expect("should lock select_errors") .pop_front() - .is_some(); + .unwrap_or(false); if should_error { return Ok(PlatformSelectResult { @@ -719,6 +840,25 @@ pub(crate) fn build_services_with_http_client( build_services_with_secret_and_http_client(NoopSecretStore, http_client) } +/// Build test services that dispatch HTTP requests with an attested client IP. +pub(crate) fn build_services_with_http_client_and_client_ip( + http_client: Arc, + client_ip: IpAddr, +) -> RuntimeServices { + RuntimeServices::builder() + .config_store(Arc::new(NoopConfigStore)) + .secret_store(Arc::new(NoopSecretStore)) + .kv_store(Arc::new(edgezero_core::key_value_store::NoopKvStore)) + .backend(Arc::new(StubBackend)) + .http_client(http_client) + .geo(Arc::new(NoopGeo)) + .client_info(ClientInfo { + client_ip: Some(client_ip), + ..ClientInfo::default() + }) + .build() +} + pub(crate) fn noop_services_with_client_ip(ip: IpAddr) -> RuntimeServices { RuntimeServices::builder() .config_store(Arc::new(NoopConfigStore)) @@ -766,11 +906,20 @@ pub(crate) fn build_services_with_backend_and_http_client( } /// Build a [`RuntimeServices`] with a custom secret store, [`StubBackend`], and HTTP client. -pub(crate) fn build_services_with_secret_and_http_client( +pub(crate) fn build_services_with_config_secret_and_http_client( + config_store: impl PlatformConfigStore + 'static, secret_store: impl PlatformSecretStore + 'static, http_client: Arc, ) -> RuntimeServices { - build_services_with_secret_http_client_and_client_ip(secret_store, http_client, None) + RuntimeServices::builder() + .config_store(Arc::new(config_store)) + .secret_store(Arc::new(secret_store)) + .kv_store(Arc::new(edgezero_core::key_value_store::NoopKvStore)) + .backend(Arc::new(StubBackend)) + .http_client(http_client) + .geo(Arc::new(NoopGeo)) + .client_info(ClientInfo::default()) + .build() } pub(crate) fn build_services_with_secret_http_client_and_client_ip( @@ -794,6 +943,14 @@ pub(crate) fn build_services_with_secret_http_client_and_client_ip( .build() } +/// Build test services with a custom secret store and the standard test config store. +pub(crate) fn build_services_with_secret_and_http_client( + secret_store: impl PlatformSecretStore + 'static, + http_client: Arc, +) -> RuntimeServices { + build_services_with_config_secret_and_http_client(NoopConfigStore, secret_store, http_client) +} + #[cfg(test)] mod tests { use crate::platform::DEFAULT_FIRST_BYTE_TIMEOUT; diff --git a/crates/trusted-server-core/src/platform/traits.rs b/crates/trusted-server-core/src/platform/traits.rs index c6af0a307..6c31cd279 100644 --- a/crates/trusted-server-core/src/platform/traits.rs +++ b/crates/trusted-server-core/src/platform/traits.rs @@ -2,7 +2,7 @@ use std::net::IpAddr; use error_stack::Report; -use super::{GeoInfo, PlatformBackendSpec, PlatformError, StoreId, StoreName}; +use super::{BackendNamingPolicy, GeoInfo, PlatformBackendSpec, PlatformError, StoreId, StoreName}; /// Synchronous, object-safe access to a key-value config store. /// @@ -94,6 +94,9 @@ pub trait PlatformSecretStore: Send + Sync { /// Synchronous, object-safe dynamic backend management. pub trait PlatformBackend: Send + Sync { + /// Return this adapter's pure backend naming and transport timer policy. + fn naming_policy(&self) -> BackendNamingPolicy; + /// Compute the deterministic backend name for the given spec without /// registering anything. /// @@ -125,12 +128,11 @@ pub trait PlatformBackend: Send + Sync { /// connection pooling nor accumulates registrations toward the per-service /// dynamic backend limit. /// - /// The default returns the exact budget-bound value - /// (`remaining_ms.min(configured_ms)`): adapters that neither register nor - /// enforce a backend-name transport timeout gain nothing from rounding and - /// must not shorten bidder deadlines for no benefit. + /// Delegates to the same pure policy used by startup validation so runtime + /// transport timers and predicted names cannot drift. fn canonicalize_transport_timeout_ms(&self, remaining_ms: u32, configured_ms: u32) -> u32 { - remaining_ms.min(configured_ms) + self.naming_policy() + .canonicalize_transport_timeout_ms(remaining_ms, configured_ms) } } diff --git a/crates/trusted-server-core/src/proxy.rs b/crates/trusted-server-core/src/proxy.rs index 59017cf97..122e4c832 100644 --- a/crates/trusted-server-core/src/proxy.rs +++ b/crates/trusted-server-core/src/proxy.rs @@ -7,9 +7,7 @@ use error_stack::{Report, ResultExt}; use futures::StreamExt as _; use http::{HeaderValue, Method, Request, Response, StatusCode, header}; use serde::{Deserialize, Serialize}; -use std::collections::HashMap; use std::io::{Cursor, Write}; -use std::sync::{Arc, LazyLock, Mutex}; use std::time::Duration; use web_time::{SystemTime, UNIX_EPOCH}; @@ -27,13 +25,10 @@ use crate::edge_cookie::get_ec_id; use crate::error::TrustedServerError; use crate::platform::{ DEFAULT_FIRST_BYTE_TIMEOUT, PlatformBackendSpec, PlatformHttpRequest, PlatformResponse, - RuntimeServices, StoreName, + RuntimeServices, }; -use crate::redacted::Redacted; use crate::s3_sigv4::{self, S3Credentials}; -use crate::settings::{ - AssetOriginAuth, OriginQueryPolicy, ProxyAssetRoute, S3SigV4AuthConfig, Settings, -}; +use crate::settings::{AssetOriginAuth, OriginQueryPolicy, ProxyAssetRoute, Settings}; use crate::streaming_processor::{Compression, PipelineConfig, StreamProcessor, StreamingPipeline}; /// Chunk size used for streaming content through the rewrite pipeline. @@ -229,17 +224,6 @@ impl AssetProxyResponse { } } -#[derive(Clone, Debug, Eq, Hash, PartialEq)] -struct S3CredentialsCacheKey { - secret_store: String, - access_key_id: String, - secret_access_key: String, - session_token: Option, -} - -static S3_CREDENTIALS_CACHE: LazyLock>>> = - LazyLock::new(|| Mutex::new(HashMap::new())); - /// Convert a platform-neutral response into a buffered [`Response`] for downstream processing. /// /// # Errors @@ -883,76 +867,7 @@ fn asset_origin_host_header( }) } -fn s3_credentials_cache_key(config: &S3SigV4AuthConfig) -> S3CredentialsCacheKey { - S3CredentialsCacheKey { - secret_store: config.secret_store.clone(), - access_key_id: config.access_key_id.clone(), - secret_access_key: config.secret_access_key.clone(), - session_token: config.session_token.clone(), - } -} - -fn load_s3_credentials( - services: &RuntimeServices, - config: &S3SigV4AuthConfig, -) -> Result, Report> { - let cache_key = s3_credentials_cache_key(config); - if let Some(credentials) = S3_CREDENTIALS_CACHE - .lock() - .expect("should lock S3 credentials cache") - .get(&cache_key) - .cloned() - { - return Ok(credentials); - } - - let store_name = StoreName::from(config.secret_store.as_str()); - let access_key_id = services - .secret_store() - .get_string(&store_name, &config.access_key_id) - .change_context(TrustedServerError::Proxy { - message: "failed to read S3 access key ID from secret store".to_string(), - })?; - let secret_access_key = services - .secret_store() - .get_string(&store_name, &config.secret_access_key) - .change_context(TrustedServerError::Proxy { - message: "failed to read S3 secret access key from secret store".to_string(), - })?; - let session_token = config - .session_token - .as_deref() - .map(|key| { - services - .secret_store() - .get_string(&store_name, key) - .change_context(TrustedServerError::Proxy { - message: "failed to read S3 session token from secret store".to_string(), - }) - }) - .transpose()?; - let credentials = Arc::new(S3Credentials { - access_key_id, - secret_access_key: Redacted::new(secret_access_key), - session_token: session_token.map(Redacted::new), - }); - - let mut cache = S3_CREDENTIALS_CACHE - .lock() - .expect("should lock S3 credentials cache"); - Ok(Arc::clone(cache.entry(cache_key).or_insert(credentials))) -} - -#[cfg(test)] -fn clear_s3_credentials_cache_for_tests() { - S3_CREDENTIALS_CACHE - .lock() - .expect("should lock S3 credentials cache") - .clear(); -} - fn apply_asset_origin_auth( - services: &RuntimeServices, method: &Method, target_url: &url::Url, headers: &mut http::HeaderMap, @@ -960,13 +875,17 @@ fn apply_asset_origin_auth( ) -> Result<(), Report> { match auth { AssetOriginAuth::S3SigV4(config) => { - let credentials = load_s3_credentials(services, config)?; + let credentials = S3Credentials { + access_key_id: config.access_key_id.expose().clone(), + secret_access_key: config.secret_access_key.clone(), + session_token: config.session_token.clone(), + }; s3_sigv4::sign_headers( method, target_url, headers, &config.region, - credentials.as_ref(), + &credentials, // s3_sigv4 converts this via chrono's `DateTime::::from`, which // only accepts `std::time::SystemTime`. `std::time::SystemTime::now()` // panics on `wasm32-unknown-unknown` (Cloudflare Workers), so derive an @@ -1078,7 +997,7 @@ async fn preflight_s3_origin_for_image_optimizer( // HEAD preflight lets missing or unauthorized objects return raw S3 errors // without invoking IO on the failure path. let mut head_headers = unsigned_headers.clone(); - apply_asset_origin_auth(services, &Method::HEAD, target_url, &mut head_headers, auth)?; + apply_asset_origin_auth(&Method::HEAD, target_url, &mut head_headers, auth)?; let head_response = send_asset_origin_request( services, backend_name, @@ -1101,7 +1020,7 @@ async fn preflight_s3_origin_for_image_optimizer( } let mut get_headers = unsigned_headers.clone(); - apply_asset_origin_auth(services, &Method::GET, target_url, &mut get_headers, auth)?; + apply_asset_origin_auth(&Method::GET, target_url, &mut get_headers, auth)?; let mut response = send_asset_origin_request( services, backend_name, @@ -1205,13 +1124,7 @@ pub async fn handle_asset_proxy_request( } if let Some(auth) = &route.auth { - apply_asset_origin_auth( - services, - req.method(), - &target_url, - &mut outbound_headers, - auth, - )?; + apply_asset_origin_auth(req.method(), &target_url, &mut outbound_headers, auth)?; } let mut platform_req = @@ -2221,10 +2134,9 @@ mod tests { use super::{ AssetProxyCachePolicy, IMAGE_FALLBACK_CONTENT_TYPE, ProxyRequestConfig, SUPPORTED_ENCODINGS, asset_origin_host_header, asset_path_skips_image_optimizer, - build_asset_proxy_target_url, clear_s3_credentials_cache_for_tests, - handle_asset_proxy_request, handle_first_party_click, handle_first_party_proxy, - handle_first_party_proxy_rebuild, handle_first_party_proxy_sign, is_host_allowed, - is_host_permitted, proxy_request, rebuild_response_with_body, + build_asset_proxy_target_url, handle_asset_proxy_request, handle_first_party_click, + handle_first_party_proxy, handle_first_party_proxy_rebuild, handle_first_party_proxy_sign, + is_host_allowed, is_host_permitted, proxy_request, rebuild_response_with_body, reconstruct_and_validate_signed_target, stream_asset_body, }; use crate::cache_policy::{CachePolicy, EdgeCacheHeader}; @@ -2239,6 +2151,7 @@ mod tests { PlatformError, PlatformHttpClient, PlatformHttpRequest, PlatformPendingRequest, PlatformResponse, PlatformSecretStore, PlatformSelectResult, StoreId, StoreName, }; + use crate::redacted::Redacted; use crate::settings::{ AssetImageOptimizerConfig, AssetOriginAuth, ImageOptimizerAspectRatioConfig, ImageOptimizerCropOffsetsConfig, ImageOptimizerProfileSet, ImageOptimizerSettings, @@ -4719,9 +4632,11 @@ mod tests { ); route.auth = Some(AssetOriginAuth::S3SigV4(S3SigV4AuthConfig { region: "us-east-1".to_string(), - secret_store: "s3-auth".to_string(), - access_key_id: "access_key_id".to_string(), - secret_access_key: "secret_access_key".to_string(), + secret_store: None, + access_key_id: Redacted::new("AKIAIOSFODNN7EXAMPLE".to_string()), + secret_access_key: Redacted::new( + "wJalrXUtnFEMI/K7MDENG+bPxRfiCYEXAMPLEKEY".to_string(), + ), session_token: None, origin_query: None, })); @@ -4763,9 +4678,11 @@ mod tests { ); route.auth = Some(AssetOriginAuth::S3SigV4(S3SigV4AuthConfig { region: "us-east-1".to_string(), - secret_store: "s3-auth".to_string(), - access_key_id: "access_key_id".to_string(), - secret_access_key: "secret_access_key".to_string(), + secret_store: None, + access_key_id: Redacted::new("AKIAIOSFODNN7EXAMPLE".to_string()), + secret_access_key: Redacted::new( + "wJalrXUtnFEMI/K7MDENG+bPxRfiCYEXAMPLEKEY".to_string(), + ), session_token: None, origin_query: Some(OriginQueryPolicy::Strip), })); @@ -4805,23 +4722,12 @@ mod tests { } #[test] - fn handle_asset_proxy_request_caches_s3_credentials_for_repeated_signing() { + fn handle_asset_proxy_request_uses_resolved_s3_credentials_without_store_reads() { futures::executor::block_on(async { - clear_s3_credentials_cache_for_tests(); let stub = Arc::new(StubHttpClient::new()); stub.push_response(200, Vec::new()); stub.push_response(200, b"optimized".to_vec()); - let secret_store = CountingSecretStore::new(HashMap::from([ - ( - "cache_access_key_id".to_string(), - b"AKIAIOSFODNN7EXAMPLE".to_vec(), - ), - ( - "cache_secret_access_key".to_string(), - b"wJalrXUtnFEMI/K7MDENG+bPxRfiCYEXAMPLEKEY".to_vec(), - ), - ("cache_session_token".to_string(), b"session-token".to_vec()), - ])); + let secret_store = CountingSecretStore::new(HashMap::new()); let observed_secret_store = secret_store.clone(); let services = build_services_with_secret_and_http_client( secret_store, @@ -4838,10 +4744,12 @@ mod tests { let mut route = test_s3_image_optimizer_route(); route.auth = Some(AssetOriginAuth::S3SigV4(S3SigV4AuthConfig { region: "us-east-1".to_string(), - secret_store: "s3-auth-cache".to_string(), - access_key_id: "cache_access_key_id".to_string(), - secret_access_key: "cache_secret_access_key".to_string(), - session_token: Some("cache_session_token".to_string()), + secret_store: None, + access_key_id: Redacted::new("AKIAIOSFODNN7EXAMPLE".to_string()), + secret_access_key: Redacted::new( + "wJalrXUtnFEMI/K7MDENG+bPxRfiCYEXAMPLEKEY".to_string(), + ), + session_token: Some(Redacted::new("session-token".to_string())), origin_query: None, })); @@ -4855,19 +4763,9 @@ mod tests { "should sign both the S3 preflight and final request" ); assert_eq!( - observed_secret_store.read_count("cache_access_key_id"), - 1, - "should read S3 access key ID once despite repeated signing" - ); - assert_eq!( - observed_secret_store.read_count("cache_secret_access_key"), - 1, - "should read S3 secret access key once despite repeated signing" - ); - assert_eq!( - observed_secret_store.read_count("cache_session_token"), - 1, - "should read S3 session token once despite repeated signing" + observed_secret_store.read_count("AKIAIOSFODNN7EXAMPLE"), + 0, + "should not read S3 credentials from the runtime secret store" ); let headers = stub.recorded_request_headers(); assert!( diff --git a/crates/trusted-server-core/src/publisher.rs b/crates/trusted-server-core/src/publisher.rs index 40bb7ea27..9525820ce 100644 --- a/crates/trusted-server-core/src/publisher.rs +++ b/crates/trusted-server-core/src/publisher.rs @@ -57,7 +57,10 @@ use crate::cache_policy::{ use crate::consent::{consent_allows_server_side_auction, gate_eids_by_consent}; use crate::constants::{COOKIE_TS_EIDS, HEADER_X_COMPRESS_HINT}; use crate::cookies::handle_request_cookies; -use crate::creative_opportunities::{AssemblyMode, CreativeOpportunitiesConfig}; +use crate::creative_opportunities::{ + AdStackGateInput, AssemblyMode, CreativeOpportunitiesConfig, RuntimeAdStackExpected, + evaluate_ad_stack_gate, +}; use crate::ec::EcContext; use crate::ec::kv::KvIdentityGraph; use crate::ec::registry::PartnerRegistry; @@ -2965,6 +2968,7 @@ fn make_collect_context<'a>( settings, request: placeholder, timeout_ms: 0, + transport_timeout_ms: 0, provider_responses: None, services, } @@ -3002,15 +3006,12 @@ pub(crate) fn is_prefetch_request(req: &Request) -> bool { header("sec-purpose") || header("purpose") } -#[derive(Debug, Clone, Copy)] -struct ServerSideAdStackConfig { - /// Dedicated `[creative_opportunities].enabled` switch. - ad_templates_enabled: bool, - /// Global `[auction].enabled` gate used by publisher/page-bids flows. - auction_enabled: bool, -} - /// Returns whether request-scoped signals permit an ad-eligible navigation. +/// +/// This is the request half of the shared ad-stack gate: the configuration +/// halves (`matched_slots`, the kill switches) are deliberately absent, because +/// the cache policy for a structurally inactive template must distinguish a +/// page that no request could activate from one this particular request skipped. fn is_server_side_ad_eligible_navigation( is_get: bool, is_navigation: bool, @@ -3021,29 +3022,6 @@ fn is_server_side_ad_eligible_navigation( is_get && is_navigation && !is_prefetch && !is_bot && consent_allows_auction } -/// Returns true only when the publisher should inject and run server-side ad templates. -/// -/// This includes auction dispatch plus initial ad-slot injection. -fn should_run_server_side_ad_stack( - is_get: bool, - is_navigation: bool, - is_prefetch: bool, - is_bot: bool, - has_matched_slots: bool, - consent_allows_auction: bool, - config: ServerSideAdStackConfig, -) -> bool { - is_server_side_ad_eligible_navigation( - is_get, - is_navigation, - is_prefetch, - is_bot, - consent_allows_auction, - ) && config.ad_templates_enabled - && has_matched_slots - && config.auction_enabled -} - /// Write winning bids from an auction result into the shared `ad_bids_state` lock. /// Build the request origin (`scheme://host`, where `host` includes any port) /// used to emit absolute first-party URLs in inline creatives. Returns an empty @@ -4150,18 +4128,18 @@ pub async fn handle_publisher_request( // (storage/access) before firing. Known non-GDPR jurisdictions are free. let consent_allows_auction = consent_allows_server_side_auction(&consent_context); - let should_run_ad_stack = should_run_server_side_ad_stack( - is_get, - is_navigation, - is_prefetch, - is_bot, - !matched_slots.is_empty(), - consent_allows_auction, - ServerSideAdStackConfig { - ad_templates_enabled, - auction_enabled: auction.orchestrator.is_enabled(), - }, - ); + let should_run_ad_stack = evaluate_ad_stack_gate(AdStackGateInput { + method_get: is_get, + navigation: is_navigation, + prefetch: is_prefetch, + bot: is_bot, + matched_slots: !matched_slots.is_empty(), + consent_allows_auction: Some(consent_allows_auction), + auction_enabled: auction.orchestrator.is_enabled(), + ad_templates_enabled, + }) + .expected + == RuntimeAdStackExpected::Yes; let should_run_auction = should_run_ad_stack; // Diagnostic: shows which gate suppresses the server-side auction. Pair with // the `EC context: ... jurisdiction=...` line from EC-context construction @@ -4254,6 +4232,7 @@ pub async fn handle_publisher_request( settings, request: &req, timeout_ms: auction_timeout_ms, + transport_timeout_ms: auction_timeout_ms, provider_responses: None, services, }; @@ -4270,8 +4249,18 @@ pub async fn handle_publisher_request( DispatchAuctionOutcome::DispatchFailed { request, provider_responses, + fatal_admission_error, + metadata, elapsed_ms, } => { + if let Some(error) = fatal_admission_error { + log::warn!( + "Auction admission failed before publisher dispatch; continuing without bids: {error:?}" + ); + } + if !metadata.is_empty() { + log::info!("Auction dispatch failure metadata: {metadata:?}"); + } emit_auction_events_best_effort_lazy(services, || { build_auction_events( observation, @@ -4637,7 +4626,7 @@ pub async fn handle_publisher_request( let gate_content_type = response .headers() .get(header::CONTENT_TYPE) - .and_then(|h| h.to_str().ok()) + .and_then(|value| value.to_str().ok()) .unwrap_or_default() .to_string(); let mut template_cache_key = template_cache_reservation.and_then(|reservation| { @@ -4735,9 +4724,8 @@ pub async fn handle_publisher_request( let origin_content_type = response .headers() .get(header::CONTENT_TYPE) - .and_then(|value| value.to_str().ok()) - .unwrap_or_default() - .to_string(); + .and_then(|h| h.to_str().ok()) + .unwrap_or_default(); // `template_cache_key` is `Some` only for a response the gate authorized, which is // exactly a response that will be assembled. Those must be private regardless of // `should_run_ad_stack`: a bot, prefetch, kill-switched or consent-denied request can @@ -4745,7 +4733,7 @@ pub async fn handle_publisher_request( // caching directives, letting a downstream cache serve it to a later eligible reader. let assembled_response_must_be_private = template_cache_key.is_some(); let is_not_modified = response.status() == StatusCode::NOT_MODIFIED; - if is_html_content_type(&origin_content_type) || is_not_modified { + if is_html_content_type(origin_content_type) || is_not_modified { if should_run_ad_stack || assembled_response_must_be_private { enforce_synthesized_html_cache_privacy(&mut response); } else if is_server_side_ad_eligible_navigation( @@ -4766,19 +4754,6 @@ pub async fn handle_publisher_request( } } } - apply_datadome_client_tag_cache_privacy( - &mut response, - &request_method, - suppress_datadome_client_side_tag, - &origin_content_type, - ); - apply_publisher_asset_cache_policy( - settings, - &request_path, - &request_method, - edge_header, - &mut response, - )?; crate::integrations::gpt_diagnostics::finalize_response(&gpt_diagnostics, &mut response); @@ -4789,6 +4764,13 @@ pub async fn handle_publisher_request( .unwrap_or_default() .to_string(); + apply_datadome_client_tag_cache_privacy( + &mut response, + &request_method, + suppress_datadome_client_side_tag, + &content_type, + ); + let status = response.status(); let content_encoding = response @@ -4803,6 +4785,14 @@ pub async fn handle_publisher_request( set_response_compression(&mut response, reader_compression); } + apply_publisher_asset_cache_policy( + settings, + &request_path, + &request_method, + edge_header, + &mut response, + )?; + match route { ResponseRoute::PassThrough => { log::debug!( @@ -5835,10 +5825,6 @@ fn surrogate_control_freshness( .map_or((directive, None), |(name, value)| (name, Some(value))); let name = name.trim().to_ascii_lowercase(); match name.as_str() { - // Deliberately not `cache_policy::cache_control_headers_are_private_or_no_store`: - // this gate additionally treats `no-cache` as non-shareable, because "revalidate - // before reuse" is correct for an HTTP cache and too permissive for a spike-owned - // one. Consolidating the two would loosen this gate rather than tidy it. "private" | "no-store" | "no-cache" => { return Err(TemplateCacheBypassReason::OriginNotShareable); } @@ -6018,7 +6004,7 @@ fn template_cache_ttl( // Core Cache has no HTTP semantics. Fastly's documented Surrogate-Control subset // is parsed by `origin_shared_ttl`; every other vendor-specific policy remains a // bypass rather than guessing that unrelated CDNs share its grammar or precedence. - if crate::cache_policy::EDGE_CACHE_HEADER_NAMES + if crate::response_privacy::CDN_CACHE_HEADERS .iter() .filter(|name| **name != "surrogate-control") .any(|name| response_headers.contains_key(*name)) @@ -6275,6 +6261,30 @@ fn page_bids_unknown_format() -> Response { /// The SPA hook sends `location.pathname`, but the parameter is /// client-controlled: strip any query string or fragment and force a leading /// `/` so slot `page_patterns` always match against a canonical path shape. +/// How the page-bids endpoint serializes its answer. +#[derive(Debug, Clone, Copy, Default, PartialEq, Eq)] +pub(crate) enum PageBidsFormat { + /// `application/json`. What the SPA navigation hook consumes. + #[default] + Json, +} + +impl PageBidsFormat { + /// Parse the `format` query parameter. + /// + /// # Errors + /// + /// Returns the offending value if it names no known format. Unknown values are + /// rejected rather than defaulting so callers cannot silently negotiate a response + /// representation the endpoint no longer supports. + fn parse(raw: Option<&str>) -> Result { + match raw { + None | Some("json") => Ok(Self::Json), + Some(other) => Err(other.to_string()), + } + } +} + fn normalize_page_bids_path(raw: &str) -> String { let path = raw.split(['?', '#']).next().unwrap_or(""); if path.starts_with('/') { @@ -6384,25 +6394,28 @@ pub async fn handle_page_bids( }) .unwrap_or_else(|| "/".to_string()); - let format = req.uri().query().and_then(|query| { - url::form_urlencoded::parse(query.as_bytes()) - .find(|(key, _)| key == "format") - .map(|(_, value)| value.into_owned()) - }); - if !matches!(format.as_deref(), None | Some("json")) { - log::warn!( - "page-bids: rejecting unknown format `{}`", - format.as_deref().unwrap_or_default() - ); - return Ok(page_bids_unknown_format()); - } + let format = match PageBidsFormat::parse( + req.uri() + .query() + .and_then(|query| { + url::form_urlencoded::parse(query.as_bytes()) + .find(|(k, _)| k == "format") + .map(|(_, v)| v.into_owned()) + }) + .as_deref(), + ) { + Ok(format) => format, + Err(unknown) => { + log::warn!("page-bids: rejecting unknown format `{unknown}`"); + return Ok(page_bids_unknown_format()); + } + }; let matched_slots = if co_config.enabled { match_renderable_slots(auction.slots, co_config, &path_param) } else { Vec::new() }; - let request_info = crate::http_util::RequestInfo::from_request(&req, services.client_info()); let ec_id = ec_context.ec_value().filter(|_| ec_context.ec_allowed()); let consent_context = ec_context.consent(); @@ -6495,6 +6508,7 @@ pub async fn handle_page_bids( settings, request: &req, timeout_ms, + transport_timeout_ms: timeout_ms, provider_responses: None, services, }; @@ -6598,6 +6612,7 @@ pub async fn handle_page_bids( Vec::new() }; + debug_assert_eq!(format, PageBidsFormat::Json); let body = serde_json::json!({ "slots": slots_json, "bids": bid_map, @@ -6675,6 +6690,7 @@ mod tests { creative: Some(creative.to_string()), adomain: None, bidder: "seat".to_string(), + returned_seat: None, width: 300, height: 250, nurl: None, @@ -7587,8 +7603,14 @@ mod tests { .integrations .insert_config("gpt_diagnostics", &serde_json::json!({ "enabled": true })) .expect("should enable diagnostics"); - let integration_registry = - IntegrationRegistry::new(&settings).expect("should create integration registry"); + let integration_registry = IntegrationRegistry::with_plan( + &settings, + Arc::new( + crate::auction::compile_auction_plan(&settings) + .expect("should compile auction plan"), + ), + ) + .expect("should create integration registry"); let mut request = HttpRequest::builder() .method(Method::GET) .uri("https://publisher.example/article?ts_console=1") @@ -7626,8 +7648,14 @@ mod tests { #[test] fn stream_publisher_body_round_trips_gzip() { let settings = create_test_settings(); - let integration_registry = - IntegrationRegistry::new(&settings).expect("should create integration registry"); + let integration_registry = IntegrationRegistry::with_plan( + &settings, + Arc::new( + crate::auction::compile_auction_plan(&settings) + .expect("should compile auction plan"), + ), + ) + .expect("should create integration registry"); let input = b"{\"asset\":\"https://origin.test-publisher.com/path/file.js\"}"; let compressed = gzip_encode(input); let params = make_stream_params(&settings, "gzip"); @@ -7657,8 +7685,14 @@ mod tests { #[test] fn stream_publisher_body_round_trips_brotli() { let settings = create_test_settings(); - let integration_registry = - IntegrationRegistry::new(&settings).expect("should create integration registry"); + let integration_registry = IntegrationRegistry::with_plan( + &settings, + Arc::new( + crate::auction::compile_auction_plan(&settings) + .expect("should compile auction plan"), + ), + ) + .expect("should create integration registry"); let input = b"{\"asset\":\"https://origin.test-publisher.com/path/file.css\"}"; let compressed = brotli_encode(input); let params = make_stream_params(&settings, "br"); @@ -7739,7 +7773,7 @@ mod tests { registry: None, }, req, - EdgeCacheHeader::SurrogateControl, + EdgeCacheHeader::SMaxageFallback, ) .await .expect("should proxy publisher request") @@ -7948,9 +7982,8 @@ mod tests { "prebid".to_string(), serde_json::json!({ "enabled": enabled, - "server_url": "https://prebid.example.com/openrtb2/auction", "external_bundle_url": "https://assets.example.com/prebid/bundle.js", - "timeout": timeout_ms, + "timeout_ms": timeout_ms, }), ); settings @@ -8107,7 +8140,7 @@ mod tests { .headers() .get(header::CACHE_CONTROL) .and_then(|v| v.to_str().ok()), - Some("no-store, private") + Some("private, no-store") ); assert!( response @@ -8734,6 +8767,7 @@ mod tests { creative: None, adomain: None, bidder: STUB_BIDDER.to_string(), + returned_seat: None, width: 728, height: 90, nurl: None, @@ -8767,7 +8801,8 @@ mod tests { /// [`settings_with_mode`], with an auction provider that actually bids. fn settings_with_bidder(mode: &str) -> Settings { let mut settings = settings_with_mode(mode); - settings.auction.providers = vec![STUB_BIDDER.to_string()]; + settings.auction.providers = + crate::auction::AuctionConfig::legacy_provider_map(&[STUB_BIDDER]); settings } @@ -10169,7 +10204,7 @@ mod tests { warm.headers() .get(header::CACHE_CONTROL) .and_then(|value| value.to_str().ok()), - Some("no-store, private"), + Some("private, no-store"), "reusing template cache must not make the assembled response browser-cacheable" ); let warm = String::from_utf8(body_of(warm).await) @@ -10364,7 +10399,7 @@ mod tests { ); assert_eq!( header_of(&warm, header::CACHE_CONTROL), - Some("no-store, private"), + Some("private, no-store"), "an assembled response is per-user even when its template is not" ); @@ -10402,7 +10437,7 @@ mod tests { ); assert_eq!( header_of(&warm, header::CACHE_CONTROL), - Some("no-store, private") + Some("private, no-store") ); } @@ -10586,9 +10621,8 @@ mod tests { "prebid".to_string(), serde_json::json!({ "enabled": true, - "server_url": "https://prebid.example.com/openrtb2/auction", "external_bundle_url": "https://assets.example.com/prebid/bundle.js", - "timeout": timeout_ms, + "timeout_ms": timeout_ms, }), ); settings @@ -10911,7 +10945,7 @@ mod tests { .headers() .get(header::CACHE_CONTROL) .and_then(|value| value.to_str().ok()), - Some("no-store, private") + Some("private, no-store") ); assert!(response.headers().contains_key(header::SET_COOKIE)); let document = String::from_utf8(body_of(response).await) @@ -11162,7 +11196,7 @@ mod tests { .headers() .get(header::CACHE_CONTROL) .and_then(|v| v.to_str().ok()), - Some("no-store, private"), + Some("private, no-store"), "an assembled response must be private whatever the ad stack decided" ); } @@ -11702,7 +11736,7 @@ mod tests { #[test] fn cdn_specific_cache_policy_cannot_be_overridden_by_public_cache_control() { - for name in crate::cache_policy::EDGE_CACHE_HEADER_NAMES { + for name in crate::response_privacy::CDN_CACHE_HEADERS { let mut split = shareable(); split.insert( header::HeaderName::from_static(name), @@ -11727,7 +11761,7 @@ mod tests { #[test] fn unsupported_vendor_freshness_does_not_authorize_template_cache() { - for name in crate::cache_policy::EDGE_CACHE_HEADER_NAMES + for name in crate::response_privacy::CDN_CACHE_HEADERS .iter() .filter(|name| **name != "surrogate-control") { @@ -12572,252 +12606,70 @@ mod tests { } } - #[tokio::test] - async fn publisher_asset_cache_policy_applies_to_non_html_response() { - let settings = Settings::from_toml(&format!( - r#"{} - - [[cache.asset_rules]] - id = "publisher-fingerprinted-assets" - enabled = true - path_globs = ["/assets/**/*.png"] - fingerprint_style = "hex" - visibility = "public" - browser_ttl_seconds = 31536000 - edge_ttl_seconds = 31536000 - immutable = true - "#, - crate_test_settings_str() - )) - .expect("should parse settings with cache rule"); - let stub = Arc::new(StubHttpClient::new()); - stub.push_response_with_headers( - 200, - b"png".to_vec(), - vec![ - (header::CONTENT_TYPE.as_str(), "image/png"), - (header::CACHE_CONTROL.as_str(), "public, max-age=60"), - ], - ); - let services = build_services_with_http_client( - Arc::clone(&stub) as Arc - ); - let request = HttpRequest::builder() - .method(Method::GET) - .uri("https://publisher.example/assets/logo.0123abcd.png") - .header(header::HOST, "publisher.example") - .body(EdgeBody::empty()) - .expect("should build request"); - - let response = run_publisher_proxy(&settings, &services, request).await; - let PublisherResponse::PassThrough { response, .. } = response else { - panic!("should pass through non-HTML asset response"); + mod ssat_cache_policy_tests { + use super::*; + use crate::auction::provider::{AuctionProvider, ProviderRequestOutcome}; + use crate::auction::telemetry::{AuctionEventBatch, AuctionTelemetrySink}; + use crate::creative_opportunities::{CreativeOpportunityFormat, CreativeOpportunitySlot}; + use crate::platform::test_support::{ + NoopConfigStore, NoopGeo, NoopSecretStore, StubBackend, + }; + use crate::platform::{ + ClientInfo, PlatformError, PlatformHttpClient, PlatformPendingRequest, + PlatformResponse, PlatformSelectResult, }; + use crate::test_support::tests::crate_test_settings_str; - assert_eq!( - response - .headers() - .get(header::CACHE_CONTROL) - .and_then(|value| value.to_str().ok()), - Some("public, max-age=31536000, immutable"), - "matched publisher asset should receive immutable browser policy" - ); - assert_eq!( - response - .headers() - .get("surrogate-control") - .and_then(|value| value.to_str().ok()), - Some("max-age=31536000"), - "matched publisher asset should receive Fastly edge policy" - ); - } + const ORIGIN_ETAG: &str = "\"origin-tag\""; + const ORIGIN_LAST_MODIFIED: &str = "Wed, 21 Oct 2015 07:28:00 GMT"; + const UNEXPECTED_304_PROVIDER: &str = "example-navigation-bidder"; + const UNEXPECTED_304_BACKEND: &str = "example-navigation-bidder-backend"; - #[tokio::test] - async fn publisher_asset_policy_response_with_cookie_is_private_after_finalization() { - let settings = Settings::from_toml(&format!( - r#"{} + struct DispatchingTestProvider; - [[cache.asset_rules]] - id = "publisher-fingerprinted-assets" - enabled = true - path_globs = ["/assets/**/*.png"] - fingerprint_style = "hex" - visibility = "public" - browser_ttl_seconds = 31536000 - edge_ttl_seconds = 31536000 - immutable = true - "#, - crate_test_settings_str() - )) - .expect("should parse settings with cache rule"); - let stub = Arc::new(StubHttpClient::new()); - stub.push_response_with_headers( - 200, - b"png".to_vec(), - vec![ - (header::CONTENT_TYPE.as_str(), "image/png"), - (header::CACHE_CONTROL.as_str(), "public, max-age=60"), - (header::SET_COOKIE.as_str(), "viewer=example; Path=/"), - ], - ); - let services = build_services_with_http_client( - Arc::clone(&stub) as Arc - ); - let request = HttpRequest::builder() - .method(Method::GET) - .uri("https://publisher.example/assets/logo.0123abcd.png") - .header(header::HOST, "publisher.example") - .body(EdgeBody::empty()) - .expect("should build request"); + struct RangeAwareHttpClient { + stub: StubHttpClient, + } - let response = run_publisher_proxy(&settings, &services, request).await; - let PublisherResponse::PassThrough { mut response, .. } = response else { - panic!("should pass through non-HTML asset response"); - }; - crate::response_privacy::apply_response_headers_with_cache_privacy( - &settings, - &mut response, - ); + impl RangeAwareHttpClient { + fn new() -> Self { + Self { + stub: StubHttpClient::new(), + } + } + } - assert_eq!( - response - .headers() - .get(header::CACHE_CONTROL) - .and_then(|value| value.to_str().ok()), - Some("private, max-age=0"), - "publisher asset with Set-Cookie must become private after finalization" - ); - assert!( - response.headers().get("surrogate-control").is_none(), - "publisher asset with Set-Cookie must not retain a shared-cache header" - ); - } + #[async_trait::async_trait(?Send)] + impl PlatformHttpClient for RangeAwareHttpClient { + async fn send( + &self, + request: PlatformHttpRequest, + ) -> Result> { + if request.request.headers().contains_key(header::RANGE) { + self.stub.push_response_with_headers( + 206, + b"partial".to_vec(), + vec![ + ("content-type", "text/html; charset=utf-8"), + ("content-range", "bytes 0-18/39"), + ], + ); + } else { + self.stub.push_response_with_headers( + 200, + b"origin".to_vec(), + vec![("content-type", "text/html; charset=utf-8")], + ); + } + self.stub.send(request).await + } - #[tokio::test] - async fn publisher_asset_cache_policy_skips_html_response() { - let settings = Settings::from_toml(&format!( - r#"{} - - [[cache.asset_rules]] - id = "broad-publisher-path" - enabled = true - path_glob = "/news/*.html" - visibility = "public" - browser_ttl_seconds = 31536000 - edge_ttl_seconds = 31536000 - immutable = true - fingerprint_style = "hex" - "#, - crate_test_settings_str() - )) - .expect("should parse settings with cache rule"); - let stub = Arc::new(StubHttpClient::new()); - stub.push_response_with_headers( - 200, - b"news".to_vec(), - vec![ - (header::CONTENT_TYPE.as_str(), "text/html; charset=utf-8"), - (header::CACHE_CONTROL.as_str(), "public, max-age=60"), - ], - ); - let services = build_services_with_http_client( - Arc::clone(&stub) as Arc - ); - let request = HttpRequest::builder() - .method(Method::GET) - .uri("https://publisher.example/news/story.0123abcd.html") - .header(header::HOST, "publisher.example") - .body(EdgeBody::empty()) - .expect("should build request"); - - let response = run_publisher_proxy(&settings, &services, request).await; - let response = match response { - PublisherResponse::Stream { response, .. } | PublisherResponse::Buffered(response) => { - response - } - PublisherResponse::PassThrough { .. } | PublisherResponse::AssembleTemplate { .. } => { - panic!("should classify HTML response for processing") - } - }; - - assert_eq!( - response - .headers() - .get(header::CACHE_CONTROL) - .and_then(|value| value.to_str().ok()), - Some("public, max-age=60"), - "asset policy must not cache publisher HTML" - ); - assert!( - response.headers().get("surrogate-control").is_none(), - "HTML response must not receive a shared-cache header" - ); - } - - mod ssat_cache_policy_tests { - use super::*; - use crate::auction::provider::{AuctionProvider, ProviderRequestOutcome}; - use crate::auction::telemetry::{AuctionEventBatch, AuctionTelemetrySink}; - use crate::creative_opportunities::{CreativeOpportunityFormat, CreativeOpportunitySlot}; - use crate::platform::test_support::{ - NoopConfigStore, NoopGeo, NoopSecretStore, StubBackend, - }; - use crate::platform::{ - ClientInfo, PlatformError, PlatformHttpClient, PlatformPendingRequest, - PlatformResponse, PlatformSelectResult, - }; - use crate::test_support::tests::crate_test_settings_str; - - const ORIGIN_ETAG: &str = "\"origin-tag\""; - const ORIGIN_LAST_MODIFIED: &str = "Wed, 21 Oct 2015 07:28:00 GMT"; - const UNEXPECTED_304_PROVIDER: &str = "example_navigation_bidder"; - const UNEXPECTED_304_BACKEND: &str = "example-navigation-bidder-backend"; - - struct DispatchingTestProvider; - - struct RangeAwareHttpClient { - stub: StubHttpClient, - } - - impl RangeAwareHttpClient { - fn new() -> Self { - Self { - stub: StubHttpClient::new(), - } - } - } - - #[async_trait::async_trait(?Send)] - impl PlatformHttpClient for RangeAwareHttpClient { - async fn send( - &self, - request: PlatformHttpRequest, - ) -> Result> { - if request.request.headers().contains_key(header::RANGE) { - self.stub.push_response_with_headers( - 206, - b"partial".to_vec(), - vec![ - ("content-type", "text/html; charset=utf-8"), - ("content-range", "bytes 0-18/39"), - ], - ); - } else { - self.stub.push_response_with_headers( - 200, - b"origin".to_vec(), - vec![("content-type", "text/html; charset=utf-8")], - ); - } - self.stub.send(request).await - } - - async fn send_async( - &self, - request: PlatformHttpRequest, - ) -> Result> { - self.stub.send_async(request).await - } + async fn send_async( + &self, + request: PlatformHttpRequest, + ) -> Result> { + self.stub.send_async(request).await + } async fn select( &self, @@ -12829,7 +12681,7 @@ mod tests { #[async_trait::async_trait(?Send)] impl AuctionProvider for DispatchingTestProvider { - fn provider_name(&self) -> &'static str { + fn provider_name(&self) -> &str { UNEXPECTED_304_PROVIDER } @@ -12930,10 +12782,9 @@ mod tests { Settings::from_toml(&crate_test_settings_str()) .expect("should parse settings without creative opportunities") } - fn settings_with_dispatching_provider() -> Settings { let toml = format!( - "{}\n[auction]\nenabled = true\nproviders = [\"{UNEXPECTED_304_PROVIDER}\"]\n\n\ + "{}\n[auction]\nenabled = true\n\n[auction.providers.{UNEXPECTED_304_PROVIDER}]\nprotocol = \"openrtb-2.6\"\nendpoint = \"https://unexpected.example/openrtb2/auction\"\nrouting = \"all_eligible\"\n\n\ [creative_opportunities]\ngam_network_id = \"12345\"\n", crate_test_settings_str() ); @@ -12989,6 +12840,10 @@ mod tests { .expect("should build conditional navigation request") } + fn queue_cacheable_html_response(stub: &StubHttpClient) { + queue_html_response_with_cache_control(stub, "public, max-age=300"); + } + fn queue_html_response_with_cache_control( stub: &StubHttpClient, cache_control: &'static str, @@ -13114,12 +12969,51 @@ mod tests { .map(|(_, value)| value.as_str()) } + #[tokio::test] + async fn signer_admission_failure_continues_publisher_origin_without_provider_io() { + let mut settings = settings_with_dispatching_provider(); + settings + .request_signing + .as_mut() + .expect("should configure request signing stores") + .enabled = true; + let plan = Arc::new( + crate::auction::compile_auction_plan(&settings) + .expect("should compile signed navigation auction"), + ); + let orchestrator = crate::auction::build_orchestrator_with_plan(plan, &settings) + .expect("should build signed plan-backed orchestrator"); + let stub = Arc::new(StubHttpClient::new()); + queue_cacheable_html_response(&stub); + let services = services_with_telemetry( + Arc::clone(&stub) as Arc, + Arc::new(RecordingTelemetrySink::default()), + ); + let slots = [article_slot()]; + + let response = run_with_orchestrator( + &settings, + &services, + &orchestrator, + &slots, + conditional_navigation_request(), + ) + .await; + + assert_eq!(response_head(response).status, StatusCode::OK); + assert_eq!( + stub.recorded_backend_names(), + vec!["stub-backend".to_string()], + "signer admission failure should skip provider I/O and still fetch the publisher origin" + ); + } + #[tokio::test] async fn eligible_navigation_bypasses_cache_and_returns_non_storable_html() { // Arrange let settings = settings_with_enabled_auction_and_creative_opportunities(); let stub = Arc::new(StubHttpClient::new()); - queue_html_response_with_cache_control(&stub, "public, max-age=300"); + queue_cacheable_html_response(&stub); let services = build_services_with_http_client( Arc::clone(&stub) as Arc ); @@ -13155,7 +13049,7 @@ mod tests { .headers .get(header::CACHE_CONTROL) .and_then(|value| value.to_str().ok()), - Some("no-store, private"), + Some("private, no-store"), "eligible HTML response should be private and non-storable" ); for header_name in [ @@ -13212,11 +13106,11 @@ mod tests { } #[tokio::test] - async fn navigation_without_matched_slots_uses_short_browser_cache_policy() { + async fn navigation_without_matched_slots_preserves_origin_cache_policy() { // Arrange let settings = settings_with_enabled_auction_and_creative_opportunities(); let stub = Arc::new(StubHttpClient::new()); - queue_html_response_with_cache_control(&stub, "public, max-age=300"); + queue_cacheable_html_response(&stub); let services = build_services_with_http_client( Arc::clone(&stub) as Arc ); @@ -13646,7 +13540,7 @@ mod tests { .headers .get(header::CACHE_CONTROL) .and_then(|value| value.to_str().ok()), - Some("no-store, private"), + Some("private, no-store"), "active GPT diagnostics should retain cache privacy when server-side ad templates are inactive" ); } @@ -13754,7 +13648,7 @@ mod tests { .headers() .get(header::CACHE_CONTROL) .and_then(|value| value.to_str().ok()), - Some("no-store, private"), + Some("private, no-store"), "eligible origin 304 should return an explicitly non-storable response" ); assert!( @@ -13926,6 +13820,188 @@ mod tests { } } + #[tokio::test] + async fn publisher_asset_cache_policy_applies_to_non_html_response() { + let settings = Settings::from_toml(&format!( + r#"{} + + [[cache.asset_rules]] + id = "publisher-fingerprinted-assets" + enabled = true + path_globs = ["/assets/**/*.png"] + fingerprint_style = "hex" + visibility = "public" + browser_ttl_seconds = 31536000 + edge_ttl_seconds = 31536000 + immutable = true + "#, + crate_test_settings_str() + )) + .expect("should parse settings with cache rule"); + let stub = Arc::new(StubHttpClient::new()); + stub.push_response_with_headers( + 200, + b"png".to_vec(), + vec![ + (header::CONTENT_TYPE.as_str(), "image/png"), + (header::CACHE_CONTROL.as_str(), "public, max-age=60"), + ], + ); + let services = build_services_with_http_client( + Arc::clone(&stub) as Arc + ); + let request = HttpRequest::builder() + .method(Method::GET) + .uri("https://publisher.example/assets/logo.0123abcd.png") + .header(header::HOST, "publisher.example") + .body(EdgeBody::empty()) + .expect("should build request"); + + let response = run_publisher_proxy(&settings, &services, request).await; + let PublisherResponse::PassThrough { response, .. } = response else { + panic!("should pass through non-HTML asset response"); + }; + + assert_eq!( + response + .headers() + .get(header::CACHE_CONTROL) + .and_then(|value| value.to_str().ok()), + Some("public, max-age=31536000, s-maxage=31536000, immutable"), + "matched publisher asset should receive immutable browser and edge policy" + ); + assert_eq!( + response + .headers() + .get("surrogate-control") + .and_then(|value| value.to_str().ok()), + None, + "S-maxage fallback should keep the edge TTL in Cache-Control" + ); + } + + #[tokio::test] + async fn publisher_asset_policy_response_with_cookie_is_private_after_finalization() { + let settings = Settings::from_toml(&format!( + r#"{} + + [[cache.asset_rules]] + id = "publisher-fingerprinted-assets" + enabled = true + path_globs = ["/assets/**/*.png"] + fingerprint_style = "hex" + visibility = "public" + browser_ttl_seconds = 31536000 + edge_ttl_seconds = 31536000 + immutable = true + "#, + crate_test_settings_str() + )) + .expect("should parse settings with cache rule"); + let stub = Arc::new(StubHttpClient::new()); + stub.push_response_with_headers( + 200, + b"png".to_vec(), + vec![ + (header::CONTENT_TYPE.as_str(), "image/png"), + (header::CACHE_CONTROL.as_str(), "public, max-age=60"), + (header::SET_COOKIE.as_str(), "viewer=example; Path=/"), + ], + ); + let services = build_services_with_http_client( + Arc::clone(&stub) as Arc + ); + let request = HttpRequest::builder() + .method(Method::GET) + .uri("https://publisher.example/assets/logo.0123abcd.png") + .header(header::HOST, "publisher.example") + .body(EdgeBody::empty()) + .expect("should build request"); + + let response = run_publisher_proxy(&settings, &services, request).await; + let PublisherResponse::PassThrough { mut response, .. } = response else { + panic!("should pass through non-HTML asset response"); + }; + crate::response_privacy::apply_response_headers_with_cache_privacy( + &settings, + &mut response, + ); + + assert_eq!( + response + .headers() + .get(header::CACHE_CONTROL) + .and_then(|value| value.to_str().ok()), + Some("private, max-age=0"), + "publisher asset with Set-Cookie must become private after finalization" + ); + assert!( + response.headers().get("surrogate-control").is_none(), + "publisher asset with Set-Cookie must not retain a shared-cache header" + ); + } + + #[tokio::test] + async fn publisher_asset_cache_policy_skips_html_response() { + let settings = Settings::from_toml(&format!( + r#"{} + + [[cache.asset_rules]] + id = "broad-publisher-path" + enabled = true + path_glob = "/news/*.html" + visibility = "public" + browser_ttl_seconds = 31536000 + edge_ttl_seconds = 31536000 + immutable = true + fingerprint_style = "hex" + "#, + crate_test_settings_str() + )) + .expect("should parse settings with cache rule"); + let stub = Arc::new(StubHttpClient::new()); + stub.push_response_with_headers( + 200, + b"news".to_vec(), + vec![ + (header::CONTENT_TYPE.as_str(), "text/html; charset=utf-8"), + (header::CACHE_CONTROL.as_str(), "public, max-age=60"), + ], + ); + let services = build_services_with_http_client( + Arc::clone(&stub) as Arc + ); + let request = HttpRequest::builder() + .method(Method::GET) + .uri("https://publisher.example/news/story.0123abcd.html") + .header(header::HOST, "publisher.example") + .body(EdgeBody::empty()) + .expect("should build request"); + + let response = run_publisher_proxy(&settings, &services, request).await; + let response = match response { + PublisherResponse::Stream { response, .. } | PublisherResponse::Buffered(response) => { + response + } + PublisherResponse::PassThrough { .. } | PublisherResponse::AssembleTemplate { .. } => { + panic!("should classify HTML response for processing") + } + }; + + assert_eq!( + response + .headers() + .get(header::CACHE_CONTROL) + .and_then(|value| value.to_str().ok()), + Some("public, max-age=60"), + "asset policy must not apply shared asset caching to publisher HTML" + ); + assert!( + response.headers().get("surrogate-control").is_none(), + "HTML response must not receive a shared-cache header" + ); + } + #[tokio::test] async fn publisher_request_uses_platform_http_client_with_http_types() { let settings = create_test_settings(); @@ -14232,6 +14308,7 @@ mod tests { &serde_json::json!({ "enabled": true, "enable_protection": true, + "server_side_key_secret_name": "server-side-key", "protection_excluded_ip_cidrs": ["192.0.2.0/24"], "client_side_key": "test-client-key", }), @@ -14359,7 +14436,7 @@ mod tests { .headers() .get(header::CACHE_CONTROL) .and_then(|value| value.to_str().ok()), - Some("no-store, private"), + Some("private, no-store"), "suppressed HTML should be private and non-storable" ); assert!( @@ -14404,7 +14481,7 @@ mod tests { .headers() .get(header::CACHE_CONTROL) .and_then(|value| value.to_str().ok()), - Some("no-store, private"), + Some("private, no-store"), "suppressed HTML should use the exact synthesized-HTML policy" ); } @@ -14574,73 +14651,6 @@ mod tests { ); } - #[test] - fn server_side_ad_stack_runs_only_when_all_auction_gates_pass() { - let enabled_config = ServerSideAdStackConfig { - ad_templates_enabled: true, - auction_enabled: true, - }; - assert!( - should_run_server_side_ad_stack(true, true, false, false, true, true, enabled_config,), - "GET, real navigation, enabled templates, matched slots, and consent should run TS ad stack" - ); - - assert!( - !should_run_server_side_ad_stack(false, true, false, false, true, true, enabled_config,), - "non-GET requests should skip TS ad stack" - ); - assert!( - !should_run_server_side_ad_stack(true, false, false, false, true, true, enabled_config,), - "non-document requests should skip TS ad stack" - ); - assert!( - !should_run_server_side_ad_stack(true, true, true, false, true, true, enabled_config,), - "prefetch requests should skip TS ad stack and injection" - ); - assert!( - !should_run_server_side_ad_stack(true, true, false, true, true, true, enabled_config,), - "bot requests should skip TS ad stack and injection" - ); - assert!( - !should_run_server_side_ad_stack(true, true, false, false, false, true, enabled_config,), - "requests with no matching slots should skip TS ad stack" - ); - assert!( - !should_run_server_side_ad_stack(true, true, false, false, true, false, enabled_config,), - "requests without required consent should skip TS ad stack and injection" - ); - assert!( - !should_run_server_side_ad_stack( - true, - true, - false, - false, - true, - true, - ServerSideAdStackConfig { - ad_templates_enabled: true, - auction_enabled: false, - }, - ), - "disabled [auction].enabled kill switch should skip TS ad stack and injection" - ); - assert!( - !should_run_server_side_ad_stack( - true, - true, - false, - false, - true, - true, - ServerSideAdStackConfig { - ad_templates_enabled: false, - auction_enabled: true, - }, - ), - "disabled [creative_opportunities].enabled switch should skip TS ad stack and injection" - ); - } - #[tokio::test] async fn body_close_hold_loop_processes_close_tail_before_reading_post_body_chunks() { let settings = create_test_settings(); @@ -15261,8 +15271,14 @@ mod tests { #[test] fn tsjs_dynamic_returns_not_found_for_unknown_filename() { let settings = create_test_settings(); - let registry = - IntegrationRegistry::new(&settings).expect("should create integration registry"); + let registry = IntegrationRegistry::with_plan( + &settings, + Arc::new( + crate::auction::compile_auction_plan(&settings) + .expect("should compile auction plan"), + ), + ) + .expect("should create integration registry"); let req = build_request( Method::GET, "https://publisher.example/static/tsjs=unknown.js", @@ -15276,8 +15292,14 @@ mod tests { #[test] fn tsjs_dynamic_serves_unified_bundle_for_known_filename() { let settings = create_test_settings(); - let registry = - IntegrationRegistry::new(&settings).expect("should create integration registry"); + let registry = IntegrationRegistry::with_plan( + &settings, + Arc::new( + crate::auction::compile_auction_plan(&settings) + .expect("should compile auction plan"), + ), + ) + .expect("should create integration registry"); let req = build_request( Method::GET, "https://publisher.example/static/tsjs=tsjs-unified.min.js", @@ -15295,8 +15317,14 @@ mod tests { .integrations .insert_config("gpt_diagnostics", &serde_json::json!({ "enabled": true })) .expect("should enable diagnostics"); - let registry = - IntegrationRegistry::new(&settings).expect("should create integration registry"); + let registry = IntegrationRegistry::with_plan( + &settings, + Arc::new( + crate::auction::compile_auction_plan(&settings) + .expect("should compile auction plan"), + ), + ) + .expect("should create integration registry"); let mut req = build_request( Method::GET, "https://publisher.example/static/tsjs=tsjs-gpt_diagnostics.min.js", @@ -15362,8 +15390,14 @@ mod tests { #[test] fn tsjs_dynamic_serves_prebid_shim_when_enabled() { let settings = create_test_settings(); - let registry = - IntegrationRegistry::new(&settings).expect("should create integration registry"); + let registry = IntegrationRegistry::with_plan( + &settings, + Arc::new( + crate::auction::compile_auction_plan(&settings) + .expect("should compile auction plan"), + ), + ) + .expect("should create integration registry"); let req = build_request( Method::GET, "https://publisher.example/static/tsjs=tsjs-prebid.min.js", @@ -15387,13 +15421,18 @@ mod tests { "prebid", &serde_json::json!({ "enabled": false, - "server_url": "https://test-prebid.com/openrtb2/auction", "external_bundle_url": "https://assets.example/prebid/trusted-prebid.js", }), ) .expect("should update prebid config"); - let registry = - IntegrationRegistry::new(&settings).expect("should create integration registry"); + let registry = IntegrationRegistry::with_plan( + &settings, + Arc::new( + crate::auction::compile_auction_plan(&settings) + .expect("should compile auction plan"), + ), + ) + .expect("should create integration registry"); let req = build_request( Method::GET, "https://publisher.example/static/tsjs=tsjs-prebid.min.js", @@ -15411,8 +15450,14 @@ mod tests { #[test] fn tsjs_dynamic_returns_not_found_for_arbitrary_module_name() { let settings = create_test_settings(); - let registry = - IntegrationRegistry::new(&settings).expect("should create integration registry"); + let registry = IntegrationRegistry::with_plan( + &settings, + Arc::new( + crate::auction::compile_auction_plan(&settings) + .expect("should compile auction plan"), + ), + ) + .expect("should create integration registry"); let req = build_request( Method::GET, "https://publisher.example/static/tsjs=tsjs-evil.min.js", @@ -15567,8 +15612,14 @@ mod tests { use std::io::Write; let settings = create_test_settings(); - let registry = - IntegrationRegistry::new(&settings).expect("should create integration registry"); + let registry = IntegrationRegistry::with_plan( + &settings, + Arc::new( + crate::auction::compile_auction_plan(&settings) + .expect("should compile auction plan"), + ), + ) + .expect("should create integration registry"); // Compress CSS containing an origin URL that should be rewritten. // CSS uses the text URL replacer (not lol_html), so inline URLs are rewritten. @@ -15631,8 +15682,14 @@ mod tests { #[test] fn stream_publisher_body_handles_empty_body() { let settings = create_test_settings(); - let registry = - IntegrationRegistry::new(&settings).expect("should create integration registry"); + let registry = IntegrationRegistry::with_plan( + &settings, + Arc::new( + crate::auction::compile_auction_plan(&settings) + .expect("should compile auction plan"), + ), + ) + .expect("should create integration registry"); let params = OwnedProcessResponseParams { csp_nonce_observed: None, @@ -15674,8 +15731,14 @@ mod tests { #[test] fn stream_publisher_body_rejects_stream_body_in_sync_path() { let settings = create_test_settings(); - let registry = - IntegrationRegistry::new(&settings).expect("should create integration registry"); + let registry = IntegrationRegistry::with_plan( + &settings, + Arc::new( + crate::auction::compile_auction_plan(&settings) + .expect("should compile auction plan"), + ), + ) + .expect("should create integration registry"); let params = OwnedProcessResponseParams { csp_nonce_observed: None, template_cache_key: None, @@ -15792,8 +15855,14 @@ mod tests { fn stream_publisher_body_async_processes_stream_without_auction() { futures::executor::block_on(async { let settings = create_test_settings(); - let registry = - IntegrationRegistry::new(&settings).expect("should create integration registry"); + let registry = IntegrationRegistry::with_plan( + &settings, + Arc::new( + crate::auction::compile_auction_plan(&settings) + .expect("should compile auction plan"), + ), + ) + .expect("should create integration registry"); let orchestrator = AuctionOrchestrator::new(settings.auction.clone()); let services = noop_services(); let mut params = OwnedProcessResponseParams { @@ -15850,8 +15919,14 @@ mod tests { fn stream_publisher_body_async_processes_gzip_stream_without_auction() { futures::executor::block_on(async { let settings = create_test_settings(); - let registry = - IntegrationRegistry::new(&settings).expect("should create integration registry"); + let registry = IntegrationRegistry::with_plan( + &settings, + Arc::new( + crate::auction::compile_auction_plan(&settings) + .expect("should compile auction plan"), + ), + ) + .expect("should create integration registry"); let orchestrator = AuctionOrchestrator::new(settings.auction.clone()); let services = noop_services(); let mut params = OwnedProcessResponseParams { @@ -15911,8 +15986,14 @@ mod tests { fn stream_publisher_body_async_processes_deflate_stream_without_auction() { futures::executor::block_on(async { let settings = create_test_settings(); - let registry = - IntegrationRegistry::new(&settings).expect("should create integration registry"); + let registry = IntegrationRegistry::with_plan( + &settings, + Arc::new( + crate::auction::compile_auction_plan(&settings) + .expect("should compile auction plan"), + ), + ) + .expect("should create integration registry"); let orchestrator = AuctionOrchestrator::new(settings.auction.clone()); let services = noop_services(); let mut params = OwnedProcessResponseParams { @@ -15972,8 +16053,14 @@ mod tests { fn stream_publisher_body_async_processes_brotli_stream_without_auction() { futures::executor::block_on(async { let settings = create_test_settings(); - let registry = - IntegrationRegistry::new(&settings).expect("should create integration registry"); + let registry = IntegrationRegistry::with_plan( + &settings, + Arc::new( + crate::auction::compile_auction_plan(&settings) + .expect("should compile auction plan"), + ), + ) + .expect("should create integration registry"); let orchestrator = AuctionOrchestrator::new(settings.auction.clone()); let services = noop_services(); let mut params = OwnedProcessResponseParams { @@ -16033,8 +16120,14 @@ mod tests { fn stream_publisher_body_async_rejects_truncated_brotli_stream() { futures::executor::block_on(async { let settings = create_test_settings(); - let registry = - IntegrationRegistry::new(&settings).expect("should create integration registry"); + let registry = IntegrationRegistry::with_plan( + &settings, + Arc::new( + crate::auction::compile_auction_plan(&settings) + .expect("should compile auction plan"), + ), + ) + .expect("should create integration registry"); let orchestrator = AuctionOrchestrator::new(settings.auction.clone()); let services = noop_services(); let mut params = OwnedProcessResponseParams { @@ -16112,8 +16205,14 @@ mod tests { fn stream_publisher_body_async_rejects_truncated_gzip_stream() { futures::executor::block_on(async { let settings = create_test_settings(); - let registry = - IntegrationRegistry::new(&settings).expect("should create integration registry"); + let registry = IntegrationRegistry::with_plan( + &settings, + Arc::new( + crate::auction::compile_auction_plan(&settings) + .expect("should compile auction plan"), + ), + ) + .expect("should create integration registry"); let orchestrator = AuctionOrchestrator::new(settings.auction.clone()); let services = noop_services(); let mut params = non_html_stream_params("gzip"); @@ -16149,8 +16248,14 @@ mod tests { fn stream_publisher_body_async_rejects_truncated_deflate_stream() { futures::executor::block_on(async { let settings = create_test_settings(); - let registry = - IntegrationRegistry::new(&settings).expect("should create integration registry"); + let registry = IntegrationRegistry::with_plan( + &settings, + Arc::new( + crate::auction::compile_auction_plan(&settings) + .expect("should compile auction plan"), + ), + ) + .expect("should create integration registry"); let orchestrator = AuctionOrchestrator::new(settings.auction.clone()); let services = noop_services(); let mut params = non_html_stream_params("deflate"); @@ -16191,8 +16296,14 @@ mod tests { // decoded expansion exceeds it — the decompression-bomb case the // raw-byte cap alone cannot catch. settings.publisher.max_buffered_body_bytes = 1024; - let registry = - IntegrationRegistry::new(&settings).expect("should create integration registry"); + let registry = IntegrationRegistry::with_plan( + &settings, + Arc::new( + crate::auction::compile_auction_plan(&settings) + .expect("should compile auction plan"), + ), + ) + .expect("should create integration registry"); let orchestrator = AuctionOrchestrator::new(settings.auction.clone()); let services = noop_services(); let mut params = non_html_stream_params("gzip"); @@ -16274,8 +16385,14 @@ mod tests { fn stream_publisher_body_async_processes_stream_with_auction_hold() { futures::executor::block_on(async { let settings = create_test_settings(); - let registry = - IntegrationRegistry::new(&settings).expect("should create integration registry"); + let registry = IntegrationRegistry::with_plan( + &settings, + Arc::new( + crate::auction::compile_auction_plan(&settings) + .expect("should compile auction plan"), + ), + ) + .expect("should create integration registry"); let orchestrator = AuctionOrchestrator::new(settings.auction.clone()); let services = noop_services(); let state = AdBidsState::default(); @@ -16343,8 +16460,14 @@ mod tests { fn stream_publisher_body_async_auction_hold_decodes_multi_member_gzip_buffered() { futures::executor::block_on(async { let settings = create_test_settings(); - let registry = - IntegrationRegistry::new(&settings).expect("should create integration registry"); + let registry = IntegrationRegistry::with_plan( + &settings, + Arc::new( + crate::auction::compile_auction_plan(&settings) + .expect("should compile auction plan"), + ), + ) + .expect("should create integration registry"); let orchestrator = AuctionOrchestrator::new(settings.auction.clone()); let services = noop_services(); let state = AdBidsState::default(); @@ -16415,8 +16538,14 @@ mod tests { fn stream_publisher_body_async_processes_non_html_stream_after_auction_collect() { futures::executor::block_on(async { let settings = create_test_settings(); - let registry = - IntegrationRegistry::new(&settings).expect("should create integration registry"); + let registry = IntegrationRegistry::with_plan( + &settings, + Arc::new( + crate::auction::compile_auction_plan(&settings) + .expect("should compile auction plan"), + ), + ) + .expect("should create integration registry"); let orchestrator = AuctionOrchestrator::new(settings.auction.clone()); let services = noop_services(); let mut params = OwnedProcessResponseParams { @@ -16474,7 +16603,14 @@ mod tests { fn drain_streaming_finalize_body(content_encoding: &str, body: EdgeBody) -> Vec { let settings = Arc::new(create_test_settings()); let registry = Arc::new( - IntegrationRegistry::new(&settings).expect("should create integration registry"), + IntegrationRegistry::with_plan( + &settings, + Arc::new( + crate::auction::compile_auction_plan(&settings) + .expect("should compile auction plan"), + ), + ) + .expect("should create integration registry"), ); let orchestrator = Arc::new(AuctionOrchestrator::new(settings.auction.clone())); let services = noop_services(); @@ -16599,7 +16735,14 @@ mod tests { ) -> EdgeBody { let settings = Arc::new(settings); let registry = Arc::new( - IntegrationRegistry::new(&settings).expect("should create integration registry"), + IntegrationRegistry::with_plan( + &settings, + Arc::new( + crate::auction::compile_auction_plan(&settings) + .expect("should compile auction plan"), + ), + ) + .expect("should create integration registry"), ); let orchestrator = Arc::new(AuctionOrchestrator::new(settings.auction.clone())); let services = noop_services(); @@ -16867,7 +17010,14 @@ mod tests { // responses must carry no body and correct framing per status. let settings = Arc::new(create_test_settings()); let registry = Arc::new( - IntegrationRegistry::new(&settings).expect("should create integration registry"), + IntegrationRegistry::with_plan( + &settings, + Arc::new( + crate::auction::compile_auction_plan(&settings) + .expect("should compile auction plan"), + ), + ) + .expect("should create integration registry"), ); let orchestrator = Arc::new(AuctionOrchestrator::new(settings.auction.clone())); @@ -16926,8 +17076,14 @@ mod tests { // The buffered finalizer (Axum/Cloudflare/Spin) must correct bodiless // framing identically to the streaming finalizer. let settings = create_test_settings(); - let registry = - IntegrationRegistry::new(&settings).expect("should create integration registry"); + let registry = IntegrationRegistry::with_plan( + &settings, + Arc::new( + crate::auction::compile_auction_plan(&settings) + .expect("should compile auction plan"), + ), + ) + .expect("should create integration registry"); let orchestrator = AuctionOrchestrator::new(settings.auction.clone()); let services = noop_services(); @@ -17006,8 +17162,14 @@ mod tests { // terminal abandonment event so the SSP work and quota consumption stay // observable instead of vanishing silently. let settings = Arc::new(create_test_settings()); - let registry = - IntegrationRegistry::new(&settings).expect("should create integration registry"); + let registry = IntegrationRegistry::with_plan( + &settings, + Arc::new( + crate::auction::compile_auction_plan(&settings) + .expect("should compile auction plan"), + ), + ) + .expect("should create integration registry"); let orchestrator = Arc::new(AuctionOrchestrator::new(settings.auction.clone())); let make_params = || { @@ -17179,7 +17341,14 @@ mod tests { fn publisher_response_streaming_finalize_holds_auction_and_keeps_gzip_tail() { let settings = Arc::new(create_test_settings()); let registry = Arc::new( - IntegrationRegistry::new(&settings).expect("should create integration registry"), + IntegrationRegistry::with_plan( + &settings, + Arc::new( + crate::auction::compile_auction_plan(&settings) + .expect("should compile auction plan"), + ), + ) + .expect("should create integration registry"), ); let orchestrator = Arc::new(AuctionOrchestrator::new(settings.auction.clone())); let services = noop_services(); @@ -17270,8 +17439,14 @@ mod tests { #[test] fn stream_publisher_body_treats_mixed_case_html_as_html() { let settings = create_test_settings(); - let registry = - IntegrationRegistry::new(&settings).expect("should create integration registry"); + let registry = IntegrationRegistry::with_plan( + &settings, + Arc::new( + crate::auction::compile_auction_plan(&settings) + .expect("should compile auction plan"), + ), + ) + .expect("should create integration registry"); let bids_script = r#""#; let state = AdBidsState::with_script(bids_script); @@ -17328,8 +17503,14 @@ mod tests { #[test] fn stream_publisher_body_surfaces_mid_stream_decode_error() { let settings = create_test_settings(); - let registry = - IntegrationRegistry::new(&settings).expect("should create integration registry"); + let registry = IntegrationRegistry::with_plan( + &settings, + Arc::new( + crate::auction::compile_auction_plan(&settings) + .expect("should compile auction plan"), + ), + ) + .expect("should create integration registry"); // Claim gzip encoding but feed non-gzip bytes. The GzDecoder will // error as soon as it tries to read the gzip header. @@ -17423,8 +17604,14 @@ mod tests { ) .expect("should update nextjs config"); - let registry = - IntegrationRegistry::new(&settings).expect("should create integration registry"); + let registry = IntegrationRegistry::with_plan( + &settings, + Arc::new( + crate::auction::compile_auction_plan(&settings) + .expect("should compile auction plan"), + ), + ) + .expect("should create integration registry"); assert!( registry.has_html_post_processors(), @@ -17503,8 +17690,14 @@ mod tests { }), ) .expect("should update nextjs config"); - let registry = - IntegrationRegistry::new(&settings).expect("should create integration registry"); + let registry = IntegrationRegistry::with_plan( + &settings, + Arc::new( + crate::auction::compile_auction_plan(&settings) + .expect("should compile auction plan"), + ), + ) + .expect("should create integration registry"); // Small, single-fragment RSC script — placeholder path (not fallback). let html = br#""#; @@ -17630,6 +17823,7 @@ mod tests { creative: None, adomain: None, bidder: bidder.to_string(), + returned_seat: None, width: 300, height: 250, nurl: Some(nurl.to_string()), @@ -18224,6 +18418,7 @@ mod tests { creative: Some(creative.to_string()), adomain: None, bidder: "prebid".to_string(), + returned_seat: None, width: 300, height: 250, nurl: None, @@ -18627,6 +18822,7 @@ mod tests { creative: None, adomain: None, bidder: "thetradedesk".to_string(), + returned_seat: None, width: 300, height: 250, nurl: None, @@ -18682,6 +18878,7 @@ mod tests { creative: None, adomain: None, bidder: "amazon-aps".to_string(), + returned_seat: None, width: 300, height: 250, nurl: None, @@ -18794,6 +18991,7 @@ mod tests { creative: None, adomain: None, bidder: "amazon-aps".to_string(), + returned_seat: None, width: 300, height: 250, nurl: None, @@ -18838,6 +19036,7 @@ mod tests { creative: None, adomain: None, bidder: "kargo".to_string(), + returned_seat: None, width: 300, height: 250, nurl: None, @@ -19170,7 +19369,7 @@ mod tests { #[async_trait::async_trait(?Send)] impl AuctionProvider for AuctionIdTestProvider { - fn provider_name(&self) -> &'static str { + fn provider_name(&self) -> &str { AUCTION_ID_TEST_PROVIDER } @@ -19215,6 +19414,7 @@ mod tests { creative: None, adomain: None, bidder: AUCTION_ID_TEST_PROVIDER.to_string(), + returned_seat: None, width: 300, height: 250, nurl: None, @@ -19470,7 +19670,8 @@ mod tests { #[tokio::test] async fn page_bids_response_includes_auction_id_only_for_winning_bids() { let mut settings = settings_with_co(); - settings.auction.providers = vec![AUCTION_ID_TEST_PROVIDER.to_string()]; + settings.auction.providers = + crate::auction::AuctionConfig::legacy_provider_map(&[AUCTION_ID_TEST_PROVIDER]); settings .integrations .insert_config("gpt_diagnostics", &serde_json::json!({ "enabled": true })) @@ -19626,7 +19827,8 @@ mod tests { } let mut settings = settings_with_co(); - settings.auction.providers = vec![AUCTION_ID_TEST_PROVIDER.to_string()]; + settings.auction.providers = + crate::auction::AuctionConfig::legacy_provider_map(&[AUCTION_ID_TEST_PROVIDER]); settings .integrations .insert_config("gpt_diagnostics", &serde_json::json!({ "enabled": true })) @@ -20118,7 +20320,7 @@ mod tests { // When consent denies the server-side auction (here: Jurisdiction // Unknown fails closed), the endpoint must return no slots so the SPA // hook does not create GPT slots client-side — matching the publisher - // navigation path's `should_run_server_side_ad_stack` gate. + // navigation path's shared `evaluate_ad_stack_gate` gate. let settings = settings_with_co(); let orchestrator = AuctionOrchestrator::new(settings.auction.clone()); let slots = article_slot(); @@ -20211,7 +20413,7 @@ mod tests { /// `[publisher] domain` from [`crate_test_settings_str`]. const CONFIGURED_DOMAIN: &str = "test-publisher.com"; - const CAPTURING_PROVIDER: &str = "request_capturing_provider"; + const CAPTURING_PROVIDER: &str = "request-capturing-provider"; /// Records the [`AuctionRequest`] the orchestrator dispatched, then /// fails its launch so no real transport handle is needed. @@ -20221,7 +20423,7 @@ mod tests { #[async_trait::async_trait(?Send)] impl AuctionProvider for RequestCapturingProvider { - fn provider_name(&self) -> &'static str { + fn provider_name(&self) -> &str { CAPTURING_PROVIDER } @@ -20280,7 +20482,7 @@ mod tests { fn settings_with_capturing_provider() -> Settings { let toml = format!( - "{}\n[auction]\nenabled = true\nproviders = [\"{CAPTURING_PROVIDER}\"]\n\n\ + "{}\n[auction]\nenabled = true\n\n[auction.providers.{CAPTURING_PROVIDER}]\nprotocol = \"openrtb-2.6\"\nendpoint = \"https://capture.example/openrtb2/auction\"\nrouting = \"all_eligible\"\n\n\ [creative_opportunities]\ngam_network_id = \"12345\"\n", crate_test_settings_str() ); diff --git a/crates/trusted-server-core/src/response_privacy.rs b/crates/trusted-server-core/src/response_privacy.rs index 8674429ea..c1a70a01a 100644 --- a/crates/trusted-server-core/src/response_privacy.rs +++ b/crates/trusted-server-core/src/response_privacy.rs @@ -12,30 +12,82 @@ use edgezero_core::http::{HeaderMap, HeaderName, HeaderValue, Response, header}; use crate::cache_policy::{ - CacheControlPolicy, EdgeCacheHeader, cache_control_headers_are_private_or_no_store, - is_edge_cache_header_name, remove_edge_cache_headers, + cache_control_headers_are_private_or_no_store, is_edge_cache_header_name, + remove_edge_cache_headers, }; use crate::settings::Settings; -/// Marks a response whose `no-store, private` policy belongs to Trusted Server. +/// Marks a response whose `private, no-store` policy belongs to Trusted Server. /// /// Platform terminal hooks use this out-of-band marker to re-enforce the policy after /// late integrations run without rewriting unrelated origin-private responses. #[derive(Clone, Copy, Debug)] pub struct TerminalPrivateResponse; +/// CDN-targeted cache headers stripped from every cookie-bearing response. +/// +/// A single source of truth so the adapter copies of the privacy downgrade +/// cannot drift apart. +pub const CDN_CACHE_HEADERS: &[&str] = &[ + "surrogate-control", + "fastly-surrogate-control", + "cdn-cache-control", + "cloudflare-cdn-cache-control", +]; + const INACTIVE_AD_STACK_BROWSER_CACHE_CONTROL: &str = "private, max-age=60"; #[derive(Debug, Clone, Copy)] struct GeneratedInactiveAdStackBrowserCachePolicy; -fn cache_control_is_private_or_no_store(response: &Response) -> bool { - cache_control_headers_are_private_or_no_store(response.headers()) +fn cache_control_segment_has_directive(segment: &[u8], target: &[u8]) -> bool { + let name_end = segment + .iter() + .position(|byte| *byte == b'=') + .unwrap_or(segment.len()); + segment[..name_end] + .trim_ascii() + .eq_ignore_ascii_case(target) +} + +fn cache_control_value_has_directive(value: &[u8], target: &[u8]) -> bool { + let mut segment_start = 0; + let mut in_quotes = false; + let mut escaped = false; + + for (index, byte) in value.iter().copied().enumerate() { + if in_quotes { + if escaped { + escaped = false; + } else if byte == b'\\' { + escaped = true; + } else if byte == b'"' { + in_quotes = false; + } + } else if byte == b'"' { + in_quotes = true; + } else if byte == b',' { + if cache_control_segment_has_directive(&value[segment_start..index], target) { + return true; + } + segment_start = index + 1; + } + } + + cache_control_segment_has_directive(&value[segment_start..], target) +} + +fn cache_control_has_directive(headers: &HeaderMap, target: &str) -> bool { + headers + .get_all(header::CACHE_CONTROL) + .iter() + .any(|value| cache_control_value_has_directive(value.as_bytes(), target.as_bytes())) } /// Returns whether `Cache-Control` prohibits storage by shared caches. pub(crate) fn cache_control_forbids_shared_storage(headers: &HeaderMap) -> bool { - cache_control_headers_are_private_or_no_store(headers) + cache_control_has_directive(headers, "private") + || cache_control_has_directive(headers, "no-store") } fn has_generated_inactive_ad_stack_browser_cache_policy(response: &Response) -> bool { @@ -64,14 +116,41 @@ pub fn apply_inactive_ad_stack_browser_cache_policy(response: &mut Response) { .insert(GeneratedInactiveAdStackBrowserCachePolicy); } +fn strip_cdn_cache_headers(response: &mut Response) { + for name in CDN_CACHE_HEADERS { + response.headers_mut().remove(*name); + } +} + +/// Whether `Cache-Control` already forbids shared caching. +/// +/// Extracted because both arms of the cookie-privacy net below need it. +/// +/// `publisher::template_cache_bypass_reason` deliberately does **not** call this and keeps its own +/// copy: it additionally treats `no-cache` as non-shareable, because "revalidate before +/// reuse" is correct for an HTTP cache and too permissive for a spike-owned one. The +/// duplicate is the stricter of the two, so consolidating them would loosen the shared +/// template-cache gate rather than tidy it. +/// +/// Directives are case-insensitive (RFC 9111 §5.2), so `No-Store` and `Private` +/// count. `no-cache` deliberately does **not**: it requires revalidation before +/// reuse, not a refusal to store, so a `no-cache` response is still shareable. +/// Callers needing the stricter reading must check it themselves. +#[must_use] +pub fn is_private_or_no_store(headers: &HeaderMap) -> bool { + cache_control_headers_are_private_or_no_store(headers) +} + /// Reassert the terminal privacy invariant for a synthesized per-reader response. /// /// Call this after every configurable response mutation. It deliberately overwrites -/// `Cache-Control` and strips validators, expiry metadata, and runtime edge-cache +/// `Cache-Control` and strips validators, expiry metadata, and CDN-specific cache /// directives so a later integration cannot turn an assembled document into C3. pub fn enforce_private_no_store(response: &mut Response) { - CacheControlPolicy::NoStorePrivate - .apply_to_headers(response.headers_mut(), EdgeCacheHeader::None); + response.headers_mut().insert( + header::CACHE_CONTROL, + HeaderValue::from_static("private, no-store"), + ); for name in [ header::ETAG.as_str(), header::LAST_MODIFIED.as_str(), @@ -80,6 +159,7 @@ pub fn enforce_private_no_store(response: &mut Response) { ] { response.headers_mut().remove(name); } + strip_cdn_cache_headers(response); } /// Marks a Trusted Server response as terminal-private and applies its cache policy. @@ -91,8 +171,8 @@ pub(crate) fn enforce_terminal_private_cache_privacy(response: &mut Response) { /// Forces synthesized HTML to be private and non-storable. /// /// Use this exact policy whenever Trusted Server changes an origin HTML -/// representation with request-specific content: force `no-store, private`, -/// remove origin validators, and remove all runtime edge-cache directives. +/// representation with request-specific content: force `private, no-store`, +/// remove origin validators, and remove all CDN-targeted cache directives. pub(crate) fn enforce_synthesized_html_cache_privacy(response: &mut Response) { enforce_terminal_private_cache_privacy(response); } @@ -103,7 +183,7 @@ pub(crate) fn enforce_synthesized_html_cache_privacy(response: &mut Response) { /// `no-store` directive cannot coexist with an independently authoritative edge /// cache header. pub fn enforce_uncacheable_cache_privacy(response: &mut Response) { - if cache_control_is_private_or_no_store(response) { + if is_private_or_no_store(response.headers()) { remove_edge_cache_headers(response.headers_mut()); } } @@ -122,11 +202,11 @@ pub fn enforce_set_cookie_cache_privacy(response: &mut Response) { if !response.headers().contains_key(header::SET_COOKIE) { return; } - // Edge-cache headers must come off every cookie-bearing response, even one - // already carrying a stricter `no-store`/`private` directive — they are + // Shared-cache control headers must come off every cookie-bearing response, even + // one already carrying a stricter `no-store`/`private` directive — they are // independent of Cache-Control and would otherwise let a shared cache store // and replay one visitor's Set-Cookie. - remove_edge_cache_headers(response.headers_mut()); + strip_cdn_cache_headers(response); // Cookie privacy takes precedence over the generated inactive-stack browser // policy, while unrelated origin private/no-store policies remain unchanged. let already_forbids_shared_storage = cache_control_forbids_shared_storage(response.headers()); @@ -149,10 +229,10 @@ pub fn enforce_set_cookie_cache_privacy(response: &mut Response) { /// First downgrades cookie-bearing responses via /// [`enforce_set_cookie_cache_privacy`], then applies operator headers — but on /// an uncacheable (`private`/`no-store`) response the cache-controlling headers -/// (`Cache-Control` and runtime edge-cache headers) are skipped so operators +/// (`Cache-Control` and the surrogate cache headers) are skipped so operators /// cannot re-enable shared caching for per-user payloads. After the operator /// headers are applied the cookie-privacy downgrade runs once more, so a -/// configured `Set-Cookie` combined with public edge-cache headers cannot +/// configured `Set-Cookie` combined with public/surrogate cache headers cannot /// produce a shared-cacheable cookie-bearing response. /// /// Invalid header names/values are logged and skipped rather than panicking, so @@ -190,9 +270,9 @@ pub fn apply_response_headers_with_cache_privacy(settings: &Settings, response: enforce_uncacheable_cache_privacy(response); // Operator headers can themselves introduce Set-Cookie (alongside public - // edge-cache headers) onto a previously cookieless response, which the - // pre-apply pass could not see. Re-run the downgrade so the final response - // can never pair Set-Cookie with shared cacheability. + // or surrogate cache headers) onto a previously cookieless response, which + // the pre-apply pass could not see. Re-run the downgrade so the final + // response can never pair Set-Cookie with shared cacheability. enforce_set_cookie_cache_privacy(response); } @@ -200,9 +280,8 @@ pub fn apply_response_headers_with_cache_privacy(settings: &Settings, response: mod tests { use super::*; - use edgezero_core::http::response_builder; - use crate::cache_policy::EDGE_CACHE_HEADER_NAMES; + use edgezero_core::http::response_builder; fn settings_with_response_headers(headers: &[(&str, &str)]) -> Settings { let mut s = Settings::from_toml( @@ -231,7 +310,7 @@ mod tests { } #[test] - fn synthesized_html_is_forced_no_store_without_validators_or_edge_headers() { + fn synthesized_html_is_forced_no_store_without_validators_or_cdn_headers() { let mut response = response_builder() .header(header::CACHE_CONTROL, "private, max-age=600") .header(header::ETAG, "\"origin\"") @@ -247,7 +326,7 @@ mod tests { assert_eq!( response.headers()[header::CACHE_CONTROL], - "no-store, private", + "private, no-store", "synthesized HTML should always be non-storable" ); assert!( @@ -259,7 +338,7 @@ mod tests { ); for header_name in [header::ETAG.as_str(), header::LAST_MODIFIED.as_str()] .into_iter() - .chain(EDGE_CACHE_HEADER_NAMES.iter().copied()) + .chain(CDN_CACHE_HEADERS.iter().copied()) { assert!( !response.headers().contains_key(header_name), @@ -351,6 +430,7 @@ mod tests { let mut response = response_builder() .header(header::SET_COOKIE, "id=abc") .header("surrogate-control", "max-age=600") + .header("fastly-surrogate-control", "max-age=600") .header("cdn-cache-control", "max-age=600") .header("cloudflare-cdn-cache-control", "max-age=600") .body(edgezero_core::body::Body::empty()) @@ -366,14 +446,12 @@ mod tests { Some("private, max-age=0"), "operator public Cache-Control must not override cookie privacy downgrade" ); - assert!( - !response.headers().contains_key("surrogate-control") - && !response.headers().contains_key("cdn-cache-control") - && !response - .headers() - .contains_key("cloudflare-cdn-cache-control"), - "edge cache headers must be stripped on cookie responses" - ); + for header_name in CDN_CACHE_HEADERS { + assert!( + !response.headers().contains_key(*header_name), + "CDN cache header {header_name} must be stripped on cookie responses" + ); + } } #[test] @@ -384,8 +462,9 @@ mod tests { ("set-cookie", "operator=abc"), ("cache-control", "public, max-age=600"), ("surrogate-control", "max-age=600"), - ("cdn-cache-control", "max-age=600"), - ("cloudflare-cdn-cache-control", "max-age=600"), + ("fastly-surrogate-control", "max-age=600"), + ("cdn-cache-control", "public, max-age=600"), + ("cloudflare-cdn-cache-control", "public, max-age=600"), ]); let mut response = response_builder() .body(edgezero_core::body::Body::empty()) @@ -401,14 +480,12 @@ mod tests { Some("private, max-age=0"), "operator Set-Cookie plus public Cache-Control must be re-downgraded to private" ); - assert!( - !response.headers().contains_key("surrogate-control") - && !response.headers().contains_key("cdn-cache-control") - && !response - .headers() - .contains_key("cloudflare-cdn-cache-control"), - "edge cache headers must be stripped when operator headers add Set-Cookie" - ); + for header_name in CDN_CACHE_HEADERS { + assert!( + !response.headers().contains_key(*header_name), + "CDN cache header {header_name} must be stripped when operator headers add Set-Cookie" + ); + } assert!( response.headers().contains_key(header::SET_COOKIE), "the operator Set-Cookie itself should still be applied" @@ -434,7 +511,7 @@ mod tests { response .headers() .get(header::CACHE_CONTROL) - .and_then(|v| v.to_str().ok()), + .and_then(|value| value.to_str().ok()), Some("private, max-age=0"), "pseudo-directives must not prevent the cookie privacy downgrade" ); @@ -494,7 +571,7 @@ mod tests { "private, no-store", "operator cache headers must not weaken an existing private response" ); - for header_name in EDGE_CACHE_HEADER_NAMES { + for header_name in CDN_CACHE_HEADERS { assert!( !response.headers().contains_key(*header_name), "operator headers must not restore shared caching through {header_name}" @@ -502,6 +579,80 @@ mod tests { } } + #[test] + fn terminal_private_stamp_removes_every_cache_and_validator_header() { + let mut response = response_builder() + .header(header::CACHE_CONTROL, "public, s-maxage=600") + .header(header::ETAG, "\"origin\"") + .header(header::LAST_MODIFIED, "Wed, 12 Aug 2026 00:00:00 GMT") + .header(header::EXPIRES, "Wed, 12 Aug 2026 01:00:00 GMT") + .header(header::AGE, "30") + .header("surrogate-control", "max-age=600") + .header("cdn-cache-control", "public, max-age=600") + .body(edgezero_core::body::Body::empty()) + .expect("should build response"); + + enforce_private_no_store(&mut response); + + assert_eq!( + response.headers()[header::CACHE_CONTROL], + "private, no-store" + ); + for name in [ + header::ETAG.as_str(), + header::LAST_MODIFIED.as_str(), + header::EXPIRES.as_str(), + header::AGE.as_str(), + "surrogate-control", + "cdn-cache-control", + ] { + assert!( + !response.headers().contains_key(name), + "terminal private stamp must strip {name}" + ); + } + } + + #[test] + fn applies_operator_headers_on_cookieless_response() { + let settings = settings_with_response_headers(&[("x-operator", "value")]); + let mut response = response_builder() + .body(edgezero_core::body::Body::empty()) + .expect("should build response"); + + apply_response_headers_with_cache_privacy(&settings, &mut response); + + assert_eq!( + response + .headers() + .get("x-operator") + .and_then(|v| v.to_str().ok()), + Some("value"), + "operator headers should still apply to cacheable responses" + ); + } + + #[test] + fn uncacheable_response_rejects_operator_cdn_cache_headers() { + let settings = settings_with_response_headers(&[ + ("cdn-cache-control", "public, max-age=600"), + ("cloudflare-cdn-cache-control", "public, max-age=600"), + ]); + let mut response = response_builder() + .header(header::CACHE_CONTROL, "private, no-store") + .body(edgezero_core::body::Body::empty()) + .expect("should build response"); + + apply_response_headers_with_cache_privacy(&settings, &mut response); + + for header_name in ["cdn-cache-control", "cloudflare-cdn-cache-control"] { + assert!( + !response.headers().contains_key(header_name), + "operator headers must not restore shared caching through {header_name}" + ); + } + } + #[test] fn strips_edge_headers_from_uncacheable_cookieless_response() { let settings = settings_with_response_headers(&[ @@ -555,58 +706,4 @@ mod tests { "final guard should remove every edge-cache header" ); } - - #[test] - fn terminal_private_stamp_removes_every_cache_and_validator_header() { - let mut response = response_builder() - .header(header::CACHE_CONTROL, "public, s-maxage=600") - .header(header::ETAG, "\"origin\"") - .header(header::LAST_MODIFIED, "Wed, 12 Aug 2026 00:00:00 GMT") - .header(header::EXPIRES, "Wed, 12 Aug 2026 01:00:00 GMT") - .header(header::AGE, "30") - .header("surrogate-control", "max-age=600") - .header("cdn-cache-control", "public, max-age=600") - .body(edgezero_core::body::Body::empty()) - .expect("should build response"); - - enforce_private_no_store(&mut response); - - assert_eq!( - response.headers()[header::CACHE_CONTROL], - "no-store, private" - ); - for name in [ - header::ETAG.as_str(), - header::LAST_MODIFIED.as_str(), - header::EXPIRES.as_str(), - header::AGE.as_str(), - ] - .into_iter() - .chain(EDGE_CACHE_HEADER_NAMES.iter().copied()) - { - assert!( - !response.headers().contains_key(name), - "terminal private stamp must strip {name}" - ); - } - } - - #[test] - fn applies_operator_headers_on_cookieless_response() { - let settings = settings_with_response_headers(&[("x-operator", "value")]); - let mut response = response_builder() - .body(edgezero_core::body::Body::empty()) - .expect("should build response"); - - apply_response_headers_with_cache_privacy(&settings, &mut response); - - assert_eq!( - response - .headers() - .get("x-operator") - .and_then(|v| v.to_str().ok()), - Some("value"), - "operator headers should still apply to cacheable responses" - ); - } } diff --git a/crates/trusted-server-core/src/secret_resolution.rs b/crates/trusted-server-core/src/secret_resolution.rs new file mode 100644 index 000000000..6b6cd7696 --- /dev/null +++ b/crates/trusted-server-core/src/secret_resolution.rs @@ -0,0 +1,408 @@ +//! Runtime resolution of `EdgeZero` app-config secret references. +//! +//! Config blobs carry secret-store key names at rest. This module walks the +//! public `EdgeZero` metadata contract and replaces those names only in the +//! in-memory value used to build runtime [`crate::settings::Settings`]. + +use edgezero_core::app_config::{AppConfigMeta, SecretField, SecretKind, SecretPathSegment}; +use error_stack::Report; +use serde_json::Value; + +use crate::error::TrustedServerError; +use crate::platform::{PlatformSecretStore, StoreName}; + +/// Resolve all secret references in a serialized Trusted Server app config. +/// +/// The input is mutated in memory; the verified envelope is never rewritten. +/// Secret values are not included in structural or platform errors. +/// +/// # Errors +/// +/// Returns [`TrustedServerError::Configuration`] when a required path or key +/// is malformed, a secret is unavailable, is not valid UTF-8, or resolves to an +/// empty value. +pub fn resolve_secret_references( + data: &mut Value, + secret_store: &dyn PlatformSecretStore, + default_store_name: &StoreName, +) -> Result<(), Report> { + let mut resolved_data = data.clone(); + for field in C::secret_fields() { + if matches!(field.kind, SecretKind::StoreRef) { + continue; + } + resolve_field( + &mut resolved_data, + &field, + &field.path, + "", + secret_store, + default_store_name, + )?; + } + *data = resolved_data; + Ok(()) +} + +fn resolve_field( + node: &mut Value, + field: &SecretField, + remaining: &[SecretPathSegment], + rendered_path: &str, + secret_store: &dyn PlatformSecretStore, + default_store_name: &StoreName, +) -> Result<(), Report> { + match remaining.split_first() { + Some((SecretPathSegment::Field(name), [])) => resolve_leaf( + node, + field, + name.as_ref(), + rendered_path, + secret_store, + default_store_name, + ), + Some((SecretPathSegment::OptionalField(name), [])) => { + if matches!(node.get(name.as_ref()), None | Some(Value::Null)) { + return Ok(()); + } + resolve_leaf( + node, + field, + name.as_ref(), + rendered_path, + secret_store, + default_store_name, + ) + } + Some((SecretPathSegment::Field(name), rest)) => { + let next_path = join_field(rendered_path, name.as_ref()); + let child = node + .as_object_mut() + .and_then(|object| object.get_mut(name.as_ref())) + .ok_or_else(|| missing_path(&next_path))?; + if child.is_null() { + return Err(missing_path(&next_path)); + } + resolve_field( + child, + field, + rest, + &next_path, + secret_store, + default_store_name, + ) + } + Some((SecretPathSegment::OptionalField(name), rest)) => { + let next_path = join_field(rendered_path, name.as_ref()); + let Some(child) = node + .as_object_mut() + .and_then(|object| object.get_mut(name.as_ref())) + else { + return Ok(()); + }; + if child.is_null() { + return Ok(()); + } + resolve_field( + child, + field, + rest, + &next_path, + secret_store, + default_store_name, + ) + } + Some((SecretPathSegment::ArrayEach, rest)) => { + let items = node.as_array_mut().ok_or_else(|| { + configuration_error(format!("expected an array at `{rendered_path}`")) + })?; + for (index, item) in items.iter_mut().enumerate() { + let indexed_path = format!("{rendered_path}[{index}]"); + resolve_field( + item, + field, + rest, + &indexed_path, + secret_store, + default_store_name, + )?; + } + Ok(()) + } + None => Ok(()), + } +} + +fn resolve_leaf( + parent: &mut Value, + field: &SecretField, + key: &str, + rendered_parent: &str, + secret_store: &dyn PlatformSecretStore, + default_store_name: &StoreName, +) -> Result<(), Report> { + let leaf_path = join_field(rendered_parent, key); + let object = parent.as_object_mut().ok_or_else(|| { + configuration_error(format!("expected an object containing `{leaf_path}`")) + })?; + + let key_name = match object.get(key) { + Some(Value::String(value)) if !value.is_empty() => value.clone(), + Some(Value::Null) | None if field.optional => return Ok(()), + Some(Value::Null) | None => return Err(missing_path(&leaf_path)), + Some(Value::String(_)) => { + return Err(configuration_error(format!( + "secret key reference at `{leaf_path}` must not be empty" + ))); + } + _ => { + return Err(configuration_error(format!( + "secret key reference at `{leaf_path}` must be a string" + ))); + } + }; + + let resolved = secret_store + .get_string(default_store_name, &key_name) + .map_err(|_| { + configuration_error(format!( + "failed to resolve secret reference at `{leaf_path}` from secret store \ + `{default_store_name}`" + )) + })?; + if resolved.is_empty() { + return Err(configuration_error(format!( + "resolved secret at `{leaf_path}` must not be empty" + ))); + } + + object.insert(key.to_owned(), Value::String(resolved)); + Ok(()) +} + +fn join_field(prefix: &str, field: &str) -> String { + if prefix.is_empty() { + field.to_owned() + } else { + format!("{prefix}.{field}") + } +} + +fn missing_path(path: &str) -> Report { + configuration_error(format!("missing required secret path `{path}`")) +} + +fn configuration_error(message: String) -> Report { + Report::new(TrustedServerError::Configuration { message }) +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::platform::{PlatformError, StoreId}; + use std::collections::BTreeMap; + + struct MemorySecretStore { + values: BTreeMap>, + } + + impl PlatformSecretStore for MemorySecretStore { + fn get_bytes( + &self, + _store_name: &StoreName, + key: &str, + ) -> Result, Report> { + self.values.get(key).cloned().ok_or_else(|| { + Report::new(PlatformError::SecretStore) + .attach(format!("missing test secret for key `{key}`")) + }) + } + + fn create( + &self, + _store_id: &StoreId, + _name: &str, + _value: &str, + ) -> Result<(), Report> { + Ok(()) + } + + fn delete(&self, _store_id: &StoreId, _name: &str) -> Result<(), Report> { + Ok(()) + } + } + + struct Fixture; + + impl AppConfigMeta for Fixture { + fn secret_fields() -> Vec { + vec![ + SecretField { + kind: SecretKind::KeyInDefault, + optional: false, + path: vec![ + SecretPathSegment::Field("outer".into()), + SecretPathSegment::ArrayEach, + SecretPathSegment::Field("token".into()), + ], + }, + SecretField { + kind: SecretKind::KeyInDefault, + optional: true, + path: vec![ + SecretPathSegment::Field("outer".into()), + SecretPathSegment::ArrayEach, + SecretPathSegment::Field("optional".into()), + ], + }, + SecretField { + kind: SecretKind::KeyInDefault, + optional: false, + path: vec![ + SecretPathSegment::OptionalField("feature".into()), + SecretPathSegment::Field("credential".into()), + ], + }, + ] + } + } + + fn store() -> MemorySecretStore { + MemorySecretStore { + values: BTreeMap::from([ + ("token-a".to_owned(), b"resolved-a".to_vec()), + ("token-b".to_owned(), b"resolved-b".to_vec()), + ("feature-key".to_owned(), b"resolved-feature".to_vec()), + ]), + } + } + + #[test] + fn resolves_nested_array_values_and_skips_optional_nulls() { + let mut data = serde_json::json!({ + "outer": [ + {"token": "token-a", "optional": null}, + {"token": "token-b"} + ] + }); + + resolve_secret_references::(&mut data, &store(), &StoreName::from("secrets")) + .expect("should resolve nested array secrets"); + + assert_eq!(data["outer"][0]["token"], "resolved-a"); + assert_eq!(data["outer"][1]["token"], "resolved-b"); + assert!(data["outer"][0]["optional"].is_null()); + } + + #[test] + fn resolves_present_and_skips_absent_optional_intermediate() { + let mut absent = serde_json::json!({ + "outer": [{"token": "token-a"}] + }); + resolve_secret_references::(&mut absent, &store(), &StoreName::from("secrets")) + .expect("should skip absent optional intermediate"); + + let mut present = serde_json::json!({ + "outer": [{"token": "token-a"}], + "feature": {"credential": "feature-key"} + }); + resolve_secret_references::(&mut present, &store(), &StoreName::from("secrets")) + .expect("should resolve present optional intermediate"); + + assert_eq!(present["feature"]["credential"], "resolved-feature"); + } + + #[test] + fn rejects_missing_required_path_without_secret_values() { + for mut data in [ + serde_json::json!({"outer": [{}]}), + serde_json::json!({"outer": [{"token": null}]}), + ] { + let err = resolve_secret_references::( + &mut data, + &store(), + &StoreName::from("secrets"), + ) + .expect_err("should reject missing required secret path"); + + assert!(err.to_string().contains("missing required secret path")); + assert!(err.to_string().contains("outer[0].token")); + assert!(!err.to_string().contains("resolved-a")); + } + } + + #[test] + fn rejects_non_string_required_leaf() { + let mut data = serde_json::json!({"outer": [{"token": true}]}); + let err = + resolve_secret_references::(&mut data, &store(), &StoreName::from("secrets")) + .expect_err("should reject non-string secret reference"); + + assert!(err.to_string().contains("must be a string")); + assert!(err.to_string().contains("outer[0].token")); + } + + #[test] + fn failed_lookup_reports_safe_reference_context_without_secret_values() { + let plaintext_blob_value = "legacy-plaintext-credential"; + let mut data = serde_json::json!({"outer": [{"token": plaintext_blob_value}]}); + let store = MemorySecretStore { + values: BTreeMap::from([( + "fixture-secret-key".to_owned(), + b"fixture-secret-value".to_vec(), + )]), + }; + + let err = + resolve_secret_references::(&mut data, &store, &StoreName::from("secrets")) + .expect_err("should reject a missing secret key"); + let diagnostic = format!("{err:?}"); + + assert!(diagnostic.contains("outer[0].token")); + assert!(diagnostic.contains("secrets")); + assert!(!diagnostic.contains(plaintext_blob_value)); + assert!(!diagnostic.contains("missing test secret")); + assert!(!diagnostic.contains("fixture-secret-value")); + } + + #[test] + fn rejects_malformed_array_path_without_resolving_values() { + let mut data = serde_json::json!({"outer": {"token": "token-a"}}); + let err = + resolve_secret_references::(&mut data, &store(), &StoreName::from("secrets")) + .expect_err("should reject a non-array intermediate path"); + + assert!(err.to_string().contains("expected an array")); + assert!(!err.to_string().contains("resolved-a")); + } + + #[test] + fn rejects_invalid_utf8_and_empty_resolved_values() { + let mut invalid = store(); + invalid.values.insert("token-a".to_owned(), vec![0xff]); + let mut data = serde_json::json!({"outer": [{"token": "token-a"}]}); + let err = + resolve_secret_references::(&mut data, &invalid, &StoreName::from("secrets")) + .expect_err("should reject invalid UTF-8"); + assert!(err.to_string().contains("outer[0].token")); + + let empty = MemorySecretStore { + values: BTreeMap::from([("token-a".to_owned(), Vec::new())]), + }; + let mut data = serde_json::json!({"outer": [{"token": "token-a"}]}); + let err = + resolve_secret_references::(&mut data, &empty, &StoreName::from("secrets")) + .expect_err("should reject empty resolved value"); + assert!(err.to_string().contains("outer[0].token")); + } + + #[test] + fn does_not_mutate_data_when_resolution_fails() { + let mut data = serde_json::json!({"outer": [{"token": "missing"}]}); + let original = data.clone(); + let result = + resolve_secret_references::(&mut data, &store(), &StoreName::from("secrets")); + assert!(result.is_err(), "should fail for missing secret key"); + assert_eq!(data, original, "should preserve unresolved data on failure"); + } +} diff --git a/crates/trusted-server-core/src/settings.rs b/crates/trusted-server-core/src/settings.rs index 0c78ab00b..8fff7917d 100644 --- a/crates/trusted-server-core/src/settings.rs +++ b/crates/trusted-server-core/src/settings.rs @@ -104,7 +104,11 @@ impl Default for Publisher { impl Publisher { /// Known placeholder values that must not be used in production. - pub const PROXY_SECRET_PLACEHOLDERS: &[&str] = &["change-me-proxy-secret", "proxy-secret"]; + pub const PROXY_SECRET_PLACEHOLDERS: &[&str] = &[ + "change-me-proxy-secret", + "proxy-secret", + "replace-with-random-proxy-secret", + ]; /// Returns the EC cookie domain, computed as `.{domain}`. /// @@ -211,14 +215,42 @@ impl Publisher { } } -#[derive(Debug, Default, Clone, Deserialize, Serialize)] +#[derive(Default, Clone, Deserialize, Serialize)] pub struct IntegrationSettings { #[serde(flatten)] entries: HashMap, } +impl std::fmt::Debug for IntegrationSettings { + fn fmt(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + let mut integration_ids = self.entries.keys().collect::>(); + integration_ids.sort_unstable(); + formatter + .debug_struct("IntegrationSettings") + .field("integration_ids", &integration_ids) + .finish() + } +} + pub trait IntegrationConfig: DeserializeOwned + Validate { fn is_enabled(&self) -> bool; + + /// Validate the public field schema for an explicitly disabled config. + /// + /// Integrations with removed fields override this hook using a disabled-safe + /// typed schema. The default preserves existing support for minimal disabled + /// blocks whose enabled-only required fields are omitted. + /// + /// # Errors + /// + /// Returns a deserialization error when the disabled public field schema is invalid. + fn validate_disabled_schema(raw: &JsonValue) -> Result<(), serde_json::Error> { + match serde_json::from_value::(raw.clone()) { + Ok(_) => Ok(()), + Err(error) if error.to_string().starts_with("missing field ") => Ok(()), + Err(error) => Err(error), + } + } } impl IntegrationSettings { @@ -251,6 +283,29 @@ impl IntegrationSettings { == Some(false) } + fn remove_legacy_static_secret_store_selectors(&mut self) { + let Some(datadome) = self + .entries + .get_mut("datadome") + .and_then(JsonValue::as_object_mut) + else { + return; + }; + + let mut removed = datadome.remove("server_side_key_secret_store").is_some(); + if let Some(bypass) = datadome + .get_mut("protection_test_bypass") + .and_then(JsonValue::as_object_mut) + { + removed |= bypass.remove("credential_secret_store").is_some(); + } + if removed { + log::warn!( + "DataDome secret-store selectors are deprecated and ignored; static credentials resolve through the default app-config secret store" + ); + } + } + /// Retrieves and validates a typed configuration for an integration. /// /// # Errors @@ -269,6 +324,11 @@ impl IntegrationSettings { }; if Self::is_explicitly_disabled(raw) { + T::validate_disabled_schema(raw).change_context(TrustedServerError::Configuration { + message: format!( + "Integration '{integration_id}' configuration could not be parsed" + ), + })?; return Ok(None); } @@ -318,7 +378,7 @@ impl DerefMut for IntegrationSettings { /// A partner (SSP, DSP, identity vendor) configured in `[[ec.partners]]`. /// /// Partners are defined statically in `trusted-server.toml` rather than -/// registered via API. At startup, each partner's `api_token` is hashed +/// registered via API. At startup, each configured `api_token` is hashed /// (SHA-256) for O(1) auth lookups; the plaintext is never stored at runtime. #[derive(Debug, Clone, Deserialize, Serialize, Validate)] #[serde(deny_unknown_fields)] @@ -340,9 +400,12 @@ pub struct EcPartner { /// Whether this partner's UIDs appear in auction `user.eids`. #[serde(default, deserialize_with = "from_value_or_str")] pub bidstream_enabled: bool, - /// Plaintext API token. Hashed at startup for auth lookups. - /// Used by batch sync (inbound) and identify (inbound). - pub api_token: Redacted, + /// Plaintext API token used by inbound batch sync and identify requests. + /// + /// When present, the token is hashed at startup for auth lookups. Omitting + /// it disables inbound partner API authentication for this partner. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub api_token: Option>, /// Max batch sync API requests per partner per minute. #[serde( default = "EcPartner::default_batch_rate_limit", @@ -515,6 +578,7 @@ impl Ec { "secret_key", "trusted-server", "trusted-server-placeholder-secret", + "replace-with-random-ec-passphrase", ]; /// Default maximum concurrent pull-sync requests. @@ -712,16 +776,12 @@ fn default_request_signing_enabled() -> bool { false } -fn default_s3_secret_store() -> String { - "s3-auth".to_string() +fn default_s3_access_key_id() -> Redacted { + Redacted::new("access_key_id".to_string()) } -fn default_s3_access_key_id() -> String { - "access_key_id".to_string() -} - -fn default_s3_secret_access_key() -> String { - "secret_access_key".to_string() +fn default_s3_secret_access_key() -> Redacted { + Redacted::new("secret_access_key".to_string()) } fn default_asset_image_optimizer_enabled() -> bool { @@ -808,25 +868,25 @@ impl AssetOriginAuth { /// AWS Signature Version 4 configuration for `S3` asset origins. /// /// The route `origin_url` must use the same `S3` host that `AWS` validates in -/// the `SigV4` canonical request. Credentials are read from the named runtime -/// secret store and cached per process by configured secret names. +/// the `SigV4` canonical request. Credential fields hold secret-store key names +/// in app config and resolved values at runtime. #[derive(Debug, Clone, Deserialize, Serialize)] #[serde(deny_unknown_fields)] pub struct S3SigV4AuthConfig { /// `AWS` region used in the credential scope. pub region: String, - /// Runtime secret store containing `S3` credentials. - #[serde(default = "default_s3_secret_store")] - pub secret_store: String, - /// Secret name containing the `AWS` access key ID. + /// Deprecated per-route store selector accepted for migration only. + #[serde(default, skip_serializing)] + pub secret_store: Option, + /// Secret reference containing the `AWS` access key ID. #[serde(default = "default_s3_access_key_id")] - pub access_key_id: String, - /// Secret name containing the `AWS` secret access key. + pub access_key_id: Redacted, + /// Secret reference containing the `AWS` secret access key. #[serde(default = "default_s3_secret_access_key")] - pub secret_access_key: String, - /// Optional secret name containing an `AWS` session token. + pub secret_access_key: Redacted, + /// Optional secret reference containing an `AWS` session token. #[serde(default)] - pub session_token: Option, + pub session_token: Option>, /// Query-string handling policy for the signed `S3` origin request. /// /// Set this to `strip` when request query parameters are transformation @@ -845,14 +905,17 @@ fn s3_region_is_valid(region: &str) -> bool { impl S3SigV4AuthConfig { fn normalize(&mut self) { self.region = self.region.trim().to_string(); - self.secret_store = self.secret_store.trim().to_string(); - self.access_key_id = self.access_key_id.trim().to_string(); - self.secret_access_key = self.secret_access_key.trim().to_string(); - self.session_token = self - .session_token - .take() - .map(|value| value.trim().to_string()) - .filter(|value| !value.is_empty()); + if self.secret_store.take().is_some() { + log::warn!( + "S3 secret_store is deprecated and ignored; static credentials resolve through the default app-config secret store" + ); + } + self.access_key_id = Redacted::new(self.access_key_id.expose().trim().to_string()); + self.secret_access_key = Redacted::new(self.secret_access_key.expose().trim().to_string()); + self.session_token = self.session_token.take().and_then(|value| { + let value = value.expose().trim().to_string(); + (!value.is_empty()).then(|| Redacted::new(value)) + }); } fn prepare_runtime(&self) -> Result<(), Report> { @@ -868,12 +931,9 @@ impl S3SigV4AuthConfig { .to_string(), })); } - if self.secret_store.is_empty() - || self.access_key_id.is_empty() - || self.secret_access_key.is_empty() - { + if self.access_key_id.expose().is_empty() || self.secret_access_key.expose().is_empty() { return Err(Report::new(TrustedServerError::Configuration { - message: "proxy.asset_routes auth s3_sigv4 secret names must not be empty" + message: "proxy.asset_routes auth s3_sigv4 credentials must not be empty after secret resolution" .to_string(), })); } @@ -1793,15 +1853,15 @@ pub struct TinybirdSettings { /// Regional Tinybird API host, without scheme or path. #[serde(default)] pub api_host: String, - /// Fastly Secret Store name containing Tinybird append tokens. - #[serde(default = "default_tinybird_secret_store")] - pub secret_store: String, + /// Deprecated feature-specific store selector accepted for migration only. + #[serde(default, skip_serializing)] + pub secret_store: Option, /// Auction Events API datasource name. #[serde(default = "default_tinybird_auction_dataset")] pub auction_dataset: String, - /// Secret key containing the auction datasource APPEND token. - #[serde(default = "default_tinybird_auction_token_secret")] - pub auction_token_secret: String, + /// Secret reference containing the auction datasource APPEND token. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub auction_token_secret: Option>, /// Reserved for future access-log telemetry. /// /// `true` is rejected until an access-log emitter is wired, so operators @@ -1811,9 +1871,9 @@ pub struct TinybirdSettings { /// Future access-log Events API datasource name. #[serde(default = "default_tinybird_access_dataset")] pub access_dataset: String, - /// Future Secret Store key containing the access-log datasource APPEND token. - #[serde(default = "default_tinybird_access_token_secret")] - pub access_token_secret: String, + /// Deprecated placeholder for the unwired access-log APPEND token. + #[serde(default, skip_serializing)] + pub access_token_secret: Option>, /// Future fraction of requests to emit for optional access telemetry. #[serde(default)] pub access_sample_rate: f64, @@ -1822,26 +1882,14 @@ pub struct TinybirdSettings { pub max_body_bytes: usize, } -fn default_tinybird_secret_store() -> String { - "ts_secrets".to_owned() -} - fn default_tinybird_auction_dataset() -> String { "auction_events_raw".to_owned() } -fn default_tinybird_auction_token_secret() -> String { - "tinybird_auction_append_token".to_owned() -} - fn default_tinybird_access_dataset() -> String { "access_logs_raw".to_owned() } -fn default_tinybird_access_token_secret() -> String { - "tinybird_access_append_token".to_owned() -} - fn default_tinybird_max_body_bytes() -> usize { 1024 * 1024 } @@ -1851,12 +1899,12 @@ impl Default for TinybirdSettings { Self { enabled: false, api_host: String::new(), - secret_store: default_tinybird_secret_store(), + secret_store: None, auction_dataset: default_tinybird_auction_dataset(), - auction_token_secret: default_tinybird_auction_token_secret(), + auction_token_secret: None, access_enabled: false, access_dataset: default_tinybird_access_dataset(), - access_token_secret: default_tinybird_access_token_secret(), + access_token_secret: None, access_sample_rate: 0.0, max_body_bytes: default_tinybird_max_body_bytes(), } @@ -1866,11 +1914,18 @@ impl Default for TinybirdSettings { impl TinybirdSettings { fn normalize(&mut self) { self.api_host = self.api_host.trim().to_ascii_lowercase(); - self.secret_store = self.secret_store.trim().to_owned(); + if self.secret_store.take().is_some() { + log::warn!( + "tinybird.secret_store is deprecated and ignored; static credentials resolve through the default app-config secret store" + ); + } self.auction_dataset = self.auction_dataset.trim().to_owned(); - self.auction_token_secret = self.auction_token_secret.trim().to_owned(); + self.auction_token_secret = self.auction_token_secret.take().and_then(|value| { + let value = value.expose().trim().to_owned(); + (!value.is_empty()).then(|| Redacted::new(value)) + }); self.access_dataset = self.access_dataset.trim().to_owned(); - self.access_token_secret = self.access_token_secret.trim().to_owned(); + self.access_token_secret = None; } fn prepare_runtime(&mut self) -> Result<(), Report> { @@ -1894,18 +1949,15 @@ impl TinybirdSettings { return Ok(()); } validate_tinybird_api_host(&self.api_host)?; - if self.secret_store.is_empty() { - return Err(Report::new(TrustedServerError::Configuration { + validate_tinybird_dataset(&self.auction_dataset, "tinybird.auction_dataset")?; + let token = self.auction_token_secret.as_ref().ok_or_else(|| { + Report::new(TrustedServerError::Configuration { message: - "tinybird.secret_store must not be empty when Tinybird telemetry is enabled" + "tinybird.auction_token_secret is required when Tinybird telemetry is enabled" .to_owned(), - })); - } - if self.enabled { - validate_tinybird_dataset(&self.auction_dataset, "tinybird.auction_dataset")?; - validate_tinybird_secret(&self.auction_token_secret, "tinybird.auction_token_secret")?; - } - Ok(()) + }) + })?; + validate_tinybird_secret(token.expose(), "tinybird.auction_token_secret") } } @@ -1946,7 +1998,7 @@ fn validate_tinybird_dataset(value: &str, setting: &str) -> Result<(), Report Result<(), Report> { if value.is_empty() || value.chars().any(char::is_control) { return Err(Report::new(TrustedServerError::Configuration { - message: format!("{setting} must be a non-empty Secret Store key"), + message: format!("{setting} must be non-empty after secret resolution"), })); } Ok(()) @@ -2899,21 +2951,30 @@ impl Settings { Self::finalize_deserialized(settings, "Build-time configuration") } + pub(crate) fn normalize_deserialized(&mut self) { + self.cache.normalize(); + self.proxy.normalize(); + self.image_optimizer.normalize(); + self.tinybird.normalize(); + self.debug.auction_html_comment_options.normalize(); + self.integrations + .remove_legacy_static_secret_store_selectors(); + self.consent.validate(); + } + pub(crate) fn finalize_deserialized( mut settings: Self, validation_label: &str, ) -> Result> { - settings.cache.normalize(); - settings.proxy.normalize(); - settings.image_optimizer.normalize(); - settings.debug.auction_html_comment_options.normalize(); - settings.consent.validate(); - + settings.normalize_deserialized(); settings.prepare_runtime()?; settings.validate().map_err(|err| { Report::new(TrustedServerError::Configuration { - message: format!("{validation_label} validation failed: {err}"), + message: format!( + "{validation_label} validation failed: {}", + validation_error_summary(&err) + ), }) })?; @@ -3021,7 +3082,11 @@ impl Settings { insecure_fields.push("trusted_client_ip.shared_secret".to_owned()); } for partner in &self.ec.partners { - if EcPartner::is_placeholder_api_token(partner.api_token.expose()) { + if partner + .api_token + .as_ref() + .is_some_and(|token| EcPartner::is_placeholder_api_token(token.expose())) + { insecure_fields.push(format!("ec.partners[{}].api_token", partner.source_domain)); } } @@ -3222,7 +3287,7 @@ impl Settings { /// /// Returns [`TrustedServerError::Configuration`] listing any uncovered /// admin endpoints. - fn validate_admin_coverage(&self) -> Result<(), Report> { + pub(crate) fn validate_admin_coverage(&self) -> Result<(), Report> { let uncovered = self.uncovered_admin_endpoints()?; if uncovered.is_empty() { return Ok(()); @@ -3244,7 +3309,9 @@ impl Settings { /// regexes, so a narrow handler can shadow the admin namespace for paths no /// probe enumerates. Handlers are Trusted Server's own basic-auth gates, so /// a placeholder password is never valid on any of them. - fn validate_admin_handler_passwords(&self) -> Result<(), Report> { + pub(crate) fn validate_admin_handler_passwords( + &self, + ) -> Result<(), Report> { for handler in &self.handlers { if is_admin_placeholder_password(handler.password.expose()) { return Err(Report::new(TrustedServerError::Configuration { @@ -3339,6 +3406,47 @@ fn validate_host_header_override(value: &str) -> Result<(), ValidationError> { Ok(()) } +fn validation_error_summary(errors: &validator::ValidationErrors) -> String { + fn walk(errors: &validator::ValidationErrors, prefix: &str, messages: &mut Vec) { + let mut fields = errors + .errors() + .keys() + .map(AsRef::as_ref) + .collect::>(); + fields.sort_unstable(); + + for field in fields { + let path = if prefix.is_empty() { + field.to_owned() + } else { + format!("{prefix}.{field}") + }; + let Some(kind) = errors.errors().get(field) else { + continue; + }; + match kind { + validator::ValidationErrorsKind::Field(validations) => { + for validation in validations { + messages.push(format!("{path}: {}", validation.code)); + } + } + validator::ValidationErrorsKind::Struct(inner) => { + walk(inner, &path, messages); + } + validator::ValidationErrorsKind::List(items) => { + for (index, inner) in items { + walk(inner, &format!("{path}[{index}]"), messages); + } + } + } + } + } + + let mut messages = Vec::new(); + walk(errors, "", &mut messages); + messages.join(", ") +} + fn validate_redacted_not_empty(value: &Redacted) -> Result<(), ValidationError> { if value.expose().is_empty() { return Err(ValidationError::new("empty_value")); @@ -3436,9 +3544,10 @@ where } // Helper: allow Vec fields to deserialize from either a JSON array or a map of numeric indices. -// This lets env vars like TRUSTED_SERVER__INTEGRATIONS__PREBID__BIDDERS__0=smartadserver work, which the config env source -// represents as an object {"0": "value"} rather than a sequence. Also supports string inputs that are -// JSON arrays or comma-separated values. +// This lets env vars such as +// TRUSTED_SERVER__INTEGRATIONS__PREBID__CLIENT_SIDE_BIDDERS__0=example-browser work; +// the config env source represents the value as an object rather than a sequence. +// String inputs may also be JSON arrays or comma-separated values. /// Deserializes a `HashMap` from either: /// - A TOML table / JSON object (standard deserialization) /// - A JSON string (e.g. from env var: `'{"Key": "value"}'`) @@ -3574,6 +3683,7 @@ mod tests { use regex::Regex; use serde_json::json; use std::collections::HashSet; + use std::sync::Arc; use crate::auction::build_orchestrator; use crate::integrations::{ @@ -4272,12 +4382,9 @@ mod tests { !settings.tinybird.enabled, "Tinybird should default disabled" ); - assert_eq!(settings.tinybird.secret_store, "ts_secrets"); + assert_eq!(settings.tinybird.secret_store, None); assert_eq!(settings.tinybird.auction_dataset, "auction_events_raw"); - assert_eq!( - settings.tinybird.auction_token_secret, - "tinybird_auction_append_token" - ); + assert!(settings.tinybird.auction_token_secret.is_none()); } #[test] @@ -4297,7 +4404,7 @@ mod tests { #[test] fn tinybird_accepts_region_host_without_scheme() { let toml = format!( - "{}\n[tinybird]\nenabled = true\napi_host = \"api.us-east.aws.tinybird.co\"\n", + "{}\n[tinybird]\nenabled = true\napi_host = \"api.us-east.aws.tinybird.co\"\nauction_token_secret = \"test-auction-token\"\n", crate_test_settings_str() ); @@ -4333,10 +4440,7 @@ mod tests { .integration_config::("prebid") .expect("Prebid config query should succeed") .expect("Prebid config should load from test settings"); - assert_eq!( - prebid_cfg.server_url, - "https://test-prebid.com/openrtb2/auction" - ); + assert_eq!(prebid_cfg.timeout_ms, 1000); assert!( settings .integration_config::("nextjs") @@ -5098,6 +5202,24 @@ origin_host_header_overide = "www.example.com""#, ); } + #[test] + fn ec_partner_api_token_can_be_omitted() { + let partner: EcPartner = toml::from_str( + r#" +name = "Example Partner" +source_domain = "partner.example.com" +"#, + ) + .expect("should deserialize partner without API token"); + + assert!(partner.api_token.is_none(), "should omit API token"); + let serialized = serde_json::to_value(partner).expect("should serialize partner"); + assert!( + serialized.get("api_token").is_none(), + "should not serialize an omitted API token" + ); + } + #[test] fn validate_passphrase_rejects_under_32_characters() { let passphrase = Redacted::new("a".repeat(31)); @@ -5291,101 +5413,6 @@ origin_host_header_overide = "www.example.com""#, assert!(settings.is_err(), "Should fail when sections are missing"); } - #[test] - fn test_prebid_bidders_override_with_json_env() { - let toml_str = crate_test_settings_str(); - let env_key = format!( - "{}{}INTEGRATIONS{}PREBID{}BIDDERS", - ENVIRONMENT_VARIABLE_PREFIX, - ENVIRONMENT_VARIABLE_SEPARATOR, - ENVIRONMENT_VARIABLE_SEPARATOR, - ENVIRONMENT_VARIABLE_SEPARATOR - ); - - // Ensure no external override interferes - let origin_key = format!( - "{}{}PUBLISHER{}ORIGIN_URL", - ENVIRONMENT_VARIABLE_PREFIX, - ENVIRONMENT_VARIABLE_SEPARATOR, - ENVIRONMENT_VARIABLE_SEPARATOR - ); - temp_env::with_var( - origin_key, - Some("https://origin.test-publisher.com"), - || { - temp_env::with_var(env_key, Some("[\"smartadserver\",\"rubicon\"]"), || { - let res = Settings::from_toml_and_env(&toml_str); - if res.is_err() { - eprintln!("JSON override error: {:?}", res.as_ref().err()); - } - let settings = res.expect("Settings should parse with JSON env override"); - let cfg = settings - .integration_config::("prebid") - .expect("Prebid config query should succeed") - .expect("Prebid config should exist with env override"); - assert_eq!( - cfg.bidders, - vec!["smartadserver".to_string(), "rubicon".to_string()] - ); - }); - }, - ); - } - - #[test] - fn test_prebid_bidders_override_with_indexed_env() { - let toml_str = crate_test_settings_str(); - - let env_key0 = format!( - "{}{}INTEGRATIONS{}PREBID{}BIDDERS{}0", - ENVIRONMENT_VARIABLE_PREFIX, - ENVIRONMENT_VARIABLE_SEPARATOR, - ENVIRONMENT_VARIABLE_SEPARATOR, - ENVIRONMENT_VARIABLE_SEPARATOR, - ENVIRONMENT_VARIABLE_SEPARATOR - ); - let env_key1 = format!( - "{}{}INTEGRATIONS{}PREBID{}BIDDERS{}1", - ENVIRONMENT_VARIABLE_PREFIX, - ENVIRONMENT_VARIABLE_SEPARATOR, - ENVIRONMENT_VARIABLE_SEPARATOR, - ENVIRONMENT_VARIABLE_SEPARATOR, - ENVIRONMENT_VARIABLE_SEPARATOR - ); - - // Also ensure origin_url env is a plain string (avoid any external env interference) - let origin_key = format!( - "{}{}PUBLISHER{}ORIGIN_URL", - ENVIRONMENT_VARIABLE_PREFIX, - ENVIRONMENT_VARIABLE_SEPARATOR, - ENVIRONMENT_VARIABLE_SEPARATOR - ); - temp_env::with_var( - origin_key, - Some("https://origin.test-publisher.com"), - || { - temp_env::with_var(env_key0, Some("smartadserver"), || { - temp_env::with_var(env_key1, Some("openx"), || { - let res = Settings::from_toml_and_env(&toml_str); - if res.is_err() { - eprintln!("Indexed override error: {:?}", res.as_ref().err()); - } - let settings = - res.expect("Settings should parse with indexed env override"); - let cfg = settings - .integration_config::("prebid") - .expect("Prebid config query should succeed") - .expect("Prebid config should exist with indexed env override"); - assert_eq!( - cfg.bidders, - vec!["smartadserver".to_string(), "openx".to_string()] - ); - }); - }); - }, - ); - } - #[test] fn test_handlers_override_with_env() { let toml_str = crate_test_settings_str(); @@ -5580,7 +5607,14 @@ origin_host_header_overide = "www.example.com""#, ); assert_eq!(settings.ec.partners[0].openrtb_atype, 571187); assert!(settings.ec.partners[0].bidstream_enabled); - assert_eq!(settings.ec.partners[0].api_token.expose(), "env-token-0"); + assert_eq!( + settings.ec.partners[0] + .api_token + .as_ref() + .map(Redacted::expose) + .map(String::as_str), + Some("env-token-0") + ); assert_eq!(settings.ec.partners[1].name, "Env Partner 1"); assert_eq!( settings.ec.partners[1].source_domain, @@ -5588,7 +5622,14 @@ origin_host_header_overide = "www.example.com""#, ); assert_eq!(settings.ec.partners[1].openrtb_atype, 3); assert!(!settings.ec.partners[1].bidstream_enabled); - assert_eq!(settings.ec.partners[1].api_token.expose(), "env-token-1"); + assert_eq!( + settings.ec.partners[1] + .api_token + .as_ref() + .map(Redacted::expose) + .map(String::as_str), + Some("env-token-1") + ); }, ); } @@ -5935,7 +5976,7 @@ origin_host_header_overide = "www.example.com""#, } #[test] - fn disabled_invalid_integration_skips_validation() { + fn disabled_integration_can_omit_enabled_required_fields_and_skip_semantic_validation() { let mut settings = create_test_settings(); settings .integrations @@ -5943,21 +5984,26 @@ origin_host_header_overide = "www.example.com""#, "gpt", &json!({ "enabled": false, - "script_url": "not a url", }), ) .expect("should insert GPT config"); let config = settings .integration_config::("gpt") - .expect("disabled GPT config should be ignored"); + .expect("minimal disabled GPT config should be ignored"); assert!(config.is_none(), "disabled GPT config should be skipped"); - IntegrationRegistry::new(&settings) - .expect("disabled invalid integration config should not fail registry startup"); + IntegrationRegistry::with_plan( + &settings, + Arc::new( + crate::auction::compile_auction_plan(&settings) + .expect("should compile auction plan"), + ), + ) + .expect("disabled invalid integration config should not fail registry startup"); } #[test] - fn disabled_invalid_default_enabled_prebid_skips_validation() { + fn minimal_disabled_prebid_deserializes_without_enabled_only_validation() { let mut settings = create_test_settings(); settings .integrations @@ -5965,7 +6011,6 @@ origin_host_header_overide = "www.example.com""#, "prebid", &json!({ "enabled": false, - "server_url": "not a url", }), ) .expect("should insert prebid config"); @@ -5974,10 +6019,47 @@ origin_host_header_overide = "www.example.com""#, .integration_config::("prebid") .expect("disabled prebid config should be ignored"); assert!(config.is_none(), "disabled prebid config should be skipped"); - IntegrationRegistry::new(&settings) - .expect("disabled default-enabled prebid config should not fail registry startup"); + IntegrationRegistry::with_plan( + &settings, + Arc::new( + crate::auction::compile_auction_plan(&settings) + .expect("should compile auction plan"), + ), + ) + .expect("disabled default-enabled prebid config should not fail registry startup"); build_orchestrator(&settings) - .expect("disabled default-enabled prebid config should not fail orchestrator startup"); + .expect("minimal disabled prebid config should not fail orchestrator startup"); + } + + #[test] + fn disabled_removed_prebid_and_aps_fields_are_rejected() { + for (integration_id, removed_field) in [("prebid", "server_url"), ("aps", "account_id")] { + let mut settings = create_test_settings(); + settings + .integrations + .insert_config( + integration_id, + &json!({ + "enabled": false, + (removed_field): "removed-value", + }), + ) + .expect("should insert removed integration config field"); + + let error = match integration_id { + "prebid" => settings + .integration_config::(integration_id) + .expect_err("should reject removed disabled Prebid field"), + "aps" => settings + .integration_config::(integration_id) + .expect_err("should reject removed disabled APS field"), + _ => unreachable!("test integration ID should be known"), + }; + assert!( + format!("{error:?}").contains(removed_field), + "should identify removed field `{removed_field}`: {error:?}" + ); + } } #[test] @@ -5994,7 +6076,13 @@ origin_host_header_overide = "www.example.com""#, ) .expect("should insert GPT config"); - let err = match IntegrationRegistry::new(&settings) { + let err = match IntegrationRegistry::with_plan( + &settings, + Arc::new( + crate::auction::compile_auction_plan(&settings) + .expect("should compile auction plan"), + ), + ) { Ok(_) => panic!("enabled invalid integration should fail registry startup"), Err(err) => err, }; @@ -6004,72 +6092,6 @@ origin_host_header_overide = "www.example.com""#, ); } - #[test] - fn disabled_invalid_provider_config_does_not_fail_orchestrator_startup() { - let mut settings = create_test_settings(); - settings - .integrations - .insert_config( - "adserver_mock", - &json!({ - "enabled": false, - "endpoint": "not a url", - }), - ) - .expect("should insert adserver mock config"); - - build_orchestrator(&settings).expect("disabled invalid provider config should be ignored"); - } - - #[test] - fn enabled_invalid_provider_config_fails_orchestrator_startup() { - let mut settings = create_test_settings(); - settings - .integrations - .insert_config( - "adserver_mock", - &json!({ - "enabled": true, - "endpoint": "not a url", - }), - ) - .expect("should insert adserver mock config"); - - let err = match build_orchestrator(&settings) { - Ok(_) => panic!("enabled invalid provider config should fail startup"), - Err(err) => err, - }; - assert!( - err.to_string().contains("Integration 'adserver_mock'"), - "should identify the invalid provider config" - ); - } - - #[test] - fn empty_prebid_server_url_fails_orchestrator_startup() { - let mut settings = create_test_settings(); - settings - .integrations - .insert_config( - "prebid", - &json!({ - "enabled": true, - "server_url": "", - }), - ) - .expect("should insert prebid config"); - - let err = match build_orchestrator(&settings) { - Ok(_) => panic!("empty prebid server_url should fail startup"), - Err(err) => err, - }; - assert!( - err.to_string() - .contains("Integration 'prebid' configuration failed validation"), - "should surface a validation error for prebid.server_url" - ); - } - /// Verifies that `from_toml` does NOT read environment variables. /// The runtime path should only use the pre-built TOML. #[test] @@ -6126,7 +6148,6 @@ origin_host_header_overide = "www.example.com""#, + r#" [auction] enabled = true - providers = [] "#; let settings = Settings::from_toml(&toml_str).expect("should parse valid TOML"); @@ -6147,7 +6168,6 @@ origin_host_header_overide = "www.example.com""#, + r#" [auction] enabled = true - providers = [] rewrite_creatives = false "#; @@ -6174,7 +6194,6 @@ origin_host_header_overide = "www.example.com""#, + r#" [auction] enabled = true - providers = [] allowed_context_keys = ["permutive_segments", "lockr_ids"] "#; let settings = Settings::from_toml(&toml_str).expect("should parse valid TOML"); @@ -6190,7 +6209,6 @@ origin_host_header_overide = "www.example.com""#, + r#" [auction] enabled = true - providers = [] allowed_context_keys = [] "#; let settings = Settings::from_toml(&toml_str).expect("should parse valid TOML"); @@ -6410,9 +6428,9 @@ origin_host_header_overide = "www.example.com""#, match route.auth.as_ref().expect("should configure route auth") { AssetOriginAuth::S3SigV4(config) => { assert_eq!(config.region, "us-east-1"); - assert_eq!(config.secret_store, "s3-auth"); - assert_eq!(config.access_key_id, "access_key_id"); - assert_eq!(config.secret_access_key, "secret_access_key"); + assert_eq!(config.secret_store, None); + assert_eq!(config.access_key_id.expose(), "access_key_id"); + assert_eq!(config.secret_access_key.expose(), "secret_access_key"); } } } @@ -7431,7 +7449,7 @@ formats = [{ width = 300, height = 250 }] } #[test] - fn legacy_settings_loader_applies_creative_opportunity_enabled_environment_override() { + fn settings_creative_opportunity_enabled_flag_supports_environment_override() { let toml = format!( "{}\n[creative_opportunities]\nenabled = true\ngam_network_id = \"21765378893\"\n", crate_test_settings_str() @@ -7451,7 +7469,7 @@ formats = [{ width = 300, height = 250 }] .creative_opportunities .expect("should have creative opportunities") .enabled, - "legacy settings loader should disable template delivery" + "environment override should disable template delivery" ); }); } diff --git a/crates/trusted-server-core/src/settings_data.rs b/crates/trusted-server-core/src/settings_data.rs index 06ea548fc..b82ec92d4 100644 --- a/crates/trusted-server-core/src/settings_data.rs +++ b/crates/trusted-server-core/src/settings_data.rs @@ -3,12 +3,14 @@ use error_stack::{Report, ResultExt}; use serde::Deserialize; use sha2::{Digest as _, Sha256}; +use crate::config_payload::DEFAULT_SECRET_STORE_ID; use crate::config_payload::settings_from_config_blob; use crate::error::TrustedServerError; -use crate::platform::{PlatformConfigStore, StoreName}; +use crate::platform::{PlatformConfigStore, PlatformSecretStore, StoreName}; use crate::settings::Settings; -const DEFAULT_CONFIG_STORE_ID: &str = "trusted_server_config"; +/// Canonical logical config store used by Trusted Server app config. +pub const DEFAULT_CONFIG_STORE_ID: &str = "trusted_server_config"; const FASTLY_CHUNK_POINTER_KIND: &str = "fastly_config_chunks"; const FASTLY_CONFIG_ENTRY_LIMIT: usize = 8_000; @@ -40,21 +42,29 @@ pub fn default_config_key() -> String { EnvConfig::from_env().store_key("config", DEFAULT_CONFIG_STORE_ID) } +/// Returns the default `EdgeZero` secret-store name for Trusted Server secrets. +#[must_use] +pub fn default_secret_store_name() -> StoreName { + StoreName::from(EnvConfig::from_env().store_name("secrets", DEFAULT_SECRET_STORE_ID)) +} + /// Loads [`Settings`] from a platform config store and key. /// /// # Errors /// /// Returns [`TrustedServerError::Configuration`] when the config blob is -/// missing, cannot be read, fails envelope verification, or fails Trusted -/// Server settings validation. +/// missing, cannot be read, fails envelope verification, secret resolution, +/// or Trusted Server settings validation. pub fn get_settings_from_config_store( config_store: &dyn PlatformConfigStore, + secret_store: &dyn PlatformSecretStore, store_name: &StoreName, key: &str, + default_secret_store_name: &StoreName, ) -> Result> { let raw_value = read_config_entry(config_store, store_name, key)?; let envelope_json = resolve_fastly_chunk_pointer(config_store, store_name, &raw_value)?; - settings_from_config_blob(&envelope_json) + settings_from_config_blob(&envelope_json, secret_store, default_secret_store_name) } fn read_config_entry( @@ -177,7 +187,7 @@ fn configuration_error(message: String) -> Result Result<(), Report> { Ok(()) } - fn delete( + fn delete(&self, _store_id: &StoreId, _key: &str) -> Result<(), Report> { + Ok(()) + } + } + + struct EchoSecretStore; + + impl PlatformSecretStore for EchoSecretStore { + fn get_bytes( &self, - _store_id: &crate::platform::StoreId, - _key: &str, + _store_name: &StoreName, + key: &str, + ) -> Result, Report> { + let value = match key { + "unit-test-proxy-secret" => "unit-test-proxy-secret-32-bytes-ok", + _ => key, + }; + Ok(value.as_bytes().to_vec()) + } + + fn create( + &self, + _store_id: &StoreId, + _name: &str, + _value: &str, ) -> Result<(), Report> { Ok(()) } + + fn delete(&self, _store_id: &StoreId, _name: &str) -> Result<(), Report> { + Ok(()) + } } fn envelope_json(settings: &Settings) -> String { @@ -219,18 +254,32 @@ mod tests { serde_json::to_string(&envelope).expect("should serialize envelope") } + fn load_settings( + config_store: &dyn PlatformConfigStore, + store_name: &StoreName, + key: &str, + ) -> Result> { + get_settings_from_config_store( + config_store, + &EchoSecretStore, + store_name, + key, + &StoreName::from("trusted_server_secrets"), + ) + } + #[test] fn loads_settings_from_config_blob_entry() { - let settings = + let mut settings = Settings::from_toml(&crate_test_settings_str()).expect("should parse test settings"); + settings.proxy.allowed_domains = vec!["*.example".to_owned(), "*.example.com".to_owned()]; let envelope_json = envelope_json(&settings); let store = MemoryConfigStore { entries: BTreeMap::from([(CONFIG_BLOB_KEY.to_string(), envelope_json)]), }; - let loaded = - get_settings_from_config_store(&store, &StoreName::from("app_config"), CONFIG_BLOB_KEY) - .expect("should load settings"); + let loaded = load_settings(&store, &StoreName::from("app_config"), CONFIG_BLOB_KEY) + .expect("should load settings"); assert_eq!( loaded.publisher.domain, settings.publisher.domain, @@ -240,8 +289,9 @@ mod tests { #[test] fn loads_settings_from_fastly_chunk_pointer() { - let settings = + let mut settings = Settings::from_toml(&crate_test_settings_str()).expect("should parse test settings"); + settings.proxy.allowed_domains = vec!["*.example".to_owned(), "*.example.com".to_owned()]; let envelope_json = envelope_json(&settings); let midpoint = envelope_json.len() / 2; let first_chunk = envelope_json[..midpoint].to_string(); @@ -275,9 +325,8 @@ mod tests { ]), }; - let loaded = - get_settings_from_config_store(&store, &StoreName::from("app_config"), CONFIG_BLOB_KEY) - .expect("should load settings"); + let loaded = load_settings(&store, &StoreName::from("app_config"), CONFIG_BLOB_KEY) + .expect("should load settings"); assert_eq!( loaded.publisher.domain, settings.publisher.domain, @@ -306,9 +355,8 @@ mod tests { entries: BTreeMap::from([(CONFIG_BLOB_KEY.to_string(), pointer)]), }; - let err = - get_settings_from_config_store(&store, &StoreName::from("app_config"), CONFIG_BLOB_KEY) - .expect_err("should reject malformed chunk length metadata"); + let err = load_settings(&store, &StoreName::from("app_config"), CONFIG_BLOB_KEY) + .expect_err("should reject malformed chunk length metadata"); assert!( err.to_string().contains("chunk lengths total mismatch"), @@ -322,9 +370,8 @@ mod tests { entries: BTreeMap::new(), }; - let err = - get_settings_from_config_store(&store, &StoreName::from("app_config"), CONFIG_BLOB_KEY) - .expect_err("should fail when blob is missing"); + let err = load_settings(&store, &StoreName::from("app_config"), CONFIG_BLOB_KEY) + .expect_err("should fail when blob is missing"); assert!( err.to_string().contains(CONFIG_BLOB_KEY), diff --git a/crates/trusted-server-core/src/test_support.rs b/crates/trusted-server-core/src/test_support.rs index 5f094c0d2..89f73534d 100644 --- a/crates/trusted-server-core/src/test_support.rs +++ b/crates/trusted-server-core/src/test_support.rs @@ -23,9 +23,11 @@ pub mod tests { [integrations.prebid] enabled = true - server_url = "https://test-prebid.com/openrtb2/auction" external_bundle_url = "https://assets.example/prebid/trusted-prebid.js" + [integrations.prebid.bundle] + adapters = ["exampleBidder"] + [integrations.nextjs] enabled = false rewrite_attributes = ["href", "link", "url"] diff --git a/crates/trusted-server-integration-tests/Cargo.toml b/crates/trusted-server-integration-tests/Cargo.toml index f2319fec8..7477fdbd1 100644 --- a/crates/trusted-server-integration-tests/Cargo.toml +++ b/crates/trusted-server-integration-tests/Cargo.toml @@ -23,6 +23,7 @@ workspace = true [dependencies] edgezero-core = { workspace = true } serde_json = { workspace = true } +toml = { workspace = true } trusted-server-core = { workspace = true } [dev-dependencies] @@ -40,7 +41,6 @@ reqwest = { workspace = true, features = ["blocking", "cookies"] } scraper = { workspace = true } testcontainers = { workspace = true } tokio = { workspace = true, features = ["rt-multi-thread"] } -toml = { workspace = true } tower = { workspace = true, features = ["util"] } trusted-server-adapter-axum = { path = "../trusted-server-adapter-axum" } trusted-server-adapter-cloudflare = { path = "../trusted-server-adapter-cloudflare" } diff --git a/crates/trusted-server-integration-tests/browser/package-lock.json b/crates/trusted-server-integration-tests/browser/package-lock.json index 39b512a1d..00f5a6d07 100644 --- a/crates/trusted-server-integration-tests/browser/package-lock.json +++ b/crates/trusted-server-integration-tests/browser/package-lock.json @@ -8,7 +8,18 @@ "name": "integration-tests-browser", "version": "1.0.0", "devDependencies": { - "@playwright/test": "^1.49.0" + "@playwright/test": "^1.49.0", + "prebid-universal-creative": "1.17.2" + } + }, + "node_modules/@gulpjs/messages": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/@gulpjs/messages/-/messages-1.1.0.tgz", + "integrity": "sha512-Ys9sazDatyTgZVb4xPlDufLweJ/Os2uHWOv+Caxvy2O85JcnT4M3vc73bi8pdLWlv3fdWQz3pdI9tVwo8rQQSg==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=10.13.0" } }, "node_modules/@playwright/test": { @@ -27,6 +38,308 @@ "node": ">=18" } }, + "node_modules/ansi-regex": { + "version": "5.0.1", + "resolved": "https://registry.npmjs.org/ansi-regex/-/ansi-regex-5.0.1.tgz", + "integrity": "sha512-quJQXlTSUGL2LH9SUXo8VwsY4soanhgo6LNSm84E1LBcE8s3O0wpdiRzyR9z/ZZJMlMWv37qOOb9pdJlMUEKFQ==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=8" + } + }, + "node_modules/ansi-styles": { + "version": "4.3.0", + "resolved": "https://registry.npmjs.org/ansi-styles/-/ansi-styles-4.3.0.tgz", + "integrity": "sha512-zbB9rCJAT1rbjiVDb2hqKFHNYLxgtk8NURxZ3IZwD3F6NtxbXZQCnnSi1Lkx+IDohdPlFp222wVALIheZJQSEg==", + "dev": true, + "license": "MIT", + "dependencies": { + "color-convert": "^2.0.1" + }, + "engines": { + "node": ">=8" + }, + "funding": { + "url": "https://github.com/chalk/ansi-styles?sponsor=1" + } + }, + "node_modules/array-each": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/array-each/-/array-each-1.0.1.tgz", + "integrity": "sha512-zHjL5SZa68hkKHBFBK6DJCTtr9sfTCPCaph/L7tMSLcTFgy+zX7E+6q5UArbtOtMBCtxdICpfTCspRse+ywyXA==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/array-slice": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/array-slice/-/array-slice-1.1.0.tgz", + "integrity": "sha512-B1qMD3RBP7O8o0H2KbrXDyB0IccejMF15+87Lvlor12ONPRHP6gTjXMNkt/d3ZuOGbAe66hFmaCfECI24Ufp6w==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/babel-plugin-transform-object-assign": { + "version": "6.22.0", + "resolved": "https://registry.npmjs.org/babel-plugin-transform-object-assign/-/babel-plugin-transform-object-assign-6.22.0.tgz", + "integrity": "sha512-N6Pddn/0vgLjnGr+mS7ttlFkQthqcnINE9EMOxB0CF8F4t6kuJXz6NUeLfSoRbLmkGh0mgDs9i2isdaZj0Ghtg==", + "dev": true, + "license": "MIT", + "dependencies": { + "babel-runtime": "^6.22.0" + } + }, + "node_modules/babel-runtime": { + "version": "6.26.0", + "resolved": "https://registry.npmjs.org/babel-runtime/-/babel-runtime-6.26.0.tgz", + "integrity": "sha512-ITKNuq2wKlW1fJg9sSW52eepoYgZBggvOAHC0u/CYu/qxQ9EVzThCgR69BnSXLHjy2f7SY5zaQ4yt7H9ZVxY2g==", + "dev": true, + "license": "MIT", + "dependencies": { + "core-js": "^2.4.0", + "regenerator-runtime": "^0.11.0" + } + }, + "node_modules/braces": { + "version": "3.0.3", + "resolved": "https://registry.npmjs.org/braces/-/braces-3.0.3.tgz", + "integrity": "sha512-yQbXgO/OSZVD2IsiLlro+7Hf6Q18EJrKSEsdoMzKePKXct3gvD8oLcOQdIzGupr5Fj+EDe8gO/lxc1BzfMpxvA==", + "dev": true, + "license": "MIT", + "dependencies": { + "fill-range": "^7.1.1" + }, + "engines": { + "node": ">=8" + } + }, + "node_modules/chalk": { + "version": "4.1.2", + "resolved": "https://registry.npmjs.org/chalk/-/chalk-4.1.2.tgz", + "integrity": "sha512-oKnbhFyRIXpUuez8iBMmyEa4nbj4IOQyuhc/wy9kY7/WVPcwIO9VA668Pu8RkO7+0G76SLROeyw9CpQ061i4mA==", + "dev": true, + "license": "MIT", + "dependencies": { + "ansi-styles": "^4.1.0", + "supports-color": "^7.1.0" + }, + "engines": { + "node": ">=10" + }, + "funding": { + "url": "https://github.com/chalk/chalk?sponsor=1" + } + }, + "node_modules/cliui": { + "version": "7.0.4", + "resolved": "https://registry.npmjs.org/cliui/-/cliui-7.0.4.tgz", + "integrity": "sha512-OcRE68cOsVMXp1Yvonl/fzkQOyjLSu/8bhPDfQt0e0/Eb283TKP20Fs2MqoPsr9SwA595rRCA+QMzYc9nBP+JQ==", + "dev": true, + "license": "ISC", + "dependencies": { + "string-width": "^4.2.0", + "strip-ansi": "^6.0.0", + "wrap-ansi": "^7.0.0" + } + }, + "node_modules/color-convert": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/color-convert/-/color-convert-2.0.1.tgz", + "integrity": "sha512-RRECPsj7iu/xb5oKYcsFHSppFNnsj/52OVTRKb4zP5onXwVF3zVmmToNcOfGC+CRDpfK/U584fMg38ZHCaElKQ==", + "dev": true, + "license": "MIT", + "dependencies": { + "color-name": "~1.1.4" + }, + "engines": { + "node": ">=7.0.0" + } + }, + "node_modules/color-name": { + "version": "1.1.4", + "resolved": "https://registry.npmjs.org/color-name/-/color-name-1.1.4.tgz", + "integrity": "sha512-dOy+3AuW3a2wNbZHIuMZpTcgjGuLU/uBL/ubcZF9OXbDo8ff4O8yVp5Bf0efS8uEoYo5q4Fx7dY9OgQGXgAsQA==", + "dev": true, + "license": "MIT" + }, + "node_modules/copy-props": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/copy-props/-/copy-props-4.0.0.tgz", + "integrity": "sha512-bVWtw1wQLzzKiYROtvNlbJgxgBYt2bMJpkCbKmXM3xyijvcjjWXEk5nyrrT3bgJ7ODb19ZohE2T0Y3FgNPyoTw==", + "dev": true, + "license": "MIT", + "dependencies": { + "each-props": "^3.0.0", + "is-plain-object": "^5.0.0" + }, + "engines": { + "node": ">= 10.13.0" + } + }, + "node_modules/core-js": { + "version": "2.6.12", + "resolved": "https://registry.npmjs.org/core-js/-/core-js-2.6.12.tgz", + "integrity": "sha512-Kb2wC0fvsWfQrgk8HU5lW6U/Lcs8+9aaYcy4ZFc6DDlo4nZ7n70dEgE5rtR0oG6ufKDUnrwfWL1mXR5ljDatrQ==", + "deprecated": "core-js@<3.23.3 is no longer maintained and not recommended for usage due to the number of issues. Because of the V8 engine whims, feature detection in old core-js versions could cause a slowdown up to 100x even if nothing is polyfilled. Some versions have web compatibility issues. Please, upgrade your dependencies to the actual version of core-js.", + "dev": true, + "hasInstallScript": true, + "license": "MIT" + }, + "node_modules/detect-file": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/detect-file/-/detect-file-1.0.0.tgz", + "integrity": "sha512-DtCOLG98P007x7wiiOmfI0fi3eIKyWiLTGJ2MDnVi/E04lWGbf+JzrRHMm0rgIIZJGtHpKpbVgLWHrv8xXpc3Q==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/each-props": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/each-props/-/each-props-3.0.0.tgz", + "integrity": "sha512-IYf1hpuWrdzse/s/YJOrFmU15lyhSzxelNVAHTEG3DtP4QsLTWZUzcUL3HMXmKQxXpa4EIrBPpwRgj0aehdvAw==", + "dev": true, + "license": "MIT", + "dependencies": { + "is-plain-object": "^5.0.0", + "object.defaults": "^1.1.0" + }, + "engines": { + "node": ">= 10.13.0" + } + }, + "node_modules/emoji-regex": { + "version": "8.0.0", + "resolved": "https://registry.npmjs.org/emoji-regex/-/emoji-regex-8.0.0.tgz", + "integrity": "sha512-MSjYzcWNOA0ewAHpz0MxpYFvwg6yjy1NG3xteoqz644VCo/RPgnr1/GGt+ic3iJTzQ8Eu3TdM14SawnVUmGE6A==", + "dev": true, + "license": "MIT" + }, + "node_modules/es-errors": { + "version": "1.3.0", + "resolved": "https://registry.npmjs.org/es-errors/-/es-errors-1.3.0.tgz", + "integrity": "sha512-Zf5H2Kxt2xjTvbJvP2ZWLEICxA6j+hAmMzIlypy4xcBg1vKVnx89Wy0GbS+kf5cwCVFFzdCFh2XSCFNULS6csw==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">= 0.4" + } + }, + "node_modules/escalade": { + "version": "3.2.0", + "resolved": "https://registry.npmjs.org/escalade/-/escalade-3.2.0.tgz", + "integrity": "sha512-WUj2qlxaQtO4g6Pq5c29GTcWGDyd8itL8zTlipgECz3JesAiiOKotd8JU6otB3PACgG6xkJUyVhboMS+bje/jA==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=6" + } + }, + "node_modules/expand-tilde": { + "version": "2.0.2", + "resolved": "https://registry.npmjs.org/expand-tilde/-/expand-tilde-2.0.2.tgz", + "integrity": "sha512-A5EmesHW6rfnZ9ysHQjPdJRni0SRar0tjtG5MNtm9n5TUvsYU8oozprtRD4AqHxcZWWlVuAmQo2nWKfN9oyjTw==", + "dev": true, + "license": "MIT", + "dependencies": { + "homedir-polyfill": "^1.0.1" + }, + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/extend": { + "version": "3.0.2", + "resolved": "https://registry.npmjs.org/extend/-/extend-3.0.2.tgz", + "integrity": "sha512-fjquC59cD7CyW6urNXK0FBufkZcoiGG80wTuPujX590cB5Ttln20E2UB4S/WARVqhXffZl2LNgS+gQdPIIim/g==", + "dev": true, + "license": "MIT" + }, + "node_modules/fill-range": { + "version": "7.1.1", + "resolved": "https://registry.npmjs.org/fill-range/-/fill-range-7.1.1.tgz", + "integrity": "sha512-YsGpe3WHLK8ZYi4tWDg2Jy3ebRz2rXowDxnld4bkQB00cc/1Zw9AWnC0i9ztDJitivtQvaI9KaLyKrc+hBW0yg==", + "dev": true, + "license": "MIT", + "dependencies": { + "to-regex-range": "^5.0.1" + }, + "engines": { + "node": ">=8" + } + }, + "node_modules/findup-sync": { + "version": "5.0.0", + "resolved": "https://registry.npmjs.org/findup-sync/-/findup-sync-5.0.0.tgz", + "integrity": "sha512-MzwXju70AuyflbgeOhzvQWAvvQdo1XL0A9bVvlXsYcFEBM87WR4OakL4OfZq+QRmr+duJubio+UtNQCPsVESzQ==", + "dev": true, + "license": "MIT", + "dependencies": { + "detect-file": "^1.0.0", + "is-glob": "^4.0.3", + "micromatch": "^4.0.4", + "resolve-dir": "^1.0.1" + }, + "engines": { + "node": ">= 10.13.0" + } + }, + "node_modules/fined": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/fined/-/fined-2.0.0.tgz", + "integrity": "sha512-OFRzsL6ZMHz5s0JrsEr+TpdGNCtrVtnuG3x1yzGNiQHT0yaDnXAj8V/lWcpJVrnoDpcwXcASxAZYbuXda2Y82A==", + "dev": true, + "license": "MIT", + "dependencies": { + "expand-tilde": "^2.0.2", + "is-plain-object": "^5.0.0", + "object.defaults": "^1.1.0", + "object.pick": "^1.3.0", + "parse-filepath": "^1.0.2" + }, + "engines": { + "node": ">= 10.13.0" + } + }, + "node_modules/flagged-respawn": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/flagged-respawn/-/flagged-respawn-2.0.0.tgz", + "integrity": "sha512-Gq/a6YCi8zexmGHMuJwahTGzXlAZAOsbCVKduWXC6TlLCjjFRlExMJc4GC2NYPYZ0r/brw9P7CpRgQmlPVeOoA==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">= 10.13.0" + } + }, + "node_modules/for-in": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/for-in/-/for-in-1.0.2.tgz", + "integrity": "sha512-7EwmXrOjyL+ChxMhmG5lnW9MPt1aIeZEwKhQzoBUdTV0N3zuwWDZYVJatDvZ2OyzPUvdIAZDsCetk3coyMfcnQ==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/for-own": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/for-own/-/for-own-1.0.0.tgz", + "integrity": "sha512-0OABksIGrxKK8K4kynWkQ7y1zounQxP+CWnyclVwj81KW3vlLlGUx57DKGcP/LH216GzqnstnPocF16Nxs0Ycg==", + "dev": true, + "license": "MIT", + "dependencies": { + "for-in": "^1.0.1" + }, + "engines": { + "node": ">=0.10.0" + } + }, "node_modules/fsevents": { "version": "2.3.2", "resolved": "https://registry.npmjs.org/fsevents/-/fsevents-2.3.2.tgz", @@ -42,6 +355,450 @@ "node": "^8.16.0 || ^10.6.0 || >=11.0.0" } }, + "node_modules/function-bind": { + "version": "1.1.2", + "resolved": "https://registry.npmjs.org/function-bind/-/function-bind-1.1.2.tgz", + "integrity": "sha512-7XHNxH7qX9xG5mIwxkhumTox/MIRNcOgDrxWsMt2pAr23WHp6MrRlN7FBSFpCpr+oVO0F744iUgR82nJMfG2SA==", + "dev": true, + "license": "MIT", + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, + "node_modules/get-caller-file": { + "version": "2.0.5", + "resolved": "https://registry.npmjs.org/get-caller-file/-/get-caller-file-2.0.5.tgz", + "integrity": "sha512-DyFP3BM/3YHTQOCUL/w0OZHR0lpKeGrxotcHWcqNEdnltqFwXVfhEBQ94eIo34AfQpo0rGki4cyIiftY06h2Fg==", + "dev": true, + "license": "ISC", + "engines": { + "node": "6.* || 8.* || >= 10.*" + } + }, + "node_modules/global-modules": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/global-modules/-/global-modules-1.0.0.tgz", + "integrity": "sha512-sKzpEkf11GpOFuw0Zzjzmt4B4UZwjOcG757PPvrfhxcLFbq0wpsgpOqxpxtxFiCG4DtG93M6XRVbF2oGdev7bg==", + "dev": true, + "license": "MIT", + "dependencies": { + "global-prefix": "^1.0.1", + "is-windows": "^1.0.1", + "resolve-dir": "^1.0.0" + }, + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/global-prefix": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/global-prefix/-/global-prefix-1.0.2.tgz", + "integrity": "sha512-5lsx1NUDHtSjfg0eHlmYvZKv8/nVqX4ckFbM+FrGcQ+04KWcWFo9P5MxPZYSzUvyzmdTbI7Eix8Q4IbELDqzKg==", + "dev": true, + "license": "MIT", + "dependencies": { + "expand-tilde": "^2.0.2", + "homedir-polyfill": "^1.0.1", + "ini": "^1.3.4", + "is-windows": "^1.0.1", + "which": "^1.2.14" + }, + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/glogg": { + "version": "2.2.0", + "resolved": "https://registry.npmjs.org/glogg/-/glogg-2.2.0.tgz", + "integrity": "sha512-eWv1ds/zAlz+M1ioHsyKJomfY7jbDDPpwSkv14KQj89bycx1nvK5/2Cj/T9g7kzJcX5Bc7Yv22FjfBZS/jl94A==", + "dev": true, + "license": "MIT", + "dependencies": { + "sparkles": "^2.1.0" + }, + "engines": { + "node": ">= 10.13.0" + } + }, + "node_modules/gulp-cli": { + "version": "3.1.0", + "resolved": "https://registry.npmjs.org/gulp-cli/-/gulp-cli-3.1.0.tgz", + "integrity": "sha512-zZzwlmEsTfXcxRKiCHsdyjZZnFvXWM4v1NqBJSYbuApkvVKivjcmOS2qruAJ+PkEHLFavcDKH40DPc1+t12a9Q==", + "dev": true, + "license": "MIT", + "dependencies": { + "@gulpjs/messages": "^1.1.0", + "chalk": "^4.1.2", + "copy-props": "^4.0.0", + "gulplog": "^2.2.0", + "interpret": "^3.1.1", + "liftoff": "^5.0.1", + "mute-stdout": "^2.0.0", + "replace-homedir": "^2.0.0", + "semver-greatest-satisfied-range": "^2.0.0", + "string-width": "^4.2.3", + "v8flags": "^4.0.0", + "yargs": "^16.2.0" + }, + "bin": { + "gulp": "bin/gulp.js" + }, + "engines": { + "node": ">=10.13.0" + } + }, + "node_modules/gulplog": { + "version": "2.2.0", + "resolved": "https://registry.npmjs.org/gulplog/-/gulplog-2.2.0.tgz", + "integrity": "sha512-V2FaKiOhpR3DRXZuYdRLn/qiY0yI5XmqbTKrYbdemJ+xOh2d2MOweI/XFgMzd/9+1twdvMwllnZbWZNJ+BOm4A==", + "dev": true, + "license": "MIT", + "dependencies": { + "glogg": "^2.2.0" + }, + "engines": { + "node": ">= 10.13.0" + } + }, + "node_modules/has-flag": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/has-flag/-/has-flag-4.0.0.tgz", + "integrity": "sha512-EykJT/Q1KjTWctppgIAgfSO0tKVuZUjhgMr17kqTumMl6Afv3EISleU7qZUzoXDFTAHTDC4NOoG/ZxU3EvlMPQ==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=8" + } + }, + "node_modules/hasown": { + "version": "2.0.4", + "resolved": "https://registry.npmjs.org/hasown/-/hasown-2.0.4.tgz", + "integrity": "sha512-T2UbfbBEF32wiepXIsMlTW9+dDYC6wMh/t/vYA4tuOMKqWz/n3vr1NFSxQiyP+zk2mXsoMA/i/7qV6LKut1t1A==", + "dev": true, + "license": "MIT", + "dependencies": { + "function-bind": "^1.1.2" + }, + "engines": { + "node": ">= 0.4" + } + }, + "node_modules/homedir-polyfill": { + "version": "1.0.3", + "resolved": "https://registry.npmjs.org/homedir-polyfill/-/homedir-polyfill-1.0.3.tgz", + "integrity": "sha512-eSmmWE5bZTK2Nou4g0AI3zZ9rswp7GRKoKXS1BLUkvPviOqs4YTN1djQIqrXy9k5gEtdLPy86JjRwsNM9tnDcA==", + "dev": true, + "license": "MIT", + "dependencies": { + "parse-passwd": "^1.0.0" + }, + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/ini": { + "version": "1.3.8", + "resolved": "https://registry.npmjs.org/ini/-/ini-1.3.8.tgz", + "integrity": "sha512-JV/yugV2uzW5iMRSiZAyDtQd+nxtUnjeLt0acNdw98kKLrvuRVyB80tsREOE7yvGVgalhZ6RNXCmEHkUKBKxew==", + "dev": true, + "license": "ISC" + }, + "node_modules/interpret": { + "version": "3.1.1", + "resolved": "https://registry.npmjs.org/interpret/-/interpret-3.1.1.tgz", + "integrity": "sha512-6xwYfHbajpoF0xLW+iwLkhwgvLoZDfjYfoFNu8ftMoXINzwuymNLd9u/KmwtdT2GbR+/Cz66otEGEVVUHX9QLQ==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=10.13.0" + } + }, + "node_modules/is-absolute": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/is-absolute/-/is-absolute-1.0.0.tgz", + "integrity": "sha512-dOWoqflvcydARa360Gvv18DZ/gRuHKi2NU/wU5X1ZFzdYfH29nkiNZsF3mp4OJ3H4yo9Mx8A/uAGNzpzPN3yBA==", + "dev": true, + "license": "MIT", + "dependencies": { + "is-relative": "^1.0.0", + "is-windows": "^1.0.1" + }, + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/is-core-module": { + "version": "2.16.2", + "resolved": "https://registry.npmjs.org/is-core-module/-/is-core-module-2.16.2.tgz", + "integrity": "sha512-evOr8xfXKxE6qSR0hSXL2r3sd7ALj8+7jQEUvPYcm5sgZFdJ+AYzT6yNmJenvIYQBgIGwfwz08sL8zoL7yq2BA==", + "dev": true, + "license": "MIT", + "dependencies": { + "hasown": "^2.0.3" + }, + "engines": { + "node": ">= 0.4" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, + "node_modules/is-extglob": { + "version": "2.1.1", + "resolved": "https://registry.npmjs.org/is-extglob/-/is-extglob-2.1.1.tgz", + "integrity": "sha512-SbKbANkN603Vi4jEZv49LeVJMn4yGwsbzZworEoyEiutsN3nJYdbO36zfhGJ6QEDpOZIFkDtnq5JRxmvl3jsoQ==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/is-fullwidth-code-point": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/is-fullwidth-code-point/-/is-fullwidth-code-point-3.0.0.tgz", + "integrity": "sha512-zymm5+u+sCsSWyD9qNaejV3DFvhCKclKdizYaJUuHA83RLjb7nSuGnddCHGv0hk+KY7BMAlsWeK4Ueg6EV6XQg==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=8" + } + }, + "node_modules/is-glob": { + "version": "4.0.3", + "resolved": "https://registry.npmjs.org/is-glob/-/is-glob-4.0.3.tgz", + "integrity": "sha512-xelSayHH36ZgE7ZWhli7pW34hNbNl8Ojv5KVmkJD4hBdD3th8Tfk9vYasLM+mXWOZhFkgZfxhLSnrwRr4elSSg==", + "dev": true, + "license": "MIT", + "dependencies": { + "is-extglob": "^2.1.1" + }, + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/is-number": { + "version": "7.0.0", + "resolved": "https://registry.npmjs.org/is-number/-/is-number-7.0.0.tgz", + "integrity": "sha512-41Cifkg6e8TylSpdtTpeLVMqvSBEVzTttHvERD741+pnZ8ANv0004MRL43QKPDlK9cGvNp6NZWZUBlbGXYxxng==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=0.12.0" + } + }, + "node_modules/is-plain-object": { + "version": "5.1.0", + "resolved": "https://registry.npmjs.org/is-plain-object/-/is-plain-object-5.1.0.tgz", + "integrity": "sha512-bUi/yjmtKYcRVUtWRGr0UA6xEFh2I6zWUwMrUXB3s7bmYCaZ8a+0ZsTRkrawh/mzlSD1Y0Ph8bp/U+TvBpWDNw==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/is-relative": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/is-relative/-/is-relative-1.0.0.tgz", + "integrity": "sha512-Kw/ReK0iqwKeu0MITLFuj0jbPAmEiOsIwyIXvvbfa6QfmN9pkD1M+8pdk7Rl/dTKbH34/XBFMbgD4iMJhLQbGA==", + "dev": true, + "license": "MIT", + "dependencies": { + "is-unc-path": "^1.0.0" + }, + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/is-unc-path": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/is-unc-path/-/is-unc-path-1.0.0.tgz", + "integrity": "sha512-mrGpVd0fs7WWLfVsStvgF6iEJnbjDFZh9/emhRDcGWTduTfNHd9CHeUwH3gYIjdbwo4On6hunkztwOaAw0yllQ==", + "dev": true, + "license": "MIT", + "dependencies": { + "unc-path-regex": "^0.1.2" + }, + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/is-windows": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/is-windows/-/is-windows-1.0.2.tgz", + "integrity": "sha512-eXK1UInq2bPmjyX6e3VHIzMLobc4J94i4AWn+Hpq3OU5KkrRC96OAcR3PRJ/pGu6m8TRnBHP9dkXQVsT/COVIA==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/isexe": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/isexe/-/isexe-2.0.0.tgz", + "integrity": "sha512-RHxMLp9lnKHGHRng9QFhRCMbYAcVpn69smSGcq3f36xjgVVWThj4qqLbTLlq7Ssj8B+fIQ1EuCEGI2lKsyQeIw==", + "dev": true, + "license": "ISC" + }, + "node_modules/isobject": { + "version": "3.0.1", + "resolved": "https://registry.npmjs.org/isobject/-/isobject-3.0.1.tgz", + "integrity": "sha512-WhB9zCku7EGTj/HQQRz5aUQEUeoQZH2bWcltRErOpymJ4boYE6wL9Tbr23krRPSZ+C5zqNSrSw+Cc7sZZ4b7vg==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/liftoff": { + "version": "5.0.1", + "resolved": "https://registry.npmjs.org/liftoff/-/liftoff-5.0.1.tgz", + "integrity": "sha512-wwLXMbuxSF8gMvubFcFRp56lkFV69twvbU5vDPbaw+Q+/rF8j0HKjGbIdlSi+LuJm9jf7k9PB+nTxnsLMPcv2Q==", + "dev": true, + "license": "MIT", + "dependencies": { + "extend": "^3.0.2", + "findup-sync": "^5.0.0", + "fined": "^2.0.0", + "flagged-respawn": "^2.0.0", + "is-plain-object": "^5.0.0", + "rechoir": "^0.8.0", + "resolve": "^1.20.0" + }, + "engines": { + "node": ">=10.13.0" + } + }, + "node_modules/map-cache": { + "version": "0.2.2", + "resolved": "https://registry.npmjs.org/map-cache/-/map-cache-0.2.2.tgz", + "integrity": "sha512-8y/eV9QQZCiyn1SprXSrCmqJN0yNRATe+PO8ztwqrvrbdRLA3eYJF0yaR0YayLWkMbsQSKWS9N2gPcGEc4UsZg==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/micromatch": { + "version": "4.0.8", + "resolved": "https://registry.npmjs.org/micromatch/-/micromatch-4.0.8.tgz", + "integrity": "sha512-PXwfBhYu0hBCPw8Dn0E+WDYb7af3dSLVWKi3HGv84IdF4TyFoC0ysxFd0Goxw7nSv4T/PzEJQxsYsEiFCKo2BA==", + "dev": true, + "license": "MIT", + "dependencies": { + "braces": "^3.0.3", + "picomatch": "^2.3.1" + }, + "engines": { + "node": ">=8.6" + } + }, + "node_modules/mute-stdout": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/mute-stdout/-/mute-stdout-2.0.0.tgz", + "integrity": "sha512-32GSKM3Wyc8dg/p39lWPKYu8zci9mJFzV1Np9Of0ZEpe6Fhssn/FbI7ywAMd40uX+p3ZKh3T5EeCFv81qS3HmQ==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">= 10.13.0" + } + }, + "node_modules/object.defaults": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/object.defaults/-/object.defaults-1.1.0.tgz", + "integrity": "sha512-c/K0mw/F11k4dEUBMW8naXUuBuhxRCfG7W+yFy8EcijU/rSmazOUd1XAEEe6bC0OuXY4HUKjTJv7xbxIMqdxrA==", + "dev": true, + "license": "MIT", + "dependencies": { + "array-each": "^1.0.1", + "array-slice": "^1.0.0", + "for-own": "^1.0.0", + "isobject": "^3.0.0" + }, + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/object.pick": { + "version": "1.3.0", + "resolved": "https://registry.npmjs.org/object.pick/-/object.pick-1.3.0.tgz", + "integrity": "sha512-tqa/UMy/CCoYmj+H5qc07qvSL9dqcs/WZENZ1JbtWBlATP+iVOe778gE6MSijnyCnORzDuX6hU+LA4SZ09YjFQ==", + "dev": true, + "license": "MIT", + "dependencies": { + "isobject": "^3.0.1" + }, + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/parse-filepath": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/parse-filepath/-/parse-filepath-1.0.2.tgz", + "integrity": "sha512-FwdRXKCohSVeXqwtYonZTXtbGJKrn+HNyWDYVcp5yuJlesTwNH4rsmRZ+GrKAPJ5bLpRxESMeS+Rl0VCHRvB2Q==", + "dev": true, + "license": "MIT", + "dependencies": { + "is-absolute": "^1.0.0", + "map-cache": "^0.2.0", + "path-root": "^0.1.1" + }, + "engines": { + "node": ">=0.8" + } + }, + "node_modules/parse-passwd": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/parse-passwd/-/parse-passwd-1.0.0.tgz", + "integrity": "sha512-1Y1A//QUXEZK7YKz+rD9WydcE1+EuPr6ZBgKecAB8tmoW6UFv0NREVJe1p+jRxtThkcbbKkfwIbWJe/IeE6m2Q==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/path-parse": { + "version": "1.0.7", + "resolved": "https://registry.npmjs.org/path-parse/-/path-parse-1.0.7.tgz", + "integrity": "sha512-LDJzPVEEEPR+y48z93A0Ed0yXb8pAByGWo/k5YYdYgpY2/2EsOsksJrq7lOHxryrVOn1ejG6oAp8ahvOIQD8sw==", + "dev": true, + "license": "MIT" + }, + "node_modules/path-root": { + "version": "0.1.1", + "resolved": "https://registry.npmjs.org/path-root/-/path-root-0.1.1.tgz", + "integrity": "sha512-QLcPegTHF11axjfojBIoDygmS2E3Lf+8+jI6wOVmNVenrKSo3mFdSGiIgdSHenczw3wPtlVMQaFVwGmM7BJdtg==", + "dev": true, + "license": "MIT", + "dependencies": { + "path-root-regex": "^0.1.0" + }, + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/path-root-regex": { + "version": "0.1.2", + "resolved": "https://registry.npmjs.org/path-root-regex/-/path-root-regex-0.1.2.tgz", + "integrity": "sha512-4GlJ6rZDhQZFE0DPVKh0e9jmZ5egZfxTkp7bcRDuPlJXbAwhxcl2dINPUAsjLdejqaLsCeg8axcLjIbvBjN4pQ==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/picomatch": { + "version": "2.3.2", + "resolved": "https://registry.npmjs.org/picomatch/-/picomatch-2.3.2.tgz", + "integrity": "sha512-V7+vQEJ06Z+c5tSye8S+nHUfI51xoXIXjHQ99cQtKUkQqqO1kO/KCJUfZXuB47h/YBlDhah2H3hdUGXn8ie0oA==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=8.6" + }, + "funding": { + "url": "https://github.com/sponsors/jonschlinkert" + } + }, "node_modules/playwright": { "version": "1.58.2", "resolved": "https://registry.npmjs.org/playwright/-/playwright-1.58.2.tgz", @@ -73,6 +830,312 @@ "engines": { "node": ">=18" } + }, + "node_modules/postscribe": { + "version": "2.0.8", + "resolved": "https://registry.npmjs.org/postscribe/-/postscribe-2.0.8.tgz", + "integrity": "sha512-Sxt6pek38NKX85Vb/PbcritqVxsgPZQFLcuf4o0f7lXRb76jM0XP79SGwCBPRTuv+U2zqByQan8EzRjqquD73A==", + "dev": true, + "license": "MIT", + "dependencies": { + "prescribe": ">=1.1.2" + } + }, + "node_modules/prebid-universal-creative": { + "version": "1.17.2", + "resolved": "https://registry.npmjs.org/prebid-universal-creative/-/prebid-universal-creative-1.17.2.tgz", + "integrity": "sha512-+1fB/eD3eXF+m8T0S4GL/wrXatx/tpeTtZ6ptFQnjQxtejiM0GuoFWdJvx5xlqkP1Z14WEE6/eRc1zWcxvg/Dg==", + "dev": true, + "license": "Apache-2.0", + "dependencies": { + "babel-plugin-transform-object-assign": "^6.22.0", + "gulp-cli": "^3.0.0", + "postscribe": "^2.0.8" + } + }, + "node_modules/prescribe": { + "version": "1.1.3", + "resolved": "https://registry.npmjs.org/prescribe/-/prescribe-1.1.3.tgz", + "integrity": "sha512-HEg0ElY5tmmCshST4tzl47+SirJO2cVo6j/+O4d6xIz+80ixNcN0GgPQsn76AgeTTIAQOrwq1rfoptubQuZ1Uw==", + "dev": true, + "license": "MIT" + }, + "node_modules/rechoir": { + "version": "0.8.0", + "resolved": "https://registry.npmjs.org/rechoir/-/rechoir-0.8.0.tgz", + "integrity": "sha512-/vxpCXddiX8NGfGO/mTafwjq4aFa/71pvamip0++IQk3zG8cbCj0fifNPrjjF1XMXUne91jL9OoxmdykoEtifQ==", + "dev": true, + "license": "MIT", + "dependencies": { + "resolve": "^1.20.0" + }, + "engines": { + "node": ">= 10.13.0" + } + }, + "node_modules/regenerator-runtime": { + "version": "0.11.1", + "resolved": "https://registry.npmjs.org/regenerator-runtime/-/regenerator-runtime-0.11.1.tgz", + "integrity": "sha512-MguG95oij0fC3QV3URf4V2SDYGJhJnJGqvIIgdECeODCT98wSWDAJ94SSuVpYQUoTcGUIL6L4yNB7j1DFFHSBg==", + "dev": true, + "license": "MIT" + }, + "node_modules/replace-homedir": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/replace-homedir/-/replace-homedir-2.0.0.tgz", + "integrity": "sha512-bgEuQQ/BHW0XkkJtawzrfzHFSN70f/3cNOiHa2QsYxqrjaC30X1k74FJ6xswVBP0sr0SpGIdVFuPwfrYziVeyw==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">= 10.13.0" + } + }, + "node_modules/require-directory": { + "version": "2.1.1", + "resolved": "https://registry.npmjs.org/require-directory/-/require-directory-2.1.1.tgz", + "integrity": "sha512-fGxEI7+wsG9xrvdjsrlmL22OMTTiHRwAMroiEeMgq8gzoLC/PQr7RsRDSTLUg/bZAZtF+TVIkHc6/4RIKrui+Q==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/resolve": { + "version": "1.22.12", + "resolved": "https://registry.npmjs.org/resolve/-/resolve-1.22.12.tgz", + "integrity": "sha512-TyeJ1zif53BPfHootBGwPRYT1RUt6oGWsaQr8UyZW/eAm9bKoijtvruSDEmZHm92CwS9nj7/fWttqPCgzep8CA==", + "dev": true, + "license": "MIT", + "dependencies": { + "es-errors": "^1.3.0", + "is-core-module": "^2.16.1", + "path-parse": "^1.0.7", + "supports-preserve-symlinks-flag": "^1.0.0" + }, + "bin": { + "resolve": "bin/resolve" + }, + "engines": { + "node": ">= 0.4" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, + "node_modules/resolve-dir": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/resolve-dir/-/resolve-dir-1.0.1.tgz", + "integrity": "sha512-R7uiTjECzvOsWSfdM0QKFNBVFcK27aHOUwdvK53BcW8zqnGdYp0Fbj82cy54+2A4P2tFM22J5kRfe1R+lM/1yg==", + "dev": true, + "license": "MIT", + "dependencies": { + "expand-tilde": "^2.0.0", + "global-modules": "^1.0.0" + }, + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/semver": { + "version": "6.3.1", + "resolved": "https://registry.npmjs.org/semver/-/semver-6.3.1.tgz", + "integrity": "sha512-BR7VvDCVHO+q2xBEWskxS6DJE1qRnb7DxzUrogb71CWoSficBxYsiAGd+Kl0mmq/MprG9yArRkyrQxTO6XjMzA==", + "dev": true, + "license": "ISC", + "optional": true, + "bin": { + "semver": "bin/semver.js" + } + }, + "node_modules/semver-greatest-satisfied-range": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/semver-greatest-satisfied-range/-/semver-greatest-satisfied-range-2.0.0.tgz", + "integrity": "sha512-lH3f6kMbwyANB7HuOWRMlLCa2itaCrZJ+SAqqkSZrZKO/cAsk2EOyaKHUtNkVLFyFW9pct22SFesFp3Z7zpA0g==", + "dev": true, + "license": "MIT", + "dependencies": { + "sver": "^1.8.3" + }, + "engines": { + "node": ">= 10.13.0" + } + }, + "node_modules/sparkles": { + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/sparkles/-/sparkles-2.1.0.tgz", + "integrity": "sha512-r7iW1bDw8R/cFifrD3JnQJX0K1jqT0kprL48BiBpLZLJPmAm34zsVBsK5lc7HirZYZqMW65dOXZgbAGt/I6frg==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">= 10.13.0" + } + }, + "node_modules/string-width": { + "version": "4.2.3", + "resolved": "https://registry.npmjs.org/string-width/-/string-width-4.2.3.tgz", + "integrity": "sha512-wKyQRQpjJ0sIp62ErSZdGsjMJWsap5oRNihHhu6G7JVO/9jIB6UyevL+tXuOqrng8j/cxKTWyWUwvSTriiZz/g==", + "dev": true, + "license": "MIT", + "dependencies": { + "emoji-regex": "^8.0.0", + "is-fullwidth-code-point": "^3.0.0", + "strip-ansi": "^6.0.1" + }, + "engines": { + "node": ">=8" + } + }, + "node_modules/strip-ansi": { + "version": "6.0.1", + "resolved": "https://registry.npmjs.org/strip-ansi/-/strip-ansi-6.0.1.tgz", + "integrity": "sha512-Y38VPSHcqkFrCpFnQ9vuSXmquuv5oXOKpGeT6aGrr3o3Gc9AlVa6JBfUSOCnbxGGZF+/0ooI7KrPuUSztUdU5A==", + "dev": true, + "license": "MIT", + "dependencies": { + "ansi-regex": "^5.0.1" + }, + "engines": { + "node": ">=8" + } + }, + "node_modules/supports-color": { + "version": "7.2.0", + "resolved": "https://registry.npmjs.org/supports-color/-/supports-color-7.2.0.tgz", + "integrity": "sha512-qpCAvRl9stuOHveKsn7HncJRvv501qIacKzQlO/+Lwxc9+0q2wLyv4Dfvt80/DPn2pqOBsJdDiogXGR9+OvwRw==", + "dev": true, + "license": "MIT", + "dependencies": { + "has-flag": "^4.0.0" + }, + "engines": { + "node": ">=8" + } + }, + "node_modules/supports-preserve-symlinks-flag": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/supports-preserve-symlinks-flag/-/supports-preserve-symlinks-flag-1.0.0.tgz", + "integrity": "sha512-ot0WnXS9fgdkgIcePe6RHNk1WA8+muPa6cSjeR3V8K27q9BB1rTE3R1p7Hv0z1ZyAc8s6Vvv8DIyWf681MAt0w==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">= 0.4" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, + "node_modules/sver": { + "version": "1.8.4", + "resolved": "https://registry.npmjs.org/sver/-/sver-1.8.4.tgz", + "integrity": "sha512-71o1zfzyawLfIWBOmw8brleKyvnbn73oVHNCsu51uPMz/HWiKkkXsI31JjHW5zqXEqnPYkIiHd8ZmL7FCimLEA==", + "dev": true, + "license": "MIT", + "optionalDependencies": { + "semver": "^6.3.0" + } + }, + "node_modules/to-regex-range": { + "version": "5.0.1", + "resolved": "https://registry.npmjs.org/to-regex-range/-/to-regex-range-5.0.1.tgz", + "integrity": "sha512-65P7iz6X5yEr1cwcgvQxbbIw7Uk3gOy5dIdtZ4rDveLqhrdJP+Li/Hx6tyK0NEb+2GCyneCMJiGqrADCSNk8sQ==", + "dev": true, + "license": "MIT", + "dependencies": { + "is-number": "^7.0.0" + }, + "engines": { + "node": ">=8.0" + } + }, + "node_modules/unc-path-regex": { + "version": "0.1.2", + "resolved": "https://registry.npmjs.org/unc-path-regex/-/unc-path-regex-0.1.2.tgz", + "integrity": "sha512-eXL4nmJT7oCpkZsHZUOJo8hcX3GbsiDOa0Qu9F646fi8dT3XuSVopVqAcEiVzSKKH7UoDti23wNX3qGFxcW5Qg==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/v8flags": { + "version": "4.0.1", + "resolved": "https://registry.npmjs.org/v8flags/-/v8flags-4.0.1.tgz", + "integrity": "sha512-fcRLaS4H/hrZk9hYwbdRM35D0U8IYMfEClhXxCivOojl+yTRAZH3Zy2sSy6qVCiGbV9YAtPssP6jaChqC9vPCg==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">= 10.13.0" + } + }, + "node_modules/which": { + "version": "1.3.1", + "resolved": "https://registry.npmjs.org/which/-/which-1.3.1.tgz", + "integrity": "sha512-HxJdYWq1MTIQbJ3nw0cqssHoTNU267KlrDuGZ1WYlxDStUtKUhOaJmh112/TZmHxxUfuJqPXSOm7tDyas0OSIQ==", + "dev": true, + "license": "ISC", + "dependencies": { + "isexe": "^2.0.0" + }, + "bin": { + "which": "bin/which" + } + }, + "node_modules/wrap-ansi": { + "version": "7.0.0", + "resolved": "https://registry.npmjs.org/wrap-ansi/-/wrap-ansi-7.0.0.tgz", + "integrity": "sha512-YVGIj2kamLSTxw6NsZjoBxfSwsn0ycdesmc4p+Q21c5zPuZ1pl+NfxVdxPtdHvmNVOQ6XSYG4AUtyt/Fi7D16Q==", + "dev": true, + "license": "MIT", + "dependencies": { + "ansi-styles": "^4.0.0", + "string-width": "^4.1.0", + "strip-ansi": "^6.0.0" + }, + "engines": { + "node": ">=10" + }, + "funding": { + "url": "https://github.com/chalk/wrap-ansi?sponsor=1" + } + }, + "node_modules/y18n": { + "version": "5.0.8", + "resolved": "https://registry.npmjs.org/y18n/-/y18n-5.0.8.tgz", + "integrity": "sha512-0pfFzegeDWJHJIAmTLRP2DwHjdF5s7jo9tuztdQxAhINCdvS+3nGINqPd00AphqJR/0LhANUS6/+7SCb98YOfA==", + "dev": true, + "license": "ISC", + "engines": { + "node": ">=10" + } + }, + "node_modules/yargs": { + "version": "16.2.2", + "resolved": "https://registry.npmjs.org/yargs/-/yargs-16.2.2.tgz", + "integrity": "sha512-Nt9ZJjXTv5R8MHbqby/wXQ6Gi0Bb3TcYZkR1bzuL4yB2OxWPkXknz513gEF0GoA6tn00UpbPvERW8rzCuWCA6w==", + "dev": true, + "license": "MIT", + "dependencies": { + "cliui": "^7.0.2", + "escalade": "^3.1.1", + "get-caller-file": "^2.0.5", + "require-directory": "^2.1.1", + "string-width": "^4.2.0", + "y18n": "^5.0.5", + "yargs-parser": "^20.2.2" + }, + "engines": { + "node": ">=10" + } + }, + "node_modules/yargs-parser": { + "version": "20.2.9", + "resolved": "https://registry.npmjs.org/yargs-parser/-/yargs-parser-20.2.9.tgz", + "integrity": "sha512-y11nGElTIV+CT3Zv9t7VKl+Q3hTQoT9a1Qzezhhl6Rp21gJ/IVTW7Z3y9EWXhuUBC2Shnf+DX0antecpAwSP8w==", + "dev": true, + "license": "ISC", + "engines": { + "node": ">=10" + } } } } diff --git a/crates/trusted-server-integration-tests/browser/package.json b/crates/trusted-server-integration-tests/browser/package.json index 13282f289..42855b5b2 100644 --- a/crates/trusted-server-integration-tests/browser/package.json +++ b/crates/trusted-server-integration-tests/browser/package.json @@ -8,6 +8,7 @@ "test:wordpress": "TEST_FRAMEWORK=wordpress npx playwright test" }, "devDependencies": { - "@playwright/test": "^1.49.0" + "@playwright/test": "^1.49.0", + "prebid-universal-creative": "1.17.2" } } diff --git a/crates/trusted-server-integration-tests/browser/tests/shared/aps-renderer.spec.ts b/crates/trusted-server-integration-tests/browser/tests/shared/aps-renderer.spec.ts index fce505d42..e7ee7c1c7 100644 --- a/crates/trusted-server-integration-tests/browser/tests/shared/aps-renderer.spec.ts +++ b/crates/trusted-server-integration-tests/browser/tests/shared/aps-renderer.spec.ts @@ -10,6 +10,13 @@ const SCRIPT_CREATIVE_URL = "https://creative.example/script.js"; const SANDBOX = "allow-forms allow-pointer-lock allow-popups allow-popups-to-escape-sandbox allow-scripts allow-top-navigation-by-user-activation"; const TSJS_CRATE = resolve(__dirname, "../../../../trusted-server-js"); +const PUC_BANNER = readFileSync( + resolve( + __dirname, + "../../node_modules/prebid-universal-creative/dist/banner.js", + ), + "utf8", +); function clientAuctionBundlePaths() { const manifestPath = resolve(TSJS_CRATE, "dist/prebid/manifest.json"); @@ -197,6 +204,171 @@ const SCRIPT_CREATIVE = `(function(){ })();`; test.describe("APS rendering", () => { + test("renders through real PUC and expands only its authenticated 1x1 shell", async ({ + page, + }) => { + const adId = "fictional-inline-ad-id"; + const publisherOrigin = new URL(runtimeUrl("/")).origin; + const outerCreativeUrl = runtimeUrl("/fictional-puc-shell"); + let creativeRequests = 0; + + await page.route(runtimeUrl("/aps-puc-topology-test"), (route) => + route.fulfill({ + status: 200, + contentType: "text/html", + body: '
', + }), + ); + await page.route(outerCreativeUrl, (route) => + route.fulfill({ + status: 200, + contentType: "text/html", + body: ``, + }), + ); + await page.route(IFRAME_CREATIVE_URL, (route) => { + creativeRequests += 1; + return route.fulfill({ + status: 200, + contentType: "text/html", + body: IFRAME_CREATIVE, + }); + }); + + await page.goto(runtimeUrl("/aps-puc-topology-test")); + await page.addScriptTag({ path: clientAuctionBundlePaths().gpt }); + await page.evaluate( + ({ creativeUrl, outerUrl, selectedAdId }) => { + const typedWindow = window as unknown as { + tsjs: Record; + pucEvents: Array>; + }; + typedWindow.tsjs = { + bids: { + "aps-slot": { + hb_adid: selectedAdId, + hb_bidder: "fictional", + hb_pb: "1.23", + adm: ``, + w: 300, + h: 250, + }, + }, + adSlots: [ + { + id: "aps-slot", + div_id: "div-aps", + gam_unit_path: "/fictional/aps", + formats: [[300, 250]], + }, + ], + }; + typedWindow.pucEvents = []; + const locator = document.createElement("iframe"); + locator.name = "__pb_locator__"; + document.body.appendChild(locator); + window.addEventListener("message", (event) => { + try { + const message = JSON.parse( + String(event.data), + ) as Record; + if (message.message === "Prebid Event") { + typedWindow.pucEvents.push(message); + } + } catch { + // Ignore unrelated publisher messages. + } + }); + + const slot = document.getElementById("div-aps")!; + slot.style.width = "1px"; + slot.style.height = "1px"; + const outerShell = document.createElement("div"); + outerShell.id = "aps-outer-shell"; + outerShell.style.width = "1px"; + outerShell.style.height = "1px"; + const innerShell = document.createElement("div"); + innerShell.id = "aps-inner-shell"; + innerShell.style.width = "1px"; + innerShell.style.height = "1px"; + const frame = document.createElement("iframe"); + frame.id = "google_ads_iframe_fictional_0"; + frame.width = "1"; + frame.height = "1"; + frame.style.width = "1px"; + frame.style.height = "1px"; + frame.src = outerUrl; + innerShell.appendChild(frame); + outerShell.appendChild(innerShell); + slot.appendChild(outerShell); + + const other = document.getElementById("div-other")!; + const otherFrame = document.createElement("iframe"); + otherFrame.width = "1"; + otherFrame.height = "1"; + otherFrame.style.width = "1px"; + otherFrame.style.height = "1px"; + other.appendChild(otherFrame); + }, + { + creativeUrl: IFRAME_CREATIVE_URL, + outerUrl: outerCreativeUrl, + selectedAdId: adId, + }, + ); + + await expect.poll(() => creativeRequests).toBe(1); + await expect + .poll(() => + page.evaluate(() => + ( + window as unknown as { + pucEvents: Array>; + } + ).pucEvents.some( + (event) => event.event === "adRenderSucceeded", + ), + ), + ) + .toBe(true); + await expect(page.locator("#google_ads_iframe_fictional_0")).toHaveCSS( + "width", + "300px", + ); + await expect(page.locator("#google_ads_iframe_fictional_0")).toHaveCSS( + "height", + "250px", + ); + await expect(page.locator("#div-aps")).toHaveCSS("width", "300px"); + await expect(page.locator("#div-aps")).toHaveCSS("height", "250px"); + await expect(page.locator("#aps-outer-shell")).toHaveCSS( + "width", + "300px", + ); + await expect(page.locator("#aps-outer-shell")).toHaveCSS( + "height", + "250px", + ); + await expect(page.locator("#aps-inner-shell")).toHaveCSS( + "width", + "300px", + ); + await expect(page.locator("#aps-inner-shell")).toHaveCSS( + "height", + "250px", + ); + await expect(page.locator("#div-other iframe")).toHaveCSS( + "width", + "1px", + ); + await expect(page.locator("#div-other iframe")).toHaveCSS( + "height", + "1px", + ); + }); + test("renders a trustedServer adapter bid using Prebid's generated GAM ad ID", async ({ page, }) => { diff --git a/crates/trusted-server-integration-tests/fixtures/configs/trusted-server.integration.toml b/crates/trusted-server-integration-tests/fixtures/configs/trusted-server.integration.toml index d8e35d179..dae3f57cc 100644 --- a/crates/trusted-server-integration-tests/fixtures/configs/trusted-server.integration.toml +++ b/crates/trusted-server-integration-tests/fixtures/configs/trusted-server.integration.toml @@ -1,16 +1,16 @@ [[handlers]] path = "^/_ts/admin" username = "admin" -password = "integration-admin-password-32-bytes-ok" +password = "integration_admin_password" [publisher] domain = "localhost" cookie_domain = "localhost" origin_url = "http://127.0.0.1:8888" -proxy_secret = "integration-test-proxy-secret" +proxy_secret = "integration_proxy_secret" [ec] -passphrase = "integration-test-ec-secret-padded-32" +passphrase = "integration_ec_passphrase" ec_store = "ec_identity_store" pull_sync_concurrency = 3 @@ -18,13 +18,13 @@ pull_sync_concurrency = 3 name = "Integration Test Partner" source_domain = "inttest.example.com" bidstream_enabled = true -api_token = "integration-test-token-alpha-32-bytes-ok" +api_token = "integration_partner_token_alpha" [[ec.partners]] name = "Integration Test Partner 2" source_domain = "inttest2.example.com" bidstream_enabled = true -api_token = "integration-test-token-bravo-32-bytes-ok" +api_token = "integration_partner_token_bravo" [request_signing] enabled = false @@ -33,9 +33,7 @@ secret_store_id = "secrets" [integrations.prebid] enabled = false -server_url = "https://prebid.example.com/openrtb2/auction" timeout_ms = 1000 -bidders = [] debug = false client_side_bidders = [] @@ -99,17 +97,38 @@ certificate_check = false [auction] enabled = false -providers = [] timeout_ms = 2000 allowed_context_keys = [] -[integrations.aps] -enabled = true -account_id = "example-aps-account-id" +[auction.providers.pbs-main] +protocol = "openrtb-2.6" +profile = "prebid-server" +endpoint = "https://prebid.example.com/openrtb2/auction" +routing = "explicit" + +[auction.providers.pbs-main.profile_config] +debug = false +test_mode = false +consent_forwarding = "both" + +[auction.providers.pbs-main.notifications] +suppress_all = false +suppress_seats = [] + +[auction.providers.aps-main] +protocol = "openrtb-2.6" +profile = "aps" endpoint = "https://aps.example.com/e/pb/bid" -timeout_ms = 1000 +routing = "all_eligible" + +[auction.providers.aps-main.profile_config] +account_id = "example-aps-account-id" +debug = false allow_script_creatives = false +[auction.bidders.example-bidder] +provider = "pbs-main" + [integrations.google_tag_manager] enabled = false container_id = "GTM-EXAMPLE" diff --git a/crates/trusted-server-integration-tests/fixtures/configs/viceroy-template.toml b/crates/trusted-server-integration-tests/fixtures/configs/viceroy-template.toml index 9f1443d20..f11a67dff 100644 --- a/crates/trusted-server-integration-tests/fixtures/configs/viceroy-template.toml +++ b/crates/trusted-server-integration-tests/fixtures/configs/viceroy-template.toml @@ -66,7 +66,28 @@ key = "api_key" data = "test-api-key" + [[local_server.secret_stores.ts_secrets]] + key = "integration_admin_password" + data = "integration-admin-password-32-bytes-ok" + [[local_server.secret_stores.ts_secrets]] + key = "integration_proxy_secret" + data = "integration-test-proxy-secret-32-bytes-ok" + [[local_server.secret_stores.ts_secrets]] + key = "integration_ec_passphrase" + data = "integration-test-ec-secret-padded-32" + [[local_server.secret_stores.ts_secrets]] + key = "integration_partner_token_alpha" + data = "integration-test-token-alpha-32-bytes-ok" + [[local_server.secret_stores.ts_secrets]] + key = "integration_partner_token_bravo" + data = "integration-test-token-bravo-32-bytes-ok" + [local_server.config_stores] + [local_server.config_stores.edgezero_runtime_env] + format = "inline-toml" + [local_server.config_stores.edgezero_runtime_env.contents] + EDGEZERO__STORES__SECRETS__TRUSTED_SERVER_SECRETS__NAME = "ts_secrets" + # Generated integration configs inject the trusted_server_config blob # into the store required by the Fastly entry point. # GENERATED_TRUSTED_SERVER_CONFIG_STORES diff --git a/crates/trusted-server-integration-tests/src/bin/generate-viceroy-config.rs b/crates/trusted-server-integration-tests/src/bin/generate-viceroy-config.rs index 85b1bcf0f..58c26736e 100644 --- a/crates/trusted-server-integration-tests/src/bin/generate-viceroy-config.rs +++ b/crates/trusted-server-integration-tests/src/bin/generate-viceroy-config.rs @@ -4,7 +4,7 @@ use std::fs; use std::path::PathBuf; use edgezero_core::blob_envelope::BlobEnvelope; -use trusted_server_core::{config::validate_settings_for_deploy, settings::Settings}; +use trusted_server_core::config::TrustedServerAppConfig; const GENERATED_AT: &str = "2026-06-23T00:00:00Z"; const GENERATED_STORES_MARKER: &str = " # GENERATED_TRUSTED_SERVER_CONFIG_STORES"; @@ -114,15 +114,16 @@ fn build_app_config_envelope( app_config_toml: &str, origin_url: Option<&str>, ) -> Result { - let mut settings = Settings::from_toml(app_config_toml) - .map_err(|report| error_box(format!("invalid Trusted Server app config: {report:?}")))?; + let app_config: TrustedServerAppConfig = toml::from_str(app_config_toml) + .map_err(|error| error_box(format!("invalid Trusted Server app config: {error}")))?; + let mut settings = app_config.into_settings(); if let Some(origin_url) = origin_url { settings.publisher.origin_url = origin_url.to_string(); } - validate_settings_for_deploy(&settings) + let app_config = TrustedServerAppConfig::new(settings) .map_err(|report| error_box(format!("invalid Trusted Server app config: {report:?}")))?; - let data = serde_json::to_value(&settings).map_err(|error| { + let data = serde_json::to_value(&app_config).map_err(|error| { error_box(format!( "failed to serialize Trusted Server app config to JSON: {error}" )) @@ -161,11 +162,71 @@ fn error_box(message: impl Into) -> DynError { #[cfg(test)] mod tests { use super::*; + use error_stack::Report; + use std::collections::HashMap; use trusted_server_core::config_payload::settings_from_config_blob; + use trusted_server_core::platform::{PlatformError, PlatformSecretStore, StoreId, StoreName}; const TEMPLATE: &str = include_str!("../../fixtures/configs/viceroy-template.toml"); const APP_CONFIG: &str = include_str!("../../fixtures/configs/trusted-server.integration.toml"); + struct IntegrationSecretStore { + values: HashMap>, + } + + impl PlatformSecretStore for IntegrationSecretStore { + fn get_bytes( + &self, + _store_name: &StoreName, + key: &str, + ) -> Result, Report> { + self.values + .get(key) + .cloned() + .ok_or_else(|| Report::new(PlatformError::SecretStore)) + } + + fn create( + &self, + _store_id: &StoreId, + _name: &str, + _value: &str, + ) -> Result<(), Report> { + Ok(()) + } + + fn delete(&self, _store_id: &StoreId, _name: &str) -> Result<(), Report> { + Ok(()) + } + } + + fn integration_secret_store() -> IntegrationSecretStore { + IntegrationSecretStore { + values: HashMap::from([ + ( + "integration_admin_password".to_owned(), + b"integration-admin-password-32-bytes-ok".to_vec(), + ), + ( + "integration_proxy_secret".to_owned(), + b"integration-test-proxy-secret-32-bytes-ok".to_vec(), + ), + ( + "integration_ec_passphrase".to_owned(), + b"integration-test-ec-secret-padded-32".to_vec(), + ), + ( + "integration_partner_token_alpha".to_owned(), + b"integration-test-token-alpha-32-bytes-ok".to_vec(), + ), + ( + "integration_partner_token_bravo".to_owned(), + b"integration-test-token-bravo-32-bytes-ok".to_vec(), + ), + ]), + } + } + #[test] fn parse_args_does_not_require_removed_rollout_switch() { let result = parse_args([ @@ -253,7 +314,12 @@ mod tests { fn generated_blob_verifies_and_applies_origin_override() { let envelope = build_app_config_envelope(APP_CONFIG, Some("http://127.0.0.1:9999")) .expect("should build envelope"); - let settings = settings_from_config_blob(&envelope).expect("should verify blob"); + let settings = settings_from_config_blob( + &envelope, + &integration_secret_store(), + &StoreName::from("trusted_server_secrets"), + ) + .expect("should verify blob"); assert_eq!( settings.publisher.origin_url, "http://127.0.0.1:9999", @@ -268,6 +334,19 @@ mod tests { assert!(result.is_err(), "should reject invalid app config"); } + #[test] + fn invalid_non_secret_app_config_fails_before_envelope_generation() { + let invalid = APP_CONFIG.replace("domain = \"localhost\"", "domain = \"invalid/domain\""); + + let err = build_app_config_envelope(&invalid, None) + .expect_err("should reject invalid non-secret config before creating an envelope"); + + assert!( + err.to_string().contains("invalid_publisher_domain"), + "error should identify the structural validation failure: {err}" + ); + } + #[test] fn missing_marker_fails() { let result = inject_generated_config_stores("[local_server]", "{}"); diff --git a/crates/trusted-server-integration-tests/tests/common/config.rs b/crates/trusted-server-integration-tests/tests/common/config.rs index 4dc971d0e..037fa4658 100644 --- a/crates/trusted-server-integration-tests/tests/common/config.rs +++ b/crates/trusted-server-integration-tests/tests/common/config.rs @@ -1,7 +1,6 @@ use edgezero_core::blob_envelope::BlobEnvelope; use error_stack::Report; -use trusted_server_core::config::validate_settings_for_deploy; -use trusted_server_core::settings::Settings; +use trusted_server_core::config::TrustedServerAppConfig; use crate::common::runtime::{TestError, TestResult}; @@ -10,18 +9,19 @@ const APP_CONFIG: &str = include_str!("../../fixtures/configs/trusted-server.int pub fn integration_app_config_envelope(origin_port: u16) -> TestResult { let origin_url = format!("http://127.0.0.1:{origin_port}"); - let mut settings = Settings::from_toml(APP_CONFIG).map_err(|report| { + let app_config: TrustedServerAppConfig = toml::from_str(APP_CONFIG).map_err(|error| { Report::new(TestError::ConfigGeneration).attach(format!( - "invalid Trusted Server integration config: {report:?}" + "invalid Trusted Server integration config: {error}" )) })?; + let mut settings = app_config.into_settings(); settings.publisher.origin_url = origin_url; - validate_settings_for_deploy(&settings).map_err(|report| { + let app_config = TrustedServerAppConfig::new(settings).map_err(|report| { Report::new(TestError::ConfigGeneration) .attach(format!("invalid generated integration config: {report:?}")) })?; - let data = serde_json::to_value(&settings).map_err(|error| { + let data = serde_json::to_value(&app_config).map_err(|error| { Report::new(TestError::ConfigGeneration) .attach(format!("failed to serialize integration settings: {error}")) })?; diff --git a/crates/trusted-server-integration-tests/tests/environments/axum.rs b/crates/trusted-server-integration-tests/tests/environments/axum.rs index 235af413f..3623d8491 100644 --- a/crates/trusted-server-integration-tests/tests/environments/axum.rs +++ b/crates/trusted-server-integration-tests/tests/environments/axum.rs @@ -10,6 +10,30 @@ use std::process::{Child, Command, Stdio}; /// Default port the Axum dev server binds to when no `PORT` env var is supplied. const AXUM_DEFAULT_PORT: u16 = 8787; +/// Secret-store entries referenced by the integration app-config fixture. +const INTEGRATION_SECRET_ENV: &[(&str, &str)] = &[ + ( + "TRUSTED_SERVER_SECRET_TRUSTED_SERVER_SECRETS_INTEGRATION_ADMIN_PASSWORD", + "integration-admin-password-32-bytes-ok", + ), + ( + "TRUSTED_SERVER_SECRET_TRUSTED_SERVER_SECRETS_INTEGRATION_PROXY_SECRET", + "integration-test-proxy-secret-32-bytes-ok", + ), + ( + "TRUSTED_SERVER_SECRET_TRUSTED_SERVER_SECRETS_INTEGRATION_EC_PASSPHRASE", + "integration-test-ec-secret-padded-32", + ), + ( + "TRUSTED_SERVER_SECRET_TRUSTED_SERVER_SECRETS_INTEGRATION_PARTNER_TOKEN_ALPHA", + "integration-test-token-alpha-32-bytes-ok", + ), + ( + "TRUSTED_SERVER_SECRET_TRUSTED_SERVER_SECRETS_INTEGRATION_PARTNER_TOKEN_BRAVO", + "integration-test-token-bravo-32-bytes-ok", + ), +]; + /// Axum native dev-server runtime environment. /// /// Spawns the pre-built `trusted-server-axum` binary directly (no WASM, no @@ -40,6 +64,7 @@ impl RuntimeEnvironment for AxumDevServer { "TRUSTED_SERVER_CONFIG_TRUSTED_SERVER_CONFIG_TRUSTED_SERVER_CONFIG", app_config, ) + .envs(INTEGRATION_SECRET_ENV.iter().copied()) .stdout(Stdio::null()) .stderr(Stdio::piped()) .spawn() diff --git a/crates/trusted-server-js/lib/src/core/first_impression.ts b/crates/trusted-server-js/lib/src/core/first_impression.ts new file mode 100644 index 000000000..05252fcc1 --- /dev/null +++ b/crates/trusted-server-js/lib/src/core/first_impression.ts @@ -0,0 +1,386 @@ +import { resolveSlotElementByDivId } from './slot_element'; +import type { + FirstImpressionPhase, + FirstImpressionPublisherAuction, + FirstImpressionSlotClaim, + FirstImpressionState, + TsjsApi, +} from './types'; + +/** Time allowed for one navigation's losing first-impression delivery. */ +export const FIRST_IMPRESSION_LEASE_MS = 5000; + +const MAX_FIRST_IMPRESSION_SLOTS = 256; +const MAX_PUBLISHER_AUCTIONS_PER_SLOT = 16; + +function currentGeneration(ts: TsjsApi): number { + return ts.navGeneration ?? 0; +} + +function claimMatchesElement( + claim: FirstImpressionSlotClaim, + element: HTMLElement, + generation: number +): boolean { + return ( + claim.generation === generation && + claim.slotElementId === element.id && + claim.element === element && + element.ownerDocument === document && + element.isConnected && + document.getElementById(element.id) === element + ); +} + +function removePublisherAuction( + state: FirstImpressionState, + claim: FirstImpressionSlotClaim, + token: string, + now: number +): void { + delete claim.publisherAuctions[token]; + if ( + claim.owner === 'publisher' && + (claim.phase === 'auctioning' || claim.phase === 'delivery_pending') && + Object.keys(claim.publisherAuctions).length === 0 && + claim.expiresAt <= now + ) { + delete state.slots[claim.slotElementId]; + } +} + +function pruneFirstImpressionState(ts: TsjsApi, now = Date.now()): FirstImpressionState { + const generation = currentGeneration(ts); + if (ts.firstImpression?.generation !== generation) { + ts.firstImpression = { generation, nextToken: 0, slots: {}, fallbackSlots: {} }; + } + + const state = ts.firstImpression; + state.slots ??= {}; + state.fallbackSlots ??= {}; + for (const [elementId, claim] of Object.entries(state.slots)) { + if ( + claim.slotElementId !== elementId || + !claimMatchesElement(claim, claim.element, generation) + ) { + delete state.slots[elementId]; + continue; + } + const hasReservedFallback = + claim.owner === 'publisher' && + (claim.phase === 'auctioning' || claim.phase === 'delivery_pending') && + state.fallbackSlots[elementId] === claim.element; + for (const [token, auction] of Object.entries(claim.publisherAuctions)) { + // A TS-owned losing publisher auction remains a fail-closed tombstone for + // this physical element and navigation. Publisher registrations also stay + // intact while an expired claim is waiting to transition to its reserved + // TS fallback, so an overlapping late callback cannot escape suppression. + if ( + auction.expiresAt <= now && + !hasReservedFallback && + !(claim.owner === 'trusted_server' && auction.suppressDelivery) + ) { + removePublisherAuction(state, claim, token, now); + } + } + if ( + claim.owner === 'publisher' && + (claim.phase === 'auctioning' || claim.phase === 'delivery_pending') && + Object.keys(claim.publisherAuctions).length === 0 && + claim.expiresAt <= now && + !hasReservedFallback + ) { + delete state.slots[elementId]; + } + } + for (const [elementId, element] of Object.entries(state.fallbackSlots)) { + if ( + !element.isConnected || + element.id !== elementId || + document.getElementById(elementId) !== element + ) { + delete state.fallbackSlots[elementId]; + } + } + return state; +} + +/** Resolve a publisher ad-unit code with the same contract GPT uses. */ +export function resolveFirstImpressionElement(adUnitCode: string): HTMLElement | undefined { + return resolveSlotElementByDivId(adUnitCode).element ?? undefined; +} + +/** Return the live ownership claim for an exact slot element. */ +export function firstImpressionClaim( + ts: TsjsApi, + element: HTMLElement +): FirstImpressionSlotClaim | undefined { + const state = pruneFirstImpressionState(ts); + const claim = state.slots[element.id]; + return claim && claimMatchesElement(claim, element, state.generation) ? claim : undefined; +} + +function storeClaim(state: FirstImpressionState, claim: FirstImpressionSlotClaim): boolean { + if ( + !state.slots[claim.slotElementId] && + Object.keys(state.slots).length >= MAX_FIRST_IMPRESSION_SLOTS + ) { + return false; + } + state.slots[claim.slotElementId] = claim; + return true; +} + +/** Atomically claim an untouched slot for Trusted Server. */ +export function claimFirstImpressionForTrustedServer( + ts: TsjsApi, + element: HTMLElement, + now = Date.now() +): FirstImpressionSlotClaim | undefined { + const state = pruneFirstImpressionState(ts, now); + const existing = state.slots[element.id]; + if (existing && claimMatchesElement(existing, element, state.generation)) { + const canTransitionPublisherFallback = + existing.owner === 'publisher' && + existing.phase !== 'requested' && + existing.phase !== 'rendered' && + existing.expiresAt <= now && + state.fallbackSlots[element.id] === element; + if (!canTransitionPublisherFallback) return undefined; + + existing.owner = 'trusted_server'; + existing.phase = 'delivery_pending'; + existing.expiresAt = now + FIRST_IMPRESSION_LEASE_MS; + for (const auction of Object.values(existing.publisherAuctions)) { + auction.suppressDelivery = true; + } + return existing; + } + + const claim: FirstImpressionSlotClaim = { + generation: state.generation, + slotElementId: element.id, + element, + owner: 'trusted_server', + phase: 'delivery_pending', + expiresAt: now + FIRST_IMPRESSION_LEASE_MS, + publisherAuctions: {}, + }; + return storeClaim(state, claim) ? claim : undefined; +} + +function schedulePublisherAuctionExpiry(ts: TsjsApi, token: string): void { + window.setTimeout(() => { + // Pruning releases ordinary publisher claims. TS-owned suppression tokens + // deliberately survive as bounded tombstones until navigation/element change. + findPublisherAuction(ts, token); + }, FIRST_IMPRESSION_LEASE_MS); +} + +/** Release a TS claim when slot setup failed before any request could start. */ +export function releaseTrustedServerFirstImpressionClaim( + ts: TsjsApi, + element: HTMLElement, + claim: FirstImpressionSlotClaim +): void { + const state = pruneFirstImpressionState(ts); + if ( + state.slots[element.id] === claim && + claim.owner === 'trusted_server' && + claim.phase === 'delivery_pending' + ) { + delete state.slots[element.id]; + if (state.fallbackSlots[element.id] === element) { + delete state.fallbackSlots[element.id]; + } + } +} + +/** Register real publisher auctions before native `requestBids()` starts. */ +export function registerPublisherFirstImpressionAuctions( + ts: TsjsApi, + adUnitCodes: Iterable, + now = Date.now() +): Map { + const state = pruneFirstImpressionState(ts, now); + const registrations = new Map(); + + for (const adUnitCode of adUnitCodes) { + const element = resolveFirstImpressionElement(adUnitCode); + if (!element) continue; + + let claim = state.slots[element.id]; + if (!claim || !claimMatchesElement(claim, element, state.generation)) { + claim = { + generation: state.generation, + slotElementId: element.id, + element, + owner: 'publisher', + phase: 'auctioning', + expiresAt: now + FIRST_IMPRESSION_LEASE_MS, + publisherAuctions: {}, + }; + if (!storeClaim(state, claim)) continue; + } + + if ( + claim.owner === 'publisher' && + (claim.phase === 'requested' || claim.phase === 'rendered') + ) { + continue; + } + if ( + claim.owner === 'trusted_server' && + (claim.publisherRegistrationClosed || claim.expiresAt <= now) + ) { + continue; + } + if (Object.keys(claim.publisherAuctions).length >= MAX_PUBLISHER_AUCTIONS_PER_SLOT) continue; + + const token = `${state.generation}:${++state.nextToken}`; + const auction: FirstImpressionPublisherAuction = { + token, + adUnitCode, + phase: 'auctioning', + expiresAt: now + FIRST_IMPRESSION_LEASE_MS, + adIds: [], + suppressDelivery: claim.owner === 'trusted_server', + }; + claim.publisherAuctions[token] = auction; + if (claim.owner === 'publisher') claim.expiresAt = Math.max(claim.expiresAt, auction.expiresAt); + registrations.set(adUnitCode, token); + schedulePublisherAuctionExpiry(ts, token); + } + + return registrations; +} + +function findPublisherAuction( + ts: TsjsApi, + token: string, + now = Date.now() +): + | { + state: FirstImpressionState; + claim: FirstImpressionSlotClaim; + auction: FirstImpressionPublisherAuction; + } + | undefined { + const state = pruneFirstImpressionState(ts, now); + for (const claim of Object.values(state.slots)) { + const auction = claim.publisherAuctions[token]; + if (auction) return { state, claim, auction }; + } + return undefined; +} + +/** Move one publisher auction to delivery-pending without disturbing overlaps. */ +export function markPublisherFirstImpressionDeliveryPending( + ts: TsjsApi, + token: string, + adIds: string[], + now = Date.now() +): void { + const found = findPublisherAuction(ts, token, now); + if (!found) return; + found.auction.phase = 'delivery_pending'; + found.auction.adIds = [...new Set(adIds)]; + if (found.claim.owner === 'publisher') found.claim.phase = 'delivery_pending'; +} + +/** Release exactly one publisher auction token after failure, timeout, or removal. */ +export function releasePublisherFirstImpressionAuction( + ts: TsjsApi, + token: string, + now = Date.now() +): void { + const found = findPublisherAuction(ts, token, now); + if (!found) return; + if (found.claim.owner === 'trusted_server' && found.auction.suppressDelivery) { + found.claim.publisherRegistrationClosed = true; + return; + } + found.auction.expiresAt = Math.min(found.auction.expiresAt, now); + if ( + found.claim.owner === 'publisher' && + Object.keys(found.claim.publisherAuctions).length === 1 + ) { + found.claim.expiresAt = now; + } + removePublisherAuction(found.state, found.claim, token, now); +} + +/** Consume one correlated publisher delivery and report whether TS owns it. */ +export function consumePublisherFirstImpressionDelivery( + ts: TsjsApi, + token: string | undefined, + now = Date.now() +): boolean { + if (!token) return false; + const found = findPublisherAuction(ts, token, now); + if (!found) return false; + + const suppress = found.claim.owner === 'trusted_server' && found.auction.suppressDelivery; + delete found.claim.publisherAuctions[token]; + if (suppress) found.claim.publisherRegistrationClosed = true; + return suppress; +} + +/** Record a GPT request or render, using publisher ownership when no claimant exists. */ +export function observeFirstImpressionGptLifecycle( + ts: TsjsApi, + element: HTMLElement, + phase: Extract, + now = Date.now() +): void { + const state = pruneFirstImpressionState(ts, now); + let claim = state.slots[element.id]; + if (!claim || !claimMatchesElement(claim, element, state.generation)) { + claim = { + generation: state.generation, + slotElementId: element.id, + element, + owner: 'publisher', + phase, + expiresAt: Number.POSITIVE_INFINITY, + publisherAuctions: {}, + }; + storeClaim(state, claim); + return; + } + + claim.phase = phase; + if (claim.owner === 'publisher') { + claim.expiresAt = Number.POSITIVE_INFINITY; + } else { + // Once TS has committed a GPT request, only publisher auctions that were + // already registered can still represent an overlapping first impression. + // New publisher refreshes are ordinary later impressions and must proceed. + claim.publisherRegistrationClosed = true; + } +} + +/** Reserve the only Trusted Server fallback allowed for this physical slot and generation. */ +export function reservePublisherFirstImpressionFallback( + ts: TsjsApi, + element: HTMLElement +): boolean { + const state = pruneFirstImpressionState(ts); + const reservedElement = state.fallbackSlots[element.id]; + if (reservedElement) return false; + state.fallbackSlots[element.id] = element; + return true; +} + +/** Delay before an abandoned publisher claim can receive one per-slot TS fallback. */ +export function publisherFirstImpressionRetryDelay( + ts: TsjsApi, + element: HTMLElement, + now = Date.now() +): number | undefined { + const claim = firstImpressionClaim(ts, element); + if (!claim) return 0; + if (claim.owner !== 'publisher') return undefined; + if (claim.phase === 'requested' || claim.phase === 'rendered') return undefined; + return Math.max(0, claim.expiresAt - now); +} diff --git a/crates/trusted-server-js/lib/src/core/slot_element.ts b/crates/trusted-server-js/lib/src/core/slot_element.ts new file mode 100644 index 000000000..b7cf47d88 --- /dev/null +++ b/crates/trusted-server-js/lib/src/core/slot_element.ts @@ -0,0 +1,83 @@ +/** Result of resolving one configured slot div ID against the live DOM. */ +export interface SlotElementResolution { + element: HTMLElement | null; + prefixMatchCount: number; + activeMatchCount: number; +} + +function isElementVisible(element: HTMLElement): boolean { + const elementWithVisibilityCheck = element as HTMLElement & { + checkVisibility?: (options?: { + checkVisibilityCSS?: boolean; + visibilityProperty?: boolean; + }) => boolean; + }; + if (typeof elementWithVisibilityCheck.checkVisibility === 'function') { + return elementWithVisibilityCheck.checkVisibility({ + checkVisibilityCSS: true, + visibilityProperty: true, + }); + } + + for (let current: HTMLElement | null = element; current; current = current.parentElement) { + const style = window.getComputedStyle(current); + if ( + style.display === 'none' || + style.visibility === 'hidden' || + style.visibility === 'collapse' + ) { + return false; + } + } + return true; +} + +function slotElementHasLayout(element: HTMLElement): boolean { + if (!isElementVisible(element)) return false; + const elementRect = element.getBoundingClientRect(); + if (elementRect.width > 0 && elementRect.height > 0) return true; + + const container = document.getElementById(`${element.id}-container`); + if (!container || !isElementVisible(container)) return false; + const containerRect = container.getBoundingClientRect(); + return containerRect.width > 0; +} + +/** Resolve an exact ID or one unambiguous visible/layout prefix match. */ +export function resolveSlotElementByDivId(divId: string): SlotElementResolution { + if (!divId) { + return { element: null, prefixMatchCount: 0, activeMatchCount: 0 }; + } + + const exact = document.getElementById(divId); + if (exact) { + return { element: exact, prefixMatchCount: 1, activeMatchCount: 1 }; + } + + const prefixMatches = Array.from(document.querySelectorAll('[id]')).filter( + (element) => element.id.startsWith(divId) && !element.id.endsWith('-container') + ); + if (prefixMatches.length === 1 && isElementVisible(prefixMatches[0]!)) { + return { + element: prefixMatches[0]!, + prefixMatchCount: 1, + activeMatchCount: 1, + }; + } + + const visibleMatches = prefixMatches.filter(isElementVisible); + if (visibleMatches.length === 1) { + return { + element: visibleMatches[0]!, + prefixMatchCount: prefixMatches.length, + activeMatchCount: 1, + }; + } + + const activeMatches = visibleMatches.filter(slotElementHasLayout); + return { + element: activeMatches.length === 1 ? activeMatches[0]! : null, + prefixMatchCount: prefixMatches.length, + activeMatchCount: activeMatches.length, + }; +} diff --git a/crates/trusted-server-js/lib/src/core/types.ts b/crates/trusted-server-js/lib/src/core/types.ts index 03ff0aca2..e49b66146 100644 --- a/crates/trusted-server-js/lib/src/core/types.ts +++ b/crates/trusted-server-js/lib/src/core/types.ts @@ -365,6 +365,41 @@ export interface GptSlotHandoff { suppressPublisherRefresh: boolean; } +export type FirstImpressionOwner = 'publisher' | 'trusted_server'; +export type FirstImpressionPhase = 'auctioning' | 'delivery_pending' | 'requested' | 'rendered'; + +/** One publisher auction participating in the current navigation's first impression. */ +export interface FirstImpressionPublisherAuction { + token: string; + adUnitCode: string; + phase: 'auctioning' | 'delivery_pending'; + expiresAt: number; + adIds: string[]; + suppressDelivery: boolean; +} + +/** First-impression ownership for one exact physical slot element. */ +export interface FirstImpressionSlotClaim { + generation: number; + slotElementId: string; + element: HTMLElement; + owner: FirstImpressionOwner; + phase: FirstImpressionPhase; + expiresAt: number; + publisherAuctions: Record; + /** No later publisher auction may join this TS-owned first impression. */ + publisherRegistrationClosed?: boolean; + targeting?: Record; +} + +/** Bounded first-impression state shared by the GPT bootstrap, GPT, and Prebid bundles. */ +export interface FirstImpressionState { + generation: number; + nextToken: number; + slots: Record; + fallbackSlots: Record; +} + export interface TsjsApi { version: string; que: Array<() => void>; @@ -436,6 +471,10 @@ export interface TsjsApi { gptSlotHandoffs?: Record; /** True only while TS calls a GPT function that the handoff wrappers observe. */ gptSlotHandoffInternal?: boolean; + /** Per-navigation first-impression ownership shared by GPT and Prebid. */ + firstImpression?: FirstImpressionState; + /** Guards the shared production GPT lifecycle listener installation. */ + firstImpressionListenersInstalled?: boolean; /** Guards SPA pushState hook installation. */ spaHookInstalled?: boolean; /** Internal one-shot state shared by bootstrap and bundle scheduler installs. */ diff --git a/crates/trusted-server-js/lib/src/integrations/aps/render.ts b/crates/trusted-server-js/lib/src/integrations/aps/render.ts index adec0b036..610271bd1 100644 --- a/crates/trusted-server-js/lib/src/integrations/aps/render.ts +++ b/crates/trusted-server-js/lib/src/integrations/aps/render.ts @@ -70,8 +70,7 @@ function sourceMatchedCandidates( source?: MessageEventSource | null ): HTMLElement[] { if (!source) return candidates; - const sourceMatches = candidates.filter((element) => sourceBelongsToElement(source, element)); - return sourceMatches.length > 0 ? sourceMatches : candidates; + return candidates.filter((element) => sourceBelongsToElement(source, element)); } function dynamicSlotCandidates( @@ -103,23 +102,26 @@ function findApsContainer(slotId: string, source?: MessageEventSource | null): H if (slotId.endsWith('-container')) { const inner = findSlot(slotId.slice(0, -'-container'.length)); - if (inner) return inner; + if (inner) return source && !sourceBelongsToElement(source, inner) ? null : inner; } const direct = findSlot(slotId); - if (direct && !direct.id.endsWith('-container')) return direct; + if (direct && !direct.id.endsWith('-container')) { + return source && !sourceBelongsToElement(source, direct) ? null : direct; + } const configuredDivId = window.tsjs?.adSlots?.find((slot) => slot.id === slotId)?.div_id; if (configuredDivId) { const configured = findSlot(configuredDivId); - if (configured) return configured; + if (configured) { + return source && !sourceBelongsToElement(source, configured) ? null : configured; + } const dynamic = uniqueSlotCandidate(dynamicSlotCandidates(configuredDivId, source)); if (dynamic) return dynamic; } - const dynamic = uniqueSlotCandidate(dynamicSlotCandidates(slotId, source)); - return dynamic ?? direct; + return uniqueSlotCandidate(dynamicSlotCandidates(slotId, source)); } catch { return null; } diff --git a/crates/trusted-server-js/lib/src/integrations/gpt/index.ts b/crates/trusted-server-js/lib/src/integrations/gpt/index.ts index 89b480c6f..eb521254d 100644 --- a/crates/trusted-server-js/lib/src/integrations/gpt/index.ts +++ b/crates/trusted-server-js/lib/src/integrations/gpt/index.ts @@ -1,4 +1,13 @@ +import { + claimFirstImpressionForTrustedServer, + firstImpressionClaim, + observeFirstImpressionGptLifecycle, + publisherFirstImpressionRetryDelay, + releaseTrustedServerFirstImpressionClaim, + reservePublisherFirstImpressionFallback, +} from '../../core/first_impression'; import { log } from '../../core/log'; +import { resolveSlotElementByDivId } from '../../core/slot_element'; import type { AuctionSlot, AuctionBidData, @@ -83,95 +92,6 @@ interface SlotRenderEndedEvent { slot: GoogleTagSlot; } -interface SlotElementResolution { - element: HTMLElement | null; - prefixMatchCount: number; - activeMatchCount: number; -} - -function isElementVisible(element: HTMLElement): boolean { - const elementWithVisibilityCheck = element as HTMLElement & { - checkVisibility?: (options?: { - checkVisibilityCSS?: boolean; - visibilityProperty?: boolean; - }) => boolean; - }; - if (typeof elementWithVisibilityCheck.checkVisibility === 'function') { - return elementWithVisibilityCheck.checkVisibility({ - checkVisibilityCSS: true, - visibilityProperty: true, - }); - } - - for (let current: HTMLElement | null = element; current; current = current.parentElement) { - const style = window.getComputedStyle(current); - if ( - style.display === 'none' || - style.visibility === 'hidden' || - style.visibility === 'collapse' - ) { - return false; - } - } - return true; -} - -function slotElementHasLayout(element: HTMLElement): boolean { - if (!isElementVisible(element)) return false; - const elementRect = element.getBoundingClientRect(); - if (elementRect.width > 0 && elementRect.height > 0) return true; - - const container = document.getElementById(`${element.id}-container`); - if (!container || !isElementVisible(container)) return false; - const containerRect = container.getBoundingClientRect(); - return containerRect.width > 0; -} - -function resolveSlotElementByDivId(divId: string): SlotElementResolution { - if (!divId) { - return { element: null, prefixMatchCount: 0, activeMatchCount: 0 }; - } - // Exact-id matches intentionally skip the visibility tiers below: a - // configured literal id is unambiguous, so a hidden match is still the - // right element (adInit defines the slot; GPT simply renders nothing while - // it is hidden). Prefix matches go through the tiers because a prefix can - // match several candidates and only visibility/layout disambiguates them — - // so a hidden exact-id match resolves while a hidden prefix match does not. - const exact = document.getElementById(divId); - if (exact) { - return { element: exact, prefixMatchCount: 1, activeMatchCount: 1 }; - } - - const prefixMatches = Array.from(document.querySelectorAll('[id]')).filter( - (element) => element.id.startsWith(divId) && !element.id.endsWith('-container') - ); - // A unique prefix match may be a lazy slot that has not been sized yet, but - // it must still be visible through its ancestor containers. - if (prefixMatches.length === 1 && isElementVisible(prefixMatches[0]!)) { - return { - element: prefixMatches[0]!, - prefixMatchCount: 1, - activeMatchCount: 1, - }; - } - - const visibleMatches = prefixMatches.filter(isElementVisible); - if (visibleMatches.length === 1) { - return { - element: visibleMatches[0]!, - prefixMatchCount: prefixMatches.length, - activeMatchCount: 1, - }; - } - - const activeMatches = visibleMatches.filter(slotElementHasLayout); - return { - element: activeMatches.length === 1 ? activeMatches[0]! : null, - prefixMatchCount: prefixMatches.length, - activeMatchCount: activeMatches.length, - }; -} - function findSlotElementByDivId(divId: string): HTMLElement | null { return resolveSlotElementByDivId(divId).element; } @@ -191,48 +111,183 @@ function candidateSlotRoots(elementId: string): HTMLElement[] { return roots; } -function candidateSlotRootsForConfiguredDivId(divId: string): HTMLElement[] { - const roots = candidateSlotRoots(divId); - const dynamicElements = Array.from(document.querySelectorAll('[id]')).filter( - (element) => element.id.startsWith(divId) && !element.id.endsWith('-container') - ); - for (const element of dynamicElements) { - if (!roots.includes(element)) roots.push(element); - const container = document.getElementById(`${element.id}-container`); - if (container && !roots.includes(container)) roots.push(container); +interface MessageSourceFrame { + iframe: HTMLIFrameElement; + root: HTMLElement; +} + +function sourceFrameInRoots( + source: MessageEventSource | null, + roots: readonly HTMLElement[] +): MessageSourceFrame | undefined { + if (!source) return undefined; + const matches = new Map(); + for (const root of roots) { + for (const iframe of root.querySelectorAll('iframe')) { + if (iframe.contentWindow === source && !matches.has(iframe)) matches.set(iframe, root); + } } - return roots; + if (matches.size !== 1) return undefined; + const [iframe, root] = matches.entries().next().value as [HTMLIFrameElement, HTMLElement]; + return { iframe, root }; } -function sourceIsInSlotRoots(source: MessageEventSource, roots: HTMLElement[]): boolean { - return roots.some((root) => - Array.from(root.querySelectorAll('iframe')).some((iframe) => iframe.contentWindow === source) - ); +function sourceFrameForConfiguredDivId( + source: MessageEventSource | null, + divId: string +): MessageSourceFrame | undefined { + const exact = document.getElementById(divId); + const candidates = exact + ? [exact] + : Array.from(document.querySelectorAll('[id]')).filter( + (element) => element.id.startsWith(divId) && !element.id.endsWith('-container') + ); + const matches = candidates + .map((element) => sourceFrameInRoots(source, candidateSlotRoots(element.id))) + .filter((frame): frame is MessageSourceFrame => frame !== undefined); + return matches.length === 1 ? matches[0] : undefined; } -function slotIdForMessageSource(source: MessageEventSource | null): string | undefined { - if (!source) return undefined; +function uniqueSourceFrame( + frames: Array +): MessageSourceFrame | undefined { + const matches = new Map(); + for (const frame of frames) { + if (frame) matches.set(frame.iframe, frame); + } + return matches.size === 1 ? matches.values().next().value : undefined; +} - const divToSlotId = window.tsjs?.divToSlotId ?? {}; - const resolvedSlotId = Object.entries(divToSlotId).find(([elementId]) => - sourceIsInSlotRoots(source, candidateSlotRoots(elementId)) - )?.[1]; - if (resolvedSlotId) return resolvedSlotId; +function sourceFrameForSlotId( + source: MessageEventSource | null, + slotId: string +): MessageSourceFrame | undefined { + const mappedFrames = Object.entries(window.tsjs?.divToSlotId ?? {}) + .filter(([, mappedSlotId]) => mappedSlotId === slotId) + .map(([elementId]) => sourceFrameInRoots(source, candidateSlotRoots(elementId))); + const configuredFrames = (window.tsjs?.adSlots ?? []) + .filter((slot) => slot.id === slotId) + .map((slot) => sourceFrameForConfiguredDivId(source, slot.div_id)); + return uniqueSourceFrame([...mappedFrames, ...configuredFrames]); +} - const slots = window.tsjs?.adSlots ?? []; - return [...slots] - .sort((left, right) => right.div_id.length - left.div_id.length) - .find((slot) => sourceIsInSlotRoots(source, candidateSlotRootsForConfiguredDivId(slot.div_id))) - ?.id; +interface MessageSourceSlotFrame extends MessageSourceFrame { + slotId: string; +} + +function slotFrameForMessageSource( + source: MessageEventSource | null +): MessageSourceSlotFrame | undefined { + const slotIds = new Set(); + for (const [elementId, slotId] of Object.entries(window.tsjs?.divToSlotId ?? {})) { + if (sourceFrameInRoots(source, candidateSlotRoots(elementId))) slotIds.add(slotId); + } + for (const slot of window.tsjs?.adSlots ?? []) { + if (sourceFrameForConfiguredDivId(source, slot.div_id)) slotIds.add(slot.id); + } + if (slotIds.size !== 1) return undefined; + const slotId = slotIds.values().next().value as string; + const frame = sourceFrameForSlotId(source, slotId); + return frame ? { ...frame, slotId } : undefined; } -function messageSourceBelongsToAdUnit( +function sourceFrameForAdUnit( source: MessageEventSource | null, adUnitCode: string +): MessageSourceFrame | undefined { + return sourceFrameForConfiguredDivId(source, adUnitCode); +} + +function hasCollapsedDimension(element: HTMLElement, dimension: 'width' | 'height'): boolean { + const value = window.getComputedStyle(element)[dimension]; + const match = /^(\d+(?:\.\d+)?)px$/.exec(value); + return match !== null && Number(match[1]) <= 1; +} + +function usesFixedPositioning(element: HTMLElement): boolean { + const position = window.getComputedStyle(element).position; + return position === 'fixed' || position === 'sticky'; +} + +const MAX_CREATIVE_SHELL_DIMENSION = 10_000; + +function creativeFrameIsCurrent( + source: MessageEventSource | null, + frame: MessageSourceFrame, + generation: number, + stillOwnsCreative: () => boolean ): boolean { - return source - ? sourceIsInSlotRoots(source, candidateSlotRootsForConfiguredDivId(adUnitCode)) - : false; + return ( + (window.tsjs?.navGeneration ?? 0) === generation && + stillOwnsCreative() && + frame.iframe.isConnected && + frame.root.isConnected && + frame.root.contains(frame.iframe) && + frame.iframe.contentWindow === source + ); +} + +/** Resize the authenticated source iframe and collapsed ancestors through its slot root. */ +function resizeCollapsedCreativeFrame( + source: MessageEventSource | null, + frame: MessageSourceFrame, + width: number, + height: number, + generation: number, + stillOwnsCreative: () => boolean +): void { + if ( + !creativeFrameIsCurrent(source, frame, generation, stillOwnsCreative) || + !Number.isFinite(width) || + !Number.isFinite(height) || + width <= 0 || + height <= 0 || + width > MAX_CREATIVE_SHELL_DIMENSION || + height > MAX_CREATIVE_SHELL_DIMENSION || + frame.iframe.getAttribute('width') !== '1' || + frame.iframe.getAttribute('height') !== '1' || + !hasCollapsedDimension(frame.iframe, 'width') || + !hasCollapsedDimension(frame.iframe, 'height') || + usesFixedPositioning(frame.iframe) || + frame.iframe.closest( + 'ins[data-anchor-status], [data-google-interstitial], [data-vignette-loaded]' + ) + ) { + return; + } + + const collapsedAncestors: HTMLElement[] = []; + let reachedRoot = false; + for (let ancestor = frame.iframe.parentElement; ancestor; ancestor = ancestor.parentElement) { + if ( + ancestor === document.body || + ancestor === document.documentElement || + !ancestor.isConnected || + usesFixedPositioning(ancestor) || + ancestor.matches( + 'ins[data-anchor-status], [data-google-interstitial], [data-vignette-loaded]' + ) + ) { + return; + } + if (hasCollapsedDimension(ancestor, 'width') || hasCollapsedDimension(ancestor, 'height')) { + collapsedAncestors.push(ancestor); + } + if (ancestor === frame.root) { + reachedRoot = true; + break; + } + } + if (!reachedRoot) return; + + frame.iframe.width = String(width); + frame.iframe.height = String(height); + frame.iframe.style.width = `${width}px`; + frame.iframe.style.height = `${height}px`; + for (const ancestor of collapsedAncestors) { + ancestor.style.width = `${width}px`; + ancestor.style.height = `${height}px`; + } } function clearTargetingKeys(slot: GoogleTagSlot, keys: Iterable): void { @@ -930,11 +985,196 @@ function installLatePublisherSlotHandoff(ts: TsjsApi): void { }); } +function installFirstImpressionLifecycleObservers(ts: TsjsApi, g: Partial): void { + if (ts.firstImpressionListenersInstalled) return; + g.cmd?.push(() => { + if (ts.firstImpressionListenersInstalled) return; + const pubads = g.pubads?.(); + if (!pubads?.addEventListener) return; + + const observe = + (phase: 'requested' | 'rendered') => + (event: SlotRenderEndedEvent): void => { + const elementId = event.slot?.getSlotElementId?.(); + const element = elementId ? document.getElementById(elementId) : null; + if (element) observeFirstImpressionGptLifecycle(ts, element, phase); + }; + pubads.addEventListener('slotRequested', observe('requested')); + pubads.addEventListener('slotRenderEnded', observe('rendered')); + ts.firstImpressionListenersInstalled = true; + }); +} + +function trustedServerTargeting( + slot: AuctionSlot, + bid: AuctionBidData +): Record { + const targeting: Record = { ...(slot.targeting ?? {}) }; + for (const key of TS_BID_TARGETING_KEYS) { + if (bid[key]) targeting[key] = String(bid[key]); + } + targeting[TS_INITIAL_TARGETING_KEY] = '1'; + return targeting; +} + +function applyTrustedServerTargeting( + ts: TsjsApi, + gptSlot: GoogleTagSlot, + slot: AuctionSlot, + bid: AuctionBidData, + elementIds: readonly string[] +): string[] { + const previousKeys = ts.prevSlotTargetingKeys ?? {}; + clearTargetingKeys(gptSlot, [ + ...TS_BASE_TARGETING_KEYS, + ...elementIds.flatMap((elementId) => previousKeys[elementId] ?? []), + ]); + const targeting = trustedServerTargeting(slot, bid); + for (const [key, value] of Object.entries(targeting)) gptSlot.setTargeting(key, value); + const element = document.getElementById(elementIds[0]!); + const claim = element ? firstImpressionClaim(ts, element) : undefined; + if (claim?.owner === 'trusted_server') claim.targeting = targeting; + return Object.keys(slot.targeting ?? {}); +} + +function clearPreviousNavigationTargeting(ts: TsjsApi, g: Partial): void { + const previousKeys = ts.prevSlotTargetingKeys ?? {}; + const touchedElementIds = new Set([ + ...Object.keys(previousKeys), + ...Object.keys(ts.divToSlotId ?? {}), + ]); + + const pubads = g.pubads?.(); + if (pubads && touchedElementIds.size > 0) { + for (const slot of pubads.getSlots?.() ?? []) { + const elementId = slot.getSlotElementId(); + if (!touchedElementIds.has(elementId)) continue; + clearTargetingKeys(slot, [...TS_BASE_TARGETING_KEYS, ...(previousKeys[elementId] ?? [])]); + } + } + + ts.prevSlotTargetingKeys = {}; + ts.divToSlotId = {}; +} + +function schedulePublisherFirstImpressionFallback( + ts: TsjsApi, + g: Partial, + slot: AuctionSlot, + bid: AuctionBidData, + element: HTMLElement, + generation: number +): void { + if (!reservePublisherFirstImpressionFallback(ts, element)) return; + + const retry = (): void => { + if ( + (ts.navGeneration ?? 0) !== generation || + !element.isConnected || + document.getElementById(element.id) !== element + ) { + return; + } + const delay = publisherFirstImpressionRetryDelay(ts, element); + if (delay === undefined) return; + if (delay > 0) { + window.setTimeout(retry, delay + 1); + return; + } + + g.cmd?.push(() => { + if ( + (ts.navGeneration ?? 0) !== generation || + !element.isConnected || + document.getElementById(element.id) !== element + ) { + return; + } + const claim = claimFirstImpressionForTrustedServer(ts, element); + if (!claim) return; + + const pubads = g.pubads?.(); + if (!pubads) { + releaseTrustedServerFirstImpressionClaim(ts, element, claim); + return; + } + let gptSlot = pubads + .getSlots?.() + .find((candidate) => candidate.getSlotElementId() === element.id); + let tsOwned = false; + if (!gptSlot) { + gptSlot = + withGptSlotHandoffInternal(ts, () => + g.defineSlot?.(slot.gam_unit_path, slot.formats, element.id) + ) ?? undefined; + if (!gptSlot) { + releaseTrustedServerFirstImpressionClaim(ts, element, claim); + return; + } + gptSlot.addService(pubads); + tsOwned = true; + (ts.gptSlotHandoffs ??= {})[element.id] = { + gamUnitPath: slot.gam_unit_path, + formats: slot.formats, + divIdPrefix: slot.div_id, + slotElementId: element.id, + publisherClaimed: false, + suppressPublisherDisplay: false, + suppressPublisherRefresh: false, + }; + } + + const slotElementId = gptSlot.getSlotElementId?.() ?? element.id; + const targetingKeys = applyTrustedServerTargeting(ts, gptSlot, slot, bid, [ + element.id, + slotElementId, + ]); + (ts.divToSlotId ??= {})[element.id] = slot.id; + if (slotElementId !== element.id) ts.divToSlotId[slotElementId] = slot.id; + (ts.prevSlotTargetingKeys ??= {})[element.id] = targetingKeys; + if (slotElementId !== element.id) ts.prevSlotTargetingKeys[slotElementId] = targetingKeys; + if (tsOwned) (ts.prevGptSlots ??= []).push(gptSlot); + + try { + ts.gptDiagnosticsRecorder?.recordTrustedServerOpportunity( + gptSlot, + slot.id, + trustedServerOpportunity(bid), + bid.hb_auction_id, + slot.formats + ); + } catch { + // Diagnostics must not alter fallback delivery. + } + + if (!ts.servicesEnabled) { + pubads.enableSingleRequest(); + g.enableServices?.(); + ts.servicesEnabled = true; + } + if (tsOwned) withGptSlotHandoffInternal(ts, () => g.display?.(slotElementId)); + syncInitialLoadDisabled(g, ts); + if (!tsOwned || ts.gptInitialLoadDisabled) { + ts.adInitRefreshInProgress = true; + try { + withGptSlotHandoffInternal(ts, () => pubads.refresh([gptSlot!])); + } finally { + ts.adInitRefreshInProgress = false; + } + } + }); + }; + + retry(); +} + export function installTsAdInit(): void { const ts = (window.tsjs ??= {} as TsjsApi); installInitialLoadDetector(ts); installScheduleInitialAdInit(ts); + const g = (window as GptWindow).googletag; + if (g) installFirstImpressionLifecycleObservers(ts, g); installLatePublisherSlotHandoff(ts); ts.adInit = function () { const slots = ts.adSlots ?? []; @@ -951,6 +1191,7 @@ export function installTsAdInit(): void { const generation = ts.navGeneration ?? 0; const g = (window as GptWindow).googletag; if (!g) return; + installFirstImpressionLifecycleObservers(ts, g); const warnedResolutionFailures = new Set(); g.cmd?.push(() => { @@ -1000,6 +1241,8 @@ export function installTsAdInit(): void { (g.pubads!().getSlots?.() ?? []).forEach((gptSlot: GoogleTagSlot) => { const elementId = gptSlot.getSlotElementId(); if (!prevTouchedDivIds.has(elementId)) return; + const element = document.getElementById(elementId); + if (element && firstImpressionClaim(ts, element)) return; clearTargetingKeys(gptSlot, [ ...TS_BASE_TARGETING_KEYS, ...(prevSlotTargetingKeys[elementId] ?? []), @@ -1037,6 +1280,14 @@ export function installTsAdInit(): void { } const actualDivId = el.id; const bid = bids[slot.id] ?? {}; + const firstImpression = claimFirstImpressionForTrustedServer(ts, el); + if (!firstImpression) { + const claim = firstImpressionClaim(ts, el); + if (claim?.owner === 'publisher') { + schedulePublisherFirstImpressionFallback(ts, g, slot, bid, el, generation); + } + return; + } const existingSlot = g.pubads!() .getSlots?.() @@ -1052,7 +1303,10 @@ export function installTsAdInit(): void { const defined = withGptSlotHandoffInternal(ts, () => g.defineSlot?.(slot.gam_unit_path, slot.formats, actualDivId) ); - if (!defined) return; + if (!defined) { + releaseTrustedServerFirstImpressionClaim(ts, el, firstImpression); + return; + } defined.addService(g.pubads!()); gptSlot = defined; tsOwned = true; @@ -1068,17 +1322,10 @@ export function installTsAdInit(): void { } const slotDivId2 = gptSlot.getSlotElementId?.() ?? actualDivId; - clearTargetingKeys(gptSlot, [ - ...TS_BASE_TARGETING_KEYS, - ...(prevSlotTargetingKeys[actualDivId] ?? []), - ...(prevSlotTargetingKeys[slotDivId2] ?? []), + const slotTargetingKeys = applyTrustedServerTargeting(ts, gptSlot, slot, bid, [ + actualDivId, + slotDivId2, ]); - - Object.entries(slot.targeting ?? {}).forEach(([k, v]) => gptSlot.setTargeting(k, v)); - TS_BID_TARGETING_KEYS.forEach((key) => { - if (bid[key]) gptSlot.setTargeting(key, String(bid[key]!)); - }); - gptSlot.setTargeting(TS_INITIAL_TARGETING_KEY, '1'); // Diagnostics are observational only. A missing or malformed debug // implementation must never interrupt slot mapping or delivery. try { @@ -1098,7 +1345,6 @@ export function installTsAdInit(): void { // injection address the same, single GPT slot. divToSlotId[actualDivId] = slot.id; if (slotDivId2 !== actualDivId) divToSlotId[slotDivId2] = slot.id; - const slotTargetingKeys = Object.keys(slot.targeting ?? {}); nextSlotTargetingKeys[actualDivId] = slotTargetingKeys; if (slotDivId2 !== actualDivId) nextSlotTargetingKeys[slotDivId2] = slotTargetingKeys; if (tsOwned) { @@ -1397,7 +1643,10 @@ export function installSpaAuctionHook(): void { async function onNavigate(path: string): Promise { if (path === currentPath) return; currentPath = path; + const g = (window as GptWindow).googletag; + if (g) clearPreviousNavigationTargeting(ts, g); ts.navGeneration = (ts.navGeneration ?? 0) + 1; + delete ts.firstImpression; // A route change invalidates hydration aliases before the new route's // publisher can define a same-prefix slot while page-bids is in flight. for (const [elementId, handoff] of Object.entries(ts.gptSlotHandoffs ?? {})) { @@ -1459,9 +1708,13 @@ export function installSpaAuctionHook(): void { patchHistoryMethod('pushState'); patchHistoryMethod('replaceState'); - window.addEventListener('popstate', () => { - void onNavigate(location.pathname); - }); + window.addEventListener( + 'popstate', + () => { + void onNavigate(location.pathname); + }, + true + ); } /** @@ -1682,6 +1935,7 @@ export function installTsRenderBridge(): void { if (!port) return; const now = Date.now(); + const generation = window.tsjs?.navGeneration ?? 0; pruneConsumedPrebidApsIds(consumedPrebidApsIds, now); const consumedPrebidAps = consumedPrebidApsIds.get(adId); if (consumedPrebidAps) { @@ -1698,7 +1952,8 @@ export function installTsRenderBridge(): void { // Prebid handles ad IDs globally and would otherwise answer a request from // an unrelated iframe when this slot-bound capability rejects it. e.stopImmediatePropagation(); - if (!messageSourceBelongsToAdUnit(e.source, prebidRendererEntry.adUnitCode)) return; + const sourceFrame = sourceFrameForAdUnit(e.source, prebidRendererEntry.adUnitCode); + if (!sourceFrame) return; const renderer = validateApsRenderer(prebidRendererEntry.renderer); if (!renderer || !hasConsumedPrebidApsIdCapacity(consumedPrebidApsIds, adId)) return; if (!consumeApsPrebidRenderer(adId, prebidRendererEntry)) return; @@ -1718,6 +1973,12 @@ export function installTsRenderBridge(): void { trustedServer: (validatedRenderer) => { const rendererUrl = apsRendererUrl(); if (!rendererUrl) return false; + const stillOwnsCreative = () => + sourceFrameForAdUnit(e.source, prebidRendererEntry.adUnitCode)?.iframe === + sourceFrame.iframe; + if (!creativeFrameIsCurrent(e.source, sourceFrame, generation, stillOwnsCreative)) { + return false; + } try { port.postMessage( JSON.stringify({ @@ -1731,7 +1992,15 @@ export function installTsRenderBridge(): void { height: validatedRenderer.height, }) ); - return true; + resizeCollapsedCreativeFrame( + e.source, + sourceFrame, + validatedRenderer.width, + validatedRenderer.height, + generation, + stillOwnsCreative + ); + return creativeFrameIsCurrent(e.source, sourceFrame, generation, stillOwnsCreative); } catch (err) { log.warn(`[tsjs-gpt] APS Prebid response post failed for '${adId}'`, err); return false; @@ -1748,8 +2017,8 @@ export function installTsRenderBridge(): void { return; } - const sourceSlotId = slotIdForMessageSource(e.source); - if (!sourceSlotId) return; + const sourceSlotFrame = slotFrameForMessageSource(e.source); + if (!sourceSlotFrame) return; // Resolve the bid by the requesting slot, not by the first bid whose hb_adid // matches. hb_adid is not unique per bid: absent PBS Cache it falls back to a @@ -1758,7 +2027,7 @@ export function installTsRenderBridge(): void { // first-match-by-adId lookup would resolve every duplicate to one slot, so all // but that slot render blank. const bids = window.tsjs?.bids ?? {}; - const slotId = sourceSlotId; + const slotId = sourceSlotFrame.slotId; const matchedBid = bids[slotId]; // Not a TS bid, or the requesting slot's bid does not own this adId — let @@ -1782,6 +2051,13 @@ export function installTsRenderBridge(): void { trustedServer: (validatedRenderer) => { const rendererUrl = apsRendererUrl(); if (!rendererUrl) return false; + const stillOwnsCreative = () => + window.tsjs?.bids?.[slotId] === matchedBid && + matchedBid.hb_adid === adId && + sourceFrameForSlotId(e.source, slotId)?.iframe === sourceSlotFrame.iframe; + if (!creativeFrameIsCurrent(e.source, sourceSlotFrame, generation, stillOwnsCreative)) { + return false; + } try { port.postMessage( JSON.stringify({ @@ -1795,7 +2071,20 @@ export function installTsRenderBridge(): void { height: validatedRenderer.height, }) ); - return true; + resizeCollapsedCreativeFrame( + e.source, + sourceSlotFrame, + validatedRenderer.width, + validatedRenderer.height, + generation, + stillOwnsCreative + ); + return creativeFrameIsCurrent( + e.source, + sourceSlotFrame, + generation, + stillOwnsCreative + ); } catch (err) { log.warn(`[tsjs-gpt] APS server response post failed for '${slotId}'`, err); return false; @@ -1825,6 +2114,15 @@ export function installTsRenderBridge(): void { if (inlineAdm) { e.stopImmediatePropagation(); + const stillOwnsCreative = () => + Boolean( + window.tsjs?.bids?.[slotId] === matchedBid && + matchedBid.hb_adid === adId && + sourceFrameForSlotId(e.source, slotId)?.iframe === sourceSlotFrame.iframe + ); + if (!creativeFrameIsCurrent(e.source, sourceSlotFrame, generation, stillOwnsCreative)) { + return; + } try { port.postMessage( JSON.stringify({ @@ -1841,6 +2139,15 @@ export function installTsRenderBridge(): void { log.warn(`[tsjs-gpt] pbRender bridge: response post failed for '${slotId}'`, err); return; } + resizeCollapsedCreativeFrame( + e.source, + sourceSlotFrame, + width, + height, + generation, + stillOwnsCreative + ); + if (!creativeFrameIsCurrent(e.source, sourceSlotFrame, generation, stillOwnsCreative)) return; safelyRecordCreativeResponse(attemptId); fireWinBillingBeacons(slotId, matchedBid); log.debug(`[tsjs-gpt] pbRender bridge served '${slotId}' from inline adm`); @@ -1890,6 +2197,15 @@ export function installTsRenderBridge(): void { cached.price !== undefined ? expandAuctionPriceMacro(cached.adm, cached.price) : cached.adm; + const cachedWidth = cached.width ?? width; + const cachedHeight = cached.height ?? height; + const stillOwnsCreative = () => + window.tsjs?.bids?.[slotId] === matchedBid && + matchedBid.hb_adid === adId && + sourceFrameForSlotId(e.source, slotId)?.iframe === sourceSlotFrame.iframe; + if (!creativeFrameIsCurrent(e.source, sourceSlotFrame, generation, stillOwnsCreative)) { + return; + } try { port.postMessage( JSON.stringify({ @@ -1897,15 +2213,26 @@ export function installTsRenderBridge(): void { adId, ad, renderer: TS_DISPLAY_RENDERER, - width: cached.width ?? width, - height: cached.height ?? height, + width: cachedWidth, + height: cachedHeight, }) ); + resizeCollapsedCreativeFrame( + e.source, + sourceSlotFrame, + cachedWidth, + cachedHeight, + generation, + stillOwnsCreative + ); } catch (err) { safelyRecordCreativeFailure(attemptId, 'response_post_failed'); log.warn(`[tsjs-gpt] pbRender bridge: response post failed for '${slotId}'`, err); return; } + if (!creativeFrameIsCurrent(e.source, sourceSlotFrame, generation, stillOwnsCreative)) { + return; + } safelyRecordCreativeResponse(attemptId); // Beacons carry the server-expanded ${AUCTION_PRICE} from the auction's // clearing price, not `cached.price` — the auction result is the diff --git a/crates/trusted-server-js/lib/src/integrations/prebid/index.ts b/crates/trusted-server-js/lib/src/integrations/prebid/index.ts index 44b47f2da..ca822bd60 100644 --- a/crates/trusted-server-js/lib/src/integrations/prebid/index.ts +++ b/crates/trusted-server-js/lib/src/integrations/prebid/index.ts @@ -13,6 +13,15 @@ import type _pbjsDefault from 'prebid.js'; +import { + consumePublisherFirstImpressionDelivery, + FIRST_IMPRESSION_LEASE_MS, + firstImpressionClaim, + markPublisherFirstImpressionDeliveryPending, + registerPublisherFirstImpressionAuctions, + releasePublisherFirstImpressionAuction, + resolveFirstImpressionElement, +} from '../../core/first_impression'; import { log } from '../../core/log'; import { buildAdRequest, parseAuctionResponse } from '../../core/auction'; import { registerApsPrebidRenderer, validateApsRenderer } from '../aps/render'; @@ -130,7 +139,7 @@ const TS_REFRESH_TARGETING_KEYS = [ ] as const; const MAX_PUBLISHER_AD_UNIT_SNAPSHOTS = 256; const MAX_PENDING_PUBLISHER_BIDS = 2048; -const PENDING_PUBLISHER_DELIVERY_TTL_MS = 5000; +const PENDING_PUBLISHER_DELIVERY_TTL_MS = FIRST_IMPRESSION_LEASE_MS; /** Configuration options for the Prebid integration. */ export interface PrebidNpmConfig { @@ -150,7 +159,8 @@ interface InjectedPrebidConfig { accountId?: string; timeout?: number; debug?: boolean; - bidders?: string[]; + /** Validated browser bidder route codes owned by the server auction plan. */ + serverSideBidders?: string[]; /** Bidders that run client-side via native Prebid.js adapters. */ clientSideBidders?: string[]; /** GAM ad-unit-path suffixes excluded from refresh auctions. */ @@ -172,6 +182,10 @@ export function getInjectedConfig(): InjectedPrebidConfig | undefined { return undefined; } +function injectedServerSideBidderCodes(config = getInjectedConfig()): string[] { + return config?.serverSideBidders ?? []; +} + /** Collect all unique bidder codes from the provided ad units. */ export function collectBidders(adUnits: Array<{ bids?: Array<{ bidder?: string }> }>): string[] { const bidders = new Set(); @@ -343,7 +357,7 @@ export function auctionBidsToPrebidBids( bidderCode: bid.seat, meta: { advertiserDomains: bid.adomain, - // Second descriptor carrier — see APS_RENDERER_FIELD for the rationale. + // Second descriptor carrier. See APS_RENDERER_FIELD for the rationale. ...(renderer ? { [APS_RENDERER_FIELD]: renderer } : {}), }, }, @@ -375,10 +389,19 @@ type PendingPublisherBid = { adUnitCode: string; expiresAt: number; registrationId: number; + generation: number; + element: HTMLElement; + retainUntilContextChange: boolean; + firstImpressionToken?: string; }; type PendingPublisherCode = { + adUnitCode: string; expiresAt: number; registrationId: number; + generation: number; + element: HTMLElement; + retainUntilContextChange: boolean; + firstImpressionToken?: string; }; type RemoveAdUnit = (adUnitCode?: string | string[]) => unknown; type PrebidWithRemoveAdUnit = { @@ -388,8 +411,9 @@ type PrebidWithRemoveAdUnit = { let publisherAdUnitSnapshots = new Map(); let pendingPublisherBids = new Map(); -let pendingPublisherCodes = new Map(); +let pendingPublisherCodes = new Map>(); let pendingPublisherRegistrationId = 0; +let publisherFirstImpressionTokens = new Map>(); let syntheticRefreshAdUnits = new WeakSet(); type TrustedServerBidRequest = { adUnitCode?: string; @@ -414,6 +438,7 @@ type RefreshGptSlot = { getSlotElementId?: () => string; getAdUnitPath?: () => string; getTargeting?: (key: string) => string[]; + setTargeting?: (key: string, value: string | string[]) => RefreshGptSlot; clearTargeting?: (key?: string) => RefreshGptSlot; getSizes?: () => unknown[]; }; @@ -647,27 +672,31 @@ function copyParams(params: Record | undefined): Record; } -/** Copy bidder params previously folded into a `trustedServer` bid. */ +/** Copy only plan-owned bidder params previously folded into a `trustedServer` bid. */ function foldedBidderParams( - bid: TrustedServerBid | undefined + bid: TrustedServerBid | undefined, + serverSideBidders: Set ): Record> { const folded = (bid?.params?.[BIDDER_PARAMS_KEY] ?? {}) as Record< string, Record >; return Object.fromEntries( - Object.entries(folded).map(([bidder, params]) => [bidder, copyParams(params)]) + Object.entries(folded) + .filter(([bidder]) => serverSideBidders.has(bidder)) + .map(([bidder, params]) => [bidder, copyParams(params)]) ); } /** Capture immutable request-scoped bidder and zone data before the shim mutates an ad unit. */ function capturePublisherAdUnitSnapshot( unit: TrustedServerAdUnit, - clientSideBidders: Set + clientSideBidders: Set, + serverSideBidders: Set ): PublisherAdUnitSnapshot | undefined { if (typeof unit.code !== 'string' || unit.code.length === 0) return undefined; - const rawBidderParams: Record> = {}; + const rawBidderParams = Object.create(null) as Record>; const clientSideBids: ClientSideBidSnapshot[] = []; let existingTsBid: TrustedServerBid | undefined; @@ -678,15 +707,19 @@ function capturePublisherAdUnitSnapshot( existingTsBid ??= bid; continue; } - if (clientSideBidders.has(bid.bidder)) { - clientSideBids.push({ bidder: bid.bidder, params: copyParams(bid.params) }); + if (!serverSideBidders.has(bid.bidder)) { + if (clientSideBidders.has(bid.bidder)) { + clientSideBids.push({ bidder: bid.bidder, params: copyParams(bid.params) }); + } continue; } rawBidderParams[bid.bidder] = copyParams(bid.params); } const bidderParams = - Object.keys(rawBidderParams).length > 0 ? rawBidderParams : foldedBidderParams(existingTsBid); + Object.keys(rawBidderParams).length > 0 + ? rawBidderParams + : foldedBidderParams(existingTsBid, serverSideBidders); const zone = unit.mediaTypes?.banner?.name; return { @@ -751,16 +784,16 @@ function serverSideBidderParamsForRefresh( if (match) { if (!Array.isArray(match.bids)) return {}; - const clientSideBidders = new Set(getInjectedConfig()?.clientSideBidders ?? []); - const params: Record> = {}; + const serverSideBidders = new Set(injectedServerSideBidderCodes()); + const params = Object.create(null) as Record>; for (const bid of match.bids) { if (!bid?.bidder) continue; if (bid.bidder === ADAPTER_CODE) { - Object.assign(params, foldedBidderParams(bid)); + Object.assign(params, foldedBidderParams(bid, serverSideBidders)); continue; } - if (clientSideBidders.has(bid.bidder)) continue; + if (!serverSideBidders.has(bid.bidder)) continue; params[bid.bidder] = copyParams(bid.params); } @@ -819,41 +852,179 @@ function clearRefreshTargeting(slot: RefreshGptSlot): void { } } +function restoreTrustedServerFirstImpressionTargeting(slot: RefreshGptSlot): void { + const ts = window.tsjs; + const injectedSlot = findInjectedSlotForRefresh(slot); + const element = [refreshSlotElementId(slot), injectedSlot?.div_id] + .filter((elementId): elementId is string => Boolean(elementId)) + .map((elementId) => document.getElementById(elementId)) + .find((candidate): candidate is HTMLElement => + Boolean(candidate && ts && firstImpressionClaim(ts, candidate)?.owner === 'trusted_server') + ); + const claim = ts && element ? firstImpressionClaim(ts, element) : undefined; + if (claim?.owner !== 'trusted_server' || !claim.targeting || !slot.setTargeting) return; + clearRefreshTargeting(slot); + for (const [key, value] of Object.entries(claim.targeting)) slot.setTargeting(key, value); +} + +/** Track a first-impression token until its exact auction is consumed or abandoned. */ +function trackPublisherFirstImpressionToken(adUnitCode: string, token: string): void { + const tokens = publisherFirstImpressionTokens.get(adUnitCode) ?? new Set(); + tokens.add(token); + publisherFirstImpressionTokens.set(adUnitCode, tokens); +} + +function forgetPublisherFirstImpressionToken(adUnitCode: string, token?: string): void { + const tokens = publisherFirstImpressionTokens.get(adUnitCode); + if (!tokens) return; + if (token === undefined) { + if (window.tsjs) { + for (const current of tokens) releasePublisherFirstImpressionAuction(window.tsjs, current); + } + publisherFirstImpressionTokens.delete(adUnitCode); + return; + } + tokens.delete(token); + if (tokens.size === 0) publisherFirstImpressionTokens.delete(adUnitCode); +} + /** Remove pending delivery state for an ad unit, optionally from one registration only. */ function removePendingPublisherBidsForCode(adUnitCode: string, registrationId?: number): void { - const pendingCode = pendingPublisherCodes.get(adUnitCode); - if (registrationId !== undefined && pendingCode?.registrationId !== registrationId) return; + const registrations = pendingPublisherCodes.get(adUnitCode); + if (registrations) { + if (registrationId === undefined) { + pendingPublisherCodes.delete(adUnitCode); + } else { + const pending = registrations.get(registrationId); + if (!pending?.retainUntilContextChange) registrations.delete(registrationId); + if (registrations.size === 0) pendingPublisherCodes.delete(adUnitCode); + } + } - pendingPublisherCodes.delete(adUnitCode); for (const [adId, pendingBid] of pendingPublisherBids) { if ( pendingBid.adUnitCode === adUnitCode && - (registrationId === undefined || pendingBid.registrationId === registrationId) + (registrationId === undefined || pendingBid.registrationId === registrationId) && + (registrationId === undefined || !pendingBid.retainUntilContextChange) ) { pendingPublisherBids.delete(adId); + if (pendingBid.firstImpressionToken) { + forgetPublisherFirstImpressionToken(adUnitCode, pendingBid.firstImpressionToken); + } + } + } +} + +function removeConsumedPublisherRegistration(adUnitCode: string, registrationId: number): void { + const registrations = pendingPublisherCodes.get(adUnitCode); + const pendingCode = registrations?.get(registrationId); + registrations?.delete(registrationId); + if (registrations?.size === 0) pendingPublisherCodes.delete(adUnitCode); + + const tokens = new Set(); + if (pendingCode?.firstImpressionToken) tokens.add(pendingCode.firstImpressionToken); + for (const [adId, pendingBid] of pendingPublisherBids) { + if (pendingBid.adUnitCode !== adUnitCode || pendingBid.registrationId !== registrationId) { + continue; } + pendingPublisherBids.delete(adId); + if (pendingBid.firstImpressionToken) tokens.add(pendingBid.firstImpressionToken); } + for (const token of tokens) forgetPublisherFirstImpressionToken(adUnitCode, token); +} + +function pendingPublisherContextIsCurrent( + pending: PendingPublisherBid | PendingPublisherCode +): boolean { + return ( + pending.generation === (window.tsjs?.navGeneration ?? 0) && + pending.element.isConnected && + document.getElementById(pending.element.id) === pending.element && + resolvePublisherDeliveryElement(pending.adUnitCode) === pending.element + ); +} + +function resolvePublisherDeliveryElement(adUnitCode: string): HTMLElement | undefined { + const direct = resolveFirstImpressionElement(adUnitCode); + if (direct) return direct; + + const gpt = ( + window as unknown as { + googletag?: { pubads?(): { getSlots?(): RefreshGptSlot[] } }; + } + ).googletag; + const matches = (gpt?.pubads?.().getSlots?.() ?? []) + .filter((slot) => { + const injectedSlot = findInjectedSlotForRefresh(slot); + return refreshSlotElementId(slot) === adUnitCode || injectedSlot?.div_id === adUnitCode; + }) + .map((slot) => { + const elementId = refreshSlotElementId(slot); + return elementId ? document.getElementById(elementId) : null; + }) + .filter((element): element is HTMLElement => Boolean(element?.isConnected)); + return matches.length === 1 ? matches[0] : undefined; +} + +function pendingPublisherContextMatchesSlot( + pending: PendingPublisherBid | PendingPublisherCode, + slot: RefreshGptSlot +): boolean { + if (!pendingPublisherContextIsCurrent(pending)) return false; + const injectedSlot = findInjectedSlotForRefresh(slot); + return [refreshSlotElementId(slot), injectedSlot?.div_id] + .filter((code): code is string => typeof code === 'string' && code.length > 0) + .some((code) => { + const exact = document.getElementById(code); + return ( + exact === pending.element || + Boolean(exact && (pending.element.contains(exact) || exact.contains(pending.element))) || + resolvePublisherDeliveryElement(code) === pending.element + ); + }); } /** Discard delivery state that outlived the publisher auction which created it. */ function prunePendingPublisherBids(now = Date.now()): void { - for (const [adUnitCode, pendingCode] of pendingPublisherCodes) { - if (pendingCode.expiresAt <= now) pendingPublisherCodes.delete(adUnitCode); + for (const [adUnitCode, registrations] of pendingPublisherCodes) { + for (const [registrationId, pendingCode] of registrations) { + if ( + !pendingPublisherContextIsCurrent(pendingCode) || + (pendingCode.expiresAt <= now && !pendingCode.retainUntilContextChange) + ) { + registrations.delete(registrationId); + } + } + if (registrations.size === 0) pendingPublisherCodes.delete(adUnitCode); } for (const [adId, pendingBid] of pendingPublisherBids) { - if (pendingBid.expiresAt <= now) pendingPublisherBids.delete(adId); + if ( + !pendingPublisherContextIsCurrent(pendingBid) || + (pendingBid.expiresAt <= now && !pendingBid.retainUntilContextChange) + ) { + pendingPublisherBids.delete(adId); + } } } -/** Store a short-lived pending publisher ad-unit code for delivery correlation. */ -function storePendingPublisherCode(adUnitCode: string, pendingCode: PendingPublisherCode): void { - pendingPublisherCodes.delete(adUnitCode); - pendingPublisherCodes.set(adUnitCode, pendingCode); - - if (pendingPublisherCodes.size > MAX_PENDING_PUBLISHER_BIDS) { - const oldestCode = pendingPublisherCodes.keys().next().value; - if (oldestCode !== undefined) removePendingPublisherBidsForCode(oldestCode); +/** Store a short-lived pending publisher ad-unit code without erasing overlaps. */ +function storePendingPublisherCode(pendingCode: PendingPublisherCode): void { + const registrations = pendingPublisherCodes.get(pendingCode.adUnitCode) ?? new Map(); + registrations.set(pendingCode.registrationId, pendingCode); + pendingPublisherCodes.set(pendingCode.adUnitCode, registrations); + + let registrationCount = 0; + for (const pending of pendingPublisherCodes.values()) registrationCount += pending.size; + if (registrationCount > MAX_PENDING_PUBLISHER_BIDS) { + for (const [adUnitCode, pendingRegistrations] of pendingPublisherCodes) { + const evictable = [...pendingRegistrations.values()].find( + (pending) => !pending.retainUntilContextChange + ); + if (!evictable) continue; + removePendingPublisherBidsForCode(adUnitCode, evictable.registrationId); + break; + } } } @@ -868,29 +1039,18 @@ function storePendingPublisherBid(adId: string, pendingBid: PendingPublisherBid) } } -/** Register every requested publisher code and any bid IDs returned for that auction. */ -function registerPendingPublisherBids( +function publisherResponseAdIds( publisherAdUnitCodes: Set, bidResponses: unknown -): number { - prunePendingPublisherBids(); - const registrationId = ++pendingPublisherRegistrationId; - const expiresAt = Date.now() + PENDING_PUBLISHER_DELIVERY_TTL_MS; - - for (const adUnitCode of publisherAdUnitCodes) { - removePendingPublisherBidsForCode(adUnitCode); - storePendingPublisherCode(adUnitCode, { expiresAt, registrationId }); - } - - if (!bidResponses || typeof bidResponses !== 'object' || Array.isArray(bidResponses)) { - return registrationId; - } +): Map { + const adIds = new Map(); + if (!bidResponses || typeof bidResponses !== 'object' || Array.isArray(bidResponses)) + return adIds; for (const [responseCode, responseGroup] of Object.entries(bidResponses)) { if (!responseGroup || typeof responseGroup !== 'object') continue; const bids = (responseGroup as { bids?: unknown }).bids; if (!Array.isArray(bids)) continue; - for (const bid of bids) { if (!bid || typeof bid !== 'object') continue; const response = bid as { adId?: unknown; adUnitCode?: unknown }; @@ -898,29 +1058,98 @@ function registerPendingPublisherBids( const adUnitCode = typeof response.adUnitCode === 'string' ? response.adUnitCode : responseCode; if (!adId || !adUnitCode || !publisherAdUnitCodes.has(adUnitCode)) continue; + adIds.set(adUnitCode, [...(adIds.get(adUnitCode) ?? []), adId]); + } + } + return adIds; +} - storePendingPublisherBid(adId, { adUnitCode, expiresAt, registrationId }); +/** Register every requested publisher code and any bid IDs returned for that auction. */ +function registerPendingPublisherBids( + publisherAdUnitCodes: Set, + bidResponses: unknown, + firstImpressionTokens: Map +): number { + prunePendingPublisherBids(); + const registrationId = ++pendingPublisherRegistrationId; + const expiresAt = Date.now() + PENDING_PUBLISHER_DELIVERY_TTL_MS; + const responseAdIds = publisherResponseAdIds(publisherAdUnitCodes, bidResponses); + + for (const adUnitCode of publisherAdUnitCodes) { + const element = resolvePublisherDeliveryElement(adUnitCode); + if (!element) continue; + const firstImpressionToken = firstImpressionTokens.get(adUnitCode); + const retainUntilContextChange = Boolean( + firstImpressionToken && + window.tsjs && + firstImpressionClaim(window.tsjs, element)?.owner === 'trusted_server' + ); + storePendingPublisherCode({ + adUnitCode, + expiresAt, + registrationId, + generation: window.tsjs?.navGeneration ?? 0, + element, + retainUntilContextChange, + firstImpressionToken, + }); + if (firstImpressionToken && window.tsjs) { + markPublisherFirstImpressionDeliveryPending( + window.tsjs, + firstImpressionToken, + responseAdIds.get(adUnitCode) ?? [] + ); + } + } + + for (const [adUnitCode, adIds] of responseAdIds) { + const element = resolvePublisherDeliveryElement(adUnitCode); + if (!element) continue; + const firstImpressionToken = firstImpressionTokens.get(adUnitCode); + const retainUntilContextChange = Boolean( + firstImpressionToken && + window.tsjs && + firstImpressionClaim(window.tsjs, element)?.owner === 'trusted_server' + ); + for (const adId of adIds) { + storePendingPublisherBid(adId, { + adUnitCode, + expiresAt, + registrationId, + generation: window.tsjs?.navGeneration ?? 0, + element, + retainUntilContextChange, + firstImpressionToken, + }); } } return registrationId; } -/** - * Partition slots by whether they belong to a pending publisher auction. - * - * A current `hb_adid` is the precise signal. When publishers intentionally - * omit that targeting, a short-lived requested-code match preserves delivery - * for no-bid and custom-targeting auctions. Without an ID, that fallback cannot - * distinguish a delayed delivery from the first independent refresh, so it may - * conservatively suppress one auction before its one-shot state is consumed. - * A non-empty unmatched ID remains independent so stale targeting cannot - * suppress a fresh auction. Every match is consumed once. - */ -function publisherDeliverySlots(targetSlots: RefreshGptSlot[]): Set { +interface PublisherDeliveryPartition { + deliverySlots: Set; + suppressedSlots: Set; +} + +/** Consume the equivalent one-shot suppression owned by the inner GPT wrapper. */ +function consumeGptPublisherRefreshSuppression(slot: RefreshGptSlot): void { + const elementId = refreshSlotElementId(slot); + const handoff = elementId ? window.tsjs?.gptSlotHandoffs?.[elementId] : undefined; + if (handoff?.suppressPublisherRefresh) handoff.suppressPublisherRefresh = false; +} + +/** Restore TS targeting and consume any equivalent GPT-wrapper handoff. */ +function prepareSuppressedPublisherSlot(slot: RefreshGptSlot): void { + restoreTrustedServerFirstImpressionTargeting(slot); + consumeGptPublisherRefreshSuppression(slot); +} + +/** Partition correlated publisher deliveries from one losing first-impression delivery. */ +function publisherDeliverySlots(targetSlots: RefreshGptSlot[]): PublisherDeliveryPartition { prunePendingPublisherBids(); const deliverySlots = new Set(); - const deliveredCodes = new Set(); + const suppressedSlots = new Set(); for (const slot of targetSlots) { const adIds = slot.getTargeting?.('hb_adid'); @@ -928,25 +1157,45 @@ function publisherDeliverySlots(targetSlots: RefreshGptSlot[]): Set typeof adId === 'string' && adId.length > 0) .map((adId) => pendingPublisherBids.get(adId)) - .find((bid): bid is PendingPublisherBid => bid !== undefined) + .find( + (bid): bid is PendingPublisherBid => + bid !== undefined && pendingPublisherContextMatchesSlot(bid, slot) + ) : undefined; const hasAdId = Array.isArray(adIds) && adIds.some((adId) => typeof adId === 'string' && adId.length > 0); const injectedSlot = findInjectedSlotForRefresh(slot); - const pendingCode = hasAdId - ? undefined - : [refreshSlotElementId(slot), injectedSlot?.div_id] + const pendingCodeCandidates = [ + ...new Map( + [refreshSlotElementId(slot), injectedSlot?.div_id] .filter((code): code is string => typeof code === 'string' && code.length > 0) - .find((code) => pendingPublisherCodes.has(code)); - const adUnitCode = pendingBid?.adUnitCode ?? pendingCode; - if (!adUnitCode) continue; + .flatMap((code) => [...(pendingPublisherCodes.get(code)?.values() ?? [])]) + .filter( + (pending) => + pendingPublisherContextMatchesSlot(pending, slot) && + (!hasAdId || pending.retainUntilContextChange) + ) + .map((pending) => [pending.registrationId, pending] as const) + ).values(), + ].sort((left, right) => left.registrationId - right.registrationId); + const pendingCode = pendingCodeCandidates.length === 1 ? pendingCodeCandidates[0] : undefined; + const pending = pendingBid ?? pendingCode; + if (!pending) { + if (pendingCodeCandidates.some((candidate) => candidate.retainUntilContextChange)) { + suppressedSlots.add(slot); + } + continue; + } - deliverySlots.add(slot); - deliveredCodes.add(adUnitCode); + const suppress = + pending.firstImpressionToken && window.tsjs + ? consumePublisherFirstImpressionDelivery(window.tsjs, pending.firstImpressionToken) + : false; + removeConsumedPublisherRegistration(pending.adUnitCode, pending.registrationId); + (suppress ? suppressedSlots : deliverySlots).add(slot); } - deliveredCodes.forEach((adUnitCode) => removePendingPublisherBidsForCode(adUnitCode)); - return deliverySlots; + return { deliverySlots, suppressedSlots }; } /** Evict publisher state after Prebid removes one or more ad units. */ @@ -955,6 +1204,9 @@ function removePublisherState(adUnitCode?: string | string[]): void { publisherAdUnitSnapshots.clear(); pendingPublisherBids.clear(); pendingPublisherCodes.clear(); + for (const code of publisherFirstImpressionTokens.keys()) { + forgetPublisherFirstImpressionToken(code); + } return; } @@ -962,6 +1214,7 @@ function removePublisherState(adUnitCode?: string | string[]): void { for (const code of adUnitCodes) { publisherAdUnitSnapshots.delete(code); removePendingPublisherBidsForCode(code); + forgetPublisherFirstImpressionToken(code); } } @@ -1004,24 +1257,34 @@ function collectAuctionEids(): AuctionEid[] | undefined { * repeat calls (double script inclusion, a bundle that still carries a * baked-in shim) a no-op instead of a double adapter registration. */ +function apsRendererCarrier(bid: Record): unknown { + const renderer = bid[APS_RENDERER_FIELD]; + if (renderer !== undefined) return renderer; + const meta = bid['meta']; + if (meta !== null && typeof meta === 'object') { + return (meta as Record)[APS_RENDERER_FIELD]; + } + return undefined; +} + +function scrubApsRendererCarrier(bid: Record): void { + delete bid[APS_RENDERER_FIELD]; + const meta = bid['meta']; + if (meta !== null && typeof meta === 'object') { + delete (meta as Record)[APS_RENDERER_FIELD]; + } +} + function installApsBidResponseRegistry(): void { const prebid = pbjs as typeof pbjs & Record; if (prebid[APS_BID_RESPONSE_LISTENER_SENTINEL] === true) return; - const registerFromBid = (rawBid: unknown): void => { + const registerRenderer = (rawBid: unknown): void => { const bid = rawBid as Record; if (bid['adapterCode'] !== ADAPTER_CODE || bid['bidderCode'] !== APS_BIDDER_CODE) { return; } - // Prefer the custom top-level field; fall back to the per-bid copy in `meta` - // — see APS_RENDERER_FIELD for why both carriers exist. Guard the `meta` - // read: a module may have overwritten it with a non-object value. - const rawMeta = bid['meta']; - const meta = - typeof rawMeta === 'object' && rawMeta !== null - ? (rawMeta as Record) - : undefined; - const renderer = bid[APS_RENDERER_FIELD] ?? meta?.[APS_RENDERER_FIELD]; + const renderer = apsRendererCarrier(bid); const adId = bid['adId']; if (renderer === undefined || typeof adId !== 'string') { return; @@ -1035,10 +1298,7 @@ function installApsBidResponseRegistry(): void { }); // Keep the executable capability only in the bounded, one-time registry. Prebid // still owns the generated ad ID and ordinary GAM targeting on this bid object. - delete bid[APS_RENDERER_FIELD]; - if (meta) { - delete meta[APS_RENDERER_FIELD]; - } + scrubApsRendererCarrier(bid); if (!registered) { // Prebid can admit zero-CPM bids when `allowZeroCpmBids` is enabled. // Its targeting selection rejects every negative CPM, so this bid cannot @@ -1048,12 +1308,11 @@ function installApsBidResponseRegistry(): void { } }; - // Register on `bidAccepted` — the first event after Prebid assigns `adId` — so - // the executable descriptor is scrubbed from the bid before `bidResponse` and - // analytics consumers of later events can observe it. The `bidResponse` pass - // is a fallback that no-ops when the `bidAccepted` pass already scrubbed. - pbjs.onEvent('bidAccepted', registerFromBid); - pbjs.onEvent('bidResponse', registerFromBid); + // Register on `bidAccepted`, the first event after Prebid assigns `adId`, so + // later event consumers cannot observe the executable descriptor. The + // `bidResponse` pass is a compatibility fallback. + pbjs.onEvent('bidAccepted', registerRenderer); + pbjs.onEvent('bidResponse', registerRenderer); prebid[APS_BID_RESPONSE_LISTENER_SENTINEL] = true; } @@ -1084,6 +1343,7 @@ export function installPrebidNpm(config?: Partial): typeof pbjs pendingPublisherBids = new Map(); pendingPublisherCodes = new Map(); pendingPublisherRegistrationId = 0; + publisherFirstImpressionTokens = new Map(); syntheticRefreshAdUnits = new WeakSet(); const prebidWithRemoveAdUnit = pbjs as unknown as PrebidWithRemoveAdUnit; @@ -1160,23 +1420,42 @@ export function installPrebidNpm(config?: Partial): typeof pbjs const originalRequestBids = pbjs.requestBids.bind(pbjs); - // Bidders that should run client-side via their native Prebid.js adapters. - // Read once from the server-injected config. + // Browser demand ownership is explicit. Only validated auction-plan route + // codes are folded; every other publisher bidder entry remains in Prebid.js. const clientSideBidders = new Set(injected?.clientSideBidders ?? []); + const serverSideBidders = new Set(injectedServerSideBidderCodes(injected)); if (clientSideBidders.size > 0) { log.info('[tsjs-prebid] client-side bidders:', [...clientSideBidders]); } + if (serverSideBidders.size > 0) { + log.info('[tsjs-prebid] server-side bidders:', [...serverSideBidders]); + } // Shim requestBids to inject the trustedServer bidder into every ad unit - // so server-side bids flow through the /auction orchestrator while - // client-side bidders are left untouched. + // so plan-owned server-side bids flow through the /auction orchestrator while + // every unowned bidder is left untouched. pbjs.requestBids = function (requestObj?: Parameters[0]) { log.debug('[tsjs-prebid] requestBids called'); recordUserIdModuleDiagnostics(); const opts = { ...(requestObj ?? {}) }; // eslint-disable-next-line @typescript-eslint/no-explicit-any - const adUnits = ((opts as any).adUnits || pbjs.adUnits || []) as TrustedServerAdUnit[]; + const explicitAdUnits = (opts as any).adUnits as TrustedServerAdUnit[] | undefined; + // eslint-disable-next-line @typescript-eslint/no-explicit-any + const requestedAdUnitCodes = Array.isArray((opts as any).adUnitCodes) + ? new Set( + // eslint-disable-next-line @typescript-eslint/no-explicit-any + ((opts as any).adUnitCodes as unknown[]).filter( + (code): code is string => typeof code === 'string' + ) + ) + : undefined; + const adUnits = (explicitAdUnits ?? (pbjs.adUnits as TrustedServerAdUnit[]) ?? []).filter( + (unit) => + explicitAdUnits !== undefined || + requestedAdUnitCodes === undefined || + requestedAdUnitCodes.has(unit.code ?? '') + ); const isSyntheticRefresh = adUnits.length > 0 && adUnits.every((unit) => syntheticRefreshAdUnits.has(unit)); const publisherAdUnitCodes = new Set( @@ -1185,11 +1464,25 @@ export function installPrebidNpm(config?: Partial): typeof pbjs .map((unit) => unit.code) .filter((code): code is string => typeof code === 'string' && code.length > 0) ); + const firstImpressionTokens = + !isSyntheticRefresh && !window.tsjs?.adInitRefreshInProgress + ? registerPublisherFirstImpressionAuctions( + (window.tsjs ??= {} as TsjsApi), + publisherAdUnitCodes + ) + : new Map(); + for (const [adUnitCode, token] of firstImpressionTokens) { + trackPublisherFirstImpressionToken(adUnitCode, token); + window.setTimeout( + () => forgetPublisherFirstImpressionToken(adUnitCode, token), + PENDING_PUBLISHER_DELIVERY_TTL_MS + ); + } // Ensure every ad unit has a trustedServer bid entry for (const unit of adUnits) { if (!syntheticRefreshAdUnits.has(unit)) { - const snapshot = capturePublisherAdUnitSnapshot(unit, clientSideBidders); + const snapshot = capturePublisherAdUnitSnapshot(unit, clientSideBidders, serverSideBidders); if (snapshot && unit.code) { storePublisherAdUnitSnapshot(unit.code, snapshot); } @@ -1199,24 +1492,21 @@ export function installPrebidNpm(config?: Partial): typeof pbjs unit.bids = []; } - // Preserve per-bidder params for server-side expansion. - // Skip client-side bidders — they remain as standalone bids and run - // via their native Prebid.js adapters in the browser. - const bidderParams: Record> = {}; + // Preserve params only for bidder codes owned by the validated auction + // plan. Provider IDs, returned seat aliases, APS renderer identity, and + // ordinary browser demand cannot enter the trustedServer envelope. + const bidderParams = Object.create(null) as Record>; for (const bid of unit.bids) { - if (!bid?.bidder || bid.bidder === ADAPTER_CODE) { - continue; - } - if (clientSideBidders.has(bid.bidder)) { + if (!bid?.bidder || !serverSideBidders.has(bid.bidder)) { continue; } bidderParams[bid.bidder] = bid.params ?? {}; } - // Keep only bids that should still execute in the browser. All other - // bidders are routed through the trustedServer adapter. + // Keep every unowned bid in browser demand, including configured native + // adapters and standard entries not claimed by the plan. unit.bids = unit.bids.filter( - (bid) => bid?.bidder === ADAPTER_CODE || clientSideBidders.has(bid?.bidder ?? '') + (bid) => bid?.bidder === ADAPTER_CODE || !serverSideBidders.has(bid?.bidder ?? '') ); // WORKAROUND: Read the zone from mediaTypes.banner.name. This is NOT a @@ -1237,10 +1527,7 @@ export function installPrebidNpm(config?: Partial): typeof pbjs // by the prior call, so `bidderParams` is now empty. Retain the // params captured on the first call instead of overwriting them with // `{}`, which would drop the publisher's inline PBS params on refresh. - const prevBidderParams = (prevParams[BIDDER_PARAMS_KEY] ?? {}) as Record< - string, - Record - >; + const prevBidderParams = foldedBidderParams(existingTsBid, serverSideBidders); const effectiveBidderParams = Object.keys(bidderParams).length > 0 ? bidderParams : prevBidderParams; @@ -1280,7 +1567,7 @@ export function installPrebidNpm(config?: Partial): typeof pbjs syncPrebidEidsCookie(); const registrationId = isSyntheticRefresh ? undefined - : registerPendingPublisherBids(publisherAdUnitCodes, args[0]); + : registerPendingPublisherBids(publisherAdUnitCodes, args[0], firstImpressionTokens); if (typeof originalBidsBack !== 'function') return; try { @@ -1291,11 +1578,23 @@ export function installPrebidNpm(config?: Partial): typeof pbjs removePendingPublisherBidsForCode(code, registrationId) ); } + for (const [adUnitCode, token] of firstImpressionTokens) { + releasePublisherFirstImpressionAuction(window.tsjs!, token); + forgetPublisherFirstImpressionToken(adUnitCode, token); + } throw error; } }; - return originalRequestBids(opts); + try { + return originalRequestBids(opts); + } catch (error) { + for (const [adUnitCode, token] of firstImpressionTokens) { + releasePublisherFirstImpressionAuction(window.tsjs!, token); + forgetPublisherFirstImpressionToken(adUnitCode, token); + } + throw error; + } }; // Apply initial configuration @@ -1403,11 +1702,16 @@ export function installRefreshHandler(timeoutMs = 1500): void { return originalRefresh(slots, opts); } - const deliverySlots = publisherDeliverySlots(targetSlots); - const independentSlots = targetSlots.filter((slot) => !deliverySlots.has(slot)); + const { deliverySlots, suppressedSlots } = publisherDeliverySlots(targetSlots); + suppressedSlots.forEach(prepareSuppressedPublisherSlot); + const remainingSlots = targetSlots.filter((slot) => !suppressedSlots.has(slot)); + if (remainingSlots.length === 0) return; + const forwardedSlots = suppressedSlots.size > 0 ? remainingSlots : slots; + const independentSlots = remainingSlots.filter((slot) => !deliverySlots.has(slot)); if (independentSlots.length === 0) { - recordPrebidRefreshForDiagnostics(targetSlots); - return dispatchPrebidRefresh(originalRefresh, slots, opts); + remainingSlots.forEach(consumeGptPublisherRefreshSuppression); + recordPrebidRefreshForDiagnostics(remainingSlots); + return dispatchPrebidRefresh(originalRefresh, forwardedSlots, opts); } // Clear stale Trusted Server/Prebid targeting from independent slots before @@ -1421,7 +1725,29 @@ export function installRefreshHandler(timeoutMs = 1500): void { (slot) => !isExcludedFromRefreshAuction(slot, excludedGamAdUnitPathSuffixes) ); if (!auctionSlots.length) { - return originalRefresh(slots, opts); + const immediateSlotCodes = new Map(); + remainingSlots.forEach((slot) => { + const elementId = refreshSlotElementId(slot); + if (elementId) immediateSlotCodes.set(slot, elementId); + }); + const immediateTokens = registerPublisherFirstImpressionAuctions( + (window.tsjs ??= {} as TsjsApi), + immediateSlotCodes.values() + ); + const immediateSuppressedSlots = new Set(); + for (const [slot, elementId] of immediateSlotCodes) { + const token = immediateTokens.get(elementId); + if (token && window.tsjs && consumePublisherFirstImpressionDelivery(window.tsjs, token)) { + immediateSuppressedSlots.add(slot); + } + } + immediateSuppressedSlots.forEach(prepareSuppressedPublisherSlot); + const immediateSlots = remainingSlots.filter((slot) => !immediateSuppressedSlots.has(slot)); + if (immediateSlots.length === 0) return; + immediateSlots.forEach(consumeGptPublisherRefreshSuppression); + const immediateForwardedSlots = + immediateSuppressedSlots.size > 0 ? immediateSlots : forwardedSlots; + return originalRefresh(immediateForwardedSlots, opts); } const adUnits = auctionSlots.map((slot) => { @@ -1464,6 +1790,20 @@ export function installRefreshHandler(timeoutMs = 1500): void { // unrelated GPT slots whose targeting this wrapper only cleared for // `targetSlots` — leaving their next request dependent on stale state. const refreshAdUnitCodes = adUnits.map((unit) => unit.code); + const refreshTs = (window.tsjs ??= {} as TsjsApi); + const refreshGeneration = refreshTs.navGeneration ?? 0; + const delayedRefreshCodes = new Map(); + const delayedRefreshElements = new Map(); + remainingSlots.forEach((slot) => { + const elementId = refreshSlotElementId(slot); + if (elementId) delayedRefreshCodes.set(slot, elementId); + const element = elementId ? resolveFirstImpressionElement(elementId) : undefined; + if (element) delayedRefreshElements.set(slot, element); + }); + const refreshFirstImpressionTokens = registerPublisherFirstImpressionAuctions( + refreshTs, + delayedRefreshCodes.values() + ); adUnits.forEach((unit) => syntheticRefreshAdUnits.add(unit)); // Preserve GPT Single Request Architecture: when a publisher refresh @@ -1477,19 +1817,56 @@ export function installRefreshHandler(timeoutMs = 1500): void { if (completed) return; completed = true; if (fallbackTimer !== undefined) clearTimeout(fallbackTimer); + + // The publisher refresh itself started before this asynchronous auction. + // Reconcile its per-slot token only when the callback is ready to issue + // GPT: TS may have won an already-overlapping first impression while the + // auction was pending, while a publisher-first token prevents TS from + // claiming the slot midway through the same refresh. + const callbackFilteredSlots = new Set(); + const callbackSuppressedSlots = new Set(); + for (const slot of remainingSlots) { + const elementId = delayedRefreshCodes.get(slot); + const token = elementId ? refreshFirstImpressionTokens.get(elementId) : undefined; + const element = delayedRefreshElements.get(slot); + const contextIsStale = Boolean( + element && + ((window.tsjs?.navGeneration ?? 0) !== refreshGeneration || + !element.isConnected || + document.getElementById(element.id) !== element) + ); + const suppress = Boolean( + token && window.tsjs && consumePublisherFirstImpressionDelivery(window.tsjs, token) + ); + if (contextIsStale) { + callbackFilteredSlots.add(slot); + } else if (suppress) { + callbackFilteredSlots.add(slot); + callbackSuppressedSlots.add(slot); + } + } + callbackSuppressedSlots.forEach(prepareSuppressedPublisherSlot); + + const completedSlots = remainingSlots.filter((slot) => !callbackFilteredSlots.has(slot)); + if (completedSlots.length === 0) return; + const completedAdUnitCodes = refreshAdUnitCodes.filter( + (_code, index) => !callbackFilteredSlots.has(auctionSlots[index]) + ); if (applyTargeting) { try { - pbjs.setTargetingForGPTAsync?.(refreshAdUnitCodes); + pbjs.setTargetingForGPTAsync?.(completedAdUnitCodes); } catch (error) { log.error('[tsjs-prebid] refresh targeting failed', error); } } - recordPrebidRefreshForDiagnostics(targetSlots); - // Preserve the publisher's original refresh form. In particular, a bare - // GPT refresh remains bare so GPT resolves its registered slot set when - // the auction completes; the dispatch wrapper only scopes the shared - // diagnostics context around the delegated call. - dispatchPrebidRefresh(originalRefresh, slots, opts); + completedSlots.forEach(consumeGptPublisherRefreshSuppression); + recordPrebidRefreshForDiagnostics(completedSlots); + // Preserve the publisher's original refresh form unless one losing + // first-impression slot was filtered. A delayed bare call must also + // become explicit so slots added after the auction snapshot cannot join. + const completedForwardedSlots = + slots === undefined || callbackFilteredSlots.size > 0 ? completedSlots : forwardedSlots; + dispatchPrebidRefresh(originalRefresh, completedForwardedSlots, opts); } try { diff --git a/crates/trusted-server-js/lib/test/integrations/gpt/ad_init.test.ts b/crates/trusted-server-js/lib/test/integrations/gpt/ad_init.test.ts index b7186518b..e034a8ba4 100644 --- a/crates/trusted-server-js/lib/test/integrations/gpt/ad_init.test.ts +++ b/crates/trusted-server-js/lib/test/integrations/gpt/ad_init.test.ts @@ -6,6 +6,10 @@ import { resolve } from 'node:path'; import { describe, it, expect, vi, beforeEach, afterEach, afterAll } from 'vitest'; import envelope from '../../fixtures/aps-renderer-v1.json'; +import { + registerPublisherFirstImpressionAuctions, + resolveFirstImpressionElement, +} from '../../../src/core/first_impression'; import type { AuctionBidData, TsjsApi } from '../../../src/core/types'; import { APS_PREBID_CREATIVE_RUNNER_URL, @@ -248,6 +252,7 @@ describe('installTsAdInit', () => { const mockSlot = { addService: vi.fn().mockReturnThis(), setTargeting: vi.fn().mockReturnThis(), + clearTargeting: vi.fn().mockReturnThis(), getSlotElementId: vi.fn().mockReturnValue('div-atf-sidebar'), getTargeting: vi.fn().mockReturnValue([]), }; @@ -282,6 +287,105 @@ describe('installTsAdInit', () => { return { mockPubads, mockSlot }; } + it('leaves a publisher-auctioned slot untouched when delayed adInit receives no candidate', async () => { + const recordTrustedServerOpportunity = vi.fn(); + const { mockPubads, mockSlot } = configureOpportunityDiagnostics( + undefined, + recordTrustedServerOpportunity + ); + const ts = (window as TestWindow).tsjs as TsjsApi; + registerPublisherFirstImpressionAuctions(ts, ['div-atf-sidebar']); + + const { installTsAdInit } = await import('../../../src/integrations/gpt/index'); + installTsAdInit(); + ts.adInit!(); + + expect(mockSlot.setTargeting).not.toHaveBeenCalled(); + expect(mockPubads.refresh).not.toHaveBeenCalled(); + expect(recordTrustedServerOpportunity).not.toHaveBeenCalled(); + expect(ts.divToSlotId).toEqual({}); + expect(ts.prevSlotTargetingKeys).toEqual({}); + }); + + it('falls back once when a publisher auction abandons its first-impression claim', async () => { + vi.useFakeTimers(); + try { + const recordTrustedServerOpportunity = vi.fn(); + const { mockPubads, mockSlot } = configureOpportunityDiagnostics( + { hb_pb: '1.10', hb_adid: 'example-fallback-ad', adm: '
Fallback
' }, + recordTrustedServerOpportunity + ); + const ts = (window as TestWindow).tsjs as TsjsApi; + registerPublisherFirstImpressionAuctions(ts, ['div-atf-sidebar']); + const { installTsAdInit } = await import('../../../src/integrations/gpt/index'); + installTsAdInit(); + ts.adInit!(); + + expect(mockPubads.refresh).not.toHaveBeenCalled(); + vi.advanceTimersByTime(5001); + + expect(mockSlot.setTargeting).toHaveBeenCalledWith('ts_initial', '1'); + expect(mockPubads.refresh).toHaveBeenCalledOnce(); + expect(mockPubads.refresh).toHaveBeenCalledWith([mockSlot]); + expect(recordTrustedServerOpportunity).toHaveBeenCalledOnce(); + + vi.advanceTimersByTime(10_000); + expect(mockPubads.refresh).toHaveBeenCalledOnce(); + } finally { + vi.useRealTimers(); + } + }); + + it('does not clear targeting or request again after TS claims an existing slot', async () => { + const recordTrustedServerOpportunity = vi.fn(); + const { mockPubads, mockSlot } = configureOpportunityDiagnostics( + undefined, + recordTrustedServerOpportunity + ); + const ts = (window as TestWindow).tsjs as TsjsApi; + const { installTsAdInit } = await import('../../../src/integrations/gpt/index'); + installTsAdInit(); + + ts.adInit!(); + const clearCalls = mockSlot.clearTargeting.mock.calls.length; + const targetingCalls = mockSlot.setTargeting.mock.calls.length; + ts.adInit!(); + + expect(mockSlot.clearTargeting).toHaveBeenCalledTimes(clearCalls); + expect(mockSlot.setTargeting).toHaveBeenCalledTimes(targetingCalls); + expect(mockPubads.refresh).toHaveBeenCalledOnce(); + expect(recordTrustedServerOpportunity).toHaveBeenCalledOnce(); + }); + + it.each(['slotRequested', 'slotRenderEnded'] as const)( + 'leaves a publisher slot untouched after an earlier %s event', + async (eventName) => { + const recordTrustedServerOpportunity = vi.fn(); + const { mockPubads, mockSlot } = configureOpportunityDiagnostics( + { hb_pb: '2.00', hb_adid: 'late-page-bid', adm: '
Late
' }, + recordTrustedServerOpportunity + ); + const ts = (window as TestWindow).tsjs as TsjsApi; + const { installTsAdInit } = await import('../../../src/integrations/gpt/index'); + installTsAdInit(); + const lifecycleListener = mockPubads.addEventListener.mock.calls.find( + ([registeredEvent]) => registeredEvent === eventName + )?.[1] as ((event: SlotRenderEvent) => void) | undefined; + expect(lifecycleListener).toBeDefined(); + lifecycleListener!({ isEmpty: false, slot: mockSlot }); + + ts.adInit!(); + + expect(mockSlot.setTargeting).not.toHaveBeenCalled(); + expect(mockPubads.refresh).not.toHaveBeenCalled(); + expect(recordTrustedServerOpportunity).not.toHaveBeenCalled(); + expect(ts.firstImpression?.slots['div-atf-sidebar']?.owner).toBe('publisher'); + expect(ts.firstImpression?.slots['div-atf-sidebar']?.phase).toBe( + eventName === 'slotRequested' ? 'requested' : 'rendered' + ); + } + ); + it.each([ [ 'inline markup', @@ -1858,7 +1962,9 @@ describe('installTsAdInit', () => { (window as TestWindow).tsjs!.adInit!(); - expect(nativeRefresh).toHaveBeenCalledWith([mockSlot]); + // The slot already spent its first impression above. Changing GPT's + // initial-load mode must not make a repeated adInit request it again. + expect(nativeRefresh).not.toHaveBeenCalled(); nativeRefresh.mockClear(); gpt.setConfig({ disableInitialLoad: false }); @@ -1877,7 +1983,7 @@ describe('installTsAdInit', () => { (window as TestWindow).tsjs!.adInit!(); - expect(nativeRefresh).toHaveBeenCalledWith([mockSlot]); + expect(nativeRefresh).not.toHaveBeenCalled(); // A later modern call can re-enable initial load after the legacy API. nativeRefresh.mockClear(); @@ -2688,6 +2794,7 @@ describe('installTsAdInit', () => { ) ); const selectedElement = selectedIndex === null ? undefined : elements[selectedIndex]; + expect(resolveFirstImpressionElement(divId)).toBe(selectedElement); const mockSlot = { addService: vi.fn().mockReturnThis(), setTargeting: vi.fn().mockReturnThis(), @@ -3034,6 +3141,23 @@ describe('installTsRenderBridge', () => { return iframe.contentWindow!; } + function createCollapsedTrustedSlotIframe(divId = 'div-header') { + const slot = document.createElement('div'); + slot.id = divId; + const wrapper = document.createElement('div'); + wrapper.style.width = '1px'; + wrapper.style.height = '1px'; + const iframe = document.createElement('iframe'); + iframe.width = '1'; + iframe.height = '1'; + iframe.style.width = '1px'; + iframe.style.height = '1px'; + wrapper.appendChild(iframe); + slot.appendChild(wrapper); + document.body.appendChild(slot); + return { iframe, slot, source: iframe.contentWindow!, wrapper }; + } + async function captureBridgeListener(): Promise<(e: MessageEvent) => unknown> { let bridgeListener: ((e: MessageEvent) => unknown) | undefined; const origAdd = window.addEventListener.bind(window); @@ -3095,6 +3219,99 @@ describe('installTsRenderBridge', () => { expect(recordTrustedServerCreativeFailure).not.toHaveBeenCalled(); }); + it('expands an authenticated collapsed inline creative shell after response delivery', async () => { + const tsjs = (window as TestWindow).tsjs!; + tsjs.bids.homepage_header.adm = '
Fictional creative
'; + tsjs.bids.homepage_header.w = 728; + tsjs.bids.homepage_header.h = 90; + delete tsjs.bids.homepage_header.nurl; + delete tsjs.bids.homepage_header.burl; + const bridgeListener = await captureBridgeListener(); + const collapsed = createCollapsedTrustedSlotIframe(); + const postMessage = vi.fn(); + + bridgeListener( + Object.assign(new Event('message'), { + data: JSON.stringify({ message: 'Prebid Request', adId: 'test-cache-uuid' }), + ports: [{ postMessage }], + source: collapsed.iframe.contentWindow!, + stopImmediatePropagation: vi.fn(), + }) as unknown as MessageEvent + ); + + expect(postMessage).toHaveBeenCalledOnce(); + expect(collapsed.iframe.width).toBe('728'); + expect(collapsed.iframe.height).toBe('90'); + expect(collapsed.wrapper.style.width).toBe('728px'); + expect(collapsed.wrapper.style.height).toBe('90px'); + }); + + it('expands every collapsed ancestor through the authenticated slot root', async () => { + const tsjs = (window as TestWindow).tsjs!; + tsjs.bids.homepage_header.adm = '
Fictional creative
'; + tsjs.bids.homepage_header.w = 728; + tsjs.bids.homepage_header.h = 90; + delete tsjs.bids.homepage_header.nurl; + delete tsjs.bids.homepage_header.burl; + const bridgeListener = await captureBridgeListener(); + const collapsed = createCollapsedTrustedSlotIframe(); + const outerWrapper = document.createElement('div'); + outerWrapper.style.width = '1px'; + outerWrapper.style.height = '1px'; + collapsed.slot.insertBefore(outerWrapper, collapsed.wrapper); + outerWrapper.appendChild(collapsed.wrapper); + + bridgeListener( + Object.assign(new Event('message'), { + data: JSON.stringify({ message: 'Prebid Request', adId: 'test-cache-uuid' }), + ports: [{ postMessage: vi.fn() }], + source: collapsed.iframe.contentWindow!, + stopImmediatePropagation: vi.fn(), + }) as unknown as MessageEvent + ); + + expect(collapsed.wrapper.style.width).toBe('728px'); + expect(collapsed.wrapper.style.height).toBe('90px'); + expect(outerWrapper.style.width).toBe('728px'); + expect(outerWrapper.style.height).toBe('90px'); + }); + + it.each(['fixed', 'anchor', 'expanded', 'oversized'] as const)( + 'does not resize a %s Universal Creative shell', + async (guard) => { + const tsjs = (window as TestWindow).tsjs!; + tsjs.bids.homepage_header.adm = '
Fictional creative
'; + tsjs.bids.homepage_header.w = guard === 'oversized' ? 10_001 : 300; + tsjs.bids.homepage_header.h = 250; + delete tsjs.bids.homepage_header.nurl; + delete tsjs.bids.homepage_header.burl; + const bridgeListener = await captureBridgeListener(); + const collapsed = createCollapsedTrustedSlotIframe(); + if (guard === 'fixed') collapsed.iframe.style.position = 'fixed'; + if (guard === 'expanded') collapsed.iframe.style.width = '300px'; + if (guard === 'anchor') { + const anchor = document.createElement('ins'); + anchor.dataset.anchorStatus = 'displayed'; + collapsed.slot.insertBefore(anchor, collapsed.wrapper); + anchor.appendChild(collapsed.wrapper); + } + + bridgeListener( + Object.assign(new Event('message'), { + data: JSON.stringify({ message: 'Prebid Request', adId: 'test-cache-uuid' }), + ports: [{ postMessage: vi.fn() }], + source: collapsed.source, + stopImmediatePropagation: vi.fn(), + }) as unknown as MessageEvent + ); + + expect(collapsed.iframe.width).toBe('1'); + expect(collapsed.iframe.height).toBe('1'); + expect(collapsed.wrapper.style.width).toBe('1px'); + expect(collapsed.wrapper.style.height).toBe('1px'); + } + ); + it('records no creative evidence for an ad ID the requesting slot does not own', async () => { const recordTrustedServerCreativeRequest = vi.fn().mockReturnValue(42); const recordTrustedServerCreativeResponse = vi.fn(); @@ -3195,7 +3412,7 @@ describe('installTsRenderBridge', () => { expect(fetchStub).not.toHaveBeenCalled(); }); - it('records response_post_failed when posting inline markup throws', async () => { + it('records response_post_failed without resizing when posting inline markup throws', async () => { const beaconSpy = vi.spyOn(navigator, 'sendBeacon').mockReturnValue(true); const recordTrustedServerCreativeRequest = vi.fn().mockReturnValue(46); const recordTrustedServerCreativeResponse = vi.fn(); @@ -3209,7 +3426,7 @@ describe('installTsRenderBridge', () => { tsjs.bids.homepage_header.adm = '
Creative
'; const bridgeListener = await captureBridgeListener(); - const source = createTrustedSlotIframe(); + const collapsed = createCollapsedTrustedSlotIframe(); const stopImmediatePropagation = vi.fn(); expect(() => bridgeListener( @@ -3222,7 +3439,7 @@ describe('installTsRenderBridge', () => { }), }, ], - source, + source: collapsed.source, stopImmediatePropagation, }) as unknown as MessageEvent ) @@ -3232,6 +3449,8 @@ describe('installTsRenderBridge', () => { expect(recordTrustedServerCreativeFailure).toHaveBeenCalledTimes(1); expect(recordTrustedServerCreativeFailure).toHaveBeenCalledWith(46, 'response_post_failed'); expect(recordTrustedServerCreativeResponse).not.toHaveBeenCalled(); + expect(collapsed.iframe.width).toBe('1'); + expect(collapsed.iframe.height).toBe('1'); expect(beaconSpy).not.toHaveBeenCalled(); beaconSpy.mockRestore(); }); @@ -3252,7 +3471,8 @@ describe('installTsRenderBridge', () => { const beaconSpy = vi.spyOn(navigator, 'sendBeacon').mockReturnValue(true); const bridgeListener = await captureBridgeListener(); - const source = createTrustedSlotIframe(); + const collapsed = createCollapsedTrustedSlotIframe(); + const source = collapsed.source; const stopSpy = vi.fn(); const portMessages: string[] = []; const fakePort = { postMessage: (message: string) => portMessages.push(message) }; @@ -3297,6 +3517,10 @@ describe('installTsRenderBridge', () => { }); expect(String(response.renderer)).not.toContain(renderer.accountId); expect(String(response.renderer)).not.toContain(renderer.aaxResponse); + expect(collapsed.iframe.width).toBe('300'); + expect(collapsed.iframe.height).toBe('250'); + expect(collapsed.wrapper.style.width).toBe('300px'); + expect(collapsed.wrapper.style.height).toBe('250px'); // Universal Creative's dynamic-renderer path evaluates the returned static // source and calls window.render(response, helper, targetWindow). Consume @@ -3417,7 +3641,8 @@ describe('installTsRenderBridge', () => { }; const bridgeListener = await captureBridgeListener(); - const source = createTrustedSlotIframe(); + const collapsed = createCollapsedTrustedSlotIframe(); + const source = collapsed.source; const stopSpy = vi.fn(); const portMessages: string[] = []; const event = Object.assign(new Event('message'), { @@ -3453,6 +3678,10 @@ describe('installTsRenderBridge', () => { ); expect(renderer.bidId).not.toBe(prebidAdId); expect((window as TestWindow).tsjs.apsPrebidRenderers[prebidAdId]).toBeUndefined(); + expect(collapsed.iframe.width).toBe('300'); + expect(collapsed.iframe.height).toBe('250'); + expect(collapsed.wrapper.style.width).toBe('300px'); + expect(collapsed.wrapper.style.height).toBe('250px'); expect(fetchStub).not.toHaveBeenCalled(); foreignIframe.remove(); }); @@ -3542,7 +3771,7 @@ describe('installTsRenderBridge', () => { } }); - it('uses the requesting frame to resolve a registered APS dynamic slot prefix', async () => { + it('uses the requesting frame to disambiguate a registered APS slot prefix', async () => { const renderer = apsRenderer(); const prebidAdId = 'native-dynamic-prebid-ad-id'; const markUsed = vi.fn(); @@ -3556,7 +3785,7 @@ describe('installTsRenderBridge', () => { }, }; const marker = enablePublisherNativeMode(); - const firstSource = createTrustedSlotIframe('div-native-first'); + createTrustedSlotIframe('div-native-first'); const source = createTrustedSlotIframe('div-native-second'); try { @@ -3569,6 +3798,7 @@ describe('installTsRenderBridge', () => { stopImmediatePropagation: vi.fn(), }) as unknown as MessageEvent ); + const native = nativeRunnerIn('div-native-second'); native.runner.dispatchEvent(new Event('load')); await Promise.resolve(); @@ -3576,11 +3806,7 @@ describe('installTsRenderBridge', () => { expect(native.frame.style.display).toBe(''); expect(markUsed).toHaveBeenCalledOnce(); - expect( - Array.from(document.querySelectorAll('#div-native-first iframe')).some( - (frame) => frame.contentWindow === firstSource - ) - ).toBe(true); + expect((window as TestWindow).tsjs.apsPrebidRenderers[prebidAdId]).toBeUndefined(); } finally { marker.remove(); document.getElementById('div-native-first')?.remove(); @@ -4274,7 +4500,86 @@ describe('installTsRenderBridge', () => { expect(fetchStub).not.toHaveBeenCalled(); }); - it('uses the adInit-resolved div when a responsive prefix becomes ambiguous', async () => { + it('uses the requesting frame to resolve inline adm under an ambiguous prefix', async () => { + const beaconSpy = vi.spyOn(navigator, 'sendBeacon').mockReturnValue(true); + const tsjs = (window as TestWindow).tsjs!; + tsjs.bids.homepage_header.adm = '
Prefix inline creative
'; + delete tsjs.bids.homepage_header.hb_cache_host; + delete tsjs.bids.homepage_header.hb_cache_path; + tsjs.adSlots = [ + { + id: 'homepage_header', + formats: [[728, 90]], + gam_unit_path: '/a/b/c', + div_id: 'div-inline-prefix-', + targeting: {}, + }, + ]; + tsjs.divToSlotId = {}; + createTrustedSlotIframe('div-inline-prefix-first'); + const source = createTrustedSlotIframe('div-inline-prefix-second'); + const bridgeListener = await captureBridgeListener(); + const postMessage = vi.fn(); + const stopImmediatePropagation = vi.fn(); + + bridgeListener( + Object.assign(new Event('message'), { + data: JSON.stringify({ message: 'Prebid Request', adId: 'test-cache-uuid' }), + ports: [{ postMessage }], + source, + stopImmediatePropagation, + }) as unknown as MessageEvent + ); + + expect(postMessage).toHaveBeenCalledOnce(); + expect(JSON.parse(postMessage.mock.calls[0]![0])).toEqual( + expect.objectContaining({ ad: '
Prefix inline creative
' }) + ); + expect(stopImmediatePropagation).toHaveBeenCalledOnce(); + expect(fetchStub).not.toHaveBeenCalled(); + beaconSpy.mockRestore(); + }); + + it('rejects a requesting frame owned by multiple prefix candidates', async () => { + const tsjs = (window as TestWindow).tsjs!; + tsjs.bids.homepage_header.adm = '
Ambiguous inline creative
'; + tsjs.adSlots = [ + { + id: 'homepage_header', + formats: [[728, 90]], + gam_unit_path: '/a/b/c', + div_id: 'div-nested-prefix-', + targeting: {}, + }, + ]; + tsjs.divToSlotId = {}; + const outer = document.createElement('div'); + outer.id = 'div-nested-prefix-outer'; + const inner = document.createElement('div'); + inner.id = 'div-nested-prefix-inner'; + const iframe = document.createElement('iframe'); + inner.appendChild(iframe); + outer.appendChild(inner); + document.body.appendChild(outer); + const bridgeListener = await captureBridgeListener(); + const postMessage = vi.fn(); + const stopImmediatePropagation = vi.fn(); + + bridgeListener( + Object.assign(new Event('message'), { + data: JSON.stringify({ message: 'Prebid Request', adId: 'test-cache-uuid' }), + ports: [{ postMessage }], + source: iframe.contentWindow, + stopImmediatePropagation, + }) as unknown as MessageEvent + ); + + expect(postMessage).not.toHaveBeenCalled(); + expect(stopImmediatePropagation).not.toHaveBeenCalled(); + expect(fetchStub).not.toHaveBeenCalled(); + }); + + it('uses the requesting frame when a responsive prefix is ambiguous', async () => { const beaconSpy = vi.spyOn(navigator, 'sendBeacon').mockReturnValue(true); fetchStub.mockResolvedValue({ ok: true, @@ -4299,9 +4604,7 @@ describe('installTsRenderBridge', () => { targeting: {}, }, ]; - (window as TestWindow).tsjs!.divToSlotId = { - 'div-responsive-a': 'homepage_header', - }; + (window as TestWindow).tsjs!.divToSlotId = {}; const bridgeListener = await captureBridgeListener(); const portMessages: string[] = []; @@ -4409,7 +4712,7 @@ describe('installTsRenderBridge', () => { beaconSpy.mockRestore(); }); - it('sizes a PBS Cache render from the cached bid dimensions', async () => { + it('sizes a PBS Cache render and its collapsed shell from cached bid dimensions', async () => { const beaconSpy = vi.spyOn(navigator, 'sendBeacon').mockReturnValue(true); // Cached bid is 300x250 while the slot's first format is 728x90 (from the // default setup). The response must use the cached dimensions. @@ -4421,13 +4724,13 @@ describe('installTsRenderBridge', () => { const bridgeListener = await captureBridgeListener(); const portMessages: string[] = []; const fakePort = { postMessage: (s: string) => portMessages.push(s) }; - const source = createTrustedSlotIframe(); + const collapsed = createCollapsedTrustedSlotIframe(); bridgeListener( Object.assign(new Event('message'), { data: JSON.stringify({ message: 'Prebid Request', adId: 'test-cache-uuid' }), ports: [fakePort], - source, + source: collapsed.source, stopImmediatePropagation: vi.fn(), }) as unknown as MessageEvent ); @@ -4438,6 +4741,52 @@ describe('installTsRenderBridge', () => { const parsed = JSON.parse(portMessages[0]) as PrebidResponseMessage; expect(parsed.width).toBe(300); expect(parsed.height).toBe(250); + expect(collapsed.iframe.width).toBe('300'); + expect(collapsed.iframe.height).toBe('250'); + expect(collapsed.wrapper.style.width).toBe('300px'); + expect(collapsed.wrapper.style.height).toBe('250px'); + beaconSpy.mockRestore(); + }); + + it('does not resize a stale cache response after navigation', async () => { + const recordTrustedServerCreativeResponse = vi.fn(); + (window as TestWindow).tsjs!.gptDiagnosticsRecorder = { + recordTrustedServerCreativeRequest: vi.fn().mockReturnValue(91), + recordTrustedServerCreativeResponse, + recordTrustedServerCreativeFailure: vi.fn(), + } as unknown as TsjsApi['gptDiagnosticsRecorder']; + const beaconSpy = vi.spyOn(navigator, 'sendBeacon').mockReturnValue(true); + let resolveText: ((body: string) => void) | undefined; + fetchStub.mockResolvedValue({ + ok: true, + text: () => + new Promise((resolve) => { + resolveText = resolve; + }), + } as Response); + const bridgeListener = await captureBridgeListener(); + const collapsed = createCollapsedTrustedSlotIframe(); + const postMessage = vi.fn(); + + bridgeListener( + Object.assign(new Event('message'), { + data: JSON.stringify({ message: 'Prebid Request', adId: 'test-cache-uuid' }), + ports: [{ postMessage }], + source: collapsed.source, + stopImmediatePropagation: vi.fn(), + }) as unknown as MessageEvent + ); + await Promise.resolve(); + expect(resolveText).toBeDefined(); + (window as TestWindow).tsjs!.navGeneration = 1; + resolveText?.(JSON.stringify({ adm: '
cached
', w: 300, h: 250 })); + await new Promise((resolve) => setTimeout(resolve, 0)); + + expect(postMessage).not.toHaveBeenCalled(); + expect(recordTrustedServerCreativeResponse).not.toHaveBeenCalled(); + expect(beaconSpy).not.toHaveBeenCalled(); + expect(collapsed.iframe.width).toBe('1'); + expect(collapsed.iframe.height).toBe('1'); beaconSpy.mockRestore(); }); diff --git a/crates/trusted-server-js/lib/test/integrations/gpt/gpt_bootstrap.test.ts b/crates/trusted-server-js/lib/test/integrations/gpt/gpt_bootstrap.test.ts index ab6d646f2..55fd61634 100644 --- a/crates/trusted-server-js/lib/test/integrations/gpt/gpt_bootstrap.test.ts +++ b/crates/trusted-server-js/lib/test/integrations/gpt/gpt_bootstrap.test.ts @@ -3,7 +3,8 @@ import path from 'node:path'; import { describe, it, expect, vi, beforeEach, afterEach } from 'vitest'; -import type { TsjsApi } from '../../../src/core/types'; +import { FIRST_IMPRESSION_LEASE_MS } from '../../../src/core/first_impression'; +import type { FirstImpressionSlotClaim, TsjsApi } from '../../../src/core/types'; /** * Executable coverage for the edge-injected `gpt_bootstrap.js` — the @@ -33,6 +34,7 @@ interface MockGoogleTag { pubads: () => unknown; enableServices: () => void; display: (divId: string) => void; + getConfig?: (key: string) => Record; setConfig?: (config: Record) => void; } @@ -233,6 +235,278 @@ describe('gpt_bootstrap.js fallback', () => { expect((window as TestWindow).tsjs!.gptInitialLoadDisabled).toBe(true); }); + it('keeps the bootstrap lease synchronized with the bundle contract', () => { + const bootstrapLease = /var FIRST_IMPRESSION_LEASE_MS = (\d+);/.exec(BOOTSTRAP_SOURCE); + + expect(Number(bootstrapLease?.[1])).toBe(FIRST_IMPRESSION_LEASE_MS); + }); + + it('clears the bootstrap fallback reservation when transitioned slot setup fails', () => { + vi.useFakeTimers(); + vi.setSystemTime(100); + try { + const pubads = { + getSlots: vi.fn(() => []), + refresh: vi.fn(), + }; + (window as TestWindow).googletag = makeGoogleTag({ + cmd: { push: (command) => command() }, + defineSlot: vi.fn(() => null), + pubads: vi.fn(() => pubads), + }); + document.body.innerHTML = '
'; + + runBootstrap(); + const ts = (window as TestWindow).tsjs!; + const element = document.getElementById('failed-bootstrap-fallback')!; + const publisherClaim: FirstImpressionSlotClaim = { + generation: 0, + slotElementId: element.id, + element, + owner: 'publisher', + phase: 'auctioning', + expiresAt: 5_100, + publisherAuctions: { + original: { + token: 'original', + adUnitCode: element.id, + phase: 'auctioning', + expiresAt: 5_100, + adIds: [], + suppressDelivery: false, + }, + }, + }; + ts.firstImpression = { + generation: 0, + nextToken: 1, + slots: { [element.id]: publisherClaim }, + fallbackSlots: {}, + }; + ts.adSlots = [ + { + id: 'failed-bootstrap-fallback-ad', + gam_unit_path: '/123/failed-bootstrap-fallback', + div_id: element.id, + formats: [[300, 250]], + targeting: {}, + }, + ]; + ts.bids = { 'failed-bootstrap-fallback-ad': { hb_pb: '1.00' } }; + + ts.adInit!(); + expect(ts.firstImpression.fallbackSlots[element.id]).toBe(element); + + vi.advanceTimersByTime(5_001); + + expect(ts.firstImpression.slots[element.id]).toBeUndefined(); + expect(ts.firstImpression.fallbackSlots[element.id]).toBeUndefined(); + } finally { + vi.clearAllTimers(); + vi.useRealTimers(); + } + }); + + it('retains an expired TS suppression tombstone in the persistent bootstrap listener', () => { + const queue: Array<() => void> = []; + const listeners = new Map void>(); + const pubads = { + addEventListener: vi.fn((name: string, listener: (event: never) => void) => { + listeners.set(name, listener as (event: { slot: { getSlotElementId(): string } }) => void); + }), + getSlots: vi.fn(() => []), + refresh: vi.fn(), + }; + (window as TestWindow).googletag = makeGoogleTag({ + cmd: queue, + pubads: vi.fn(() => pubads), + }); + document.body.innerHTML = '
'; + + runBootstrap(); + [...queue].forEach((command) => command()); + const element = document.getElementById('persistent-slot')!; + const claim: FirstImpressionSlotClaim = { + generation: 0, + slotElementId: element.id, + element, + owner: 'trusted_server', + phase: 'delivery_pending', + expiresAt: 0, + publisherAuctions: { + late: { + token: 'late', + adUnitCode: element.id, + phase: 'delivery_pending', + expiresAt: 0, + adIds: ['late-ad'], + suppressDelivery: true, + }, + }, + }; + (window as TestWindow).tsjs!.firstImpression = { + generation: 0, + nextToken: 1, + slots: { [element.id]: claim }, + fallbackSlots: {}, + }; + + listeners.get('slotRequested')!({ slot: { getSlotElementId: () => element.id } }); + + expect(claim.publisherAuctions.late).toBeDefined(); + expect(claim.publisherRegistrationClosed).toBe(true); + }); + + it('prunes a malformed bootstrap registry key before recording the main-document slot', () => { + const queue: Array<() => void> = []; + const listeners = new Map void>(); + const pubads = { + addEventListener: vi.fn((name: string, listener: (event: never) => void) => { + listeners.set(name, listener as (event: { slot: { getSlotElementId(): string } }) => void); + }), + getSlots: vi.fn(() => []), + refresh: vi.fn(), + }; + (window as TestWindow).googletag = makeGoogleTag({ + cmd: queue, + pubads: vi.fn(() => pubads), + }); + document.body.innerHTML = '
'; + + runBootstrap(); + [...queue].forEach((command) => command()); + const element = document.getElementById('malformed-bootstrap-slot')!; + const malformedClaim: FirstImpressionSlotClaim = { + generation: 0, + slotElementId: element.id, + element, + owner: 'trusted_server', + phase: 'delivery_pending', + expiresAt: Number.POSITIVE_INFINITY, + publisherAuctions: {}, + }; + (window as TestWindow).tsjs!.firstImpression = { + generation: 0, + nextToken: 0, + slots: { 'wrong-registry-key': malformedClaim }, + fallbackSlots: {}, + }; + + listeners.get('slotRequested')!({ slot: { getSlotElementId: () => element.id } }); + + const slots = (window as TestWindow).tsjs!.firstImpression!.slots; + expect(slots['wrong-registry-key']).toBeUndefined(); + expect(slots[element.id]).toEqual( + expect.objectContaining({ element, owner: 'publisher', phase: 'requested' }) + ); + }); + + it('rejects a connected same-ID bootstrap claim from a foreign document', () => { + const queue: Array<() => void> = []; + const listeners = new Map void>(); + const pubads = { + addEventListener: vi.fn((name: string, listener: (event: never) => void) => { + listeners.set(name, listener as (event: { slot: { getSlotElementId(): string } }) => void); + }), + getSlots: vi.fn(() => []), + refresh: vi.fn(), + }; + (window as TestWindow).googletag = makeGoogleTag({ + cmd: queue, + pubads: vi.fn(() => pubads), + }); + document.body.innerHTML = '
'; + + runBootstrap(); + [...queue].forEach((command) => command()); + const element = document.getElementById('foreign-bootstrap-slot')!; + const foreignDocument = document.implementation.createHTMLDocument('foreign'); + const foreignElement = foreignDocument.createElement('div'); + foreignElement.id = element.id; + foreignDocument.body.appendChild(foreignElement); + const foreignClaim: FirstImpressionSlotClaim = { + generation: 0, + slotElementId: element.id, + element: foreignElement, + owner: 'trusted_server', + phase: 'delivery_pending', + expiresAt: 0, + publisherAuctions: { + foreign: { + token: 'foreign', + adUnitCode: element.id, + phase: 'delivery_pending', + expiresAt: 0, + adIds: ['foreign-ad'], + suppressDelivery: true, + }, + }, + }; + (window as TestWindow).tsjs!.firstImpression = { + generation: 0, + nextToken: 1, + slots: { [element.id]: foreignClaim }, + fallbackSlots: {}, + }; + + expect(foreignElement.isConnected).toBe(true); + listeners.get('slotRequested')!({ slot: { getSlotElementId: () => element.id } }); + + const currentClaim = (window as TestWindow).tsjs!.firstImpression!.slots[element.id]; + expect(currentClaim).toEqual( + expect.objectContaining({ element, owner: 'publisher', phase: 'requested' }) + ); + expect(currentClaim!.publisherAuctions).toEqual({}); + }); + + it('refuses a 257th bootstrap lifecycle claim without evicting live claims', () => { + const queue: Array<() => void> = []; + const listeners = new Map void>(); + const pubads = { + addEventListener: vi.fn((name: string, listener: (event: never) => void) => { + listeners.set(name, listener as (event: { slot: { getSlotElementId(): string } }) => void); + }), + getSlots: vi.fn(() => []), + refresh: vi.fn(), + }; + (window as TestWindow).googletag = makeGoogleTag({ + cmd: queue, + pubads: vi.fn(() => pubads), + }); + + runBootstrap(); + [...queue].forEach((command) => command()); + const slots: Record = {}; + for (let index = 0; index < 256; index += 1) { + const element = document.createElement('div'); + element.id = `bounded-slot-${index}`; + document.body.appendChild(element); + slots[element.id] = { + generation: 0, + slotElementId: element.id, + element, + owner: 'publisher', + phase: 'rendered', + expiresAt: Number.POSITIVE_INFINITY, + publisherAuctions: {}, + }; + } + (window as TestWindow).tsjs!.firstImpression = { + generation: 0, + nextToken: 0, + slots, + fallbackSlots: {}, + }; + const overflow = document.createElement('div'); + overflow.id = 'bounded-slot-overflow'; + document.body.appendChild(overflow); + + listeners.get('slotRequested')!({ slot: { getSlotElementId: () => overflow.id } }); + + expect(Object.keys(slots)).toHaveLength(256); + expect(slots[overflow.id]).toBeUndefined(); + }); + it('installs fallback adInit and scheduleInitialAdInit when the bundle is absent', () => { runBootstrap(); const ts = (window as TestWindow).tsjs!; @@ -419,6 +693,7 @@ describe('gpt_bootstrap.js fallback', () => { gam_unit_path: '/123/atf', div_id: 'div-atf-sidebar', formats: [[300, 250]], + targeting: { ts_route: 'home' }, }, ]; ts.bids = { atf_sidebar_ad: { hb_pb: '1.00' } }; @@ -428,10 +703,71 @@ describe('gpt_bootstrap.js fallback', () => { expect(defineSlot).toHaveBeenCalledWith('/123/atf', [[300, 250]], 'div-atf-sidebar'); expect(mockSlot.setTargeting).toHaveBeenCalledWith('hb_pb', '1.00'); expect(mockSlot.setTargeting).toHaveBeenCalledWith('ts_initial', '1'); + expect(mockSlot.setTargeting).toHaveBeenCalledWith('ts_route', 'home'); + expect(ts.prevSlotTargetingKeys).toEqual({ 'div-atf-sidebar': ['ts_route'] }); expect(display).toHaveBeenCalledWith('div-atf-sidebar'); expect(ts.servicesEnabled).toBe(true); }); + it('fallback adInit leaves a publisher-rendered slot untouched', () => { + const mockSlot = { + addService: vi.fn().mockReturnThis(), + setTargeting: vi.fn().mockReturnThis(), + getSlotElementId: vi.fn().mockReturnValue('div-atf-sidebar'), + }; + const mockPubads = { + addEventListener: vi.fn(), + enableSingleRequest: vi.fn(), + getSlots: vi.fn().mockReturnValue([mockSlot]), + refresh: vi.fn(), + }; + const nativeRefresh = mockPubads.refresh; + const defineSlot = vi.fn(); + (window as TestWindow).googletag = { + cmd: { push: vi.fn((fn: () => void) => fn()) }, + defineSlot, + pubads: vi.fn().mockReturnValue(mockPubads), + enableServices: vi.fn(), + display: vi.fn(), + }; + document.body.innerHTML = '
'; + runBootstrap(); + const ts = (window as TestWindow).tsjs!; + const element = document.getElementById('div-atf-sidebar')!; + ts.firstImpression = { + generation: 0, + nextToken: 0, + fallbackSlots: {}, + slots: { + 'div-atf-sidebar': { + generation: 0, + slotElementId: 'div-atf-sidebar', + element, + owner: 'publisher', + phase: 'rendered', + expiresAt: Number.POSITIVE_INFINITY, + publisherAuctions: {}, + }, + }, + }; + ts.adSlots = [ + { + id: 'atf_sidebar_ad', + gam_unit_path: '/123/atf', + div_id: 'div-atf-sidebar', + formats: [[300, 250]], + }, + ]; + ts.bids = { atf_sidebar_ad: { hb_pb: '1.00' } }; + + ts.adInit!(); + + expect(mockSlot.setTargeting).not.toHaveBeenCalled(); + expect(nativeRefresh).not.toHaveBeenCalled(); + expect(defineSlot).not.toHaveBeenCalled(); + expect(ts.servicesEnabled).not.toBe(true); + }); + it('fallback adInit cancels queued work when the generation advances before the queue drains', () => { const commandQueue: Array<() => void> = []; const nativeRefresh = vi.fn(); diff --git a/crates/trusted-server-js/lib/test/integrations/gpt/index.test.ts b/crates/trusted-server-js/lib/test/integrations/gpt/index.test.ts index f684d7188..1b6488f73 100644 --- a/crates/trusted-server-js/lib/test/integrations/gpt/index.test.ts +++ b/crates/trusted-server-js/lib/test/integrations/gpt/index.test.ts @@ -612,7 +612,7 @@ describe('GPT GAM attribution bundle fallback', () => { expect(typeof win.tsjs?.adInit).toBe('function'); expect(typeof win.tsjs?.scheduleInitialAdInit).toBe('function'); expect(win.tsjs?.spaHookInstalled).toBe(true); - expect(addEventListenerSpy).toHaveBeenCalledWith('popstate', expect.any(Function)); + expect(addEventListenerSpy).toHaveBeenCalledWith('popstate', expect.any(Function), true); expect(addEventListenerSpy).toHaveBeenCalledWith('load', expect.any(Function)); expect(addEventListenerSpy).toHaveBeenCalledWith('message', expect.any(Function)); if (setConfig) { diff --git a/crates/trusted-server-js/lib/test/integrations/gpt/spa_hook.test.ts b/crates/trusted-server-js/lib/test/integrations/gpt/spa_hook.test.ts index 314348fa8..979b5b0c7 100644 --- a/crates/trusted-server-js/lib/test/integrations/gpt/spa_hook.test.ts +++ b/crates/trusted-server-js/lib/test/integrations/gpt/spa_hook.test.ts @@ -61,7 +61,7 @@ describe('installSpaAuctionHook', () => { // Drop any ad containers inserted by a test so DOM state does not leak. document.body.innerHTML = ''; // Remove this test's popstate listener(s) so they do not fire in later tests. - popstateHandlers.forEach((handler) => window.removeEventListener('popstate', handler)); + popstateHandlers.forEach((handler) => window.removeEventListener('popstate', handler, true)); popstateHandlers = []; vi.restoreAllMocks(); vi.unstubAllGlobals(); @@ -237,9 +237,9 @@ describe('installSpaAuctionHook', () => { expect(adInit).not.toHaveBeenCalled(); }); - it('runs adInit on an empty page-bids response when prior TS state exists', async () => { - // When TS touched slots on a previous navigation, an empty response still - // needs adInit() to sweep the stale TS targeting from those slots. + it('does not defer cleanup to adInit when an empty response has only prior targeting', async () => { + // Navigation clears prior targeting synchronously, so an empty response + // does not need adInit when TS owns no slots that still require destruction. fetchStub.mockResolvedValue({ ok: true, json: async () => ({ slots: [], bids: {} }), @@ -255,7 +255,103 @@ describe('installSpaAuctionHook', () => { await flushAsync(); expect(ts.adSlots).toEqual([]); - expect(adInit).toHaveBeenCalledTimes(1); + expect(adInit).not.toHaveBeenCalled(); + }); + + it('clears prior targeting before page-bids resolves without touching new publisher targeting', async () => { + let resolveFetch: ((response: Response) => void) | undefined; + fetchStub.mockImplementation( + () => + new Promise((resolve) => { + resolveFetch = resolve; + }) + ); + const element = document.createElement('div'); + element.id = 'div-route-slot'; + document.body.appendChild(element); + const clearTargeting = vi.fn(); + const gptSlot = { + addService: vi.fn().mockReturnThis(), + clearTargeting, + getSlotElementId: vi.fn().mockReturnValue(element.id), + getTargeting: vi.fn().mockReturnValue([]), + setTargeting: vi.fn().mockReturnThis(), + }; + const pubads = { + addEventListener: vi.fn(), + enableSingleRequest: vi.fn(), + getSlots: vi.fn().mockReturnValue([gptSlot]), + refresh: vi.fn(), + }; + (window as TestWindow).googletag = { + cmd: { push: vi.fn((fn: () => void) => fn()) }, + defineSlot: vi.fn().mockReturnValue(gptSlot), + destroySlots: vi.fn(), + display: vi.fn(), + enableServices: vi.fn(), + pubads: vi.fn().mockReturnValue(pubads), + }; + + const { installSpaAuctionHook, installTsAdInit } = await importGptModule(); + installTsAdInit(); + installSpaAuctionHook(); + const ts = (window as TestWindow).tsjs!; + ts.prevSlotTargetingKeys = { [element.id]: ['ts_route'] }; + ts.divToSlotId = { [element.id]: 'route_slot' }; + + history.pushState({}, '', '/publisher-route'); + + expect(clearTargeting.mock.calls.map(([key]) => key)).toEqual([ + 'hb_pb', + 'hb_bidder', + 'hb_adid', + 'hb_cache_host', + 'hb_cache_path', + 'ts_initial', + 'ts_route', + ]); + expect(ts.prevSlotTargetingKeys).toEqual({}); + expect(ts.divToSlotId).toEqual({}); + const cleanupCallCount = clearTargeting.mock.calls.length; + + ts.firstImpression = { + generation: 1, + nextToken: 0, + fallbackSlots: {}, + slots: { + [element.id]: { + generation: 1, + slotElementId: element.id, + element, + owner: 'publisher', + phase: 'auctioning', + expiresAt: Date.now() + 5000, + publisherAuctions: {}, + }, + }, + }; + gptSlot.setTargeting('hb_adid', 'publisher-current'); + resolveFetch!( + new Response( + JSON.stringify({ + slots: [ + { + id: 'route_slot', + gam_unit_path: '/123/route', + div_id: element.id, + formats: [[300, 250]], + targeting: {}, + }, + ], + bids: {}, + }), + { status: 200, headers: { 'Content-Type': 'application/json' } } + ) + ); + await flushAsync(); + + expect(clearTargeting).toHaveBeenCalledTimes(cleanupCallCount); + expect(gptSlot.setTargeting).toHaveBeenCalledWith('hb_adid', 'publisher-current'); }); it('defers applying bids until the route ad container is inserted', async () => { diff --git a/crates/trusted-server-js/lib/test/integrations/prebid/index.test.ts b/crates/trusted-server-js/lib/test/integrations/prebid/index.test.ts index 8ead01aa8..e98a7d711 100644 --- a/crates/trusted-server-js/lib/test/integrations/prebid/index.test.ts +++ b/crates/trusted-server-js/lib/test/integrations/prebid/index.test.ts @@ -44,7 +44,7 @@ interface InjectedPrebidTestConfig { accountId?: string; timeout?: number; debug?: boolean; - bidders?: string[]; + serverSideBidders?: string[]; clientSideBidders?: string[]; excludedGamAdUnitPathSuffixes?: unknown; } @@ -200,8 +200,19 @@ import { installPrebidNpm, installRefreshHandler, } from '../../../src/integrations/prebid/index'; +import { installTsAdInit } from '../../../src/integrations/gpt/index'; import type { AuctionBid } from '../../../src/core/auction'; +import { + claimFirstImpressionForTrustedServer, + consumePublisherFirstImpressionDelivery, + firstImpressionClaim, + observeFirstImpressionGptLifecycle, + registerPublisherFirstImpressionAuctions, + releaseTrustedServerFirstImpressionClaim, + reservePublisherFirstImpressionFallback, +} from '../../../src/core/first_impression'; import { log } from '../../../src/core/log'; +import type { TsjsApi } from '../../../src/core/types'; import { GptDiagnosticsObserver } from '../../../src/integrations/gpt_diagnostics/observer'; import { GptDiagnosticsStore } from '../../../src/integrations/gpt_diagnostics/store'; import envelope from '../../fixtures/aps-renderer-v1.json'; @@ -317,6 +328,7 @@ describe('prebid/auctionBidsToPrebidBids', () => { bidderCode: 'aps', ad: '', trustedServerRenderer: renderer, + meta: expect.objectContaining({ trustedServerRenderer: renderer }), }) ); }); @@ -439,23 +451,24 @@ describe('prebid/installPrebidNpm', () => { ); }); - it('registers accepted APS descriptors under Prebid generated ad IDs', () => { + it('registers normalized APS descriptors at bidAccepted under Prebid generated ad IDs', () => { installPrebidNpm(); - const bidResponseListener = mockOnEvent.mock.calls.find( - ([eventName]) => eventName === 'bidResponse' + const bidAcceptedListener = mockOnEvent.mock.calls.find( + ([eventName]) => eventName === 'bidAccepted' )?.[1] as ((bid: Record) => void) | undefined; - expect(bidResponseListener).toBeTypeOf('function'); + expect(bidAcceptedListener).toBeTypeOf('function'); const renderer = apsRenderer(); - bidResponseListener!({ + const normalizedBid: Record = { adapterCode: 'trustedServer', bidderCode: 'aps', adId: 'prebid-generated-ad-id', adUnitCode: 'div-aps', ttl: 300, - trustedServerRenderer: renderer, - }); + meta: { trustedServerRenderer: renderer }, + }; + bidAcceptedListener!(normalizedBid); const entry = testWindow.tsjs?.apsPrebidRenderers?.['prebid-generated-ad-id']; expect(entry).toEqual( @@ -467,6 +480,8 @@ describe('prebid/installPrebidNpm', () => { }) ); + expect(normalizedBid).not.toHaveProperty('trustedServerRenderer'); + expect(normalizedBid['meta']).not.toHaveProperty('trustedServerRenderer'); entry?.markUsed(); expect(mockMarkWinningBidAsUsed).toHaveBeenCalledWith({ adId: 'prebid-generated-ad-id', @@ -474,6 +489,24 @@ describe('prebid/installPrebidNpm', () => { }); }); + it('keeps bidResponse as a top-level renderer compatibility fallback', () => { + installPrebidNpm(); + const bidResponseListener = mockOnEvent.mock.calls.find( + ([eventName]) => eventName === 'bidResponse' + )?.[1] as ((bid: Record) => void) | undefined; + const renderer = apsRenderer(); + + bidResponseListener!({ + adapterCode: 'trustedServer', + bidderCode: 'aps', + adId: 'fallback-ad-id', + adUnitCode: 'div-aps', + trustedServerRenderer: renderer, + }); + + expect(testWindow.tsjs?.apsPrebidRenderers?.['fallback-ad-id']?.renderer).toEqual(renderer); + }); + it('makes failed APS renderer registrations ineligible when zero-CPM bids are allowed', () => { const warnSpy = vi.spyOn(log, 'warn').mockImplementation(() => {}); installPrebidNpm(); @@ -488,7 +521,7 @@ describe('prebid/installPrebidNpm', () => { adUnitCode: 'div-aps', ttl: 300, cpm: 1.23, - trustedServerRenderer: { ...apsRenderer(), aaxResponse: 'invalid' }, + meta: { trustedServerRenderer: { ...apsRenderer(), aaxResponse: 'invalid' } }, }; bidResponseListener!(malformedBid); bidResponseListener!({ @@ -502,6 +535,7 @@ describe('prebid/installPrebidNpm', () => { expect(testWindow.tsjs?.apsPrebidRenderers?.['malformed-ad-id']).toBeUndefined(); expect(testWindow.tsjs?.apsPrebidRenderers?.['foreign-ad-id']).toBeUndefined(); expect(malformedBid).not.toHaveProperty('trustedServerRenderer'); + expect(malformedBid['meta']).not.toHaveProperty('trustedServerRenderer'); // Prebid's allowZeroCpmBids path still requires cpm >= 0. expect(malformedBid['cpm']).toBe(-1); expect(warnSpy).toHaveBeenCalledWith( @@ -1132,6 +1166,40 @@ describe('prebid/installPrebidNpm', () => { }); describe('requestBids shim', () => { + beforeEach(() => { + testWindow.__tsjs_prebid = { + serverSideBidders: ['appnexus', 'rubicon', 'kargo', 'openx'], + }; + }); + + it('limits a global request to opts.adUnitCodes', () => { + const selected = document.createElement('div'); + selected.id = 'selected-global-unit'; + const unselected = document.createElement('div'); + unselected.id = 'unselected-global-unit'; + document.body.append(selected, unselected); + const selectedUnit = { + code: selected.id, + bids: [{ bidder: 'appnexus', params: { placementId: 1 } }], + }; + const unselectedUnit = { + code: unselected.id, + bids: [{ bidder: 'rubicon', params: { accountId: 2 } }], + }; + mockPbjs.adUnits = [selectedUnit, unselectedUnit]; + const pbjs = installPrebidNpm(); + + pbjs.requestBids({ adUnitCodes: [selected.id] } as unknown as RequestBidsArg); + + expect(selectedUnit.bids.map((bid) => bid.bidder)).toEqual(['trustedServer']); + expect(unselectedUnit.bids).toEqual([{ bidder: 'rubicon', params: { accountId: 2 } }]); + expect((testWindow.tsjs as TsjsApi).firstImpression?.slots[selected.id]).toBeDefined(); + expect((testWindow.tsjs as TsjsApi).firstImpression?.slots[unselected.id]).toBeUndefined(); + + selected.remove(); + unselected.remove(); + }); + it('preserves publisher ts adserverTargeting while adding trustedServer settings', () => { const publisherTargeting = [{ key: 'ts', val: () => 'publisher-value' }]; mockPbjs.bidderSettings = { @@ -1195,26 +1263,84 @@ describe('prebid/installPrebidNpm', () => { expect(tsCount).toBe(1); }); - it('captures per-bidder params on trustedServer bid', () => { + it('folds only authoritative routes across mixed client, PBS, APS, and standard demand', () => { + testWindow.__tsjs_prebid = { + serverSideBidders: ['pbsRoute', 'standardRoute'], + clientSideBidders: ['exampleBrowser'], + }; const pbjs = installPrebidNpm(); const adUnits = [ { bids: [ - { bidder: 'appnexus', params: { placementId: 123 } }, - { bidder: 'rubicon', params: { accountId: 'abc' } }, + { bidder: 'exampleBrowser', params: { placement: 'browser' } }, + { bidder: 'pbsRoute', params: { placement: 'pbs' } }, + { bidder: 'aps', params: { slot: 'aps' } }, + { bidder: 'standardRoute', params: { placement: 'standard' } }, + { bidder: 'pbs-provider-id', params: { forbidden: true } }, ], }, ]; pbjs.requestBids({ adUnits } as unknown as RequestBidsArg); const trustedServerBid = adUnits[0].bids.find((b: TestBid) => b.bidder === 'trustedServer'); - expect(trustedServerBid).toBeDefined(); - expect(trustedServerBid.params.bidderParams).toEqual({ - appnexus: { placementId: 123 }, - rubicon: { accountId: 'abc' }, + expect(trustedServerBid?.params?.bidderParams).toEqual({ + pbsRoute: { placement: 'pbs' }, + standardRoute: { placement: 'standard' }, }); - expect(adUnits[0].bids.map((b: TestBid) => b.bidder)).toEqual(['trustedServer']); + expect(adUnits[0].bids.map((b: TestBid) => b.bidder)).toEqual([ + 'exampleBrowser', + 'aps', + 'pbs-provider-id', + 'trustedServer', + ]); + }); + + it('preserves prototype-named server-side bidders as owned JSON properties', () => { + testWindow.__tsjs_prebid = { serverSideBidders: ['__proto__'] }; + const pbjs = installPrebidNpm(); + const adUnits = [ + { + bids: [{ bidder: '__proto__', params: { placement: 'server-owned' } }], + }, + ]; + + pbjs.requestBids({ adUnits } as unknown as RequestBidsArg); + + const trustedServerBid = adUnits[0].bids.find((bid) => bid.bidder === 'trustedServer'); + const bidderParams = trustedServerBid?.params?.bidderParams as Record; + expect(Object.prototype.hasOwnProperty.call(bidderParams, '__proto__')).toBe(true); + expect(bidderParams['__proto__']).toEqual({ placement: 'server-owned' }); + expect(JSON.parse(JSON.stringify(bidderParams))).toEqual( + Object.fromEntries([['__proto__', { placement: 'server-owned' }]]) + ); + expect(adUnits[0].bids.map((bid) => bid.bidder)).toEqual(['trustedServer']); + }); + + it('does not let returned bidder aliases or APS renderer aliases affect folding', () => { + testWindow.__tsjs_prebid = { serverSideBidders: ['configuredRoute'] }; + const pbjs = installPrebidNpm(); + const adUnits = [ + { + bids: [ + { bidder: 'configuredRoute', params: { placement: 1 } }, + { bidder: 'alternateReturnedSeat', params: { placement: 2 } }, + { bidder: 'apsRendererAlias', params: { placement: 3 } }, + ], + }, + ]; + + pbjs.requestBids({ adUnits } as unknown as RequestBidsArg); + + const trustedServerBid = adUnits[0].bids.find((bid) => bid.bidder === 'trustedServer'); + expect(trustedServerBid?.params?.bidderParams).toEqual({ + configuredRoute: { placement: 1 }, + }); + expect(adUnits[0].bids.map((bid) => bid.bidder)).toEqual([ + 'alternateReturnedSeat', + 'apsRendererAlias', + 'trustedServer', + ]); }); it('preserves captured bidder params when requestBids runs twice on the same ad unit', () => { @@ -1267,6 +1393,21 @@ describe('prebid/installPrebidNpm', () => { expect(adUnits[0].bids).toEqual([{ bidder: 'trustedServer', params: { bidderParams: {} } }]); }); + it('preserves the empty stored-request envelope on initial and repeated requests', () => { + const pbjs = installPrebidNpm(); + const adUnits = [ + { + code: 'stored-slot', + bids: [{ bidder: 'trustedServer', params: { bidderParams: {} } }], + }, + ]; + + pbjs.requestBids({ adUnits } as unknown as RequestBidsArg); + pbjs.requestBids({ adUnits } as unknown as RequestBidsArg); + + expect(adUnits[0].bids).toEqual([{ bidder: 'trustedServer', params: { bidderParams: {} } }]); + }); + it('includes zone from mediaTypes.banner.name in trustedServer params', () => { const pbjs = installPrebidNpm(); @@ -1431,6 +1572,18 @@ describe('prebid/installPrebidNpm with server-injected config', () => { ); }); + it('keeps browser timeout and debug independent from multiple PBS routes', () => { + testWindow.__tsjs_prebid = { + timeout: 1750, + debug: false, + serverSideBidders: ['pbsPrimaryRoute', 'pbsSecondaryRoute'], + }; + + installPrebidNpm(); + + expect(mockSetConfig).toHaveBeenCalledWith({ debug: false, bidderTimeout: 1750 }); + }); + it('explicit config overrides server-injected values', () => { testWindow.__tsjs_prebid = { timeout: 1500, debug: true }; @@ -1458,15 +1611,25 @@ describe('prebid/installRefreshHandler', () => { mockPbjs.setTargetingForGPTAsync = undefined; testWindow.tsjs = undefined; delete testWindow.googletag; - delete testWindow.__tsjs_prebid; + testWindow.__tsjs_prebid = { + serverSideBidders: ['appnexus', 'rubicon', 'kargo', 'openx', 'exampleServer'], + }; + document.body.replaceChildren(); }); afterEach(() => { testWindow.tsjs = undefined; delete testWindow.googletag; delete testWindow.__tsjs_prebid; + document.body.replaceChildren(); }); + function attachTestSlot(code: string): void { + const element = document.createElement('div'); + element.id = code; + document.body.appendChild(element); + } + it('builds refresh ad units from injected slot metadata', () => { const originalRefresh = vi.fn(); const gptSlot = { @@ -1630,7 +1793,10 @@ describe('prebid/installRefreshHandler', () => { }); it('includes configured client-side bidders in refresh ad units', () => { - testWindow.__tsjs_prebid = { clientSideBidders: ['rubicon'] }; + testWindow.__tsjs_prebid = { + clientSideBidders: ['rubicon'], + serverSideBidders: ['appnexus', 'exampleServer', 'kargo'], + }; // Original publisher ad unit carries a client-side rubicon bid. mockPbjs.adUnits = [ { @@ -1751,7 +1917,10 @@ describe('prebid/installRefreshHandler', () => { // publisher's Prebid ad unit is keyed by the inner div_id. The synthetic // refresh code stays the GPT element id (so GPT can match it), while params // and client-side bids are recovered from the injected div_id candidate. - testWindow.__tsjs_prebid = { clientSideBidders: ['rubicon'] }; + testWindow.__tsjs_prebid = { + clientSideBidders: ['rubicon'], + serverSideBidders: ['appnexus', 'exampleServer', 'kargo'], + }; mockPbjs.adUnits = [ { code: 'div-ad-x', @@ -2086,7 +2255,7 @@ describe('prebid/installRefreshHandler', () => { }) ); expect(setTargetingForGPTAsync).toHaveBeenCalledWith(['div-ad-display']); - expect(originalRefresh).toHaveBeenCalledWith(undefined, undefined); + expect(originalRefresh).toHaveBeenCalledWith(targetSlots, undefined); mockPbjs.setTargetingForGPTAsync = undefined; }); @@ -2277,6 +2446,7 @@ describe('prebid/installRefreshHandler', () => { const pbjs = installPrebidNpm(); const prepareDelivery = (code: string) => { + if (!document.getElementById(code)) attachTestSlot(code); mockRequestBids.mockImplementationOnce((options) => { options.bidsBackHandler?.(); }); @@ -2357,6 +2527,7 @@ describe('prebid/installRefreshHandler', () => { new GptDiagnosticsObserver(store).install(); } const pbjs = installPrebidNpm(); + attachTestSlot('install-order'); mockRequestBids.mockImplementationOnce((options) => options.bidsBackHandler?.()); pbjs.requestBids({ adUnits: [{ code: 'install-order', bids: [{ bidder: 'exampleServer', params: {} }] }], @@ -2405,6 +2576,7 @@ describe('prebid/installRefreshHandler', () => { const pbjs = installPrebidNpm(); installRefreshHandler(750); + attachTestSlot('nested-reentrant'); mockRequestBids.mockImplementation((options) => options.bidsBackHandler?.()); pbjs.requestBids({ adUnits: [{ code: 'nested-reentrant', bids: [{ bidder: 'exampleServer', params: {} }] }], @@ -2484,21 +2656,31 @@ describe('prebid publisher snapshots and delivery refreshes', () => { (window as unknown as { __tsjs_prebid_bundle?: unknown }).__tsjs_prebid_bundle = DEFAULT_BUNDLE_MANIFEST; mockPbjs.setTargetingForGPTAsync = undefined; - delete testWindow.__tsjs_prebid; + testWindow.__tsjs_prebid = { + serverSideBidders: ['exampleServer', 'exampleFallback'], + }; testWindow.tsjs = undefined; delete testWindow.googletag; + document.body.replaceChildren(); }); afterEach(() => { delete testWindow.__tsjs_prebid; testWindow.tsjs = undefined; delete testWindow.googletag; + document.body.replaceChildren(); }); function installGpt(slots: Array>) { installedGptSlots = slots; for (const slot of slots) { if (!slot || typeof slot !== 'object') continue; + const elementId = slot.getSlotElementId?.(); + if (typeof elementId === 'string' && elementId && !document.getElementById(elementId)) { + const element = document.createElement('div'); + element.id = elementId; + document.body.appendChild(element); + } const originalGetTargeting = slot.getTargeting?.bind(slot); slot.getTargeting = (key: string) => { const deliveryAdId = deliveryAdIds.get(slot); @@ -2552,175 +2734,937 @@ describe('prebid publisher snapshots and delivery refreshes', () => { opts?.bidsBackHandler?.(bidResponses, false, auctionId); } - function installPrebidRefreshDiagnostics( - implementation?: (slots: Array>) => void - ) { - const recordPrebidRefresh = vi.fn(implementation); - testWindow.tsjs = { gptDiagnosticsRecorder: { recordPrebidRefresh } }; - return recordPrebidRefresh; - } + it('suppresses every publisher auction registered before the first TS delivery', () => { + const element = document.createElement('div'); + element.id = 'overlapping-first-impression'; + document.body.appendChild(element); + const ts = {} as TsjsApi; + claimFirstImpressionForTrustedServer(ts, element, 100); + const first = registerPublisherFirstImpressionAuctions(ts, [element.id], 101).get(element.id); + const second = registerPublisherFirstImpressionAuctions(ts, [element.id], 102).get(element.id); - it('records a publisher delivery refresh immediately before its GPT request', () => { - const slot = { - getSlotElementId: () => 'example-delivery-marker', - getTargeting: () => [], - clearTargeting: vi.fn(), - }; - const recordPrebidRefresh = installPrebidRefreshDiagnostics(); - const { originalRefresh, pubads } = installGpt([slot]); - mockRequestBids.mockImplementation((opts) => completePublisherAuction(opts)); - const pbjs = installPrebidNpm(); + expect(consumePublisherFirstImpressionDelivery(ts, first, 103)).toBe(true); + expect(consumePublisherFirstImpressionDelivery(ts, second, 104)).toBe(true); + expect(registerPublisherFirstImpressionAuctions(ts, [element.id], 105)).toEqual(new Map()); - pbjs.requestBids({ - adUnits: [ - { code: 'example-delivery-marker', bids: [{ bidder: 'exampleServer', params: {} }] }, - ], - bidsBackHandler: () => pubads.refresh([slot]), - } as unknown as RequestBidsArg); + element.remove(); + }); - expect(recordPrebidRefresh).toHaveBeenCalledTimes(1); - expect(recordPrebidRefresh).toHaveBeenCalledWith([slot]); - expect(recordPrebidRefresh.mock.invocationCallOrder[0]).toBeLessThan( - originalRefresh.mock.invocationCallOrder[0] - ); - expect(originalRefresh).toHaveBeenCalledTimes(1); - expect(originalRefresh).toHaveBeenCalledWith([slot], undefined); + it('suppresses a correlated TS-owned delivery after the five-second lease', () => { + const element = document.createElement('div'); + element.id = 'late-first-impression'; + document.body.appendChild(element); + const ts = {} as TsjsApi; + claimFirstImpressionForTrustedServer(ts, element, 100); + const token = registerPublisherFirstImpressionAuctions(ts, [element.id], 101).get(element.id); + + expect(consumePublisherFirstImpressionDelivery(ts, token, 5_102)).toBe(true); + + element.remove(); }); - it('records a completed synthetic refresh immediately before its GPT request', () => { - const slot = { - getSlotElementId: () => 'example-synthetic-marker', - getTargeting: () => [], - clearTargeting: vi.fn(), - }; - const recordPrebidRefresh = installPrebidRefreshDiagnostics(); - const { originalRefresh, pubads } = installGpt([slot]); - mockRequestBids.mockImplementation((opts) => completePublisherAuction(opts)); - installPrebidNpm(); + it('rejects a connected claim whose element is no longer canonical for its ID', () => { + const element = document.createElement('div'); + element.id = 'replaced-canonical-element'; + document.body.appendChild(element); + const ts = {} as TsjsApi; + claimFirstImpressionForTrustedServer(ts, element, 100); + const token = registerPublisherFirstImpressionAuctions(ts, [element.id], 101).get(element.id); + const replacement = document.createElement('div'); + replacement.id = element.id; + document.body.insertBefore(replacement, element); - pubads.refresh([slot]); + expect(document.getElementById(element.id)).toBe(replacement); + expect(consumePublisherFirstImpressionDelivery(ts, token, 102)).toBe(false); + expect(ts.firstImpression?.slots[element.id]).toBeUndefined(); - expect(recordPrebidRefresh).toHaveBeenCalledTimes(1); - expect(recordPrebidRefresh).toHaveBeenCalledWith([slot]); - expect(recordPrebidRefresh.mock.invocationCallOrder[0]).toBeLessThan( - originalRefresh.mock.invocationCallOrder[0] - ); - expect(originalRefresh).toHaveBeenCalledTimes(1); - expect(originalRefresh).toHaveBeenCalledWith([slot], undefined); + replacement.remove(); + element.remove(); }); - it('records every slot in a mixed SRA refresh before its GPT request', () => { - const deliverySlot = { - getSlotElementId: () => 'example-mixed-delivery-marker', - getTargeting: () => [], - clearTargeting: vi.fn(), - }; - const independentSlot = { - getSlotElementId: () => 'example-mixed-independent-marker', - getTargeting: () => [], - clearTargeting: vi.fn(), - }; - const targetSlots = [deliverySlot, independentSlot]; - const recordPrebidRefresh = installPrebidRefreshDiagnostics(); - const { originalRefresh, pubads } = installGpt(targetSlots); - mockRequestBids.mockImplementation((opts) => completePublisherAuction(opts)); - const pbjs = installPrebidNpm(); + it('prunes a claim stored under a registry key that does not match its slot element ID', () => { + const element = document.createElement('div'); + element.id = 'malformed-registry-key-slot'; + document.body.appendChild(element); + const ts = {} as TsjsApi; + const claim = claimFirstImpressionForTrustedServer(ts, element, 100)!; + delete ts.firstImpression!.slots[element.id]; + ts.firstImpression!.slots['wrong-registry-key'] = claim; - pbjs.requestBids({ - adUnits: [ - { - code: 'example-mixed-delivery-marker', - bids: [{ bidder: 'exampleServer', params: {} }], - }, - ], - bidsBackHandler: () => pubads.refresh(targetSlots), - } as unknown as RequestBidsArg); + expect(firstImpressionClaim(ts, element)).toBeUndefined(); + expect(ts.firstImpression!.slots['wrong-registry-key']).toBeUndefined(); - expect(recordPrebidRefresh).toHaveBeenCalledTimes(1); - expect(recordPrebidRefresh).toHaveBeenCalledWith(targetSlots); - expect(recordPrebidRefresh.mock.calls[0][0][0]).toBe(deliverySlot); - expect(recordPrebidRefresh.mock.calls[0][0][1]).toBe(independentSlot); - expect(recordPrebidRefresh.mock.invocationCallOrder[0]).toBeLessThan( - originalRefresh.mock.invocationCallOrder[0] - ); - expect(originalRefresh).toHaveBeenCalledTimes(1); - expect(originalRefresh).toHaveBeenCalledWith(targetSlots, undefined); + element.remove(); }); - it('records one synthetic timeout fallback before one GPT request', () => { + it('rejects a connected same-ID TS claim from a foreign document', () => { + const element = document.createElement('div'); + element.id = 'foreign-document-claim-slot'; + document.body.appendChild(element); + const foreignDocument = document.implementation.createHTMLDocument('foreign'); + const foreignElement = foreignDocument.createElement('div'); + foreignElement.id = element.id; + foreignDocument.body.appendChild(foreignElement); + const ts = {} as TsjsApi; + const claim = claimFirstImpressionForTrustedServer(ts, element, 100)!; + const token = registerPublisherFirstImpressionAuctions(ts, [element.id], 101).get(element.id); + claim.element = foreignElement; + + expect(foreignElement.isConnected).toBe(true); + expect(consumePublisherFirstImpressionDelivery(ts, token, 102)).toBe(false); + expect(ts.firstImpression?.slots[element.id]).toBeUndefined(); + expect(claimFirstImpressionForTrustedServer(ts, element, 103)?.element).toBe(element); + + element.remove(); + }); + + it('prunes an ordinary expired publisher registration without a reserved fallback', () => { + const element = document.createElement('div'); + element.id = 'ordinary-expired-publisher-slot'; + document.body.appendChild(element); + const ts = {} as TsjsApi; + const token = registerPublisherFirstImpressionAuctions(ts, [element.id], 100).get(element.id); + + expect(consumePublisherFirstImpressionDelivery(ts, token, 5_101)).toBe(false); + expect(ts.firstImpression?.slots[element.id]).toBeUndefined(); + + element.remove(); + }); + + it('clears a failed fallback reservation before a later ordinary publisher claim expires', () => { vi.useFakeTimers(); + vi.setSystemTime(100); try { - const slot = { - getSlotElementId: () => 'example-timeout-marker', - getTargeting: () => [], - clearTargeting: vi.fn(), - }; - const recordPrebidRefresh = installPrebidRefreshDiagnostics(); - const { originalRefresh, pubads } = installGpt([slot]); - mockRequestBids.mockImplementation(() => undefined); - installPrebidNpm(); + const element = document.createElement('div'); + element.id = 'failed-fallback-reservation-slot'; + document.body.appendChild(element); + const ts = {} as TsjsApi; + const originalToken = registerPublisherFirstImpressionAuctions(ts, [element.id]).get( + element.id + ); + expect(originalToken).toBeDefined(); + expect(reservePublisherFirstImpressionFallback(ts, element)).toBe(true); - pubads.refresh([slot]); - expect(recordPrebidRefresh).not.toHaveBeenCalled(); - expect(originalRefresh).not.toHaveBeenCalled(); + vi.advanceTimersByTime(5_001); + const fallbackClaim = claimFirstImpressionForTrustedServer(ts, element)!; + expect(fallbackClaim.owner).toBe('trusted_server'); + expect(fallbackClaim.publisherAuctions[originalToken!]?.suppressDelivery).toBe(true); - vi.advanceTimersByTime(640); + releaseTrustedServerFirstImpressionClaim(ts, element, fallbackClaim); + expect(ts.firstImpression?.slots[element.id]).toBeUndefined(); + expect(ts.firstImpression?.fallbackSlots[element.id]).toBeUndefined(); - expect(recordPrebidRefresh).toHaveBeenCalledTimes(1); - expect(recordPrebidRefresh).toHaveBeenCalledWith([slot]); - expect(recordPrebidRefresh.mock.invocationCallOrder[0]).toBeLessThan( - originalRefresh.mock.invocationCallOrder[0] - ); - expect(originalRefresh).toHaveBeenCalledTimes(1); + const laterToken = registerPublisherFirstImpressionAuctions(ts, [element.id]).get(element.id); + expect(laterToken).toBeDefined(); + vi.advanceTimersByTime(5_001); + expect(consumePublisherFirstImpressionDelivery(ts, laterToken)).toBe(false); + expect(ts.firstImpression?.slots[element.id]).toBeUndefined(); + + const freshClaim = claimFirstImpressionForTrustedServer(ts, element)!; + expect(freshClaim.publisherAuctions).toEqual({}); + + element.remove(); } finally { - vi.runOnlyPendingTimers(); + vi.clearAllTimers(); vi.useRealTimers(); } }); - it('records a caught synthetic auction failure before one GPT fallback request', () => { + it('reserves first impression while a publisher refresh auction is pending', () => { + const code = 'pending-publisher-refresh-slot'; const slot = { - getSlotElementId: () => 'example-failure-marker', + getSlotElementId: () => code, getTargeting: () => [], + getSizes: () => [[300, 250]], clearTargeting: vi.fn(), }; - const recordPrebidRefresh = installPrebidRefreshDiagnostics(); const { originalRefresh, pubads } = installGpt([slot]); - mockRequestBids.mockImplementation(() => { - throw new Error('example auction failure'); + let completeRefresh: (() => void) | undefined; + mockRequestBids.mockImplementation((opts) => { + completeRefresh = opts.bidsBackHandler; }); installPrebidNpm(); pubads.refresh([slot]); - expect(recordPrebidRefresh).toHaveBeenCalledTimes(1); - expect(recordPrebidRefresh).toHaveBeenCalledWith([slot]); - expect(recordPrebidRefresh.mock.invocationCallOrder[0]).toBeLessThan( - originalRefresh.mock.invocationCallOrder[0] - ); - expect(originalRefresh).toHaveBeenCalledTimes(1); + const ts = (testWindow.tsjs ??= {}) as unknown as TsjsApi; + expect( + claimFirstImpressionForTrustedServer(ts, document.getElementById(code)!) + ).toBeUndefined(); + expect(originalRefresh).not.toHaveBeenCalled(); + + completeRefresh?.(); + + expect(originalRefresh).toHaveBeenCalledOnce(); expect(originalRefresh).toHaveBeenCalledWith([slot], undefined); }); - it('does not record or refresh again for a late callback after timeout', () => { + it('suppresses an original publisher delivery after the lease-boundary TS fallback', () => { vi.useFakeTimers(); try { + const code = 'lease-boundary-fallback-slot'; const slot = { - getSlotElementId: () => 'example-late-marker', + getSlotElementId: () => code, getTargeting: () => [], + getSizes: () => [[300, 250]], clearTargeting: vi.fn(), + setTargeting: vi.fn(), }; - const recordPrebidRefresh = installPrebidRefreshDiagnostics(); const { originalRefresh, pubads } = installGpt([slot]); - let bidsBackHandler: (() => void) | undefined; - mockRequestBids.mockImplementation((opts) => { - bidsBackHandler = opts.bidsBackHandler; + let originalPublisherAuction: Parameters[0]; + mockRequestBids.mockImplementation((options) => { + if (!originalPublisherAuction) { + originalPublisherAuction = options; + return; + } + completePublisherAuction(options); }); - installPrebidNpm(); - - pubads.refresh([slot]); - vi.advanceTimersByTime(640); - bidsBackHandler?.(); + const pbjs = installPrebidNpm(); + const ts = (testWindow.tsjs ??= {}) as unknown as TsjsApi; + ts.servicesEnabled = true; + ts.adSlots = [ + { + id: 'lease-boundary-fallback-ad', + gam_unit_path: '/123/lease-boundary', + div_id: code, + formats: [[300, 250]], + targeting: {}, + }, + ]; + ts.bids = { + 'lease-boundary-fallback-ad': { + hb_pb: '1.00', + hb_adid: 'trusted-server-fallback-ad', + }, + }; + + pbjs.requestBids({ + adUnits: [{ code, bids: [{ bidder: 'exampleServer', params: {} }] }], + bidsBackHandler: () => pubads.refresh([slot]), + } as unknown as RequestBidsArg); + installTsAdInit(); + ts.adInit!(); + + vi.advanceTimersByTime(5001); + observeFirstImpressionGptLifecycle(ts, document.getElementById(code)!, 'requested'); + expect(originalRefresh).toHaveBeenCalledOnce(); + expect(ts.firstImpression?.slots[code]?.owner).toBe('trusted_server'); + expect(ts.firstImpression?.fallbackSlots[code]).toBe(document.getElementById(code)); + expect(Object.values(ts.firstImpression?.slots[code]?.publisherAuctions ?? {})).toEqual([ + expect.objectContaining({ suppressDelivery: true }), + ]); + + completePublisherAuction(originalPublisherAuction); + expect(originalRefresh).toHaveBeenCalledOnce(); + + deliveryAdIds.delete(slot); + pubads.refresh([slot]); + expect(mockRequestBids).toHaveBeenCalledTimes(2); + expect(originalRefresh).toHaveBeenCalledTimes(2); + } finally { + vi.clearAllTimers(); + vi.useRealTimers(); + } + }); + + it('suppresses a delayed publisher refresh when TS already owns first impression', () => { + const code = 'pending-ts-owned-refresh-slot'; + const slot = { + getSlotElementId: () => code, + getTargeting: () => [], + getSizes: () => [[300, 250]], + clearTargeting: vi.fn(), + setTargeting: vi.fn(), + }; + const { originalRefresh, pubads } = installGpt([slot]); + const ts = (testWindow.tsjs ??= {}) as unknown as TsjsApi; + claimFirstImpressionForTrustedServer(ts, document.getElementById(code)!); + let completeRefresh: (() => void) | undefined; + mockRequestBids.mockImplementation((opts) => { + completeRefresh = opts.bidsBackHandler; + }); + installPrebidNpm(); + + pubads.refresh([slot]); + expect(originalRefresh).not.toHaveBeenCalled(); + + completeRefresh?.(); + + expect(originalRefresh).not.toHaveBeenCalled(); + }); + + it('filters only the TS-owned slot from a delayed mixed publisher refresh', () => { + const tsCode = 'pending-mixed-ts-slot'; + const publisherCode = 'pending-mixed-publisher-slot'; + const tsSlot = { + getSlotElementId: () => tsCode, + getTargeting: () => [], + getSizes: () => [[300, 250]], + clearTargeting: vi.fn(), + setTargeting: vi.fn(), + }; + const publisherSlot = { + getSlotElementId: () => publisherCode, + getTargeting: () => [], + getSizes: () => [[300, 250]], + clearTargeting: vi.fn(), + }; + const { originalRefresh, pubads } = installGpt([tsSlot, publisherSlot]); + const ts = (testWindow.tsjs ??= {}) as unknown as TsjsApi; + claimFirstImpressionForTrustedServer(ts, document.getElementById(tsCode)!); + let completeRefresh: (() => void) | undefined; + mockRequestBids.mockImplementation((opts) => { + completeRefresh = opts.bidsBackHandler; + }); + installPrebidNpm(); + + pubads.refresh([tsSlot, publisherSlot]); + completeRefresh?.(); + + expect(originalRefresh).toHaveBeenCalledOnce(); + expect(originalRefresh).toHaveBeenCalledWith([publisherSlot], undefined); + }); + + it('filters a TS-owned excluded slot from a delayed mixed publisher refresh', () => { + const eligibleCode = 'pending-mixed-eligible-slot'; + const excludedCode = 'pending-mixed-excluded-slot'; + const eligibleSlot = { + getSlotElementId: () => eligibleCode, + getAdUnitPath: () => '/123/content', + getTargeting: () => [], + getSizes: () => [[300, 250]], + clearTargeting: vi.fn(), + }; + const excludedSlot = { + getSlotElementId: () => excludedCode, + getAdUnitPath: () => '/123/trackingonly', + getTargeting: () => [], + getSizes: () => [[1, 1]], + clearTargeting: vi.fn(), + setTargeting: vi.fn(), + }; + const { originalRefresh, pubads } = installGpt([eligibleSlot, excludedSlot]); + const ts = (testWindow.tsjs ??= {}) as unknown as TsjsApi; + claimFirstImpressionForTrustedServer(ts, document.getElementById(excludedCode)!); + testWindow.__tsjs_prebid = { excludedGamAdUnitPathSuffixes: ['/trackingonly'] }; + let completeRefresh: (() => void) | undefined; + mockRequestBids.mockImplementation((opts) => { + completeRefresh = opts.bidsBackHandler; + }); + installPrebidNpm(); + + pubads.refresh([eligibleSlot, excludedSlot]); + completeRefresh?.(); + + expect(originalRefresh).toHaveBeenCalledOnce(); + expect(originalRefresh).toHaveBeenCalledWith([eligibleSlot], undefined); + }); + + it('drops delayed delivery and auction slots together after SPA navigation', () => { + const deliveryCode = 'pending-navigation-delivery-slot'; + const auctionCode = 'pending-navigation-auction-slot'; + const deliverySlot = { + getSlotElementId: () => deliveryCode, + getTargeting: () => [], + getSizes: () => [[300, 250]], + clearTargeting: vi.fn(), + }; + const auctionSlot = { + getSlotElementId: () => auctionCode, + getTargeting: () => [], + getSizes: () => [[300, 250]], + clearTargeting: vi.fn(), + }; + const { originalRefresh, pubads } = installGpt([deliverySlot, auctionSlot]); + let completeRefresh: (() => void) | undefined; + mockRequestBids.mockImplementation((opts) => { + if (opts?.adUnits?.[0]?.code === deliveryCode) { + completePublisherAuction(opts); + } else { + completeRefresh = opts.bidsBackHandler; + } + }); + const pbjs = installPrebidNpm(); + + pbjs.requestBids({ + adUnits: [{ code: deliveryCode, bids: [{ bidder: 'exampleServer', params: {} }] }], + bidsBackHandler: () => pubads.refresh([deliverySlot, auctionSlot]), + } as unknown as RequestBidsArg); + ((testWindow.tsjs ??= {}) as unknown as TsjsApi).navGeneration = 1; + completeRefresh?.(); + + expect(originalRefresh).not.toHaveBeenCalled(); + }); + + it('drops a delayed publisher refresh after SPA navigation', () => { + const code = 'pending-previous-navigation-refresh-slot'; + const slot = { + getSlotElementId: () => code, + getTargeting: () => [], + getSizes: () => [[300, 250]], + clearTargeting: vi.fn(), + }; + const { originalRefresh, pubads } = installGpt([slot]); + let completeRefresh: (() => void) | undefined; + mockRequestBids.mockImplementation((opts) => { + completeRefresh = opts.bidsBackHandler; + }); + installPrebidNpm(); + + pubads.refresh([slot]); + ((testWindow.tsjs ??= {}) as unknown as TsjsApi).navGeneration = 1; + completeRefresh?.(); + + expect(originalRefresh).not.toHaveBeenCalled(); + }); + + it('drops a delayed publisher refresh after physical element replacement', () => { + const code = 'pending-replaced-refresh-slot'; + const slot = { + getSlotElementId: () => code, + getTargeting: () => [], + getSizes: () => [[300, 250]], + clearTargeting: vi.fn(), + }; + const { originalRefresh, pubads } = installGpt([slot]); + let completeRefresh: (() => void) | undefined; + mockRequestBids.mockImplementation((opts) => { + completeRefresh = opts.bidsBackHandler; + }); + installPrebidNpm(); + + pubads.refresh([slot]); + document.getElementById(code)?.remove(); + const replacement = document.createElement('div'); + replacement.id = code; + document.body.appendChild(replacement); + completeRefresh?.(); + + expect(originalRefresh).not.toHaveBeenCalled(); + }); + + it('keeps a delayed bare refresh scoped to its captured slot list', () => { + const firstCode = 'pending-bare-first-slot'; + const laterCode = 'pending-bare-later-slot'; + const firstSlot = { + getSlotElementId: () => firstCode, + getTargeting: () => [], + getSizes: () => [[300, 250]], + clearTargeting: vi.fn(), + }; + const laterSlot = { + getSlotElementId: () => laterCode, + getTargeting: () => [], + getSizes: () => [[300, 250]], + clearTargeting: vi.fn(), + }; + const slots = [firstSlot]; + const { originalRefresh, pubads } = installGpt(slots); + let completeRefresh: (() => void) | undefined; + mockRequestBids.mockImplementation((opts) => { + completeRefresh = opts.bidsBackHandler; + }); + installPrebidNpm(); + + pubads.refresh(); + slots.push(laterSlot); + completeRefresh?.(); + + expect(originalRefresh).toHaveBeenCalledOnce(); + expect(originalRefresh).toHaveBeenCalledWith([firstSlot], undefined); + }); + + it('allows publisher refreshes that start after the TS first impression request', () => { + const code = 'requested-ts-owned-refresh-slot'; + const element = document.createElement('div'); + element.id = code; + document.body.appendChild(element); + const ts = {} as TsjsApi; + claimFirstImpressionForTrustedServer(ts, element); + observeFirstImpressionGptLifecycle(ts, element, 'requested'); + + expect(registerPublisherFirstImpressionAuctions(ts, [code])).toEqual(new Map()); + expect(ts.firstImpression?.slots[code]?.publisherRegistrationClosed).toBe(true); + }); + + it('clears a stale GPT handoff when delegating a post-request publisher refresh', () => { + const code = 'post-request-handoff-slot'; + const slot = { + getSlotElementId: () => code, + getTargeting: () => [], + getSizes: () => [[300, 250]], + clearTargeting: vi.fn(), + }; + installedGptSlots = [slot]; + const nativeRefresh = vi.fn(); + const ts = (testWindow.tsjs = {} as unknown as PrebidTestWindow['tsjs']) as unknown as TsjsApi; + const handoff = { + gamUnitPath: '/123/post-request', + formats: [[300, 250] as [number, number]], + divIdPrefix: code, + slotElementId: code, + publisherClaimed: true, + suppressPublisherDisplay: false, + suppressPublisherRefresh: true, + }; + ts.gptSlotHandoffs = { [code]: handoff }; + const innerRefresh = vi.fn((slots?: (typeof slot)[]) => { + if (handoff.suppressPublisherRefresh) { + handoff.suppressPublisherRefresh = false; + return; + } + nativeRefresh(slots); + }); + const pubads = { refresh: innerRefresh, getSlots: () => [slot] }; + testWindow.googletag = { + cmd: { push: (fn: () => void) => fn() }, + pubads: () => pubads, + }; + const element = document.createElement('div'); + element.id = code; + document.body.appendChild(element); + claimFirstImpressionForTrustedServer(ts, element); + observeFirstImpressionGptLifecycle(ts, element, 'requested'); + installRefreshHandler(640); + mockRequestBids.mockImplementation((opts) => completePublisherAuction(opts)); + installPrebidNpm(); + + pubads.refresh([slot]); + + expect(handoff.suppressPublisherRefresh).toBe(false); + expect(nativeRefresh).toHaveBeenCalledWith([slot]); + }); + + it('suppresses an all-excluded refresh while the TS first impression is pending', () => { + const code = 'pending-all-excluded-slot'; + const slot = { + getSlotElementId: () => code, + getAdUnitPath: () => '/123/trackingonly', + getTargeting: () => [], + getSizes: () => [[1, 1]], + clearTargeting: vi.fn(), + setTargeting: vi.fn(), + }; + const { originalRefresh, pubads } = installGpt([slot]); + const ts = (testWindow.tsjs ??= {}) as unknown as TsjsApi; + claimFirstImpressionForTrustedServer(ts, document.getElementById(code)!); + testWindow.__tsjs_prebid = { excludedGamAdUnitPathSuffixes: ['/trackingonly'] }; + installPrebidNpm(); + + pubads.refresh([slot]); + + expect(mockRequestBids).not.toHaveBeenCalled(); + expect(originalRefresh).not.toHaveBeenCalled(); + }); + + it('delegates an all-excluded refresh after the TS first impression request', () => { + const code = 'requested-all-excluded-slot'; + const slot = { + getSlotElementId: () => code, + getAdUnitPath: () => '/123/trackingonly', + getTargeting: () => [], + getSizes: () => [[1, 1]], + clearTargeting: vi.fn(), + }; + installedGptSlots = [slot]; + const nativeRefresh = vi.fn(); + const ts = (testWindow.tsjs = {} as unknown as PrebidTestWindow['tsjs']) as unknown as TsjsApi; + const handoff = { + gamUnitPath: '/123/trackingonly', + formats: [[1, 1] as [number, number]], + divIdPrefix: code, + slotElementId: code, + publisherClaimed: true, + suppressPublisherDisplay: false, + suppressPublisherRefresh: true, + }; + ts.gptSlotHandoffs = { [code]: handoff }; + const innerRefresh = vi.fn((slots?: (typeof slot)[]) => { + if (handoff.suppressPublisherRefresh) { + handoff.suppressPublisherRefresh = false; + return; + } + nativeRefresh(slots); + }); + const pubads = { refresh: innerRefresh, getSlots: () => [slot] }; + testWindow.googletag = { + cmd: { push: (fn: () => void) => fn() }, + pubads: () => pubads, + }; + const element = document.createElement('div'); + element.id = code; + document.body.appendChild(element); + claimFirstImpressionForTrustedServer(ts, element); + observeFirstImpressionGptLifecycle(ts, element, 'requested'); + testWindow.__tsjs_prebid = { excludedGamAdUnitPathSuffixes: ['/trackingonly'] }; + installRefreshHandler(640); + installPrebidNpm(); + + pubads.refresh([slot]); + + expect(mockRequestBids).not.toHaveBeenCalled(); + expect(handoff.suppressPublisherRefresh).toBe(false); + expect(nativeRefresh).toHaveBeenCalledWith([slot]); + }); + + it('consumes late-handoff suppression when Prebid suppresses the same delivery', () => { + const code = 'composed-suppression-slot'; + const element = document.createElement('div'); + element.id = code; + document.body.appendChild(element); + const slot = { + getSlotElementId: () => code, + getTargeting: () => [], + getSizes: () => [[300, 250]], + clearTargeting: vi.fn(), + setTargeting: vi.fn(), + }; + installedGptSlots = [slot]; + const nativeRefresh = vi.fn(); + const ts = (testWindow.tsjs = {} as unknown as PrebidTestWindow['tsjs']) as unknown as TsjsApi; + const handoff = { + gamUnitPath: '/123/composed', + formats: [[300, 250] as [number, number]], + divIdPrefix: code, + slotElementId: code, + publisherClaimed: true, + suppressPublisherDisplay: false, + suppressPublisherRefresh: true, + }; + ts.gptSlotHandoffs = { [code]: handoff }; + const innerRefresh = vi.fn((slots?: (typeof slot)[]) => { + if (handoff.suppressPublisherRefresh) { + handoff.suppressPublisherRefresh = false; + return; + } + nativeRefresh(slots); + }); + const pubads = { refresh: innerRefresh, getSlots: () => [slot] }; + testWindow.googletag = { + cmd: { push: (fn: () => void) => fn() }, + pubads: () => pubads, + }; + claimFirstImpressionForTrustedServer(ts, element); + installRefreshHandler(640); + mockRequestBids.mockImplementation((opts) => completePublisherAuction(opts)); + const pbjs = installPrebidNpm(); + + pbjs.requestBids({ + adUnits: [{ code, bids: [{ bidder: 'exampleServer', params: {} }] }], + bidsBackHandler: () => pubads.refresh([slot]), + } as unknown as RequestBidsArg); + + expect(handoff.suppressPublisherRefresh).toBe(false); + expect(innerRefresh).not.toHaveBeenCalled(); + + pubads.refresh([slot]); + + expect(nativeRefresh).toHaveBeenCalledWith([slot]); + }); + + it('forwards only unsuppressed excluded slots', () => { + const suppressedCode = 'mixed-suppressed-slot'; + const excludedCode = 'mixed-excluded-slot'; + const suppressedSlot = { + getSlotElementId: () => suppressedCode, + getAdUnitPath: () => '/123/content', + getTargeting: () => [], + getSizes: () => [[300, 250]], + clearTargeting: vi.fn(), + setTargeting: vi.fn(), + }; + const excludedSlot = { + getSlotElementId: () => excludedCode, + getAdUnitPath: () => '/123/trackingonly', + getTargeting: () => [], + getSizes: () => [[1, 1]], + clearTargeting: vi.fn(), + }; + const { originalRefresh, pubads } = installGpt([suppressedSlot, excludedSlot]); + const ts = (testWindow.tsjs ??= {}) as unknown as TsjsApi; + claimFirstImpressionForTrustedServer(ts, document.getElementById(suppressedCode)!); + testWindow.__tsjs_prebid = { excludedGamAdUnitPathSuffixes: ['/trackingonly'] }; + mockRequestBids.mockImplementation((opts) => completePublisherAuction(opts)); + const pbjs = installPrebidNpm(); + + pbjs.requestBids({ + adUnits: [{ code: suppressedCode, bids: [{ bidder: 'exampleServer', params: {} }] }], + bidsBackHandler: () => pubads.refresh([suppressedSlot, excludedSlot]), + } as unknown as RequestBidsArg); + + expect(originalRefresh).toHaveBeenCalledWith([excludedSlot], undefined); + }); + + it('rejects pending delivery state from a previous navigation', () => { + const code = 'previous-navigation-slot'; + const slot = { + getSlotElementId: () => code, + getTargeting: () => [], + getSizes: () => [[300, 250]], + clearTargeting: vi.fn(), + }; + const { originalRefresh, pubads } = installGpt([slot]); + mockRequestBids.mockImplementation((opts) => completePublisherAuction(opts)); + const pbjs = installPrebidNpm(); + pbjs.requestBids({ + adUnits: [{ code, bids: [{ bidder: 'exampleServer', params: {} }] }], + } as unknown as RequestBidsArg); + ((testWindow.tsjs ??= {}) as unknown as TsjsApi).navGeneration = 1; + + pubads.refresh([slot]); + + expect(mockRequestBids).toHaveBeenCalledTimes(2); + expect(originalRefresh).toHaveBeenCalledWith([slot], undefined); + }); + + it('rejects pending delivery state after physical element replacement', () => { + const code = 'replaced-physical-slot'; + const slot = { + getSlotElementId: () => code, + getTargeting: () => [], + getSizes: () => [[300, 250]], + clearTargeting: vi.fn(), + }; + const { originalRefresh, pubads } = installGpt([slot]); + mockRequestBids.mockImplementation((opts) => completePublisherAuction(opts)); + const pbjs = installPrebidNpm(); + pbjs.requestBids({ + adUnits: [{ code, bids: [{ bidder: 'exampleServer', params: {} }] }], + } as unknown as RequestBidsArg); + document.getElementById(code)?.remove(); + const replacement = document.createElement('div'); + replacement.id = code; + document.body.appendChild(replacement); + + pubads.refresh([slot]); + + expect(mockRequestBids).toHaveBeenCalledTimes(2); + expect(originalRefresh).toHaveBeenCalledWith([slot], undefined); + }); + + function installPrebidRefreshDiagnostics( + implementation?: (slots: Array>) => void + ) { + const recordPrebidRefresh = vi.fn(implementation); + testWindow.tsjs = { gptDiagnosticsRecorder: { recordPrebidRefresh } }; + return recordPrebidRefresh; + } + + it('suppresses one publisher delivery after TS claims first and allows a later refresh', () => { + const code = 'example-ts-first-slot'; + const element = document.createElement('div'); + element.id = code; + document.body.appendChild(element); + try { + const targeting = new Map([ + ['ts_initial', '1'], + ['hb_adid', 'example-ts-ad-id'], + ['hb_pb', '1.25'], + ]); + const slot = { + getSlotElementId: () => code, + getTargeting: (key: string) => { + const value = targeting.get(key); + return value === undefined ? [] : Array.isArray(value) ? value : [value]; + }, + setTargeting: vi.fn((key: string, value: string | string[]) => { + targeting.set(key, value); + return slot; + }), + clearTargeting: vi.fn((key: string) => { + targeting.delete(key); + return slot; + }), + getSizes: () => [[300, 250]], + }; + const ts = (testWindow.tsjs = {} as TsjsApi) as TsjsApi; + const claim = claimFirstImpressionForTrustedServer(ts, element)!; + claim.targeting = Object.fromEntries(targeting); + const { originalRefresh, pubads } = installGpt([slot]); + mockRequestBids.mockImplementation((opts) => completePublisherAuction(opts)); + const pbjs = installPrebidNpm(); + + pbjs.requestBids({ + adUnits: [{ code, bids: [{ bidder: 'exampleServer', params: {} }] }], + bidsBackHandler: () => pubads.refresh([slot], { changeCorrelator: false }), + } as unknown as RequestBidsArg); + + expect(originalRefresh).not.toHaveBeenCalled(); + expect(slot.setTargeting).toHaveBeenCalledWith('ts_initial', '1'); + expect(slot.setTargeting).toHaveBeenCalledWith('hb_adid', 'example-ts-ad-id'); + expect(ts.firstImpression?.slots[code]?.publisherRegistrationClosed).toBe(true); + + pubads.refresh([slot], { changeCorrelator: false }); + + expect(mockRequestBids).toHaveBeenCalledTimes(2); + expect(originalRefresh).toHaveBeenCalledOnce(); + expect(originalRefresh).toHaveBeenCalledWith([slot], { changeCorrelator: false }); + } finally { + element.remove(); + } + }); + + it('records a publisher delivery refresh immediately before its GPT request', () => { + const slot = { + getSlotElementId: () => 'example-delivery-marker', + getTargeting: () => [], + clearTargeting: vi.fn(), + }; + const recordPrebidRefresh = installPrebidRefreshDiagnostics(); + const { originalRefresh, pubads } = installGpt([slot]); + mockRequestBids.mockImplementation((opts) => completePublisherAuction(opts)); + const pbjs = installPrebidNpm(); + + pbjs.requestBids({ + adUnits: [ + { code: 'example-delivery-marker', bids: [{ bidder: 'exampleServer', params: {} }] }, + ], + bidsBackHandler: () => pubads.refresh([slot]), + } as unknown as RequestBidsArg); + + expect(recordPrebidRefresh).toHaveBeenCalledTimes(1); + expect(recordPrebidRefresh).toHaveBeenCalledWith([slot]); + expect(recordPrebidRefresh.mock.invocationCallOrder[0]).toBeLessThan( + originalRefresh.mock.invocationCallOrder[0] + ); + expect(originalRefresh).toHaveBeenCalledTimes(1); + expect(originalRefresh).toHaveBeenCalledWith([slot], undefined); + }); + + it('records a completed synthetic refresh immediately before its GPT request', () => { + const slot = { + getSlotElementId: () => 'example-synthetic-marker', + getTargeting: () => [], + clearTargeting: vi.fn(), + }; + const recordPrebidRefresh = installPrebidRefreshDiagnostics(); + const { originalRefresh, pubads } = installGpt([slot]); + mockRequestBids.mockImplementation((opts) => completePublisherAuction(opts)); + installPrebidNpm(); + + pubads.refresh([slot]); + + expect(recordPrebidRefresh).toHaveBeenCalledTimes(1); + expect(recordPrebidRefresh).toHaveBeenCalledWith([slot]); + expect(recordPrebidRefresh.mock.invocationCallOrder[0]).toBeLessThan( + originalRefresh.mock.invocationCallOrder[0] + ); + expect(originalRefresh).toHaveBeenCalledTimes(1); + expect(originalRefresh).toHaveBeenCalledWith([slot], undefined); + }); + + it('records every slot in a mixed SRA refresh before its GPT request', () => { + const deliverySlot = { + getSlotElementId: () => 'example-mixed-delivery-marker', + getTargeting: () => [], + clearTargeting: vi.fn(), + }; + const independentSlot = { + getSlotElementId: () => 'example-mixed-independent-marker', + getTargeting: () => [], + clearTargeting: vi.fn(), + }; + const targetSlots = [deliverySlot, independentSlot]; + const recordPrebidRefresh = installPrebidRefreshDiagnostics(); + const { originalRefresh, pubads } = installGpt(targetSlots); + mockRequestBids.mockImplementation((opts) => completePublisherAuction(opts)); + const pbjs = installPrebidNpm(); + + pbjs.requestBids({ + adUnits: [ + { + code: 'example-mixed-delivery-marker', + bids: [{ bidder: 'exampleServer', params: {} }], + }, + ], + bidsBackHandler: () => pubads.refresh(targetSlots), + } as unknown as RequestBidsArg); + + expect(recordPrebidRefresh).toHaveBeenCalledTimes(1); + expect(recordPrebidRefresh).toHaveBeenCalledWith(targetSlots); + expect(recordPrebidRefresh.mock.calls[0][0][0]).toBe(deliverySlot); + expect(recordPrebidRefresh.mock.calls[0][0][1]).toBe(independentSlot); + expect(recordPrebidRefresh.mock.invocationCallOrder[0]).toBeLessThan( + originalRefresh.mock.invocationCallOrder[0] + ); + expect(originalRefresh).toHaveBeenCalledTimes(1); + expect(originalRefresh).toHaveBeenCalledWith(targetSlots, undefined); + }); + + it('records one synthetic timeout fallback before one GPT request', () => { + vi.useFakeTimers(); + try { + const slot = { + getSlotElementId: () => 'example-timeout-marker', + getTargeting: () => [], + clearTargeting: vi.fn(), + }; + const recordPrebidRefresh = installPrebidRefreshDiagnostics(); + const { originalRefresh, pubads } = installGpt([slot]); + mockRequestBids.mockImplementation(() => undefined); + installPrebidNpm(); + + pubads.refresh([slot]); + expect(recordPrebidRefresh).not.toHaveBeenCalled(); + expect(originalRefresh).not.toHaveBeenCalled(); + + vi.advanceTimersByTime(640); + + expect(recordPrebidRefresh).toHaveBeenCalledTimes(1); + expect(recordPrebidRefresh).toHaveBeenCalledWith([slot]); + expect(recordPrebidRefresh.mock.invocationCallOrder[0]).toBeLessThan( + originalRefresh.mock.invocationCallOrder[0] + ); + expect(originalRefresh).toHaveBeenCalledTimes(1); + } finally { + vi.runOnlyPendingTimers(); + vi.useRealTimers(); + } + }); + + it('records a caught synthetic auction failure before one GPT fallback request', () => { + const slot = { + getSlotElementId: () => 'example-failure-marker', + getTargeting: () => [], + clearTargeting: vi.fn(), + }; + const recordPrebidRefresh = installPrebidRefreshDiagnostics(); + const { originalRefresh, pubads } = installGpt([slot]); + mockRequestBids.mockImplementation(() => { + throw new Error('example auction failure'); + }); + installPrebidNpm(); + + pubads.refresh([slot]); + + expect(recordPrebidRefresh).toHaveBeenCalledTimes(1); + expect(recordPrebidRefresh).toHaveBeenCalledWith([slot]); + expect(recordPrebidRefresh.mock.invocationCallOrder[0]).toBeLessThan( + originalRefresh.mock.invocationCallOrder[0] + ); + expect(originalRefresh).toHaveBeenCalledTimes(1); + expect(originalRefresh).toHaveBeenCalledWith([slot], undefined); + }); + + it('does not record or refresh again for a late callback after timeout', () => { + vi.useFakeTimers(); + try { + const slot = { + getSlotElementId: () => 'example-late-marker', + getTargeting: () => [], + clearTargeting: vi.fn(), + }; + const recordPrebidRefresh = installPrebidRefreshDiagnostics(); + const { originalRefresh, pubads } = installGpt([slot]); + let bidsBackHandler: (() => void) | undefined; + mockRequestBids.mockImplementation((opts) => { + bidsBackHandler = opts.bidsBackHandler; + }); + installPrebidNpm(); + + pubads.refresh([slot]); + vi.advanceTimersByTime(640); + bidsBackHandler?.(); expect(recordPrebidRefresh).toHaveBeenCalledTimes(1); expect(recordPrebidRefresh).toHaveBeenCalledWith([slot]); @@ -2851,7 +3795,10 @@ describe('prebid publisher snapshots and delivery refreshes', () => { }); it('recovers inline params, ordered client bids, and zone when pbjs.adUnits is empty', () => { - testWindow.__tsjs_prebid = { clientSideBidders: ['exampleBrowser'] }; + testWindow.__tsjs_prebid = { + clientSideBidders: ['exampleBrowser'], + serverSideBidders: ['exampleServer', 'appnexus'], + }; const runtimeInstance = 'example-runtime-instance'; const code = `example-slot-${runtimeInstance}`; const slot = { @@ -2903,7 +3850,10 @@ describe('prebid publisher snapshots and delivery refreshes', () => { }); it('isolates nested bidder-param objects and arrays from later publisher mutation', () => { - testWindow.__tsjs_prebid = { clientSideBidders: ['exampleBrowser'] }; + testWindow.__tsjs_prebid = { + clientSideBidders: ['exampleBrowser'], + serverSideBidders: ['exampleServer', 'appnexus'], + }; const code = 'example-nested-params-slot'; const slot = { getSlotElementId: () => code, @@ -3076,7 +4026,10 @@ describe('prebid publisher snapshots and delivery refreshes', () => { }); it('prefers a rich live unit when a fresh same-code request overwrites the snapshot with empty bids', () => { - testWindow.__tsjs_prebid = { clientSideBidders: ['exampleBrowser'] }; + testWindow.__tsjs_prebid = { + clientSideBidders: ['exampleBrowser'], + serverSideBidders: ['exampleServer', 'appnexus'], + }; const code = 'example-live-rich-slot'; const slot = { getSlotElementId: () => code, @@ -3108,6 +4061,51 @@ describe('prebid publisher snapshots and delivery refreshes', () => { ]); }); + it('filters unowned stored bidder params before snapshot, reuse, and refresh recovery', () => { + testWindow.__tsjs_prebid = { serverSideBidders: ['exampleServer'] }; + const code = 'example-stored-envelope-slot'; + const slot = { + getSlotElementId: () => code, + getTargeting: () => [], + getSizes: () => [[300, 250]], + clearTargeting: vi.fn(), + }; + const { pubads } = installGpt([slot]); + const pbjs = installPrebidNpm(); + const adUnits = [ + { + code, + bids: [ + { + bidder: 'trustedServer', + params: { + bidderParams: { + exampleServer: { placement: 'authoritative' }, + pbsProviderId: { placement: 'provider' }, + returnedSeatAlias: { placement: 'alias' }, + }, + }, + }, + ], + }, + ]; + + pbjs.requestBids({ adUnits } as unknown as RequestBidsArg); + expect(adUnits[0].bids[0].params?.bidderParams).toEqual({ + exampleServer: { placement: 'authoritative' }, + }); + + pbjs.requestBids({ adUnits } as unknown as RequestBidsArg); + expect(adUnits[0].bids[0].params?.bidderParams).toEqual({ + exampleServer: { placement: 'authoritative' }, + }); + + pubads.refresh([slot]); + expect(refreshAdUnitFromLastRequest().bids[0].params?.bidderParams).toEqual({ + exampleServer: { placement: 'authoritative' }, + }); + }); + it('does not resurrect an older snapshot when the live unit is intentionally empty', () => { const code = 'example-live-empty-slot'; const slot = { @@ -3335,7 +4333,7 @@ describe('prebid publisher snapshots and delivery refreshes', () => { expect(coveredSlot.clearTargeting).not.toHaveBeenCalled(); expect(gamOnlySlot.clearTargeting).toHaveBeenCalledWith('hb_adid'); expect(originalRefresh).toHaveBeenCalledTimes(1); - expect(originalRefresh).toHaveBeenCalledWith(undefined, undefined); + expect(originalRefresh).toHaveBeenCalledWith([coveredSlot, gamOnlySlot], undefined); }); it('keeps explicit unrelated lists synthetic and partitions mixed delivery lists', () => { @@ -3521,6 +4519,10 @@ describe('prebid publisher snapshots and delivery refreshes', () => { clearTargeting: vi.fn(), }; const { originalRefresh, pubads } = installGpt([slot]); + const publisherElement = document.createElement('div'); + publisherElement.id = code; + publisherElement.appendChild(document.getElementById('example-different-gpt-slot')!); + document.body.appendChild(publisherElement); let auctionId = 'example-null-auction'; const setTargetingForGPTAsync = vi.fn(() => { deliveryAdIds.set(slot, `${auctionId}-${code}`); @@ -3786,7 +4788,7 @@ describe('prebid publisher snapshots and delivery refreshes', () => { expect(originalRefresh).toHaveBeenCalledWith([slot], undefined); }); - it('consumes all overlapping pending bids for the same ad-unit code', () => { + it('preserves a sibling registration after consuming an exact overlapping delivery', () => { const code = 'example-overlapping-code'; const slot = { getSlotElementId: () => code, @@ -3798,26 +4800,89 @@ describe('prebid publisher snapshots and delivery refreshes', () => { mockRequestBids.mockImplementation((opts) => completePublisherAuction(opts)); const pbjs = installPrebidNpm(); - pbjs.requestBids({ - adUnits: [{ code, bids: [{ bidder: 'exampleServer', params: {} }] }], - bidsBackHandler: () => {}, - } as unknown as RequestBidsArg); - pbjs.requestBids({ - adUnits: [{ code, bids: [{ bidder: 'exampleServer', params: {} }] }], - bidsBackHandler: () => {}, - } as unknown as RequestBidsArg); + for (let index = 0; index < 2; index += 1) { + pbjs.requestBids({ + adUnits: [{ code, bids: [{ bidder: 'exampleServer', params: {} }] }], + bidsBackHandler: () => {}, + } as unknown as RequestBidsArg); + } + deliveryAdIds.set(slot, `example-auction-0-${code}`); + pubads.refresh([slot]); + deliveryAdIds.set(slot, `example-auction-1-${code}`); pubads.refresh([slot]); + expect(mockRequestBids).toHaveBeenCalledTimes(2); expect(slot.clearTargeting).not.toHaveBeenCalled(); + expect(originalRefresh).toHaveBeenNthCalledWith(1, [slot], undefined); + expect(originalRefresh).toHaveBeenNthCalledWith(2, [slot], undefined); + }); + + it('does not guess between ordinary overlapping code-only registrations', () => { + const code = 'example-ambiguous-code-only'; + const slot = { + getSlotElementId: () => code, + getTargeting: () => [], + getSizes: () => [[300, 250]], + clearTargeting: vi.fn(), + }; + const { originalRefresh, pubads } = installGpt([slot]); + mockRequestBids.mockImplementation((opts) => completePublisherAuction(opts)); + const pbjs = installPrebidNpm(); + + for (let index = 0; index < 2; index += 1) { + pbjs.requestBids({ + adUnits: [{ code, bids: [{ bidder: 'exampleServer', params: {} }] }], + bidsBackHandler: () => {}, + } as unknown as RequestBidsArg); + } + + deliveryAdIds.delete(slot); + pubads.refresh([slot]); + expect(mockRequestBids).toHaveBeenCalledTimes(3); deliveryAdIds.set(slot, `example-auction-0-${code}`); pubads.refresh([slot]); expect(mockRequestBids).toHaveBeenCalledTimes(3); - expect(slot.clearTargeting).toHaveBeenCalledWith('hb_adid'); - expect(originalRefresh).toHaveBeenNthCalledWith(1, [slot], undefined); - expect(originalRefresh).toHaveBeenNthCalledWith(2, [slot], undefined); + expect(originalRefresh).toHaveBeenCalledTimes(2); + }); + + it('fails closed without consuming TS-owned ambiguous code-only registrations', () => { + const code = 'example-ts-ambiguous-code-only'; + const element = document.createElement('div'); + element.id = code; + document.body.appendChild(element); + const slot = { + getSlotElementId: () => code, + getTargeting: () => [], + getSizes: () => [[300, 250]], + clearTargeting: vi.fn(), + setTargeting: vi.fn(), + }; + const { originalRefresh, pubads } = installGpt([slot]); + const ts = (testWindow.tsjs ??= {}) as unknown as TsjsApi; + claimFirstImpressionForTrustedServer(ts, element); + mockRequestBids.mockImplementation((opts) => completePublisherAuction(opts)); + const pbjs = installPrebidNpm(); + + for (let index = 0; index < 2; index += 1) { + pbjs.requestBids({ + adUnits: [{ code, bids: [{ bidder: 'exampleServer', params: {} }] }], + bidsBackHandler: () => {}, + } as unknown as RequestBidsArg); + } + + deliveryAdIds.delete(slot); + pubads.refresh([slot]); + deliveryAdIds.set(slot, `example-auction-0-${code}`); + pubads.refresh([slot]); + deliveryAdIds.set(slot, `example-auction-1-${code}`); + pubads.refresh([slot]); + + expect(mockRequestBids).toHaveBeenCalledTimes(2); + expect(originalRefresh).not.toHaveBeenCalled(); + element.remove(); }); it('filters invalid explicit entries without duplicating or leaking a valid delivery', () => { @@ -4092,7 +5157,10 @@ describe('prebid/client-side bidders', () => { }); it('excludes client-side bidders from trustedServer bidderParams', () => { - testWindow.__tsjs_prebid = { clientSideBidders: ['rubicon'] }; + testWindow.__tsjs_prebid = { + clientSideBidders: ['rubicon'], + serverSideBidders: ['appnexus', 'exampleServer', 'kargo'], + }; const pbjs = installPrebidNpm(); @@ -4117,7 +5185,10 @@ describe('prebid/client-side bidders', () => { }); it('preserves client-side bidder bids as standalone entries', () => { - testWindow.__tsjs_prebid = { clientSideBidders: ['rubicon'] }; + testWindow.__tsjs_prebid = { + clientSideBidders: ['rubicon'], + serverSideBidders: ['appnexus', 'exampleServer', 'kargo'], + }; const pbjs = installPrebidNpm(); @@ -4139,7 +5210,10 @@ describe('prebid/client-side bidders', () => { }); it('handles multiple client-side bidders', () => { - testWindow.__tsjs_prebid = { clientSideBidders: ['rubicon', 'openx'] }; + testWindow.__tsjs_prebid = { + clientSideBidders: ['rubicon', 'openx'], + serverSideBidders: ['appnexus', 'exampleServer'], + }; const pbjs = installPrebidNpm(); @@ -4166,8 +5240,8 @@ describe('prebid/client-side bidders', () => { expect(adUnits[0].bids.find((b: TestBid) => b.bidder === 'appnexus')).toBeUndefined(); }); - it('behaves normally when no client-side bidders are configured', () => { - // No __tsjs_prebid at all — all bidders go server-side + it('leaves all unowned bidders in browser demand when no routes are configured', () => { + testWindow.__tsjs_prebid = { serverSideBidders: [] }; const pbjs = installPrebidNpm(); const adUnits = [ @@ -4181,14 +5255,19 @@ describe('prebid/client-side bidders', () => { pbjs.requestBids({ adUnits } as unknown as RequestBidsArg); const tsBid = adUnits[0].bids.find((b: TestBid) => b.bidder === 'trustedServer') as TestBid; - expect(tsBid.params.bidderParams).toEqual({ - appnexus: { placementId: 123 }, - rubicon: { accountId: 'abc' }, - }); + expect(tsBid.params?.bidderParams).toEqual({}); + expect(adUnits[0].bids.map((bid) => bid.bidder)).toEqual([ + 'appnexus', + 'rubicon', + 'trustedServer', + ]); }); it('behaves normally when client-side bidders list is empty', () => { - testWindow.__tsjs_prebid = { clientSideBidders: [] }; + testWindow.__tsjs_prebid = { + clientSideBidders: [], + serverSideBidders: ['appnexus', 'rubicon'], + }; const pbjs = installPrebidNpm(); @@ -4210,7 +5289,10 @@ describe('prebid/client-side bidders', () => { }); it('still injects trustedServer when all bidders are client-side', () => { - testWindow.__tsjs_prebid = { clientSideBidders: ['rubicon', 'appnexus'] }; + testWindow.__tsjs_prebid = { + clientSideBidders: ['rubicon', 'appnexus'], + serverSideBidders: ['openx', 'exampleServer'], + }; const pbjs = installPrebidNpm(); @@ -4237,7 +5319,10 @@ describe('prebid/client-side bidders', () => { adapters: ['rubicon'], bidderCodes: ['rubicon'], }; - testWindow.__tsjs_prebid = { clientSideBidders: ['rubicon', 'openx'] }; + testWindow.__tsjs_prebid = { + clientSideBidders: ['rubicon', 'openx'], + serverSideBidders: ['appnexus', 'exampleServer'], + }; const errorSpy = vi.spyOn(console, 'error').mockImplementation(() => {}); @@ -4281,7 +5366,10 @@ describe('prebid/client-side bidders', () => { adapters: ['adf'], bidderCodes: ['adf', 'adform', 'adformOpenRTB'], }; - testWindow.__tsjs_prebid = { clientSideBidders: ['adform'] }; + testWindow.__tsjs_prebid = { + clientSideBidders: ['adform'], + serverSideBidders: ['appnexus', 'exampleServer'], + }; const errorSpy = vi.spyOn(console, 'error').mockImplementation(() => {}); @@ -4306,7 +5394,10 @@ describe('prebid/client-side bidders', () => { adapters: ['a1Media'], bidderCodes: ['a1media'], }; - testWindow.__tsjs_prebid = { clientSideBidders: ['a1Media'] }; + testWindow.__tsjs_prebid = { + clientSideBidders: ['a1Media'], + serverSideBidders: ['appnexus', 'exampleServer'], + }; const errorSpy = vi.spyOn(console, 'error').mockImplementation(() => {}); @@ -4328,7 +5419,10 @@ describe('prebid/client-side bidders', () => { it('treats a malformed manifest as unstamped instead of throwing', () => { // The manifest is a plain window global any page script can overwrite. testWindow.__tsjs_prebid_bundle = { adapters: 'rubicon', userIdModules: 42 }; - testWindow.__tsjs_prebid = { clientSideBidders: ['rubicon'] }; + testWindow.__tsjs_prebid = { + clientSideBidders: ['rubicon'], + serverSideBidders: ['appnexus', 'exampleServer', 'kargo'], + }; const warnSpy = vi.spyOn(console, 'warn').mockImplementation(() => {}); @@ -4345,7 +5439,10 @@ describe('prebid/client-side bidders', () => { it('warns when the external bundle stamped no adapter manifest', () => { delete testWindow.__tsjs_prebid_bundle; - testWindow.__tsjs_prebid = { clientSideBidders: ['rubicon'] }; + testWindow.__tsjs_prebid = { + clientSideBidders: ['rubicon'], + serverSideBidders: ['appnexus', 'exampleServer', 'kargo'], + }; const warnSpy = vi.spyOn(console, 'warn').mockImplementation(() => {}); @@ -4361,7 +5458,10 @@ describe('prebid/client-side bidders', () => { }); it('does not log errors when all client-side bidders have adapters', () => { - testWindow.__tsjs_prebid = { clientSideBidders: ['rubicon'] }; + testWindow.__tsjs_prebid = { + clientSideBidders: ['rubicon'], + serverSideBidders: ['appnexus', 'exampleServer', 'kargo'], + }; const errorSpy = vi.spyOn(console, 'error').mockImplementation(() => {}); diff --git a/crates/trusted-server-js/lib/test/prebid-artifact-integration.test.mjs b/crates/trusted-server-js/lib/test/prebid-artifact-integration.test.mjs index 6ee858568..6f4be0caa 100644 --- a/crates/trusted-server-js/lib/test/prebid-artifact-integration.test.mjs +++ b/crates/trusted-server-js/lib/test/prebid-artifact-integration.test.mjs @@ -86,7 +86,7 @@ describe('tsjs-prebid shim artifact', () => { // A value-import of Prebid or a private rendering helper would multiply // the shim size; retain a margin above the normal compact shim output. expect(bundleCode.length).toBeGreaterThan(200_000); - expect(shimCode.length).toBeLessThan(30_000); + expect(shimCode.length).toBeLessThan(32_000); expect(shimCode).toContain('markWinningBidAsUsed'); }); }); @@ -132,7 +132,10 @@ describe('external bundle + served shim evaluated together', () => { // Mirror the server's head-injected state, which always precedes the // bundle script in document order. pageWindow.eval('window.pbjs = { que: [], cmd: [] };'); - pageWindow.__tsjs_prebid = { clientSideBidders: [] }; + pageWindow.__tsjs_prebid = { + clientSideBidders: [], + serverSideBidders: ['appnexus'], + }; pageWindow.eval(bundleCode); @@ -176,7 +179,18 @@ describe('external bundle + served shim evaluated together', () => { { code: 'ad-slot-1', mediaTypes: { banner: { sizes: [[300, 250]] } }, - bids: [{ bidder: 'appnexus', params: { placementId: 1 } }], + bids: [ + { + bidder: 'trustedServer', + params: { + bidderParams: { + appnexus: { placementId: 1 }, + pbsProviderId: { placementId: 2 }, + returnedSeatAlias: { placementId: 3 }, + }, + }, + }, + ], }, ], timeout: 1000, @@ -203,8 +217,8 @@ describe('external bundle + served shim evaluated together', () => { const payload = JSON.parse(body); const adUnit = payload.adUnits[0]; expect(adUnit.code).toBe('ad-slot-1'); - // The server-side bidder was folded into the trustedServer request - // instead of running client-side. + // Stored trustedServer params retain only authoritative server-side route + // codes; provider IDs and returned aliases cannot reach /auction. const trustedServerBid = adUnit.bids.find((bid) => bid.bidder === 'trustedServer'); expect(trustedServerBid.params.bidderParams).toEqual({ appnexus: { placementId: 1 } }); diff --git a/docs/guide/api-reference.md b/docs/guide/api-reference.md index 184915f0a..d280368a1 100644 --- a/docs/guide/api-reference.md +++ b/docs/guide/api-reference.md @@ -83,38 +83,38 @@ curl -i "https://edge.example.com/_ts/clear-tester" ## First-Party Endpoints -### GET /first-party/ad +### POST /auction -Server-side ad rendering endpoint. Returns complete HTML for a single ad slot. +Browser and programmatic auction endpoint. It accepts the Trusted Server ad-unit +request shape and returns an OpenRTB response with sanitized creatives. -**Query Parameters:** -| Parameter | Type | Required | Description | -|-----------|------|----------|-------------| -| `slot` | string | Yes | Ad slot identifier (matches ad unit code) | -| `w` | integer | Yes | Ad width in pixels | -| `h` | integer | Yes | Ad height in pixels | - -**Response:** +**Request Body:** -- **Content-Type:** `text/html; charset=utf-8` -- **Body:** Complete HTML creative with first-party proxying applied +```json +{ + "adUnits": [ + { + "code": "header-banner", + "mediaTypes": { "banner": { "sizes": [[728, 90]] } }, + "bids": [ + { + "bidder": "example-server-bidder", + "params": { "placement": "example-placement" } + } + ] + } + ] +} +``` **Example:** ```bash -curl "https://edge.example.com/first-party/ad?slot=header-banner&w=728&h=90" +curl -X POST https://edge.example.com/auction \ + -H "Content-Type: application/json" \ + -d '{"adUnits":[{"code":"banner","mediaTypes":{"banner":{"sizes":[[300,250]]}}}]}' ``` -**Response Headers:** - -No EC ID response header is emitted. EC identity is maintained with the `ts-ec` cookie. - -**Use Cases:** - -- Server-side ad rendering -- Direct iframe embedding -- First-party ad delivery - --- ## Edge Cookie Endpoints @@ -140,10 +140,10 @@ Returns EC identity plus the authenticated partner's UID and EID for the current "ec": "954d...e0c3.nZ1GxL", "consent": "ok", "degraded": false, - "source_domain": "formally-vital-lion.edgecompute.app", + "source_domain": "ssp.example.com", "uid": "mock-user-123", "eid": { - "source": "formally-vital-lion.edgecompute.app", + "source": "ssp.example.com", "uids": [{ "id": "mock-user-123", "atype": 3 }] }, "cluster_size": 3 @@ -188,63 +188,6 @@ Server-to-server batch sync endpoint for writing EC ID to partner UID mappings. --- -### POST /third-party/ad - -Client-side auction endpoint for TSJS library. - -**Request Body:** - -```json -{ - "adUnits": [ - { - "code": "header-banner", - "mediaTypes": { - "banner": { - "sizes": [ - [728, 90], - [970, 250] - ] - } - } - } - ], - "config": { - "debug": false - } -} -``` - -**Response:** - -```json -{ - "seatbid": [ - { - "bid": [ - { - "impid": "header-banner", - "adm": "...", - "price": 1.5, - "w": 728, - "h": 90 - } - ] - } - ] -} -``` - -**Example:** - -```bash -curl -X POST https://edge.example.com/third-party/ad \ - -H "Content-Type: application/json" \ - -d '{"adUnits":[{"code":"banner","mediaTypes":{"banner":{"sizes":[[300,250]]}}}]}' -``` - ---- - ### GET /first-party/proxy Unified proxy for resources referenced by creatives (images, scripts, CSS, etc.). @@ -626,10 +569,10 @@ The auction preview validates the stored record and partner configuration, but c | `5xx` | Unexpected configuration or KV failure (plaintext) | ```bash -curl -u admin:secure-password \ +curl -u 'admin:' \ "https://edge.example.com/_ts/admin/ec/aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa.abc123" -curl -u admin:secure-password \ +curl -u 'admin:' \ --cookie "ts-ec=aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa.abc123" \ "https://edge.example.com/_ts/admin/ec" ``` @@ -660,7 +603,7 @@ After successful authentication this endpoint always returns `200 OK`; missing o ``` ```bash -curl -u admin:secure-password \ +curl -u 'admin:' \ --cookie "sharedId=fictional-shared-id" \ "https://edge.example.com/_ts/admin/eids" ``` @@ -711,30 +654,21 @@ All integration modules are built at compile time. At runtime, the server concat ### Prebid Integration -#### GET /first-party/ad - -See [First-Party Endpoints](#get-first-party-ad) above. +#### POST /auction -#### POST /third-party/ad +See [First-Party Endpoints](#post-auction) above. -See [First-Party Endpoints](#post-third-party-ad) above. +#### GET /integrations/prebid/bundle.js -#### GET /prebid.js (Optional) +Proxies the configured external Prebid bundle through the first-party domain. +The optional `v` query value is the configured SHA-256 cache key. -Returns empty JavaScript to override Prebid.js when `script_handler` is configured. +#### GET `` (Optional) -**Configuration:** - -```toml -[integrations.prebid] -script_handler = "/prebid.js" -``` - -**Response:** - -- **Content-Type:** `application/javascript; charset=utf-8` -- **Body:** `// Prebid.js override by Trusted Server` -- **Cache:** `immutable, max-age=31536000` +Each configured Prebid script pattern registers an endpoint that returns empty +JavaScript, preventing the publisher's original Prebid bundle from loading. +The defaults include `/prebid.js`, `/prebid.min.js`, `/prebidjs.js`, and +`/prebidjs.min.js`; set `script_patterns = []` to disable interception. --- @@ -844,13 +778,16 @@ Endpoints under protected paths require HTTP Basic Authentication: [[handlers]] path = "^/_ts/admin" username = "admin" -password = "secure-password" +password = "admin_password" ``` +`password` is a key in the Trusted Server secret store. Provision the actual +Basic Authentication password under `admin_password`. + **Usage:** ```bash -curl -u admin:secure-password https://edge.example.com/_ts/admin/keys/rotate +curl -u 'admin:' https://edge.example.com/_ts/admin/keys/rotate ``` **Protected Endpoints:** diff --git a/docs/guide/architecture.md b/docs/guide/architecture.md index da1a58bcd..3aa1b8857 100644 --- a/docs/guide/architecture.md +++ b/docs/guide/architecture.md @@ -68,7 +68,7 @@ Fermyon Spin adapter (`wasm32-wasip1` component): - Production-capable deployment target for the Spin runtime - Platform services (config store, secret store, KV) backed by Spin component variables and the EdgeZero KV handle - Outbound HTTP via `spin_sdk::http::send` — no configurable per-request timeout (see rustdoc) -- Single auction provider only; multi-provider fan-out requires the Fastly adapter +- Single auction provider only; enabled multi-provider plans fail target validation at startup ```bash # Check (native) @@ -107,6 +107,13 @@ pub trait RequestWrapper { External configuration via `trusted-server.toml` allows deployment-time customization without code changes. +Server-side auctions are configuration-first. `[auction.providers.]` declares +provider instances and `[auction.bidders.]` maps browser-visible bidders to +exactly one provider. Startup compiles these maps into one immutable +`AuctionPlan` shared by orchestration and integration registration. Provider IDs +remain distinct from upstream returned seats and browser delivery bidder codes. +The optional mediator is selected separately by `[auction].mediator`. + ### Consent-Aware Design Data collection operations are subject to available consent signals (TCF v2 format, GPP, GPC). Enforcement follows built-in per-jurisdiction rules, with publisher configuration tuning jurisdiction lists, signal interpretation, and conflict resolution. @@ -160,6 +167,12 @@ Page content and request bodies are processed in-flight and are not persisted. E The workspace has multiple WASM runtimes with runtime-specific SDKs. Use target-matched clippy aliases (`cargo clippy-fastly`, `cargo clippy-spin-native`, etc.) rather than broad `--all-features` workspace clippy — the latter is not a reliable gate across adapters. +Fastly and Axum support concurrent auction provider fan-out. Cloudflare and Spin +currently accept at most one provider in an enabled auction. Every adapter runs +target-aware fan-out and backend-name checks at startup. No current adapter +claims an abortable provider-wide total-request deadline, so configured auction +and provider timeouts are logical budgets rather than hard wall-clock ceilings. + ## Next Steps - Learn about [Configuration](/guide/configuration) diff --git a/docs/guide/asset-routes.md b/docs/guide/asset-routes.md index 8ac9b25cd..405bc1d1b 100644 --- a/docs/guide/asset-routes.md +++ b/docs/guide/asset-routes.md @@ -64,10 +64,9 @@ origin_url = "https://bucket.s3.us-east-1.amazonaws.com" type = "s3_sigv4" region = "us-east-1" origin_query = "strip" -secret_store = "s3-auth" -access_key_id = "access_key_id" -secret_access_key = "secret_access_key" -# session_token = "session_token" +access_key_id = "s3_access_key_id" +secret_access_key = "s3_secret_access_key" +# session_token = "s3_session_token" ``` ### S3 requirements @@ -77,22 +76,21 @@ secret_access_key = "secret_access_key" - S3 support is for `GET` and `HEAD` asset reads. - Signing uses header-based AWS SigV4, not presigned URLs. - The signer uses `x-amz-content-sha256: UNSIGNED-PAYLOAD`. -- Credentials are loaded from the configured runtime secret store and cached per process by configured secret names. +- Credential references resolve from the logical `trusted_server_secrets` store while runtime settings are built. Signing performs no request-time secret-store reads. - Successful authenticated S3 responses preserve the origin `Cache-Control`; configure object cache headers intentionally. - Existing client `Authorization` and `x-amz-*` signing headers are replaced before signing. ### Secret store values -The default secret store and key names are: +Credential fields contain secret key references: -| Config field | Default value | Secret value | +| Config field | Default key | Resolved value | | ------------------- | ------------------- | ------------------------------------ | -| `secret_store` | `s3-auth` | Secret store name | | `access_key_id` | `access_key_id` | AWS access key ID | | `secret_access_key` | `secret_access_key` | AWS secret access key | | `session_token` | unset | Optional AWS temporary session token | -Use private deployment configuration for environment-specific store names or profile tables. +Place those values in the logical `trusted_server_secrets` store. Adapter configuration maps that logical ID to an environment-specific physical store such as Fastly `ts_secrets`. The legacy `secret_store` field is accepted for one migration release but ignored and omitted from newly pushed config. ## Origin query policy diff --git a/docs/guide/auction-orchestration.md b/docs/guide/auction-orchestration.md index a47dbf155..fe2e54fdf 100644 --- a/docs/guide/auction-orchestration.md +++ b/docs/guide/auction-orchestration.md @@ -213,22 +213,26 @@ The orchestrator is composed of several modules: | `endpoints.rs` | `crates/trusted-server-core/src/auction/` | HTTP handler for `POST /auction` | | `config.rs` | `crates/trusted-server-core/src/auction/` | Auction configuration types | -### Provider Auto-Discovery +### Configuration-first plan -Providers register themselves at startup via builder functions. The `build_orchestrator()` function in `auction/mod.rs` iterates all registered builders, passes the application settings, and each builder returns zero or more providers depending on whether its config section is present and enabled: +At startup, Trusted Server compiles `[auction.providers]` and +`[auction.bidders]` through one registry into an immutable `AuctionPlan`. +Provider IDs, endpoints, profile defaults, routes, static extensions, and +notification policy are resolved once. The same `Arc` is shared by +the orchestrator and integration registry; request handling does not reinterpret +raw provider configuration. -```rust -// Each integration registers its own builder -fn provider_builders() -> &'static [ProviderBuilder] { - &[ - prebid::register_auction_provider, - aps::register_providers, - adserver_mock::register_providers, - ] -} -``` +The first version registers three OpenRTB 2.6 profiles in Rust: + +- `standard` for the common banner subset and bounded static extensions; +- `prebid-server` for PBS request, response, cache, override, and diagnostics + behavior; and +- `aps` for APS account/SDK fields, response eligibility, and renderer output. -This means you only need to add a config section to `trusted-server.toml` for a provider to be automatically discovered and registered. +Each configured provider is an instance of the generic planned OpenRTB path. +Multiple instances may select the same profile or endpoint and remain distinct +through their provider IDs. The existing `adserver_mock` mediator stays in a +separate static integration path selected by `[auction].mediator`. ## Auction Strategies @@ -241,9 +245,25 @@ When no mediator is set, the orchestrator runs all providers in parallel and sel ```toml [auction] enabled = true -providers = ["prebid", "aps"] -# No mediator — direct price comparison timeout_ms = 2000 + +[auction.providers.pbs-main] +protocol = "openrtb-2.6" +profile = "prebid-server" +endpoint = "https://prebid.example.com/openrtb2/auction" +routing = "explicit" + +[auction.providers.aps-main] +protocol = "openrtb-2.6" +profile = "aps" +endpoint = "https://aps.example.com/e/pb/bid" +routing = "all_eligible" +profile_config = { account_id = "example-aps-account" } + +[auction.bidders.example-server] +provider = "pbs-main" + +# No mediator — direct price comparison ``` **How winner selection works:** @@ -263,9 +283,29 @@ When a `mediator` is configured, provider responses are forwarded to the mediato ```toml [auction] enabled = true -providers = ["prebid", "aps"] -mediator = "adserver_mock" # Enables mediation timeout_ms = 2000 +mediator = "adserver_mock" # Enables mediation + +[auction.providers.pbs-main] +protocol = "openrtb-2.6" +profile = "prebid-server" +endpoint = "https://prebid.example.com/openrtb2/auction" +routing = "explicit" + +[auction.providers.aps-main] +protocol = "openrtb-2.6" +profile = "aps" +endpoint = "https://aps.example.com/e/pb/bid" +routing = "all_eligible" +profile_config = { account_id = "example-aps-account" } + +[auction.bidders.example-server] +provider = "pbs-main" + +[integrations.adserver_mock] +enabled = true +endpoint = "https://mediator.example.com/mediate" +timeout_ms = 500 ``` **How mediation works:** @@ -287,7 +327,7 @@ All demand sources implement the `AuctionProvider` trait: ```rust pub trait AuctionProvider: Send + Sync { - fn provider_name(&self) -> &'static str; + fn provider_name(&self) -> &str; fn request_bids( &self, @@ -336,11 +376,17 @@ Transforms auction requests into OpenRTB 2.x format and sends them to a Prebid S - When `debug` is enabled, PBS debug payload and per-bid status (`bidstatus`) also included ```toml -[integrations.prebid] -enabled = true -server_url = "https://prebid-server.example.com" -timeout_ms = 1000 -bidders = ["appnexus", "rubicon"] +[auction.providers.pbs-main] +protocol = "openrtb-2.6" +profile = "prebid-server" +endpoint = "https://prebid.example.com/openrtb2/auction" +routing = "explicit" + +[auction.providers.pbs-main.profile_config] +debug = false + +[auction.bidders.example-server] +provider = "pbs-main" ``` ### APS Provider @@ -363,10 +409,14 @@ Builds an independent banner OpenRTB request for Amazon Publisher Services. - a minimized typed renderer is preserved instead of creative markup or APS notifications. ```toml -[integrations.aps] -enabled = true -account_id = "example-account" -timeout_ms = 800 +[auction.providers.aps-main] +protocol = "openrtb-2.6" +profile = "aps" +endpoint = "https://aps.example.com/e/pb/bid" +routing = "all_eligible" + +[auction.providers.aps-main.profile_config] +account_id = "example-aps-account" debug = false allow_script_creatives = false ``` @@ -652,127 +702,142 @@ Each proxied URL includes a `tstoken` HMAC signature for tamper protection. See ## Configuration -### Full Example +### Full example ```toml [auction] enabled = true sanitize_creatives = false # Opt-in; blanks script-based creatives when enabled rewrite_creatives = true -providers = ["prebid", "aps"] -mediator = "adserver_mock" # Remove for parallel_only strategy timeout_ms = 2000 +mediator = "adserver_mock" -[integrations.prebid] -enabled = true -server_url = "https://prebid-server.example.com" -timeout_ms = 1000 -bidders = ["appnexus", "rubicon"] -auto_configure = true +[auction.providers.pbs-main] +protocol = "openrtb-2.6" +profile = "prebid-server" +endpoint = "https://prebid.example.com/openrtb2/auction" +timeout_ms = 900 +routing = "explicit" + +[auction.providers.pbs-main.profile_config] debug = false +test_mode = false +consent_forwarding = "both" -[integrations.aps] -enabled = true -account_id = "example-account" -timeout_ms = 800 +[auction.providers.pbs-main.notifications] +suppress_all = false +suppress_seats = ["example-seat"] + +[auction.providers.aps-main] +protocol = "openrtb-2.6" +profile = "aps" +endpoint = "https://aps.example.com/e/pb/bid" +routing = "all_eligible" + +[auction.providers.aps-main.profile_config] +account_id = "example-aps-account" debug = false allow_script_creatives = false -[integrations.adserver_mock] +[integrations.aps] enabled = true -endpoint = "https://your-mediator.example.com/adserver/mediate" -timeout_ms = 500 -price_floor = 0.50 -``` +rendering_mode = "trusted_server" -### Configuration Reference - -#### `[auction]` - -| Field | Type | Default | Description | -| -------------------- | -------- | ------- | --------------------------------------------------------------- | -| `enabled` | bool | `false` | Enable the auction system | -| `sanitize_creatives` | bool | `false` | Strip executable markup from winning-bid `adm` before delivery | -| `rewrite_creatives` | bool | `true` | Rewrite winning-bid `adm` through first-party endpoints | -| `providers` | string[] | `[]` | Ordered list of provider names to call | -| `mediator` | string? | `null` | Provider name to use as mediator (enables `parallel_mediation`) | -| `timeout_ms` | u32 | `2000` | Overall auction timeout in milliseconds | - -Both creative-processing fields must be present in the TOML for their -environment overrides to apply; see -[Environment Variable Overrides](#environment-variable-overrides). - -#### `[integrations.prebid]` - -| Field | Type | Default | Description | -| ---------------- | -------- | ----------------- | -------------------------------------------------------------------------------------- | -| `enabled` | bool | `true` | Enable Prebid provider | -| `server_url` | string | — | Prebid Server URL (required) | -| `timeout_ms` | u32 | `1000` | Request timeout | -| `bidders` | string[] | `["mocktioneer"]` | Default bidders when not specified per-slot | -| `auto_configure` | bool | `true` | Auto-remove client-side prebid.js scripts | -| `debug` | bool | `false` | Enable Prebid debug mode (sets `ext.prebid.debug` and `ext.prebid.returnallbidstatus`) | -| `test_mode` | bool | `false` | Set OpenRTB `test: 1` for non-billable test traffic | - -#### `[integrations.aps]` - -| Field | Type | Default | Description | -| ------------------------ | ------ | ----------------------------- | ----------------------------------------------------------------- | -| `enabled` | bool | `false` | Enable APS provider | -| `account_id` | string | — | APS account ID (required; `pub_id` is an alias) | -| `endpoint` | string | Built-in APS OpenRTB endpoint | Optional APS OpenRTB endpoint override | -| `timeout_ms` | u32 | `800` | Request timeout | -| `debug` | bool | `false` | Include the raw APS HTTP exchange in `/auction` provider metadata | -| `inventory_domain` | string | — | Override `site.domain` for APS-authorized inventory | -| `inventory_page_origin` | string | — | HTTPS origin paired with `inventory_domain` for `site.page` | -| `allow_script_creatives` | bool | `false` | Admit script bids before APS candidate reduction | - -#### `[integrations.adserver_mock]` - -| Field | Type | Default | Description | -| ------------- | ------ | ---------------------------------------- | ------------------------- | -| `enabled` | bool | `false` | Enable mediator | -| `endpoint` | string | `http://localhost:6767/adserver/mediate` | Mediator service endpoint | -| `timeout_ms` | u32 | `500` | Request timeout | -| `price_floor` | f64? | `null` | Global price floor CPM | - -### Timeout Tuning - -The orchestrator timeout should exceed the sum of provider timeouts to allow all providers to respond. Providers that exceed their individual timeouts are collected as they finish — the orchestrator doesn't wait indefinitely. - -```toml -[auction] -timeout_ms = 2000 # Overall ceiling +[auction.bidders.example-server] +provider = "pbs-main" [integrations.prebid] -timeout_ms = 1000 # Prebid Server budget +enabled = true +timeout_ms = 1000 +debug = false +client_side_bidders = ["example-browser"] +external_bundle_url = "https://assets.example.com/prebid/trusted-prebid.js" -[integrations.aps] -timeout_ms = 800 # APS budget +[proxy] +allowed_domains = ["assets.example.com"] [integrations.adserver_mock] -timeout_ms = 500 # Mediator budget (called after providers) +enabled = true +endpoint = "https://mediator.example.com/mediate" +timeout_ms = 500 ``` -### Environment Variable Overrides - -The typed `ts config validate`, `ts config diff`, and `ts config push` flows can -override auction values that already exist in the TOML. EdgeZero v0.0.4 does -not create missing leaves, so existing configs must add **both** -`rewrite_creatives = true` and `sanitize_creatives = false` under `[auction]` -before relying on the corresponding environment overrides — an override for a -missing leaf is silently ignored. +`[auction.providers]` is a map, not a provider-name list. Each provider ID owns +endpoint/backend correlation and telemetry. `[auction.bidders]` maps each +client-visible bidder ID to one provider. The mediator remains a separately +registered integration selected by `[auction].mediator`. + +Creative processing is independent of provider routing. `sanitize_creatives` +is opt-in and strips executable markup. `rewrite_creatives` defaults to `true` +and rewrites eligible URLs through first-party endpoints. + +Common provider fields and defaults: + +| Field | Default | Meaning | +| ---------------- | --------------- | -------------------------------------------------------------- | +| `protocol` | Required | `openrtb-2.6` | +| `profile` | `standard` | Typed OpenRTB behavior | +| `endpoint` | Required | Fixed absolute HTTPS endpoint | +| `timeout_ms` | Profile default | PBS 1000 ms, APS 800 ms, standard inherits auction timeout | +| `routing` | `explicit` | `explicit` or `all_eligible` | +| `profile_config` | `{}` | Profile-owned typed settings | +| `notifications` | No suppression | Common `nurl`/`burl` suppression by all bids or returned seats | + +APS normally uses `all_eligible`, which sends every compatible banner slot but +never another provider's bidder parameters. `explicit` providers receive only +centrally routed or trusted stored-request demand. + +Provider IDs must match `^[a-z][a-z0-9-]{0,62}$`. Bidder IDs are limited to 128 +UTF-8 bytes and cannot be the exact reserved browser envelope ID +`trustedServer`. Static standard-profile `request_ext` and `imp_ext` objects +are each limited to 16 KiB, eight container levels, and 256 keys at one object +level. Notification seat lists are limited to 128 unique entries of at most 128 +UTF-8 bytes each. + +### Validation and target capability + +Target-independent `ts config validate` compiles profiles, defaults, routes, +endpoints, bounds, signing structure, and mediator selection. Every adapter +startup compiles the same plan and then validates backend-name prediction, +fan-out capability, and target resource limits. Fastly and Axum allow +multi-provider fan-out; Cloudflare and Spin currently reject enabled auctions +with more than one provider. Fastly reserves 40 of its default 200 dynamic +backend names for non-auction traffic and rejects auction plans whose provider +IDs and reachable timeout buckets could require more than the remaining 160. + +This tree does not yet have the EdgeZero callback required to run target-aware +validation before `ts config push --adapter ` performs remote work. +Until that callback lands, startup remains the mandatory target-aware gate. + +### Timeout behavior + +For each provider, Trusted Server uses the smaller of its resolved timeout and +the remaining auction budget for launch decisions and OpenRTB `tmax`. The +mediator is not launched after the logical auction budget is exhausted. + +No current adapter claims an abortable provider-wide total-request deadline. +Already-launched work may complete after the logical budget, and a completed +late response can remain eligible. Local decision and delivery also finish +after network launch closes, so `timeout_ms` is not a hard wall-clock ceiling +and an auction can exceed it. + +Browser Prebid `timeout_ms` and `debug` stay under `[integrations.prebid]` and +are independent of all server provider values. Server endpoint, timeout, +routes, profile debug/test/overrides/consent, and notification suppression do +not belong to the browser integration. + +### Environment variable overrides + +Environment overlays replace leaves that already exist in TOML: ```bash TRUSTED_SERVER__AUCTION__ENABLED=true -TRUSTED_SERVER__AUCTION__REWRITE_CREATIVES=true TRUSTED_SERVER__AUCTION__SANITIZE_CREATIVES=false -TRUSTED_SERVER__AUCTION__PROVIDERS=prebid,aps -TRUSTED_SERVER__AUCTION__MEDIATOR=adserver_mock +TRUSTED_SERVER__AUCTION__REWRITE_CREATIVES=true TRUSTED_SERVER__AUCTION__TIMEOUT_MS=2000 -TRUSTED_SERVER__INTEGRATIONS__PREBID__SERVER_URL=https://pbs.example.com -TRUSTED_SERVER__INTEGRATIONS__APS__ACCOUNT_ID=example-account -TRUSTED_SERVER__INTEGRATIONS__APS__DEBUG=false +TRUSTED_SERVER__AUCTION__PROVIDERS__PBS-MAIN__ENDPOINT=https://prebid.example.com/openrtb2/auction +TRUSTED_SERVER__AUCTION__PROVIDERS__PBS-MAIN__TIMEOUT_MS=900 +TRUSTED_SERVER__AUCTION__MEDIATOR=adserver_mock ``` Before rolling back to a binary that does not know a creative-processing field, diff --git a/docs/guide/cli.md b/docs/guide/cli.md index b6829895e..0e0ee9553 100644 --- a/docs/guide/cli.md +++ b/docs/guide/cli.md @@ -76,6 +76,68 @@ Trusted Server settings JSON. This blob model is intentional because full Trusted Server configs can exceed Fastly limits when split into one config-store entry per setting. +Reclaim orphaned chunk entries leaked from prior oversized pushes: + +```bash +ts config gc --adapter fastly +``` + +Without `--yes`, `config gc` only previews: it reports what it would delete and +deletes nothing. `--dry-run` states that intent explicitly and conflicts with +`--yes`. To actually delete, pass `--yes` together with `--older-than ` +(`s`/`m`/`h`/`d` suffixes, e.g. `7d`; a bare number means seconds): + +```bash +ts config gc --adapter fastly --yes --older-than 7d +``` + +`config gc` sweeps every root in the selected physical store, so `--older-than` +is a safety assertion about the whole store: nothing in it changed within the +window and no writer is targeting it. Unlike the other `config` subcommands, +`gc` never loads the typed app config; its `--no-env` flag instead ignores +`EDGEZERO__STORES__CONFIG____NAME` when resolving which physical store to +sweep, and `--store ` overrides the manifest's config-store id outright. +Both change which store gets swept, so on a destructive run check the store id +`gc` reports before passing `--yes`. + +### Diagnose ad-template configuration + +The static `ts config ad-templates` commands evaluate local configuration +without launching a browser: + +| Command | Purpose | +| ---------------------------------------- | ------------------------------------------------------------------------- | +| `lint` | Summarize configuration and report invalid slot page patterns. | +| `match [--details]` | List matching slots; `--details` includes divs, paths, formats/providers. | +| `check --expected-slot ID` | Assert the exact matching slot set; repeat `--expected-slot`. | +| `check --expect-no-slots` | Assert that no slots match. | +| `explain ` | Print every runtime ad-stack gate and its final yes/no verdict. | + +`check --allow-extra-slots` permits matches beyond the repeated +`--expected-slot` values. It conflicts with `--expect-no-slots`. + +`explain` models a GET navigation with consent allowed by default. Use +`--method `, `--non-navigation`, `--prefetch`, `--bot`, or +`--consent-denied` to model another request. Provider configuration is printed +as a separate advisory; it does not change the runtime gate verdict. + +Every `ts config ad-templates ...` and `ts audit ad-templates ...` command +accepts the same config-location flags: + +| Flag | Behavior | +| --------------------- | ----------------------------------------------------------------------------- | +| `--app-config ` | Read this app config instead of deriving `.toml` from the manifest. | +| `--manifest ` | Read this manifest; defaults to `edgezero.toml`. | +| `--no-env` | Disable `TRUSTED_SERVER__...` overlays for read-only commands. | + +The mutating audit generator always edits file-backed values and never writes +environment-only overlays into TOML, including during `--dry-run`. + +For CI-oriented assertions, exit code 0 means the assertion passed, 1 means the +command ran and found drift (`config ad-templates check` or audit verification +with `--strict`), and 2 means argument parsing, configuration, browser launch, +or another tool operation failed. + ## Lifecycle commands Lifecycle commands delegate to the selected EdgeZero adapter: @@ -88,6 +150,71 @@ ts deploy --adapter fastly ts serve --adapter fastly ``` +`ts deploy` accepts `--staging` (Fastly only) to build and upload a staged +draft version cloned from the active one instead of activating a production +deploy. Adapter passthrough arguments must now follow a `--` separator; unknown +flags before `--` (including the renamed-away `--stage`) are rejected at parse +time rather than forwarded. This is a change: passthrough args previously +worked without the separator, so existing runbooks and CI jobs that pass +adapter flags directly need the `--` added: + +```bash +ts deploy --adapter fastly --service-id --staging +ts deploy --adapter fastly -- --comment "release" +``` + +A staged deploy only redirects the staged version's config selector at the +`_staging` key — it does not copy the production config blob +there. Push the staged config before probing the staged version: + +```bash +ts config push --adapter fastly --staging +ts config diff --adapter fastly --staging +``` + +> **Known limitation:** Trusted Server's Fastly entry point does not yet read +> the version-linked `edgezero_runtime_env` selectors, so a staged version +> currently loads the **production** config blob rather than the staged one — +> a staging healthcheck exercises the new binary against production config. +> Selector resolution for custom entry points is tracked upstream in EdgeZero; +> until it lands, do not rely on `--staging` to validate a config change. + +`--staging` on `config push` / `config diff` writes and compares the +`_staging` key in the same store. It is mutually exclusive +with `--key`: the staging key is derived from the store's logical id, so an +explicit key would be written where nothing reads it. + +Inspect and verify deployments with the deploy lifecycle commands. All three are +Fastly-only — the axum, cloudflare, and spin adapters reject them: + +```bash +# Capture the production rollback target BEFORE deploying: after a deploy this +# prints the NEW version, and Fastly keeps no record of which version was live +# before it, so the target is then unrecoverable. +ts active-version --adapter fastly --service-id + +# Probe a deployed version until it reports healthy. `` is the version +# the deploy activated; pass `--service-id` to `ts deploy` and it emits that as +# a machine-readable `version=` line. +ts healthcheck --adapter fastly --service-id \ + --version --domain edge.example + +# Re-activate the version captured before the deploy +ts rollback --adapter fastly --service-id \ + --version --rollback-to +``` + +`healthcheck` probes `/` by default (`--path` overrides) and makes 3 total +attempts — not 3 retries after a first try — with a 5 second delay between +attempts and a 10 second per-attempt timeout (`--retry`, `--retry-delay`, +`--timeout`). With `--staging` it resolves the staged version's IP from the +service id and probes that instead of the production endpoint. + +`rollback` cannot infer the production rollback target: Fastly exposes no +metadata to tell a previously live version from a staged one, so pass the +version to re-activate via `--rollback-to`. With `--staging`, it deactivates +the staged `--version` instead and needs no `--rollback-to`. + ## Audit a public page `ts audit` loads a public page in a fresh headless Chrome/Chromium session, @@ -98,7 +225,7 @@ Chrome or Chromium must be installed locally. The command checks common PATH names and standard macOS/Linux install locations. ```bash -ts audit https://publisher.example +ts audit generate https://publisher.example ``` By default, the command writes: @@ -118,13 +245,13 @@ ts config validate If a config already exists, avoid overwriting it: ```bash -ts audit https://publisher.example --no-config +ts audit generate https://publisher.example --no-config ``` Use custom output paths when reviewing artifacts first: ```bash -ts audit https://publisher.example \ +ts audit generate https://publisher.example \ --js-assets audit/js-assets.toml \ --config audit/trusted-server.toml ``` @@ -132,9 +259,308 @@ ts audit https://publisher.example \ Use `--force` only when replacing existing output files is intentional: ```bash -ts audit https://publisher.example --force +ts audit generate https://publisher.example --force +``` + +The legacy `ts audit ` form remains a compatibility alias for artifact +generation. New automation should use `ts audit generate `. + +## Generate ad-template slots from a live site + +`ts audit ad-templates generate ` discovers the publisher's ad slots and +rewrites the `[creative_opportunities]` slot array in `trusted-server.toml` in +place, preserving every other section and comment. + +```bash +ts audit ad-templates generate https://publisher.example/ +``` + +It samples the site rather than a single page. Ad slots repeat per site +section, so the crawl is sized by the publisher's taxonomy — a dozen sections — +not its catalogue: + +1. Load the requested page and read its links and, from `robots.txt`, its + sitemap. +2. Group both into candidate sections, keeping one landing page and one article + per section. +3. Load those pages, recording each slot's div, sizes, and GAM ad-unit path. +4. Reconcile every slot across the pages it appeared on. +5. Infer a `{section}` ad-unit template if the evidence proves one. +6. Verify the result loads, then write it. + +### What it writes + +Given a site whose ad units track the section, the run produces: + +```toml +[creative_opportunities] +gam_network_id = "99999" +section_root = "homepage" +section_segment = 0 + +[[creative_opportunities.slot]] +id = "ad-header-0" +div_id = "ad-header-0" +gam_unit_path = "/{network_id}/example/{section}" +page_patterns = ["/", "/deals", "/deals/*", "/news", "/news/*"] +formats = [{ width = 728, height = 90 }] +``` + +Each section contributes **two** patterns. `*` crosses `/` in this glob +dialect, so `/news/*` matches `/news/a/b` but not the bare `/news` landing +page; emitting only the star form would drop the landing page from the slot. + +Sizes are unioned across pages, so a format that renders only on articles +survives alongside the homepage's. + +### When it keeps literal paths, and when it refuses + +A wrong ad-unit template makes the publisher bid against inventory that does not +exist, so the command prefers a narrow literal path over a plausible guess. + +| Situation | Result | +| --------------------------------------------------------------------------------------- | ----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| Only one page was crawled | Literal path. One observation cannot distinguish a literal from a template. | +| The ad unit never varied by section | Literal path. | +| A section's slug is not derivable from its URL (`/site-news` requesting `.../sitenews`) | The slot is omitted; the note lists the ad-unit paths it used and says none generalized. | +| No crawled page lacked a section segment, so `section_root` is unwitnessed | No template is written and the reason names the crawl gap. A slot that merely never appears on the root (a sidebar, an in-article unit) still templates, borrowing the `section_root` another slot witnessed; a note says so. | +| Two path segments could both be the section | No template; the ambiguity is reported. | +| The ad unit varies by device, geo, or anything the URL cannot supply | The refused slot is omitted and the reason is written as a note. | +| Crawled pages report different GAM network ids | The run fails; the pages are not one property. | +| More than a quarter of crawled pages return no slots | The run fails. That is the signature of bot protection serving challenge pages, and writing from it would silently narrow the slot set. | +| Several live elements normalize onto one div-id prefix | The whole group is omitted, on every page of the crawl. A prefix resolves to at most one element and the exact ids change per render; the prefix is named in a note. | +| A per-render token sits before the placement part of a div id | The slot is omitted from a single observation and the family prefix is named in a note; no stable prefix identifies one element. | +| A crawled page redirects off the audited origin | The page is skipped on that profile, its path is named in a note, and it stops counting toward profile coverage. There is no override for generation; another site's evidence is never folded into the config. | + +Every run checks that the config it produced still loads before replacing the +file, and `--dry-run` runs the same check — a clean preview is evidence the +config loads, not just that it parses. Dry-run stdout is a zero-context unified +diff containing only the managed creative-opportunity fields; notes and refusal +reasons go to stderr, so unrelated config and secrets are not printed. Crawl +progress also goes to stderr, one line per phase and page — for example +`Auditing desktop [2/17]: /news`. Progress renders the path only, never the +origin, userinfo, query, or fragment, and there is no flag to suppress it. A +`--dry-run` that changes nothing says so on stderr too, leaving stdout an empty +diff. + +### Bounding and steering the crawl + +```bash +# Cover more of a large site. +ts audit ad-templates generate https://publisher.example/ --max-sections 20 --max-pages 41 + +# Audit exactly one page, as earlier releases did. +ts audit ad-templates generate https://publisher.example/ --max-pages 1 + +# Trigger lazy-loaded inventory on every crawled page. +ts audit ad-templates generate https://publisher.example/ --scroll + +# Set the patterns yourself; this disables pattern inference entirely, and the +# run fails outright if any slot's template had to borrow section_root. +ts audit ad-templates generate https://publisher.example/ \ + --page-pattern '/' --page-pattern '/news' --page-pattern '/news/*' + +# Preview without writing. +ts audit ad-templates generate https://publisher.example/ --dry-run +``` + +Re-running merges into the existing slots: a slot seen again keeps its +hand-tuned fields and gains this run's patterns and newly observed formats, and +a hand-written `gam_unit_path` template is preserved. A configured `div_id` is +matched exactly when the crawl observed that exact id; it is treated as a +runtime prefix only when it was never observed as a literal element, so a +configured `ad-sidebar-1` no longer absorbs a discovered `ad-sidebar-10` — the +sibling is appended as its own slot. A prefix that does claim several +discovered divs is named in a stderr note, because the runtime resolves a +prefix to at most one element. `--replace` discards existing slots instead, +which also discards any template you wrote by hand. + +`--scroll` performs the same deterministic stepped scroll on every page and +device profile after the initial settle, then waits for the page to settle again +before collecting evidence. It is opt-in because it increases crawl time, ad +requests, and publisher-page side effects. + +During a normal merge, configured slots missing from the current crawl are +preserved and named in a stderr note. Absence is not proof that a slot is stale: +the crawl may have missed a page type, device target, or lazy-loaded placement. +Review coverage and re-run with `--scroll` when appropriate. Only use +`--replace` when intentionally pruning every slot the run did not rediscover. + +A slot that never appeared without a section segment can borrow a +`section_root` witnessed by another slot only while its patterns are derived +from the paths where it was observed. If `--page-pattern` would override those +patterns, generation fails and names the affected div ids; remove the explicit +patterns so the safe per-slot patterns can be derived. + +A merge refuses to change the section policy that preserved `{section}` slots +were written against. If the config has a non-empty `section_root`, an inferred +root or segment mismatch fails and asks for `--replace` as an explicit +migration. An explicitly configured `section_segment` is preserved even when +`section_root` is unset. When the root is unset and the segment is either unset +or agrees with inference, the first merge adopts the inferred root and makes +the otherwise unloadable `{section}` config valid. + +Locale-prefixed sites are inferred at their observed section depth. Only real +ISO 639-1 language codes are read as a locale prefix, so a two-letter _section_ +root such as `/tv` or `/us` keeps sections at the first segment. For +example, `/en/news/story` can produce `section_segment = 1`; generated patterns +retain the locale prefix (`/en/news` and `/en/news/*`). The crawler never +invents an unwitnessed locale or section. + +Behind bot protection, pass a valid clearance cookie. The crawl reuses one +browser session, so clearance earned on the first page carries to the rest, and +`--page-delay-ms` spaces the requests — an unpaced crawl is both discourteous to +the origin and likelier to be challenged partway through: + +```bash +ts audit ad-templates generate https://publisher.example/ \ + --cookie '=' --page-delay-ms 1500 +``` + +Some origins refuse a headless browser outright regardless of the cookie. +`--headful` runs a visible one, which is also the quickest way to _see_ whether +a challenge is being shown: + +```bash +ts audit ad-templates generate https://publisher.example/ --headful +``` + +### Sites behind a consent platform + +Publishers gate slot definition behind their consent platform, and the audit +runs in a throwaway browser profile with no consent cookie. Left alone, such a +site defines no slots at all and looks identical to a site with no ad stack. + +The crawl therefore answers the two IAB interfaces every compliant platform +exposes — TCF v2 and US Privacy — as a consenting, out-of-scope reader, before +any page script runs. This changes only what the audit browser sees; it does not +affect the publisher's own readers. Pass `--no-assume-consent` to observe the +un-consented page instead. + +When a page still yields no slots, the run reports GPT's observable state — +whether the library reached `apiReady`, how many queued commands never drained, +how many scripts ran. An empty slot registry has several very different causes, +and that line distinguishes them. + +### Auditing a production hostname served locally + +`ts dev proxy` serves a production hostname from a local Trusted Server. +Auditing through it keeps the page's origin, cookie scope, and any origin checks +in the ad stack matching production rather than `localhost`: + +```bash +ts dev proxy --map www.publisher.example=127.0.0.1:7676 --upstream-plaintext --rewrite-host + +ts audit ad-templates generate https://www.publisher.example/ \ + --browser-proxy 127.0.0.1:18080 --danger-accept-invalid-certs +``` + +`--danger-accept-invalid-certs` covers the proxy's MITM certificate when the +throwaway browser profile does not trust its CA; installing that CA +(`ts dev proxy ca`) is preferable. Against a real origin the flag is dangerous — +the audit sends any `--cookie` session upstream and treats the response as +evidence, so an invalid certificate could mean an impersonator is both +harvesting the session and fabricating the result. + +Note that a local Trusted Server injects its own configured slots into the page, +so a run through the proxy can rediscover config it already has. Slot ids that +are absent from the current config are the publisher's own. + +### Slots that change div id on every render + +Some ad stacks build div ids from a per-render token, so one placement arrives +under a new id on every page. Those ids match nothing at runtime, so the run +declines to write them and reports the group instead: + +```text +note: skipped 3 slot(s) that look like one placement under a per-render div id + on `/123456789/publisher/overlay` (ex_slot_a1_overlay_1, …); + they share the prefix `ex_slot`. Add it once by hand with a div_id prefix + that is stable across renders +``` + +This particular group is detected by evidence, not by recognising token shapes: +candidates share an ad-unit path and formats, and what separates a fragmented +placement from two legitimate siblings on one unit is co-occurrence — real +siblings appear together on a page, fragments never do. (A single id whose +per-render token sits _before_ the placement part is refused on shape alone, +from one observation, as the table above notes.) The suggested prefix is a +starting point only, not written as a `div_id`, because it reaches only as far +as the observed tokens happen to agree. + +### Checking for a device split + +Publishers often serve a different ad unit per device +(`/network/desktop/news` against `/network/mobile/news`). A desktop-only crawl +cannot see that — it infers a template correct for desktop and silently wrong +for every mobile impression. + +```bash +ts audit ad-templates generate https://publisher.example/ --profiles desktop,mobile ``` +Each page is loaded once per profile. Where the profiles disagree, the slot is +omitted and the diagnostic explains the conflicting paths. The generator does +not fall back to a fabricated default ad unit. + +### Deploy ordering for templated config + +> **A config containing `section_root` or `section_segment` is not +> rollback-safe.** These keys are rejected outright by a Trusted Server binary +> that predates ad-unit templating, and the rejection fails the _entire_ +> configuration load — not just the ad-template section — so every route serves +> an error. This is a full-site outage, not a degraded ad stack. + +When a run reports that it wrote a `{section}` template: + +1. Deploy the template-aware binary **first**. +2. Then `ts config push`. +3. Do **not** roll that binary back while the config is live. + +A run that did not template writes neither key, and leaves the config exactly as +rollback-safe as it was. + +### Audit safety defaults + +Every `ts audit` browser session validates TLS certificates. This matters +because `--cookie` sends a real session to the origin and the page's own +response becomes the audit's evidence, so a certificate-invalid host could both +harvest the session and fabricate what the audit reports. Override only for a +host you control with a known self-signed certificate: + +```bash +ts audit page https://staging.publisher.example --danger-accept-invalid-certs +``` + +`ts audit ad-templates verify` matches configured slots against the +**post-redirect** path, so it refuses a redirect that leaves the requested +origin rather than accepting another site's evidence as verification. Allow it +for a known redirect between your own properties (for example apex to `www`): + +```bash +ts audit ad-templates verify https://publisher.example/ --allow-cross-origin-redirect +``` + +Verification accepts multiple URLs and reuses one browser/profile. Add +`--strict` to return exit 1 when a confirmable slot is missing or partially +confirmed, and `--json` for the stable machine-readable report. Video- and +native-only slots are reported as `unconfirmable`; that records a checker +limitation and does not fail strict mode. A live out-of-page slot with no sizes +against banner-configured formats is reported `partial` and does fail strict +mode. `--scroll` enables the optional second evidence phase and labels evidence +first seen after the deterministic scroll. + +Browser-backed ad-template generation and verification share `--chrome`, +`--headful`, `--browser-proxy`, `--no-assume-consent`, `--scroll`, +`--settle-quiet-ms`, `--settle-max-ms`, and `--danger-accept-invalid-certs`; +`--scroll` runs the same deterministic scroll pass in both, and in verification +it additionally labels the second evidence phase. Verification also accepts +`--browser-profile desktop|mobile`; generation uses `--profiles desktop,mobile` +to compare both profiles. `--cookie NAME=VALUE` is repeatable and creates +host-only, root-path cookies; HTTPS targets also mark them Secure. Verification +refuses cookies when URLs span multiple origins. The quiet settle window must +not exceed the maximum. + `ts audit` is not an EdgeZero adapter command. It has no `--adapter` option and it does not provision resources, push config, build, deploy, or contact platform APIs. diff --git a/docs/guide/configuration.md b/docs/guide/configuration.md index 7bf7c38fa..313599048 100644 --- a/docs/guide/configuration.md +++ b/docs/guide/configuration.md @@ -6,25 +6,27 @@ Learn how to configure Trusted Server for your deployment. Trusted Server uses a flexible configuration system based on: -1. **TOML Files** - `trusted-server.toml` for base configuration +1. **TOML Files** - `trusted-server.toml` for ordinary configuration and secret key names 2. **Environment Variables** - Typed CLI overrides with the `TRUSTED_SERVER__` prefix -3. **Fastly Stores** - KV/Config/Secret stores for runtime data +3. **EdgeZero Stores** - Config and secret stores for the pushed blob and runtime secret values ## Quick Start ### Minimal Configuration -Create `trusted-server.toml` in your project root: +Create `trusted-server.toml` in your project root. Generate both secret values +first with `openssl rand -base64 32`; the placeholders below are intentionally +rejected until replaced. ```toml [publisher] domain = "publisher.com" cookie_domain = ".publisher.com" origin_url = "https://origin.publisher.com" -proxy_secret = "your-secure-secret-here" +proxy_secret = "publisher_proxy_secret" [ec] -passphrase = "replace-with-32-plus-byte-random-secret" +passphrase = "ec_passphrase" ``` ### Environment Variable Overrides @@ -37,16 +39,90 @@ read by the deployed application at request time. # Format: TRUSTED_SERVER__SECTION__FIELD export TRUSTED_SERVER__PUBLISHER__DOMAIN=publisher.com export TRUSTED_SERVER__PUBLISHER__ORIGIN_URL=https://origin.publisher.com -export TRUSTED_SERVER__EC__PASSPHRASE=replace-with-32-plus-byte-random-secret +# Secret overrides, when needed, are key names, not secret values. +export TRUSTED_SERVER__PUBLISHER__PROXY_SECRET=publisher_proxy_secret +export TRUSTED_SERVER__EC__PASSPHRASE=ec_passphrase +# Replace the rejected placeholder values in trusted-server.toml, then validate. ts config validate ts config push --adapter fastly ``` +### Secret-store migration + +Static app-config credentials contain stable key names only. This includes +publisher, EC, handler, Tinybird, DataDome, and S3 credential fields: + +- `publisher.proxy_secret` +- `ec.passphrase` +- `ec.partners[*].api_token`, when inbound identify or batch sync is used +- `ec.partners[*].ts_pull_token`, when pull sync is enabled +- `handlers[*].password` +- `tinybird.auction_token_secret`, when Tinybird auction telemetry is enabled +- `integrations.datadome.server_side_key_secret_name`, when protection is enabled +- `integrations.datadome.protection_test_bypass.credential_secret_name`, when the bypass is enabled +- `proxy.asset_routes[*].auth.access_key_id`, `secret_access_key`, and optional `session_token` + +Their values belong in the logical `trusted_server_secrets` store and are +resolved only while an instance builds runtime settings. An adapter can map the +logical ID to a different physical name. For example, Fastly commonly maps +`trusted_server_secrets` to physical store `ts_secrets`. + +Migrate an existing deployment in this order: + +1. Populate the physical store mapped from `trusted_server_secrets` with the + existing credential values without printing them in shell history, logs, or + CI output. +2. Replace each active credential value with a stable key name and remove the + legacy Tinybird, DataDome, and S3 `secret_store` selectors. +3. Run `ts config validate`, then `ts config push --adapter fastly --no-diff`. +4. Restart/redeploy instances as needed to load the new values. Rotation is + startup-scoped; changing a store value does not alter already-built state. + +`--no-diff` prevents `config push` from rendering the previous plaintext +configuration during this migration. + +Keep `publisher.proxy_secret` and `ec.passphrase` stable unless intentionally +rotating signed URLs or EC identifiers. On Spin, the app-config blob is stored +under the `trusted_server_config` key in Spin's built-in `default` key-value +store. Set the corresponding CLI store mapping before pushing so the write +matches the runtime lookup: + +```bash +export EDGEZERO__STORES__CONFIG__TRUSTED_SERVER_CONFIG__NAME=default +ts config push --adapter spin +``` + +For local Spin development, add `--local` to the push command. Also declare a +component variable for each chosen secret key name using the encoder documented +in `spin.toml`. Missing stores, keys, invalid UTF-8, and empty values fail +closed; inline plaintext fallback is not supported. + +### Tinybird auction telemetry + +Tinybird uses the same typed secret-reference path as the other static +credentials. Do not configure a feature-specific store: + +```toml +[tinybird] +enabled = true +api_host = "api.example.com" +auction_dataset = "auction_events_raw" +auction_token_secret = "tinybird_auction_append_token" +``` + +Store the APPEND token value under `tinybird_auction_append_token` in the +physical store mapped from `trusted_server_secrets`. The token is resolved once +at startup. Disabled Tinybird telemetry does not require or resolve the token. +The legacy `tinybird.secret_store` field is accepted for one migration release, +but it is ignored and omitted from newly pushed config. + ### Generate Secure Secrets +Generate values locally and write them directly to the platform secret store; +do not put the generated output in `trusted-server.toml` or the app-config blob. + ```bash -# Generate cryptographically random secrets openssl rand -base64 32 ``` @@ -81,15 +157,18 @@ fail and the service will return its startup-error response. ## Example: Production Setup +Generate and substitute every `replace-with-*` value before validation or +deployment. + ```toml [publisher] domain = "publisher.com" cookie_domain = ".publisher.com" origin_url = "https://origin.publisher.com" -proxy_secret = "change-me-to-secure-value" +proxy_secret = "publisher_proxy_secret" [ec] -passphrase = "replace-with-32-plus-byte-random-secret" +passphrase = "ec_passphrase" [request_signing] enabled = true @@ -98,10 +177,28 @@ secret_store_id = "01GYYY" [integrations.prebid] enabled = true -server_url = "https://prebid-server.example.com/openrtb2/auction" +client_side_bidders = ["example-browser-bidder"] +external_bundle_url = "https://assets.example.com/prebid/trusted-prebid.js" + +[proxy] +allowed_domains = ["assets.example.com"] + +[auction] +enabled = true +timeout_ms = 2000 + +[auction.providers.pbs-main] +protocol = "openrtb-2.6" +profile = "prebid-server" +endpoint = "https://prebid.example.com/openrtb2/auction" timeout_ms = 1200 -bidders = ["kargo", "appnexus", "openx"] -client_side_bidders = ["rubicon"] +routing = "explicit" + +[auction.providers.pbs-main.profile_config] +debug = false + +[auction.bidders.example-server-bidder] +provider = "pbs-main" ``` ## Detailed Reference @@ -116,9 +213,10 @@ base TOML configuration by `ts config validate`, `ts config diff`, and stored in the app-config blob. Changing an environment variable requires rerunning validation and pushing the resolved config, not rebuilding the binary. -EdgeZero v0.0.4 only overrides leaves that already exist in the parsed TOML; it -does not create missing fields. Add newly introduced defaulted fields to an -existing config before relying on their environment overrides. Pass `--no-env` +The pinned EdgeZero loader only overrides leaves that already exist in the +parsed TOML; it does not create missing fields. Add newly introduced defaulted +fields to an existing config before relying on their environment overrides. +Secret overlays still contain key names, never secret values. Pass `--no-env` to use file values without the overlay. ### Format @@ -145,26 +243,26 @@ TRUSTED_SERVER__PUBLISHER__DOMAIN=publisher.com **Nested Field**: ```bash -TRUSTED_SERVER__INTEGRATIONS__PREBID__SERVER_URL=https://prebid.example/auction +TRUSTED_SERVER__AUCTION__PROVIDERS__PBS-MAIN__ENDPOINT=https://prebid.example.com/openrtb2/auction ``` **Array Field (JSON)**: ```bash -TRUSTED_SERVER__INTEGRATIONS__PREBID__BIDDERS='["kargo","rubicon"]' +TRUSTED_SERVER__INTEGRATIONS__PREBID__CLIENT_SIDE_BIDDERS='["example-browser-a","example-browser-b"]' ``` **Array Field (Indexed)**: ```bash -TRUSTED_SERVER__INTEGRATIONS__PREBID__BIDDERS__0=kargo -TRUSTED_SERVER__INTEGRATIONS__PREBID__BIDDERS__1=rubicon +TRUSTED_SERVER__INTEGRATIONS__PREBID__CLIENT_SIDE_BIDDERS__0=example-browser-a +TRUSTED_SERVER__INTEGRATIONS__PREBID__CLIENT_SIDE_BIDDERS__1=example-browser-b ``` **Array Field (Comma-Separated)**: ```bash -TRUSTED_SERVER__INTEGRATIONS__PREBID__BIDDERS=kargo,rubicon,appnexus +TRUSTED_SERVER__INTEGRATIONS__PREBID__CLIENT_SIDE_BIDDERS=example-browser-a,example-browser-b ``` ## Publisher Configuration @@ -179,13 +277,13 @@ Core publisher settings for domain, origin, and proxy configuration. | `cookie_domain` | String | Yes | Domain for non-EC cookies (typically with leading dot) | | `origin_url` | String | Yes | Full URL of publisher origin server | | `origin_host_header_override` | String | No | Outbound Host header to send while connecting to `origin_url` | -| `proxy_secret` | String | Yes | Secret key for encrypting/signing proxy URLs | +| `proxy_secret` | String | Yes | Secret-store key name for the proxy URL secret | | `max_buffered_body_bytes` | Integer | No | Buffered-body cap / Fastly stream raw+decoded byte ceiling (default 16 MiB) | > **Note:** EC cookies (`ts-ec`) derive their domain automatically as `.{domain}` and > do not use `cookie_domain`. The `cookie_domain` field is used by other cookie helpers. -**Example**: +**Example** (replace the rejected secret placeholder before validation): ```toml [publisher] @@ -194,7 +292,7 @@ cookie_domain = ".publisher.com" origin_url = "https://origin.publisher.com" # Optional: connect to origin_url but send this outbound Host header. # origin_host_header_override = "www.publisher.com" -proxy_secret = "change-me-to-secure-random-value" +proxy_secret = "publisher_proxy_secret"},{ ``` **Environment Override**: @@ -204,7 +302,7 @@ TRUSTED_SERVER__PUBLISHER__DOMAIN=publisher.com TRUSTED_SERVER__PUBLISHER__COOKIE_DOMAIN=.publisher.com TRUSTED_SERVER__PUBLISHER__ORIGIN_URL=https://origin.publisher.com TRUSTED_SERVER__PUBLISHER__ORIGIN_HOST_HEADER_OVERRIDE=www.publisher.com -TRUSTED_SERVER__PUBLISHER__PROXY_SECRET=your-secret-here +TRUSTED_SERVER__PUBLISHER__PROXY_SECRET=publisher_proxy_secret TRUSTED_SERVER__PUBLISHER__MAX_BUFFERED_BODY_BYTES=16777216 ``` @@ -283,21 +381,12 @@ connecting to the host in `origin_url`. #### `proxy_secret` -**Purpose**: Secret key for HMAC-SHA256 signing of proxy URLs. +**Purpose**: Secret-store key name for the HMAC-SHA256 value used to sign proxy URLs. -**Security**: - -- Keep confidential and secure -- Rotate periodically (90 days recommended) -- Use cryptographically random values (32+ bytes) -- Never commit to version control - -**Generation**: - -```bash -# Generate secure random secret -openssl rand -base64 32 -``` +The referenced value is resolved from `trusted_server_secrets` at startup. It +must be at least 32 bytes, so generate it with a cryptographically secure random +source. Keep that value confidential, rotate it only intentionally, and never +put it in the TOML file or pushed app-config blob. **Usage**: @@ -502,6 +591,9 @@ Settings for Edge Cookie identifier generation. The `ec_store` KV store is the o ### `[ec]` +`passphrase` is a key name in `trusted_server_secrets`; the resolved value must +be at least 32 bytes. Keep it stable to preserve EC identifier continuity. + | Field | Type | Required | Description | | ------------------------- | -------------- | -------- | ----------------------------------------------------------------------- | | `passphrase` | String | Yes | Publisher passphrase used as HMAC key | @@ -515,24 +607,30 @@ Settings for Edge Cookie identifier generation. The `ec_store` KV store is the o `source_domain` is the canonical partner key. It matches incoming OpenRTB EID `source` values and is also used as the EC KV `ids` map key. ::: +`api_token` is optional. Set it to a key in `trusted_server_secrets` only when +the partner calls the inbound identify or batch-sync APIs. A partner without +`api_token` remains available for source-domain lookup, bidstream EIDs, and +outbound pull sync, but cannot authenticate to those inbound APIs. + **Example**: ```toml [ec] -passphrase = "replace-with-32-plus-byte-random-secret" +passphrase = "ec_passphrase" ec_store = "ec_identity_store" [[ec.partners]] name = "Mocktioneer SSP" source_domain = "mocktioneer.example" -api_token = "partner-api-token-32-bytes-minimum" bidstream_enabled = true +# api_token = "partner_api_token" # only for inbound identify or batch sync +# ts_pull_token = "partner_ts_pull_token" # required when pull sync is enabled ``` **Environment Override**: ```bash -TRUSTED_SERVER__EC__PASSPHRASE=your-secret +TRUSTED_SERVER__EC__PASSPHRASE=ec_passphrase TRUSTED_SERVER__EC__EC_STORE=ec_identity_store ``` @@ -540,20 +638,13 @@ TRUSTED_SERVER__EC__EC_STORE=ec_identity_store #### `passphrase` -**Purpose**: Publisher passphrase used as HMAC key for EC ID generation. +**Purpose**: Secret-store key name whose resolved value is the HMAC key for EC ID generation. **Security**: -- Must be non-empty -- Rotate periodically for security -- Store securely (environment variable recommended) - -**Generation**: - -```bash -# Generate secure random key -openssl rand -hex 32 -``` +- The key name is stored in app config; the value is stored in `trusted_server_secrets` +- Keep the value stable unless intentionally rotating EC identifiers +- Do not place the value in environment overlays or the pushed blob **Validation**: Application startup fails if: @@ -690,18 +781,18 @@ Path-based HTTP Basic Authentication. [[handlers]] path = "^/_ts/admin" username = "admin" -password = "secure-password" +password = "admin_password"},{ # Multiple handlers [[handlers]] path = "^/secure" username = "user1" -password = "pass1" +password = "secure_handler_password"},{ [[handlers]] path = "^/api/private" username = "api-user" -password = "api-pass" +password = "api_handler_password"},{ ``` **Environment Override**: @@ -710,12 +801,12 @@ password = "api-pass" # Handler 0 TRUSTED_SERVER__HANDLERS__0__PATH="^/_ts/admin" TRUSTED_SERVER__HANDLERS__0__USERNAME="admin" -TRUSTED_SERVER__HANDLERS__0__PASSWORD="secure-password" +TRUSTED_SERVER__HANDLERS__0__PASSWORD="admin_password" # Handler 1 TRUSTED_SERVER__HANDLERS__1__PATH="^/api/private" TRUSTED_SERVER__HANDLERS__1__USERNAME="api-user" -TRUSTED_SERVER__HANDLERS__1__PASSWORD="api-pass" +TRUSTED_SERVER__HANDLERS__1__PASSWORD="api_handler_password" ``` ### Path Patterns @@ -794,10 +885,9 @@ scheduled for removal **Password Storage**: -- Stored in plain text in config -- Use environment variables in production -- Rotate passwords regularly -- Consider using Fastly Secret Store +- `handlers[*].password` is a key name in `trusted_server_secrets` +- Store the resolved password only in the platform secret store +- Rotate passwords through the store and restart/redeploy instances **Limitations**: @@ -807,12 +897,9 @@ scheduled for removal - No rate limiting (add at edge) ::: warning Production Use -For production, store credentials in environment variables: - -```bash -TRUSTED_SERVER__HANDLERS__0__PASSWORD=$(cat /run/secrets/admin_password) -``` - +Do not put handler passwords in `trusted-server.toml`, environment overlays, or +app-config blobs. Provision the referenced key in `trusted_server_secrets` +before pushing the config. ::: ## URL Rewrite Configuration @@ -1031,15 +1118,14 @@ target_path = "/image/upload/$1.$2" The first supported origin auth type is `s3_sigv4`. -| Field | Type | Required | Default | Description | -| ------------------- | ------ | -------- | ------------------- | ----------------------------------------------- | -| `type` | String | Yes | none | Must be `s3_sigv4` | -| `region` | String | Yes | none | AWS region used in the SigV4 credential scope | -| `secret_store` | String | No | `s3-auth` | Runtime secret store containing AWS credentials | -| `access_key_id` | String | No | `access_key_id` | Secret key containing the AWS access key ID | -| `secret_access_key` | String | No | `secret_access_key` | Secret key containing the AWS secret access key | -| `session_token` | String | No | unset | Optional secret key containing a session token | -| `origin_query` | String | No | route default | `preserve` or `strip` | +| Field | Type | Required | Default | Description | +| ------------------- | ------ | -------- | ------------------- | ------------------------------------------------------------ | +| `type` | String | Yes | none | Must be `s3_sigv4` | +| `region` | String | Yes | none | AWS region used in the SigV4 credential scope | +| `access_key_id` | String | No | `access_key_id` | Default-store secret reference for the AWS access key ID | +| `secret_access_key` | String | No | `secret_access_key` | Default-store secret reference for the AWS secret access key | +| `session_token` | String | No | unset | Optional secret key containing a session token | +| `origin_query` | String | No | route default | `preserve` or `strip` | **Example**: @@ -1052,13 +1138,12 @@ origin_url = "https://bucket.s3.us-east-1.amazonaws.com" type = "s3_sigv4" region = "us-east-1" origin_query = "strip" -secret_store = "s3-auth" -access_key_id = "access_key_id" -secret_access_key = "secret_access_key" -# session_token = "session_token" +access_key_id = "s3_access_key_id" +secret_access_key = "s3_secret_access_key" +# session_token = "s3_session_token" ``` -S3 auth uses header-based AWS SigV4 with `UNSIGNED-PAYLOAD`. It is scoped to read-only asset requests and expects `origin_url` to use the S3 host that AWS validates. Credentials are cached per process by configured secret names after the first successful read. +S3 auth uses header-based AWS SigV4 with `UNSIGNED-PAYLOAD`. It is scoped to read-only asset requests and expects `origin_url` to use the S3 host that AWS validates. Credential references resolve from `trusted_server_secrets` at startup, and request signing performs no secret-store reads. Effective `origin_query` precedence is auth-level `origin_query`, then enabled Image Optimizer `origin_query`, then the route default. @@ -1285,78 +1370,82 @@ apply when the integration section exists in `trusted-server.toml`. ### Prebid Integration -**Section**: `[integrations.prebid]` - -| Field | Type | Default | Description | -| -------------------------- | ------------- | ---------------------------------------------------------------------- | ----------------------------------------------------------------------------------------------------------------------------------------------------- | -| `enabled` | Boolean | `true` | Enable Prebid integration | -| `server_url` | String | Required | Prebid Server endpoint URL | -| `timeout_ms` | Integer | `1000` | Request timeout in milliseconds | -| `bidders` | Array[String] | `["mocktioneer"]` | List of enabled bidders | -| `bid_param_overrides` | Table | `{}` | Static per-bidder param overrides; normalized into the canonical override-rule engine and shallow-merged into bidder params | -| `bid_param_zone_overrides` | Table | `{}` | Per-bidder, per-zone param overrides; normalized into the canonical override-rule engine and shallow-merged into bidder params | -| `bid_param_override_rules` | Array[Table] | `[]` | Canonical ordered override rules with `when` matchers and `set` objects; evaluated after compatibility fields so later rules win on conflicts | -| `suppress_nurl` | Boolean | `false` | Strip `nurl` and `burl` from every PBS bid when the PBS deployment fires win/billing notifications server-side | -| `suppress_nurl_bidders` | Array[String] | `[]` | Bidder seats whose `nurl` and `burl` should be stripped while preserving client-side win/billing pixels for other bidders | -| `debug` | Boolean | `false` | Enable debug mode (sets `ext.prebid.debug` and `returnallbidstatus`; surfaces debug metadata in responses) | -| `test_mode` | Boolean | `false` | Set OpenRTB `test: 1` flag for non-billable test traffic (independent of `debug`) | -| `debug_query_params` | String | `None` | Extra query params appended for debugging | -| `client_side_bidders` | Array[String] | `[]` | Bidders that run client-side via native Prebid.js adapters instead of server-side (see [Prebid docs](/guide/integrations/prebid#client-side-bidders)) | -| `script_patterns` | Array[String] | `["/prebid.js", "/prebid.min.js", "/prebidjs.js", "/prebidjs.min.js"]` | URL patterns for Prebid script interception | - -APS is configured exclusively under `[integrations.aps]`. `aps` entries in -`bidders` or `client_side_bidders` are logged and removed case-insensitively so -an upgrade does not prevent Trusted Server from starting. Remove those entries -from operator configuration; this guard prevents APS demand from reaching -Prebid Server or the client-side Prebid bundle. +`[integrations.prebid]` owns browser behavior only. Server endpoint, provider +timeout, routing, profile debug/test controls, consent forwarding, bidder-param +overrides, and notification suppression belong under `[auction]`. + +| Browser field | Type | Default | Description | +| ------------------------------------- | ------------- | ---------------------------------------------------------------------- | ------------------------------------------------------------------------------ | +| `enabled` | Boolean | `true` | Enable browser bundle injection, interception, and the `trustedServer` adapter | +| `account_id` | String | `None` | Optional account value injected into browser Prebid configuration | +| `timeout_ms` | Integer | `1000` | Browser Prebid.js timeout; independent of every server provider timeout | +| `debug` | Boolean | `false` | Browser Prebid.js debug flag; independent of server profile debug | +| `client_side_bidders` | Array[String] | `[]` | Bidders kept on native browser adapters | +| `excluded_gam_ad_unit_path_suffixes` | Array[String] | `[]` | GAM suffixes excluded from Trusted Server refresh auctions | +| `script_patterns` | Array[String] | `["/prebid.js", "/prebid.min.js", "/prebidjs.js", "/prebidjs.min.js"]` | Publisher Prebid script paths intercepted by Trusted Server | +| `external_bundle_url` | String | Required when enabled | HTTPS publisher-specific Prebid.js bundle URL | +| `external_bundle_sha256` / `*_sri` | String | `None` | Optional bundle integrity and cache metadata | +| `bundle.adapters` / `user_id_modules` | Array[String] | CLI selection | Inputs used by `ts prebid bundle` | + +Server-side bidder codes are derived from validated `[auction.bidders.*]` +routes and injected into the browser. There is no second server bidder list in +`[integrations.prebid]`. A browser bidder stays client-side only when named in +`client_side_bidders` and its adapter is present in the generated bundle. **Example**: ```toml [integrations.prebid] enabled = true -server_url = "https://prebid-server.example/openrtb2/auction" -timeout_ms = 1200 -bidders = ["kargo", "appnexus", "openx"] +timeout_ms = 1000 debug = false -# test_mode = false +client_side_bidders = ["example-browser"] +external_bundle_url = "https://assets.example.com/prebid/trusted-prebid.js" +script_patterns = ["/prebid.js", "/prebid.min.js"] -# Bidders that run client-side via native Prebid.js adapters -client_side_bidders = ["rubicon"] +[proxy] +allowed_domains = ["assets.example.com"] -# Customize script interception (optional) -script_patterns = ["/prebid.js", "/prebid.min.js"] +[integrations.prebid.bundle] +adapters = ["example-browser"] -[integrations.prebid.bid_param_overrides.criteo] -networkId = 99999 -pubid = "server-pub" +[auction.providers.pbs-main] +protocol = "openrtb-2.6" +profile = "prebid-server" +endpoint = "https://prebid.example.com/openrtb2/auction" +routing = "explicit" -[integrations.prebid.bid_param_zone_overrides.kargo] -header = { placementId = "_s2sHeaderPlacement" } +[auction.providers.pbs-main.profile_config] +debug = false +test_mode = false +consent_forwarding = "both" +bid_param_overrides = { example-server = { placement = "example-placement" } } -[[integrations.prebid.bid_param_override_rules]] -when.bidder = "kargo" +[[auction.providers.pbs-main.profile_config.bid_param_override_rules]] +when.bidder = "example-server" when.zone = "header" -set = { placementId = "_s2sHeaderPlacement" } +set = { placement = "example-header-placement" } + +[auction.providers.pbs-main.notifications] +suppress_all = false +suppress_seats = ["example-seat"] + +[auction.bidders.example-server] +provider = "pbs-main" ``` **Environment Override**: ```bash TRUSTED_SERVER__INTEGRATIONS__PREBID__ENABLED=true -TRUSTED_SERVER__INTEGRATIONS__PREBID__SERVER_URL=https://prebid.example/auction -TRUSTED_SERVER__INTEGRATIONS__PREBID__TIMEOUT_MS=1200 -TRUSTED_SERVER__INTEGRATIONS__PREBID__BIDDERS=kargo,appnexus,openx -TRUSTED_SERVER__INTEGRATIONS__PREBID__BID_PARAM_OVERRIDES='{"criteo":{"networkId":99999,"pubid":"server-pub"}}' -TRUSTED_SERVER__INTEGRATIONS__PREBID__BID_PARAM_ZONE_OVERRIDES='{"kargo":{"header":{"placementId":"_s2sHeaderPlacement"}}}' -TRUSTED_SERVER__INTEGRATIONS__PREBID__BID_PARAM_OVERRIDE_RULES='[{"when":{"bidder":"kargo","zone":"header"},"set":{"placementId":"_s2sHeaderPlacement"}}]' -TRUSTED_SERVER__INTEGRATIONS__PREBID__CLIENT_SIDE_BIDDERS=rubicon -TRUSTED_SERVER__INTEGRATIONS__PREBID__DEBUG=false -TRUSTED_SERVER__INTEGRATIONS__PREBID__TEST_MODE=false -TRUSTED_SERVER__INTEGRATIONS__PREBID__DEBUG_QUERY_PARAMS=debug=1 -TRUSTED_SERVER__INTEGRATIONS__PREBID__SCRIPT_PATTERNS='["/prebid.js","/prebid.min.js"]' +TRUSTED_SERVER__INTEGRATIONS__PREBID__TIMEOUT_MS=1000 +TRUSTED_SERVER__INTEGRATIONS__PREBID__CLIENT_SIDE_BIDDERS=example-browser +TRUSTED_SERVER__AUCTION__PROVIDERS__PBS-MAIN__ENDPOINT=https://prebid.example.com/openrtb2/auction +TRUSTED_SERVER__AUCTION__PROVIDERS__PBS-MAIN__PROFILE_CONFIG='{"debug":false,"test_mode":false,"consent_forwarding":"both"}' ``` +Environment overlays only replace leaves already present in TOML. + **Script Pattern Matching**: The `script_patterns` configuration determines which Prebid scripts are intercepted and replaced with empty JavaScript responses. This prevents client-side Prebid.js from loading when using server-side bidding. @@ -1367,13 +1456,19 @@ The `script_patterns` configuration determines which Prebid scripts are intercep See [Prebid Integration](/guide/integrations/prebid) for full details. -**Bid Param Override Surfaces**: +**Server Bid Param Override Surfaces**: + +These fields belong under +`[auction.providers..profile_config]` for a `prebid-server` provider: -- `bid_param_overrides`: Static per-bidder shallow-merge overrides. -- `bid_param_zone_overrides`: Per-bidder, per-zone shallow-merge overrides. -- `bid_param_override_rules`: Canonical ordered rules with `when` matchers and `set` objects. +- `bid_param_overrides`: static per-bidder shallow-merge overrides; +- `bid_param_zone_overrides`: per-bidder, per-zone shallow-merge overrides; and +- `bid_param_override_rules`: canonical ordered rules with `when` matchers and + `set` objects. -Compatibility fields are normalized into the same runtime engine as canonical rules. Explicit `bid_param_override_rules` run after compatibility-derived rules, so later canonical rules win on conflicts. +Compatibility-shaped fields are normalized into the same profile-local runtime +engine. Explicit rules run after compatibility-derived rules, so later rules +win on conflicts. ### Next.js Integration @@ -1478,19 +1573,22 @@ rewrite_scripts = true ## Auction Configuration -Settings for the auction orchestrator that coordinates multiple bid providers. +`[auction.providers.*]` is the only server-side provider inventory, and +`[auction.bidders.*]` is the only client-visible bidder route map. The optional +`[auction].mediator` remains a separate integration selection; it is not a +provider or bidder route. ### `[auction]` -| Field | Type | Default | Description | -| -------------------- | ------------- | ------------------ | -------------------------------------------------------------- | -| `enabled` | Boolean | `false` | Enable the auction orchestrator | -| `sanitize_creatives` | Boolean | `false` | Strip executable markup from winning-bid `adm` before delivery | -| `rewrite_creatives` | Boolean | `true` | Rewrite winning-bid `adm` through first-party endpoints | -| `providers` | Array[String] | `[]` | Provider names that participate (e.g., `["prebid", "aps"]`) | -| `mediator` | String | Optional | Mediator provider name (runs parallel mediation when set) | -| `timeout_ms` | Integer | `2000` | Auction timeout in milliseconds | -| `creative_store` | String | `"creative_store"` | Deprecated; creatives are now delivered inline | +| Field | Type | Default | Description | +| ---------------------- | ------- | ------------------ | -------------------------------------------------------------- | +| `enabled` | Boolean | `false` | Enable the auction orchestrator | +| `sanitize_creatives` | Boolean | `false` | Strip executable markup from winning-bid `adm` before delivery | +| `rewrite_creatives` | Boolean | `true` | Rewrite winning-bid `adm` through first-party endpoints | +| `timeout_ms` | Integer | `2000` | Logical auction budget in milliseconds | +| `mediator` | String | `None` | Optional separate `adserver_mock` mediator | +| `creative_store` | String | `"creative_store"` | Deprecated; creatives are delivered inline | +| `allowed_context_keys` | Array | `[]` | Request context keys admitted into the auction | Creative markup delivered by `POST /auction` and the publisher SSAT/page-bids path is processed by two independent passes. With `sanitize_creatives = true` @@ -1530,7 +1628,7 @@ remove that field's non-default value (and any environment override), run `ts config validate`, push the resulting default-compatible blob, and only then roll back the binary. -**Environment overlays:** EdgeZero v0.0.4 overlays cannot create missing TOML +**Environment overlays:** The pinned EdgeZero loader cannot create missing TOML leaves. Existing configs must add **both** leaves under `[auction]` (`rewrite_creatives` and `sanitize_creatives`) before `TRUSTED_SERVER__AUCTION__REWRITE_CREATIVES` / @@ -1538,43 +1636,176 @@ leaves. Existing configs must add **both** leaves under `[auction]` a missing leaf is silently ignored. ::: -**Example**: +### Provider map + +::: danger Breaking migration from the provider list +The former `[auction].providers = ["prebid", ...]` list and server-owned fields +under `[integrations.prebid]` and `[integrations.aps]` are no longer accepted, +even when an integration is disabled. Replace them with provider instances and +bidder routes before deployment. + +For Prebid Server, move `server_url` to provider `endpoint`, server timeout to +provider `timeout_ms`, request controls and bidder-parameter overrides to the +`prebid-server` `profile_config`, notification suppression to `notifications`, +and each server bidder to `[auction.bidders.]`. Origin-only legacy +`server_url` values compile to `/openrtb2/auction`; query parameters survive, +and configured non-root custom endpoint paths remain exact. Browser timeout, +debug, bundle, script interception, refresh exclusions, and +`client_side_bidders` remain under `[integrations.prebid]`. Configure timeout or +debug under both owners when both browser and server behavior should retain the +old value. + +For APS, move endpoint and timeout to the provider, then move account, +inventory, debug, and creative controls to the `aps` `profile_config`. + +Only bidder codes listed in `[auction.bidders]` are folded into Trusted Server +requests. Unlisted publisher bids remain native browser demand. All provider +endpoints must be absolute HTTPS URLs. + +The old and new blobs are mutually incompatible. Activate the new binary and +map-shaped config together. A binary-first or config-first rolling deployment +will put one version on a schema it rejects. Roll back by restoring the old +binary and old-schema blob together. +::: + +Each table name is the provider ID used for configuration, backend correlation, +health, response metadata, and telemetry. Provider IDs must match +`^[a-z][a-z0-9-]{0,62}$`. Multiple instances may select the same profile and +endpoint because the provider ID remains their distinct runtime identity. ```toml [auction] enabled = true sanitize_creatives = false rewrite_creatives = true -providers = ["aps", "prebid"] timeout_ms = 2000 +mediator = "adserver_mock" -[integrations.aps] -enabled = true -account_id = "example-account" +[auction.providers.pbs-main] +protocol = "openrtb-2.6" +profile = "prebid-server" +endpoint = "https://prebid.example.com/openrtb2/auction" +routing = "explicit" + +[auction.providers.pbs-main.profile_config] +debug = false +test_mode = false +consent_forwarding = "both" + +[auction.providers.pbs-main.notifications] +suppress_all = false +suppress_seats = ["example-seat"] + +[auction.providers.aps-main] +protocol = "openrtb-2.6" +profile = "aps" +endpoint = "https://aps.example.com/e/pb/bid" +routing = "all_eligible" + +[auction.providers.aps-main.profile_config] +account_id = "example-aps-account" debug = false -# Optional pair for deployments hosted away from APS-authorized inventory. -# inventory_domain = "publisher.example" -# inventory_page_origin = "https://www.publisher.example" allow_script_creatives = false -[integrations.prebid] +[auction.bidders.example-server] +provider = "pbs-main" + +[integrations.adserver_mock] enabled = true -server_url = "https://prebid-server.example.com/openrtb2/auction" -``` +endpoint = "https://mediator.example.com/mediate" +timeout_ms = 500 +``` + +| Provider field | Required | Default | Description | +| ---------------- | -------- | --------------- | ------------------------------------------------------------- | +| `protocol` | Yes | None | Must be `openrtb-2.6` | +| `profile` | No | `standard` | `standard`, `prebid-server`, or `aps` | +| `endpoint` | Yes | None | Absolute HTTPS URL with host and no credentials or fragment | +| `timeout_ms` | No | Profile default | Provider logical budget before the remaining-auction cap | +| `routing` | No | `explicit` | `explicit` or `all_eligible` | +| `profile_config` | No | `{}` | Typed object owned by the selected profile | +| `notifications` | No | No suppression | Common `nurl`/`burl` suppression after response normalization | + +Timeout defaults are 1000 ms for `prebid-server`, 800 ms for `aps`, and the +auction timeout for `standard`. An explicit provider timeout overrides the +profile default. Runtime uses `min(provider timeout, auction time remaining)` +for launch decisions and OpenRTB `tmax`. + +`routing = "explicit"` sends only slots carrying a bidder assigned to that +provider (plus trusted stored-request routes). `routing = "all_eligible"` sends +every banner-compatible slot to the provider, regardless of bidder routes. It +does not disclose bidder parameters assigned to another provider. APS commonly +uses `all_eligible` to preserve its whole-inventory participation. + +### Bidder routes and bounds + +Each `[auction.bidders.]` maps one client-visible bidder ID to exactly +one provider. Bidder IDs must be nonempty, no more than 128 UTF-8 bytes, contain +no control characters or surrounding whitespace, and cannot be the reserved +exact ID `trustedServer`. Browser `trustedServer.bidderParams` accepts at most +128 bidder entries; its optional `zone` is at most 256 UTF-8 bytes. + +For the `standard` profile, `profile_config.request_ext` and `imp_ext` must be +JSON objects. Each object is limited to 16 KiB serialized, eight container +levels, and 256 keys at any one object level. Reserved driver, profile, and +signing fields cannot be overwritten. + +Common notification suppression uses exact returned OpenRTB seat values, not +bidder route IDs: -**Environment Override**: +```toml +[auction.providers.pbs-main.notifications] +suppress_all = false +suppress_seats = ["example-seat"] +``` + +`suppress_seats` permits at most 128 unique nonempty entries, each at most 128 +UTF-8 bytes and without ASCII control characters. + +### Validation timing and target limits + +`ts config validate` and ordinary deploy validation compile the complete +target-independent plan: profiles and defaults, routes, endpoint ownership, +extension bounds, notifications, signing structure, and mediator selection. +Target-specific checks are deferred to adapter startup. Startup uses the same +compiled plan and additionally validates backend-name prediction/collisions and +provider fan-out capability. + +Fastly and Axum support multiple configured providers. Cloudflare and Spin +currently reject an enabled auction with more than one provider because those +adapters do not support concurrent provider fan-out. Disabled auctions may keep +dormant multi-provider maps without target rejection. + +A target-aware pre-write `ts config push --adapter ` callback is **not +available in this tree** because the required EdgeZero callback is not yet +available. Until it lands, push performs target-independent validation and +adapter startup is the mandatory target-aware gate. Do not treat a successful +push as proof that a Cloudflare or Spin multi-provider plan can start. + +### Deadline behavior + +Configured timeouts are logical budgets, not hard wall-clock guarantees. No +current adapter claims an abortable provider-wide total-request deadline. +Already-launched work may complete after the logical budget and a completed late +response can remain eligible. Once the logical auction budget is exhausted, +Trusted Server starts no additional provider or mediator network work, then +finishes local decision and delivery. An auction can therefore exceed its +configured wall-clock timeout. + +Creative sanitization is opt-in. URL rewriting runs independently when +`rewrite_creatives = true`. See +[Creative Processing](/guide/creative-processing#auction-rewrite-control). + +**Environment overrides** replace map leaves that already exist in TOML: ```bash TRUSTED_SERVER__AUCTION__ENABLED=true TRUSTED_SERVER__AUCTION__SANITIZE_CREATIVES=false TRUSTED_SERVER__AUCTION__REWRITE_CREATIVES=true -TRUSTED_SERVER__AUCTION__PROVIDERS=aps,prebid -TRUSTED_SERVER__AUCTION__PROVIDERS__0=aps -TRUSTED_SERVER__AUCTION__PROVIDERS__1=prebid -TRUSTED_SERVER__AUCTION__MEDIATOR=adserver_mock TRUSTED_SERVER__AUCTION__TIMEOUT_MS=2000 -TRUSTED_SERVER__AUCTION__CREATIVE_STORE=creative_store -TRUSTED_SERVER__INTEGRATIONS__APS__DEBUG=false +TRUSTED_SERVER__AUCTION__PROVIDERS__PBS-MAIN__ENDPOINT=https://prebid.example.com/openrtb2/auction +TRUSTED_SERVER__AUCTION__PROVIDERS__PBS-MAIN__TIMEOUT_MS=900 +TRUSTED_SERVER__AUCTION__MEDIATOR=adserver_mock ``` ## Creative Opportunities Configuration @@ -1929,14 +2160,15 @@ Configuration is validated at startup: **EC Validation**: -- `passphrase` ≥ 1 character -- `passphrase` ≠ known placeholders (`"secret-key"`, `"secret_key"`, `"trusted-server"` — case-insensitive) +- The `passphrase` key name is non-empty at push time +- The resolved passphrase is at least 32 bytes at runtime +- Known placeholder values are rejected after resolution **Handler Validation**: - `path` is valid regex -- `username` non-empty -- `password` non-empty +- `username` is ordinary configuration and non-empty +- The resolved `password` is non-empty and is checked for placeholders at runtime **Integration Validation**: @@ -1956,8 +2188,7 @@ Configuration is validated at startup: **Error Format**: ``` -Configuration error: Integration 'prebid' configuration failed validation: -server_url: must not be empty +Configuration error: provider `pbs-main` endpoint must be an absolute HTTPS URL ``` ## Best Practices @@ -1971,41 +2202,35 @@ server_url: must not be empty [publisher] domain = "localhost" origin_url = "http://localhost:3000" -proxy_secret = "dev-secret" -``` - -**Staging**: - -```bash -# .env.staging -TRUSTED_SERVER__PUBLISHER__ORIGIN_URL=https://staging.publisher.com -TRUSTED_SERVER__PUBLISHER__PROXY_SECRET=$(cat /run/secrets/proxy_secret_staging) +proxy_secret = "publisher_proxy_secret" ``` -**Production**: +**Staging and production**: -```bash -# All secrets from environment -TRUSTED_SERVER__PUBLISHER__PROXY_SECRET=$(cat /run/secrets/proxy_secret) -TRUSTED_SERVER__EC__PASSPHRASE=$(cat /run/secrets/ec_secret) -TRUSTED_SERVER__HANDLERS__0__PASSWORD=$(cat /run/secrets/admin_password) -``` +- Provision the same key names in the target `trusted_server_secrets` store. +- Keep only the key names in `trusted-server.toml` and environment overlays. +- Push the config after provisioning and restart/redeploy after rotation. ### Secret Management **Do**: -✅ Use environment variables for secrets -✅ Rotate secrets periodically -✅ Generate cryptographically random values -✅ Store in secure secret management (Fastly Secret Store, Vault) -✅ Use different secrets per environment + +- ✅ Store values in the platform secret store +- ✅ Rotate values deliberately and restart/redeploy instances +- ✅ Generate values locally without printing them to logs +- ✅ Use different values per environment when appropriate +- ✅ Keep stable key names for rotation **Don't**: -❌ Commit secrets to version control -❌ Use default/placeholder values -❌ Share secrets across environments -❌ Log secret values -❌ Expose in error messages + +- ❌ Commit secret values to version control +- ❌ Put secret values in environment overlays +- ❌ Put secret values in config diff output or app-config blobs +- ❌ Treat missing secret-store keys as inline values +- ❌ Use default/placeholder values +- ❌ Share secrets across environments +- ❌ Log secret values +- ❌ Expose in error messages ### File Organization @@ -2041,10 +2266,10 @@ trusted-server.dev.toml # Development overrides **"Configuration field '...' is set to a known placeholder value"**: -- `ec.passphrase` cannot be `"secret-key"`, `"secret_key"`, or `"trusted-server"` (case-insensitive) -- `publisher.proxy_secret` cannot be `"change-me-proxy-secret"` (case-insensitive) -- Must be non-empty -- Change to a secure random value (see generation commands above) +- Confirm the referenced key exists in `trusted_server_secrets` +- Ensure the resolved value is non-empty, meets its length requirement, and is not a known placeholder +- Do not replace the key name with a plaintext value in the app config +- Rotate the value in the platform secret store, then restart/redeploy **"Invalid regex"**: @@ -2061,7 +2286,7 @@ trusted-server.dev.toml # Development overrides **Environment Variables Not Applied**: - Run the override through `ts config validate`, `ts config diff`, or `ts config push` -- Verify the target leaf already exists in `trusted-server.toml`; EdgeZero v0.0.4 does not create missing fields +- Verify the target leaf already exists in `trusted-server.toml`; the pinned EdgeZero loader does not create missing fields - Verify prefix: `TRUSTED_SERVER__` - Check separator: `__` (double underscore) - Confirm the variable is exported: `echo $VARIABLE_NAME` diff --git a/docs/guide/ec-setup-guide.md b/docs/guide/ec-setup-guide.md index a11a352a8..c721b1eaf 100644 --- a/docs/guide/ec-setup-guide.md +++ b/docs/guide/ec-setup-guide.md @@ -12,30 +12,38 @@ This guide covers: ## Prerequisites -- Trusted Server deployed and reachable (example: `https://getpurpose.ai`) +- Trusted Server deployed and reachable (example: `https://trusted-server.example.com`) - Access to update `trusted-server.toml` / deployment configuration - Fastly CLI authenticated (for store verification) - A valid TCF v2 format string (`euconsent-v2`) for consent-required requests ## 1) Required Configuration -Set EC configuration in `trusted-server.toml`: +Configure secret-store key names in `trusted-server.toml`, then provision the +passphrase and partner token independently with `openssl rand -base64 32`: ```toml [ec] -passphrase = "replace-with-32-plus-byte-random-secret" +passphrase = "ec_passphrase" ec_store = "ec_identity_store" [[ec.partners]] name = "Mocktioneer SSP" -source_domain = "formally-vital-lion.edgecompute.app" -api_token = "test-batch-sync-key-2026" +source_domain = "ssp.example.com" +api_token = "partner_api_token" bidstream_enabled = true ``` +The `passphrase` and `api_token` fields contain keys in the Trusted Server +secret store, not the credential values. This workflow calls the inbound +identify and batch-sync APIs, so its partner needs `api_token`. Partners that +do not call either API may omit it. Provision high-entropy values under +`ec_passphrase` and `partner_api_token`; see +[Configuration](/guide/configuration#secret-store-migration). + Required behavior assumptions: -- `passphrase` is long-lived HMAC-SHA256 keying material for EC ID derivation; use a high-entropy random value of at least 32 characters +- The value stored under `ec_passphrase` is long-lived HMAC-SHA256 keying material for EC ID derivation; use a high-entropy random value of at least 32 characters - `ec_store` is linked to the active Fastly service version - `ec_store` is the only KV-backed EC lifecycle store; it contains identity graph state, minimal consent metadata, source-domain keyed partner UIDs, and withdrawal tombstones - Live consent is interpreted from request cookies, headers, geolocation, and policy defaults rather than a separate consent KV store @@ -46,12 +54,13 @@ Required behavior assumptions: ## 2) Configure Demo Variables ```bash -TS_BASE_URL="https://getpurpose.ai" -MOCK_SSP_URL="https://formally-vital-lion.edgecompute.app" +TS_BASE_URL="https://trusted-server.example.com" +MOCK_SSP_URL="https://ssp.example.com" -PARTNER_SOURCE_DOMAIN="formally-vital-lion.edgecompute.app" +PARTNER_SOURCE_DOMAIN="ssp.example.com" PARTNER_NAME="Mocktioneer SSP" -PARTNER_API_KEY="test-batch-sync-key-2026" +# Use the value provisioned under the partner_api_token secret-store key. +PARTNER_API_KEY="" # Optional: use a real browser EC if already present EC_ID="<64hex.6chars>" @@ -67,12 +76,14 @@ Partners are configured in `trusted-server.toml` and loaded at startup: ```toml [[ec.partners]] name = "Mocktioneer SSP" -source_domain = "formally-vital-lion.edgecompute.app" -api_token = "test-batch-sync-key-2026" +source_domain = "ssp.example.com" +api_token = "partner_api_token" bidstream_enabled = true ``` -Deploy/restart after changing partner configuration. +Provision the bearer token value under `partner_api_token`, then deploy or +restart after changing partner configuration. The token is required for this +demo because it exercises the inbound partner APIs. ## 4) Acquire or Reuse EC Cookie @@ -138,10 +149,10 @@ Expected shape: "ec": "", "consent": "ok", "degraded": false, - "source_domain": "formally-vital-lion.edgecompute.app", + "source_domain": "ssp.example.com", "uid": "mock-user-123", "eid": { - "source": "formally-vital-lion.edgecompute.app", + "source": "ssp.example.com", "uids": [{ "id": "mock-user-123", "atype": 3 }] }, "cluster_size": 12 @@ -174,7 +185,7 @@ echo "" | base64 -d | python3 -m json.tool Expected decoded payload contains: -- `source = formally-vital-lion.edgecompute.app` +- `source = ssp.example.com` - `uids[0].id = ` ## 8) Fastly KV Operational Checks diff --git a/docs/guide/error-reference.md b/docs/guide/error-reference.md index b5348ed9f..b45cac63a 100644 --- a/docs/guide/error-reference.md +++ b/docs/guide/error-reference.md @@ -55,15 +55,20 @@ Missing required field: publisher.domain **Cause:** Required configuration field not provided -**Solution:** Add the missing field to `trusted-server.toml`: +**Solution:** Add the missing field to `trusted-server.toml`. Secret fields name +entries in the Trusted Server secret store. ```toml [publisher] domain = "your-publisher-domain.com" origin_url = "https://origin.your-publisher-domain.com" -proxy_secret = "change-me-to-random-string" +proxy_secret = "publisher_proxy_secret" ``` +`proxy_secret` names an entry in the Trusted Server secret store. Provision a +high-entropy value under `publisher_proxy_secret`; do not put that value in the +TOML file. + **Required Fields:** - `publisher.domain` @@ -78,21 +83,26 @@ proxy_secret = "change-me-to-random-string" **Error Message:** ``` -Invalid URL in integrations.prebid.server_url +provider `pbs-main` endpoint must be an absolute HTTPS URL ``` -**Cause:** Malformed URL in configuration +**Cause:** Malformed auction provider endpoint. -**Solution:** Ensure URLs are well-formed with scheme: +**Solution:** Configure an absolute HTTPS endpoint with a host and no embedded +credentials or fragment: ```toml # ❌ Wrong -[integrations.prebid] -server_url = "prebid-server.example.com" +[auction.providers.pbs-main] +protocol = "openrtb-2.6" +profile = "prebid-server" +endpoint = "prebid.example.com/openrtb2/auction" # ✅ Correct -[integrations.prebid] -server_url = "https://prebid-server.example.com" +[auction.providers.pbs-main] +protocol = "openrtb-2.6" +profile = "prebid-server" +endpoint = "https://prebid.example.com/openrtb2/auction" ``` --- @@ -114,13 +124,13 @@ Failed to parse environment variable: TRUSTED_SERVER__PUBLISHER__DOMAIN TRUSTED_SERVER__PUBLISHER__DOMAIN="example.com" # For numbers -TRUSTED_SERVER__INTEGRATIONS__PREBID__TIMEOUT_MS=1000 +TRUSTED_SERVER__AUCTION__PROVIDERS__PBS-MAIN__TIMEOUT_MS=1000 # For booleans TRUSTED_SERVER__INTEGRATIONS__PREBID__ENABLED=true -# For arrays (comma-separated) -TRUSTED_SERVER__INTEGRATIONS__PREBID__BIDDERS="appnexus,rubicon" +# For browser-side bidder arrays (comma-separated) +TRUSTED_SERVER__INTEGRATIONS__PREBID__CLIENT_SIDE_BIDDERS="exampleBidder,exampleBrowserBidder" ``` See [Configuration Reference](./configuration.md) for complete patterns. @@ -141,19 +151,23 @@ Failed to generate EC ID: HMAC error **Solution:** -1. Ensure `passphrase` is set in `trusted-server.toml`: +1. Ensure `passphrase` names a secret-store entry in `trusted-server.toml`: ```toml [ec] -passphrase = "replace-with-32-plus-byte-random-secret" +passphrase = "ec_passphrase" ``` -2. Or set via environment variable: +2. If using a typed CLI environment override, set the key name rather than the + passphrase value: ```bash -TRUSTED_SERVER__EC__PASSPHRASE=replace-with-32-plus-byte-random-secret +TRUSTED_SERVER__EC__PASSPHRASE=ec_passphrase ``` +3. Provision a high-entropy value of at least 32 characters under + `ec_passphrase` in the Trusted Server secret store. + --- ### Backend not found @@ -164,21 +178,16 @@ TRUSTED_SERVER__EC__PASSPHRASE=replace-with-32-plus-byte-random-secret Backend not found: prebid-server ``` -**Cause:** Dynamic backend creation failed or backend not configured +**Cause:** Dynamic backend creation for a configured provider endpoint failed. +Provider backends are derived from `[auction.providers.]`; they are not +manually named static Fastly backends. **Solution:** -For integrations using dynamic backends (Prebid, Testlight): - -- Ensure the integration is enabled -- Verify the URL is accessible from Fastly edge -- Check Fastly service limits (backend count) - -For static backends, configure in Fastly dashboard: - -1. Go to Origins → Hosts -2. Add backend with name matching configuration -3. Redeploy service +- Verify the provider endpoint is canonical HTTPS and reachable from the edge +- Check the provider ID and target-specific backend-name validation error +- Check platform backend-count limits +- Run `ts config validate`, then verify target-aware startup validation on the selected adapter --- @@ -220,13 +229,16 @@ Upstream request timeout after 1000ms **Solution:** -1. Increase timeout in configuration: +1. Increase the affected server provider timeout: ```toml -[integrations.prebid] -timeout_ms = 2000 # Increase from default 1000ms +[auction.providers.pbs-main] +timeout_ms = 2000 ``` +Browser `[integrations.prebid].timeout_ms` is independent and does not control +Prebid Server transport. + 2. Verify upstream service is responsive: ```bash @@ -279,13 +291,15 @@ Prebid Server returned 400: Invalid OpenRTB request **Solution:** -1. Enable debug mode: +1. Enable debug mode on the Prebid Server profile: ```toml -[integrations.prebid] -debug = true +[auction.providers.pbs-main] +profile_config = { debug = true } ``` +`[integrations.prebid].debug` controls browser Prebid.js only. + 2. Check logs for request/response details 3. Verify bidders are supported by your Prebid Server 4. Ensure ad unit format is correct: @@ -630,14 +644,18 @@ cargo install viceroy --version 0.17.0 --locked --force ### Enable Debug Logging -**In configuration:** +Browser Prebid.js debug remains under `[integrations.prebid]`: ```toml [integrations.prebid] debug = true +``` + +For Prebid Server diagnostics, enable debug in that provider's profile: -# Or via environment variable -TRUSTED_SERVER__INTEGRATIONS__PREBID__DEBUG=true +```toml +[auction.providers.pbs-main] +profile_config = { debug = true } ``` **Check Fastly logs:** @@ -654,8 +672,10 @@ fastly log-tail # Start local server fastly compute serve -# Test endpoint -curl http://localhost:7676/first-party/ad?slot=test&w=300&h=250 +# Test the auction endpoint +curl -X POST http://localhost:7676/auction \ + -H "Content-Type: application/json" \ + -d '{"adUnits":[{"code":"test","mediaTypes":{"banner":{"sizes":[[300,250]]}}}]}' ``` --- @@ -663,10 +683,10 @@ curl http://localhost:7676/first-party/ad?slot=test&w=300&h=250 ### Validate Configuration ```bash -# Test configuration load -cargo run --bin trusted-server-adapter-fastly -- --validate-config +# Validate the resolved deployment configuration +ts config validate -# Or check startup logs +# Then check startup logs when exercising the runtime fastly compute serve 2>&1 | grep -i "settings" ``` diff --git a/docs/guide/fastly.md b/docs/guide/fastly.md index 7ef1e8ae8..13f82742b 100644 --- a/docs/guide/fastly.md +++ b/docs/guide/fastly.md @@ -258,15 +258,41 @@ Used for storing public configuration (e.g., public keys, key metadata): fastly config-store create --name jwks_store ``` -### Secret Store +### Secret Stores -Used for storing sensitive data (e.g., private signing keys): +Trusted Server keeps static app-config credentials under logical store ID +`trusted_server_secrets`. The physical Fastly store can use another name, such +as `ts_secrets`. Request-signing private keys remain in their separate, +runtime-managed store. + +Set the physical mapping before provisioning: + +```bash +export EDGEZERO__STORES__SECRETS__TRUSTED_SERVER_SECRETS__NAME=ts_secrets +ts provision --adapter fastly +``` + +Provisioning creates or reuses the physical store and persists this runtime +mapping in Fastly Config Store `edgezero_runtime_env`: + +```text +EDGEZERO__STORES__SECRETS__TRUSTED_SERVER_SECRETS__NAME=ts_secrets +``` + +The Fastly service must link both `ts_secrets` and `edgezero_runtime_env` to the +active service version. The custom streaming entry point reads the mapping +before loading app config, so every startup and reload resolves static +credentials from `ts_secrets` while the portable manifest continues to declare +`trusted_server_secrets`. + +Create the separate request-signing store when that feature is enabled: ```bash fastly secret-store create --name signing_keys ``` -Note the store IDs - you'll need them for your `trusted-server.toml` configuration. +Do not copy the same app credential store under a second hardcoded +`trusted_server_secrets` Fastly link. Configure the mapping instead. ## Create EC KV Store @@ -282,14 +308,24 @@ Create it: fastly kv-store create --name ec_identity_store ``` -Configure in `trusted-server.toml`: +Configure the secret-store key name in `trusted-server.toml`: ```toml [ec] -passphrase = "replace-with-32-plus-byte-random-secret" +passphrase = "ec_passphrase" ec_store = "ec_identity_store" ``` +Store the high-entropy passphrase under that key in `ts_secrets`. The resolved +value, rather than the key name, must contain at least 32 characters: + +```bash +fastly secret-store-entry create \ + --store-id= \ + --name=ec_passphrase \ + --secret= +``` + Verify stores exist: ```bash diff --git a/docs/guide/first-party-proxy.md b/docs/guide/first-party-proxy.md index 8c2d35a35..8b9056f0e 100644 --- a/docs/guide/first-party-proxy.md +++ b/docs/guide/first-party-proxy.md @@ -443,9 +443,12 @@ Configure proxy behavior in `trusted-server.toml`: domain = "publisher.com" cookie_domain = ".publisher.com" origin_url = "https://origin.publisher.com" -proxy_secret = "your-secure-random-secret" +proxy_secret = "publisher_proxy_secret" ``` +`proxy_secret` is the key name in the Trusted Server secret store. Provision a +high-entropy value of at least 32 characters under `publisher_proxy_secret`. + ### Asset Routes Use `[[proxy.asset_routes]]` when a first-party path prefix should proxy directly to another asset origin. diff --git a/docs/guide/getting-started.md b/docs/guide/getting-started.md index 9314f983b..7d73235b0 100644 --- a/docs/guide/getting-started.md +++ b/docs/guide/getting-started.md @@ -65,18 +65,29 @@ The server will be available at `http://localhost:7676`. No Fastly account, CLI, or Viceroy needed. Runs natively on your machine. -The Axum adapter reads configuration from environment variables — it does **not** -auto-load `.env` files. You must export the variables into your shell before starting -the server. +The Axum adapter reads the EdgeZero config blob and secret store from +environment variables — it does **not** auto-load `.env` files. You must export +the variables into your shell before starting the server. ```bash -# Copy and edit the environment file +# Create the local app config and apply the non-secret development overlay. +cp trusted-server.example.toml trusted-server.toml cp .env.dev .env - -# Export the variables into your current shell session set -a && source .env && set +a -# Build and start the dev server +# Create the local blob-backed config-store entry. +ts config push --adapter axum --local --yes +export TRUSTED_SERVER_CONFIG_TRUSTED_SERVER_CONFIG_TRUSTED_SERVER_CONFIG="$( + jq -r '.trusted_server_config' .edgezero/local-config-trusted_server_config.json +)" + +# Populate the three secret references from the starter config for this shell. +# Use stable values only if you need existing proxy URLs or EC IDs to remain valid. +export TRUSTED_SERVER_SECRET_TRUSTED_SERVER_SECRETS_PUBLISHER_PROXY_SECRET="$(openssl rand -base64 32)" +export TRUSTED_SERVER_SECRET_TRUSTED_SERVER_SECRETS_EC_PASSPHRASE="$(openssl rand -base64 32)" +export TRUSTED_SERVER_SECRET_TRUSTED_SERVER_SECRETS_HANDLER_PASSWORD="$(openssl rand -base64 32)" + +# Build and start the dev server in the same shell. cargo run -p trusted-server-adapter-axum ``` @@ -85,12 +96,16 @@ The server will be available at `http://localhost:8787`. Set `PORT=` befor **Environment variable conventions used by the Axum adapter:** -| Purpose | Pattern | Example | -| ------------------ | ------------------------------------- | -------------------------------------------------------- | -| Config store value | `TRUSTED_SERVER_CONFIG_{STORE}_{KEY}` | `TRUSTED_SERVER_CONFIG_SETTINGS_AD_SERVER_URL=https://…` | -| Secret store value | `TRUSTED_SERVER_SECRET_{STORE}_{KEY}` | `TRUSTED_SERVER_SECRET_KEYS_SIGNING_KEY=abc123` | +| Purpose | Pattern | Example | +| ------------------ | ------------------------------------- | --------------------------------------------------------------------- | +| Config store value | `TRUSTED_SERVER_CONFIG_{STORE}_{KEY}` | `TRUSTED_SERVER_CONFIG_TRUSTED_SERVER_CONFIG_TRUSTED_SERVER_CONFIG=…` | +| Secret store value | `TRUSTED_SERVER_SECRET_{STORE}_{KEY}` | `TRUSTED_SERVER_SECRET_TRUSTED_SERVER_SECRETS_PROXY_KEY=…` | -Store names and key names are uppercased with hyphens and dots replaced by underscores. +The config-store value is the verified app-config blob. Secret-store values are +looked up by the key names in that blob. Store names and key names are uppercased +with hyphens and dots replaced by underscores. The quick-start exports ephemeral +secret-store values only into the current shell; do not put secret values in the +TOML config, config-store blob, or a source-controlled environment file. > **Dev server limitations:** The Axum adapter does not support KV store, > geo lookup, config/secret-store writes, or admin key-management routes. @@ -127,26 +142,45 @@ ts config init To bootstrap from a public publisher page, run an audit first: ```bash -ts audit https://publisher.example +ts audit generate https://publisher.example ``` The audit command writes `js-assets.toml` plus a draft `trusted-server.toml`. -Review the draft, replace placeholders/secrets, then validate it. +Review the draft, replace placeholders with stable secret key names, then +validate it. Edit `trusted-server.toml` to configure: -- Ad server integrations +- browser integrations under `[integrations.*]` +- server auction providers under map-shaped `[auction.providers.]` +- server bidder routes under `[auction.bidders.]` +- ad server integrations - KV store mappings - EC configuration -- Consent settings (`[gdpr]`) +- consent settings under `[gdpr]` +- stable key names for `trusted_server_secrets` -Validate the config before pushing it to platform storage: +Do not put a Prebid Server URL or server bidder list under +`[integrations.prebid]`, and do not put APS account, endpoint, or timeout fields +under `[integrations.aps]`. Those server values belong to auction provider +common fields and `profile_config`. + +Provision the physical store mapped from logical `trusted_server_secrets` with +the existing credential values before pushing a migrated config. On Fastly, +`ts_secrets` is the documented example physical name. Then validate and push: ```bash ts config validate +ts config push --adapter fastly ``` -See [Configuration](/guide/configuration) and [Trusted Server CLI](/guide/cli) for details. +The validation command performs target-independent plan validation. Each adapter +performs mandatory target-aware fan-out and backend-name validation at startup. +The EdgeZero callback needed for target-aware pre-write push validation is not +yet available in this tree, so startup remains the final target gate. + +Restart or redeploy instances after secret rotation. See +[Configuration](/guide/configuration) and [Trusted Server CLI](/guide/cli) for details. ## Deploy to Fastly diff --git a/docs/guide/integration-guide.md b/docs/guide/integration-guide.md index 4346fd7e0..cf7da71f6 100644 --- a/docs/guide/integration-guide.md +++ b/docs/guide/integration-guide.md @@ -308,22 +308,33 @@ Prebid applies the same steps outlined above with a few notable patterns: ```toml [integrations.prebid] enabled = true -server_url = "https://prebid.example/openrtb2/auction" timeout_ms = 1200 -bidders = ["equativ", "sampleBidder"] -external_bundle_url = "https://assets.example/prebid/trusted-prebid.js" +client_side_bidders = ["example-browser"] +external_bundle_url = "https://assets.example.com/prebid/trusted-prebid.js" # external_bundle_sha256 = "..." # external_bundle_sri = "sha384-..." # script_patterns = ["/static/prebid/*"] +[auction.providers.pbs-main] +protocol = "openrtb-2.6" +profile = "prebid-server" +endpoint = "https://prebid.example.com/openrtb2/auction" +routing = "explicit" + +[auction.bidders.example-server] +provider = "pbs-main" + [proxy] -allowed_domains = ["assets.example"] +allowed_domains = ["assets.example.com"] ``` The `proxy.allowed_domains` entry is required for `external_bundle_url` and must cover the bundle host plus any HTTPS redirect targets used by that host. -Tests or scaffolding can inject configs by calling `settings.integrations.insert_config("prebid", &serde_json::json!({...}))`, the same helper that other integrations use. +Browser integration tests can inject `[integrations.prebid]` settings with the +same registry helper as other integrations. Server provider and bidder behavior +must be constructed from the compiled auction plan rather than integration-owned +endpoint or bidder fields. **2. Routes Owned by the Integration** diff --git a/docs/guide/integrations-overview.md b/docs/guide/integrations-overview.md index 95f3d2011..d0d132f28 100644 --- a/docs/guide/integrations-overview.md +++ b/docs/guide/integrations-overview.md @@ -18,7 +18,9 @@ Trusted Server provides built-in integrations with third-party services for firs ### Prebid -**What it does:** Enables server-side header bidding through Prebid Server while maintaining first-party context. +**What it does:** Supplies the browser Prebid.js bundle and `trustedServer` +adapter while auction provider maps independently configure server-side Prebid +Server demand. **Key Features:** @@ -34,18 +36,29 @@ Trusted Server provides built-in integrations with third-party services for firs ```toml [integrations.prebid] enabled = true -server_url = "https://prebid-server.example.com" timeout_ms = 1000 -bidders = ["appnexus", "rubicon"] -auto_configure = true debug = false +client_side_bidders = ["example-browser"] +external_bundle_url = "https://assets.example.com/prebid/trusted-prebid.js" + +[proxy] +allowed_domains = ["assets.example.com"] + +[auction.providers.pbs-main] +protocol = "openrtb-2.6" +profile = "prebid-server" +endpoint = "https://prebid.example.com/openrtb2/auction" +routing = "explicit" + +[auction.bidders.example-server] +provider = "pbs-main" ``` **Endpoints:** -- `GET /first-party/ad` - Server-side ad rendering -- `POST /third-party/ad` - Client-side auction endpoint -- `GET /prebid.js` - Optional empty script override +- `POST /auction` - Browser and programmatic auction endpoint +- `GET /integrations/prebid/bundle.js` - First-party external bundle proxy +- `GET ` - Configured empty-script interception routes **When to use:** You want to monetize your site with programmatic advertising while maintaining first-party context. @@ -325,9 +338,12 @@ All integrations can be configured via environment variables: ```bash # Pattern: TRUSTED_SERVER__INTEGRATIONS__{INTEGRATION}__{SETTING} -# Prebid -TRUSTED_SERVER__INTEGRATIONS__PREBID__SERVER_URL="https://new-server.com" +# Existing Prebid browser-map leaves TRUSTED_SERVER__INTEGRATIONS__PREBID__TIMEOUT_MS=2000 +TRUSTED_SERVER__INTEGRATIONS__PREBID__DEBUG=true + +# Existing provider-map leaves use the validated provider ID segment +TRUSTED_SERVER__AUCTION__PROVIDERS__PBS_MAIN__ENDPOINT="https://prebid.example.com/openrtb2/auction" # Next.js TRUSTED_SERVER__INTEGRATIONS__NEXTJS__ENABLED=true diff --git a/docs/guide/integrations/aps.md b/docs/guide/integrations/aps.md index bde759c45..d25db5af6 100644 --- a/docs/guide/integrations/aps.md +++ b/docs/guide/integrations/aps.md @@ -10,7 +10,7 @@ Trusted Server can request banner bids from Amazon Publisher Services (APS) thro The integration supports: - banner impressions; -- APS OpenRTB requests to the integration's built-in production endpoint; +- APS OpenRTB requests to the provider's configured HTTPS endpoint; - decoded-CPM winner selection with or without a mediator; - direct `/auction` rendering; - client-side `trustedServer` Prebid adapter auctions through GAM; and @@ -25,33 +25,32 @@ The integration does not implement: ## Configuration +APS server ownership is entirely under an auction provider. The optional +`[integrations.aps]` table controls browser renderer ownership only. It does not +own the APS account, endpoint, timeout, debug behavior, or script policy. APS +renderer support is registered whenever the compiled auction plan contains an +`aps` profile, even if `[integrations.aps]` is absent or disabled. + ```toml [integrations.aps] enabled = true -account_id = "example-aps-account-id" -timeout_ms = 800 -# Include raw APS request/response data in /auction metadata on test sites only. -debug = false -# Set both when the deployment hostname differs from APS-authorized inventory. -# inventory_domain = "publisher.example" -# inventory_page_origin = "https://www.publisher.example" -allow_script_creatives = false -# Default. Set publisher_native only for the controlled friendly-frame experiment below. +# Default. Use publisher_native only for the controlled experiment below. rendering_mode = "trusted_server" +``` +```toml [auction] enabled = true -providers = ["aps", "prebid"] timeout_ms = 2000 ``` -`account_id` is the canonical field. `pub_id` remains a compatibility alias for migration, including integer values, but new configuration should not use it. Supplying both names is an error. +`profile_config.account_id` is the canonical field. The config-first schema does not accept the legacy `pub_id` alias. `debug` defaults to `false`. Enable it only on controlled test sites because it includes the raw APS request and response, including identity, consent, device, page, account, bid, and creative data, in the client-visible `/auction` response. `allow_script_creatives` defaults to `false`. While disabled, APS script bids are rejected before per-impression reduction, floors, mediation, and winner selection. Enable it only for a controlled cohort after the browser-security checks in [Rollout](#rollout) pass. -`rendering_mode` is a strict enum: `trusted_server` (the default) retains the opaque static renderer route, and `publisher_native` disables that route and adds `data-ts-aps-rendering-mode="publisher_native"` to the server-generated TSJS bundle tag. TSJS captures this server-owned attribute when the bundle executes, so markup added later cannot change the mode. The attribute works under a publisher CSP that blocks inline scripts. Unknown values fail configuration deserialization. +`integrations.aps.rendering_mode` is a strict enum. `trusted_server` is the default and retains the opaque static renderer route. `publisher_native` disables that route and adds `data-ts-aps-rendering-mode="publisher_native"` to the server-generated TSJS bundle tag. TSJS captures this server-owned attribute when the bundle executes, so markup added later cannot change the mode. The attribute works under a publisher CSP that blocks inline scripts. Unknown values fail configuration deserialization. ### Publisher-native runner experiment @@ -82,11 +81,76 @@ The APS provider may also participate through a configured mediator: ```toml [auction] enabled = true -providers = ["aps", "prebid"] mediator = "adserver_mock" -timeout_ms = 2000 + +[auction.providers.aps-main] +protocol = "openrtb-2.6" +profile = "aps" +endpoint = "https://aps.example.com/e/pb/bid" +routing = "all_eligible" + +[auction.providers.aps-main.profile_config] +account_id = "example-aps-account" +debug = false +allow_script_creatives = false +# Configure both only when authorized inventory differs from the deployment host. +# inventory_domain = "inventory.example.com" +# inventory_page_origin = "https://www.inventory.example.com" + +[integrations.adserver_mock] +enabled = true +endpoint = "https://mediator.example.com/mediate" +timeout_ms = 500 ``` +The common provider `endpoint` is required and must be an absolute HTTPS URL +with a host and no credentials or fragment. The legacy `/e/dtb/bid` path is +rejected. `timeout_ms` belongs beside `endpoint`; when omitted, the `aps` +profile default is 800 ms. Runtime caps it by the remaining auction budget. + +`profile_config.account_id` is required, nonempty, and at most 1024 bytes. It is +the canonical field; integration-owned `account_id`, `pub_id`, endpoint, and +timeout fields are not part of the public schema. `debug` and +`allow_script_creatives` both default to `false`. + +Enable `debug` only on controlled test sites because it includes the raw APS +request and response—including identity, consent, device, page, account, bid, +and creative data—in client-visible `/auction` metadata. + +Set `inventory_domain` and `inventory_page_origin` together only when the public +deployment hostname differs from APS-authorized inventory. The domain becomes +`site.domain`. The HTTPS page origin replaces the current page's scheme and host +while preserving its path; query and fragment are removed. The origin must be +the inventory domain or a subdomain and cannot contain credentials, a port, +path, query, or fragment. + +`routing = "all_eligible"` is the usual APS configuration: every +banner-compatible slot is eligible without a synthetic APS bidder entry. It +does not expose bidder parameters routed to another provider. Use +`routing = "explicit"` only when APS participation should require a central +bidder route: + +```toml +[auction.providers.aps-main] +protocol = "openrtb-2.6" +profile = "aps" +endpoint = "https://aps.example.com/e/pb/bid" +routing = "explicit" + +[auction.providers.aps-main.profile_config] +account_id = "example-aps-account" + +[auction.bidders.aps] +provider = "aps-main" +``` + +The optional mediator stays separate under `[auction].mediator`; never declare +it under `[auction.providers]` or `[auction.bidders]`. + +APS uses ordinary auction slot IDs and banner formats. Legacy creative- +opportunity APS `slot_id` values are ignored, and `bidders.aps.slotID` is not +required. + ## OpenRTB request Trusted Server builds the APS request independently from its Prebid Server request. The request includes: @@ -103,7 +167,9 @@ Raw outbound and inbound payloads are logged only at TRACE level. With debug dis ## Debug mode -Set `debug = true` under `[integrations.aps]` to include the direct APS HTTP exchange in the APS provider summary returned by `POST /auction`: +Set `debug = true` under +`[auction.providers..profile_config]` to include the direct APS HTTP +exchange in that provider's summary returned by `POST /auction`: ```json { @@ -126,7 +192,7 @@ Set `debug = true` under `[integrations.aps]` to include the direct APS HTTP exc } ``` -This follows the Prebid Server `metadata.debug.httpcalls` representation. APS makes one direct HTTP call per auction, so the map uses the provider key `aps` with one entry. Request and captured response bodies are strings, and header values are arrays so repeated headers are preserved. If a non-success response body cannot be read within the existing 2 MiB upstream limit, `responsebody` is omitted rather than reported as an empty body. APS does not add PBS-only `resolvedrequest` or `bidstatus` fields. +This follows the Prebid Server `metadata.debug.httpcalls` representation. APS makes one direct HTTP call per provider per auction, and the map preserves the legacy `aps` key with one entry. Request and captured response bodies are strings, and header values are arrays so repeated headers are preserved. If a non-success response body cannot be read within the existing 2 MiB upstream limit, `responsebody` is omitted rather than reported as an empty body. APS does not add PBS-only `resolvedrequest` or `bidstatus` fields. The debug exchange is emitted for successful responses, `204 No Content`, malformed response bodies, and non-success HTTP statuses. Transport failures and auction timeouts happen before an HTTP response reaches the parser and continue to use the orchestrator's normal error metadata. @@ -195,7 +261,9 @@ In `trusted_server` mode, the TSJS auction client validates the typed renderer d ### GAM and Universal Creative -For initial navigation and page-bids, Trusted Server publishes the same descriptor in `window.tsjs.bids`. The source-checked Prebid Universal Creative bridge accepts requests only from the iframe that owns the matching `hb_adid` and validates the complete envelope. In `trusted_server` mode it returns a static dynamic-renderer program that creates the same opaque renderer iframe. In `publisher_native` mode it instead resolves the publisher div and starts the friendly-frame runner without sending a Universal Creative renderer response. +For initial navigation and page-bids, Trusted Server publishes the same descriptor in `window.tsjs.bids`. The source-checked Prebid Universal Creative bridge accepts requests only from the iframe that owns the matching `hb_adid` and validates the complete envelope. In `trusted_server` mode it returns a static dynamic-renderer program that creates the same opaque renderer iframe. After the response is delivered, the bridge expands an authenticated ordinary display iframe only when its width and height attributes and computed geometry are still 1x1. It resizes that source iframe and every collapsed clipping ancestor through the authenticated slot root to the validated winning dimensions. Ambiguous sources, stale navigation or refresh completions, anchors, interstitials, fixed or sticky frames, invalid dimensions, and already-expanded frames remain unchanged. The same guard applies to APS capabilities, inline `adm`, and PBS Cache responses. + +In `publisher_native` mode the bridge instead resolves the publisher div and starts the friendly-frame runner without sending a Universal Creative renderer response. That renderer replaces the slot through a different owner and does not run the collapsed-shell helper. After the native runner loads, Trusted Server replaces the existing children of the resolved publisher div with the friendly frame. This removes the GAM or Universal Creative iframe when it is inside that div. If the runner fails, the existing iframe remains, but its Universal Creative request receives no response because Trusted Server has already claimed the selected bid. This one-owner behavior avoids a second render path, but GAM impression and viewability reporting must be validated with the APS account team for the controlled cohort. @@ -225,15 +293,22 @@ If script rendering requires weakening the outer sandbox, leave `allow_script_cr ## Migration from the legacy APS integration -This release is a direct protocol cutover: - -1. Replace the legacy `/e/dtb/bid` endpoint with `/e/pb/bid`. -2. Rename `pub_id` to `account_id`. -3. Remove APS-specific slot ID configuration and remove `aps` from Prebid Server bidder lists. Trusted Server also filters APS from PBS requests for this path. -4. Prepare GAM line items and Universal Creative for `hb_bidder=aps` and the selected APS `hb_adid`. +This release is a direct configuration and protocol cutover: + +1. Move `endpoint` and `timeout_ms` to `[auction.providers.]` and use + `/e/pb/bid`; `/e/dtb/bid` remains rejected. +2. Move `account_id`, `debug`, `allow_script_creatives`, and inventory overrides + to the provider's `profile_config`; `pub_id` is not part of the new schema. +3. Remove APS-specific slot ID configuration and any APS entry from old Prebid + Server bidder lists. Use `routing = "all_eligible"` or an explicit + `[auction.bidders.aps]` route. +4. Prepare GAM line items and Universal Creative for `hb_bidder=aps` and the + selected APS `hb_adid`. 5. Disable publisher-native APS demand for the Trusted Server test cohort. -There is no legacy runtime switch. Roll back by disabling `[integrations.aps]`, restoring native APS for the cohort, or deploying the prior binary. +There is no legacy runtime switch. Roll back by disabling `[auction]` or +removing the APS provider, restoring native APS for the cohort, or deploying +the prior binary. Changing `rendering_mode` does not update pages that are already loaded or stored in an HTML cache. A cached `trusted_server` page can continue requesting `/integrations/aps/renderer` after a native-mode deployment removes that route. A cached `publisher_native` page continues using its captured native mode after rollback. Coordinate the mode change with HTML cache expiry or purge and reload active test sessions before judging the result. @@ -257,10 +332,12 @@ Use fictional values in source-controlled configuration and fixtures. Supply con - Confirm `account_id` and account eligibility with APS. - Confirm the endpoint is `/e/pb/bid` and uses HTTPS without credentials. - If the deployment hostname differs from APS-authorized inventory, configure both `inventory_domain` and `inventory_page_origin` with the APS-approved identity. -- Ensure `aps` appears in `auction.providers`. +- Ensure an `[auction.providers.]` entry selects `profile = "aps"`. - Check aggregate APS drop reasons for currency, dimensions, render source, URL, tag type, or script-gate rejection. - Confirm the provider timeout fits inside the auction timeout. -- On a controlled test site, set `debug = true` and inspect `ext.orchestrator.provider_details[].metadata.debug.httpcalls.aps` in the `/auction` response. +- On a controlled test site, set profile `debug = true` and inspect + `ext.orchestrator.provider_details[].metadata.debug.httpcalls.aps` in the + `/auction` response. ### Winner targets but does not render diff --git a/docs/guide/integrations/datadome.md b/docs/guide/integrations/datadome.md index 0c2d8ae6c..ed671b4a2 100644 --- a/docs/guide/integrations/datadome.md +++ b/docs/guide/integrations/datadome.md @@ -43,7 +43,7 @@ rewrite_sdk = true # Server-side Protection API layer enable_protection = false -server_side_key_secret_store = "ts_secrets" +# Required only when enable_protection = true. server_side_key_secret_name = "datadome_server_side_key" protection_api_origin = "https://api-fastly.datadome.co" timeout_ms = 1500 @@ -68,30 +68,29 @@ patterns = ["(?i)\\.(avi|flv|mka|mkv|mov|mp4|mpeg|mpg|mp3|flac|ogg|ogm|opus|wav| ### Configuration options -| Option | Type | Default | Description | -| -------------------------------------- | ------- | -------------------------------- | ----------------------------------------------------------------------- | -| `enabled` | boolean | `false` | Enable the DataDome integration | -| `sdk_origin` | string | `https://js.datadome.co` | DataDome SDK origin URL for `tags.js` | -| `api_origin` | string | `https://api-js.datadome.co` | DataDome signal collection API origin URL for `/js/*` | -| `cache_ttl_seconds` | integer | `3600` | Cache TTL for `tags.js` | -| `rewrite_sdk` | boolean | `true` | Rewrite DataDome script URLs in HTML to first-party paths | -| `enable_protection` | boolean | `false` | Call the Protection API before route matching | -| `server_side_key_secret_store` | string | `ts_secrets` | Runtime secret store containing the DataDome server-side key | -| `server_side_key_secret_name` | string | `datadome_server_side_key` | Secret name containing the DataDome server-side key | -| `protection_api_origin` | string | `https://api-fastly.datadome.co` | Protection API origin | -| `timeout_ms` | integer | `1500` | Dynamic backend first-byte timeout for Protection API calls | -| `protection_excluded_methods` | array | `["OPTIONS"]` | HTTP methods skipped before the Protection API call | -| `protection_excluded_asns` | array | `[]` | Client autonomous system numbers skipped before the Protection API call | -| `protection_excluded_ip_cidrs` | array | `[]` | Inline client IP CIDR ranges skipped before the Protection API call | -| `protection_excluded_ip_cidr_sources` | array | `[]` | Config Store sources containing dynamic client IP CIDR bypass lists | -| `protection_ip_list_cache_ttl_seconds` | integer | `300` | Process-local cache TTL for Config Store-backed IP CIDR bypass lists | -| `protection_exclusion_rules` | array | Static asset path regex | Structured method/path/query/IP/ASN exclusion rules | -| `protection_test_bypass` | object | omitted | Staging-only fixed-header bypass; secret must contain at least 32 bytes | -| `enable_graphql_support` | boolean | `false` | Reserved for future GraphQL body inspection; ignored in v1 | -| `client_side_key` | string | `""` | DataDome client-side JavaScript key used for tag injection | -| `inject_client_side_tag` | boolean | `true` | Auto-inject the browser tag when `client_side_key` is non-empty | -| `client_side_tag_url` | string | `/integrations/datadome/tags.js` | Root-relative or HTTPS script URL used by auto-injection | -| `client_side_configuration` | object | `{ ajaxListenerPath = true }` | Options assigned to `window.ddoptions` | +| Option | Type | Default | Description | +| -------------------------------------- | ------- | -------------------------------- | ------------------------------------------------------------------------------ | +| `enabled` | boolean | `false` | Enable the DataDome integration | +| `sdk_origin` | string | `https://js.datadome.co` | DataDome SDK origin URL for `tags.js` | +| `api_origin` | string | `https://api-js.datadome.co` | DataDome signal collection API origin URL for `/js/*` | +| `cache_ttl_seconds` | integer | `3600` | Cache TTL for `tags.js` | +| `rewrite_sdk` | boolean | `true` | Rewrite DataDome script URLs in HTML to first-party paths | +| `enable_protection` | boolean | `false` | Call the Protection API before route matching | +| `server_side_key_secret_name` | string | none | Default-store secret reference required when protection is enabled | +| `protection_api_origin` | string | `https://api-fastly.datadome.co` | Protection API origin | +| `timeout_ms` | integer | `1500` | Dynamic backend first-byte timeout for Protection API calls | +| `protection_excluded_methods` | array | `["OPTIONS"]` | HTTP methods skipped before the Protection API call | +| `protection_excluded_asns` | array | `[]` | Client autonomous system numbers skipped before the Protection API call | +| `protection_excluded_ip_cidrs` | array | `[]` | Inline client IP CIDR ranges skipped before the Protection API call | +| `protection_excluded_ip_cidr_sources` | array | `[]` | Config Store sources containing dynamic client IP CIDR bypass lists | +| `protection_ip_list_cache_ttl_seconds` | integer | `300` | Process-local cache TTL for Config Store-backed IP CIDR bypass lists | +| `protection_exclusion_rules` | array | Static asset path regex | Structured method/path/query/IP/ASN exclusion rules | +| `protection_test_bypass` | object | omitted | Configuration-gated fixed-header bypass; secret must contain at least 32 bytes | +| `enable_graphql_support` | boolean | `false` | Reserved for future GraphQL body inspection; ignored in v1 | +| `client_side_key` | string | `""` | DataDome client-side JavaScript key used for tag injection | +| `inject_client_side_tag` | boolean | `true` | Auto-inject the browser tag when `client_side_key` is non-empty | +| `client_side_tag_url` | string | `/integrations/datadome/tags.js` | Root-relative or HTTPS script URL used by auto-injection | +| `client_side_configuration` | object | `{ ajaxListenerPath = true }` | Options assigned to `window.ddoptions` | ## Client-side setup @@ -156,7 +155,7 @@ When `enable_protection = true`, Trusted Server calls DataDome before normal rou - **Challenge**: return the DataDome response directly without contacting the publisher origin. - **Fail-open condition**: continue routing without DataDome effects when the Protection API times out, returns malformed instructions, or returns an unexpected status. -The configured `server_side_key_secret_store` and `server_side_key_secret_name` must resolve to a non-empty secret when server-side protection is enabled. If the secret cannot be read, DataDome protection fails open for that request. +`server_side_key_secret_name` is a key reference in the logical `trusted_server_secrets` store. It must resolve to a non-empty value when server-side protection is enabled. Missing or invalid credentials fail startup before requests are served. Protection API transport and response failures continue to fail open per request. ### Protected traffic @@ -169,46 +168,40 @@ A request is protected when all of the following are true: 5. The client IP does not match `protection_excluded_ip_cidrs` or any Config Store-backed CIDR source. 6. The client ASN is not listed in `protection_excluded_asns`. 7. No `protection_exclusion_rules` match. -8. The request does not contain a matching enabled `protection_test_bypass` credential while `FASTLY_IS_STAGING=1`. +8. The request does not contain a matching enabled `protection_test_bypass` credential. Static assets are excluded by default using a case-insensitive file-extension regex. Trusted Server internal routes such as `/static/tsjs=`, `/integrations/`, `/first-party/`, admin routes, discovery routes, and signature-verification routes are also excluded by default. Auction traffic at `/auction` is protected by default. -### Staging test bypass +### Test bypass For short-lived browser automation on an access-controlled staging site, you can configure a static header credential that skips only the server-side Protection API: ```toml -# Runtime activation also requires FASTLY_IS_STAGING=1. [integrations.datadome.protection_test_bypass] enabled = true -credential_secret_store = "ts_secrets" credential_secret_name = "datadome_test_bypass" ``` `protection_test_bypass` requires `enable_protection = true`; it is disabled -when omitted and is runtime-active only when `FASTLY_IS_STAGING=1`. -`FASTLY_IS_STAGING` is supplied at runtime by Fastly (`1` in staging and `0` in -production); it is not compiled into or promoted with the Wasm artifact. Verify -staging through the `X-TS-ENV: staging` response signal and the integration -activation log, and verify production omits that response signal. A retained -section cannot bypass protection in a production or other non-staging runtime. -Store a randomly generated credential containing at least 32 bytes of -high-entropy material in the configured Secret Store, configure this section -only while needed, protect the site with an outer access control such as Basic -Auth, and remove the section when testing finishes. +when omitted or when its `enabled` field is false. Store a randomly generated +credential containing at least 32 bytes of high-entropy material under the +referenced key in the logical default Secret Store (`trusted_server_secrets` by +default), configure this section only while needed, and protect the site with an +outer access control such as Basic Auth. Remove the section when testing +finishes. Whenever the enabled DataDome request filter runs on the Fastly adapter, the fixed `x-ts-datadome-bypass` header is removed before configuration or credential checks. It therefore cannot reach DataDome or the publisher origin -through that path when the bypass is absent, disabled, inactive, or invalid. -Active credentials are compared in constant time and never logged. Duplicate -header values fail closed. Scope the header to the staging origin; do not attach -it to every request in a browser context because that can disclose the -credential to third-party origins. With Playwright: +through that path when the bypass is absent, disabled, or invalid. Active +credentials are compared in constant time and never logged. Duplicate header +values fail closed. Scope the header to the test origin; do not attach it to +every request in a browser context because that can disclose the credential to +third-party origins. With Playwright: ```ts await context.route('https://staging.example.com/**', async (route) => { @@ -233,7 +226,7 @@ This behavior applies to: - `protection_excluded_ip_cidr_sources`; - structured `ip_cidr` rules; - structured `ip_cidr_source` rules; and -- a matching enabled `protection_test_bypass` credential in a staging runtime. +- a matching enabled `protection_test_bypass` credential. Method, ASN, path, query-parameter, static-asset, and internal-route exclusions alone do not suppress the client-side tag. However, a simultaneous matching IP @@ -375,7 +368,6 @@ TRUSTED_SERVER__INTEGRATIONS__DATADOME__API_ORIGIN=https://api-js.datadome.co TRUSTED_SERVER__INTEGRATIONS__DATADOME__CACHE_TTL_SECONDS=3600 TRUSTED_SERVER__INTEGRATIONS__DATADOME__REWRITE_SDK=true TRUSTED_SERVER__INTEGRATIONS__DATADOME__ENABLE_PROTECTION=true -TRUSTED_SERVER__INTEGRATIONS__DATADOME__SERVER_SIDE_KEY_SECRET_STORE=ts_secrets TRUSTED_SERVER__INTEGRATIONS__DATADOME__SERVER_SIDE_KEY_SECRET_NAME=datadome_server_side_key TRUSTED_SERVER__INTEGRATIONS__DATADOME__CLIENT_SIDE_KEY=your-client-side-key ``` @@ -421,7 +413,6 @@ Check that both fields are configured: [integrations.datadome] enabled = true enable_protection = true -server_side_key_secret_store = "ts_secrets" server_side_key_secret_name = "datadome_server_side_key" ``` diff --git a/docs/guide/integrations/prebid.md b/docs/guide/integrations/prebid.md index 32f2827fb..675ed02ae 100644 --- a/docs/guide/integrations/prebid.md +++ b/docs/guide/integrations/prebid.md @@ -14,82 +14,129 @@ Prebid is the leading open-source header bidding solution that allows publishers ## Configuration +Prebid configuration has two independent owners: + +- `[integrations.prebid]` owns browser Prebid.js behavior: bundle selection and + injection, browser timeout/debug, account injection, script interception, + client-side bidders, and refresh exclusions. +- `[auction.providers.]`, its `profile_config`, `notifications`, and + `[auction.bidders]` own every Prebid Server request. + ```toml [integrations.prebid] enabled = true -server_url = "https://prebid-server.example.com/openrtb2/auction" -timeout_ms = 1200 -bidders = ["kargo", "appnexus", "openx"] +timeout_ms = 1000 debug = false -# test_mode = false - -# Generated external Prebid bundle served through /integrations/prebid/bundle.js. -external_bundle_url = "https://assets.example/prebid/trusted-prebid.js" -# external_bundle_sha256 = "..." -# external_bundle_sri = "sha384-..." - -# Bidders that run client-side via native Prebid.js adapters instead of -# being routed through the server-side auction. -client_side_bidders = ["rubicon"] - -# Keep matching GAM inventory out of Trusted Server's Prebid refresh auctions. -# GAM still refreshes these slots. -excluded_gam_ad_unit_path_suffixes = ["/trackingonly"] - -# Script interception patterns (optional - defaults shown below) -script_patterns = ["/prebid.js", "/prebid.min.js", "/prebidjs.js", "/prebidjs.min.js"] - -# Required when external_bundle_url is configured. Include the bundle host and -# any HTTPS redirect targets used by that host. -[proxy] -allowed_domains = ["assets.example"] +client_side_bidders = ["example-browser"] +excluded_gam_ad_unit_path_suffixes = ["/example-tracking-only"] +script_patterns = ["/prebid.js", "/prebid.min.js"] +external_bundle_url = "https://assets.example.com/prebid/trusted-prebid.js" +# external_bundle_sha256 = "" +# external_bundle_sri = "sha384-" -# External bundle generation inputs used by `ts prebid bundle`. [integrations.prebid.bundle] -adapters = ["rubicon"] +adapters = ["example-browser"] user_id_modules = ["sharedIdSystem"] -# Optional static per-bidder param overrides (shallow merge) -[integrations.prebid.bid_param_overrides.criteo] -networkId = 99999 -pubid = "server-pub" +[proxy] +allowed_domains = ["assets.example.com"] -# Optional per-bidder, per-zone param overrides (shallow merge) -[integrations.prebid.bid_param_zone_overrides.kargo] -header = {placementId = "_s2sHeaderPlacement"} -in_content = {placementId = "_s2sContentPlacement"} +[auction] +enabled = true +timeout_ms = 2000 -# Optional canonical ordered override rules -[[integrations.prebid.bid_param_override_rules]] -when.bidder = "kargo" +[auction.providers.pbs-main] +protocol = "openrtb-2.6" +profile = "prebid-server" +endpoint = "https://prebid.example.com/openrtb2/auction" +timeout_ms = 900 +routing = "explicit" + +[auction.providers.pbs-main.profile_config] +debug = false +test_mode = false +debug_query_params = "example-debug=1" +consent_forwarding = "both" +bid_param_overrides = { example-server = { placement = "example-placement" } } +bid_param_zone_overrides = { example-server = { header = { placement = "example-header" } } } + +[[auction.providers.pbs-main.profile_config.bid_param_override_rules]] +when.bidder = "example-server" when.zone = "header" -set = { placementId = "_s2sHeaderPlacement" } +set = { placement = "example-rule-placement" } + +[auction.providers.pbs-main.notifications] +suppress_all = false +suppress_seats = ["example-seat"] + +[auction.bidders.example-server] +provider = "pbs-main" ``` -### Configuration Options - -| Field | Type | Default | Description | -| ------------------------------------ | ------------- | ---------------------------------------------------------------------- | ---------------------------------------------------------------------------------------------------------------------------------------------------------------- | -| `enabled` | Boolean | `true` | Enable Prebid integration | -| `server_url` | String | Required | Prebid Server endpoint URL | -| `timeout_ms` | Integer | `1000` | Request timeout in milliseconds | -| `bidders` | Array[String] | `["mocktioneer"]` | List of enabled bidders | -| `external_bundle_url` | String | Required when enabled | Absolute HTTPS URL of the generated external Prebid bundle, proxied through `/integrations/prebid/bundle.js`; its host must be listed in `proxy.allowed_domains` | -| `external_bundle_sha256` | String | `None` | Optional 64-character hex SHA-256 used for versioned first-party URLs, immutable cache headers, and `sha256:` ETags | -| `external_bundle_sri` | String | `None` | Optional Subresource Integrity metadata added to the same-origin bundle script tag when configured | -| `bid_param_overrides` | Table | `{}` | Static per-bidder param overrides; normalized into the canonical override-rule engine and shallow-merged into bidder params | -| `bid_param_zone_overrides` | Table | `{}` | Per-bidder, per-zone param overrides; normalized into the canonical override-rule engine and shallow-merged into bidder params | -| `bid_param_override_rules` | Array[Table] | `[]` | Canonical ordered override rules with `when` matchers and `set` objects; evaluated after compatibility fields so later rules win on conflicts | -| `suppress_nurl` | Boolean | `false` | Strip `nurl` and `burl` from every PBS bid when the PBS deployment fires win/billing notifications server-side | -| `suppress_nurl_bidders` | Array[String] | `[]` | Bidder seats whose `nurl` and `burl` should be stripped while preserving client-side win/billing pixels for other bidders | -| `debug` | Boolean | `false` | Enable Prebid debug mode (sets `ext.prebid.debug` and `ext.prebid.returnallbidstatus`; surfaces debug metadata in auction responses) | -| `test_mode` | Boolean | `false` | Set the OpenRTB `test: 1` flag so bidders treat the auction as non-billable test traffic. Separate from `debug` to avoid suppressing real demand | -| `debug_query_params` | String | `None` | Extra query params appended for debugging | -| `client_side_bidders` | Array[String] | `[]` | Bidders that run client-side via native Prebid.js adapters instead of server-side. See [Client-Side Bidders](#client-side-bidders) | -| `excluded_gam_ad_unit_path_suffixes` | Array[String] | `[]` | Exact, case-sensitive GAM ad-unit-path suffixes excluded from Trusted Server's Prebid refresh auction; matching slots still refresh through GAM | -| `script_patterns` | Array[String] | `["/prebid.js", "/prebid.min.js", "/prebidjs.js", "/prebidjs.min.js"]` | URL patterns for Prebid script interception | -| `bundle.adapters` | Array[String] | Required for `ts prebid bundle` | Prebid.js bidder adapter modules imported into the generated external browser bundle | -| `bundle.user_id_modules` | Array[String] | Generator default preset when omitted | Prebid User ID modules imported into the generated external browser bundle | +### Browser configuration options + +| Field | Default | Ownership and behavior | +| ------------------------------------ | ---------------------------------------------------------------------- | ----------------------------------------------------------------------------------- | +| `enabled` | `true` | Enables browser bundle injection/interception; it does not create a server provider | +| `account_id` | `None` | Optional browser-injected account value | +| `timeout_ms` | `1000` | Browser Prebid.js timeout only | +| `debug` | `false` | Browser Prebid.js debug only | +| `client_side_bidders` | `[]` | Native browser adapters that are not folded into `trustedServer` | +| `excluded_gam_ad_unit_path_suffixes` | `[]` | GAM suffixes omitted from Trusted Server refresh auctions | +| `script_patterns` | `["/prebid.js", "/prebid.min.js", "/prebidjs.js", "/prebidjs.min.js"]` | Publisher Prebid scripts intercepted to prevent duplicate instances | +| `external_bundle_url` | Required when enabled | HTTPS generated bundle URL; host and redirects must be in `proxy.allowed_domains` | +| `external_bundle_sha256` | `None` | Optional content hash used for versioning, cache policy, and ETag | +| `external_bundle_sri` | `None` | Optional SRI metadata | +| `bundle.adapters` | Required for `ts prebid bundle` | Browser bidder adapters compiled into the external bundle | +| `bundle.user_id_modules` | Generator preset | Browser User ID modules compiled into the external bundle | + +### Server provider options + +Common fields are `protocol`, `profile`, required HTTPS `endpoint`, optional +`timeout_ms`, and `routing`. The `prebid-server` timeout defaults to 1000 ms; +an explicit provider value overrides it, and the remaining auction budget caps +runtime `tmax`. + +When migrating an origin-only legacy `server_url`, use that origin as the +provider `endpoint`. The compiler adds `/openrtb2/auction` and preserves query +parameters. A configured non-root path, such as `/bid` or `/custom/pbs`, stays +exact. `/openrtb2/auction/` is normalized to `/openrtb2/auction`. + +The typed `profile_config` fields are: + +| Field | Default | Behavior | +| -------------------------- | ------- | --------------------------------------------------- | +| `debug` | `false` | PBS request/response diagnostics | +| `test_mode` | `false` | Top-level OpenRTB `test: 1`; independent of debug | +| `debug_query_params` | `None` | Optional page-URL debug query fragment | +| `bid_param_overrides` | `{}` | Static per-bidder shallow merges | +| `bid_param_zone_overrides` | `{}` | Per-bidder/per-zone shallow merges | +| `bid_param_override_rules` | `[]` | Ordered exact-match rules; later matching rules win | +| `consent_forwarding` | `both` | `openrtb_only`, `cookies_only`, or `both` | + +`notifications.suppress_all` replaces the old global notification switch. +`notifications.suppress_seats` removes `nurl` and `burl` only for exact returned +`seatbid.seat` values. It does not match bidder route IDs. See +[Configuration](/guide/configuration#auction-configuration) for bounds. + +### Browser/server bidder ownership + +Every server-side bidder code comes from `[auction.bidders.]`; the browser +integration has no server bidder list. The validated route keys are injected as +`serverSideBidders`. On initial and refresh auctions, only matching publisher +bids are folded into the `trustedServer.bidderParams` envelope. Configured +`client_side_bidders` and other unowned demand remain native browser bids. Both +paths compete in the same Prebid.js auction. + +The reserved `trustedServer` envelope cannot select a provider or endpoint. Its +nested bidder keys resolve through `[auction.bidders]`, and one envelope accepts +at most 128 bidder entries. The optional `zone` fact is limited to 256 UTF-8 +bytes. Missing, `null`, or empty `bidderParams` invokes Prebid stored-request +routing; malformed envelopes do not. + +Browser `timeout_ms`/`debug` never inherit a server provider timeout or profile +debug value. Enabling the browser integration does not create a server provider, +and a `prebid-server` provider can exist independently from browser injection. ## External Bundle Generation @@ -255,15 +302,15 @@ Use `bid_param_overrides` for static per-bidder param overrides when the same ov **Example**: ```toml -[integrations.prebid.bid_param_overrides.criteo] +[auction.providers.pbs-main.profile_config.bid_param_overrides.example-server] networkId = 99999 -pubid = "server-pub" +pubid = "example-server-pub" ``` **Environment variable**: ```text -TRUSTED_SERVER__INTEGRATIONS__PREBID__BID_PARAM_OVERRIDES='{"criteo":{"networkId":99999,"pubid":"server-pub"}}' +TRUSTED_SERVER__AUCTION__PROVIDERS__PBS-MAIN__PROFILE_CONFIG__BID_PARAM_OVERRIDES='{"example-server":{"networkId":99999,"pubid":"example-server-pub"}}' ``` ### Bid Param Zone Overrides @@ -283,10 +330,10 @@ The JS adapter reads the zone from `mediaTypes.banner.name` on each Prebid ad un **Example**: ```toml -[integrations.prebid.bid_param_zone_overrides.kargo] -header = {placementId = "_s2sHeaderPlacement"} -in_content = {placementId = "_s2sContentPlacement"} -fixed_bottom = {placementId = "_s2sBottomPlacement"} +[auction.providers.pbs-main.profile_config.bid_param_zone_overrides.example-server] +header = { placementId = "example-header-placement" } +in_content = { placementId = "example-content-placement" } +fixed_bottom = { placementId = "example-bottom-placement" } ``` If the incoming request for zone `header` has: @@ -306,7 +353,7 @@ For an unrecognized zone (e.g., `sidebar`), the incoming params are left unchang **Environment variable**: ```text -TRUSTED_SERVER__INTEGRATIONS__PREBID__BID_PARAM_ZONE_OVERRIDES='{"kargo":{"header":{"placementId":"_s2sHeaderPlacement"}}}' +TRUSTED_SERVER__AUCTION__PROVIDERS__PBS-MAIN__PROFILE_CONFIG__BID_PARAM_ZONE_OVERRIDES='{"example-server":{"header":{"placementId":"example-header-placement"}}}' ``` ### Bid Param Override Rules @@ -326,16 +373,16 @@ Use `bid_param_override_rules` for the canonical ordered override format. Each r **Example**: ```toml -[[integrations.prebid.bid_param_override_rules]] -when.bidder = "kargo" +[[auction.providers.pbs-main.profile_config.bid_param_override_rules]] +when.bidder = "example-server" when.zone = "header" -set = { placementId = "_s2sHeaderPlacement", keep = "server" } +set = { placementId = "example-header-placement", keep = "example" } ``` **Environment variable**: ```text -TRUSTED_SERVER__INTEGRATIONS__PREBID__BID_PARAM_OVERRIDE_RULES='[{"when":{"bidder":"kargo","zone":"header"},"set":{"placementId":"_s2sHeaderPlacement","keep":"server"}}]' +TRUSTED_SERVER__AUCTION__PROVIDERS__PBS-MAIN__PROFILE_CONFIG__BID_PARAM_OVERRIDE_RULES='[{"when":{"bidder":"example-server","zone":"header"},"set":{"placementId":"example-header-placement","keep":"example"}}]' ``` ## Refresh Auction GAM-Path Opt-Out @@ -378,25 +425,33 @@ external Prebid adapters or User ID modules. ## Client-Side Bidders -Some Prebid.js bid adapters do not work well through Prebid Server (e.g. Magnite/Rubicon). The `client_side_bidders` config field lets you keep these bidders running natively in the browser while routing all other bidders through the server-side auction. +The `client_side_bidders` config field keeps selected demand on native +Prebid.js adapters while validated `[auction.bidders]` routes identify demand +owned by Trusted Server. ### How it works 1. The server injects the `clientSideBidders` list into the page via `window.__tsjs_prebid`. 2. When `pbjs.requestBids()` is called, the TSJS shim checks each bid against the list. 3. **Client-side bidders** are left as standalone bids — their native Prebid.js adapters handle them in the browser. -4. **All other bidders** are absorbed into the `trustedServer` adapter and routed through the `/auction` orchestrator to Prebid Server. +4. **Bidders present in `[auction.bidders]`** are absorbed into the + `trustedServer` adapter and routed through `/auction` to their configured + provider. Unowned bidders remain native browser demand. 5. Both sets of bids compete in the same Prebid.js auction. ### Configuration ```toml [integrations.prebid] -bidders = ["kargo", "appnexus", "openx"] # server-side via PBS -client_side_bidders = ["rubicon"] # native browser adapters +client_side_bidders = ["example-browser"] + +[auction.bidders.example-server] +provider = "pbs-main" ``` -The two lists are independent — the operator manages both explicitly. If a bidder appears in both lists, a warning is logged at startup (the bidder will run in both paths, which is likely unintended). +Do not route the same bidder through `[auction.bidders]` while also listing it in +`client_side_bidders`; choose one owner. Include every client-side adapter in +the generated external bundle. ### External bundle adapter selection @@ -405,7 +460,7 @@ Client-side bidders need their Prebid.js adapter modules included in the generat ```bash cd crates/trusted-server-js/lib npm run build:prebid-external -- \ - --adapters=rubicon,appnexus,openx \ + --adapters=example-browser \ --user-id-modules=sharedIdSystem,uid2IdSystem \ --out=dist/prebid ``` @@ -514,25 +569,18 @@ In practice, this gives operators both: ## Endpoints -### GET /first-party/ad - -Server-side ad rendering for single ad slot. +### POST /auction -**Query Parameters**: - -- `slot` - Ad unit code -- `w` - Width in pixels -- `h` - Height in pixels - -**Response**: Complete HTML creative with first-party proxying. - -### POST /third-party/ad - -Client-side auction endpoint for TSJS library. +Browser and programmatic auction endpoint used by the Trusted Server Prebid adapter. **Request Body**: Ad units configuration **Response**: OpenRTB bid response with creatives +### GET /integrations/prebid/bundle.js + +First-party proxy route for the configured `external_bundle_url`. An optional +`?v=` query enables content-addressed caching. + ### GET `` (Dynamic) Routes are registered dynamically based on the `script_patterns` configuration. Each pattern creates an endpoint that returns an empty JavaScript file to prevent client-side Prebid.js loading. diff --git a/docs/guide/proxy-signing.md b/docs/guide/proxy-signing.md index 2f34678c3..701a7621b 100644 --- a/docs/guide/proxy-signing.md +++ b/docs/guide/proxy-signing.md @@ -19,9 +19,13 @@ Signatures use HMAC-SHA256 with the publisher's `proxy_secret`: ```toml [publisher] -proxy_secret = "your-secret-key-here" # Must be secure random string +proxy_secret = "publisher_proxy_secret" ``` +The config value is a key in the Trusted Server secret store. Provision a +secure random signing value under `publisher_proxy_secret`; at least 32 random +bytes are recommended. + ## Signature Validation On incoming requests: @@ -37,7 +41,7 @@ On incoming requests: ## Security Notes -- Keep `proxy_secret` confidential and secure -- Rotate secrets periodically -- Never expose the secret in client-side code -- Use strong random values (32+ bytes) +- Keep the resolved signing value confidential +- Rotate the stored value periodically +- Never expose the resolved value in client-side code +- Use a strong random value of at least 32 characters diff --git a/docs/package-lock.json b/docs/package-lock.json index e31c9db5e..e6003067a 100644 --- a/docs/package-lock.json +++ b/docs/package-lock.json @@ -14,7 +14,7 @@ "eslint": "^10", "mermaid": "^11.12.3", "prettier": "^3.4.2", - "typescript-eslint": "8.57", + "typescript-eslint": "8.66", "vitepress": "^1.5.0", "vitepress-plugin-mermaid": "^2.0.17" } @@ -180,7 +180,6 @@ "integrity": "sha512-Jc360x4yqb3eEg4OY4KEIdGePBxZogivKI+OGIU8aLXgAYPTECvzeOBc90312yHA1hr3AeRlAFl0rIc8lQaIrQ==", "dev": true, "license": "MIT", - "peer": true, "dependencies": { "@algolia/client-common": "5.50.0", "@algolia/requester-browser-xhr": "5.50.0", @@ -1886,7 +1885,6 @@ "integrity": "sha512-GYDxsZi3ChgmckRT9HPU0WEhKLP08ev/Yfcq2AstjrDASOYCSXeyjDsHg4v5t4jOj7cyDX3vmprafKlWIG9MXQ==", "dev": true, "license": "MIT", - "peer": true, "dependencies": { "undici-types": "~7.16.0" } @@ -1914,20 +1912,20 @@ "license": "MIT" }, "node_modules/@typescript-eslint/eslint-plugin": { - "version": "8.57.2", - "resolved": "https://registry.npmjs.org/@typescript-eslint/eslint-plugin/-/eslint-plugin-8.57.2.tgz", - "integrity": "sha512-NZZgp0Fm2IkD+La5PR81sd+g+8oS6JwJje+aRWsDocxHkjyRw0J5L5ZTlN3LI1LlOcGL7ph3eaIUmTXMIjLk0w==", + "version": "8.66.0", + "resolved": "https://registry.npmjs.org/@typescript-eslint/eslint-plugin/-/eslint-plugin-8.66.0.tgz", + "integrity": "sha512-p088eaGrzYz1s+7cov0aMOCkNGTJlVxF4jgubf28c8L0Cv9Rloj8YBHnv4hXLq6IIEE1AsjNWavO+k+8kP2Y0A==", "dev": true, "license": "MIT", "dependencies": { "@eslint-community/regexpp": "^4.12.2", - "@typescript-eslint/scope-manager": "8.57.2", - "@typescript-eslint/type-utils": "8.57.2", - "@typescript-eslint/utils": "8.57.2", - "@typescript-eslint/visitor-keys": "8.57.2", + "@typescript-eslint/scope-manager": "8.66.0", + "@typescript-eslint/type-utils": "8.66.0", + "@typescript-eslint/utils": "8.66.0", + "@typescript-eslint/visitor-keys": "8.66.0", "ignore": "^7.0.5", "natural-compare": "^1.4.0", - "ts-api-utils": "^2.4.0" + "ts-api-utils": "^2.5.0" }, "engines": { "node": "^18.18.0 || ^20.9.0 || >=21.1.0" @@ -1937,15 +1935,15 @@ "url": "https://opencollective.com/typescript-eslint" }, "peerDependencies": { - "@typescript-eslint/parser": "^8.57.2", + "@typescript-eslint/parser": "^8.66.0", "eslint": "^8.57.0 || ^9.0.0 || ^10.0.0", - "typescript": ">=4.8.4 <6.0.0" + "typescript": ">=4.8.4 <6.1.0" } }, "node_modules/@typescript-eslint/eslint-plugin/node_modules/ignore": { - "version": "7.0.5", - "resolved": "https://registry.npmjs.org/ignore/-/ignore-7.0.5.tgz", - "integrity": "sha512-Hs59xBNfUIunMFgWAbGX5cq6893IbWg4KnrjbYwX3tx0ztorVgTDA6B2sxf8ejHJ4wz8BqGUMYlnzNBer5NvGg==", + "version": "7.0.6", + "resolved": "https://registry.npmjs.org/ignore/-/ignore-7.0.6.tgz", + "integrity": "sha512-BAg6QkE8W+TuQLrrw0Ugr7HegXduRuuj8/ti2kSOc+jz1dmx8/WNcjr6XGnq5YpDWxFwwaavqD0+jIUOKelTsw==", "dev": true, "license": "MIT", "engines": { @@ -1953,17 +1951,16 @@ } }, "node_modules/@typescript-eslint/parser": { - "version": "8.57.2", - "resolved": "https://registry.npmjs.org/@typescript-eslint/parser/-/parser-8.57.2.tgz", - "integrity": "sha512-30ScMRHIAD33JJQkgfGW1t8CURZtjc2JpTrq5n2HFhOefbAhb7ucc7xJwdWcrEtqUIYJ73Nybpsggii6GtAHjA==", + "version": "8.66.0", + "resolved": "https://registry.npmjs.org/@typescript-eslint/parser/-/parser-8.66.0.tgz", + "integrity": "sha512-X6ypGChaWYk6PBtUg2BwuTZEFFcHJAtGTVJ9/lCTOufhZ4i9fNolQNnktq+kkMCwMj7V8Svsq7+TxSDslmhE0g==", "dev": true, "license": "MIT", - "peer": true, "dependencies": { - "@typescript-eslint/scope-manager": "8.57.2", - "@typescript-eslint/types": "8.57.2", - "@typescript-eslint/typescript-estree": "8.57.2", - "@typescript-eslint/visitor-keys": "8.57.2", + "@typescript-eslint/scope-manager": "8.66.0", + "@typescript-eslint/types": "8.66.0", + "@typescript-eslint/typescript-estree": "8.66.0", + "@typescript-eslint/visitor-keys": "8.66.0", "debug": "^4.4.3" }, "engines": { @@ -1975,18 +1972,18 @@ }, "peerDependencies": { "eslint": "^8.57.0 || ^9.0.0 || ^10.0.0", - "typescript": ">=4.8.4 <6.0.0" + "typescript": ">=4.8.4 <6.1.0" } }, "node_modules/@typescript-eslint/project-service": { - "version": "8.57.2", - "resolved": "https://registry.npmjs.org/@typescript-eslint/project-service/-/project-service-8.57.2.tgz", - "integrity": "sha512-FuH0wipFywXRTHf+bTTjNyuNQQsQC3qh/dYzaM4I4W0jrCqjCVuUh99+xd9KamUfmCGPvbO8NDngo/vsnNVqgw==", + "version": "8.66.0", + "resolved": "https://registry.npmjs.org/@typescript-eslint/project-service/-/project-service-8.66.0.tgz", + "integrity": "sha512-7MthGPTt4BP69lSryqpqq8HQqxuzynssckL/jyDyk3+TNMQ3y2jFWkptCrktWvBrP+EH787Nl5N5Qpw7WZg+5g==", "dev": true, "license": "MIT", "dependencies": { - "@typescript-eslint/tsconfig-utils": "^8.57.2", - "@typescript-eslint/types": "^8.57.2", + "@typescript-eslint/tsconfig-utils": "^8.66.0", + "@typescript-eslint/types": "^8.66.0", "debug": "^4.4.3" }, "engines": { @@ -1997,18 +1994,18 @@ "url": "https://opencollective.com/typescript-eslint" }, "peerDependencies": { - "typescript": ">=4.8.4 <6.0.0" + "typescript": ">=4.8.4 <6.1.0" } }, "node_modules/@typescript-eslint/scope-manager": { - "version": "8.57.2", - "resolved": "https://registry.npmjs.org/@typescript-eslint/scope-manager/-/scope-manager-8.57.2.tgz", - "integrity": "sha512-snZKH+W4WbWkrBqj4gUNRIGb/jipDW3qMqVJ4C9rzdFc+wLwruxk+2a5D+uoFcKPAqyqEnSb4l2ULuZf95eSkw==", + "version": "8.66.0", + "resolved": "https://registry.npmjs.org/@typescript-eslint/scope-manager/-/scope-manager-8.66.0.tgz", + "integrity": "sha512-8TGcH25j9zqJ/IULB/ppyhRvxA8QYfFEZ7nfbg6/BN9spDgb8fPWQXlE5l8TWBL50EtUx007uZ1o9VOwrq2/9g==", "dev": true, "license": "MIT", "dependencies": { - "@typescript-eslint/types": "8.57.2", - "@typescript-eslint/visitor-keys": "8.57.2" + "@typescript-eslint/types": "8.66.0", + "@typescript-eslint/visitor-keys": "8.66.0" }, "engines": { "node": "^18.18.0 || ^20.9.0 || >=21.1.0" @@ -2019,9 +2016,9 @@ } }, "node_modules/@typescript-eslint/tsconfig-utils": { - "version": "8.57.2", - "resolved": "https://registry.npmjs.org/@typescript-eslint/tsconfig-utils/-/tsconfig-utils-8.57.2.tgz", - "integrity": "sha512-3Lm5DSM+DCowsUOJC+YqHHnKEfFh5CoGkj5Z31NQSNF4l5wdOwqGn99wmwN/LImhfY3KJnmordBq/4+VDe2eKw==", + "version": "8.66.0", + "resolved": "https://registry.npmjs.org/@typescript-eslint/tsconfig-utils/-/tsconfig-utils-8.66.0.tgz", + "integrity": "sha512-9D5gLYZG4rOjcoag8MQ/fWI8WqA9wcPDyOGyWtWFhvM1lHRbliqUSPIY5J3zqCU1tvSwzXxnnjhQhz5Ne7mJ4g==", "dev": true, "license": "MIT", "engines": { @@ -2032,21 +2029,21 @@ "url": "https://opencollective.com/typescript-eslint" }, "peerDependencies": { - "typescript": ">=4.8.4 <6.0.0" + "typescript": ">=4.8.4 <6.1.0" } }, "node_modules/@typescript-eslint/type-utils": { - "version": "8.57.2", - "resolved": "https://registry.npmjs.org/@typescript-eslint/type-utils/-/type-utils-8.57.2.tgz", - "integrity": "sha512-Co6ZCShm6kIbAM/s+oYVpKFfW7LBc6FXoPXjTRQ449PPNBY8U0KZXuevz5IFuuUj2H9ss40atTaf9dlGLzbWZg==", + "version": "8.66.0", + "resolved": "https://registry.npmjs.org/@typescript-eslint/type-utils/-/type-utils-8.66.0.tgz", + "integrity": "sha512-LG2dWfjZQQp0ADtAu/EWJVayefGL2UEZ3CDeI44D9v3rXB/WYUqE/jpO28KrEKul5AySrmI+Zh1v6v+xW2U9+g==", "dev": true, "license": "MIT", "dependencies": { - "@typescript-eslint/types": "8.57.2", - "@typescript-eslint/typescript-estree": "8.57.2", - "@typescript-eslint/utils": "8.57.2", + "@typescript-eslint/types": "8.66.0", + "@typescript-eslint/typescript-estree": "8.66.0", + "@typescript-eslint/utils": "8.66.0", "debug": "^4.4.3", - "ts-api-utils": "^2.4.0" + "ts-api-utils": "^2.5.0" }, "engines": { "node": "^18.18.0 || ^20.9.0 || >=21.1.0" @@ -2057,13 +2054,13 @@ }, "peerDependencies": { "eslint": "^8.57.0 || ^9.0.0 || ^10.0.0", - "typescript": ">=4.8.4 <6.0.0" + "typescript": ">=4.8.4 <6.1.0" } }, "node_modules/@typescript-eslint/types": { - "version": "8.57.2", - "resolved": "https://registry.npmjs.org/@typescript-eslint/types/-/types-8.57.2.tgz", - "integrity": "sha512-/iZM6FnM4tnx9csuTxspMW4BOSegshwX5oBDznJ7S4WggL7Vczz5d2W11ecc4vRrQMQHXRSxzrCsyG5EsPPTbA==", + "version": "8.66.0", + "resolved": "https://registry.npmjs.org/@typescript-eslint/types/-/types-8.66.0.tgz", + "integrity": "sha512-H6gcYaSDOyvL3AD/jHUtUFo2jqGgn/F6nuyuZSu0QTesxL+cP4dQoIMrODRofuJC09g64+WgZ6tE19Y1N2YIFQ==", "dev": true, "license": "MIT", "engines": { @@ -2075,21 +2072,21 @@ } }, "node_modules/@typescript-eslint/typescript-estree": { - "version": "8.57.2", - "resolved": "https://registry.npmjs.org/@typescript-eslint/typescript-estree/-/typescript-estree-8.57.2.tgz", - "integrity": "sha512-2MKM+I6g8tJxfSmFKOnHv2t8Sk3T6rF20A1Puk0svLK+uVapDZB/4pfAeB7nE83uAZrU6OxW+HmOd5wHVdXwXA==", + "version": "8.66.0", + "resolved": "https://registry.npmjs.org/@typescript-eslint/typescript-estree/-/typescript-estree-8.66.0.tgz", + "integrity": "sha512-8/x4INiiQb10jGgXYD7116/zQ+OL84ZIFn0za68wwFHCanT/VLbBEroWht8RV8fn0/ZCAoazHLQgwUC0UQcDfg==", "dev": true, "license": "MIT", "dependencies": { - "@typescript-eslint/project-service": "8.57.2", - "@typescript-eslint/tsconfig-utils": "8.57.2", - "@typescript-eslint/types": "8.57.2", - "@typescript-eslint/visitor-keys": "8.57.2", + "@typescript-eslint/project-service": "8.66.0", + "@typescript-eslint/tsconfig-utils": "8.66.0", + "@typescript-eslint/types": "8.66.0", + "@typescript-eslint/visitor-keys": "8.66.0", "debug": "^4.4.3", "minimatch": "^10.2.2", "semver": "^7.7.3", "tinyglobby": "^0.2.15", - "ts-api-utils": "^2.4.0" + "ts-api-utils": "^2.5.0" }, "engines": { "node": "^18.18.0 || ^20.9.0 || >=21.1.0" @@ -2099,20 +2096,20 @@ "url": "https://opencollective.com/typescript-eslint" }, "peerDependencies": { - "typescript": ">=4.8.4 <6.0.0" + "typescript": ">=4.8.4 <6.1.0" } }, "node_modules/@typescript-eslint/utils": { - "version": "8.57.2", - "resolved": "https://registry.npmjs.org/@typescript-eslint/utils/-/utils-8.57.2.tgz", - "integrity": "sha512-krRIbvPK1ju1WBKIefiX+bngPs+odIQUtR7kymzPfo1POVw3jlF+nLkmexdSSd4UCbDcQn+wMBATOOmpBbqgKg==", + "version": "8.66.0", + "resolved": "https://registry.npmjs.org/@typescript-eslint/utils/-/utils-8.66.0.tgz", + "integrity": "sha512-jasearZPolBw5NJNYGMwxzHMF83niVWmMU1VdHzG1CyfI2VS7f7nZltnKtHcg20hW+7Uo5GfK4MeDPoU3qI8EA==", "dev": true, "license": "MIT", "dependencies": { "@eslint-community/eslint-utils": "^4.9.1", - "@typescript-eslint/scope-manager": "8.57.2", - "@typescript-eslint/types": "8.57.2", - "@typescript-eslint/typescript-estree": "8.57.2" + "@typescript-eslint/scope-manager": "8.66.0", + "@typescript-eslint/types": "8.66.0", + "@typescript-eslint/typescript-estree": "8.66.0" }, "engines": { "node": "^18.18.0 || ^20.9.0 || >=21.1.0" @@ -2123,17 +2120,17 @@ }, "peerDependencies": { "eslint": "^8.57.0 || ^9.0.0 || ^10.0.0", - "typescript": ">=4.8.4 <6.0.0" + "typescript": ">=4.8.4 <6.1.0" } }, "node_modules/@typescript-eslint/visitor-keys": { - "version": "8.57.2", - "resolved": "https://registry.npmjs.org/@typescript-eslint/visitor-keys/-/visitor-keys-8.57.2.tgz", - "integrity": "sha512-zhahknjobV2FiD6Ee9iLbS7OV9zi10rG26odsQdfBO/hjSzUQbkIYgda+iNKK1zNiW2ey+Lf8MU5btN17V3dUw==", + "version": "8.66.0", + "resolved": "https://registry.npmjs.org/@typescript-eslint/visitor-keys/-/visitor-keys-8.66.0.tgz", + "integrity": "sha512-dkKR8q+lKciskj1Y3vthHktl+3cMLWGyVUP23bRiPZ5O9BRT++4EqDDV+TVeIKBL1VXVEqrJlz8MYbcnvJcAlg==", "dev": true, "license": "MIT", "dependencies": { - "@typescript-eslint/types": "8.57.2", + "@typescript-eslint/types": "8.66.0", "eslint-visitor-keys": "^5.0.0" }, "engines": { @@ -2433,7 +2430,6 @@ "integrity": "sha512-UVJyE9MttOsBQIDKw1skb9nAwQuR5wuGD3+82K6JgJlm/Y+KI92oNsMNGZCYdDsVtRHSak0pcV5Dno5+4jh9sw==", "dev": true, "license": "MIT", - "peer": true, "bin": { "acorn": "bin/acorn" }, @@ -2474,7 +2470,6 @@ "integrity": "sha512-yE5I83Q2s8euVou8Y3feXK08wyZInJWLYXgWO6Xti9jBUEZAGUahyeQ7wSZWkifLWVnQVKEz5RAmBlXG5nqxog==", "dev": true, "license": "MIT", - "peer": true, "dependencies": { "@algolia/abtesting": "1.16.0", "@algolia/client-abtesting": "5.50.0", @@ -2567,7 +2562,6 @@ "integrity": "sha512-opLQzEVriiH1uUQ4Kctsd49bRoFDXGGSC4GUqj7pGyxM3RehRhvTlZJc1FL/Flew2p5uwxa1tUDWKzI4wNM8pg==", "dev": true, "license": "Apache-2.0", - "peer": true, "dependencies": { "@chevrotain/cst-dts-gen": "11.1.2", "@chevrotain/gast": "11.1.2", @@ -2672,7 +2666,6 @@ "integrity": "sha512-iJc4TwyANnOGR1OmWhsS9ayRS3s+XQ185FmuHObThD+5AeJCakAAbWv8KimMTt08xCCLNgneQwFp+JRJOr9qGQ==", "dev": true, "license": "MIT", - "peer": true, "engines": { "node": ">=0.10" } @@ -3107,7 +3100,6 @@ "integrity": "sha512-fmTRWbNMmsmWq6xJV8D19U/gw/bwrHfNXxrIN+HfZgnzqTHp9jOmKMhsTUjXOJnZOdZY9Q28y4yebKzqDKlxlQ==", "dev": true, "license": "ISC", - "peer": true, "engines": { "node": ">=12" } @@ -3363,7 +3355,6 @@ "integrity": "sha512-S9jlY/ELKEUwwQnqWDO+f+m6sercqOPSqXM5Go94l7DOmxHVDgmSFGWEzeE/gwgTAr0W103BWt0QLe/7mabIvA==", "dev": true, "license": "MIT", - "peer": true, "dependencies": { "@eslint-community/eslint-utils": "^4.8.0", "@eslint-community/regexpp": "^4.12.2", @@ -3613,7 +3604,6 @@ "integrity": "sha512-/yNdlIkpWbM0ptxno3ONTuf+2g318kh2ez3KSeZN5dZ8YC6AAmgeWz+GasYYiBJPFaYcSAPeu4GfhUaChzIJXA==", "dev": true, "license": "MIT", - "peer": true, "dependencies": { "tabbable": "^6.4.0" } @@ -3979,7 +3969,6 @@ "integrity": "sha512-fEnci+Immw6lKMFI8sqzjlATTyjLkRa6axrEgLV2yHTfv8r+h1wjFbV6xeRtd4rUV1cS4EpR9rwp3Rci7TRWDw==", "dev": true, "license": "MIT", - "peer": true, "dependencies": { "@braintree/sanitize-url": "^7.1.1", "@iconify/utils": "^3.0.2", @@ -4300,12 +4289,11 @@ "license": "ISC" }, "node_modules/picomatch": { - "version": "4.0.4", - "resolved": "https://registry.npmjs.org/picomatch/-/picomatch-4.0.4.tgz", - "integrity": "sha512-QP88BAKvMam/3NxH6vj2o21R6MjxZUAd6nlwAS/pnGvN9IVLocLHxGYIzFhg6fUQ+5th6P4dv4eW9jX3DSIj7A==", + "version": "4.0.5", + "resolved": "https://registry.npmjs.org/picomatch/-/picomatch-4.0.5.tgz", + "integrity": "sha512-RvwwcruNjI1ncT5xRakeyS9Lf8lcItv34KD+aif+VH9kduAyfYBipGh12274xtenIPZ119/R9BdTBa8gAwSh0A==", "dev": true, "license": "MIT", - "peer": true, "engines": { "node": ">=12" }, @@ -4552,9 +4540,9 @@ "peer": true }, "node_modules/semver": { - "version": "7.7.4", - "resolved": "https://registry.npmjs.org/semver/-/semver-7.7.4.tgz", - "integrity": "sha512-vFKC2IEtQnVhpT78h1Yp8wzwrf8CM+MzKMHGJZfBtzhZNycRFnXsHk6E5TxIkkMsgNS7mdX3AGB7x2QM2di4lA==", + "version": "7.8.5", + "resolved": "https://registry.npmjs.org/semver/-/semver-7.8.5.tgz", + "integrity": "sha512-Y7/KDsb8LjooZpwaqGyulO6DQlksgCncchHGk+sZIY4SBvUocMBEFH5Ur1fI4dV+Jvl0w6cjvucaIi40puRioA==", "dev": true, "license": "ISC", "bin": { @@ -4688,14 +4676,14 @@ } }, "node_modules/tinyglobby": { - "version": "0.2.15", - "resolved": "https://registry.npmjs.org/tinyglobby/-/tinyglobby-0.2.15.tgz", - "integrity": "sha512-j2Zq4NyQYG5XMST4cbs02Ak8iJUdxRM0XI5QyxXuZOzKOINmWurp3smXu3y5wDcJrptwpSjgXHzIQxR0omXljQ==", + "version": "0.2.17", + "resolved": "https://registry.npmjs.org/tinyglobby/-/tinyglobby-0.2.17.tgz", + "integrity": "sha512-wXR/dYpcqKmfWpEdZjiKJOwCNFndD0DMnrW/cYjVGttEkBfVgcLFHoNrlj47mjOVic9yyNu65alsgF4NQyTa2g==", "dev": true, "license": "MIT", "dependencies": { "fdir": "^6.5.0", - "picomatch": "^4.0.3" + "picomatch": "^4.0.4" }, "engines": { "node": ">=12.0.0" @@ -4767,16 +4755,16 @@ } }, "node_modules/typescript-eslint": { - "version": "8.57.2", - "resolved": "https://registry.npmjs.org/typescript-eslint/-/typescript-eslint-8.57.2.tgz", - "integrity": "sha512-VEPQ0iPgWO/sBaZOU1xo4nuNdODVOajPnTIbog2GKYr31nIlZ0fWPoCQgGfF3ETyBl1vn63F/p50Um9Z4J8O8A==", + "version": "8.66.0", + "resolved": "https://registry.npmjs.org/typescript-eslint/-/typescript-eslint-8.66.0.tgz", + "integrity": "sha512-QlEbBPz/RuJ1XUHj29nm3t0F/O/cSlEnntozqPOYHnnTGAXFamnMBu5i9Vn6vhUPHGAjR+Vl+5J8vPN/BMUrJw==", "dev": true, "license": "MIT", "dependencies": { - "@typescript-eslint/eslint-plugin": "8.57.2", - "@typescript-eslint/parser": "8.57.2", - "@typescript-eslint/typescript-estree": "8.57.2", - "@typescript-eslint/utils": "8.57.2" + "@typescript-eslint/eslint-plugin": "8.66.0", + "@typescript-eslint/parser": "8.66.0", + "@typescript-eslint/typescript-estree": "8.66.0", + "@typescript-eslint/utils": "8.66.0" }, "engines": { "node": "^18.18.0 || ^20.9.0 || >=21.1.0" @@ -4787,7 +4775,7 @@ }, "peerDependencies": { "eslint": "^8.57.0 || ^9.0.0 || ^10.0.0", - "typescript": ">=4.8.4 <6.0.0" + "typescript": ">=4.8.4 <6.1.0" } }, "node_modules/ufo": { @@ -4937,7 +4925,6 @@ "integrity": "sha512-o5a9xKjbtuhY6Bi5S3+HvbRERmouabWbyUcpXXUA1u+GNUKoROi9byOJ8M0nHbHYHkYICiMlqxkg1KkYmm25Sw==", "dev": true, "license": "MIT", - "peer": true, "dependencies": { "esbuild": "^0.21.3", "postcss": "^8.4.43", @@ -4998,7 +4985,6 @@ "integrity": "sha512-+2ym1/+0VVrbhNyRoFFesVvBvHAVMZMK0rw60E3X/5349M1GuVdKeazuksqopEdvkKwKGs21Q729jX81/bkBJg==", "dev": true, "license": "MIT", - "peer": true, "dependencies": { "@docsearch/css": "3.8.2", "@docsearch/js": "3.8.2", @@ -5110,7 +5096,6 @@ "integrity": "sha512-iV/sU9SzOlmA/0tygSmjkEN6Jbs3nPoIPFhCMLD2STrjgOU8DX7ZtzMhg4ahVwf5Rp9KoFzcXeB1ZrVbLBp5/Q==", "dev": true, "license": "MIT", - "peer": true, "dependencies": { "@vue/compiler-dom": "3.5.31", "@vue/compiler-sfc": "3.5.31", diff --git a/docs/package.json b/docs/package.json index f74991a4c..df1bd66d0 100644 --- a/docs/package.json +++ b/docs/package.json @@ -20,7 +20,7 @@ "eslint": "^10", "mermaid": "^11.12.3", "prettier": "^3.4.2", - "typescript-eslint": "8.57", + "typescript-eslint": "8.66", "vitepress": "^1.5.0", "vitepress-plugin-mermaid": "^2.0.17" } diff --git a/docs/superpowers/plans/2026-06-26-server-side-ad-template-cli.md b/docs/superpowers/plans/2026-06-26-server-side-ad-template-cli.md new file mode 100644 index 000000000..df781e518 --- /dev/null +++ b/docs/superpowers/plans/2026-06-26-server-side-ad-template-cli.md @@ -0,0 +1,2142 @@ +# Server-Side Ad Template CLI Implementation Plan + +> **For agentic workers:** REQUIRED SUB-SKILL: Use superpowers:subagent-driven-development (recommended) or superpowers:executing-plans to implement this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking. + +**Goal:** Build the unified `ts` CLI support for server-side ad-template static diagnostics and browser-backed verification described in `docs/superpowers/specs/2026-06-26-server-side-ad-template-cli-design.md`. + +**Architecture:** Keep the CLI host-only and thin: Clap parsing stays in `run.rs` / command adapter modules, shared app-config loading moves to `app_config.rs`, pure ad-template logic lives under `ad_templates/`, and Chrome/Chromium collection lives under `audit/`. Runtime gate rules are extracted into a small pure helper in `trusted-server-core` so the CLI does not duplicate server behavior. + +**Tech Stack:** Rust 2024 workspace, host-target `trusted-server-cli`, `clap`, EdgeZero typed app-config loader, `serde`/`serde_json` for stable JSON, `chromiumoxide` for browser-backed audit collection, local HTML fixture tests, and existing `trusted-server-core::creative_opportunities` matching. + +--- + +## Current State + +- Branch: `feature/ts-cli-ad-templates`. +- Static ad-template commands already exist in `crates/trusted-server-cli/src/config_ad_templates.rs`. +- The current branch does not contain #800 audit files. Port useful #800 pieces into the current #799 code shape; do not resurrect stale `args.rs` or `config_command.rs`. +- The spec was updated after review and is the source of truth: + `docs/superpowers/specs/2026-06-26-server-side-ad-template-cli-design.md`. +- Keep `.env` and operator-owned `trusted-server.toml` out of commits. + +## File Map + +### New files + +| File | Responsibility | +| -------------------------------------------------------------- | -------------------------------------------------------------------------------------------------------- | +| `crates/trusted-server-cli/src/app_config.rs` | Shared effective app-config loader and shared `AppConfigArgs`. | +| `crates/trusted-server-cli/src/ad_templates/mod.rs` | Re-export focused ad-template CLI modules. | +| `crates/trusted-server-cli/src/ad_templates/expected.rs` | Path/URL normalization and expected-slot projection from runtime slot matching. | +| `crates/trusted-server-cli/src/ad_templates/compare.rs` | Pure DOM/GPT/APS evidence comparison, statuses, warnings, runtime gate output, strict failure decisions. | +| `crates/trusted-server-cli/src/ad_templates/output.rs` | Human and JSON rendering for static diagnostics and browser verification. | +| `crates/trusted-server-cli/src/audit/mod.rs` | Audit namespace entry point. | +| `crates/trusted-server-cli/src/audit/page.rs` | Generic page audit command ported from #800. | +| `crates/trusted-server-cli/src/audit/collector.rs` | Browser collector trait plus collected page/evidence structs. | +| `crates/trusted-server-cli/src/audit/browser.rs` | Chromiumoxide-backed browser collector, init scripts, optional scroll, page-level collection errors. | +| `crates/trusted-server-cli/src/audit/ad_templates.rs` | `ts audit ad-templates verify` orchestration. | +| `crates/trusted-server-cli/src/audit/ad_template_collector.js` | Read-only init script for GPT/APS/DOM evidence collection, included via `include_str!`. | + +### Modified files + +| File | Change summary | +| ---------------------------------------------------------- | --------------------------------------------------------------------------------------------------------------- | +| `Cargo.toml` | Add workspace dependencies missing from this branch: `chromiumoxide`, `serde`, and `serde_json` if not present. | +| `crates/trusted-server-cli/Cargo.toml` | Add host-only CLI dependencies for browser audit and JSON output. | +| `crates/trusted-server-cli/src/lib.rs` | Register new `app_config`, `ad_templates`, and `audit` modules under `cfg(not(target_arch = "wasm32"))`. | +| `crates/trusted-server-cli/src/run.rs` | Add `Audit` command namespace, parser tests, and dispatch. | +| `crates/trusted-server-cli/src/config_ad_templates.rs` | Shrink to Clap adapter using shared loader/expected/output modules. | +| `crates/trusted-server-core/src/creative_opportunities.rs` | Add pure runtime gate helper types/functions shared by runtime and CLI. | +| `crates/trusted-server-core/src/publisher.rs` | Route existing server-side ad-stack gate through the shared helper without changing behavior. | + +## Implementation Rules + +- Use TDD for each task: write a failing test first, run it, implement the minimal code, re-run, then commit. +- Commit after each task using repo style: sentence case, imperative, no semantic prefix. +- Keep `trusted-server-cli` host-only. Do not introduce `tokio`, `chromiumoxide`, or filesystem/browser dependencies into core runtime or wasm adapter crates. +- Do not write real publisher domains or secrets in tests. Use `example.com`, `publisher.example`, and fictional IDs only. +- Prefer pure module tests over browser tests. Browser-backed fixture tests should use local HTML only and no GPT/APS network. + +## Task 0: Baseline And Branch Hygiene + +**Files:** + +- Verify: `docs/superpowers/specs/2026-06-26-server-side-ad-template-cli-design.md` +- Verify: `docs/superpowers/plans/2026-06-26-server-side-ad-template-cli.md` + +- [ ] **Step 1: Confirm branch and working tree** + + Run: + + ```bash + git status --short --branch + git log --oneline --decorate -5 + ``` + + Expected: on `feature/ts-cli-ad-templates`; no unrelated modified files besides the approved spec/plan docs. + +- [ ] **Step 2: Run docs format check before code work** + + Run: + + ```bash + cd docs && npm run format + ``` + + Expected: `All matched files use Prettier code style!` + +- [ ] **Step 3: Commit reviewed spec and plan** + + Run: + + ```bash + git add docs/superpowers/specs/2026-06-26-server-side-ad-template-cli-design.md docs/superpowers/plans/2026-06-26-server-side-ad-template-cli.md + git commit -m "Add server-side ad-template CLI implementation plan" + ``` + + Expected: docs-only commit. If the spec commit already exists separately, commit only the plan. + +## Task 1: Share Runtime Ad-Stack Gate Logic + +**Files:** + +- Modify: `crates/trusted-server-core/src/creative_opportunities.rs` +- Modify: `crates/trusted-server-core/src/publisher.rs` + +- [ ] **Step 1: Write failing core tests for the shared gate helper** + + Add tests near the existing `creative_opportunities` tests: + + ```rust + #[test] + fn ad_stack_gate_passes_for_eligible_navigation() { + let result = evaluate_ad_stack_gate(AdStackGateInput { + method_get: true, + navigation: true, + prefetch: false, + bot: false, + matched_slots: true, + consent_allows_auction: Some(true), + auction_enabled: true, + }); + + assert_eq!(result.expected, RuntimeAdStackExpected::Yes); + assert!(result.blocking_gates().is_empty()); + } + + #[test] + fn ad_stack_gate_blocks_known_kill_switch() { + let result = evaluate_ad_stack_gate(AdStackGateInput { + method_get: true, + navigation: true, + prefetch: false, + bot: false, + matched_slots: true, + consent_allows_auction: Some(true), + auction_enabled: false, + }); + + assert_eq!(result.expected, RuntimeAdStackExpected::No); + assert!(result.blocking_gates().contains(&AdStackGateName::AuctionEnabled)); + } + + #[test] + fn ad_stack_gate_is_unknown_when_consent_is_unknown() { + let result = evaluate_ad_stack_gate(AdStackGateInput { + method_get: true, + navigation: true, + prefetch: false, + bot: false, + matched_slots: true, + consent_allows_auction: None, + auction_enabled: true, + }); + + assert_eq!(result.expected, RuntimeAdStackExpected::Unknown); + } + + // Locks the spec §5.2 mirror invariant: with Some(consent) supplied for every + // input combination, `expected == Yes` must equal the legacy all-AND boolean. + #[test] + fn ad_stack_gate_with_known_consent_matches_legacy_boolean() { + for bits in 0u8..64 { + let input = AdStackGateInput { + method_get: bits & 1 != 0, + navigation: bits & 2 != 0, + prefetch: bits & 4 != 0, + bot: bits & 8 != 0, + matched_slots: bits & 16 != 0, + consent_allows_auction: Some(bits & 32 != 0), + auction_enabled: bits & 1 == 0, + }; + // Legacy semantics: all positive gates true, both negative gates false. + let legacy = input.method_get + && input.navigation + && !input.prefetch + && !input.bot + && input.matched_slots + && input.consent_allows_auction == Some(true) + && input.auction_enabled; + let got = evaluate_ad_stack_gate(input).expected == RuntimeAdStackExpected::Yes; + assert_eq!(got, legacy, "gate mismatch for bits={bits}"); + } + } + ``` + +- [ ] **Step 2: Run the focused test and verify it fails** + + Run: + + ```bash + # NOTE: trusted-server-core links the `fastly` crate and CANNOT build for the host + # triple — run core tests on the DEFAULT target (wasm32-wasip1 + viceroy runner), + # i.e. no `--target`. Only the host-only `trusted-server-cli` uses `--target `. + cargo test -p trusted-server-core creative_opportunities::tests::ad_stack_gate + ``` + + Expected: compile failure because `AdStackGateInput` / `evaluate_ad_stack_gate` do not exist. + +- [ ] **Step 3: Implement pure gate types and helper** + + Add public, serde-free types to `creative_opportunities.rs`: + + ```rust + #[derive(Debug, Clone, Copy, Eq, PartialEq)] + pub enum RuntimeAdStackExpected { + Yes, + No, + Unknown, + } + + #[derive(Debug, Clone, Copy, Eq, PartialEq)] + pub enum AdStackGateName { + MethodGet, + Navigation, + NotPrefetch, + NotBot, + MatchedSlots, + ConsentAllowsAuction, + AuctionEnabled, + } + + #[derive(Debug, Clone, Copy)] + pub struct AdStackGateInput { + pub method_get: bool, + pub navigation: bool, + pub prefetch: bool, + pub bot: bool, + pub matched_slots: bool, + pub consent_allows_auction: Option, + pub auction_enabled: bool, + } + + #[derive(Debug, Clone, Eq, PartialEq)] + pub struct AdStackGateResult { + pub expected: RuntimeAdStackExpected, + blocking_gates: Vec, + } + + impl AdStackGateResult { + pub fn blocking_gates(&self) -> &[AdStackGateName] { + &self.blocking_gates + } + } + ``` + + Implement `evaluate_ad_stack_gate(input)` so any known blocking boolean gate returns `No`, all known pass plus `Some(true)` consent returns `Yes`, and all known pass plus `None` consent returns `Unknown`. + + Mind the gate polarity, mirroring `should_run_server_side_ad_stack`: `method_get`, + `navigation`, `matched_slots`, and `auction_enabled` block when **false**, while + `prefetch` and `bot` block when **true** (their gate names `NotPrefetch` / `NotBot` + pass when the input bool is false). `consent_allows_auction` is the only tri-state + input: `Some(false)` blocks (No), `Some(true)` passes, `None` yields Unknown only + when no other gate already blocks. + +- [ ] **Step 4: Route `publisher.rs` through the helper** + + Replace the body of `should_run_server_side_ad_stack` with a call to `evaluate_ad_stack_gate`, preserving the existing function signature for low-risk runtime compatibility: + + ```rust + crate::creative_opportunities::evaluate_ad_stack_gate( + crate::creative_opportunities::AdStackGateInput { + method_get: is_get, + navigation: is_navigation, + prefetch: is_prefetch, + bot: is_bot, + matched_slots: has_matched_slots, + consent_allows_auction: Some(consent_allows_auction), + auction_enabled, + }, + ) + .expected + == crate::creative_opportunities::RuntimeAdStackExpected::Yes + ``` + +- [ ] **Step 5: Run focused tests** + + Run: + + ```bash + # Core tests run on the default wasm target via viceroy (no --target). + cargo test -p trusted-server-core publisher::tests + cargo test -p trusted-server-core creative_opportunities + ``` + + Expected: all focused tests pass (including the existing `should_run_server_side_ad_stack` truth-table tests in `publisher::tests`). + +- [ ] **Step 6: Commit** + + ```bash + git add crates/trusted-server-core/src/creative_opportunities.rs crates/trusted-server-core/src/publisher.rs + git commit -m "Share server-side ad stack gate evaluation" + ``` + +## Task 2: Extract Shared CLI App Config Loader + +**Files:** + +- Create: `crates/trusted-server-cli/src/app_config.rs` +- Modify: `crates/trusted-server-cli/src/lib.rs` +- Modify: `crates/trusted-server-cli/src/config_ad_templates.rs` + +- [ ] **Step 1: Write failing loader tests** + + Move the existing temp-project helpers from `config_ad_templates.rs` tests into `app_config.rs` tests and add: + + ```rust + #[test] + fn explicit_missing_app_config_does_not_fall_back() { + let temp = TempDir::new().expect("should create temp dir"); + let manifest_path = temp.path().join("edgezero.toml"); + fs::write(&manifest_path, "[app]\nname = \"trusted-server\"\n") + .expect("should write manifest"); + let missing_path = temp.path().join("missing.toml"); + + let args = AppConfigArgs { + app_config: Some(missing_path.clone()), + manifest: manifest_path, + no_env: true, + }; + + let err = load_settings(&args).expect_err("should reject missing explicit config"); + assert!( + err.contains(missing_path.to_string_lossy().as_ref()), + "error should mention the explicit missing path" + ); + } + ``` + +- [ ] **Step 2: Run focused test and verify it fails** + + Run: + + ```bash + cargo test -p trusted-server-cli app_config --target $(rustc -vV | sed -n 's/^host: //p') + ``` + + Expected: compile failure because `app_config` module is not registered. + +- [ ] **Step 3: Implement `app_config.rs`** + + Move these items out of `config_ad_templates.rs`: + - `AppConfigArgs` + - `LoadedSettings` + - `load_settings` + - `resolve_app_config_path` + + Make the API explicit: + + ```rust + #[derive(Clone, Debug, Args)] + pub struct AppConfigArgs { + #[arg(long)] + pub app_config: Option, + #[arg(long, default_value = "edgezero.toml")] + pub manifest: PathBuf, + #[arg(long)] + pub no_env: bool, + } + + pub struct LoadedSettings { + pub app_config_path: PathBuf, + pub settings: Settings, + } + + pub fn load_settings(args: &AppConfigArgs) -> Result { + let manifest_loader = ManifestLoader::from_path(&args.manifest) + .map_err(|err| format!("failed to load {}: {err}", args.manifest.display()))?; + let app_name = manifest_loader.manifest().app.name.clone().ok_or_else(|| { + format!( + "{} has no [app].name; cannot resolve trusted-server.toml", + args.manifest.display() + ) + })?; + let app_config_path = + resolve_app_config_path(args.app_config.as_deref(), &args.manifest, &app_name); + + let mut opts = AppConfigLoadOptions::default(); + opts.env_overlay = !args.no_env; + let app_config = app_config::deserialize_app_config_with_options::( + &app_config_path, + &app_name, + &opts, + ) + .map_err(|err| format!("failed to load {}: {err}", app_config_path.display()))?; + + Ok(LoadedSettings { + app_config_path, + settings: app_config.into_settings(), + }) + } + + fn resolve_app_config_path( + explicit: Option<&Path>, + manifest_path: &Path, + app_name: &str, + ) -> PathBuf { + if let Some(path) = explicit { + return path.to_path_buf(); + } + let file_name = format!("{app_name}.toml"); + if let Some(parent) = manifest_path + .parent() + .filter(|parent| !parent.as_os_str().is_empty()) + { + parent.join(file_name) + } else { + PathBuf::from(file_name) + } + } + ``` + + Include the same top-level imports currently used by these helpers: + `std::path::{Path, PathBuf}`, `clap::Args`, + `edgezero_core::app_config::{self, AppConfigLoadOptions}`, + `edgezero_core::manifest::ManifestLoader`, + `trusted_server_core::config::TrustedServerAppConfig`, and + `trusted_server_core::settings::Settings`. + +- [ ] **Step 4: Register module and update imports** + + In `lib.rs`, add: + + ```rust + #[cfg(not(target_arch = "wasm32"))] + mod app_config; + ``` + + In `config_ad_templates.rs`, import: + + ```rust + use crate::app_config::{load_settings, AppConfigArgs}; + ``` + +- [ ] **Step 5: Run focused CLI tests** + + Run: + + ```bash + cargo test -p trusted-server-cli config_ad_templates --target $(rustc -vV | sed -n 's/^host: //p') + cargo test -p trusted-server-cli app_config --target $(rustc -vV | sed -n 's/^host: //p') + ``` + + Expected: existing static command behavior remains unchanged. + +- [ ] **Step 6: Commit** + + ```bash + git add crates/trusted-server-cli/src/app_config.rs crates/trusted-server-cli/src/config_ad_templates.rs crates/trusted-server-cli/src/lib.rs + git commit -m "Extract shared CLI app config loader" + ``` + +## Task 3: Add Expected-Slot Model + +**Files:** + +- Create: `crates/trusted-server-cli/src/ad_templates/mod.rs` +- Create: `crates/trusted-server-cli/src/ad_templates/expected.rs` +- Modify: `crates/trusted-server-cli/Cargo.toml` +- Modify: `crates/trusted-server-cli/src/lib.rs` +- Modify: `crates/trusted-server-cli/src/config_ad_templates.rs` + +- [ ] **Step 1: Write failing expected-slot tests** + + Add a test-only dependency to `crates/trusted-server-cli/Cargo.toml` so tests can + deserialize core slot config instead of constructing `CreativeOpportunitySlot` with + its `pub(crate)` `compiled_patterns` cache: + + ```toml + [target.'cfg(not(target_arch = "wasm32"))'.dev-dependencies] + toml = { workspace = true } + ``` + + In `expected.rs`, add tests for path normalization, full URL normalization, config-order preservation, resolved div ID, resolved GAM unit path, provider names, and matching page patterns: + + ```rust + fn creative_config_with_slots(patterns: &[&str]) -> CreativeOpportunitiesConfig { + let page_patterns = patterns + .iter() + .map(|pattern| format!("\"{pattern}\"")) + .collect::>() + .join(", "); + let toml = format!( + r#" + gam_network_id = "123" + auction_timeout_ms = 500 + price_granularity = "dense" + + [[slot]] + id = "atf" + gam_unit_path = "/123/news/atf" + div_id = "ad-atf-" + page_patterns = [{page_patterns}] + formats = [{{ width = 300, height = 250 }}] + floor_price = 0.50 + targeting = {{ zone = "atf" }} + + [slot.providers.prebid] + bidders = {{}} + "# + ); + let mut config = toml::from_str::(&toml) + .expect("should deserialize creative opportunities config"); + config.compile_slots(); + config + } + + #[test] + fn expected_slots_use_runtime_matcher_and_config_order() { + let config = creative_config_with_slots(["/news/*", "/"].as_slice()); + let expected = expected_slots_for_path("/news/story", &config) + .expect("should build expected slots"); + + assert_eq!(expected.path, "/news/story"); + assert_eq!(expected.slots.iter().map(|slot| slot.id.as_str()).collect::>(), ["atf"]); + assert_eq!(expected.slots[0].div_id, "ad-atf-"); + assert_eq!(expected.slots[0].gam_unit_path, "/123/news/atf"); + assert_eq!(expected.slots[0].providers, ["prebid"]); + } + + #[test] + fn normalize_path_or_url_strips_query_and_fragment() { + assert_eq!(normalize_path_or_url("https://www.example.com/news/story?x=1#top").expect("should normalize"), "/news/story"); + assert_eq!(normalize_path_or_url("news/story?x=1").expect("should normalize"), "/news/story"); + } + ``` + +- [ ] **Step 2: Run focused test and verify it fails** + + Run: + + ```bash + cargo test -p trusted-server-cli ad_templates::expected --target $(rustc -vV | sed -n 's/^host: //p') + ``` + + Expected: compile failure because module/types do not exist. + +- [ ] **Step 3: Implement expected-slot structs** + + Define pure structs that own strings and are stable for output: + + ```rust + #[derive(Debug, Clone, PartialEq)] + pub struct ExpectedSlots { + pub path: String, + pub slots: Vec, + } + + #[derive(Debug, Clone, PartialEq)] + pub struct ExpectedSlot { + pub id: String, + pub div_id: String, + pub gam_unit_path: String, + pub formats: Vec, + pub providers: Vec, + pub page_patterns: Vec, + } + + #[derive(Debug, Clone, PartialEq)] + pub struct ExpectedFormat { + pub width: u32, + pub height: u32, + // Mirrors `MediaType` rendered as a stable string (`"banner"`, `"video"`, `"native"`). + pub media_type: String, + } + ``` + + `div_id` and `gam_unit_path` are resolved (non-optional) strings. The core + `CreativeOpportunitySlot` stores `div_id` / `gam_unit_path` as `Option` + and the GAM unit path is composed with the configured GAM network ID; mirror the + existing `format_slot` resolution in `config_ad_templates.rs` so the CLI does not + invent a second resolution rule. Use + `trusted_server_core::creative_opportunities::match_slots`. Do not compile globs in CLI. + +- [ ] **Step 4: Register `ad_templates` and update static commands** + + In `lib.rs`, add: + + ```rust + #[cfg(not(target_arch = "wasm32"))] + mod ad_templates; + ``` + + Rewire `config_ad_templates.rs` onto the shared module, and remove the now-duplicated + local code so there is no name collision or dead `normalize_path_or_url`: + - delete the private `fn normalize_path_or_url` (currently `config_ad_templates.rs:448`) + and add `use crate::ad_templates::expected::{expected_slots_for_path, normalize_path_or_url};`; + - the existing `config_ad_templates::tests::normalizes_path_or_url_like_runtime_request_path` + test (currently `:661`) calls the local fn via `super::*` — either delete it (Task 3 + Step 1 already adds normalization tests in `expected.rs`) or repoint it at + `crate::ad_templates::expected::normalize_path_or_url`. Pick one so the test crate + still compiles at this commit. + +- [ ] **Step 5: Run focused tests** + + Run: + + ```bash + cargo test -p trusted-server-cli ad_templates::expected --target $(rustc -vV | sed -n 's/^host: //p') + cargo test -p trusted-server-cli config_ad_templates --target $(rustc -vV | sed -n 's/^host: //p') + ``` + + Expected: all pass. + +- [ ] **Step 6: Commit** + + ```bash + git add crates/trusted-server-cli/Cargo.toml crates/trusted-server-cli/src/ad_templates/mod.rs crates/trusted-server-cli/src/ad_templates/expected.rs crates/trusted-server-cli/src/config_ad_templates.rs crates/trusted-server-cli/src/lib.rs + git commit -m "Add shared ad-template expected slot model" + ``` + +## Task 4: Add Stable Output And JSON Types + +**Files:** + +- Create: `crates/trusted-server-cli/src/ad_templates/output.rs` +- Modify: `crates/trusted-server-cli/Cargo.toml` +- Modify: `crates/trusted-server-cli/src/ad_templates/mod.rs` + +- [ ] **Step 1: Add CLI JSON dependencies** + + The CLI crate has **no plain `[dependencies]` table** — every runtime dep lives + under `[target.'cfg(not(target_arch = "wasm32"))'.dependencies]` (the workspace + default build target is `wasm32-wasip1` per `.cargo/config.toml`). Add the new deps + to that existing table; do **not** create a `[dependencies]` table, or they compile + for wasm and leak host-only crates into the wasm build: + + ```toml + [target.'cfg(not(target_arch = "wasm32"))'.dependencies] + # ... existing clap/url/etc ... + serde = { workspace = true } + serde_json = { workspace = true } + ``` + + Add workspace dependency `chromiumoxide = "0.9.1"` in `Cargo.toml` only in Task 7 when browser code is introduced. + +- [ ] **Step 2: Write failing JSON output tests** + + In `output.rs`, add tests that construct an in-memory verification result and assert exact JSON values: + + ```rust + #[test] + fn verification_json_contains_gate_state_and_extra_evidence() { + let result = VerificationReport::example_confirmed_with_extra_evidence(); + let value = serde_json::to_value(&result).expect("should serialize"); + + assert_eq!(value["ok"], true); + assert_eq!(value["pages"][0]["requested_path"], "/news/story"); + assert_eq!(value["pages"][0]["runtime_ad_stack_expected"], "unknown"); + assert_eq!(value["pages"][0]["extra_evidence"][0]["kind"], "gpt"); + assert_eq!(value["pages"][0]["warnings"][0]["code"], "redirected"); + } + + // Pins the spec §8 navigation_failed shape: error present, runtime/gates/ + // matched_slot_count keys ABSENT (skipped), final_url/path null. + #[test] + fn page_error_json_matches_navigation_failed_shape() { + let result = VerificationReport::example_navigation_failed(); + let value = serde_json::to_value(&result).expect("should serialize"); + let page = &value["pages"][0]; + + assert_eq!(page["error"]["code"], "navigation_failed"); + assert!(page["final_url"].is_null(), "final_url should be null"); + assert!(page["path"].is_null(), "path should be null"); + assert!(page.get("runtime_ad_stack_expected").is_none(), "runtime field absent on error page"); + assert!(page.get("gates").is_none(), "gates absent on error page"); + assert!(page.get("matched_slot_count").is_none(), "matched_slot_count absent on error page"); + assert_eq!(value["ok"], false); + } + ``` + +- [ ] **Step 3: Run focused test and verify it fails** + + Run: + + ```bash + cargo test -p trusted-server-cli ad_templates::output --target $(rustc -vV | sed -n 's/^host: //p') + ``` + + Expected: compile failure because output model does not exist. + +- [ ] **Step 4: Implement serializable output types** + + Model the **entire** `--json` wire tree from spec §8 (this is the single source of + truth for field names and ordering). Use owned `String` / `Vec` fields and + `#[serde(rename_all = "snake_case")]` so output is stable. Leaf enums: + + ```rust + #[derive(Debug, Clone, Copy, Eq, PartialEq, Serialize)] + #[serde(rename_all = "snake_case")] + pub enum SlotStatus { Confirmed, Partial, Missing } + + #[derive(Debug, Clone, Copy, Eq, PartialEq, Serialize)] + #[serde(rename_all = "snake_case")] + pub enum RuntimeAdStackExpectedJson { Yes, No, Unknown } + + impl From for RuntimeAdStackExpectedJson { + fn from(value: trusted_server_core::creative_opportunities::RuntimeAdStackExpected) -> Self { + use trusted_server_core::creative_opportunities::RuntimeAdStackExpected as Core; + match value { + Core::Yes => Self::Yes, + Core::No => Self::No, + Core::Unknown => Self::Unknown, + } + } + } + + #[derive(Debug, Clone, Copy, Eq, PartialEq, Serialize)] + #[serde(rename_all = "snake_case")] + pub enum GateState { Pass, Fail, Unknown } + ``` + + Top-level tree (field names and nesting must match §8 exactly): + + ```rust + #[derive(Debug, Clone, PartialEq, Serialize)] + pub struct VerificationReport { + pub ok: bool, + pub strict: bool, + pub pages: Vec, + pub warnings: Vec, + } + + #[derive(Debug, Clone, PartialEq, Serialize)] + pub struct PageJson { + pub url: String, + pub final_url: Option, + pub requested_path: String, + pub path: Option, + // Field ORDER matters: serde serializes in declaration order. Spec §8 places + // `error` immediately after `path` on the navigation_failed shape, so it must + // be declared here (not last). On normal pages `error` is None and skipped, so + // the runtime/gates/slots run in §8 order; on error pages the runtime/gates/ + // matched_slot_count are None and skipped, leaving url..path, error, slots, + // extra_evidence, warnings — exactly the §8 error shape. + #[serde(skip_serializing_if = "Option::is_none")] + pub error: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub runtime_ad_stack_expected: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub gates: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub matched_slot_count: Option, + pub slots: Vec, + pub extra_evidence: Vec, + pub warnings: Vec, + } + + // One field per gate name from spec §5.2 / §8, each a GateState. + #[derive(Debug, Clone, PartialEq, Serialize)] + pub struct Gates { + pub method_get: GateState, + pub navigation: GateState, + pub not_prefetch: GateState, + pub not_bot: GateState, + pub matched_slots: GateState, + pub auction_enabled: GateState, + pub consent_allows_auction: GateState, + } + + // Serialize for output JSON; Deserialize because the browser collector payload + // (Task 8) carries warning objects decoded into `BrowserAdEvidence.warnings`. + #[derive(Debug, Clone, Eq, PartialEq, Serialize, serde::Deserialize)] + pub struct Warning { + pub code: String, + pub message: String, + } + ``` + + Define the remaining nested JSON structs **explicitly** — do not serialize the + compare-module types directly. The compare types (`SlotResult`, `SlotEvidence`, + `GptSlotEvidence`, `ExtraEvidence`) carry a `phase` field and are not `Serialize`; + spec §8's `evidence.gpt` has **no** `phase` key and `configured` excludes `id` + and `page_patterns`. Mismatched reuse would emit extra keys. Wire structs: + + ```rust + #[derive(Debug, Clone, PartialEq, Serialize)] + pub struct SlotJson { + pub id: String, + pub status: SlotStatus, + pub phase: EvidencePhaseJson, + pub configured: ConfiguredJson, + pub evidence: SlotEvidenceJson, + pub warnings: Vec, + } + + // §8 `configured`: div_id, gam_unit_path, formats, providers — NO id/page_patterns. + #[derive(Debug, Clone, PartialEq, Serialize)] + pub struct ConfiguredJson { + pub div_id: String, + pub gam_unit_path: String, + pub formats: Vec, + pub providers: Vec, + } + + #[derive(Debug, Clone, PartialEq, Serialize)] + pub struct FormatJson { + pub width: u32, + pub height: u32, + pub media_type: String, + } + + #[derive(Debug, Clone, PartialEq, Serialize)] + pub struct SlotEvidenceJson { + pub dom_id: Option, + pub gpt: Option, + } + + // §8 `evidence.gpt`: gam_unit_path, div_id, sizes — NO phase. + #[derive(Debug, Clone, PartialEq, Serialize)] + pub struct GptEvidenceJson { + pub gam_unit_path: String, + pub div_id: String, + pub sizes: Vec<[u32; 2]>, + } + + #[derive(Debug, Clone, Copy, PartialEq, Serialize)] + #[serde(rename_all = "snake_case")] + pub enum EvidencePhaseJson { InitialLoad, Scroll } + + #[derive(Debug, Clone, PartialEq, Serialize)] + pub struct ExtraEvidenceJson { + pub kind: String, + pub phase: EvidencePhaseJson, + pub dom_id: Option, + pub gam_unit_path: Option, + pub sizes: Vec<[u32; 2]>, + pub reason: String, + } + ``` + + Note `sizes` serialize as `[[300,250]]` (arrays of two ints), matching §8 — use + `[u32; 2]` here even though the compare module uses `(u32, u32)` tuples; the + Task 9 assembly maps tuple → `[w, h]`. The conversion from the compare + `SlotResult`/`SlotEvidence`/`ExtraEvidence` to these JSON types (dropping `phase` + from `gpt`, dropping `id`/`page_patterns` from `configured`) lives in Task 9 Step 7. + `Warning` is the single warning type for the whole CLI; defined here and re-exported + from `ad_templates::mod` so `compare.rs` reuses it (plain data, not JSON logic). + Keep `example_confirmed_with_extra_evidence()` and similar fixtures behind + `#[cfg(test)]`. + +- [ ] **Step 5: Add verification human-render helpers** + + Add only the **browser-verification** page summary writers here (used by + `audit::ad_templates` in Task 9), writing to `&mut dyn Write`; no `println!` / + `eprintln!`. Do **not** add static match/check/explain writers in this task — + those are the existing `write_match_result`/`format_slot`/etc. functions that + Task 6 Step 3 **moves** out of `config_ad_templates.rs`. Keeping the static + relocation solely in Task 6 avoids two competing copies of the same helpers in + `output.rs`. The verification writers added here may be unused until Task 9 (a + warn-level `dead_code` lint that does not fail `cargo test`); add + `#[allow(dead_code)]` if clippy is run between Task 4 and Task 9. + +- [ ] **Step 6: Run focused tests** + + Run: + + ```bash + cargo test -p trusted-server-cli ad_templates::output --target $(rustc -vV | sed -n 's/^host: //p') + cargo test -p trusted-server-cli config_ad_templates --target $(rustc -vV | sed -n 's/^host: //p') + ``` + + Expected: all pass. + +- [ ] **Step 7: Commit** + + ```bash + git add Cargo.toml crates/trusted-server-cli/Cargo.toml crates/trusted-server-cli/src/ad_templates/mod.rs crates/trusted-server-cli/src/ad_templates/output.rs + git commit -m "Add ad-template CLI output models" + ``` + +## Task 5: Add Pure Evidence Comparison + +**Files:** + +- Create: `crates/trusted-server-cli/src/ad_templates/compare.rs` +- Modify: `crates/trusted-server-cli/src/ad_templates/mod.rs` + +- [ ] **Step 1: Write failing comparison tests** + + Cover every spec status and warning case without launching Chrome. Define small + test constructors so tests do not couple to the full `BrowserAdEvidence` field + list (`page_bids` and `warnings` default empty, evidence items default to + `EvidencePhase::InitialLoad`): + + ```rust + fn dom(id: &str) -> DomEvidence { + DomEvidence { dom_id: id.to_string(), phase: EvidencePhase::InitialLoad } + } + + fn gpt_slot(gam_unit_path: &str, div_id: &str, sizes: &[(u32, u32)]) -> GptSlotEvidence { + GptSlotEvidence { + gam_unit_path: gam_unit_path.to_string(), + div_id: div_id.to_string(), + sizes: sizes.to_vec(), + phase: EvidencePhase::InitialLoad, + } + } + + fn aps(slot_id: &str, sizes: &[(u32, u32)]) -> ApsFetchBidsEvidence { + ApsFetchBidsEvidence { slot_id: slot_id.to_string(), sizes: sizes.to_vec(), phase: EvidencePhase::InitialLoad } + } + + // Non-banner format helper for the unsupported-format test. + fn expected_slot_video(id: &str, div_id: &str, gam_unit_path: &str) -> ExpectedSlot { + ExpectedSlot { + id: id.to_string(), + div_id: div_id.to_string(), + gam_unit_path: gam_unit_path.to_string(), + formats: vec![ExpectedFormat { width: 0, height: 0, media_type: "video".to_string() }], + providers: Vec::new(), + page_patterns: Vec::new(), + } + } + + fn evidence(doms: Vec, gpts: Vec, aps: Vec) -> BrowserAdEvidence { + BrowserAdEvidence { + dom_ids: doms, + gpt_slots: gpts, + aps_calls: aps, + page_bids: Vec::new(), + warnings: Vec::new(), + } + } + + #[test] + fn gpt_path_div_and_size_overlap_confirms_slot() { + let expected = expected_slot("atf", "ad-atf-", "/123/news/atf", &[(300, 250)], &["aps"]); + let evidence = evidence( + vec![dom("ad-atf-0")], + vec![gpt_slot("/123/news/atf", "ad-atf-0", &[(300, 250)])], + Vec::new(), + ); + + let result = compare_page_evidence(&[expected], &evidence, RuntimeGateSummary::unknown_allowed()); + + assert_eq!(result.slots[0].status, SlotStatus::Confirmed, "GPT path+div+size overlap should confirm"); + assert!(result.slots[0].warnings.is_empty(), "confirmed slot should carry no warnings"); + } + + #[test] + fn dom_only_is_partial() { + let expected = expected_slot("atf", "ad-atf-", "/123/news/atf", &[(300, 250)], &[]); + let evidence = evidence(vec![dom("ad-atf-0")], Vec::new(), Vec::new()); + + let result = compare_page_evidence(&[expected], &evidence, RuntimeGateSummary::unknown_allowed()); + + assert_eq!(result.slots[0].status, SlotStatus::Partial, "DOM-only evidence should be partial"); + assert!( + result.slots[0].warnings.iter().any(|w| w.code == "dom_without_gpt"), + "DOM-only slot should warn dom_without_gpt" + ); + } + + #[test] + fn no_dom_or_gpt_is_missing() { + let expected = expected_slot("atf", "ad-atf-", "/123/news/atf", &[(300, 250)], &[]); + let evidence = evidence(Vec::new(), Vec::new(), Vec::new()); + + let result = compare_page_evidence(&[expected], &evidence, RuntimeGateSummary::unknown_allowed()); + + assert_eq!(result.slots[0].status, SlotStatus::Missing, "no DOM/GPT evidence should be missing"); + } + + #[test] + fn prefix_dom_resolution_ignores_container_suffix() { + let expected = expected_slot("header", "ad-header-0-", "/123/homepage/header", &[(728, 90)], &[]); + // First candidate ends with `-container` and must be skipped; the framework-suffixed ID resolves. + let evidence = evidence( + vec![dom("ad-header-0--container"), dom("ad-header-0-_R_abc123")], + Vec::new(), + Vec::new(), + ); + + let result = compare_page_evidence(&[expected], &evidence, RuntimeGateSummary::unknown_allowed()); + + assert_eq!(result.slots[0].evidence.dom_id.as_deref(), Some("ad-header-0-_R_abc123"), "prefix match should skip -container"); + assert_eq!(result.slots[0].status, SlotStatus::Partial, "DOM-only prefix match is partial without GPT"); + } + + #[test] + fn unmatched_gpt_slot_becomes_extra_evidence() { + let expected = expected_slot("atf", "ad-atf-", "/123/news/atf", &[(300, 250)], &[]); + let evidence = evidence( + vec![dom("ad-atf-0")], + vec![ + gpt_slot("/123/news/atf", "ad-atf-0", &[(300, 250)]), + gpt_slot("/123/publisher/right-rail", "ad-right-rail-0", &[(300, 250)]), + ], + Vec::new(), + ); + + let result = compare_page_evidence(&[expected], &evidence, RuntimeGateSummary::unknown_allowed()); + + assert_eq!(result.slots[0].status, SlotStatus::Confirmed, "matched slot still confirms"); + assert_eq!(result.extra_evidence.len(), 1, "unmatched GPT slot becomes extra evidence"); + assert_eq!(result.extra_evidence[0].kind, "gpt"); + assert!(!result.strict_failed(), "extra evidence alone must not fail strict"); + } + + #[test] + fn auction_disabled_skips_strict_missing_failure() { + let expected = expected_slot("atf", "ad-atf-", "/123/news/atf", &[(300, 250)], &[]); + let evidence = evidence(Vec::new(), Vec::new(), Vec::new()); + + let result = compare_page_evidence(&[expected], &evidence, RuntimeGateSummary::auction_disabled()); + + assert_eq!(result.runtime_ad_stack_expected, RuntimeAdStackExpected::No, "auction disabled should set No"); + assert_eq!(result.slots[0].status, SlotStatus::Missing, "static status is still reported"); + assert!(!result.strict_failed(), "missing slot must not fail strict when ad stack expected is No"); + } + + // §5.4: GPT path+div match but no numeric size overlap -> partial + warning. + #[test] + fn gpt_incompatible_sizes_is_partial() { + let expected = expected_slot("atf", "ad-atf-", "/123/news/atf", &[(300, 250)], &[]); + let evidence = evidence( + vec![dom("ad-atf-0")], + vec![gpt_slot("/123/news/atf", "ad-atf-0", &[(728, 90)])], + Vec::new(), + ); + + let result = compare_page_evidence(&[expected], &evidence, RuntimeGateSummary::unknown_allowed()); + + assert_eq!(result.slots[0].status, SlotStatus::Partial, "no size overlap should be partial"); + assert!(result.slots[0].warnings.iter().any(|w| w.code == "incompatible_sizes")); + } + + // §5.4/§5.6: matched slot with only non-banner formats -> partial + unsupported_format. + #[test] + fn non_banner_only_slot_is_partial() { + let expected = expected_slot_video("video", "ad-video-", "/123/news/video"); + let evidence = evidence( + vec![dom("ad-video-0")], + vec![gpt_slot("/123/news/video", "ad-video-0", &[(640, 480)])], + Vec::new(), + ); + + let result = compare_page_evidence(&[expected], &evidence, RuntimeGateSummary::unknown_allowed()); + + assert_eq!(result.slots[0].status, SlotStatus::Partial, "non-banner-only should be partial"); + assert!(result.slots[0].warnings.iter().any(|w| w.code == "unsupported_format")); + } + + // §5.4: GPT element ID may be `${resolved_dom_id}-container` and still confirm. + #[test] + fn gpt_container_element_id_confirms() { + let expected = expected_slot("atf", "ad-atf-0", "/123/news/atf", &[(300, 250)], &[]); + let evidence = evidence( + vec![dom("ad-atf-0"), dom("ad-atf-0-container")], + vec![gpt_slot("/123/news/atf", "ad-atf-0-container", &[(300, 250)])], + Vec::new(), + ); + + let result = compare_page_evidence(&[expected], &evidence, RuntimeGateSummary::unknown_allowed()); + + assert_eq!(result.slots[0].status, SlotStatus::Confirmed, "container element id is a valid GPT div match"); + } + + // §5.4: out-of-page GPT slot is partial (so it fails strict) plus a warning. + #[test] + fn out_of_page_gpt_slot_warns_and_is_partial() { + let expected = expected_slot("interstitial", "ad-oop-", "/123/news/oop", &[(300, 250)], &[]); + // gpt_slot with empty sizes models an out-of-page slot (no numeric sizes). + let evidence = evidence(vec![dom("ad-oop-0")], vec![gpt_slot("/123/news/oop", "ad-oop-0", &[])], Vec::new()); + + let result = compare_page_evidence(&[expected], &evidence, RuntimeGateSummary::unknown_allowed()); + + assert_eq!(result.slots[0].status, SlotStatus::Partial, "a sizeless slot against banner formats is partial"); + assert!(result.slots[0].warnings.iter().any(|w| w.code == "out_of_page_slot")); + } + + // §5.5: matching APS fetchBids -> no provider warning. + #[test] + fn aps_match_adds_no_warning() { + let expected = expected_slot("atf", "ad-atf-", "/123/news/atf", &[(300, 250)], &["aps"]); + let evidence = evidence( + vec![dom("ad-atf-0")], + vec![gpt_slot("/123/news/atf", "ad-atf-0", &[(300, 250)])], + vec![aps("atf", &[(300, 250)])], + ); + + let result = compare_page_evidence(&[expected], &evidence, RuntimeGateSummary::unknown_allowed()); + + assert_eq!(result.slots[0].status, SlotStatus::Confirmed); + assert!(!result.slots[0].warnings.iter().any(|w| w.code.starts_with("aps_")), "matching APS should not warn"); + } + + // §5.5: configured aps provider but no APS evidence -> provider warning, still confirmed, strict not failed. + #[test] + fn aps_missing_warns_but_keeps_confirmed() { + let expected = expected_slot("atf", "ad-atf-", "/123/news/atf", &[(300, 250)], &["aps"]); + let evidence = evidence( + vec![dom("ad-atf-0")], + vec![gpt_slot("/123/news/atf", "ad-atf-0", &[(300, 250)])], + Vec::new(), + ); + + let result = compare_page_evidence(&[expected], &evidence, RuntimeGateSummary::unknown_allowed()); + + assert_eq!(result.slots[0].status, SlotStatus::Confirmed, "missing APS does not flip status"); + assert!(result.slots[0].warnings.iter().any(|w| w.code == "aps_evidence_missing")); + assert!(!result.strict_failed(), "provider warning alone must not fail strict"); + } + ``` + + Add a `#[cfg(test)]` constructor in `compare.rs` tests that builds a real + `ExpectedSlot` (the Task 3 type) so comparison tests stay readable: + + ```rust + fn expected_slot(id: &str, div_id: &str, gam_unit_path: &str, sizes: &[(u32, u32)], providers: &[&str]) -> ExpectedSlot { + ExpectedSlot { + id: id.to_string(), + div_id: div_id.to_string(), + gam_unit_path: gam_unit_path.to_string(), + formats: sizes.iter().map(|&(width, height)| ExpectedFormat { width, height, media_type: "banner".to_string() }).collect(), + providers: providers.iter().map(|p| p.to_string()).collect(), + page_patterns: Vec::new(), + } + } + ``` + +- [ ] **Step 2: Run focused test and verify it fails** + + Run: + + ```bash + cargo test -p trusted-server-cli ad_templates::compare --target $(rustc -vV | sed -n 's/^host: //p') + ``` + + Expected: compile failure because comparison module does not exist. + +- [ ] **Step 3: Implement browser evidence structs** + + Define the minimum collector-independent input shape plus the comparison result + shape the tests assert against: + + All browser-evidence input structs derive `Debug, Clone` and `serde::Deserialize` + (Task 8 decodes them from the collector's `window.__tsAdTemplateEvidence` JSON); + `EvidencePhase` deserializes from `"initial_load"` / `"scroll"`. The comparison- + result structs derive `Debug` (so the Step 1 `assert_eq!`/`matches!` assertions + compile) and `Clone`. Sizes are `(u32, u32)` tuples internally; deserialize them + from JSON `[w, h]` arrays. + + ```rust + #[derive(Debug, Clone, Copy, Eq, PartialEq, serde::Deserialize)] + #[serde(rename_all = "snake_case")] + pub enum EvidencePhase { + InitialLoad, + Scroll, + } + + #[derive(Debug, Clone, serde::Deserialize)] + pub struct DomEvidence { + pub dom_id: String, + pub phase: EvidencePhase, + } + + #[derive(Debug, Clone, serde::Deserialize)] + pub struct GptSlotEvidence { + pub gam_unit_path: String, + pub div_id: String, + pub sizes: Vec<(u32, u32)>, + pub phase: EvidencePhase, + } + + // APS `apstag.fetchBids` evidence (spec §5.5): configured slot ID + observed sizes. + #[derive(Debug, Clone, serde::Deserialize)] + pub struct ApsFetchBidsEvidence { + pub slot_id: String, + pub sizes: Vec<(u32, u32)>, + pub phase: EvidencePhase, + } + + // DEFERRED in this implementation: `/__ts/page-bids` SPA observation (spec §5.2 + // "when available"). The struct/field are forward scaffolding so the collector and + // JSON can grow it later; Task 8 does NOT populate it and Task 4 JSON does NOT + // surface it in Phase 1. Tracked as a deferred item in Risks. Keep the field so + // `BrowserAdEvidence` deserialization stays forward-compatible (default empty). + #[derive(Debug, Clone, serde::Deserialize)] + pub struct PageBidsEvidence { + pub slot_id: String, + pub phase: EvidencePhase, + } + + // `Warning` is the shared CLI warning type defined in Task 4 (`output.rs`) and + // re-exported from `ad_templates::mod`. It is plain data reused here (not JSON + // logic). Because the collector payload carries warnings, give `Warning` BOTH + // `Serialize` (Task 4 output) and `Deserialize` (Task 8 decode) derives. + use crate::ad_templates::output::Warning; + + #[derive(Debug, Clone, serde::Deserialize)] + pub struct BrowserAdEvidence { + pub dom_ids: Vec, + pub gpt_slots: Vec, + pub aps_calls: Vec, + #[serde(default)] + pub page_bids: Vec, + #[serde(default)] + pub warnings: Vec, + } + + // Comparison output. Uses the core `RuntimeAdStackExpected` enum from Task 1 so + // pure comparison logic does not depend on the output/JSON module. Task 4's + // `RuntimeAdStackExpectedJson` is produced only at serialization time. + #[derive(Debug, Clone)] + pub struct PageVerificationResult { + pub runtime_ad_stack_expected: trusted_server_core::creative_opportunities::RuntimeAdStackExpected, + pub slots: Vec, + pub extra_evidence: Vec, + } + + #[derive(Debug, Clone)] + pub struct SlotResult { + pub id: String, + pub status: SlotStatus, + pub phase: EvidencePhase, + pub evidence: SlotEvidence, + pub warnings: Vec, + } + + #[derive(Debug, Clone)] + pub struct SlotEvidence { + pub dom_id: Option, + pub gpt: Option, + } + + #[derive(Debug, Clone)] + pub struct ExtraEvidence { + pub kind: String, + pub phase: EvidencePhase, + pub dom_id: Option, + pub gam_unit_path: Option, + pub sizes: Vec<(u32, u32)>, + pub reason: String, + } + ``` + + `RuntimeGateSummary` is the third argument to `compare_page_evidence`; it wraps + the core gate result. Provide `RuntimeGateSummary::unknown_allowed()` (expected + `Unknown`) and `RuntimeGateSummary::auction_disabled()` (expected `No`) test + constructors so comparison tests do not rebuild gate inputs by hand. + +- [ ] **Step 4: Implement DOM/GPT/APS rules** + + Status rules: + - DOM exact ID first, then first prefix match, **excluding `-container`** wrappers + (slot-root resolution, spec §5.3). + - GPT confirms when: GAM unit path matches, the GPT slot element ID equals the + resolved DOM ID **or** an existing `${resolved_dom_id}-container` element + (spec §5.4 — note this is the GPT element-ID match, distinct from the §5.3 DOM + root resolution that skips `-container`), and at least one numeric banner size + overlaps. + - GPT path/div match with no numeric size overlap → `partial` (warn `incompatible_sizes`). + - Matched slot whose configured formats are **all non-banner** (video/native) → + `partial` (warn `unsupported_format`); banner is the only Phase-1 confirmable type. + - DOM-only (no GPT) → `partial` (warn `dom_without_gpt`). + - No DOM and no GPT → `missing`. + + Size-compatibility warnings (spec §5.4 — all are warnings, none flip a confirmed + slot to fail): emit a `Warning` for each of: + - `fluid_size_ignored` — non-numeric observed sizes like `"fluid"` ignored for matching; + - `extra_observed_size` — observed GPT sizes not in the configured set; + - `configured_size_not_observed` — configured sizes never observed (when ≥1 was); + - `out_of_page_slot` — out-of-page GPT slot with no sizes observed; the slot is + reported `partial`, which fails `--strict`. + + Provider + extra evidence: + - APS: configured `providers.aps.slot_id` with matching `fetchBids` → no warning; + missing/ambiguous APS evidence → provider warning only (`aps_evidence_missing` / + `aps_evidence_ambiguous`), never flips status or fails `--strict` in Phase 1. + - Unmatched live DOM/GPT/APS evidence → structured `extra_evidence` (never fails strict). + + Define each warning `code` as a stable string constant so output and tests share them. + +- [ ] **Step 5: Implement strict decision method** + + Add an inherent method on the result so tests can call `result.strict_failed()`: + + ```rust + impl PageVerificationResult { + pub fn strict_failed(&self) -> bool { + use trusted_server_core::creative_opportunities::RuntimeAdStackExpected; + if self.runtime_ad_stack_expected == RuntimeAdStackExpected::No { + return false; + } + self.slots + .iter() + .any(|slot| matches!(slot.status, SlotStatus::Missing | SlotStatus::Partial)) + } + } + ``` + + - false when `runtime_ad_stack_expected == No`; + - true for any `missing` or `partial` slot when expected is `Yes` or `Unknown`; + - false for provider warnings and extra evidence alone (they are not slot statuses). + +- [ ] **Step 6: Run focused tests** + + Run: + + ```bash + cargo test -p trusted-server-cli ad_templates::compare --target $(rustc -vV | sed -n 's/^host: //p') + ``` + + Expected: all comparison tests pass. + +- [ ] **Step 7: Commit** + + ```bash + git add crates/trusted-server-cli/src/ad_templates/mod.rs crates/trusted-server-cli/src/ad_templates/compare.rs + git commit -m "Add pure ad-template evidence comparison" + ``` + +## Task 6: Refactor Static Commands Onto Shared Modules + +**Files:** + +- Modify: `crates/trusted-server-cli/src/config_ad_templates.rs` +- Modify: `crates/trusted-server-cli/src/ad_templates/output.rs` +- Modify: `crates/trusted-server-cli/src/run.rs` + +- [ ] **Step 1: Add characterization tests before refactor** + + These guard behavior across the Step 3 move, so each must assert **exact output + substrings** (capture the command's `Vec`/`String` output and + `assert!(out.contains("..."))`), not just run without panicking — a bare + smoke test cannot catch a wording regression. Mirror the existing assertion style + at `config_ad_templates.rs:570-657`. Pin, with concrete expected strings: + - `lint` not configured → e.g. `"creative_opportunities: not configured"`; + - `lint` with slots + auction disabled → slot count line + `"auction: disabled"`; + - `match --details` → slot div ID, GAM unit path, formats, providers lines; + - `check --expect-no-slots` → success message; + - `check` failure with missing and unexpected slots → the exact failure lines; + - `explain` → each gate line (including `"auction providers configured"`) and the + EdgeZero legacy-fallback warning text. + + Run the existing tests first and copy the real emitted strings so the + characterization assertions match current behavior exactly before refactoring. + +- [ ] **Step 2: Run tests before refactor** + + Run: + + ```bash + cargo test -p trusted-server-cli config_ad_templates --target $(rustc -vV | sed -n 's/^host: //p') + ``` + + Expected: characterization tests pass against the current implementation. + +- [ ] **Step 3: Move formatting into `ad_templates::output`** + + Move these helpers out of `config_ad_templates.rs`: + - `write_match_result` + - `write_gate` + - `format_slot` + - `format_format` + - `format_providers` + - `join_set` + - `plural` + + Keep command functions small: parse args, load config, call expected/gate logic, render. + +- [ ] **Step 4: Reuse shared gate helper in `explain`** + + Build `AdStackGateInput` from explain flags and config: + + ```rust + let gate = evaluate_ad_stack_gate(AdStackGateInput { + method_get, + navigation: !args.non_navigation, + prefetch: args.prefetch, + bot: args.bot, + matched_slots: !expected.slots.is_empty(), + consent_allows_auction: Some(!args.consent_denied), + auction_enabled: loaded.settings.auction.enabled, + }); + ``` + + Render the seven shared gate names from `gate` rather than hand-rolled boolean chains. + + **Preserve the explain-only provider gate.** The current `run_explain` + (`config_ad_templates.rs:270`) renders an eighth gate, + `"auction providers configured"` (`!loaded.settings.auction.providers.is_empty()`, + line 299), and ANDs it into its local `runs_ad_stack` decision (line 302). The + shared `evaluate_ad_stack_gate` helper and runtime `should_run_server_side_ad_stack` + intentionally have no provider-configured gate. Do not fold this into + `AdStackGateInput`. Keep `"auction providers configured"` as an explain-only + supplementary `write_gate(...)` line rendered alongside the shared result, and + keep it in `explain`'s own `runs_ad_stack` decision: + + ```rust + let providers_configured = !loaded.settings.auction.providers.is_empty(); + render_shared_gates(out, &gate)?; + write_gate(out, "auction providers configured", providers_configured)?; + let runs_ad_stack = + gate.expected == RuntimeAdStackExpected::Yes && providers_configured; + ``` + + This keeps `explain` output and behavior identical to the current implementation + (verified by the Step 1 characterization test) while still sharing the seven core + runtime gates with `publisher.rs`. + +- [ ] **Step 5: Run focused tests** + + Run: + + ```bash + cargo test -p trusted-server-cli config_ad_templates --target $(rustc -vV | sed -n 's/^host: //p') + cargo test -p trusted-server-cli ad_templates --target $(rustc -vV | sed -n 's/^host: //p') + ``` + + Expected: no output regressions except intentional wording updates covered by tests. + +- [ ] **Step 6: Commit** + + ```bash + git add crates/trusted-server-cli/src/config_ad_templates.rs crates/trusted-server-cli/src/ad_templates/output.rs crates/trusted-server-cli/src/run.rs + git commit -m "Refactor static ad-template commands" + ``` + +## Task 7: Port Generic Audit Namespace And Browser Collector + +**Files:** + +- Modify: `Cargo.toml` +- Modify: `crates/trusted-server-cli/Cargo.toml` +- Create: `crates/trusted-server-cli/src/audit/mod.rs` +- Create: `crates/trusted-server-cli/src/audit/page.rs` +- Create: `crates/trusted-server-cli/src/audit/collector.rs` +- Create: `crates/trusted-server-cli/src/audit/browser.rs` +- Modify: `crates/trusted-server-cli/src/lib.rs` +- Modify: `crates/trusted-server-cli/src/run.rs` + +- [ ] **Step 1: Add browser dependencies** + + Add `chromiumoxide` to the root `[workspace.dependencies]` table (inert until a + crate references it via `{ workspace = true }`): + + ```toml + [workspace.dependencies] + # ... existing entries ... + chromiumoxide = "0.9.1" + ``` + + Add the host deps to the CLI crate under its existing + `[target.'cfg(not(target_arch = "wasm32"))'.dependencies]` table — NOT a plain + `[dependencies]` table (workspace default target is wasm32; an unconditional dep + compiles for wasm and breaks the build / leaks host-only crates): + + ```toml + [target.'cfg(not(target_arch = "wasm32"))'.dependencies] + # ... existing clap/url/serde/etc ... + chromiumoxide = { workspace = true } + futures = { workspace = true } + tempfile = { workspace = true } + tokio = { workspace = true } + which = { workspace = true } + ``` + + Verify the root workspace already provides `futures`, `tempfile`, `tokio`, `which` + (it does on this branch); only `chromiumoxide` is a new workspace entry. + +- [ ] **Step 2: Write failing audit parser tests** + + In `run.rs` tests: + + ```rust + #[test] + fn audit_legacy_url_parses_as_page_alias() { + let args = parse(&["ts", "audit", "https://www.example.com/"]); + assert!(matches!(args.command, Command::Audit(_))); + } + + #[test] + fn audit_page_subcommand_parses() { + let args = parse(&["ts", "audit", "page", "https://www.example.com/"]); + assert!(matches!(args.command, Command::Audit(_))); + } + + #[test] + fn audit_ad_templates_verify_parses() { + let args = parse(&["ts", "audit", "ad-templates", "verify", "https://www.example.com/"]); + assert!(matches!(args.command, Command::Audit(_))); + } + + #[test] + fn audit_ad_templates_is_not_legacy_url() { + assert!(Args::try_parse_from(["ts", "audit", "ad-templates"]).is_err()); + } + ``` + +- [ ] **Step 3: Run parser tests and verify failure** + + Run: + + ```bash + cargo test -p trusted-server-cli audit_ --target $(rustc -vV | sed -n 's/^host: //p') + ``` + + Expected: compile failure because `Audit` command does not exist. + +- [ ] **Step 4: Implement audit Clap namespace in current `run.rs` shape** + + Do not add stale #800 `args.rs`. Add a `Command::Audit(AuditArgs)` variant to the + existing `Command` enum, plus the full audit arg surface. The parser tests in + Step 2 exercise `ad-templates verify`, so the **entire** command surface (including + the verify args) must be defined here for those tests to compile. Task 9 implements + the verify _behavior_ only — it does not redefine these arg types. + + **Visibility:** `audit::run_audit` lives in `audit/mod.rs` and must name these + types in its signature and match their variants, so every audit arg type and its + fields are `pub(crate)` (not private). `PageAuditArgs` (from `audit/page.rs`, Task 7 + Step 5) and `AuditAdTemplatesVerifyArgs` are likewise `pub(crate)`/`pub`. `run.rs` + imports `AuditArgs` for the `Command::Audit(AuditArgs)` variant; everything else is + read by `audit/mod.rs`. (This mirrors `config_ad_templates::AdTemplatesCommand`, + which is `pub` and consumed by `run.rs`.) + + ```rust + // value parser shared by legacy_url and verify urls; rejects non-HTTP(S) schemes. + pub(crate) fn parse_http_url(raw: &str) -> Result { + let url = url::Url::parse(raw).map_err(|error| format!("invalid URL `{raw}`: {error}"))?; + match url.scheme() { + "http" | "https" => Ok(url), + other => Err(format!("unsupported URL scheme `{other}` (expected http or https)")), + } + } + + #[derive(Debug, clap::Args)] + pub(crate) struct AuditArgs { + #[command(subcommand)] + pub(crate) command: Option, + #[arg(value_parser = parse_http_url, hide = true)] + pub(crate) legacy_url: Option, + } + + #[derive(Debug, Subcommand)] + pub(crate) enum AuditSubcommand { + Page(PageAuditArgs), + #[command(name = "ad-templates", subcommand)] + AdTemplates(AuditAdTemplatesCommand), + } + + #[derive(Debug, Subcommand)] + pub(crate) enum AuditAdTemplatesCommand { + Verify(AuditAdTemplatesVerifyArgs), + } + + // Defined here (not Task 9) so parser tests compile. Task 9 fills in the handler. + #[derive(Debug, clap::Args)] + pub(crate) struct AuditAdTemplatesVerifyArgs { + #[command(flatten)] + pub config: AppConfigArgs, + #[arg(required = true, value_parser = parse_http_url)] + pub urls: Vec, + #[arg(long)] + pub strict: bool, + #[arg(long)] + pub json: bool, + #[arg(long)] + pub scroll: bool, + } + ``` + + Dispatch `Command::Audit(args)` to a single `audit::run_audit(args: AuditArgs)` + entry point (in `audit/mod.rs`) that normalizes the namespace: `legacy_url` (if + present) and `Page` both route to the generic page audit; `AdTemplates(Verify(..))` + routes to the verifier (a stub returning `Ok(())` until Task 9). If Clap cannot make + the optional-subcommand-plus-hidden-positional contract unambiguous, implement a + small `AuditArgs::normalize()` that rejects `legacy_url` values that are not HTTP(S). + Decide arg-type home consistently: keep them in `run.rs` as `pub(crate)` (as shown) + and import into `audit/mod.rs`, or move them next to `run_audit` in `audit/mod.rs` + and import `AuditArgs` into `run.rs` — either works, but do not split them. + +- [ ] **Step 5: Port minimal generic page audit** + + Port useful #800 concepts into `audit/page.rs`, but keep output read-only by default for now: + - parse/validate URL; + - call `AuditCollector::collect_page`; + - print summary with final URL, title, script/resource counts, warnings; + - no draft config generation in this PR unless #800 rebase keeps it explicitly. + +- [ ] **Step 6: Implement collector trait and browser collector base** + + `audit/collector.rs` — define the trait plus its concrete request/response types so + Task 9's `FakeCollector` and the verify orchestration have a contract to assert on: + + ```rust + pub trait AuditCollector { + fn collect_page(&self, request: BrowserCollectRequest) -> Result; + } + + pub struct BrowserCollectRequest { + pub url: url::Url, + // Pre-navigation init scripts (evaluate-on-new-document). Empty for plain page audit; + // Task 8 passes the ad-template collector script here. + pub init_scripts: Vec, + pub scroll: bool, + } + + pub struct CollectedPage { + pub final_url: url::Url, + pub title: String, + // Generic page-audit signals (counts only; no page HTML/cookies/storage). + pub script_count: usize, + pub resource_count: usize, + pub warnings: Vec, + // Present only when an ad-template init script was injected (Task 8); None for + // plain `ts audit page`. This is how `BrowserAdEvidence` rides on a CollectedPage. + pub ad_evidence: Option, + } + ``` + + `BrowserCollectRequest` carries `init_scripts` + `scroll` so ad-template verification + enables evidence hooks without changing the trait later. + + `audit/browser.rs` should port #800's: + - `which` browser lookup; + - isolated `TempDir` profile; + - current-thread Tokio runtime; + - `Browser::launch`; + - `page.goto`; + - `wait_for_navigation_response`; + - settle loop. + +- [ ] **Step 7: Run compile-focused CLI tests** + + Run: + + ```bash + cargo test -p trusted-server-cli audit_ --target $(rustc -vV | sed -n 's/^host: //p') + cargo test -p trusted-server-cli --target $(rustc -vV | sed -n 's/^host: //p') + ``` + + Expected: parser and non-browser unit tests pass. No test should require installed Chrome yet. + +- [ ] **Step 8: Commit** + + ```bash + git add Cargo.toml crates/trusted-server-cli/Cargo.toml crates/trusted-server-cli/src/audit crates/trusted-server-cli/src/lib.rs crates/trusted-server-cli/src/run.rs + git commit -m "Add audit namespace and browser collector base" + ``` + +## Task 8: Add Browser Ad-Template Evidence Collector + +**Files:** + +- Create: `crates/trusted-server-cli/src/audit/ad_template_collector.js` +- Modify: `crates/trusted-server-cli/src/audit/browser.rs` +- Modify: `crates/trusted-server-cli/src/audit/collector.rs` +- Modify: `crates/trusted-server-cli/src/ad_templates/compare.rs` + +- [ ] **Step 1: Write JS collector contract fixture tests** + + Add Rust unit tests that inspect generated init-script text and decode a mocked `window.__tsAdTemplateEvidence` JSON payload. These should not launch Chrome. + + Prefer **behavioral** assertions over brittle substring matching: where possible, + assert by decoding a mocked `window.__tsAdTemplateEvidence` payload into + `BrowserAdEvidence` and checking fields. For the few structural checks that must + inspect the script text, pin **exact** marker substrings (no "or equivalent", so + the pass condition is deterministic) — choose the markers to match the strings the + implementation will actually emit: + - `build_ad_template_init_script` output contains the literal `__TS_CONFIG` injection; + - contains the chosen googletag-hook marker (pick ONE and pin it, e.g. + `Object.defineProperty(window, "googletag"`); + - contains the `cmd.push` wrap marker; + - contains the `defineSlot` record marker; + - contains the `apstag.fetchBids` wrap marker; + - embeds only the configured div prefixes / provider IDs passed via `__TS_CONFIG` + (assert a non-configured prefix is absent). + +- [ ] **Step 2: Run tests and verify failure** + + Run: + + ```bash + cargo test -p trusted-server-cli ad_template_collector --target $(rustc -vV | sed -n 's/^host: //p') + ``` + + Expected: failure because collector script/builder does not exist. + +- [ ] **Step 3: Implement init script builder** + + In Rust, build script as: + + ```rust + pub fn build_ad_template_init_script(config: &AdTemplateCollectorConfig) -> Result { + let config_json = serde_json::to_string(config) + .map_err(|error| format!("failed to serialize ad-template collector config: {error}"))?; + Ok(format!(";(() => {{ const __TS_CONFIG = {config_json};\n{}\n}})();", include_str!("ad_template_collector.js"))) + } + ``` + + Keep the JS file generic; pass configured prefixes and APS slot IDs through `__TS_CONFIG`. + +- [ ] **Step 4: Implement read-only JS evidence collection** + + In `ad_template_collector.js`, write to `window.__tsAdTemplateEvidence`: + - `dom_ids`: matched IDs from configured prefixes, excluding `-container`; + - `gpt_slots`: record `defineSlot` calls observed **both** directly **and** when + dispatched from the `googletag.cmd` queue (wrap `cmd.push` so queued callbacks are + instrumented without changing their order — spec §7), **plus** a post-settle + `googletag.pubads().getSlots()` scrape. For each scraped slot capture + `getAdUnitPath()`, `getSlotElementId()`, and `getSizes()` so `getSlots()`-only + slots still carry numeric `sizes` for the §5.4 overlap rule. Normalize sizes from + both `defineSlot` input and `getSizes()` output: `[300,250]` → one `(300,250)`; + `[[300,250],[728,90]]` → two pairs; non-numeric (`"fluid"`) dropped from numeric + sizes and surfaced as a `fluid_size_ignored` warning; + - `aps_calls`: `fetchBids` payloads (configured slot IDs + sizes); + - `warnings`: collector warnings only ({code, message}), no page HTML/cookies/storage. + + Always call original page functions with unchanged arguments, and never override + `navigator.webdriver` (spec §7). + +- [ ] **Step 5: Add browser collector extraction** + + After settle and after optional scroll, evaluate: + + ```javascript + ;() => window.__tsAdTemplateEvidence || null + ``` + + Decode into `BrowserAdEvidence`. If decode fails, return a page warning rather than failing navigation. + +- [ ] **Step 6: Add deterministic scroll** + + In `audit/browser.rs`, implement `scroll` by evaluating: + + ```javascript + ;async () => { + const height = Math.max( + document.body.scrollHeight, + document.documentElement.scrollHeight + ) + for (const y of [ + Math.floor(height * 0.33), + Math.floor(height * 0.66), + height, + ]) { + window.scrollTo(0, y) + await new Promise((resolve) => setTimeout(resolve, 250)) + } + window.scrollTo(0, 0) + } + ``` + + Then wait for the same settle quiet period and collect evidence with `phase = "scroll"` where the JS script marks new observations. + +- [ ] **Step 7: Run focused tests** + + Run: + + ```bash + cargo test -p trusted-server-cli ad_template_collector --target $(rustc -vV | sed -n 's/^host: //p') + cargo test -p trusted-server-cli audit::browser --target $(rustc -vV | sed -n 's/^host: //p') + ``` + + Expected: unit tests pass without launching Chrome. + +- [ ] **Step 8: Commit** + + ```bash + git add crates/trusted-server-cli/src/audit/ad_template_collector.js crates/trusted-server-cli/src/audit/browser.rs crates/trusted-server-cli/src/audit/collector.rs crates/trusted-server-cli/src/ad_templates/compare.rs + git commit -m "Collect browser ad-template evidence" + ``` + +## Task 9: Implement `ts audit ad-templates verify` + +**Files:** + +- Create: `crates/trusted-server-cli/src/audit/ad_templates.rs` +- Modify: `crates/trusted-server-cli/src/audit/mod.rs` +- Modify: `crates/trusted-server-cli/src/run.rs` +- Modify: `crates/trusted-server-cli/src/ad_templates/output.rs` + +- [ ] **Step 1: Write failing orchestration tests with a fake collector** + + Build a fake collector implementing `AuditCollector` and test: + - one confirmed page exits success in default mode; + - strict missing slot returns error; + - `[auction].enabled = false` returns runtime skipped and does not strict-fail missing evidence; + - one page navigation error plus one success sets JSON `ok = false`; + - invalid `ftp://` URL fails before fake collector is called; + - redirect uses final path for expected slots and emits redirect warning. + + Define the test scaffolding explicitly (no dangling helpers): + + ```rust + // Maps each requested URL to a canned outcome so orchestration is tested without Chrome. + struct FakeCollector { + pages: std::collections::HashMap>, + } + + impl FakeCollector { + // Success page: requested -> final_url, carrying the given ad evidence. + fn page(requested: &str, final_url: &str, evidence: BrowserAdEvidence) -> Self { + let mut pages = std::collections::HashMap::new(); + pages.insert( + requested.to_string(), + Ok(CollectedPage { + final_url: url::Url::parse(final_url).expect("valid final url"), + title: String::new(), + script_count: 0, + resource_count: 0, + warnings: Vec::new(), + ad_evidence: Some(evidence), + }), + ); + Self { pages } + } + // Helper to add a failing page for multi-URL tests. + fn with_error(mut self, requested: &str, message: &str) -> Self { + self.pages.insert(requested.to_string(), Err(message.to_string())); + self + } + } + + impl AuditCollector for FakeCollector { + fn collect_page(&self, request: BrowserCollectRequest) -> Result { + self.pages + .get(request.url.as_str()) + .cloned() + .unwrap_or_else(|| Err(format!("no fake page for {}", request.url))) + } + } + + impl BrowserAdEvidence { + // #[cfg(test)] fixture: one confirmed news slot (atf / ad-atf-0 / /123/news/atf, 300x250). + fn confirmed_news_slot() -> Self { + BrowserAdEvidence { + dom_ids: vec![DomEvidence { dom_id: "ad-atf-0".into(), phase: EvidencePhase::InitialLoad }], + gpt_slots: vec![GptSlotEvidence { + gam_unit_path: "/123/news/atf".into(), + div_id: "ad-atf-0".into(), + sizes: vec![(300, 250)], + phase: EvidencePhase::InitialLoad, + }], + aps_calls: Vec::new(), + page_bids: Vec::new(), + warnings: Vec::new(), + } + } + } + + // Runs the verify orchestration with `--json` over `urls` and returns parsed JSON. + // Loads a #[cfg(test)] effective config whose `/news/*` slot is the atf slot above. + fn run_verify_json(collector: &dyn AuditCollector, urls: impl IntoIterator) -> serde_json::Value { /* impl in test module */ } + + #[test] + fn verify_uses_final_url_for_matching_after_redirect() { + let collector = FakeCollector::page( + "https://www.example.com/", + "https://www.example.com/news/story", + BrowserAdEvidence::confirmed_news_slot(), + ); + let json = run_verify_json(&collector, ["https://www.example.com/"]); + + assert_eq!(json["pages"][0]["path"], "/news/story"); + // Warning order is unspecified; assert presence, not index 0. + let warnings = json["pages"][0]["warnings"].as_array().expect("warnings array"); + assert!( + warnings.iter().any(|w| w["code"] == "redirected"), + "redirect should emit a `redirected` warning" + ); + } + ``` + + `run_verify_json` calls the same `run_verify` entry point used in production but + with the fake collector injected and output captured; define it in the test module + so all six listed cases share it. + +- [ ] **Step 2: Run focused tests and verify failure** + + Run: + + ```bash + cargo test -p trusted-server-cli audit::ad_templates --target $(rustc -vV | sed -n 's/^host: //p') + ``` + + Expected: compile failure because verifier module does not exist. + +- [ ] **Step 3: Wire the verifier handler** + + `AuditAdTemplatesVerifyArgs` already exists from Task 7, Step 4. Replace the Task 7 + stub so `audit::run_audit` routes `AdTemplates(Verify(args))` into a new + `audit::ad_templates::run_verify(args)`. Do not redefine the arg struct. + +- [ ] **Step 4: Implement verification orchestration** + + For each URL: + 1. Collect browser page with ad-template init script and optional scroll. + 2. Parse final URL and normalize final path. + 3. Build expected slots for final path. + 4. Build gate summary using shared core gate helper with `consent_allows_auction = None`. + 5. Add redirect warning (`code = "redirected"`) if requested path differs from final path. + 6. Compare evidence (`compare_page_evidence`) to get a `PageVerificationResult`. + 7. **Assemble the wire `PageJson`** (Task 4 type) from the pieces the comparison + result does not carry: `url` / `final_url` / `requested_path` / `path`, + `gates` (map the gate summary's per-gate states to `GateState`), + `matched_slot_count`, `runtime_ad_stack_expected` (via the `From` impl on + `RuntimeAdStackExpectedJson`), then the `slots` / `extra_evidence` / `warnings` + from the comparison result. `PageVerificationResult` is intentionally URL- and + gate-agnostic; this step is where per-page request context is joined in. + 8. Preserve page-level errors as a `PageJson` with `error: Some(..)` and continue + remaining URLs. + +- [ ] **Step 5: Implement exit behavior** + - Default auditor-assist mode: return `Ok(())` for missing/partial evidence when no page-level collection errors occur. + - `--strict`: return `Err(String)` when any non-skipped page has missing/partial slot. + - Multi-URL page errors: JSON `ok=false`; command returns `Err(String)` after writing JSON/human output. + - Invalid schemes: fail before browser launch and before any output. + +- [ ] **Step 6: Run focused tests** + + Run: + + ```bash + cargo test -p trusted-server-cli audit::ad_templates --target $(rustc -vV | sed -n 's/^host: //p') + cargo test -p trusted-server-cli ad_templates --target $(rustc -vV | sed -n 's/^host: //p') + ``` + + Expected: verifier and pure comparison tests pass. + +- [ ] **Step 7: Commit** + + ```bash + git add crates/trusted-server-cli/src/audit/ad_templates.rs crates/trusted-server-cli/src/audit/mod.rs crates/trusted-server-cli/src/run.rs crates/trusted-server-cli/src/ad_templates/output.rs + git commit -m "Verify ad-template slots from browser evidence" + ``` + +## Task 10: Add Local Browser Fixture Tests + +**Files:** + +- Modify: `crates/trusted-server-cli/src/audit/browser.rs` +- Modify: `crates/trusted-server-cli/src/audit/ad_templates.rs` + +- [ ] **Step 1: Add test-only local HTTP fixture helper** + + In `audit::browser` tests, create a `TcpListener` serving static HTML from strings. Keep it test-only and host-target only. + + Fixture pages: + - direct `googletag.defineSlot`; + - `googletag.cmd.push`; + - late `window.googletag = { cmd: [] }`; + - late `window.apstag`; + - lazy slot created after scroll; + - redirect from `/` to `/news/story`; + - navigation returning 500. + +- [ ] **Step 2: Gate tests when Chrome is unavailable** + + Add helper: + + ```rust + fn chrome_available() -> bool { + ["chrome", "chromium", "google-chrome", "google-chrome-stable"] + .iter() + .any(|name| which::which(name).is_ok()) + } + ``` + + Each browser fixture test should early-return when unavailable. Do not use + `println!` / `eprintln!`; keep the skip reason in the helper name or a skipped + assertion message so clippy stays clean. This keeps CI portable unless Chrome is + installed. + +- [ ] **Step 3: Write fixture tests** + + Tests should assert the collector sees evidence, not real ad network behavior: + - direct GPT evidence confirms; + - command-queue GPT evidence confirms; + - APS `fetchBids` evidence removes APS provider warning; + - lazy slot appears only when `--scroll` is set; + - redirect result uses final path; + - failed page produces page-level error while other pages continue. + +- [ ] **Step 4: Run fixture tests locally** + + Run: + + ```bash + cargo test -p trusted-server-cli browser_fixture --target $(rustc -vV | sed -n 's/^host: //p') -- --nocapture + ``` + + Expected: pass when Chrome/Chromium exists; otherwise tests skip with explicit message. + +- [ ] **Step 5: Commit** + + ```bash + git add crates/trusted-server-cli/src/audit/browser.rs crates/trusted-server-cli/src/audit/ad_templates.rs + git commit -m "Add browser fixtures for ad-template verification" + ``` + +## Task 11: Update Documentation And Help Snapshots + +**Files:** + +- Modify: `docs/superpowers/specs/2026-06-26-server-side-ad-template-cli-design.md` if implementation decisions differ. +- Modify: `trusted-server.example.toml` only if command examples need harmless fictional config comments. +- Modify: `CLAUDE.md` only if verification commands or CLI command surface need to be documented. + +- [ ] **Step 1: Run CLI help manually** + + Run: + + ```bash + cargo run -p trusted-server-cli --target $(rustc -vV | sed -n 's/^host: //p') -- audit --help + cargo run -p trusted-server-cli --target $(rustc -vV | sed -n 's/^host: //p') -- audit ad-templates verify --help + cargo run -p trusted-server-cli --target $(rustc -vV | sed -n 's/^host: //p') -- config ad-templates --help + ``` + + Expected: nested audit commands are discoverable; hidden legacy `ts audit ` does not dominate help text. + +- [ ] **Step 2: Update docs if help text or behavior differs from spec** + + Keep examples using `https://www.example.com/` only. Do not mention real publisher sites. + +- [ ] **Step 3: Run docs format** + + Run: + + ```bash + cd docs && npm run format + ``` + + Expected: Prettier passes. + +- [ ] **Step 4: Commit** + + ```bash + git add docs trusted-server.example.toml CLAUDE.md + git commit -m "Document ad-template CLI verification" + ``` + + If no docs changed, skip the commit. + +## Task 12: Final Verification + +**Files:** + +- Verify all touched files. + +- [ ] **Step 1: Rust format** + + Run: + + ```bash + cargo fmt --all -- --check + ``` + + Expected: pass. + +- [ ] **Step 2: Host CLI tests** + + Run: + + ```bash + cargo test -p trusted-server-cli --target $(rustc -vV | sed -n 's/^host: //p') + ``` + + Expected: pass. Browser fixture tests either pass or explicitly skip when Chrome/Chromium is unavailable. + +- [ ] **Step 3: Workspace tests** + + Run: + + ```bash + cargo test --workspace + ``` + + Expected: pass. + +- [ ] **Step 4: Clippy** + + Run: + + ```bash + cargo clippy --workspace --all-targets --all-features -- -D warnings + ``` + + Expected: pass. + +- [ ] **Step 5: Wasm isolation proof** + + `trusted-server-adapter-fastly` does **not** depend on `trusted-server-cli`, so the + adapter build never compiles the CLI crate and cannot detect a CLI-crate dep leak. + The real proof is building the **CLI crate itself** for the wasm target (its modules + are `#[cfg(not(target_arch = "wasm32"))]`, so a wasm build must succeed with the + host-only deps compiled out). Note the workspace default target is already + `wasm32-wasip1`, so Steps 3–4 (`cargo test/clippy --workspace`) also build the CLI + crate for wasm — but make the isolation check explicit: + + ```bash + # Real CLI isolation proof: CLI crate must build for wasm with host deps excluded. + cargo build --package trusted-server-cli --target wasm32-wasip1 + # Adapter still built to confirm the production artifact is unaffected. + cargo build --package trusted-server-adapter-fastly --release --target wasm32-wasip1 + ``` + + Expected: both pass. If `chromiumoxide`/`tokio`/etc. leaked into a non-target-cfg + dependency table, the first command fails — that is the guard. + +- [ ] **Step 6: Docs format** + + Run: + + ```bash + cd docs && npm run format + ``` + + Expected: pass. + +- [ ] **Step 7: Inspect final diff** + + Run: + + ```bash + git status --short + git diff --stat origin/server-side-ad-templates-impl...HEAD + git log --oneline origin/server-side-ad-templates-impl..HEAD + ``` + + Expected: only intended CLI/core/doc files changed; no `.env`, operator `trusted-server.toml`, or generated browser artifacts included. + +## Risks And Watch Points + +- `chromiumoxide` must remain a host-only `trusted-server-cli` dependency. Any wasm build failure here means the dependency leaked. +- `ts audit ` compatibility must not swallow `ts audit ad-templates` as a URL. +- Runtime gate extraction (Task 1) only touches `should_run_server_side_ad_stack` + (the navigation gate). `/__ts/page-bids` is **intentionally NOT routed** through + `evaluate_ad_stack_gate` — its gate semantics differ (bot/prefetch skip the auction + but keep slots; no `is_navigation`/`is_get` gate). Its parity is preserved by + leaving it untouched, not by sharing the helper. Do not reroute page-bids. Keep + existing publisher and page-bids tests passing. +- The browser collector must not capture page HTML, cookies, storage, request bodies, or arbitrary DOM. Only collect configured-prefix DOM IDs and ad-related evidence. Never override `navigator.webdriver`. +- `runtime_ad_stack_expected = "unknown"` is normal for live consent state; do not over-model consent unless the collector can prove it. +- Browser fixture tests must not depend on real GPT/APS network calls. +- **Deferred in this implementation:** `/__ts/page-bids` SPA observation (spec §5.2 + "when available"). `PageBidsEvidence` exists as forward scaffolding but is not + collected (Task 8), surfaced in JSON (Task 4), or tested. Revisit if SPA route + verification is prioritized. +- Keep generation (`ts audit ad-templates generate`) out of this PR. diff --git a/docs/superpowers/plans/2026-08-10-1009-esi-validation-spike.md b/docs/superpowers/plans/2026-08-10-1009-esi-validation-spike.md new file mode 100644 index 000000000..5b9a455e7 --- /dev/null +++ b/docs/superpowers/plans/2026-08-10-1009-esi-validation-spike.md @@ -0,0 +1,967 @@ +# #1009 ESI Validation Spike + +> **HISTORICAL SPIKE — DO NOT IMPLEMENT.** This document records the investigation, +> including executable ESI tags, parser/subrequests, and a client-fill arm that were all +> removed. Every unchecked item below is historical, not remaining work. The accepted +> implementation keeps the public `esi` spelling but uses Fastly C2 plus exact byte-seam +> assembly. See +> [the merge-hardening design](../specs/2026-08-12-1009-esi-merge-hardening-design.md) and +> [implementation plan](./2026-08-12-1009-esi-merge-hardening.md). + +> **For agentic workers:** REQUIRED SUB-SKILL: Use superpowers:subagent-driven-development +> (recommended) or superpowers:executing-plans to implement this plan task-by-task. Steps +> use checkbox (`- [ ]`) syntax for tracking. + +**Goal:** Decide #1009 on evidence. Build a shared-template pipeline behind a flag, run +ESI and client-fill against it, and produce a decision record that either adopts ESI, +adopts client-fill, or rejects both — with the Fastly-only maintenance cost priced in. + +**Architecture:** +`origin → lol_html transform → fastly::cache::core → finalize headers → stream assembly`. + +Headers finalize **before** assembly, not after — streaming responses on this adapter +commit headers first and then pipe chunks, so nothing can be set once assembly starts. + +The transform emits **one unconditional marker at the body-close seam**. Not two: the +head seam is not a template hole, because `tsjs.adSlots` presence is request-gated +(Task 3 Step 2). The cached object is a shared template with no per-user bytes and no +request-dependent decisions. Assembly is either the `esi` crate (edge) or a client fetch +(browser), selected per request by the arm allocator so both are measured on one build. + +**Tech Stack:** Rust 2024, `wasm32-wasip1`, `fastly` 0.12.1 (`cache::core`, `http::purge`), +`esi` 0.7, `lol_html`, a real Fastly test service for cache behaviour. + +**Spec:** `docs/superpowers/specs/2026-08-08-esi-cacheable-root-validation-design.md` — +read the 2026-08-10 correction at the top and +[§6.6](../specs/2026-08-08-esi-cacheable-root-validation-design.md#66-the-esi-pipeline-corrected) +before writing any code. + +**Control:** [the Stage 0 plan](./2026-08-08-1009-measurement-and-stage-0.md). Its +instrumentation and its bypass flag are prerequisites — this plan compares against them +and does not duplicate them. + +--- + +## Why this plan exists + +An earlier revision of the spec concluded ESI was structurally impossible. It was wrong: +`fastly::cache::core` provides the cache boundary natively, and purge runs inside Compute. +That correction reopens #1009 as an empirical question, and this plan is how it gets +answered. + +**What is genuinely uncertain**, and what each arm is for: + +1. Does a shared template plus per-request assembly beat today's inline path enough to + matter? +2. Does **edge** assembly (ESI) beat **client** assembly (a fetch) by enough to justify a + Fastly-only rendering path that must be maintained alongside the portable one? +3. Can per-user leakage be excluded across cold MISS, warm HIT, stale revalidation, + transform failure, and fragment failure? + +Question 3 is a gate, not a metric. A win on 1 and 2 with a failure on 3 is a rejection. + +## Three caches, never conflated + +The original error came from treating these as one thing. Every task below names which it +means. + +| # | Cache | Contents | Status | +| --- | --------------------------------- | ----------------------------- | ----------------------------------- | +| C1 | Origin read-through | raw origin bytes | Exists. Stage 0 turns it back on. | +| C2 | Shared transformed template | post-`lol_html`, pre-assembly | **New.** What this plan builds. | +| C3 | Assembled-response delivery cache | final per-user output | **Must never exist.** Not proposed. | + +If a task appears to require C3, stop — that is the leakage failure mode, not a design +option. + +## Arms + +Five, but only four are treatable as equivalent. + +| Arm | Root | Bids | Notes | +| ------- | ----------------------- | ---------------- | ---------------------------------------------------------- | +| **A0** | inline, C1 bypassed | inline `` | Today. The baseline. | +| **A1** | inline, C1 on | inline `` | Stage 0. Isolates the bypass from the template change. | +| **A2** | shared template from C2 | client fetch | Portable. Works on all four adapters. | +| **A3** | shared template from C2 | ESI at the edge | Fastly-only. The thing #1009 proposed. | +| **REF** | origin direct, TS off | publisher's own | **Reference, not an arm.** Different work, not comparable. | + +A0→A1 measures the bypass. A1→A2 measures the template split. A2→A3 measures edge versus +client assembly — **that difference is the entire case for ESI**, and it is the number +this plan exists to produce. + +**Do not compare A2 and A3 on root TTFB.** They serve the same C2 template, so their root +timings should be near-identical by construction; a null result there proves nothing. +ESI's claimed advantage is that bids arrive without a client round-trip, so measure: +**bids-ready time**, **`adInit` fire time**, and **first TS-attributed creative paint**. +Root TTFB stays as a guard that the template path did not regress, not as the comparison. + +REF is included because #1009 anchors on it, and excluded from pass/fail because TS-off +does no auction and no injection. Comparing against it measures the feature's existence, +not its implementation. + +--- + +## Task order and dependencies + +``` +Task 1 (esi compiles) ── DONE, PASS ──┐ + ├──> Task 3 (C2 cache) ─┬──> Task 4 (A2 client-fill) +Stage 0 plan (flag + instrumentation) ┘ ├──> Task 5 (A3 ESI) + └──> Task 6 (safety gates) + │ + Task 2 (real service) ─────────────────────────────┴──> Task 7 (decision) +``` + +**Task 2 is not a blocker on Tasks 3–6.** Everything those tasks need is exercisable under +Viceroy 0.17 — verified, see Task 2. The real service is required only for the +measurements Task 7 decides on, so provision it once there is something worth measuring. + +Task 6 runs against every arm, not once at the end. + +--- + +## Task 1: Confirm `esi` 0.7 builds on this toolchain + +Cheapest possible falsification. Do this before anything else. + +**Files:** `crates/trusted-server-adapter-fastly/Cargo.toml` + +- [ ] **Step 1: Add the dependency** + +```bash +cargo add esi@0.7 --package trusted-server-adapter-fastly +``` + +It belongs in the **Fastly adapter**, never in `trusted-server-core` — the crate is +hard-bound to `fastly::{Request, Response, Backend}` and core must stay portable. + +- [ ] **Step 2: Check it compiles for the real target** + +```bash +cargo check-fastly +``` + +Expected: clean. The crate declares edition 2021 with no `rust-version`, and pulls recent +`rand` and `nom`, so this is a genuine question on Rust 1.95.0 / `wasm32-wasip1`. + +- [ ] **Step 3: Check no shared dependency was forced to move** + +```bash +git diff --stat Cargo.lock +cargo check --manifest-path crates/trusted-server-integration-tests/Cargo.toml --tests \ + --target "$(rustc -vV | sed -n 's/^host: //p')" +``` + +**Correction, verified 2026-08-10:** an earlier revision of this step warned about a +desync between the root `Cargo.lock` and `crates/trusted-server-integration-tests/Cargo.lock`. +**That second lockfile does not exist** — the crate is a workspace member (root +`Cargo.toml:10`) and shares the root lockfile. The hazard cannot arise in that form. + +What does matter is whether adding `esi` forces an **existing** shared dependency to a new +version, since `regex`, `bytes`, and `log` are used across the workspace. Adding a new +major that coexists is harmless; moving an existing one is not. If one moves, fix with a +targeted `cargo update -p --precise ` — **never a full update**. + +**Already run and recorded** in [the findings](./2026-08-08-1009-measurement-findings.md): +no existing shared dependency moved. + +- [ ] **Step 4: Record and commit, or stop** + +**Task 1 is complete — verdict PASS, recorded 2026-08-10.** `esi` 0.7.1 compiles clean on +Rust 1.95.0 / `wasm32-wasip1`, all six clippy targets pass, and no existing shared +dependency moved. See [the findings](./2026-08-08-1009-measurement-findings.md). + +Had Step 2 failed, this plan would have stopped here with #1009 answered "not on this +toolchain." It did not. + +```bash +git add crates/trusted-server-adapter-fastly/Cargo.toml Cargo.lock +git commit -m "Add the esi crate to the Fastly adapter for the #1009 validation spike" +``` + +--- + +## Task 2: Local validation first, real service only for what needs it + +**Verified 2026-08-10 under Viceroy 0.17: the entire Core Cache surface this spike uses +works locally.** A probe exercised `cache::core::insert`, `lookup`, `finish`, `to_stream`, +and — the shape Task 3 Step 4 actually specifies — `Transaction::lookup`, +`must_insert_or_update`, `insert(...).surrogate_keys(...).execute_and_stream_back()`, and +hit-after-insert semantics. All passed. Recorded in +[the findings](./2026-08-08-1009-measurement-findings.md). + +That reorders this plan. An earlier revision made provisioning a Fastly service Task 2 and +a blocker on everything after it. It is not a blocker: **almost all of the correctness and +safety work is local**, and only the numbers and the cache topology need real +infrastructure. + +| Work | Where | +| ------------------------------------------------------------ | ------------ | +| C2 insert / lookup / transaction logic (Task 3) | **Local** | +| The `lol_html` transform and template byte-identity (Task 3) | **Local** | +| ESI assembly — the crate is pure Rust over `BufRead`/`Write` | **Local** | +| DCA off, dispatcher allowlist, injection refusal (Task 5) | **Local** | +| Fragment-failure degradation (Task 5) | **Local** | +| Header-finalization ordering, no-C3 assertions (Task 6) | **Local** | +| Cross-user leakage / request-neutrality gates (Task 6) | **Local** | +| Shielding behaviour | Real service | +| POP-level cache tiering (`x-cache`, `hit-state`, `age`) | Real service | +| Request collapsing under genuine concurrency | Real service | +| Stale revalidation timing at the edge | Real service | +| **Every performance number in Task 7's decision rule** | Real service | + +**So: build and prove correctness locally through Tasks 3, 5, and 6 before provisioning +anything.** If the design is wrong or leaks, that surfaces locally for free, and the +service is only needed once there is something worth measuring. + +Two caveats on the local scope. Viceroy is a single instance, so a passing `Transaction` +test proves the API works, **not** that collapsing behaves correctly under load. And local +timings are meaningless for the decision — do not let a fast local run substitute for +Task 7 evidence. + +### When the real service is needed + +- [ ] **Step 1: Provision it — after local correctness passes, not before** + +Separate from production. Confirm and record: whether the publisher backend is +**shielded**, and whether any Delivery service fronts the Compute service. Both change +what the numbers mean. + +```bash +fastly service list +fastly backend list --service-id --version latest +``` + +The shielding answer also settles an open question from the Stage 0 findings: #1009's +off-TS win came from a shield HIT, so whether the test service has one determines whether +its numbers transfer to production at all. + +- [ ] **Step 2: Extend the harness for lineage, not just correlation** + +The existing tester-cookie A/B has no way to join server timings to browser timings. A +root-only request ID is not enough either: under A3 the auction happens in a **fragment +subrequest**, so a root ID never reaches the auction telemetry. + +Propagate a **lineage ID plus the experiment arm** through the whole chain: + +``` +root request → C2 lookup → fragment subrequest → auction telemetry → browser render event +``` + +Generated at TS entry, forwarded into the fragment request, attached to the +`auction_events_raw` row, echoed as `x-ts-request-id`, and exposed to the browser harness +so render events carry it. Every timing log line includes both fields. + +Without this the experiment cannot join hold time, origin time, auction telemetry, browser +TTFB, and render outcome for the same pageview. **That is the difference between an +experiment and a pile of numbers.** + +- [ ] **Step 3: Capture C1 and C2 status separately** + +`x-cache`, `hit-state`, and `age` describe the **HTTP read-through cache (C1)**. They say +nothing about the **transformed-template cache (C2)**, which is a `cache::core` object +with no HTTP semantics. Recording only the former and calling it "cache status" would +attribute C2 hits and misses to the wrong tier. + +Emit both: the C1 headers as-is, plus an explicit `x-ts-c2` field carrying HIT / MISS / +STALE / BYPASS from the transaction outcome. Record the serving POP alongside. A median +that mixes cold-MISS and warm-HIT requests is meaningless, and arms cannot be compared +unless the mix is known — per tier. + +- [ ] **Step 4: Build a request-scoped arm allocator** + +`AssemblyMode` as specified in Task 3 is a **global** setting, but the sample plan below +requires randomized, non-sequential allocation. A global flip gives sequential blocks +instead, which confounds arm with time of day, cache warmth, and traffic mix. + +Allocate per request: hash the lineage ID into buckets, or key off the tester cookie. +The global setting stays as the kill switch and as the way to force a single arm; the +allocator is what the experiment actually uses. Record the assigned arm on every log line +and every telemetry row. + +- [ ] **Step 5: Define the sample plan before collecting anything** + +Write all of this into the findings document **before** the first measurement, and treat +it as fixed: + +| Element | What to state | +| -------------------- | -------------------------------------------------------------------------------------------------------------------------- | +| Allocation | Requests per arm per route, and how arms are assigned | +| Randomization | Randomized or blocked by route and cache state — not sequential runs | +| Pilot variance | A small pilot to estimate variance, before sizing the real run | +| MDE and power | The smallest difference worth detecting, and the N that detects it | +| CI method | Which interval, computed how | +| Warmup and carryover | How cold MISS is forced, how warm HIT is confirmed, and how one arm's cache state is prevented from contaminating the next | + +Rationale: this whole effort exists because #1009 drew a causal conclusion from N=4 that +did not survive contact with the code. Repeating that with more arms and no power +calculation would be worse, not better — it would look rigorous while being equally +unfalsifiable. + +--- + +## Task 3: Build C2 — the shared transformed-template cache + +The core of the spike. Behind a flag, default off. + +**Files:** + +- `crates/trusted-server-core/src/publisher.rs` — emit **one** unconditional marker at the body-close seam (see Step 2; the head seam is not a template hole) +- `crates/trusted-server-core/src/settings.rs` — the mode flag +- `crates/trusted-server-adapter-fastly/src/` — the `cache::core` read/write + +- [ ] **Step 1: Add the assembly-mode setting** + +```rust +/// How per-user ad state reaches the page. +/// +/// `Inline` is today's behaviour: bids injected before ``, root uncacheable. +/// `ClientFill` and `Esi` both serve a shared template from the transformed-template +/// cache and fill the holes afterwards. Spike-only — remove with the spike. +#[derive(Debug, Clone, Copy, Default, PartialEq, Eq, Deserialize, Serialize)] +#[serde(rename_all = "snake_case")] +pub enum AssemblyMode { + #[default] + Inline, + ClientFill, + Esi, +} +``` + +Default `Inline` so the flag is a no-op until set. Note the hazards the Stage 0 plan +already documents: `Settings` carries `#[serde(deny_unknown_fields)]`, `ts config push` is +typed, and `Publisher` has a hand-written `Default` plus eight exhaustive test literals +and a live doctest. + +- [ ] **Step 2: Make the template strictly request-neutral** + +**The obvious design is wrong and would leak.** An earlier draft kept `tsjs.adSlots` in +the shared template on the grounds that it is per-URL. Its _content_ is per-URL; its +_presence_ is not. It is gated on `should_run_ad_stack` (`publisher.rs:2920-2927`), which +is `is_get && is_navigation && !is_prefetch && !is_bot && has_matched_slots && +consent_allows_auction && auction_enabled`. + +So the first request to fill C2 would freeze **its own** consent decision, bot +classification, prefetch status, and kill-switch state into an object every later visitor +reads. A consent-denied first fill serves a no-ads template to consenting users; a +consenting first fill serves ad markup to a user who refused. + +**Rule: the template contains an unconditional inert placeholder and nothing else.** + +| Element | Where it lives | +| ------------------------- | -------------------------------------------------- | +| tsjs bundle script tag | Template — content-hashed, genuinely per-URL | +| URL rewrites | Template — per-host, in the cache key | +| `tsjs.adSlots` | **Fragment** — its presence is request-dependent | +| `tsjs.bids` | **Fragment** | +| GPT diagnostics bootstrap | **Fragment** — gated on a per-request cookie/query | + +Emit **one** unconditional marker at the body-close seam, identical on every request that +reaches the transform. Under `Esi` it is an executable ESI include tag; under +`ClientFill` it is nothing at all, with the client fetching unprompted. + +- [ ] **Step 3: Bypass C2 for anything that must not be shared** + +`cache::core` is not an HTTP cache — it will happily store whatever you hand it. Nothing +rejects private or authenticated responses for you. Refuse to insert when **any** holds: + +- The origin response carries `Set-Cookie`. +- The origin response is `private`, `no-store`, or `no-cache`. +- The request carried `Authorization`. +- The response is not 200 with an HTML content type. +- DataDome's request filter replaced the document. + +Audit every request-dependent rewrite before declaring the template neutral — the +integration head-inserts and the GPT-diagnostics bootstrap are both request-scoped and +must not reach C2. + +**Assert it, do not assume it.** A unit test over the transform output must fail on any +of: a bid value, an EC ID, a consent string, a geo value, a diagnostics bootstrap, or a +`Set-Cookie`. Then a second test must assert the template is **byte-identical** for two +requests differing in consent, bot classification, and prefetch status. That second test +is the one that catches this class of bug; the first would have passed on the broken +design. + +- [ ] **Step 4: Write and read C2 — with the real API** + +The builder is move-based and the insert and read handles are different objects. Naïve +code does not compile: + +```rust +// WRONG — surrogate_keys consumes the builder and returns it; this discards the +// return value and then uses a moved binding. And execute() gives a WRITE stream, +// so there is nothing to read back from it. +let mut insert = cache::core::insert(key, ttl); +insert.surrogate_keys(["ts-template"]); +let body = insert.execute()?; +``` + +Correct shape, using a transaction so a cold cache under load transforms once: + +```rust +use fastly::cache::core::{Transaction, CacheKey}; + +let tx = Transaction::lookup(CacheKey::from(key_bytes)).execute()?; + +// Order matters: a STALE entry sets BOTH found() and must_insert_or_update(). +// Testing found() first would serve the stale bytes and silently never fulfil the +// update obligation, leaving every concurrent waiter blocked until timeout. +let template: Body = if tx.must_insert_or_update() { + // Fetch and prepare BEFORE consuming `tx`. After `insert()` the transaction is + // gone and `cancel_insert_or_update()` is unreachable, so anything that can fail + // and does not need the writer belongs here. + let origin = match fetch_and_prepare_origin() { + Ok(origin) => origin, + Err(e) => { + tx.cancel_insert_or_update()?; // releases the obligation to a waiter + return fallback_uncached(e); + } + }; + + // `Transaction::insert(self)` consumes `tx` from this line on. + let (mut writer, found) = tx + .insert(template_ttl) + .surrogate_keys(["ts-template", &url_surrogate_key]) // chained, not discarded + .user_metadata(metadata_envelope) + .execute_and_stream_back()?; + + match stream_lol_html_output(origin, &mut writer) { + Ok(()) => { + writer.finish()?; // REQUIRED, and consumes `writer` + found.to_stream()? // fallible; there is no `to_body()` + } + Err(e) => { + // Also consumes `writer`, marking an unsuccessful end so no partial + // template is served. (A `StreamingBody` dropped without `finish()` is + // aborted anyway, but say it explicitly.) + writer.abandon()?; + return fallback_uncached(e); + } + } +} else if let Some(found) = tx.found() { + found.to_stream()? // C2 HIT — skip origin fetch and transform +} else { + unreachable!("a transaction is either obliged to insert or has found an item") +}; +``` + +Two ownership rules this shape exists to respect, both of which an earlier draft broke: +`Transaction::insert(self)` **consumes** the transaction, so a helper taking `&tx` cannot +call it and `cancel_insert_or_update` is unreachable afterwards; and `finish`/`abandon` +each consume the writer, so neither can be referenced from an arm that did not bind it. + +**Decide the stale policy explicitly.** `Found::is_stale()` and `is_usable()` exist, and +`stale_while_revalidate` can be set at insert. Serving stale while revalidating is a real +option — but it is a state machine, and `cache::core` implements none of it for you. The +spike should start by treating stale as a miss and only add stale-serve if the numbers +justify it. + +**`cache::core` carries no HTTP semantics.** Status, headers, content encoding, and +revalidation are all yours. Serialize what you need into `user_metadata` — at minimum the +content encoding, the transform schema version, and the origin `Vary` values the key was +built from — and decide explicitly whether the stored template is compressed. + +**Cache key must include**, beyond the origin's declared `Vary` (`rsc`, +`next-router-state-tree`, `next-router-prefetch`, `next-router-segment-prefetch`, +`Accept-Encoding` — measured, see the Stage 0 findings): + +- The full URL, explicitly. Do not rely on an ambient request key. +- **The assembly mode.** A2 and A3 emit different template bytes and would otherwise + poison each other's entries. +- **A template schema version**, bumped whenever the transform changes, so a deploy does + not read yesterday's shape. +- Request host and scheme, the enabled-integration set, and the tsjs content hash. + +Per-user signals must never appear in the key. If a signal cannot be excluded from the +template, it does not belong in C2 at all. + +### Design decided 2026-08-10: `cache::core`. Do not revisit read-through. + +An earlier revision left this open between `cache::core` and read-through caching with +`after_send` + `set_body_transform`. Investigated and verified against the pinned SDK and +Viceroy 0.17 source. **Read-through is not viable here** — not on preference, on three +hard blockers: + +1. **Viceroy stubs the entire HTTP Cache ABI**, and the SDK converts that into a _send + error_ rather than a fallback. `is_request_cacheable` returns + `Err(NotAvailable("HTTP Cache API primitives"))` + (`viceroy-lib-0.17.0/src/wiggle_abi/http_cache.rs:108-114`; 26 such stubs in that + file), which makes `must_use_host_caching()` true, which with a send hook set returns + `Err(SendErrorCause::HttpCacheApiUnsupported)` + (`fastly-0.12.1/src/http/request.rs:626-632`). **Setting `after_send` makes every + publisher origin fetch fail** under `fastly compute serve`, `cargo test-fastly`, and + the parity suite. The whole local loop dies. +2. **`with_cache_bypass` makes the hook silently dead.** `get_caching_mode` checks + `cache_override.is_pass()` **first** (`request.rs:612-615`) and returns host caching, so + `after_send` is never invoked and no error is raised. On exactly the requests in scope, + today, the hook would do nothing quietly. +3. **The closure bounds are incompatible with this codebase.** `with_after_send` requires + `Fn + Send + Sync + 'static` (`request.rs:545-550`). Everything the rewriter needs is + `!Send` by construction — `edgezero_core::body::Body` wraps a `LocalBoxStream` + deliberately, which is why the platform layer is `#[async_trait(?Send)]` throughout. + And `set_body_transform` is synchronous, so it could never await the auction collect. + +Read-through's appeal was real — `CandidateResponse::apply_and_stream_back` is +`execute_and_stream_back` with HTTP semantics attached, and TTL/SWR/vary/surrogate keys +derived from origin headers for free. It is simply unreachable from here. + +**Also settled: core cannot reach it at all.** `PlatformHttpRequest` +(`platform/http.rs:16-37`) is a plain data struct with no callback slot, and carrying one +would name `fastly::http::CandidateResponse` in portable core, breaking the other three +adapters. + +### Follow the existing null-object pattern + +`cache::core` fits the shape the repo already uses four times for a Fastly-only capability +behind a portable trait: `UnavailableHttpClient` (`platform/http.rs:216-243`), +`UnavailableKvStore` (`platform/kv.rs:14-17`), and the `RuntimeServices.kv_store` +field/accessor/builder (`platform/types.rs:170,222,269,330`). Add +`PlatformTemplateCache` the same way, and follow +`crates/trusted-server-adapter-fastly/src/ec_kv.rs` — 140 lines, the repo's only real +edge-storage read/write — rather than inventing a shape. + +**Return `EdgeBody`, not `Vec`.** `EdgeBody::Stream` exists, +`fastly_body_to_edge_stream` (`adapter-fastly/src/platform.rs:503`) already converts, and +`PublisherResponse::Buffered` tolerates a live stream (`publisher.rs:1019-1022`). + +### Exact insertion point + +**Immediately before `let mut platform_request = PlatformHttpRequest::new(...)`** — the +last line before `req` is consumed, and a few lines before the origin send. Everything +needed is in scope there: `settings`, `services`, the final URI and Host, `backend_name`, +`request_path`, `matched_slots`, `should_run_ad_stack`, `request_had_authorization`, +`request_host`, `request_scheme`. + +**One required move:** `assembly_mode` is currently computed _after_ the send, for the +logging call site. It depends only on `settings`, so hoist it above the insertion point. + +**Tee-ing is not needed.** With any post-processor registered — and the Next.js +integration always registers one — `HtmlWithPostProcessing` emits nothing until the final +chunk and then returns the whole transformed document as one contiguous buffer +(`html_processor.rs:92-97,148`). Two `write_all` calls on the same slice; no tee +abstraction, no extra copy. Still use `execute_and_stream_back`, but for transaction +correctness and request collapsing rather than for memory. On a hit the processor is never +built at all. + +- [ ] **Step 4b: close the risks the design investigation surfaced** + +Four, all specific to this codebase rather than to `cache::core` in general. + +**`Vary` is in the key list but nothing consumes it.** `c2_bypass_reason` checks +`Set-Cookie`, `Cache-Control`, `Authorization`, status and content type — **not `Vary`**. +Viceroy supports `WriteOptions.vary_rule`, so the mechanism exists; the gate has to use +it. Until then the key is missing a signal the origin explicitly declares, and Step A's +verdict is a `PROVISIONAL PASS`, not a release gate. + +**Resolved — `VarySpec`, commit `b688d667`.** Building the key exposed a problem this +plan states but does not solve: the key must cover everything the origin varies on, but +**a lookup happens before the fetch**, so on a cold key the origin's `Vary` is not yet +known. Three ways out — configure the list; two-phase lookup against a URL-keyed record +holding the last-seen `Vary`; or store the list alongside and re-key on mismatch. The +latter two are correct and double the lookups on every request. + +Configured is taken, **as a spike-grade choice rather than a production one**: Step A +already measured the origin's actual `Vary`, and a 60s TTL bounds drift to a minute +rather than indefinitely. + +The drift is guarded rather than merely accepted. `VarySpec::uncovered_by` runs _after_ +the origin responds, when its `Vary` is finally known, and names which headers the +configured spec missed. A template built under a key that did not cover something the +origin varies on **must not be stored** — a request differing only in that header would +read it. Naming the specific headers makes a stale config identifiable instead of +producing a generic refusal. + +Two decisions worth their tests. An absent header and a present-but-empty one key the +same, because the origin sees no difference between them. And `Vary: *` is not reported +as a named gap — it means uncacheable, which the eligibility gate handles, and reporting +it would produce a nonsense instruction to configure a header called `*`. + +Still open: wiring `uncovered_by` into `c2_bypass_reason` as a bypass reason, which +happens with the store call site. + +**Store bytes plus a metadata envelope; rebuild every header on a hit.** The publisher +path forces `private, no-store` and strips `ETag`/`Last-Modified`/CDN headers _after_ the +send. Replaying stored origin headers would fight that. Store only the transformed body +and a small `user_metadata` envelope — content encoding, content type, schema version, +tsjs hash — and construct every response header from scratch on a hit. Then no origin +header is ever replayed and the `Set-Cookie` privacy net is trivially safe. +`get_user_metadata` is implemented in Viceroy. + +**Content-Encoding belongs in the key.** The streaming pipeline pairs input encoding to +the same output encoding, so the transformed bytes inherit whatever the origin negotiated +from the client's `Accept-Encoding` — still gzip, deflate, br or identity after +`restrict_accept_encoding` narrows it. Either key on the negotiated encoding or normalize +to identity in the cache and re-encode on read. Getting this wrong serves brotli bytes to +a client that asked for gzip. + +**Host and scheme belong in the key.** The post-processed output is host-dependent by +construction: `request_host` and `request_scheme` reach `IntegrationHtmlContext`. + +- [ ] **Step 4c: file the wasted-dispatch follow-up** + +The auction is dispatched _before_ the insertion point. Under `Esi` and `ClientFill` the +root injects nothing, so that dispatch is already pure waste on this branch — and on a C2 +hit it is waste that must be cleaned up via `emit_abandoned_auction` or it leaks +telemetry. + +Keeping the lookup at the insertion point above is right for the spike: minimal diff, and +lookup latency overlaps the in-flight auction. Moving it earlier would eliminate the +wasted dispatch but serialize the lookup ahead of dispatch. **File it; do not fix it +here.** Suppressing root-level dispatch under the shared modes is Task 4's job, where it +also has to be reconciled with the exactly-one-auction gate. + +- [ ] **Step 5: Unit tests, then the target suite** + +```bash +cargo test -p trusted-server-core --target aarch64-apple-darwin assembly_mode +cargo test-fastly && cargo test-axum && cargo test-cloudflare && cargo test-spin +cargo fmt --all -- --check && cargo clippy-fastly +``` + +`ClientFill` must work on all four adapters. `Esi` is Fastly-only and must not break the +others' compilation. + +- [x] **Step 6: the call site — DONE.** `2db10639` (store), `2a2e6c6a` (lookup). + +The cache now engages end to end: a second request for the same URL is served without +touching the origin, and is byte-identical to what was stored. Verified by mutation — +disabling the lookup fails the hit test, so the hit is the cache answering rather than +the fixture answering twice. + +**Wiring the lookup corrected the key.** It carried the content encoding the _origin_ +chose, which does not exist at lookup time. That meant storing under `br` and looking up +under `gzip, br` — a cache that never hits. The field is now the `Accept-Encoding` sent +to the origin. Sound because negotiation is a function of what the origin was offered, +so identical offers yield identical choices; the chosen encoding stays in the metadata +and is what the served response declares. + +That made every key field request-derived, so **the key is built before the fetch** and +the response gate only authorizes storing it. A key that needed the response could only +ever authorize a store, never satisfy a read. + +**The lookup re-checks the request-derived disqualifications, and only those.** The +store gate is response-derived and cannot re-run, but need not: anything in the cache +passed it on the way in. What must re-run are properties of the _reader_ rather than of +the bytes — an authenticated request must not be served a shared template even when that +template is perfectly cacheable. + +**Shared modes take the buffered finalizer.** Storing needs every transformed byte and +streaming does not collect them. The branch keys on the store authorization rather than +on the assembly mode, so `Inline` never reaches it and the spike cannot regress the +shipped path by construction. A C2 _miss_ therefore buffers — the right trade, since a +miss is already paying an origin fetch and a full transform, and what the spike measures +is the hit, where there is no origin fetch to stream from at all. + +Every response header on a hit is constructed, never replayed, so no origin header can +reach a second visitor through the cache. + +The publisher tests use an in-memory cache double, so they prove the wiring rather than +the backing. The join they leave untested is the one `app.rs` makes: the publisher +reaches the cache as a `dyn PlatformTemplateCache` behind `RuntimeServices`, never as +the concrete type the Fastly tests exercise. That join is now executed under Viceroy +against the real Core Cache rather than only type-checked. + +**What this does not establish.** `ClientFill` and `Esi` still render a template with a +hole and nothing filling it. Task 4 and Task 5 remain the blockers on anything +deployable — a cache that works is necessary, not sufficient. + +--- + +## Task 4: Arm A2 — client-fill + +Mostly already specified. See +[the spec's Appendix B](../specs/2026-08-08-esi-cacheable-root-validation-design.md#appendix-b--stage-1-plumbing-condensed) +for the client plumbing, the two-condition join gate, and the server contract; and +[§5](../specs/2026-08-08-esi-cacheable-root-validation-design.md#5-the-trap-in-the-deferred-work--read-this-before-scheduling-stages-12) +for the silent-empty-bids trap, which applies in full. + +- [ ] **Step 1: Hoist the closure-trapped client state** — `pageBidsEndpoint`, + `requestPageBids`, and the `inflight`/`currentPath`/`lastAppliedPath` state, per + Appendix B. Do **not** route the initial load through `onNavigate`. +- [ ] **Step 2: Make `installScheduleInitialAdInit` a hydration-ready AND bids-settled + join**, with a bounded timeout that fires `adInit` untargeted rather than stranding + the slot. Derive the timeout from measured fetch latency, not a constant. +- [ ] **Step 3: Suppress the navigation-path dispatch** so exactly one auction runs per + pageview. Add a new `AuctionSource` for initial loads **plus the mechanism that + delivers it** — a header behind the same-origin gate, not a query parameter. +- [ ] **Step 4: Relocate terminal telemetry.** Navigation `Completed` is emitted only from + the collect functions; the `ts-debug` dump rides the same string. Both move. +- [ ] **Step 5: Verify exactly one auction per pageview** in `auction_events_raw`. Two is + a doubling of SSP spend and an immediate fail. + +--- + +## Task 5: Arm A3 — ESI at the edge + +- [x] **Step 0: the mechanism works — DONE.** `9539061e`, hardened in `0597f54e`. + +Verified under Viceroy with the real `esi` 0.7 crate rather than argued from docs: a +template carrying the `` seam's own ESI include tag comes back with the fragment +spliced in its place and no unresolved tag left. + +**The async/sync obstacle is dissolved, not worked around.** `esi`'s fragment dispatcher +is synchronous and this codebase's fragment producer is `async`; calling one from the +other means a nested executor, which panics. +`PendingFragmentContent::CompletedRequest` lets the dispatcher hand back an +already-built response, so the caller resolves the fragment in the normal async flow and +the dispatcher performs **no I/O at all** — no subrequest, no backend, no self-call, +nothing for Viceroy to stub. That also removes the need for a self-referencing backend +this plan would otherwise have required. + +**Step 2's instruction was right, and reading the crate showed why.** +`CacheConfig::is_includes_cacheable` defaults to **`true`**. A fragment carries one +visitor's bids, so the default caches per-user data and serves it to the next visitor — +silently, on a hit. `includes_force_ttl` is worse where set: it caches everything, +ignoring `private`, `no-store` and `Set-Cookie` alike. Both now stated explicitly, along +with `default_dca`/`inherit_parent_dca` (fragment bytes are data, never re-parsed as +ESI), `max_include_depth = 1`, and rendered caching / `edge_control` off because the +publisher path owns those headers. + +Nine tests. Four assert the configuration; the rest assert behaviour, including that a +fragment containing its own nested ESI include is spliced as text rather than dispatched, so +auction data cannot drive fragment requests. + +**What remains is the call site**, below. Emitting the include and resolving it are both +proven; connecting them is not done. + +- [ ] **Step 1: Wire `process_stream`, not the wrappers** + +`process_response` and `process_response_streaming` consume `self` _and_ send the response +themselves, which takes ownership away from the finalize / `ec_finalize` / apply-effects +ordering. `process_stream(&mut self, src: impl BufRead, out: &mut impl Write, …)` keeps it. + +Source is the C2 body. Sink is the client response body. + +**The ordering an earlier draft described is impossible.** It said EC cookie, geo, and the +privacy net run _after_ assembly. They cannot: streaming responses on this adapter +**commit headers first and then pipe chunks** +(`adapter-fastly/src/main.rs`, `send_edgezero_response`). Once ESI starts writing, no +header can change. + +The correct invariant: + +> **Finalize every header before a single body byte is written** — EC `Set-Cookie`, geo +> suppression, and an unconditional `Cache-Control: private, no-store` — **then** stream +> the assembly with no further header mutation. + +That means `private, no-store` is set unconditionally up front rather than derived from +what the assembly turns out to contain. Deriving it after the fact is not available, and +assuming it was is how a per-user response ends up shared-cacheable. + +- [ ] **Step 2: Disable DCA explicitly and allowlist the dispatcher** + +```rust +let config = esi::Configuration::default() + .with_escaped(false) + .with_default_dca(esi::DcaMode::None) // call the setter; do not rely on the default + .with_inherit_parent_dca(false); +``` + +Comments are not configuration. An earlier draft said DCA "stays at its default" — on a +pre-1.0 crate whose default could move in a patch release, and where this setting fails +**open**, that is not good enough. Call the setters. + +Also disable **fragment caching** explicitly, or mark the include `no-store="on"`. A +cached auction fragment is a per-user object in a shared cache — the C3 failure mode by +another route. + +The dispatcher must be **exact-path allowlisted**: a fragment URL that is not the bids +endpoint is refused, not fetched. The built-in dispatcher builds a dynamic backend per URL +host and panics on a hostless URL — never use it. + +Rationale in the spec's §2: bid payloads carry partner-controlled creative markup, so a +recursive parse would let an SSP make the edge fetch an arbitrary URL. **Add a unit test +that feeds a partner-controlled ESI include targeting `http://attacker.example/` through +a creative payload and asserts no fetch is attempted.** + +- [ ] **Step 3: The fragment must be a script, not the JSON endpoint** + +**`/_ts/page-bids` cannot be the ESI target.** It returns +`serde_json::json!({"slots":…, "bids":…})` (`publisher.rs:3987`), and ESI splices fragment +bytes in literally — the page would contain raw JSON where an executable script belongs. +Nothing would call `scheduleInitialAdInit`. + +Add a **dedicated fragment endpoint** returning the executable script — the same shape +`build_bids_script` produces today, plus the `adSlots` assignment that moved out of the +template in Task 3 Step 2. Either that, or use the `esi` crate's fragment-response +processor to wrap the JSON; the dedicated endpoint is simpler and easier to assert on. + +Three more things the naïve marker gets wrong: + +- **The same-origin gate will reject it.** `page_bids_request_allowed` + (`publisher.rs:3644`) requires `Sec-Fetch-Site: same-origin` or the `X-TSJS-Page-Bids` + header. An internal ESI subrequest carries neither. Give the fragment endpoint an + internal contract and a fixed backend rather than weakening that gate — it exists to + stop third parties burning SSP quota. +- **Parent context does not propagate.** EC identity, consent state, client IP, geo, User + Agent, and the correlation ID all live on the parent request. Forward an **explicitly + approved allowlist** of them into the fragment request. Forwarding everything is how a + fragment ends up more privileged than the parent. +- **Root dispatch must be suppressed.** The navigation path already dispatches an + auction. If A3 does not suppress it, every pageview runs two — doubling SSP and APS + spend. This applies to **A2 and A3 alike**. + +- [ ] **Step 4: Validate the whole URL, not the path** + +An exact-path allowlist alone permits `https://attacker.example/_ts/page-bids`. Validate +**scheme, authority, method, path, and query** — or better, ignore the marker's URL +entirely and dispatch to a fixed internal backend, treating the ESI include as a signal +rather than an address. + +Add a test that feeds an ESI include targeting +`https://attacker.example/_ts/page-bids` through a creative payload and asserts no +outbound fetch is attempted. + +- [ ] **Step 5: Deterministic synthetic fragment first** + +Before wiring the real auction, point the include at a fixed-content endpoint. This +separates "does the pipeline assemble correctly" from "does the auction behave," and the +two fail very differently. Only once assembly is proven does the fragment become the real +one. + +- [ ] **Step 6: Handle the flush hazard** + +`esi` flushes its output writer after each parse batch. Fastly's `StreamingBody` is a +`BufWriter`, so anything between esi and it must propagate `flush()` or nothing leaves the +Wasm heap. + +- [ ] **Step 7: Fragment failure must degrade, not break** + +Assert that a fragment timeout or non-2xx yields a page with empty bids rather than a 5xx +or a truncated document. Note the crate's non-obvious semantics: `alt` is attempted before +`onerror="continue"`, and `` runs **all** attempts and concatenates every +non-failed output — it is not first-success-wins. + +--- + +## Task 6: Safety gates — run against every arm + +Not a phase. Every one of these is a hard fail, independent of any performance result. + +- [x] **Zero cross-user leakage.** DONE — `76df2469`. Two synthetic users differing in EC + identity, consent jurisdiction and geo store a byte-identical template, each against + a fresh cache so the first cannot answer for the second. Forbidden-substring checks + are the second layer, since byte-identity also holds if both leak the same thing. + Mutation-verified: leaking `adSlots` through the head seam fails it. +- [x] **Cold MISS, warm HIT, stale revalidation** DONE — `76df2469`, and end to end under + `viceroy serve` (below). Stale reads as a miss; serving stale would mean serving a + template built by an older transform or bundle. + + The first stale test passed for the wrong reason and had to be rewritten: a zero TTL + produces an *absent* entry, not a stale one, so `is_stale()` was never reached — + confirmed by reverting the check and watching it stay green. Only a + `stale_while_revalidate` window makes an entry present-and-stale. + +- [x] **Transform failure** DONE — `76df2469`. A partial template in C2 is the worst + outcome available: a truncated document served to every later visitor, indefinitely, + with no error after the first request. Mutation-verified by storing before the cap + check. +- [ ] **Request collapsing** works: concurrent cold requests transform once. +- [x] **DCA disabled** DONE — `0597f54e`. Config asserted _and_ behaviour: a fragment + carrying its own nested ESI include is spliced as text rather than dispatched. + +- [ ] **Request collapsing** — not tested, and not testable here. Viceroy is + single-threaded, so the concurrent cold-request case cannot be produced. The racing + _writer_ path is covered (`a_second_put_on_a_fresh_entry_is_a_no_op`), which is the + correctness half; the collapsing half needs real concurrency. +- [ ] **Exactly one auction per pageview**, from `auction_events_raw`. +- [ ] **Cookie and privacy finalization ran BEFORE assembly**, not after — EC + `Set-Cookie` on first visit, geo suppression, and an unconditional + `Cache-Control: private, no-store`. Headers commit before the body streams on this + adapter, so "finalize after assembly" is not available; asserting it that way is how + a per-user response ends up shared-cacheable. ESI's streaming mode dropping + `$add_header` is a consequence of the same constraint, not a separate hazard. +- [ ] **Slot and bid attribution unchanged.** Same slots matched, same bids applied, same + renders attributed. Use TS-attributed renders — the SSAT line item, non-empty + `ts.bids`, `hb_adid` presence — **never slot fill**, which is blind to empty bids + because `adInit` defines slots regardless. +- [x] **No C3 — assert positively, not by absence.** DONE — `0adb578e`, and this gate's + wording caught a live bug. A C2 hit returns before the point where the publisher path + stamps `private, no-store`, so it served HTML with **no `Cache-Control` at all** — + heuristically cacheable, and therefore a shared cache of an assembled per-user + response. Checking for the _absence_ of `public`/`s-maxage`/`Surrogate-Control` would + have reported it as safe, because there was nothing present to forbid. Covered for + returning visitors specifically, where the cookie-privacy net never fires. + + Original wording, retained because it is what made the difference: Forbidding `public`, `s-maxage`, and + `Surrogate-Control` is **not sufficient**: a bare `Cache-Control: max-age=60` passes + that check and is still shared-cacheable, and that is exactly what the measured + origin sends. Require instead that every assembled response carries + `Cache-Control: private, no-store` and that `Expires`, `ETag`, `Last-Modified`, and + all four CDN cache directives are stripped. Test it for **returning** users + specifically — they set no EC cookie, so the cookie privacy net never fires and is + not a backstop here. + +--- + +## Task 7: The decision record + +**Files:** `docs/superpowers/plans/2026-08-10-1009-esi-decision-record.md` + +- [ ] **Step 1: Record every arm** with N, confidence interval, cache-tier mix, route mix, + and POP. Any arm missing those is not reportable. + +- [ ] **Step 2: Apply the decision rule, stated here before the data exists** + +**Adopt ESI only if all three hold:** + +1. Every Task 6 gate passes on A3. +2. A3 beats A2 on **bids-ready time, `adInit` fire time, and first TS-attributed creative + paint** — by a margin the reviewers ratify **before** collection, not chosen after + seeing the numbers. **Not root TTFB:** A2 and A3 serve the same C2 template, so their + root timings are near-identical by construction and a difference there would be noise. + Root TTFB is a non-regression guard only. +3. Render outcomes on A3 are non-inferior to A0. + +**Otherwise adopt A2 (client-fill)** if its gates pass and it beats A1. It is portable +across all four adapters and carries no Fastly-only maintenance burden. + +**Otherwise keep A1** — Stage 0 alone — and record #1009 as answered in the negative with +evidence. + +The margin in (2) exists because A3's cost is not its diff. It is a second rendering +architecture, Fastly-only, on a pre-1.0 crate, in the critical render path. A small win +does not pay for that. + +- [ ] **Step 3: Record what would change the answer**, so this does not get re-litigated + from scratch. At minimum: React #418 / [#938](https://github.com/IABTechLab/trusted-server/issues/938) + being fixed such that `adInit` can run synchronously, which is what would make edge + assembly's round-trip saving actually worth something. + +- [ ] **Step 4: Clean up.** Remove the spike flag or promote it to a real setting; purge + C2 (`purge_surrogate_key` on `ts-template`); remove the synthetic fragment endpoint; + and either land or delete the `esi` dependency. **A spike flag left in place becomes + permanent configuration surface.** + +--- + +## Reproducibility metadata + +Record with every result, or it cannot be re-run or trusted: commit SHA; `esi` and +`fastly` crate versions; Fastly service and version IDs; whether the backend is shielded; +`template_ttl`; the origin's `Cache-Control` and `Vary` at collection time; assembly mode; +routes; N per arm; and the cache-tier mix. + +## Out of scope + +- **Stages 1–2 of the spec** as production work. This spike may build parts of the + client-fill path to measure it; shipping it is a separate decision behind the + correctness defects. +- **Full RSC/flight partitioning.** `rsc_flight.rs` has no static/dynamic split. +- **Publisher-authored ESI.** Breaks the no-origin-changes promise. +- **A C3 delivery cache.** Not a deferred item — a thing that must not exist. + +## Definition of done + +- [ ] Task 1 verdict recorded: `esi` 0.7 builds on Rust 1.95.0 / `wasm32-wasip1`, or it + does not and the spike stopped. +- [ ] All four arms measured on one build, with correlation IDs joining server and browser + timings, and cache tier recorded per request. +- [ ] Every Task 6 gate has an explicit pass/fail per arm. +- [ ] Decision record exists, applies the pre-ratified rule, and names what would change + the answer. +- [ ] Cleanup complete: flag resolved, C2 purged, synthetic endpoint removed, dependency + landed or dropped. +- [ ] All CI gates pass: `cargo fmt --all -- --check`; the six clippy targets; the four + adapter test suites; the parity suite; JS build, test, and format; docs format. diff --git a/docs/superpowers/plans/2026-08-11-config-first-auction-provider-architecture-implementation-plan.md b/docs/superpowers/plans/2026-08-11-config-first-auction-provider-architecture-implementation-plan.md new file mode 100644 index 000000000..a3c83d4b4 --- /dev/null +++ b/docs/superpowers/plans/2026-08-11-config-first-auction-provider-architecture-implementation-plan.md @@ -0,0 +1,751 @@ +# Config-First Auction Provider Architecture Implementation Plan + +**Date:** 2026-08-11 +**Status:** Draft for implementation review +**Spec:** `docs/superpowers/specs/2026-08-10-config-first-auction-provider-architecture-design.md` +**Implementation baseline:** `main` at `af67d8c2`, including merged APS PR #918 + +## Baseline and branch requirement + +The design-spec branch predates the merged APS OpenRTB implementation. Implementation must start from current `main` at or after `af67d8c2`, not from the current documentation branch's Rust tree. Rebase or create a fresh implementation worktree after the specification is merged so the work preserves the current APS OpenRTB request, response, renderer, diagnostics, and parity tests. + +The implementation must not restore the pre-#918 APS `/e/dtb/bid` protocol or use the older encoded-price mediator-only path as its baseline. + +## Decisions locked for this plan + +- The generic `standard` compatibility endpoint is a fictional local mock available only under automated tests. It adds no runtime endpoint, profile, authentication option, or production-support claim. +- Provider ID, returned upstream seat, and browser-facing delivery bidder code are separate identities. + - Provider ID owns configuration, backend correlation, health, and telemetry. + - A valid string `seatbid.seat` is retained as `returned_seat`. + - Existing `Bid.bidder` remains the delivery bidder code for compatibility. + - PBS uses the returned seat or `unknown`; APS continues to use `aps`. +- Disabled global signing removes only signature-bearing fields. PBS retains its existing `request_host`/`request_scheme` object; APS and `standard` omit `ext.trusted_server`. +- Enabled global signing loads one auction-local signer before dispatch and applies the existing version 1.1 extension to every OpenRTB provider after profile augmentation. +- On an adapter with a future enforceable total-request deadline, late completions are timeouts and are discarded. No current adapter claims that capability; an already-launched completed response remains eligible, but no further provider or mediator network work starts after logical budget exhaustion. +- Existing mock mediation remains a statically registered, separately selected path. It is not represented by `ProviderPlan`, a profile, or a new generic mediator trait. +- `BTreeMap` or sorted vectors provide deterministic plan, route, request, and validation order. Runtime correctness must not depend on TOML or `HashMap` iteration order. +- Provider IDs use the spec's lowercase ASCII grammar and 63-byte limit. +- First-version bounds introduced by this work are constants with tests: + - at most 128 `notifications.suppress_seats` entries, each at most 128 UTF-8 bytes; + - at most 128 bidder entries in one `trustedServer.bidderParams` object; + - bidder IDs at most 128 UTF-8 bytes and `trustedServer.zone` at most 256 UTF-8 bytes; + - each static `request_ext` or `imp_ext` object at most 16 KiB serialized, at most eight object/array levels deep, and at most 256 keys at any one object level. +- The existing 256 KiB `/auction` body limit remains authoritative. Header snapshots use the exact first `HeaderMap` value already selected by the request layer and add no truncation behavior. +- No temporary old/new public configuration compatibility mode ships. Internal staging adapters may exist while the branch is under development, but they must be removed before the configuration schema switch is merged. + +## Definition of done + +- `[auction.providers]` and `[auction.bidders]` are the only server-side bidder-provider inventory and routing source. +- `standard`, `prebid-server`, and `aps` are compiled through one Rust registry into an immutable `AuctionPlan`. +- Deploy validation and every adapter startup use the same target-independent compiler; target-aware paths also run the same adapter capability and backend-name validation. +- Runtime request handling uses normalized auction data, a transport-owned Prebid header snapshot, and filtered `ProviderAuctionInput` values. Profiles do not receive the raw HTTP request or unrestricted runtime services. +- A generic OpenRTB 2.6 driver owns standard fields, common response decoding, request finalization, signing, and common notification suppression. +- PBS and APS singleton bidder-provider registration is removed after parity fixtures pass. +- Multiple instances of one profile dispatch and correlate by provider ID. +- Empty trusted Prebid envelopes, explicit bidder routing, APS `all_eligible`, unknown bidders, no-slot skips, and mixed browser demand follow the specification. +- PBS and APS request, response, privacy, creative, timeout, debug, and diagnostics behavior matches `main`, except for the intentional all-provider signing expansion. +- Provider ID, returned seat, and delivery bidder code remain independent through ranking, mediation, delivery, and telemetry. +- Existing ranking, floors, USD assumptions, mock mediation, creative sanitization, APS rendering, Prebid Cache, and telemetry remain intact. +- Example configuration, operator docs, and browser-injected Prebid configuration use the new ownership model. +- All target-specific tests, JS tests, integration parity tests, formatting, and clippy gates pass. + +## Proposed architecture + +### Raw and compiled configuration + +Add raw serde types under `auction_config_types.rs` and compile them into types that cannot represent unresolved references: + +```rust +pub struct AuctionConfig { + pub enabled: bool, + pub timeout_ms: u32, + pub providers: BTreeMap, + pub bidders: BTreeMap, + pub mediator: Option, +} + +pub struct AuctionPlan { + providers: Vec, + bidder_routes: BTreeMap, + signing_enabled: bool, +} + +pub struct ProviderPlan { + id: ProviderId, + endpoint: CanonicalProviderEndpoint, + timeout_ms: u32, + routing: RoutingMode, + notifications: NotificationPolicy, + protocol: ProtocolPlan, + profile: CompiledOpenRtbProfile, +} +``` + +Use newtypes for `ProviderId` and `BidderId`. Sort compiled providers by provider ID and store bidder routes as validated provider indexes or IDs. `AuctionPlan` stores only the enabled signing policy; it never stores loaded keys. + +The first-version protocol representation is a closed `ProtocolPlan::OpenRtb26` enum. Do not introduce scripting, dynamic loading, or a generalized protocol trait before a second protocol exists. + +### Profile registry and dispatch + +Use a small compile-time registration table rather than runtime plugins: + +```rust +pub struct OpenRtbProfileRegistration { + id: &'static str, + default_timeout: ProfileTimeoutDefault, + compile: fn(&serde_json::Value) -> Result>, +} + +enum CompiledOpenRtbProfile { + Standard(StandardProfilePlan), + PrebidServer(PrebidProfilePlan), + Aps(ApsProfilePlan), +} +``` + +The enum supplies typed methods for field policy, request augmentation, request-local parse state, response interpretation, and renderer capabilities. This is intentionally simpler than boxed `Any` state or a public capability-composition system. Adding a repository-owned profile requires a registration entry and enum variant, which is acceptable for the first version. + +Keep PBS-specific compilation and behavior in `integrations/prebid.rs` initially and APS-specific behavior in `integrations/aps.rs`. Expose narrow `pub(crate)` registration/compile hooks to the auction module. Do not convert either large integration file into a directory tree solely for this refactor. + +### Generic provider execution + +Replace singleton bidder providers with one generic planned OpenRTB provider execution path. It owns endpoint selection, backend specification, one-request-per-provider dispatch, common driver invocation, and response association. It dispatches profile behavior through `CompiledOpenRtbProfile` without matching profile names in the orchestrator. + +Adapt the existing `AuctionProvider` dispatch/parse seam rather than replacing the split dispatch/collect mechanism wholesale: + +- change static provider-name APIs to borrow the validated dynamic provider ID; +- let the generic planned provider carry one `Arc`; +- route and filter slots before invoking it; +- retain request-local profile parse state in the existing pending response token; +- split bidder-provider storage from the statically selected mock mediator so the mediator never enters the compiled plan. + +Delete PBS and APS singleton registration only after their planned profiles pass parity tests. + +### Admission and execution context + +Introduce an admitted request representation that separates canonical data from transport-only data: + +```rust +struct AdmittedAuction { + request: AuctionRequest, + prebid_headers: PrebidTransportHeaders, + signer: Option, +} + +struct ProviderAuctionInput { + provider_id: ProviderId, + slots: Vec, + canonical: Arc, + logical_budget_ms: u32, +} +``` + +The exact ownership can use references or `Arc` to avoid copying publisher/user/device data. Profiles receive `ProviderAuctionInput`, never the current raw-request-bearing `AuctionContext`. Common transport receives the private Prebid header snapshot. The existing mediator may continue receiving the broader legacy context until a separate mediation design changes it. + +Keep timeout values explicit and separate: + +- `logical_budget_ms = min(provider_timeout_ms, remaining_auction_ms)` controls launch and OpenRTB `tmax`. +- `transport_timeout_ms = backend.canonicalize_transport_timeout_ms(remaining_auction_ms, provider_timeout_ms)` controls backend identity and the adapter's available transport timers. + +Fastly quantization of `transport_timeout_ms` must never replace or shorten `logical_budget_ms`. + +### Bid identity + +Preserve `Bid.bidder` as the serialized delivery bidder code and add: + +```rust +pub returned_seat: Option +``` + +`AuctionResponse.provider` remains the provider ID. Update comments and constructors so these three identities cannot be accidentally interchanged. Keep `returned_seat` out of unchanged external mediator/client wire shapes where necessary and restore it from the original provider bid after mock mediation. Notification suppression matches only `returned_seat`; response serialization and the APS browser renderer continue to use `Bid.bidder`. The existing telemetry seat carrier uses `returned_seat` when present and falls back to `Bid.bidder` when absent. + +### Target validation + +Add a pure adapter validation description passed into the compiler's second stage: + +```rust +pub struct AuctionTargetCapabilities<'a> { + pub target_name: &'static str, + pub supports_concurrent_fanout: bool, + pub enforces_total_request_deadline: bool, + pub backend_name_predictor: &'a dyn PlatformBackend, +} +``` + +Reuse `PlatformBackend::predict_name` for startup validation. Where CLI validation cannot instantiate the runtime backend, extract one shared pure naming helper/descriptor and make both `predict_name` and CLI validation call it; do not create a second independently implemented codec. Cloudflare and Spin skip registration but still use their real deterministic names containing canonical backend-spec fields and the provider discriminator. + +Target-independent `ts config validate` runs stage one and emits an explicit message that adapter checks are deferred. Adapter startup always runs both stages. + +Target-aware `ts config push --adapter ` must validate before any remote read, prompt, or write. Add an EdgeZero typed-push callback API that receives the already deserialized, environment-overlaid `TrustedServerAppConfig` plus the selected adapter ID between EdgeZero's ordinary typed validation and its first remote operation. Trusted Server maps that ID to the same target capability/name-prediction descriptor used at startup and runs stage two in the callback. Pin the EdgeZero revision containing this hook. This keeps config loading, overlays, platform writes, and adapter selection inside EdgeZero while making target validation mandatory rather than duplicating a loader in Trusted Server. + +## Stage 1 — Pin behavioral parity before refactoring + +Files: + +- `crates/trusted-server-core/src/integrations/prebid.rs` +- `crates/trusted-server-core/src/integrations/aps.rs` +- `crates/trusted-server-core/src/auction/orchestrator.rs` +- `crates/trusted-server-core/src/auction/formats.rs` +- `crates/trusted-server-js/lib/test/integrations/prebid/index.test.ts` + +Steps: + +1. Add or consolidate golden helpers that serialize current PBS and APS requests from the same canonical fixture. +2. Pin PBS behavior for: + - standard field ownership differences; + - raw `Referer` in both HTTP forwarding and `site.ref`; + - `User-Agent`, `Accept-Language`, and platform-attested `X-Forwarded-For`; + - every Cookie mode, malformed/non-UTF-8 Cookie fallback, and removal-to-empty; + - TCF, jurisdiction-derived GDPR, USP, GPP/SID, Google Additional Consent, EIDs, and KV/policy body fallback; + - stored requests, bidder-parameter merge precedence, override rules, test/debug fields, Cache coordinates, and response diagnostics; + - exact signing-enabled and signing-disabled JSON, including the disabled host/scheme-only object and absence of every signature-bearing key. +3. Pin APS behavior from merged main for: + - request field differences and account/SDK extensions; + - inventory identity overrides; + - consent and EID placement; + - response shape validation and sibling-bid isolation; + - renderer envelopes and script policy; + - highest-price-per-impression reduction and bid-ID tie-breaking; + - default 800 ms timeout and debug diagnostics; + - signing-disabled absence of `ext.trusted_server` before the intentional coverage expansion. +4. Pin synchronous and split dispatch/collect behavior, including current late completions and mediator fallback. +5. Use only fictional request, endpoint, account, bidder, seat, and creative data. + +This stage establishes the comparison baseline; it must not change production behavior. + +Verification: + +```bash +cargo test-fastly +cargo test-axum +cd crates/trusted-server-js/lib && npx vitest run +``` + +## Stage 2 — Add config-first types, profile registry, and plan compiler + +Files: + +- `crates/trusted-server-core/src/auction_config_types.rs` +- `crates/trusted-server-core/src/auction/mod.rs` +- new `crates/trusted-server-core/src/auction/plan.rs` +- new `crates/trusted-server-core/src/auction/profile.rs` +- internal compiler test modules beside the new plan/profile code + +Steps: + +1. Add `ProviderId`, `BidderId`, `ProviderConfig`, `BidderRouteConfig`, `RoutingMode`, `NotificationConfig`, and typed common validation. +2. Parse profile configuration as an object and immediately hand it to the selected registered profile compiler. Do not retain untyped `serde_json::Value` in the runtime plan. +3. Add the compile-time `standard`, `prebid-server`, and `aps` registration table independent of browser integration enablement. +4. Implement target-independent compilation: + - provider ID grammar and uniqueness; + - only `openrtb-2.6`; + - profile resolution and typed profile config; + - HTTPS endpoint parsing, canonicalization, credentials/fragment rejection, and profile-specific endpoint restrictions; + - profile timeout defaults and explicit overrides; + - bidder-to-provider resolution; + - static extension type, size, depth, key-count, and reserved-field ownership checks; + - notification bounds and duplicate rejection; + - signing configuration structure; + - deterministic provider and route order. +5. Keep `[auction].mediator` validation separate and restricted to the existing static mock mediator. +6. Add compiler tests for every rejection, defaults, two same-profile instances, and browser integration independence. + +This stage is internal scaffolding only: do not replace `AuctionConfig.providers`, change `Settings`, or wire `validate_settings_for_deploy`. Compiler tests construct raw provider maps directly. Public schema and validation switch together in Stage 11, so no committed state exposes dual schemas or makes deploy validation disagree with the live runtime. + +Verification: + +```bash +cargo test-fastly +``` + +## Stage 3 — Add adapter capability and backend-name validation + +Files: + +- `crates/trusted-server-core/src/platform/traits.rs` +- `crates/trusted-server-core/src/platform/types.rs` +- `crates/trusted-server-core/src/integrations/mod.rs` +- `crates/trusted-server-core/src/auction/plan.rs` +- `crates/trusted-server-adapter-fastly/src/backend.rs` +- `crates/trusted-server-adapter-fastly/src/platform.rs` +- `crates/trusted-server-adapter-fastly/src/app.rs` +- `crates/trusted-server-adapter-axum/src/platform.rs` +- `crates/trusted-server-adapter-axum/src/app.rs` +- `crates/trusted-server-adapter-cloudflare/src/platform.rs` +- `crates/trusted-server-adapter-cloudflare/src/app.rs` +- `crates/trusted-server-adapter-spin/src/platform.rs` +- `crates/trusted-server-cli/src/run.rs` +- `crates/trusted-server-cli` target-validation tests +- root EdgeZero dependency pin/lockfile +- coordinated EdgeZero CLI typed-push callback API + +Steps: + +1. Add `AuctionTargetCapabilities` and shared pure backend-name prediction input to target validation. +2. Prepare provider ID as every planned `PlatformBackendSpec.discriminator`; production backend naming switches in Stage 11. +3. Reuse or extract the algorithms behind `PlatformBackend::predict_name` so target validation and runtime construction cannot drift: + - Fastly's canonical backend specification and digest naming; + - Axum's environment/backend normalization; + - Cloudflare's deterministic no-registration backend name; + - Spin's deterministic no-registration backend name. +4. Reject two provider plans whose predicted names collide before either can register or overwrite a correlation map. +5. Declare capabilities from current behavior: + - Fastly: concurrent fan-out, first-byte/between-byte transport timers, and no enforceable total-request deadline; + - Axum: concurrent fan-out and no provider-specific total-request deadline; + - Cloudflare: no concurrent fan-out and no provider-specific total-request deadline; + - Spin: no concurrent fan-out and no provider-specific total-request deadline. +6. Reject more than one active provider for Cloudflare and Spin at target validation. Do not infer that only one will be requested at runtime. +7. Keep existing runtime collision assertions as defense in depth. +8. Add the EdgeZero typed-push validation callback, update the dependency pin, and test that the callback runs after overlays/typed validation but before any remote read or write. +9. Add unit tests that map each CLI adapter ID to the same capability/prediction descriptor used by startup and validate directly constructed plans. End-to-end map-shaped config-push rejection waits for the public schema cutover in Stage 11. +10. Add target prediction/capability tests and same-profile/same-endpoint independent-correlation tests. Actual adapter startup and target-aware push wiring remains part of the atomic Stage 11 cutover. + +Verification: + +```bash +cargo test-fastly +cargo test-axum +cargo test-cloudflare +cargo test-spin +./scripts/test-cli.sh +``` + +## Stage 4 — Normalize admission and route provider-local inputs + +Files: + +- new `crates/trusted-server-core/src/auction/routing.rs` +- `crates/trusted-server-core/src/auction/types.rs` +- `crates/trusted-server-core/src/auction/formats.rs` +- `crates/trusted-server-core/src/auction/endpoints.rs` +- `crates/trusted-server-core/src/creative_opportunities.rs` +- `crates/trusted-server-core/src/publisher.rs` + +Steps: + +1. Split canonical slot demand into bidder parameters, trusted provider routes, and bounded Prebid `zone` facts. +2. Normalize the reserved `trustedServer` envelope before central routing: + - missing/null/empty `bidderParams` produces stored-request intent; + - non-object, bad key/value, partial malformed, and bound violations reject admission and never fan out stored requests; + - valid unknown bidders route to `unroutable_bidder` and do not trigger fallback; + - usable direct objects win collisions, while unusable direct values cannot overwrite usable envelope objects; + - deterministic merge order is independent of map iteration. +3. Snapshot the first accepted `Cookie`, `User-Agent`, `Referer`, and `Accept-Language` values into a private `PrebidTransportHeaders`. Ignore client-supplied XFF and preserve the platform-attested IP separately. +4. Build one `ProviderAuctionInput` per provider: + - remove non-banner formats before routing; + - `explicit` receives only centrally routed or trusted slots; + - `all_eligible` receives every banner-compatible slot; + - parameters assigned to another provider are never copied; + - no valid banner formats means no provider input; + - no eligible slots produces a skip result without dispatch. +5. Convert creative-opportunity construction: + - explicit bidder parameters remain bidder IDs and use the central registry; + - empty Prebid stored-request intent expands to every compiled PBS plan; + - remove hard-coded APS provider selection and rely on `all_eligible` or explicit central/trusted routing. +6. Use the same admission/router helper for `/auction`, initial navigation, refresh/page-bids, and other server-generated auction entry points. +7. Add table-driven tests for routing, mixed providers, unknown bidders, empty/malformed envelopes, collisions, bounds, non-banner slots, and no parameter leakage. + +Verification: + +```bash +cargo test-fastly +cargo test-axum +``` + +## Stage 5 — Build the common OpenRTB driver and test-only standard profile + +Files: + +- new `crates/trusted-server-core/src/auction/openrtb.rs` +- `crates/trusted-server-core/src/openrtb.rs` +- `crates/trusted-server-core/src/auction/profile.rs` +- `crates/trusted-server-core/src/request_signing/signing.rs` +- `crates/trusted-server-core/src/auction/types.rs` + +Steps: + +1. Extract common banner request construction into a driver that owns: + - request/impression IDs and banner formats; + - site, publisher, user, device, consent, EID, floor, secure, currency, and `tmax` fields; + - application of a typed per-profile field policy; + - common request extension ownership and collision checks. +2. Implement fixed standard/PBS/APS field policies. Treat Stage 1 golden fixtures as normative, especially for consent and privacy differences. +3. Implement `StandardProfilePlan` with bounded static `request.ext` and `imp.ext`. It does not invent a wire location for bidder params. +4. Build response decoding around independent bid validation and preserve current profile-specific whole-response versus sibling-bid behavior. +5. Treat response `id` as informational for PBS/APS parity; keep transport association as the correlation boundary. +6. Add `Bid.returned_seat` while retaining `Bid.bidder` as delivery code. +7. Move signing to common finalization: + - profiles finish augmentation first; + - finalization freezes request ID and signing-owned fields; + - enabled signing inserts the existing v1.1 extension; + - disabled PBS can retain host/scheme only; + - disabled APS/standard omit the object. +8. Add exact serialized signing-on and signing-off fixtures for all three profiles. Assert each signature-bearing key separately: disabled PBS retains only host/scheme, disabled APS/standard omit the object, and every enabled request contains the full v1.1 extension after augmentation. +9. Add a `#[cfg(test)]` fictional standard endpoint mock using the existing stub HTTP-client machinery. Test ordinary bid, no-bid/204, malformed response isolation, static extensions, signed request compatibility, and no-auth behavior. No fixture handler or endpoint is compiled into production. +10. Add a common-transport redirect fixture: return 3xx with `Location`, assert no second request occurs, classify the original provider outcome, and prove the canonical endpoint used for request dispatch is the same one used for backend prediction/construction. + +Verification: + +```bash +cargo test-fastly +cargo test-axum +``` + +## Stage 6 — Build plan-backed orchestrator execution under test construction + +Files: + +- `crates/trusted-server-core/src/auction/provider.rs` +- `crates/trusted-server-core/src/auction/orchestrator.rs` +- `crates/trusted-server-core/src/auction/mod.rs` +- `crates/trusted-server-core/src/auction/endpoints.rs` + +Steps: + +1. Change static provider identity APIs to dynamic validated provider IDs. +2. Add one generic planned OpenRTB provider that combines a `ProviderPlan`, common driver, profile enum, and existing platform transport. +3. Add an internal/test constructor through which the orchestrator owns immutable plan-backed bidder providers and routes filtered inputs before launch. Production startup remains on the legacy constructor until Stage 11. +4. Preserve the existing split request/parse token and carry typed profile-local parse state without exposing another provider's state. +5. Prepare split mock mediator registration/lookup from bidder-provider storage. Keep its invocation and fallback unchanged, and restore `returned_seat` from the selected original provider bid rather than deriving it from mediator output. +6. Compute exact `logical_budget_ms` once for launch and `tmax`, then separately compute adapter-canonicalized `transport_timeout_ms` for backend naming/timers. Add Fastly tests proving timeout quantization never changes `tmax`. +7. Load the current signer once during common admission before routing/dispatch. A load failure returns before any `send_async` call. Do not cache signer keys at startup. +8. Ensure one request per eligible provider, no request for skipped providers, and no zero-impression request. +9. Keep provider failures isolated and preserve current local ranking/floor logic after normalization. +10. Add multi-instance tests for two providers with the same profile, endpoint, and timeout, proving independent backend correlation and outcome metadata. + +Stages 2 through 10 use internal construction scaffolding and tests without changing the public configuration/runtime path. They are development order, not independently mergeable public migrations. Stage 11 performs the one atomic cutover; the merged result has one config-first runtime path and no public legacy provider list. + +Verification: + +```bash +cargo test-fastly +cargo test-axum +cargo test-cloudflare +cargo test-spin +``` + +## Stage 7 — Extract and migrate the Prebid Server profile + +Files: + +- `crates/trusted-server-core/src/integrations/prebid.rs` +- `crates/trusted-server-core/src/auction/openrtb.rs` +- `crates/trusted-server-core/src/auction/profile.rs` +- `crates/trusted-server-core/src/auction/orchestrator.rs` + +Steps: + +1. Define typed `PrebidProfileConfig` for debug, test mode, debug query params, override fields/rules, and consent forwarding. Prepare server endpoint, timeout, bidders, and notification suppression for common ownership, but keep the live legacy config fields until Stage 11. +2. Compile the current override engine once into `PrebidProfilePlan`. +3. Move PBS-only request behavior behind the profile: + - `imp.ext.prebid.bidder` from routed params; + - stored-request fallback for trusted routes; + - zone and generic override merge behavior; + - test/debug request fields and page query behavior; + - PBS-specific consent and Google Additional Consent policy; + - disabled-signing host/scheme extension fields. +4. Keep raw HTTP values out of the profile. Common Prebid transport applies the snapshot matrix for Cookie, UA, Referer, Accept-Language, and attested XFF. +5. Move PBS-only response behavior behind the profile: + - Prebid Cache coordinates; + - existing bid-status/debug metadata; + - request-local parse facts; + - current valid-sibling behavior and delivery bidder fallback. +6. Apply common notification suppression after PBS normalization and before ranking/mediation. +7. Exercise the profile through the internal planned-provider constructor while the live singleton remains unchanged. +8. Run the Stage 1 golden matrix against the new profile and compare serialized requests plus normalized outcomes. +9. Mark singleton registration and static backend discriminator for atomic removal in Stage 11. + +Verification: + +```bash +cargo test-fastly +cargo test-axum +``` + +## Stage 8 — Extract and migrate the APS profile + +Files: + +- `crates/trusted-server-core/src/integrations/aps.rs` +- `crates/trusted-server-core/src/auction/openrtb.rs` +- `crates/trusted-server-core/src/auction/profile.rs` +- `crates/trusted-server-core/src/auction/types.rs` +- APS renderer registration call sites under adapter/core startup + +Steps: + +1. Define typed `ApsProfileConfig` for account ID, debug, script opt-in, and inventory identity overrides. Prepare endpoint and timeout for common provider ownership, but keep the live legacy config fields until Stage 11. +2. Express current APS request differences as its fixed field policy and owned account/SDK extensions. +3. Preserve current response validation, exact minimized renderer envelope, script gate, creative URL policy, diagnostics, and deterministic candidate reduction. +4. Capture a valid string `seatbid.seat` as `returned_seat`; use no returned seat for missing/non-string values. Keep `Bid.bidder = "aps"` so the existing renderer path remains active. +5. Preserve current APS removal/non-exposure of notification URLs before common configurable suppression. +6. Add a narrow validated-plan query for APS renderer activation and test it through the internal constructor; production registry wiring switches in Stage 11. +7. Make `all_eligible` the parity migration configuration and test optional `explicit` routing separately. +8. Exercise the profile through the internal planned-provider constructor while the live singleton remains unchanged. +9. Run the Stage 1 APS matrix against the profile, including exact enabled/disabled signing fixtures. +10. Mark singleton registration and static backend discriminator for atomic removal in Stage 11. + +Verification: + +```bash +cargo test-fastly +cargo test-axum +cd crates/trusted-server-js/lib && npx vitest run +``` + +## Stage 9 — Prepare browser Prebid separation and shim migration + +Files: + +- `crates/trusted-server-core/src/integrations/prebid.rs` +- `crates/trusted-server-js/lib/src/integrations/prebid/index.ts` +- `crates/trusted-server-js/lib/test/integrations/prebid/index.test.ts` +- `crates/trusted-server-core/src/creative_opportunities.rs` +- `crates/trusted-server-core/src/publisher.rs` + +Steps: + +1. Define the post-cutover browser-only `[integrations.prebid]` shape, retaining `timeout_ms` and `debug` with 1000 ms/false defaults. +2. Prepare removal of server endpoint, server bidder allowlist, server debug, overrides, consent-forwarding, and suppression ownership, but do not change the live serde shape before Stage 11. +3. Keep browser bundle, account injection, script patterns, client-side bidders, and excluded GAM suffixes under `[integrations.prebid]`. +4. Add an injection path that consumes validated server-side bidder codes from `AuctionPlan`, not a second Prebid allowlist. Provider timeout/profile debug must never affect `pbjs.setConfig`; production startup passes the plan in Stage 11. +5. Preserve the current requestBids shim's initial and refresh snapshots, folding only server-side entries into `trustedServer.bidderParams` while leaving configured client-side bidders in the browser. +6. Add JS tests for: + - browser timeout/debug independence with multiple PBS plans; + - browser integration disabled while PBS profile compilation remains valid; + - mixed client-side, PBS, APS, and standard-provider demand; + - initial and refresh parameter preservation; + - empty stored-request envelopes; + - alternate returned bidder codes and APS renderer alias. +7. Update creative-opportunity tests to prove no hard-coded provider ID reaches client-controlled input. + +Verification: + +```bash +(cd crates/trusted-server-js/lib && npx vitest run) +(cd crates/trusted-server-js/lib && node build-all.mjs) +cargo test-fastly +``` + +## Stage 10 — Finalize diagnostics, notifications, deadlines, and mediation parity + +Files: + +- `crates/trusted-server-core/src/auction/orchestrator.rs` +- `crates/trusted-server-core/src/auction/types.rs` +- `crates/trusted-server-core/src/auction/telemetry.rs` +- `crates/trusted-server-core/src/auction/formats.rs` +- `crates/trusted-server-core/src/integrations/adserver_mock.rs` +- all adapter platform test modules + +Steps: + +1. Add the fixed `routing` metadata object: + - auction-level saturating `unroutable_bidder_count` in `OrchestrationResult.metadata` plus bounded structured logging; + - provider-level `skipped_no_eligible_slots = true`; + - provider-level saturating `unused_bidder_params_count`. +2. Carry only booleans/counts. Do not include parameter values or bidder-ID lists. +3. Apply `suppress_all` and exact `returned_seat` suppression after profile normalization and before ranking/mediation. +4. Verify that missing/non-string seats do not match suppression, PBS still serializes `unknown`, and APS still serializes `aps`. +5. Restore `returned_seat` from the original provider bid after mock mediation. Make the existing telemetry seat field prefer `returned_seat` and fall back to delivery bidder code; add direct and mediated APS tests proving provider ID, upstream seat, and `aps` remain distinct. +6. Implement capability-dependent late completion behavior: + - a future hard-total-deadline adapter would discard/classify timeout; + - every current adapter accepts an already-completed late response because none claims an enforceable total-request deadline; + - neither path launches new provider or mediator network work at zero remaining budget; + - local ranking/delivery still finishes; + - synchronous and split dispatch/collect agree. +7. Preserve response times as actual elapsed times and document wall-clock overrun limitations in adapter docs/tests. +8. Run regression tests for no mediator, mock mediation, mediator timeout/fallback, floors, winner selection, creative sanitization, Prebid Cache, APS renderer delivery, and telemetry provider IDs/seats. +9. Confirm routing metadata uses existing maps and does not add OpenRTB response or telemetry schema fields. + +Verification: + +```bash +cargo test-fastly +cargo test-axum +cargo test-cloudflare +cargo test-spin +cargo test --manifest-path crates/trusted-server-integration-tests/Cargo.toml --test parity +``` + +## Stage 11 — Perform the atomic schema and runtime cutover + +Stages 2 through 10 prepare and test the new architecture internally. This stage is one non-partial cutover: do not commit or merge a state in which public settings, deploy validation, runtime startup, browser injection, examples, or fixtures disagree. + +Files: + +- `crates/trusted-server-core/src/auction_config_types.rs` +- `crates/trusted-server-core/src/config.rs` +- `crates/trusted-server-core/src/settings.rs` +- `crates/trusted-server-core/src/auction/mod.rs` +- `crates/trusted-server-core/src/integrations/registry.rs` +- `crates/trusted-server-core/src/integrations/prebid.rs` +- `crates/trusted-server-core/src/integrations/aps.rs` +- all four adapter `app.rs` startup files +- `crates/trusted-server-cli/src/run.rs` and target-validation tests +- `crates/trusted-server-integration-tests/fixtures/configs/trusted-server.integration.toml` +- relevant adapter startup/route fixtures +- `trusted-server.example.toml` +- `docs/guide/configuration.md` +- `docs/guide/integrations/prebid.md` +- `docs/guide/integrations/aps.md` +- relevant README/getting-started pages +- configuration/env-overlay/config-push tests + +Steps: + +1. Replace the public legacy provider list with map-shaped `[auction.providers]` and `[auction.bidders]` settings. +2. Compile one `Arc` per process startup, run target validation once, and pass that same plan to both `AuctionOrchestrator` and `IntegrationRegistry`. Do not recompile or inspect unresolved provider config in either consumer. +3. Switch all four adapters to the generic plan-backed provider constructor and provider-ID backend discriminator. +4. Make `validate_settings_for_deploy` run the same target-independent compiler. Make target-aware `config push` run the EdgeZero pre-write callback with the selected adapter capabilities; keep `config validate` target-independent with an explicit deferred-check message. Add end-to-end tests proving Cloudflare/Spin multi-provider configs fail before any remote read, prompt, or write. +5. Remove PBS/APS singleton bidder-provider builders and their static backend discriminators. Retain the mock mediator in its separate static registration path. +6. Apply the prepared Prebid browser/server config split. Pass validated browser server-side bidder codes from the shared plan into integration injection. +7. Use `AuctionPlan::has_profile(ProfileId::Aps)` or an equivalent narrow query to register APS renderer support even when `[integrations.aps]` is disabled. Test renderer presence on every adapter. +8. Remove obsolete integration-owned server fields and update all serde, env-overlay, config-push, adapter, integration, and browser fixtures in the same cutover. +9. Replace examples/docs with `[auction.providers.*]`, `profile_config`, and `[auction.bidders.*]`; keep `[auction].mediator` separate. +10. Document browser/server Prebid ownership, APS `all_eligible`, provider ID and extension bounds, target validation, and current no-hard-total-deadline limitations. Use only fictional/example values. +11. Run the complete cutover gate before committing: + +```bash +cargo test-fastly +cargo test-axum +cargo test-cloudflare +cargo test-spin +./scripts/test-cli.sh +cargo test --manifest-path crates/trusted-server-integration-tests/Cargo.toml --test parity +(cd crates/trusted-server-js/lib && npx vitest run) +(cd crates/trusted-server-js/lib && node build-all.mjs) +(cd docs && npm run format) +``` + +No committed stage may expose both public schemas. + +## Stage 12 — Full verification and cleanup + +1. Delete all internal staging adapters, duplicate legacy server configuration fields, singleton provider builders, unused static provider constants, and obsolete tests. +2. Confirm no bidder-provider profile receives `AuctionContext.request` or unrestricted `RuntimeServices`. +3. Confirm only common transport handles endpoint, backend, headers, redirects, response bounds, and dispatch. +4. Confirm the test-only standard endpoint mock is under `#[cfg(test)]` or integration-test code and absent from production binaries/config schema. +5. Run the complete repository gates: + +```bash +cargo fmt --all -- --check +cargo clippy-fastly +cargo clippy-axum +cargo clippy-cloudflare +cargo clippy-cloudflare-wasm +cargo clippy-spin-native +cargo clippy-spin-wasm +cargo test-fastly +cargo test-axum +cargo test-cloudflare +cargo test-spin +./scripts/test-cli.sh +cargo test --manifest-path crates/trusted-server-integration-tests/Cargo.toml --test parity +(cd crates/trusted-server-js/lib && npx vitest run) +(cd crates/trusted-server-js/lib && npm run format) +(cd crates/trusted-server-js/lib && node build-all.mjs) +(cd docs && npm run format) +``` + +6. Run `git diff --check` and verify no generated, staged, or `.pi-subagents` artifacts are included. + +## Test matrix summary + +| Area | Required coverage | +| ------------------ | ---------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| Config compiler | IDs, protocol/profile lookup, typed profile config, defaults, routes, endpoint canonicalization, static ownership/bounds, notifications, signing structure. | +| Target validation | shared backend-name prediction parity, encoded collisions, fan-out rejection, deadline capability claims, all four adapters. | +| Admission | envelope missing/null/empty/malformed/partial/unknown/collision/bounds, zone, raw header snapshot, attested XFF. | +| Routing | explicit, all-eligible, trusted routes, mixed providers, unknown bidders, no banners, no parameter leakage, deterministic order. | +| Standard profile | request/response, static extensions, unused params, 204, malformed bids, exact signing on/off, redirects, test-only no-auth endpoint. | +| PBS parity | all standard-field differences, consent matrix, headers/Cookies, stored requests, overrides, debug/test, Cache, diagnostics, seats, suppression, exact signing on/off. | +| APS parity | account/SDK fields, inventory identity, consent, renderer/script policy, response validation, candidate reduction, seats versus `aps`, exact signing on/off. | +| Orchestration | one request/provider, skip/no dispatch, partial failures, dynamic correlation, synchronous/split, logical/hard/late deadlines. | +| Decision/delivery | local ranking, floors, USD, mock mediation/fallback, sanitization, PBS Cache, APS renderer. | +| Browser JS | browser config ownership, mixed demand, folding, client-side preservation, refresh snapshots, aliases/renderers. | +| Diagnostics | fixed metadata carrier, saturating counts, no params/IDs, provider IDs in telemetry, no schema expansion. | +| Configuration/docs | TOML maps, env overlays, example-only values, CLI validation, adapter limitations. | + +## Primary file checklist + +### Core control plane + +- [ ] `auction_config_types.rs`: raw provider/bidder schema and newtypes. +- [ ] `auction/plan.rs`: compiler, immutable plan, target validation. +- [ ] `auction/profile.rs`: profile registration and typed compiled enum. +- [ ] `config.rs`: shared deploy validation. +- [ ] `auction/mod.rs`: registry wiring and mediator separation. + +### Core runtime + +- [ ] `auction/routing.rs`: envelope normalization and provider-local inputs. +- [ ] `auction/openrtb.rs`: common request/response driver and standard profile. +- [ ] `auction/provider.rs`: dynamic generic planned provider seam. +- [ ] `auction/orchestrator.rs`: plan execution, deadlines, diagnostics, unchanged decision flow. +- [ ] `auction/types.rs`: admitted input and returned-seat identity. +- [ ] `auction/formats.rs`: normalized admission and delivery alias serialization. +- [ ] `auction/endpoints.rs`: header snapshot and auction-local signer load. +- [ ] `auction/telemetry.rs`: provider-ID parity, returned-seat preference with delivery-code fallback, and existing metadata consumption only. +- [ ] `integrations/adserver_mock.rs`: restore original returned seat without generalizing mediation. + +### Profiles and browser integration + +- [ ] `integrations/prebid.rs`: typed profile, browser/server split, header/Cookie parity. +- [ ] `integrations/aps.rs`: typed profile, renderer activation, seat/alias parity. +- [ ] `creative_opportunities.rs`: config-neutral demand and trusted stored intent. +- [ ] `publisher.rs`: shared admission/plan use for page auctions. +- [ ] `integrations/registry.rs`: consume the shared compiled plan for renderer and browser capability queries. +- [ ] `trusted-server-js/lib/src/integrations/prebid/index.ts`: browser config and routing shim. + +### Platforms and tooling + +- [ ] Core platform types/traits: capability contract and shared backend-name prediction. +- [ ] Fastly backend/platform/app: provider discriminator, logical/quantized timeout separation, and no total-deadline claim. +- [ ] Axum platform/app: name prediction, fan-out, and no total-deadline claim. +- [ ] Cloudflare platform/app: deterministic name prediction, one-provider validation, and late completion behavior. +- [ ] Spin platform/app: deterministic name prediction, one-provider validation, and late completion behavior. +- [ ] EdgeZero callback/dependency pin plus CLI tests: target-aware pre-write push validation, target-independent validation, and map-shaped overlays. + +### Documentation + +- [ ] `trusted-server.example.toml`. +- [ ] Configuration guide. +- [ ] Prebid guide. +- [ ] APS guide. +- [ ] Adapter timeout/fan-out notes where maintained. + +## Risk register + +| Risk | Mitigation | +| -------------------------------------------------------------------- | ---------------------------------------------------------------------------------------------------------------------------------- | +| Implementing from the stale spec branch regresses merged APS | Require implementation worktree from `main` at/after `af67d8c2`; run APS goldens first. | +| Common field extraction broadens privacy exposure | Fixed typed policies plus exhaustive consent/header golden matrices; profiles can omit but never restore centrally removed data. | +| Dynamic profile state becomes an unsafe type-erasure layer | Use a closed typed enum and typed request-state variants for v1. | +| Multiple provider instances collide in an adapter | Provider ID discriminator, shared prediction parity tests, compile-time target collision rejection, runtime guard retained. | +| Browser Prebid behavior silently follows one server profile | Keep browser timeout/debug explicit; derive only server bidder identities from the central registry. | +| Empty or malformed envelope unexpectedly fans out stored requests | Exact fallback table and admission tests; malformed input never triggers fallback. | +| Header refactor truncates or broadens Cookie/Referer disclosure | Snapshot the same first accepted header values with no new truncation; transport-only access; parity fixtures for malformed bytes. | +| APS seat preservation breaks renderer activation | Keep `Bid.bidder = "aps"`; add independent `returned_seat`; test direct and page delivery. | +| Disabled signing changes PBS wire shape | Preserve host/scheme-only object and pin exact enabled/disabled JSON fixtures. | +| Non-abortable adapters claim a wall-clock guarantee they cannot meet | Capability-specific semantics, adapter docs, synchronous/split late-response tests. | +| Mock mediator is accidentally generalized or routed | Separate storage/construction; retain existing `[auction].mediator` path and regression fixtures. | +| Public diagnostics leak bidder data or expand telemetry schema | Fixed existing metadata maps with booleans/saturating counts only. | +| Temporary dual architecture survives the refactor | Final cleanup stage and searches for singleton builders, legacy list fields, and static PBS/APS backend IDs. | +| Plan grows into unrelated privacy/auth/network policy work | Keep consent behavior, auth omission, and endpoint-network deferrals exactly as specified. | + +## Explicitly deferred + +Do not add during implementation: + +- real generic endpoint onboarding or endpoint credentials; +- a generic mediator/profile system; +- new signing protocol fields or body binding; +- new consent minimization or Cookie behavior; +- video/native support; +- currency conversion; +- label/group/multi-route routing; +- request splitting; +- new adapter abort/deadline mechanisms; +- new telemetry fields; +- broader private-network, custom-port, or DNS-rebinding policy. diff --git a/docs/superpowers/plans/2026-08-18-contiguous-generated-slot-tables.md b/docs/superpowers/plans/2026-08-18-contiguous-generated-slot-tables.md new file mode 100644 index 000000000..0fb8977a0 --- /dev/null +++ b/docs/superpowers/plans/2026-08-18-contiguous-generated-slot-tables.md @@ -0,0 +1,46 @@ +# Contiguous Generated Slot Tables Implementation Plan + +> **For agentic workers:** REQUIRED SUB-SKILL: Use superpowers:subagent-driven-development (recommended) or superpowers:executing-plans to implement this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking. + +**Goal:** Keep generated creative-opportunity slot and provider tables contiguous with their parent section. + +**Architecture:** Normalize the document positions carried by generated `toml_edit` tables before inserting them into the target document. Anchor the whole generated subtree at the target creative section and rely on stable serialization order. + +**Tech Stack:** Rust, `toml_edit`, Cargo tests + +--- + +### Task 1: Reproduce the position collision + +**Files:** + +- Modify/Test: `crates/trusted-server-cli/src/commands/audit/generate/slot_toml.rs` + +- [ ] Add `splice_keeps_generated_slots_and_providers_contiguous` with a late creative section and unrelated tables at colliding positions. +- [ ] Assert no unrelated table header occurs between `[creative_opportunities]`, all generated slots, and their provider subtables. +- [ ] Add `splice_groups_a_new_creative_section_with_its_slots` for an input that has no creative section, proving the newly created parent and generated subtree share the final anchor. +- [ ] Run each focused test with `cargo test_cli_macos -- --exact` and confirm both ordering assertions fail. + +### Task 2: Normalize imported table positions + +**Files:** + +- Modify: `crates/trusted-server-cli/src/commands/audit/generate/slot_toml.rs` + +- [ ] Add a small recursive helper using `Table::set_position`, `Table::iter_mut`, and `ArrayOfTables::iter_mut` to assign one anchor position to every table in the generated slot subtree. +- [ ] Use the existing creative table's position; for a newly created section, allocate one greater than the greatest parsed position and explicitly assign that anchor to both the new parent and its generated subtree. +- [ ] Run `cargo test_cli_macos commands::audit::generate::slot_toml::tests::splice_keeps_generated_slots_and_providers_contiguous -- --exact` and confirm it passes. +- [ ] Run `cargo test_cli_macos commands::audit::generate::slot_toml::tests::splice_groups_a_new_creative_section_with_its_slots -- --exact` and confirm it passes. +- [ ] Run `cargo test_cli_macos commands::audit::generate::slot_toml::tests` and confirm the complete module suite passes. + +### Task 3: Verify and deliver + +**Files:** + +- Modify: `crates/trusted-server-cli/src/commands/audit/generate/slot_toml.rs` + +- [ ] Run `./scripts/test-cli.sh`. +- [ ] Run `cargo fmt --all -- --check`. +- [ ] Run `cargo clippy --package trusted-server-cli --target aarch64-apple-darwin --all-targets -- -D warnings`. +- [ ] Confirm `trusted-server.toml` and the user's existing `fastly.toml` change remain untouched. +- [ ] Commit the verified generator fix on the current feature branch. diff --git a/docs/superpowers/plans/2026-08-18-pr-823-review-resolution.md b/docs/superpowers/plans/2026-08-18-pr-823-review-resolution.md new file mode 100644 index 000000000..873168438 --- /dev/null +++ b/docs/superpowers/plans/2026-08-18-pr-823-review-resolution.md @@ -0,0 +1,723 @@ +# PR 823 Review Resolution Implementation Plan + +> **For agentic workers:** REQUIRED SUB-SKILL: Use superpowers:subagent-driven-development (recommended) or superpowers:executing-plans to implement this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking. + +**Goal:** Resolve every actionable finding in PR 823 review `4958563121`, verify the branch, publish it, and answer all 28 inline threads. + +**Architecture:** Correct the review findings at four existing seams: core runtime gate APIs, pure CLI projection/comparison, crawl generation and TOML persistence, and the shared browser session. Keep page-controlled work bounded, use one source of truth for runtime/browser behavior, and preserve operator-authored configuration outside the managed creative-opportunities fields (`slot`, `gam_network_id`, `section_root`, and `section_segment`). + +**Tech Stack:** Rust 2024, clap 4, toml_edit 0.23, chromiumoxide 0.9, Tokio current-thread runtime, serde/serde_json, embedded JavaScript collector, mdBook documentation, GitHub CLI. + +--- + +## File Map + +- `crates/trusted-server-core/src/creative_opportunities.rs`: allocation-free gate evaluation, gate diagnostics, pattern validation, consent semantics. +- `crates/trusted-server-core/src/publisher.rs`: named gate input at the runtime call site. +- `crates/trusted-server-cli/src/ad_templates/{expected,compare,output}.rs`: runtime-equivalent projection, typed formats, confirmability, safe output. +- `crates/trusted-server-cli/src/commands/config/ad_templates.rs`: static command validation, gate parity, lint, escaping. +- `crates/trusted-server-cli/src/commands/audit/{collector,browser,ad_templates,ad_template_collector.js}.rs`: shared browser options/session and verifier behavior. +- `crates/trusted-server-cli/src/commands/audit/generate/{browser_collector,evidence,gpt_slots,crawl_plan,page_patterns,unit_template,slot_toml,mod,validate}.rs`: crawl evidence, inference, persistence, and dry-run safety. +- `crates/trusted-server-cli/src/commands/audit/{mod,page}.rs`, `crates/trusted-server-cli/src/run.rs`, `crates/trusted-server-cli/src/main.rs`: clap contracts and exit outcomes. +- `docs/guide/cli.md`, `scripts/test-cli.sh`, `.github/workflows/test.yml`: operator contract and enforced browser CI. + +## Task 1: Make the runtime gate API allocation-free and reusable + +**Files:** + +- Modify: `crates/trusted-server-core/src/creative_opportunities.rs` +- Modify: `crates/trusted-server-core/src/publisher.rs` + +- [ ] **Step 1: Add failing core tests** + +Add tests that sweep all 64 boolean combinations with `consent_allows_auction: None`, assert the expected `No`/`Unknown` result, assert `blocking_gates()` derives diagnostics without an owned `Vec`, and exercise the specific page-pattern validation error. + +Use a borrowed/static iterator contract: + +```rust +pub fn blocking_gates(self) -> impl Iterator { + AdStackGateName::ALL + .into_iter() + .filter(move |gate| gate.blocks(self.input)) +} + +pub fn validate_page_pattern(pattern: &str) -> Result<(), String> { + compile_page_pattern(pattern).map(|_| ()) +} +``` + +- [ ] **Step 2: Run the narrow tests and confirm RED** + +Run: + +```bash +cargo test --package trusted-server-core --target "$(rustc -vV | awk '/host:/ {print $2}')" ad_stack_gate -- --nocapture +``` + +Expected: failure because the unknown-consent sweep and allocation-free diagnostic API are not implemented. + +- [ ] **Step 3: Implement the minimal core change** + +Store the original `AdStackGateInput` in `AdStackGateResult`, compute `expected` with boolean expressions rather than `Vec::push`, expose a zero-allocation iterator over a `const ALL`, make `compile_page_pattern` crate-private, and add `validate_page_pattern`. Document that `None` means unknown and differs from denied (`Some(false)`). Preserve the detailed glob error in `compile_patterns`. + +Delete `should_run_server_side_ad_stack`; construct `AdStackGateInput` with named fields in `publisher.rs`. Import the gate types at module scope. + +- [ ] **Step 4: Verify GREEN** + +Run the narrow command again, then: + +```bash +cargo test-fastly creative_opportunities +cargo test-axum creative_opportunities +``` + +Expected: all selected tests pass. + +- [ ] **Step 5: Commit** + +```bash +git add crates/trusted-server-core/src/creative_opportunities.rs crates/trusted-server-core/src/publisher.rs +git commit -m "Align ad stack gate diagnostics with runtime" +``` + +## Task 2: Align expected-slot projection and comparison with runtime behavior + +**Files:** + +- Modify: `crates/trusted-server-cli/src/ad_templates/expected.rs` +- Modify: `crates/trusted-server-cli/src/ad_templates/compare.rs` +- Modify: `crates/trusted-server-cli/src/ad_templates/output.rs` +- Modify: `crates/trusted-server-cli/src/commands/audit/ad_templates.rs` + +- [ ] **Step 1: Add failing projection and comparison tests** + +Cover: + +- an unrenderable dynamic slot is omitted from expected slots and does not make `matched_slots` pass; +- the diagnostic says the runtime omits the slot for that path; +- `MediaType` remains typed through comparison; +- video/native-only slots produce `Unconfirmable` and do not fail strict; +- a sizeless out-of-page slot against banner-configured formats is `Partial` and fails strict; +- an incompatible banner is still `Partial` and fails strict; +- a missing slot has `phase: None` and JSON omits `phase`; +- server-side APS configuration alone does not emit `aps_evidence_missing`; +- collector warnings are appended to page warnings; +- human output contains expectation, gates, matched count, extra evidence, and warnings; +- bidi override/isolate characters are escaped. + +The central type changes are: + +```rust +pub struct ExpectedFormat { + pub width: u32, + pub height: u32, + pub media_type: MediaType, +} + +pub enum SlotStatus { + Confirmed, + Partial, + Missing, + Unconfirmable, +} + +pub struct SlotResult { + pub phase: Option, + // existing fields +} +``` + +- [ ] **Step 2: Run the narrow tests and confirm RED** + +Run: + +```bash +HOST_TARGET="$(rustc -vV | awk '/host:/ {print $2}')" +cargo test --package trusted-server-cli --target "$HOST_TARGET" ad_templates::expected +cargo test --package trusted-server-cli --target "$HOST_TARGET" ad_templates::compare +cargo test --package trusted-server-cli --target "$HOST_TARGET" commands::audit::ad_templates +``` + +Expected: new assertions fail on current projection/status/warning behavior. + +- [ ] **Step 3: Implement projection, comparison, and output changes** + +Filter `match_slots` with `render_gam_unit_path(...).map(...)` while building `ExpectedSlot`. Remove the unconditional client-side APS check. Compute confirmability before assigning status. Map typed media values to strings only in `to_slot_json`. Make JSON phase `Option` with `skip_serializing_if = "Option::is_none"`. Extend warnings with `evidence.warnings` after decode. + +Extend `is_terminal_control` with `0x202A..=0x202E` and `0x2066..=0x2069`. Apply `escape_terminal_text` to every human-facing page/config-derived field. + +- [ ] **Step 4: Verify GREEN** + +Run all three narrow commands again. + +Expected: all selected tests pass with no warnings. + +- [ ] **Step 5: Commit** + +```bash +git add crates/trusted-server-cli/src/ad_templates crates/trusted-server-cli/src/commands/audit/ad_templates.rs +git commit -m "Match ad template verification to runtime behavior" +``` + +## Task 3: Correct static CLI contracts and process exit semantics + +**Files:** + +- Modify: `crates/trusted-server-cli/src/commands/config/ad_templates.rs` +- Modify: `crates/trusted-server-cli/src/commands/audit/mod.rs` +- Modify: `crates/trusted-server-cli/src/run.rs` +- Modify: `crates/trusted-server-cli/src/main.rs` +- Modify: `crates/trusted-server-cli/Cargo.toml` +- Modify: `Cargo.lock` + +- [ ] **Step 1: Add failing parser, normalization, lint, and outcome tests** + +Add tests proving: + +- bare and full-URL forms normalize spaces, dot segments, tabs, queries, and fragments identically; +- `/r?to=https://example.com` remains a bare path; +- `check` requires exactly one expectation mode and rejects `--allow-extra-slots --expect-no-slots` through clap; +- `--method` accepts a valid `http::Method` and uses exact GET semantics; +- `lint` reports each invalid configured pattern; +- `explain` uses `gate.expected` even when providers are empty and prints provider state separately; +- `--edgezero-enabled` is rejected because the unsupported model is removed; +- bare `ts audit` displays help rather than a drifting manual error; +- parser coverage includes lint, explain, generate, verify profiles/options, and the no-`--adapter` contract; +- an assertion outcome maps to exit 1 and a tool error maps to exit 2. + +Use an explicit process outcome: + +```rust +#[derive(Debug, Clone, Copy, Eq, PartialEq)] +pub enum RunOutcome { + Success, + AssertionFailed, +} + +impl RunOutcome { + pub const fn exit_code(self) -> i32 { + match self { + Self::Success => 0, + Self::AssertionFailed => 1, + } + } +} +``` + +Tool failures remain `Err(String)` and therefore exit 2. Assertion commands write their failure to stderr before returning `AssertionFailed`, avoiding `log::error!` filtering. + +- [ ] **Step 2: Run parser/static tests and confirm RED** + +Run: + +```bash +HOST_TARGET="$(rustc -vV | awk '/host:/ {print $2}')" +cargo test --package trusted-server-cli --target "$HOST_TARGET" commands::config::ad_templates +cargo test --package trusted-server-cli --target "$HOST_TARGET" run::tests +``` + +Expected: current hand-rolled validation, normalization, and exit behavior fail the new tests. + +- [ ] **Step 3: Implement the CLI contract** + +Use a dummy HTTPS base with `Url::options().base_url(...)` for bare paths after anchored scheme detection on the pre-query slice. Add clap `ArgGroup`, `conflicts_with`, `arg_required_else_help`, typed `http::Method`, and browser settle validation. Add `http = { workspace = true }` to the CLI host dependencies. + +Return `RunOutcome` from dispatchable CI commands. Keep edgezero delegated errors as tool errors. Remove the unsupported EdgeZero flag/text and route gate output through `blocking_gates()`. + +- [ ] **Step 4: Verify GREEN** + +Run the two narrow commands again and confirm all tests pass. + +- [ ] **Step 5: Commit** + +```bash +git add Cargo.lock crates/trusted-server-cli/Cargo.toml crates/trusted-server-cli/src/main.rs crates/trusted-server-cli/src/run.rs crates/trusted-server-cli/src/commands/audit/mod.rs crates/trusted-server-cli/src/commands/config/ad_templates.rs +git commit -m "Define ad template CLI assertion contracts" +``` + +## Task 4: Make the injected collector bounded and behavior-preserving + +**Files:** + +- Modify: `crates/trusted-server-cli/src/commands/audit/ad_template_collector.js` +- Modify: `crates/trusted-server-cli/src/commands/audit/collector.rs` +- Modify: `crates/trusted-server-cli/src/commands/audit/browser.rs` + +- [ ] **Step 1: Add failing JavaScript-contract and decoder tests** + +Add tests/fixtures for an out-of-`u32` size beside a valid slot, a truthy `googletag.cmd` without `push`, multiple `cmd.push` arguments, 512-character capture limits, and non-enumerable/closure-local wrapping. Replace the existing `contains("cmd.push")` assertion with assertions that the no-op wrapper is absent. + +The JavaScript bounds are: + +```javascript +const __TS_MAX_STRING = 512 +function __ts_text(value) { + return String(value).slice(0, __TS_MAX_STRING) +} + +if (width > 4294967295 || height > 4294967295) return null +``` + +The setter must always retain the publisher value: + +```javascript +set(value) { + try { + internal = wrap(value) + } catch (error) { + internal = value + __ts_push(__ts_ev.warnings, { + code: "wrap_failed", + message: __ts_text(error), + }) + } +} +``` + +- [ ] **Step 2: Run the narrow tests and confirm RED** + +Run: + +```bash +HOST_TARGET="$(rustc -vV | awk '/host:/ {print $2}')" +cargo test --package trusted-server-cli --target "$HOST_TARGET" commands::audit::collector +cargo test --package trusted-server-cli --target "$HOST_TARGET" collector_payload +``` + +Expected: current script permits oversized integers and retains the behavior-changing wrapper. + +- [ ] **Step 3: Implement minimal collector changes** + +Guard all page-derived strings through `__ts_text`, enforce numeric upper bounds, delete the `cmd.push` wrapper, use a closure-local `WeakSet` for wrapped objects, and install wrapped functions with non-enumerable `Object.defineProperty`. Soften the header claim to “observes without capturing page data.” + +Before serde decode, stringify the evidence inside the page and return a small +sentinel instead of the payload when the serialized string exceeds 1 MiB +(`MAX_EVIDENCE_PAYLOAD_BYTES = 1_048_576`). On the Rust side, the sentinel +produces an `ad_evidence_too_large` warning and `ad_evidence: None`; it does not +fail navigation or the whole collection. This bounds CDP transfer and Rust +decode/allocation while preserving a precise operator diagnostic. + +- [ ] **Step 4: Verify GREEN** + +Run the narrow commands again and confirm all tests pass. + +- [ ] **Step 5: Commit** + +```bash +git add crates/trusted-server-cli/src/commands/audit/ad_template_collector.js crates/trusted-server-cli/src/commands/audit/collector.rs crates/trusted-server-cli/src/commands/audit/browser.rs +git commit -m "Bound browser ad template evidence collection" +``` + +## Task 5: Unify browser launch, session reuse, and settling + +**Files:** + +- Modify: `crates/trusted-server-cli/src/commands/audit/collector.rs` +- Modify: `crates/trusted-server-cli/src/commands/audit/browser.rs` +- Modify: `crates/trusted-server-cli/src/commands/audit/ad_templates.rs` +- Modify: `crates/trusted-server-cli/src/commands/audit/page.rs` +- Modify: `crates/trusted-server-cli/src/commands/audit/mod.rs` +- Modify: `crates/trusted-server-cli/src/commands/audit/generate/browser_collector.rs` +- Modify: `crates/trusted-server-cli/src/commands/audit/generate/collector.rs` +- Modify: `crates/trusted-server-cli/src/commands/audit/generate/mod.rs` + +- [ ] **Step 1: Add failing fake-collector and browser configuration tests** + +Cover one launch/session for multiple URLs, root included in the profile batch, page close on success/error, host-only `Path=/` cookies, explicit final-URL failure, same-host HTTP-to-HTTPS acceptance, host/downgrade/port refusal, new-headless 1280x800 defaults, headful/profile/proxy/consent parity, `$CHROME` parity, and generic/legacy default-on consent. + +Extend the trait with a default batch method so fakes remain simple: + +```rust +pub trait AuditCollector { + fn collect_page(&self, request: BrowserCollectRequest) -> Result; + + fn collect_pages( + &self, + requests: &[BrowserCollectRequest], + ) -> Vec> { + requests.iter().cloned().map(|request| self.collect_page(request)).collect() + } +} +``` + +The real browser implementation overrides `collect_pages` to create one runtime, +temporary profile, browser, handler, and sequentially closed pages. + +- [ ] **Step 2: Run narrow tests and confirm RED** + +Run: + +```bash +HOST_TARGET="$(rustc -vV | awk '/host:/ {print $2}')" +cargo test --package trusted-server-cli --target "$HOST_TARGET" commands::audit::browser +cargo test --package trusted-server-cli --target "$HOST_TARGET" commands::audit::ad_templates +cargo test --package trusted-server-cli --target "$HOST_TARGET" commands::audit::generate::browser_collector +``` + +Expected: verifier launches per URL, browser defaults diverge, and tabs/cookies/final URL handling fail new assertions. + +- [ ] **Step 3: Implement shared browser configuration and batching** + +Move executable resolution and launch-option construction into `browser.rs` as crate-visible helpers used by both collectors. Flatten shared browser options into generate and verify, while keeping generation-only pacing/crawl flags local. Build cookies with explicit domain from `url.host_str()` and `path = Some("/".to_string())`; do not set `url` simultaneously. + +In each page collector, capture the inner result, always call bounded `page.close().await`, then return the captured result. Batch verify requests via `collect_pages`. Include the root in each profile's batch rather than collecting it in a throwaway session. Use `spawn_blocking` for scraper analysis before folding results. + +- [ ] **Step 4: Bound post-navigation work and correct settle semantics** + +Install `performance.setResourceTimingBufferSize(100000)` before navigation. Make `settle` return warnings and wrap every `evaluate`, URL/title read, scroll operation, and evidence read in a per-operation timeout. Accrue quiet only after `document.readyState` is `interactive` or `complete`; sleep `min(remaining_quiet, 250ms)` so short quiet values are honored. Treat `wait_for_navigation` timeout as a warning after successful `goto`. + +Propagate GPT/link/sitemap evaluation errors as notes, set `await_promise` for sitemap discovery, and warn when only the main frame is inspected while child frames exist. + +- [ ] **Step 5: Verify GREEN** + +Run all three narrow commands again. If Chrome is available, also run: + +```bash +HOST_TARGET="$(rustc -vV | awk '/host:/ {print $2}')" +cargo test --package trusted-server-cli --target "$HOST_TARGET" commands::audit::browser::tests:: -- --ignored --test-threads=1 +``` + +Expected: unit/fake tests pass; browser fixtures execute and pass when Chrome exists. + +- [ ] **Step 6: Commit** + +```bash +git add crates/trusted-server-cli/src/commands/audit +git commit -m "Share browser sessions across ad template audits" +``` + +## Task 6: Preserve crawl evidence and make inference conservative + +**Files:** + +- Modify: `crates/trusted-server-cli/src/commands/audit/generate/evidence.rs` +- Modify: `crates/trusted-server-cli/src/commands/audit/generate/gpt_slots.rs` +- Modify: `crates/trusted-server-cli/src/commands/audit/generate/page_patterns.rs` +- Modify: `crates/trusted-server-cli/src/commands/audit/generate/crawl_plan.rs` +- Modify: `crates/trusted-server-cli/src/commands/audit/generate/unit_template.rs` +- Modify: `crates/trusted-server-cli/src/commands/audit/generate/mod.rs` + +- [ ] **Step 1: Add failing inference tests** + +Add focused tests for: + +- `annonsü1`/`annonsü2` and `ünicode-ad-a`/`ünicode-ad-b` prefixes; +- desktop-empty/mobile-present and the inverse; +- two disjoint unrelated placements retained, two with a useful prefix or three fragments refused; +- same-page normalized UUID collisions retained with raw div IDs and all formats; +- 16+ digit numeric stable segments retained; +- comma-separated SRA `dids` ignored; +- locale `/en` pattern emitted as `/en` and every emitted glob matches its source path; +- glob metacharacters escaped with `glob::Pattern::escape`; +- percent-encoded noise/extension paths and `.html`/`.htm`/`.php` treatment; +- dropped-section notes capped at ten plus “and N more”; +- both ambiguous template rows result in explicit `Refuse`; +- real crawl evidence can infer `section_segment = 1`; +- refused slots do not appear in rendered output and their reasons appear in notes. + +- [ ] **Step 2: Run narrow tests and confirm RED** + +Run: + +```bash +HOST_TARGET="$(rustc -vV | awk '/host:/ {print $2}')" +cargo test --package trusted-server-cli --target "$HOST_TARGET" commands::audit::generate::evidence +cargo test --package trusted-server-cli --target "$HOST_TARGET" commands::audit::generate::gpt_slots +cargo test --package trusted-server-cli --target "$HOST_TARGET" commands::audit::generate::page_patterns +cargo test --package trusted-server-cli --target "$HOST_TARGET" commands::audit::generate::crawl_plan +cargo test --package trusted-server-cli --target "$HOST_TARGET" commands::audit::generate::unit_template +``` + +Expected: each new regression reproduces its review finding. + +- [ ] **Step 3: Implement evidence-preserving discovery** + +Use the last matching `char_indices` byte boundary for shared prefixes. Remove an empty-page marker whenever a later profile yields slots. Require `(useful shared prefix || group size >= 3)` before classifying disjoint same-shape slots as fragments; emit an ambiguity diagnostic otherwise. + +Group normalized collisions within a page before deduplication. When a group has multiple raw div IDs, keep raw entries, make their generated IDs unique, and attach a collision note. Restrict ephemeral hex matching to tokens containing at least one `a..f`, or an explicit UUID shape; never treat all-digit identifiers as hashes. Reject gampad fallback when parsed `dids` contains a comma. + +- [ ] **Step 4: Implement conservative patterns/templates** + +Emit the observed short path for locale landing pages, escape literal prefixes, decode only for filtering while retaining encoded request paths for matching, and cap notes. Teach crawl planning to carry/infer the section depth used by page-pattern generation. + +Delete the tautological witness check and move its explanatory invariant into `analyse_slot` docs. Keep the existing conservative `Refuse` result for non-derivable slugs and unwitnessed roots. Filter all `Refuse` decisions before `RenderSlot` creation and push each reason into notes. + +- [ ] **Step 5: Verify GREEN** + +Run all five narrow commands again, then: + +```bash +HOST_TARGET="$(rustc -vV | awk '/host:/ {print $2}')" +cargo test --package trusted-server-cli --target "$HOST_TARGET" commands::audit::generate +``` + +Expected: the generate module suite passes. + +- [ ] **Step 6: Commit** + +```bash +git add crates/trusted-server-cli/src/commands/audit/generate +git commit -m "Preserve ad template crawl evidence" +``` + +## Task 7: Make slot persistence and dry-run output safe + +**Files:** + +- Modify: `Cargo.toml` +- Modify: `Cargo.lock` +- Modify: `crates/trusted-server-cli/Cargo.toml` +- Modify: `crates/trusted-server-cli/src/commands/audit/generate/slot_toml.rs` +- Modify: `crates/trusted-server-cli/src/commands/audit/generate/mod.rs` +- Modify: `crates/trusted-server-cli/src/commands/audit/generate/validate.rs` + +- [ ] **Step 1: Add failing persistence tests** + +Cover: + +- trailing comments after the final slot; +- a multiline string line beginning `[foo]`; +- an array continuation beginning `[300, 250]`; +- non-contiguous slot tables; +- byte-identical unrelated sections/comments and CRLF preservation; +- end-to-end `--replace` through `run_update_slots`; +- dry-run source file byte identity; +- stdout contains only a zero-context unified diff of managed + creative-opportunities changes and does not contain `admin_password` or + unrelated config; +- notes/rollback warning go to stderr; +- a concurrent source edit between initial read and write is refused; +- rerun unions formats and reports broad-prefix collapse. + +Change `run_update_slots` to accept separate writers: + +```rust +pub(crate) fn run_update_slots( + request: &UpdateSlotsRequest<'_>, + collectors: &[(&str, &dyn AuditCollector)], + out: &mut dyn Write, + err: &mut dyn Write, +) -> CliResult<()>; +``` + +- [ ] **Step 2: Run persistence tests and confirm RED** + +Run: + +```bash +HOST_TARGET="$(rustc -vV | awk '/host:/ {print $2}')" +cargo test --package trusted-server-cli --target "$HOST_TARGET" commands::audit::generate::slot_toml +cargo test --package trusted-server-cli --target "$HOST_TARGET" update_slots +``` + +Expected: current line scanner corrupts/preserves incorrectly and dry-run leaks the complete config. + +- [ ] **Step 3: Implement a TOML-aware managed edit** + +Parse the source as `DocumentMut` and update the complete managed field set: +`creative_opportunities.slot`, `gam_network_id`, `section_root`, and +`section_segment`. Insert the generated array-of-tables and upsert only scalar +values that generation actually inferred. A generated `None` preserves the +existing scalar on both merge and `--replace`; absence of fresh evidence is +never an instruction to delete operator configuration. Retain decorations on +all other items. Before returning, parse both documents and compare canonical +clones with all four managed fields removed; return an error if any other item +differs. Preserve CRLF after serialization. Add regression cases in Step 1 for +an unresolved network ID and literal-only rerun retaining existing +`gam_network_id`/section policy. + +Document `splice_creative_slots` at its definition and remove the orphaned comments. Replace the `let _ = network_id` presence check with `keys.network_id.is_none()` logic. + +- [ ] **Step 4: Implement secret-safe dry-run and stale-read protection** + +Add `similar` as a workspace/CLI dependency and render a zero-context unified +diff between the old and new managed creative-opportunities projection. The +projection contains only `gam_network_id`, `section_root`, `section_segment`, +and the slot array, so every generated scalar change is visible without +including unrelated operator keys: + +```rust +let diff = similar::TextDiff::from_lines(old_managed, new_managed); +writeln!(out, "{}", diff.unified_diff().context_radius(0).header("configured creative opportunities", "generated creative opportunities"))?; +``` + +Send all notes to `err`. Immediately before atomic rename, re-read the config and compare it with the original bytes; refuse on mismatch. Do not perform this check on dry-run because no write occurs. + +In `merge_render_slots`, union discovered formats into a matching existing slot and count how many discovered slots map to each existing prefix; report counts greater than one. + +- [ ] **Step 5: Verify GREEN** + +Run both narrow commands again and confirm all tests pass. + +- [ ] **Step 6: Commit** + +```bash +git add Cargo.toml Cargo.lock crates/trusted-server-cli/Cargo.toml crates/trusted-server-cli/src/commands/audit/generate +git commit -m "Preserve operator config during slot generation" +``` + +## Task 8: Complete documentation, test hygiene, and CI enforcement + +**Files:** + +- Modify: `docs/guide/cli.md` +- Modify: `scripts/test-cli.sh` +- Modify: `.github/workflows/test.yml` +- Modify: `crates/trusted-server-cli/src/lib.rs` +- Modify: touched Rust tests and comments under `crates/trusted-server-cli/src/` + +- [ ] **Step 1: Add/restore parser and CI guard tests** + +Restore the `audit` no-`--adapter` parser test. Add a script contract that sets `TS_AUDIT_BROWSER_TESTS=1`; browser fixture tests panic when that variable is set and Chrome cannot be resolved. Configure the workflow with a browser setup action or the runner's installed Chrome path and export `CHROME` before `scripts/test-cli.sh`. + +- [ ] **Step 2: Replace sensitive-looking fixtures and stale assertions** + +Replace sensitive or customer-shaped fixtures introduced by this PR with fictional network IDs, publisher names, URL shapes, and neutral div tokens. Update comments to describe shapes rather than customers. + +Correct all touched `expect` messages to start with `should`, remove redundant crate/file `dead_code` allowances and annotate only genuinely deferred fields, reorder `Audit`, simplify the Prebid query parser so keys—not substrings—are matched, and bind legacy URLs directly without an impossible `expect`. + +- [ ] **Step 3: Document the complete operator contract** + +In `docs/guide/cli.md`, document: + +- `config ad-templates lint|match|check|explain` and every flag; +- shared `--app-config`, `--manifest`, and `--no-env` behavior; +- `audit ad-templates generate|verify` browser/profile/proxy/consent/settle flags; +- dry-run stdout diff versus stderr notes; +- exit 0 success, exit 1 assertion drift, exit 2 tool/configuration error; +- refused slots are omitted with reasons; +- locale-prefixed inference and section depth; +- `Unconfirmable` strict behavior and optional evidence phase. + +Update the existing design/output examples where the wire contract changed. + +- [ ] **Step 4: Run format and focused checks** + +Run: + +```bash +cargo fmt --all -- --check +cd docs && npm run format +``` + +Expected: both commands exit 0. + +- [ ] **Step 5: Commit** + +```bash +git add .github/workflows/test.yml scripts/test-cli.sh docs crates/trusted-server-cli/src +git commit -m "Document and enforce ad template audit contracts" +``` + +## Task 9: Run full verification and repair regressions + +**Files:** + +- Modify only files implicated by a failing check. + +- [ ] **Step 1: Run format and CLI/browser tests** + +```bash +cargo fmt --all -- --check +./scripts/test-cli.sh +``` + +Expected: exit 0; browser fixture output shows tests executed rather than skipped. + +- [ ] **Step 2: Run repository target suites** + +```bash +cargo test-fastly +cargo test-axum +cargo test-cloudflare +cargo test-spin +``` + +Expected: all suites exit 0. + +- [ ] **Step 3: Run all target-matched clippy gates** + +```bash +cargo clippy-fastly +cargo clippy-axum +cargo clippy-cloudflare +cargo clippy-cloudflare-wasm +cargo clippy-spin-native +cargo clippy-spin-wasm +cargo clippy --manifest-path crates/trusted-server-cli/Cargo.toml --target "$(rustc -vV | sed -n 's/host: //p')" --all-targets -- -D warnings +``` + +Expected: all commands exit 0 with no warnings. + +- [ ] **Step 4: Run cross-adapter parity gates** + +```bash +cargo fmt --manifest-path crates/trusted-server-integration-tests/Cargo.toml -- --check +cargo test --manifest-path crates/trusted-server-integration-tests/Cargo.toml --test parity +cargo clippy --manifest-path crates/trusted-server-integration-tests/Cargo.toml --all-targets -- -D warnings +``` + +Expected: formatting, parity tests, and integration-test clippy exit 0. + +- [ ] **Step 5: Run JavaScript and documentation checks** + +```bash +cd crates/trusted-server-js/lib && npx vitest run && npm run format && node build-all.mjs +cd ../../.. && cd docs && npm run format +``` + +Expected: tests/build/format exit 0. + +- [ ] **Step 6: Inspect the final diff against the review** + +Run: + +```bash +git diff --check origin/main...HEAD +git status --short +``` + +Walk the 28-thread traceability table and every summary category in the design spec. Confirm each has a code/doc/test resolution or an evidence-backed response. + +- [ ] **Step 7: Commit any verification-only corrections** + +If verification required changes, inspect `git diff --name-only`, stage each +listed path explicitly (never `git add .`), and commit them as `Resolve ad +template review regressions`. Record those exact paths in the execution log. +Skip this commit when verification required no changes. + +## Task 10: Publish and answer GitHub review threads + +**Files:** + +- No repository files unless publication reveals a conflict. + +- [ ] **Step 1: Push the verified branch** + +```bash +git push origin feature/ts-cli-ad-templates +``` + +Expected: push succeeds and PR 823 shows the verified head commit. + +- [ ] **Step 2: Correct the PR description** + +Change the legacy alias statement to say bare `ts audit ` aliases to `ts audit generate `. Preserve all unrelated PR-body content. + +- [ ] **Step 3: Reply to every inline thread** + +For each ID in the spec traceability table, post through: + +```bash +gh api repos/IABTechLab/trusted-server/pulls/823/comments//replies -f body='' +``` + +Each reply must name the concrete behavior changed and, where useful, the focused test. For question threads, state the chosen behavior: union formats and diagnose broad prefixes; default consent assumption on; keep conservative refusal and align docs; allow only same-host HTTP-to-HTTPS upgrades; remove the unsupported EdgeZero model. + +- [ ] **Step 4: Verify publication** + +Query PR 823's head SHA, review comments, checks, and unresolved threads. Confirm all 28 inline comments have one reply and no reply claims a fix absent from the pushed diff. + +- [ ] **Step 5: Report the result** + +Summarize commits, verification commands, any environment limitation, PR link, and thread reply count. Do not claim checks pass without fresh output from Task 9. diff --git a/docs/superpowers/plans/2026-08-18-pre-navigation-cookie-install.md b/docs/superpowers/plans/2026-08-18-pre-navigation-cookie-install.md new file mode 100644 index 000000000..c574c4104 --- /dev/null +++ b/docs/superpowers/plans/2026-08-18-pre-navigation-cookie-install.md @@ -0,0 +1,49 @@ +# Pre-navigation Cookie Installation Implementation Plan + +> **For agentic workers:** REQUIRED SUB-SKILL: Use superpowers:subagent-driven-development (recommended) or superpowers:executing-plans to implement this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking. + +**Goal:** Allow domain/path-scoped operator cookies to be installed before the audit's first navigation. + +**Architecture:** Add one browser-level cookie installation helper beside `host_cookie`, and call it before creating each audit page. Preserve explicit host-only and root-path scope while avoiding `Page::set_cookie`'s `about:blank` validation. + +**Tech Stack:** Rust, chromiumoxide/CDP, Tokio, Cargo tests + +--- + +### Task 1: Reproduce the pre-navigation failure + +**Files:** + +- Modify: `crates/trusted-server-cli/src/commands/audit/browser.rs` + +- [ ] Add a Chrome-backed test that installs a host-only cookie before navigating away from `about:blank` and asserts it reaches the first document. +- [ ] Exercise the existing `BrowserCollector` end-to-end against a local HTTP fixture, supplying the cookie through `BrowserCollectRequest`, so the RED test compiles before the fix exists. +- [ ] Run `cargo test_cli_macos commands::audit::browser::tests::supplied_cookie_reaches_first_navigation -- --ignored --exact --nocapture` and confirm it fails with `Blank page can not have cookie`. +- [ ] Add a Chrome-backed error test against the wished-for `set_browser_cookies` API, using an invalid cookie name, and assert the error contains the name but not the secret value. +- [ ] Run that error test and confirm RED because `set_browser_cookies` does not exist yet. + +### Task 2: Install cookies at browser scope + +**Files:** + +- Modify: `crates/trusted-server-cli/src/commands/audit/browser.rs` +- Modify: `crates/trusted-server-cli/src/commands/audit/generate/browser_collector.rs` + +- [ ] Add `set_browser_cookies(&Browser, &[(String, String)], &Url) -> Result<(), String>` beside `host_cookie`; install one cookie per browser call so failures retain name-only context without exposing values. +- [ ] Invoke it before page creation in both collectors and remove page-level cookie installation. +- [ ] Run `cargo test_cli_macos commands::audit::browser::tests::supplied_cookie_reaches_first_navigation -- --ignored --exact --nocapture` and confirm it passes. +- [ ] Run the focused error test and confirm it passes. + +### Task 3: Verify the change + +**Files:** + +- Modify: `crates/trusted-server-cli/src/commands/audit/browser.rs` +- Modify: `crates/trusted-server-cli/src/commands/audit/generate/browser_collector.rs` + +- [ ] Run `cargo test_cli_macos commands::audit::browser::tests`. +- [ ] Run `cargo test_cli_macos commands::audit::generate::browser_collector::tests`. +- [ ] Run `./scripts/test-cli.sh` to exercise the portable host-target suite and ignored browser fixtures. +- [ ] Run `cargo fmt --all -- --check`. +- [ ] Run `cargo clippy --package trusted-server-cli --target aarch64-apple-darwin --all-targets -- -D warnings`. +- [ ] Inspect the diff to confirm no cookie values are logged and `fastly.toml` remains untouched. diff --git a/docs/superpowers/plans/2026-08-19-ad-template-generation-progress.md b/docs/superpowers/plans/2026-08-19-ad-template-generation-progress.md new file mode 100644 index 000000000..ca4245314 --- /dev/null +++ b/docs/superpowers/plans/2026-08-19-ad-template-generation-progress.md @@ -0,0 +1,167 @@ +# Ad-template Generation Progress Implementation Plan + +> **For agentic workers:** REQUIRED SUB-SKILL: Use superpowers:subagent-driven-development (recommended) or superpowers:executing-plans to implement this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking. + +**Goal:** Show immediate, safe, profile-aware progress while `ts audit ad-templates generate` performs a long browser crawl. + +**Architecture:** Add typed progress events to the `AuditCollector` boundary so the browser can report work before buffered page results are returned. Render and flush those events from `run_update_slots` on stderr, using only URL paths. Preserve crawl/progress errors over teardown errors while always closing and waiting for Chrome. + +**Tech Stack:** Rust 2024, `std::io::Write`, existing `url`, `tokio`, `chromiumoxide`, and CLI test helpers; no new dependency. + +--- + +### Task 1: Define and render safe progress events + +**Files:** + +- Modify: `crates/trusted-server-cli/src/commands/audit/generate/collector.rs` +- Modify: `crates/trusted-server-cli/src/commands/audit/generate/mod.rs` + +- [ ] **Step 1: Write failing renderer and writer tests** + +Add tests in `generate/mod.rs` for events covering launch, `1/?`, `2/17`, and finalization. Assert that `https://user:pass@publisher.example/news?token=secret#fragment` renders only `/news`, terminal control bytes are escaped, stdout remains untouched, and a counting writer records an explicit `flush()`. Add writers that fail independently on `write()` and `flush()` and assert a CLI output error. + +- [ ] **Step 2: Run the focused tests and confirm RED** + +Run: + +```bash +cargo test --package trusted-server-cli --target aarch64-apple-darwin progress -- --nocapture +``` + +Expected: FAIL because the progress event and renderer do not exist. + +- [ ] **Step 3: Add the progress model and renderer** + +In `collector.rs`, define a small event enum and callback type: + +```rust +pub(crate) enum CollectionProgress<'a> { + Launching, + Loading { + current: usize, + total: Option, + url: &'a Url, + }, + Planning, + Finalizing, +} + +pub(crate) type ProgressSink<'a> = + &'a mut dyn FnMut(CollectionProgress<'_>) -> CliResult<()>; +``` + +Add concise doc comments to the enum, every variant, and the callback alias. The +callback documentation must state that returning an error stops new collection +work but does not bypass an already-launched browser's finalization/close/wait. + +In `generate/mod.rs`, add a `write_collection_progress` helper that accepts a profile label, formats only `url.path()` (or `/` when empty), sanitizes it with `escape_terminal_text`, writes one line to stderr, and immediately calls `flush()`. Render and test `Planning` between the root load and subsequent page loads. + +- [ ] **Step 4: Run the focused tests and confirm GREEN** + +Run the command from Step 2. Expected: all progress renderer/writer tests pass. + +### Task 2: Propagate progress through collectors with teardown-safe failures + +**Files:** + +- Modify: `crates/trusted-server-cli/src/commands/audit/generate/collector.rs` +- Modify: `crates/trusted-server-cli/src/commands/audit/generate/browser_collector.rs` +- Modify: `scripts/test-cli.sh` + +- [ ] **Step 1: Write failing collector tests** + +Test default `collect_pages` and `collect_site` count semantics, including an attempted page whose collection fails. The exact dynamic-site sequence is root `1/?`, planning, then follow-ups `2/total` through `total/total`; totals include the root and failed attempts advance the count. Add a Chrome-backed test whose progress callback fails during collection. It must return the progress error only after the browser teardown path completes. Extend the existing result-combination unit tests to cover first-error preservation across a collection/planning error, a later finalization-progress error, close error, and wait error, while proving finalization, close, and wait were all attempted. + +- [ ] **Step 2: Run the focused tests and confirm RED** + +Run: + +```bash +cargo test --package trusted-server-cli --target aarch64-apple-darwin commands::audit::generate::browser_collector::tests -- --nocapture +cargo test --package trusted-server-cli --target aarch64-apple-darwin commands::audit::generate::collector::tests -- --nocapture +``` + +Expected: FAIL because collectors do not accept or emit progress callbacks. + +- [ ] **Step 3: Add callbacks to the collector boundary** + +Extend `collect_pages` and `collect_site` with `ProgressSink`. Default collectors emit `Loading` before each page. The root of a dynamically planned site emits `current: 1, total: None`, followed by `Planning`; after planning, default `collect_site` iterates follow-ups itself with an explicit offset so they report `2/total` onward. Fixed batches emit totals including the root, and failed attempts still consume their position. + +Pass the callback into `with_browser`. Adapt `BrowserAuditCollector::collect_page` with an explicit no-op progress sink because single-page artifact generation has no command progress writer. Emit `Launching` before browser launch, `Loading` immediately before each navigation, `Planning` immediately before invoking the root planner, and `Finalizing` before close/wait. Track only the first crawl/progress error: on callback failure, stop scheduling pages, still attempt finalization, `browser.close()`, and `browser.wait()`, then return that first error ahead of teardown errors. + +Extend `scripts/test-cli.sh` with a second ignored-test filter for +`commands::audit::generate::browser_collector::tests::` so the new Chrome-backed +progress-failure test is actually executed under `TS_AUDIT_BROWSER_TESTS=1` and +single-threaded, alongside the existing three browser audit fixtures. + +- [ ] **Step 4: Run unit and Chrome-backed tests and confirm GREEN** + +Run the focused command, then: + +```bash +./scripts/test-cli.sh +``` + +Expected: collector unit tests and all four Chrome-backed tests pass. + +### Task 3: Wire profile-aware progress into generation + +**Files:** + +- Modify: `crates/trusted-server-cli/src/commands/audit/generate/mod.rs` + +- [ ] **Step 1: Write failing generation tests** + +Update `run_update_slots` tests to assert the stderr buffer contains progress for the first profile's `1/?` root, planning, later known totals, the second profile's `1/total` root, and finalization. Assert dry-run diff/success output on stdout contains no progress lines. Add an ordering test with a shared observable writer and fake collector: from inside `collect_site`, after invoking and flushing the progress callback but before returning, assert the progress bytes are already visible. + +- [ ] **Step 2: Run the focused tests and confirm RED** + +Run: + +```bash +cargo test --package trusted-server-cli --target aarch64-apple-darwin update_slots -- --nocapture +``` + +Expected: FAIL because `run_update_slots` does not provide progress callbacks. + +- [ ] **Step 3: Connect callbacks and profiles** + +Create a progress closure for the first profile and pass it to `collect_site`. Pass `err` through `crawl_sections`, create a closure for each later profile, and pass it to `collect_pages`. Keep notes and final summary behavior unchanged. + +- [ ] **Step 4: Run the focused tests and confirm GREEN** + +Run the command from Step 2. Expected: all generation tests pass and progress appears only in stderr. + +### Task 4: Verify and ship + +**Files:** + +- Verify all modified files plus the two design documents. + +- [ ] **Step 1: Format and lint** + +```bash +cargo fmt --all -- --check +cargo clippy --package trusted-server-cli --target aarch64-apple-darwin --all-targets --all-features -- -D warnings +git diff --check +cd docs && npm run format +``` + +Expected: all commands exit 0. + +- [ ] **Step 2: Run the complete local CLI suite** + +```bash +./scripts/test-cli.sh +``` + +Expected: unit, config, proxy, documentation, and Chrome-backed tests pass. + +- [ ] **Step 3: Review the scoped diff** + +Confirm no cookie values, real publisher data, or changes to the pre-existing `fastly.toml` modification are included. Request an independent code review and address concrete findings. + +- [ ] **Step 4: Commit and push** + +Stage only the progress implementation and its design/plan documents. Commit with `Show ad-template generation progress`, push `feature/ts-cli-ad-templates`, and confirm local HEAD matches the remote branch. diff --git a/docs/superpowers/plans/2026-08-19-refuse-volatile-div-collisions.md b/docs/superpowers/plans/2026-08-19-refuse-volatile-div-collisions.md new file mode 100644 index 000000000..1dca98982 --- /dev/null +++ b/docs/superpowers/plans/2026-08-19-refuse-volatile-div-collisions.md @@ -0,0 +1,78 @@ +# Refuse Volatile Div-ID Collisions Implementation Plan + +> **For agentic workers:** REQUIRED SUB-SKILL: Use superpowers:subagent-driven-development (recommended) or superpowers:executing-plans to implement this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking. + +**Goal:** Prevent `ts audit ad-templates generate --replace` from writing exact per-render div IDs when several live elements normalize to one runtime prefix. + +**Architecture:** Keep collision detection in GPT discovery, where normalized and raw IDs are both available. On the first distinct collision, remove the tentatively accepted normalized slot and mark the group ambiguous; suppress all later members and emit one actionable diagnostic. Carry a separate evidence-present bit into `EvidenceTable` so collision-only pages are not classified as bot challenges. + +**Tech Stack:** Rust, Chromium GPT evidence model, built-in Rust test framework. + +--- + +### Task 1: Specify refusal behavior + +**Files:** + +- Modify: `crates/trusted-server-cli/src/commands/audit/generate/gpt_slots.rs` +- Modify: `crates/trusted-server-cli/src/commands/audit/generate/evidence.rs` + +- [ ] Add assertions for documented `DiscoveredSlots::had_slot_evidence` and run one focused test to observe the expected missing-field compile failure. +- [ ] Add only the documented field with its derived/default false value so behavioral tests can compile; do not wire discovery or classification yet. +- [ ] Change the same-page collision test to require zero emitted slots and one refusal diagnostic. +- [ ] Assert the diagnostic names `ad-in_content`, explains that a broad prefix resolves only one element and raw IDs are volatile, and tells the operator to expose distinct stable IDs. +- [ ] Rename the test to `same_page_hex_normalization_collision_is_refused`. +- [ ] Extend `repeated_raw_div_after_a_normalization_collision_is_deduplicated` with repeats of both initial raw IDs and a third distinct ID; require zero slots and one diagnostic. +- [ ] Add `request_normalization_collision_is_refused`; require zero slots, one diagnostic with the same prefix/safety/action content, true evidence, and a surviving request-derived network ID. +- [ ] Add `ambiguous_registry_stem_still_suppresses_request_fallback` and require no slot resurrection. +- [ ] Require every registry/request collision test to assert `had_slot_evidence` is true. +- [ ] Add `collision_only_page_is_not_classified_as_empty` using a discovered collision result. +- [ ] Run `cargo test --package trusted-server-cli --target aarch64-apple-darwin same_page_hex_normalization_collision_is_refused` and confirm RED because two raw slots remain. +- [ ] Run `cargo test --package trusted-server-cli --target aarch64-apple-darwin repeated_raw_div_after_a_normalization_collision_is_deduplicated` and confirm RED because raw slots remain. +- [ ] Run `cargo test --package trusted-server-cli --target aarch64-apple-darwin request_normalization_collision_is_refused` and confirm RED because request-derived raw slots remain. +- [ ] Run `cargo test --package trusted-server-cli --target aarch64-apple-darwin ambiguous_registry_stem_still_suppresses_request_fallback` and confirm RED because the ambiguous registry group remains deployable. +- [ ] Run `cargo test --package trusted-server-cli --target aarch64-apple-darwin collision_only_page_is_not_classified_as_empty` and confirm RED because collision-only evidence is classified as empty. + +### Task 2: Refuse ambiguous collision groups + +**Files:** + +- Modify: `crates/trusted-server-cli/src/commands/audit/generate/gpt_slots.rs` +- Modify: `crates/trusted-server-cli/src/commands/audit/generate/evidence.rs` + +- [ ] Replace raw-ID preservation with a collision result that distinguishes first ambiguity from later members. +- [ ] Remove the initially accepted normalized slot when ambiguity is first proven. +- [ ] Suppress the colliding and subsequent raw members. +- [ ] Emit one message naming the prefix, both unsafe representations, and the publisher-markup action. +- [ ] Set `had_slot_evidence` for any otherwise usable registry/request candidate and use it in empty-page classification. +- [ ] Run `cargo test --package trusted-server-cli --target aarch64-apple-darwin normalization_collision` and confirm the registry and request collision tests GREEN. +- [ ] Run `cargo test --package trusted-server-cli --target aarch64-apple-darwin ambiguous_registry_stem_still_suppresses_request_fallback` and confirm registry precedence GREEN. +- [ ] Run `cargo test --package trusted-server-cli --target aarch64-apple-darwin collision_only_page_is_not_classified_as_empty` and confirm GREEN. + +### Task 3: Verify and deliver + +**Files:** + +- Verify: `crates/trusted-server-cli/src/commands/audit/generate/gpt_slots.rs` +- Verify: `crates/trusted-server-cli/src/commands/audit/generate/evidence.rs` +- Verify: `docs/superpowers/specs/2026-08-19-refuse-volatile-div-collisions-design.md` +- Verify: `docs/superpowers/plans/2026-08-19-refuse-volatile-div-collisions.md` + +- [ ] Run `cargo fmt --all -- --check`. +- [ ] Run `./scripts/test-cli.sh aarch64-apple-darwin`. +- [ ] Run `cargo clippy --package trusted-server-cli --target aarch64-apple-darwin --all-targets --all-features -- -D warnings`. +- [ ] Run `cd docs && npm run format`. +- [ ] Run `git diff --check` and inspect the scoped diff. +- [ ] Commit and push the fix to `feature/ts-cli-ad-templates`. + +### Task 4: Refuse a known single-observation volatile family + +**Files:** + +- Modify: `crates/trusted-server-cli/src/commands/audit/generate/gpt_slots.rs` + +- [ ] Add failing registry and request tests for a single `__` observation. +- [ ] Add a recognizer keyed on the token shape — ten or more leading digits followed by more alphanumerics — in any position that has a non-empty family prefix before it and placement content after it. +- [ ] Omit matching slots while preserving evidence/network discovery and emit one deduplicated actionable diagnostic naming the family prefix. +- [ ] Add negative tests proving IDs with no token, a bare digit run, or a trailing token remain eligible. +- [ ] Run the focused tests, then repeat Task 3 verification and delivery. diff --git a/docs/superpowers/plans/2026-08-21-pr-823-round-5-review-resolution.md b/docs/superpowers/plans/2026-08-21-pr-823-round-5-review-resolution.md new file mode 100644 index 000000000..9f3735d91 --- /dev/null +++ b/docs/superpowers/plans/2026-08-21-pr-823-round-5-review-resolution.md @@ -0,0 +1,307 @@ +# PR 823 Round-5 Review Resolution Implementation Plan + +> **For agentic workers:** REQUIRED SUB-SKILL: Use superpowers:subagent-driven-development (recommended) or superpowers:executing-plans to implement this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking. + +**Goal:** Resolve every actionable finding in PR 823 review `4989897698` while preserving generation compatibility and enforcing root-less template safety. + +**Architecture:** Keep browser-option defaults and legacy clap compatibility at the CLI boundary, carry borrowed-root evidence through template inference, and reject unsafe overrides before rendering. Improve diagnostics and validation at their existing seams, then pin cross-language and documentation invariants with focused tests. + +**Tech Stack:** Rust 2024, clap 4 derive, `url`, `toml`, embedded JavaScript, mdBook/VitePress documentation. + +--- + +## File Map + +- `crates/trusted-server-cli/src/commands/audit/collector.rs`: generation browser default constants and option defaults. +- `crates/trusted-server-cli/src/commands/audit/mod.rs`: hidden legacy browser arguments, early TOML validation, conversion to generation arguments. +- `crates/trusted-server-cli/src/run.rs`: clap contract tests. +- `crates/trusted-server-cli/src/commands/audit/generate/browser_collector.rs`: collector defaults and formatting. +- `crates/trusted-server-cli/src/commands/audit/generate/unit_template.rs`: borrowed-root inference metadata and root-gap refusal reasons. +- `crates/trusted-server-cli/src/commands/audit/generate/mod.rs`: redirect output, profile-scoped notes, merge-policy validation, explicit-pattern refusal. +- `crates/trusted-server-cli/src/commands/audit/generate/gpt_slots.rs`: timestamp-shaped volatile token recognition. +- `crates/trusted-server-cli/src/commands/audit/browser.rs`: Rust/JavaScript evidence-cap invariant test. +- `crates/trusted-server-cli/src/commands/audit/page.rs`: accurate final-URL/terminal-escaping test claims. +- `docs/guide/cli.md` and the volatile-collision design/plan: operator and historical documentation corrections. + +### Task 1: Restore generation browser defaults and legacy clap isolation + +**Files:** + +- Modify: `crates/trusted-server-cli/src/commands/audit/collector.rs` +- Modify: `crates/trusted-server-cli/src/commands/audit/generate/browser_collector.rs` +- Modify: `crates/trusted-server-cli/src/commands/audit/mod.rs` +- Modify: `crates/trusted-server-cli/src/run.rs` + +- [ ] **Step 1: Add failing clap and default tests** + +Add parser coverage proving that `ts audit --help` does not advertise generation +browser flags, `ts audit --chrome /tmp/chrome generate ...` is rejected, and the +legacy `ts audit --chrome ... --settle-max-ms ...` form still parses and +reaches `GenerateArgs`. Add a generation-option default assertion for 750 ms and +12,000 ms. + +- [ ] **Step 2: Run the focused tests and confirm RED** + +Run: + +```bash +cargo test --package trusted-server-cli --target aarch64-apple-darwin run::tests::audit_ -- --nocapture +cargo test --package trusted-server-cli --target aarch64-apple-darwin commands::audit::tests::legacy_ -- --nocapture +``` + +Expected: the hidden/help and 12-second assertions fail on the current branch. + +- [ ] **Step 3: Implement one generation-default source and legacy mirror** + +Define generation-specific constants in `collector.rs` and use them in clap +attributes and `GenerateBrowserOpts::default`: + +```rust +pub(crate) const GENERATE_SETTLE_QUIET_MS: u64 = 750; +pub(crate) const GENERATE_SETTLE_MAX_MS: u64 = 12_000; +``` + +Use those constants in `BrowserAuditCollector::default`. Replace the flattened +`GenerateBrowserOpts` under `LegacyGenerateArgs` with `LegacyBrowserOpts`, whose +seven fields each use `hide = true, requires = "legacy_url"`. Implement +`From<&LegacyBrowserOpts> for GenerateBrowserOpts` and use it in +`legacy_generate_args`. Add the missing blank line between collector methods. + +- [ ] **Step 4: Re-run focused tests and confirm GREEN** + +Run the Step 2 commands and the focused collector default test. + +- [ ] **Step 5: Commit** + +```bash +git add crates/trusted-server-cli/src/commands/audit/collector.rs crates/trusted-server-cli/src/commands/audit/generate/browser_collector.rs crates/trusted-server-cli/src/commands/audit/mod.rs crates/trusted-server-cli/src/run.rs +git commit -m "Preserve generation browser option contracts" +``` + +### Task 2: Enforce borrowed-root and merge-policy safety + +**Files:** + +- Modify: `crates/trusted-server-cli/src/commands/audit/generate/unit_template.rs` +- Modify: `crates/trusted-server-cli/src/commands/audit/generate/mod.rs` + +- [ ] **Step 1: Add failing inference and end-to-end tests** + +Add tests proving: + +- `InferenceOutcome` identifies `ad-sidebar` as borrowing the root witnessed by + another slot; +- explicit `--page-pattern` values cause `run_update_slots` to fail before the + source config changes when any rendered template borrowed the root; +- no-policy inference gives affected multi-path slots the root-witness reason; +- a configured `section_segment = 1` with no `section_root` refuses inferred + segment 0 when preserved `{section}` slots exist; +- the same segment, or an unset segment, allows adopting the inferred root. + +- [ ] **Step 2: Run the focused tests and confirm RED** + +Run: + +```bash +cargo test --package trusted-server-cli --target aarch64-apple-darwin commands::audit::generate::unit_template::tests -- --nocapture +cargo test --package trusted-server-cli --target aarch64-apple-darwin commands::audit::generate::tests::merge_ -- --nocapture +cargo test --package trusted-server-cli --target aarch64-apple-darwin commands::audit::generate::tests::explicit_ -- --nocapture +``` + +Expected: borrowed stems are unavailable, explicit patterns are accepted, and +the configured-segment mismatch is accepted. + +- [ ] **Step 3: Carry borrowed stems and reject unsafe overrides** + +Add an ordered `borrowed_section_root: Vec` field to +`InferenceOutcome`. Populate it only when `RootUnwitnessed` successfully becomes +a template. Before building render slots, reject non-empty explicit patterns if +that vector is non-empty: + +```rust +return cli_error(format!( + "cannot apply --page-pattern to slot(s) {} because their {{section}} templates borrow section_root; remove --page-pattern so patterns can be derived from observed paths", + borrowed.join(", ") +)); +``` + +On the no-policy path, replace the generic multi-path refusal reason for +structurally valid root-unwitnessed slots with the specific missing-root-witness +reason. Preserve structural refusal reasons unchanged. + +Update `validate_merge_policy` so an explicit configured segment is compared +before the empty-root adoption return. Keep the guard limited to preserved +`{section}` slots and allow `--replace`. + +- [ ] **Step 4: Re-run focused tests and confirm GREEN** + +Run all Step 2 commands. + +- [ ] **Step 5: Commit** + +```bash +git add crates/trusted-server-cli/src/commands/audit/generate/unit_template.rs crates/trusted-server-cli/src/commands/audit/generate/mod.rs +git commit -m "Protect borrowed section templates during generation" +``` + +### Task 3: Make redirects, warnings, and config errors actionable + +**Files:** + +- Modify: `crates/trusted-server-cli/src/commands/audit/generate/mod.rs` +- Modify: `crates/trusted-server-cli/src/commands/audit/mod.rs` + +- [ ] **Step 1: Add failing diagnostic tests** + +Strengthen the HTTPS-upgrade assertion to require +`http://publisher.example/` and `https://publisher.example/`. Add a two-profile +warning test whose output names desktop and mobile separately. Add a malformed +whole-document TOML test while retaining tests for unknown valid settings and an +unreadable `[creative_opportunities]` section. + +- [ ] **Step 2: Run the focused tests and confirm RED** + +Run: + +```bash +cargo test --package trusted-server-cli --target aarch64-apple-darwin update_slots_accepts_a_same_host_https_upgrade -- --nocapture +cargo test --package trusted-server-cli --target aarch64-apple-darwin profile_warning -- --nocapture +cargo test --package trusted-server-cli --target aarch64-apple-darwin creative_config -- --nocapture +``` + +- [ ] **Step 3: Implement scoped diagnostics and early parse failure** + +Render redirect endpoints as `origin.ascii_serialization() + path`. Thread the +profile label into `fold_collected`; keep the consent-stub warning global, label +page warnings/interstitials with path and profile, and retain the existing +site-wide discovery-warning dedupe. + +Replace `.ok()` in `creative_config` with an error mapping that identifies a +malformed existing TOML document and explains that generation did not start. +Continue parsing into `toml::Value`, not runtime `Settings`, so valid unknown +settings remain tolerated. + +- [ ] **Step 4: Re-run focused tests and confirm GREEN** + +Run all Step 2 commands. + +- [ ] **Step 5: Commit** + +```bash +git add crates/trusted-server-cli/src/commands/audit/generate/mod.rs crates/trusted-server-cli/src/commands/audit/mod.rs +git commit -m "Clarify audit generation diagnostics" +``` + +### Task 4: Pin detector and embedded-collector invariants + +**Files:** + +- Modify: `crates/trusted-server-cli/src/commands/audit/generate/gpt_slots.rs` +- Modify: `crates/trusted-server-cli/src/commands/audit/browser.rs` +- Modify: `crates/trusted-server-cli/src/commands/audit/page.rs` + +- [ ] **Step 1: Add failing invariant tests** + +Add a negative volatile-token test for `promo-20260820a-sidebar`, retain a +positive timestamp-shaped control with at least ten leading digits, and add the +embedded-JavaScript constant assertion: + +```rust +assert!( + AD_TEMPLATE_COLLECTOR_JS.contains(&format!( + "const __ts_max_entries = {MAX_EVIDENCE_ENTRIES}" + )), + "should keep the JS cap equal to MAX_EVIDENCE_ENTRIES" +); +``` + +In the page summary test, assert the exact percent-encoded final URL line and +limit the raw-control assertion's comment to title and warning fields. + +- [ ] **Step 2: Run the focused tests and confirm RED** + +Run: + +```bash +cargo test --package trusted-server-cli --target aarch64-apple-darwin per_render_token -- --nocapture +cargo test --package trusted-server-cli --target aarch64-apple-darwin evidence_entries -- --nocapture +cargo test --package trusted-server-cli --target aarch64-apple-darwin page_controlled_text -- --nocapture +``` + +- [ ] **Step 3: Tighten the token shape and correct the test claim** + +Require at least ten leading digits in `is_per_render_token`. Keep the rest of +the recognizer unchanged. Add the evidence-cap test and page assertion without +removing final-URL escaping. + +- [ ] **Step 4: Re-run focused tests and confirm GREEN** + +Run all Step 2 commands. + +- [ ] **Step 5: Commit** + +```bash +git add crates/trusted-server-cli/src/commands/audit/generate/gpt_slots.rs crates/trusted-server-cli/src/commands/audit/browser.rs crates/trusted-server-cli/src/commands/audit/page.rs +git commit -m "Pin audit evidence recognition invariants" +``` + +### Task 5: Align documentation and local style + +**Files:** + +- Modify: `docs/guide/cli.md` +- Modify: `docs/superpowers/specs/2026-08-19-refuse-volatile-div-collisions-design.md` +- Modify: `docs/superpowers/plans/2026-08-19-refuse-volatile-div-collisions.md` +- Modify: `crates/trusted-server-cli/src/commands/audit/generate/mod.rs` +- Modify: `crates/trusted-server-cli/src/commands/audit/generate/browser_collector.rs` + +- [ ] In the volatile-collision design example, remove the section-varying + sidebar from the list of omitted/explained slots because root-less templating + now writes it with a borrowed-root diagnostic. +- [ ] In the volatile-collision implementation plan, state that a recognized + render token must have a non-empty family prefix before it and placement + content after it; remove the broader "in any position" claim. +- [ ] Update the guide to say that a configured segment without a root is + preserved for existing templates, and document the explicit-pattern refusal + for borrowed-root slots. +- [ ] Add the missing `GenerateArgs.browser` doc comment, change the `expect` + message to the required `"should ..."` form, and retain the method-separation + blank line from Task 1. +- [ ] Run `cd docs && npm run format` and `cargo fmt --all -- --check`. +- [ ] Commit: + +```bash +git add docs/guide/cli.md docs/superpowers/specs/2026-08-19-refuse-volatile-div-collisions-design.md docs/superpowers/plans/2026-08-19-refuse-volatile-div-collisions.md crates/trusted-server-cli/src/commands/audit/generate/mod.rs crates/trusted-server-cli/src/commands/audit/generate/browser_collector.rs +git commit -m "Align ad-template generation documentation" +``` + +### Task 6: Verify the complete review resolution + +**Files:** + +- Verify all files above. + +- [ ] Run focused audit generation tests: + +```bash +cargo test --package trusted-server-cli --target aarch64-apple-darwin commands::audit::generate -- --nocapture +``` + +- [ ] Run the complete host CLI suite: + +```bash +./scripts/test-cli.sh aarch64-apple-darwin +``` + +- [ ] Run lint and formatting gates: + +```bash +cargo clippy --package trusted-server-cli --target aarch64-apple-darwin --all-targets --all-features -- -D warnings +cargo fmt --all -- --check +cd docs && npm run format +git diff --check +``` + +- [ ] Inspect `git status --short`, `git log --oneline -6`, and the complete + diff from `073d5644` to ensure only the approved review resolution is present. +- [ ] Do not push or post GitHub replies without separate user authorization. diff --git a/docs/superpowers/plans/2026-08-24-ad-template-div-id-reconciliation.md b/docs/superpowers/plans/2026-08-24-ad-template-div-id-reconciliation.md new file mode 100644 index 000000000..2ec2f7979 --- /dev/null +++ b/docs/superpowers/plans/2026-08-24-ad-template-div-id-reconciliation.md @@ -0,0 +1,295 @@ +# Ad-template div-ID reconciliation implementation plan + +> **For agentic workers:** REQUIRED SUB-SKILL: Use superpowers:subagent-driven-development (recommended) or superpowers:executing-plans to implement this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking. + +**Goal:** Preserve observed numeric sibling creative opportunities during config merge and refuse singleton div IDs containing shorter high-entropy per-render tokens. + +**Architecture:** Reconciliation will use the normalized identities already retained by `EvidenceTable` to distinguish observed literals from intentional configured prefixes. GPT discovery will keep its vendor-neutral, position-aware volatile-family classifier and add a conservative eight-leading-digit/eight-character-suffix alternative without changing existing ten-digit behavior. + +**Tech Stack:** Rust 2024, `BTreeSet`, existing Trusted Server CLI evidence/merge pipeline, Cargo unit and browser integration tests. + +--- + +### Task 1: Preserve observed literal siblings during merge + +**Files:** + +- Modify: `crates/trusted-server-cli/src/commands/audit/generate/slot_toml.rs` + +- [ ] **Step 1: Write failing numeric-sibling merge tests** + +Add focused tests beside the existing prefix tests: + +```rust +#[test] +fn observed_literal_does_not_claim_numeric_siblings() { + let existing = existing_config( + "gam_network_id = \"222\"\n\n\ + [[slot]]\nid = \"ad-sidebar-1\"\ndiv_id = \"ad-sidebar-1\"\n\ + gam_unit_path = \"/222/sidebar\"\npage_patterns = [\"/\"]\n\ + formats = [{ width = 300, height = 250 }]\n", + ); + let discovered = ["ad-sidebar-1", "ad-sidebar-10", "ad-sidebar-11"] + .into_iter() + .map(|div_id| { + RenderSlot::from_evidence( + div_id, + div_id, + Some("/222/sidebar".to_string()), + [(300, 250)], + vec!["/news/*".to_string()], + false, + ) + }) + .collect(); + + let (merged, diagnostics) = + merge_render_slots_with_diagnostics(Some(&existing), discovered, false); + + assert_eq!(merged.len(), 3); + assert!(merged.iter().any(|slot| slot.id == "ad-sidebar-10")); + assert!(merged.iter().any(|slot| slot.id == "ad-sidebar-11")); + assert!(diagnostics.notes.is_empty()); +} +``` + +Add a second regression with an unrelated existing slot and discovered +`ad-sidebar-1` followed by `ad-sidebar-10`. It must prove a newly appended +observed literal cannot absorb a later sibling. Keep +`merge_reports_when_a_broad_prefix_claims_multiple_discovered_divs` unchanged as +the positive intentional-prefix control. + +- [ ] **Step 2: Run the tests and verify RED** + +Run: + +```bash +cargo test -p trusted-server-cli observed_literal_does_not_claim_numeric_siblings -- --nocapture +cargo test -p trusted-server-cli newly_appended_literal_does_not_claim_numeric_sibling -- --nocapture +``` + +Expected: both fail because `ad-sidebar-1` absorbs the longer discovered IDs. + +- [ ] **Step 3: Implement exact-first, evidence-aware prefix matching** + +In `merge_render_slots_with_observed_diagnostics`, build a borrowed set from +`observed_div_ids` once: + +```rust +let observed_literals = observed_div_ids + .iter() + .map(String::as_str) + .collect::>(); +``` + +Thread `&observed_literals` through discovered-slot reconciliation and +observed/unobserved classification. Refactor the matcher so it: + +1. searches all merged slots for an exact stable-key match; +2. returns that exact match immediately; +3. searches for the longest prefix only among prefixes absent from + `observed_literals`; and +4. retains configuration order for equal-length prefix ties. + +Use the same helper for seeding `observed_existing`, so merge behavior and stale +diagnostics cannot disagree. Keep exact matching available for configured slots +that omit `div_id` and therefore resolve through `id`. + +Update the `MergeDiagnostics` field comment from “raw crawl” to “normalized +evidence.” + +- [ ] **Step 4: Add and run the normalization-boundary regression** + +Use `discover_gpt_slots` plus `merge_slots` to show that a live +`ad-header-0-_R_3f_` identity normalizes to `ad-header-0`, and therefore makes +configured `ad-header-0` an observed literal rather than a prefix for a distinct +`ad-header-01` slot. Do not pass collector-level raw IDs into the merge. + +Run: + +```bash +cargo test -p trusted-server-cli normalized_stem_is_the_literal_merge_boundary -- --nocapture +``` + +Expected after implementation: PASS. + +- [ ] **Step 5: Run focused merge tests and verify GREEN** + +Run: + +```bash +cargo test -p trusted-server-cli slot_toml::tests -- --nocapture +``` + +Expected: all merge tests pass, including the existing intentional broad-prefix +test. + +- [ ] **Step 6: Commit** + +```bash +git add crates/trusted-server-cli/src/commands/audit/generate/slot_toml.rs +git commit -m "Preserve observed literal ad slot siblings" +``` + +### Task 2: Refuse eight-digit, long-suffix volatile tokens + +**Files:** + +- Modify: `crates/trusted-server-cli/src/commands/audit/generate/gpt_slots.rs` + +- [ ] **Step 1: Add failing shorter-token registry and request tests** + +Add singleton cases using a synthetic shape: + +```rust +const SHORT_VOLATILE_DIV: &str = + "vendor-tag_12345678AbCdEfGhIjKl_slot_overlay_1"; +``` + +Assert both registry and GAMPAD request discovery: + +- retain `had_slot_evidence`; +- produce no writable slots; and +- emit the existing volatile-family warning naming `vendor-tag`. + +- [ ] **Step 2: Run the tests and verify RED** + +Run: + +```bash +cargo test -p trusted-server-cli shorter_high_entropy_singleton -- --nocapture +``` + +Expected: FAIL because the current classifier requires ten leading digits and +accepts the eight-digit token literally. + +- [ ] **Step 3: Add failing classifier boundary tests** + +Extend the table-driven tests so these remain eligible: + +```text +vendor-tag_1234567AbCdEfGh_slot_inarticle_1 # seven leading digits +vendor-tag_12345678AbCdEfG_slot_inarticle_1 # seven-character suffix +promo-20260820a-sidebar # short calendar suffix +vendor-tag_1234567890123456_slot_inarticle_1 # bare numeric segment +``` + +Add `vendor-tag_12345678AbCdEfGh_slot_inarticle_1` to the volatile table. Run +the two boundary tests and confirm only the new 8+8 volatile assertion fails. + +- [ ] **Step 4: Implement the conservative alternative token shape** + +Keep the current all-ASCII-alphanumeric requirement and compute the suffix +length after the leading digit run. A segment is per-render when either: + +```rust +(leading_digits >= 10 && suffix_length >= 1) + || (leading_digits >= 8 && suffix_length >= 8) +``` + +Keep the existing requirement that the token occurs before another div-ID +segment. Do not add a vendor name or family-specific regular expression. + +- [ ] **Step 5: Run GPT discovery tests and verify GREEN** + +Run: + +```bash +cargo test -p trusted-server-cli gpt_slots::tests -- --nocapture +``` + +Expected: all discovery, normalization, collision, registry, request, and +boundary tests pass. + +- [ ] **Step 6: Commit** + +```bash +git add crates/trusted-server-cli/src/commands/audit/generate/gpt_slots.rs +git commit -m "Reject shorter high-entropy ad slot tokens" +``` + +### Task 3: Verify the complete change + +**Files:** + +- No source changes expected. + +- [ ] **Step 1: Run formatting and diff checks** + +```bash +cargo fmt --all -- --check +git diff --check +cd docs && npm run format +``` + +Expected: all exit zero and formatting makes no changes. + +- [ ] **Step 2: Run the complete CLI suite** + +```bash +./scripts/test-cli.sh +``` + +Expected: all unit, config overlay, proxy E2E, and ignored real-Chrome fixtures +pass. The browser portions require permission to bind loopback listeners. + +- [ ] **Step 3: Run host-target CLI clippy** + +```bash +cargo clippy \ + --manifest-path crates/trusted-server-cli/Cargo.toml \ + --target "$(rustc -vV | sed -n 's/^host: //p')" \ + --all-targets -- -D warnings +``` + +Expected: the changed CLI crate and all of its test targets lint without +warnings. The adapter-scoped aliases below do not include this crate. + +- [ ] **Step 4: Run repository target-specific Rust gates** + +```bash +cargo clippy-fastly +cargo clippy-axum +cargo clippy-cloudflare +cargo clippy-cloudflare-wasm +cargo clippy-spin-native +cargo clippy-spin-wasm +cargo test-fastly +cargo test-axum +cargo test-cloudflare +cargo test-spin +``` + +Expected: every command exits zero with no warnings promoted to errors. + +- [ ] **Step 5: Run parity and JavaScript/docs gates** + +```bash +cargo test --manifest-path crates/trusted-server-integration-tests/Cargo.toml --test parity +(cd crates/trusted-server-js/lib && npx vitest run) +(cd crates/trusted-server-js/lib && npm run format) +(cd docs && npm run format) +``` + +Expected: parity, Vitest, and formatting checks pass. + +- [ ] **Step 6: Review branch state** + +```bash +git status --short +git log --oneline --decorate -10 +``` + +Expected: clean feature worktree with the two implementation commits above the +approved design/plan commits. + +- [ ] **Step 7: Validate against the operator's dry-run output** + +Ask the operator to rerun the established desktop/mobile `--scroll --dry-run` +command with a current DataDome cookie. Confirm: + +- there is no `ad-sidebar-1` broad-prefix collision note; +- numeric sidebar siblings are emitted as distinct slots; +- the singleton mobile volatile-family slot is refused; and +- older configured volatile-family slots remain named as preserved but + unobserved until the operator deliberately prunes them. diff --git a/docs/superpowers/plans/2026-08-24-ad-template-generate-scroll-staleness.md b/docs/superpowers/plans/2026-08-24-ad-template-generate-scroll-staleness.md new file mode 100644 index 000000000..db6f22ce2 --- /dev/null +++ b/docs/superpowers/plans/2026-08-24-ad-template-generate-scroll-staleness.md @@ -0,0 +1,358 @@ +# Ad-template Generate Scroll and Staleness Implementation Plan + +> **For agentic workers:** REQUIRED SUB-SKILL: Use superpowers:subagent-driven-development (recommended) or superpowers:executing-plans to implement this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking. + +**Goal:** Add opt-in scrolling to `ts audit ad-templates generate` and warn when a normal merge preserves configured slots that the current crawl did not observe. + +**Architecture:** Thread one parsed `--scroll` value through the generation browser session and reuse a shared deterministic scroll primitive before the generator's final evidence scrape. Extend merge reconciliation with structured diagnostics that record unmatched pre-existing slot IDs; format the warning at the command layer so it can account for whether scrolling was already enabled without changing merge behavior. + +**Tech Stack:** Rust 2024, clap, chromiumoxide/CDP, Tokio, existing CLI and Chrome-fixture test harnesses, rustfmt, clippy, Prettier. + +--- + +## File map + +- Create `crates/trusted-server-cli/src/commands/audit/browser_scroll.rs`: shared deterministic scroll primitive. +- Modify `crates/trusted-server-cli/src/commands/audit/mod.rs`: declare the shared module, parse `--scroll`, and wire it into generation. +- Modify `crates/trusted-server-cli/src/commands/audit/browser.rs`: reuse shared scrolling while retaining verifier-only phase marking. +- Modify `crates/trusted-server-cli/src/commands/audit/generate/browser_collector.rs`: carry scroll state, scroll and re-settle, and test lazy GPT discovery. +- Modify `crates/trusted-server-cli/src/commands/audit/generate/mod.rs`: carry scroll context and render contextual stale-slot notes. +- Modify `crates/trusted-server-cli/src/commands/audit/generate/slot_toml.rs`: report unmatched preserved slots from the authoritative merge matcher. +- Modify `crates/trusted-server-cli/src/run.rs`: test parsing and defaults. +- Modify `scripts/test-cli.sh`: run the new ignored Chrome fixture. +- Modify `docs/guide/cli.md`: document both behaviors. + +### Task 1: Parse and wire generation scrolling + +**Files:** + +- Modify: `crates/trusted-server-cli/src/run.rs` +- Modify: `crates/trusted-server-cli/src/commands/audit/mod.rs` +- Modify: `crates/trusted-server-cli/src/commands/audit/generate/browser_collector.rs` +- Modify: `crates/trusted-server-cli/src/commands/audit/generate/mod.rs` + +- [ ] **Step 1: Write failing parsing tests** + +Extend `audit_generate_subcommands_use_generation_settle_defaults` with +`assert!(!generate.scroll)`. Add: + +```rust +#[test] +fn audit_ad_templates_generate_parses_scroll() { + let args = parse(&[ + "ts", "audit", "ad-templates", "generate", + "https://www.example.com/", "--scroll", + ]); + let Command::Audit(audit) = args.command else { + panic!("expected audit command"); + }; + let Some(crate::commands::audit::AuditSubcommand::AdTemplates( + crate::commands::audit::AuditAdTemplatesCommand::Generate(generate), + )) = audit.command else { + panic!("expected audit ad-templates generate command"); + }; + assert!(generate.scroll, "--scroll should enable generation scrolling"); +} +``` + +- [ ] **Step 2: Run the focused test and verify it fails** + +```bash +HOST_TARGET="$(rustc -vV | awk '/host:/ { print $2 }')" +cargo test --package trusted-server-cli --target "$HOST_TARGET" audit_ad_templates_generate_parses_scroll +``` + +Expected: compilation fails because `AuditAdTemplatesGenerateArgs` has no +`scroll` field. + +- [ ] **Step 3: Add the flag and session wiring** + +Add to `AuditAdTemplatesGenerateArgs`: + +```rust +/// Perform a deterministic scroll pass after each page initially settles. +#[arg(long)] +pub scroll: bool, +``` + +Add `scroll: bool` to `BrowserAuditCollector` and `SessionSettings`, default it +to false, and add `with_scroll(bool)`. Thread it through `session()`, +`with_browser`, `collect_page_from_browser`, and `collect_open_page`; Task 2 +will use it. + +Add `scroll: bool` to `UpdateSlotsRequest`. In `run_audit`, set both the +collector option and request field from `gen_args.scroll`. Update every test +fixture constructing `UpdateSlotsRequest` with `scroll: false`, except the later +contextual-warning test. + +- [ ] **Step 4: Run parsing/default tests** + +```bash +HOST_TARGET="$(rustc -vV | awk '/host:/ { print $2 }')" +cargo test --package trusted-server-cli --target "$HOST_TARGET" audit_generate_subcommands_use_generation_settle_defaults +cargo test --package trusted-server-cli --target "$HOST_TARGET" audit_ad_templates_generate_parses_scroll +``` + +Expected: both pass. + +- [ ] **Step 5: Commit** + +```bash +git add crates/trusted-server-cli/src/run.rs crates/trusted-server-cli/src/commands/audit/mod.rs crates/trusted-server-cli/src/commands/audit/generate/browser_collector.rs crates/trusted-server-cli/src/commands/audit/generate/mod.rs +git commit -m "Add scroll option to ad-template generation" +``` + +### Task 2: Share and execute deterministic scrolling + +**Files:** + +- Create: `crates/trusted-server-cli/src/commands/audit/browser_scroll.rs` +- Modify: `crates/trusted-server-cli/src/commands/audit/mod.rs` +- Modify: `crates/trusted-server-cli/src/commands/audit/browser.rs` +- Modify: `crates/trusted-server-cli/src/commands/audit/generate/browser_collector.rs` +- Modify: `scripts/test-cli.sh` + +- [ ] **Step 1: Add a failing Chrome fixture** + +Add a self-contained tall HTML page whose scroll listener installs a stub GPT +registry and defines `/123/lazy` in `ad-lazy-0` only after `window.scrollY > 0`. +Add this ignored test: + +```rust +#[test] +#[ignore = "requires local Chrome/Chromium; run through scripts/test-cli.sh"] +fn collects_lazy_gpt_slot_only_when_scroll_is_enabled() { + if !browser_fixture_available() { + return; + } + let url = lazy_gpt_fixture_url(); + let without_scroll = BrowserAuditCollector::default() + .collect_page(&url, &[]) + .expect("should collect without scrolling"); + let with_scroll = BrowserAuditCollector::default() + .with_scroll(true) + .collect_page(&url, &[]) + .expect("should collect with scrolling"); + + assert!(without_scroll.gpt_slots.is_empty()); + assert!(with_scroll.gpt_slots.iter().any(|slot| { + slot.gam_unit_path == "/123/lazy" && slot.div_id == "ad-lazy-0" + })); +} +``` + +Use loopback HTTP instead of `file://` if Chrome requires it for reliable scroll +events. Change `scripts/test-cli.sh` to run the ignored +`commands::audit::generate::browser_collector::tests::` prefix so lifecycle and +lazy-slot fixtures are both covered. + +- [ ] **Step 2: Run the fixture and verify it fails** + +```bash +HOST_TARGET="$(rustc -vV | awk '/host:/ { print $2 }')" +TS_AUDIT_BROWSER_TESTS=1 cargo test --package trusted-server-cli --target "$HOST_TARGET" collects_lazy_gpt_slot_only_when_scroll_is_enabled -- --ignored --test-threads=1 +``` + +Expected: the scrolled result still lacks `/123/lazy`. + +- [ ] **Step 3: Implement the shared primitive** + +Create `browser_scroll.rs` with a `ScrollFailure` enum (evaluation failure and +timeout) and: + +```rust +pub(crate) async fn scroll_page(page: &chromiumoxide::Page) -> Vec { + let mut failures = Vec::new(); + for fraction in ["0.33", "0.66", "1"] { + let script = format!( + "window.scrollTo(0, Math.floor(Math.max(document.body.scrollHeight, document.documentElement.scrollHeight) * {fraction}))" + ); + evaluate(page, script, &mut failures).await; + tokio::time::sleep(Duration::from_millis(250)).await; + } + evaluate(page, "window.scrollTo(0, 0)".to_string(), &mut failures).await; + failures +} +``` + +Bound each evaluation at five seconds. Declare the module in `audit/mod.rs`. +In `browser.rs`, leave the pre-scroll evidence snapshot and +`window.__tsScrollPhase = true` marker in place, replace the local step loop with +the shared function, and map failures to existing `Warning` output. + +In the generation collector, after initial settle but before final HTML/GPT/ +network/link scraping, call the shared function when `scroll` is true, append +its failures as page warnings, and call `wait_for_page_settle` again. A second +settle timeout is a warning, not a discarded page. + +- [ ] **Step 4: Run browser tests** + +```bash +HOST_TARGET="$(rustc -vV | awk '/host:/ { print $2 }')" +cargo test --package trusted-server-cli --target "$HOST_TARGET" commands::audit::browser::tests:: +TS_AUDIT_BROWSER_TESTS=1 cargo test --package trusted-server-cli --target "$HOST_TARGET" collects_lazy_gpt_slot_only_when_scroll_is_enabled -- --ignored --test-threads=1 +``` + +Expected: all pass and `/123/lazy` appears only with scrolling. + +- [ ] **Step 5: Commit** + +```bash +git add crates/trusted-server-cli/src/commands/audit/browser_scroll.rs crates/trusted-server-cli/src/commands/audit/mod.rs crates/trusted-server-cli/src/commands/audit/browser.rs crates/trusted-server-cli/src/commands/audit/generate/browser_collector.rs scripts/test-cli.sh +git commit -m "Collect lazy ad slots during generation scroll" +``` + +### Task 3: Report unmatched slots preserved by merge + +**Files:** + +- Modify: `crates/trusted-server-cli/src/commands/audit/generate/slot_toml.rs` +- Modify: `crates/trusted-server-cli/src/commands/audit/generate/mod.rs` + +- [ ] **Step 1: Write failing diagnostic tests** + +Next to `merge_keeps_existing_only_slots`, assert that diagnostic merging marks +preserved `sidebar` but not rediscovered `header`; multiple missing IDs retain +configuration order; and full rediscovery, empty existing slots, and +`--replace` produce no stale IDs. + +Add command tests with fake collectors and in-memory writers. Assert non-scroll +wording contains `or --scroll`, scroll wording omits that retry, stdout remains +only diff/summary content, and preserved slots remain in candidate TOML. + +- [ ] **Step 2: Run focused tests and verify they fail** + +```bash +HOST_TARGET="$(rustc -vV | awk '/host:/ { print $2 }')" +cargo test --package trusted-server-cli --target "$HOST_TARGET" merge_reports_preserved_unobserved_slots +cargo test --package trusted-server-cli --target "$HOST_TARGET" update_slots_reports_preserved_unobserved_slots +``` + +Expected: failures because unmatched existing slots are not exposed. + +- [ ] **Step 3: Add structured merge diagnostics** + +Define: + +```rust +#[derive(Debug, Default, PartialEq, Eq)] +pub(super) struct MergeDiagnostics { + pub(super) notes: Vec, + pub(super) unobserved_existing_slot_ids: Vec, +} +``` + +Change `merge_render_slots_with_diagnostics` to return this structure with the +merged slots. Record every matched existing index in a `BTreeSet`, then +collect unmatched existing IDs by enumerating configuration order. Preserve the +current broad-prefix messages in `notes`. The `replace || existing.is_empty()` +early path returns default diagnostics. Keep `merge_render_slots` returning only +the slot vector. + +- [ ] **Step 4: Format the contextual note in `run_update_slots`** + +Extend pending notes with `merge_diagnostics.notes`. If unmatched IDs exist, +append their count and comma-separated IDs. End with: + +```rust +let follow_up = if request.scroll { + "Re-run with broader page/profile coverage; `--replace` prunes them but also discards every hand-written field on the slots the run did rediscover." +} else { + "Re-run with broader coverage or --scroll; `--replace` prunes them but also discards every hand-written field on the slots the run did rediscover." +}; +``` + +Do not change the merged configuration. `emit_notes` remains the only terminal +sanitization/output boundary. + +- [ ] **Step 5: Run merge and command tests** + +```bash +HOST_TARGET="$(rustc -vV | awk '/host:/ { print $2 }')" +cargo test --package trusted-server-cli --target "$HOST_TARGET" commands::audit::generate::slot_toml::tests::merge_ +cargo test --package trusted-server-cli --target "$HOST_TARGET" update_slots_reports_preserved_unobserved_slots +``` + +Expected: all pass, with unchanged merged TOML and warnings only on stderr. + +- [ ] **Step 6: Commit** + +```bash +git add crates/trusted-server-cli/src/commands/audit/generate/slot_toml.rs crates/trusted-server-cli/src/commands/audit/generate/mod.rs +git commit -m "Warn about preserved unobserved ad slots" +``` + +### Task 4: Document and verify + +**Files:** + +- Modify: `docs/guide/cli.md` + +- [ ] **Step 1: Document both behaviors** + +Add a generation `--scroll` example under “Bounding and steering the crawl.” +Explain that every page/profile scrolls after initial settle and settles again, +and that it is opt-in because it adds time, requests, and publisher side effects. + +Update merge documentation: missing existing slots are preserved and named on +stderr; absence may reflect coverage, targeting, or lazy loading; only +`--replace` intentionally prunes them. + +- [ ] **Step 2: Format docs and inspect scope** + +```bash +cd docs && npm run format +git diff --check +git diff -- docs/guide/cli.md +``` + +Expected: formatting passes and only intended docs change. + +- [ ] **Step 3: Run the full CLI harness, including Chrome fixtures** + +```bash +./scripts/test-cli.sh +``` + +Expected: all host CLI and configured ignored browser tests pass. + +- [ ] **Step 4: Run formatting and lint gates** + +```bash +cargo fmt --all -- --check +cargo clippy-fastly +cargo clippy-axum +cargo clippy-cloudflare +cargo clippy-cloudflare-wasm +cargo clippy-spin-native +cargo clippy-spin-wasm +``` + +Expected: all exit zero without warnings. + +- [ ] **Step 5: Run adapter regression suites** + +```bash +cargo test-fastly +cargo test-axum +cargo test-cloudflare +cargo test-spin +``` + +Expected: all pass. Do not use bare `cargo test --workspace`. + +- [ ] **Step 6: Review scope and commit docs** + +```bash +git status --short +git diff --check +git diff HEAD -- crates/trusted-server-cli scripts/test-cli.sh docs/guide/cli.md +``` + +Confirm `fastly.toml` remains untouched and issue #1059 produced no code changes. +Then: + +```bash +git add docs/guide/cli.md +git commit -m "Document generation scroll and stale-slot warnings" +``` diff --git a/docs/superpowers/plans/2026-08-27-pr-1079-review-remediation.md b/docs/superpowers/plans/2026-08-27-pr-1079-review-remediation.md new file mode 100644 index 000000000..33c6a2832 --- /dev/null +++ b/docs/superpowers/plans/2026-08-27-pr-1079-review-remediation.md @@ -0,0 +1,154 @@ +# PR 1079 Review Remediation Implementation Plan + +> **For agentic workers:** REQUIRED SUB-SKILL: Use superpowers:subagent-driven-development (recommended) or superpowers:executing-plans to implement this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking. + +**Goal:** Resolve every review finding on PR 1079 and produce an `rc/202608`-based staging branch containing the corrected implementation. + +**Architecture:** Keep the first-claimant state machine, but make suppression token-local and correlation navigation/element-local. The first suppressed delivery closes registration while preserving every already-registered losing token until navigation or element replacement. Compose GPT/Prebid refresh wrappers explicitly, and centralize pre-response creative freshness validation plus safe authenticated-shell expansion. + +**Tech Stack:** TypeScript, Vitest/jsdom, Playwright, esbuild, Rust workspace validation, Git. + +--- + +### Task 1: First-impression token semantics + +**Files:** + +- Modify: `crates/trusted-server-js/lib/src/core/types.ts` +- Modify: `crates/trusted-server-js/lib/src/core/first_impression.ts` +- Test: `crates/trusted-server-js/lib/test/integrations/prebid/index.test.ts` + +- [ ] **Step 1: Add failing overlap and late-token tests** + +Add tests named `suppresses every publisher auction registered before the first TS delivery` and `suppresses a correlated TS-owned delivery after the five-second lease`. Assert two pre-registered callbacks are both suppressed, a later auction proceeds, and a fake-timer callback after 5 seconds remains suppressed. + +- [ ] **Step 2: Run the focused tests and verify RED** + +Run: `cd crates/trusted-server-js/lib && npx vitest run test/integrations/prebid/index.test.ts -t "registered before|five-second lease"` + +Expected: FAIL because `suppressionConsumed` permits the second delivery and expiry deletes the late token. + +- [ ] **Step 3: Implement token-local suppression** + +Replace `suppressionConsumed` with a claim-level `publisherRegistrationClosed` flag. Set it on the first suppressed delivery; do not consult it when consuming tokens already registered. Retain unresolved TS-owned suppressing tokens as non-evictable tombstones while generation and exact element identity match, including across timeout and auction failure; prune publisher-owned expired tokens and remove suppressing tombstones only on navigation or element replacement. + +- [ ] **Step 4: Run focused tests and verify GREEN** + +Run the Step 2 command. Expected: PASS. + +- [ ] **Step 5: Commit the state-machine checkpoint** + +Run: `git add crates/trusted-server-js/lib/src/core/types.ts crates/trusted-server-js/lib/src/core/first_impression.ts crates/trusted-server-js/lib/test/integrations/prebid/index.test.ts && git commit -m "fix(js): make first impression suppression auction local"` + +### Task 2: Prebid request and delivery correlation + +**Files:** + +- Modify: `crates/trusted-server-js/lib/src/integrations/prebid/index.ts` +- Test: `crates/trusted-server-js/lib/test/integrations/prebid/index.test.ts` + +- [ ] **Step 1: Add five failing Prebid regressions** + +Add tests named `consumes late-handoff suppression when Prebid suppresses the same delivery`, `limits a global request to opts.adUnitCodes`, `forwards only unsuppressed excluded slots`, `rejects pending delivery state from a previous navigation`, and `rejects pending delivery state after physical element replacement`. Assert the next legitimate refresh survives composed wrappers; only the selected global unit is mutated/claimed/correlated; a suppressed slot is absent from the native mixed refresh; and stale records neither suppress nor directly forward the new physical slot. + +- [ ] **Step 2: Run focused tests and verify RED** + +Run: `cd crates/trusted-server-js/lib && npx vitest run test/integrations/prebid/index.test.ts -t "late-handoff|opts.adUnitCodes|unsuppressed excluded|previous navigation|physical element replacement"` + +Expected: FAIL on the current wrapper, scoping, forwarding, and stale-correlation behavior. + +- [ ] **Step 3: Implement scoped, physical correlation** + +When `opts.adUnits` is absent and `opts.adUnitCodes` is an array, filter `pbjs.adUnits` before snapshotting, mutation, claiming, and correlation. Stamp `PendingPublisherBid` and `PendingPublisherCode` with `navGeneration` and the exact resolved `HTMLElement`; accept them only if generation, element identity, connectivity, DOM lookup, and target-slot resolution still match. Retain still-current suppressing correlations as tombstones. When Prebid suppresses a slot, clear the matching `gptSlotHandoffs` one-shot flag. In the no-auction/excluded branch call native GPT with `forwardedSlots`, not the original list. + +- [ ] **Step 4: Run the full Prebid test file and verify GREEN** + +Run: `cd crates/trusted-server-js/lib && npx vitest run test/integrations/prebid/index.test.ts`. Expected: PASS. + +- [ ] **Step 5: Commit the Prebid checkpoint** + +Run: `git add crates/trusted-server-js/lib/src/integrations/prebid/index.ts crates/trusted-server-js/lib/test/integrations/prebid/index.test.ts && git commit -m "fix(js): scope publisher delivery correlation"` + +### Task 3: Creative freshness and nested shell repair + +**Files:** + +- Modify: `crates/trusted-server-js/lib/src/integrations/gpt/index.ts` +- Test: `crates/trusted-server-js/lib/test/integrations/gpt/ad_init.test.ts` +- Test: `crates/trusted-server-integration-tests/browser/tests/shared/aps-renderer.spec.ts` + +- [ ] **Step 1: Add failing stale-response and nested-shell tests** + +Change `does not resize a stale cache response after navigation` to assert zero port posts, zero successful-response evidence, and zero billing beacons. Add `expands every collapsed ancestor through the authenticated slot root`, with iframe -> 1x1 inner wrapper -> 1x1 outer wrapper -> authenticated root. Add/extend the browser scenario to assert all clipping ancestors have the winning dimensions. + +- [ ] **Step 2: Run focused GPT tests and verify RED** + +Run: `cd crates/trusted-server-js/lib && npx vitest run test/integrations/gpt/ad_init.test.ts -t "stale cache response|every collapsed ancestor"` + +Expected: FAIL because stale cache data is posted and only the immediate parent is resized. + +- [ ] **Step 3: Validate before creative side effects** + +Create one helper that checks current generation, winning bid/renderer ownership, authenticated source iframe identity, connectivity, and containment. Invoke it immediately before every APS or ADM `postMessage`; return before successful-response diagnostics, `markUsed`, or billing on failure. + +- [ ] **Step 4: Expand the authenticated shell safely** + +Require finite positive dimensions no larger than 10,000. Require the source iframe to retain its 1x1 attributes and collapsed computed dimensions. Preflight every ancestor through the authenticated root, rejecting detached/foreign roots, `body`/`html`, fixed/sticky positioning, and anchor/vignette/interstitial markers. Then resize the iframe and each ancestor whose width or height remains collapsed; never mutate outside the authenticated root. + +- [ ] **Step 5: Run GPT unit and browser tests** + +Run: `cd crates/trusted-server-js/lib && npx vitest run test/integrations/gpt/ad_init.test.ts` + +Run: `cd crates/trusted-server-integration-tests/browser && npx playwright test tests/shared/aps-renderer.spec.ts` + +Expected: PASS. + +- [ ] **Step 6: Commit the renderer checkpoint** + +Run: `git add crates/trusted-server-js/lib/src/integrations/gpt/index.ts crates/trusted-server-js/lib/test/integrations/gpt/ad_init.test.ts crates/trusted-server-integration-tests/browser/tests/shared/aps-renderer.spec.ts && git commit -m "fix(js): reject stale creatives and expand nested shells"` + +### Task 4: Full verification + +- [ ] **Step 0: Commit the reviewed design and plan** + +Run: `git add docs/superpowers/specs/2026-08-27-pr-1079-review-remediation-design.md docs/superpowers/plans/2026-08-27-pr-1079-review-remediation.md && git commit -m "docs: plan PR 1079 review remediation"`. + +- [ ] **Step 1: Run JS gates** + +Run from `crates/trusted-server-js/lib`: `npm run format && npm run lint && npx vitest run && node build-all.mjs`. Run the relevant Playwright suite with the command established in Task 3. Expected: every command exits 0. + +- [ ] **Step 2: Run repository Rust gates** + +Run: `cargo fmt --all -- --check`, `cargo test-fastly`, `cargo test-axum`, `cargo test-cloudflare`, `cargo test-spin`, `./scripts/test-cli.sh`, `cargo clippy-fastly`, `cargo clippy-axum`, `cargo clippy-cloudflare`, `cargo clippy-cloudflare-wasm`, `cargo clippy-spin-native`, and `cargo clippy-spin-wasm`. Expected: every command exits 0. + +- [ ] **Step 3: Commit formatting or test-only adjustments** + +If verification changed tracked files, review them and commit only scoped changes as `chore: finalize PR 1079 remediation verification`. + +### Task 5: Build the staging branch + +- [ ] **Step 1: Confirm a clean repair branch** + +Run: `git status --short --branch` and record `git rev-parse HEAD`. Expected: branch `fix/gpt-first-impression-aps-shell-review`, no uncommitted changes. + +- [ ] **Step 2: Refresh the remote RC ref** + +Run: `git fetch origin refs/heads/rc/202608:refs/remotes/origin/rc/202608 refs/heads/fix/gpt-first-impression-aps-shell:refs/remotes/origin/fix/gpt-first-impression-aps-shell`. + +- [ ] **Step 3: Create and merge the staging branch** + +Run: `git switch -c staging/202608-pr1079-review origin/rc/202608` then `git merge --no-ff fix/gpt-first-impression-aps-shell-review -m "Merge PR 1079 review remediation for staging"`. Expected: merge succeeds without unresolved conflicts. + +- [ ] **Step 4: Re-run critical post-merge gates** + +Run: `cd crates/trusted-server-js/lib && npm run format && npm run lint && npx vitest run && node build-all.mjs`. + +Run: `cd crates/trusted-server-integration-tests/browser && npx playwright test tests/shared/aps-renderer.spec.ts`. + +Run from the repository root: `cargo fmt --all -- --check && cargo check-fastly && cargo check-axum && cargo check-cloudflare`. + +Expected: every command exits 0 and `git status --short --branch` is clean on `staging/202608-pr1079-review`. + +- [ ] **Step 5: Report deployable refs** + +Record the repair-branch hash, staging merge hash, exact test results, and any non-blocking environment limitations. Do not push unless separately requested. diff --git a/docs/superpowers/specs/2026-04-15-server-side-ad-templates-design.md b/docs/superpowers/specs/2026-04-15-server-side-ad-templates-design.md index 8617ef877..147323675 100644 --- a/docs/superpowers/specs/2026-04-15-server-side-ad-templates-design.md +++ b/docs/superpowers/specs/2026-04-15-server-side-ad-templates-design.md @@ -68,10 +68,14 @@ across every navigation in the user's clickstream rather than once per session. pipeline. The GAM call (`securepubads.g.doubleclick.net`) moving server-side is aspirational, contingent on Google agreement, and is not committed for any phase (see §9.6). -- Eliminating Prebid entirely — a stripped-down Prebid bundle (_slim-Prebid_) is +- Eliminating Prebid entirely. A stripped-down Prebid bundle (_slim-Prebid_) is lazy-loaded post-`window.load` to handle scroll/refresh auctions and userID - enrichment. **TS owns the first impression; Prebid owns subsequent refresh - auctions.** + enrichment. **The first valid claimant owns each navigation's first impression.** + A publisher auction, GPT request, or GPT render consumes the claim before late + page-bids data can target or refresh that slot. If TS claims first, it suppresses + one correlated losing publisher delivery during a bounded lease. Later publisher + refresh auctions proceed normally. Strict TS-first delivery would require holding + publisher delivery and remains a separate design choice. - Dynamic slot discovery (reading the DOM) — this design commits to pre-defined, URL-matched slot templates. Smart Slots' dynamic injection behavior is replaced by server knowledge. diff --git a/docs/superpowers/specs/2026-06-26-server-side-ad-template-cli-design.md b/docs/superpowers/specs/2026-06-26-server-side-ad-template-cli-design.md new file mode 100644 index 000000000..98af3e176 --- /dev/null +++ b/docs/superpowers/specs/2026-06-26-server-side-ad-template-cli-design.md @@ -0,0 +1,842 @@ +# Server-Side Ad Template CLI Design + +**Date:** 2026-06-26 +**Status:** Draft design +**Scope:** Static and browser-backed CLI diagnostics for server-side ad templates + +## 1. Goal + +Add Trusted Server CLI support for server-side ad-template onboarding and +verification without resurrecting the stale standalone `ts-config` design. + +The CLI must answer two operator questions: + +1. Given an effective `trusted-server.toml`, which configured ad-template slots + match this path? +2. Given one or more live publisher URLs, are the configured slots for the + final navigated paths actually present on the page according to DOM, GPT, + and provider evidence, and do any runtime gates explain why Trusted Server + would not inject or auction for that page? + +The command surface is split by whether the command is local-config-only or +browser-backed: + +```bash +ts config ad-templates lint +ts config ad-templates match +ts config ad-templates check +ts config ad-templates explain + +ts audit ad-templates verify ... +``` + +Static commands live under `ts config` because they only load local effective +app config. Browser-backed verification lives under `ts audit` because it loads +public publisher pages in Chrome/Chromium and observes live page behavior. + +## 2. Context + +This design replaces the stale PR #724 direction. + +PR #724 designed a standalone `ts-config` binary around a +`creative-opportunities.toml` file. That is no longer the project shape: + +- Trusted Server configuration now flows through the unified `ts` CLI from PR + #799. +- Server-side ad-template slots live under `[creative_opportunities]` / + `[[creative_opportunities.slot]]` in `trusted-server.toml`. +- Effective config can include EdgeZero app-config environment overlays unless + `--no-env` is passed. +- Operator-owned `trusted-server.toml` is ignored; the repository tracks + `trusted-server.example.toml`. + +PR #799 is the CLI base. It owns the `ts` binary, EdgeZero lifecycle delegates, +and typed app-config validation/push/diff behavior. + +PR #800 is the audit dependency. It adds the generic browser-backed +`ts audit ` collector using local Chrome/Chromium. At the time this spec +was written, PR #800 was stale relative to the latest #799 head, so this work +depends on the #800 audit collector after it is rebased onto the latest #799 +typed blob-config model. + +## 3. Non-Goals + +- Do not add a standalone `ts-config` binary. +- Do not reintroduce `creative-opportunities.toml`. +- Do not implement browser-backed generation in Phase 1. +- Do not mutate `trusted-server.toml` from `verify`. +- Do not probe PBS, GAM, or APS management APIs. +- Do not require EdgeZero platform adapters for local static diagnostics. +- Do not make `ts audit ad-templates verify` push, provision, deploy, or update + platform resources. +- Do not rely on real GPT or APS network calls in tests. + +Browser-backed generation is a later phase: + +```bash +ts audit ad-templates generate ... +``` + +That phase needs separate rules for slot ID derivation, page-pattern inference, +multi-URL merging, TOML ordering, and whether the command emits a patch, a draft +file, or full config blocks. + +## 4. Command Surface + +### 4.1 Shared Config Flags + +All `ts config ad-templates ...` commands and +`ts audit ad-templates verify` accept the same local app-config flags: + +```bash +--app-config +--manifest +--no-env +``` + +Defaults match PR #799: + +| Option | Default | +| -------------- | ------------------------------------------------ | +| `--app-config` | `.toml`, resolved from `edgezero.toml` | +| `--manifest` | `edgezero.toml` | +| `--no-env` | `false`; app-config env overlay is applied | + +If an explicit `--app-config` path is supplied and missing, the command reports +that path as the error. It must not silently fall back to an environment or +manifest-derived path. + +### 4.2 Static Config Diagnostics + +```bash +ts config ad-templates lint [--app-config ] [--manifest ] [--no-env] +``` + +Reports whether `[creative_opportunities]` is configured, how many slots exist, +GAM network ID, auction timeout, auction enablement, configured auction +providers, and whether current EdgeZero routing will fall back to the legacy +path when configured slots are present. + +```bash +ts config ad-templates match [--details] ... +``` + +Normalizes a path or full URL to a path and reports the slots matched by the +runtime `creative_opportunities::match_slots` logic. `--details` includes slot +div ID, GAM unit path, page patterns, formats, and configured providers. + +```bash +ts config ad-templates check \ + (--expected-slot ... | --expect-no-slots) \ + [--allow-extra-slots] ... +``` + +CI-friendly assertion wrapper around the same matching logic. + +```bash +ts config ad-templates explain \ + [--method GET] \ + [--non-navigation] \ + [--prefetch] \ + [--bot] \ + [--consent-denied] \ + [--edgezero-enabled] ... +``` + +Explains the major runtime gates that decide whether the server-side ad stack +would run for a page request. This is a local model, not a live request replay. + +### 4.3 Browser-Backed Verification + +```bash +ts audit ad-templates verify ... \ + [--app-config ] \ + [--manifest ] \ + [--no-env] \ + [--strict] \ + [--json] \ + [--scroll] +``` + +Behavior: + +- Accept one or more `http` or `https` URLs. +- Reject all other schemes before launching a browser. +- Load the effective Trusted Server app config. +- For each URL, navigate first, collect the final URL, normalize the final URL + to a path, and call `creative_opportunities::match_slots`. +- Preserve the requested URL/path separately from the final URL/path. +- Emit a redirect warning when the final path differs from the requested path. +- Expect only the slots matched for the final URL path to be present on that + live page. +- Report live DOM/GPT/APS ad-slot evidence that does not correspond to a + matched configured slot as structured extra evidence. +- Launch Chrome/Chromium through the audit collector from the rebased #800 work. +- Inject a read-only ad-template collector before publisher scripts run. +- Compare configured matched slots against DOM, GPT, and APS evidence. +- Report runtime ad-stack gate evidence separately from placement evidence. +- Print human output by default. +- Emit stable machine-readable output with `--json`. +- Exit `0` by default for missing or partial live evidence; this is an + auditor-assist mode. +- Exit non-zero under `--strict` when a matched configured slot is missing or + only partially confirmed. + +`--scroll` performs a deterministic scroll pass after initial load and settle. +It is opt-in because it is slower and can trigger additional page behavior. +Slots first observed during scroll count as confirmed when the GPT evidence is +otherwise sufficient. + +## 5. Confirmation Model + +The verifier compares configured expected slots to live page evidence. + +It must keep three concepts separate: + +1. **Static slot matching:** which configured slots match a URL path according + to `creative_opportunities::match_slots`. +2. **Runtime ad-stack eligibility:** whether Trusted Server would run its + server-side ad stack for the audited navigation. This mirrors + `should_run_server_side_ad_stack` for the initial publisher request and the + `/__ts/page-bids` kill-switch/consent behavior for SPA route updates. +3. **Live placement evidence:** what the browser actually observes on the + rendered page through DOM, GPT, and APS evidence. + +`verify` is primarily a live placement verifier. `--strict` fails when matched +configured slots for an eligible page are missing or partial. Runtime gates are +reported so operators can distinguish "the slot is not on the page" from "the +current request/config would intentionally suppress Trusted Server ad-template +injection or page-bids slot output". + +### 5.1 Expected Slots + +For each input URL: + +1. Navigate the browser to the requested URL. +2. Record `requested_url`, `requested_path`, `final_url`, and `final_path`. +3. Match configured slots through the core runtime matcher using `final_path`. +4. Build an expected-slot record for each matched slot: + - slot ID; + - resolved div ID; + - resolved GAM unit path; + - configured formats; + - configured providers; + - matching page patterns. + +Only these expected slots are verified for that page. For example, slots whose +only pattern is `/` are expected for the homepage path, not for `/news/story`. + +When a navigation redirects, `verify` uses the final path for expected slots and +reports the requested path in output. This matches runtime behavior: Trusted +Server evaluates the actual publisher request path it handles, not the URL the +operator typed before redirects. + +### 5.2 Runtime Gate Evidence + +For each page result, `verify` reports a local runtime-gate model: + +| Gate | Source | +| ------------------------ | -------------------------------------------------------------------------------------------------------- | +| `method_get` | Browser navigation request; expected to pass for normal `verify`. | +| `navigation` | Browser navigation request; expected to pass for normal `verify`. | +| `not_prefetch` | Browser request headers; expected to pass unless the collector is extended with prefetch simulation. | +| `not_bot` | Browser User-Agent checked against the runtime bot fragments. | +| `matched_slots` | Final-path slot matching. | +| `auction_enabled` | Effective `[auction].enabled` / orchestrator enablement from app config. | +| `consent_allows_auction` | `unknown` unless the collector can prove a consent-allowed or consent-denied state for the live request. | + +`runtime_ad_stack_expected` is a three-state value: `yes`, `no`, or `unknown`. +Known blocking gates produce page warnings and set +`runtime_ad_stack_expected = "no"`. Unknown gates set +`runtime_ad_stack_expected = "unknown"` but do not by themselves fail +`--strict`. + +If `runtime_ad_stack_expected = "no"` because of a known config/request gate +such as `[auction].enabled = false`, strict mode does not fail missing GPT/APS +evidence for that page. The page result is reported as skipped for runtime +verification while still showing the static expected slots and any live +placement evidence that was observed. + +If `runtime_ad_stack_expected = "yes"` or `"unknown"`, strict mode applies the +normal missing/partial placement rules from §5.6. + +For SPA routes, `/__ts/page-bids` returns no slots when the ad-stack kill switch +or consent gate blocks the stack. Browser verification should report observed +page-bids responses when available, but it must not require real partner bids in +tests. + +Live ad-slot evidence that does not map to a matched expected slot is reported +as structured extra evidence. Extra evidence can identify publisher-owned slots +that have not yet moved into server-side ad templates, slots whose +`page_patterns` are too narrow, or slots that should stay outside Trusted +Server. It does not make `--strict` fail in Phase 1. + +### 5.3 DOM Slot Resolution + +The verifier must mirror the runtime GPT bootstrap's slot-root resolution: + +1. Try `document.getElementById(slot.div_id)`. +2. If absent, find the first element with an ID that starts with `slot.div_id`. +3. Ignore elements whose ID ends with `-container`. + +This is required because `div_id` may intentionally be a stable prefix for +framework-generated IDs, for example `ad-header-0-`. + +### 5.4 GPT Evidence + +A slot is confirmed by GPT evidence when the live page exposes a GPT slot whose: + +- ad unit path equals the configured resolved GAM unit path; +- slot element ID equals the resolved DOM element ID or an existing + `${resolved_dom_id}-container` element used by Trusted Server when defining + its own slot; +- configured sizes are compatible with the observed GPT sizes. + +The collector should observe both direct `googletag.defineSlot` calls and +post-load `googletag.pubads().getSlots()` state. + +Size compatibility is defined for Phase 1 as follows: + +- Normalize configured sizes from `CreativeOpportunityFormat` values where + `media_type = "banner"` into `(width, height)` pairs. +- Normalize observed GPT sizes from `defineSlot` input and `getSizes()` output: + - `[300, 250]` becomes one `(300, 250)` pair. + - `[[300, 250], [728, 90]]` becomes two pairs. + - non-numeric values such as `"fluid"` are ignored for numeric matching and + reported as warnings. +- A GPT slot's sizes are compatible when the configured banner size set and the + observed numeric GPT size set have at least one pair in common. +- Extra observed GPT sizes do not block confirmation, but they are reported as + warnings so auditors can decide whether to add formats to config. +- Configured banner sizes that are not observed do not block confirmation when + at least one configured size was observed, but they are reported as warnings. +- If ad unit path and div match but no numeric size overlap exists, the slot is + `partial`, not `confirmed`. +- Configured `video` and `native` formats are not used for Phase 1 GPT size + confirmation. A matched slot with only non-banner formats is `unconfirmable` + with an unsupported-format warning and does not fail `--strict`. +- A sizeless live GPT slot is `partial` when the config declares banner sizes, + because that is observable drift and must fail `--strict`. + +### 5.5 APS Evidence + +Phase 1 does not wrap or collect `apstag.fetchBids`: APS is server-side provider +configuration and client-side calls are neither required nor authoritative for +the runtime ad-template decision. + +### 5.6 Statuses + +| Status | Meaning | +| --------------- | -------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| `confirmed` | GPT evidence matches the configured GAM unit path, div resolution, and compatible sizes. | +| `partial` | The page has some evidence for the configured slot, but not enough to confirm it. This includes DOM-only evidence, GPT path/div matches with incompatible sizes, GPT path/div matches for unsupported non-banner-only configured formats, and other non-confirming GPT evidence. | +| `missing` | No DOM or GPT evidence confirms the configured slot. | +| `unconfirmable` | The checker cannot evaluate the configured format with Phase 1 evidence, such as a non-banner-only slot. This is reported but does not fail strict mode. | + +In `--strict` mode: + +- `missing` fails. +- `partial` fails. +- `unconfirmable` does not fail. + +Provider issues are not statuses. They are warnings attached to the slot result. +For example, a slot can be `confirmed` and still carry a warning that configured +APS evidence was missing or ambiguous. Provider warnings do not fail `--strict` +unless a future `--strict-providers` flag is added. + +## 6. Architecture + +The architecture should keep command parsing thin and move ad-template behavior +into pure, testable modules. + +```text +crates/trusted-server-cli/src/ + app_config.rs + ad_templates/ + mod.rs + expected.rs + compare.rs + output.rs + config_ad_templates.rs + audit/ + page.rs + browser.rs + ad_templates.rs +``` + +### 6.1 `app_config.rs` + +Shared loader for effective Trusted Server app config. + +Responsibilities: + +- read `edgezero.toml` through EdgeZero manifest helpers; +- resolve the default `.toml` path; +- apply EdgeZero app-config env overlay unless `--no-env`; +- return `TrustedServerAppConfig` / `Settings`; +- report errors in the same terms as #799 config commands. + +This avoids duplicating config path and env-overlay behavior between +`ts config ad-templates ...` and `ts audit ad-templates verify`. + +The current branch already has a private loader in `config_ad_templates.rs`. +Before adding browser-backed verification, move that behavior into this shared +module and route the existing static commands through it so both command +families load the same effective config. + +### 6.2 `ad_templates::expected` + +Pure local expected-slot model. + +Responsibilities: + +- normalize path-or-URL input; +- call `creative_opportunities::match_slots`; +- convert matched slots into stable expected-slot structs; +- preserve deterministic ordering by slot order from config. + +This module must not compile glob patterns independently or duplicate matching +semantics. + +If richer pattern diagnostics are needed, add a small helper to +`trusted-server-core::creative_opportunities` and use it from both runtime and +CLI. + +### 6.3 `ad_templates::compare` + +Pure comparison between expected slots and collected browser evidence. + +Responsibilities: + +- implement DOM prefix matching rules; +- compare GPT path, div, and size evidence; +- compare APS evidence; +- collect unmatched live DOM/GPT/APS ad-slot evidence as structured + `extra_evidence`; +- assign `confirmed`, `partial`, `missing`, and provider warning details; +- decide strict failure status. + +This module should be testable without launching Chrome. + +### 6.4 `ad_templates::output` + +Human and JSON output model. + +Responsibilities: + +- serialize stable JSON output; +- keep arrays ordered by input URL, then configured slot order, then provider + name; +- render concise human summaries; +- avoid leaking page HTML, cookies, local storage, or arbitrary page data. + +### 6.5 `config_ad_templates.rs` + +Thin Clap adapter for `ts config ad-templates ...`. + +Responsibilities: + +- parse command arguments; +- call `app_config` and `ad_templates::expected`; +- delegate formatting to `ad_templates::output`; +- keep no browser-specific logic. + +### 6.6 `audit::browser` + +Shared browser utility extracted from or aligned with the rebased #800 audit +collector. + +Responsibilities: + +- locate Chrome/Chromium; +- launch an isolated profile; +- reject non-HTTP(S) URLs before navigation; +- set bounded navigation and settle timeouts; +- run optional init scripts; +- perform optional deterministic scroll; +- collect final URL, title, rendered scripts, resource entries, and optional + ad-template evidence. + +The generic `ts audit ` command from #800 should continue to work without +ad-template verification enabled. + +### 6.7 `audit::ad_templates` + +Browser-backed verifier orchestration. + +Responsibilities: + +- parse `ts audit ad-templates verify`; +- load effective config through `app_config`; +- compute expected slots for each URL; +- run the browser collector with ad-template evidence enabled; +- call `ad_templates::compare`; +- print human or JSON output; +- apply default auditor-assist exit behavior and `--strict` behavior. + +## 7. Browser Collector + +The ad-template collector is injected before page scripts run. It is read-only: +it records evidence and calls original page functions with unchanged arguments. + +The rebased #800 collector must grow a pre-navigation init-script hook before it +can satisfy this spec. The stale #800 collector only navigates, waits, and reads +post-load page state; that is insufficient for GPT/APS call evidence. + +Instrumentation requirements: + +- install the collector through the browser's "evaluate on new document" / + init-script mechanism before navigation; +- serialize only configured div prefixes and provider IDs needed for matching; +- observe pages that create `window.googletag = { cmd: [] }` after injection; +- wrap `googletag.cmd.push` callbacks without changing callback order; +- record direct `googletag.defineSlot` calls and calls executed from the GPT + command queue; +- read final `googletag.pubads().getSlots()` state after settle and after + scroll; +- observe pages that assign `window.apstag` after injection and wrap + `apstag.fetchBids` when present; +- tolerate pages that never load GPT or APS and report warnings instead of + throwing collector errors. + +Evidence to collect: + +- DOM elements with IDs relevant to configured slot div prefixes; +- calls to `googletag.defineSlot`; +- final `googletag.pubads().getSlots()` state after settle and after scroll; +- calls to `apstag.fetchBids`; +- timestamps or phases indicating whether evidence was observed during + `initial_load` or `scroll`. + +The collector must not: + +- block, rewrite, or suppress publisher scripts; +- override `navigator.webdriver`; +- capture cookies, local storage, session storage, request bodies, or arbitrary + page data; +- require real GPT/APS network calls in test fixtures. + +## 8. JSON Output Contract + +`--json` emits deterministic JSON. Shape: + +```json +{ + "ok": true, + "strict": false, + "pages": [ + { + "url": "https://www.example.com/news/story", + "final_url": "https://www.example.com/news/story", + "requested_path": "/news/story", + "path": "/news/story", + "runtime_ad_stack_expected": "unknown", + "gates": { + "method_get": "pass", + "navigation": "pass", + "not_prefetch": "pass", + "not_bot": "pass", + "matched_slots": "pass", + "auction_enabled": "pass", + "consent_allows_auction": "unknown" + }, + "matched_slot_count": 1, + "slots": [ + { + "id": "atf", + "status": "confirmed", + "phase": "initial_load", + "configured": { + "div_id": "ad-atf-", + "gam_unit_path": "/123/news/atf", + "formats": [ + { "width": 300, "height": 250, "media_type": "banner" } + ], + "providers": ["aps"] + }, + "evidence": { + "dom_id": "ad-atf-0", + "gpt": { + "gam_unit_path": "/123/news/atf", + "div_id": "ad-atf-0", + "sizes": [[300, 250]] + } + }, + "warnings": [] + } + ], + "extra_evidence": [], + "warnings": [] + } + ], + "warnings": [] +} +``` + +Warning entries are objects with stable `code` and human-readable `message` +fields. Human output may print only the message. JSON consumers must not need to +parse warning strings. + +Extra live evidence is structured: + +```json +{ + "kind": "gpt", + "phase": "initial_load", + "dom_id": "ad-right-rail-0", + "gam_unit_path": "/123/publisher/right-rail", + "sizes": [[300, 250]], + "reason": "no_configured_slot_matched" +} +``` + +Allowed `kind` values for Phase 1 are `dom` and `gpt`. + +Strict-mode failures with page results use the same shape and set `ok` to +`false`. Example partial slot: + +```json +{ + "ok": false, + "strict": true, + "pages": [ + { + "url": "https://www.example.com/", + "final_url": "https://www.example.com/", + "requested_path": "/", + "path": "/", + "runtime_ad_stack_expected": "unknown", + "gates": { + "method_get": "pass", + "navigation": "pass", + "not_prefetch": "pass", + "not_bot": "pass", + "matched_slots": "pass", + "auction_enabled": "pass", + "consent_allows_auction": "unknown" + }, + "matched_slot_count": 1, + "slots": [ + { + "id": "homepage-header", + "status": "partial", + "phase": "initial_load", + "configured": { + "div_id": "ad-header-0-", + "gam_unit_path": "/123/homepage/header", + "formats": [{ "width": 728, "height": 90, "media_type": "banner" }], + "providers": ["aps"] + }, + "evidence": { + "dom_id": "ad-header-0-_R_abc123", + "gpt": null + }, + "warnings": [ + { + "code": "dom_without_gpt", + "message": "DOM element matched, but no GPT slot evidence was observed" + } + ] + } + ], + "extra_evidence": [], + "warnings": [] + } + ], + "warnings": [] +} +``` + +For errors that occur before any page result can be produced, the command exits +non-zero and prints the normal CLI error. JSON error output can be added later +if the base CLI standardizes it. + +For multi-URL runs, browser/navigation failures after argument validation are +page-level failures when possible. The command continues to the remaining URLs, +sets top-level `ok` to `false`, and includes a page result: + +```json +{ + "url": "https://www.example.com/broken", + "final_url": null, + "requested_path": "/broken", + "path": null, + "error": { + "code": "navigation_failed", + "message": "failed to read main document navigation response" + }, + "slots": [], + "extra_evidence": [], + "warnings": [] +} +``` + +Invalid schemes are still rejected before browser launch for the whole command, +because they are argument errors rather than page collection results. + +## 9. Error Handling + +Static commands fail when: + +- config cannot be loaded; +- `[creative_opportunities]` is malformed; +- CLI assertions in `check` fail. + +Browser verification fails when: + +- config cannot be loaded; +- any URL is not HTTP(S); +- Chrome/Chromium cannot be found or launched; +- all navigations fail before any page result can be collected; +- at least one page-level error occurs in a multi-URL run; +- command output cannot be written; +- `--strict` is set, runtime verification is not skipped by a known gate, and + at least one matched slot is missing or partial. `unconfirmable` is excluded. + +Browser collection can still produce a page result with warnings when: + +- page settle times out; +- a navigation redirects before final URL matching; +- scroll evidence is incomplete; +- GPT is not loaded; +- extra live DOM/GPT ad-slot evidence has no matched configured slot; +- no slots match the URL. + +## 10. Testing + +Static tests: + +- parse every `ts config ad-templates` command; +- load temp `edgezero.toml` and temp `trusted-server.toml`; +- verify `--app-config`, `--manifest`, and `--no-env` behavior; +- verify `/`, `/news/*`, and full URL normalization behavior; +- verify `check` success and failure output. +- verify the existing static command loader uses the shared `app_config` module. + +Pure comparison tests: + +- exact DOM ID match; +- prefix DOM ID match for framework-generated suffixes; +- ignore `-container` elements; +- GPT confirms by GAM unit path, div ID, and compatible sizes; +- DOM-only creates `partial`; +- no DOM/GPT creates `missing`; +- APS match creates no provider warning; +- APS missing/ambiguous creates provider warnings; +- `--strict` fails only missing and partial slots. + +Browser fixture tests: + +- local HTML fixture with direct `googletag.defineSlot`; +- fixture using `googletag.cmd.push`; +- fixture assigning `window.googletag` after collector injection; +- fixture with delayed/lazy slot observed only with `--scroll`; +- fixture with APS `fetchBids`; +- fixture assigning `window.apstag` after collector injection; +- redirect fixture that matches expected slots on final path; +- multi-URL fixture where one URL fails and one URL returns page results; +- fixture where `[auction].enabled = false` reports runtime skipped instead of + strict missing-slot failure; +- invalid non-HTTP(S) URL rejection before browser launch; +- JSON contract tests for warning codes, `extra_evidence`, page errors, + deterministic ordering, `partial`, `missing`, and strict failures; +- fixture with no real GPT/APS network dependency. + +Verification commands: + +```bash +cargo test --workspace +cargo fmt --all -- --check +cargo clippy --workspace --all-targets --all-features -- -D warnings +cargo test --package trusted-server-cli --target +``` + +## 11. Branch And PR Plan + +The implementation should not be built on stale #724. + +Recommended dependency order: + +1. Land or rebase PR #799 as the CLI base. +2. Rebase PR #800 onto the latest #799 head so `ts audit` uses the current typed + blob app-config model. +3. Harden and refactor the existing static `ts config ad-templates ...` + diagnostics on top of the current server-side ad-template branch and #799: + extract the private config loader into `app_config`, move pure expected-slot + logic into `ad_templates::expected`, and keep existing behavior covered by + tests. +4. Extend the rebased #800 collector with pre-navigation init scripts, + ad-template evidence hooks, optional scroll, page-level errors, and bounded + structured output. +5. Build `ts audit ad-templates verify` on top of that collector and the + server-side ad-template branch. +6. Keep `generate` for a separate Phase 2 spec and PR. + +If delivery needs to be split, static diagnostics can land before browser-backed +verification. Browser-backed verification should not duplicate the #800 browser +collector. + +## 12. CLI Namespace Decision + +`ts audit ad-templates verify` is the final command shape for browser-backed +ad-template verification. + +When this work is combined with the rebased #800 audit command, `ts audit` +should become a subcommand namespace: + +```bash +ts audit page +ts audit generate +ts audit ad-templates verify ... +``` + +The existing #800 `ts audit ` behavior should be preserved as a +compatibility alias for `ts audit generate ` during the transition, +including its artifact output flags. This avoids a successful but silent +behavior change for existing onboarding scripts. + +Parsing contract: + +- `ts audit page ` is the canonical generic page-audit command. +- `ts audit generate ` is the canonical artifact-generation command. +- `ts audit ad-templates verify ...` is the canonical ad-template verifier. +- `ts audit ` is a hidden compatibility alias for + `ts audit generate ` and is accepted only when `` parses as `http` + or `https`. +- `ts audit ad-templates` must never be treated as a legacy URL positional. +- `ts audit page` without a URL must fail with the normal Clap missing-argument + error. + +Implementation shape: + +```rust +#[derive(Debug, clap::Args)] +struct AuditArgs { + #[command(subcommand)] + command: Option, + #[arg(value_parser = parse_http_url, hide = true)] + legacy_url: Option, +} + +#[derive(Debug, clap::Subcommand)] +enum AuditSubcommand { + Page(PageAuditArgs), + #[command(name = "ad-templates", subcommand)] + AdTemplates(AuditAdTemplatesCommand), +} +``` + +If Clap cannot enforce the optional-subcommand plus hidden positional contract +cleanly, implement a small custom dispatcher for the `audit` argv tail and test +it directly. Required parser tests: + +- `ts audit https://www.example.com/` dispatches to artifact generation; +- `ts audit page https://www.example.com/` dispatches to page audit; +- `ts audit ad-templates verify https://www.example.com/` dispatches to + ad-template verification; +- `ts audit ad-templates` does not parse as a URL; +- `ts audit ftp://www.example.com/` fails before browser launch. + +JSON error output is intentionally left to the broader CLI output contract. This +spec only standardizes successful verification result JSON and strict-mode +verification failure JSON where page results exist. diff --git a/docs/superpowers/specs/2026-07-24-prevent-duplicate-gpt-slot-requests-design.md b/docs/superpowers/specs/2026-07-24-prevent-duplicate-gpt-slot-requests-design.md index 68e1cf75e..befdcd427 100644 --- a/docs/superpowers/specs/2026-07-24-prevent-duplicate-gpt-slot-requests-design.md +++ b/docs/superpowers/specs/2026-07-24-prevent-duplicate-gpt-slot-requests-design.md @@ -21,9 +21,10 @@ A fix must keep both implementations in sync. 1. A configured placement has at most one initial GPT slot and ad request when TS runs before a publisher defines its inner div. -2. Apply TS targeting and the `ts_initial=1` marker before that single initial - request. -3. Continue reusing a slot that the publisher has already defined. +2. Apply TS targeting and the `ts_initial=1` marker only when TS owns that single + initial request. +3. Continue reusing a slot that the publisher has already defined without changing + its targeting after a publisher auction, GPT request, or GPT render claims it. 4. Keep the TS-only fallback: if the publisher never defines the placement, TS still displays it and makes exactly one initial request. 5. Preserve `disableInitialLoad()`, SPA targeting cleanup, and the rule that TS does @@ -35,11 +36,36 @@ A fix must keep both implementations in sync. - Deduplicating by GAM ad-unit path. Multiple visible placements may validly share a path. - Changing publisher GAM configuration, line items, or refresh policy. -- Delaying the initial TS request while waiting an arbitrary amount of time for - framework hydration. A time-based grace period cannot distinguish a slow - publisher-owned slot from a placement that the publisher will never define. +- Delaying the initial TS request while waiting for a publisher that has not made a + concrete claim. A time-based grace period cannot distinguish a slow publisher-owned + slot from a placement that the publisher will never define. An actual publisher + `requestBids()` call receives a bounded lease instead. - General interception of unrelated GPT slots. +## Decision: first claimant owns delivery + +The first valid claimant owns each physical slot's first impression for the current +navigation. A real publisher `requestBids()` call claims before native Prebid starts. +A GPT `slotRequested` or `slotRenderEnded` event also claims for the publisher when TS +has not claimed first. `adInit()` may write `ts_initial=1`, apply `hb_*` targeting, and +request an existing slot only after it atomically claims an untouched slot. + +Publisher auction claims use unique, expiring registration tokens. The matching +callback moves only its token to delivery-pending and attaches returned ad IDs. +Overlapping auctions cannot clear each other's tokens. Exact ad-ID delivery consumes +only its matching registration. A code-only delivery consumes a registration only +when exactly one current candidate matches; ambiguous ordinary deliveries run a new +auction, while ambiguous TS-owned suppressing deliveries fail closed without deleting +their tombstones. If TS claimed first, the GPT refresh wrapper filters one correlated +losing publisher delivery and restores the TS targeting snapshot. It forwards every +unaffected slot and the original refresh options exactly once. The one-shot state is +then consumed, so later publisher refresh auctions remain eligible. + +If a publisher claim expires without a GPT request, `adInit()` retries only that slot +after checking the navigation generation, DOM element identity, and ownership again. +It never reruns whole-page initialization. Strict TS-first delivery is outside this +design because it would require holding publisher delivery while page-bids settles. + ## Decision: one inner-div slot with late-definition handoff TS will define its fallback slot on the **actual inner div**, never on its outer diff --git a/docs/superpowers/specs/2026-08-08-esi-cacheable-root-validation-design.md b/docs/superpowers/specs/2026-08-08-esi-cacheable-root-validation-design.md new file mode 100644 index 000000000..031ecc50c --- /dev/null +++ b/docs/superpowers/specs/2026-08-08-esi-cacheable-root-validation-design.md @@ -0,0 +1,864 @@ +# The Cacheable Root: Latency Diagnosis and Stage 0 Design + +_Filename retains its original `esi-` prefix; the commit history and every +cross-reference point at it. The subject moved, the path did not._ + +**Issue:** IABTechLab/trusted-server#1009 · **Date:** 2026-08-08 · + +> **HISTORICAL RECORD — NOT THE CURRENT IMPLEMENTATION.** This document preserves the +> measurement and feasibility investigation. Every executable ESI tag, parser, and +> subrequest described below belongs to a rejected spike; do not use those sections to +> infer current runtime behavior. The final branch retains `assembly_mode = "esi"` only as +> the operator spelling for Fastly C2 plus exact byte-seam assembly. See +> [the merge-hardening design](./2026-08-12-1009-esi-merge-hardening-design.md). + +**Revised:** 2026-08-10 +**Baseline:** citations verified at `cfb98f4`; unchanged as of `b0ce56c3`. + +> ## ⚠️ Correction, 2026-08-10 — this document's original ESI verdict was wrong +> +> The first revision concluded that ESI was **structurally blocked**: that it +> presupposed a TS-owned template cache which did not exist, and that such a cache was +> in turn blocked on purge capability the platform did not offer. **Both claims are +> false**, and an external review was right to reject them. +> +> Verified against the pinned `fastly` 0.12.1: +> +> - **The cache boundary is native.** `fastly::cache::core` provides +> `insert(key, max_age).execute() -> StreamingBody` for arbitrary bytes, `lookup()` / +> `found()` to read them back, and `Transaction` with `must_insert()` for request +> collapsing. The two-stage design needs no separate KV or template service. +> - **Purge exists in-process.** `fastly::http::purge::purge_surrogate_key` purges from +> inside Compute; the management-API token scope cited in the original is irrelevant to +> it. Note which cache, though: `InsertBuilder::surrogate_keys([...])` is the **Core +> Cache** API and keys the transformed-template cache (C2). It does **not** key the HTTP +> read-through cache (C1) that Stage 0 turns on — purging that needs origin-supplied +> keys or the HTTP cache's own surrogate-key surface. +> - **The original pipeline ordering was backwards.** It said "order esi → lol*html, +> never the reverse." `lol_html` \_emits* the ESI include tags, so ESI must run after it. +> Correct order is in [§6.6](#66-the-esi-pipeline-corrected). +> +> The error was inspecting what this repository does and reporting it as what the +> platform permits — the same mistake this document criticises #1009 for making in the +> other direction. +> +> **ESI is therefore feasible and unvalidated, not rejected.** Validating it is +> [a separate plan](../plans/2026-08-10-1009-esi-validation-spike.md). What survives +> here is the latency re-diagnosis and the Stage 0 optimisation, which are worth doing +> and are **not** an answer to #1009. + +## Document map — read this first + +#1009 is answered across three documents, not one. This is the only place that says +which owns what. + +| Document | Owns | +| ------------------------------------------------------------------------ | --------------------------------------------------------------------------------------------------- | +| **This spec** | Why the TTFB regression happens, what Stage 0 is and why, and the corrected ESI feasibility verdict | +| [Stage 0 plan](../plans/2026-08-08-1009-measurement-and-stage-0.md) | Implementing the measurement and the cache-bypass flag. **Does not close #1009.** | +| [ESI validation spike](../plans/2026-08-10-1009-esi-validation-spike.md) | **Where #1009 is actually decided.** Four arms, safety gates, decision rule. | +| [Findings](../plans/2026-08-08-1009-measurement-findings.md) | Recorded results. Currently: Step A only, at `PROVISIONAL PASS`. | + +**If you want the ESI answer**, it is [§2](#2-why--the-three-findings) for the verdict, +[§6.6](#66-the-esi-pipeline-corrected) for the pipeline, and the spike plan for how it +gets validated. Everything else here is Stage 0 and the latency analysis behind it. + +**Decision requested:** approve the four items in §1. + +> **Orientation.** #1009 asks whether Edge Side Includes (ESI) can cache page fragments +> so that cacheable publisher HTML is separated from per-user ad state, recovering a +> TTFB regression that Trusted Server (TS) adds to navigations on a Next.js App Router +> publisher running on Fastly Compute. ESI can do this; whether it should is not settled +> here. Separately, the regression has a cheaper cause than the issue assumes. +> +> **This document deliberately carries no performance measurements.** Every conclusion +> below is derived from code at the pinned baseline, so it can be checked by reading the +> repository rather than by trusting a benchmark. Where a quantity is needed and unknown, +> it is named as unknown and [§3](#3-monday-morning) says how to obtain it. +> +> Terms used throughout: **the hold** = TS holding the HTTP response open at `` +> until the server-side auction (SSAT) resolves. **React #418** = the React +> hydration-mismatch error raised when `adInit()` mutates ad-slot subtrees during +> hydration; it is why bid application is deferred to `window.load`. It is a React error +> number, **not** a repository issue — the tracker is +> [#938](https://github.com/IABTechLab/trusted-server/issues/938). **The SSAT price +> defect** = a live mispricing bug named in #1009 (prices reading 100× high) — cited +> from #1009 and prior investigation, not re-verified here. + +--- + +## 1. Decision requested + +| # | Decision | Owner needed | +| --- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | ------------- | +| D1 | **ESI is feasible and unvalidated.** Validate it via [the spike plan](../plans/2026-08-10-1009-esi-validation-spike.md), not by deferring it. | Eng + product | +| D2 | **Fund ~3 days of measurement** (§3). No dependencies. Can start immediately. | Eng | +| D3 | **Approve Stage 0** — an operator flag disabling the origin cache bypass, subject to a **`FINAL PASS`** in §3. A `Vary` check alone is a `PROVISIONAL PASS` and is not a release gate. Rollback needs a purge path, not only a config push. | Eng | +| D4 | **Stages 1–2 queue behind the SSAT price defect and #938.** Stages 3b–4 unscheduled. ESI is not in this queue — see §7. | Product | + +Rationale for D4 in [§8](#8-priority). Everything this document recommends _against_ +doing is in [§7](#7-deferred-work-specified-not-scheduled), at deliberately lower +detail than the work it recommends. + +--- + +## 2. Why — the three findings + +**ESI is buildable on the pinned SDK, and unvalidated.** `lol_html` emits executable ESI +include tags into a shared template; `fastly::cache::core` stores that template; the +`esi` crate assembles per request on the way out. Everything that requires is +already a dependency. The real open questions are empirical, not architectural: does it +beat a plain client fetch by enough to justify a Fastly-only rendering path, and can +per-user leakage be excluded under cold MISS, warm HIT, stale revalidation, and fragment +failure. [The spike plan](../plans/2026-08-10-1009-esi-validation-spike.md) answers +those; [§6.6](#66-the-esi-pipeline-corrected) gives the pipeline. + +Two constraints stay true regardless. ESI is **Fastly-only at every API level**, so it +is a per-platform accelerator rather than the architecture, and its maintenance cost +belongs in the decision. And its Dynamic Content Assembly must be **explicitly disabled** +— bid payloads carry partner-controlled creative markup, so under `DcaMode::Esi` an SSP +could embed an ESI include targeting an arbitrary URL and make the edge fetch it. Details in +[Appendix E](#appendix-e--esi-notes-condensed). + +**The auction is already out of band; the hold is ~free.** It is dispatched _before_ +the origin fetch and does not block — dispatched at `publisher.rs:2751-2755`, sent at `:2870` — +with a 500 ms budget. The actual cost is `with_cache_bypass` +(`publisher.rs:2867`), +which forces every ad-eligible navigation to miss the Fastly readthrough cache. + +**The two fixes are multiplicative.** Removing the bypass alone lets the previously +hidden auction surface as the new bottleneck. Removing the hold alone changes nothing, +because the auction was never the bottleneck. **Shipping the hold removal without the +bypass removal will measure no improvement and will read as the effort having failed** — +the most likely way this work gets judged unfairly. + +**Ordering is established; magnitude is not.** The ordering above follows from code and +needs no measurement. The _size_ of the win does — and the one quantity it depends on, +the origin build time under `Pass`, has never been measured. #1009's timings do not +supply it: they compare cached fetches against each other, not against an origin build. +**Quote no figure to a publisher until §3 Step C runs.** Full reasoning in +[§6](#6-the-analysis). + +--- + +## 3. Monday morning + +Three checks, ordered cheapest-first. Each needs a named owner before starting. + +**Step A — origin `Vary` and cookie check (minutes for the first pass).** `curl` the +origin with and without `RSC`, `Next-Router-*`, and the experiment header; inspect `Vary`, +`Cache-Control`, and `Set-Cookie`. **This first pass yields a `PROVISIONAL PASS` only** — +it is not what gates the flip. A `FINAL PASS` additionally requires a real authenticated +session, Basic Auth through TS, the experiment variant, representative routes, and +cached-hit render attribution. Do the cheap pass first because it is the +cheapest thing that unblocks anything. + +**Step B — what consumes TS's own response headers (under a day).** Request a TS-served +path that already emits `public, s-maxage` +(`http_util.rs:294-311`) +twice and look for `x-cache`/`age` on TS's _own_ response. **Gates the Stage 3a/3b +split** — see [§7](#7-deferred-work-specified-not-scheduled). + +**Step C — measure the hold directly (1 day + a measurement window).** + +The hold's cost is literally the duration of one `.await`: `collect_stream_auction` at +`publisher.rs:793`, plus the +two EOF variants in `hold_finish_ready_segments` and `hold_finish_tail_segments`. Two +`Instant`s around it yield **`hold_wait_ms`** — the number this entire document is +arguing about, measured rather than modelled. + +Emit two timings per ad-eligible navigation: + +| Metric | Why | +| ----------------- | ---------------------------------------------------------------------- | +| `hold_wait_ms` | **The decision.** How long the response was actually held for bids. | +| `origin_fetch_ms` | Attribution — how much of the win Stage 0 can claim. Origin TTFB only. | + +`hold_wait_ms` replaces the proxy comparison an earlier draft proposed. Comparing `O` +against `A` was an indirect way of asking "does the hold block?"; this asks it directly, +costs less to build, and removes the modelling error corrected in +[§6.2](#62-what-the-hold-actually-costs). + +Deliberately not measured: auction collect duration is already instrumented +(`OrchestrationResult::total_time_ms`, `auction/orchestrator.rs:285`, flowing to +`auction_events_raw`) — read it, don't rebuild it. Rewrite duration decides nothing and +would mean touching two finalizers. + +- **Mechanism: a `log::info!` line behind a debug flag, not `Server-Timing`.** A response + header would in fact work for the origin-fetch figure — that value is known before + headers commit — but a server-side log needs no browser harness to collect it, `log` is + this project's instrumentation crate, and the auction path already measures itself with + `web_time::Instant`. Gate it behind config: one line per eligible navigation is real log + spend and the instrumentation is temporary. +- **Sample: enough navigations per arm to separate the medians with confidence**, across + both page types, and state the N alongside any result. #1009's sample was small enough + that its conclusion did not survive contact with the code; replacing it with another + underpowered sample would repeat the error. + +**Step C has two outcomes, both actionable:** + +| `hold_wait_ms` median | Meaning | Effect on staging | +| --------------------- | ----------------------- | ------------------------------------------------------------ | +| Near zero | The hold is free | Proceed as staged: Stage 0 primary, Stage 2 protects its win | +| Materially non-zero | The hold **is** costing | **Staging inverts** — Stage 2 primary, Stage 0 secondary | + +The work does not change; its order and justification do. **The staging in §7 is +conditional on this measurement**, and the second outcome is a live possibility rather +than a formality — §6.2's argument for the first is weaker than an earlier draft claimed. + +Stage 1's bids-fetch timeout still needs a measured client-side figure rather than an +invented constant, but Step C is server-side and does not supply it. Capture it from the +browser harness when Stage 1 is actually scheduled. + +--- + +## 4. Stage 0 — the only build item recommended now + +Stop bypassing the read-through cache on ad-eligible navigations +(`publisher.rs:2867`). + +**Ship it as an operator flag, not a deletion.** Add +`publisher.bypass_origin_cache`, defaulting to today's behaviour, in the same release as +the Step C instrumentation. Then turn it off with `ts config push`. + +The diff is slightly larger than deleting a line, and that is the point. The risk being +gated here is **cache poisoning** — serving one representation in response to a request +for another. For that class of failure, rollback speed dominates diff size: a config push +reverts the read path in seconds where a release does not — but a config push **evicts +nothing**, so full rollback is flip, then purge or roll a versioned key namespace, then +observe past the origin TTL. The flag also buys an A/B on a byte-identical +build, removing build difference as a confound in the very measurement this depends on, +and allows flipping for a tester-cookie population before all traffic. + +Retire the flag once the change has held: flip the default, then delete the setting and +its branch. A temporary flag left in place becomes permanent configuration surface. + +### What to watch after the flip + +Two regression signals, both checked before the win is: + +- **`unexpected_origin_304` abandonment rate.** That reason + (`publisher.rs:2894-2916`, + emitted via `emit_abandoned_auction` at `:2360`) exists precisely because the ad-stack + path refuses cached and conditional origin responses. Re-enabling the cache is what + could revive it. **Any non-zero rate is a rollback signal** — it means a 304 is reaching + TS that the conditional-header strip was supposed to make impossible. +- **Representation mixing.** Spot-check that HTML navigations still return HTML and RSC + fetches still return `text/x-component`. A mismatch means the `Vary` risk materialized + despite a PASS verdict. Roll back immediately; this is cache poisoning, not a + performance regression. + +**Why it is safe in principle.** The conditional-header strip runs 34 lines earlier +under the same gate (`publisher.rs:2832-2836`, +which also strips `Range`/`If-Range`), so the request already reaches the cache +unconditional and a HIT returns a full body. [The 304-prevention design](./2026-07-22-ssat-root-document-304-prevention-design.md) +added the bypass as belt-and-braces and listed the TTFB cost under its own Risks. The +strip alone satisfies its invariant. + +**But it carries a risk that design never considered — and this is the blocking +precondition.** RSC fetches are not navigations +(`is_navigation_request` +requires `Sec-Fetch-Dest: document`), so they never set the bypass and **already flow +through the readthrough cache**, while HTML navigations are `PASS`. Removing the bypass +puts both representations under one cache key. #1009 states the origin varies on +`rsc`, `next-router-*`, and a publisher-specific experiment header — if that variance is +not declared via `Vary`, the +cache can serve a flight payload to an HTML navigation. + +The classification is also not airtight: `is_navigation_request` falls back to the +`Accept` header when Fetch Metadata is absent, and its own comment warns _"this path is +weaker — `fetch()` can set Accept: text/html"_ +(`http_util.rs:84-88`). + +**A FAIL is not merely a Stage 0 blocker — it is a live production defect.** RSC fetches +already transit the read-through cache today, because they never set the bypass. If the +origin varies on `Next-Router-*` without declaring it, TS is cross-serving RSC variants +right now. On a FAIL, file that immediately and treat "ask the origin to declare `Vary`" +as urgent rather than as the cheaper of two options. + +**The `Vary` check is necessary but not sufficient.** Turning the read-through cache on +for HTML navigations exposes three things a representation check does not cover, and all +three are a larger class than the RSC split: + +- **Client `Cookie`.** TS forwards client cookies to origin unchanged — there is no + `COOKIE` strip on the publisher path. Any cookie-personalized HTML (logged-in state, + paywall meter, publisher-side A/B assignment) becomes cross-servable unless the origin + declares `Vary: Cookie` or marks those responses private. +- **Origin `Set-Cookie`.** If the origin emits `Set-Cookie` alongside a shared-cacheable + `Cache-Control`, the read-through cache can replay one visitor's cookie to the next. + TS's own privacy net downgrades **TS's** response — it runs after the cache has already + stored the origin's. +- **`Authorization`.** #1009 describes a basic-auth-gated deployment. Responses to + authorized requests entering a shared cache needs its own check. + +So Step A must capture `Cache-Control` and `Set-Cookie` too, and repeat each request with +and without a session cookie. Same minutes of work; closes the bigger hole. + +**Two effort branches, and Step A's `Vary` result decides which** — note this selects the +_shape_ of Stage 0, while the `FINAL PASS` conditions decide _whether it ships at all_: + +| Step A result | Stage 0 is… | Effort | +| ---------------------- | --------------------------------------------- | ------ | +| Origin declares `Vary` | the flag, its tests, then a config push | 1–2 d | +| Origin does **not** | a TS-side cache-key discriminator — a feature | 4–8 d | + +The discriminator is the safer design either way, because it keys on the headers that +actually distinguish the representations rather than on the navigation classification. + +**Two benefits beyond TTFB, worth stating to a publisher:** + +- **Origin load drops.** The 304-prevention design explicitly accepted _"increasing + origin load"_ as a cost. This reverses it. +- **`stale-if-error` becomes reachable.** Under `Pass` an origin outage is a hard + failure. This needs a decision rather than a default: stale HTML carries stale slot + markup, and whether that beats an error is a product call. + +--- + +## 5. The trap in the deferred work — read this before scheduling Stages 1–2 + +The hold is load-bearing for something other than latency. The invariant is: + +> `ad_bids_state` must be `Some(..)` when `lol_html` processes the `` end tag. + +The end-tag handler (`html_processor.rs:381-395`) +locks that mutex once and falls back to `build_empty_bids_script()` on `None`. + +**Removing the hold without relocating collection renders a normal page with +`tsjs.bids = {}` and no server-side ads** — no error, no non-2xx, no ERROR log. On +Axum, Cloudflare, and Spin the loss is fully silent: +`publisher.rs:2248` holds a +bare `Option` with no guard, so not even a drop warning fires. **The +SSPs are billed regardless.** + +This is why Stage 2 is gated on three companions and a production soak, and why slot +fill cannot be the canary — see [§7](#7-deferred-work-specified-not-scheduled). + +--- + +## 6. The analysis + +### 6.1 Corrections to #1009's premises + +| # | #1009 states | Verified against `cfb98f4` | +| --- | -------------------------------------- | -------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| 1 | Two per-user injection seams | Partly. `tsjs.adSlots` **content** is per-URL — `build_slot_json` emits config- and path-derived fields only. But its **presence** is gated on `should_run_ad_stack` (consent, bot, prefetch, kill switch), so it is request-dependent and **must not live in a shared template**. See §6.7. | +| 2 | Identity off-inline is a prerequisite | Privacy net is cookie-gated and returning navs set no cookie (`ec/finalize.rs:86-94`). **First-visit only** — but note the corollary: because returning navigations set no cookie, that net never fires for them and is **not** a backstop against shared-caching a per-user response. | +| 3 | Stamp at `:2882-2888` | `:2945-2963`, `private, no-store`, also removing `ETag`/`Last-Modified`/four CDN headers. **Seven headers.** | +| 4 | Three cacheability killers | Two more: `bypass_cache` and the `304→502 guard`. **The bypass is the cost.** | +| 5 | Two `!Send` pipelines is the hard part | `?Send` already pervasive. **Not the obstacle.** | +| 6 | Goal: root as a shared Fastly HIT | Nothing caches TS's own response on Compute; the A/B's `x-cache` is the **backend readthrough** cache. **Reframes the goal.** | + +Rows on #1009's `esi` compatibility check (holds), its drifted line numbers, and its +two broken `#1`/`#3` cross-references are in [Appendix A](#appendix-a--full-1009-correction-table). + +**Credit where due.** #1009 names the hold as blocker 1 and states it correctly. What +changes here is its _causal weight_. Likewise, #1009's own observation that TS _"shifts +the auction cost from client-side to server-side rather than adding new work"_ is the +argument for client-fill, which the issue then declines in favour of ESI. + +### 6.2 What the hold actually costs + +**An earlier draft of this section claimed a stronger argument than the code supports. +It was wrong, and the correction matters.** + +The hold does not key off `lol_html` at all. `BodyCloseHoldBuffer::push` +(`publisher.rs:2190-2202`) +scans the **decoded origin input** for ` Dispatch precedes the origin fetch, so the hold costs `max(0, A − T)`, where `A` is the +> auction collect duration and `T` is origin TTFB plus body transfer up to the `` +> byte. Since `` sits at the end of a document, `T` is close to the full download. + +`A` is bounded by `auction_timeout_ms`, resolved as +`creative_opportunities.auction_timeout_ms` falling back to `auction.timeout_ms` +(`publisher.rs:2680-2684`) +— check the resolution order against your own config rather than trusting a number; the +shipped example sets different values at each level. + +**This is a claim requiring measurement, not a proof.** §3 Step C measures the hold's +cost directly rather than inferring it. + +A finding that does survive, and belongs with [the ceiling](#64-the-ceiling): because +`HtmlWithPostProcessing` withholds all output until the final chunk, the streaming-prefix +design at `publisher.rs:1343-1348` +— whose comment promises "the client receives the document up to `` while the +auction rides alongside transfer" — is **inert on a Next.js publisher**. Every +`step.ready` yields empty bytes. That comment is misleading on exactly the publisher +under discussion. + +### 6.3 The quantity nobody has measured + +Write the fetch time under `Pass` as `O`. Recovery depends on it, and it has never been +captured. #1009's timings cannot supply it: they compare a POP hit against a +shield-served fetch, both of which are _cached_ paths, whereas `CacheOverride::Pass` +bypasses TS's read-through cache and its shield. + +Note `Pass` bypasses **TS's** caches only. It has no authority over any CDN the publisher +runs in front of their own origin — and #1009's `x-cache: MISS, MISS` on the TS-on arm +hints one may exist. So `O` may not be origin build time at all. Since `O` is the single +quantity this model depends on, that ambiguity is worth resolving in Step C rather than +assuming. + +What follows from code alone, without any number: + +| Configuration | Long pole after the change | Recovery | +| ------------------- | -------------------------- | ------------------------------ | +| Hold removal only | origin (still `PASS`) | **none** | +| Bypass removal only | the auction budget | partial — the auction surfaces | +| **Both** | the rewrite | **the full available win** | + +That ordering is what the staging rests on, and it is measurement-independent. The +magnitude of each row is not, and §3 Step C supplies it. + +### 6.4 The ceiling + +#1009 targets "approach the TS-off warm numbers." **Unreachable, structurally.** Those +numbers are TS-off _streaming_ a POP HIT. TS buffers the whole document before emitting +a byte (16 MB cap), so its floor is `full origin body download + full rewrite` — above a +streamed hit by construction, whatever the timings turn out to be. Set the target from +Step C's measured rewrite cost rather than from the TS-off baseline. Going below the +floor requires true origin streaming (#849), out of scope. A non-Next.js publisher with +no post-processor takes the streaming path and would see a lower floor. + +### 6.5 Confidence + +**High on the structural claims.** §6.2's argument, the bypass forcing a cache miss, the +the silent-empty-bids failure mode, the geo and `Vary` blockers, +and the fill-canary blindness are all read directly out of the code at `cfb98f4`. Anyone +can check them without running anything. + +**None on magnitude.** `O` is unmeasured and the rewrite cost is unmeasured. This +document does not estimate them, and no figure in it should be quoted as one. + +Worth stating plainly: #1009 reached the opposite causal conclusion from a small sample. +That is a caution about small samples generally, not only about that one — which is why +§3 Step C specifies the measurement rather than this document supplying a substitute +for it. + +### 6.6 The ESI pipeline, corrected + +An earlier revision of this document said "order esi → lol*html, never the reverse." +That is backwards. `lol_html` is what \_emits* the ESI include tags; ESI cannot process +tags that do not exist yet. The correct order: + +``` +origin → lol_html transform → fastly::cache::core → finalize headers → stream esi assembly → client + (one unconditional marker (shared template, (EC cookie, geo, (per request, + at the body-close seam; surrogate-keyed, unconditional fetch the + the head seam is NOT a TS-chosen TTL) private/no-store) fragment) + hole — adSlots presence + is request-gated, §6.7) nothing may change + after this point +``` + +The push/pull mismatch that the earlier revision treated as a blocker is real but +irrelevant: `lol_html` pushes, `esi` pulls, and **the cache is the buffer between them**. +That is not an obstacle to the two-stage design — it _is_ the two-stage design, which is +what #1009 proposed in the first place. + +Mechanism, all present in the pinned `fastly` 0.12.1: + +| Need | API | +| ----------------------- | ------------------------------------------------------------------------------------------------------------------------------------------ | +| Store the template | `cache::core::insert(key, max_age).execute() -> StreamingBody` | +| Read it back | `cache::core::lookup(key)` → `found()` | +| Avoid a thundering herd | `cache::core::Transaction` — `must_insert()` / `must_insert_or_update()` | +| Invalidate **C2 only** | `InsertBuilder::surrogate_keys([...])` (Core Cache) + `fastly::http::purge::purge_surrogate_key`. Does **not** key C1 — see the row below. | +| Invalidate C1 | Origin-supplied surrogate keys, or the HTTP cache's own surrogate-key surface. Not the Core Cache API. | + +Purge runs **inside Compute**. The management-API token scope cited under +[Stage 4](#7-deferred-work-specified-not-scheduled) governs a different surface and does +not gate this. + +**Three caches, kept distinct.** Conflating them is what produced the original error: + +1. **Origin read-through** — raw origin bytes. What Stage 0 turns back on. +2. **Shared transformed template** — post-`lol_html`, pre-ESI, no per-user data. The ESI + target, and new. +3. **Assembled-response delivery cache** — the final per-user output. **Must never + exist.** Nothing in this document or the spike proposes one. + +**Validation constraint.** Viceroy 0.17 cannot exercise the customized read-through hooks +end to end. Unit tests can cover the transform and the security properties; MISS / HIT / +stale / shielding behaviour must run against a real Fastly test service. + +--- + +### 6.7 What may and may not live in a shared template + +A correction to §6.1 row 1, and the constraint that governs any shared-template design. + +The original framing — "`adSlots` is per-URL, so there is one per-user hole, not two" — +is half right and dangerously so. `build_slot_json` really does emit only config- and +path-derived fields. But whether the script is emitted **at all** is gated on +`should_run_ad_stack` (`publisher.rs:2920-2927`), which is +`is_get && is_navigation && !is_prefetch && !is_bot && has_matched_slots && +consent_allows_auction && auction_enabled`. + +So the _content_ is per-URL and the _presence_ is per-request. A shared object filled by +the first request would freeze that request's consent decision, bot classification, +prefetch status, and kill-switch state for every later reader. A consent-denied fill +serves a no-ads template to consenting users; a consenting fill serves ad markup to +someone who refused. + +**The rule for anything cached and shared:** + +| May live in the template | Must live in the per-request fragment | +| --------------------------------------- | ---------------------------------------------- | +| tsjs bundle script tag (content-hashed) | `tsjs.adSlots` — presence is request-gated | +| URL rewrites (per-host, in the key) | `tsjs.bids` | +| | GPT diagnostics bootstrap (cookie/query-gated) | +| | Integration head-inserts (request-scoped) | + +The test that catches this class is **byte-identity of the template across requests +differing in consent, bot classification, and prefetch status** — not an absence-of- +per-user-values scan, which the broken design would have passed. + +This applies to any shared-template work, ESI or client-fill alike. The +[spike plan](../plans/2026-08-10-1009-esi-validation-spike.md) implements it. + +--- + +## 7. Deferred work, specified not scheduled + +**The full sequence, in one place.** Stage 0 is specified in [§4](#4-stage-0--the-only-build-item-recommended-now) +rather than repeated here; everything below it is deferred. + +| Stage | What | Status | +| ----- | ----------------------------------------------- | ---------------------------------------------- | +| **0** | Operator flag disabling the origin cache bypass | Recommended now. Gated on a `FINAL PASS`. §4. | +| 1 | Bid delivery off the response body | Deferred behind the correctness defects | +| 2 | Delete the `` hold | Deferred; one-way, needs a Stage 1 soak | +| 3a | Browser caching (`private, max-age` + `ETag`) | Specified, low risk, unscheduled | +| 3b | Shared cacheability | Blocked on geo suppression, `Vary`, and Step B | +| 4 | Purge wiring | Prerequisite for any TS-owned cache | + +**ESI is not a stage here.** It was Stage 5 in an earlier revision, queued behind the +rest. It no longer queues: it is feasible on the pinned SDK and is decided by +[its own spike plan](../plans/2026-08-10-1009-esi-validation-spike.md), which runs +independently of Stages 1–4. The shared template cache it needs is `fastly::cache::core` +([§6.6](#66-the-esi-pipeline-corrected)), not a new service. + +Lower detail below is deliberate. Full specifications are in the appendices. + +**Stage 1 — bid delivery off the response body.** The client fetches `/_ts/page-bids` at +navigation generation 0. Endpoint, same-origin gate, wire shape, and client consumer +already exist. Three decisions must be made before planning: the `slots: []` precedence +rule when head-open already injected a non-empty `ts.adSlots`; the new terminal-event +emission point; and whether the dispatch/collect split survives at all. Plumbing detail +in [Appendix B](#appendix-b--stage-1-plumbing-condensed). Estimated 8–13 d, low-to-medium +confidence, uncertainty concentrated client-side. + +Three companions are mandatory, not optional: **suppress the server bids script +entirely** (not an empty one), **fail loud** (the end-tag handler takes bids by value so +a missing auction is a compile error), and **relocate telemetry** (navigation +`Completed` rows are emitted only from the collect functions, and the `ts-debug` dump +rides the same string). Behaviour change to accept: under client-fill the auction runs +only if the browser executes the fetch, so bots and JS-disabled clients stop triggering +server-side auctions — revenue-relevant, sign unknown. + +**Stage 2 — delete the hold.** 5–8 d. **Rollback is one-way**: it deletes the hold, the +dispatch/collect split, and twelve tests, so the only revert is a release. Ships only +after Stage 1 has run flag-on in production for a window defined _before_ Stage 1 +starts, with TS-attributed renders flat and `auction_events_raw` navigation rows intact. +Secondary wins: removes the duplicated per-codec decoder/encoder wiring, six compression +imports, and the non-parser-context `` runs _all_ attempts and concatenates every non-failed output** — not + first-success-wins, so primary/fallback pairs render both. Least obvious behaviour in + the crate. +- Single include, not per-slot: the auction is one operation producing all slots' bids. + +--- + +## Appendix F — deferred open items (condensed) + +Implementation-level, for unscheduled work only. Decisions needing a human are in +[§9](#9-decisions-needed-from-this-review). + +Should `collect_non_html_auction` (`publisher.rs:2388`) go with the hold or stay? Is +`body_close_hold_loop_stream` (`:2109`, no production caller) safe to delete, or is the +buffered-adapter streaming cutover (#495) still live? Does hidden-tab rAF behaviour +interact badly with a bids timeout? What are Fastly's pending-request semantics when a +`DispatchedAuction` drops mid-flight? Does `stale-if-error` on a cached root serve +acceptable content given stale slot markup? And the googletag shim discards listeners +queued before it loads (#1009 Part 1) — not filed, should be. + +--- + +## Appendix G — code-grounded seams + +All pinned to `cfb98f4`. + +| Concern | Location | +| --------------------------------------------------------- | ------------------------------------------------------------------------------- | +| Eligibility decision | `publisher.rs:2651`, `:2660` | +| `is_navigation_request` | `http_util.rs:73-98` | +| Auction dispatch (pre-origin, non-blocking) | `publisher.rs:2698-2760` | +| Auction overlap intent | `auction/orchestrator.rs:950-952` | +| Auction timeout resolution | `publisher.rs:2680-2684` (creative_opportunities, else auction.timeout_ms) | +| Conditional/range header strip | `publisher.rs:2832-2836` | +| Origin cache bypass | `publisher.rs:2866-2868` | +| Origin 304 → 502 guard | `publisher.rs:2894-2916` | +| `adSlots` build (content per-URL, presence request-gated) | `publisher.rs:2920`, `:3558-3577`, `:3501-3525` | +| Uncacheable stamp | `publisher.rs:2945-2963` | +| `` hold — sync / async / Fastly lazy | `publisher.rs:2235` / `:2109` / `:1318-1390` | +| Hold buffer | `publisher.rs:2177-2218` | +| Auction collect (HTML / non-HTML) | `publisher.rs:2431` (emits `:2456`) / `:2388` (emits `:2410`) | +| Abandonment emitter | `publisher.rs:2360` | +| Bids script build | `publisher.rs:3438-3491` | +| `/_ts/page-bids` | `publisher.rs:3611`, handler `:3723`, auction `:3903` | +| Injection seams (head / body-close) | `html_processor.rs:310-363` / `:381-395` | +| Post-processor buffering | `html_processor.rs:62-94` | +| Next.js post-processor registration | `integrations/nextjs/mod.rs:107` | +| Max buffered body (16 MB) | `settings.rs:77-79` | +| EC cookie issuance policy | `ec/finalize.rs:86-107` | +| Cookie-privacy net | `response_privacy.rs:20-61` | +| Geo response headers | `adapter-fastly/src/middleware.rs:194-200` | +| Cacheable-header precedent | `http_util.rs:294-311` | +| Management token lacks purge (wrong surface) | `adapter-fastly/src/management_api.rs:12` | +| In-process purge / surrogate keys | `fastly` 0.12.1 `cache::core`, `http::purge::purge_surrogate_key` | +| Client initial-ad gate | `js/lib/src/integrations/gpt/index.ts:536-555` | +| `adInit` bid application | `js/lib/src/integrations/gpt/index.ts:566`, `:652`, `:657-661` | +| Client SPA auction hook | `js/lib/src/integrations/gpt/index.ts:806`, `:859`, `:892-949` | +| Prior design that introduced the killers | `docs/superpowers/specs/2026-07-22-ssat-root-document-304-prevention-design.md` | diff --git a/docs/superpowers/specs/2026-08-10-config-first-auction-provider-architecture-design.md b/docs/superpowers/specs/2026-08-10-config-first-auction-provider-architecture-design.md new file mode 100644 index 000000000..ca13e13e0 --- /dev/null +++ b/docs/superpowers/specs/2026-08-10-config-first-auction-provider-architecture-design.md @@ -0,0 +1,1092 @@ +# Config-First Auction Provider Architecture + +**Date:** 2026-08-10 +**Status:** Draft +**Scope:** Configuration-first OpenRTB 2.6 auction providers with Prebid Server and APS parity + +## Summary + +Redesign Trusted Server auction-provider registration around configuration-defined provider instances. + +The auction orchestrator will no longer contain statically configured Prebid Server and APS provider instances. Instead: + +- Provider instances are defined under `[auction.providers.*]`. +- OpenRTB 2.6 is the only protocol implemented in the first version. +- A provider selects an OpenRTB profile that owns nonstandard request and response semantics. +- Trusted Server integrations may register profiles through a compile-time Rust registry. +- A central bidder registry routes each client-requested bidder to exactly one provider. +- Provider configuration is compiled and validated at startup into an immutable auction plan. +- The runtime bidder-provider path operates only on compiled provider plans and normalized auction data. + +This redesign changes provider registration, routing, request construction, and response normalization. It deliberately preserves existing auction economics, privacy enforcement, signing behavior, creative delivery, the existing statically registered mock mediation path, and telemetry unless a structural change is required to support the new provider architecture. It does not introduce a generic mediator type or mediator profile. + +> **Core principle:** Configuration defines provider instances. Rust code registers +> tested protocols and profiles. A compiler turns configuration into an immutable +> plan. The orchestrator executes that plan without knowing about Prebid Server, +> APS, or any specific exchange. + +## Goal + +Allow operators to register any endpoint that conforms to the supported OpenRTB 2.6 subset without adding a new provider implementation. + +The same architecture must fully replace the current Prebid Server and APS auction providers while preserving their required behavior through OpenRTB profiles. + +## Problem + +The current system models Prebid Server and APS as statically named provider implementations. Each implementation combines several concerns: + +- Provider identity and enablement. +- OpenRTB request construction. +- Provider-specific request extensions. +- HTTP transport and backend registration. +- Timeout handling. +- OpenRTB response parsing. +- Provider-specific validation. +- Creative interpretation. +- Diagnostics. + +This causes several constraints: + +- Only one instance of each statically named provider can be registered. +- Adding a standards-compliant OpenRTB endpoint still requires Rust provider code. +- Provider identity, protocol behavior, and upstream seat identity are not cleanly separated. +- Slot routing is implicit and differs by provider. +- Deploy-time validation and runtime registration maintain separate provider inventories. +- Prebid browser integration concerns are coupled to server-side provider configuration. + +The auction orchestrator itself already has useful behavior for parallel execution, deadlines, partial failures, winner selection, mediation, and telemetry. The redesign should preserve those strengths while replacing provider construction and routing. + +## Design Decisions + +### Configuration-first boundary + +“Configuration-first” means a standards-compliant endpoint within the supported OpenRTB 2.6 subset can be onboarded using configuration alone. + +It does not mean: + +- Arbitrary OpenRTB variants can be programmed through configuration. +- New protocols can be implemented through templates or scripts. +- Provider-specific behavior can use unrestricted JSON transformations. + +If an endpoint requires nonstandard semantics, those semantics are implemented as ordinary Rust profile behavior registered in this repository. + +### Protocol scope + +The architecture may support additional protocols later, but the first version implements only: + +```text +openrtb-2.6 +``` + +No runtime plugin, sandboxed WASM, dynamic native extension, or configuration scripting system is included. + +### Provider configuration location + +All server-side auction-provider instances are configured under: + +```toml +[auction.providers.*] +``` + +Browser and page integrations remain under: + +```toml +[integrations.*] +``` + +A browser integration may be disabled while its server-side auction profile is used by a provider. + +### Profile model + +The first version exposes one profile selection per provider rather than a configurable list of capabilities. + +Profiles are registered Rust implementations with narrow responsibility for protocol-specific request and response semantics. The initial profiles are: + +- `standard` +- `prebid-server` +- `aps` + +Internally, profile implementations may share smaller components. Those components are not exposed as a public capability-composition language in the first version. + +### Current behavior preservation + +The redesign preserves the current behavior of: + +- Trusted Server request signing, using the existing signing protocol, while intentionally expanding enabled signing coverage to every OpenRTB provider. +- Consent extraction, privacy enforcement, and identity gating. +- Highest decoded-price winner selection. +- Current floor enforcement. +- Current USD assumptions. +- The existing statically registered mock mediation path. +- Creative sanitization and delivery. +- APS rendering behavior. +- Prebid Cache handling. +- Existing auction telemetry and provider outcome semantics. + +The project does not redesign these systems. + +### Media scope + +The first version supports banner inventory only. + +Non-banner formats are excluded before provider routing and are never emitted upstream. A slot with no valid banner format is skipped. The canonical model and protocol boundary may remain extensible to video and native, but video and native request construction, validation, ranking, and rendering are outside this specification. + +## Non-Goals + +This specification does not include: + +- A new auction pricing or ranking model. +- Currency conversion or a new money representation. +- Changes to floor behavior. +- A generic mediator type, mediator profile, or configuration-first mediator architecture. +- Changes to the existing mock mediation behavior. +- A new request-signing protocol. +- A new privacy or consent system. +- A new telemetry schema. +- A new creative renderer architecture. +- Video or native media support. +- Multiple routes for the same bidder. +- Request splitting across multiple upstream calls for one provider. +- Runtime-loaded profiles or plugins. +- Sandboxed WASM extensions. +- Arbitrary JSONPath, templates, scripts, or response expressions. +- Speculative endpoint authentication mechanisms. +- Label-based routing, provider groups, or a general routing rule language. +- Arbitrary overrides of standard OpenRTB fields. +- Migration compatibility with the current provider configuration schema. + +Breaking configuration changes are acceptable for this design. + +## Terminology + +### Provider ID + +A unique operator-defined provider instance, such as: + +```text +pbs-primary +aps-primary +fictional-direct +``` + +Provider IDs must match `^[a-z][a-z0-9-]{0,62}$`. This lowercase ASCII grammar is part of the configuration contract: it excludes control characters and adapter-specific punctuation aliases, keeps backend names bounded, and still requires target-encoded collision validation. + +Provider health, runtime correlation, backend discrimination, configuration, and telemetry use the provider ID. + +### Bidder ID + +A demand source requested by the publisher or browser integration, such as: + +```text +rubicon +pubmatic +appnexus +``` + +Trusted Server maps each bidder ID to one provider ID. + +### Protocol + +The wire contract used by a provider. The only first-version value is `openrtb-2.6`. + +### Profile + +A registered Rust implementation that augments generic OpenRTB request construction and interprets nonstandard response semantics. + +### Seat + +The buyer identity returned in `seatbid.seat`. A seat is not a provider ID and must not be used for transport correlation. + +### Delivery bidder code + +The bidder code serialized to the browser-facing auction response. It is distinct from provider ID and returned seat because current delivery contracts differ: Prebid Server uses a valid returned seat or the fallback `unknown`, while APS must continue to use `aps` so the existing browser renderer activates. + +### Provider plan + +An immutable, validated runtime representation compiled from one provider's configuration. + +## High-Level Architecture + +```mermaid +flowchart TD + Config[Trusted Server configuration] --> Compiler[Auction plan compiler] + Registry[Protocol and profile registry] --> Compiler + Compiler --> Plan[Immutable AuctionPlan] + Request[Canonical AuctionRequest] --> Router[Bidder and slot router] + Plan --> Router + Router --> Inputs[Per-provider ProviderAuctionInput] + Inputs --> Encoder[OpenRTB 2.6 driver and selected profile] + Encoder --> Transport[Existing platform HTTP transport] + Transport --> Decoder[OpenRTB decoder and selected profile] + Decoder --> Outcomes[Normalized provider outcomes] + Outcomes --> Decision[Existing ranking or mediation] + Decision --> Delivery[Existing creative delivery] +``` + +### Control plane + +The control plane parses configuration, registers available profiles, validates provider and bidder references, and compiles an immutable `AuctionPlan` during startup. + +### Runtime plane + +The runtime plane receives a canonical auction request, routes its bidder demand to provider plans, creates one provider-specific input per provider, executes the existing concurrent auction flow, and normalizes responses before the existing decision stage. + +Raw configuration is not repeatedly interpreted during auctions. + +## Configuration Schema + +### Auction configuration + +The provider blocks are the source of truth for bidder providers. A separate ordered bidder-provider name list is not required. + +The existing `[auction].mediator` reference may continue to select the current statically registered mock mediator. That mediator is not configured under `[auction.providers.*]`, is not bidder-routed, and is not compiled through the protocol and profile registry. This specification does not generalize mediation. + +```toml +[auction] +enabled = true +timeout_ms = 2000 + +[auction.providers.pbs-primary] +protocol = "openrtb-2.6" +profile = "prebid-server" +endpoint = "https://pbs.example/openrtb2/auction" +timeout_ms = 900 +routing = "explicit" + +[auction.providers.aps-primary] +protocol = "openrtb-2.6" +profile = "aps" +endpoint = "https://aps.example/bid" +timeout_ms = 700 +routing = "all_eligible" + +[auction.providers.aps-primary.profile_config] +account_id = "example-account" +allow_script_creatives = false + +[auction.providers.rubicon-direct] +protocol = "openrtb-2.6" +profile = "standard" +endpoint = "https://rubicon.example/bid" +timeout_ms = 650 +routing = "explicit" + +[auction.providers.rubicon-direct.profile_config] +request_ext = { account = "example-account" } +imp_ext = { placementGroup = "display" } +``` + +### Common provider fields + +| Field | Required | Default | Meaning | +| ------------ | -------- | --------------- | -------------------------------------------------------------------------- | +| `protocol` | Yes | None | Registered protocol identifier. First version supports only `openrtb-2.6`. | +| `profile` | No | `standard` | Registered OpenRTB profile identifier. | +| `endpoint` | Yes | None | Fixed operator-configured HTTPS endpoint. | +| `timeout_ms` | No | Profile default | Maximum provider timeout, capped by the remaining auction deadline. | +| `routing` | No | `explicit` | `explicit` or `all_eligible`. | + +Each provider may also define a `profile_config` table. The selected profile parses that table as typed configuration; an omitted table is treated as empty configuration. + +Standard OpenRTB notification suppression is common provider configuration: + +```toml +[auction.providers.pbs-primary.notifications] +suppress_all = false +suppress_seats = ["example-seat"] +``` + +- `suppress_all` removes `nurl` and `burl` from every normalized bid returned by the provider. +- `suppress_seats` removes those URLs only when the exact returned `seatbid.seat` value matches an entry. +- Seat suppression is independent of the bidder registry because returned seats may be aliases or originate from stored requests. +- Suppression entries must be nonempty, unique strings without ASCII control characters. A provider may configure at most 128 entries, and each entry may contain at most 128 UTF-8 bytes. +- Suppression is applied after response parsing and before bids reach ranking, mediation, or delivery. + +When `timeout_ms` is omitted, the compiler resolves it from the selected profile: `prebid-server` uses 1000 ms, `aps` uses 800 ms, and `standard` uses the auction timeout. An explicit provider value overrides that default. Runtime still caps the resolved timeout by the remaining auction deadline. + +Provider presence under `[auction.providers.*]` means the provider is configured for the enabled auction. The implementation may add a conventional enablement field only if required by the broader settings system; it must not reintroduce a separate provider inventory. + +### Bidder registry + +The central bidder registry maps each client-visible bidder to exactly one provider: + +```toml +[auction.bidders.rubicon] +provider = "rubicon-direct" + +[auction.bidders.pubmatic] +provider = "pbs-primary" + +[auction.bidders.appnexus] +provider = "pbs-primary" + +[auction.bidders.aps] +provider = "aps-primary" +``` + +The client requests bidders. It does not select providers or endpoints. + +### Browser integrations + +Browser-specific behavior remains separate: + +```toml +[integrations.prebid] +# Browser bundle, injection, adapter, and script behavior only. +``` + +Enabling or disabling a browser integration does not register, enable, or disable an auction provider. + +## Configuration Validation + +The auction plan compiler must reject configuration when: + +- A provider ID is duplicated or invalid. +- Two provider IDs collide after target-adapter backend-name encoding. +- A protocol is unknown. +- A profile is unknown. +- A profile configuration cannot be parsed or validated. +- A bidder references an unknown provider. +- A bidder has more than one provider route. +- A profile cannot support banner inventory. +- A provider endpoint is not an absolute HTTPS URL with a nonempty host, contains URL credentials or a fragment, or violates stricter selected-profile endpoint requirements. +- A provider's static extension configuration is not an object. +- Static extensions exceed bounded size or nesting limits. +- Static extensions collide with reserved fields owned by the OpenRTB driver, signing, or profile. +- Request signing is enabled but its structural configuration is missing or invalid. +- More than one active provider is configured for a platform adapter that cannot perform concurrent fan-out. This target-specific validation is conservative because one auction may request bidders routed to different providers, and any `all_eligible` provider may participate alongside them. + +Compilation has two explicit stages: + +1. Target-independent compilation parses settings, resolves profiles and defaults, validates routes and field ownership, canonicalizes endpoints, and produces the immutable plan. +2. Target validation receives that plan plus an adapter capability and shared pure backend-name prediction description. It validates fan-out support, deadline claims, and encoded backend-name uniqueness without rebuilding provider configuration or duplicating runtime naming algorithms. + +The same compiler, registry, and target validator must be used by deploy tooling and runtime startup. A target-aware deploy or push passes the selected adapter description and must reject target-specific failures before publication. A target-agnostic `config validate` command runs the complete first stage and clearly reports that target checks are deferred; adapter startup remains the final mandatory target check. Configuration schema or documentation generation may consume the same registry where supported. + +## Profile Registry + +### Registration + +Profiles are registered through ordinary Rust code compiled into Trusted Server. + +Conceptually: + +```text +auction core module → registers "standard" +prebid module → registers "prebid-server" +aps module → registers "aps" +``` + +A profile's availability does not depend on its corresponding browser integration being enabled. + +### Factory responsibility + +A profile factory: + +1. Parses its typed configuration. +2. Validates its configuration. +3. Reports supported media and creative representations. +4. Declares its fixed typed standard-field policy. +5. Compiles immutable runtime profile behavior. + +### Runtime responsibility + +A profile may: + +- Declare a fixed typed policy for standard OpenRTB fields. +- Augment a generic OpenRTB request within fields reserved to that profile. +- Interpret provider-specific response extensions. +- Apply provider-specific bid validation. +- Perform deterministic provider-local candidate reduction when required to preserve registered profile behavior. +- Produce the existing normalized creative or renderer representation. +- Extract provider-specific metadata required to preserve current behavior. + +A profile must not directly overwrite fields owned by the OpenRTB driver, central privacy enforcement, or signing. The common driver applies the profile's compiled standard-field policy while constructing those fields. Each profile also declares the request extensions and response fields it owns, and the compiler rejects ownership collisions. + +A profile may not: + +- Select its endpoint. +- Send HTTP requests. +- Register platform backends. +- Resolve secrets. +- Route other providers' bidders. +- Compare bids across providers, apply auction floors, or choose final auction winners. +- Invoke mediation. +- Override central privacy enforcement. +- Modify another provider plan. + +### Typed standard-field policy + +Central privacy enforcement defines the maximum data permitted for an auction. A compiled profile field policy may omit data from that approved view, but it cannot restore, derive, or request data that central enforcement removed. + +The common driver remains the only component that constructs standard OpenRTB fields. Each registered profile declares a fixed Rust policy for differences such as: + +- `imp.tagid`. +- Primary `banner.w` and `banner.h` fields. +- `banner.topframe`. +- `site.ref` forwarding. +- Precise latitude and longitude. + +These policies are registered, reviewed, and tested Rust behavior. They are not operator-configurable arbitrary field overrides. The `prebid-server` and `aps` profiles preserve their current field behavior through their respective policies. + +The `standard` profile uses the shared PBS and APS baseline. It includes request and impression IDs, banner formats, site domain and sanitized page URL, consent-approved user ID and EIDs, user agent, IP address, coarse geo, DNT, language, consent fields, floors, secure-impression requirements, effective timeout, and current USD currency assumptions. By default it omits `site.ref`, precise latitude and longitude, `imp.tagid`, primary `banner.w` and `banner.h`, and `banner.topframe`. + +Consent fields have a fixed wire policy rather than a profile-defined arbitrary map: + +- `standard` emits an admitted TCF string as `user.consent`; applies the current jurisdiction and applicability rules to `regs.gdpr` and `regs.ext.gdpr`; mirrors admitted USP, GPP, and GPP SID values in their current top-level `regs` and compatibility `regs.ext` placements; and omits `user.ext.ConsentedProvidersSettings`. +- `prebid-server` preserves that current OpenRTB policy plus its existing consent-forwarding mode and Google Additional Consent mapping in `user.ext.ConsentedProvidersSettings`. +- `aps` preserves its current OpenRTB consent placements and deliberately omits Google Additional Consent. + +Golden field-matrix tests are normative for absent consent, GDPR applicability and jurisdiction combinations, TCF, USP, GPP and section IDs, Google Additional Consent, and cookie-sourced versus KV- or policy-sourced consent. A common-driver extraction may not broaden one profile to fields currently exposed only by another. + +### Runtime inputs + +A profile receives only: + +- Its compiled profile configuration. +- The provider ID. +- The provider's routed slots and bidder parameters. +- Canonical publisher, user, device, consent, and context data already approved for the auction. +- The effective provider timeout. +- Request-local parse state where required. + +A profile does not receive the raw downstream HTTP request or unrestricted runtime services. Browser values needed by OpenRTB are normalized into canonical auction data before profile execution. + +Request admission also retains a transport-owned Prebid header snapshot containing exactly the first values selected by the current HTTP header API for `Cookie`, `User-Agent`, `Referer`, and `Accept-Language`. It preserves accepted header bytes without introducing a second truncation rule; the existing inbound request/header limit remains authoritative. The snapshot is not exposed to profiles or other unrestricted runtime code. Common Prebid transport forwards the current `User-Agent`, `Referer`, and `Accept-Language` values and synthesizes `X-Forwarded-For` only from the platform-attested client IP, never from a client-supplied `X-Forwarded-For` value. The Prebid field policy may also use the raw accepted `Referer` for its existing `site.ref` behavior; the canonical `site.page` remains sanitized separately. APS and `standard` do not receive these raw browser headers. + +To preserve current Prebid consent-forwarding behavior, the compiled Prebid profile selects `openrtb_only`, `cookies_only`, or `both`, and common transport applies the existing behavior to the snapshotted `Cookie` value: + +- `both` and `cookies_only` forward the complete selected `Cookie` header value unchanged. +- `openrtb_only` removes the existing allowlisted consent-cookie names and forwards the remaining cookies. +- When the header cannot be parsed by the existing stripping path, current fallback behavior is preserved, including forwarding the original non-UTF-8 value. +- If stripping removes every cookie, the upstream `Cookie` header is omitted. +- Consent originating from KV or policy state remains in the OpenRTB body when no browser consent cookie can carry it, including in `cookies_only` mode. + +No other first-version profile receives a browser `Cookie` header. Reducing Prebid forwarding to consent cookies only, changing malformed-header handling, or otherwise redesigning these modes requires a separate consent-focused specification. + +## Canonical Auction Model + +The canonical auction model remains independent of the OpenRTB wire format. + +Conceptually: + +```rust +AuctionRequest { + id, + slots, + publisher, + user, + device, + privacy, + context, +} +``` + +A slot separates requested demand from provider routing and provider-specific input: + +```rust +AuctionSlot { + id, + banner_formats, + floor, + bidder_params, + trusted_provider_routes, +} +``` + +- `bidder_params` is keyed by bidder ID and originates from client or server auction input. +- `trusted_provider_routes` is produced only by trusted server-side opportunity construction or admission normalization of a recognized integration envelope. +- Client input cannot choose provider IDs directly. + +## Routing Model + +### Prebid browser-envelope normalization + +The reserved `trustedServer` browser-adapter entry is an admission envelope, not a bidder ID. Before central routing, request admission unpacks its bounded `bidderParams` object into the canonical slot bidder map: + +```text +trustedServer.bidderParams.rubicon → bidder_params.rubicon +trustedServer.bidderParams.pubmatic → bidder_params.pubmatic +``` + +Each nested key becomes the client-requested bidder ID and is resolved only through `[auction.bidders]`. Nested values become that bidder's parameters. The envelope cannot name provider IDs or endpoints. Its optional `zone` value is preserved as a bounded Prebid slot-matching fact for existing override rules; it does not participate in provider routing. + +A usable `bidderParams` value is a bounded JSON object containing at least one nonempty bidder key whose value is an object accepted by the existing bidder-parameter admission rules. The fallback cases are exact: + +- Missing, `null`, or an empty `bidderParams` object creates stored-request routes to every configured `prebid-server` plan. +- A non-object `bidderParams`, an invalid bidder key or value, or a bounds violation is malformed input and does not trigger stored-request fan-out. +- An object containing a structurally valid but unregistered bidder is usable; it produces `unroutable_bidder` during central routing and does not trigger stored-request fallback. +- A partially malformed object is rejected rather than partially routed. + +A programmatic request may contain both a direct bidder entry and the same bidder inside the envelope. To preserve the existing Prebid merge rule, a usable direct object wins; an unusable direct value cannot overwrite a usable envelope value. Admission applies this rule deterministically before central routing. It never relies on map iteration order. + +When fallback applies, request admission derives a server-controlled stored-request route to each configured `prebid-server` provider plan. The client still does not select those provider IDs. Each routed Prebid plan receives the slot without inline bidder parameters and applies the existing stored-request fallback. This preserves initial and refresh auction behavior that currently uses an empty synthetic `trustedServer` bid. + +### Client-originated demand + +For each bidder requested on a slot: + +1. Look up the bidder in `[auction.bidders]`. +2. Resolve its single provider ID. +3. Add the slot and only that bidder's parameters to the provider's routed view. +4. Record an `unroutable_bidder` outcome when no route exists. +5. Continue the auction for other routable bidders and providers. + +Example client demand: + +```text +Slot: header +Bidders: rubicon, pubmatic, appnexus +``` + +Configured routes: + +```text +rubicon → rubicon-direct +pubmatic → pbs-primary +appnexus → pbs-primary +``` + +Resulting provider inputs: + +```text +rubicon-direct +└── header + └── rubicon parameters + +pbs-primary +└── header + ├── pubmatic parameters + └── appnexus parameters +``` + +### Trusted server-generated demand + +Server-generated opportunities that intentionally rely on stored requests may name trusted provider routes without supplying bidder parameters. + +For migration, existing creative-opportunity construction expresses empty Prebid stored-request intent without a provider ID; the trusted router expands it to every configured `prebid-server` plan, matching the browser-envelope rule. Explicit creative-opportunity bidder parameters continue through `[auction.bidders]`. Creative opportunities no longer hard-code an APS provider instance: an `aps` plan configured as `all_eligible` receives every compatible slot, while an explicitly routed APS plan participates only through a centrally routed bidder or a trusted server-generated route. + +This supports the existing Prebid stored-request and APS paths without allowing the browser to choose an endpoint. + +### Routing modes + +#### `explicit` + +The provider receives only slots routed through: + +- The central bidder registry. +- Trusted server-generated provider routes. + +This is the default. + +#### `all_eligible` + +The provider receives every banner-compatible slot, regardless of bidder routes. + +This mode must be explicitly configured. It is the migration-equivalent routing mode for the current APS provider, which receives every banner-compatible slot. Operators may deliberately choose `explicit` for narrower APS participation. + +### No eligible slots + +When a provider has no eligible slots: + +- No upstream request is sent. +- The provider is recorded as `skipped_no_eligible_slots`. +- This is distinct from no-bid because the provider was not called. + +## Provider Auction Input + +Routing produces one immutable `ProviderAuctionInput` per provider. + +It contains only: + +- Slots admitted by explicit routes or by that provider's `all_eligible` mode. +- Bidder parameters assigned to that provider through the central bidder registry. +- Privacy-approved canonical auction data. +- Provider identity. +- Effective timeout. + +`all_eligible` admits additional banner slots but does not grant access to bidder parameters assigned to another provider. A profile never receives another provider's bidder parameters and therefore does not need provider-specific exclusion logic. + +## OpenRTB 2.6 Driver + +The generic driver owns standard banner OpenRTB behavior and applies the selected profile's compiled standard-field policy. + +### Request responsibilities + +- Request and impression IDs. +- Banner formats. +- Site and publisher data currently supplied by Trusted Server. +- Device and user data currently supplied by Trusted Server. +- Existing consent and EID forwarding behavior. +- Floors and floor currency. +- `tmax` using the effective timeout. +- Current secure-impression requirements. +- Current auction currency assumptions. +- Common Trusted Server signing finalization when enabled, plus the documented PBS host/scheme-only behavior when disabled. + +### Response responsibilities + +- HTTP 204 and ordinary empty responses as no-bid where currently supported. +- Standard OpenRTB response decoding. +- Transport association between the dispatched provider request and its response. For parity, omission or mismatch of the OpenRTB response `id` alone does not reject a PBS or APS response in the first version; profiles may preserve stricter existing behavior where one already exists. +- `seatbid.seat` preservation independently from delivery bidder code. +- Standard bid ID, impression ID, price, dimensions, domains, creative markup, and notification URLs. +- Current banner compatibility checks. +- Existing response-size bounds. +- Existing error and outcome classifications where applicable. + +### Bidder parameters + +Client-supplied bidder parameters are profile input, not generic OpenRTB fields. + +The generic driver does not invent a location for them. + +- The `prebid-server` profile consumes them. +- Another profile may consume them in a provider-specific way. +- The `standard` profile does not forward nonempty bidder parameters by default. +- Unconsumed nonempty parameters produce bounded `unused_bidder_params` diagnostics. + +### Static extensions + +The standard profile may accept validated static objects for: + +- `request.ext` +- `imp.ext` + +Static extensions: + +- Cannot contain secrets. +- Cannot contain templates. +- Cannot read request data. +- Cannot use JSONPath or arbitrary expressions. +- Are bounded by size and nesting depth. +- Cannot overwrite fields reserved by signing, the OpenRTB driver, or another profile responsibility. + +### Ordinary field overrides + +The first version does not support operator-configured arbitrary overrides of fields such as `site.domain`, `device.ip`, `user.id`, or `imp.tagid`. + +A registered profile's fixed typed standard-field policy is not an arbitrary override. Typed operator configuration for additional standard fields should be added only when a concrete endpoint requires it. + +## Prebid Server Profile + +The `prebid-server` profile preserves required Prebid Server behavior while delegating standard fields to the OpenRTB driver. + +### Profile responsibilities + +- Construct `imp.ext.prebid.bidder` from routed bidder parameters. +- Preserve current deterministic bidder-parameter merging and validation semantics where still applicable. +- Support stored-request fallback for trusted server-generated provider routes and admission-generated empty `trustedServer` envelopes. +- Add Prebid-specific request extensions and test/debug fields. +- Preserve Prebid Cache coordinate extraction. +- Preserve Prebid response diagnostics required by current behavior. +- Preserve notification suppression behavior through the common provider outcome model. +- Preserve current request-local data needed to parse responses. + +### Responsibilities moved to common architecture + +- Endpoint and timeout configuration. +- Provider identity. +- Bidder routing. +- Standard OpenRTB request fields. +- Trusted Server signing invocation. +- Standard consent and EID forwarding. +- HTTP transport and backend correlation. +- Standard response parsing and validation. +- Winner selection and mediation. + +### Profile configuration parity + +The central bidder registry is the sole server-side bidder allowlist and route source. The Prebid profile does not define a second `bidders` list. + +The first version must preserve these server-side controls and defaults: + +| Current control | New owner | Default and validation | +| -------------------------- | ---------------------------------------------------------------- | ------------------------------------------------------------------------------------------ | +| `server_url` | Common provider `endpoint` | Required fixed HTTPS endpoint. | +| `timeout_ms` | Common provider `timeout_ms` | Defaults to 1000 ms for `prebid-server`; explicit values override it. | +| `bidders` | Central `[auction.bidders]` registry | Every bidder route is explicit and unique. | +| `debug` | `auction.providers..profile_config.debug` | `false`; preserves current request and response debug behavior. | +| `test_mode` | `auction.providers..profile_config.test_mode` | `false`; preserves the current OpenRTB test flag. | +| `debug_query_params` | `auction.providers..profile_config.debug_query_params` | Absent by default; preserves current page-URL behavior when configured. | +| `bid_param_zone_overrides` | `auction.providers..profile_config.bid_param_zone_overrides` | Empty by default; preserves current typed validation and merge behavior. | +| `bid_param_overrides` | `auction.providers..profile_config.bid_param_overrides` | Empty by default; preserves current typed validation and merge behavior. | +| `bid_param_override_rules` | `auction.providers..profile_config.bid_param_override_rules` | Empty by default; preserves current rule validation, ordering, and shallow-merge behavior. | +| `consent_forwarding` | `auction.providers..profile_config.consent_forwarding` | `both`; preserves the existing `openrtb_only`, `cookies_only`, and `both` behavior. | +| `suppress_nurl` | Common `auction.providers..notifications.suppress_all` | `false`; preserves global `nurl` and `burl` suppression. | +| `suppress_nurl_bidders` | Common `auction.providers..notifications.suppress_seats` | Empty; exact returned seat IDs, validated independently of bidder routes. | + +Stored-request fallback remains built-in Prebid profile behavior rather than another configuration switch. Existing browser-only fields, including bundle configuration, script patterns, client-side bidders, account injection, and excluded GAM ad-unit suffixes, remain under `[integrations.prebid]`. + +Parity tests must cover defaults and non-default values for every field in this table. + +### Browser integration separation + +The Prebid browser integration continues to own: + +- Browser bundle construction. +- JavaScript injection. +- Browser adapter behavior. +- Client-side bidder configuration. +- Script interception and rewriting. +- Browser `timeout_ms` and `debug`, with their current defaults of 1000 ms and `false`, for the injected global Prebid.js configuration. + +Browser `timeout_ms` and `debug` are independent from every server-side provider's common timeout and `profile_config.debug`. No value is selected from multiple provider plans for browser injection. The browser integration does not own the server-side Prebid provider endpoint or bidder route map. + +## APS Profile + +The `aps` profile preserves APS-specific OpenRTB and rendering behavior. + +### Profile responsibilities + +- Add APS account and SDK request extensions. +- Preserve APS inventory identity behavior. +- Interpret the APS response shape and extension fields. +- Preserve APS-specific bid validation. +- Preserve the current highest-price-per-impression candidate reduction and bid-ID tie-breaker, including displaced-bid diagnostics. +- Extract creative URL and tag type. +- Produce the existing typed APS renderer descriptor. +- Preserve script-creative opt-in behavior. +- Preserve APS diagnostics required by current behavior. +- Preserve a valid returned `seatbid.seat` separately while continuing to mark accepted APS bids with delivery bidder code `aps` for the existing browser renderer. + +### Responsibilities moved to common architecture + +- Endpoint and timeout configuration. +- Provider identity. +- Bidder routing or explicit `all_eligible` routing. +- Standard OpenRTB request fields. +- Trusted Server signing invocation. +- Standard consent and EID forwarding. +- HTTP transport and backend correlation. +- Winner selection and mediation. + +### Profile configuration parity + +The first version must preserve these APS controls and defaults: + +| Current control | New owner | Default and validation | +| ------------------------ | -------------------------------------------------------------- | ------------------------------------------------------------------------------------------------------------ | +| `endpoint` | Common provider `endpoint` | Required fixed HTTPS endpoint; legacy unsupported endpoint forms remain rejected. | +| `timeout_ms` | Common provider `timeout_ms` | Defaults to 800 ms for `aps`; explicit values override it. | +| `account_id` | `auction.providers..profile_config.account_id` | Required, nonempty, and subject to the current size and input validation. | +| `debug` | `auction.providers..profile_config.debug` | `false`; preserves current debug behavior. | +| `allow_script_creatives` | `auction.providers..profile_config.allow_script_creatives` | `false`; preserves the existing explicit script opt-in. | +| `inventory_domain` | `auction.providers..profile_config.inventory_domain` | Absent by default; preserves current domain validation. | +| `inventory_page_origin` | `auction.providers..profile_config.inventory_page_origin` | Absent by default; must be configured with `inventory_domain` and preserve current origin/domain validation. | + +Parity tests must cover defaults, `all_eligible` routing as the current-behavior migration path, optional narrower `explicit` routing, debug behavior, inventory override validation, iframe creatives, permitted script creatives, and rejected script creatives. + +Using the APS profile must activate any server-side renderer support it requires independently of browser integration enablement. + +## Request Signing + +Request signing remains controlled by the existing optional global configuration. When global signing is enabled, signing is an auction-wide requirement and every OpenRTB provider request contains the complete version 1.1 Trusted Server signing extension. + +When global signing is disabled, no provider request contains signature-bearing fields (`version`, `signature`, `kid`, or `ts`). To preserve current Prebid wire behavior, the `prebid-server` profile may still emit `ext.trusted_server` containing only its existing `request_host` and `request_scheme` fields. APS and `standard` emit no `ext.trusted_server` object while signing is disabled. Static extension configuration cannot claim this reserved object. + +Applying the complete extension to APS and configuration-defined standard providers is an intentional coverage expansion from the current PBS-only behavior. It is not request-body signing and is not described as wire-parity with the current APS request. + +The project will: + +- Reuse the existing signing implementation and version 1.1 wire contract. +- Avoid cryptographic or protocol redesign. +- Keep signing configuration global rather than repeated under providers. +- Compile only the enabled signing policy into the immutable auction plan; loaded key material is never stored in the plan. +- Load the current signer once during auction admission, before provider routing and dispatch, when global signing is enabled. +- Fail the auction before any provider request is dispatched when the current signer cannot be loaded. +- Reuse that auction-local signer for every provider request in the fan-out. +- Preserve live key rotation by loading the current key for each admitted auction rather than only at process startup. + +Signing version 1.1 authenticates only its existing canonical payload: version, key ID, publisher host, publisher scheme, OpenRTB request ID, and timestamp. It does not authenticate the serialized OpenRTB body, provider ID, endpoint, bidder parameters, or profile extensions. Body or provider binding requires a future signing-protocol version and is outside this specification. + +Profiles augment their owned request fields before common request finalization. The common finalizer then inserts the signing extension and freezes the request ID and signing-owned fields. This ordering prevents profiles or static extensions from overwriting signing fields; it does not imply that version 1.1 authenticates the augmented body. + +Parity and compatibility tests must prove that Prebid Server preserves the existing signing contract and that APS accepts requests containing the extension. The standard-profile compatibility endpoint is a fictional local mock that exists only in automated tests. It implements the documented standard OpenRTB subset, requires no authentication or additional typed fields, and must not become a runtime endpoint, built-in provider, production-support claim, or new profile. + +## Transport and Execution + +The existing platform transport abstractions remain responsible for: + +- Backend registration and naming. +- Asynchronous request dispatch. +- Response correlation. +- Existing request and response bounds. +- Existing timeout behavior. +- Existing platform-specific fan-out capability checks. + +The compiler canonicalizes each provider endpoint once. The endpoint must be an absolute HTTPS URL with a nonempty host and no embedded username, password, or fragment. The same canonical endpoint supplies both the outbound request URI and the platform backend specification. Automatic redirects are not followed; a different destination requires a configuration change and recompilation. Existing TLS certificate and hostname verification remain enabled. Registered profiles may impose stricter endpoint validation, including the APS legacy-endpoint rejection. + +Every provider backend specification uses the validated provider ID as its discriminator. A profile ID is never sufficient for backend discrimination because multiple provider instances may use the same profile, endpoint, and timeout. Target-specific validation must reject provider IDs that collide after any adapter-specific backend-name encoding; lossy normalization may not silently merge them. + +Dispatch state associates the resulting backend identity with exactly one provider ID and compiled profile within an auction. A collision must fail before either mapping can overwrite the other. This first-version contract preserves the existing backend-name correlation mechanism without introducing a new cross-adapter per-request token system. + +At most one outbound request is sent per provider per auction. Exactly one request is sent for each provider with eligible slots, containing every slot admitted by its routing mode. + +The first version does not split one provider's slots across multiple requests and does not send one request per slot. + +The runtime always computes the logical provider budget as: + +```text +min(provider timeout, auction time remaining) +``` + +That exact logical budget controls whether a provider may launch, the OpenRTB `tmax` value, and whether later upstream or mediator network work may launch. It is distinct from a transport timeout used for backend construction and from a hard transport deadline. An adapter may canonicalize or quantize its transport timeout for stable backend identity, but that derived value must not replace the exact logical budget or shorten `tmax`. + +A hard network deadline is enforced only when the target adapter exposes an abortable total-request deadline. On such an adapter, a completion after the enforced deadline is discarded and classified as a provider timeout. + +Fastly currently provides first-byte and between-byte backend timeout controls, not an absolute total-request deadline. Axum has broader task/client cancellation behavior, and Cloudflare and Spin may use eager or broader platform HTTP execution, but no current adapter claims an enforceable provider-wide total-request deadline for this capability. To preserve current behavior, an already-launched call may therefore complete after its logical budget and its completed response remains eligible for local ranking and delivery. No additional provider or mediator network work may launch once the auction has no remaining logical budget, but local decision and delivery still complete. This rule applies equally when split dispatch/collect observes a completed response after the logical deadline. Documentation and tests must state that such an auction can exceed the configured wall-clock budget. + +The plan compiler and target-specific validator use an adapter capability description that distinguishes: + +- Concurrent fan-out support. +- Enforceable total-request transport deadlines. + +Adapters without concurrent fan-out continue to reject configurations with more than one active bidder provider. An adapter without an enforceable outbound deadline may still run one provider, preserving current behavior, but its documentation and tests must identify the transport limitation. + +Provider failures remain isolated from other provider outcomes. + +The automated standard-profile endpoint fixture is unauthenticated and test-only. This specification does not define bearer-token, custom-header, secret-store, or other endpoint-authentication schemas. + +## Response Normalization + +Every provider response is normalized into the existing shared auction response and bid model, or its clean architectural equivalent. + +The normalized result must preserve: + +- Provider ID. +- Valid returned seat, when present. +- Delivery bidder code. +- Slot/impression ID. +- Bid ID. +- Decoded price and existing currency assumptions. +- Banner dimensions. +- Standard creative markup or existing typed renderer. +- Existing notification URL behavior. +- Provider-specific metadata required for current diagnostics. + +Profile-specific response interpretation occurs before bids reach auction ranking or mediation. A registered profile may deterministically reduce its own provider response when required for parity, but it cannot compare bids across providers, apply auction floors, select final winners, or invoke mediation. + +Identity normalization is explicit: + +- A valid string `seatbid.seat` becomes `returned_seat`. +- A missing or non-string seat becomes no returned seat and cannot match `notifications.suppress_seats`. +- Prebid Server uses the valid returned seat as its delivery bidder code and otherwise preserves the current `unknown` fallback. +- APS always uses `aps` as its delivery bidder code, independently of its returned seat. +- Provider ID remains the only backend, health, and transport-correlation identity. + +One provider's malformed response does not fail another provider. Existing behavior for whether an invalid individual bid or full response is dropped should be preserved unless the common driver can enforce an equivalent stricter check without changing externally visible behavior. + +## Decision, Mock Mediation, and Delivery + +This project does not redesign the decision or delivery stages and does not introduce a generic mediator type. The existing `[auction].mediator` reference and statically registered mock mediator remain outside the compiled bidder-provider plan. Provider profiles cannot invoke mediation. + +After normalization, the existing system continues to: + +- Select the highest decoded-price bid per slot when no mediator is configured. +- Apply existing floors. +- Use existing USD assumptions. +- Invoke the existing mediator path when configured. +- Fall back according to existing mediation behavior. +- Sanitize and rewrite creatives according to existing settings. +- Serialize standard creatives and APS renderer descriptors according to existing contracts. + +Provider profiles do not perform cross-provider ranking or choose final auction winners. Provider-local candidate reduction remains part of response normalization where explicitly registered for parity. + +## Telemetry and Diagnostics + +Existing auction telemetry and provider result reporting remain in scope for parity. + +The new architecture must preserve the ability to report: + +- Provider instance ID. +- Provider outcome. +- Response time. +- Bid count. +- Returned seats. +- Existing error classifications. +- Winner status. +- Existing profile-specific diagnostics when enabled. + +The redesign may centralize how diagnostics are carried, but it must not introduce a new telemetry product or schema as part of this work. The existing telemetry seat carrier uses `returned_seat` when present and otherwise falls back to the delivery bidder code to preserve missing-seat behavior. If mock mediation reconstructs a bid, it restores the original provider bid's returned seat rather than deriving it from the mediator or delivery alias. + +New routing outcomes should be distinguishable: + +- `unroutable_bidder` +- `skipped_no_eligible_slots` +- `unused_bidder_params` + +They use a fixed `routing` object in existing metadata maps rather than new response or telemetry fields: + +- Auction-level `OrchestrationResult.metadata["routing"]` carries `unroutable_bidder_count` for internal diagnostics and bounded structured logging. +- A skipped provider produces its ordinary provider result with `AuctionResponse.metadata["routing"].skipped_no_eligible_slots = true`, so the existing `ProviderSummary.metadata` carrier remains usable. +- A called provider that receives but does not consume routed parameter objects records `AuctionResponse.metadata["routing"].unused_bidder_params_count`. + +Only booleans and saturating counts are carried. Bidder parameter values and bidder-ID lists are never included. Existing telemetry may consume these existing metadata carriers, but this work adds no telemetry columns or new client response fields. + +## Runtime Flow + +```text +1. Receive or construct canonical banner auction request. +2. Apply existing consent, identity, and privacy enforcement. +3. Resolve each client bidder through the central bidder registry. +4. Add trusted provider routes for server-generated opportunities. +5. Build one filtered ProviderAuctionInput per provider. +6. Skip providers with no eligible slots. +7. Use the OpenRTB 2.6 driver to construct the standard request. +8. Invoke the selected profile to augment the request. +9. Apply the existing Trusted Server request signature. +10. Dispatch one request per provider through existing transport. +11. Decode the standard OpenRTB response. +12. Invoke the selected profile for provider-specific normalization. +13. Produce normalized provider outcomes. +14. Run existing local ranking or mediation. +15. Run existing creative delivery and telemetry. +``` + +## Failure Behavior + +The system must preserve partial-auction behavior. + +| Condition | Required outcome | +| ----------------------------------------------------------------------------------- | ------------------------------------------------------------------------------------------------------------------ | +| Unknown bidder in runtime request | Record `unroutable_bidder`; continue other demand. | +| Provider has no eligible slots after applying its routing mode and banner filtering | Record `skipped_no_eligible_slots`; do not call provider. | +| Profile configuration invalid | Reject configuration at deploy/startup. | +| Enabled auction signing cannot load the current signer | Fail the auction before any provider request is dispatched. | +| Provider cannot build a valid request | Provider-local launch/build failure. | +| Provider transport fails | Provider-local transport failure. | +| Provider times out | Provider-local timeout. | +| Provider returns valid no-bid | Provider no-bid. | +| Provider response cannot be decoded | Provider-local parse failure. | +| Individual bid is invalid | Preserve current profile/common validation behavior; valid sibling bids remain eligible where currently supported. | +| No providers produce valid bids | Existing auction no-winner behavior. | + +No request with zero eligible impressions should be sent upstream. + +## Security Requirements + +- Provider endpoints are fixed operator configuration, never client-derived. +- Clients cannot select provider IDs or endpoint URLs. +- Each bidder has one server-controlled provider route. +- Provider endpoints are canonical absolute HTTPS URLs with nonempty hosts and no URL credentials or fragments. +- The same canonical endpoint is used for the request URI and backend registration, redirects are not followed automatically, and existing TLS certificate and hostname verification remain enabled. +- Static extensions cannot contain secrets or request templates. +- Profiles cannot bypass central privacy enforcement. +- Profiles cannot access raw browser requests or unrestricted runtime services. +- Provider parameters must not be logged or returned in unbounded diagnostics. +- Existing response-body limits remain enforced. +- Existing creative sanitization remains enforced after winner selection. + +## Acceptance Criteria + +### Configuration and compilation + +- [ ] Provider instances are configured under `[auction.providers.*]`, with the selected profile's typed settings under `profile_config`. +- [ ] The first version recognizes only `openrtb-2.6`. +- [ ] `standard`, `prebid-server`, and `aps` profiles are registered through Rust profile factories. +- [ ] Profile availability is independent of browser integration enablement. +- [ ] Provider and profile configuration is compiled once at startup. +- [ ] Omitted provider timeouts resolve to 1000 ms for `prebid-server`, 800 ms for `aps`, and the auction timeout for `standard`; explicit values override those defaults. +- [ ] Deploy validation and runtime startup use the same two-stage provider compiler, profile registry, adapter capability descriptions, and shared backend-name prediction algorithms; target-agnostic validation reports deferred target checks. +- [ ] Provider IDs enforce the documented lowercase ASCII grammar and length; duplicate IDs, target-encoded backend-name collisions, unknown profiles, invalid endpoints, and invalid bidder routes fail validation. +- [ ] Endpoint tests require canonical absolute HTTPS URLs, reject missing hosts, credentials, and fragments, use the same URL for request and backend construction, and prove redirects are not followed automatically. +- [ ] Every backend specification uses the provider ID rather than the profile ID as its discriminator. +- [ ] Two provider IDs using the same profile, endpoint, and timeout dispatch and correlate independently. +- [ ] Target-specific validation rejects more than one active provider on adapters without concurrent fan-out support. +- [ ] Adapter capabilities distinguish concurrent fan-out from enforceable total-request transport deadlines. +- [ ] Invalid enabled signing configuration fails deploy/startup validation. +- [ ] When signing is enabled, inability to load the current signer fails auction admission before any provider dispatch. + +### Routing + +- [ ] Clients submit bidder identities and parameters without selecting providers. +- [ ] Request admission unfolds the reserved `trustedServer.bidderParams` envelope into canonical bidder IDs before routing. +- [ ] The reserved `trustedServer` envelope cannot name provider IDs or endpoints, and its `zone` value cannot influence routing. +- [ ] Missing, null, and empty recognized `trustedServer.bidderParams` create server-controlled stored-request routes to configured `prebid-server` plans; malformed, unknown-bidder, partial-validity, bounds, and direct/envelope collision cases follow the documented deterministic rules. +- [ ] Initial and refresh auction tests cover mixed PBS, APS, direct server-side, and client-side bidders, proving that only server-side entries are unfolded and routed. +- [ ] `[auction.bidders]` routes each bidder to exactly one provider. +- [ ] Unknown runtime bidders are recorded as `unroutable_bidder` without failing other demand. +- [ ] `explicit` is the default provider routing mode. +- [ ] `all_eligible` is available only through explicit provider configuration and is documented as the current-behavior migration mode for APS. +- [ ] Trusted server-generated slots may route directly to providers without inline bidder parameters. +- [ ] Provider inputs contain only slots admitted by the provider's routing mode and only bidder parameters assigned to that provider. +- [ ] `all_eligible` does not expose bidder parameters assigned to another provider. +- [ ] Providers with no eligible slots are skipped without an upstream request. + +### OpenRTB and profiles + +- [ ] The common OpenRTB driver constructs current standard banner request fields by applying the selected profile's fixed typed field policy. +- [ ] Central privacy enforcement defines the maximum permitted data, and profile policies may only omit from that approved view. +- [ ] The `prebid-server` and `aps` field policies preserve their current request differences, including the complete consent-field matrix. +- [ ] The `standard` profile uses the documented shared PBS and APS baseline, consent placements, and omissions. +- [ ] Prebid transport preserves current `User-Agent`, raw `Referer`, `Accept-Language`, platform-attested `X-Forwarded-For`, and exact selected `Cookie` header behavior without exposing the raw request to profiles. +- [ ] The common driver preserves existing consent, identity, floor, timeout, currency, and signing wire semantics while intentionally expanding enabled signing coverage to every OpenRTB provider. +- [ ] Tests prove that standard, Prebid Server, and APS requests receive the version 1.1 signing extension after profile request augmentation when global signing is enabled. +- [ ] Tests prove that global signing disabled removes all signature-bearing fields, preserves the existing PBS-only host/scheme object, and omits `ext.trusted_server` from APS and `standard`. +- [ ] Signing tests document that version 1.1 does not bind the request body, provider ID, endpoint, bidder parameters, or profile extensions. +- [ ] APS and the automated, fictional standard-profile mock endpoint have signed-request compatibility fixtures; the latter exists only in tests and requires no runtime feature or authentication. +- [ ] Standard OpenRTB endpoints can be configured without adding a provider implementation. +- [ ] Static `request.ext` and `imp.ext` objects are bounded and validated. +- [ ] Client bidder parameters are not assigned an invented generic wire location. +- [ ] The Prebid profile preserves current bidder parameters, stored requests, cache handling, diagnostics, and notification behavior. +- [ ] Common notification suppression preserves `nurl` and `burl` by default, supports provider-wide suppression, and matches per-seat suppression against exact returned seat IDs independently of bidder routes. +- [ ] Prebid parity tests cover the exact existing `openrtb_only`, `cookies_only`, and `both` Cookie-header behavior, including KV/policy-sourced body-consent fallback. +- [ ] The APS profile preserves current account extensions, inventory identity, response validation, renderer, script policy, diagnostics, and deterministic highest-price-per-impression reduction with its bid-ID tie-breaker. +- [ ] Prebid and APS no longer register singleton auction-provider instances. + +### Runtime behavior + +- [ ] At most one outbound request is sent per provider per auction, and none is sent when the provider has no eligible slots. +- [ ] The exact logical provider budget always controls launch eligibility and OpenRTB `tmax`; adapter-specific transport-timeout canonicalization does not replace or shorten it. +- [ ] Hard total-request deadlines discard late completions where supported and are not claimed for any current adapter without an abortable absolute deadline. +- [ ] On adapters without enforceable provider deadlines, completed late responses remain eligible, no new network work launches after logical budget exhaustion, and synchronous plus split execution tests document the possible wall-clock overrun. +- [ ] Existing adapter-specific concurrent fan-out and timeout behavior remains intact. +- [ ] Provider failures remain isolated. +- [ ] Provider ID, returned seat, and delivery bidder code remain distinct through direct and mediated outcomes, browser delivery, and the existing telemetry seat carrier; PBS and APS preserve their documented delivery aliases. +- [ ] Existing winner selection, floors, mock mediation, creative delivery, and telemetry continue to behave as before. +- [ ] Banner behavior has parity with the existing Prebid and APS paths. +- [ ] Non-banner formats are excluded before routing and are never emitted upstream. +- [ ] A slot with no banner formats is skipped. +- [ ] Video and native are not introduced by this work. + +## Deferred Design Areas + +The following require separate requirements before implementation: + +- Additional protocols. +- Video and native support. +- Multiple provider routes for one bidder. +- Provider groups and label/rule-based routing. +- Request splitting for large auctions. +- New money or currency-conversion models. +- New signing protocol versions. +- New privacy or data-sharing controls. +- Consent-cookie forwarding minimization or changes to existing malformed-header behavior. +- New diagnostics and telemetry schemas. +- General endpoint authentication configuration. +- Arbitrary standard-field mappings. +- Runtime or sandboxed profile extensions. +- Generic mediator types, mediator profiles, and configuration-first mediation. +- New abortable outbound-timeout implementations for adapters that do not currently provide them. +- Broader outbound-network policy for IP literals, private or reserved networks, custom ports, and DNS rebinding. + +## Open Questions + +No unresolved product decisions currently block this specification. + +Implementation planning must still identify: + +- The exact Rust profile-factory and compiled-profile interfaces. +- The minimal changes required to separate the current Prebid and APS logic into common OpenRTB and profile-owned behavior. +- The internal organization of the unauthenticated, automated standard-profile mock endpoint and its fixtures. +- The complete parity test fixture set for Prebid, APS, routing, signing, and split dispatch/collect execution. +- The exact configuration representation needed by existing environment override and app-config tooling. + +These are implementation-planning questions and must not expand the product scope defined above. + +## Related Designs + +- [Auction Orchestration Flow](./2026-03-19-auction-orchestration-flow-design.md) +- [Prebid Generic Bid Parameter Override Rules](./2026-04-08-prebid-generic-bid-param-override-rules-design.md) +- [APS OpenRTB First-Class Integration](./2026-07-15-aps-openrtb-first-class-integration-design.md) diff --git a/docs/superpowers/specs/2026-08-18-contiguous-generated-slot-tables-design.md b/docs/superpowers/specs/2026-08-18-contiguous-generated-slot-tables-design.md new file mode 100644 index 000000000..200a8abbe --- /dev/null +++ b/docs/superpowers/specs/2026-08-18-contiguous-generated-slot-tables-design.md @@ -0,0 +1,15 @@ +# Contiguous Generated Slot Tables Design + +## Problem + +`splice_creative_slots` parses rendered slots in a temporary `toml_edit::DocumentMut` and moves its `ArrayOfTables` into the target document. Parsed tables retain document-local numeric positions. Those positions collide with positions in the target document, so serialization can interleave generated slot and provider tables with unrelated top-level tables even though the resulting TOML remains semantically valid. + +## Design + +Before insertion, assign the generated slot tables and all nested provider tables the target `[creative_opportunities]` table's document position. `toml_edit` performs a stable position sort, so equal positions retain traversal order: the creative table, each slot, and that slot's provider tables remain contiguous. For a newly created creative section, allocate an anchor after the greatest existing parsed-table position. + +The update continues to preserve unrelated values, comments, line endings, and semantic table ownership. It does not reformat existing operator-authored content or modify slot inference. + +## Testing + +Add a regression fixture with a late `[creative_opportunities]` section and unrelated tables whose positions overlap those from the temporary generated document. Assert that the parent, generated slots, and provider subtables serialize contiguously before the next unrelated table. Retain the existing semantic-preservation and CRLF tests, then run the CLI test suite, formatting, and native CLI clippy. diff --git a/docs/superpowers/specs/2026-08-18-pr-823-review-resolution-design.md b/docs/superpowers/specs/2026-08-18-pr-823-review-resolution-design.md new file mode 100644 index 000000000..5ff4d0707 --- /dev/null +++ b/docs/superpowers/specs/2026-08-18-pr-823-review-resolution-design.md @@ -0,0 +1,254 @@ +# PR 823 Review Resolution Design + +## Goal + +Resolve the actionable findings in review `4958563121` on PR 823 without +unrelated refactoring, verify the complete branch, publish the fixes, and reply +to every inline review thread with concrete resolution evidence. + +## Scope + +The implementation covers all 28 inline threads and all actionable items in the +review summary. The summary's explicitly out-of-scope pre-existing +partially-invalid `page_patterns` behavior is not expanded into this PR unless a +fix is required by another in-scope change. The PR description's stale legacy +alias sentence is corrected after the branch changes are published. + +Each reviewer suggestion is verified against the current code. A suggestion is +implemented when it is correct for this repository. Where repository evidence +contradicts a suggestion, the implementation retains the correct behavior and +the review response explains the evidence. + +## Design Principles + +- Preserve operator-authored configuration, comments, ordering, and unrelated + sections byte-for-byte wherever possible. +- Never print secrets or whole effective configuration documents as diagnostic + output. +- Never turn uncertain crawl evidence into a runnable fabricated ad-unit path. +- Treat browser navigation as a session, not a sequence of isolated launches. +- Keep `generate`, `verify`, static CLI commands, and runtime matching on shared + domain rules instead of parallel reimplementations. +- Bound all page-controlled data and browser operations. +- Use test-first changes for behavior corrections and minimal annotations for + code-quality-only corrections. + +## Component Design + +### 1. Configuration integrity and command output + +`slot_toml` will replace the line-oriented slot-boundary heuristic with a +TOML-aware edit strategy. The resulting document must preserve every top-level +item outside the managed creative-opportunity fields and preserve comments +adjacent to or between operator sections. Non-contiguous slot declarations, +multiline values, arrays whose continuation lines begin with `[`, trailing +comments, CRLF input, and inline-slot conversion receive regression coverage. +The updater will reject a candidate if preservation cannot be proven. + +Generation will re-read the source config immediately before the atomic write +and refuse to overwrite a concurrently edited file. `--dry-run` will emit only +the managed creative-opportunities change, never the complete config. Notes and +rollback warnings go to stderr so machine-readable stdout remains clean. Tests +will prove that dry-run leaves the source file byte-identical and does not expose +unrelated secret-bearing keys. + +Merge behavior remains add-only for operator-authored data: existing templated +unit paths are retained, newly observed formats are unioned, and multiple +discovered placements absorbed by one broad configured div prefix produce an +operator note. + +### 2. Crawl evidence and inference + +Inference will preserve evidence instead of silently collapsing it: + +- Non-ASCII shared-prefix computation uses UTF-8 byte boundaries. +- Same-page normalization collisions retain distinct raw placements and emit a + diagnostic rather than silently dropping formats. Numeric-only stable tokens + are not classified as hexadecimal hash noise. +- Multi-slot SRA request fallbacks are ignored when `dids` names more than one + slot. +- A page is considered empty only when no audited profile found slots there. +- Fragment detection requires stronger evidence: a useful shared prefix, or at + least three disjoint fragments. Ambiguous two-slot groups are retained with a + note. +- Locale landing paths are emitted literally when they are shorter than the + inferred section depth, and literal path segments are escaped before being + interpolated into globs. +- Refused template decisions are omitted from generated slots and surfaced with + their reasons. The documentation and tests will consistently describe these + cases as refusal, not literal fallback. +- The redundant witness rule is removed or made independently meaningful. The + actual crawler will support the section depth that inference can produce; + locale-prefixed behavior will not exist only in hand-built evidence tests. +- Dropped-section diagnostics are capped, percent-encoded paths are normalized + before filtering, and page-like extensions are classified consistently. + +The root page and section pages for a device profile are collected in one +browser session. Page analysis that parses full HTML is moved off the +current-thread CDP event pump. Each page/tab is closed on every success and +error path. + +### 3. Shared browser behavior + +The browser collectors will share executable discovery and launch/session +configuration. Browser options exposed to operators will have one meaning in +`page`, `verify`, and `generate`: Chrome override, settling, headful/headless +mode, device profile/viewport, proxy, consent assumption, cookies, and TLS +policy. + +`verify` will reuse one browser/runtime/profile across its URLs so clearance and +session state survive. The generic/legacy generator will default to the same +consent assumption as ad-template generation and expose the opt-out rather than +depending on `derive(Default)`. + +Cookie parameters are explicitly host-only with `Path=/`. A same-host +`http`-to-`https` upgrade is accepted with a redirect note; host changes, +downgrades, and unexpected port changes remain cross-origin refusals. Failure to +read or parse the final browser URL fails closed instead of substituting the +requested URL. + +Every post-navigation evaluation is time-bounded. The collector enlarges the +resource timing buffer before navigation, waits for an interactive or complete +document before accruing quiet time, honors sub-poll quiet windows, validates +`quiet <= max`, and reports saturation. Navigation load-event timeout is a +warning after a successful `goto`; it does not discard readable page evidence. +Evidence payload bytes and captured string lengths are capped before expensive +decode/allocation. + +Init-script and page-evaluation failures become explicit warnings or errors +rather than empty evidence. Promise-returning sitemap evaluation awaits its +result. Main-frame-only collection is disclosed when frames are skipped. + +The injected collector will be behavior-preserving: size pairs enforce the +`u32` range, the `googletag` setter is total, the unused non-variadic `cmd.push` +wrapper is removed, wrapping markers are closure-local/non-enumerable, and +page-derived warning text is terminal-safe. + +### 4. Runtime and static-command parity + +Expected-slot projection uses the runtime's renderability rule. Slots the +runtime omits for a path do not count as matched verification slots; diagnostics +state that the runtime omits the slot on that path rather than claiming the +whole config is rejected. + +Configured media type remains a typed `MediaType` through comparison and is +rendered to a string only at the output boundary. Slots that the phase-one +checker cannot confirm (video/native-only) are represented as unconfirmable and +do not fail `--strict`; genuinely partial or missing confirmable slots still +fail, including a live out-of-page slot with no sizes matched against +banner-configured formats, which is partial. Slot phase is absent when no +evidence exists. +The server-side APS compatibility field no longer creates unconditional +client-side `fetchBids` warnings. + +Collector warnings are included in page results. Human output includes the +runtime expectation, gate summary, matched count, extra evidence, and warnings +already present in JSON. Output escaping covers Unicode bidi controls and all +config-derived strings. + +`explain` reports exactly the shared runtime gate result. Provider configuration +is a separate advisory. The unsupported `--edgezero-enabled` model and stale +legacy-fallback claim are removed because no runtime condition backs them. +Gate diagnostics consume the shared gate result instead of rebuilding lists by +hand. The hot runtime gate avoids heap allocation, the seven-boolean wrapper is +removed, and the consent tri-state is documented and exhaustively tested. + +`compile_page_pattern` becomes crate-private and a public validation-only API is +used by the CLI. `lint` explicitly reports every configured page pattern the +runtime would drop, while the broader pre-existing runtime acceptance policy +remains out of scope. Specific compile failures are retained in logs. HTTP +methods use `http::Method` parsing so CLI semantics match the runtime. + +Full URLs and bare path inputs pass through the same URL normalization rules: +percent-encoding, dot-segment resolution, query/fragment removal, and leading +slash behavior must be identical. Scheme detection is anchored to the path +portion before `?`, so an absolute URL inside a query value does not cause a +bare path to be parsed as a full URL. + +### 5. CLI contracts, documentation, and CI + +Clap owns argument validation: URL parsing happens at the value parser, the +audit namespace uses help-on-missing-subcommand, `check` uses an argument group +and conflicts, and settle bounds are rejected during parsing. Parser tests cover +the visible command shapes and legacy restrictions. + +CI-oriented assertion failures exit 1; tool/configuration/navigation failures +exit 2. Assertion text is written directly and cannot disappear behind a log +filter. The guide documents all four `ts config ad-templates` commands, all +flags, shared config-loading flags, browser flags, consent/profile behavior, +dry-run output, and exit codes. + +Browser fixture CI either installs/resolves Chrome and requires the tests to +execute, or explicitly opts into a mode that fails when Chrome is unavailable; +it may not report success after silently skipping every browser assertion. + +All real-looking customer identifiers and names introduced by this PR are +replaced with fictional values in tests, comments, and documentation. Stale +module-level lint suppressions, inaccurate docs, assertion messages, enum +ordering, dead query matching, and orphaned comments are corrected without +unrelated cleanup. + +## Inline Review Traceability + +| Thread | Resolution area | +| -------------------------- | ------------------------------------------------------------------ | +| `3802056460`, `3802056470` | TOML-aware splice and comment/value preservation | +| `3802056474` | Secret-safe dry-run and stderr diagnostics | +| `3802056481` | Omit and explain refused slots | +| `3802056488` | UTF-8-safe div prefix calculation | +| `3802056494` | Same-page normalized-div collisions | +| `3802056497` | Locale landing-page patterns | +| `3802056502` | Multi-profile empty-page accounting | +| `3802056508` | Close every browser tab | +| `3802056513` | Enforce JavaScript-to-Rust `u32` bounds | +| `3802056521`, `3802056529` | Total GPT hook and removal of behavior-changing `cmd.push` wrapper | +| `3802056539` | Shared faithful browser launch configuration | +| `3802056549`, `3802056555` | Correct settling and load-timeout handling | +| `3802056559` | Preserve injected collector warnings | +| `3802056564`, `3802056571` | Runtime renderability parity and accurate diagnostics | +| `3802056580`, `3802056584` | Unconfirmable status and removal of false APS warning | +| `3802056586` | Identical URL and bare-path normalization | +| `3802056593` | Fictional committed examples | +| `3802056599` | Browser fixture CI must execute or fail loudly | +| `3802056605` | Add-only merge of formats with broad-prefix diagnostics | +| `3802056614` | Consent parity for generic and legacy generation | +| `3802056623` | Refusal behavior, tests, and documentation agree | +| `3802056628` | Safe same-host HTTP-to-HTTPS redirect handling | +| `3802056638` | Remove ungrounded EdgeZero fallback model | + +## Error Handling and Compatibility + +All new Rust fallible paths use the repository's existing `CliResult` / +`error-stack` conventions. Browser failures identify the operation and URL but +do not include cookies, configuration values, or page payloads. Best-effort +cleanup must not replace an earlier collection error. + +JSON compatibility is preserved where possible. New distinctions are additive +or correct semantically invalid fields: unconfirmable status is explicit, and +phase may be omitted when there was no evidence. Documentation is updated with +the exact wire behavior. + +## Verification Strategy + +Each behavioral issue follows red-green-refactor: + +1. Add the smallest unit, parser, orchestration, or fixture test reproducing the + review finding. +2. Run the narrow test and confirm the expected failure. +3. Implement the minimal correction. +4. Re-run the narrow test and the affected crate suite. + +Final verification runs the repository-required commands relevant to the +changed surface: CLI tests through `scripts/test-cli.sh`, target-matched Rust +tests, JS tests when the collector script changes, `cargo fmt --all -- --check`, +all target-matched clippy aliases, documentation formatting, and browser fixture +tests with an available Chrome. Any environment-dependent test that cannot run +is reported explicitly and is not described as passing. + +## Review Replies and Publication + +Changes are grouped into reviewable commits by component, then pushed to the PR +branch after final verification. Each inline reply is posted in its existing +thread and states the concrete change, relevant test, or evidence-backed reason +for retaining behavior. Replies avoid generic acknowledgements. Threads are not +replied to as fixed until the corresponding commit is visible on GitHub. diff --git a/docs/superpowers/specs/2026-08-18-pre-navigation-cookie-install-design.md b/docs/superpowers/specs/2026-08-18-pre-navigation-cookie-install-design.md new file mode 100644 index 000000000..e9025174d --- /dev/null +++ b/docs/superpowers/specs/2026-08-18-pre-navigation-cookie-install-design.md @@ -0,0 +1,15 @@ +# Pre-navigation Cookie Installation Design + +## Problem + +The audit collectors open `about:blank` so initialization scripts can be installed before publisher code runs. Cookies are explicitly scoped by domain and `/`, but `chromiumoxide::Page::set_cookie` rejects cookies without a URL while the page is still `about:blank`. Consequently, any audit using `--cookie` fails before navigation; audits without cookies are unaffected. + +## Design + +Build the same host-only, root-scoped `CookieParam` values, then install them through `Browser::set_cookies` before creating the page. Browser-level installation sends the explicit domain/path cookie directly to Chrome without deriving scope from the current page URL. Both verification and generation collectors use one shared helper so their behavior cannot drift. + +Cookie-installation errors remain fatal and identify the affected cookie without logging its value. Page initialization, first-request authentication, browser-session reuse, and cookie scope remain unchanged. + +## Testing + +Add a Chrome-backed regression test that starts with `about:blank`, installs a cookie through the shared browser helper, navigates to a local HTTP fixture, and verifies the cookie is visible on the first loaded document. Run the focused CLI tests, formatting, and lint checks required for the touched crate. diff --git a/docs/superpowers/specs/2026-08-19-ad-template-generation-progress-design.md b/docs/superpowers/specs/2026-08-19-ad-template-generation-progress-design.md new file mode 100644 index 000000000..8f75d0dcf --- /dev/null +++ b/docs/superpowers/specs/2026-08-19-ad-template-generation-progress-design.md @@ -0,0 +1,59 @@ +# Ad-template generation progress design + +## Problem + +`ts audit ad-templates generate` audits up to the configured page budget for +each selected device profile (17 pages by default). Navigation and page settling +are intentionally bounded but can still take tens of seconds per page. The +browser collector buffers page results until the browser session closes, so the +command currently emits no output during most of that work and appears stuck. + +## Design + +Emit line-oriented progress on stderr while collection is running. Progress +must identify the device profile, current page, known total, and safe page +location. It must also identify non-page phases where a noticeable pause can +occur: launching the browser, planning the crawl after the root page, and +finalizing the browser session. + +Progress is an explicit collector callback rather than direct terminal output +inside the browser implementation. This keeps output policy in the command +layer, makes the behavior testable with in-memory writers, and lets non-browser +collectors preserve the same contract. Each line is flushed immediately. + +The first profile's root navigation has no final total because follow-up pages +are planned from the rendered root. It is reported as `1/?`; once planning +finishes, subsequent pages use a stable `current/total` count. Later profiles +receive the complete target list and report the root as `1/total`. Totals include +the root, and every attempted page advances the current count even if collection +fails. + +Progress never prints a full URL. It renders only the origin-free path, omitting +userinfo, query, and fragment data, then applies the CLI's existing terminal-text +sanitizer. An empty path is rendered as `/`. + +Stdout remains reserved for the generated diff or success summary. This keeps +`--dry-run` and shell redirection stable. Progress is intentionally plain text, +not an animated spinner, so it remains useful in logs and does not add a terminal +UI dependency. + +## Error handling + +Failure to write or flush progress is returned as a normal CLI output error. A +callback failure during a browser session stops further collection but does not +skip finalization, browser close, or process wait. An earlier collection or +planning error takes precedence over a later progress error; either takes +precedence over teardown errors. Close and wait are still attempted +independently. No cookie values, URL credentials, query values, fragments, or +browser credentials are included in progress. + +## Tests + +Unit tests will verify that progress is emitted before collection completes, +contains the specified profile-aware page counts, keeps stdout unchanged, +redacts URL credentials/query/fragment data, sanitizes paths, and reports +finalization. Writer tests will cover write failure, flush failure, and explicit +flush invocation. Collector tests will verify teardown still runs after progress +failure and that collection/planning errors, progress errors, and teardown errors +retain the stated precedence. The existing CLI and Chrome-backed suites will +verify the collector behavior and browser lifecycle remain intact. diff --git a/docs/superpowers/specs/2026-08-19-refuse-volatile-div-collisions-design.md b/docs/superpowers/specs/2026-08-19-refuse-volatile-div-collisions-design.md new file mode 100644 index 000000000..bdc9c5b9c --- /dev/null +++ b/docs/superpowers/specs/2026-08-19-refuse-volatile-div-collisions-design.md @@ -0,0 +1,95 @@ +# Refuse Volatile Div-ID Collisions + +## Problem + +GPT discovery normalizes per-render div IDs such as +`ad-in_content--in_content-0` to the stable prefix `ad-in_content`. +When several live elements on the same page normalize to that prefix, the +runtime cannot represent them safely: one prefix resolves at most one element, +while each exact raw ID changes on a later render. The current collision path +preserves the raw IDs, causing `--replace` to write unusable literal slots. + +## Design + +Treat a source-local normalized collision as ambiguous and refuse the entire +group. The first observation remains tentatively accepted. When a second raw div +ID that describes a _different element_ normalizes to the same prefix, remove +the first slot, record the group as ambiguous, and suppress every later member. +Emit one diagnostic when the group first becomes ambiguous, naming the +normalized prefix and explaining that neither a single prefix nor volatile exact +IDs are safe. Tell the operator to expose distinct stable div IDs or prefixes in +publisher markup before configuring the placements. + +Two raw IDs sharing a stem are not by themselves two elements. One element +re-rendered under a fresh framework token produces exactly that shape, and +absorbing it is what normalization is for: a React publisher reports +`ad-header-0-_R_3f_` from the server render and `ad-header-0-_r_0_` from the +client one, and refusing that pair would generate no slots at all. The two cases +are separated by comparing what the ephemeral markers did _not_ cover — the +marker spans are excised and the remaining parts compared, so identical +residues mean one element observed twice, while `-in_content-0` against +`-in_content-1` means two siblings and is refused. + +The verdict is site-wide, not page-local. Article pages carry several in-content +units and refuse the shared prefix while a landing page carries one, so a +page-local refusal would let crawl sampling decide whether the ambiguous prefix +reaches the config. `DiscoveredSlots` therefore carries the refused stems, +`EvidenceTable` unions them across pages, and the slot iterator the writer reads +suppresses them regardless of which page contributed them. + +Registry and request-derived evidence retain separate collision maps, matching +the current source precedence: even an ambiguous registry stem continues to +suppress request fallback for that stem. Network-ID discovery is unaffected. + +`DiscoveredSlots` records whether any otherwise usable GPT slot evidence was +seen independently of how many safe slots remain. `EvidenceTable::fold_page` +uses that signal when classifying empty pages, so a collision-only page is not +mistaken for a bot challenge. Cross-page slot inference, merging, and +`--replace` otherwise remain unchanged because ambiguous slots never enter +those stages. + +Some ad stacks build IDs as `__`, where the +render token — at least ten leading digits followed by more alphanumerics, +that is, a millisecond timestamp plus entropy — sits _before_ the part that +distinguishes one placement from the next. Such an ID can be written neither +literally nor as a prefix: the only stable prefix stops at the token and reaches +every placement in the family at once. Discovery refuses a single otherwise +usable registry or request observation of that shape, preserves the page/network +evidence, and emits one diagnostic naming the family prefix. The shape decides +rather than a vendor name, so any stack with this layout is covered without a +code change, and every placement after the token is covered rather than an +enumerated few. A token in trailing position is _not_ this case — everything +before it still identifies the element — and is left to normalization and the +collision check. + +## Safety and Output + +The generator prefers omission over a configuration that cannot match future +renders. For an observed desktop crawl of a site with this mix, replacement +output should therefore contain the stable `ad-header-0` and `ad-fixed_bottom-0` +slots, while the in-content collision group and the volatile-token family are +explained in notes. + +## Tests + +- A two-element same-page normalization collision yields no slots and one + diagnostic containing the prefix, both unsafe alternatives, and operator + action. +- Two renders of one element (identical residues either side of the marker, + including a React server/client pair) collapse to one slot with no diagnostic. +- Repeats of the first and second IDs plus a third distinct ID after a collision + remain suppressed and do not create additional diagnostics. +- Request-derived collisions follow the same policy. +- An ambiguous registry stem still suppresses request fallback, and network-ID + discovery survives when every collided slot is omitted. +- A stem refused on one page stays refused after a later page contributes a + single member of the group. +- A collision-only page is recorded as having evidence rather than as an empty + challenge page. +- Single registry- and request-derived render-token observations are omitted + while retaining evidence and any parseable network ID, for every placement + suffix after the token. +- IDs with no render token, with a bare digit run, or with a trailing token stay + eligible. +- Existing normalization, request fallback, fragment detection, and full CLI + tests remain green. diff --git a/docs/superpowers/specs/2026-08-21-pr-823-round-5-review-resolution-design.md b/docs/superpowers/specs/2026-08-21-pr-823-round-5-review-resolution-design.md new file mode 100644 index 000000000..8962f4862 --- /dev/null +++ b/docs/superpowers/specs/2026-08-21-pr-823-round-5-review-resolution-design.md @@ -0,0 +1,98 @@ +# PR 823 Round-5 Review Resolution + +## Goal + +Resolve review `4989897698` on PR 823 without weakening the generator's safety +rules, silently changing existing CLI defaults, or expanding the change beyond +the audit CLI and its documentation. + +## Browser and CLI Compatibility + +The hidden `ts audit ` compatibility form keeps accepting the same browser +flags as `ts audit generate `, but those flags must remain hidden and must +require the legacy URL positional. A dedicated `LegacyBrowserOpts` mirrors the +seven generation browser fields and converts into `GenerateBrowserOpts` when the +legacy command is dispatched. Consequently, flags placed before a real audit +subcommand are rejected instead of parsed and ignored. + +Generation retains its established 750 ms quiet period and 12-second maximum +settle wait. Generation defaults have one source of truth shared by clap, +`GenerateBrowserOpts::default`, and `BrowserAuditCollector::default`; applying +parsed options must not silently shorten the collector's maximum. The generic +page/verification collector keeps its existing independent 10-second default. + +Redirect notes show the origin and path for both requested and final URLs. This +makes scheme and host changes visible without exposing URL userinfo, queries, or +fragments. + +## Root-Less Template Safety + +Template inference records which slot stems borrowed the config-level +`section_root` because those slots were never witnessed on a path without the +configured section segment. Such a template is safe only while its page patterns +are derived from the paths where the slot was observed. + +Operator-supplied `--page-pattern` values replace those derived patterns for +every slot. If inference contains any borrowed-root slot and explicit patterns +were supplied, generation fails before rendering or writing a candidate config. +The error identifies the affected slots, explains that explicit patterns cannot +prove the borrowed-root invariant, and directs the operator to remove +`--page-pattern`. Failing the command is preferable to silently omitting real +inventory or attempting an unsound glob intersection. + +When no config-level section policy can be inferred because every otherwise +templatable slot lacks a root witness, each affected slot's refusal reason names +that crawl gap rather than claiming that its paths failed to generalize. + +## Merge Policy + +An explicitly configured `section_segment` is operator intent even when +`section_root` is currently unset. If preserved `{section}` slots exist and an +inferred policy would change that configured segment, merge fails and requires +`--replace` for the migration. If the configured segment matches, or is unset, +the inferred `section_root` may be adopted so the previously incomplete config +becomes loadable. + +## Diagnostics and Early Validation + +Warnings produced while folding a collected page include the device-profile +label as well as the path. Identical warnings from desktop and mobile therefore +remain distinguishable. The consent-stub warning remains a single unscoped +run-level note, and site-wide discovery warnings remain deduplicated. + +The existing config is parsed as TOML before Chrome starts. A whole-document +syntax error is returned immediately; a valid document with settings unknown to +the CLI still permits extraction of `[creative_opportunities]`; and a present +but unreadable creative section remains an error. + +The volatile div-id token recognizer requires at least ten leading digits plus +an alphanumeric suffix. This continues to recognize timestamp-like generated +tokens while preventing an eight-digit calendar date followed by a stable +letter from causing a single-observation family refusal. + +## Consistency Corrections + +Tests pin the Rust evidence cap to the embedded JavaScript collector constant. +The terminal-escaping test claims only controls it can actually inject; URL's +own percent-encoding is covered by an exact final-URL assertion rather than +presented as evidence for terminal escaping. Existing code escaping the final +URL remains as defense in depth. + +The affected guide, prior volatile-collision spec and plan, documentation +comments, `expect` message, and method spacing are corrected to describe the +implemented behavior exactly. The root-less templating behavior and this review +resolution are documented by this design and its paired implementation plan. + +## Testing and Delivery + +Every behavioral correction starts with a focused regression test that fails on +the current branch. Tests cover hidden legacy flags, the 12-second generation +default, complete redirect notes, borrowed-root rejection with explicit +patterns, configured-segment preservation, profile-specific warnings, +whole-document TOML failure, the evidence-cap invariant, and the calendar-date +token control. + +After focused tests pass, verification runs the host-target CLI suite and +audit/generate tests, CLI clippy with warnings denied, Rust formatting, docs +formatting, and `git diff --check`. No GitHub replies or push are part of this +change unless separately requested. diff --git a/docs/superpowers/specs/2026-08-24-ad-template-div-id-reconciliation-design.md b/docs/superpowers/specs/2026-08-24-ad-template-div-id-reconciliation-design.md new file mode 100644 index 000000000..9290e6e5c --- /dev/null +++ b/docs/superpowers/specs/2026-08-24-ad-template-div-id-reconciliation-design.md @@ -0,0 +1,118 @@ +# Ad-template div-ID reconciliation design + +## Goal + +Prevent `ts audit ad-templates generate` from losing numeric sibling creative +opportunities during merge or persisting a singleton div ID whose middle token +is demonstrably per-render. + +This follows a live validation crawl. The crawl observed +`ad-sidebar-1`, `ad-sidebar-10`, and other siblings, but the merge treated the +configured literal `ad-sidebar-1` as a prefix and absorbed the longer IDs. It +also proposed one `vendor-tag_12345678AbCdEfGhIjKl_slot_overlay_1`-shaped slot +because the volatile-token classifier recognizes ten leading digits but this +token has eight. + +## Scope + +The change is limited to div-ID identity and volatility classification during +generation: + +- Preserve every distinct normalized, usable div identity retained by the + evidence table when an existing configured div ID was itself observed + exactly. +- Preserve intentional configured prefix behavior when that prefix was not + observed as a literal element ID. +- Refuse singleton IDs with a conservative eight-digit-plus-long-suffix token + shape in a non-trailing segment. +- Keep the existing warning and refusal behavior for ambiguous and fragmented + placements. + +This does not implement the broader cross-page-type preservation requested by +GitHub issue #1059, change crawl planning, or change runtime slot resolution. + +## Exact versus prefix reconciliation + +The generator already carries the div identities from `EvidenceTable::slots()` +into the TOML merge. This is intentionally not collector-level raw DOM input: +the identities have passed per-page normalization and usability checks, while +slots later rejected by template inference or cross-page fragmentation remain +present. Page-local volatile and ambiguous identities already refused by GPT +discovery do not re-enter reconciliation. + +The merge will classify a configured or newly appended slot as an observed +literal when its resolved div identity appears exactly in that normalized +evidence set. + +Matching proceeds in this order: + +1. Prefer an exact stable-key match. +2. Otherwise consider configured-prefix matches whose prefix was not observed + as a literal normalized div identity during this crawl. +3. Choose the longest remaining prefix, retaining configuration order for + equal-length ties. +4. Append the discovered slot when neither exact nor eligible prefix matching + succeeds. + +Consequently, `ad-sidebar-1` matches itself but cannot claim +`ad-sidebar-10`. A hand-authored broad prefix such as `ad-`, absent as a literal +DOM ID, retains its existing merge behavior. Newly appended discovered slots +are also protected because the decision is based on the normalized evidence +set, not only the original configuration indexes. + +The same reconciliation rules will drive observed/unobserved diagnostics so a +slot cannot be merged one way and classified for staleness another way. + +## Volatile token classification + +The existing vendor-neutral classifier refuses a div ID when a non-trailing +segment contains a per-render token before the placement suffix. It currently +recognizes a segment with at least ten leading digits followed by alphanumerics. + +Retain that rule and add a narrower alternative for shorter counters: + +- at least eight leading ASCII digits; and +- at least eight trailing ASCII alphanumeric characters in the same segment. + +The token must still occur before another div-ID segment. This catches the +`12345678AbCdEfGhIjKl` shape without claiming: + +- bare numeric placement IDs; +- seven-digit counters with long suffixes; +- eight-digit values with fewer than eight trailing characters, including + calendar-like `20260820a`; or +- trailing tokens whose preceding prefix can still identify the element. + +The warning remains vendor-neutral and names the stable family prefix. The slot +continues to count as evidence of an ad stack but is not rendered into config. + +## Diagnostics and failure behavior + +No new command failure is introduced. Unsafe singleton volatile slots are +skipped with the existing volatile-family note. Literal numeric siblings are +written separately and no longer produce the broad-prefix collision note. +Truly intentional broad prefixes can still produce that note when they claim +multiple observed divs. + +Normal merge continues to preserve configured slots. `--replace` retains its +existing replacement semantics. + +## Testing + +Use test-driven development with focused regressions: + +- A merge containing configured `ad-sidebar-1` and normalized observations for + `ad-sidebar-1`, `ad-sidebar-10`, and `ad-sidebar-11` must produce three slots. +- A configured `ad-` prefix that was not observed literally must continue to + merge multiple matching discovered divs and emit its collision note. +- Newly appended observed literals must not absorb later numeric siblings. +- A framework-bearing DOM ID normalized to a stable stem must classify the + matching configured stem as literal; identities refused during per-page GPT + discovery must not be reintroduced solely for merge classification. +- Registry and request evidence containing a singleton shorter high-entropy token + must be refused with the volatile-family warning. +- Boundary tests cover seven leading digits, eight digits with a seven-character + suffix, eight digits with an eight-character suffix, bare digits, and the + existing calendar-shaped example. +- Run the complete CLI suite, including the real-Chrome scrolling fixture, plus + formatting and the repository's target-specific verification gates. diff --git a/docs/superpowers/specs/2026-08-24-ad-template-generate-scroll-staleness-design.md b/docs/superpowers/specs/2026-08-24-ad-template-generate-scroll-staleness-design.md new file mode 100644 index 000000000..8709f6330 --- /dev/null +++ b/docs/superpowers/specs/2026-08-24-ad-template-generate-scroll-staleness-design.md @@ -0,0 +1,98 @@ +# Ad-template generation scroll and staleness diagnostics design + +## Problem + +`ts audit ad-templates generate` currently collects each page only after its +initial settle. Unlike `ts audit page` and `ts audit ad-templates verify`, it +cannot request the deterministic scroll pass that triggers lazy ad inventory. +On a lazy-loading publisher site this produced fewer observable frames than a +scrolled page audit of the same page. + +Generation also merges by default, deliberately preserving configured slots +that the current crawl did not rediscover. That safety behavior is correct, but +it is silent: stale slots look as though the latest crawl confirmed them. + +## Scope + +Add opt-in scrolling to `ts audit ad-templates generate` and report configured +slots that a merge preserved without observing during the current crawl. + +This change does not prune slots automatically, enable scrolling by default, +alter crawl planning or budgets, change volatile-div refusal, or implement +GitHub issue #1059. `--replace` remains the only intentional pruning mode. + +## Command behavior + +`ts audit ad-templates generate` accepts a boolean `--scroll` option. Its +default is false, preserving current crawl cost and side effects. When enabled, +every page on every selected device profile performs the same deterministic +stepped scroll used by the existing page audit: scroll to 33%, 66%, and 100% of +the document, pause between steps, return to the top, then wait for the page to +settle again before reading HTML, GPT registry entries, and network evidence. + +The browser collector carries the option as session configuration so root, +planned section, desktop, and mobile page loads all behave consistently. Scroll +evaluation failures are best-effort page warnings; they do not discard evidence +that was already available after the initial settle. + +The implementation will share the deterministic scroll primitive with the +existing browser audit rather than maintain a second sequence of scroll steps. +Verifier-only evidence-phase bookkeeping remains in the verifier call path. + +## Merge diagnostics + +During a normal merge, generation tracks which pre-existing configured slots +matched at least one discovered slot. After processing all discovered slots, it +reports every unmatched pre-existing slot in configuration order. Those slots +remain unchanged in the output. + +The diagnostic is explicit about the limits of negative crawl evidence. Its +human-readable form for a non-scrolling run is equivalent to: + +```text +note: preserved 2 configured slot(s) not observed during this crawl: ad-header-0, ad-fixed_bottom-0. Re-run with broader coverage or --scroll; `--replace` prunes them but also discards every hand-written field on the slots the run did rediscover. +``` + +When the current run already used `--scroll`, the follow-up omits that redundant +suggestion and recommends broader page/profile coverage before intentional +pruning. + +No staleness diagnostic is emitted when all configured slots were rediscovered, +when there were no existing slots, or under `--replace`, because that mode does +not preserve unmatched slots. Matching uses the same reconciliation logic as +the merge itself, avoiding a second definition of slot identity. + +Diagnostics go to stderr through the existing generation-note path. Stdout +remains limited to the dry-run diff or successful write summary, so redirection +and machine comparison remain stable. + +## Safety and compatibility + +The default command behavior, merge result, and generated TOML remain unchanged +unless `--scroll` discovers additional evidence. The warning never mutates or +deletes operator configuration. It names only configured slot IDs and does not +include cookies, URL credentials, query strings, or fragments. + +Scrolling can trigger additional ad requests and publisher behavior, which is +why it remains explicit. Existing page-delay, settle-window, browser-proxy, +certificate, cookie, and device-profile behavior applies unchanged. + +## Tests + +CLI parsing tests cover `--scroll` and its false default. Browser-collector tests +use a deterministic local page that defines a GPT slot only after scrolling and +prove that generation captures it with the option enabled but not without it. +Existing browser lifecycle and settle tests continue to cover teardown and +timeouts. + +Merge unit tests cover multiple unmatched configured slots, stable diagnostic +ordering, partial rediscovery, full rediscovery, an empty existing config, and +`--replace`. Command-level tests verify that the warning reaches stderr while +stdout and the preserved generated configuration retain their existing +contracts. + +Verification will run the host CLI test suite and relevant Chrome-backed CLI +tests, followed by the repository-required formatting and CLI lint gates. A +manual dry run against a live publisher site may be used when a fresh +bot-protection cookie and proxy are available, but network-dependent behavior +is not a required CI test. diff --git a/docs/superpowers/specs/2026-08-27-pr-1079-review-remediation-design.md b/docs/superpowers/specs/2026-08-27-pr-1079-review-remediation-design.md new file mode 100644 index 000000000..f3603e767 --- /dev/null +++ b/docs/superpowers/specs/2026-08-27-pr-1079-review-remediation-design.md @@ -0,0 +1,80 @@ +# PR 1079 Review Remediation Design + +## Goal + +Make the first-impression ownership and APS creative bridge safe under overlapping +publisher auctions, late callbacks, SPA navigation, mixed GPT refresh lists, and +nested 1x1 GAM shells. Preserve PR 1079's first-claimant policy: Trusted Server may +win an untouched physical slot, but must neither overwrite a publisher impression +nor let a stale response affect a later navigation. + +## Ownership model + +First-impression state remains keyed by navigation generation and exact physical +element identity. Each publisher auction gets an independent token whose +suppression decision is fixed when the auction is registered. When Trusted Server +commits its request, registration closes for new losing publisher auctions, while +already-registered losing tokens remain suppressible. Those tokens remain as +tombstones for the lifetime of the same navigation and exact physical element. +Unresolved suppressing tombstones are never evicted or removed by timeout or +auction failure; only navigation change or physical element replacement removes +them. The existing per-slot registration limit bounds the set before registration +closes, so an arbitrarily late correlated callback cannot become unrelated. + +Prebid's pending bid/code correlation records carry the navigation generation and +physical element identity captured at registration. A record is usable only while +both still match, and consuming one exact ad-ID delivery removes only its auction's +registration. A code-only delivery consumes a record only when exactly one current +registration matches. Ambiguous ordinary code-only deliveries run an independent +auction rather than guessing; ambiguous TS-owned suppressing deliveries fail closed +without deleting their tombstones. Scoped `requestBids({ adUnitCodes })` calls +inspect, mutate, claim, and correlate only those requested global ad units. + +## Refresh suppression + +The Prebid delivery wrapper is the owner of first-impression delivery suppression. +When it suppresses a GPT slot, it also consumes any equivalent late-handoff +one-shot flag so the inner GPT wrapper cannot suppress the next legitimate +refresh. When it delegates a permitted GPT request, it consumes that flag at the +delegation boundary so the inner wrapper cannot silently drop the request. Mixed +refresh calls always forward the already-filtered slot list, including the path +where every remaining slot is excluded from a Prebid auction. That all-excluded +path performs the same ownership registration and consumption synchronously +before delegating. A bare refresh delayed by an auction becomes an explicit list +at callback time, preventing slots added after the snapshot from joining it. + +A publisher-triggered GPT refresh that starts a synthetic Prebid auction registers +its own per-slot first-impression tokens before waiting for the asynchronous +callback. A publisher-first token reserves the slot so TS cannot claim it while +the auction is pending. A token registered against an earlier TS claim is consumed +at callback time, filtering that slot from the eventual GPT request. When TS emits +its first GPT request, registration closes for new losing publisher tokens so +ordinary later publisher refreshes continue normally. Mixed callbacks forward +only their unsuppressed slots and scope Prebid targeting to the same filtered set. +The callback also revalidates the captured navigation generation and exact +physical element, dropping stale work rather than refreshing a replacement slot. + +## Creative bridge + +Every asynchronous renderer/cache result is revalidated before posting a creative +response or recording successful response/billing evidence. A stale result may be +recorded as safe failure telemetry, but is never recorded as a response or win. +Validation covers navigation generation, winning bid identity, authenticated +source iframe identity, DOM connectivity, and containment in the authenticated +slot root. When a configured prefix matches several roots, the requesting frame +may disambiguate them only when exactly one candidate root owns that source. + +After a valid response is posted, a collapsed 1x1 source iframe is expanded to the +winning creative size. The bridge walks all collapsed ancestors through the +authenticated slot root and expands each clipping shell. It refuses all resizing +for fixed/sticky, anchor, vignette, interstitial, detached, oversized, or +otherwise unauthenticated shells. + +## Verification + +Regression tests cover all seven review findings, including wrapper composition, +scoped ad-unit requests, mixed excluded refreshes, stale SPA callbacks, +overlapping auctions, stale cache responses with no successful response/billing +evidence, and two nested +collapsed ancestors. Existing JS unit/browser suites, formatting, lint, build, +and repository Rust verification remain the completion gates. diff --git a/fastly.toml b/fastly.toml index 56002bc5a..ca8bce8d3 100644 --- a/fastly.toml +++ b/fastly.toml @@ -57,11 +57,18 @@ build = """ key = "tinybird_auction_append_token" data = "test-tinybird-auction-append-token" + # App-config references use logical `trusted_server_secrets`; the + # edgezero_runtime_env mapping below resolves it to physical `ts_secrets`. [[local_server.secret_stores.ts_secrets]] - key = "tinybird_access_append_token" - data = "test-tinybird-access-append-token" + key = "placeholder" + data = "placeholder" [local_server.config_stores] + [local_server.config_stores.edgezero_runtime_env] + format = "inline-toml" + [local_server.config_stores.edgezero_runtime_env.contents] + EDGEZERO__STORES__SECRETS__TRUSTED_SERVER_SECRETS__NAME = "ts_secrets" + [local_server.config_stores.trusted_server_config] format = "inline-toml" [local_server.config_stores.trusted_server_config.contents] diff --git a/scripts/template-cache-local-test.sh b/scripts/template-cache-local-test.sh index cc7e9eb87..01da8ef04 100755 --- a/scripts/template-cache-local-test.sh +++ b/scripts/template-cache-local-test.sh @@ -24,6 +24,7 @@ esac REPO_ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" WORK="$(mktemp -d)" ORIGIN_PORT="${ORIGIN_PORT:-9099}" +BID_PORT="${BID_PORT:-9100}" TS_PORT="${TS_PORT:-7788}" HOST_TRIPLE="$(rustc -vV | sed -n 's/^host: //p')" @@ -31,6 +32,7 @@ HOST_TRIPLE="$(rustc -vV | sed -n 's/^host: //p')" # observable in the timings: with an instant auction, buffered and streaming # assembly are indistinguishable. BID_DELAY="${BID_DELAY:-1.5}" +REQUEST_TIMEOUT_SECONDS="${REQUEST_TIMEOUT_SECONDS:-30}" PASS=0 FAIL=0 @@ -45,8 +47,16 @@ check() { # check cleanup() { local status=$? - [ -n "${VICEROY_PID:-}" ] && kill "$VICEROY_PID" 2>/dev/null || true - [ -n "${ORIGIN_PID:-}" ] && kill "$ORIGIN_PID" 2>/dev/null || true + if [ -n "${VICEROY_PID:-}" ]; then + kill "$VICEROY_PID" 2>/dev/null || true + wait "$VICEROY_PID" 2>/dev/null || true + fi + if [ -n "${ORIGIN_PID:-}" ]; then + # macOS may launch the framework Python process as a child of the shim. + pkill -TERM -P "$ORIGIN_PID" 2>/dev/null || true + kill "$ORIGIN_PID" 2>/dev/null || true + wait "$ORIGIN_PID" 2>/dev/null || true + fi rm -rf "$WORK" exit $status } @@ -60,13 +70,17 @@ command -v node >/dev/null || { echo "node not found. The harness executes the real GPT bundle to verify slot setup." >&2 exit 1 } +command -v openssl >/dev/null || { + echo "openssl not found. The harness needs it for the local HTTPS bid endpoint." >&2 + exit 1 +} # A port already in use means requests would go to something else entirely — most # likely a leftover run, whose warm cache and stale config would read as a result. -for port in "$ORIGIN_PORT" "$TS_PORT"; do +for port in "$ORIGIN_PORT" "$BID_PORT" "$TS_PORT"; do if lsof -nP -iTCP:"$port" -sTCP:LISTEN >/dev/null 2>&1; then echo "Port $port is already in use. Stop the process, or set" >&2 - echo "ORIGIN_PORT / TS_PORT to something free." >&2 + echo "ORIGIN_PORT / BID_PORT / TS_PORT to something free." >&2 lsof -nP -iTCP:"$port" -sTCP:LISTEN >&2 exit 1 fi @@ -78,19 +92,35 @@ cargo build -p trusted-server-cli --target "$HOST_TRIPLE" >/dev/null WASM="$REPO_ROOT/target/wasm32-wasip1/debug/trusted-server-adapter-fastly.wasm" TS="$REPO_ROOT/target/$HOST_TRIPLE/debug/ts" -info "Starting stub origin on :$ORIGIN_PORT" +info "Generating a local CA and HTTPS bid certificate" +openssl req -x509 -newkey rsa:2048 -sha256 -days 1 -nodes \ + -subj "/CN=Trusted Server local harness CA" \ + -keyout "$WORK/ca-key.pem" -out "$WORK/ca-cert.pem" >/dev/null 2>&1 +openssl req -newkey rsa:2048 -sha256 -nodes -subj "/CN=localhost" \ + -keyout "$WORK/server-key.pem" -out "$WORK/server.csr" >/dev/null 2>&1 +cat > "$WORK/server.ext" <<'EOF' +basicConstraints=CA:FALSE +keyUsage=digitalSignature,keyEncipherment +extendedKeyUsage=serverAuth +subjectAltName=DNS:localhost,IP:127.0.0.1 +EOF +openssl x509 -req -sha256 -days 1 -in "$WORK/server.csr" \ + -CA "$WORK/ca-cert.pem" -CAkey "$WORK/ca-key.pem" -CAcreateserial \ + -extfile "$WORK/server.ext" -out "$WORK/server-cert.pem" >/dev/null 2>&1 + +info "Starting stub origin on :$ORIGIN_PORT and HTTPS bidder on :$BID_PORT" cat > "$WORK/origin.py" < "$WORK/origin.log" 2>&1 & ORIGIN_PID=$! sleep 1 info "Generating stub config (mode: $MODE)" -python3 - "$REPO_ROOT/trusted-server.example.toml" "$WORK/app.toml" "$MODE" "$ORIGIN_PORT" <<'PYEOF' -import sys, re -src, out, mode, port = sys.argv[1:5] +python3 - "$REPO_ROOT/trusted-server.example.toml" "$WORK/app.toml" "$MODE" \ + "$ORIGIN_PORT" "$BID_PORT" <<'PYEOF' +import sys + +src, out, mode, origin_port, bid_port = sys.argv[1:6] s = open(src).read() -s = s.replace('origin_url = "https://origin.example.com"', f'origin_url = "http://127.0.0.1:{port}"', 1) + +def replace_once(content, old, new, description): + if content.count(old) != 1: + raise SystemExit(f"expected one {description} replacement target") + return content.replace(old, new, 1) + + +s = replace_once( + s, + 'origin_url = "https://origin.example.com"', + f'origin_url = "http://127.0.0.1:{origin_port}"', + "publisher origin", +) # The example config ships placeholders that validation rejects outright, # including the reserved publisher domain/cookie_domain. -s = s.replace('domain = "example.com"', 'domain = "local-harness.example"', 1) -s = s.replace('cookie_domain = ".example.com"', 'cookie_domain = ".local-harness.example"', 1) -s = s.replace('password = "replace-with-admin-password-32-bytes"', - 'password = "local-harness-admin-password-not-a-real-one"', 1) -s = s.replace('proxy_secret = "change-me-proxy-secret"', - 'proxy_secret = "local-harness-proxy-secret-not-a-real-one"', 1) -s = re.sub(r'passphrase = "[^"]*"', - 'passphrase = "local-harness-ec-passphrase-not-a-real-one"', s, count=1) - -# A real auction, pointed at the stub's slow endpoint, so the timings mean something. -s = s.replace('[integrations.prebid]\nenabled = false\nserver_url = "https://prebid.example.com/openrtb2/auction"', - f'[integrations.prebid]\nenabled = true\nserver_url = "http://127.0.0.1:{port}/bid"\n' - 'external_bundle_url = "https://assets.example.com/prebid/trusted-prebid-stub.js"', 1) -s = s.replace('providers = []', 'providers = ["prebid"]', 1) -s = s.replace('\n[proxy]\n', '\n[proxy]\nallowed_domains = ["assets.example.com", "127.0.0.1"]\n', 1) -s = s.replace('[auction]\nenabled = false', '[auction]\nenabled = true', 1) -s = s.replace('auction_timeout_ms = 500', 'auction_timeout_ms = 3000', 1) -s = s.replace('timeout_ms = 2000', 'timeout_ms = 3000', 1) +s = replace_once( + s, + 'domain = "example.com"', + 'domain = "local-harness.example"', + "publisher domain", +) +s = replace_once( + s, + 'cookie_domain = ".example.com"', + 'cookie_domain = ".local-harness.example"', + "publisher cookie domain", +) +# A real auction points at the slow HTTPS stub so the timings mean something. +s = replace_once( + s, + '[integrations.prebid]\nenabled = false', + '[integrations.prebid]\nenabled = true\n' + 'external_bundle_url = "https://assets.example.com/prebid/trusted-prebid-stub.js"', + "Prebid integration", +) +s = replace_once( + s, + 'endpoint = "https://prebid.example.com/openrtb2/auction"', + f'endpoint = "https://localhost:{bid_port}/bid"\ntimeout_ms = 5000', + "Prebid provider endpoint", +) +s = replace_once( + s, + '\n[proxy]\n', + '\n[proxy]\nallowed_domains = ["assets.example.com", "127.0.0.1"]\n', + "proxy table", +) +s = replace_once( + s, + '[auction]\n# Keep disabled until provider endpoints, routes, and profile values below are\n' + '# replaced with deployment-specific settings.\nenabled = false', + '[auction]\n# Keep disabled until provider endpoints, routes, and profile values below are\n' + '# replaced with deployment-specific settings.\nenabled = true', + "auction enablement", +) +s = replace_once( + s, + 'sanitize_creatives = false\ntimeout_ms = 2000', + 'sanitize_creatives = false\ntimeout_ms = 10000', + "auction timeout", +) +s = replace_once( + s, + 'auction_timeout_ms = 500', + 'auction_timeout_ms = 10000', + "creative opportunity auction timeout", +) # The template-cache keys go directly under the table header. The slot is a table of its own # and must go at the end: inserted here it would swallow every scalar key that @@ -230,6 +322,70 @@ info "Seeding an isolated config store (tracked fastly.toml remains untouched)" # pointed at this checkout without copying the workspace. cp "$REPO_ROOT/edgezero.toml" "$WORK/edgezero.toml" cp "$REPO_ROOT/fastly.toml" "$WORK/fastly.toml" + +# The application registers provider backends dynamically. Pre-register the exact +# deterministic name so Viceroy reuses a local backend that trusts the temporary CA. +python3 - "$WORK/fastly.toml" "$WORK/ca-cert.pem" "$BID_PORT" <<'PYEOF' +import hashlib +import json +import sys + +manifest, ca_certificate, port = sys.argv[1:4] +provider_id = "pbs-main" +timeout_ms = "5000" + + +def field(value): + return f"{len(value)}:{value}" + + +canonical = "".join([ + field("https"), + field("localhost"), + field(port), + field("1"), + "n", + "s", + field(provider_id), + field(timeout_ms), + field(timeout_ms), +]) +digest = hashlib.sha256(canonical.encode()).hexdigest()[:32] +readable = f"https_localhost_{port}_p_{provider_id}_fb{timeout_ms}_bb{timeout_ms}" +backend_name = f"backend_{readable}_{digest}" +backend = f'''[local_server.backends.{backend_name}] +url = "https://localhost:{port}" +cert_host = "localhost" +ca_certificate.file = {json.dumps(ca_certificate)} +''' + +content = open(manifest).read() +marker = "[local_server.backends]\n\n" +if content.count(marker) != 1: + raise SystemExit("expected one local backend insertion target") +content = content.replace(marker, f"{marker}{backend}\n", 1) +open(manifest, "w").write(content) +PYEOF + +python3 - "$WORK/fastly.toml" <<'PYEOF' +import sys + +with open(sys.argv[1], "a") as manifest: + manifest.write(''' +[[local_server.secret_stores.ts_secrets]] +key = "publisher_proxy_secret" +data = "fictional-local-publisher-proxy-secret-value" + +[[local_server.secret_stores.ts_secrets]] +key = "ec_passphrase" +data = "fictional-local-ec-passphrase-secret-value" + +[[local_server.secret_stores.ts_secrets]] +key = "handler_password" +data = "fictional-local-handler-password-secret-value" +''') +PYEOF + ln -s "$REPO_ROOT/crates" "$WORK/crates" (cd "$WORK" && "$TS" config push --adapter fastly --local \ --manifest "$WORK/edgezero.toml" --app-config "$WORK/app.toml" \ @@ -256,7 +412,7 @@ fi req() { # req [extra curl args...] local out="$1"; shift - curl -sS -D "$out.headers" -o "$out" \ + curl -sS --max-time "$REQUEST_TIMEOUT_SECONDS" -D "$out.headers" -o "$out" \ -w '%{time_starttransfer} %{time_total} %{http_code}' \ -H "Host: ts.example.com" \ -H "Accept-Encoding: gzip" \ @@ -326,18 +482,20 @@ assembly_state() { # Shared by the ESI assertions below. check_hit_is_private() { local hdrs - hdrs=$(curl -s -D- -o /dev/null -H "Host: ts.example.com" \ + hdrs=$(curl -sS --max-time "$REQUEST_TIMEOUT_SECONDS" -D- -o /dev/null \ + -H "Host: ts.example.com" \ -H "Accept-Encoding: gzip" \ -H "sec-fetch-dest: document" -H "sec-fetch-mode: navigate" \ "http://127.0.0.1:$TS_PORT/article") check "cache hit is not shared-cacheable" \ - "$(echo "$hdrs" | grep -ci 'cache-control: no-store, private' || true)" "1" + "$(echo "$hdrs" | grep -ci 'cache-control: private, no-store' || true)" "1" } check_post_reaches_origin() { local before before=$(grep -cF "origin: received POST /article" "$WORK/origin.log" || true) - curl -s -o /dev/null -X POST -d 'x=1' -H "Host: ts.example.com" \ + curl -sS --max-time "$REQUEST_TIMEOUT_SECONDS" -o /dev/null \ + -X POST -d 'x=1' -H "Host: ts.example.com" \ -H "Accept-Encoding: gzip" \ "http://127.0.0.1:$TS_PORT/article" check "a POST still reaches the origin" \ @@ -523,16 +681,17 @@ first chunk looks identical to one that does not. import socket, sys, time host, port, path = sys.argv[1], int(sys.argv[2]), sys.argv[3] -extra = sys.argv[4] if len(sys.argv) > 4 else "" +timeout = float(sys.argv[4]) req = ( f"GET {path} HTTP/1.1\r\nHost: ts.example.com\r\n" "sec-fetch-dest: document\r\nsec-fetch-mode: navigate\r\n" "accept-encoding: gzip\r\n" - f"{extra}Connection: close\r\n\r\n" + "Connection: close\r\n\r\n" ).encode() -s = socket.create_connection((host, port)) +s = socket.create_connection((host, port), timeout=timeout) +s.settimeout(timeout) t0 = time.time() s.sendall(req) @@ -569,7 +728,9 @@ cat <<'EOF' one that does not. EOF echo -probe_body_ms() { python3 "$WORK/probe.py" 127.0.0.1 "$TS_PORT" /article; } +probe_body_ms() { + python3 "$WORK/probe.py" 127.0.0.1 "$TS_PORT" /article "$REQUEST_TIMEOUT_SECONDS" +} echo " request A: $(probe_body_ms)" B_LINE="$(probe_body_ms)" echo " request B: $B_LINE" diff --git a/scripts/test-cli.sh b/scripts/test-cli.sh index eef9e2f7d..7b562c96e 100755 --- a/scripts/test-cli.sh +++ b/scripts/test-cli.sh @@ -19,3 +19,20 @@ if ! rustup target list --installed | awk -v target="$HOST_TARGET" '$0 == target fi cargo test --package trusted-server-cli --target "$HOST_TARGET" +export TS_AUDIT_BROWSER_TESTS=1 +AUDIT_BROWSER_TEST_FILTERS=( + "commands::audit::browser::tests::" + "commands::audit::generate::browser_collector::tests::" +) +for AUDIT_BROWSER_TEST_FILTER in "${AUDIT_BROWSER_TEST_FILTERS[@]}"; do + AUDIT_BROWSER_TEST_COUNT="$({ + cargo test --package trusted-server-cli --target "$HOST_TARGET" \ + "$AUDIT_BROWSER_TEST_FILTER" -- --ignored --list + } | awk '/: test$/ { count += 1 } END { print count + 0 }')" + if [ "$AUDIT_BROWSER_TEST_COUNT" -eq 0 ]; then + echo "No ignored browser audit fixtures matched $AUDIT_BROWSER_TEST_FILTER" >&2 + exit 1 + fi + cargo test --package trusted-server-cli --target "$HOST_TARGET" \ + "$AUDIT_BROWSER_TEST_FILTER" -- --ignored --test-threads=1 +done diff --git a/tinybird/datasources/auction_events_raw.datasource b/tinybird/datasources/auction_events_raw.datasource index d62f8ae5d..32959bbd2 100644 --- a/tinybird/datasources/auction_events_raw.datasource +++ b/tinybird/datasources/auction_events_raw.datasource @@ -12,6 +12,7 @@ SCHEMA > `region` Nullable(String), `is_mobile` UInt8, `is_known_browser` UInt8, + `browser_family` LowCardinality(Nullable(String)), `gdpr_applies` UInt8, `consent_present` UInt8, `terminal_status` LowCardinality(Nullable(String)), diff --git a/trusted-server.example.toml b/trusted-server.example.toml index b0e359cb4..48794494c 100644 --- a/trusted-server.example.toml +++ b/trusted-server.example.toml @@ -31,22 +31,22 @@ # REQUIRED — Admin authentication # ----------------------------------------------------------------------------- # HTTP Basic-auth handler(s). At least one handler whose `path` regex covers -# the /_ts/admin endpoints is mandatory; startup fails without it. Each handler -# needs a non-placeholder username/password (deploy validation rejects the -# sample password below). +# the /_ts/admin endpoints is mandatory; startup fails without it. Each +# handler's `password` names a key in the Trusted Server secrets store +# (lowercase letters, digits, and underscores only); provision the real admin +# password under that key before deploying. [[handlers]] # Regex matched against the request path. This one guards the admin surface. path = "^/_ts/admin" username = "admin" -password = "replace-with-admin-password-32-bytes" +password = "handler_password" -# You can add more handlers to basic-auth-protect other path prefixes. The -# sample password below is a known placeholder that deploy validation rejects, -# so it forces a real secret before push: +# You can add more handlers to basic-auth-protect other path prefixes. Each +# handler's password references its own secrets-store key: # [[handlers]] # path = "^/secure" # username = "user" -# password = "replace-with-admin-password" +# password = "secure_handler_password" # ----------------------------------------------------------------------------- @@ -59,8 +59,10 @@ domain = "example.com" cookie_domain = ".example.com" # Upstream origin to proxy publisher content from. No trailing slash. origin_url = "https://origin.example.com" -# HMAC secret for signing first-party proxy URLs. Replace before deploying. -proxy_secret = "change-me-proxy-secret" +# Secrets-store key holding the HMAC secret for signing first-party proxy URLs. +# Provision the referenced key in the Trusted Server secrets store before +# validating/pushing. +proxy_secret = "publisher_proxy_secret" # Optional: override the outbound Host header sent to origin_url. # origin_host_header_override = "www.example.com" # Optional: max bytes buffered when a response is post-processed in full (HTML @@ -68,14 +70,23 @@ proxy_secret = "change-me-proxy-secret" # return 502. Raise for deployments serving very large publisher pages. # max_buffered_body_bytes = 16777216 # 16 MiB +# Optional: trust a fronting CDN's reader IP only when the same request also +# supplies the matching shared secret. The front door must set both headers on +# requests it sends here and remove client-supplied copies on its other routes. +# Fastly-only; see docs/guide/fastly.md for the required front-door setup. +# [trusted_client_ip] +# ip_header = "x-ts-client-ip" +# auth_header = "x-ts-client-ip-auth" +# shared_secret = "replace-with-a-random-shared-secret" + # ----------------------------------------------------------------------------- # REQUIRED — Edge Cookie (EC) identity # ----------------------------------------------------------------------------- [ec] -# Secret used to derive EC identifiers. Must be >= 32 chars and non-placeholder -# in production (deploy validation rejects known placeholders). -passphrase = "trusted-server-placeholder-secret" +# Secrets-store key holding the secret used to derive EC identifiers. The +# resolved secret must be >= 32 chars and non-placeholder in production. +passphrase = "ec_passphrase" # KV store that persists EC identity state. This is the physical store name # bound per adapter (e.g. `ec_identity_store` in fastly.toml); edgezero.toml's # logical KV id is `trusted_server_kv`. @@ -86,9 +97,10 @@ pull_sync_concurrency = 3 # cluster_trust_threshold = 10 # entries with cluster_size <= this are individual users # cluster_recheck_secs = 3600 # re-evaluate cluster_size after this many seconds -# Optional identity partners (SSP/DSP/identity vendors). Each needs a real, -# non-placeholder api_token (>= 32 bytes) at deploy. Configure real partners via -# private config, not this template. +# Optional identity partners (SSP/DSP/identity vendors). `api_token` and +# `ts_pull_token` name keys in the Trusted Server secrets store; provision the +# referenced keys (real tokens, >= 32 bytes) before validating/pushing. +# Configure real partners via private config, not this template. # [[ec.partners]] # name = "Example Partner" # source_domain = "partner.example.com" @@ -97,7 +109,10 @@ pull_sync_concurrency = 3 # openrtb_atype = 3 # include this partner's UIDs in auction user.eids # bidstream_enabled = true -# api_token = "replace-with-partner-api-token-32-bytes-minimum" +# Only for inbound identify or batch-sync API access: +# api_token = "partner_api_token" +# Required when pull sync is enabled: +# ts_pull_token = "partner_ts_pull_token" # batch_rate_limit = 60 # max batch-sync requests/min (default 60) # pull_sync_enabled = false # default false @@ -238,10 +253,13 @@ pull_sync_concurrency = 3 # edge_ttl_seconds = 31536000 # immutable = true -# Server-side auction. Provider/mediator names must match enabled integrations. -# Kept active with the creative-processing leaves present so the EdgeZero -# environment override can apply to them. +# Server-side auction. Providers are declared as [auction.providers.] +# tables below; a mediator name must match an enabled integration. Kept active +# with the creative-processing leaves present so the EdgeZero environment +# override can apply to them. [auction] +# Keep disabled until provider endpoints, routes, and profile values below are +# replaced with deployment-specific settings. enabled = false # Rewrite winning-bid creative HTML to first-party endpoints (default true). Set # false to skip proxy/click-URL conversion and creative TSJS injection. @@ -259,13 +277,51 @@ rewrite_creatives = true # server's iframe), since it removes script-based creatives entirely and would # blank slots on a script-heavy demand stack. sanitize_creatives = false -providers = [] timeout_ms = 2000 # mediator = "adserver_mock" # optional mediator integration # Context keys the JS client may forward into auction requests (allowlist; # empty blocks all). allowed_context_keys = [] +# Example server-side provider declarations. Remove or customize before enabling auctions. +[auction.providers.pbs-main] +protocol = "openrtb-2.6" +profile = "prebid-server" +endpoint = "https://prebid.example.com/openrtb2/auction" +# `prebid-server` defaults to 1000 ms when omitted. +routing = "explicit" + +[auction.providers.pbs-main.profile_config] +debug = false +test_mode = false +consent_forwarding = "both" + +[auction.providers.pbs-main.notifications] +suppress_all = false +suppress_seats = [] + +[auction.bidders.example-bidder] +provider = "pbs-main" + +# APS server behavior is also provider/profile-owned. Use routing = "all_eligible" +# when every banner-compatible slot should be eligible. The APS profile timeout +# defaults to 800 ms; debug and script creatives default to false. +# [auction.providers.aps-main] +# protocol = "openrtb-2.6" +# profile = "aps" +# endpoint = "https://aps.example.com/e/pb/bid" +# routing = "all_eligible" +# +# [auction.providers.aps-main.profile_config] +# account_id = "example-aps-account-id" +# debug = false +# allow_script_creatives = false +# inventory_domain = "publisher.example" +# inventory_page_origin = "https://www.publisher.example" +# +# Browser renderer ownership is configured separately via [integrations.aps] +# (see the Integrations section below). + # Server-side ad slot templates + creative-opportunity auction. Kept active. [creative_opportunities] # Set false to disable server-side ad templates while keeping slot definitions @@ -399,14 +455,13 @@ auction_timeout_ms = 500 # their section is commented out. Required fields are noted per block. # ============================================================================= -# Prebid Server-side auction + first-party Prebid.js bundle. -# When enabled: `server_url` is required, and `external_bundle_url` is required -# (its host must be listed in [proxy].allowed_domains). Kept active but disabled. +# First-party Prebid.js bundle (browser side). Server-side Prebid endpoints are +# declared as [auction.providers.] tables above, not here. When enabled, +# `external_bundle_url` is required (its host must be listed in +# [proxy].allowed_domains). Kept active but disabled. [integrations.prebid] enabled = false -server_url = "https://prebid.example.com/openrtb2/auction" timeout_ms = 1000 -bidders = [] debug = false client_side_bidders = [] # bidders running via native Prebid.js adapters # Keep selected GAM inventory out of Trusted Server's Prebid refresh auctions. @@ -527,24 +582,15 @@ gam_attribution_enabled = false # [integrations.gpt_diagnostics] # enabled = true -# Amazon Publisher Services (APS/TAM) OpenRTB. `account_id` required when -# enabled (`pub_id` is accepted as a deserialization alias only). -# [integrations.aps] -# enabled = true -# account_id = "example-aps-account-id" # required (non-empty); your APS account -# endpoint = "https://aps.example.com/e/pb/bid" -# timeout_ms = 1000 -# Include raw APS request/response data in /auction metadata on test sites only. -# debug = false -# Script creatives require separate security validation before opt-in. -# allow_script_creatives = false +# Amazon Publisher Services (APS/TAM) browser renderer ownership. APS server +# behavior (endpoint, account id, timeouts, script creatives) is declared on an +# `aps`-profile [auction.providers.] table in the auction section above. # Winning-bid renderer. Default `trusted_server` uses TS's opaque static renderer # route; set `publisher_native` only for the controlled publisher-origin # friendly-frame experiment. +# [integrations.aps] +# enabled = true # rendering_mode = "trusted_server" -# Set both when the deployment hostname differs from APS-authorized inventory. -# inventory_domain = "publisher.example" -# inventory_page_origin = "https://www.publisher.example" # Google Tag Manager first-party proxy. Kept active but disabled so `ts audit` # can fill container_id and flip `enabled` when GTM is detected. `container_id`