diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 95654ecdd..7b374bcff 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -1,4 +1,4 @@ -name: "Run Tests" +name: 'Run Tests' permissions: contents: read @@ -18,19 +18,19 @@ jobs: - name: Retrieve Rust version id: rust-version - run: echo "rust-version=$(grep '^rust ' .tool-versions | awk '{print $2}')" >> $GITHUB_OUTPUT + run: echo "rust-version=$(grep '^rust ' .tool-versions | awk '{print $2}')" >> "$GITHUB_OUTPUT" shell: bash - name: Retrieve Viceroy version id: viceroy-version # `.tool-versions` is the single source of truth so this workflow and # `.github/actions/setup-integration-test-env/action.yml` can't drift. - run: echo "viceroy-version=$(grep '^viceroy ' .tool-versions | awk '{print $2}')" >> $GITHUB_OUTPUT + run: echo "viceroy-version=$(grep '^viceroy ' .tool-versions | awk '{print $2}')" >> "$GITHUB_OUTPUT" shell: bash - name: Retrieve Node.js version id: node-version - run: echo "node-version=$(grep '^nodejs ' .tool-versions | awk '{print $2}')" >> $GITHUB_OUTPUT + run: echo "node-version=$(grep '^nodejs ' .tool-versions | awk '{print $2}')" >> "$GITHUB_OUTPUT" shell: bash - name: Set up Rust toolchain @@ -71,6 +71,65 @@ jobs: - name: Run cache purge harness run: BID_DELAY=3 ./scripts/template-cache-local-test.sh purge + test-fastly-eof: + name: HTML EOF runtime (Fastly) + runs-on: ubuntu-latest + timeout-minutes: 25 + steps: + - uses: actions/checkout@v4 + + - name: Read pinned build tool versions + id: versions + shell: bash + run: | + echo "rust=$(awk '$1 == "rust" { print $2 }' .tool-versions)" >> "$GITHUB_OUTPUT" + echo "node=$(awk '$1 == "nodejs" { print $2 }' .tool-versions)" >> "$GITHUB_OUTPUT" + + - uses: actions-rust-lang/setup-rust-toolchain@v1 + with: + toolchain: ${{ steps.versions.outputs.rust }} + target: wasm32-wasip1 + cache-shared-key: cargo-${{ runner.os }} + + - uses: actions/setup-node@v4 + with: + node-version: ${{ steps.versions.outputs.node }} + + - name: Cache isolated EOF fixture Viceroy + id: cache-eof-viceroy + uses: actions/cache@v4 + with: + path: ${{ runner.temp }}/viceroy-eof/bin/viceroy + key: viceroy-eof-${{ runner.os }}-amd64-v0.19.0-sha256-23b2af7d + + - name: Install isolated Viceroy 0.19 for backend timeout simulation + if: steps.cache-eof-viceroy.outputs.cache-hit != 'true' + shell: bash + run: | + curl -fsSL --retry 3 https://github.com/fastly/Viceroy/releases/download/v0.19.0/viceroy_v0.19.0_linux-amd64.tar.gz -o "$RUNNER_TEMP/viceroy-eof.tar.gz" + echo "23b2af7d8dce8f4a217716d6bd8b15c4f41494316a0d4e80f179ffb4b6c74744 $RUNNER_TEMP/viceroy-eof.tar.gz" | sha256sum --check --strict + mkdir -p "$RUNNER_TEMP/viceroy-eof/bin" + tar -xzf "$RUNNER_TEMP/viceroy-eof.tar.gz" -C "$RUNNER_TEMP/viceroy-eof/bin" viceroy + + - name: Select isolated fixture runtime + shell: bash + run: echo "$RUNNER_TEMP/viceroy-eof/bin" >> "$GITHUB_PATH" + + - name: Verify actual Fastly HTML EOF completion and auction assembly + timeout-minutes: 15 + env: + CARGO_TARGET_DIR: ${{ github.workspace }}/target + TMPDIR: ${{ runner.temp }} + run: ./scripts/async-html-finalization-local-test.sh fastly --release --auction + + - name: Upload EOF runtime diagnostics on failure + if: failure() + uses: actions/upload-artifact@v4 + with: + name: fastly-html-eof-diagnostics + path: ${{ runner.temp }}/trusted-server-finalize-fastly-*/ + if-no-files-found: ignore + test-axum: name: cargo test (axum native) runs-on: ubuntu-latest @@ -79,7 +138,7 @@ jobs: - name: Retrieve Rust version id: rust-version - run: echo "rust-version=$(grep '^rust ' .tool-versions | awk '{print $2}')" >> $GITHUB_OUTPUT + run: echo "rust-version=$(grep '^rust ' .tool-versions | awk '{print $2}')" >> "$GITHUB_OUTPUT" shell: bash - name: Set up Rust toolchain @@ -89,6 +148,8 @@ jobs: # wasm32-wasip1 is required by the "Verify Fastly WASM release # build" step below; the axum build and tests are native. target: wasm32-wasip1 + # clippy is required by the "Lint native finalization harness" step. + components: clippy cache-shared-key: cargo-${{ runner.os }} - name: Read pinned Node.js version for CLI fixtures @@ -106,6 +167,27 @@ jobs: - name: Run Axum adapter tests run: cargo test-axum + - name: Verify finalization harness process cleanup + run: cargo test --manifest-path crates/trusted-server-integration-tests/Cargo.toml --features html-eof-runtime --test html_eof_runtime + + - name: Lint native finalization harness + run: cargo clippy --manifest-path crates/trusted-server-integration-tests/Cargo.toml --features html-eof-runtime --test html_eof_runtime -- -D warnings + + - name: Verify actual Axum HTML finalization and auction assembly + timeout-minutes: 10 + env: + CARGO_TARGET_DIR: ${{ github.workspace }}/target + TMPDIR: ${{ runner.temp }} + run: ./scripts/async-html-finalization-local-test.sh axum --auction + + - name: Upload finalization runtime diagnostics on failure + if: failure() + uses: actions/upload-artifact@v4 + with: + name: axum-html-finalization-diagnostics + path: ${{ runner.temp }}/trusted-server-finalize-axum-*/ + if-no-files-found: ignore + - name: Test template build digest inputs (native) run: cargo test -p trusted-server-core --test template_build_digest @@ -133,7 +215,7 @@ jobs: - name: Retrieve Rust version id: rust-version - run: echo "rust-version=$(grep '^rust ' .tool-versions | awk '{print $2}')" >> $GITHUB_OUTPUT + run: echo "rust-version=$(grep '^rust ' .tool-versions | awk '{print $2}')" >> "$GITHUB_OUTPUT" shell: bash - name: Set up Rust toolchain (native + wasm32-unknown-unknown) @@ -160,7 +242,7 @@ jobs: - name: Retrieve Rust version id: rust-version - run: echo "rust-version=$(grep '^rust ' .tool-versions | awk '{print $2}')" >> $GITHUB_OUTPUT + run: echo "rust-version=$(grep '^rust ' .tool-versions | awk '{print $2}')" >> "$GITHUB_OUTPUT" shell: bash - name: Set up Rust toolchain (native + wasm32-wasip1) @@ -185,7 +267,7 @@ jobs: TRUSTED_SERVER__PUBLISHER__ORIGIN_URL: http://127.0.0.1:8080 TRUSTED_SERVER__PUBLISHER__PROXY_SECRET: integration-test-proxy-secret TRUSTED_SERVER__EC__PASSPHRASE: integration-test-ec-secret-padded-32 - TRUSTED_SERVER__PROXY__CERTIFICATE_CHECK: "false" + TRUSTED_SERVER__PROXY__CERTIFICATE_CHECK: 'false' run: cargo build --package trusted-server-adapter-spin --target wasm32-wasip1 --features spin --release - name: Run Spin adapter tests (native host) @@ -199,7 +281,7 @@ jobs: - name: Retrieve Rust version id: rust-version - run: echo "rust-version=$(grep '^rust ' .tool-versions | awk '{print $2}')" >> $GITHUB_OUTPUT + run: echo "rust-version=$(grep '^rust ' .tool-versions | awk '{print $2}')" >> "$GITHUB_OUTPUT" shell: bash - name: Set up Rust toolchain @@ -230,14 +312,14 @@ jobs: - name: Retrieve Rust version id: rust-version - run: echo "rust-version=$(grep '^rust ' .tool-versions | awk '{print $2}')" >> $GITHUB_OUTPUT + run: echo "rust-version=$(grep '^rust ' .tool-versions | awk '{print $2}')" >> "$GITHUB_OUTPUT" shell: bash - name: Set up Rust toolchain uses: actions-rust-lang/setup-rust-toolchain@v1 with: toolchain: ${{ steps.rust-version.outputs.rust-version }} - components: "clippy, rustfmt" + components: 'clippy, rustfmt' cache-shared-key: cargo-cli-${{ runner.os }} - name: Read pinned Node.js version for CLI fixtures @@ -304,14 +386,14 @@ jobs: - name: Retrieve Node.js version id: node-version working-directory: . - run: echo "node-version=$(grep '^nodejs ' .tool-versions | awk '{print $2}')" >> $GITHUB_OUTPUT + run: echo "node-version=$(grep '^nodejs ' .tool-versions | awk '{print $2}')" >> "$GITHUB_OUTPUT" shell: bash - name: Use Node.js uses: actions/setup-node@v4 with: node-version: ${{ steps.node-version.outputs.node-version }} - cache: "npm" + cache: 'npm' cache-dependency-path: crates/trusted-server-js/lib/package.json - name: Install dependencies diff --git a/AGENTS.md b/AGENTS.md index dd1dd5233..1be970f03 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -363,6 +363,9 @@ Every PR must pass: 8. Markdown format outside `docs/` (requires `cd docs && npm ci` first): `docs/node_modules/.bin/prettier --config docs/.prettierrc --check "*.md" ".claude/**/*.md" ".github/**/*.md" "crates/**/*.md" "scripts/**/*.md" "tinybird/**/*.md"`; fix with `--write` in place of `--check` 9. Native core build-digest test and lint (`cargo test -p trusted-server-core --test template_build_digest` and `cargo clippy -p trusted-server-core --test template_build_digest -- -D warnings`) 10. Wasm clock lint: `scripts/lint-wasm-clock.sh` (crates built into the Cloudflare Worker must use `web_time`, not the `std::time::{Instant,SystemTime}` `now`/`elapsed` methods, which panic there) +11. Native Rust HTML EOF harness configuration, timing and cleanup: `cargo test --manifest-path crates/trusted-server-integration-tests/Cargo.toml --features html-eof-runtime --test html_eof_runtime` and `cargo clippy --manifest-path crates/trusted-server-integration-tests/Cargo.toml --features html-eof-runtime --test html_eof_runtime -- -D warnings` +12. Actual Axum EOF/auction controls: `./scripts/async-html-finalization-local-test.sh axum --auction` +13. Actual Fastly EOF/auction controls with isolated Viceroy 0.19+: `PATH="/tmp/example-eof-viceroy/bin:$PATH" ./scripts/async-html-finalization-local-test.sh fastly --release --auction`; install with `cargo install viceroy --version 0.19.0 --locked --root /tmp/example-eof-viceroy`. Ordinary tests retain pinned Viceroy 0.17. See `scripts/README.md` for retained diagnostics and other runtimes. --- diff --git a/Cargo.lock b/Cargo.lock index e915a0ea8..6097394bf 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -6686,12 +6686,14 @@ version = "0.1.0" dependencies = [ "async-trait", "axum", + "base64", "bytes", "derive_more", "edgezero-adapter-axum", "edgezero-core", "env_logger", "error-stack", + "futures", "http", "http-body-util", "libc", @@ -6699,6 +6701,7 @@ dependencies = [ "reqwest 0.12.28", "scraper", "serde_json", + "tempfile", "testcontainers", "tokio", "toml", diff --git a/crates/trusted-server-adapter-axum/src/app.rs b/crates/trusted-server-adapter-axum/src/app.rs index caba714d6..cff5bc47e 100644 --- a/crates/trusted-server-adapter-axum/src/app.rs +++ b/crates/trusted-server-adapter-axum/src/app.rs @@ -246,6 +246,7 @@ async fn dispatch_fallback( // slots; `handle_publisher_request` matches them against the request path. let mut ec_context = build_ec_context(state, services, &req); let auction = AuctionDispatch { + integration_registry: &state.registry, orchestrator: &state.orchestrator, slots: state.settings.creative_opportunity_slots(), registry: None, @@ -526,6 +527,7 @@ fn named_route_handler( } else { let mut ec_context = build_ec_context(&state, &services, &req); let auction = AuctionDispatch { + integration_registry: &state.registry, orchestrator: &state.orchestrator, slots: state.settings.creative_opportunity_slots(), registry: None, diff --git a/crates/trusted-server-adapter-axum/src/platform.rs b/crates/trusted-server-adapter-axum/src/platform.rs index e2713f47f..c91a422d1 100644 --- a/crates/trusted-server-adapter-axum/src/platform.rs +++ b/crates/trusted-server-adapter-axum/src/platform.rs @@ -552,6 +552,7 @@ pub fn build_runtime_services(ctx: &edgezero_core::context::RequestContext) -> R static GEO: std::sync::OnceLock> = std::sync::OnceLock::new(); RuntimeServices::builder() + .timer(Arc::new(AxumTimer)) .config_store(Arc::clone(CONFIG_STORE.get_or_init(|| { Arc::new(AxumPlatformConfigStore) as Arc }))) @@ -581,12 +582,35 @@ pub fn build_runtime_services(ctx: &edgezero_core::context::RequestContext) -> R .build() } +struct AxumTimer; + +impl trusted_server_core::platform::PlatformTimer for AxumTimer { + fn delay(&self, duration: std::time::Duration) -> futures::future::LocalBoxFuture<'static, ()> { + Box::pin(async move { + tokio::time::sleep(duration).await; + }) + } +} + // --------------------------------------------------------------------------- // Tests // --------------------------------------------------------------------------- #[cfg(test)] mod tests { + #[test] + fn html_finalization_timer_is_request_scoped() { + let request = edgezero_core::http::request_builder() + .body(edgezero_core::body::Body::empty()) + .expect("should build a request"); + let context = edgezero_core::context::RequestContext::new(request, Default::default()); + let services = build_runtime_services(&context); + assert!( + services.timer().is_some(), + "should install a cooperative timer without polling it" + ); + } + use super::*; use edgezero_core::body::Body as EdgeBody; use std::time::Duration; diff --git a/crates/trusted-server-adapter-cloudflare/Cargo.toml b/crates/trusted-server-adapter-cloudflare/Cargo.toml index 2ce969ef7..495dfe633 100644 --- a/crates/trusted-server-adapter-cloudflare/Cargo.toml +++ b/crates/trusted-server-adapter-cloudflare/Cargo.toml @@ -17,6 +17,8 @@ crate-type = ["cdylib", "rlib"] [features] default = [] +# Opt-in local runtime fixture and its diagnostic console sink. +test-utils = ["trusted-server-core/test-utils"] # Keep for explicit `cargo check --features cloudflare --target wasm32-unknown-unknown` cloudflare = ["edgezero-adapter-cloudflare/cloudflare", "dep:worker"] diff --git a/crates/trusted-server-adapter-cloudflare/src/app.rs b/crates/trusted-server-adapter-cloudflare/src/app.rs index 87b9567e7..b6e49f561 100644 --- a/crates/trusted-server-adapter-cloudflare/src/app.rs +++ b/crates/trusted-server-adapter-cloudflare/src/app.rs @@ -555,6 +555,7 @@ fn build_router(state: &Arc) -> RouterService { } else { let mut ec_context = build_ec_context(&state.settings, &services, &req); let auction = AuctionDispatch { + integration_registry: &state.registry, orchestrator: &state.orchestrator, slots: state.settings.creative_opportunity_slots(), registry: None, @@ -724,6 +725,7 @@ fn build_router(state: &Arc) -> RouterService { let page_bids = make_handler(Arc::clone(&state), |s, services, req| async move { let mut ec_context = build_ec_context(&s.settings, &services, &req); let auction = AuctionDispatch { + integration_registry: &s.registry, orchestrator: &s.orchestrator, slots: s.settings.creative_opportunity_slots(), registry: None, diff --git a/crates/trusted-server-adapter-cloudflare/src/lib.rs b/crates/trusted-server-adapter-cloudflare/src/lib.rs index 980a9b209..80c24e8f1 100644 --- a/crates/trusted-server-adapter-cloudflare/src/lib.rs +++ b/crates/trusted-server-adapter-cloudflare/src/lib.rs @@ -24,6 +24,8 @@ use worker::{Context, Env, Request, Response, Result, event}; /// Returns a Workers runtime error when the fallback error response cannot be /// constructed. pub async fn main(req: Request, env: Env, ctx: Context) -> Result { + #[cfg(feature = "test-utils")] + install_fixture_logger(); if let Ok(config) = env.var("TRUSTED_SERVER_CONFIG") { app::set_cloudflare_config_json(config.to_string()); } @@ -37,3 +39,32 @@ pub async fn main(req: Request, env: Env, ctx: Context) -> Result { } } } + +#[cfg(all(feature = "test-utils", target_arch = "wasm32"))] +struct FixtureLogger; + +#[cfg(all(feature = "test-utils", target_arch = "wasm32"))] +impl log::Log for FixtureLogger { + fn enabled(&self, metadata: &log::Metadata<'_>) -> bool { + metadata.level() <= log::Level::Warn + } + + fn log(&self, record: &log::Record<'_>) { + if self.enabled(record.metadata()) { + worker::console_warn!("{}", record.args()); + } + } + + fn flush(&self) {} +} + +#[cfg(all(feature = "test-utils", target_arch = "wasm32"))] +fn install_fixture_logger() { + static LOGGER: FixtureLogger = FixtureLogger; + if log::set_logger(&LOGGER).is_ok() { + std::panic::set_hook(Box::new(|information| { + worker::console_error!("{}", information); + })); + log::set_max_level(log::LevelFilter::Warn); + } +} diff --git a/crates/trusted-server-adapter-cloudflare/src/platform.rs b/crates/trusted-server-adapter-cloudflare/src/platform.rs index 94616b203..09d1f2ce6 100644 --- a/crates/trusted-server-adapter-cloudflare/src/platform.rs +++ b/crates/trusted-server-adapter-cloudflare/src/platform.rs @@ -631,7 +631,7 @@ pub fn build_runtime_services(ctx: &edgezero_core::context::RequestContext) -> R // simply absent on the native host target, producing Ok(None) from lookup(). let geo = build_geo(ctx); - RuntimeServices::builder() + let builder = RuntimeServices::builder() .config_store(config_store) .secret_store(secret_store) .kv_store(kv_store) @@ -643,8 +643,10 @@ pub fn build_runtime_services(ctx: &edgezero_core::context::RequestContext) -> R tls_protocol: None, tls_cipher: None, ..ClientInfo::default() - }) - .build() + }); + #[cfg(target_arch = "wasm32")] + let builder = builder.timer(Arc::new(CloudflareTimer)); + builder.build() } // --------------------------------------------------------------------------- @@ -754,6 +756,18 @@ fn reject_multi_provider_fanout(len: usize) -> Result<(), Report> Ok(()) } +#[cfg(target_arch = "wasm32")] +struct CloudflareTimer; + +#[cfg(target_arch = "wasm32")] +impl trusted_server_core::platform::PlatformTimer for CloudflareTimer { + fn delay(&self, duration: std::time::Duration) -> futures::future::LocalBoxFuture<'static, ()> { + Box::pin(async move { + worker::Delay::from(duration).await; + }) + } +} + #[cfg(test)] mod tests { use super::*; diff --git a/crates/trusted-server-adapter-fastly/src/app.rs b/crates/trusted-server-adapter-fastly/src/app.rs index d1da9b61f..b96bcccbf 100644 --- a/crates/trusted-server-adapter-fastly/src/app.rs +++ b/crates/trusted-server-adapter-fastly/src/app.rs @@ -255,6 +255,8 @@ fn warn_if_certificate_check_disabled(settings: &Settings) { /// continues to rely on the trusted `fastly-ssl` header injected by /// `edgezero_main` after sanitization. fn build_per_request_services(state: &AppState, ctx: &RequestContext) -> RuntimeServices { + // No cooperative timer is installed: Fastly host calls block. EOF consumers + // must bound their own transport calls using the remaining finish budget. let client_info = ctx .request() .extensions() @@ -696,6 +698,7 @@ async fn run_named_route( Some(&partner_registry) }; let auction = AuctionDispatch { + integration_registry: &state.registry, orchestrator: &state.orchestrator, slots: state.settings.creative_opportunity_slots(), registry: registry_ref, @@ -860,6 +863,7 @@ async fn dispatch_fallback( match PartnerRegistry::from_config(&state.settings.ec.partners) { Ok(partner_registry) => { let auction = AuctionDispatch { + integration_registry: &state.registry, orchestrator: &state.orchestrator, slots, registry: Some(&partner_registry), @@ -3030,6 +3034,7 @@ mod tests { None, &mut ec_context, AuctionDispatch { + integration_registry: ®istry, orchestrator: &orchestrator, slots: settings.creative_opportunity_slots(), registry: None, diff --git a/crates/trusted-server-adapter-spin/Cargo.toml b/crates/trusted-server-adapter-spin/Cargo.toml index 81eeb9430..b0a5403d1 100644 --- a/crates/trusted-server-adapter-spin/Cargo.toml +++ b/crates/trusted-server-adapter-spin/Cargo.toml @@ -17,6 +17,8 @@ crate-type = ["cdylib", "rlib"] [features] default = [] +# Opt-in local runtime fixture and its diagnostic stderr sink. +test-utils = ["trusted-server-core/test-utils"] spin = ["edgezero-adapter-spin/spin"] [dependencies] diff --git a/crates/trusted-server-adapter-spin/src/app.rs b/crates/trusted-server-adapter-spin/src/app.rs index 804a23b62..674b1ba0b 100644 --- a/crates/trusted-server-adapter-spin/src/app.rs +++ b/crates/trusted-server-adapter-spin/src/app.rs @@ -713,6 +713,7 @@ fn build_router(state: &Arc) -> RouterService { } let mut ec_context = build_ec_context(&s.settings, &services, &req); let auction = AuctionDispatch { + integration_registry: &s.registry, orchestrator: &s.orchestrator, slots: s.settings.creative_opportunity_slots(), registry: None, @@ -836,6 +837,7 @@ fn build_router(state: &Arc) -> RouterService { } else { let mut ec_context = build_ec_context(&state.settings, &services, &req); let auction = AuctionDispatch { + integration_registry: &state.registry, orchestrator: &state.orchestrator, slots: state.settings.creative_opportunity_slots(), registry: None, diff --git a/crates/trusted-server-adapter-spin/src/lib.rs b/crates/trusted-server-adapter-spin/src/lib.rs index f47877ff2..adbf61c9d 100644 --- a/crates/trusted-server-adapter-spin/src/lib.rs +++ b/crates/trusted-server-adapter-spin/src/lib.rs @@ -13,5 +13,36 @@ use spin_sdk::http_service; #[http_service] // FORCED: edgezero_adapter_spin::run_app returns anyhow::Result — EdgeZero SDK constraint, not a project choice. async fn handle(req: Request) -> anyhow::Result { + #[cfg(feature = "test-utils")] + install_fixture_logger(); edgezero_adapter_spin::run_app::(req).await } + +#[cfg(all(feature = "test-utils", feature = "spin", target_arch = "wasm32"))] +use std::io::Write as _; + +#[cfg(all(feature = "test-utils", feature = "spin", target_arch = "wasm32"))] +struct FixtureLogger; + +#[cfg(all(feature = "test-utils", feature = "spin", target_arch = "wasm32"))] +impl log::Log for FixtureLogger { + fn enabled(&self, metadata: &log::Metadata<'_>) -> bool { + metadata.level() <= log::Level::Warn + } + + fn log(&self, record: &log::Record<'_>) { + if self.enabled(record.metadata()) { + let _ = writeln!(std::io::stderr(), "{}", record.args()); + } + } + + fn flush(&self) {} +} + +#[cfg(all(feature = "test-utils", feature = "spin", target_arch = "wasm32"))] +fn install_fixture_logger() { + static LOGGER: FixtureLogger = FixtureLogger; + if log::set_logger(&LOGGER).is_ok() { + log::set_max_level(log::LevelFilter::Warn); + } +} diff --git a/crates/trusted-server-adapter-spin/src/platform.rs b/crates/trusted-server-adapter-spin/src/platform.rs index b1ced0f70..d46170bfc 100644 --- a/crates/trusted-server-adapter-spin/src/platform.rs +++ b/crates/trusted-server-adapter-spin/src/platform.rs @@ -727,7 +727,7 @@ pub fn build_runtime_services(ctx: &edgezero_core::context::RequestContext) -> R #[cfg(not(all(feature = "spin", target_arch = "wasm32")))] let secret_store: Arc = Arc::new(NoopSecretStore); - RuntimeServices::builder() + let builder = RuntimeServices::builder() .config_store(config_store) .secret_store(secret_store) .kv_store(kv_store) @@ -744,8 +744,10 @@ pub fn build_runtime_services(ctx: &edgezero_core::context::RequestContext) -> R h2_fingerprint: None, server_hostname: None, server_region: None, - }) - .build() + }); + #[cfg(all(feature = "spin", target_arch = "wasm32"))] + let builder = builder.timer(Arc::new(SpinTimer)); + builder.build() } // --------------------------------------------------------------------------- @@ -778,6 +780,18 @@ fn extract_client_ip(ctx: &edgezero_core::context::RequestContext) -> Option futures::future::LocalBoxFuture<'static, ()> { + Box::pin(async move { + spin_sdk::time::sleep(duration).await; + }) + } +} + #[cfg(test)] mod tests { use super::*; diff --git a/crates/trusted-server-core/src/config.rs b/crates/trusted-server-core/src/config.rs index e48f5c2f0..f8a439c6e 100644 --- a/crates/trusted-server-core/src/config.rs +++ b/crates/trusted-server-core/src/config.rs @@ -37,6 +37,8 @@ use crate::settings::{AssetOriginAuth, IntegrationConfig, Settings}; const DEPLOY_VALIDATION_FIELD: &str = "trusted_server"; #[cfg(test)] const DEPLOY_VALIDATED_INTEGRATION_IDS: &[&str] = &[ + #[cfg(feature = "test-utils")] + crate::test_support::async_html_finalization::ID, "prebid", "aps", "adserver_mock", @@ -308,6 +310,11 @@ fn validate_enabled_integrations( validate_prebid(settings, plan)?; validate_integration::(settings, "aps")?; validate_integration::(settings, "adserver_mock")?; + #[cfg(feature = "test-utils")] + validate_integration::( + settings, + crate::test_support::async_html_finalization::ID, + )?; validate_integration::(settings, "testlight")?; validate_integration::(settings, "nextjs")?; validate_integration::(settings, "permutive")?; diff --git a/crates/trusted-server-core/src/html_processor.rs b/crates/trusted-server-core/src/html_processor.rs index 3889d7d59..a6e4855de 100644 --- a/crates/trusted-server-core/src/html_processor.rs +++ b/crates/trusted-server-core/src/html_processor.rs @@ -24,25 +24,44 @@ use crate::settings::Settings; use crate::streaming_processor::{HtmlRewriterAdapter, StreamProcessor}; use crate::tsjs; -struct HtmlWithStreamingProcessors { +pub(crate) mod finish; + +use finish::HtmlProcessorStage; + +pub(crate) struct HtmlWithStreamingProcessors { inner: Box, - processors: Vec>, + stages: Vec, + finish_index: Option, + marker_length: usize, } impl StreamProcessor for HtmlWithStreamingProcessors { fn process_chunk(&mut self, chunk: &[u8], is_last: bool) -> Result, io::Error> { + self.validate_capabilities()?; + if is_last && self.requires_async_finish() { + return Err(io::Error::other( + "HTML processors require asynchronous EOF completion", + )); + } let mut output = self.inner.process_chunk(chunk, is_last)?; - for processor in &mut self.processors { - output = processor.process_chunk(&output, is_last)?; + for stage in &mut self.stages { + output = stage.process(&output, is_last, self.marker_length)?; } Ok(output) } fn reset(&mut self) { self.inner.reset(); - for processor in &mut self.processors { - processor.reset(); + for stage in &mut self.stages { + stage.reset(); } + self.finish_index = None; + } + + fn requires_async_finish(&self) -> bool { + self.stages + .iter() + .any(HtmlProcessorStage::requires_async_finish) } } @@ -194,8 +213,19 @@ impl HtmlProcessorConfig { reason = "the returned processor owns request configuration captured by its handlers" )] pub fn create_html_processor(config: HtmlProcessorConfig) -> impl StreamProcessor { + create_html_processor_with_document_state(config, &IntegrationDocumentState::default()) +} + +#[allow( + clippy::needless_pass_by_value, + reason = "static parser callbacks capture owned request configuration" +)] +pub(crate) fn create_html_processor_with_document_state( + config: HtmlProcessorConfig, + document_state: &IntegrationDocumentState, +) -> HtmlWithStreamingProcessors { let stream_processor_factories = config.integrations.html_stream_processor_factories(); - let document_state = IntegrationDocumentState::default(); + if config.suppress_datadome_client_side_tag { document_state.get_or_insert_with(DATADOME_INTEGRATION_ID, || DataDomeClientTagSuppressed); } @@ -708,14 +738,19 @@ pub fn create_html_processor(config: HtmlProcessorConfig) -> impl StreamProcesso origin_host: config.origin_host.clone(), document_state: document_state.clone(), }; - let processors = stream_processor_factories + let stages = stream_processor_factories .into_iter() - .map(|factory| factory.create(stream_context.clone())) + .map(|factory| { + HtmlProcessorStage::new( + factory.integration_id(), + factory.create(stream_context.clone()), + factory.requires_async_finish(), + factory.buffer_output_until_eof(), + config.max_buffered_body_bytes, + ) + }) .collect(); - HtmlWithStreamingProcessors { - inner: Box::new(inner), - processors, - } + HtmlWithStreamingProcessors::from_stages(Box::new(inner), stages) } #[cfg(test)] @@ -723,7 +758,8 @@ mod tests { use super::*; use crate::integrations::{ AttributeRewriteAction, IntegrationAttributeContext, IntegrationAttributeRewriter, - IntegrationHeadInjector, IntegrationHtmlContext, + IntegrationHeadInjector, IntegrationHtmlContext, IntegrationScriptContext, + IntegrationScriptRewriter, ScriptRewriteAction, }; use crate::streaming_processor::{Compression, PipelineConfig, StreamingPipeline}; use crate::test_support::tests::create_test_settings; @@ -752,6 +788,53 @@ mod tests { } } + #[test] + fn terminal_document_state_observes_parser_callbacks_and_isolates_requests() { + struct ArticleObserver; + impl IntegrationScriptRewriter for ArticleObserver { + fn integration_id(&self) -> &'static str { + "example_article" + } + fn selector(&self) -> &'static str { + "p" + } + fn rewrite( + &self, + content: &str, + context: &IntegrationScriptContext<'_>, + ) -> ScriptRewriteAction { + context + .document_state + .get_or_insert_with("example_article", || content.to_string()); + ScriptRewriteAction::Keep + } + } + for text in ["Example first article", "Example second article"] { + let state = IntegrationDocumentState::default(); + let mut config = create_test_config(); + config.integrations = + IntegrationRegistry::from_rewriters(Vec::new(), vec![Arc::new(ArticleObserver)]); + let mut processor = create_html_processor_with_document_state(config, &state); + let _ = processor + .process_chunk( + format!("

{text}

").as_bytes(), + false, + ) + .expect("should process article HTML"); + let _ = processor + .process_chunk(&[], true) + .expect("should close the synchronous parser"); + let article = state + .get::("example_article") + .expect("should expose parser state to the terminal phase"); + assert_eq!( + article.as_str(), + text, + "should share this document's observed text without leaking another request" + ); + } + } + #[test] fn integration_attribute_rewriter_can_remove_elements() { struct RemovingLinkRewriter; @@ -1501,22 +1584,34 @@ mod tests { let inner_final_calls = Arc::new(std::sync::atomic::AtomicUsize::new(0)); let first_final_calls = Arc::new(std::sync::atomic::AtomicUsize::new(0)); let second_final_calls = Arc::new(std::sync::atomic::AtomicUsize::new(0)); - let mut processor = HtmlWithStreamingProcessors { - inner: Box::new(DecoratingProcessor { + let mut processor = HtmlWithStreamingProcessors::from_stages( + Box::new(DecoratingProcessor { prefix: b'I', final_calls: Arc::clone(&inner_final_calls), }), - processors: vec![ - Box::new(DecoratingProcessor { - prefix: b'A', - final_calls: Arc::clone(&first_final_calls), - }), - Box::new(DecoratingProcessor { - prefix: b'B', - final_calls: Arc::clone(&second_final_calls), - }), + vec![ + HtmlProcessorStage::new( + "A", + Box::new(DecoratingProcessor { + prefix: b'A', + final_calls: Arc::clone(&first_final_calls), + }), + false, + false, + 64, + ), + HtmlProcessorStage::new( + "B", + Box::new(DecoratingProcessor { + prefix: b'B', + final_calls: Arc::clone(&second_final_calls), + }), + false, + false, + 64, + ), ], - }; + ); assert_eq!( processor diff --git a/crates/trusted-server-core/src/html_processor/finish.rs b/crates/trusted-server-core/src/html_processor/finish.rs new file mode 100644 index 000000000..05b1705a9 --- /dev/null +++ b/crates/trusted-server-core/src/html_processor/finish.rs @@ -0,0 +1,1721 @@ +//! Ordered HTML processor completion and optional bounded output retention. + +use std::io; + +use futures::FutureExt as _; +use futures::future::{Fuse, LocalBoxFuture}; +use http::StatusCode; +use web_time::Instant; + +use super::HtmlWithStreamingProcessors; +use crate::platform::RuntimeServices; +use crate::streaming_processor::StreamProcessor; +use crate::streaming_processor::finish::{ + StreamFinish, StreamFinishDeadline, StreamFinishError, StreamFinishInput, StreamOutputWriter, +}; + +pub(crate) struct HtmlProcessorStage { + integration_id: &'static str, + processor: Box, + async_capable: bool, + retained: Option, + skip_async: bool, + buffer_output: bool, + limit: usize, + previous_suffix: Vec, +} + +impl HtmlProcessorStage { + pub(super) fn new( + integration_id: &'static str, + processor: Box, + async_capable: bool, + buffered: bool, + limit: usize, + ) -> Self { + Self { + integration_id, + processor, + async_capable, + retained: buffered.then(|| StreamOutputWriter::new(limit)), + skip_async: false, + buffer_output: buffered, + limit, + previous_suffix: Vec::new(), + } + } + + pub(super) fn reset(&mut self) { + self.processor.reset(); + self.retained = self + .buffer_output + .then(|| StreamOutputWriter::new(self.limit)); + self.skip_async = false; + self.previous_suffix.clear(); + } + + pub(super) fn validate(&self) -> io::Result<()> { + if self.async_capable != self.processor.requires_async_finish() + || (self.retained.is_some() && !self.async_capable) + { + return Err(io::Error::other(format!( + "Integration {} declares inconsistent asynchronous EOF capabilities", + self.integration_id, + ))); + } + Ok(()) + } + + pub(super) fn requires_async_finish(&self) -> bool { + self.async_capable || self.processor.requires_async_finish() + } + + fn output(&mut self, chunk: &[u8], last: bool) -> io::Result> { + let output = self.processor.process_chunk(chunk, last)?; + let Some(retained) = self.retained.as_mut() else { + return Ok(output); + }; + if retained.append(&output).is_ok() { + return if last { + Ok(self + .retained + .take() + .expect("should own retained stage output") + .into_bytes()) + } else { + Ok(Vec::new()) + }; + } + self.skip_async = true; + let mut ready = self + .retained + .take() + .expect("should own overflow prefix") + .into_bytes(); + ready.extend_from_slice(&output); + log::warn!( + "HTML EOF processor {} bypasses optional I/O after output collection overflow", + self.integration_id + ); + Ok(ready) + } + + fn remember_suffix(&mut self, output: &[u8], marker_length: usize) { + let keep = marker_length.saturating_sub(1); + if keep == 0 || output.is_empty() { + return; + } + if output.len() >= keep { + self.previous_suffix.clear(); + self.previous_suffix + .extend_from_slice(&output[output.len() - keep..]); + } else { + self.previous_suffix.extend_from_slice(output); + if self.previous_suffix.len() > keep { + self.previous_suffix + .drain(..self.previous_suffix.len() - keep); + } + } + } + + pub(super) fn process( + &mut self, + bytes: &[u8], + last: bool, + marker_length: usize, + ) -> io::Result> { + let output = self.output(bytes, last)?; + self.remember_suffix(&output, marker_length); + Ok(output) + } +} + +pub(crate) struct HtmlFinishContext<'a> { + pub response_status: StatusCode, + pub services: &'a RuntimeServices, + pub deadline: &'a StreamFinishDeadline, + pub timer: &'a mut Option>>, + pub limit: usize, + pub marker: Option<&'a [u8]>, +} + +fn marker_count(prefix: &[u8], bytes: &[u8], marker: &[u8]) -> usize { + if marker.is_empty() { + return 0; + } + let ordinary = bytes + .windows(marker.len()) + .filter(|window| *window == marker) + .count(); + let mut boundary = prefix.to_vec(); + boundary.extend_from_slice(&bytes[..bytes.len().min(marker.len().saturating_sub(1))]); + ordinary + + boundary + .windows(marker.len()) + .enumerate() + .filter(|(start, window)| *start < prefix.len() && *window == marker) + .count() +} + +async fn finish_output( + stage: &mut HtmlProcessorStage, + fallback: Vec, + context: &mut HtmlFinishContext<'_>, +) -> Vec { + if stage.skip_async || !stage.async_capable { + return fallback; + } + if fallback.len() > context.limit { + log::warn!( + "HTML EOF processor {} skips optional I/O for {} bytes exceeding the {}-byte limit", + stage.integration_id, + fallback.len(), + context.limit + ); + return fallback; + } + if context.deadline.is_expired_at(Instant::now()) { + log::debug!( + "HTML EOF processor {} skips optional I/O after deadline expiry", + stage.integration_id + ); + return fallback; + } + let mut candidate = StreamOutputWriter::new(context.limit); + let result = { + let Some(work) = stage.processor.async_finish(StreamFinishInput { + response_status: context.response_status, + output: &fallback, + replacement: &mut candidate, + services: context.services, + deadline: context.deadline, + }) else { + return fallback; + }; + let work = work.fuse(); + futures::pin_mut!(work); + if context.deadline.is_expired_at(Instant::now()) { + None + } else { + if context.timer.is_none() + && let Some(timer) = context.services.timer() + { + *context.timer = Some(timer.delay(context.deadline.remaining()).fuse()); + } + if let Some(mut timer) = context.timer.as_mut() { + futures::select_biased! { + _ = timer => None, + result = work => Some(result), + } + } else { + Some(work.await) + } + } + }; + if result.is_none() { + context.deadline.mark_expired(); + } + if result.is_none() || context.deadline.is_expired_at(Instant::now()) { + log::warn!( + "HTML EOF processor {} times out after {} ms; retains {} bytes", + stage.integration_id, + context.deadline.elapsed().as_millis(), + fallback.len() + ); + return fallback; + } + if let Some(Err(error)) = &result { + let outcome = match error.current_context() { + StreamFinishError::Unavailable => "unavailable", + StreamFinishError::Rejected => "rejected", + StreamFinishError::OutputLimitExceeded { .. } => "output_limit", + }; + log::warn!( + "HTML EOF processor {} retains input: {outcome}", + stage.integration_id + ); + } + if !matches!(result, Some(Ok(StreamFinish::Replace))) || candidate.is_poisoned() { + return fallback; + } + let replacement = candidate.into_bytes(); + if context.marker.is_some_and(|marker| { + marker_count(&stage.previous_suffix, &fallback, marker) + != marker_count(&stage.previous_suffix, &replacement, marker) + }) { + log::warn!( + "HTML EOF processor {} retains input after invalid assembly markers", + stage.integration_id + ); + return fallback; + } + if context.deadline.is_expired_at(Instant::now()) { + fallback + } else { + replacement + } +} + +impl HtmlWithStreamingProcessors { + pub(crate) fn from_stages( + inner: Box, + stages: Vec, + ) -> Self { + Self { + inner, + stages, + finish_index: None, + marker_length: 0, + } + } + + pub(crate) fn validate_capabilities(&self) -> io::Result<()> { + for stage in &self.stages { + stage.validate()?; + } + Ok(()) + } + + pub(crate) fn set_finish_marker(&mut self, marker: Option<&[u8]>) { + self.marker_length = marker.map_or(0, <[u8]>::len); + } + + pub(crate) fn next_finish_requires_async(&self) -> bool { + self.finish_index + .and_then(|index| self.stages.get(index)) + .is_some_and(|stage| stage.async_capable && !stage.skip_async) + } + + pub(crate) fn begin_finish(&mut self) -> io::Result> { + self.validate_capabilities()?; + if self.finish_index.is_some() { + return Err(io::Error::other("HTML parser already receives EOF")); + } + let mut output = self.inner.process_chunk(&[], true)?; + for stage in &mut self.stages { + output = stage.process(&output, false, self.marker_length)?; + } + self.finish_index = Some(0); + Ok(output) + } + + pub(crate) async fn finish_next( + &mut self, + mut context: HtmlFinishContext<'_>, + ) -> io::Result>> { + let index = self + .finish_index + .ok_or_else(|| io::Error::other("HTML parser must finish before processor EOF"))?; + let Some(stage) = self.stages.get_mut(index) else { + return Ok(None); + }; + let fallback = stage.output(&[], true)?; + let mut output = finish_output(stage, fallback, &mut context).await; + stage.remember_suffix(&output, self.marker_length); + for downstream in &mut self.stages[index + 1..] { + output = downstream.process(&output, false, self.marker_length)?; + } + self.finish_index = Some(index + 1); + Ok(Some(output)) + } +} + +#[cfg(test)] +mod tests { + use std::cell::{Cell, RefCell}; + use std::io; + use std::rc::Rc; + #[cfg(not(target_arch = "wasm32"))] + use std::sync::{Arc, Mutex}; + use std::time::Duration; + + use error_stack::Report; + use futures::executor::block_on; + use futures::future::{LocalBoxFuture, poll_fn}; + use http::StatusCode; + use std::task::Poll; + + use super::*; + use crate::html_processor::HtmlWithStreamingProcessors; + use crate::platform::test_support::noop_services; + #[cfg(not(target_arch = "wasm32"))] + use crate::platform::test_support::{ + NoopBackend, NoopConfigStore, NoopGeo, NoopHttpClient, NoopSecretStore, + }; + #[cfg(not(target_arch = "wasm32"))] + use crate::platform::{PlatformTimer, RuntimeServices}; + use crate::streaming_processor::StreamProcessor; + use crate::streaming_processor::finish::{ + StreamFinish, StreamFinishDeadline, StreamFinishError, StreamFinishInput, + }; + + struct Stage { + name: &'static str, + tail: &'static [u8], + replacement: Option<&'static [u8]>, + events: Rc>>, + calls: Rc>, + } + + impl StreamProcessor for Stage { + fn process_chunk(&mut self, chunk: &[u8], last: bool) -> io::Result> { + if last { + self.events.borrow_mut().push(format!("{}:eof", self.name)); + Ok(self.tail.to_vec()) + } else { + if !chunk.is_empty() { + self.events.borrow_mut().push(format!( + "{}:chunk:{}", + self.name, + String::from_utf8_lossy(chunk) + )); + } + if self.name == "C" { + Ok(chunk.to_ascii_uppercase()) + } else { + Ok(chunk.to_vec()) + } + } + } + + fn requires_async_finish(&self) -> bool { + self.replacement.is_some() + } + + fn async_finish<'a>( + &'a mut self, + input: StreamFinishInput<'a>, + ) -> Option>>> { + let replacement = self.replacement?; + Some(Box::pin(async move { + self.calls.set(self.calls.get() + 1); + self.events + .borrow_mut() + .push(format!("{}:async", self.name)); + input.replacement.append(replacement)?; + Ok(StreamFinish::Replace) + })) + } + } + + fn stage( + name: &'static str, + tail: &'static [u8], + replacement: Option<&'static [u8]>, + events: &Rc>>, + calls: &Rc>, + ) -> Box { + Box::new(Stage { + name, + tail, + replacement, + events: Rc::clone(events), + calls: Rc::clone(calls), + }) + } + + #[test] + fn async_eof_output_reaches_downstream_before_its_eof() { + let events = Rc::new(RefCell::new(Vec::new())); + let calls = Rc::new(Cell::new(0)); + let mut chain = HtmlWithStreamingProcessors::from_stages( + stage("parser", b"parser-tail", None, &events, &calls), + vec![ + HtmlProcessorStage::new( + "B", + stage("B", b"old", Some(b"late"), &events, &calls), + true, + false, + 64, + ), + HtmlProcessorStage::new( + "C", + stage("C", b"end", None, &events, &calls), + false, + false, + 64, + ), + ], + ); + assert_eq!( + chain + .process_chunk(b"page", false) + .expect("should stream ordinary bytes"), + b"PAGE" + ); + assert_eq!( + chain.begin_finish().expect("should release parser tail"), + b"PARSER-TAIL" + ); + assert_eq!( + calls.get(), + 0, + "should expose ready prefix before optional I/O" + ); + let services = noop_services(); + let deadline = StreamFinishDeadline::new(Duration::from_secs(1), false); + let mut timer = None; + let first = block_on(chain.finish_next(HtmlFinishContext { + response_status: StatusCode::OK, + services: &services, + deadline: &deadline, + timer: &mut timer, + limit: 64, + marker: None, + })) + .expect("should finish upstream stage") + .expect("should produce a stage result"); + assert_eq!(first, b"LATE", "should transform async output downstream"); + assert!( + !events.borrow().iter().any(|event| event == "C:eof"), + "should not close downstream early" + ); + let second = block_on(chain.finish_next(HtmlFinishContext { + response_status: StatusCode::OK, + services: &services, + deadline: &deadline, + timer: &mut timer, + limit: 64, + marker: None, + })) + .expect("should finish downstream stage") + .expect("should return its tail"); + assert_eq!(second, b"end"); + for name in ["parser", "B", "C"] { + assert_eq!( + events + .borrow() + .iter() + .filter(|event| **event == format!("{name}:eof")) + .count(), + 1, + "should finish every stage once" + ); + } + } + + #[test] + fn async_eof_buffer_overflow_replays_once_and_skips_io() { + let events = Rc::new(RefCell::new(Vec::new())); + let calls = Rc::new(Cell::new(0)); + let mut chain = HtmlWithStreamingProcessors::from_stages( + stage("parser", b"", None, &events, &calls), + vec![HtmlProcessorStage::new( + "B", + stage("B", b"end", Some(b"wrong"), &events, &calls), + true, + true, + 4, + )], + ); + assert!( + chain + .process_chunk(b"page", false) + .expect("should hold under cap") + .is_empty() + ); + assert_eq!( + chain + .process_chunk(b"!", false) + .expect("should replay cap crossing"), + b"page!" + ); + assert_eq!( + chain + .process_chunk(b"next", false) + .expect("should continue delivery"), + b"next" + ); + assert!( + chain + .begin_finish() + .expect("should finish parser") + .is_empty() + ); + let services = noop_services(); + let deadline = StreamFinishDeadline::new(Duration::from_secs(1), false); + let mut timer = None; + let tail = block_on(chain.finish_next(HtmlFinishContext { + response_status: StatusCode::OK, + services: &services, + deadline: &deadline, + timer: &mut timer, + limit: 4, + marker: None, + })) + .expect("should finish bypassed stage") + .expect("should emit sync tail"); + assert_eq!(tail, b"end"); + assert_eq!( + calls.get(), + 0, + "should skip optional I/O after collection overflow" + ); + } + + #[test] + fn async_eof_rejects_marker_removal_across_streamed_boundary() { + let events = Rc::new(RefCell::new(Vec::new())); + let calls = Rc::new(Cell::new(0)); + let mut chain = HtmlWithStreamingProcessors::from_stages( + stage("parser", b"", None, &events, &calls), + vec![HtmlProcessorStage::new( + "B", + stage("B", b"EN", Some(b"wrong"), &events, &calls), + true, + false, + 64, + )], + ); + chain.set_finish_marker(Some(b"TOKEN")); + assert_eq!( + chain + .process_chunk(b"TOK", false) + .expect("should stream prefix"), + b"TOK" + ); + chain.begin_finish().expect("should finish parser"); + let services = noop_services(); + let deadline = StreamFinishDeadline::new(Duration::from_secs(1), false); + let mut timer = None; + let tail = block_on(chain.finish_next(HtmlFinishContext { + response_status: StatusCode::OK, + services: &services, + deadline: &deadline, + timer: &mut timer, + limit: 64, + marker: Some(b"TOKEN"), + })) + .expect("should retain marker fallback") + .expect("should return EOF output"); + assert_eq!( + tail, b"EN", + "should retain a marker completed across the earlier prefix" + ); + } + + fn finish_marker_stage(chain: &mut HtmlWithStreamingProcessors, limit: usize) -> Vec { + let services = noop_services(); + let deadline = StreamFinishDeadline::new(Duration::from_secs(1), false); + let mut timer = None; + block_on(chain.finish_next(HtmlFinishContext { + response_status: StatusCode::OK, + services: &services, + deadline: &deadline, + timer: &mut timer, + limit, + marker: Some(b"TOKEN"), + })) + .expect("should finish the marker-aware stage") + .expect("should return stage EOF output") + } + + #[test] + fn async_eof_rejects_marker_creation_across_streamed_boundary() { + let events = Rc::new(RefCell::new(Vec::new())); + let calls = Rc::new(Cell::new(0)); + let mut chain = HtmlWithStreamingProcessors::from_stages( + stage("parser", b"", None, &events, &calls), + vec![HtmlProcessorStage::new( + "B", + stage("B", b"ordinary", Some(b"EN"), &events, &calls), + true, + false, + 64, + )], + ); + chain.set_finish_marker(Some(b"TOKEN")); + assert_eq!( + chain + .process_chunk(b"TOK", false) + .expect("should stream the prefix"), + b"TOK", + "should release ordinary output before EOF" + ); + chain.begin_finish().expect("should finish the parser"); + assert_eq!( + finish_marker_stage(&mut chain, 64), + b"ordinary", + "should reject a candidate that creates a token across emitted output" + ); + assert_eq!(calls.get(), 1, "should evaluate the candidate once"); + } + + #[test] + fn async_eof_retained_output_preserves_a_marker_completed_at_eof() { + let events = Rc::new(RefCell::new(Vec::new())); + let calls = Rc::new(Cell::new(0)); + let mut chain = HtmlWithStreamingProcessors::from_stages( + stage("parser", b"", None, &events, &calls), + vec![HtmlProcessorStage::new( + "B", + stage("B", b"EN", Some(b"wrong"), &events, &calls), + true, + true, + 64, + )], + ); + chain.set_finish_marker(Some(b"TOKEN")); + assert!( + chain + .process_chunk(b"TOK", false) + .expect("should retain the prefix") + .is_empty(), + "should withhold a declaring stage's output" + ); + chain.begin_finish().expect("should finish the parser"); + assert_eq!( + finish_marker_stage(&mut chain, 64), + b"TOKEN", + "should retain the complete collected fallback when replacement removes its marker" + ); + assert_eq!(calls.get(), 1, "should evaluate retained output once"); + } + + #[test] + fn async_eof_overflow_replays_a_split_marker_and_skips_invalid_replacement() { + let events = Rc::new(RefCell::new(Vec::new())); + let calls = Rc::new(Cell::new(0)); + let mut chain = HtmlWithStreamingProcessors::from_stages( + stage("parser", b"", None, &events, &calls), + vec![HtmlProcessorStage::new( + "B", + stage("B", b"!", Some(b"wrong"), &events, &calls), + true, + true, + 4, + )], + ); + chain.set_finish_marker(Some(b"TOKEN")); + assert!( + chain + .process_chunk(b"TOK", false) + .expect("should retain the prefix") + .is_empty() + ); + assert_eq!( + chain + .process_chunk(b"EN", false) + .expect("should replay cap crossing"), + b"TOKEN", + "should replay the split marker exactly once" + ); + chain.begin_finish().expect("should finish the parser"); + assert_eq!( + finish_marker_stage(&mut chain, 4), + b"!", + "should emit synchronous EOF after overflow" + ); + assert_eq!( + calls.get(), + 0, + "should never start the invalid optional replacement after overflow" + ); + assert_eq!( + events + .borrow() + .iter() + .filter(|event| event.as_str() == "B:eof") + .count(), + 1, + "should finish the bypassed stage exactly once" + ); + } + + #[test] + fn async_eof_upstream_replacement_updates_downstream_marker_boundary() { + let events = Rc::new(RefCell::new(Vec::new())); + let calls = Rc::new(Cell::new(0)); + let mut chain = HtmlWithStreamingProcessors::from_stages( + stage("parser", b"", None, &events, &calls), + vec![ + HtmlProcessorStage::new( + "A", + stage("A", b"old", Some(b"TOK"), &events, &calls), + true, + false, + 64, + ), + HtmlProcessorStage::new( + "B", + stage("B", b"EN", Some(b"wrong"), &events, &calls), + true, + false, + 64, + ), + ], + ); + chain.set_finish_marker(Some(b"TOKEN")); + chain.begin_finish().expect("should finish the parser"); + assert_eq!( + finish_marker_stage(&mut chain, 64), + b"TOK", + "should accept upstream ordinary output" + ); + assert_eq!( + finish_marker_stage(&mut chain, 64), + b"EN", + "should preserve the downstream token completed from accepted upstream output" + ); + assert_eq!(calls.get(), 2, "should evaluate both stage candidates once"); + } + + struct ResettableStage(usize); + impl StreamProcessor for ResettableStage { + fn process_chunk(&mut self, _: &[u8], _: bool) -> io::Result> { + self.0 += 1; + Ok(vec![ + u8::try_from(self.0).expect("should keep test counts in a byte"), + ]) + } + fn reset(&mut self) { + self.0 = 0; + } + } + + #[test] + fn async_eof_revision_preserves_synchronous_reset_forwarding() { + let mut chain = HtmlWithStreamingProcessors::from_stages( + Box::new(ResettableStage(0)), + vec![HtmlProcessorStage::new( + "reset", + Box::new(ResettableStage(0)), + false, + false, + 8, + )], + ); + assert_eq!( + chain + .process_chunk(b"page", false) + .expect("should process the first document"), + vec![1], + "should start with fresh state" + ); + chain.reset(); + assert_eq!( + chain + .process_chunk(b"next", false) + .expect("should process after reset"), + vec![1], + "should reset parser and processor state" + ); + } + + #[cfg(not(target_arch = "wasm32"))] + struct RecordingTimer(Arc>>); + #[cfg(not(target_arch = "wasm32"))] + impl PlatformTimer for RecordingTimer { + fn delay(&self, duration: Duration) -> LocalBoxFuture<'static, ()> { + self.0 + .lock() + .expect("should lock recorded timer requests") + .push(duration); + Box::pin(futures::future::pending()) + } + } + + #[cfg(not(target_arch = "wasm32"))] + #[test] + fn async_eof_timer_uses_remaining_budget_when_first_callback_starts() { + let requested = Arc::new(Mutex::new(Vec::new())); + let services = RuntimeServices::builder() + .config_store(Arc::new(NoopConfigStore)) + .secret_store(Arc::new(NoopSecretStore)) + .kv_store(Arc::new(edgezero_core::key_value_store::NoopKvStore)) + .backend(Arc::new(NoopBackend)) + .http_client(Arc::new(NoopHttpClient)) + .geo(Arc::new(NoopGeo)) + .client_info(crate::platform::ClientInfo::default()) + .timer(Arc::new(RecordingTimer(Arc::clone(&requested)))) + .build(); + let events = Rc::new(RefCell::new(Vec::new())); + let calls = Rc::new(Cell::new(0)); + let mut chain = HtmlWithStreamingProcessors::from_stages( + stage("parser", b"ready", None, &events, &calls), + vec![HtmlProcessorStage::new( + "B", + stage("B", b"tail", Some(b"late"), &events, &calls), + true, + false, + 64, + )], + ); + let deadline = StreamFinishDeadline::new(Duration::from_secs(5), true); + chain + .begin_finish() + .expect("should make parser output ready"); + std::thread::sleep(Duration::from_millis(20)); + let mut timer = None; + block_on(chain.finish_next(HtmlFinishContext { + response_status: StatusCode::OK, + services: &services, + deadline: &deadline, + timer: &mut timer, + limit: 64, + marker: None, + })) + .expect("should finish a callback after ready-prefix delivery"); + let requested = requested + .lock() + .expect("should read recorded timer request"); + assert_eq!( + requested.len(), + 1, + "should initialize one cooperative timer at first callback" + ); + assert!( + requested[0] <= Duration::from_millis(4980), + "should subtract time already spent before callback polling" + ); + } + + struct PendingFinish { + tail: Vec, + dropped: Rc>, + polls: Rc>, + fallback_pointer: Rc>, + } + struct BorrowProbe<'a> { + _bytes: &'a [u8], + dropped: Rc>, + } + impl Drop for BorrowProbe<'_> { + fn drop(&mut self) { + self.dropped.set(true); + } + } + impl StreamProcessor for PendingFinish { + fn process_chunk(&mut self, chunk: &[u8], last: bool) -> io::Result> { + if last { + Ok(std::mem::take(&mut self.tail)) + } else { + Ok(chunk.to_vec()) + } + } + fn requires_async_finish(&self) -> bool { + true + } + fn async_finish<'a>( + &'a mut self, + input: StreamFinishInput<'a>, + ) -> Option>>> { + Some(Box::pin(async move { + let _probe = BorrowProbe { + _bytes: input.output, + dropped: Rc::clone(&self.dropped), + }; + self.fallback_pointer.set(input.output.as_ptr() as usize); + self.polls.set(self.polls.get() + 1); + input.replacement.append(b"partial")?; + futures::future::pending().await + })) + } + } + + fn timer_after_callback_poll() -> Option>> { + let mut polled = false; + Some( + Box::pin(poll_fn(move |context| { + if polled { + Poll::Ready(()) + } else { + polled = true; + context.waker().wake_by_ref(); + Poll::Pending + } + })) + .boxed_local() + .fuse(), + ) + } + + #[test] + fn async_eof_timeout_drops_borrow_and_preserves_fallback_allocation() { + let events = Rc::new(RefCell::new(Vec::new())); + let calls = Rc::new(Cell::new(0)); + let dropped = Rc::new(Cell::new(false)); + let pointer = Rc::new(Cell::new(0)); + let mut chain = HtmlWithStreamingProcessors::from_stages( + stage("parser", b"", None, &events, &calls), + vec![HtmlProcessorStage::new( + "pending", + Box::new(PendingFinish { + tail: b"held".to_vec(), + dropped: Rc::clone(&dropped), + polls: Rc::clone(&calls), + fallback_pointer: Rc::clone(&pointer), + }), + true, + false, + 64, + )], + ); + chain.begin_finish().expect("should finish parser"); + let services = noop_services(); + let deadline = StreamFinishDeadline::new(Duration::from_secs(1), true); + let mut timer = timer_after_callback_poll(); + let output = block_on(chain.finish_next(HtmlFinishContext { + response_status: StatusCode::OK, + services: &services, + deadline: &deadline, + timer: &mut timer, + limit: 64, + marker: None, + })) + .expect("should retain output on timeout") + .expect("should return held output"); + assert_eq!(output, b"held", "should discard partial candidate"); + assert_eq!( + output.as_ptr() as usize, + pointer.get(), + "should keep the original allocation outside the dropped future" + ); + assert!( + dropped.get(), + "should drop callback borrows before selecting fallback" + ); + assert_eq!(calls.get(), 1, "should poll the pending callback"); + } + + #[test] + fn async_eof_timer_expiry_skips_later_io_but_completes_downstream_eof() { + let events = Rc::new(RefCell::new(Vec::new())); + let calls = Rc::new(Cell::new(0)); + let mut chain = HtmlWithStreamingProcessors::from_stages( + stage("parser", b"", None, &events, &calls), + vec![ + HtmlProcessorStage::new( + "pending", + Box::new(PendingFinish { + tail: b"held".to_vec(), + dropped: Rc::new(Cell::new(false)), + polls: Rc::clone(&calls), + fallback_pointer: Rc::new(Cell::new(0)), + }), + true, + false, + 64, + ), + HtmlProcessorStage::new( + "C", + stage("C", b"ctail", Some(b"wrong"), &events, &calls), + true, + false, + 64, + ), + ], + ); + chain.begin_finish().expect("should finish parser"); + let services = noop_services(); + let deadline = StreamFinishDeadline::new(Duration::from_secs(1), true); + let mut timer = timer_after_callback_poll(); + let first = block_on(chain.finish_next(HtmlFinishContext { + response_status: StatusCode::OK, + services: &services, + deadline: &deadline, + timer: &mut timer, + limit: 64, + marker: None, + })) + .expect("should finish timed-out upstream") + .expect("should return transformed fallback"); + assert_eq!( + first, b"HELD", + "should send fallback downstream as normal input" + ); + let second = block_on(chain.finish_next(HtmlFinishContext { + response_status: StatusCode::OK, + services: &services, + deadline: &deadline, + timer: &mut timer, + limit: 64, + marker: None, + })) + .expect("should complete downstream EOF") + .expect("should return its synchronous tail"); + assert_eq!( + second, b"ctail", + "should skip later optional I/O after timer expiry" + ); + assert_eq!(calls.get(), 1, "should not poll a later optional callback"); + assert_eq!( + events + .borrow() + .iter() + .filter(|event| event.as_str() == "C:eof") + .count(), + 1, + "should complete downstream synchronous EOF once" + ); + } + + #[derive(Clone, Copy)] + enum LeafOutcome { + Error, + PoisonedReplacement, + LateReplacement, + #[cfg(not(target_arch = "wasm32"))] + ElapsedReplacement, + EchoReplacement, + EmptyReplacement, + Replacement, + } + + #[derive(Clone, Default)] + struct LeafObservation { + calls: Rc>, + eof_calls: Rc>, + fallback_pointer: Rc>, + dropped: Rc>, + } + + struct LeafProbe { + tail: Vec, + outcome: LeafOutcome, + observation: LeafObservation, + } + + impl StreamProcessor for LeafProbe { + fn process_chunk(&mut self, chunk: &[u8], last: bool) -> io::Result> { + if last { + self.observation + .eof_calls + .set(self.observation.eof_calls.get() + 1); + Ok(std::mem::take(&mut self.tail)) + } else { + Ok(chunk.to_vec()) + } + } + + fn requires_async_finish(&self) -> bool { + true + } + + fn async_finish<'a>( + &'a mut self, + input: StreamFinishInput<'a>, + ) -> Option>>> { + Some(Box::pin(async move { + let _probe = BorrowProbe { + _bytes: input.output, + dropped: Rc::clone(&self.observation.dropped), + }; + self.observation.calls.set(self.observation.calls.get() + 1); + self.observation + .fallback_pointer + .set(input.output.as_ptr() as usize); + match self.outcome { + LeafOutcome::Error => { + input.replacement.append(b"partial")?; + Err(Report::new(StreamFinishError::Unavailable)) + } + LeafOutcome::PoisonedReplacement => { + input.replacement.append(b"partial")?; + let _ = input + .replacement + .append(&[b'x'; 65]) + .expect_err("should poison output beyond the 64-byte cap"); + Ok(StreamFinish::Replace) + } + LeafOutcome::LateReplacement => { + input.replacement.append(b"candidate")?; + input.deadline.mark_expired(); + Ok(StreamFinish::Replace) + } + #[cfg(not(target_arch = "wasm32"))] + LeafOutcome::ElapsedReplacement => { + std::thread::sleep(input.deadline.remaining() + Duration::from_millis(5)); + input.replacement.append(b"candidate")?; + Ok(StreamFinish::Replace) + } + LeafOutcome::EchoReplacement => { + input.replacement.append(input.output)?; + Ok(StreamFinish::Replace) + } + LeafOutcome::EmptyReplacement => Ok(StreamFinish::Replace), + LeafOutcome::Replacement => { + input.replacement.append(b"candidate")?; + Ok(StreamFinish::Replace) + } + } + })) + } + } + + fn complete_leaf_probe( + outcome: LeafOutcome, + mut timer: Option>>, + ) -> (Vec, LeafObservation, usize) { + let events = Rc::new(RefCell::new(Vec::new())); + let calls = Rc::new(Cell::new(0)); + let observation = LeafObservation::default(); + let fallback = b"fallback".to_vec(); + let fallback_pointer = fallback.as_ptr() as usize; + let mut chain = HtmlWithStreamingProcessors::from_stages( + stage("parser", b"", None, &events, &calls), + vec![HtmlProcessorStage::new( + "probe", + Box::new(LeafProbe { + tail: fallback, + outcome, + observation: observation.clone(), + }), + true, + false, + 64, + )], + ); + chain.begin_finish().expect("should finish the parser"); + let services = noop_services(); + let deadline = StreamFinishDeadline::new(Duration::from_secs(30), timer.is_some()); + let output = block_on(chain.finish_next(HtmlFinishContext { + response_status: StatusCode::OK, + services: &services, + deadline: &deadline, + timer: &mut timer, + limit: 64, + marker: None, + })) + .expect("should complete synchronous EOF despite optional failure") + .expect("should return the selected stage output"); + assert_eq!( + observation.eof_calls.get(), + 1, + "should complete the processor EOF exactly once" + ); + (output, observation, fallback_pointer) + } + + #[test] + fn async_eof_callback_error_discards_partial_candidate_without_copying_fallback() { + let (output, observation, fallback_pointer) = complete_leaf_probe(LeafOutcome::Error, None); + assert_eq!(output, b"fallback", "should discard the partial candidate"); + assert_eq!( + output.as_ptr() as usize, + fallback_pointer, + "should retain the driver-owned fallback allocation" + ); + assert_eq!( + observation.fallback_pointer.get(), + fallback_pointer, + "should lend the original fallback to optional work" + ); + assert!( + observation.dropped.get(), + "should drop callback borrows before returning fallback" + ); + assert_eq!(observation.calls.get(), 1, "should run optional work once"); + } + + #[test] + fn async_eof_poisoned_candidate_cannot_commit_even_when_callback_returns_replace() { + let (output, observation, fallback_pointer) = + complete_leaf_probe(LeafOutcome::PoisonedReplacement, None); + assert_eq!( + output, b"fallback", + "should reject partial output after an ignored append error" + ); + assert_eq!( + output.as_ptr() as usize, + fallback_pointer, + "should retain the original fallback allocation after poisoning" + ); + assert!(observation.dropped.get(), "should release callback borrows"); + assert_eq!(observation.calls.get(), 1, "should run optional work once"); + } + + #[test] + fn async_eof_timer_wins_when_timer_and_replacement_are_both_ready() { + let timer = Some(futures::future::ready(()).boxed_local().fuse()); + let (output, observation, fallback_pointer) = + complete_leaf_probe(LeafOutcome::Replacement, timer); + assert_eq!( + output, b"fallback", + "should prefer expiry over ready output" + ); + assert_eq!( + output.as_ptr() as usize, + fallback_pointer, + "should preserve fallback allocation when the timer wins" + ); + assert_eq!( + observation.calls.get(), + 0, + "should not poll callback work when the timer is already ready" + ); + } + + #[test] + fn async_eof_replacement_completed_after_expiry_retains_original_fallback() { + let (output, observation, fallback_pointer) = + complete_leaf_probe(LeafOutcome::LateReplacement, None); + assert_eq!( + output, b"fallback", + "should discard a late successful result" + ); + assert_eq!( + output.as_ptr() as usize, + fallback_pointer, + "should preserve the original fallback after late completion" + ); + assert!( + observation.dropped.get(), + "should drop the completed callback before selecting fallback" + ); + assert_eq!( + observation.calls.get(), + 1, + "should exercise expiry during callback execution" + ); + } + + #[test] + fn async_eof_empty_replace_intentionally_removes_nonempty_fallback() { + let (output, observation, _) = complete_leaf_probe(LeafOutcome::EmptyReplacement, None); + assert!( + output.is_empty(), + "should accept an empty replacement instead of retaining fallback" + ); + assert_eq!(observation.calls.get(), 1, "should run optional work once"); + assert!(observation.dropped.get(), "should release callback borrows"); + } + + #[test] + fn async_eof_failed_later_stage_preserves_accepted_upstream_replacement() { + let events = Rc::new(RefCell::new(Vec::new())); + let upstream_calls = Rc::new(Cell::new(0)); + let observation = LeafObservation::default(); + let mut chain = HtmlWithStreamingProcessors::from_stages( + stage("parser", b"", None, &events, &upstream_calls), + vec![ + HtmlProcessorStage::new( + "upstream", + stage("A", b"old", Some(b"accepted"), &events, &upstream_calls), + true, + false, + 64, + ), + HtmlProcessorStage::new( + "later", + Box::new(LeafProbe { + tail: b"tail".to_vec(), + outcome: LeafOutcome::Error, + observation: observation.clone(), + }), + true, + true, + 64, + ), + ], + ); + assert!( + chain + .process_chunk(b"page", false) + .expect("should process ordinary input") + .is_empty(), + "should retain only the explicitly buffered downstream stage" + ); + chain.begin_finish().expect("should finish the parser"); + let services = noop_services(); + let deadline = StreamFinishDeadline::new(Duration::from_secs(30), false); + let mut timer = None; + let first = block_on(chain.finish_next(HtmlFinishContext { + response_status: StatusCode::OK, + services: &services, + deadline: &deadline, + timer: &mut timer, + limit: 64, + marker: None, + })) + .expect("should commit the upstream replacement") + .expect("should return the downstream stage's ready output"); + assert!( + first.is_empty(), + "should deliver accepted upstream bytes into downstream retention" + ); + assert_eq!(upstream_calls.get(), 1, "should finish upstream I/O once"); + assert_eq!( + observation.eof_calls.get(), + 0, + "should preserve downstream EOF ordering" + ); + let output = block_on(chain.finish_next(HtmlFinishContext { + response_status: StatusCode::OK, + services: &services, + deadline: &deadline, + timer: &mut timer, + limit: 64, + marker: None, + })) + .expect("should retain downstream fallback on optional failure") + .expect("should release downstream retained output"); + assert_eq!( + output, b"pageacceptedtail", + "should retain accepted upstream output while discarding the later candidate" + ); + assert_eq!( + output.as_ptr() as usize, + observation.fallback_pointer.get(), + "should move the retained fallback without a rollback copy" + ); + assert_eq!(observation.calls.get(), 1, "should run later I/O once"); + assert_eq!( + observation.eof_calls.get(), + 1, + "should finish later EOF once" + ); + assert!(observation.dropped.get(), "should release callback borrows"); + } + + #[cfg(not(target_arch = "wasm32"))] + #[test] + fn async_eof_timerless_callback_elapsed_past_deadline_preserves_fallback() { + let events = Rc::new(RefCell::new(Vec::new())); + let calls = Rc::new(Cell::new(0)); + let observation = LeafObservation::default(); + let fallback = b"fallback".to_vec(); + let fallback_pointer = fallback.as_ptr() as usize; + let mut chain = HtmlWithStreamingProcessors::from_stages( + stage("parser", b"", None, &events, &calls), + vec![HtmlProcessorStage::new( + "elapsed", + Box::new(LeafProbe { + tail: fallback, + outcome: LeafOutcome::ElapsedReplacement, + observation: observation.clone(), + }), + true, + false, + 64, + )], + ); + chain.begin_finish().expect("should finish the parser"); + let services = noop_services(); + let deadline = StreamFinishDeadline::new(Duration::from_millis(100), false); + let mut timer = None; + let output = block_on(chain.finish_next(HtmlFinishContext { + response_status: StatusCode::OK, + services: &services, + deadline: &deadline, + timer: &mut timer, + limit: 64, + marker: None, + })) + .expect("should finish timerless work that completes late") + .expect("should retain synchronous output"); + assert_eq!( + observation.calls.get(), + 1, + "should exercise callback execution rather than pre-poll expiry" + ); + assert!( + deadline.elapsed() >= Duration::from_millis(100), + "should cross the actual wall-clock deadline" + ); + assert_eq!(output, b"fallback", "should reject the late replacement"); + assert_eq!( + output.as_ptr() as usize, + fallback_pointer, + "should preserve the original fallback allocation" + ); + assert!(observation.dropped.get(), "should release callback borrows"); + } + + #[test] + fn async_eof_oversized_streaming_fallback_skips_io_without_truncation_or_copy() { + let events = Rc::new(RefCell::new(Vec::new())); + let calls = Rc::new(Cell::new(0)); + let observation = LeafObservation::default(); + let fallback = vec![b'x'; 65]; + let fallback_pointer = fallback.as_ptr() as usize; + let mut chain = HtmlWithStreamingProcessors::from_stages( + stage("parser", b"", None, &events, &calls), + vec![HtmlProcessorStage::new( + "oversized", + Box::new(LeafProbe { + tail: fallback, + outcome: LeafOutcome::Replacement, + observation: observation.clone(), + }), + true, + false, + 64, + )], + ); + chain.begin_finish().expect("should finish the parser"); + let services = noop_services(); + let deadline = StreamFinishDeadline::new(Duration::from_secs(30), false); + let mut timer = None; + let output = block_on(chain.finish_next(HtmlFinishContext { + response_status: StatusCode::OK, + services: &services, + deadline: &deadline, + timer: &mut timer, + limit: 64, + marker: None, + })) + .expect("should deliver an oversized synchronous tail") + .expect("should return the complete tail"); + assert_eq!( + output.as_slice(), + &[b'x'; 65], + "should not truncate fallback" + ); + assert_eq!( + output.as_ptr() as usize, + fallback_pointer, + "should deliver the original allocation" + ); + assert_eq!(observation.calls.get(), 0, "should skip optional I/O"); + assert_eq!( + observation.eof_calls.get(), + 1, + "should finish synchronous EOF once" + ); + } + + #[test] + fn async_eof_small_chunks_keep_a_bounded_marker_suffix() { + let events = Rc::new(RefCell::new(Vec::new())); + let calls = Rc::new(Cell::new(0)); + let mut chain = HtmlWithStreamingProcessors::from_stages( + stage("parser", b"", None, &events, &calls), + vec![HtmlProcessorStage::new( + "B", + stage("B", b"EN", Some(b"wrong"), &events, &calls), + true, + false, + 64, + )], + ); + chain.set_finish_marker(Some(b"TOKEN")); + for chunk in [b"xxxx".as_slice(), b"TO", b"K"] { + assert_eq!( + chain + .process_chunk(chunk, false) + .expect("should stream each small chunk"), + chunk, + "should preserve ordinary output" + ); + assert!( + chain.stages[0].previous_suffix.len() <= 4, + "should bound marker history after every chunk" + ); + } + assert_eq!( + chain.stages[0].previous_suffix, b"xTOK", + "should retain the trailing bytes across small chunks" + ); + chain.begin_finish().expect("should finish the parser"); + assert_eq!( + finish_marker_stage(&mut chain, 64), + b"EN", + "should reject removal of a marker completed after trimmed history" + ); + assert_eq!(calls.get(), 1, "should evaluate the candidate once"); + } + + #[test] + fn async_eof_single_byte_marker_rejects_creation_and_removal() { + for (tail, replacement) in [ + (b"|".as_slice(), b"wrong".as_slice()), + (b"tail".as_slice(), b"|".as_slice()), + ] { + let events = Rc::new(RefCell::new(Vec::new())); + let calls = Rc::new(Cell::new(0)); + let mut chain = HtmlWithStreamingProcessors::from_stages( + stage("parser", b"", None, &events, &calls), + vec![HtmlProcessorStage::new( + "marker", + stage("B", tail, Some(replacement), &events, &calls), + true, + false, + 64, + )], + ); + chain.set_finish_marker(Some(b"|")); + assert_eq!( + chain + .process_chunk(b"page|", false) + .expect("should release ordinary output"), + b"page|", + "should preserve the already emitted marker" + ); + chain.begin_finish().expect("should finish the parser"); + let services = noop_services(); + let deadline = StreamFinishDeadline::new(Duration::from_secs(30), false); + let mut timer = None; + let output = block_on(chain.finish_next(HtmlFinishContext { + response_status: StatusCode::OK, + services: &services, + deadline: &deadline, + timer: &mut timer, + limit: 64, + marker: Some(b"|"), + })) + .expect("should retain output after invalid marker replacement") + .expect("should return the fallback tail"); + assert_eq!( + output, tail, + "should reject changing single-byte marker count" + ); + assert_eq!(calls.get(), 1, "should evaluate optional output once"); + } + } + + #[test] + fn async_eof_duplicate_parser_finish_rejects_before_repeating_effects() { + let events = Rc::new(RefCell::new(Vec::new())); + let calls = Rc::new(Cell::new(0)); + let mut chain = HtmlWithStreamingProcessors::from_stages( + stage("parser", b"parser-tail", None, &events, &calls), + vec![HtmlProcessorStage::new( + "B", + stage("B", b"tail", Some(b"candidate"), &events, &calls), + true, + false, + 64, + )], + ); + assert_eq!( + chain.begin_finish().expect("should finish the parser once"), + b"parser-tail", + "should release its tail once" + ); + let original_events = events.borrow().clone(); + chain + .begin_finish() + .expect_err("should reject a repeated parser EOF"); + assert_eq!( + *events.borrow(), + original_events, + "should not repeat parser or downstream chunk effects" + ); + assert_eq!( + calls.get(), + 0, + "should not start optional work at parser EOF" + ); + } + + #[test] + fn async_eof_exact_retention_and_replacement_cap_succeed_at_eof() { + let events = Rc::new(RefCell::new(Vec::new())); + let calls = Rc::new(Cell::new(0)); + let observation = LeafObservation::default(); + let mut chain = HtmlWithStreamingProcessors::from_stages( + stage("parser", b"", None, &events, &calls), + vec![HtmlProcessorStage::new( + "exact", + Box::new(LeafProbe { + tail: b"tail".to_vec(), + outcome: LeafOutcome::EchoReplacement, + observation: observation.clone(), + }), + true, + true, + 12, + )], + ); + for chunk in [b"page", b"more"] { + assert!( + chain + .process_chunk(chunk, false) + .expect("should retain ordinary output") + .is_empty(), + "should withhold output until EOF" + ); + } + chain.begin_finish().expect("should finish the parser"); + let services = noop_services(); + let deadline = StreamFinishDeadline::new(Duration::from_secs(30), false); + let mut timer = None; + let output = block_on(chain.finish_next(HtmlFinishContext { + response_status: StatusCode::OK, + services: &services, + deadline: &deadline, + timer: &mut timer, + limit: 12, + marker: None, + })) + .expect("should accept exact-cap EOF completion") + .expect("should return retained output through its replacement"); + assert_eq!( + output, b"pagemoretail", + "should include the whole retained output" + ); + assert_eq!( + observation.calls.get(), + 1, + "should not bypass exact-cap I/O" + ); + assert_ne!( + output.as_ptr() as usize, + observation.fallback_pointer.get(), + "should commit the exact-cap replacement instead of retaining fallback" + ); + assert_eq!( + observation.eof_calls.get(), + 1, + "should finish synchronous EOF once" + ); + } + + #[cfg(not(target_arch = "wasm32"))] + struct SlowConstructor(Rc>); + #[cfg(not(target_arch = "wasm32"))] + impl StreamProcessor for SlowConstructor { + fn process_chunk(&mut self, chunk: &[u8], _: bool) -> io::Result> { + Ok(chunk.to_vec()) + } + fn requires_async_finish(&self) -> bool { + true + } + fn async_finish<'a>( + &'a mut self, + _: StreamFinishInput<'a>, + ) -> Option>>> { + std::thread::sleep(Duration::from_millis(5)); + Some(Box::pin(async move { + self.0.set(self.0.get() + 1); + Ok(StreamFinish::Unchanged) + })) + } + } + + #[cfg(not(target_arch = "wasm32"))] + #[test] + fn async_eof_does_not_poll_work_after_constructor_consumes_budget() { + let events = Rc::new(RefCell::new(Vec::new())); + let calls = Rc::new(Cell::new(0)); + let mut chain = HtmlWithStreamingProcessors::from_stages( + stage("parser", b"", None, &events, &calls), + vec![HtmlProcessorStage::new( + "slow", + Box::new(SlowConstructor(Rc::clone(&calls))), + true, + false, + 64, + )], + ); + chain.begin_finish().expect("should finish the parser"); + let services = noop_services(); + let deadline = StreamFinishDeadline::new(Duration::from_millis(1), false); + let mut timer = None; + block_on(chain.finish_next(HtmlFinishContext { + response_status: StatusCode::OK, + services: &services, + deadline: &deadline, + timer: &mut timer, + limit: 64, + marker: None, + })) + .expect("should retain fallback after constructor consumes budget"); + assert_eq!( + calls.get(), + 0, + "should check deadline again before polling optional work" + ); + } +} diff --git a/crates/trusted-server-core/src/integrations/html_finalization.rs b/crates/trusted-server-core/src/integrations/html_finalization.rs new file mode 100644 index 000000000..460770197 --- /dev/null +++ b/crates/trusted-server-core/src/integrations/html_finalization.rs @@ -0,0 +1,71 @@ +//! Test-only adapter preserving existing buffered-output regression scenarios. + +use std::io; +use std::sync::Arc; + +use error_stack::Report; +use futures::future::LocalBoxFuture; + +use super::{IntegrationHtmlStreamContext, IntegrationHtmlStreamProcessorFactory}; +use crate::streaming_processor::StreamProcessor; +use crate::streaming_processor::finish::StreamFinishInput; + +pub(crate) use crate::streaming_processor::finish::{ + StreamFinish as HtmlFinalization, StreamFinishError as HtmlFinalizationError, + StreamOutputWriter as HtmlReplacementWriter, +}; + +pub(crate) trait IntegrationHtmlFinalizerFactory: Send + Sync { + fn integration_id(&self) -> &'static str; + fn create(&self, context: IntegrationHtmlStreamContext) -> Box; +} + +pub(crate) trait IntegrationHtmlFinalizer { + fn finalize<'a>( + &'a mut self, + input: HtmlFinalizationInput<'a>, + ) -> LocalBoxFuture<'a, Result>>; +} + +pub(crate) struct HtmlFinalizationInput<'a> { + pub response_status: http::StatusCode, + pub html: &'a [u8], + pub replacement: &'a mut HtmlReplacementWriter, +} + +pub(super) struct TestFinishFactory(pub Arc); +struct TestFinishProcessor(Box); + +impl IntegrationHtmlStreamProcessorFactory for TestFinishFactory { + fn integration_id(&self) -> &'static str { + self.0.integration_id() + } + fn requires_async_finish(&self) -> bool { + true + } + fn buffer_output_until_eof(&self) -> bool { + true + } + fn create(&self, context: IntegrationHtmlStreamContext) -> Box { + Box::new(TestFinishProcessor(self.0.create(context))) + } +} + +impl StreamProcessor for TestFinishProcessor { + fn process_chunk(&mut self, chunk: &[u8], _: bool) -> io::Result> { + Ok(chunk.to_vec()) + } + fn requires_async_finish(&self) -> bool { + true + } + fn async_finish<'a>( + &'a mut self, + input: StreamFinishInput<'a>, + ) -> Option>>> { + Some(self.0.finalize(HtmlFinalizationInput { + response_status: input.response_status, + html: input.output, + replacement: input.replacement, + })) + } +} diff --git a/crates/trusted-server-core/src/integrations/mod.rs b/crates/trusted-server-core/src/integrations/mod.rs index 820e2c24b..2ff464f5a 100644 --- a/crates/trusted-server-core/src/integrations/mod.rs +++ b/crates/trusted-server-core/src/integrations/mod.rs @@ -18,6 +18,8 @@ pub mod didomi; pub mod google_tag_manager; pub mod gpt; pub mod gpt_diagnostics; +#[cfg(test)] +mod html_finalization; pub mod js_asset_proxy; pub mod lockr; pub mod nextjs; @@ -28,6 +30,12 @@ mod registry; pub mod sourcepoint; pub mod testlight; +#[cfg(test)] +pub(crate) use html_finalization::{ + HtmlFinalization, HtmlFinalizationError, HtmlFinalizationInput, IntegrationHtmlFinalizer, + IntegrationHtmlFinalizerFactory, +}; + pub use registry::{ AttributeRewriteAction, AttributeRewriteOutcome, HeaderMutation, HeaderMutationMode, IntegrationAttributeContext, IntegrationAttributeRewriter, IntegrationDocumentState, @@ -296,6 +304,11 @@ pub(crate) fn builders() -> &'static [IntegrationBuilder] { id: js_asset_proxy::JS_ASSET_PROXY_INTEGRATION_ID, build: js_asset_proxy::register, }, + #[cfg(feature = "test-utils")] + IntegrationBuilder { + id: crate::test_support::async_html_finalization::ID, + build: crate::test_support::async_html_finalization::register, + }, IntegrationBuilder { id: "testlight", build: testlight::register, diff --git a/crates/trusted-server-core/src/integrations/registry.rs b/crates/trusted-server-core/src/integrations/registry.rs index d3c09cd7d..5bfdd5450 100644 --- a/crates/trusted-server-core/src/integrations/registry.rs +++ b/crates/trusted-server-core/src/integrations/registry.rs @@ -8,6 +8,9 @@ use error_stack::Report; use http::{Method, Request, Response}; use matchit::Router; +#[cfg(test)] +use super::IntegrationHtmlFinalizerFactory; + use crate::auction::AuctionPlan; use crate::constants::HEADER_X_TS_EC; use crate::ec::EcContext; @@ -601,6 +604,18 @@ pub trait IntegrationHtmlStreamProcessorFactory: Send + Sync { /// Identifier for logging and diagnostics. fn integration_id(&self) -> &'static str; + /// Declare optional asynchronous EOF capability before cache admission. + fn requires_async_finish(&self) -> bool { + false + } + + /// Declare that core must retain this stage's synchronous output until EOF. + /// + /// Requires asynchronous completion and explicit operator permission. + fn buffer_output_until_eof(&self) -> bool { + false + } + /// Create a request-local streaming processor. fn create(&self, context: IntegrationHtmlStreamContext) -> Box; } @@ -931,10 +946,12 @@ impl IntegrationRegistry { } } - Ok(Self { + let registry = Self { inner: Arc::new(inner), plan: Some(plan), - }) + }; + registry.ensure_async_html_finish_allowed(settings)?; + Ok(registry) } /// Return whether this registry and another consumer share the same plan allocation. @@ -1111,6 +1128,78 @@ impl IntegrationRegistry { self.inner.html_stream_processors.clone() } + /// Report async EOF capability before any processed-template lookup. + #[must_use] + pub fn has_async_html_finish(&self) -> bool { + self.inner + .html_stream_processors + .iter() + .any(|factory| factory.requires_async_finish()) + } + + /// Validate operator permission and factory retention declarations. + /// + /// # Errors + /// + /// Returns a configuration error for missing permission or inconsistent + /// factory metadata. Requested integration work is never silently omitted. + pub fn ensure_async_html_finish_allowed( + &self, + settings: &Settings, + ) -> Result<(), Report> { + for factory in &self.inner.html_stream_processors { + if factory.buffer_output_until_eof() && !factory.requires_async_finish() { + return Err(Report::new(TrustedServerError::Configuration { + message: format!( + "Integration {} declares EOF buffering without async completion", + factory.integration_id() + ), + })); + } + if factory.requires_async_finish() && !settings.publisher.allow_async_html_finish { + return Err(Report::new(TrustedServerError::Configuration { + message: format!( + "Integration {} requires publisher.allow_async_html_finish = true; EOF I/O can delay HTML delivery", + factory.integration_id() + ), + })); + } + } + Ok(()) + } + + /// Construct an isolated processor registry for regression tests. + #[cfg(test)] + #[must_use] + pub(crate) fn from_html_finalizers( + factories: Vec>, + ) -> Self { + Self::from_html_stream_processors( + factories + .into_iter() + .map(|factory| { + Arc::new(super::html_finalization::TestFinishFactory(factory)) + as Arc + }) + .collect(), + ) + } + + /// Construct an isolated ordered processor registry for fixture controls. + #[cfg(any(test, feature = "test-utils"))] + #[must_use] + pub fn from_html_stream_processors( + factories: Vec>, + ) -> Self { + Self { + inner: Arc::new(IntegrationRegistryInner { + html_stream_processors: factories, + ..Default::default() + }), + plan: None, + } + } + /// Collect HTML snippets for insertion at the start of ``. #[must_use] pub fn head_inserts(&self, ctx: &IntegrationHtmlContext<'_>) -> Vec { @@ -1421,6 +1510,113 @@ mod tests { use crate::platform::test_support::noop_services; use http::{HeaderValue, StatusCode, header}; + #[cfg(feature = "test-utils")] + #[test] + fn async_eof_requires_explicit_operator_permission() { + let mut settings = crate::test_support::tests::create_test_settings(); + assert!( + !settings.publisher.allow_async_html_finish, + "should default to no EOF I/O permission" + ); + settings + .integrations + .insert_config( + "html_finalization_fixture", + &serde_json::json!({ + "enabled": true, + "lookup_url": "https://lookup.example.com/approve", + "transport_timeout_ms": 1000, + }), + ) + .expect("should configure the fictional EOF integration"); + let error = IntegrationRegistry::new(&settings) + .err() + .expect("should reject enabled EOF I/O without operator permission"); + let message = format!("{error:?}"); + assert!( + message.contains("allow_async_html_finish"), + "should name the required setting" + ); + assert!( + message.contains("html_finalization_fixture"), + "should identify the enabled integration" + ); + settings.publisher.allow_async_html_finish = true; + let registry = IntegrationRegistry::new(&settings) + .expect("should register EOF I/O after explicit permission"); + assert!( + registry.has_async_html_finish(), + "should expose capability before body processing" + ); + } + + #[test] + fn async_eof_permission_alone_adds_no_processing_path() { + let mut settings = crate::test_support::tests::create_test_settings(); + settings.publisher.allow_async_html_finish = true; + let registry = IntegrationRegistry::new(&settings).expect("should build ordinary registry"); + assert!( + !registry.has_async_html_finish(), + "should retain the normal path without EOF factories" + ); + } + + struct RetentionWithoutAsyncFactory; + + impl IntegrationHtmlStreamProcessorFactory for RetentionWithoutAsyncFactory { + fn integration_id(&self) -> &'static str { + "example_inconsistent_retention" + } + fn buffer_output_until_eof(&self) -> bool { + true + } + fn create( + &self, + _: IntegrationHtmlStreamContext, + ) -> Box { + Box::new(crate::streaming_replacer::StreamingReplacer::new(Vec::new())) + } + } + + #[test] + fn async_eof_rejects_retention_without_async_capability() { + let mut settings = crate::test_support::tests::create_test_settings(); + settings.publisher.allow_async_html_finish = true; + let registry = IntegrationRegistry::from_html_stream_processors(vec![Arc::new( + RetentionWithoutAsyncFactory, + )]); + let error = registry + .ensure_async_html_finish_allowed(&settings) + .expect_err("should reject retention without declared EOF capability"); + let message = format!("{error:?}"); + assert!( + message.contains("example_inconsistent_retention"), + "should identify the invalid factory" + ); + assert!( + message.contains("buffering without async completion"), + "should explain the invalid metadata even with permission" + ); + } + + #[test] + fn html_finalizers_are_empty_by_default() { + let registry = IntegrationRegistry::from_request_filters(Vec::new()); + assert!( + !registry.has_async_html_finish(), + "should retain the ordinary streaming path" + ); + assert!( + registry.html_stream_processor_factories().is_empty(), + "should register no default finalizers" + ); + let registration = IntegrationRegistration::builder("example_finalizer").build(); + assert!( + registration.html_stream_processors.is_empty(), + "should make finalization opt-in" + ); + } + struct DefaultMetadataHeadInjector; impl IntegrationHeadInjector for DefaultMetadataHeadInjector { diff --git a/crates/trusted-server-core/src/platform/mod.rs b/crates/trusted-server-core/src/platform/mod.rs index 63f5c2a94..71258e9c5 100644 --- a/crates/trusted-server-core/src/platform/mod.rs +++ b/crates/trusted-server-core/src/platform/mod.rs @@ -43,6 +43,7 @@ mod template_assembly; mod template_cache; #[cfg(test)] pub(crate) mod test_support; +mod timer; mod traits; mod types; @@ -72,6 +73,7 @@ pub use template_cache::{ TemplateCookieValue, TemplateEntry, TemplateMetadata, TemplateMetadataEncodeError, UnavailableTemplateCache, VaryHeaderValues, VarySpec, reader_url_surrogate_key, }; +pub use timer::PlatformTimer; pub use traits::{PlatformBackend, PlatformConfigStore, PlatformGeo, PlatformSecretStore}; pub use types::{ ClientInfo, GeoInfo, PlatformBackendSpec, RuntimeServices, RuntimeServicesBuilder, StoreId, @@ -94,6 +96,15 @@ mod tests { use super::test_support::{noop_services, noop_services_with_client_ip}; use super::*; + #[test] + fn runtime_timer_is_optional() { + let services = noop_services(); + assert!( + services.timer().is_none(), + "should allow platforms without cooperative timers" + ); + } + struct MarkerKvStore(&'static str); #[async_trait(?Send)] @@ -163,6 +174,43 @@ mod tests { ); } + struct MarkerTimer; + impl PlatformTimer for MarkerTimer { + fn delay(&self, _: Duration) -> futures::future::LocalBoxFuture<'static, ()> { + Box::pin(futures::future::ready(())) + } + } + + #[test] + fn runtime_timer_survives_cloning_and_service_replacement() { + let mut services = noop_services(); + services.timer = Some(Arc::new(MarkerTimer)); + let replaced = services + .clone() + .with_kv_store(Arc::new(MarkerKvStore("example"))) + .with_template_cache(Arc::new(UnavailableTemplateCache)) + .with_template_assembler(Arc::new(UnavailableTemplateAssembler)); + assert!( + Arc::ptr_eq( + services + .timer + .as_ref() + .expect("should retain the original timer"), + replaced + .timer + .as_ref() + .expect("should preserve the timer through service replacement") + ), + "should preserve the same timer capability" + ); + futures::executor::block_on( + replaced + .timer() + .expect("should expose the installed timer") + .delay(Duration::ZERO), + ); + } + #[test] fn runtime_services_geo_lookup_returns_none_for_no_ip() { let services = noop_services(); diff --git a/crates/trusted-server-core/src/platform/timer.rs b/crates/trusted-server-core/src/platform/timer.rs new file mode 100644 index 000000000..fcb78bc05 --- /dev/null +++ b/crates/trusted-server-core/src/platform/timer.rs @@ -0,0 +1,14 @@ +//! Cooperative timer capability supplied by supporting adapters. + +use futures::future::LocalBoxFuture; +use std::time::Duration; + +/// Creates local delay futures without requiring a platform executor in core. +/// +/// A delay cannot preempt blocking calls or work that never yields. Fastly does +/// not supply this service because its host calls block. Consumers on Fastly +/// must independently bound their calls using the remaining EOF budget. +pub trait PlatformTimer: Send + Sync { + /// Wait for the duration on the adapter's request executor. + fn delay(&self, duration: Duration) -> LocalBoxFuture<'static, ()>; +} diff --git a/crates/trusted-server-core/src/platform/types.rs b/crates/trusted-server-core/src/platform/types.rs index 3570bdc10..760dba0e5 100644 --- a/crates/trusted-server-core/src/platform/types.rs +++ b/crates/trusted-server-core/src/platform/types.rs @@ -7,7 +7,7 @@ use crate::auction::telemetry::{AuctionTelemetrySink, NoopAuctionTelemetrySink}; use super::{ PlatformBackend, PlatformConfigStore, PlatformGeo, PlatformHttpClient, PlatformKvStore, - PlatformSecretStore, + PlatformSecretStore, PlatformTimer, }; /// Geographic information extracted from a request. @@ -188,6 +188,8 @@ pub struct RuntimeServices { pub(crate) auction_telemetry_sink: Arc, /// Per-request client metadata extracted at the entry point. pub(crate) client_info: ClientInfo, + /// Optional cooperative timer; absent on blocking host runtimes. + pub(crate) timer: Option>, } impl RuntimeServices { @@ -215,6 +217,12 @@ impl RuntimeServices { RuntimeServicesBuilder::new() } + /// Return the request's optional cooperative timer. + #[must_use] + pub fn timer(&self) -> Option<&dyn PlatformTimer> { + self.timer.as_deref() + } + /// Returns the config store service. #[must_use] pub fn config_store(&self) -> &dyn PlatformConfigStore { @@ -340,6 +348,7 @@ pub struct RuntimeServicesBuilder { geo: Option>, auction_telemetry_sink: Option>, client_info: Option, + timer: Option>, } impl RuntimeServicesBuilder { @@ -355,9 +364,17 @@ impl RuntimeServicesBuilder { geo: None, auction_telemetry_sink: None, client_info: None, + timer: None, } } + /// Set the cooperative timer for request-time yielding work. + #[must_use] + pub fn timer(mut self, timer: Arc) -> Self { + self.timer = Some(timer); + self + } + /// Set the config store implementation. #[must_use] pub fn config_store(mut self, config_store: Arc) -> Self { @@ -442,6 +459,7 @@ impl RuntimeServicesBuilder { #[must_use] pub fn build(self) -> RuntimeServices { RuntimeServices { + timer: self.timer, config_store: self .config_store .expect("should set config_store before building RuntimeServices"), diff --git a/crates/trusted-server-core/src/publisher.rs b/crates/trusted-server-core/src/publisher.rs index 682ecc0f1..d58239662 100644 --- a/crates/trusted-server-core/src/publisher.rs +++ b/crates/trusted-server-core/src/publisher.rs @@ -19,6 +19,7 @@ //! the streaming route). It is not a content-rewriting concern. use std::borrow::Cow; +mod html_finalization; use std::io::Write; use std::sync::atomic::{AtomicBool, Ordering}; use std::sync::{Arc, Mutex}; @@ -68,9 +69,11 @@ use crate::ec::kv::KvIdentityGraph; use crate::ec::registry::PartnerRegistry; use crate::ec::{EcContext, EidSyncSource}; use crate::error::TrustedServerError; -use crate::html_processor::BodyCloseInjection; +use crate::html_processor::{ + BodyCloseInjection, HtmlProcessorConfig, create_html_processor_with_document_state, +}; use crate::http_util::{RequestInfo, is_navigation_request, serve_static_with_etag}; -use crate::integrations::IntegrationRegistry; +use crate::integrations::{IntegrationDocumentState, IntegrationRegistry}; use crate::platform::{ GeoInfo, PlatformBackendSpec, PlatformHttpRequest, RuntimeServices, TEMPLATE_CACHE_PURGE_ALL_SURROGATE_KEY, VarySpec, contains_publisher_esi_directive, @@ -91,6 +94,7 @@ use crate::streaming_processor::{ STREAM_CHUNK_SIZE, StreamProcessor, StreamingPipeline, }; use crate::streaming_replacer::create_url_replacer; +use html_finalization::HtmlEofDriver; include!(concat!(env!("OUT_DIR"), "/template_build_digest.rs")); @@ -706,6 +710,9 @@ impl PublisherBodyProcessor { } impl StreamProcessor for PublisherBodyProcessor { + fn requires_async_finish(&self) -> bool { + self.inner.requires_async_finish() + } fn process_chunk(&mut self, chunk: &[u8], is_last: bool) -> Result, std::io::Error> { self.inner.process_chunk(chunk, is_last) } @@ -1079,7 +1086,6 @@ async fn hold_step_decoded_chunk( state: &mut AuctionHoldState, collect_refs: &AuctionCollectDeps<'_>, ) -> Result> { - let mut ready = Vec::new(); let processed = match processor .process_chunk(chunk, false) @@ -1092,21 +1098,41 @@ async fn hold_step_decoded_chunk( return Err(err); } }; - let bytes: Cow<'_, [u8]> = match state.hold.as_mut() { - None => Cow::Borrowed(&processed), - Some(seam) => Cow::Owned(seam.push(&processed)), - }; - match encoder.encode_chunk(bytes.into_owned()) { - Ok(encoded) if !encoded.is_empty() => ready.push(bytes::Bytes::from(encoded)), - Ok(_) => {} - Err(err) => { + match encode_processed_html(encoder, processed, Some(state)) { + Ok(step) => Ok(step), + Err(error) => { abandon_hold_auction(state, collect_refs.services, "stream_encode_error").await; - return Err(err.change_context(TrustedServerError::Proxy { - message: "Failed to encode processed chunk".to_string(), - })); + Err(error) } } - let close_found = state.hold.as_ref().is_some_and(InlineBodyCloseSeam::found); +} + +fn encode_processed_html( + encoder: &mut BodyStreamEncoder, + processed: Vec, + state: Option<&mut AuctionHoldState>, +) -> Result> { + let mut close_found = false; + let bytes = if let Some(state) = state { + let bytes = match state.hold.as_mut() { + Some(seam) => seam.push(&processed), + None => processed, + }; + close_found = state.hold.as_ref().is_some_and(InlineBodyCloseSeam::found); + bytes + } else { + processed + }; + let encoded = encoder + .encode_chunk(bytes) + .change_context(TrustedServerError::Proxy { + message: "Failed to encode processed chunk".to_string(), + })?; + let ready = if encoded.is_empty() { + Vec::new() + } else { + vec![bytes::Bytes::from(encoded)] + }; Ok(HoldStepSegments { ready, close_found }) } @@ -1496,8 +1522,13 @@ fn deferred_inline_seam_token( fn create_html_stream_processor( params: HtmlStreamProcessorParams<'_>, ) -> Result, Report> { - use crate::html_processor::{HtmlProcessorConfig, create_html_processor}; + create_html_stream_processor_with_document_state(params, &IntegrationDocumentState::default()) +} +fn create_html_stream_processor_with_document_state( + params: HtmlStreamProcessorParams<'_>, + document_state: &IntegrationDocumentState, +) -> Result> { let config = HtmlProcessorConfig::from_settings( params.settings, params.integration_registry, @@ -1530,7 +1561,13 @@ fn create_html_stream_processor( .with_csp_nonce_observer(csp_nonce_observed) .with_datadome_client_tag_suppression(params.suppress_datadome_client_side_tag); - Ok(create_html_processor(config)) + let processor = create_html_processor_with_document_state(config, document_state); + processor + .validate_capabilities() + .change_context(TrustedServerError::Configuration { + message: "HTML processor EOF capabilities are inconsistent".to_string(), + })?; + Ok(processor) } /// Result of publisher request handling, indicating whether the response body @@ -1707,6 +1744,10 @@ pub struct OwnedProcessResponseParams { pub(crate) request_host: String, pub(crate) request_scheme: String, pub(crate) content_type: String, + /// Authoritative origin status exposed to terminal integrations. + pub(crate) response_status: StatusCode, + /// Whether status or headers identify a range-associated representation. + pub(crate) range_associated: bool, pub(crate) ad_slots_script: Option, pub(crate) ad_bids_state: AdBidsState, /// Observation context for the in-flight auction. @@ -1732,6 +1773,12 @@ pub struct OwnedProcessResponseParams { pub(crate) csp_nonce_observed: Option>, } +impl OwnedProcessResponseParams { + fn is_range_associated(&self) -> bool { + self.range_associated || self.response_status == StatusCode::PARTIAL_CONTENT + } +} + /// Response-authorized template cache insert inputs. The key is built before origin lookup; the /// lifetime is only known after the origin proves this representation is shareable. pub(crate) struct AuthorizedTemplateStore { @@ -1777,13 +1824,19 @@ enum TemplateStoreOutcome { /// Panics if the `ad_bids_state` mutex is poisoned, which requires a prior panic while /// it was held. Every holder is a short, infallible read or write of an `Option`. pub async fn buffer_publisher_response_async( - publisher_response: PublisherResponse, + mut publisher_response: PublisherResponse, method: &Method, settings: &Settings, integration_registry: &IntegrationRegistry, orchestrator: &AuctionOrchestrator, services: &RuntimeServices, ) -> Result, Report> { + if let Err(error) = integration_registry.ensure_async_html_finish_allowed(settings) { + if let PublisherResponse::Stream { params, .. } = &mut publisher_response { + abandon_response_auction(params, services, "processor_init_error").await; + } + return Err(error); + } match publisher_response { PublisherResponse::Buffered(mut response) => { // A buffered-unmodified response can carry an origin body (a stream @@ -1823,6 +1876,21 @@ pub async fn buffer_publisher_response_async( } return Ok(response); } + params.response_status = response.status(); + params.range_associated = response_is_range_associated(&response); + if params.is_range_associated() + && is_html_content_type(¶ms.content_type) + && integration_registry.has_async_html_finish() + { + abandon_response_auction(&mut params, services, "partial_response").await; + *response.body_mut() = body; + return Ok(response); + } + if is_html_content_type(¶ms.content_type) + && integration_registry.has_async_html_finish() + { + enforce_synthesized_html_cache_privacy(&mut response); + } let mut output = BoundedWriter::new(settings.publisher.max_buffered_body_bytes); stream_publisher_body_async( body, @@ -2141,6 +2209,8 @@ fn build_template_assembly_params( ad_bids_state: AdBidsState, ) -> OwnedProcessResponseParams { OwnedProcessResponseParams { + response_status: StatusCode::OK, + range_associated: false, csp_nonce_observed: None, // Already stored; storing again on a hit would be pointless work. template_cache_key: None, @@ -2399,13 +2469,19 @@ async fn store_template_if_authorized( /// Panics if the `ad_bids_state` mutex is poisoned, which requires a prior panic while /// it was held. Every holder is a short, infallible read or write of an `Option`. pub async fn publisher_response_into_streaming_response( - publisher_response: PublisherResponse, + mut publisher_response: PublisherResponse, method: &Method, settings: Arc, integration_registry: &IntegrationRegistry, orchestrator: Arc, services: RuntimeServices, ) -> Result, Report> { + if let Err(error) = integration_registry.ensure_async_html_finish_allowed(&settings) { + if let PublisherResponse::Stream { params, .. } = &mut publisher_response { + abandon_response_auction(params, &services, "processor_init_error").await; + } + return Err(error); + } // A template can only be stored once the transform has produced every byte, and // streaming hands bytes to the client as they are produced rather than collecting // them. Shared modes therefore take the buffered finalizer, which already @@ -2423,14 +2499,17 @@ pub async fn publisher_response_into_streaming_response( &publisher_response, PublisherResponse::Stream { params, .. } if params.template_cache_key.is_some() ) { - return buffer_publisher_response_async( + // This already-buffered cache miss can nest large codec futures. + // Keep its future off the WASI call stack; ordinary inline streaming + // never takes this branch. + return Box::pin(buffer_publisher_response_async( publisher_response, method, &settings, integration_registry, &orchestrator, &services, - ) + )) .await; } @@ -2567,8 +2646,47 @@ pub async fn publisher_response_into_streaming_response( return Ok(response); } + params.response_status = response.status(); + params.range_associated = response_is_range_associated(&response); + if params.is_range_associated() + && is_html_content_type(¶ms.content_type) + && integration_registry.has_async_html_finish() + { + abandon_response_auction(&mut params, &services, "partial_response").await; + *response.body_mut() = body; + return Ok(response); + } response.headers_mut().remove(header::CONTENT_LENGTH); let mut params = *params; + if is_html_content_type(¶ms.content_type) + && integration_registry.has_async_html_finish() + { + enforce_synthesized_html_cache_privacy(&mut response); + let mut driver = Box::pin(build_html_eof_driver( + body, + &mut params, + &settings, + integration_registry, + &services, + )) + .await?; + let stream = async_stream::try_stream! { + let deps = AuctionCollectDeps { + price_granularity: params.price_granularity, ad_bids_state: ¶ms.ad_bids_state, + orchestrator: &orchestrator, services: &services, settings: &settings, + request_origin: request_origin(¶ms.request_scheme, ¶ms.request_host), + }; + while let Some(step) = driver.next_step(&deps).await.map_err(publisher_stream_error)? { + for segment in step.ready { yield segment; } + if step.close_found { + for segment in driver.collect_close(&deps).await.map_err(publisher_stream_error)? { yield segment; } + } + } + }; + *response.body_mut() = EdgeBody::from_stream::<_, std::io::Error>(stream); + return Ok(response); + } + let (mut processor, inline_seam_token) = match PublisherBodyProcessor::new(¶ms, &settings, integration_registry) { Ok(built) => built, @@ -2784,6 +2902,11 @@ fn response_carries_body(method: &Method, status: StatusCode) -> bool { && status != StatusCode::NOT_MODIFIED } +fn response_is_range_associated(response: &Response) -> bool { + response.status() == StatusCode::PARTIAL_CONTENT + || response.headers().contains_key(header::CONTENT_RANGE) +} + /// Prevent shared caches from replaying tag-suppressed HTML to other clients. fn apply_datadome_client_tag_cache_privacy( response: &mut Response, @@ -2892,6 +3015,17 @@ impl Write for BoundedWriter { /// I/O happened upstream in [`handle_publisher_request`]. Writes processed /// chunks directly to `output`. /// +/// This helper performs synchronous transformations only. Registered HTML +/// finalizers require [`stream_publisher_body_async`], +/// [`buffer_publisher_response_async`], or +/// [`publisher_response_into_streaming_response`]. Adapters use those async +/// entry points when terminal integrations are enabled. +/// +/// Callers with a pending auction must use an async entry point so it can be +/// collected or abandoned. This helper borrows `params` and leaves auction +/// ownership with the caller, including when range-associated HTML bypasses +/// processing. +/// /// # Errors /// /// Returns an error if processing fails mid-stream. Since headers are already @@ -2904,6 +3038,24 @@ pub fn stream_publisher_body( settings: &Settings, integration_registry: &IntegrationRegistry, ) -> Result<(), Report> { + if params.is_range_associated() + && is_html_content_type(¶ms.content_type) + && integration_registry.has_async_html_finish() + { + let mut input = body_as_reader(body)?; + std::io::copy(&mut input, output).change_context(TrustedServerError::Proxy { + message: "Failed to copy range-associated HTML".to_string(), + })?; + return output.flush().change_context(TrustedServerError::Proxy { + message: "Failed to flush range-associated HTML".to_string(), + }); + } + if is_html_content_type(¶ms.content_type) && integration_registry.has_async_html_finish() { + return Err(Report::new(TrustedServerError::Configuration { + message: "Registered HTML processors require an asynchronous publisher entry point" + .to_string(), + })); + } let borrowed = ProcessResponseParams { content_encoding: ¶ms.content_encoding, origin_host: ¶ms.origin_host, @@ -2929,6 +3081,103 @@ pub fn stream_publisher_body( process_response_streaming(body, output, &borrowed, output_compression) } +async fn abandon_response_auction( + params: &mut OwnedProcessResponseParams, + services: &RuntimeServices, + reason: &'static str, +) { + if let Some(dispatched) = params.dispatched_auction.take() { + emit_abandoned_auction( + services, + params.auction_observation.take(), + dispatched, + reason, + ) + .await; + } +} + +async fn build_html_eof_driver( + body: EdgeBody, + params: &mut OwnedProcessResponseParams, + settings: &Settings, + registry: &IntegrationRegistry, + services: &RuntimeServices, +) -> Result, Report> { + match HtmlEofDriver::new(body, params, settings, registry) { + Ok(driver) => Ok(Box::new(driver)), + Err(error) => { + if let Some(dispatched) = params.dispatched_auction.take() { + emit_abandoned_auction( + services, + params.auction_observation.take(), + dispatched, + "processor_init_error", + ) + .await; + } + Err(error) + } + } +} + +async fn stream_html_eof( + body: EdgeBody, + output: &mut W, + params: &mut OwnedProcessResponseParams, + settings: &Settings, + registry: &IntegrationRegistry, + orchestrator: &AuctionOrchestrator, + services: &RuntimeServices, +) -> Result<(), Report> { + let mut driver = build_html_eof_driver(body, params, settings, registry, services).await?; + let deps = AuctionCollectDeps { + price_granularity: params.price_granularity, + ad_bids_state: ¶ms.ad_bids_state, + orchestrator, + services, + settings, + request_origin: request_origin(¶ms.request_scheme, ¶ms.request_host), + }; + while let Some(step) = driver.next_step(&deps).await? { + for segment in step.ready { + if let Err(error) = output.write_all(&segment) { + driver.abandon(&deps, "stream_write_error").await; + return Err( + Report::new(error).change_context(TrustedServerError::Proxy { + message: "Failed to write finalized HTML".to_string(), + }), + ); + } + } + if (step.close_found || driver.at_finish_barrier()) + && let Err(error) = output.flush() + { + driver.abandon(&deps, "stream_write_error").await; + return Err( + Report::new(error).change_context(TrustedServerError::Proxy { + message: "Failed to flush finalized HTML".to_string(), + }), + ); + } + if step.close_found { + for segment in driver.collect_close(&deps).await? { + if let Err(error) = output.write_all(&segment) { + driver.abandon(&deps, "stream_write_error").await; + return Err( + Report::new(error).change_context(TrustedServerError::Proxy { + message: "Failed to write HTML auction tail".to_string(), + }), + ); + } + } + } + } + output.flush().change_context(TrustedServerError::Proxy { + message: "Failed to flush finalized HTML".to_string(), + }) +} + /// Stream publisher body with parser-confirmed live bid injection. /// /// Drives the origin body through the HTML pipeline one chunk at a time. The @@ -2958,6 +3207,40 @@ pub async fn stream_publisher_body_async( orchestrator: &AuctionOrchestrator, services: &RuntimeServices, ) -> Result<(), Report> { + if let Err(error) = integration_registry.ensure_async_html_finish_allowed(settings) { + abandon_response_auction(params, services, "processor_init_error").await; + return Err(error); + } + if params.is_range_associated() + && is_html_content_type(¶ms.content_type) + && integration_registry.has_async_html_finish() + { + abandon_response_auction(params, services, "partial_response").await; + let mut source = BodyChunkSource::new(body, STREAM_CHUNK_SIZE) + .with_max_bytes(settings.publisher.max_buffered_body_bytes); + while let Some(bytes) = source.next_chunk().await? { + output + .write_all(&bytes) + .change_context(TrustedServerError::Proxy { + message: "Failed to copy range-associated HTML".to_string(), + })?; + } + return output.flush().change_context(TrustedServerError::Proxy { + message: "Failed to flush range-associated HTML".to_string(), + }); + } + if is_html_content_type(¶ms.content_type) && integration_registry.has_async_html_finish() { + return Box::pin(stream_html_eof( + body, + output, + params, + settings, + integration_registry, + orchestrator, + services, + )) + .await; + } let Some(dispatched) = params.dispatched_auction.take() else { if body.is_stream() { return process_response_streaming_async( @@ -3904,6 +4187,80 @@ impl InlineBodyCloseSeam { self.buffered.drain(..split_at).collect() } + fn push_bounded(&mut self, chunk: &[u8], limit: usize) -> std::io::Result> { + let overflow = || std::io::Error::other("HTML auction suffix exceeds retained-byte limit"); + if self.found { + let next = self + .buffered + .len() + .checked_add(chunk.len()) + .filter(|size| *size <= limit) + .ok_or_else(overflow)?; + if next > self.buffered.capacity() { + let capacity = self + .buffered + .capacity() + .saturating_mul(2) + .max(next) + .min(limit); + self.buffered.reserve_exact(capacity - self.buffered.len()); + } + self.buffered.extend_from_slice(chunk); + return Ok(Vec::new()); + } + let mut boundary = self.buffered.clone(); + boundary.extend_from_slice(&chunk[..chunk.len().min(self.token.len().saturating_sub(1))]); + let crossing = + find_bytes(&boundary, &self.token).filter(|position| *position < self.buffered.len()); + let inside = find_bytes(chunk, &self.token); + if let Some((old_prefix, chunk_prefix, suffix_start)) = crossing + .map(|position| { + ( + position, + 0, + position + self.token.len() - self.buffered.len(), + ) + }) + .or_else(|| { + inside.map(|position| (self.buffered.len(), position, position + self.token.len())) + }) + { + let suffix = &chunk[suffix_start..]; + if suffix.len() > limit { + return Err(overflow()); + } + let mut ready = self.buffered[..old_prefix].to_vec(); + ready.extend_from_slice(&chunk[..chunk_prefix]); + self.buffered = Vec::new(); + self.buffered.reserve_exact(suffix.len()); + self.buffered.extend_from_slice(suffix); + self.found = true; + return Ok(ready); + } + let tail = if chunk.len() >= self.token.len().saturating_sub(1) { + chunk[chunk.len() - self.token.len().saturating_sub(1)..].to_vec() + } else { + let mut tail = self.buffered.clone(); + tail.extend_from_slice(chunk); + tail + }; + let keep = longest_suffix_prefix(&tail, &self.token); + if keep > limit { + return Err(overflow()); + } + let old_ready = self + .buffered + .len() + .saturating_sub(keep.saturating_sub(chunk.len())); + let chunk_ready = chunk.len().saturating_sub(keep); + let mut ready = self.buffered[..old_ready].to_vec(); + ready.extend_from_slice(&chunk[..chunk_ready]); + self.buffered = Vec::new(); + self.buffered.reserve_exact(keep); + self.buffered.extend_from_slice(&tail[tail.len() - keep..]); + Ok(ready) + } + fn found(&self) -> bool { self.found } @@ -4311,6 +4668,11 @@ pub struct AuctionDispatch<'a> { pub slots: &'a [crate::creative_opportunities::CreativeOpportunitySlot], /// Partner registry for KV-backed EID resolution. `None` skips KV enrichment. pub registry: Option<&'a PartnerRegistry>, + /// Enabled integration registry used before origin and shared-cache decisions. + /// + /// Pass the same registry to response finalization so request admission and + /// document processing use the same enabled integration set. + pub integration_registry: &'a IntegrationRegistry, } /// Request-side conditions that decide whether this request's origin response may be @@ -4438,7 +4800,11 @@ pub async fn handle_publisher_request( mut req: Request, edge_header: EdgeCacheHeader, ) -> Result> { + auction + .integration_registry + .ensure_async_html_finish_allowed(settings)?; log::debug!("Proxying request to publisher_origin"); + let async_html_finish_present = auction.integration_registry.has_async_html_finish(); // Adapter fallbacks prepare this before EC/cookie handling. Keep this // idempotent call as a direct-handler safety net and for focused tests. @@ -4680,7 +5046,12 @@ pub async fn handle_publisher_request( // a panic-prone invariant in the public request handler. let reader_compression = reader_compression.unwrap_or(Compression::None); - if should_run_ad_stack || datadome_suppression_requires_full_body { + if should_run_ad_stack + || datadome_suppression_requires_full_body + || (async_html_finish_present + && req.method() == Method::GET + && is_html_document_request(&req)) + { // HTML document contexts whose output may be synthesized must not // receive a cached 304 or partial 206. Non-document subresources contain // no executable injected tag, so retain their validators and ranges. @@ -4727,12 +5098,13 @@ pub async fn handle_publisher_request( .creative_opportunities .as_ref() .is_some_and(CreativeOpportunitiesConfig::origin_readthrough_enabled); - let request_can_use_shared_template = request_can_use_shared_template( - shared_request_inputs, - matches!(assembly_mode, AssemblyMode::Esi), - reader_supports_assembly, - reader_requires_origin, - ); + let request_can_use_shared_template = !async_html_finish_present + && request_can_use_shared_template( + shared_request_inputs, + matches!(assembly_mode, AssemblyMode::Esi), + reader_supports_assembly, + reader_requires_origin, + ); // Only advertise encodings the rewrite pipeline can decode and re-encode. This // remains unconditional when template cache negotiation fails: that request bypasses shared @@ -5334,7 +5706,12 @@ pub async fn handle_publisher_request( // caching directives, letting a downstream cache serve it to a later eligible reader. let assembled_response_must_be_private = template_cache_key.is_some(); let is_not_modified = response.status() == StatusCode::NOT_MODIFIED; - if is_html_content_type(&origin_content_type) || is_not_modified { + let range_associated_html_bypass = async_html_finish_present + && is_html_content_type(&origin_content_type) + && response_is_range_associated(&response); + if !range_associated_html_bypass + && (is_html_content_type(&origin_content_type) || is_not_modified) + { if should_run_ad_stack || assembled_response_must_be_private { enforce_synthesized_html_cache_privacy(&mut response); } else if is_server_side_ad_eligible_navigation( @@ -5355,12 +5732,14 @@ pub async fn handle_publisher_request( } } } - apply_datadome_client_tag_cache_privacy( - &mut response, - &request_method, - suppress_datadome_client_side_tag, - &origin_content_type, - ); + if !range_associated_html_bypass { + apply_datadome_client_tag_cache_privacy( + &mut response, + &request_method, + suppress_datadome_client_side_tag, + &origin_content_type, + ); + } apply_publisher_asset_cache_policy( settings, &request_path, @@ -5387,7 +5766,18 @@ pub async fn handle_publisher_request( .unwrap_or_default() .trim() .to_ascii_lowercase(); - let route = classify_response_route(status, &content_type, &content_encoding, request_host); + let route = if range_associated_html_bypass { + ResponseRoute::BufferedUnmodified + } else { + classify_response_route(status, &content_type, &content_encoding, request_host) + }; + if async_html_finish_present + && route == ResponseRoute::Stream + && is_html_content_type(&content_type) + && response_carries_body(&request_method, status) + { + enforce_synthesized_html_cache_privacy(&mut response); + } if template_cache_key.is_some() { set_response_compression(&mut response, reader_compression); } @@ -5472,6 +5862,8 @@ pub async fn handle_publisher_request( response, body, params: Box::new(OwnedProcessResponseParams { + response_status: status, + range_associated: false, // The transform writes here; the post-transform gate reads it. Always // present on the live path so no future caller has to remember to // supply it — the handlers themselves stay gated on authorization. @@ -7759,6 +8151,7 @@ mod tests { Some(&graph), &mut ec_context, AuctionDispatch { + integration_registry: &IntegrationRegistry::empty_for_tests(), orchestrator: &orchestrator, slots: &slots, registry: None, @@ -7969,6 +8362,7 @@ mod tests { Some(&graph), &mut ec_context, AuctionDispatch { + integration_registry: &IntegrationRegistry::empty_for_tests(), orchestrator: &orchestrator, slots: &slots, registry: Some(®istry), @@ -8982,6 +9376,8 @@ mod tests { content_encoding: &str, ) -> OwnedProcessResponseParams { OwnedProcessResponseParams { + response_status: StatusCode::OK, + range_associated: false, csp_nonce_observed: None, template_cache_key: None, seam_ad_slots: None, @@ -9212,6 +9608,7 @@ mod tests { None, &mut ec_context, AuctionDispatch { + integration_registry: &IntegrationRegistry::empty_for_tests(), orchestrator: &orchestrator, slots: &[], registry: None, @@ -10488,146 +10885,504 @@ mod tests { .await } - async fn run_with_orchestrator( - settings: &Arc, - services: &RuntimeServices, - request: Request, - orchestrator: AuctionOrchestrator, - finalizer: Finalizer, - ) -> Response { + #[tokio::test] + async fn html_finalization_preserves_a_winning_bid_before_the_structural_body_end() { + let settings = Arc::new(crate::publisher::allow_eof_test_settings( + settings_with_bidder("inline"), + )); + let stub = Arc::new(StubHttpClient::new()); + let cache = Arc::new(MemoryTemplateCache::default()); + let services = services(Arc::clone(&stub), cache); + let registry = IntegrationRegistry::from_html_finalizers(vec![Arc::new( + super::TerminalTestFactory, + )]); + let mut orchestrator = AuctionOrchestrator::new(settings.auction.clone()); + orchestrator.register_provider(Arc::new(WinningBidProvider { + price: STUB_BID_CPM, + })); let orchestrator = Arc::new(orchestrator); - let registry = - IntegrationRegistry::new(settings).expect("should create integration registry"); + queue_bid_response(&stub); + queue_shareable_html(&stub); let consent = crate::consent::ConsentContext { jurisdiction: crate::consent::jurisdiction::Jurisdiction::NonRegulated, ..Default::default() }; let mut ec_context = EcContext::new_for_test(None, consent); - let publisher_response = handle_publisher_request( - settings, - services, + let response = handle_publisher_request( + &settings, + &services, None, &mut ec_context, AuctionDispatch { + integration_registry: ®istry, orchestrator: &orchestrator, slots: &[article_slot()], registry: None, }, - request, + navigation_request(), EdgeCacheHeader::SMaxageFallback, ) .await - .expect("should proxy publisher request"); - - finalize_test_publisher_response( - publisher_response, + .expect("should launch the winning auction"); + let response = publisher_response_into_streaming_response( + response, + &Method::GET, settings, - services, ®istry, orchestrator, - finalizer, + services, ) .await + .expect("should retain terminal and auction phases"); + let html = + String::from_utf8(body_of(response).await).expect("should emit example UTF-8 HTML"); + let bids = seam_bids(&html); + assert_eq!( + bids.get("test-slot") + .and_then(|bid| bid.get("hb_pb")) + .and_then(serde_json::Value::as_str), + Some("3.50"), + "should retain the winning bid after terminal I/O" + ); + assert!(html.contains("origin"), "should preserve the article bytes"); + assert!( + html.ends_with(""), + "should commit the terminal output" + ); + assert_eq!( + html.matches("var b=JSON.parse(").count(), + 1, + "should inject bids once" + ); + assert!( + html.find("var b=JSON.parse(") + .expect("should find the bids") + < html + .find("") + .expect("should retain the structural body end"), + "should assemble bids before the parsed closing tag" + ); + assert!( + !html.contains("ts-inline-body-close-"), + "should consume the parser seam token" + ); } - async fn finalize_test_publisher_response( - publisher_response: PublisherResponse, - settings: &Arc, - services: &RuntimeServices, - registry: &IntegrationRegistry, - orchestrator: Arc, - finalizer: Finalizer, - ) -> Response { - match finalizer { - Finalizer::Streaming => publisher_response_into_streaming_response( - publisher_response, - &Method::GET, - Arc::clone(settings), - registry, - orchestrator, - services.clone(), - ) - .await - .expect("should finalize publisher response"), - Finalizer::Buffered => buffer_publisher_response_async( - publisher_response, - &Method::GET, - settings, - registry, - &orchestrator, - services, - ) - .await - .expect("should finalize publisher response"), - } - } - - /// Collects a response body whether it was buffered or streamed. - /// - /// Both occur here by design: a response the gate authorized is buffered so it - /// can be stored, and one it refused keeps streaming exactly like `inline`. A - /// helper that assumed either would silently only ever test one of them. - async fn body_of(response: Response) -> Vec { - match response.into_body() { - EdgeBody::Stream(mut stream) => { - let mut out = Vec::new(); - while let Some(chunk) = stream.next().await { - out.extend_from_slice(&chunk.expect("stream chunk should read")); + #[tokio::test] + async fn html_finalization_range_bypass_preserves_origin_policy_with_live_or_suppressed_ads() + { + for (live, suppressed) in [(true, false), (false, false), (false, true)] { + for (status, content_range) in [(206, false), (200, true), (416, true)] { + let mut settings = + crate::publisher::allow_eof_test_settings(settings_with_bidder("inline")); + settings.auction.enabled = live; + let settings = Arc::new(settings); + let stub = Arc::new(StubHttpClient::new()); + let sink = Arc::new(RecordingTelemetrySink::default()); + let services = services_with_cache_and_telemetry( + Arc::clone(&stub), + Arc::new(MemoryTemplateCache::default()), + Arc::clone(&sink), + ); + let calls = Arc::new(AtomicUsize::new(0)); + let registry = IntegrationRegistry::from_html_finalizers(vec![Arc::new( + super::CountingTerminalFactory(Arc::clone(&calls)), + )]); + let mut orchestrator = AuctionOrchestrator::new(settings.auction.clone()); + orchestrator.register_provider(Arc::new(WinningBidProvider { + price: STUB_BID_CPM, + })); + if live { + queue_bid_response(&stub); } - out + let raw = b"

Example partial article

"; + let mut headers = vec![ + ("content-type", "text/html"), + ("content-length", "30"), + ("cache-control", "public, max-age=60"), + ("etag", "example-origin-tag"), + ("last-modified", "Mon, 01 Jan 2024 00:00:00 GMT"), + ("surrogate-control", "max-age=60"), + ]; + if content_range { + headers.push(("content-range", "bytes 0-29/100")); + } + stub.push_response_with_headers(status, raw.to_vec(), headers); + let consent = crate::consent::ConsentContext { + jurisdiction: crate::consent::jurisdiction::Jurisdiction::NonRegulated, + ..Default::default() + }; + let mut ec_context = EcContext::new_for_test(None, consent); + let mut request = navigation_request(); + if suppressed { + request + .extensions_mut() + .insert(crate::integrations::datadome::DataDomeClientTagSuppressed); + } + let publisher = handle_publisher_request( + &settings, + &services, + None, + &mut ec_context, + AuctionDispatch { + integration_registry: ®istry, + orchestrator: &orchestrator, + slots: &[article_slot()], + registry: None, + }, + request, + EdgeCacheHeader::SMaxageFallback, + ) + .await + .expect("should classify partial HTML before synthesized policy"); + let response = publisher_response_into_streaming_response( + publisher, + &Method::GET, + settings, + ®istry, + Arc::new(orchestrator), + services, + ) + .await + .expect("should preserve partial origin delivery"); + for (name, expected) in [ + (header::CONTENT_LENGTH, "30"), + (header::CACHE_CONTROL, "public, max-age=60"), + (header::ETAG, "example-origin-tag"), + (header::LAST_MODIFIED, "Mon, 01 Jan 2024 00:00:00 GMT"), + ] { + assert_eq!( + response + .headers() + .get(name) + .and_then(|value| value.to_str().ok()), + Some(expected), + "should preserve origin policy for live={live}, suppressed={suppressed}, status={status}" + ); + } + assert_eq!( + response + .headers() + .get("surrogate-control") + .and_then(|value| value.to_str().ok()), + Some("max-age=60"), + "should preserve origin edge cache policy" + ); + assert_eq!( + body_of(response).await, + raw, + "should preserve partial article bytes" + ); + assert_eq!( + calls.load(Ordering::SeqCst), + 0, + "should perform no terminal I/O" + ); + let batches = sink.batches.lock().expect("should inspect range telemetry"); + let summaries: Vec<_> = batches + .iter() + .flat_map(crate::auction::telemetry::AuctionEventBatch::rows) + .filter(|row| row.event_kind == "summary") + .collect(); + assert_eq!( + summaries.len(), + 1, + "should record one terminal auction outcome" + ); + assert_eq!( + summaries[0].terminal_status.as_deref(), + Some(if live { "abandoned" } else { "skipped" }), + "should abandon dispatched bids once and preserve ordinary skipped telemetry" + ); } - other => other - .into_bytes() - .expect("a non-stream body has its bytes in hand") - .to_vec(), + } + } + + #[derive(Default)] + struct RawOriginCacheClient { + cached: Mutex>>, + misses: AtomicUsize, + hits: AtomicUsize, + } + + #[async_trait::async_trait(?Send)] + impl crate::platform::PlatformHttpClient for RawOriginCacheClient { + async fn send( + &self, + request: crate::platform::PlatformHttpRequest, + ) -> Result> + { + assert!( + matches!( + request.cache_intent, + crate::platform::PlatformCacheIntent::Shared { .. } + ), + "should retain raw-origin cache eligibility" + ); + let mut cached = self.cached.lock().expect("should lock raw origin cache"); + let bytes = if let Some(bytes) = cached.as_ref() { + self.hits.fetch_add(1, Ordering::SeqCst); + bytes.clone() + } else { + self.misses.fetch_add(1, Ordering::SeqCst); + let bytes = + b"example raw article".to_vec(); + *cached = Some(bytes.clone()); + bytes + }; + Ok(crate::platform::PlatformResponse::new( + Response::builder() + .header(header::CONTENT_TYPE, "text/html") + .header(header::CACHE_CONTROL, "public, max-age=60") + .body(EdgeBody::from(bytes)) + .expect("should construct raw origin response"), + )) + } + + async fn send_async( + &self, + _: crate::platform::PlatformHttpRequest, + ) -> Result< + crate::platform::PlatformPendingRequest, + Report, + > { + panic!("should not dispatch an auction in the raw-origin cache fixture") + } + + async fn select( + &self, + _: Vec, + ) -> Result> + { + panic!("should not collect an auction in the raw-origin cache fixture") } } #[tokio::test] - async fn harness_emits_a_summary_row_for_an_ad_serving_navigation() { - let stub = Arc::new(StubHttpClient::new()); - let sink = Arc::new(RecordingTelemetrySink::default()); - let services = services_with_cache_and_telemetry( - Arc::clone(&stub), - Arc::new(MemoryTemplateCache::default()), - Arc::clone(&sink), + async fn html_finalization_runs_again_on_a_raw_origin_cache_hit() { + let client = Arc::new(RawOriginCacheClient::default()); + let template_cache = Arc::new(MemoryTemplateCache::default()); + let mut settings = + crate::publisher::allow_eof_test_settings(settings_with_readthrough_enabled("esi")); + settings.auction.enabled = false; + let settings = Arc::new(settings); + let services = RuntimeServices::builder() + .config_store(Arc::new(NoopConfigStore)) + .secret_store(Arc::new(NoopSecretStore)) + .kv_store(Arc::new(edgezero_core::key_value_store::NoopKvStore)) + .backend(Arc::new(StubBackend)) + .http_client(client.clone()) + .geo(Arc::new(NoopGeo)) + .client_info(ClientInfo::default()) + .template_cache(template_cache.clone()) + .build(); + let registry = IntegrationRegistry::from_html_finalizers(vec![Arc::new( + super::TerminalTestFactory, + )]); + let orchestrator = Arc::new(AuctionOrchestrator::new(settings.auction.clone())); + for _ in 0..2 { + let response = handle_publisher_request( + &settings, + &services, + None, + &mut EcContext::default(), + AuctionDispatch { + integration_registry: ®istry, + orchestrator: &orchestrator, + slots: &[], + registry: None, + }, + navigation_request(), + EdgeCacheHeader::SMaxageFallback, + ) + .await + .expect("should obtain raw origin HTML"); + let response = publisher_response_into_streaming_response( + response, + &Method::GET, + Arc::clone(&settings), + ®istry, + Arc::clone(&orchestrator), + services.clone(), + ) + .await + .expect("should finalize cached or uncached raw HTML"); + assert_eq!( + header_of(&response, header::CACHE_CONTROL), + Some(crate::cache_policy::NO_STORE_PRIVATE_CACHE_CONTROL), + "should keep both synthesized responses private" + ); + let body = body_of(response).await; + assert_eq!( + body.windows(b"".len()) + .filter(|bytes| *bytes == b"") + .count(), + 1, + "should run finalization once for each raw response" + ); + } + assert_eq!( + client.misses.load(Ordering::SeqCst), + 1, + "should fetch origin once" ); - let settings = Arc::new(settings_with_mode("esi")); - queue_shareable_html(&stub); - - let _ = run(&settings, &services, navigation_request()).await; + assert_eq!( + client.hits.load(Ordering::SeqCst), + 1, + "should serve a raw cache hit" + ); + assert!( + !client + .cached + .lock() + .expect("should lock raw origin cache") + .as_ref() + .expect("should retain raw cached HTML") + .windows(b"example-finalized".len()) + .any(|bytes| bytes == b"example-finalized"), + "should cache raw bytes without terminal replacements" + ); + assert!( + template_cache + .lookups + .lock() + .expect("should lock template lookups") + .is_empty(), + "should skip template lookup on both requests" + ); + assert!( + template_cache + .stored_keys + .lock() + .expect("should lock template stores") + .is_empty(), + "should not store finalized HTML as a shared template" + ); + } + #[tokio::test] + async fn html_finalizers_bypass_warm_and_cold_templates_but_keep_origin_readthrough() { + let stub = Arc::new(StubHttpClient::new()); + let cache = Arc::new(MemoryTemplateCache::default()); + let settings = Arc::new(crate::publisher::allow_eof_test_settings( + settings_with_readthrough_enabled("esi"), + )); + let services = services(Arc::clone(&stub), Arc::clone(&cache)); + queue_shareable_html(&stub); + let _ = body_of(run(&settings, &services, navigation_request()).await).await; + let lookups_before = cache.lookups.lock().expect("should lock lookups").len(); + let stores_before = cache + .stored_keys + .lock() + .expect("should lock stored keys") + .len(); + assert!(stores_before > 0, "should seed a warm shared template"); + let registry = IntegrationRegistry::from_html_finalizers(vec![Arc::new( + super::TerminalTestFactory, + )]); + let orchestrator = Arc::new(AuctionOrchestrator::new(settings.auction.clone())); + for path in ["/article", "/article-cold"] { + queue_shareable_html(&stub); + let mut request = navigation_request(); + *request.uri_mut() = format!("https://ts.example.com{path}") + .parse() + .expect("should parse an example URL"); + let mut ec_context = EcContext::default(); + let response = handle_publisher_request( + &settings, + &services, + None, + &mut ec_context, + AuctionDispatch { + integration_registry: ®istry, + orchestrator: &orchestrator, + slots: &[], + registry: None, + }, + request, + EdgeCacheHeader::SMaxageFallback, + ) + .await + .expect("should request raw origin HTML"); + let PublisherResponse::Stream { params, .. } = &response else { + panic!("should bypass shared templates"); + }; + assert!( + params.template_cache_key.is_none(), + "should carry no shared-store authorization" + ); + let response = publisher_response_into_streaming_response( + response, + &Method::GET, + Arc::clone(&settings), + ®istry, + Arc::clone(&orchestrator), + services.clone(), + ) + .await + .expect("should finalize origin HTML"); + assert_eq!( + header_of(&response, header::CACHE_CONTROL), + Some(crate::cache_policy::NO_STORE_PRIVATE_CACHE_CONTROL), + "should make terminal output private" + ); + assert!( + body_of(response) + .await + .ends_with(b""), + "should finish both warm and cold documents" + ); + } + assert_eq!( + cache.lookups.lock().expect("should lock lookups").len(), + lookups_before, + "should skip lookup and reservation entirely" + ); + assert_eq!( + cache + .stored_keys + .lock() + .expect("should lock stored keys") + .len(), + stores_before, + "should never store terminal output" + ); assert!( - last_summary_row(&sink).is_some(), - "the harness must emit a summary row, or every assertion built on it is vacuous" + stub.recorded_cache_intents().iter().all(|intent| matches!( + intent, + crate::platform::PlatformCacheIntent::Shared { .. } + )), + "should retain configured raw-origin readthrough" + ); + let _ = body_of(run(&settings, &services, navigation_request()).await).await; + assert!( + cache.lookups.lock().expect("should lock lookups").len() > lookups_before, + "should restore cache eligibility when finalizers are disabled" ); } - /// Drive `handle_publisher_request` through the **EC-preload** fetch path. - /// - /// `run_with_orchestrator` passes `kv: None` and no EC id, so - /// `should_preload_ec_snapshot` is false for every other test in this file and the - /// first of the two origin-fetch call sites is never exercised. Supplying both - /// reaches it. - async fn run_through_ec_preload( + async fn run_with_orchestrator( settings: &Arc, services: &RuntimeServices, request: Request, - ) { - let orchestrator = Arc::new(AuctionOrchestrator::new(settings.auction.clone())); - let kv = crate::ec::kv::KvIdentityGraph::in_memory("test-store"); + orchestrator: AuctionOrchestrator, + finalizer: Finalizer, + ) -> Response { + let orchestrator = Arc::new(orchestrator); + let registry = + IntegrationRegistry::new(settings).expect("should create integration registry"); let consent = crate::consent::ConsentContext { jurisdiction: crate::consent::jurisdiction::Jurisdiction::NonRegulated, ..Default::default() }; - let mut ec_context = EcContext::new_for_test(Some("test-ec-id".to_owned()), consent); - - let _ = handle_publisher_request( + let mut ec_context = EcContext::new_for_test(None, consent); + let publisher_response = handle_publisher_request( settings, services, - Some(&kv), + None, &mut ec_context, AuctionDispatch { + integration_registry: ®istry, orchestrator: &orchestrator, slots: &[article_slot()], registry: None, @@ -10637,15 +11392,135 @@ mod tests { ) .await .expect("should proxy publisher request"); + + finalize_test_publisher_response( + publisher_response, + settings, + services, + ®istry, + orchestrator, + finalizer, + ) + .await } - #[tokio::test] - async fn the_ec_preload_fetch_path_applies_the_same_cache_gate() { - // The call site a naive revert missed. Reverting only this one previously - // passed all 2,697 tests; this is the behavioral cover for it. - let stub = Arc::new(StubHttpClient::new()); - let services = services_with_cache_and_telemetry( - Arc::clone(&stub), + async fn finalize_test_publisher_response( + publisher_response: PublisherResponse, + settings: &Arc, + services: &RuntimeServices, + registry: &IntegrationRegistry, + orchestrator: Arc, + finalizer: Finalizer, + ) -> Response { + match finalizer { + Finalizer::Streaming => publisher_response_into_streaming_response( + publisher_response, + &Method::GET, + Arc::clone(settings), + registry, + orchestrator, + services.clone(), + ) + .await + .expect("should finalize publisher response"), + Finalizer::Buffered => buffer_publisher_response_async( + publisher_response, + &Method::GET, + settings, + registry, + &orchestrator, + services, + ) + .await + .expect("should finalize publisher response"), + } + } + + /// Collects a response body whether it was buffered or streamed. + /// + /// Both occur here by design: a response the gate authorized is buffered so it + /// can be stored, and one it refused keeps streaming exactly like `inline`. A + /// helper that assumed either would silently only ever test one of them. + async fn body_of(response: Response) -> Vec { + match response.into_body() { + EdgeBody::Stream(mut stream) => { + let mut out = Vec::new(); + while let Some(chunk) = stream.next().await { + out.extend_from_slice(&chunk.expect("stream chunk should read")); + } + out + } + other => other + .into_bytes() + .expect("a non-stream body has its bytes in hand") + .to_vec(), + } + } + + #[tokio::test] + async fn harness_emits_a_summary_row_for_an_ad_serving_navigation() { + let stub = Arc::new(StubHttpClient::new()); + let sink = Arc::new(RecordingTelemetrySink::default()); + let services = services_with_cache_and_telemetry( + Arc::clone(&stub), + Arc::new(MemoryTemplateCache::default()), + Arc::clone(&sink), + ); + let settings = Arc::new(settings_with_mode("esi")); + queue_shareable_html(&stub); + + let _ = run(&settings, &services, navigation_request()).await; + + assert!( + last_summary_row(&sink).is_some(), + "the harness must emit a summary row, or every assertion built on it is vacuous" + ); + } + + /// Drive `handle_publisher_request` through the **EC-preload** fetch path. + /// + /// `run_with_orchestrator` passes `kv: None` and no EC id, so + /// `should_preload_ec_snapshot` is false for every other test in this file and the + /// first of the two origin-fetch call sites is never exercised. Supplying both + /// reaches it. + async fn run_through_ec_preload( + settings: &Arc, + services: &RuntimeServices, + request: Request, + ) { + let orchestrator = Arc::new(AuctionOrchestrator::new(settings.auction.clone())); + let kv = crate::ec::kv::KvIdentityGraph::in_memory("test-store"); + let consent = crate::consent::ConsentContext { + jurisdiction: crate::consent::jurisdiction::Jurisdiction::NonRegulated, + ..Default::default() + }; + let mut ec_context = EcContext::new_for_test(Some("test-ec-id".to_owned()), consent); + + let _ = handle_publisher_request( + settings, + services, + Some(&kv), + &mut ec_context, + AuctionDispatch { + integration_registry: &IntegrationRegistry::empty_for_tests(), + orchestrator: &orchestrator, + slots: &[article_slot()], + registry: None, + }, + request, + EdgeCacheHeader::SMaxageFallback, + ) + .await + .expect("should proxy publisher request"); + } + + #[tokio::test] + async fn the_ec_preload_fetch_path_applies_the_same_cache_gate() { + // The call site a naive revert missed. Reverting only this one previously + // passed all 2,697 tests; this is the behavioral cover for it. + let stub = Arc::new(StubHttpClient::new()); + let services = services_with_cache_and_telemetry( + Arc::clone(&stub), Arc::new(MemoryTemplateCache::default()), Arc::new(RecordingTelemetrySink::default()), ); @@ -11393,6 +12268,7 @@ mod tests { Some(&graph), &mut ec_context, AuctionDispatch { + integration_registry: &IntegrationRegistry::empty_for_tests(), orchestrator: &orchestrator, slots: &[article_slot()], registry: None, @@ -12707,6 +13583,7 @@ mod tests { None, &mut ec_context, AuctionDispatch { + integration_registry: &IntegrationRegistry::empty_for_tests(), orchestrator: &orchestrator, slots: &[article_slot()], registry: None, @@ -12875,6 +13752,7 @@ mod tests { None, &mut ec_context, AuctionDispatch { + integration_registry: &IntegrationRegistry::empty_for_tests(), orchestrator: &orchestrator, slots: &[article_slot()], registry: None, @@ -13697,6 +14575,7 @@ mod tests { None, &mut ec_context, AuctionDispatch { + integration_registry: &IntegrationRegistry::empty_for_tests(), orchestrator: &orchestrator, slots: &[article_slot()], registry: None, @@ -14098,6 +14977,7 @@ mod tests { Some(&graph), &mut ec_context, AuctionDispatch { + integration_registry: &IntegrationRegistry::empty_for_tests(), orchestrator: &orchestrator, slots: &[article_slot()], registry: Some(&partners), @@ -15077,6 +15957,7 @@ mod tests { None, &mut ec_context, AuctionDispatch { + integration_registry: &IntegrationRegistry::empty_for_tests(), orchestrator: &orchestrator, slots: &[article_slot()], registry: None, @@ -16628,6 +17509,7 @@ mod tests { None, &mut ec_context, AuctionDispatch { + integration_registry: &IntegrationRegistry::empty_for_tests(), orchestrator, slots, registry: None, @@ -16689,6 +17571,7 @@ mod tests { Some(&graph), &mut ec_context, AuctionDispatch { + integration_registry: &IntegrationRegistry::empty_for_tests(), orchestrator: &orchestrator, slots: &slots, registry: None, @@ -16760,6 +17643,7 @@ mod tests { Some(&graph), &mut ec_context, AuctionDispatch { + integration_registry: &IntegrationRegistry::empty_for_tests(), orchestrator: &orchestrator, slots: &slots, registry: None, @@ -17916,6 +18800,7 @@ mod tests { None, &mut ec_context, AuctionDispatch { + integration_registry: &IntegrationRegistry::empty_for_tests(), orchestrator: &orchestrator, slots: &[], registry: None, @@ -19495,6 +20380,8 @@ mod tests { let body = EdgeBody::from(compressed); let params = OwnedProcessResponseParams { + response_status: StatusCode::OK, + range_associated: false, csp_nonce_observed: None, template_cache_key: None, seam_ad_slots: None, @@ -19554,6 +20441,8 @@ mod tests { .expect("should create integration registry"); let params = OwnedProcessResponseParams { + response_status: StatusCode::OK, + range_associated: false, csp_nonce_observed: None, template_cache_key: None, seam_ad_slots: None, @@ -19602,6 +20491,8 @@ mod tests { ) .expect("should create integration registry"); let params = OwnedProcessResponseParams { + response_status: StatusCode::OK, + range_associated: false, csp_nonce_observed: None, template_cache_key: None, seam_ad_slots: None, @@ -19728,6 +20619,8 @@ mod tests { let orchestrator = AuctionOrchestrator::new(settings.auction.clone()); let services = noop_services(); let mut params = OwnedProcessResponseParams { + response_status: StatusCode::OK, + range_associated: false, csp_nonce_observed: None, template_cache_key: None, seam_ad_slots: None, @@ -19792,6 +20685,8 @@ mod tests { let orchestrator = AuctionOrchestrator::new(settings.auction.clone()); let services = noop_services(); let mut params = OwnedProcessResponseParams { + response_status: StatusCode::OK, + range_associated: false, csp_nonce_observed: None, template_cache_key: None, seam_ad_slots: None, @@ -19859,6 +20754,8 @@ mod tests { let orchestrator = AuctionOrchestrator::new(settings.auction.clone()); let services = noop_services(); let mut params = OwnedProcessResponseParams { + response_status: StatusCode::OK, + range_associated: false, csp_nonce_observed: None, template_cache_key: None, seam_ad_slots: None, @@ -19926,6 +20823,8 @@ mod tests { let orchestrator = AuctionOrchestrator::new(settings.auction.clone()); let services = noop_services(); let mut params = OwnedProcessResponseParams { + response_status: StatusCode::OK, + range_associated: false, csp_nonce_observed: None, template_cache_key: None, seam_ad_slots: None, @@ -19993,6 +20892,8 @@ mod tests { let orchestrator = AuctionOrchestrator::new(settings.auction.clone()); let services = noop_services(); let mut params = OwnedProcessResponseParams { + response_status: StatusCode::OK, + range_associated: false, csp_nonce_observed: None, template_cache_key: None, seam_ad_slots: None, @@ -20042,6 +20943,8 @@ mod tests { fn non_html_stream_params(content_encoding: &str) -> OwnedProcessResponseParams { OwnedProcessResponseParams { + response_status: StatusCode::OK, + range_associated: false, csp_nonce_observed: None, template_cache_key: None, seam_ad_slots: None, @@ -20259,6 +21162,8 @@ mod tests { let services = noop_services(); let state = AdBidsState::default(); let mut params = OwnedProcessResponseParams { + response_status: StatusCode::OK, + range_associated: false, csp_nonce_observed: None, template_cache_key: None, seam_ad_slots: None, @@ -20334,6 +21239,8 @@ mod tests { let services = noop_services(); let state = AdBidsState::default(); let mut params = OwnedProcessResponseParams { + response_status: StatusCode::OK, + range_associated: false, csp_nonce_observed: None, template_cache_key: None, seam_ad_slots: None, @@ -20411,6 +21318,8 @@ mod tests { let orchestrator = AuctionOrchestrator::new(settings.auction.clone()); let services = noop_services(); let mut params = OwnedProcessResponseParams { + response_status: StatusCode::OK, + range_associated: false, csp_nonce_observed: None, template_cache_key: None, seam_ad_slots: None, @@ -20433,161 +21342,1721 @@ mod tests { gpt_diagnostics: None, suppress_datadome_client_side_tag: false, }; - let body = EdgeBody::stream(futures::stream::iter(vec![bytes::Bytes::from_static( - b"body{background:url('https://origin.example.com/asset.png')}", - )])); - let mut output = Vec::new(); - - stream_publisher_body_async( + let body = EdgeBody::stream(futures::stream::iter(vec![bytes::Bytes::from_static( + b"body{background:url('https://origin.example.com/asset.png')}", + )])); + let mut output = Vec::new(); + + stream_publisher_body_async( + body, + &mut output, + &mut params, + &settings, + ®istry, + &orchestrator, + &services, + ) + .await + .expect("non-html stream body should process after auction collection"); + + let css = String::from_utf8(output).expect("should be valid UTF-8"); + assert!( + css.contains("proxy.example.com"), + "should rewrite non-html stream after auction collection. Got: {css}" + ); + assert!( + !css.contains("origin.example.com"), + "should not leave origin host after rewrite. Got: {css}" + ); + }); + } + + fn drain_streaming_finalize_body(content_encoding: &str, body: EdgeBody) -> Vec { + let settings = Arc::new(create_test_settings()); + let registry = Arc::new( + IntegrationRegistry::with_plan( + &settings, + Arc::new( + crate::auction::compile_auction_plan(&settings) + .expect("should compile auction plan"), + ), + ) + .expect("should create integration registry"), + ); + let orchestrator = Arc::new(AuctionOrchestrator::new(settings.auction.clone())); + let services = noop_services(); + let response = Response::builder() + .status(StatusCode::OK) + .header(header::CONTENT_TYPE, "text/css") + .body(EdgeBody::empty()) + .expect("should build response"); + let params = OwnedProcessResponseParams { + response_status: StatusCode::OK, + range_associated: false, + csp_nonce_observed: None, + template_cache_key: None, + seam_ad_slots: None, + policy_headers: Vec::new(), + content_encoding: content_encoding.to_string(), + origin_host: "origin.example.com".to_string(), + origin_url: "https://origin.example.com".to_string(), + request_host: "proxy.example.com".to_string(), + request_scheme: "https".to_string(), + content_type: "text/css".to_string(), + ad_slots_script: None, + ad_bids_state: AdBidsState::default(), + auction_observation: None, + auction_request: None, + dispatched_auction: None, + price_granularity: crate::price_bucket::PriceGranularity::default(), + gpt_diagnostics: None, + suppress_datadome_client_side_tag: false, + }; + let publisher_response = PublisherResponse::Stream { + response, + body, + params: Box::new(params), + }; + + let response = futures::executor::block_on(publisher_response_into_streaming_response( + publisher_response, + &Method::GET, + Arc::clone(&settings), + registry.as_ref(), + orchestrator, + services, + )) + .expect("should build streaming response"); + + assert!( + matches!(response.body(), EdgeBody::Stream(_)), + "streaming finalize should keep a lazy Body::Stream" + ); + + futures::executor::block_on( + response + .into_body() + .into_bytes_bounded(settings.publisher.max_buffered_body_bytes), + ) + .expect("streaming body should drain") + .to_vec() + } + + #[test] + fn publisher_response_streaming_finalize_keeps_stream_body_lazy() { + let body_bytes = drain_streaming_finalize_body( + "", + EdgeBody::stream(futures::stream::iter(vec![bytes::Bytes::from_static( + b"body{background:url('https://origin.example.com/asset.png')}", + )])), + ); + let css = String::from_utf8(body_bytes).expect("should be valid UTF-8"); + assert!( + css.contains("proxy.example.com"), + "streaming response body should still run publisher rewriting. Got: {css}" + ); + assert!( + !css.contains("origin.example.com"), + "streaming response body should not leave origin URLs unrewritten. Got: {css}" + ); + } + + /// An origin body stream that yields `chunk` once and then stays `Pending` + /// forever without ever signalling EOF — modelling an origin that has sent + /// the document head but not yet finished the response. + fn origin_chunk_then_pending(chunk: bytes::Bytes) -> EdgeBody { + let mut sent = false; + EdgeBody::from_stream(futures::stream::poll_fn(move |_cx| { + if sent { + return std::task::Poll::Pending; + } + sent = true; + std::task::Poll::Ready(Some(Ok::<_, io::Error>(chunk.clone()))) + })) + } + + /// Poll a lazy `Body::Stream` exactly once and return the first emitted + /// chunk. Panics if the first poll is `Pending` (nothing emitted before the + /// origin would need its next chunk) or the stream ends. + fn first_lazy_body_chunk(body: EdgeBody) -> bytes::Bytes { + let mut stream = body + .into_stream() + .expect("streaming finalize should keep a lazy Body::Stream"); + let waker = futures::task::noop_waker(); + let mut cx = std::task::Context::from_waker(&waker); + let next = futures::StreamExt::next(&mut stream); + let mut next = std::pin::pin!(next); + match std::future::Future::poll(next.as_mut(), &mut cx) { + std::task::Poll::Ready(Some(Ok(bytes))) => bytes, + std::task::Poll::Ready(other) => { + panic!("first poll must emit a chunk, got Ready({other:?})") + } + std::task::Poll::Pending => { + panic!("first poll must emit rewritten content before origin EOF, got Pending") + } + } + } + + struct AsyncTailFactory(Arc); + struct AsyncTailProcessor(Arc); + + impl crate::integrations::IntegrationHtmlStreamProcessorFactory for AsyncTailFactory { + fn integration_id(&self) -> &'static str { + "example_async_tail" + } + fn requires_async_finish(&self) -> bool { + true + } + fn create( + &self, + _: crate::integrations::IntegrationHtmlStreamContext, + ) -> Box { + Box::new(AsyncTailProcessor(Arc::clone(&self.0))) + } + } + + impl StreamProcessor for AsyncTailProcessor { + fn process_chunk(&mut self, chunk: &[u8], _: bool) -> std::io::Result> { + Ok(chunk.to_vec()) + } + fn requires_async_finish(&self) -> bool { + true + } + fn async_finish<'a>( + &'a mut self, + _: crate::streaming_processor::finish::StreamFinishInput<'a>, + ) -> Option< + futures::future::LocalBoxFuture< + 'a, + Result< + crate::streaming_processor::finish::StreamFinish, + Report, + >, + >, + > { + Some(Box::pin(async move { + self.0.fetch_add(1, Ordering::SeqCst); + futures::future::pending().await + })) + } + } + + struct UndeclaredAsyncFactory(Arc); + impl crate::integrations::IntegrationHtmlStreamProcessorFactory for UndeclaredAsyncFactory { + fn integration_id(&self) -> &'static str { + "example_inconsistent_factory" + } + fn create( + &self, + _: crate::integrations::IntegrationHtmlStreamContext, + ) -> Box { + Box::new(AsyncTailProcessor(Arc::clone(&self.0))) + } + } + + #[test] + fn async_eof_factory_mismatch_rejects_before_stream_reads() { + let settings = create_test_settings(); + let reads = Arc::new(AtomicUsize::new(0)); + let calls = Arc::new(AtomicUsize::new(0)); + let registry = IntegrationRegistry::from_html_stream_processors(vec![Arc::new( + UndeclaredAsyncFactory(Arc::clone(&calls)), + )]); + let observed_reads = Arc::clone(&reads); + let body = EdgeBody::stream(futures::stream::poll_fn(move |_| { + observed_reads.fetch_add(1, Ordering::SeqCst); + std::task::Poll::Ready(Some(bytes::Bytes::from_static(b"

Example

"))) + })); + let mut output = Vec::new(); + let mut params = html_stream_params("", None); + let _ = futures::executor::block_on(stream_publisher_body_async( + body, + &mut output, + &mut params, + &settings, + ®istry, + &AuctionOrchestrator::new(settings.auction.clone()), + &noop_services(), + )) + .expect_err("should reject inconsistent EOF declarations before input effects"); + assert_eq!( + reads.load(Ordering::SeqCst), + 0, + "should not consume a stream before capability validation" + ); + assert!( + output.is_empty(), + "should not commit output for inconsistent declarations" + ); + assert_eq!( + calls.load(Ordering::SeqCst), + 0, + "should not start undeclared EOF I/O" + ); + } + + #[test] + fn async_eof_lazy_delivery_releases_page_before_pending_io() { + for streamed in [false, true] { + let settings = Arc::new(crate::publisher::allow_eof_test_settings( + create_test_settings(), + )); + let calls = Arc::new(AtomicUsize::new(0)); + let registry = IntegrationRegistry::from_html_stream_processors(vec![Arc::new( + AsyncTailFactory(Arc::clone(&calls)), + )]); + let html = b"

Example renderable page

"; + let body = if streamed { + EdgeBody::stream(futures::stream::iter(vec![bytes::Bytes::copy_from_slice( + html, + )])) + } else { + EdgeBody::from(html.to_vec()) + }; + let publisher = PublisherResponse::Stream { + response: Response::builder() + .status(200) + .header(header::CONTENT_TYPE, "text/html") + .body(EdgeBody::empty()) + .expect("should build example response"), + body, + params: Box::new(html_stream_params("", None)), + }; + let response = futures::executor::block_on(publisher_response_into_streaming_response( + publisher, + &Method::GET, + Arc::clone(&settings), + ®istry, + Arc::new(AuctionOrchestrator::new(settings.auction.clone())), + noop_services(), + )) + .expect("should prepare lazy EOF output"); + let EdgeBody::Stream(mut stream) = response.into_body() else { + panic!("should return a lazy body"); + }; + let first = futures::executor::block_on(stream.next()) + .expect("should yield ready HTML") + .expect("should deliver prefix"); + assert!( + String::from_utf8_lossy(&first).contains("Example renderable page"), + "should release the page before optional EOF I/O" + ); + assert_eq!( + calls.load(Ordering::SeqCst), + 0, + "should not start the callback before ready bytes are yielded" + ); + let waker = futures::task::noop_waker(); + let mut context = std::task::Context::from_waker(&waker); + let mut pending = false; + for _ in 0..16 { + let mut next = Box::pin(stream.next()); + match core::future::Future::poll(next.as_mut(), &mut context) { + std::task::Poll::Pending => { + pending = true; + break; + } + std::task::Poll::Ready(Some(Ok(_))) => {} + _ => panic!("should reach pending optional EOF work after delivering the page"), + } + } + assert!(pending, "should wait only after ready output is delivered"); + assert_eq!(calls.load(Ordering::SeqCst), 1, "should start EOF I/O once"); + } + } + + #[test] + fn html_finalization_applies_privacy_and_full_body_policy_without_an_auction() { + let settings = crate::publisher::allow_eof_test_settings(create_test_settings()); + let registry = + IntegrationRegistry::from_html_finalizers(vec![Arc::new(TerminalTestFactory)]); + let http = Arc::new(StubHttpClient::new()); + http.push_response_with_headers( + 404, + b"Example missing article".to_vec(), + vec![ + ("content-type", "text/html"), + ("cache-control", "public, max-age=60"), + ("etag", "example-tag"), + ], + ); + let services = build_services_with_http_client( + Arc::clone(&http) as Arc + ); + let orchestrator = AuctionOrchestrator::new(settings.auction.clone()); + let request = Request::builder() + .method(Method::GET) + .uri("https://proxy.example.com/article") + .header(header::HOST, "proxy.example.com") + .header(header::ACCEPT, "text/html") + .header(header::IF_NONE_MATCH, "example-tag") + .header(header::IF_MODIFIED_SINCE, "Mon, 01 Jan 2024 00:00:00 GMT") + .header(header::RANGE, "bytes=0-10") + .header(header::IF_RANGE, "example-tag") + .header(header::IF_MATCH, "example-precondition") + .body(EdgeBody::empty()) + .expect("should build a navigation"); + let result = futures::executor::block_on(handle_publisher_request( + &settings, + &services, + None, + &mut EcContext::default(), + AuctionDispatch { + orchestrator: &orchestrator, + slots: &[], + registry: None, + integration_registry: ®istry, + }, + request, + EdgeCacheHeader::SurrogateControl, + )) + .expect("should fetch the origin"); + let PublisherResponse::Stream { response, .. } = result else { + panic!("should select processable HTML"); + }; + assert_eq!( + response.status(), + StatusCode::NOT_FOUND, + "should preserve non-2xx status" + ); + assert_eq!( + response + .headers() + .get(header::CACHE_CONTROL) + .and_then(|value| value.to_str().ok()), + Some(crate::cache_policy::NO_STORE_PRIVATE_CACHE_CONTROL), + "should make finalized HTML private even without ads" + ); + assert!( + !response.headers().contains_key(header::ETAG), + "should strip origin validators" + ); + let requests = http.recorded_request_headers(); + let forwarded = &requests[0]; + for name in ["if-none-match", "if-modified-since", "range", "if-range"] { + assert!( + !forwarded.iter().any(|(header, _)| header == name), + "should request complete HTML" + ); + } + assert!( + forwarded.iter().any(|(header, _)| header == "if-match"), + "should preserve mutation preconditions" + ); + } + + struct CountingTerminalFactory(Arc); + struct CountingTerminalSession(Arc); + + impl crate::integrations::IntegrationHtmlFinalizerFactory for CountingTerminalFactory { + fn integration_id(&self) -> &'static str { + "example_counter" + } + fn create( + &self, + _: crate::integrations::IntegrationHtmlStreamContext, + ) -> Box { + Box::new(CountingTerminalSession(Arc::clone(&self.0))) + } + } + impl crate::integrations::IntegrationHtmlFinalizer for CountingTerminalSession { + fn finalize<'a>( + &'a mut self, + input: crate::integrations::HtmlFinalizationInput<'a>, + ) -> futures::future::LocalBoxFuture< + 'a, + Result< + crate::integrations::HtmlFinalization, + Report, + >, + > { + Box::pin(async move { + self.0.fetch_add(1, Ordering::SeqCst); + input.replacement.append(b"discarded candidate")?; + Ok(crate::integrations::HtmlFinalization::Unchanged) + }) + } + } + + #[test] + fn html_finalization_preserves_range_associated_responses_on_both_sinks() { + for (status, content_range) in [(206, true), (206, false), (200, true), (416, true)] { + for streamed in [false, true] { + for buffered in [false, true] { + for encoding in ["", "gzip"] { + for auction in [false, true] { + let settings = Arc::new(crate::publisher::allow_eof_test_settings( + create_test_settings(), + )); + let calls = Arc::new(AtomicUsize::new(0)); + let registry = + IntegrationRegistry::from_html_finalizers(vec![Arc::new( + CountingTerminalFactory(Arc::clone(&calls)), + )]); + let raw = b"

Example range fragment

"; + let encoded = if encoding == "gzip" { + gzip_encode(raw) + } else { + raw.to_vec() + }; + let body = if streamed { + EdgeBody::stream(futures::stream::iter(vec![bytes::Bytes::from( + encoded.clone(), + )])) + } else { + EdgeBody::from(encoded.clone()) + }; + let mut builder = Response::builder() + .status(status) + .header(header::CONTENT_TYPE, "text/html") + .header(header::CONTENT_ENCODING, encoding) + .header(header::CONTENT_LENGTH, encoded.len()) + .header(header::ETAG, "example-range-tag") + .header(header::CACHE_CONTROL, "public, max-age=60"); + if content_range { + builder = builder.header(header::CONTENT_RANGE, "bytes 0-29/100"); + } + let response = builder + .body(EdgeBody::empty()) + .expect("should build range metadata"); + let headers = response.headers().clone(); + let mut params = html_stream_params( + encoding, + auction.then(|| { + DispatchedAuction::empty_for_test(test_auction_request(), 500) + }), + ); + let sink = Arc::new(RecordingTelemetrySink::default()); + let services = + noop_services_with_telemetry_sink(Arc::clone(&sink) as _); + if auction { + params.auction_observation = + Some(AuctionObservationContext::from_parts( + AuctionSource::InitialNavigation, + "proxy.example.com", + "/article", + 1, + None, + &EcContext::default(), + )); + } + let publisher = PublisherResponse::Stream { + response, + body, + params: Box::new(params), + }; + let orchestrator = + Arc::new(AuctionOrchestrator::new(settings.auction.clone())); + let response = if buffered { + futures::executor::block_on(buffer_publisher_response_async( + publisher, + &Method::GET, + &settings, + ®istry, + &orchestrator, + &services, + )) + } else { + futures::executor::block_on( + publisher_response_into_streaming_response( + publisher, + &Method::GET, + settings, + ®istry, + orchestrator, + services, + ), + ) + } + .expect("should preserve range-associated HTML"); + assert_eq!( + response.status().as_u16(), + status, + "should preserve range status" + ); + assert_eq!( + response.headers(), + &headers, + "should preserve range framing and policy" + ); + let actual = futures::executor::block_on( + response.into_body().into_bytes_bounded(1024), + ) + .expect("should drain the original range bytes"); + assert_eq!( + actual.as_ref(), + encoded.as_slice(), + "should preserve encoded range bytes" + ); + assert_eq!( + calls.load(Ordering::SeqCst), + 0, + "should never finalize a range-associated response" + ); + let batches = sink.batches.lock().expect("should lock range telemetry"); + let summaries: Vec<_> = batches + .iter() + .flat_map(crate::auction::telemetry::AuctionEventBatch::rows) + .filter(|row| row.event_kind == "summary") + .collect(); + assert_eq!( + summaries.len(), + usize::from(auction), + "should abandon a dispatched auction exactly once" + ); + if auction { + assert_eq!( + summaries[0].terminal_status.as_deref(), + Some("abandoned"), + "should report range bypass abandonment" + ); + } + } + } + } + } + } + } + + #[test] + fn html_finalization_bypasses_ranged_fetches_and_unsolicited_partial_origins() { + for destination in ["empty", "document"] { + for (status, content_range) in [(206, false), (206, true), (200, true), (416, true)] { + let settings = Arc::new(crate::publisher::allow_eof_test_settings( + create_test_settings(), + )); + let calls = Arc::new(AtomicUsize::new(0)); + let registry = IntegrationRegistry::from_html_finalizers(vec![Arc::new( + CountingTerminalFactory(Arc::clone(&calls)), + )]); + let http = Arc::new(StubHttpClient::new()); + let raw = b"

Example fragment

"; + let mut headers = vec![("content-type", "text/html")]; + if content_range { + headers.push(("content-range", "bytes 0-22/100")); + } + http.push_response_with_headers(status, raw.to_vec(), headers); + let services = build_services_with_http_client(http.clone()); + let orchestrator = Arc::new(AuctionOrchestrator::new(settings.auction.clone())); + let request = Request::builder() + .method(Method::GET) + .uri("https://proxy.example.com/article") + .header(header::HOST, "proxy.example.com") + .header(header::ACCEPT, "text/html") + .header("sec-fetch-dest", destination) + .header(header::RANGE, "bytes=0-22") + .body(EdgeBody::empty()) + .expect("should build a ranged fetch"); + let publisher = futures::executor::block_on(handle_publisher_request( + &settings, + &services, + None, + &mut EcContext::default(), + AuctionDispatch { + integration_registry: ®istry, + orchestrator: &orchestrator, + slots: &[], + registry: None, + }, + request, + EdgeCacheHeader::SurrogateControl, + )) + .expect("should fetch a range-associated response"); + let response = + futures::executor::block_on(publisher_response_into_streaming_response( + publisher, + &Method::GET, + settings, + ®istry, + orchestrator, + services, + )) + .expect("should bypass terminal work on partial origin HTML"); + let actual = + futures::executor::block_on(response.into_body().into_bytes_bounded(1024)) + .expect("should drain the range fragment"); + assert_eq!( + calls.load(Ordering::SeqCst), + 0, + "should skip optional I/O on partial origin HTML" + ); + assert_eq!(actual.as_ref(), raw, "should preserve the origin fragment"); + assert_eq!( + http.recorded_request_headers()[0] + .iter() + .any(|(name, _)| name == "range"), + destination == "empty", + "should preserve non-document range semantics" + ); + } + } + } + + #[test] + fn html_finalization_direct_helpers_copy_encoded_ranges_without_optional_work() { + for streamed in [false, true] { + let settings = crate::publisher::allow_eof_test_settings(create_test_settings()); + let calls = Arc::new(AtomicUsize::new(0)); + let registry = IntegrationRegistry::from_html_finalizers(vec![Arc::new( + CountingTerminalFactory(Arc::clone(&calls)), + )]); + let encoded = gzip_encode(b"

Example partial response

"); + let body = if streamed { + EdgeBody::stream(futures::stream::iter(vec![bytes::Bytes::from( + encoded.clone(), + )])) + } else { + EdgeBody::from(encoded.clone()) + }; + let mut params = html_stream_params("gzip", None); + params.response_status = StatusCode::PARTIAL_CONTENT; + params.range_associated = streamed; + let orchestrator = AuctionOrchestrator::new(settings.auction.clone()); + let mut output = Vec::new(); + futures::executor::block_on(stream_publisher_body_async( + body, + &mut output, + &mut params, + &settings, + ®istry, + &orchestrator, + &noop_services(), + )) + .expect("should copy an encoded partial representation"); + assert_eq!( + output, encoded, + "should avoid decoding or rewriting range bytes" + ); + assert_eq!( + calls.load(Ordering::SeqCst), + 0, + "should skip terminal I/O on direct range delivery" + ); + if !streamed { + output.clear(); + stream_publisher_body( + EdgeBody::from(encoded.clone()), + &mut output, + ¶ms, + &settings, + ®istry, + ) + .expect("should copy an encoded range on the synchronous helper"); + assert_eq!( + output, encoded, + "should preserve the direct synchronous range bytes" + ); + } + } + } + + #[test] + fn html_finalization_flushes_at_the_terminal_barrier_instead_of_each_replay_chunk() { + struct FlushCounter { + bytes: Vec, + flushes: usize, + } + impl Write for FlushCounter { + fn write(&mut self, bytes: &[u8]) -> io::Result { + self.bytes.extend_from_slice(bytes); + Ok(bytes.len()) + } + fn flush(&mut self) -> io::Result<()> { + self.flushes += 1; + Ok(()) + } + } + let settings = crate::publisher::allow_eof_test_settings(create_test_settings()); + let calls = Arc::new(AtomicUsize::new(0)); + let registry = IntegrationRegistry::from_html_finalizers(vec![Arc::new( + CountingTerminalFactory(Arc::clone(&calls)), + )]); + let mut source = b"

".to_vec(); + source.extend(vec![b'x'; 5 * STREAM_CHUNK_SIZE]); + source.extend_from_slice(b"

"); + let orchestrator = AuctionOrchestrator::new(settings.auction.clone()); + let mut output = FlushCounter { + bytes: Vec::new(), + flushes: 0, + }; + futures::executor::block_on(stream_publisher_body_async( + EdgeBody::from(source), + &mut output, + &mut html_stream_params("", None), + &settings, + ®istry, + &orchestrator, + &noop_services(), + )) + .expect("should replay a complete large HTML document"); + assert!( + output.bytes.len() > 5 * STREAM_CHUNK_SIZE, + "should replay multiple bounded chunks" + ); + assert_eq!( + output.flushes, 2, + "should flush the EOF barrier and final output, not every replay chunk" + ); + assert_eq!( + calls.load(Ordering::SeqCst), + 1, + "should retain one terminal callback" + ); + } + + #[test] + fn html_finalization_flushes_all_ready_prefix_before_a_pending_auction() { + struct FlushProbe { + bytes: Arc>>, + flushed_bytes: Arc, + flushes: Arc, + } + impl Write for FlushProbe { + fn write(&mut self, bytes: &[u8]) -> io::Result { + self.bytes + .lock() + .expect("should record ready bytes") + .extend_from_slice(bytes); + Ok(bytes.len()) + } + fn flush(&mut self) -> io::Result<()> { + self.flushed_bytes.store( + self.bytes + .lock() + .expect("should inspect flushed bytes") + .len(), + Ordering::SeqCst, + ); + self.flushes.fetch_add(1, Ordering::SeqCst); + Ok(()) + } + } + for closed in [false, true] { + let mut settings = crate::publisher::allow_eof_test_settings(create_test_settings()); + settings.auction.enabled = true; + settings.auction.providers = + crate::auction::AuctionConfig::legacy_provider_map(&["seam-test"]); + settings.auction.timeout_ms = 60_000; + settings.auction.mediator = None; + let client = Arc::new(GatedAuctionHttpClient { + inner: StubHttpClient::new(), + released: AtomicBool::new(false), + collections: AtomicUsize::new(0), + }); + client.inner.push_response(200, Vec::new()); + let services = build_services_with_http_client(client.clone()); + let mut orchestrator = AuctionOrchestrator::new(settings.auction.clone()); + orchestrator.register_provider(Arc::new(GatedAuctionProvider)); + let request = Request::new(EdgeBody::empty()); + let dispatched = futures::executor::block_on(orchestrator.dispatch_auction( + &test_auction_request(), + &AuctionContext { + settings: &settings, + request: &request, + timeout_ms: 60_000, + transport_timeout_ms: 60_000, + provider_responses: None, + services: &services, + }, + )); + let DispatchAuctionOutcome::Dispatched(dispatched) = dispatched else { + panic!("should dispatch a controlled pending auction"); + }; + let calls = Arc::new(AtomicUsize::new(0)); + let registry = IntegrationRegistry::from_html_finalizers(vec![Arc::new( + CountingTerminalFactory(Arc::clone(&calls)), + )]); + let mut source = b"".to_vec(); + source.extend(vec![b'x'; 5 * STREAM_CHUNK_SIZE]); + source.extend_from_slice(b"Example barrier end"); + if closed { + source.extend_from_slice(b""); + } + let bytes = Arc::new(Mutex::new(Vec::new())); + let flushed = Arc::new(AtomicUsize::new(0)); + let flushes = Arc::new(AtomicUsize::new(0)); + let mut writer = FlushProbe { + bytes: Arc::clone(&bytes), + flushed_bytes: Arc::clone(&flushed), + flushes: Arc::clone(&flushes), + }; + let mut params = html_stream_params("", Some(dispatched)); + let mut work = Box::pin(stream_publisher_body_async( + EdgeBody::from(source), + &mut writer, + &mut params, + &settings, + ®istry, + &orchestrator, + &services, + )); + let waker = futures::task::noop_waker(); + let mut context = std::task::Context::from_waker(&waker); + assert!( + work.as_mut().poll(&mut context).is_pending(), + "should await the controlled auction" + ); + let prefix = bytes + .lock() + .expect("should inspect the delivered prefix") + .clone(); + assert!( + prefix + .windows(b"Example barrier end".len()) + .any(|window| window == b"Example barrier end"), + "should deliver the whole ready prefix before collection" + ); + assert_eq!( + flushed.load(Ordering::SeqCst), + prefix.len(), + "should flush every ready prefix byte before collection" + ); + assert_eq!( + flushes.load(Ordering::SeqCst), + 1, + "should use one pre-auction flush barrier" + ); + assert_eq!( + client.collections.load(Ordering::SeqCst), + 1, + "should poll the pending auction once" + ); + client.released.store(true, Ordering::SeqCst); + futures::executor::block_on(work) + .expect("should complete once the auction is released"); + assert_eq!( + calls.load(Ordering::SeqCst), + 1, + "should finish HTML before auction collection" + ); + } + } + + struct StatusTerminalFactory(Arc>>); + struct StatusTerminalSession(Arc>>); + + impl crate::integrations::IntegrationHtmlFinalizerFactory for StatusTerminalFactory { + fn integration_id(&self) -> &'static str { + "example_status" + } + + fn create( + &self, + _: crate::integrations::IntegrationHtmlStreamContext, + ) -> Box { + Box::new(StatusTerminalSession(Arc::clone(&self.0))) + } + } + + impl crate::integrations::IntegrationHtmlFinalizer for StatusTerminalSession { + fn finalize<'a>( + &'a mut self, + input: crate::integrations::HtmlFinalizationInput<'a>, + ) -> futures::future::LocalBoxFuture< + 'a, + Result< + crate::integrations::HtmlFinalization, + Report, + >, + > { + Box::pin(async move { + self.0 + .lock() + .expect("should record the response status") + .push(input.response_status); + Ok(crate::integrations::HtmlFinalization::Unchanged) + }) + } + } + + #[test] + fn html_finalization_exposes_actual_success_and_error_status_on_both_sinks() { + for buffered in [false, true] { + let statuses = Arc::new(Mutex::new(Vec::new())); + let registry = IntegrationRegistry::from_html_finalizers(vec![Arc::new( + StatusTerminalFactory(Arc::clone(&statuses)), + )]); + for status in [200, 404, 500] { + let settings = Arc::new(crate::publisher::allow_eof_test_settings( + create_test_settings(), + )); + let http = Arc::new(StubHttpClient::new()); + http.push_response_with_headers( + status, + b"Example complete response".to_vec(), + vec![("content-type", "text/html")], + ); + let services = build_services_with_http_client(http); + let orchestrator = Arc::new(AuctionOrchestrator::new(settings.auction.clone())); + let request = Request::builder() + .uri("https://proxy.example.com/article") + .header(header::HOST, "proxy.example.com") + .header(header::ACCEPT, "text/html") + .body(EdgeBody::empty()) + .expect("should build a complete document request"); + let publisher = futures::executor::block_on(handle_publisher_request( + &settings, + &services, + None, + &mut EcContext::default(), + AuctionDispatch { + integration_registry: ®istry, + orchestrator: &orchestrator, + slots: &[], + registry: None, + }, + request, + EdgeCacheHeader::SurrogateControl, + )) + .expect("should fetch complete success or error HTML"); + let response = if buffered { + futures::executor::block_on(buffer_publisher_response_async( + publisher, + &Method::GET, + &settings, + ®istry, + &orchestrator, + &services, + )) + } else { + futures::executor::block_on(publisher_response_into_streaming_response( + publisher, + &Method::GET, + settings, + ®istry, + orchestrator, + services, + )) + } + .expect("should finalize complete success or error HTML"); + assert_eq!( + response.status().as_u16(), + status, + "should retain the origin status" + ); + let _ = futures::executor::block_on( + response.into_body().into_bytes_bounded(1024 * 1024), + ) + .expect("should drive the complete terminal response"); + } + assert_eq!( + *statuses.lock().expect("should inspect observed statuses"), + [ + StatusCode::OK, + StatusCode::NOT_FOUND, + StatusCode::INTERNAL_SERVER_ERROR + ], + "should expose status without excluding complete error documents" + ); + } + } + + #[test] + fn html_finalization_handles_eligible_and_excluded_responses() { + for (method, status, content_type, encoding) in [ + (Method::GET, 200, "text/html", ""), + (Method::HEAD, 200, "text/html", ""), + (Method::GET, 204, "text/html", ""), + (Method::GET, 205, "text/html", ""), + (Method::GET, 304, "text/html", ""), + (Method::GET, 200, "text/plain", ""), + (Method::GET, 200, "text/html", "example-unsupported"), + ] { + let settings = Arc::new(crate::publisher::allow_eof_test_settings( + create_test_settings(), + )); + let calls = Arc::new(AtomicUsize::new(0)); + let registry = IntegrationRegistry::from_html_finalizers(vec![Arc::new( + CountingTerminalFactory(Arc::clone(&calls)), + )]); + let http = Arc::new(StubHttpClient::new()); + http.push_response_with_headers( + status, + b"Example body".to_vec(), + vec![ + ("content-type", content_type), + ("content-encoding", encoding), + ], + ); + let services = build_services_with_http_client(http); + let orchestrator = Arc::new(AuctionOrchestrator::new(settings.auction.clone())); + let request = Request::builder() + .method(method.clone()) + .uri("https://proxy.example.com/article") + .header(header::HOST, "proxy.example.com") + .header(header::ACCEPT, "text/html") + .body(EdgeBody::empty()) + .expect("should build a fixture request"); + let response = futures::executor::block_on(handle_publisher_request( + &settings, + &services, + None, + &mut EcContext::default(), + AuctionDispatch { + integration_registry: ®istry, + orchestrator: &orchestrator, + slots: &[], + registry: None, + }, + request, + EdgeCacheHeader::SurrogateControl, + )) + .expect("should classify the origin response"); + let response = futures::executor::block_on(publisher_response_into_streaming_response( + response, + &method, + Arc::clone(&settings), + ®istry, + orchestrator, + services, + )) + .expect("should preserve excluded response behavior"); + let _ = + futures::executor::block_on(response.into_body().into_bytes_bounded(1024 * 1024)) + .expect("should drain excluded output"); + let eligible = method == Method::GET + && status == 200 + && content_type == "text/html" + && encoding.is_empty(); + assert_eq!( + calls.load(Ordering::SeqCst), + usize::from(eligible), + "should invoke only eligible callbacks: method={method}, status={status}, type={content_type}, encoding={encoding}" + ); + } + } + + #[test] + fn html_finalization_collection_overflow_replays_rewritten_expansion_and_stays_private() { + let mut settings = crate::publisher::allow_eof_test_settings(create_test_settings()); + settings.publisher.max_buffered_body_bytes = 128; + let settings = Arc::new(settings); + let calls = Arc::new(AtomicUsize::new(0)); + let registry = IntegrationRegistry::from_html_finalizers(vec![Arc::new( + CountingTerminalFactory(Arc::clone(&calls)), + )]); + let source = b"Example"; + let response = PublisherResponse::Stream { + response: Response::builder() + .status(200) + .header(header::CONTENT_TYPE, "text/html") + .body(EdgeBody::empty()) + .expect("should build overflow output"), + body: EdgeBody::stream(futures::stream::iter( + source + .chunks(7) + .map(bytes::Bytes::copy_from_slice) + .collect::>(), + )), + params: Box::new(html_stream_params("", None)), + }; + let orchestrator = Arc::new(AuctionOrchestrator::new(settings.auction.clone())); + let response = futures::executor::block_on(publisher_response_into_streaming_response( + response, + &Method::GET, + Arc::clone(&settings), + ®istry, + orchestrator, + noop_services(), + )) + .expect("should create lazy overflow output"); + assert_eq!( + response + .headers() + .get(header::CACHE_CONTROL) + .expect("should retain privacy"), + crate::cache_policy::NO_STORE_PRIVATE_CACHE_CONTROL, + "should prevent caching even when collection bypasses I/O" + ); + let body = + futures::executor::block_on(response.into_body().into_bytes_bounded(1024 * 1024)) + .expect("should replay overflow output"); + let control_registry = IntegrationRegistry::from_html_stream_processors(Vec::new()); + let mut processor = create_html_stream_processor(HtmlStreamProcessorParams { + origin_host: "origin.example.com", + request_host: "proxy.example.com", + request_scheme: "https", + settings: &settings, + integration_registry: &control_registry, + ad_slots_script: html_stream_params("", None).ad_slots_script, + ad_bids_state: Arc::new(Mutex::new(None)), + suppress_datadome_client_side_tag: false, + gpt_diagnostics: None, + shared_template_authorized: false, + csp_nonce_observed: None, + deferred_inline_marker: None, + }) + .expect("should build a synchronous control"); + let mut expected = Vec::new(); + for chunk in source.chunks(7) { + expected.extend( + processor + .process_chunk(chunk, false) + .expect("should process the control chunk"), + ); + } + expected.extend( + processor + .process_chunk(&[], true) + .expect("should finish the control"), + ); + assert_eq!( + body.as_ref(), + expected, + "should emit every transformed byte exactly once after bypass" + ); + assert_eq!( + calls.load(Ordering::SeqCst), + 0, + "should skip terminal I/O after retention overflow" + ); + } + + #[test] + fn html_finalization_rejects_corrupt_codecs_without_starting_optional_io() { + let page = b"Example"; + for encoding in ["gzip", "deflate", "br"] { + let mut encoded = match encoding { + "gzip" => gzip_encode(page), + "deflate" => deflate_encode(page), + _ => brotli_encode(page), + }; + encoded.truncate(encoded.len() / 2); + for streamed in [false, true] { + let settings = Arc::new(crate::publisher::allow_eof_test_settings( + create_test_settings(), + )); + let calls = Arc::new(AtomicUsize::new(0)); + let registry = IntegrationRegistry::from_html_finalizers(vec![Arc::new( + CountingTerminalFactory(Arc::clone(&calls)), + )]); + let body = if streamed { + EdgeBody::stream(futures::stream::iter(vec![bytes::Bytes::from( + encoded.clone(), + )])) + } else { + EdgeBody::from(encoded.clone()) + }; + let response = PublisherResponse::Stream { + response: Response::builder() + .status(200) + .header(header::CONTENT_TYPE, "text/html") + .body(EdgeBody::empty()) + .expect("should build corrupt-codec response"), + body, + params: Box::new(html_stream_params(encoding, None)), + }; + let orchestrator = Arc::new(AuctionOrchestrator::new(settings.auction.clone())); + let response = + futures::executor::block_on(publisher_response_into_streaming_response( + response, + &Method::GET, + settings, + ®istry, + orchestrator, + noop_services(), + )) + .expect("should create a lazy decode path"); + let _ = futures::executor::block_on( + response.into_body().into_bytes_bounded(1024 * 1024), + ) + .expect_err("should fail corrupt input before terminal EOF"); + assert_eq!( + calls.load(Ordering::SeqCst), + 0, + "should validate {encoding} before outside work: streamed={streamed}" + ); + } + } + } + + std::thread_local! { + static CANCELLATION_WARNINGS: std::cell::Cell> = const { std::cell::Cell::new(None) }; + } + + struct CancellationLogger; + impl log::Log for CancellationLogger { + fn enabled(&self, metadata: &log::Metadata<'_>) -> bool { + metadata.level() <= log::Level::Warn + } + fn log(&self, record: &log::Record<'_>) { + if record + .args() + .to_string() + .starts_with("Dispatched server-side auction dropped without collection") + { + CANCELLATION_WARNINGS.with(|count| { + if let Some(previous) = count.get() { + count.set(Some(previous + 1)); + } + }); + } + } + fn flush(&self) {} + } + + struct HungTerminalFactory { + started: Arc, + dropped: Arc, + } + struct HungTerminalSession { + started: Arc, + dropped: Arc, + } + struct TerminalDropProbe(Arc); + impl Drop for TerminalDropProbe { + fn drop(&mut self) { + self.0.fetch_add(1, Ordering::SeqCst); + } + } + impl crate::integrations::IntegrationHtmlFinalizerFactory for HungTerminalFactory { + fn integration_id(&self) -> &'static str { + "example_hung" + } + fn create( + &self, + _: crate::integrations::IntegrationHtmlStreamContext, + ) -> Box { + Box::new(HungTerminalSession { + started: Arc::clone(&self.started), + dropped: Arc::clone(&self.dropped), + }) + } + } + impl crate::integrations::IntegrationHtmlFinalizer for HungTerminalSession { + fn finalize<'a>( + &'a mut self, + input: crate::integrations::HtmlFinalizationInput<'a>, + ) -> futures::future::LocalBoxFuture< + 'a, + Result< + crate::integrations::HtmlFinalization, + Report, + >, + > { + Box::pin(async move { + self.started.fetch_add(1, Ordering::SeqCst); + let _probe = TerminalDropProbe(Arc::clone(&self.dropped)); + input.replacement.append(b"partial example candidate")?; + futures::future::pending().await + }) + } + } + + #[test] + fn html_finalization_client_cancellation_drops_work_and_warns_for_auction_once() { + static LOGGER: CancellationLogger = CancellationLogger; + let _ = log::set_logger(&LOGGER); + log::set_max_level(log::LevelFilter::Warn); + CANCELLATION_WARNINGS.with(|count| count.set(Some(0))); + let settings = Arc::new(crate::publisher::allow_eof_test_settings( + create_test_settings(), + )); + let started = Arc::new(AtomicUsize::new(0)); + let dropped = Arc::new(AtomicUsize::new(0)); + let registry = + IntegrationRegistry::from_html_finalizers(vec![Arc::new(HungTerminalFactory { + started: Arc::clone(&started), + dropped: Arc::clone(&dropped), + })]); + let response = PublisherResponse::Stream { + response: Response::builder() + .status(200) + .header(header::CONTENT_TYPE, "text/html") + .body(EdgeBody::empty()) + .expect("should build the cancellation response"), + body: EdgeBody::from(&b"Example"[..]), + params: Box::new(html_stream_params( + "", + Some(DispatchedAuction::empty_for_test( + test_auction_request(), + 500, + )), + )), + }; + let orchestrator = Arc::new(AuctionOrchestrator::new(settings.auction.clone())); + let response = futures::executor::block_on(publisher_response_into_streaming_response( + response, + &Method::GET, + settings, + ®istry, + orchestrator, + noop_services(), + )) + .expect("should create a lazy terminal body"); + let mut stream = response + .into_body() + .into_stream() + .expect("should retain lazy output"); + { + let work = futures::StreamExt::next(&mut stream); + futures::pin_mut!(work); + let waker = futures::task::noop_waker(); + let mut context = std::task::Context::from_waker(&waker); + assert!( + work.as_mut().poll(&mut context).is_pending(), + "should await terminal work with the auction still owned" + ); + } + assert_eq!( + started.load(Ordering::SeqCst), + 1, + "should start the callback once" + ); + assert_eq!( + dropped.load(Ordering::SeqCst), + 0, + "should keep its candidate alive while pending" + ); + drop(stream); + assert_eq!( + dropped.load(Ordering::SeqCst), + 1, + "should drop the callback when the reader disconnects" + ); + let warnings = CANCELLATION_WARNINGS.with(std::cell::Cell::take); + assert_eq!( + warnings, + Some(1), + "should preserve exactly one existing auction cancellation warning" + ); + } + + #[test] + fn html_finalization_fatal_read_decode_and_sink_failures_abandon_auction_once() { + struct TestWriter { + write_fails: bool, + flush_fails: bool, + } + impl Write for TestWriter { + fn write(&mut self, bytes: &[u8]) -> io::Result { + if self.write_fails { + Err(io::Error::other("example write failure")) + } else { + Ok(bytes.len()) + } + } + fn flush(&mut self) -> io::Result<()> { + if self.flush_fails { + Err(io::Error::other("example flush failure")) + } else { + Ok(()) + } + } + } + for failure in ["read", "decode", "write", "flush"] { + let settings = crate::publisher::allow_eof_test_settings(create_test_settings()); + let calls = Arc::new(AtomicUsize::new(0)); + let registry = IntegrationRegistry::from_html_finalizers(vec![Arc::new( + CountingTerminalFactory(Arc::clone(&calls)), + )]); + let sink = Arc::new(RecordingTelemetrySink::default()); + let services = noop_services_with_telemetry_sink(Arc::clone(&sink) as _); + let orchestrator = AuctionOrchestrator::new(settings.auction.clone()); + let encoding = if failure == "decode" { "gzip" } else { "" }; + let mut params = html_stream_params( + encoding, + Some(DispatchedAuction::empty_for_test( + test_auction_request(), + 500, + )), + ); + params.auction_observation = Some(AuctionObservationContext::from_parts( + AuctionSource::InitialNavigation, + "proxy.example.com", + "/article", + 1, + None, + &EcContext::default(), + )); + let source = b"Example"; + let body = match failure { + "read" => { + EdgeBody::from_stream(futures::stream::iter(vec![Err::( + io::Error::other("example origin read failure"), + )])) + } + "decode" => { + let mut encoded = gzip_encode(source); + encoded.truncate(encoded.len() / 2); + EdgeBody::from(encoded) + } + _ => EdgeBody::from(&source[..]), + }; + let mut writer = TestWriter { + write_fails: failure == "write", + flush_fails: failure == "flush", + }; + let _ = futures::executor::block_on(stream_publisher_body_async( body, - &mut output, + &mut writer, &mut params, &settings, ®istry, &orchestrator, &services, - ) - .await - .expect("non-html stream body should process after auction collection"); - - let css = String::from_utf8(output).expect("should be valid UTF-8"); - assert!( - css.contains("proxy.example.com"), - "should rewrite non-html stream after auction collection. Got: {css}" + )) + .expect_err("should preserve fatal pipeline errors"); + let batches = sink.batches.lock().expect("should lock terminal telemetry"); + let summaries: Vec<_> = batches + .iter() + .flat_map(crate::auction::telemetry::AuctionEventBatch::rows) + .filter(|row| row.event_kind == "summary") + .collect(); + assert_eq!( + summaries.len(), + 1, + "should abandon exactly once after {failure} failure" ); - assert!( - !css.contains("origin.example.com"), - "should not leave origin host after rewrite. Got: {css}" + assert_eq!( + summaries[0].terminal_status.as_deref(), + Some("abandoned"), + "should retain fatal-error abandonment semantics" ); - }); + assert_eq!( + calls.load(Ordering::SeqCst), + usize::from(matches!(failure, "write" | "flush")), + "should start I/O only after valid input EOF" + ); + } } - fn drain_streaming_finalize_body(content_encoding: &str, body: EdgeBody) -> Vec { - let settings = Arc::new(create_test_settings()); - let registry = Arc::new( - IntegrationRegistry::with_plan( - &settings, - Arc::new( - crate::auction::compile_auction_plan(&settings) - .expect("should compile auction plan"), - ), - ) - .expect("should create integration registry"), + #[test] + fn async_eof_fatal_auction_suffix_overflow_abandons_once() { + let mut settings = crate::publisher::allow_eof_test_settings(create_test_settings()); + settings.publisher.max_buffered_body_bytes = 256; + let calls = Arc::new(AtomicUsize::new(0)); + let registry = IntegrationRegistry::from_html_stream_processors(vec![Arc::new( + AsyncTailFactory(Arc::clone(&calls)), + )]); + let sink = Arc::new(RecordingTelemetrySink::default()); + let services = noop_services_with_telemetry_sink(Arc::clone(&sink) as _); + let orchestrator = AuctionOrchestrator::new(settings.auction.clone()); + let mut params = html_stream_params( + "", + Some(DispatchedAuction::empty_for_test( + test_auction_request(), + 500, + )), ); - let orchestrator = Arc::new(AuctionOrchestrator::new(settings.auction.clone())); - let services = noop_services(); - let response = Response::builder() - .status(StatusCode::OK) - .header(header::CONTENT_TYPE, "text/css") - .body(EdgeBody::empty()) - .expect("should build response"); - let params = OwnedProcessResponseParams { - csp_nonce_observed: None, - template_cache_key: None, - seam_ad_slots: None, - policy_headers: Vec::new(), - content_encoding: content_encoding.to_string(), - origin_host: "origin.example.com".to_string(), - origin_url: "https://origin.example.com".to_string(), - request_host: "proxy.example.com".to_string(), - request_scheme: "https".to_string(), - content_type: "text/css".to_string(), - ad_slots_script: None, - ad_bids_state: AdBidsState::default(), - auction_observation: None, - auction_request: None, - dispatched_auction: None, - price_granularity: crate::price_bucket::PriceGranularity::default(), - gpt_diagnostics: None, - suppress_datadome_client_side_tag: false, - }; - let publisher_response = PublisherResponse::Stream { - response, - body, - params: Box::new(params), - }; - - let response = futures::executor::block_on(publisher_response_into_streaming_response( - publisher_response, - &Method::GET, - Arc::clone(&settings), - registry.as_ref(), - orchestrator, - services, + params.auction_observation = Some(AuctionObservationContext::from_parts( + AuctionSource::InitialNavigation, + "proxy.example.com", + "/article", + 1, + None, + &EcContext::default(), + )); + let source = format!( + "Example{}", + "x".repeat(settings.publisher.max_buffered_body_bytes + 1) + ); + let mut output = Vec::new(); + let error = futures::executor::block_on(stream_publisher_body_async( + EdgeBody::from(source.into_bytes()), + &mut output, + &mut params, + &settings, + ®istry, + &orchestrator, + &services, )) - .expect("should build streaming response"); - + .expect_err("should preserve fatal suffix overflow"); assert!( - matches!(response.body(), EdgeBody::Stream(_)), - "streaming finalize should keep a lazy Body::Stream" + format!("{error:?}").contains("HTML auction suffix exceeds its retained-byte limit"), + "should fail at the auction holder rather than optional replacement" ); - - futures::executor::block_on( - response - .into_body() - .into_bytes_bounded(settings.publisher.max_buffered_body_bytes), - ) - .expect("streaming body should drain") - .to_vec() - } - - #[test] - fn publisher_response_streaming_finalize_keeps_stream_body_lazy() { - let body_bytes = drain_streaming_finalize_body( - "", - EdgeBody::stream(futures::stream::iter(vec![bytes::Bytes::from_static( - b"body{background:url('https://origin.example.com/asset.png')}", - )])), + assert_eq!( + calls.load(Ordering::SeqCst), + 0, + "should not start EOF I/O after fatal suffix overflow" ); - let css = String::from_utf8(body_bytes).expect("should be valid UTF-8"); - assert!( - css.contains("proxy.example.com"), - "streaming response body should still run publisher rewriting. Got: {css}" + let batches = sink + .batches + .lock() + .expect("should inspect abandonment telemetry"); + let summaries: Vec<_> = batches + .iter() + .flat_map(crate::auction::telemetry::AuctionEventBatch::rows) + .filter(|row| row.event_kind == "summary") + .collect(); + assert_eq!( + summaries.len(), + 1, + "should abandon the dispatched auction exactly once" + ); + assert_eq!( + summaries[0].terminal_status.as_deref(), + Some("abandoned"), + "should record fatal body abandonment" + ); + assert_eq!( + summaries[0].terminal_reason.as_deref(), + Some("auction_suffix_overflow"), + "should distinguish suffix bounds from encoder failures" ); assert!( - !css.contains("origin.example.com"), - "streaming response body should not leave origin URLs unrewritten. Got: {css}" + params.dispatched_auction.is_none(), + "should transfer auction ownership to the failing driver" ); } - /// An origin body stream that yields `chunk` once and then stays `Pending` - /// forever without ever signalling EOF — modelling an origin that has sent - /// the document head but not yet finished the response. - fn origin_chunk_then_pending(chunk: bytes::Bytes) -> EdgeBody { - let mut sent = false; - EdgeBody::from_stream(futures::stream::poll_fn(move |_cx| { - if sent { - return std::task::Poll::Pending; - } - sent = true; - std::task::Poll::Ready(Some(Ok::<_, io::Error>(chunk.clone()))) - })) + struct TerminalTestFactory; + + struct TerminalTestSession; + + impl crate::integrations::IntegrationHtmlFinalizerFactory for TerminalTestFactory { + fn integration_id(&self) -> &'static str { + "example_terminal" + } + fn create( + &self, + _context: crate::integrations::IntegrationHtmlStreamContext, + ) -> Box { + Box::new(TerminalTestSession) + } + } + + impl crate::integrations::IntegrationHtmlFinalizer for TerminalTestSession { + fn finalize<'a>( + &'a mut self, + input: crate::integrations::HtmlFinalizationInput<'a>, + ) -> futures::future::LocalBoxFuture< + 'a, + Result< + crate::integrations::HtmlFinalization, + Report, + >, + > { + Box::pin(async move { + let mut yielded = false; + futures::future::poll_fn(|context| { + if yielded { + std::task::Poll::Ready(()) + } else { + yielded = true; + context.waker().wake_by_ref(); + std::task::Poll::Pending + } + }) + .await; + input.replacement.append(input.html)?; + input.replacement.append(b"")?; + Ok(crate::integrations::HtmlFinalization::Replace) + }) + } } - /// Poll a lazy `Body::Stream` exactly once and return the first emitted - /// chunk. Panics if the first poll is `Pending` (nothing emitted before the - /// origin would need its next chunk) or the stream ends. - fn first_lazy_body_chunk(body: EdgeBody) -> bytes::Bytes { - let mut stream = body - .into_stream() - .expect("streaming finalize should keep a lazy Body::Stream"); - let waker = futures::task::noop_waker(); - let mut cx = std::task::Context::from_waker(&waker); - let next = futures::StreamExt::next(&mut stream); - let mut next = std::pin::pin!(next); - match std::future::Future::poll(next.as_mut(), &mut cx) { - std::task::Poll::Ready(Some(Ok(bytes))) => bytes, - std::task::Poll::Ready(other) => { - panic!("first poll must emit a chunk, got Ready({other:?})") - } - std::task::Poll::Pending => { - panic!("first poll must emit rewritten content before origin EOF, got Pending") + #[test] + fn html_finalization_runs_on_materialized_and_streamed_publishers() { + for source in [ + &b"

Example article

"[..], + &b"

Example article

"[..], + "Example café".as_bytes(), + &b""[..], + ] { + for encoding in ["", "gzip", "deflate", "br"] { + let encoded = match encoding { + "gzip" => { + let middle = source.len() / 2; + let mut members = gzip_encode(&source[..middle]); + members.extend(gzip_encode(&source[middle..])); + members + } + "deflate" => deflate_encode(source), + "br" => brotli_encode(source), + _ => source.to_vec(), + }; + for auction in [false, true] { + for lazy in [false, true] { + for streamed in [false, true] { + let settings = Arc::new(crate::publisher::allow_eof_test_settings( + create_test_settings(), + )); + let registry = + IntegrationRegistry::from_html_finalizers(vec![Arc::new( + TerminalTestFactory, + )]); + let orchestrator = + Arc::new(AuctionOrchestrator::new(settings.auction.clone())); + let body = if streamed { + let chunks: Vec<_> = encoded + .chunks(7) + .map(|chunk| { + Ok::<_, io::Error>(bytes::Bytes::copy_from_slice(chunk)) + }) + .collect(); + EdgeBody::from_stream(futures::stream::iter(chunks)) + } else { + EdgeBody::from(encoded.clone()) + }; + let dispatched = auction.then(|| { + DispatchedAuction::empty_for_test(test_auction_request(), 10) + }); + let response = PublisherResponse::Stream { + response: Response::builder() + .status(StatusCode::OK) + .header(header::CONTENT_TYPE, "text/html") + .body(EdgeBody::empty()) + .expect("should build response"), + body, + params: Box::new(html_stream_params(encoding, dispatched)), + }; + let response = if lazy { + futures::executor::block_on( + publisher_response_into_streaming_response( + response, + &Method::GET, + Arc::clone(&settings), + ®istry, + Arc::clone(&orchestrator), + noop_services(), + ), + ) + .expect("should build lazy response") + } else { + futures::executor::block_on(buffer_publisher_response_async( + response, + &Method::GET, + &settings, + ®istry, + &orchestrator, + &noop_services(), + )) + .expect("should build buffered response") + }; + let body = futures::executor::block_on( + response.into_body().into_bytes_bounded(1024 * 1024), + ) + .expect("should drain body"); + let decoded = match encoding { + "gzip" => gzip_decode(&body), + "deflate" => deflate_decode(&body), + "br" => brotli_decode(&body), + _ => body.to_vec(), + }; + assert!( + decoded.ends_with(b""), + "should await finalization: encoding={encoding}, auction={auction}, lazy={lazy}, streamed={streamed}" + ); + assert!( + !String::from_utf8_lossy(&decoded) + .contains("ts-inline-body-close-"), + "should remove the parser seam after finalization" + ); + } + } + } } } } fn streaming_finalize_response(params: OwnedProcessResponseParams, body: EdgeBody) -> EdgeBody { - streaming_finalize_response_with_settings(params, body, create_test_settings()) + streaming_finalize_response_with_settings( + params, + body, + crate::publisher::allow_eof_test_settings(create_test_settings()), + ) } fn streaming_finalize_response_with_settings( @@ -20635,6 +23104,8 @@ mod tests { dispatched_auction: Option, ) -> OwnedProcessResponseParams { OwnedProcessResponseParams { + response_status: StatusCode::OK, + range_associated: false, csp_nonce_observed: None, template_cache_key: None, seam_ad_slots: None, @@ -21316,6 +23787,8 @@ mod tests { let ec_context = EcContext::new_for_test(None, crate::consent::types::ConsentContext::default()); OwnedProcessResponseParams { + response_status: StatusCode::OK, + range_associated: false, csp_nonce_observed: None, template_cache_key: None, seam_ad_slots: None, @@ -21511,6 +23984,8 @@ mod tests { .map(bytes::Bytes::copy_from_slice) .collect(); let params = OwnedProcessResponseParams { + response_status: StatusCode::OK, + range_associated: false, csp_nonce_observed: None, template_cache_key: None, seam_ad_slots: None, @@ -21592,6 +24067,8 @@ mod tests { r#""#; let state = AdBidsState::with_script(bids_script); let params = OwnedProcessResponseParams { + response_status: StatusCode::OK, + range_associated: false, csp_nonce_observed: None, template_cache_key: None, seam_ad_slots: None, @@ -21656,6 +24133,8 @@ mod tests { // Claim gzip encoding but feed non-gzip bytes. The GzDecoder will // error as soon as it tries to read the gzip header. let params = OwnedProcessResponseParams { + response_status: StatusCode::OK, + range_associated: false, csp_nonce_observed: None, template_cache_key: None, seam_ad_slots: None, @@ -21774,6 +24253,8 @@ mod tests { let body = EdgeBody::from(html.to_vec()); let params = OwnedProcessResponseParams { + response_status: StatusCode::OK, + range_associated: false, csp_nonce_observed: None, template_cache_key: None, seam_ad_slots: None, @@ -21841,6 +24322,8 @@ mod tests { // Small, single-fragment RSC script — placeholder path (not fallback). let html = br#""#; let params = OwnedProcessResponseParams { + response_status: StatusCode::OK, + range_associated: false, csp_nonce_observed: None, template_cache_key: None, seam_ad_slots: None, @@ -23786,6 +26269,7 @@ mod tests { &services, None, AuctionDispatch { + integration_registry: &IntegrationRegistry::empty_for_tests(), orchestrator, slots, registry: None, @@ -23841,6 +26325,7 @@ mod tests { &winning_services, None, AuctionDispatch { + integration_registry: &IntegrationRegistry::empty_for_tests(), orchestrator: &winning_orchestrator, slots: &slots, registry: None, @@ -23896,6 +26381,7 @@ mod tests { &no_winner_services, None, AuctionDispatch { + integration_registry: &IntegrationRegistry::empty_for_tests(), orchestrator: &no_winner_orchestrator, slots: &slots, registry: None, @@ -23948,6 +26434,7 @@ mod tests { &services, None, AuctionDispatch { + integration_registry: &IntegrationRegistry::empty_for_tests(), orchestrator: &orchestrator, slots: &slots, registry: None, @@ -24804,6 +27291,7 @@ mod tests { None, &mut ec_context, AuctionDispatch { + integration_registry: &IntegrationRegistry::empty_for_tests(), orchestrator: &orchestrator, slots: &article_slot(), registry: None, @@ -24846,6 +27334,7 @@ mod tests { &services, None, AuctionDispatch { + integration_registry: &IntegrationRegistry::empty_for_tests(), orchestrator: &orchestrator, slots: &article_slot(), registry: None, @@ -24888,6 +27377,7 @@ mod tests { None, &mut ec_context, AuctionDispatch { + integration_registry: &IntegrationRegistry::empty_for_tests(), orchestrator: &orchestrator, slots: &slots, registry: None, @@ -24932,6 +27422,7 @@ mod tests { &services, None, AuctionDispatch { + integration_registry: &IntegrationRegistry::empty_for_tests(), orchestrator: &orchestrator, slots: &slots, registry: None, @@ -24946,3 +27437,104 @@ mod tests { } } } + +#[cfg(test)] +mod async_eof_seam_tests { + use super::*; + + #[test] + fn async_eof_seam_caps_suffix_before_growth() { + let mut seam = InlineBodyCloseSeam::new(b"TOKEN".to_vec()); + assert_eq!( + seam.push_bounded(b"prefixTOKEN1234", 4) + .expect("should release the prefix and hold exact cap"), + b"prefix", + "should release ordinary HTML" + ); + assert_eq!( + seam.buffered, b"1234", + "should retain only the closing suffix" + ); + seam.push_bounded(b"!", 4) + .expect_err("should reject held suffix cap crossing"); + assert_eq!( + seam.buffered, b"1234", + "should reject before growing retained bytes" + ); + } + + #[test] + fn async_eof_seam_splits_large_ready_prefix_before_retention() { + let mut seam = InlineBodyCloseSeam::new(b"TOKEN".to_vec()); + let mut input = vec![b'x'; 100_000]; + input.extend_from_slice(b"TOKENtail"); + let ready = seam + .push_bounded(&input, 4) + .expect("should not count ready prefix against held cap"); + assert_eq!(ready.len(), 100_000, "should release the full ready prefix"); + assert_eq!( + seam.buffered, b"tail", + "should retain only bounded tail bytes" + ); + assert!( + seam.buffered.capacity() <= 4, + "should not allocate the whole ready prefix in the holder" + ); + } + + #[test] + fn async_eof_seam_grows_suffix_amortized_without_exceeding_non_power_of_two_cap() { + let limit = 1031; + let mut seam = InlineBodyCloseSeam::new(b"TOKEN".to_vec()); + seam.push_bounded(b"TOKEN", limit) + .expect("should locate the marker before collecting its suffix"); + let mut capacity_changes = 0; + for _ in 0..limit { + let previous_capacity = seam.buffered.capacity(); + assert!( + seam.push_bounded(b"x", limit) + .expect("should retain the suffix within its cap") + .is_empty(), + "should withhold suffix output until auction assembly" + ); + capacity_changes += usize::from(seam.buffered.capacity() != previous_capacity); + assert!( + seam.buffered.capacity() <= limit, + "should cap allocation even when the byte cap is not a power of two" + ); + } + assert!( + capacity_changes < 32, + "should avoid a capacity increase for every small suffix chunk" + ); + assert_eq!( + seam.finish(), + vec![b'x'; limit], + "should retain the exact-cap suffix without losing bytes" + ); + } + + #[test] + fn async_eof_seam_handles_split_tokens_under_the_cap() { + let mut seam = InlineBodyCloseSeam::new(b"TOKEN".to_vec()); + assert_eq!( + seam.push_bounded(b"pageTOK", 4) + .expect("should retain a possible split marker"), + b"page", + "should stream the prefix" + ); + assert!( + seam.push_bounded(b"ENtail", 4) + .expect("should finish split marker") + .is_empty(), + "should hold the closing suffix" + ); + assert_eq!(seam.finish(), b"tail", "should remove the marker once"); + } +} + +#[cfg(test)] +fn allow_eof_test_settings(mut settings: Settings) -> Settings { + settings.publisher.allow_async_html_finish = true; + settings +} diff --git a/crates/trusted-server-core/src/publisher/html_finalization.rs b/crates/trusted-server-core/src/publisher/html_finalization.rs new file mode 100644 index 000000000..699042619 --- /dev/null +++ b/crates/trusted-server-core/src/publisher/html_finalization.rs @@ -0,0 +1,487 @@ +//! Ordered optional HTML EOF work with driver-owned fallback and streaming output. + +use std::collections::VecDeque; +use std::io::{self, Read}; +use std::sync::Arc; +use std::time::Duration; + +use brotli::Decompressor; +use bytes::Bytes; +use edgezero_core::body::Body as EdgeBody; +use error_stack::{Report, ResultExt as _}; +use flate2::{Decompress, FlushDecompress, Status}; +use futures::future::{Fuse, LocalBoxFuture}; +use http::StatusCode; + +use super::{ + AuctionCollectDeps, AuctionHoldState, AuctionTelemetryCarry, BodyChunkSource, + DispatchedAuctionGuard, HoldStepSegments, HtmlStreamProcessorParams, InlineBodyCloseSeam, + OwnedProcessResponseParams, abandon_hold_auction, body_as_reader, + create_html_stream_processor_with_document_state, deferred_inline_seam_token, + hold_collect_close_tail, hold_finish_tail_segments, +}; +use crate::error::TrustedServerError; +use crate::html_processor::HtmlWithStreamingProcessors; +use crate::html_processor::finish::HtmlFinishContext; +use crate::integrations::{IntegrationDocumentState, IntegrationRegistry}; +use crate::settings::Settings; +use crate::streaming_processor::finish::StreamFinishDeadline; +use crate::streaming_processor::{ + BodyStreamDecoder, BodyStreamEncoder, Compression, GzipDecodeReader, STREAM_CHUNK_SIZE, + StreamProcessor, +}; + +// The reader codec must prove StreamEnd before terminal work starts. The +// ordinary flate2 Read wrapper also returns EOF for an incomplete zlib stream. +struct ValidatedZlibReader { + reader: Box, + decoder: Decompress, + input: [u8; STREAM_CHUNK_SIZE], + offset: usize, + available: usize, + eof: bool, + complete: bool, +} + +impl ValidatedZlibReader { + fn new(reader: Box) -> Self { + Self { + reader, + decoder: Decompress::new(true), + input: [0; STREAM_CHUNK_SIZE], + offset: 0, + available: 0, + eof: false, + complete: false, + } + } +} + +impl Read for ValidatedZlibReader { + fn read(&mut self, output: &mut [u8]) -> io::Result { + if output.is_empty() || self.complete { + return Ok(0); + } + loop { + if self.offset == self.available && !self.eof { + self.available = self.reader.read(&mut self.input)?; + self.offset = 0; + self.eof = self.available == 0; + } + let before_in = self.decoder.total_in(); + let before_out = self.decoder.total_out(); + let status = self + .decoder + .decompress( + &self.input[self.offset..self.available], + output, + if self.eof { + FlushDecompress::Finish + } else { + FlushDecompress::None + }, + ) + .map_err(io::Error::other)?; + let consumed = + usize::try_from(self.decoder.total_in() - before_in).map_err(io::Error::other)?; + let produced = + usize::try_from(self.decoder.total_out() - before_out).map_err(io::Error::other)?; + self.offset += consumed; + self.complete = status == Status::StreamEnd; + if produced > 0 || self.complete { + return Ok(produced); + } + if self.eof || (consumed == 0 && self.offset < self.available) { + return Err(io::Error::new( + io::ErrorKind::UnexpectedEof, + "incomplete publisher zlib stream", + )); + } + } + } +} + +enum TerminalInput { + Stream { + source: BodyChunkSource, + decoder: Box, + }, + Materialized(Box), +} + +impl TerminalInput { + fn new( + body: EdgeBody, + compression: Compression, + limit: usize, + ) -> Result> { + if body.is_stream() { + return Ok(Self::Stream { + source: BodyChunkSource::new(body, STREAM_CHUNK_SIZE).with_max_bytes(limit), + decoder: Box::new(BodyStreamDecoder::new(compression, limit)), + }); + } + // Preserve the existing reader codecs and per-step gzip decode budget. + let reader = body_as_reader(body)?; + let reader: Box = match compression { + Compression::None => Box::new(reader), + Compression::Gzip => Box::new(GzipDecodeReader::new(reader, limit)), + Compression::Deflate => Box::new(ValidatedZlibReader::new(Box::new(reader))), + Compression::Brotli => Box::new(Decompressor::new(reader, STREAM_CHUNK_SIZE)), + }; + Ok(Self::Materialized(reader)) + } + + async fn next( + &mut self, + failure_reason: &mut &'static str, + ) -> Result<(Bytes, bool), Report> { + match self { + Self::Stream { source, decoder } => { + *failure_reason = "stream_read_error"; + let raw = source.next_chunk().await?; + *failure_reason = "stream_decode_error"; + match raw { + Some(raw) => Ok((decoder.decode_chunk(raw)?, false)), + None => Ok((Bytes::from(decoder.finish()?), true)), + } + } + Self::Materialized(reader) => { + *failure_reason = "stream_read_error"; + let mut bytes = vec![0; STREAM_CHUNK_SIZE]; + let size = reader + .read(&mut bytes) + .change_context(TrustedServerError::Proxy { + message: "Failed to read decoded HTML terminal input".to_string(), + })?; + bytes.truncate(size); + Ok((Bytes::from(bytes), size == 0)) + } + } + } +} + +pub(super) struct HtmlEofDriver { + response_status: StatusCode, + input: TerminalInput, + encoder: BodyStreamEncoder, + processor: HtmlWithStreamingProcessors, + replay: VecDeque, + marker: Option>, + hold: Option, + limit: usize, + timeout: Duration, + source_finished: bool, + stages_finished: bool, + finish_deadline: Option, + finish_timer: Option>>, + had_ready_bytes: bool, + callback_barrier_issued: bool, + flush_barrier: bool, + finished: bool, + seam_flushed: bool, + failure_reason: &'static str, +} + +impl HtmlEofDriver { + pub(super) fn new( + body: EdgeBody, + params: &mut OwnedProcessResponseParams, + settings: &Settings, + registry: &IntegrationRegistry, + ) -> Result> { + registry.ensure_async_html_finish_allowed(settings)?; + let document_state = IntegrationDocumentState::default(); + let marker = deferred_inline_seam_token( + settings, + false, + params.ad_slots_script.is_some(), + params.dispatched_auction.is_some(), + ); + let mut processor = create_html_stream_processor_with_document_state( + HtmlStreamProcessorParams { + origin_host: ¶ms.origin_host, + request_host: ¶ms.request_host, + request_scheme: ¶ms.request_scheme, + settings, + integration_registry: registry, + ad_slots_script: params.ad_slots_script.clone(), + ad_bids_state: Arc::clone(params.ad_bids_state.script_cell()), + suppress_datadome_client_side_tag: params.suppress_datadome_client_side_tag, + gpt_diagnostics: params.gpt_diagnostics.clone(), + shared_template_authorized: false, + csp_nonce_observed: None, + deferred_inline_marker: marker + .as_ref() + .map(|marker| String::from_utf8_lossy(marker).into_owned()), + }, + &document_state, + )?; + processor + .validate_capabilities() + .change_context(TrustedServerError::Configuration { + message: "HTML processor EOF capabilities are inconsistent".to_string(), + })?; + processor.set_finish_marker(marker.as_deref()); + let compression = Compression::from_content_encoding(¶ms.content_encoding); + let limit = settings.publisher.max_buffered_body_bytes; + let input = TerminalInput::new(body, compression, limit)?; + let hold = params.dispatched_auction.take().map(|dispatched| { + AuctionHoldState::new( + DispatchedAuctionGuard::new(dispatched), + AuctionTelemetryCarry { + observation: params.auction_observation.take(), + auction_request: params.auction_request.take(), + }, + marker.clone(), + ) + }); + Ok(Self { + response_status: params.response_status, + input, + encoder: BodyStreamEncoder::new(compression), + processor, + replay: VecDeque::new(), + marker, + hold, + limit, + timeout: Duration::from_millis(u64::from( + settings.publisher.html_finalization_timeout_ms, + )), + source_finished: false, + stages_finished: false, + finish_deadline: None, + finish_timer: None, + had_ready_bytes: false, + callback_barrier_issued: false, + flush_barrier: false, + finished: false, + seam_flushed: false, + failure_reason: "stream_process_error", + }) + } + + pub(super) async fn next_step( + &mut self, + deps: &AuctionCollectDeps<'_>, + ) -> Result, Report> { + self.flush_barrier = false; + let result = self.next_inner(deps).await; + if result.is_err() { + self.abandon(deps, self.failure_reason).await; + } + result + } + + async fn next_inner( + &mut self, + deps: &AuctionCollectDeps<'_>, + ) -> Result, Report> { + loop { + if let Some(bytes) = self.replay.pop_front() { + if bytes.is_empty() { + continue; + } + let end = bytes.len().min(STREAM_CHUNK_SIZE); + let chunk = bytes.slice(..end); + if end < bytes.len() { + self.replay.push_front(bytes.slice(end..)); + } + self.failure_reason = "auction_suffix_overflow"; + let processed = + if let Some(seam) = self.hold.as_mut().and_then(|hold| hold.hold.as_mut()) { + seam.push_bounded(&chunk, self.limit).change_context( + TrustedServerError::Proxy { + message: "HTML auction suffix exceeds its retained-byte limit" + .to_string(), + }, + )? + } else { + chunk.to_vec() + }; + self.failure_reason = "stream_encode_error"; + let encoded = self.encoder.encode_chunk(processed)?; + self.had_ready_bytes |= !encoded.is_empty(); + return Ok(Some(HoldStepSegments { + ready: if encoded.is_empty() { + Vec::new() + } else { + vec![Bytes::from(encoded)] + }, + close_found: false, + })); + } + if self.finished { + return Ok(None); + } + if self.source_finished && !self.stages_finished { + if self.had_ready_bytes + && !self.callback_barrier_issued + && self.processor.next_finish_requires_async() + { + self.callback_barrier_issued = true; + self.flush_barrier = true; + return Ok(Some(HoldStepSegments { + ready: Vec::new(), + close_found: false, + })); + } + self.failure_reason = "stream_process_error"; + let output = self + .processor + .finish_next(HtmlFinishContext { + response_status: self.response_status, + services: deps.services, + deadline: self + .finish_deadline + .as_ref() + .expect("should start EOF deadline after parser completion"), + timer: &mut self.finish_timer, + limit: self.limit, + marker: self.marker.as_deref(), + }) + .await + .change_context(TrustedServerError::Proxy { + message: "Failed to finish HTML processor".to_string(), + })?; + self.callback_barrier_issued = false; + if let Some(output) = output { + self.replay.push_back(Bytes::from(output)); + } else { + self.stages_finished = true; + } + continue; + } + if self.stages_finished && !self.seam_flushed { + self.seam_flushed = true; + self.failure_reason = "stream_encode_error"; + if self + .hold + .as_ref() + .and_then(|hold| hold.hold.as_ref()) + .is_some_and(InlineBodyCloseSeam::found) + { + return Ok(Some(HoldStepSegments { + ready: Vec::new(), + close_found: true, + })); + } + let suffix = self.hold.as_mut().and_then(|hold| { + if hold.hold.as_ref().is_some_and(|seam| !seam.found()) { + hold.hold.take().map(InlineBodyCloseSeam::finish) + } else { + None + } + }); + let encoded = self.encoder.encode_chunk(suffix.unwrap_or_default())?; + return Ok(Some(HoldStepSegments { + ready: if encoded.is_empty() { + Vec::new() + } else { + vec![Bytes::from(encoded)] + }, + close_found: false, + })); + } + if self.stages_finished { + self.finished = true; + self.failure_reason = "stream_encode_error"; + let ready = if let Some(hold) = self.hold.as_mut() { + hold_finish_tail_segments(&mut self.encoder, hold, deps).await? + } else { + let trailer = self.encoder.finish()?; + if trailer.is_empty() { + Vec::new() + } else { + vec![Bytes::from(trailer)] + } + }; + return Ok(Some(HoldStepSegments { + ready, + close_found: false, + })); + } + let (decoded, ended) = self.input.next(&mut self.failure_reason).await?; + if !decoded.is_empty() { + self.accept(&decoded, false)?; + } + if ended { + self.failure_reason = "stream_process_error"; + let output = + self.processor + .begin_finish() + .change_context(TrustedServerError::Proxy { + message: "Failed to finish HTML parser".to_string(), + })?; + self.source_finished = true; + self.finish_deadline = Some(StreamFinishDeadline::new( + self.timeout, + deps.services.timer().is_some(), + )); + + self.replay.push_back(Bytes::from(output)); + } + } + } + + fn accept(&mut self, bytes: &[u8], is_last: bool) -> Result<(), Report> { + self.failure_reason = "stream_process_error"; + let output = self + .processor + .process_chunk(bytes, is_last) + .change_context(TrustedServerError::Proxy { + message: "Failed to process HTML terminal input".to_string(), + })?; + self.replay.push_back(Bytes::from(output)); + Ok(()) + } + + pub(super) fn at_finish_barrier(&self) -> bool { + self.flush_barrier || (self.stages_finished && self.seam_flushed && !self.finished) + } + + pub(super) async fn collect_close( + &mut self, + deps: &AuctionCollectDeps<'_>, + ) -> Result, Report> { + if let Some(hold) = self.hold.as_mut() { + hold_collect_close_tail(&mut self.encoder, hold, deps).await + } else { + Ok(Vec::new()) + } + } + + pub(super) async fn abandon(&mut self, deps: &AuctionCollectDeps<'_>, reason: &'static str) { + if let Some(hold) = self.hold.as_mut() { + abandon_hold_auction(hold, deps.services, reason).await; + } + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn materialized_deflate_requires_validated_end_before_terminal_eof() { + let mut encoder = + flate2::write::ZlibEncoder::new(Vec::new(), flate2::Compression::default()); + std::io::Write::write_all(&mut encoder, b"Example") + .expect("should encode an example page"); + let compressed = encoder.finish().expect("should finish zlib input"); + let mut valid = + ValidatedZlibReader::new(Box::new(std::io::Cursor::new(compressed.clone()))); + let mut output = Vec::new(); + valid + .read_to_end(&mut output) + .expect("should accept a validated zlib trailer"); + for missing in [1, 4, 8] { + let mut reader = ValidatedZlibReader::new(Box::new(std::io::Cursor::new( + compressed[..compressed.len() - missing].to_vec(), + ))); + let _ = reader + .read_to_end(&mut Vec::new()) + .expect_err("should reject a truncated trailer before terminal EOF"); + } + } +} diff --git a/crates/trusted-server-core/src/settings.rs b/crates/trusted-server-core/src/settings.rs index 3efc15770..ba06c807d 100644 --- a/crates/trusted-server-core/src/settings.rs +++ b/crates/trusted-server-core/src/settings.rs @@ -78,6 +78,24 @@ pub struct Publisher { #[serde(default = "default_max_buffered_body_bytes")] #[validate(range(min = 1, message = "must be at least 1 byte"))] pub max_buffered_body_bytes: usize, + /// Explicit operator permission for asynchronous HTML EOF work. + /// + /// Defaults to false. Enabled work can delay response delivery, especially + /// on buffered adapters and processors retaining output until EOF. + #[serde(default)] + pub allow_async_html_finish: bool, + /// Whole-chain HTML EOF budget in milliseconds; inert without an async processor. + #[serde(default = "default_html_finalization_timeout_ms")] + #[validate(range( + min = 1, + max = 60000, + message = "must be between 1 and 60000 milliseconds" + ))] + pub html_finalization_timeout_ms: u32, +} + +fn default_html_finalization_timeout_ms() -> u32 { + 5000 } fn default_max_buffered_body_bytes() -> usize { @@ -98,6 +116,8 @@ impl Default for Publisher { origin_host_header_override: None, proxy_secret: Redacted::default(), max_buffered_body_bytes: default_max_buffered_body_bytes(), + html_finalization_timeout_ms: default_html_finalization_timeout_ms(), + allow_async_html_finish: false, } } } @@ -189,6 +209,8 @@ impl Publisher { /// origin_host_header_override: None, /// proxy_secret: Redacted::new("proxy-secret".to_string()), /// max_buffered_body_bytes: 16 * 1024 * 1024, + /// html_finalization_timeout_ms: 5000, + /// allow_async_html_finish: false, /// }; /// assert_eq!(publisher.origin_host(), "origin.example.com:8080"); /// ``` @@ -3051,9 +3073,14 @@ impl Settings { /// - [`TrustedServerError::Configuration`] if the TOML is invalid or missing required fields #[cfg(test)] pub fn from_toml_and_env(toml_str: &str) -> Result> { + // Cargo supplies build metadata to test processes; it is not app config. + let settings_environment = std::env::vars() + .filter(|(name, _)| !name.eq_ignore_ascii_case("TRUSTED_SERVER__GIT_VERSION")) + .collect(); let environment = Environment::default() .prefix(ENVIRONMENT_VARIABLE_PREFIX) - .separator(ENVIRONMENT_VARIABLE_SEPARATOR); + .separator(ENVIRONMENT_VARIABLE_SEPARATOR) + .source(Some(settings_environment)); let toml = File::from_str(toml_str, FileFormat::Toml); let config = Config::builder() @@ -3813,11 +3840,55 @@ where #[cfg(test)] mod tests { use super::*; + use crate::auction::build_orchestrator; + use crate::integrations::{ + IntegrationRegistry, gpt::GptConfig, nextjs::NextJsIntegrationConfig, + prebid::PrebidIntegrationConfig, + }; + use crate::redacted::Redacted; + use crate::test_support::tests::{crate_test_settings_str, create_test_settings}; use regex::Regex; use serde_json::json; use std::collections::BTreeSet; use std::sync::Arc; + #[test] + fn html_finalization_timeout_defaults_and_boundaries() { + let parsed = Settings::from_toml(&crate_test_settings_str()) + .expect("should parse an omitted terminal timeout"); + assert_eq!( + parsed.publisher.html_finalization_timeout_ms, + Publisher::default().html_finalization_timeout_ms, + "should match TOML omission and the hand-written default" + ); + assert_eq!( + Publisher::default().html_finalization_timeout_ms, + 5000, + "should default the chain budget to five seconds" + ); + for (milliseconds, invalid) in [ + (0, true), + (1, false), + (5000, false), + (60000, false), + (60001, true), + ] { + let publisher = Publisher { + html_finalization_timeout_ms: milliseconds, + ..Publisher::default() + }; + let errors = publisher + .validate() + .expect_err("should reject the other empty publisher fields"); + assert_eq!( + errors + .field_errors() + .contains_key("html_finalization_timeout_ms"), + invalid, + "should enforce finish timeout boundaries" + ); + } + } /// `DebugConfig` denies unknown fields, so a binary built before /// `sandbox_metrics_enabled` existed must still accept a default blob. /// That only holds while the flag is skipped during serialization. @@ -3869,14 +3940,6 @@ mod tests { ); } - use crate::auction::build_orchestrator; - use crate::integrations::{ - IntegrationRegistry, gpt::GptConfig, nextjs::NextJsIntegrationConfig, - prebid::PrebidIntegrationConfig, - }; - use crate::redacted::Redacted; - use crate::test_support::tests::{crate_test_settings_str, create_test_settings}; - fn trusted_client_ip_toml(ip_header: &str, auth_header: &str, shared_secret: &str) -> String { format!( "{}\n[trusted_client_ip]\nip_header = \"{ip_header}\"\nauth_header = \"{auth_header}\"\nshared_secret = \"{shared_secret}\"\n", @@ -6280,6 +6343,26 @@ source_domain = "partner.example.com" ); } + #[test] + fn legacy_settings_loader_ignores_build_version_metadata() { + temp_env::with_var( + "TRUSTED_SERVER__GIT_VERSION", + Some("example-revision"), + || { + let settings = Settings::from_toml_and_env(&crate_test_settings_str()) + .expect("should keep build metadata outside application settings"); + assert_eq!( + settings.publisher.domain, + Settings::from_toml(&crate_test_settings_str()) + .expect("should parse the control configuration") + .publisher + .domain, + "should preserve configured publisher settings" + ); + }, + ); + } + #[test] fn test_override_env() { let toml_str = crate_test_settings_str(); @@ -6354,6 +6437,8 @@ source_domain = "partner.example.com" origin_host_header_override: None, proxy_secret: Redacted::new("test-secret".to_string()), max_buffered_body_bytes: 16 * 1024 * 1024, + html_finalization_timeout_ms: default_html_finalization_timeout_ms(), + allow_async_html_finish: false, }; assert_eq!(publisher.origin_host(), "origin.example.com:8080"); @@ -6365,6 +6450,8 @@ source_domain = "partner.example.com" origin_host_header_override: None, proxy_secret: Redacted::new("test-secret".to_string()), max_buffered_body_bytes: 16 * 1024 * 1024, + html_finalization_timeout_ms: default_html_finalization_timeout_ms(), + allow_async_html_finish: false, }; assert_eq!(publisher.origin_host(), "origin.example.com"); @@ -6376,6 +6463,8 @@ source_domain = "partner.example.com" origin_host_header_override: None, proxy_secret: Redacted::new("test-secret".to_string()), max_buffered_body_bytes: 16 * 1024 * 1024, + html_finalization_timeout_ms: default_html_finalization_timeout_ms(), + allow_async_html_finish: false, }; assert_eq!(publisher.origin_host(), "localhost:9090"); @@ -6387,6 +6476,8 @@ source_domain = "partner.example.com" origin_host_header_override: None, proxy_secret: Redacted::new("test-secret".to_string()), max_buffered_body_bytes: 16 * 1024 * 1024, + html_finalization_timeout_ms: default_html_finalization_timeout_ms(), + allow_async_html_finish: false, }; assert_eq!(publisher.origin_host(), "localhost:9090"); @@ -6398,6 +6489,8 @@ source_domain = "partner.example.com" origin_host_header_override: None, proxy_secret: Redacted::new("test-secret".to_string()), max_buffered_body_bytes: 16 * 1024 * 1024, + html_finalization_timeout_ms: default_html_finalization_timeout_ms(), + allow_async_html_finish: false, }; assert_eq!(publisher.origin_host(), "192.168.1.1:8080"); @@ -6409,6 +6502,8 @@ source_domain = "partner.example.com" origin_host_header_override: None, proxy_secret: Redacted::new("test-secret".to_string()), max_buffered_body_bytes: 16 * 1024 * 1024, + html_finalization_timeout_ms: default_html_finalization_timeout_ms(), + allow_async_html_finish: false, }; assert_eq!(publisher.origin_host(), "[::1]:8080"); } @@ -6422,6 +6517,8 @@ source_domain = "partner.example.com" origin_host_header_override: None, proxy_secret: Redacted::new("test-secret".to_string()), max_buffered_body_bytes: 16 * 1024 * 1024, + html_finalization_timeout_ms: default_html_finalization_timeout_ms(), + allow_async_html_finish: false, }; assert_eq!(publisher.origin_host_header(), "origin.example.com:8443"); @@ -6436,6 +6533,8 @@ source_domain = "partner.example.com" origin_host_header_override: Some("www.example.com".to_string()), proxy_secret: Redacted::new("test-secret".to_string()), max_buffered_body_bytes: 16 * 1024 * 1024, + html_finalization_timeout_ms: default_html_finalization_timeout_ms(), + allow_async_html_finish: false, }; assert_eq!(publisher.origin_host_header(), "www.example.com"); diff --git a/crates/trusted-server-core/src/streaming_processor.rs b/crates/trusted-server-core/src/streaming_processor.rs index d95e6ebf8..dbb5d0821 100644 --- a/crates/trusted-server-core/src/streaming_processor.rs +++ b/crates/trusted-server-core/src/streaming_processor.rs @@ -19,6 +19,8 @@ //! streaming interface. See `crate::platform` module doc for the //! authoritative note. +pub mod finish; + use std::cell::{Cell, RefCell}; use std::io::{self, Read, Write}; use std::rc::Rc; @@ -27,6 +29,9 @@ use brotli::Decompressor; use brotli::enc::BrotliEncoderParams; use brotli::enc::writer::CompressorWriter; use error_stack::{Report, ResultExt as _}; +use futures::future::LocalBoxFuture; + +use finish::{StreamFinish, StreamFinishError, StreamFinishInput}; use flate2::read::ZlibDecoder; use flate2::write::{GzEncoder, ZlibEncoder}; @@ -48,6 +53,29 @@ pub trait StreamProcessor { /// Returns an error if processing fails (e.g., I/O errors, encoding issues). fn process_chunk(&mut self, chunk: &[u8], is_last: bool) -> Result, io::Error>; + /// Report whether this processor requires an asynchronous EOF driver. + /// + /// Synchronous drivers reject this capability before consuming input. + fn requires_async_finish(&self) -> bool { + false + } + + /// Return optional borrowing EOF work after synchronous stage completion. + /// + /// The driver retains fallback output outside the future. Implementations + /// return promptly and perform I/O inside the future. The default creates + /// no future. Returned output reaches downstream stages before their EOF. + /// + /// # Errors + /// + /// Callback errors discard the candidate and preserve synchronous output. + fn async_finish<'a>( + &'a mut self, + _input: StreamFinishInput<'a>, + ) -> Option>>> { + None + } + /// Reset the processor state (useful for reuse) fn reset(&mut self) {} } @@ -164,6 +192,11 @@ impl StreamingPipeline

{ input: R, output: W, ) -> Result<(), Report> { + if self.processor.requires_async_finish() { + return Err(Report::new(TrustedServerError::Proxy { + message: "Processor requires asynchronous EOF completion".to_string(), + })); + } match ( self.config.input_compression, self.config.output_compression, @@ -2321,3 +2354,137 @@ mod tests { ); } } + +#[cfg(test)] +mod async_eof_contract_tests { + use std::cell::Cell; + use std::rc::Rc; + use std::time::Duration; + + use futures::executor::block_on; + use futures::future::LocalBoxFuture; + use http::StatusCode; + + use super::*; + use crate::platform::test_support::noop_services; + use crate::streaming_processor::finish::{ + StreamFinish, StreamFinishDeadline, StreamFinishError, StreamFinishInput, + StreamOutputWriter, + }; + + struct OptionalProcessor { + calls: Rc>, + } + + impl StreamProcessor for OptionalProcessor { + fn process_chunk(&mut self, chunk: &[u8], _: bool) -> io::Result> { + Ok(chunk.to_vec()) + } + + fn requires_async_finish(&self) -> bool { + true + } + + fn async_finish<'a>( + &'a mut self, + input: StreamFinishInput<'a>, + ) -> Option>>> { + Some(Box::pin(async move { + self.calls.set(self.calls.get() + 1); + input.replacement.append(input.output)?; + input.replacement.append(b"!")?; + Ok(StreamFinish::Replace) + })) + } + } + + struct ObservedReader(Rc>); + + impl Read for ObservedReader { + fn read(&mut self, _: &mut [u8]) -> io::Result { + self.0.set(self.0.get() + 1); + Ok(0) + } + } + + #[test] + fn async_eof_synchronous_pipeline_rejects_before_reading() { + let reads = Rc::new(Cell::new(0)); + let calls = Rc::new(Cell::new(0)); + let mut pipeline = StreamingPipeline::new( + PipelineConfig::default(), + OptionalProcessor { + calls: Rc::clone(&calls), + }, + ); + let mut output = Vec::new(); + let error = pipeline + .process(ObservedReader(Rc::clone(&reads)), &mut output) + .expect_err("should reject an async-capable processor before input effects"); + assert!( + format!("{error:?}").contains("async"), + "should explain async requirement" + ); + assert_eq!(reads.get(), 0, "should not consume the reader"); + assert_eq!(calls.get(), 0, "should not start the callback"); + assert!(output.is_empty(), "should not write output"); + } + + #[test] + fn async_eof_callback_borrows_non_send_state_and_driver_output() { + let calls = Rc::new(Cell::new(0)); + let mut processor = OptionalProcessor { + calls: Rc::clone(&calls), + }; + let services = noop_services(); + let deadline = StreamFinishDeadline::new(Duration::from_secs(1), false); + let mut replacement = StreamOutputWriter::new(8); + let outcome = block_on( + processor + .async_finish(StreamFinishInput { + response_status: StatusCode::OK, + output: b"tail", + replacement: &mut replacement, + services: &services, + deadline: &deadline, + }) + .expect("should provide optional EOF work"), + ) + .expect("should complete borrowed callback"); + assert_eq!( + outcome, + StreamFinish::Replace, + "should accept the candidate" + ); + assert_eq!( + replacement.into_bytes(), + b"tail!", + "should append to un-emitted output" + ); + assert_eq!(calls.get(), 1, "should call the finish once"); + } + + #[test] + fn async_eof_ordinary_processor_creates_no_future() { + let mut processor = StreamingReplacer::new(Vec::new()); + let services = noop_services(); + let deadline = StreamFinishDeadline::new(Duration::from_secs(1), false); + let mut replacement = StreamOutputWriter::new(8); + assert!( + !processor.requires_async_finish(), + "should default to synchronous processing" + ); + assert!( + processor + .async_finish(StreamFinishInput { + response_status: StatusCode::OK, + output: b"tail", + replacement: &mut replacement, + services: &services, + deadline: &deadline, + }) + .is_none(), + "should create no future by default" + ); + } +} diff --git a/crates/trusted-server-core/src/streaming_processor/finish.rs b/crates/trusted-server-core/src/streaming_processor/finish.rs new file mode 100644 index 000000000..2b4c98a3a --- /dev/null +++ b/crates/trusted-server-core/src/streaming_processor/finish.rs @@ -0,0 +1,246 @@ +//! Optional asynchronous processor completion contracts. + +use std::cell::Cell; +use std::time::Duration; + +use error_stack::Report; +use http::StatusCode; +use web_time::Instant; + +use crate::platform::RuntimeServices; + +/// Borrows un-emitted stage output and request services for optional EOF work. +/// +/// `output` is a streaming stage's tail or a deliberately buffered stage's +/// retained output. It is not universally the complete document. +pub struct StreamFinishInput<'a> { + /// Actual response status of the processed HTML. + pub response_status: StatusCode, + /// Driver-owned synchronous output still eligible for replacement. + pub output: &'a [u8], + /// Separate bounded storage for an atomic replacement candidate. + pub replacement: &'a mut StreamOutputWriter, + /// Request-scoped HTTP, KV and other runtime capabilities. + pub services: &'a RuntimeServices, + /// Absolute deadline shared across optional callbacks. + pub deadline: &'a StreamFinishDeadline, +} + +/// Selects whether to retain input or commit the bounded replacement candidate. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum StreamFinish { + /// Retain the current input without copying it. + Unchanged, + /// Commit bytes appended to [`StreamOutputWriter`]. + Replace, +} + +/// Controlled failure categories for optional document finalization. +#[derive(Debug, derive_more::Display)] +pub enum StreamFinishError { + /// The required outside service is unavailable. + #[display("Stream completion service is unavailable")] + Unavailable, + /// Outside work cannot produce an acceptable result. + #[display("Stream completion result is rejected")] + Rejected, + /// The bounded candidate cannot accept more bytes. + #[display("Stream replacement exceeds the {limit}-byte limit")] + OutputLimitExceeded { + /// Maximum candidate size in bytes. + limit: usize, + }, +} + +impl core::error::Error for StreamFinishError {} + +/// Separate bounded storage for a finalizer's replacement candidate. +/// +/// A failed append poisons the candidate, preventing partial commitment even +/// when an integration ignores the returned error. +pub struct StreamOutputWriter { + bytes: Vec, + limit: usize, + poisoned: bool, +} + +impl StreamOutputWriter { + pub(crate) fn new(limit: usize) -> Self { + Self { + bytes: Vec::new(), + limit, + poisoned: false, + } + } + + /// Append bytes without growing the candidate beyond its configured limit. + /// + /// # Errors + /// + /// Returns [`StreamFinishError::OutputLimitExceeded`] when the candidate + /// is poisoned, length arithmetic overflows, or the append exceeds the limit. + /// + /// # Examples + /// + /// ```ignore + /// input.replacement.append(b"

Example article

")?; + /// ``` + pub fn append(&mut self, chunk: &[u8]) -> Result<(), Report> { + let next = self + .bytes + .len() + .checked_add(chunk.len()) + .filter(|length| *length <= self.limit); + let next = match next { + Some(next) if !self.poisoned => next, + _ => { + self.poisoned = true; + return Err(Report::new(StreamFinishError::OutputLimitExceeded { + limit: self.limit, + })); + } + }; + if next > self.bytes.capacity() { + let capacity = self + .bytes + .capacity() + .saturating_mul(2) + .max(next) + .min(self.limit); + self.bytes.reserve_exact(capacity - self.bytes.len()); + } + self.bytes.extend_from_slice(chunk); + Ok(()) + } + + pub(crate) fn is_poisoned(&self) -> bool { + self.poisoned + } + + pub(crate) fn into_bytes(self) -> Vec { + self.bytes + } +} + +/// Absolute finish budget shared by all finalizers for one HTML document. +/// +/// A cooperative timer can interrupt yielding work, but cannot preempt CPU work +/// or blocking host calls. Consumers must bound their own calls on Fastly. +pub struct StreamFinishDeadline { + started_at: Instant, + timeout: Duration, + cooperative_timer: bool, + timer_expired: Cell, +} + +impl StreamFinishDeadline { + pub(crate) fn new(timeout: Duration, cooperative_timer: bool) -> Self { + Self { + started_at: Instant::now(), + timeout, + cooperative_timer, + timer_expired: Cell::new(false), + } + } + + /// Return the remaining whole-chain budget, saturating at zero. + #[must_use] + pub fn remaining(&self) -> Duration { + self.remaining_at(Instant::now()) + } + + pub(crate) fn elapsed(&self) -> Duration { + self.started_at.elapsed() + } + + /// Report whether the request has a cooperative timer for yielding work. + #[must_use] + pub fn has_cooperative_timer(&self) -> bool { + self.cooperative_timer + } + + pub(crate) fn mark_expired(&self) { + self.timer_expired.set(true); + } + + pub(crate) fn remaining_at(&self, now: Instant) -> Duration { + if self.timer_expired.get() { + return Duration::ZERO; + } + self.timeout + .saturating_sub(now.saturating_duration_since(self.started_at)) + } + + pub(crate) fn is_expired_at(&self, now: Instant) -> bool { + self.remaining_at(now).is_zero() + } +} + +#[cfg(test)] +mod tests { + use std::time::Duration; + + use super::*; + + #[test] + fn replacement_limit_poison_preserves_the_candidate() { + let mut writer = StreamOutputWriter::new(4); + writer.append(b"page").expect("should accept the exact cap"); + let _ = writer + .append(b"!") + .expect_err("should reject output beyond the cap"); + assert!(writer.is_poisoned(), "should poison an oversized candidate"); + assert_eq!(writer.bytes, b"page", "should preserve the accepted prefix"); + let _ = writer + .append(b"") + .expect_err("should reject writes after poisoning"); + } + + #[test] + fn replacement_capacity_stays_under_the_limit() { + let mut writer = StreamOutputWriter::new(17); + assert_eq!(writer.bytes.capacity(), 0, "should allocate lazily"); + for _ in 0..17 { + writer.append(b"x").expect("should append within the limit"); + assert!(writer.bytes.capacity() <= 17, "should bound reserve growth"); + } + } + + #[test] + fn empty_replacement_is_distinct_from_unchanged() { + let writer = StreamOutputWriter::new(4); + assert!( + writer.into_bytes().is_empty(), + "should allow an empty replacement" + ); + assert_ne!( + StreamFinish::Unchanged, + StreamFinish::Replace, + "should distinguish outcomes" + ); + } + + #[test] + fn deadline_expires_at_the_boundary() { + let deadline = StreamFinishDeadline::new(Duration::from_millis(5), false); + assert_eq!( + deadline.remaining_at(deadline.started_at), + Duration::from_millis(5), + "should start with the complete budget" + ); + let expired = deadline.started_at + Duration::from_millis(5); + assert!( + deadline.is_expired_at(expired), + "should expire at the exact boundary" + ); + assert_eq!( + deadline.remaining_at(expired + Duration::from_millis(1)), + Duration::ZERO, + "should saturate elapsed time" + ); + assert!( + !deadline.has_cooperative_timer(), + "should expose the absence of preemption" + ); + } +} diff --git a/crates/trusted-server-core/src/test_support.rs b/crates/trusted-server-core/src/test_support.rs index 37b8a9e65..3b961cfbb 100644 --- a/crates/trusted-server-core/src/test_support.rs +++ b/crates/trusted-server-core/src/test_support.rs @@ -399,6 +399,12 @@ pub mod nextjs_auction { /// stubbed out. #[must_use] pub fn services(client: Arc) -> RuntimeServices { + services_with_http_client(client) + } + + /// Build fixture runtime services with a custom asynchronous upstream. + #[must_use] + pub fn services_with_http_client(client: Arc) -> RuntimeServices { RuntimeServices::builder() .config_store(Arc::new(FixtureStore)) .secret_store(Arc::new(FixtureStore)) @@ -410,3 +416,6 @@ pub mod nextjs_auction { .build() } } + +#[cfg(feature = "test-utils")] +pub(crate) mod async_html_finalization; diff --git a/crates/trusted-server-core/src/test_support/async_html_finalization.rs b/crates/trusted-server-core/src/test_support/async_html_finalization.rs new file mode 100644 index 000000000..9e488c4ef --- /dev/null +++ b/crates/trusted-server-core/src/test_support/async_html_finalization.rs @@ -0,0 +1,243 @@ +//! Fictional terminal HTTP consumer, available only with `test-utils`. + +use std::io; +use std::sync::Arc; +use std::time::Duration; + +use edgezero_core::body::Body; +use edgezero_core::http::{Uri, request_builder}; +use error_stack::{Report, ResultExt as _}; +use futures::future::LocalBoxFuture; +use serde::Deserialize; +use serde_json::json; +use sha2::{Digest as _, Sha256}; +use url::Url; +use validator::Validate; + +use crate::error::TrustedServerError; +use crate::integrations::{ + IntegrationHtmlStreamContext, IntegrationHtmlStreamProcessorFactory, IntegrationRegistration, +}; +use crate::platform::{PlatformBackendSpec, PlatformHttpRequest}; +use crate::settings::{IntegrationConfig, Settings}; +use crate::streaming_processor::StreamProcessor; +use crate::streaming_processor::finish::{StreamFinish, StreamFinishError, StreamFinishInput}; + +pub(crate) const ID: &str = "html_finalization_fixture"; +const EOF_SENTINEL: &[u8] = b""; +const ORIGINAL: &[u8] = b"EXAMPLE_UNSIGNED"; +const REPLACEMENT: &[u8] = b"EXAMPLE_FINALIZED"; + +#[derive(Clone, Deserialize, Validate)] +pub(crate) struct FixtureConfig { + enabled: bool, + #[validate(url)] + lookup_url: String, + #[validate(range(min = 1, max = 60000))] + transport_timeout_ms: u32, + #[serde(default)] + streaming: bool, +} + +impl IntegrationConfig for FixtureConfig { + fn is_enabled(&self) -> bool { + self.enabled + } +} + +struct FixtureFactory(FixtureConfig); +struct FixtureSession { + config: FixtureConfig, + digest: Sha256, + bytes: usize, + tail: Vec, + saw_eof_sentinel: bool, + synchronous_eof: bool, +} + +pub(crate) fn register( + settings: &Settings, +) -> Result, Report> { + let Some(config) = settings.integration_config::(ID)? else { + return Ok(None); + }; + Ok(Some( + IntegrationRegistration::builder(ID) + .without_js() + .with_html_stream_processor(Arc::new(FixtureFactory(config))) + .build(), + )) +} + +impl IntegrationHtmlStreamProcessorFactory for FixtureFactory { + fn integration_id(&self) -> &'static str { + ID + } + + fn requires_async_finish(&self) -> bool { + true + } + fn buffer_output_until_eof(&self) -> bool { + !self.0.streaming + } + + fn create(&self, _: IntegrationHtmlStreamContext) -> Box { + Box::new(FixtureSession { + config: self.0.clone(), + digest: Sha256::new(), + bytes: 0, + tail: Vec::new(), + saw_eof_sentinel: false, + synchronous_eof: false, + }) + } +} + +impl StreamProcessor for FixtureSession { + fn process_chunk(&mut self, chunk: &[u8], last: bool) -> io::Result> { + self.digest.update(chunk); + self.bytes = self + .bytes + .checked_add(chunk.len()) + .ok_or_else(|| io::Error::other("Fixture byte count exceeds its range"))?; + let mut observed = std::mem::take(&mut self.tail); + observed.extend_from_slice(chunk); + self.saw_eof_sentinel |= observed + .windows(EOF_SENTINEL.len()) + .any(|bytes| bytes == EOF_SENTINEL); + let keep = observed.len().min(EOF_SENTINEL.len() - 1); + self.tail + .extend_from_slice(&observed[observed.len() - keep..]); + self.synchronous_eof = last; + Ok(chunk.to_vec()) + } + fn requires_async_finish(&self) -> bool { + true + } + + fn async_finish<'a>( + &'a mut self, + input: StreamFinishInput<'a>, + ) -> Option>>> { + Some(Box::pin(async move { + if !self.saw_eof_sentinel || !self.synchronous_eof { + return Err(Report::new(StreamFinishError::Rejected)); + } + let url = + Url::parse(&self.config.lookup_url).change_context(StreamFinishError::Rejected)?; + // Fastly cannot interrupt a blocking host call. Bound its transport + // by the remaining chain budget; timer adapters deliberately test + // cancellation against a longer transport timeout. + let timeout = if input.deadline.has_cooperative_timer() { + Duration::from_millis(u64::from(self.config.transport_timeout_ms)) + } else { + input + .deadline + .remaining() + .min(Duration::from_millis(u64::from( + self.config.transport_timeout_ms, + ))) + }; + let backend = input + .services + .backend() + .ensure(&PlatformBackendSpec { + scheme: url.scheme().to_string(), + host: url + .host_str() + .ok_or_else(|| Report::new(StreamFinishError::Rejected))? + .to_string(), + port: url.port(), + host_header_override: None, + certificate_check: true, + first_byte_timeout: timeout, + between_bytes_timeout: timeout, + discriminator: Some(ID.to_string()), + }) + .change_context(StreamFinishError::Unavailable)?; + let payload = serde_json::to_vec(&json!({ + "complete": true, + "sha256": hex::encode(self.digest.clone().finalize()), + "bytes": self.bytes, + "transport_timeout_ms": timeout.as_millis(), + })) + .change_context(StreamFinishError::Rejected)?; + let request = request_builder() + .method(http::Method::POST) + .uri( + self.config + .lookup_url + .parse::() + .change_context(StreamFinishError::Rejected)?, + ) + .header(http::header::CONTENT_TYPE, "application/json") + .body(Body::from(payload)) + .change_context(StreamFinishError::Rejected)?; + let response = input + .services + .http_client() + .send(PlatformHttpRequest::new(request, backend).with_cache_bypass()) + .await + .change_context(StreamFinishError::Unavailable)?; + if !response.response.status().is_success() { + return Err(Report::new(StreamFinishError::Unavailable)); + } + let reply = response + .response + .into_body() + .into_bytes_bounded(64) + .await + .change_context(StreamFinishError::Rejected)?; + if reply.as_ref() != b"EXAMPLE_APPROVED" { + return Err(Report::new(StreamFinishError::Rejected)); + } + if self.config.streaming { + input.replacement.append(input.output)?; + input + .replacement + .append(b"")?; + return Ok(StreamFinish::Replace); + } + let offset = input + .output + .windows(ORIGINAL.len()) + .position(|bytes| bytes == ORIGINAL) + .ok_or_else(|| Report::new(StreamFinishError::Rejected))?; + input.replacement.append(&input.output[..offset])?; + input.replacement.append(REPLACEMENT)?; + input + .replacement + .append(&input.output[offset + ORIGINAL.len()..])?; + Ok(StreamFinish::Replace) + })) + } +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::integrations::IntegrationRegistry; + use crate::test_support::tests::create_test_settings; + + #[test] + fn fictional_consumer_registers_only_when_enabled_and_adds_no_browser_bundle() { + for enabled in [false, true] { + let mut settings = create_test_settings(); + settings.publisher.allow_async_html_finish = true; + settings.integrations.insert_config(ID, &json!({ + "enabled": enabled, "lookup_url": "https://lookup.example.com/approve", "transport_timeout_ms": 1000, + })).expect("should configure the fictional consumer"); + let registry = + IntegrationRegistry::new(&settings).expect("should build fixture metadata"); + assert_eq!( + registry.has_async_html_finish(), + enabled, + "should exclude disabled terminal capabilities" + ); + assert!( + !registry.js_module_ids().contains(&ID), + "should never add a fictional browser bundle" + ); + } + } +} diff --git a/crates/trusted-server-integration-tests/Cargo.toml b/crates/trusted-server-integration-tests/Cargo.toml index 39f985860..b2369d9a6 100644 --- a/crates/trusted-server-integration-tests/Cargo.toml +++ b/crates/trusted-server-integration-tests/Cargo.toml @@ -7,6 +7,14 @@ license = { workspace = true } publish = { workspace = true } version = { workspace = true } +[features] +html-eof-runtime = [] + +[[test]] +name = "html_eof_runtime" +path = "tests/html_eof_runtime.rs" +required-features = ["html-eof-runtime"] + [[test]] name = "integration" path = "tests/integration.rs" @@ -29,11 +37,13 @@ trusted-server-core = { workspace = true, features = ["test-utils"] } [dev-dependencies] async-trait = { workspace = true } axum = { workspace = true } +base64 = { workspace = true } bytes = { workspace = true } derive_more = { workspace = true } edgezero-adapter-axum = { workspace = true, features = ["axum"] } env_logger = { workspace = true } error-stack = { workspace = true } +futures = { workspace = true } http = { workspace = true } http-body-util = { workspace = true } libc = { workspace = true } @@ -41,7 +51,8 @@ log = { workspace = true } reqwest = { workspace = true, features = ["blocking", "cookies"] } scraper = { workspace = true } testcontainers = { workspace = true } -tokio = { workspace = true, features = ["rt-multi-thread"] } +tempfile = { workspace = true } +tokio = { workspace = true, features = ["net", "rt-multi-thread", "signal"] } tower = { workspace = true, features = ["util"] } trusted-server-adapter-axum = { path = "../trusted-server-adapter-axum" } trusted-server-adapter-cloudflare = { path = "../trusted-server-adapter-cloudflare" } diff --git a/crates/trusted-server-integration-tests/tests/documentation_snippets.rs b/crates/trusted-server-integration-tests/tests/documentation_snippets.rs index d9c36d233..bac7d0b4a 100644 --- a/crates/trusted-server-integration-tests/tests/documentation_snippets.rs +++ b/crates/trusted-server-integration-tests/tests/documentation_snippets.rs @@ -3,15 +3,24 @@ use std::process::Command; use std::time::{SystemTime, UNIX_EPOCH}; const GUIDE: &str = include_str!("../../../docs/guide/integration-guide.md"); -const START: &str = ""; -const END: &str = ""; #[test] fn integration_guide_runtime_services_fixture_compiles() { + check_guide_snippet("runtime-services"); +} + +#[test] +fn integration_guide_html_eof_fixture_compiles() { + check_guide_snippet("html-eof"); +} + +fn check_guide_snippet(name: &str) { + let start = format!(""); + let end = format!(""); let marked = GUIDE - .split_once(START) - .and_then(|(_, tail)| tail.split_once(END).map(|(body, _)| body)) - .expect("should contain one bounded runtime-services snippet"); + .split_once(&start) + .and_then(|(_, tail)| tail.split_once(&end).map(|(body, _)| body)) + .expect("should contain one bounded integration snippet"); let source = marked .trim() .strip_prefix("```rust\n") @@ -49,7 +58,7 @@ fn integration_guide_runtime_services_fixture_compiles() { }) .expect("should find the workspace error-stack requirement"); let manifest = format!( - "[package]\nname = \"documentation-snippet\"\nversion = \"0.0.0\"\nedition = \"2024\"\n\n[workspace]\n\n[dependencies]\nerror-stack = \"{error_stack_requirement}\"\ntrusted-server-core = {{ path = {:?} }}\n", + "[package]\nname = \"documentation-snippet\"\nversion = \"0.0.0\"\nedition = \"2024\"\n\n[workspace]\n\n[dependencies]\nerror-stack = \"{error_stack_requirement}\"\nfutures = \"0.3\"\ntrusted-server-core = {{ path = {:?} }}\n", core_path ); fs::write(fixture_root.join("Cargo.toml"), manifest) diff --git a/crates/trusted-server-integration-tests/tests/html_eof_runtime.rs b/crates/trusted-server-integration-tests/tests/html_eof_runtime.rs new file mode 100644 index 000000000..32146b127 --- /dev/null +++ b/crates/trusted-server-integration-tests/tests/html_eof_runtime.rs @@ -0,0 +1,871 @@ +//! Native verification of ordered HTML EOF work on actual local runtimes. + +#[path = "html_eof_runtime/config.rs"] +mod config; +#[path = "html_eof_runtime/origin.rs"] +mod origin; +#[path = "html_eof_runtime/process.rs"] +mod process; + +use std::collections::BTreeMap; +use std::ffi::OsString; +use std::fs::{self, File}; +use std::net::{Ipv4Addr, TcpListener}; +use std::path::{Path, PathBuf}; +use std::process::{Command, Stdio}; +use std::thread; +use std::time::{Duration, Instant, SystemTime, UNIX_EPOCH}; + +use base64::Engine as _; +use base64::engine::general_purpose::URL_SAFE_NO_PAD; +use error_stack::{Report, ResultExt as _}; +use reqwest::blocking::Client; +use serde_json::{Value, json}; + +use process::{Cancellation, ManagedChild}; + +#[derive(Debug, derive_more::Display)] +enum HarnessError { + #[display("Invalid isolated runtime configuration")] + Configuration, + #[display("Runtime command fails")] + Runtime, + #[display("Runtime HTTP request fails")] + Http, + #[display("Runtime fixture I/O fails")] + Io, + #[display("Runtime verification is cancelled")] + Cancelled, +} + +impl core::error::Error for HarnessError {} + +type HarnessResult = Result>; + +struct Harness<'a> { + origin: &'a origin::Origin, + root: PathBuf, + workspace: PathBuf, + target: PathBuf, + environment: BTreeMap, + cancellation: Cancellation, +} + +impl<'a> Harness<'a> { + fn new(adapter: &str, origin: &'a origin::Origin) -> HarnessResult { + let root = PathBuf::from(env!("CARGO_MANIFEST_DIR")) + .join("../..") + .canonicalize() + .change_context(HarnessError::Io)?; + let workspace = tempfile::Builder::new() + .prefix(&format!("trusted-server-finalize-{adapter}-")) + .tempdir() + .change_context(HarnessError::Io)? + .keep(); + log::info!("Artifacts: {}", workspace.display()); + let mut environment = process::build_environment(std::env::vars_os()); + let target = environment + .get(&OsString::from("CARGO_TARGET_DIR")) + .map(PathBuf::from) + .unwrap_or_else(|| root.join("target")); + let target = if target.is_absolute() { + target + } else { + root.join(target) + }; + for (name, value) in [ + ("CARGO_TARGET_DIR", target.as_os_str().to_owned()), + ( + "XDG_CONFIG_HOME", + workspace.join("cli-config").into_os_string(), + ), + ("WRANGLER_SEND_METRICS", OsString::from("false")), + ( + "WRANGLER_LOG_PATH", + workspace.join("wrangler.log").into_os_string(), + ), + ( + "TRUSTED_SERVER__PUBLISHER__DOMAIN", + OsString::from("127.0.0.1"), + ), + ( + "TRUSTED_SERVER__PUBLISHER__COOKIE_DOMAIN", + OsString::from("127.0.0.1"), + ), + ( + "TRUSTED_SERVER__PUBLISHER__ORIGIN_URL", + OsString::from(format!("http://127.0.0.1:{}", origin.port)), + ), + ] { + environment.insert(OsString::from(name), value); + } + let cancellation = Cancellation::default(); + cancellation.watch(origin.runtime.handle())?; + Ok(Self { + origin, + root, + workspace, + target, + environment, + cancellation, + }) + } + + fn command(&self, program: impl AsRef) -> Command { + let mut command = Command::new(program); + command + .current_dir(&self.root) + .env_clear() + .envs(&self.environment); + command + } + + fn run(&self, command: &mut Command, name: &str) -> HarnessResult<()> { + process::logged_command( + command, + &self.workspace.join(format!("{name}.log")), + &self.cancellation, + ) + } + + fn preflight(&self, adapter: &str) -> HarnessResult<()> { + if adapter == "fastly" { + self.run(self.command("viceroy").arg("--version"), "viceroy-version")?; + let version = fs::read_to_string(self.workspace.join("viceroy-version.log")) + .change_context(HarnessError::Io)?; + validate_viceroy(&version)?; + } + if adapter == "cloudflare" { + self.run( + self.command("worker-build").arg("--version"), + "worker-version", + )?; + let actual = fs::read_to_string(self.workspace.join("worker-version.log")) + .change_context(HarnessError::Io)?; + let lock = config::parse( + &fs::read_to_string(self.root.join("Cargo.lock")) + .change_context(HarnessError::Io)?, + )?; + let expected = lock["package"] + .as_array() + .and_then(|packages| { + packages + .iter() + .find(|package| package["name"].as_str() == Some("worker")) + }) + .and_then(|package| package["version"].as_str()) + .ok_or_else(|| Report::new(HarnessError::Configuration))?; + if actual.trim() != expected { + return Err(Report::new(HarnessError::Runtime).attach(format!( + "worker-build {expected} required; found {}", + actual.trim() + ))); + } + } + Ok(()) + } + + fn build(&self, adapter: &str, release: bool) -> HarnessResult { + self.run( + self.command("cargo").args([ + "build", + "-p", + "trusted-server-cli", + "--bin", + "ts", + "--locked", + ]), + "build-cli", + )?; + let crate_root = self + .root + .join(format!("crates/trusted-server-adapter-{adapter}")); + let profile = if release { "release" } else { "debug" }; + let artifact = if adapter == "cloudflare" { + let mut command = self.command("worker-build"); + command + .current_dir(crate_root) + .arg(if release { "--release" } else { "--dev" }) + .arg("--out-dir") + .arg(self.workspace.join("build")) + .args(["--features", "cloudflare,test-utils", "--locked"]); + if !release { + command.arg("--no-opt"); + } + self.run(&mut command, "build-runtime")?; + self.workspace.join("build/index.js") + } else { + let mut command = self.command("cargo"); + command.args([ + "build", + "-p", + &format!("trusted-server-adapter-{adapter}"), + "--locked", + "--features", + if adapter == "spin" { + "trusted-server-core/test-utils,spin,test-utils" + } else { + "trusted-server-core/test-utils" + }, + ]); + if adapter != "axum" { + command.args(["--target", "wasm32-wasip1"]); + } + if release { + command.arg("--release"); + } + self.run(&mut command, "build-runtime")?; + match adapter { + "axum" => self.target.join(format!("{profile}/trusted-server-axum")), + "spin" => self.target.join(format!( + "wasm32-wasip1/{profile}/trusted_server_adapter_spin.wasm" + )), + _ => self.target.join(format!( + "wasm32-wasip1/{profile}/trusted-server-adapter-fastly.wasm" + )), + } + }; + if !artifact.is_file() { + return Err(Report::new(HarnessError::Runtime) + .attach(format!("fresh artifact missing: {}", artifact.display()))); + } + Ok(artifact) + } + + fn push(&self, adapter: &str) -> HarnessResult<()> { + let mut command = self.command(self.target.join("debug/ts")); + command + .current_dir(&self.workspace) + .args([ + "config", + "push", + "--adapter", + adapter, + "--local", + "--manifest", + ]) + .arg(self.workspace.join("edgezero.toml")) + .arg("--app-config") + .arg(self.workspace.join("trusted-server.toml")) + .args(["--yes", "--no-diff"]); + if adapter == "spin" { + command.env( + "EDGEZERO__STORES__CONFIG__TRUSTED_SERVER_CONFIG__NAME", + "default", + ); + } + self.run(&mut command, &format!("push-{adapter}")) + } + + fn runtime_command(&self, adapter: &str, artifact: &Path, port: u16) -> HarnessResult { + let mut command; + let raw = fs::read_to_string( + self.workspace + .join(".edgezero/local-config-trusted_server_config.json"), + ) + .change_context(HarnessError::Io)?; + let value: Value = + serde_json::from_str(&raw).change_context(HarnessError::Configuration)?; + let envelope = value["trusted_server_config"] + .as_str() + .ok_or_else(|| Report::new(HarnessError::Configuration))?; + match adapter { + "axum" => { + command = self.command(artifact); + command.env("PORT", port.to_string()).env( + "TRUSTED_SERVER_CONFIG_TRUSTED_SERVER_CONFIG_TRUSTED_SERVER_CONFIG", + envelope, + ); + for (key, value) in config::SECRETS { + command.env( + format!( + "TRUSTED_SERVER_SECRET_TRUSTED_SERVER_SECRETS_{}", + key.to_uppercase() + ), + value, + ); + } + } + "cloudflare" => { + let mut manifest = config::parse( + &fs::read_to_string( + self.root + .join("crates/trusted-server-adapter-cloudflare/wrangler.ci.toml"), + ) + .change_context(HarnessError::Io)?, + )?; + config::replace( + &mut manifest, + &["main"], + toml::Value::String(artifact.to_string_lossy().into_owned()), + )?; + config::replace( + &mut manifest, + &["vars", "TRUSTED_SERVER_CONFIG"], + toml::Value::String(json!({"app_config": envelope}).to_string()), + )?; + let vars = manifest["vars"] + .as_table_mut() + .ok_or_else(|| Report::new(HarnessError::Configuration))?; + for (key, value) in config::SECRETS { + vars.insert((*key).to_owned(), toml::Value::String((*value).to_owned())); + } + fs::write( + self.workspace.join("wrangler.toml"), + config::serialize(&manifest)?, + ) + .change_context(HarnessError::Io)?; + command = self.command("wrangler"); + command + .args(["dev", "--local", "--config"]) + .arg(self.workspace.join("wrangler.toml")) + .args([ + "--ip", + "127.0.0.1", + "--port", + &port.to_string(), + "--persist-to", + ]) + .arg(self.workspace.join("worker-state")); + } + "fastly" => { + let mut manifest = config::parse( + &fs::read_to_string(self.workspace.join("fastly.toml")) + .change_context(HarnessError::Io)?, + )?; + let secrets: Vec<_> = config::SECRETS + .iter() + .map(|(key, value)| { + let mut entry = toml::map::Map::new(); + entry.insert("key".to_owned(), toml::Value::String((*key).to_owned())); + entry.insert("data".to_owned(), toml::Value::String((*value).to_owned())); + toml::Value::Table(entry) + }) + .collect(); + let local = manifest["local_server"] + .as_table_mut() + .ok_or_else(|| Report::new(HarnessError::Configuration))?; + let mut stores = toml::map::Map::new(); + stores.insert( + "trusted_server_secrets".to_owned(), + toml::Value::Array(secrets), + ); + local.insert("secret_stores".to_owned(), toml::Value::Table(stores)); + fs::write( + self.workspace.join("fastly.toml"), + config::serialize(&manifest)?, + ) + .change_context(HarnessError::Io)?; + command = self.command("viceroy"); + command + .arg("--config") + .arg(self.workspace.join("fastly.toml")) + .args(["--addr", &format!("127.0.0.1:{port}")]) + .arg(artifact); + } + "spin" => { + command = self.command("spin"); + command + .args(["up", "--from"]) + .arg(self.workspace.join("spin.toml")) + .args(["--listen", &format!("127.0.0.1:{port}"), "--state-dir"]) + .arg(self.workspace.join(".spin")) + .arg("--cache-dir") + .arg(self.workspace.join("spin-cache")) + .arg("--log-dir") + .arg(self.workspace.join("spin-logs")); + for (key, value) in config::SECRETS { + command.arg("--variable").arg(format!( + "v_trusted_x5fserver_x5fsecrets_v_{}={value}", + key.replace('_', "_x5f") + )); + } + } + _ => { + return Err( + Report::new(HarnessError::Configuration).attach("unknown EOF runtime adapter") + ); + } + } + command.current_dir(&self.workspace); + Ok(command) + } +} + +impl Drop for Harness<'_> { + fn drop(&mut self) { + if let Err(error) = self + .origin + .write_events(&self.workspace.join("events.json")) + { + log::warn!("EOF harness cannot retain event diagnostics: {error:?}"); + } + } +} + +fn available_port() -> HarnessResult { + TcpListener::bind((Ipv4Addr::LOCALHOST, 0)) + .and_then(|listener| listener.local_addr()) + .map(|address| address.port()) + .change_context(HarnessError::Io) +} + +fn validate_viceroy(output: &str) -> HarnessResult<()> { + let version = output + .trim() + .strip_prefix("viceroy ") + .and_then(|version| version.split_whitespace().next()) + .and_then(|version| { + version + .split('.') + .map(str::parse::) + .collect::, _>>() + .ok() + }); + if !version + .is_some_and(|version| version.len() == 3 && version.as_slice() >= [0, 19, 0].as_slice()) + { + return Err(Report::new(HarnessError::Runtime) + .attach(format!("Viceroy >= 0.19.0 required; found {output:?}"))); + } + Ok(()) +} + +fn measured_terminal_ms( + adapter: &str, + diagnostic: &str, + lookup_elapsed: Duration, +) -> HarnessResult> { + let measured = diagnostic + .split("times out after ") + .nth(1) + .and_then(|tail| tail.split_once(" ms")) + .and_then(|(millis, _)| millis.parse::().ok()); + if let Some(millis) = measured { + if !(700..2500).contains(&millis) { + return Err( + Report::new(HarnessError::Runtime).attach("terminal budget escaped its tolerance") + ); + } + return Ok(Some(millis)); + } + if adapter != "fastly" + || !diagnostic.contains("retains input: unavailable") + || !(Duration::from_millis(700)..Duration::from_millis(2500)).contains(&lookup_elapsed) + { + return Err( + Report::new(HarnessError::Runtime).attach("missing measured bounded timeout evidence") + ); + } + Ok(None) +} + +fn consent_cookie() -> String { + let mut bits = [0_u8; 29]; + let now = SystemTime::now() + .duration_since(UNIX_EPOCH) + .expect("should have a current fixture clock") + .as_millis() + / 100; + for (offset, width, value) in [ + (0, 6, 2), + (6, 36, now), + (42, 36, now), + (108, 6, 4), + (114, 6, 13), + (152, 1, 1), + ] { + for index in 0..width { + if value & (1 << (width - index - 1)) != 0 { + bits[(offset + index) / 8] |= 1 << (7 - (offset + index) % 8); + } + } + } + format!("euconsent-v2={}", URL_SAFE_NO_PAD.encode(bits)) +} + +struct CapturedBody { + bytes: Vec, + first_byte_at: Instant, + result: std::io::Result<()>, +} + +fn capture_body(reader: &mut impl std::io::Read) -> CapturedBody { + let mut first = [0_u8; 1]; + let result = reader.read_exact(&mut first); + let first_byte_at = Instant::now(); + let mut bytes = if result.is_ok() { + Vec::from(first) + } else { + Vec::new() + }; + let result = result.and_then(|()| reader.read_to_end(&mut bytes).map(|_| ())); + CapturedBody { + bytes, + first_byte_at, + result, + } +} + +#[test] +fn truncated_response_preserves_received_bytes() { + struct TruncatedResponse(usize); + impl std::io::Read for TruncatedResponse { + fn read(&mut self, bytes: &mut [u8]) -> std::io::Result { + if bytes.is_empty() { + return Ok(0); + } + if self.0 >= 2 { + return Err(std::io::Error::from(std::io::ErrorKind::UnexpectedEof)); + } + bytes[0] = b"ab"[self.0]; + self.0 += 1; + Ok(1) + } + } + let captured = capture_body(&mut TruncatedResponse(0)); + assert!( + captured.result.is_err(), + "should report the truncated transfer" + ); + assert_eq!( + captured.bytes, b"ab", + "should retain bytes for failure diagnostics" + ); +} + +#[test] +#[ignore = "requires the selected local runtime and fresh adapter builds"] +fn actual_runtime() -> HarnessResult<()> { + let _ = env_logger::Builder::new() + .filter_level(log::LevelFilter::Info) + .try_init(); + let adapter = std::env::var("HTML_EOF_ADAPTER").change_context(HarnessError::Configuration)?; + if !["axum", "fastly", "cloudflare", "spin"].contains(&adapter.as_str()) { + return Err(Report::new(HarnessError::Configuration)); + } + let release = std::env::var("HTML_EOF_RELEASE").as_deref() == Ok("true"); + let auction = std::env::var("HTML_EOF_AUCTION").as_deref() == Ok("true"); + let origin = origin::Origin::start()?; + let harness = Harness::new(&adapter, &origin)?; + harness.preflight(&adapter)?; + let artifact = harness.build(&adapter, release)?; + harness.run( + harness + .command(harness.target.join("debug/ts")) + .args(["config", "init", "--app-config"]) + .arg(harness.workspace.join("trusted-server.toml")), + "config-init", + )?; + let template = fs::read_to_string(harness.workspace.join("trusted-server.toml")) + .change_context(HarnessError::Io)?; + fs::write( + harness.workspace.join("edgezero.toml"), + config::fixture_manifest( + include_str!("../../../edgezero.toml"), + &harness.root, + &harness.workspace, + )?, + ) + .change_context(HarnessError::Io)?; + if adapter == "spin" { + let mut manifest = + config::parse(include_str!("../../trusted-server-adapter-spin/spin.toml"))?; + config::replace( + &mut manifest, + &["component", "trusted-server", "source"], + toml::Value::String(artifact.to_string_lossy().into_owned()), + )?; + fs::write( + harness.workspace.join("spin.toml"), + config::serialize(&manifest)?, + ) + .change_context(HarnessError::Io)?; + } + let client = Client::builder() + .timeout(Duration::from_secs(10)) + .build() + .change_context(HarnessError::Http)?; + let mut cases = vec!["disabled", "success", "hung"]; + if auction { + cases.push("auction"); + } + if adapter == "fastly" { + cases.push("streaming"); + } + let mut baseline = Vec::new(); + for case in cases { + harness.cancellation.check()?; + let config = config::fixture_config(&template, case, origin.port, available_port()?)?; + fs::write(harness.workspace.join("trusted-server.toml"), config) + .change_context(HarnessError::Io)?; + fs::write(harness.workspace.join("fastly.toml"), "manifest_version = 3\nname = 'example-finalization'\nlanguage = 'rust'\n[local_server]\n").change_context(HarnessError::Io)?; + harness.push("axum")?; + if matches!(adapter.as_str(), "fastly" | "spin") { + harness.push(&adapter)?; + } + let port = available_port()?; + let log = harness.workspace.join(format!("{case}.log")); + let output = File::create(&log).change_context(HarnessError::Io)?; + let mut command = harness.runtime_command(&adapter, &artifact, port)?; + command + .stdout(Stdio::from( + output.try_clone().change_context(HarnessError::Io)?, + )) + .stderr(Stdio::from(output)); + let mut process = ManagedChild::spawn(&mut command, &harness.cancellation)?; + let base_url = format!("http://127.0.0.1:{port}"); + let ready_client = Client::builder() + .timeout(Duration::from_secs(1)) + .build() + .change_context(HarnessError::Http)?; + let mut ready = false; + for _ in 0..80 { + if let Some(status) = process.status()? { + return Err(Report::new(HarnessError::Runtime) + .attach(format!("runtime exits {status}; see {}", log.display()))); + } + if ready_client + .head(format!("{base_url}/health")) + .send() + .is_ok_and(|response| response.status().is_success()) + { + ready = true; + break; + } + thread::sleep(Duration::from_millis(100)); + } + if !ready { + return Err(Report::new(HarnessError::Runtime) + .attach(format!("runtime not ready; see {}", log.display()))); + } + let started = Instant::now(); + let mut request = client + .get(format!("{base_url}/article?case={case}")) + .header("accept", "text/html") + .header("accept-encoding", "identity") + .header("sec-fetch-dest", "document") + .header("sec-fetch-mode", "navigate"); + if case == "auction" { + request = request.header("cookie", consent_cookie()); + } + let mut response = request.send().change_context(HarnessError::Http)?; + let headers = response.headers().clone(); + let status = response.status(); + fs::write( + harness.workspace.join(format!("{case}.headers")), + format!("status={status}\n{headers:?}"), + ) + .change_context(HarnessError::Io)?; + let captured = capture_body(&mut response); + let first_byte_at = captured.first_byte_at; + let body = captured.bytes; + let completed = Instant::now(); + fs::write(harness.workspace.join(format!("{case}.body")), &body) + .change_context(HarnessError::Io)?; + captured + .result + .change_context(HarnessError::Http) + .attach(format!( + "HTTP {status} during {case}; response artifacts in {}", + harness.workspace.display() + ))?; + assert!( + status.is_success(), + "should serve {case} HTML; see {}", + log.display() + ); + match case { + "disabled" => { + assert!( + String::from_utf8_lossy(&body).contains("EXAMPLE_UNSIGNED") + && !String::from_utf8_lossy(&body).contains("EXAMPLE_FINALIZED"), + "should preserve disabled output" + ); + baseline = body.clone(); + } + "success" => { + assert_eq!( + body, + String::from_utf8_lossy(&baseline) + .replace("EXAMPLE_UNSIGNED", "EXAMPLE_FINALIZED") + .as_bytes(), + "should emit the exact approved rewrite" + ); + assert!( + completed.duration_since(started) >= Duration::from_millis(150), + "should perform delayed I/O" + ); + } + "streaming" => { + let mut expected = baseline.clone(); + expected.extend_from_slice(b""); + assert_eq!(body, expected, "should append only the EOF output"); + } + "auction" => { + let body = String::from_utf8_lossy(&body); + assert!( + body.contains("EXAMPLE_FINALIZED") + && body.contains("adSlots") + && body.contains("bids") + && !body.contains("ts-inline-body-close-"), + "should finish EOF work and auction assembly" + ); + } + _ => { + assert_eq!( + body, baseline, + "should emit byte-identical timeout fallback" + ); + assert!( + (Duration::from_millis(600)..Duration::from_secs(3)) + .contains(&completed.duration_since(started)), + "should bound the hung control" + ); + } + } + if case != "disabled" { + assert_eq!( + headers["cache-control"], "no-store, private", + "should prohibit shared caching" + ); + } + let events = origin.events(); + let lookup = origin::lookup_for_request(&events, started, completed, case != "disabled")?; + if let Some(lookup) = lookup { + assert_eq!( + lookup.data["complete"], true, + "should observe complete HTML" + ); + assert_eq!( + lookup.data["sha256"].as_str().map(str::len), + Some(64), + "should hash complete output" + ); + let timeout = lookup.data["transport_timeout_ms"] + .as_u64() + .expect("should report the transport budget"); + if adapter == "fastly" { + assert!( + (1..=750).contains(&timeout), + "should bound blocking host calls" + ); + } else { + assert_eq!( + timeout, 30000, + "should time out before the SDK transport deadline" + ); + } + } + if matches!(case, "success" | "auction" | "streaming") { + let finish = events + .iter() + .find(|event| { + event.kind == "lookup_complete" && event.at >= started && event.at <= completed + }) + .expect("should complete the current lookup"); + if case == "streaming" { + assert!( + first_byte_at + Duration::from_millis(50) < finish.at, + "should deliver actual HTML while EOF I/O remains pending" + ); + } + } + let lookup_elapsed = lookup.map(|event| completed.duration_since(event.at)); + let measured = if case == "hung" { + let mut diagnostic = fs::read_to_string(&log).change_context(HarnessError::Io)?; + if adapter == "spin" { + for path in fs::read_dir(harness.workspace.join("spin-logs")) + .change_context(HarnessError::Io)? + { + let path = path.change_context(HarnessError::Io)?.path(); + if path + .file_name() + .is_some_and(|name| name.to_string_lossy().contains("stderr")) + { + diagnostic + .push_str(&fs::read_to_string(path).change_context(HarnessError::Io)?); + } + } + } + measured_terminal_ms( + &adapter, + &diagnostic, + lookup_elapsed.expect("should measure the hung lookup"), + )? + } else { + None + }; + let timing = json!({"request_started_seconds": started.duration_since(origin.started).as_secs_f64(), "request_seconds": completed.duration_since(started).as_secs_f64(), "first_body_byte_seconds": first_byte_at.duration_since(started).as_secs_f64(), "terminal_ms": measured, "lookup_to_response_seconds": lookup_elapsed.map(|elapsed| elapsed.as_secs_f64()), "configured_budget_ms": 750, "cooperative_timer": adapter != "fastly"}); + fs::write( + harness.workspace.join(format!("{case}.timing.json")), + timing.to_string(), + ) + .change_context(HarnessError::Io)?; + origin.write_events(&harness.workspace.join("events.json"))?; + log::info!("PASS {adapter} {case}: {timing}"); + drop(process); + } + Ok(()) +} + +#[test] +fn viceroy_preflight_rejects_old_or_invalid_versions() { + for version in ["viceroy 0.17.0", "unknown"] { + assert!( + validate_viceroy(version).is_err(), + "should reject {version}" + ); + } + for version in ["viceroy 0.19.0", "viceroy 0.20.1", "viceroy 1.0.0"] { + assert!(validate_viceroy(version).is_ok(), "should accept {version}"); + } +} + +#[test] +fn timeout_proof_requires_measured_evidence() { + for adapter in ["axum", "cloudflare", "spin"] { + assert!( + measured_terminal_ms(adapter, "missing", Duration::from_millis(750)).is_err(), + "should require a timer diagnostic" + ); + assert_eq!( + measured_terminal_ms( + adapter, + "times out after 751 ms", + Duration::from_millis(750) + ) + .expect("should parse measured expiry"), + Some(751), + "should report actual expiry" + ); + } + assert!( + measured_terminal_ms( + "fastly", + "retains input: unavailable", + Duration::from_millis(100) + ) + .is_err(), + "should reject early unmeasured failure" + ); + assert!( + measured_terminal_ms( + "fastly", + "retains input: unavailable", + Duration::from_secs(3) + ) + .is_err(), + "should reject escaped budget" + ); + assert_eq!( + measured_terminal_ms( + "fastly", + "retains input: unavailable", + Duration::from_millis(749) + ) + .expect("should accept measured transport fallback"), + None, + "should not invent a cooperative timer measurement" + ); +} diff --git a/crates/trusted-server-integration-tests/tests/html_eof_runtime/config.rs b/crates/trusted-server-integration-tests/tests/html_eof_runtime/config.rs new file mode 100644 index 000000000..0b848c96b --- /dev/null +++ b/crates/trusted-server-integration-tests/tests/html_eof_runtime/config.rs @@ -0,0 +1,325 @@ +use std::path::Path; + +use error_stack::{Report, ResultExt as _}; +use toml::Value; + +use super::{HarnessError, HarnessResult}; + +pub(super) const SECRETS: &[(&str, &str)] = &[ + ("handler_password", "example-handler-password-32-bytes-ok"), + ( + "publisher_proxy_secret", + "example-publisher-secret-32-bytes-ok", + ), + ("ec_passphrase", "example-cookie-passphrase-32-bytes-ok"), +]; + +fn invalid(field: &str) -> Report { + Report::new(HarnessError::Configuration) + .attach(format!("missing or invalid template field: {field}")) +} + +pub(super) fn parse(template: &str) -> HarnessResult { + toml::from_str(template).change_context(HarnessError::Configuration) +} + +pub(super) fn serialize(value: &Value) -> HarnessResult { + toml::to_string(value).change_context(HarnessError::Configuration) +} + +pub(super) fn replace(value: &mut Value, path: &[&str], replacement: Value) -> HarnessResult<()> { + let mut field = value; + for key in path { + field = field + .get_mut(*key) + .ok_or_else(|| invalid(&path.join(".")))?; + } + if field.type_str() != replacement.type_str() { + return Err(invalid(&path.join("."))); + } + *field = replacement; + Ok(()) +} + +fn table<'a>( + value: &'a mut Value, + path: &[&str], +) -> HarnessResult<&'a mut toml::map::Map> { + let mut field = value; + for key in path { + field = field + .get_mut(*key) + .ok_or_else(|| invalid(&path.join(".")))?; + } + field.as_table_mut().ok_or_else(|| invalid(&path.join("."))) +} + +pub(super) fn fixture_config( + template: &str, + case: &str, + origin: u16, + bid: u16, +) -> HarnessResult { + let mut value = parse(template)?; + for (key, replacement) in [ + ("domain", "127.0.0.1".to_owned()), + ("cookie_domain", "127.0.0.1".to_owned()), + ("origin_url", format!("http://127.0.0.1:{origin}")), + ] { + replace(&mut value, &["publisher", key], Value::String(replacement))?; + } + let publisher = table(&mut value, &["publisher"])?; + publisher.insert( + "allow_async_html_finish".to_owned(), + Value::Boolean(case != "disabled"), + ); + publisher.insert( + "html_finalization_timeout_ms".to_owned(), + Value::Integer(750), + ); + replace( + &mut value, + &["auction", "enabled"], + Value::Boolean(case == "auction"), + )?; + if case == "auction" { + let providers = table(&mut value, &["auction", "providers"])?; + if providers.is_empty() { + return Err(invalid("auction.providers")); + } + for (_, provider) in providers.iter_mut() { + replace( + provider, + &["endpoint"], + Value::String(format!("https://127.0.0.1:{bid}/bid")), + )?; + } + let slot = parse( + "id = 'example-slot'\npage_patterns = ['/article']\nformats = [{ width = 300, height = 250 }]\n", + )?; + table(&mut value, &["creative_opportunities"])? + .insert("slot".to_owned(), Value::Array(vec![slot])); + } + let lookup = if matches!(case, "auction" | "streaming") { + "success" + } else { + case + }; + let fixture = parse(&format!( + "enabled = {}\nlookup_url = 'http://127.0.0.1:{origin}/{lookup}'\ntransport_timeout_ms = 30000\nstreaming = {}\n", + case != "disabled", + case == "streaming" + ))?; + table(&mut value, &["integrations"])?.insert("html_finalization_fixture".to_owned(), fixture); + validate_fixture(&value, case, origin, bid)?; + serialize(&value) +} + +pub(super) fn validate_fixture( + value: &Value, + case: &str, + origin: u16, + bid: u16, +) -> HarnessResult<()> { + let lookup = if matches!(case, "auction" | "streaming") { + "success" + } else { + case + }; + let required = [ + ( + vec!["publisher", "domain"], + Value::String("127.0.0.1".to_owned()), + ), + ( + vec!["publisher", "cookie_domain"], + Value::String("127.0.0.1".to_owned()), + ), + ( + vec!["publisher", "origin_url"], + Value::String(format!("http://127.0.0.1:{origin}")), + ), + ( + vec!["publisher", "allow_async_html_finish"], + Value::Boolean(case != "disabled"), + ), + ( + vec!["publisher", "html_finalization_timeout_ms"], + Value::Integer(750), + ), + ( + vec!["integrations", "html_finalization_fixture", "enabled"], + Value::Boolean(case != "disabled"), + ), + ( + vec!["integrations", "html_finalization_fixture", "lookup_url"], + Value::String(format!("http://127.0.0.1:{origin}/{lookup}")), + ), + ( + vec![ + "integrations", + "html_finalization_fixture", + "transport_timeout_ms", + ], + Value::Integer(30000), + ), + ( + vec!["integrations", "html_finalization_fixture", "streaming"], + Value::Boolean(case == "streaming"), + ), + ( + vec!["auction", "enabled"], + Value::Boolean(case == "auction"), + ), + ]; + for (path, expected) in required { + let actual = path.iter().try_fold(value, |field, key| field.get(*key)); + if actual != Some(&expected) { + return Err(invalid(&path.join("."))); + } + } + if case == "auction" { + let providers = value + .get("auction") + .and_then(|v| v.get("providers")) + .and_then(Value::as_table) + .ok_or_else(|| invalid("auction.providers"))?; + if providers.is_empty() + || providers.values().any(|provider| { + provider.get("endpoint").and_then(Value::as_str) + != Some(format!("https://127.0.0.1:{bid}/bid").as_str()) + }) + { + return Err(invalid("auction endpoint")); + } + } + Ok(()) +} + +pub(super) fn fixture_manifest( + template: &str, + root: &Path, + workspace: &Path, +) -> HarnessResult { + let mut value = parse(template)?; + let entry = value + .get("app") + .and_then(|v| v.get("entry")) + .and_then(Value::as_str) + .ok_or_else(|| invalid("app.entry"))?; + let entry = root.join(entry); + replace( + &mut value, + &["app", "entry"], + Value::String(entry.to_string_lossy().into_owned()), + )?; + for (name, adapter) in table(&mut value, &["adapters"])? { + let adapter = table(adapter, &["adapter"])?; + for key in ["crate", "manifest"] { + let old = adapter + .get(key) + .and_then(Value::as_str) + .ok_or_else(|| invalid(key))?; + let path = if key == "manifest" && matches!(name.as_str(), "fastly" | "spin") { + workspace.join(format!("{name}.toml")) + } else { + root.join(old) + }; + adapter.insert( + key.to_owned(), + Value::String(path.to_string_lossy().into_owned()), + ); + } + } + serialize(&value) +} + +#[cfg(test)] +mod tests { + use super::*; + + const CONFIG: &str = include_str!("../../../../trusted-server.example.toml"); + const MANIFEST: &str = include_str!("../../../../edgezero.toml"); + + #[test] + fn fixture_config_is_loopback_only_with_explicit_permission() { + for case in ["disabled", "success", "hung", "auction", "streaming"] { + let config = fixture_config(CONFIG, case, 12345, 12346) + .expect("should generate isolated configuration"); + let parsed: toml::Value = toml::from_str(&config).expect("should parse configuration"); + assert_eq!( + parsed["publisher"]["domain"].as_str(), + Some("127.0.0.1"), + "should isolate the publisher" + ); + assert_eq!( + parsed["publisher"]["origin_url"].as_str(), + Some("http://127.0.0.1:12345"), + "should isolate the origin" + ); + assert_eq!( + parsed["publisher"]["allow_async_html_finish"].as_bool(), + Some(case != "disabled"), + "should set explicit permission" + ); + assert_eq!( + parsed["publisher"]["html_finalization_timeout_ms"].as_integer(), + Some(750), + "should configure the fixture budget" + ); + } + } + + #[test] + fn fixture_config_rejects_missing_required_template_fields() { + let result = fixture_config( + "[publisher]\ndomain = 'example.com'", + "success", + 12345, + 12346, + ); + assert!(result.is_err(), "should reject incomplete templates"); + } + + #[test] + fn fixture_validation_rejects_external_endpoints() { + let config = fixture_config(CONFIG, "auction", 12345, 12346) + .expect("should generate auction configuration"); + let mut value = parse(&config).expect("should parse generated configuration"); + let provider = table(&mut value, &["auction", "providers"]) + .expect("should contain a provider") + .iter_mut() + .next() + .expect("should have a provider") + .1; + replace( + provider, + &["endpoint"], + Value::String("https://auction.example.com/bid".to_owned()), + ) + .expect("should change the controlled endpoint"); + assert!( + validate_fixture(&value, "auction", 12345, 12346).is_err(), + "should reject non-loopback auction endpoints" + ); + } + + #[test] + fn fixture_manifest_round_trips_paths() { + let root = Path::new("/tmp/example 🚀 \"quoted\"\\path\nnext"); + let workspace = root.join("workspace"); + let manifest = + fixture_manifest(MANIFEST, root, &workspace).expect("should isolate manifests"); + let parsed: toml::Value = toml::from_str(&manifest).expect("should parse quoted paths"); + assert_eq!( + parsed["app"]["entry"].as_str(), + root.join("crates/trusted-server-core").to_str(), + "should preserve all path characters" + ); + assert_eq!( + parsed["adapters"]["spin"]["adapter"]["manifest"].as_str(), + workspace.join("spin.toml").to_str(), + "should use an isolated runtime manifest" + ); + } +} diff --git a/crates/trusted-server-integration-tests/tests/html_eof_runtime/origin.rs b/crates/trusted-server-integration-tests/tests/html_eof_runtime/origin.rs new file mode 100644 index 000000000..828a6e8fd --- /dev/null +++ b/crates/trusted-server-integration-tests/tests/html_eof_runtime/origin.rs @@ -0,0 +1,257 @@ +use std::fs; +use std::io; +use std::net::{Ipv4Addr, TcpListener}; +use std::path::Path; +use std::sync::{Arc, Mutex}; +use std::time::{Duration, Instant}; + +use axum::body::Body; +use axum::extract::{DefaultBodyLimit, OriginalUri, State}; +use axum::http::StatusCode; +use axum::response::Response; +use axum::routing::{get, post}; +use axum::{Json, Router}; +use bytes::Bytes; +use error_stack::{Report, ResultExt as _}; +use futures::stream; +use serde_json::{Value, json}; +use tokio::runtime::Runtime; +use tokio::task::JoinHandle; + +use super::{HarnessError, HarnessResult}; + +const PREFIX: &[u8] = + b"

EXAMPLE_UNSIGNED

"; +const EOF: &[u8] = b""; + +#[derive(Clone)] +pub(super) struct Event { + pub kind: &'static str, + pub at: Instant, + pub data: Value, +} + +#[derive(Clone, Default)] +struct Events(Arc>>); + +impl Events { + fn record(&self, kind: &'static str, data: Value) { + self.0 + .lock() + .expect("should lock fixture events") + .push(Event { + kind, + at: Instant::now(), + data, + }); + } +} + +pub(super) struct Origin { + pub port: u16, + pub started: Instant, + pub runtime: Runtime, + events: Events, + server: JoinHandle<()>, +} + +impl Origin { + pub(super) fn start() -> HarnessResult { + let started = Instant::now(); + let listener = + TcpListener::bind((Ipv4Addr::LOCALHOST, 0)).change_context(HarnessError::Io)?; + let port = listener + .local_addr() + .change_context(HarnessError::Io)? + .port(); + listener + .set_nonblocking(true) + .change_context(HarnessError::Io)?; + let runtime = Runtime::new().change_context(HarnessError::Runtime)?; + let _entered = runtime.enter(); + let listener = + tokio::net::TcpListener::from_std(listener).change_context(HarnessError::Io)?; + let events = Events::default(); + let app = Router::new() + .route("/health", axum::routing::head(|| async { StatusCode::OK })) + .route("/article", get(article)) + .route("/success", post(lookup)) + .route("/hung", post(lookup)) + .layer(DefaultBodyLimit::max(2048)) + .with_state(events.clone()); + let server = runtime.spawn(async move { + if let Err(error) = axum::serve(listener, app).await { + log::error!("EOF origin fails: {error}"); + } + }); + drop(_entered); + Ok(Self { + port, + started, + runtime, + events, + server, + }) + } + + pub(super) fn events(&self) -> Vec { + self.events + .0 + .lock() + .expect("should lock fixture events") + .clone() + } + + pub(super) fn write_events(&self, path: &Path) -> HarnessResult<()> { + let events = self.events().iter().map(|event| json!({ + "event": event.kind, + "seconds_from_origin_start": event.at.duration_since(self.started).as_secs_f64(), + "data": event.data, + })).collect::>(); + fs::write(path, json!(events).to_string()).change_context(HarnessError::Io) + } +} + +impl Drop for Origin { + fn drop(&mut self) { + // Runtime teardown cancels pending lookup handlers, including the hung control. + self.server.abort(); + } +} + +async fn article(State(events): State) -> Response { + let body = stream::unfold(0, move |index| { + let events = events.clone(); + async move { + let bytes = match index { + 0 => Bytes::from_static(PREFIX), + 1 => { + tokio::time::sleep(Duration::from_millis(50)).await; + events.record("origin_eof", json!({})); + Bytes::from_static(EOF) + } + _ => return None, + }; + Some((Ok::<_, io::Error>(bytes), index + 1)) + } + }); + Response::builder() + .header("content-type", "text/html") + .header("cache-control", "public, max-age=60") + .header("content-length", PREFIX.len() + EOF.len()) + .body(Body::from_stream(body)) + .expect("should build the controlled origin response") +} + +async fn lookup( + State(events): State, + OriginalUri(uri): OriginalUri, + Json(mut payload): Json, +) -> &'static str { + payload["path"] = Value::String(uri.path().to_owned()); + events.record("lookup_start", payload); + let delay = if uri.path() == "/hung" { + Duration::from_secs(30) + } else { + Duration::from_millis(150) + }; + tokio::time::sleep(delay).await; + events.record("lookup_complete", json!({"path": uri.path()})); + "EXAMPLE_APPROVED" +} + +pub(super) fn lookup_for_request( + events: &[Event], + started: Instant, + completed: Instant, + enabled: bool, +) -> HarnessResult> { + let lookups: Vec<_> = events + .iter() + .filter(|event| { + event.kind == "lookup_start" && started <= event.at && event.at <= completed + }) + .collect(); + if lookups.len() != usize::from(enabled) { + return Err(Report::new(HarnessError::Runtime) + .attach("each request must issue exactly one enabled EOF lookup")); + } + let Some(lookup) = lookups.first().copied() else { + return Ok(None); + }; + if !events + .iter() + .any(|event| event.kind == "origin_eof" && started <= event.at && event.at <= lookup.at) + { + return Err(Report::new(HarnessError::Runtime) + .attach("EOF lookup starts before its own origin EOF")); + } + Ok(Some(lookup)) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn earlier_request_eof_cannot_satisfy_lookup_ordering() { + let start = Instant::now(); + let event = |kind, millis| Event { + kind, + at: start + Duration::from_millis(millis), + data: json!({}), + }; + let events = [ + event("origin_eof", 1), + event("lookup_start", 20), + event("origin_eof", 25), + ]; + assert!( + lookup_for_request( + &events, + start + Duration::from_millis(10), + start + Duration::from_millis(30), + true + ) + .is_err(), + "should reject an EOF from an earlier request" + ); + } + + #[test] + fn request_scoped_lookup_requires_exactly_one_enabled_call() { + let start = Instant::now(); + let events = [ + Event { + kind: "origin_eof", + at: start, + data: json!({}), + }, + Event { + kind: "lookup_start", + at: start, + data: json!({}), + }, + ]; + assert!( + lookup_for_request(&events, start, start, true) + .expect("should match the current EOF") + .is_some(), + "should return the current lookup" + ); + assert!( + lookup_for_request(&events, start, start, false).is_err(), + "should reject lookup with disabled capability" + ); + assert!( + lookup_for_request(&[], start, start, true).is_err(), + "should reject missing lookup" + ); + assert!( + lookup_for_request(&[], start, start, false) + .expect("should accept disabled work") + .is_none(), + "should accept no disabled lookup" + ); + } +} diff --git a/crates/trusted-server-integration-tests/tests/html_eof_runtime/process.rs b/crates/trusted-server-integration-tests/tests/html_eof_runtime/process.rs new file mode 100644 index 000000000..642db0f86 --- /dev/null +++ b/crates/trusted-server-integration-tests/tests/html_eof_runtime/process.rs @@ -0,0 +1,445 @@ +use std::collections::{BTreeMap, BTreeSet}; +use std::ffi::OsString; +use std::fs::File; +use std::path::Path; +use std::process::{Child, Command, ExitStatus, Stdio}; +use std::sync::atomic::{AtomicBool, Ordering}; +use std::sync::{Arc, Mutex}; +use std::thread; +use std::time::{Duration, Instant}; + +#[cfg(unix)] +use std::os::unix::process::CommandExt as _; + +use error_stack::{Report, ResultExt as _}; +use tokio::runtime::Handle; + +use super::{HarnessError, HarnessResult}; + +pub(super) fn build_environment( + source: impl IntoIterator, +) -> BTreeMap { + source + .into_iter() + .filter(|(name, _)| { + let Some(name) = name.to_str() else { + return false; + }; + name != "CARGO_BUILD_TARGET" + && (matches!( + name, + "PATH" + | "RUSTUP_HOME" + | "CARGO_HOME" + | "CARGO_TARGET_DIR" + | "TMPDIR" + | "WASM_BINDGEN_BIN" + | "WASM_OPT_BIN" + | "ESBUILD_BIN" + | "CARGO_NET_OFFLINE" + | "RUSTFLAGS" + | "CARGO_ENCODED_RUSTFLAGS" + | "RUSTDOCFLAGS" + | "CARGO_INCREMENTAL" + | "RUSTC" + | "RUSTDOC" + | "RUSTC_WRAPPER" + | "RUSTC_WORKSPACE_WRAPPER" + | "TSJS_PREBUILT_DIR" + ) || name.starts_with("CARGO_PROFILE_") + || name.starts_with("CARGO_BUILD_") + || name.starts_with("CARGO_TARGET_")) + }) + .collect() +} + +#[derive(Clone, Default)] +pub(super) struct Cancellation { + cancelled: Arc, + groups: Arc>>, +} + +impl Cancellation { + pub(super) fn check(&self) -> HarnessResult<()> { + if self.cancelled.load(Ordering::SeqCst) { + Err(Report::new(HarnessError::Cancelled)) + } else { + Ok(()) + } + } + + fn cancel(&self) { + self.cancelled.store(true, Ordering::SeqCst); + for id in self + .groups + .lock() + .expect("should lock owned process groups") + .iter() + { + signal_group(*id, libc::SIGTERM); + } + } + + pub(super) fn watch(&self, handle: &Handle) -> HarnessResult<()> { + #[cfg(unix)] + { + let _entered = handle.enter(); + let mut term = + tokio::signal::unix::signal(tokio::signal::unix::SignalKind::terminate()) + .change_context(HarnessError::Runtime)?; + let mut interrupt = + tokio::signal::unix::signal(tokio::signal::unix::SignalKind::interrupt()) + .change_context(HarnessError::Runtime)?; + // Cargo owns the test process. Detect its disappearance as well as + // direct signals so terminating the shell entry point leaves no runtimes. + let parent = unsafe { libc::getppid() }; + let cancellation = self.clone(); + handle.spawn(async move { + loop { + tokio::select! { + _ = term.recv() => break, + _ = interrupt.recv() => break, + _ = tokio::time::sleep(Duration::from_millis(100)) => { + if unsafe { libc::getppid() } != parent { break; } + } + } + } + cancellation.cancel(); + tokio::time::sleep(Duration::from_secs(5)).await; + for id in cancellation + .groups + .lock() + .expect("should lock owned process groups") + .iter() + { + signal_group(*id, libc::SIGKILL); + } + }); + } + #[cfg(not(unix))] + { + let _ = handle; + return Err(Report::new(HarnessError::Runtime) + .attach("EOF runtime controls require Unix process groups")); + } + Ok(()) + } +} + +fn signal_group(id: u32, signal: i32) { + #[cfg(unix)] + if let Ok(id) = libc::pid_t::try_from(id) { + // Every registered ID belongs to a child started in its own process group. + unsafe { + libc::killpg(id, signal); + } + } + #[cfg(not(unix))] + let _ = (id, signal); +} + +pub(super) struct ManagedChild { + child: Child, + cancellation: Cancellation, +} + +impl ManagedChild { + pub(super) fn spawn(command: &mut Command, cancellation: &Cancellation) -> HarnessResult { + cancellation.check()?; + #[cfg(unix)] + command.process_group(0); + let child = command.spawn().change_context(HarnessError::Runtime)?; + cancellation + .groups + .lock() + .expect("should lock owned process groups") + .insert(child.id()); + Ok(Self { + child, + cancellation: cancellation.clone(), + }) + } + + pub(super) fn status(&mut self) -> HarnessResult> { + self.cancellation.check()?; + self.child.try_wait().change_context(HarnessError::Runtime) + } + + pub(super) fn wait(&mut self) -> HarnessResult { + loop { + if let Some(status) = self.status()? { + return Ok(status); + } + thread::sleep(Duration::from_millis(50)); + } + } +} + +impl Drop for ManagedChild { + fn drop(&mut self) { + let id = self.child.id(); + signal_group(id, libc::SIGTERM); + let until = Instant::now() + Duration::from_secs(5); + while matches!(self.child.try_wait(), Ok(None)) && Instant::now() < until { + thread::sleep(Duration::from_millis(20)); + } + signal_group(id, libc::SIGKILL); + #[cfg(not(unix))] + let _ = self.child.kill(); + let _ = self.child.wait(); + self.cancellation + .groups + .lock() + .expect("should lock owned process groups") + .remove(&id); + } +} + +pub(super) fn logged_command( + command: &mut Command, + log: &Path, + cancellation: &Cancellation, +) -> HarnessResult<()> { + let output = File::create(log).change_context(HarnessError::Io)?; + command + .stdout(Stdio::from( + output.try_clone().change_context(HarnessError::Io)?, + )) + .stderr(Stdio::from(output)); + let status = ManagedChild::spawn(command, cancellation)?.wait()?; + if !status.success() { + return Err(Report::new(HarnessError::Runtime).attach(format!( + "command exits with {status}; see {}", + log.display() + ))); + } + Ok(()) +} + +#[cfg(test)] +mod tests { + use super::*; + use std::fs; + use std::path::PathBuf; + + #[test] + fn compiler_settings_are_preserved_without_operator_settings() { + let source = [ + ("PATH", "/tmp/example-bin"), + ("RUSTFLAGS", "-D warnings"), + ("CARGO_PROFILE_DEV_DEBUG", "0"), + ("CARGO_INCREMENTAL", "0"), + ("CARGO_TARGET_DIR", "/tmp/example-target"), + ("CARGO_BUILD_TARGET", "wasm32-wasip1"), + ("TRUSTED_SERVER__PUBLISHER__DOMAIN", "example.com"), + ("TSJS_SKIP_BUILD", "1"), + ]; + let selected = build_environment( + source.map(|(name, value)| (OsString::from(name), OsString::from(value))), + ); + for name in [ + "PATH", + "RUSTFLAGS", + "CARGO_PROFILE_DEV_DEBUG", + "CARGO_INCREMENTAL", + "CARGO_TARGET_DIR", + ] { + assert!( + selected.contains_key(&OsString::from(name)), + "should preserve {name}" + ); + } + for name in [ + "CARGO_BUILD_TARGET", + "TRUSTED_SERVER__PUBLISHER__DOMAIN", + "TSJS_SKIP_BUILD", + ] { + assert!( + !selected.contains_key(&OsString::from(name)), + "should isolate {name}" + ); + } + } + + #[test] + #[cfg(unix)] + fn owned_process_is_reaped_on_drop() { + let cancellation = Cancellation::default(); + let mut command = Command::new("sleep"); + command.arg("300"); + let child = + ManagedChild::spawn(&mut command, &cancellation).expect("should launch an owned child"); + let id = child.child.id(); + drop(child); + assert_eq!( + unsafe { libc::kill(libc::pid_t::try_from(id).expect("should fit pid"), 0) }, + -1, + "should reap the terminated process" + ); + assert!( + cancellation + .groups + .lock() + .expect("should lock groups") + .is_empty(), + "should release ownership" + ); + } + + #[test] + #[cfg(unix)] + fn completed_process_is_safe_to_drop() { + let cancellation = Cancellation::default(); + let child = ManagedChild::spawn(&mut Command::new("true"), &cancellation) + .expect("should launch a completed command"); + let mut child = child; + assert!( + child.wait().expect("should wait for completion").success(), + "should complete normally" + ); + drop(child); + assert!( + cancellation + .groups + .lock() + .expect("should lock groups") + .is_empty(), + "should release completed ownership" + ); + } + + #[test] + #[cfg(unix)] + fn sigterm_cleans_owned_descendants() { + signal_cleanup(libc::SIGTERM); + } + + #[test] + #[cfg(unix)] + fn sigint_cleans_owned_descendants() { + signal_cleanup(libc::SIGINT); + } + + #[cfg(unix)] + fn signal_cleanup(signal: i32) { + let directory = tempfile::tempdir().expect("should create probe workspace"); + let cancellation = Cancellation::default(); + let executable = std::env::current_exe().expect("should locate Rust probe executable"); + let mut command = Command::new(executable); + command + .args([ + "--exact", + "process::tests::cleanup_owner_probe", + "--ignored", + ]) + .env("HTML_EOF_PROBE_DIR", directory.path()) + .stdout(Stdio::null()) + .stderr(Stdio::null()); + let mut parent = ManagedChild::spawn(&mut command, &cancellation) + .expect("should launch the owner probe"); + let ready = directory.path().join("owner.ready"); + let until = Instant::now() + Duration::from_secs(5); + while !ready.exists() { + assert!( + parent + .status() + .expect("should inspect the owner probe") + .is_none(), + "should keep the probe alive until ready" + ); + assert!(Instant::now() < until, "should start the cleanup probe"); + thread::sleep(Duration::from_millis(20)); + } + let leaf: libc::pid_t = fs::read_to_string(&ready) + .expect("should read owned PID") + .parse() + .expect("should parse owned PID"); + assert_eq!( + unsafe { + libc::kill( + libc::pid_t::try_from(parent.child.id()).expect("should fit pid"), + signal, + ) + }, + 0, + "should signal the owner" + ); + let until = Instant::now() + Duration::from_secs(15); + let status = loop { + if let Some(status) = parent.status().expect("should inspect cleanup progress") { + break status; + } + assert!( + Instant::now() < until, + "should finish cancellation cleanup within its deadline" + ); + thread::sleep(Duration::from_millis(20)); + }; + assert!(status.success(), "should complete cancellation cleanup"); + assert!( + directory.path().join("cleaned").exists(), + "should unwind owned-process guards" + ); + assert_eq!( + unsafe { libc::kill(leaf, 0) }, + -1, + "should reap the stubborn descendant" + ); + } + + #[test] + #[ignore = "internal subprocess probe, selected only by cleanup tests"] + #[cfg(unix)] + fn cleanup_leaf_probe() { + let directory = PathBuf::from( + std::env::var_os("HTML_EOF_PROBE_DIR").expect("should supply probe workspace"), + ); + unsafe { + libc::signal(libc::SIGTERM, libc::SIG_IGN); + } + fs::write(directory.join("leaf.ready"), "ready") + .expect("should report installed signal handler"); + thread::sleep(Duration::from_secs(300)); + } + + #[test] + #[ignore = "internal subprocess probe, selected only by cleanup tests"] + #[cfg(unix)] + fn cleanup_owner_probe() { + let directory = PathBuf::from( + std::env::var_os("HTML_EOF_PROBE_DIR").expect("should supply probe workspace"), + ); + let runtime = tokio::runtime::Runtime::new().expect("should create signal runtime"); + let cancellation = Cancellation::default(); + cancellation + .watch(runtime.handle()) + .expect("should watch cancellation"); + let mut command = Command::new(std::env::current_exe().expect("should locate leaf probe")); + command + .args(["--exact", "process::tests::cleanup_leaf_probe", "--ignored"]) + .env("HTML_EOF_PROBE_DIR", &directory) + .stdout(Stdio::null()) + .stderr(Stdio::null()); + let mut leaf = ManagedChild::spawn(&mut command, &cancellation) + .expect("should spawn the stubborn descendant"); + let until = Instant::now() + Duration::from_secs(5); + while !directory.join("leaf.ready").exists() { + assert!( + Instant::now() < until, + "should start the stubborn descendant" + ); + thread::sleep(Duration::from_millis(20)); + } + fs::write(directory.join("owner.ready"), leaf.child.id().to_string()) + .expect("should report owned descendant"); + let result = leaf + .wait() + .expect_err("should interrupt an active owned command"); + assert!( + matches!(result.current_context(), HarnessError::Cancelled), + "should report cancellation" + ); + drop(leaf); + fs::write(directory.join("cleaned"), "cleaned").expect("should report completed cleanup"); + } +} diff --git a/crates/trusted-server-integration-tests/tests/parity.rs b/crates/trusted-server-integration-tests/tests/parity.rs index 2428d7799..dc26666ef 100644 --- a/crates/trusted-server-integration-tests/tests/parity.rs +++ b/crates/trusted-server-integration-tests/tests/parity.rs @@ -8,7 +8,13 @@ // Both adapters define `TrustedServerApp` — alias both to avoid name collision. // axum::http re-exports from the `http` crate, so HeaderMap types are identical. +use error_stack::Report; use std::sync::Arc; +use std::sync::atomic::{AtomicUsize, Ordering}; +use trusted_server_core::platform::{ + PlatformError, PlatformHttpClient, PlatformHttpRequest, PlatformPendingRequest, + PlatformResponse, PlatformSelectResult, +}; use axum::body::Body as AxumBody; use axum::http::Request as AxumRequest; @@ -1119,3 +1125,191 @@ async fn adapter_buffers_nextjs_auction_output() { } } } + +#[derive(Default)] +struct TerminalFixtureOrigin { + lookup_calls: AtomicUsize, + fail_lookup: bool, +} + +#[async_trait::async_trait(?Send)] +impl PlatformHttpClient for TerminalFixtureOrigin { + async fn send( + &self, + request: PlatformHttpRequest, + ) -> Result> { + let (content_type, body) = if request.request.uri().host() == Some("lookup.example.com") { + tokio::task::yield_now().await; + let payload = request + .request + .into_body() + .into_bytes() + .expect("should send a bounded fixture payload"); + let payload: serde_json::Value = serde_json::from_slice(&payload) + .expect("should encode completed-document metadata"); + assert_eq!( + payload["complete"], true, + "should observe the EOF sentinel before I/O" + ); + assert_eq!( + payload["sha256"] + .as_str() + .expect("should hash transformed HTML") + .len(), + 64, + "should send a complete digest" + ); + self.lookup_calls.fetch_add(1, Ordering::SeqCst); + if self.fail_lookup { + return Err(Report::new(PlatformError::HttpClient)); + } + ("text/plain", "EXAMPLE_APPROVED") + } else { + ( + "text/html", + "

EXAMPLE_UNSIGNED

", + ) + }; + Ok(PlatformResponse::new( + http::Response::builder() + .status(200) + .header("content-type", content_type) + .header("cache-control", "public, max-age=60") + .body(edgezero_core::body::Body::from(body)) + .expect("should build an example upstream response"), + )) + } + async fn send_async( + &self, + _: PlatformHttpRequest, + ) -> Result> { + Err(Report::new(PlatformError::Unsupported)) + } + async fn select( + &self, + _: Vec, + ) -> Result> { + Err(Report::new(PlatformError::Unsupported)) + } +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn asynchronous_html_finalization_and_privacy_match_host_adapters() { + let mut disabled_output = None; + for case in ["disabled", "failed", "success"] { + let enabled = case != "disabled"; + let mut settings = test_settings(); + settings.publisher.allow_async_html_finish = enabled; + settings.integrations.insert_config("html_finalization_fixture", &serde_json::json!({ + "enabled": enabled, "lookup_url": format!("https://lookup.example.com/{case}"), "transport_timeout_ms": 1000, + })).expect("should enable the fictional terminal consumer"); + let client = Arc::new(TerminalFixtureOrigin { + lookup_calls: AtomicUsize::new(0), + fail_lookup: case == "failed", + }); + let services = nextjs_auction::services_with_http_client( + Arc::clone(&client) as Arc + ); + let routers = [ + ( + "Axum", + AxumApp::routes_with_settings_and_services(settings.clone(), services.clone()), + ), + ( + "Cloudflare", + CloudflareApp::routes_with_settings_and_services( + settings.clone(), + services.clone(), + ), + ), + ( + "Spin", + SpinApp::routes_with_settings_and_services(settings, services), + ), + ]; + let mut expected = None; + for (adapter, router) in routers { + client.lookup_calls.store(0, Ordering::SeqCst); + let request = request_builder() + .method("GET") + .uri("https://test-publisher.example.com/article") + .header("host", "test-publisher.example.com") + .header("accept", "text/html") + .body(edgezero_core::body::Body::empty()) + .expect("should build an example navigation"); + let response = router + .expect("should build fixture routes") + .oneshot(request) + .await + .expect("should serve finalized HTML"); + assert_eq!( + response.status(), + 200, + "{adapter} should preserve the origin status" + ); + if enabled { + assert_eq!( + response + .headers() + .get("cache-control") + .expect("should set terminal privacy"), + "no-store, private", + "{adapter} should prevent downstream caching" + ); + } else { + assert_ne!( + response + .headers() + .get("cache-control") + .expect("should preserve the ordinary cache policy"), + "no-store, private", + "{adapter} should not apply EOF privacy to the inactive fixture" + ); + } + let body = response + .into_body() + .into_bytes_bounded(1024 * 1024) + .await + .expect("should drain finalized HTML"); + assert!( + String::from_utf8_lossy(&body).contains(if case == "success" { + "EXAMPLE_FINALIZED" + } else { + "EXAMPLE_UNSIGNED" + }), + "{adapter} should select the correct EOF result for {case}" + ); + if case != "success" { + assert!( + !String::from_utf8_lossy(&body).contains("EXAMPLE_FINALIZED"), + "{adapter} should not emit a finalized marker for {case}" + ); + } + if case == "failed" { + assert_eq!( + &body, + disabled_output + .as_ref() + .expect("should have the disabled baseline"), + "{adapter} should preserve the exact baseline after failed EOF I/O" + ); + } + assert_eq!( + client.lookup_calls.load(Ordering::SeqCst), + usize::from(enabled), + "{adapter} should call EOF lookup only for enabled work in {case}" + ); + if let Some(expected) = &expected { + assert_eq!( + &body, expected, + "{adapter} should produce identical terminal HTML" + ); + } else { + expected = Some(body); + } + } + if case == "disabled" { + disabled_output = expected; + } + } +} diff --git a/docs/guide/configuration.md b/docs/guide/configuration.md index 81d936ec2..70eafda92 100644 --- a/docs/guide/configuration.md +++ b/docs/guide/configuration.md @@ -309,14 +309,16 @@ Core publisher settings for domain, origin, and proxy configuration. ### `[publisher]` -| Field | Type | Required | Description | -| ----------------------------- | ------- | -------- | --------------------------------------------------------------------------- | -| `domain` | String | Yes | Publisher's apex domain name | -| `cookie_domain` | String | Yes | Domain for non-EC cookies (typically with leading dot) | -| `origin_url` | String | Yes | Full URL of publisher origin server | -| `origin_host_header_override` | String | No | Outbound Host header to send while connecting to `origin_url` | -| `proxy_secret` | String | Yes | Secret-store key name for the proxy URL secret | -| `max_buffered_body_bytes` | Integer | No | Buffered-body cap / Fastly stream raw+decoded byte ceiling (default 16 MiB) | +| Field | Type | Required | Description | +| ------------------------------ | ------- | -------- | ------------------------------------------------------------------------------------ | +| `domain` | String | Yes | Publisher's apex domain name | +| `cookie_domain` | String | Yes | Domain for non-EC cookies (typically with leading dot) | +| `origin_url` | String | Yes | Full URL of publisher origin server | +| `origin_host_header_override` | String | No | Outbound Host header to send while connecting to `origin_url` | +| `proxy_secret` | String | Yes | Secret-store key name for the proxy URL secret | +| `max_buffered_body_bytes` | Integer | No | Buffered-body cap / Fastly stream raw+decoded byte ceiling (default 16 MiB) | +| `allow_async_html_finish` | Boolean | No | Explicit permission for async HTML EOF work and its delivery cost (default false) | +| `html_finalization_timeout_ms` | Integer | No | Shared optional EOF budget after decoder/parser EOF (default 5000 ms; range 1–60000) | > **Note:** EC cookies (`ts-ec`) derive their domain automatically as `.{domain}` and > do not use `cookie_domain`. The `cookie_domain` field is used by other cookie helpers. @@ -342,6 +344,8 @@ TRUSTED_SERVER__PUBLISHER__ORIGIN_URL=https://origin.publisher.com TRUSTED_SERVER__PUBLISHER__ORIGIN_HOST_HEADER_OVERRIDE=www.publisher.com TRUSTED_SERVER__PUBLISHER__PROXY_SECRET=publisher_proxy_secret TRUSTED_SERVER__PUBLISHER__MAX_BUFFERED_BODY_BYTES=16777216 +TRUSTED_SERVER__PUBLISHER__ALLOW_ASYNC_HTML_FINISH=false +TRUSTED_SERVER__PUBLISHER__HTML_FINALIZATION_TIMEOUT_MS=5000 ``` ### Field Details @@ -437,11 +441,43 @@ intentionally, and never put it in the TOML file or pushed app-config blob. Changing `proxy_secret` invalidates all existing signed URLs. Plan rotations carefully and use graceful transition periods. ::: +#### `allow_async_html_finish` + +Explicit permission for integrations declaring asynchronous HTML processor completion. +Defaults to `false`. An enabled integration requiring EOF I/O without permission is +rejected with a configuration error; its requested work is not silently skipped. +Permission alone does not change processing when no async factories are registered. + +Non-retaining processors can release ready HTML before EOF I/O on Fastly. A factory +requesting output retention delays initial delivery until EOF or overflow. Axum, +Workers and Spin currently buffer publisher responses, so enabled EOF I/O can add to +TTFB there even without stage retention. Enable only when accepting that delivery cost. + +#### `html_finalization_timeout_ms` + +Shared budget for optional asynchronous processor EOF work in milliseconds. The default +is `5000`; accepted values are `1` through `60000`. It begins after decoder/parser EOF +and does not change the auction deadline. The initial 5-second default follows the +issue example; 60 seconds is a configuration ceiling, not a recommended delivery +latency or a measured optimum. Timer expiry skips later optional I/O while +still completing downstream synchronous EOF. Timeout retains the current un-emitted +stage output; it cannot undo bytes already delivered or remote side effects. + +The setting is inert without async processors. Axum, Workers and Spin can cancel +yielding callbacks. Fastly consumers must independently bound blocking host calls. +Opted-in processed HTML bypasses shared-template caching and uses private/no-store +policy; raw-origin readthrough remains separate. Opted-in HTML with 206 or any +`Content-Range` bypasses rewriting and EOF work while preserving encoded bytes and +range/framing metadata. Full error HTML remains eligible and exposes actual status. +See [Asynchronous HTML processor completion](/guide/integration-guide#asynchronous-html-processor-completion). + #### `max_buffered_body_bytes` -**Purpose**: Upper bound on how much of a publisher origin body the rewrite -pipeline holds in memory — the post-rewrite output buffer on buffered adapters, -and the per-stream raw/decoded byte ceiling on the Fastly streaming path. +**Purpose**: Limit individual publisher buffers and per-stream raw/decoded byte +totals. This is not an aggregate request-memory budget. Async retention collectors, +replacement candidates, auction suffixes, codec state and adapter response buffers +can coexist; total memory may exceed this setting. Integration-owned allocations +are outside the framework's bound. **Usage**: diff --git a/docs/guide/integration-guide.md b/docs/guide/integration-guide.md index f127d4207..5ece3878a 100644 --- a/docs/guide/integration-guide.md +++ b/docs/guide/integration-guide.md @@ -203,6 +203,175 @@ Streaming support is an adapter capability. Request checked `supports_streaming_responses()`; unsupported adapters must reject the request instead of silently buffering it. +## Asynchronous HTML processor completion + +An integration can register an HTML stream processor that optionally awaits HTTP, +KV or other request services after its synchronous EOF call. Use the existing +`IntegrationHtmlStreamProcessorFactory` and `.with_html_stream_processor(...)`. +Article signing is only one possible consumer; core implements the processor +lifecycle, not a signing protocol or background job system. + +Factories declare `requires_async_finish()`. Sessions implement the matching +`StreamProcessor::requires_async_finish()` and optional `async_finish(...)`. +The default callback returns `None`, creating no future. Factory metadata is +available before template lookup, so requested EOF work cannot be skipped by a +processed cache hit. The factory and session share request-local +`IntegrationDocumentState` with the existing parser hooks. + +A non-retaining session can make a KV lookup at EOF and append a fixed footer: + + + +```rust +use error_stack::Report; +use futures::future::LocalBoxFuture; +use std::io; +use trusted_server_core::streaming_processor::StreamProcessor; +use trusted_server_core::streaming_processor::finish::{ + StreamFinish, StreamFinishError, StreamFinishInput, +}; + +/// Append a fixed footer when an example KV key is present. +pub struct ExampleFooterProcessor; + +impl StreamProcessor for ExampleFooterProcessor { + fn process_chunk(&mut self, bytes: &[u8], _is_last: bool) -> io::Result> { + Ok(bytes.to_vec()) + } + + fn requires_async_finish(&self) -> bool { + true + } + + fn async_finish<'a>( + &'a mut self, + input: StreamFinishInput<'a>, + ) -> Option>>> { + Some(Box::pin(async move { + let enabled = input + .services + .kv_store() + .get_bytes("example:footer") + .await + .map_err(|_| Report::new(StreamFinishError::Unavailable))? + .is_some(); + if !enabled { + return Ok(StreamFinish::Unchanged); + } + input.replacement.append(input.output)?; + input.replacement.append(b"")?; + Ok(StreamFinish::Replace) + })) + } +} +``` + + + +The factory also returns `true` from `requires_async_finish()` and creates this +session. Leave `buffer_output_until_eof()` at its default `false` for this append +operation. KV must be available on the runtime; unavailable lookup falls back to +the synchronous tail. Fastly consumers must use host operations with independently +bounded latency and cap configurable transport timeouts with `input.deadline.remaining()`; +this generic KV interface does not provide a hard timeout. Checking the remaining +budget alone cannot interrupt a blocking KV call. + +Operators must explicitly accept the possible delivery cost: + +```toml +[publisher] +allow_async_html_finish = true +html_finalization_timeout_ms = 5000 +``` + +Permission defaults to false. Enabling an integration requiring async completion +without permission produces a configuration error naming the integration and the +setting. Work is not silently omitted. Permission alone, with no async factories, +keeps ordinary processing unchanged. + +Chunk callbacks remain synchronous. At EOF, core validates/drains the decoder and +finishes the HTML parser, sending ready bytes through downstream processors as +ordinary chunks. It then completes each processor in registration order: synchronous +EOF, optional awaited finish, accepted output passed downstream, then downstream EOF. +The auction and encoder finish afterward. Ready bytes are released before optional +I/O; a callback is never run inside a parser handler. + +`StreamFinishInput` supplies actual `response_status`, borrowed `output`, `services`, +`deadline` and a separate bounded `replacement` writer. The output is the un-emitted +synchronous tail of this stage. It is the complete retained output only when the +factory explicitly requests `buffer_output_until_eof()`. Return `StreamFinish::Unchanged` +to retain it, or write a complete candidate and return `Replace`. To append a tail, +write the existing `output` followed by the new bytes. Empty replacement is intentional. +The session and future may be non-`Send`; start I/O inside the returned future. + +Core buffers a stage's output only when its factory explicitly requests retention. +The holder and replacement candidate each use `max_buffered_body_bytes` as their cap. +These caps apply per holder; simultaneous holders and codec/adapter buffers can +use more memory in total. On collection overflow, core releases prefix/crossing/later output exactly once and +skips that stage's optional I/O, while still completing its synchronous EOF. It cannot +bound arbitrary private allocations inside an integration. Streaming stages cannot +replace bytes already delivered; EOF append does not imply safe insertion into an +already emitted head or before an earlier body close. + +Callback errors, poisoned candidates, invalid assembly-marker multiplicity and timeout +discard the candidate and retain current stage output. Previously accepted stages stay +accepted. One absolute deadline covers every callback, starting after decoder/parser +EOF. On expiry, core drops the callback, skips later optional I/O, and still completes +all downstream synchronous EOF calls. Origin, parser, synchronous processing, codec and +sink errors remain fatal. Callback panics propagate. Cancellation does not undo remote +writes or guarantee that transport work is cancelled. + +Axum, Workers and Spin provide cooperative timers. Fastly has none because its host +calls block; bound those calls with `deadline.remaining()` and transport timeouts. +A timer cannot preempt CPU work or blocking host calls. Avoid logging HTML, lookup URLs +or external response contents. + +The latency contract depends on the adapter and the stage: + +- Without async factories, delivery and auction timing keep their existing path. +- On Fastly, a non-retaining stage releases ready HTML before optional EOF I/O. + With an auction seam, the body-close and following tail wait for all hooks before + auction assembly; initial bytes can arrive promptly while page completion waits. +- A retaining stage delays first HTML delivery until EOF or overflow release. +- Axum, Workers and Spin currently buffer publisher responses. Enabled EOF I/O can + add to TTFB there even without stage retention. This feature does not add streaming + support to those adapters. + +Use `stream_publisher_body_async`, `buffer_publisher_response_async`, or +`publisher_response_into_streaming_response` for opted-in HTML. The synchronous +pipeline and `stream_publisher_body` reject async processing before input/output +effects. The raw partial-HTML copy path performs no callback; the synchronous helper +leaves pending auction ownership with its caller, so pending auctions still require +an async entry point. Async cancellation retains the auction guard's warning; error +paths that can await emit abandonment telemetry. + +Opted-in HTML bypasses shared-template reads/reservations/stores and receives +`Cache-Control: no-store, private`, including unchanged, failed and overflow-bypassed +work. Raw-origin readthrough remains independent. Cacheable asynchronous output needs +separate rules for output identity, external-data freshness, expiry, invalidation, +failure caching and callback execution on hits. + +When async factories are registered, HTML with 206 or any `Content-Range` bypasses +rewriting and EOF I/O while preserving encoded bytes, framing, validators and origin +cache policy. Independent cookie/diagnostics privacy still applies. HEAD, bodiless, +non-HTML and unsupported encodings run no callbacks. Complete error HTML is eligible; +inspect `response_status` to decide whether the integration applies. Without async +factories, existing partial-response handling continues. + +Callers of `handle_publisher_request` supply the same enabled registry in +`AuctionDispatch::integration_registry` and response processing. This required field +allows cache admission to see async capabilities before lookup. + +Replacement output passes through later stream processors, but is not parsed again. +Preserve CSP, Next.js/RSC semantics and opaque assembly markers. Core checks marker +multiplicity including split boundaries, not HTML structure. Each real consumer must +verify winning-auction placement after its changes and preserve the structural body +end; neighboring bytes alone cannot establish a marker's parsing context. + +A fictional HTTP processor is available only with `test-utils`. See +[HTML EOF runtime checks](/guide/testing#html-eof-runtime-checks) for the runtime +harness, runtime/tooling prerequisites and CI coverage. + ## Browser and script guards Add a browser module only when browser state is required. Immediate modules diff --git a/docs/guide/testing.md b/docs/guide/testing.md index 86c025908..61e2802eb 100644 --- a/docs/guide/testing.md +++ b/docs/guide/testing.md @@ -42,6 +42,40 @@ tests are in `crates/trusted-server-core/src/edge_cookie.rs`, creative rewrite tests are in `crates/trusted-server-core/src/creative.rs`, and proxy tests are in `crates/trusted-server-core/src/proxy.rs`. +## HTML EOF runtime checks + +The EOF harness builds fresh `test-utils` artifacts, creates isolated configuration +and runs disabled, successful, hung and enabled-auction controls against a loopback +origin. It uses the native Rust integration-test target and requires Rust and pinned Node. Compiler/profile +settings and an explicit `CARGO_TARGET_DIR` are preserved; operator configuration and `CARGO_BUILD_TARGET` are +not inherited because the harness selects native and adapter targets. Logs, bodies, headers and timing JSON remain in its printed temporary +directory. Owned process groups are stopped on normal exit, failure and SIGTERM. + +```bash +cargo test --manifest-path crates/trusted-server-integration-tests/Cargo.toml --features html-eof-runtime --test html_eof_runtime +./scripts/async-html-finalization-local-test.sh axum --auction +./scripts/async-html-finalization-local-test.sh cloudflare --release --auction +./scripts/async-html-finalization-local-test.sh spin --release --auction +``` + +Workers requires `worker-build` and `wasm-bindgen` matching the locked SDK plus Wrangler. +Spin requires the Spin CLI. `--release` selects production-profile fixture artifacts. + +The Fastly EOF timeout fixture requires **Viceroy 0.19 or newer** for backend timeout +simulation. Install it in an isolated location; ordinary WASI tests continue to use +repository-pinned Viceroy 0.17: + +```bash +cargo install viceroy --version 0.19.0 --locked --root /tmp/example-eof-viceroy +PATH="/tmp/example-eof-viceroy/bin:$PATH" ./scripts/async-html-finalization-local-test.sh fastly --release --auction +``` + +The harness checks this prerequisite before building adapter fixtures. Fastly additionally proves +that an actual body byte arrives before a non-retaining EOF lookup completes. +CI runs Axum, native Rust helper tests and lint, and an isolated Fastly EOF job. Workers/Spin +actual EOF timeout controls remain local commands; their target builds, native tests +and shared parity controls run in CI. Hosted results are separate from local runs. + ## Documentation checks Check the documentation surfaces from the repository root: diff --git a/docs/superpowers/plans/2026-10-12-1241-async-html-finalization.md b/docs/superpowers/plans/2026-10-12-1241-async-html-finalization.md new file mode 100644 index 000000000..4c5da824e --- /dev/null +++ b/docs/superpowers/plans/2026-10-12-1241-async-html-finalization.md @@ -0,0 +1,100 @@ +# Ordered HTML EOF Completion Implementation Plan + +> **For agentic workers:** REQUIRED SUB-SKILL: Use superpowers:executing-plans or superpowers:subagent-driven-development. Follow repository instructions for centralized implementation and independent review. + +**Goal:** Support optional ordered EOF I/O with explicit operator permission, bounded held output and predictable failure behavior. + +**Architecture:** Keep synchronous chunk processing and one optional `StreamProcessor::async_finish` callback. Complete each stage before downstream EOF, stream ordinary output, retain output only on explicit declaration, and finish auction/encoding afterward. Limits apply per holder; this is not a total request-memory guarantee. + +**Tech Stack:** Rust 2024, futures local futures, error-stack, lol_html, adapter timers, native Rust integration tests and GitHub Actions. + +**Design:** [Ordered asynchronous HTML processor completion](../specs/2026-10-12-1241-async-html-finalization-design.md). + +## Implemented lifecycle + +- [x] Add default-absent borrowing callback contracts, bounded candidate writer and shared absolute deadline in `streaming_processor.rs` and `streaming_processor/finish.rs`. +- [x] Declare async/retention capabilities in the existing registry sequence; reject missing operator permission and inconsistent metadata before input effects. +- [x] Implement stepped parser/stage EOF, optional retention, overflow replay, downstream propagation and cancellation in `html_processor.rs` and `html_processor/finish.rs`. +- [x] Migrate the publisher driver without universal buffering; preserve auction/codec ordering, cache/privacy and partial-response safeguards. +- [x] Add production-lifecycle fictional fixtures, native/WASI regression coverage and actual four-runtime controls. + +## Review hardening + +### 1. Auction suffix storage and diagnostics + +**Files:** `crates/trusted-server-core/src/publisher.rs`, `crates/trusted-server-core/src/publisher/html_finalization.rs`. + +- [x] Extend the fatal-overflow regression to require `auction_suffix_overflow` telemetry. Add repeated-chunk growth coverage with a non-power-of-two cap. +- [x] Run `cargo test -p trusted-server-core --all-features async_eof` and observe the reason/growth failures. +- [x] Reserve only when needed, doubling capacity up to the checked cap. Classify suffix overflow before scanning and encoding errors immediately before encoding. +- [x] Repeat the focused tests and relevant adapter checks. + +### 2. Harness configuration, build settings and termination + +**Files:** `crates/trusted-server-integration-tests/tests/html_eof_runtime.rs` and `tests/html_eof_runtime/{config,origin,process}.rs`. + +- [x] Add failing tests for retained compiler/profile settings, missing template matches, invalid generated configuration, unsupported Viceroy and parent SIGTERM cleanup. +- [x] Run `cargo test --manifest-path crates/trusted-server-integration-tests/Cargo.toml --features html-eof-runtime --test html_eof_runtime` and record failures before fixes. +- [x] Preserve explicit Rust build settings without inheriting operator configuration; check template replacements and parsed timeout/permission/loopback targets before configuration push. +- [x] Safely quote TOML paths, validate Fastly tooling before builds, unwind on SIGTERM, and require cooperative timeout diagnostics. +- [x] Repeat unit/real-process tests and all four actual-runtime commands in `scripts/README.md`. + +### 3. Remaining EOF acceptance boundaries + +**File:** `crates/trusted-server-core/src/html_processor/finish.rs`. + +- [x] Cover elapsed timerless completion after polling, oversized non-retained fallback, one-byte markers, duplicate parser EOF and successful EOF at the retention cap. +- [x] Verify fallback ownership, no late commitment, no optional polling after overflow and exactly-once EOF. +- [x] Keep existing cancellation, poisoning, ties, split markers and accepted-upstream-output tests; verify disabled, failed and successful EOF parity across all host adapters. + +### 4. CI and maintainer documentation + +**Files:** `.github/workflows/test.yml`, `AGENTS.md`, `scripts/README.md`, `docs/guide/testing.md`, configuration/integration guides, canonical spec/plan. + +- [x] Add isolated Viceroy 0.19 Fastly EOF controls to CI, preserving pinned 0.17 ordinary tests. Cache the isolated tool and retain failure artifacts. +- [x] Document Rust/runtime prerequisites, exact cleanup/Axum/Fastly commands and compiler setting preservation. Validate workflow syntax and the actual commands locally. +- [x] Explain per-holder memory limits, tail latency, Fastly consumer transport bounds and the configured timeout policy. Add a minimal Rust callback example and environment override documentation. +- [x] Keep the current spec and plan; remove superseded archived designs, hardening logs and execution histories. Preserve executed evidence in the PR description. +- [x] State the deferred processed-template cache requirement explicitly in the draft PR, and document new required public fields. + +### 5. Verification and publication + +- [x] Obtain independent source/spec/plan review and resolve concrete findings. +- [x] Run `cargo fmt --all -- --check` and the integration crate format check, all eight target-matched Clippy aliases, all adapter test aliases and Fastly reuse, native core/digest gates, CLI/codegen, parity and exact integration CI selections. +- [x] Run existing ESI/inline/purge controls, JS build/tests/format, docs lint/build/format and repository Markdown formatting. +- [x] Record fresh four-runtime success, disabled, hung and auction results, including Fastly early-body-byte proof. + +### 6. Runtime evidence and regression assertions + +- [x] Correlate EOF and lookup events within each request; prove an earlier request's EOF cannot satisfy the check. +- [x] Require measured lookup duration for Fastly transport fallback without assuming a cooperative timeout log. +- [x] Assert disabled/failed parity has no finalized marker and failed output equals the disabled baseline; preserve the fixture's ordinary cache policy. +- [x] Verify bounded marker history across multiple small chunks and EOF marker protection. +- [x] Check the isolated Viceroy archive against its pinned SHA-256 before extraction; invalidate the earlier unchecked tool cache. +- [x] Preserve the default-disabled permission example, format the Rust example and keep the spec independent of superseded drafts. + +**Publication:** After validation, commit and push to the existing draft PR. Confirm its remote head and report hosted CI separately from local results. + +### 7. Native Rust runtime harness + +**Files:** `crates/trusted-server-integration-tests/tests/html_eof_runtime.rs`, +`tests/html_eof_runtime/{config,origin,process}.rs`, integration-test manifest, +shell entry point, workflow, verification guides and this plan. + +- [x] Add a feature-gated Rust test target so the existing integration CI selection cannot accidentally launch external runtime controls. +- [x] Port configuration, compiler environment, event ordering, timing and real process cleanup regressions; observe failures before completing helpers. +- [x] Use typed TOML edits and validated loopback configuration; an Axum origin records monotonic events in process and streams delayed EOF. +- [x] Own spawned process groups with bounded TERM/KILL/wait cleanup and cancellation handling, including active build commands and runtime descendants. +- [x] Preserve all four adapter controls and Fastly early-body proof through the existing shell command, selecting the host target explicitly. +- [x] Remove the three added Python files and their CI/tooling prerequisites; use Rust helper tests and lint gates instead. +- [x] Review this replacement independently, run the full gate matrix and all four actual runtime controls, then update the existing draft PR. + +## Acceptance mapping + +| Requirement | Coverage | +| ------------------------------------------------- | ------------------------------------------------------------------- | +| Ordered generic lifecycle and explicit permission | Implemented lifecycle plus EOF boundary tests | +| Per-holder limits, ownership and cancellation | Suffix growth, fallback and deadline tests | +| No-hook delivery/cache behavior | Existing publisher and template-cache controls | +| Honest opt-in latency and cache scope | Canonical spec, configuration/integration guides and PR description | +| Reliable runtime evidence and repeatability | Harness regressions, actual runtimes and isolated Fastly CI job | diff --git a/docs/superpowers/specs/2026-10-12-1241-async-html-finalization-design.md b/docs/superpowers/specs/2026-10-12-1241-async-html-finalization-design.md new file mode 100644 index 000000000..7fcd4f1ef --- /dev/null +++ b/docs/superpowers/specs/2026-10-12-1241-async-html-finalization-design.md @@ -0,0 +1,240 @@ +# Ordered asynchronous HTML processor completion + +**Issue:** #1241. **Status:** draft implementation; review hardening and local verification complete. + +The [implementation plan](../plans/2026-10-12-1241-async-html-finalization.md) maps this contract to implementation and verification. Fresh executed evidence is recorded in the draft PR. + +## 1. Requirement and scope + +HTML integrations receive synchronous `process_chunk(bytes, is_last)` calls but have +no supported opportunity to await HTTP or KV after document EOF. Provide optional +asynchronous processor completion. Keep chunk processing synchronous. A processor's +completed output must reach downstream processors before their own EOF. Article +signing is an example, not an API requirement or a framework responsibility. + +Trusted Server owns processor composition, publisher driving, auction assembly, +cache admission and request services. No EdgeZero repin is needed for EOF completion. +Changing buffered adapters into streaming adapters is separate work. + +This revision does not implement signing, verification endpoints, queues, key +management, background jobs, publisher CMS changes, async parser callbacks, +per-chunk I/O, a second parser, or insertion into already emitted HTML positions. + +## 2. One processor lifecycle + +`StreamProcessor` gains an object-safe optional `async_finish` returning a +`LocalBoxFuture`. The default returns `None` without allocating a future. Its input +borrows driver-owned synchronous EOF output, a separate bounded replacement writer, +actual response status, request `RuntimeServices`, and a shared absolute deadline. +It may borrow processor state and need not be `Send`. A capability method lets +synchronous drivers reject unsupported async processing before effects. + +`StreamFinishInput.output` is un-emitted output eligible for replacement. It is a +streaming processor's EOF tail or an explicitly buffered processor's retained +output; it is not universally the complete document. Outcomes are `Unchanged` and +`Replace`. Empty replacement is intentional. Return the future promptly and perform +I/O inside it. Bytes already delivered cannot be replaced or rolled back. + +Use the existing `IntegrationHtmlStreamProcessorFactory` and one registration +sequence. Factory metadata declares async EOF capability and optional retention of +its synchronous output, both false by default. Reject retention without async +capability. Metadata and the created session's capability must agree; undeclared +async work must never silently disappear through a synchronous driver. + +Use one production processor lifecycle without a separate finalizer factory, +session trait or registration list. Tests can use a test-only adapter to this +lifecycle. + +Factory metadata is available before template lookup, without creating a session, +reading HTML or polling a future. Every session uses the existing request-local +`IntegrationDocumentState`; state must not survive requests or reusable sandboxes. + +## 3. Operator acceptance of latency + +Add `publisher.allow_async_html_finish`, default `false`. An enabled integration +requiring async EOF without this permission produces an actionable configuration +error identifying the integration and setting. Do not silently omit required work. +A true setting with no async factories leaves the old processing path intact. + +Check permission at registry construction and defend direct publisher entry points +using manually constructed registries before consuming a body or committing +processed output. Factory retention metadata specifies whether core withholds output; +permission accepts the possible delivery cost of async EOF, including on buffered +adapters. It is not a guarantee of background execution after request completion. + +## 4. Ordered EOF driver + +For no-hook pages, preserve existing composition, chunk boundaries, streaming and +auction seam timing. Create no future, collector or timer. + +For opted-in HTML: + +1. Decode and validate origin bytes under existing limits. Run parser and processor + chunks synchronously in their existing order. +2. Complete decoder/parser EOF. Feed their remaining output downstream with + `is_last = false` and release ready bytes before optional I/O. +3. Finish the next integration processor exactly once with `is_last = true`. + Keep its synchronous EOF output outside its optional borrowing future. +4. Await optional finish under the remaining shared deadline. Commit an accepted + bounded candidate or retain the synchronous fallback. +5. Feed the selected output through later processors as ordinary input. Release + ready bytes before finishing the next stage. Repeat steps 3–5. +6. After all processors complete, assemble the auction and finish the encoder. + +Use a stepped concrete HTML chain in the async publisher driver. Do not await one +composite future that holds all EOF output. Race only a leaf callback, never the +whole chain: timeout must still complete downstream synchronous EOF. + +The generic `StreamingPipeline` remains synchronous. Its entry point, synchronous +publisher helpers and direct composite EOF reject async-capable use before reader, +parser or sink effects. Both materialized and streamed bodies reach the ordered driver. +Raw partial-response copying remains a documented exception because it executes no +processor or callback; pending auctions must still use an async entry point. + +## 5. Optional bounded retention + +For factories requesting retention, core owns a collector for that stage's synchronous +output. Use `publisher.max_buffered_body_bytes` (existing 16 MiB default), checked +arithmetic and incremental allocation. At EOF, move the retained output into the +fallback and lend a slice to the callback. Candidate storage has the same cap. + +These are per-holder limits, not an aggregate request-memory budget. Retained stage output, +fallback/candidate, auction suffix, codec state and buffered adapter responses can coexist. +Total memory may exceed this setting; integrations also own allocations core cannot bound. +Candidates run one at a time and replay moves/slices selected output. Do not infer a fixed +peak-memory multiplier or a total heap bound from the body-byte limit. + +Overflow permanently disables that stage's optional EOF I/O. Replay retained prefix, +crossing chunk and later output exactly once through downstream stages, then finish +its synchronous EOF normally. It does not authorize an oversized outer buffered +response. Core bounds its holder and candidate, not arbitrary private allocations +inside integration implementations. + +After overflow, the collector is consumed into ordinary ready output. The existing +`Vec` chunk interface may coalesce the prefix and crossing chunk above this +optional retention cap; ordinary output and outer buffered-response limits still +apply. The cap bounds withheld output, not every transient streaming chunk. + +Without retention, emit ordinary stage output immediately and keep only its EOF +fallback while awaiting. If fallback exceeds the optional-work cap, deliver it under +the ordinary output policy and skip that stage's optional I/O. Do not truncate it. + +## 6. Deadlines, failures and cancellation + +Retain `publisher.html_finalization_timeout_ms`: default 5000, valid 1–60000. +The initial default follows the 5-second budget in the issue example; 60 seconds is +a configuration ceiling. Neither value is a measured optimum or a response latency +guarantee. Operators explicitly accept and tune this budget when enabling EOF work. +One absolute budget begins after decoder/parser EOF and covers every optional finish. +Share one timer across stages, initializing its delay with the remaining budget +immediately before the first callback is polled. The deadline wins readiness ties. Check before polling +callbacks and before accepting output. + +Callback errors, poisoned candidates, invalid seam-marker multiplicity and late results +discard only that candidate. For each stage, retain at most marker-length minus one +bytes of its previously emitted output and validate token counts across that boundary +plus the EOF fallback/candidate. This catches split-token creation/removal without +retaining the page. Buffered and overflow-bypassed stages use the same boundary rule. +Preserve the current fallback allocation and earlier +accepted output. On expiry, drop the leaf future, skip later optional I/O, and still +finish all later synchronous stages. Access borrowed buffers only after dropping +that future. Callback panics propagate. + +Origin, parser, synchronous processor, codec and sink failures remain fatal. Client +cancellation drops local work; preserve existing auction diagnostics. No detached +futures, retries or background execution are implied. + +Axum, Workers and Spin supply cooperative timers. Fastly supplies none: consumers +must bound blocking host calls independently. Timers cannot preempt blocking calls +or CPU work that never yields; do not promise universal hard deadline release. + +## 7. Auction and encoding + +Keep auction dispatch and its transport deadline unchanged. On enabled EOF paths, +complete processor finishes before auction collection/assembly. Release available +prefix bytes before waiting. Preserve exact-token seam handling, missing-body-end +fallback, winning bids, encoder ordering and abandonment telemetry. + +A structural body-close may precede substantial trailing content. Bound the held +auction suffix using `publisher.max_buffered_body_bytes`; count every retained byte +after the marker (and an incomplete marker prefix before it). Check before allocation. +Split and release a large chunk's ready prefix before retaining its bounded suffix; +do not first copy the entire chunk into the holder. Exact-cap retention succeeds; +cap crossing is a fatal body error with abandonment, not an unbounded +allocation. No-hook auction seam timing remains unchanged. + +Marker multiplicity protects cardinality, not HTML structure. Async output passes +through later stream processors but does not pass through the HTML parser again. +Consumers preserve markers, CSP and application semantics. EOF append does not +promise insertion into an earlier head or before an already emitted body end. + +## 8. Cache, privacy and response eligibility + +Use active async factory metadata before the first processed-template lookup to skip +shared-template reads, reservations and stores. Raw-origin readthrough is independent. +This intentionally defers the issue's shared-template cold-miss storage behavior until +maintainers accept an output identity, freshness and hit-execution policy. Keep the +PR in draft and describe it as partial coverage of #1241 until that scope is accepted. +Processed opted-in HTML uses `Cache-Control: no-store, private` even when work fails, +returns unchanged, or is bypassed after overflow. Cacheable async output needs a +separate identity, freshness, invalidation and hit-execution contract. + +Preserve `AuctionDispatch.integration_registry` and same-registry dispatch/finish. +Keep early synthesized document-GET stripping of conditional/range headers; preserve +non-document fetch semantics. + +Opted-in HTML with status 206 or any `Content-Range` bypasses all rewriting and EOF +I/O. Preserve encoded bytes, framing, validators and ordinary origin policy. +Independent cookie/diagnostics privacy remains. Async helpers abandon unneeded +auctions. HEAD, bodiless, non-HTML and unsupported encodings run no callbacks. +Complete error HTML remains eligible and exposes its actual status. + +## 9. TTFB contract + +| Path | Contract | +| ----------------------------------------- | -------------------------------------------------------- | +| No async factories | Existing delivery and auction timing; no new EOF wait | +| Permission enabled, no async factories | Same as default | +| Async stage without retention on Fastly | Ready bytes released before EOF I/O; completion can wait | +| Explicit retention | Initial delivery waits until EOF or overflow release | +| Async stage on buffered Axum/Workers/Spin | EOF wait can add to TTFB before response return | + +Do not claim zero CPU overhead or enabled-hook TTFB neutrality on buffered adapters. +Their streaming support, or reliable post-response work, is separate runtime work. + +## 10. Acceptance criteria + +- Default permission is false; disabled integrations need no permission; enabled + async integrations fail configuration without explicit permission. +- Absent callbacks allocate no future and preserve the no-hook paths. +- Yielding non-`Send` callbacks work with materialized and streamed input. +- Upstream async output reaches downstream normal chunks before downstream EOF. +- Every parser/processor receives EOF once, including after timeout. +- Ready decoder/parser/stage prefixes are released while later EOF I/O is pending. +- Buffered overflow replays prefix/crossing/suffix once and starts no optional I/O. +- Cancellation, ties, callback errors, poisoned candidates and late results preserve + driver-owned fallback without a rollback copy. +- Synchronous helpers reject async use before effects. +- Codecs, ranges, actual statuses, privacy and early cache admission retain previous + regression protections; corrupt input starts no optional work. +- Auction assembly follows processor finishes; held suffix overflow abandons safely. +- All target-matched CI gates and actual runtime fixture controls pass on final source. + +The fictional fixture demonstrates generic EOF HTTP I/O, not a production signing +integration. Verification records identify checked source, real runtimes, hosted CI, +expected skips and tooling limits separately. Historical evidence does not prove this +revised implementation. + +## 11. Native Rust verification tooling + +Keep EOF runtime verification in the existing Rust integration-test crate. A +feature-gated native test target owns its loopback origin, request-scoped monotonic +events and external runtime process groups. An ignored test runs the selected +adapter; normal helper tests need no adapter tools. The shell entry point selects +the adapter/profile and invokes Cargo with the explicit native target. + +Preserve disabled/success/hung/auction controls on all four adapters and actual +early-body delivery on Fastly. Validate typed, isolated configuration before use, +retain diagnostics, preserve compiler/profile settings, and clean process groups +on normal return, failure, SIGTERM and SIGINT. Use existing workspace dependencies +only for tests. No added verification path requires Python. diff --git a/scripts/README.md b/scripts/README.md index ad4ecf926..89d37e5df 100644 --- a/scripts/README.md +++ b/scripts/README.md @@ -16,9 +16,38 @@ their documented orchestration handles a probe deliberately. | `smoke-cloudflare.sh` | Pinned Wrangler, `cargo`, `curl`, `jq`, and Python | Uses isolated Wrangler manifests, KV state, ports, and logs; stops owned processes and removes its workspace. | | `smoke-spin.sh` | Spin CLI, `cargo`, `curl`, and Python | Uses an isolated Spin manifest and SQLite KV store; stops owned processes and removes its workspace. | | `smoke-common.sh` | Sourced by the four smoke scripts | Defines bounded port, process, config, secret, and response assertions. Do not invoke it as a standalone smoke. | +| `async-html-finalization-local-test.sh` | Rust, pinned Node and selected runtime; Fastly Viceroy 0.19+ | Builds feature-gated fixtures, preserves compiler settings, retains diagnostics, and stops owned process groups on failure or SIGTERM. | | `template-cache-local-test.sh` | Viceroy, Node, OpenSSL, `curl`, `lsof`; optional ports and mode | Builds local artifacts, generates certificates and fixtures in a temporary directory, stops owned servers, and removes the directory. | | `test-cli.sh` | Rustup and the host toolchain; optional host triple | May install the selected Rust target, then runs native CLI and browser-audit tests. Cargo artifacts persist. | +## HTML EOF runtime checks + +Requires Rust, pinned Node and the selected runtime CLI. Run from the +repository root: + +```bash +cargo test --manifest-path crates/trusted-server-integration-tests/Cargo.toml --features html-eof-runtime --test html_eof_runtime +./scripts/async-html-finalization-local-test.sh axum --auction +./scripts/async-html-finalization-local-test.sh cloudflare --release --auction +./scripts/async-html-finalization-local-test.sh spin --release --auction +``` + +**Fastly requires isolated Viceroy 0.19 or newer** for the hung-backend fixture; +ordinary tests retain the repository's 0.17 pin: + +```bash +cargo install viceroy --version 0.19.0 --locked --root /tmp/example-eof-viceroy +PATH="/tmp/example-eof-viceroy/bin:$PATH" ./scripts/async-html-finalization-local-test.sh fastly --release --auction +``` + +Workers requires matching `worker-build`/`wasm-bindgen` versions and Wrangler. +The harness builds fresh feature-gated artifacts, checks generated timeout, +permission and loopback configuration, preserves explicit compiler/profile settings +and `CARGO_TARGET_DIR`, selects build targets itself, and stops owned process groups on failure or SIGTERM. +Artifacts and diagnostics persist in the printed temporary directory for inspection. +CI runs native Rust helper tests and lint, Axum and isolated Fastly EOF controls. See the +[testing guide](../docs/guide/testing.md#html-eof-runtime-checks) for coverage details. + The four adapter smoke contracts are documented in the [deployment guides](../docs/guide/integrations-overview.md#adapter-support). Repository-wide verification policy lives in [TESTING.md](../TESTING.md). diff --git a/scripts/async-html-finalization-local-test.sh b/scripts/async-html-finalization-local-test.sh new file mode 100755 index 000000000..f941ab017 --- /dev/null +++ b/scripts/async-html-finalization-local-test.sh @@ -0,0 +1,21 @@ +#!/usr/bin/env bash +# Build a fresh test-utils artifact and exercise terminal I/O on a local runtime. +set -euo pipefail +REPO_ROOT=$(cd "$(dirname "$0")/.." && pwd) +case "${1:-}" in + axum|fastly|cloudflare|spin) export HTML_EOF_ADAPTER="$1"; shift ;; + *) echo "Usage: $0 {axum|fastly|cloudflare|spin} [--release] [--auction]" >&2; exit 2 ;; +esac +export HTML_EOF_RELEASE=false HTML_EOF_AUCTION=false +for argument in "$@"; do + case "$argument" in + --release) export HTML_EOF_RELEASE=true ;; + --auction) export HTML_EOF_AUCTION=true ;; + *) echo "Unknown argument: $argument" >&2; exit 2 ;; + esac +done +cd "$REPO_ROOT" +EOF_HOST_TARGET=$(rustc -vV | sed -n 's/host: //p') +exec cargo test --manifest-path crates/trusted-server-integration-tests/Cargo.toml \ + --target "$EOF_HOST_TARGET" --features html-eof-runtime --test html_eof_runtime \ + actual_runtime -- --ignored --exact --nocapture --test-threads=1 diff --git a/trusted-server.example.toml b/trusted-server.example.toml index 066ad659e..18cc26310 100644 --- a/trusted-server.example.toml +++ b/trusted-server.example.toml @@ -67,6 +67,11 @@ proxy_secret = "publisher_proxy_secret" # rewriting/injection) instead of streamed. Default 16 MiB; larger responses # return 502. Raise for deployments serving very large publisher pages. # max_buffered_body_bytes = 16777216 # 16 MiB +# Explicit permission for integration EOF I/O that can delay HTML delivery. +# Defaults to false; enabled async processors require true. +# allow_async_html_finish = false +# Optional asynchronous HTML processors share this budget after decoder/parser EOF. +# html_finalization_timeout_ms = 5000 # Valid range: 1..60000 ms # -----------------------------------------------------------------------------