From b750c879a59f8306b97f65c939a094747b2684c2 Mon Sep 17 00:00:00 2001 From: prk-Jr Date: Mon, 12 Oct 2026 15:04:32 +0530 Subject: [PATCH 01/11] Specify and plan asynchronous HTML finalization --- ...2026-10-12-1241-async-html-finalization.md | 670 ++++++++++++++++++ ...-12-1241-async-html-finalization-design.md | 667 +++++++++++++++++ 2 files changed, 1337 insertions(+) create mode 100644 docs/superpowers/plans/2026-10-12-1241-async-html-finalization.md create mode 100644 docs/superpowers/specs/2026-10-12-1241-async-html-finalization-design.md diff --git a/docs/superpowers/plans/2026-10-12-1241-async-html-finalization.md b/docs/superpowers/plans/2026-10-12-1241-async-html-finalization.md new file mode 100644 index 000000000..4617c5cd2 --- /dev/null +++ b/docs/superpowers/plans/2026-10-12-1241-async-html-finalization.md @@ -0,0 +1,670 @@ +# Optional asynchronous HTML finalization implementation plan + +> **For agentic workers:** REQUIRED SUB-SKILL: Use superpowers:subagent-driven-development (recommended) or superpowers:executing-plans to implement this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking. + +**Goal:** Allow explicitly registered HTML integrations to await I/O after synchronous document processing, with bounded core-owned buffers and exact fallback on optional failure. + +**Architecture:** Add an HTML-only terminal phase after existing synchronous processors and before auction-seam handling and encoding. One coordinator owns retained HTML, bounded replacement storage, and the chain deadline; the no-finalizer route remains unchanged. Enabled finalizers bypass shared-template caching before lookup and use synthesized HTML privacy rules. + +**Tech Stack:** Rust 2024, existing `futures` local futures, `error-stack`, `derive_more`, `web_time`, `lol_html`, existing compression codecs, and adapter-local Workers/Spin/Tokio timers. + +--- + +**Status:** Proposed implementation plan; no implementation started + +**Date:** 2026-10-12 + +**Branch:** `spec/1241-async-html-finalization` + +**Baseline:** `2a0cb0444dc335bd09105abe5952329bbbdfb621` + +**Design:** [Optional asynchronous HTML document finalization](../specs/2026-10-12-1241-async-html-finalization-design.md) + +## 1. Execution boundaries + +This plan implements the selected terminal-phase design, not the original proposal to add +`async_finish` to `StreamProcessor`. Read the spec before implementing. Use +@superpowers:test-driven-development for runtime changes and +@superpowers:verification-before-completion before recording results or committing. + +The tasks below describe future implementation work. Their commands and expected outcomes +are not claims that the feature exists or has already passed tests. Keep intermediate +commits in a draft branch; do not release an API that registrations can enable before all +publisher entry points support it. + +Keep the EdgeZero revision, generic streaming traits, JS runtime, and production platform +manifests unchanged. KV wiring under #1139, clock fixes under #1240, and a real signing +integration are separate work. Do not mask a live-runtime failure by replacing actual +SDK services with stubs or by reporting a native test as a runtime smoke. + +## 2. File responsibilities + +| File | Responsibility | +| --------------------------------------------------------------------------------------------------------------- | -------------------------------------------------------------------------------------------------------------- | +| Create `crates/trusted-server-core/src/integrations/html_finalization.rs` | Public factory/session/input/outcome/error contracts, bounded replacement writer, and finish deadline handle | +| Modify `crates/trusted-server-core/src/integrations/mod.rs` | Export the new contracts; later register a feature-gated fictional fixture | +| Modify `crates/trusted-server-core/src/integrations/registry.rs` | Empty-by-default finalizer metadata, builder method, ordered factory access, and test construction seam | +| Modify `crates/trusted-server-core/src/html_processor.rs` | Share one document-state instance through parser, synchronous processors, and finalizer construction | +| Create `crates/trusted-server-core/src/platform/timer.rs` | Optional platform-neutral timer contract | +| Modify `crates/trusted-server-core/src/platform/mod.rs` and `types.rs` | Timer exports, default-absent runtime service, builder/accessor support | +| Modify `crates/trusted-server-core/src/settings.rs` | Global terminal-phase timeout, matching defaults, and validation | +| Create `crates/trusted-server-core/src/publisher/html_finalization.rs` | Private retained-document collector, ordered async driver, candidate validation, and bounded replay | +| Modify `crates/trusted-server-core/src/publisher.rs` | Route opted-in HTML through the coordinator on every body/sink path; early cache/privacy/conditional decisions | +| Modify adapter `src/platform.rs` files and Fastly `src/app.rs` tests | Concrete timers and explicit Fastly timer absence | +| Create `crates/trusted-server-core/src/test_support/async_html_finalization.rs`; modify `src/test_support.rs` | Fictional integration compiled only under existing `test-utils` | +| Modify `crates/trusted-server-integration-tests/tests/parity.rs` | Host-adapter finalizer behavior and privacy parity | +| Create `scripts/fixtures/async-html-finalization-origin.py` and `scripts/async-html-finalization-local-test.sh` | Delayed/hung local HTTP lookup and actual-runtime smoke driver | +| Modify `trusted-server.example.toml` and canonical guide files | Explain the timeout, registration API, buffer/cache effects, and adapter limitations | + +Do not split or reorganize unrelated sections of `publisher.rs`. The focused child module +contains the new state machine; existing decoding, auction collection, telemetry guards, +and framing helpers remain authoritative. + +## 3. Dependency order and verification rhythm + +Tasks 1–4 establish dormant contracts and services. Task 5 implements asynchronous execution; +Task 6 implements bounded collection/replay. Task 7 establishes request-time policy before +Task 8 enables production routing. Tasks 9–11 verify codecs, adapter paths, and real runtimes. +Tasks 12–13 finish documentation and the complete gate run. + +Use native core filters for fast feedback, then the target-matched gates associated with +the touched adapters. Before each commit, run formatting, the task's tests, and its applicable +lint gate. Before PR handoff, run the entire gate list in Task 13. An unavailable runtime +is a reported verification gap, not permission to mark the check passing. + +For core-only native checkpoints, run +`cargo clippy -p trusted-server-core --all-targets --all-features -- -D warnings`. +An adapter lint alias can compile core as a dependency but does not directly lint it as +a primary package. Retain target-matched Fastly/core and adapter gates where listed. + +### Task 1: Define the terminal callback and bounded replacement writer + +**Spec coverage:** Sections 6.2, 8, 9.3, 10, and 16.3. + +**Files:** Create `crates/trusted-server-core/src/integrations/html_finalization.rs`; +modify `crates/trusted-server-core/src/integrations/mod.rs`. + +- [ ] Write failing unit tests in the new module for exact-cap append, cap overflow, + overflow-safe length checking, bounded capacity growth, poisoned-writer rejection, + and explicit `Unchanged` versus empty `Replace` outcomes. +- [ ] Define/export the spec's object-safe factory/session/input/outcome/error types. + The factory exposes `integration_id()` and `create(IntegrationHtmlStreamContext)`; + it is `Send + Sync`. A session is single-use and returns a local borrowing future. + Define `HtmlFinalizationDeadline` in this same task so the input contract compiles: + use `web_time::Instant`, a timeout `Duration`, and cooperative-timer availability; + expose remaining time and keep expiry-at-explicit-time helpers private for tests. + Cover before/exactly-at/after expiry and saturating zero remaining time. + +The callback signature is: + +```rust +pub trait IntegrationHtmlFinalizer { + fn finalize<'a>( + &'a mut self, + input: HtmlFinalizationInput<'a>, + ) -> LocalBoxFuture<'a, Result>>; +} +``` + +The input fields and result variants must match spec section 6.2 exactly. Keep the retained +HTML immutable. The writer exposes fallible `append(&[u8])`; only core can consume it to +commit a candidate. Use a concrete `derive_more::Display` error and `core::error::Error`. + +- [ ] Implement capped append: compute the next length with `checked_add`; poison and + return a controlled limit error before mutation on overflow or excess. For growth, + use an amortized capacity target bounded by the cap and reserve exactly that target. + Do not allocate the full cap up front or accept an arbitrary replacement `Vec`. +- [ ] Run `cargo test -p trusted-server-core --lib integrations::html_finalization`. + Confirm the new tests actually run, reproduce the limit failure before the fix, then pass. + Also run `cargo fmt --all -- --check`, the focused native core clippy command above, + and `cargo clippy-axum`. +- [ ] Commit the contract/writer change as `Add bounded HTML finalizer contracts`. + +The essential overflow regression has this behavior: + +```rust +let mut writer = HtmlReplacementWriter::new(4); +writer.append(b"page").expect("should accept the exact cap"); +writer.append(b"!").expect_err("should reject output beyond the cap"); +assert!(writer.is_poisoned(), "should reject later replacement commitment"); +``` + +Keep writer construction/poison inspection crate-private where only core needs them. Public +items require summaries, intra-doc links, fallible-function error documentation, and a +fictional usage example. + +### Task 2: Add the shared finish budget and optional timer contract + +**Spec coverage:** Sections 9 and 16.4. + +**Files:** Modify `integrations/html_finalization.rs`, `settings.rs`, +`platform/types.rs`, and `platform/mod.rs`; create +`crates/trusted-server-core/src/platform/timer.rs`. + +- [ ] Write failing tests for settings omission/default parity, `1`/`5000`/`60000`, + rejected `0`/`60001`, default-absent timer construction, and services clone/replacement + methods preserving an installed timer. +- [ ] Add the publisher setting with this exact default and validation: + +```rust +#[serde(default = "default_html_finalization_timeout_ms")] +#[validate(range(min = 1, max = 60000))] +pub html_finalization_timeout_ms: u32, + +fn default_html_finalization_timeout_ms() -> u32 { + 5000 +} +``` + +Update the hand-written `Publisher::default()` and only compiler-required publisher +settings literals. Do not modify unrelated OpenRTB types also named `Publisher`. + +- [ ] Add a `Send + Sync` timer trait with `delay(Duration) -> LocalBoxFuture<'static, ()>`. + Add optional `Arc` storage, a builder method, and an accessor to + `RuntimeServices`; missing timer configuration must never cause builder failure. +- [ ] Connect the timer-presence flag to Task 1's deadline type. Keep any deterministic + clock injection private to tests; do not introduce a public platform clock service. + Add settings-to-deadline tests proving the whole chain gets the configured duration, + rather than a duration independently supplied by each finalizer. +- [ ] Run `cargo test -p trusted-server-core --lib html_finalization`, + `cargo test -p trusted-server-core --lib settings::tests`, and + `cargo test -p trusted-server-core --lib platform::tests`. + Expected: new default/validation/deadline tests pass and existing service tests remain green. + Run `cargo fmt --all -- --check`, the focused native core clippy command above, + and `cargo clippy-axum`. +- [ ] Commit as `Add HTML finalization budgets and optional timers`. + +### Task 3: Register finalizers and share the parser's document state + +**Spec coverage:** Sections 6.1, 7.1, 7.2, and 14.1. + +**Files:** Modify `integrations/registry.rs`, `integrations/mod.rs`, and +`html_processor.rs`. + +- [ ] Write failing registry tests for empty defaults, disabled integration exclusion, + multiple finalizers preserving registration order, and presence checks that never + instantiate sessions. Extend the existing single-EOF processor test. +- [ ] Add `.with_html_finalizer(...)`, `has_html_finalizers()`, and ordered factory access. + Update existing hand-written registration/registry literals with empty collections. + Follow the current registry's integration enablement/build order; do not sort by ID. +- [ ] Add a `#[cfg(any(test, feature = "test-utils"))]` registry constructor from finalizer + factories for isolated core tests. Keep it out of ordinary production API builds. +- [ ] Introduce a crate-private construction helper taking an explicit document-state + instance, and keep `create_html_processor(config)` as a synchronous wrapper that + constructs state and delegates. Publisher assembly creates one state and passes clones + of its shared handle to parser hooks, streaming factories, and finalizer factories. +- [ ] Test parser-written state reaching the finalizer, two independent requests, and + retained-registry/reusable-sandbox isolation. Assert synchronous EOF is delivered once + before the first finalizer. No replacement is ever reprocessed by that chain. +- [ ] Run the registry and HTML filters: + `cargo test -p trusted-server-core --lib integrations::registry::tests` and + `cargo test -p trusted-server-core --lib html_processor::tests`. + Run `cargo test-fastly-reuse`, `cargo fmt --all -- --check`, and `cargo clippy-fastly`. +- [ ] Commit as `Register HTML finalizers with shared document state`. + +### Task 4: Wire concrete adapter timers + +**Spec coverage:** Sections 9.2, 9.3, and 14.6. + +**Files:** Modify `crates/trusted-server-adapter-axum/src/platform.rs`, +`crates/trusted-server-adapter-cloudflare/src/platform.rs`, +`crates/trusted-server-adapter-spin/src/platform.rs`, and +`crates/trusted-server-adapter-fastly/src/app.rs` tests. + +- [ ] Add service-construction tests: Axum installs a timer; production Workers and Spin + install their SDK timer under the existing runtime cfg guards; native Workers/Spin + construction remains safe without those SDKs; Fastly installs no timer. +- [ ] Implement stateless timer service objects locally. Return a boxed local future from + `worker::Delay::from(duration)`, `spin_sdk::time::sleep(duration)`, or + `tokio::time::sleep(duration)` as applicable. Delay construction is inside the returned + async future where required to avoid needing a runtime merely to build services. +- [ ] Add the timer to each supporting adapter's runtime-services builder. Preserve config, + secret, KV, HTTP, telemetry, and template service wiring. No manifest or EdgeZero changes. +- [ ] Run `cargo test-axum`, `cargo test-cloudflare`, `cargo test-spin`, and + `cargo test-fastly-reuse`. Native tests prove construction contracts, not SDK behavior. +- [ ] Run all touched adapter lint targets: `cargo clippy-axum`, + `cargo clippy-cloudflare`, `cargo clippy-cloudflare-wasm`, + `cargo clippy-spin-native`, `cargo clippy-spin-wasm`, and `cargo clippy-fastly`. + Expected: runtime cfg separation compiles on every relevant target. +- [ ] Commit as `Wire adapter timers for HTML finalization`. + +### Task 5: Implement ordered, cancellation-safe terminal execution + +**Spec coverage:** Sections 7.2–7.4, 9.3, 10, 13, and 14.2. + +**Files:** Create `crates/trusted-server-core/src/publisher/html_finalization.rs`; +declare the child module in `publisher.rs`. + +- [ ] Write failing coordinator tests using sessions that yield at least once, write partial + candidates, return unchanged/error/replacement, and record their input and drop events. + Use a local `Rc` inside a session/future to prove no accidental `Send` requirement. +- [ ] Give the coordinator ownership of current HTML outside every awaited future. Each + stage borrows current bytes and a separate fresh writer. A valid replacement moves + writer storage into current HTML; an unchanged/error result discards the writer. +- [ ] Construct one deadline and retain one pinned timer across the entire loop. Poll expiry + first, check absolute expiry before each stage and after completion, and make expiry win + ties. With no timer, await work and check elapsed budget afterward; no preemption claim. +- [ ] On timeout, explicitly drop the active future before accessing its borrowed writer or + moving current HTML; discard the candidate and stop the chain. Do not put current HTML + exclusively inside an async chain future that is dropped by a race. + +The driver-level ownership invariant is: + +```text +driver owns current HTML and current replacement writer + -> callback borrows both + -> result/timeout + -> callback future is dropped, releasing both borrows + -> driver either commits replacement or emits current HTML +``` + +- [ ] Validate poisoned writers and required marker cardinality before commitment. Preserve + the marker as an opaque integration contract; do not add a parser or pretend cardinality + proves structural placement. Callback errors remain soft; upstream/codec/sink errors + remain fatal and retain existing `Report` mappings. +- [ ] Test A-to-B output propagation, B timeout preserving A's accepted bytes, ignored writer + errors, invalid markers, one chain budget, late Fastly results, exact tie handling, and + input-buffer identity on fallback. Capture pointer identity at stage input and compare + before sink encoding, since writing to a sink necessarily copies into that sink. +- [ ] Emit controlled outcome/elapsed/byte-count logs; never format callback attachments, + HTML, signatures, tokens, or request data. Test callback drop on cancellation and ensure + no finalizer starts afterward. Panics retain existing runtime behavior. +- [ ] Run `cargo test -p trusted-server-core --lib publisher::html_finalization`, + `cargo test-fastly html_finalization`, `cargo fmt --all -- --check`, and + `cargo clippy-fastly`. Expected: all deterministic coordinator cases pass on native + and Fastly test targets. +- [ ] Commit as `Coordinate ordered HTML finalization and fallback`. + +### Task 6: Collect bounded transformed HTML and replay optional bypasses + +**Spec coverage:** Sections 7.2, 8, 12, and 14.3. + +**Files:** Modify `publisher/html_finalization.rs` and minimal collector call sites in +`publisher.rs`. + +- [ ] Add collector tests for cap minus one/exact/plus one, many chunks, a large crossing + chunk, synchronous EOF output crossing the cap, and integer-overflow rejection. +- [ ] Model collection states explicitly: collecting, bypassing, and finished. Append + transformed bytes under the existing `publisher.max_buffered_body_bytes` cap; enforce + bounded capacity growth without cloning the retained document. +- [ ] On optional collection overflow, move out the retained prefix and forward it, then the + unretained crossing chunk, then later synchronous output to ordinary seam/encoding + delivery. Skip every finalizer and create no timer or outside lookup for that document. +- [ ] Reuse existing auction hold state during bypass; a private seam token can straddle + the retained/crossing boundary. Test exact order, no duplicated EOF, no raw-token leak, + and prefix-before-auction collection. +- [ ] Retain the outer buffered writer cap and all existing raw/decoded/parser limits. A + decoder cap error is not an optional-finalizer bypass. Test both failure categories. + In route tests, reach the optional cap through rewritten-output expansion while raw + and decoded input remain within their limits; an oversized origin is an old fatal case. +- [ ] Run `cargo test -p trusted-server-core --lib publisher::html_finalization` and + `cargo test-fastly html_finalization`; run `cargo fmt --all -- --check` and + `cargo clippy-fastly`. Expected: collector/replay cases preserve bytes and old fatal limits. +- [ ] Commit as `Bound retained HTML and preserve streaming bypass`. + +### Task 7: Gate cache lookup and preserve synthesized-document policy + +**Spec coverage:** Sections 11, 14.5, and 16.5. + +**Files:** Modify `publisher.rs` and its existing cache/privacy test modules. Modify +`cache_policy.rs` only if reusing its current public helpers is insufficient. + +- [ ] Write failing request tests showing an enabled finalizer bypasses a prepopulated warm + template before lookup and excludes reservation/storage, while disabling it restores + existing eligibility. Include ESI configuration, no matched slots, consent denial, + bots/prefetch, and ordinary HTML without an auction. +- [ ] Derive the early finalizer-presence decision from enabled registry metadata. Combine + it with shared-template admission before key creation/lookup. Keep origin readthrough + eligibility separate; do not accidentally force every raw origin cache hit to miss. +- [ ] Use the existing non-shared inline assembly fallback consistently: no authorized + shared-template key, no shared-template token, and matching head/body script choices. + Do not add a first-version `shared_safe` override or a new public cache header. +- [ ] Apply `enforce_synthesized_html_cache_privacy` before response headers are committed for + HTML entering the opted-in path, including unchanged/failure/collection bypass. Preserve + duplicate cookies/CSP and existing EC/privacy finalization. +- [ ] For document GETs, extend the existing full-body predicate so + `strip_conditional_and_range_headers` removes its same four headers. Preserve `If-Match` + and `If-Unmodified-Since`. Do not broaden this stripping to assets or other methods. +- [ ] Test raw origin readthrough eligibility with finalizer metadata; validator/header + stripping; unexpected origin 304 behavior with/without the current ad-stack guard; + bodyless, unsupported-encoding, and non-HTML route selection. This task proves policy + admission and header decisions only. Assertions requiring callback execution, callback + outcome, or callback invocation counts belong to Task 8 after routing is wired. +- [ ] Run `cargo test -p trusted-server-core --lib publisher::tests`, + `cargo test-fastly publisher::tests`, `cargo fmt --all -- --check`, and + `cargo clippy-fastly`. Expected: new policy tests and all existing publisher tests pass. +- [ ] Commit as `Gate shared caching for asynchronous HTML finalizers`. + +### Task 8: Route every enabled publisher body through the terminal coordinator + +**Spec coverage:** Sections 7, 12, 14.4, and 16.1–16.2. + +**Files:** Modify `publisher.rs`, `publisher/html_finalization.rs`, and +`html_processor.rs` only where construction data must be threaded. + +- [ ] Build failing route tests across `Once`/`Stream`, lazy/buffered output, and auction + present/absent. Assert every enabled body calls its finalizer once after decoder and + synchronous EOF, and no disabled/bodiless path calls it. + Explicitly include processable 404/500 HTML and preserved method/status behavior. +- [ ] Extend relevant publisher parameters with request-local finalizer factories/session + construction data and shared document state. Keep ownership compatible with the existing + lazy response stream and borrowed buffered driver. +- [ ] Wire enabled HTML in both `publisher_response_into_streaming_response` and + `stream_publisher_body_async`/`buffer_publisher_response_async` through one coordinator. + In particular, prevent opted-in `Once` bodies without an auction from falling through + the synchronous `stream_publisher_body` path, and route enabled reader-based auction + bodies through the same terminal sequence. +- [ ] Reuse the existing decoder semantics on each body representation: drain/validate tail, + finish the synchronous chain once, run finalizers, replay accepted bytes in bounded chunks + through the existing seam controller, emit ready prefix, collect auction, finish encoder. + Do not feed candidates through `lol_html` or Next.js a second time. +- [ ] Keep current no-finalizer branches and public synchronous helper behavior intact. Add + API documentation that finalizers require supported publisher entry points; synchronous + helpers provide synchronous transformations only. +- [ ] Preserve dispatched-auction guards through collection and every new await. Test + cancellation during finalizer I/O and each fatal error category for exactly-once terminal + telemetry. Auction dispatch/deadlines must not restart or expand. +- [ ] Complete Task 7's execution-dependent assertions: raw origin readthrough hits run + finalizers on every request; success, unchanged, callback error, timeout, and collection + bypass retain private/no-store headers; HEAD/204/205/304 and unsupported routes invoke + no callback. Include enabled pages with no auction or matched slot. +- [ ] Run `cargo test-fastly`, `cargo test-fastly-reuse`, `cargo test-axum`, + `cargo test-cloudflare`, and `cargo test-spin`; then touched adapter lint aliases. + Expected: all enabled route combinations work and existing early-streaming tests pass. +- [ ] Commit as `Route opted-in HTML through terminal finalization`. + +Tasks 1–8 are one coherent feature. No release/PR readiness claim is allowed until this task +passes on all applicable paths, even if earlier dormant API commits compile. + +### Task 9: Complete the codec and legacy-behavior regression matrix + +**Spec coverage:** Sections 14.3–14.5 and 16.2–16.3. + +**Files:** Modify core tests in `publisher.rs`, `publisher/html_finalization.rs`, and +`html_processor.rs`. Keep `streaming_processor.rs` implementation unchanged. + +- [ ] Parameterize the new route tests over identity, gzip, deflate, and Brotli; test + concatenated gzip members and output released only in decoder/processor tails. +- [ ] Add malformed/truncated origin, empty HTML, split UTF-8, absent body end, marker + loss/duplication, oversized candidate, and optional collection bypass cases. +- [ ] Assert decoder validation precedes I/O, synchronous EOF precedes callback, finalizer + output precedes auction token removal, and encoder trailers follow accepted output. +- [ ] Retain and run these existing regressions unchanged: + +```text +html_stream_processors_compose_in_order_and_receive_final_once +streaming_html_with_stream_processors_rewrites_body +document_state_placeholders_substitute_through_streaming_path +streaming_finalize_emits_compressed_html_before_origin_eof +streaming_finalize_auction_hold_emits_prefix_before_origin_eof +streaming_finalize_auction_hold_emits_small_compressed_page_before_collection +parser_confirmed_auction_seam_streams_nextjs_for_every_encoding +stream_publisher_body_async_auction_hold_decodes_multi_member_gzip_buffered +body_chunk_source_resumes_after_cancelled_poll +finalizers_emit_abandoned_auction_for_bodiless_dispatched_response +``` + +- [ ] Compare no-finalizer results against the baseline using equivalent deterministic + fixtures: status, critical/repeated headers, decoded body, emission before source EOF, + and collection/telemetry counts. Do not compare request-random cookies or IDs literally. + Preserve exact encoded output where fixtures and chunking are deterministic. +- [ ] Run the three focused native filters from spec section 17 and `cargo test-fastly`. + Run `cargo fmt --all -- --check` and `cargo clippy-fastly`. +- [ ] Commit as `Verify HTML finalization across body and codec paths`. + +### Task 10: Add a feature-gated fictional consumer and host parity tests + +**Spec coverage:** Sections 4, 14.1, 14.5, and 14.6. + +**Files:** Create `crates/trusted-server-core/src/test_support/async_html_finalization.rs`; +modify `src/test_support.rs`, `integrations/mod.rs`, and +`crates/trusted-server-integration-tests/tests/parity.rs`. + +- [ ] Add a fictional `html_finalization_fixture` builder only under the existing + `test-utils` feature. Keep its Rust module and builder entry absent from ordinary builds. + Use `.without_js()` so no nonexistent JS module enters the bundle. +- [ ] Configure only fixture `enabled`, `lookup_url`, and `transport_timeout_ms` values in + the existing integration settings map. The finish budget comes exclusively from + `[publisher].html_finalization_timeout_ms`; do not introduce a second fixture deadline. +- [ ] Implement a terminal finalizer, not a buffering `StreamProcessor`. It receives + `input.html`, confirms a fictional EOF sentinel, sends a digest/EOF observation using + `json!` through `RuntimeServices::http_client`, then appends prefix/replacement/suffix + to the bounded writer. It performs no signing and needs no KV handle. +- [ ] Ensure its backend through `PlatformBackendSpec`, deriving TLS policy from the URL + scheme. Existing integration convenience helpers force TLS; do not use them for the + harness's generated loopback HTTP endpoint. Derive deterministic backend names from the + same spec where Viceroy needs a local backend mapping. +- [ ] Cap response collection for the fixture lookup. Honor remaining finish budget when + bounding Fastly first-byte/between-byte transport calls. When a transport cannot offer + the required bound without a timer, return `Unchanged` instead of an unbounded call. +- [ ] Use the normal enabled registry builder in host parity tests and inject deterministic + services through existing adapter testing seams. Verify yielding completion, unchanged + error/timeout bytes, no-finalizer output, privacy, and no mandatory KV. Do not add public + adapter constructors only to inject a custom registry. +- [ ] Run `cargo test --manifest-path crates/trusted-server-integration-tests/Cargo.toml --test parity`, + plus `cargo test-axum`, `cargo test-cloudflare`, and `cargo test-spin`. + Run `cargo clippy-axum`, `cargo clippy-cloudflare`, and `cargo clippy-spin-native`. +- [ ] Build/check normal artifacts without `test-utils` and prove the fixture builder ID is + absent using feature-off registry tests. Commit as `Add fictional HTML finalization parity fixtures`. + +### Task 11: Verify finalization on the four actual local runtimes + +**Spec coverage:** Sections 9.2–9.3 and 14.6. + +**Files:** Create `scripts/fixtures/async-html-finalization-origin.py` and +`scripts/async-html-finalization-local-test.sh`; use existing `scripts/smoke-common.sh` +helpers and provisioning examples. Do not alter production manifests. + +- [ ] Write the threaded local fixture origin with `/article/success`, `/article/hang`, and + `/lookup`. Generate fictional HTML containing `ASYNC_FINALIZATION_PENDING` and + `ARTICLE_EOF_SENTINEL`; serve the EOF sentinel in a later chunk. A lookup records receipt + of a complete-document digest/EOF observation before replying after 150 ms or remaining + pending for 30 seconds. Record start/completion counts without request-sensitive data. +- [ ] Add a driver accepting `fastly|cloudflare|spin|axum`. Generate all configs, fake secrets, + endpoints, runtime manifests, response captures, and logs in a temporary workspace. + Reuse each adapter's smoke-script provisioning pattern and `smoke-common.sh` cleanup. + Do not read operator-owned `trusted-server.toml`, `.env`, or cloud credentials. +- [ ] Build a fresh fixture artifact with the feature explicitly enabled and isolated output + paths. Do not reuse an ordinary cached Worker bundle, and do not call the existing + Cloudflare `build.sh`, which sources operator env files and rebuilds without fixture flags. + +Build commands used by the driver, with its own `CARGO_TARGET_DIR`, are: + +```bash +cargo build -p trusted-server-adapter-fastly --target wasm32-wasip1 --release --features trusted-server-core/test-utils +cargo build -p trusted-server-adapter-axum --features trusted-server-core/test-utils +cargo build -p trusted-server-adapter-spin --target wasm32-wasip1 --release --features spin,trusted-server-core/test-utils +``` + +For Workers, resolve the `worker-build` version from `Cargo.lock`, use that installed +version, and invoke from `crates/trusted-server-adapter-cloudflare`. The driver sets +`FINALIZATION_WORKER_OUTPUT` to an absolute directory inside its temporary workspace: + +```bash +worker-build --release --out-dir "$FINALIZATION_WORKER_OUTPUT" . --features trusted-server-core/test-utils +``` + +Point the temporary Wrangler manifest directly at that fixture bundle and disable implicit +rebuild. Use temporary manifest copies for Viceroy and Spin; point them to the fresh +fixture artifacts. Provision only existing application config/secrets infrastructure. +The new consumer must not require an integration KV namespace or label. + +- [ ] Run disabled/enabled cases on the same artifact, host, and port with auctions and + request-random body content disabled. Capture baseline transformed bodies. Success changes + only the expected sentinel; a hung lookup returns exact baseline body bytes. +- [ ] Set the fixture publisher finish timeout to 750 ms. On Axum/Workers/Spin, keep the + transport hang much longer and verify release within a documented cooperative scheduling + tolerance, well before 30 seconds. Record elapsed values and lookup invocation counts; + do not make native tests stand in for SDK timers. + Require a controlled timeout outcome so an early transport error cannot pass the test. + Measure terminal-phase elapsed time separately from origin transfer and server readiness. +- [ ] For Fastly, use an independently bounded transport timeout within the remaining finish + budget and verify unchanged fallback after the host timeout. Report the host bound and + elapsed checks; never describe this as timer-driven interruption. +- [ ] Run the actual runtime entry points through the driver: + +```bash +./scripts/async-html-finalization-local-test.sh fastly +./scripts/async-html-finalization-local-test.sh cloudflare +./scripts/async-html-finalization-local-test.sh spin +./scripts/async-html-finalization-local-test.sh axum +``` + +The driver launches `viceroy serve`, `wrangler dev --local`, `spin up`, or the freshly built +Axum binary with its generated manifests/environment, waits for readiness, captures +headers/body/timing, and always terminates child process groups. Missing tools fail with +an explicit prerequisite message; a script cannot silently skip a selected adapter. + +- [ ] Exercise auction-enabled runtime cases separately where supported. If the existing + Cloudflare clock panic prevents that case, record the exact failure and #1240 dependency; + no change to this feature's scope and no passing claim for the blocked case. +- [ ] Check script syntax and fixture server tests; run + `bash -n scripts/async-html-finalization-local-test.sh` and + `python3 -m py_compile scripts/fixtures/async-html-finalization-origin.py`. + Commit as `Add local runtime checks for HTML finalization`. + +### Task 12: Document the operator and integration contracts + +**Spec coverage:** Sections 6–13 and 15. + +**Files:** Modify `trusted-server.example.toml`, `docs/guide/configuration.md`, +`docs/guide/integration-guide.md`, `docs/guide/cloudflare.md`, +`docs/guide/spin.md`, `docs/guide/axum-dev.md`, and +`crates/trusted-server-integration-tests/README.md` only in relevant sections. + +- [ ] Add the optional publisher setting to the example configuration and canonical settings + table: default 5000, range 1–60000, chain scope after EOF, no effect without registration. + Put comments above configuration values, following repository conventions. +- [ ] Document `.with_html_finalizer`, object-safe local borrowing futures, terminal ordering, + shared document capture state, bounded replacement writes, unchanged fallback, and the + distinction between stage-input bytes and origin bytes. Include a fictional example. +- [ ] Explain opted-in whole-page latency, unchanged no-finalizer streaming, auction budget + interactions, shared-template bypass, `private, no-store`, raw readthrough separation, + and preservation of internal markers/CSP/RSC semantics. +- [ ] Describe timer availability, cooperative scheduling, Fastly host bounds, remote-side-effect + limitations, and independent KV/clock work without copying volatile adapter matrices into + several guides. Link to the canonical integration contract from each affected guide. +- [ ] Document the feature-gated fixture and exact runtime commands, prerequisite versions, + generated temporary workspace behavior, and how to record blocked smoke cases. +- [ ] Run `cargo doc -p trusted-server-core --no-deps --all-features`, `npm run lint` and + `npm run format` from `docs`, plus the repository Markdown check from Task 13. + Validate new local links. Commit as `Document asynchronous HTML finalizer behavior`. + +### Task 13: Run complete gates and review the implemented change against both documents + +**Spec coverage:** Sections 14, 16, and every coverage row below. + +**Files:** No additional production changes unless a check exposes a defect in this feature. + +- [ ] Inspect the diff for scope: no generic streaming trait change, EdgeZero repin, mandatory + store, real signing consumer, JS behavior change, or unrelated refactor. Check feature-off + fixture exclusion and request-state isolation. +- [ ] Run these Rust gates from the workspace root, using the target-matched aliases: + +```bash +cargo fmt --all -- --check +cargo clippy-fastly +cargo clippy-axum +cargo clippy-cloudflare +cargo clippy-cloudflare-wasm +cargo clippy-spin-native +cargo clippy-spin-wasm +cargo clippy-cli +cargo clippy-codegen +cargo test-fastly +cargo test-fastly-reuse +cargo test-axum +cargo test-cloudflare +cargo test-spin +cargo test --manifest-path crates/trusted-server-integration-tests/Cargo.toml --test parity +cargo test -p trusted-server-core --test template_build_digest +cargo clippy -p trusted-server-core --test template_build_digest -- -D warnings +``` + +- [ ] From `crates/trusted-server-js/lib`, run `node build-all.mjs`, `npx vitest run`, and + `npm run format`. From `docs`, run `npm run lint` and `npm run format`. + Install locked docs dependencies with `npm ci` if absent; then from root run: + +```bash +docs/node_modules/.bin/prettier --config docs/.prettierrc --check "*.md" ".claude/**/*.md" ".github/**/*.md" "crates/**/*.md" "scripts/**/*.md" "tinybird/**/*.md" +``` + +- [ ] Run the existing shared-template controls: + +```bash +./scripts/template-cache-local-test.sh +./scripts/template-cache-local-test.sh inline +``` + +Also run the feature's four runtime commands from Task 11 and the existing integration +jobs described in `scripts/integration-tests.sh` and `scripts/integration-tests-browser.sh` +where the feature changes their exercised routes. Record setup and exact outcomes; do not +substitute an empty-auction fixture for the existing winning-bid checks. + +- [ ] Map implementation and tests to every coverage row below. Have a reviewer inspect + the spec and plan together with the final diff; resolve behavioral contradictions centrally. + Do not weaken a regression assertion merely to match a new implementation. +- [ ] Record passed gates and all unavailable/blocked runtime cases in the PR handoff. If any + required gate is unresolved, keep the PR draft. No merge/deploy authorization is implied. + +## 4. Bidirectional spec/plan coverage + +Every requirement must have a task and evidence, and every task must have a requirement. +Use this table during review, not just as a task checklist. + +| Spec contract | Implementation tasks | Required evidence | +| -------------------------------------------------------------------------------------- | -------------------- | ------------------------------------------------------------------------------- | +| Sections 1, 4, 5: HTML terminal phase; generic streaming unchanged; no unrelated work | 1, 3, 8, 13 | API/diff inspection; unchanged legacy streaming tests | +| Section 6.1: ordered registration and one request-local document state | 3, 10 | factory-order and capture-state tests; request/reuse isolation | +| Section 6.2: local borrowing future and bounded replacement sink | 1, 5 | non-`Send` execution; poisoned writer tests | +| Section 7.1: no-finalizer behavior unchanged | 3, 8, 9 | prefix before EOF; one synchronous EOF; no timer/finalizer I/O | +| Section 7.2: drain decoder, synchronous EOF, terminal chain, seam, encoder | 5, 8, 9 | all body/sink/auction/codec combinations; exact event ordering | +| Section 7.3: auction deadline unchanged; opted-in delay explicit | 8, 9, 11, 12 | unchanged dispatch deadline; prefix-before-collection; expired-auction behavior | +| Section 7.4: cancellation releases borrows and does not lose owned input | 5, 8 | drop observer, buffer identity, exactly-once auction abandonment | +| Section 8: capped collection/candidate capacity; safe optional overflow bypass | 1, 6, 9 | boundary/overflow/capacity tests; exact replay; old fatal caps | +| Section 9.1: default 5000 and validated 1–60000 chain budget | 2, 12 | defaults/TOML/range tests; canonical configuration docs | +| Sections 9.2–9.3: optional SDK timers, one absolute budget, Fastly limitations | 1, 2, 4, 5, 11 | deterministic deadline tests; actual-runtime timings; bounded host calls | +| Section 10: stage-local fallback, error categories, markers, panic behavior | 1, 5, 9, 12 | each outcome/candidate failure; late/tie behavior; reviewed callback contract | +| Section 11.1: template bypass before lookup/storage and consistent inline fallback | 7, 9 | warm-hit bypass; no store/reservation; disabled cache behavior restored | +| Section 11.2: downstream privacy, raw-origin cache separation, conditional/range rules | 7, 8, 10 | private headers with no auction; raw-hit execution; four-header stripping | +| Section 11.3: existing body eligibility, status/method/framing/encoding | 7, 8, 9 | bodiless/no-HTML/unsupported cases; non-2xx HTML; header preservation | +| Section 12: every materialized/streamed and lazy/buffered branch | 8, 9, 10 | route matrix and host parity; no synchronous-only skip | +| Section 13: controlled logging and preserved auction telemetry | 5, 8, 11 | no sensitive log fields; outcome counts and abandonment tests | +| Section 14: deterministic, real-runtime, and full repository verification | 9, 10, 11, 13 | named regressions, runtime reports, complete gates | +| Sections 15–16: scoped delivery and readiness conditions | 12, 13 | guides, diff review, requirements/evidence mapping | + +## 5. Review and handoff record + +Before implementation planning is considered ready, review both documents for: + +- Spec requirements omitted from the plan or tasks that contradict the spec. +- Plan additions without a spec requirement, especially fixtures becoming production + consumers, private buffering reappearing, or deadlines multiplying per callback. +- Unrealistic runtime commands, unsupported cfg combinations, and existing behavior that + the plan mistakenly assumes instead of checking. +- Ownership of current HTML during timeout, shared state during factory construction, and + early cache admission that an EOF-only decision could not influence. + +Document review results describe planning readiness only. They do not prove the feature +is implemented, regression-free, or ready for deployment. Execution begins only after +the user requests implementation. + +### Review completed on 2026-10-12 + +The standalone spec review and both directions of the independent spec/plan alignment +review approved planning readiness with no unresolved substantive findings. + +The review corrected these points before handoff: + +- Policy-admission tests in Task 7 no longer require callback execution first wired in + Task 8; execution-dependent assertions now follow the routing implementation. +- The deadline type referenced by the callback API is defined in Task 1, avoiding an + unbuildable intermediate contract commit. +- Core edits have a direct primary-package lint checkpoint; non-2xx HTML is an explicit + acceptance test rather than implicit coverage. +- Overflow route tests distinguish rewrite expansion from preexisting fatal decoder caps. +- Workers fixture output stays in the temporary workspace; runtime timeout tests verify + timer expiry rather than accidentally passing on an early transport error. + +This record covers document review only. The unchecked tasks, runtime requirements, and +implementation acceptance criteria remain outstanding. diff --git a/docs/superpowers/specs/2026-10-12-1241-async-html-finalization-design.md b/docs/superpowers/specs/2026-10-12-1241-async-html-finalization-design.md new file mode 100644 index 000000000..99ba38687 --- /dev/null +++ b/docs/superpowers/specs/2026-10-12-1241-async-html-finalization-design.md @@ -0,0 +1,667 @@ +# Optional asynchronous HTML document finalization + +**Date:** 2026-10-12 + +**Status:** Proposed; design only, awaiting maintainer review + +**Issue:** trusted-server #1241 + +**Inspected baseline:** `2a0cb0444dc335bd09105abe5952329bbbdfb621` + +**Implementation plan:** [Task-by-task implementation and verification](../plans/2026-10-12-1241-async-html-finalization.md) + +## 1. Decision + +Add an opt-in asynchronous final-document phase to Trusted Server's HTML integration +pipeline. An enabled finalizer receives the completed, synchronously transformed HTML +document and may await request-time HTTP or KV services before producing replacement +HTML. Core owns both the retained document and a separate bounded replacement writer. + +Keep `StreamProcessor::process_chunk` synchronous and leave `StreamProcessor` and +`StreamingPipeline::process` unchanged. The new capability is registered explicitly as +an HTML finalizer, rather than being an optional method that some generic synchronous +drivers cannot execute. + +Only processable HTML documents with an enabled finalizer take this path. Existing +integrations retain their streaming behavior. Finalizers execute in their own registration +order, after all existing synchronous HTML processors. This is a terminal phase, not an +interleaved asynchronous extension to every streaming stage. + +One absolute finish deadline covers the entire finalizer chain. On optional-finalizer +failure, retain the bytes entering that finalizer; on deadline expiry, also skip all +remaining finalizers. Preserve existing fatal handling for origin, parser, synchronous +processor, compression, and downstream-write failures. + +For the first version, enabled HTML finalizers bypass shared-template cache reads and +writes before lookup. Their processed HTML responses use the existing synthesized HTML +`private, no-store` policy. Supporting cacheable asynchronous finalizers requires a +separate design for output identity, freshness, and invalidation. + +No EdgeZero revision change is required. Trusted Server owns this document lifecycle and +its request service container. A reusable EdgeZero timer can be considered separately. + +## 2. Verified problem and limits of the evidence + +At the inspected baseline: + +- `streaming_processor.rs::StreamProcessor` provides synchronous `process_chunk` and + `reset` methods. It offers no future-returning operation. +- `html_processor.rs::HtmlWithStreamingProcessors` forwards each chunk and the same + `is_last` value through the entire synchronous chain immediately. +- `IntegrationHtmlStreamProcessorFactory` creates per-document processors. Its context + contains hosts, scheme, and document-local state, but no request services. +- `publisher.rs` awaits auction collection outside synchronous processing callbacks. + The auction already has an application-specific asynchronous seam. +- Buffered materialized responses still use `StreamingPipeline::process` or the separate + reader-based auction hold loop. An asynchronous stream-helper change alone would miss + those paths. + +A local probe using the actual `StreamingPipeline` verified that `now_or_never()` completes +an immediately ready lookup but discards a lookup that returns `Pending`. Awaiting the same +yielding lookup completes it. This establishes the API gap; it does not establish that +every operation yields on every adapter. A local or already available KV result can be +immediately ready. + +No shipped integration at this baseline uses the problematic workaround. The reported +article-signing consumer and the contributor's unsubmitted patch, runtime timings, and +test results have not been independently validated. This is an extension capability, +not a claim that current synchronous integrations are broken. + +The first demonstrated requirement is whole-document text signing: obtain a signature +after reading the article and annotate the page. Such a consumer already withholds the +document. Making its buffering an explicit, bounded core responsibility adds no new +whole-document waiting requirement to that consumer. + +## 3. Goals + +1. Allow a registered HTML integration to await I/O after synchronous document processing + ends, on all four adapters. +2. Make buffer ownership, limits, cancellation, and unchanged-byte fallback enforceable + by core rather than implicit integration conventions. +3. Preserve early streaming, chunk boundaries, processor EOF counts, and auction-seam + timing when no finalizer is enabled. +4. Support both `EdgeBody::Once` and `EdgeBody::Stream`, with and without an auction. +5. Keep decoder validation ahead of finalization, and auction-seam replacement and encoder + finalization after it on the opted-in path. +6. Prevent warm shared-cache hits or downstream response caches from bypassing required + per-request finalization. +7. Specify observable, deterministic failure behavior and tests before implementation. + +## 4. Non-goals + +- Implementing an article-signing integration, an external API protocol, signing-key + management, or a new KV binding. +- Making `lol_html` callbacks await, making every chunk method asynchronous, or inserting + asynchronous processors between arbitrary synchronous processors. +- Adding another framework HTML parser or reparsing the finalizer's output. +- Changing the standalone RSC Flight processor, asset rewriting, EC behavior, consent + gates, or auction admission and transport guarantees. +- Guaranteeing interruption of synchronous Fastly host calls or CPU-heavy callback work. +- Guaranteeing cancellation or rollback of remote HTTP/KV side effects. +- Allowing finalizers to edit HTTP headers or alter the assembly protocol. +- Supporting shared-template storage of asynchronous output in the first version. +- Adding deployment manifests, a general task scheduler, background retries, persistent + queues, or a new platform clock abstraction. + +## 5. Alternatives and tradeoffs + +| Approach | Benefit | Cost or limitation | Decision | +| ------------------------------------------------ | --------------------------------------------------------------------------------------------------------------------------- | ---------------------------------------------------------------------------------------------------------------------------------------------- | ---------------------------------------------------- | +| Optional async finish on every `StreamProcessor` | Supports asynchronous completion inside a mixed streaming chain | Generic synchronous drivers cannot await it; downstream EOF propagation, wrappers, holding behavior, and services become broader contracts | Defer until an actual consumer requires interleaving | +| Make `process_chunk` asynchronous | Supports waits at every chunk boundary | Changes every processor and driver for an EOF-only need; boxed async trait methods can allocate per chunk; parser callbacks still cannot await | Reject | +| Explicit asynchronous HTML terminal phase | Matches the known whole-document consumer; leaves existing streaming traits intact; core can own and bound fallback buffers | Enabled pages wait for EOF; cannot interleave with synchronous stages | Select | +| Perform integration I/O before HTML rewriting | Produces metadata before transformations | Still requires buffering enabled pages and delays parsing; lacks the completed transformed document | Reject for this requirement | + +Opt-in buffering does not mean buffering every page. A registry without finalizers keeps +the current path. Within an opted-in page, all existing synchronous processors run first, +in their existing order, and receive exactly one EOF notification. Finalizers then run in +their own order. Registration order does not imply interleaving between these two phases. + +The framework can provide completed HTML and existing document-local capture state without +adding a parser. The consumer must separately explain how it extracts text and preserves +HTML semantics; this specification does not validate its extraction implementation. + +## 6. Ownership and registration + +### 6.1 Integration registration + +Add `IntegrationHtmlFinalizerFactory` and `IntegrationHtmlFinalizer` contracts alongside +the existing integration factory contracts. Add an empty-by-default finalizer collection +to `IntegrationRegistration` and `IntegrationRegistry`, plus the builder method +`with_html_finalizer`. Preserve factory registration order; do not sort finalizers by ID +or deduplicate registrations silently. + +A factory is `Send + Sync` and can live in the retained application registry. A factory +creates a mutable, single-use finalizer instance for each processed document using the +existing owned `IntegrationHtmlStreamContext`, including its `IntegrationDocumentState`. +The finalizer instance and its returned future need not be `Send`. + +The same document-state instance must be available to parser hooks, synchronous streaming +processors, and finalizers for that document. It must not be recreated between phases or +retained across requests, including reusable Fastly sandboxes. + +Currently `create_html_processor` constructs that state internally. The implementation +must expose the same state through a request-local construction helper or an explicit +construction input; creating a second state for finalizers would disconnect their capture +data from the parser. Preserve the current public helper's synchronous behavior. + +Finalizer presence must be discoverable from enabled registration metadata before shared +template lookup. It must not depend on polling a future, inspecting completed HTML, or +creating a finalizer instance. A factory may later return `Unchanged` when no relevant +article is present, but that cannot retroactively authorize a shared-cache lookup. + +Disabled integrations contribute no finalizers and cause no finalizer-related buffering, +privacy changes, timer creation, or cache bypass. No global setting enables this feature +without an integration explicitly registering a finalizer. + +### 6.2 Callback contract + +Use a regular, object-safe method returning `LocalBoxFuture`, with an explicit lifetime +covering mutable session state and borrowed inputs. Do not add native `async fn` to a trait +used behind `dyn` without an object-safety strategy, and do not require `Send` futures. + +The following is the intended API shape, not implementation code: + +```rust +pub trait IntegrationHtmlFinalizer { + fn finalize<'a>( + &'a mut self, + input: HtmlFinalizationInput<'a>, + ) -> LocalBoxFuture<'a, Result>>; +} + +pub struct HtmlFinalizationInput<'a> { + pub html: &'a [u8], + pub replacement: &'a mut HtmlReplacementWriter, + pub services: &'a RuntimeServices, + pub deadline: &'a HtmlFinalizationDeadline, +} + +pub enum HtmlFinalization { + Unchanged, + Replace, +} +``` + +`Unchanged` emits the driver-owned input without cloning it. `Replace` commits only bytes +successfully written into `HtmlReplacementWriter`. Neither the original buffer nor an +unbounded replacement `Vec` is transferred into the future. + +The method must return its future promptly and perform I/O inside that future. Factory +creation is synchronous, request-local setup; it must not perform an outside lookup or +hide whole-document buffering. Public implementations must follow repository documentation +and error conventions; `HtmlFinalizationError` is a concrete error type with `Report`. + +The replacement writer is a core-owned sink, separate from `html`. Its append operations +are fallible and enforce the byte cap before extending storage. A failed append poisons +that attempt: a later `Replace` cannot commit truncated output even if the callback ignored +the writer error. Empty replacement output is valid only when no required seam token would +be lost; `Unchanged` is the explicit no-replacement result. + +The framework does not make arbitrary integration allocations safe. The cap covers its +retained HTML and replacement storage. Consumer extraction buffers and any external +response buffering must have their own bounds and be reviewed. + +## 7. Processing lifecycle + +### 7.1 No finalizers + +```text +origin -> existing decoder -> existing HTML parser and synchronous processors + -> existing auction seam controller -> existing encoder -> client +``` + +Do not collect a whole document, allocate a replacement writer or boxed finish future, +create a timer, or move auction collection to EOF. Keep existing synchronous helper +semantics. A cheap registration-presence check is acceptable. + +### 7.2 Enabled finalizers + +```text +origin -> existing decoder -> existing HTML parser and synchronous processors + -> bounded retained HTML + -> ordered asynchronous HTML finalizers + -> existing auction seam controller -> existing encoder -> client +``` + +The driver performs these steps: + +1. Create normal document-local parser/processor state and finalizer sessions. +2. Pull and decode origin chunks, run synchronous transformations, and retain the resulting + HTML under the cap. No finalizer-visible HTML is emitted before this phase completes. +3. Drain and validate decoder tails, then finish the existing synchronous chain exactly + once. Include its final output in the retained HTML. +4. Start one absolute finalization deadline and, on supporting adapters, one timer. +5. For each finalizer, lend the current retained HTML immutably and a fresh bounded + replacement writer mutably. Race its future against the same remaining deadline. +6. For an accepted `Replace`, validate the candidate and move it into current retained + storage. Drop the previous stage's input. For `Unchanged` or a soft failure, keep the + current input and discard the candidate. Finalizer B sees the accepted output of A. +7. On deadline expiry, drop the active future and candidate, skip the remaining finalizers, + and retain the current input. Earlier accepted replacements remain accepted. +8. Feed the accepted HTML into the existing auction seam controller in bounded chunks. + Emit its available prefix before collecting bids. Preserve no-seam fallback and terminal + auction telemetry. Then emit the encoder trailer and finish the response. + +Already completed synchronous processors are never re-entered with replacement output. +This prevents duplicate parsing, script injection, Next.js processing, and EOF calls. + +Callbacks observe post-rewrite HTML before reader-specific auction insertion. They cannot +claim to sign the final assembled response or content added later by the auction. A text +signing consumer must document its article scope, exclusions, and canonicalization so +framework scripts, placeholders, and later ad content do not invalidate its claims. + +### 7.3 Auction behavior + +Auction dispatch and its existing deadline remain unchanged. They are not restarted or +extended for finalization. Full-document withholding can make an opted-in page's auction +older by the time its seam reaches collection; an expired auction follows today's +empty-bid behavior. This is an explicit latency/monetization tradeoff for enabled pages. + +When no finalizer is enabled, retain the existing ability to collect at the structural +body seam before origin EOF. The terminal phase must not become a universal EOF barrier. +Missing structural body ends, bot/prefetch/consent suppression, non-HTML bodies, and +abandonment remain governed by existing auction policy. + +### 7.4 Cancellation + +The driver must own current HTML outside the pending finalizer future. Keep a losing +future's lifetime separate from the retained buffer's ownership; explicitly drop it before +moving that buffer or inspecting the borrowed replacement writer. + +Reusing a single deadline across finalizers must not put the only fallback buffer inside +a chain future that is destroyed on timeout. The driver remains the owner at every await. + +A client disconnect drops active work and request-local buffers. It does not invoke another +finalizer, persist partial output, or initiate a background retry. Preserve the existing +dispatched-auction guard so terminal abandonment is recorded exactly once on supported +drop paths. Dropping local work is not a guarantee that a remote write did not occur. + +## 8. Memory and output limits + +Use `publisher.max_buffered_body_bytes` as the cap for both retained transformed HTML and +each replacement writer. Its existing default is 16 MiB. Do not add a second independently +configured page-size limit in this change. + +Check lengths with overflow-safe arithmetic before append or reserve. Avoid capacity +growth beyond the configured cap, including `Vec` geometric reserve growth. Allocate +incrementally; do not reserve the entire cap for every small page. + +At a finalizer await, core may hold one current HTML buffer and one replacement buffer, +each capped at that limit. It must not retain previous successful candidates or clone the +input for rollback. Codec state, pending chunks, consumer-owned state, and platform response +buffers are additional memory; this is not a claim that total request heap use is 16 MiB. + +If collection for the optional terminal phase would exceed the cap: + +- Disable all finalizers for that document before starting I/O. +- On a streaming sink, release the retained prefix, the chunk that crossed the limit, and + subsequent synchronous output through the normal auction/encoding path in order. Avoid + duplicate bytes and a fallback copy of the retained prefix. +- On a buffered sink, preserve the existing outer response-size cap. Optional-finalizer + bypass does not authorize an oversized buffered response. +- Existing raw-input, decoder, parser, or response-output limit failures remain fatal. + This fallback applies only when the new optional collection layer reaches its own cap. + +A replacement-writer overflow is a soft failure of that finalizer. Discard the candidate +and continue with its input, subject to the same remaining finish budget. Never raise the +cap to accept an integration's output. + +## 9. Deadline and adapter services + +### 9.1 Configuration + +Add `publisher.html_finalization_timeout_ms`, defaulting to `5000`. It covers only the +terminal phase after decoded/synchronously processed EOF, across all finalizers together. +It is not an origin-body timeout, auction timeout, or end-to-end response deadline. + +Accept values from `1` through `60000` inclusive and reject invalid settings at startup. +The upper bound keeps the adapter timer duration comfortably within its representable +range and gives the new optional phase a finite operator-controlled budget. Match TOML +deserialization and programmatic `Publisher::default()` behavior. The field has no runtime +effect when no finalizer is registered. + +The five-second default is a proposed product default based on the issue's example, not a +claim that all adapter execution budgets permit or guarantee five seconds. Operators must +also respect their platform's invocation limits. + +### 9.2 Timer contract + +Add an optional `PlatformTimer` service to `RuntimeServices`, defaulting to absent. The +service object is `Send + Sync`; its delay future is local and need not be `Send`. A regular +object-safe method can return `LocalBoxFuture<'static, ()>` for a `Duration`. + +| Adapter | Timer implementation | Finish deadline semantics | +| ---------- | ----------------------- | ----------------------------------------------------------------------------------- | +| Cloudflare | `worker::Delay` | Cooperative race against JS-backed I/O | +| Spin | `spin_sdk::time::sleep` | Cooperative race against runtime-backed I/O | +| Axum | `tokio::time::sleep` | Cooperative race on the existing Tokio executor | +| Fastly | No timer | Check elapsed budget at control boundaries; bound host calls within the integration | + +Keep SDK imports and concrete timer implementations in Trusted Server adapters. These +primitives exist in the pinned dependencies; do not add a Tokio dependency to core or +repin EdgeZero to obtain a timer. + +Use `web_time::Instant` for absolute budget accounting and elapsed observations in core. +Do not introduce `std::time::Instant::now()` or `SystemTime::now()` on the Cloudflare path. +The existing clock sites reported in #1240 are separate work and may block a real-runtime +auction smoke test until corrected; fixing them is not part of this design. + +### 9.3 Race and fallback rules + +Create one absolute deadline and retain one pinned timer across the entire chain. Poll +the deadline before active work at decision points. Check absolute expiry before a stage +starts and after it resolves; if completion and expiry coincide, deadline expiry wins. +Do not restart a full-duration sleep for each finalizer. + +Callbacks receive a deadline handle exposing remaining time and whether a cooperative +timer is available. Bound individual network requests and external response reads to the +remaining budget where the service permits it. Successful but late candidates are +discarded even if a timer could not run while the callback was polling. + +Timers cannot preempt a blocking host call or a callback that never yields. On Fastly, +consumer acceptance requires evidence that its calls are bounded independently, or that +it returns `Unchanged` without performing unsupported unbounded operations. Core cannot +promise a hard five-second release for arbitrary Fastly HTTP/KV operations. + +Cancellation frees borrowed buffers and releases the publisher's wait. It does not +universally abort transport work or reverse external writes. Consumers must tolerate +duplicate requests/retries and use idempotent external operations where required. + +## 10. Failure and validation contract + +Fallback bytes are the synchronously transformed input of the current finalizer, including +accepted earlier finalizers. They are not necessarily the origin's original bytes. No +rollback of prior accepted HTML or remote side effects is promised. + +| Event | Document behavior | Remaining finalizers | +| ----------------------------------------------------------------------------- | ---------------------------------------------------------------- | ------------------------------- | +| `Unchanged` | Keep current input; discard any candidate writes | Continue | +| Valid `Replace` within budget | Commit bounded candidate | Continue with replacement | +| Callback reports unavailable/rejected external work | Keep current input; discard candidate | Continue while budget remains | +| Replacement writer overflow or invalid required marker | Keep current input; discard candidate | Continue while budget remains | +| Overall deadline expires | Drop active future and candidate; keep current input | Skip all remaining | +| Optional collection limit reached | Release retained synchronous output through ordinary delivery | Skip all finalizers | +| Origin read, decoder, parser, synchronous processor, encoder, or sink failure | Existing fatal error/truncation behavior and auction abandonment | Stop | +| Callback panics | Existing panic behavior | No new panic-catching mechanism | + +Define controlled error categories for external failure and candidate validation. Keep +`Report` at orchestration boundaries and existing `io::Error` where +streaming I/O uses it. Do not turn unrelated processor errors into silent passthrough. + +Finalizers must preserve the parser-generated control marker, when present, as opaque +bytes. They must not remove, duplicate, or relocate it into script/text content. Core +checks the active token's required cardinality before committing a replacement; missing +or duplicated tokens reject that candidate. A count check cannot prove structural +placement. This remaining invariant is a reviewed integration contract, not a claim of +automatic HTML validation without parsing. + +Core does not inspect arbitrary replacement HTML for semantic validity or reconstruct +parser observations. Finalizers must preserve valid document structure, existing CSP +requirements, URL/RSC encodings, and the meaning of upstream rewrite decisions. Header +changes and transformations needing new parser-derived security observations require a +separate capability. The first consumer should add signature metadata without changing +executable markup or article text after canonicalization. + +## 11. Cache, privacy, conditional requests, and framing + +### 11.1 Shared-template cache + +When enabled finalizer metadata is present for the request's document pipeline, disallow +shared-template lookup/reservation before the first cache read. The same decision must +exclude later storage. A factory deciding `Unchanged` at EOF does not restore eligibility. + +Use the existing non-shared inline fallback consistently across slot scripts, body-seam +selection, transformation, and assembly. Do not generate a shared-template marker for a +response that has no shared-template consumer. Record the existing request-bypass outcome +where cache response-state reporting is enabled. + +Warm entries may remain stored but must not be read while the finalizer is enabled. No +purge is required merely to activate it. Disabling an integration can restore ordinary +eligibility, subject to existing settings/build fingerprints and template validation. + +There is no first-version `shared_safe` escape hatch. Later cache support must specify at +least reader-neutral output, external-data/signing-key identity, expiration, invalidation, +and whether failures can be cached. Cold-miss ordering alone does not settle these rules. + +### 11.2 Downstream privacy and origin readthrough + +Apply the existing synthesized HTML `private, no-store` policy to body-carrying HTML +responses that enter the enabled finalizer pipeline, regardless of auction eligibility or +the finalizer outcome. Strip validators and stale downstream cache directives using the +same existing helper as other synthesized HTML. Apply this before streaming headers are +committed. An unchanged or size-bypassed result remains private for that response. + +This prevents a browser or downstream cache from treating a per-request result as the +origin's public representation. Origin readthrough remains a separate raw-byte cache: it +may still provide a complete shareable origin body under existing policy, because every +request subsequently runs transformation/finalization. Do not cache assembled output as +an origin body or modify EC-cookie privacy rules. + +For document GETs requiring finalization, reuse +`strip_conditional_and_range_headers` so client revalidation or range requests cannot +make the origin return a bodyless `304` or partial `206` instead of the full document. +That helper removes `If-None-Match`, `If-Modified-Since`, `Range`, and `If-Range`. +Do not discard mutation preconditions such as `If-Match` or `If-Unmodified-Since`, +or broaden GET behavior to other methods. Unexpected bodyless responses retain existing +status/error handling and invoke no finalizer, including the existing invalid-origin-304 +guard when the ad stack is active. This feature does not turn an existing 502 into a 304. + +### 11.3 Response selection and framing + +Run finalizers only on HTML that the existing routing rules already process, including +processable non-2xx HTML. Do not rewrite unsupported encodings, unmodified pass-through +responses, assets, or standalone `text/x-component` bodies. + +HEAD and `204`, `205`, or `304` responses invoke no finalizer or finalizer I/O. Preserve +their existing framing and any dispatched-auction abandonment. Preserve method/status +handling for other requests instead of introducing a GET-only body transformer silently. + +Finalizers do not change the selected content encoding. Strip stale `Content-Length` as +the existing transformed-body path requires; stream encoders finalize only after accepted +HTML and auction output are emitted. Preserve repeated headers, cookies, and CSP policy. + +## 12. Integration into publisher paths + +The first implementation must deliberately cover these existing branches: + +| Source and sink | Auction | Required integration | +| ----------------------------------------------- | ------- | ------------------------------------------------------------------------------------- | +| Lazy response stream, `Once` or `Stream` source | None | Retain opted-in synchronous HTML output, await finalizers, then encode | +| Lazy response stream, `Once` or `Stream` source | Pending | Same phase before existing seam collection; retain auction guard | +| Buffered response, streamed source | None | Same terminal phase before completing bounded response output | +| Buffered response, streamed source | Pending | Same phase before seam/encoder finish | +| Buffered response, materialized source | None | Opted-in path cannot fall through `stream_publisher_body` and skip finalization | +| Buffered response, materialized source | Pending | Reader-based hold path must route opted-in HTML through the same terminal coordinator | + +Introduce one private HTML terminal coordinator shared by the enabled paths. Feed it the +same synchronous transformation semantics on materialized and streamed inputs. Keep +ordinary no-finalizer branches intact; avoid replacing the entire generic compression +architecture to obtain this capability. + +Do not add finalizers to a generic synchronous helper that cannot await them. Public +synchronous helpers continue to implement synchronous transformations only; document that +HTML finalizers require the supported publisher finalization entry points. Adapters must +never select a synchronous-only path for a registered finalizer silently. + +Thread document state and finalizer metadata explicitly through the relevant publisher +parameters. Do not reconstruct a separate registry/session inside a lazy stream or copy +request services into global state. Reuse existing services ownership for request-lifetime +and lazy response futures. + +## 13. Observability + +Use existing `log` instrumentation with integration ID, controlled outcome, elapsed time, +and bounded byte counts. Log ordinary completion/unchanged results at debug level and soft +failure, timeout, or optional-collection bypass at warn level. Emit at most one outcome +per attempted finalizer and one terminal collection/deadline bypass for the document. + +Never log HTML, article text, signatures, external response bodies, control tokens, client +identifiers, secrets, URLs containing request data, or arbitrary error-report attachments. +Log controlled categories instead of formatting an integration-supplied report wholesale. + +No new public response headers, persistent telemetry store, or metrics subsystem is needed. +Existing auction terminal telemetry retains its meanings and exactly-once behavior. + +## 14. Verification requirements + +The checks below are acceptance requirements for the implementation, not claims that the +unsubmitted feature has already passed them. + +### 14.1 API, registration, and state + +- Default registration has no finalizers; a disabled integration contributes none. +- Factory and finalizer ordering is deterministic, including multiple factories for one ID. +- Sessions and document capture state are isolated between requests and reused sandboxes. +- A non-`Send` future borrowing input, replacement writer, and session state compiles and + executes through each supported publisher entry point. +- Existing processors retain chunk behavior and receive EOF once before any finalizer. +- Synchronous helpers retain their documented scope; no adapter silently skips a finalizer. + +### 14.2 Real yielding, fallback, and deadlines + +- A stub that returns `Pending` at least once completes when awaited and changes output. +- Finalizer A's accepted output is exactly B's input; B never receives partial candidates. +- A timeout after partial candidate writes drops the future, emits exact input bytes, and + invokes no later finalizer. Use a drop observer and allocation/buffer identity assertions + to prove no fallback document clone. +- `Unchanged`, callback error, invalid marker, replacement overflow, and a late result + preserve exact stage input and discard partial candidate state. +- Two or more delayed finalizers share one deadline; the budget does not multiply. +- Deadline/completion ties follow the documented timeout-wins rule. Use an injectable + deterministic timer/clock in tests rather than flaky wall-clock sleeps. +- Client cancellation drops active work and retains exactly-once auction abandonment. +- Fastly tests demonstrate no timer dependency, budget checks, and bounded-call consumer + behavior; they do not pretend a synchronous call can be preempted. + +### 14.3 Bounds and byte ordering + +- Test cap minus one, exact cap, cap plus one, and length-overflow rejection. +- Test multiple chunks and one large chunk crossing the optional collection cap. Prefix, + crossing chunk, and suffix emerge once in order on the streaming bypass path. +- Buffered output caps and existing decoder/input caps retain their existing failures. +- Replacement sink overflow cannot commit even when a callback ignores its write error. +- Input and replacement capacities remain bounded; no full-cap allocation for tiny pages. +- Accepted prior stages are released as ownership advances; timeout does not restore the + origin or undo an earlier successful finalizer. + +### 14.4 Publisher and codec matrix + +Exercise `Once` and `Stream` sources, lazy and buffered sinks, auction present/absent, +and identity, gzip, deflate, and Brotli encoding. Include concatenated gzip members, +decoder tails released only at EOF, truncated/corrupt streams, empty HTML, split UTF-8, +missing body ends, and malformed-but-renderable HTML. + +Verify decoded/synchronous EOF precedes finalizer I/O, candidates preserve private marker +cardinality, ready prefixes precede auction collection, and encoder trailers follow all +accepted bytes. Finalizer failure must not leave a raw private marker in client output. + +Retain the current tests for early compressed output, parser-confirmed Next.js seams, +placeholder substitution, body-source cancellation, and auction abandonment. Those tests +must continue to use the no-finalizer path and protect its existing streaming behavior. + +### 14.5 Cache, headers, and eligibility + +- Populate a template entry with finalization disabled, enable a finalizer, and prove the + request bypasses the warm entry and runs its finalizer. Assert no lookup reservation or + store is made; cover ESI configuration falling back to inline. +- Disable the finalizer and verify ordinary cache miss/store/hit behavior is restored. +- Responses remain `private, no-store` for success, `Unchanged`, timeout, external failure, + and collection bypass, including HTML without an auction or matched ad slot. +- Raw origin readthrough hits still execute finalizers per request; no assembled output + enters the raw origin cache. +- Test synthesized GET revalidation/range handling and preserved mutation preconditions, + including unexpected origin 304 behavior with and without an active ad stack. +- Verify non-2xx HTML is processed, while HEAD, bodiless statuses, non-HTML, unsupported + encodings, and unmodified responses make no finalizer call. Preserve framing and cookies. +- Existing shared-template validation, assembly, CSP nonce, and cache-privacy tests pass + unchanged with no finalizer enabled. + +### 14.6 Actual adapters and repository gates + +Native Cloudflare/Spin tests cannot establish JS or WASI timer behavior. Add fictional +fixtures and run a bounded delayed/hung lookup on Viceroy, local Workers, Spin, and Axum. +Record the exact setup, SDK/runtime versions, observed behavior, and limitations. Do not +claim a universal exact wall-clock release; cooperative scheduling adds tolerance. + +The Cloudflare clock issue #1240 must be resolved separately, or its affected auction smoke +must be explicitly reported as blocked rather than marked passing. Keep KV wiring under +#1139 separate; HTTP-backed and unavailable-KV fixtures can verify this lifecycle without +requiring new mandatory store bindings. + +Before implementation PR handoff, run every CI gate in [AGENTS.md](../../../AGENTS.md#ci-gates), +including target-matched adapter checks/tests, parity, JS/docs formatting and tests, and +native build-digest checks. Run existing template-cache ESI/inline harnesses as appropriate. +Use only fictional fixture hosts, text, configuration, and credentials. + +## 15. Implementation scope and delivery + +Expected production touch points: + +- Core integration registration: terminal factory/session contracts and presence metadata. +- Core HTML/publisher processing: shared document state, bounded collection/replacement, + terminal coordinator, path selection, cache/conditional/privacy integration, and errors. +- Core platform services: optional timer and deadline helper using existing clock support. +- Core settings: the finish timeout default/validation and canonical configuration docs. +- Adapter service construction: Cloudflare, Spin, and Axum timer implementations; Fastly + explicitly retains no cooperative timer. +- Tests, integration fixtures, example configuration, and canonical integration/adapter + guides documenting opt-in behavior and limitations. + +Keep the generic streaming trait, synchronous pipeline, JS runtime, platform manifests, +and EdgeZero pin unchanged. Do not bundle the real signing consumer, KV-store wiring, +Cloudflare clock fixes, or unrelated publisher refactoring into this feature. + +The implementation plan should separate contract/registration work, terminal coordination, +adapter timers, cache/privacy behavior, and verification into small reviewable changes, +while not shipping an intermediate state in which enabled finalizers can be skipped. + +This document authorizes no deployment, external API use, or publication. After maintainer +review, write the implementation plan separately. The current branch contains design work +only. + +## 16. Acceptance criteria + +The feature is ready only when: + +1. A yielding finalizer works through every supported publisher response path. +2. No-finalizer output, streaming timing, auction sequencing, and error handling retain + existing behavior under their regression tests. +3. Fallback is exact at the failed stage, candidate writes are bounded and isolated, and + timeout drops the active future without cloning the retained document. +4. The deadline covers the entire terminal chain, with Fastly's non-preemption limitation + explicitly tested and documented. +5. Shared-template reads and writes cannot bypass or persist the new per-request phase; + downstream privacy and origin readthrough remain correctly separated. +6. No unsupported body, method/status, encoding, or synchronous-only adapter path silently + invokes or skips the new capability. +7. Real runtime evidence and all required repository gates are recorded with honest limits. + +## 17. References and investigation record + +- Issue #1241: requested asynchronous completion for HTML stream processors. +- PR #1135 and the [parser-aware body hold design](2026-09-07-850-parser-aware-body-hold-nextjs-streaming-design.md): + synchronous stream processors and parser-generated auction seam. +- Issues #1240 and #1139: related clock and runtime KV findings; independent scope. +- [Streaming processor](../../../crates/trusted-server-core/src/streaming_processor.rs), + [HTML processor](../../../crates/trusted-server-core/src/html_processor.rs), and + [integration registry](../../../crates/trusted-server-core/src/integrations/registry.rs). +- [Publisher](../../../crates/trusted-server-core/src/publisher.rs), + [runtime services](../../../crates/trusted-server-core/src/platform/types.rs), + [template cache contract](../../../crates/trusted-server-core/src/platform/template_cache.rs), + and [publisher settings](../../../crates/trusted-server-core/src/settings.rs). + +During investigation at the baseline, the following native core test filters passed: + +```text +cargo test -p trusted-server-core --lib streaming_processor::tests --offline +34 passed +cargo test -p trusted-server-core --lib html_processor::tests --offline +32 passed +cargo test -p trusted-server-core --lib publisher::tests --offline +369 passed +``` + +Two temporary probe tests also passed: the current synchronous pipeline discards a +yielding one-poll lookup, and awaiting the same lookup completes it. These are investigation +evidence, not implementation regression tests or proof of the contributor's runtime claims. From 35a28e3a919af552b2b203bc11661f3dd0dd6808 Mon Sep 17 00:00:00 2001 From: prk-Jr Date: Mon, 12 Oct 2026 17:33:45 +0530 Subject: [PATCH 02/11] Add optional asynchronous HTML finalization --- crates/trusted-server-adapter-axum/src/app.rs | 2 + .../src/platform.rs | 24 + .../Cargo.toml | 2 + .../src/app.rs | 2 + .../src/lib.rs | 31 + .../src/platform.rs | 20 +- .../trusted-server-adapter-fastly/src/app.rs | 3 + crates/trusted-server-adapter-spin/Cargo.toml | 2 + crates/trusted-server-adapter-spin/src/app.rs | 2 + crates/trusted-server-adapter-spin/src/lib.rs | 31 + .../src/platform.rs | 20 +- crates/trusted-server-core/src/config.rs | 7 + .../trusted-server-core/src/html_processor.rs | 63 +- .../src/integrations/html_finalization.rs | 269 +++ .../src/integrations/mod.rs | 11 + .../src/integrations/registry.rs | 65 + .../trusted-server-core/src/platform/mod.rs | 48 + .../trusted-server-core/src/platform/timer.rs | 12 + .../trusted-server-core/src/platform/types.rs | 20 +- crates/trusted-server-core/src/publisher.rs | 1528 +++++++++++++++-- .../src/publisher/html_finalization.rs | 1090 ++++++++++++ crates/trusted-server-core/src/settings.rs | 56 + .../trusted-server-core/src/test_support.rs | 9 + .../test_support/async_html_finalization.rs | 193 +++ .../tests/parity.rs | 146 ++ docs/guide/configuration.md | 26 +- docs/guide/integration-guide.md | 87 + ...2026-10-12-1241-async-html-finalization.md | 19 +- ...-async-html-finalization-implementation.md | 178 ++ ...-12-1241-async-html-finalization-design.md | 4 +- scripts/async-html-finalization-local-test.sh | 5 + .../async-html-finalization-origin.py | 84 + .../async-html-finalization-runtime.py | 275 +++ trusted-server.example.toml | 2 + 34 files changed, 4185 insertions(+), 151 deletions(-) create mode 100644 crates/trusted-server-core/src/integrations/html_finalization.rs create mode 100644 crates/trusted-server-core/src/platform/timer.rs create mode 100644 crates/trusted-server-core/src/publisher/html_finalization.rs create mode 100644 crates/trusted-server-core/src/test_support/async_html_finalization.rs create mode 100644 docs/superpowers/reviews/2026-10-12-1241-async-html-finalization-implementation.md create mode 100755 scripts/async-html-finalization-local-test.sh create mode 100644 scripts/fixtures/async-html-finalization-origin.py create mode 100644 scripts/fixtures/async-html-finalization-runtime.py diff --git a/crates/trusted-server-adapter-axum/src/app.rs b/crates/trusted-server-adapter-axum/src/app.rs index caba714d6..cff5bc47e 100644 --- a/crates/trusted-server-adapter-axum/src/app.rs +++ b/crates/trusted-server-adapter-axum/src/app.rs @@ -246,6 +246,7 @@ async fn dispatch_fallback( // slots; `handle_publisher_request` matches them against the request path. let mut ec_context = build_ec_context(state, services, &req); let auction = AuctionDispatch { + integration_registry: &state.registry, orchestrator: &state.orchestrator, slots: state.settings.creative_opportunity_slots(), registry: None, @@ -526,6 +527,7 @@ fn named_route_handler( } else { let mut ec_context = build_ec_context(&state, &services, &req); let auction = AuctionDispatch { + integration_registry: &state.registry, orchestrator: &state.orchestrator, slots: state.settings.creative_opportunity_slots(), registry: None, diff --git a/crates/trusted-server-adapter-axum/src/platform.rs b/crates/trusted-server-adapter-axum/src/platform.rs index e2713f47f..402ff2753 100644 --- a/crates/trusted-server-adapter-axum/src/platform.rs +++ b/crates/trusted-server-adapter-axum/src/platform.rs @@ -552,6 +552,7 @@ pub fn build_runtime_services(ctx: &edgezero_core::context::RequestContext) -> R static GEO: std::sync::OnceLock> = std::sync::OnceLock::new(); RuntimeServices::builder() + .timer(Arc::new(AxumTimer)) .config_store(Arc::clone(CONFIG_STORE.get_or_init(|| { Arc::new(AxumPlatformConfigStore) as Arc }))) @@ -585,8 +586,31 @@ pub fn build_runtime_services(ctx: &edgezero_core::context::RequestContext) -> R // Tests // --------------------------------------------------------------------------- +struct AxumTimer; + +impl trusted_server_core::platform::PlatformTimer for AxumTimer { + fn delay(&self, duration: std::time::Duration) -> futures::future::LocalBoxFuture<'static, ()> { + Box::pin(async move { + tokio::time::sleep(duration).await; + }) + } +} + #[cfg(test)] mod tests { + #[test] + fn html_finalization_timer_is_request_scoped() { + let request = edgezero_core::http::request_builder() + .body(edgezero_core::body::Body::empty()) + .expect("should build a request"); + let context = edgezero_core::context::RequestContext::new(request, Default::default()); + let services = build_runtime_services(&context); + assert!( + services.timer().is_some(), + "should install a cooperative timer without polling it" + ); + } + use super::*; use edgezero_core::body::Body as EdgeBody; use std::time::Duration; diff --git a/crates/trusted-server-adapter-cloudflare/Cargo.toml b/crates/trusted-server-adapter-cloudflare/Cargo.toml index 2ce969ef7..495dfe633 100644 --- a/crates/trusted-server-adapter-cloudflare/Cargo.toml +++ b/crates/trusted-server-adapter-cloudflare/Cargo.toml @@ -17,6 +17,8 @@ crate-type = ["cdylib", "rlib"] [features] default = [] +# Opt-in local runtime fixture and its diagnostic console sink. +test-utils = ["trusted-server-core/test-utils"] # Keep for explicit `cargo check --features cloudflare --target wasm32-unknown-unknown` cloudflare = ["edgezero-adapter-cloudflare/cloudflare", "dep:worker"] diff --git a/crates/trusted-server-adapter-cloudflare/src/app.rs b/crates/trusted-server-adapter-cloudflare/src/app.rs index 87b9567e7..b6e49f561 100644 --- a/crates/trusted-server-adapter-cloudflare/src/app.rs +++ b/crates/trusted-server-adapter-cloudflare/src/app.rs @@ -555,6 +555,7 @@ fn build_router(state: &Arc) -> RouterService { } else { let mut ec_context = build_ec_context(&state.settings, &services, &req); let auction = AuctionDispatch { + integration_registry: &state.registry, orchestrator: &state.orchestrator, slots: state.settings.creative_opportunity_slots(), registry: None, @@ -724,6 +725,7 @@ fn build_router(state: &Arc) -> RouterService { let page_bids = make_handler(Arc::clone(&state), |s, services, req| async move { let mut ec_context = build_ec_context(&s.settings, &services, &req); let auction = AuctionDispatch { + integration_registry: &s.registry, orchestrator: &s.orchestrator, slots: s.settings.creative_opportunity_slots(), registry: None, diff --git a/crates/trusted-server-adapter-cloudflare/src/lib.rs b/crates/trusted-server-adapter-cloudflare/src/lib.rs index 980a9b209..80c24e8f1 100644 --- a/crates/trusted-server-adapter-cloudflare/src/lib.rs +++ b/crates/trusted-server-adapter-cloudflare/src/lib.rs @@ -24,6 +24,8 @@ use worker::{Context, Env, Request, Response, Result, event}; /// Returns a Workers runtime error when the fallback error response cannot be /// constructed. pub async fn main(req: Request, env: Env, ctx: Context) -> Result { + #[cfg(feature = "test-utils")] + install_fixture_logger(); if let Ok(config) = env.var("TRUSTED_SERVER_CONFIG") { app::set_cloudflare_config_json(config.to_string()); } @@ -37,3 +39,32 @@ pub async fn main(req: Request, env: Env, ctx: Context) -> Result { } } } + +#[cfg(all(feature = "test-utils", target_arch = "wasm32"))] +struct FixtureLogger; + +#[cfg(all(feature = "test-utils", target_arch = "wasm32"))] +impl log::Log for FixtureLogger { + fn enabled(&self, metadata: &log::Metadata<'_>) -> bool { + metadata.level() <= log::Level::Warn + } + + fn log(&self, record: &log::Record<'_>) { + if self.enabled(record.metadata()) { + worker::console_warn!("{}", record.args()); + } + } + + fn flush(&self) {} +} + +#[cfg(all(feature = "test-utils", target_arch = "wasm32"))] +fn install_fixture_logger() { + static LOGGER: FixtureLogger = FixtureLogger; + if log::set_logger(&LOGGER).is_ok() { + std::panic::set_hook(Box::new(|information| { + worker::console_error!("{}", information); + })); + log::set_max_level(log::LevelFilter::Warn); + } +} diff --git a/crates/trusted-server-adapter-cloudflare/src/platform.rs b/crates/trusted-server-adapter-cloudflare/src/platform.rs index 94616b203..09d1f2ce6 100644 --- a/crates/trusted-server-adapter-cloudflare/src/platform.rs +++ b/crates/trusted-server-adapter-cloudflare/src/platform.rs @@ -631,7 +631,7 @@ pub fn build_runtime_services(ctx: &edgezero_core::context::RequestContext) -> R // simply absent on the native host target, producing Ok(None) from lookup(). let geo = build_geo(ctx); - RuntimeServices::builder() + let builder = RuntimeServices::builder() .config_store(config_store) .secret_store(secret_store) .kv_store(kv_store) @@ -643,8 +643,10 @@ pub fn build_runtime_services(ctx: &edgezero_core::context::RequestContext) -> R tls_protocol: None, tls_cipher: None, ..ClientInfo::default() - }) - .build() + }); + #[cfg(target_arch = "wasm32")] + let builder = builder.timer(Arc::new(CloudflareTimer)); + builder.build() } // --------------------------------------------------------------------------- @@ -754,6 +756,18 @@ fn reject_multi_provider_fanout(len: usize) -> Result<(), Report> Ok(()) } +#[cfg(target_arch = "wasm32")] +struct CloudflareTimer; + +#[cfg(target_arch = "wasm32")] +impl trusted_server_core::platform::PlatformTimer for CloudflareTimer { + fn delay(&self, duration: std::time::Duration) -> futures::future::LocalBoxFuture<'static, ()> { + Box::pin(async move { + worker::Delay::from(duration).await; + }) + } +} + #[cfg(test)] mod tests { use super::*; diff --git a/crates/trusted-server-adapter-fastly/src/app.rs b/crates/trusted-server-adapter-fastly/src/app.rs index fbcd5735d..80f394722 100644 --- a/crates/trusted-server-adapter-fastly/src/app.rs +++ b/crates/trusted-server-adapter-fastly/src/app.rs @@ -696,6 +696,7 @@ async fn run_named_route( Some(&partner_registry) }; let auction = AuctionDispatch { + integration_registry: &state.registry, orchestrator: &state.orchestrator, slots: state.settings.creative_opportunity_slots(), registry: registry_ref, @@ -860,6 +861,7 @@ async fn dispatch_fallback( match PartnerRegistry::from_config(&state.settings.ec.partners) { Ok(partner_registry) => { let auction = AuctionDispatch { + integration_registry: &state.registry, orchestrator: &state.orchestrator, slots, registry: Some(&partner_registry), @@ -3029,6 +3031,7 @@ mod tests { None, &mut ec_context, AuctionDispatch { + integration_registry: ®istry, orchestrator: &orchestrator, slots: settings.creative_opportunity_slots(), registry: None, diff --git a/crates/trusted-server-adapter-spin/Cargo.toml b/crates/trusted-server-adapter-spin/Cargo.toml index 81eeb9430..b0a5403d1 100644 --- a/crates/trusted-server-adapter-spin/Cargo.toml +++ b/crates/trusted-server-adapter-spin/Cargo.toml @@ -17,6 +17,8 @@ crate-type = ["cdylib", "rlib"] [features] default = [] +# Opt-in local runtime fixture and its diagnostic stderr sink. +test-utils = ["trusted-server-core/test-utils"] spin = ["edgezero-adapter-spin/spin"] [dependencies] diff --git a/crates/trusted-server-adapter-spin/src/app.rs b/crates/trusted-server-adapter-spin/src/app.rs index b8e8a2492..5cdf6bc5e 100644 --- a/crates/trusted-server-adapter-spin/src/app.rs +++ b/crates/trusted-server-adapter-spin/src/app.rs @@ -713,6 +713,7 @@ fn build_router(state: &Arc) -> RouterService { } let mut ec_context = build_ec_context(&s.settings, &services, &req); let auction = AuctionDispatch { + integration_registry: &s.registry, orchestrator: &s.orchestrator, slots: s.settings.creative_opportunity_slots(), registry: None, @@ -836,6 +837,7 @@ fn build_router(state: &Arc) -> RouterService { } else { let mut ec_context = build_ec_context(&state.settings, &services, &req); let auction = AuctionDispatch { + integration_registry: &state.registry, orchestrator: &state.orchestrator, slots: state.settings.creative_opportunity_slots(), registry: None, diff --git a/crates/trusted-server-adapter-spin/src/lib.rs b/crates/trusted-server-adapter-spin/src/lib.rs index f47877ff2..adbf61c9d 100644 --- a/crates/trusted-server-adapter-spin/src/lib.rs +++ b/crates/trusted-server-adapter-spin/src/lib.rs @@ -13,5 +13,36 @@ use spin_sdk::http_service; #[http_service] // FORCED: edgezero_adapter_spin::run_app returns anyhow::Result — EdgeZero SDK constraint, not a project choice. async fn handle(req: Request) -> anyhow::Result { + #[cfg(feature = "test-utils")] + install_fixture_logger(); edgezero_adapter_spin::run_app::(req).await } + +#[cfg(all(feature = "test-utils", feature = "spin", target_arch = "wasm32"))] +use std::io::Write as _; + +#[cfg(all(feature = "test-utils", feature = "spin", target_arch = "wasm32"))] +struct FixtureLogger; + +#[cfg(all(feature = "test-utils", feature = "spin", target_arch = "wasm32"))] +impl log::Log for FixtureLogger { + fn enabled(&self, metadata: &log::Metadata<'_>) -> bool { + metadata.level() <= log::Level::Warn + } + + fn log(&self, record: &log::Record<'_>) { + if self.enabled(record.metadata()) { + let _ = writeln!(std::io::stderr(), "{}", record.args()); + } + } + + fn flush(&self) {} +} + +#[cfg(all(feature = "test-utils", feature = "spin", target_arch = "wasm32"))] +fn install_fixture_logger() { + static LOGGER: FixtureLogger = FixtureLogger; + if log::set_logger(&LOGGER).is_ok() { + log::set_max_level(log::LevelFilter::Warn); + } +} diff --git a/crates/trusted-server-adapter-spin/src/platform.rs b/crates/trusted-server-adapter-spin/src/platform.rs index b1ced0f70..d46170bfc 100644 --- a/crates/trusted-server-adapter-spin/src/platform.rs +++ b/crates/trusted-server-adapter-spin/src/platform.rs @@ -727,7 +727,7 @@ pub fn build_runtime_services(ctx: &edgezero_core::context::RequestContext) -> R #[cfg(not(all(feature = "spin", target_arch = "wasm32")))] let secret_store: Arc = Arc::new(NoopSecretStore); - RuntimeServices::builder() + let builder = RuntimeServices::builder() .config_store(config_store) .secret_store(secret_store) .kv_store(kv_store) @@ -744,8 +744,10 @@ pub fn build_runtime_services(ctx: &edgezero_core::context::RequestContext) -> R h2_fingerprint: None, server_hostname: None, server_region: None, - }) - .build() + }); + #[cfg(all(feature = "spin", target_arch = "wasm32"))] + let builder = builder.timer(Arc::new(SpinTimer)); + builder.build() } // --------------------------------------------------------------------------- @@ -778,6 +780,18 @@ fn extract_client_ip(ctx: &edgezero_core::context::RequestContext) -> Option futures::future::LocalBoxFuture<'static, ()> { + Box::pin(async move { + spin_sdk::time::sleep(duration).await; + }) + } +} + #[cfg(test)] mod tests { use super::*; diff --git a/crates/trusted-server-core/src/config.rs b/crates/trusted-server-core/src/config.rs index 79f100e2f..a0df557e1 100644 --- a/crates/trusted-server-core/src/config.rs +++ b/crates/trusted-server-core/src/config.rs @@ -37,6 +37,8 @@ use crate::settings::{AssetOriginAuth, IntegrationConfig, Settings}; const DEPLOY_VALIDATION_FIELD: &str = "trusted_server"; #[cfg(test)] const DEPLOY_VALIDATED_INTEGRATION_IDS: &[&str] = &[ + #[cfg(feature = "test-utils")] + crate::test_support::async_html_finalization::ID, "prebid", "aps", "adserver_mock", @@ -306,6 +308,11 @@ fn validate_enabled_integrations( validate_prebid(settings, plan)?; validate_integration::(settings, "aps")?; validate_integration::(settings, "adserver_mock")?; + #[cfg(feature = "test-utils")] + validate_integration::( + settings, + crate::test_support::async_html_finalization::ID, + )?; validate_integration::(settings, "testlight")?; validate_integration::(settings, "nextjs")?; validate_integration::(settings, "permutive")?; diff --git a/crates/trusted-server-core/src/html_processor.rs b/crates/trusted-server-core/src/html_processor.rs index 3889d7d59..2f6bc5b48 100644 --- a/crates/trusted-server-core/src/html_processor.rs +++ b/crates/trusted-server-core/src/html_processor.rs @@ -194,8 +194,19 @@ impl HtmlProcessorConfig { reason = "the returned processor owns request configuration captured by its handlers" )] pub fn create_html_processor(config: HtmlProcessorConfig) -> impl StreamProcessor { + create_html_processor_with_document_state(config, &IntegrationDocumentState::default()) +} + +#[allow( + clippy::needless_pass_by_value, + reason = "static parser callbacks capture owned request configuration" +)] +pub(crate) fn create_html_processor_with_document_state( + config: HtmlProcessorConfig, + document_state: &IntegrationDocumentState, +) -> impl StreamProcessor + use<> { let stream_processor_factories = config.integrations.html_stream_processor_factories(); - let document_state = IntegrationDocumentState::default(); + if config.suppress_datadome_client_side_tag { document_state.get_or_insert_with(DATADOME_INTEGRATION_ID, || DataDomeClientTagSuppressed); } @@ -723,7 +734,8 @@ mod tests { use super::*; use crate::integrations::{ AttributeRewriteAction, IntegrationAttributeContext, IntegrationAttributeRewriter, - IntegrationHeadInjector, IntegrationHtmlContext, + IntegrationHeadInjector, IntegrationHtmlContext, IntegrationScriptContext, + IntegrationScriptRewriter, ScriptRewriteAction, }; use crate::streaming_processor::{Compression, PipelineConfig, StreamingPipeline}; use crate::test_support::tests::create_test_settings; @@ -752,6 +764,53 @@ mod tests { } } + #[test] + fn terminal_document_state_observes_parser_callbacks_and_isolates_requests() { + struct ArticleObserver; + impl IntegrationScriptRewriter for ArticleObserver { + fn integration_id(&self) -> &'static str { + "example_article" + } + fn selector(&self) -> &'static str { + "p" + } + fn rewrite( + &self, + content: &str, + context: &IntegrationScriptContext<'_>, + ) -> ScriptRewriteAction { + context + .document_state + .get_or_insert_with("example_article", || content.to_string()); + ScriptRewriteAction::Keep + } + } + for text in ["Example first article", "Example second article"] { + let state = IntegrationDocumentState::default(); + let mut config = create_test_config(); + config.integrations = + IntegrationRegistry::from_rewriters(Vec::new(), vec![Arc::new(ArticleObserver)]); + let mut processor = create_html_processor_with_document_state(config, &state); + let _ = processor + .process_chunk( + format!("

{text}

").as_bytes(), + false, + ) + .expect("should process article HTML"); + let _ = processor + .process_chunk(&[], true) + .expect("should close the synchronous parser"); + let article = state + .get::("example_article") + .expect("should expose parser state to the terminal phase"); + assert_eq!( + article.as_str(), + text, + "should share this document's observed text without leaking another request" + ); + } + } + #[test] fn integration_attribute_rewriter_can_remove_elements() { struct RemovingLinkRewriter; diff --git a/crates/trusted-server-core/src/integrations/html_finalization.rs b/crates/trusted-server-core/src/integrations/html_finalization.rs new file mode 100644 index 000000000..6b2dab09e --- /dev/null +++ b/crates/trusted-server-core/src/integrations/html_finalization.rs @@ -0,0 +1,269 @@ +//! Contracts for optional, bounded asynchronous HTML document finalization. + +use std::time::Duration; + +use error_stack::Report; +use futures::future::LocalBoxFuture; +use web_time::Instant; + +use super::IntegrationHtmlStreamContext; +use crate::platform::RuntimeServices; + +/// Creates a single-use finalizer for each processed HTML document. +/// +/// Factories may be shared across requests; the returned session is request-local. +/// Creation performs synchronous setup only and must not start outside I/O. +pub trait IntegrationHtmlFinalizerFactory: Send + Sync { + /// Return the integration identifier used for controlled diagnostic messages. + fn integration_id(&self) -> &'static str; + + /// Create a session using the parser's shared document state. + fn create(&self, context: IntegrationHtmlStreamContext) -> Box; +} + +/// Finalizes complete, synchronously transformed HTML before auction assembly. +/// +/// Sessions and their futures need not be `Send`. Output is committed atomically. +/// The method returns its future promptly, with I/O inside the future. Replacements +/// preserve document structure, CSP/RSC semantics, and opaque parser seam markers. +pub trait IntegrationHtmlFinalizer { + /// Await optional external work and select unchanged or replacement output. + /// + /// # Errors + /// + /// Return [`HtmlFinalizationError`] when optional work cannot complete. Core + /// discards partial replacement output and retains the input on failure. + /// + /// # Examples + /// + /// ```ignore + /// Box::pin(async move { + /// input.replacement.append(b"

Example article

")?; + /// Ok(HtmlFinalization::Replace) + /// }) + /// ``` + fn finalize<'a>( + &'a mut self, + input: HtmlFinalizationInput<'a>, + ) -> LocalBoxFuture<'a, Result>>; +} + +/// Borrows the current document, separate candidate storage, and request services. +pub struct HtmlFinalizationInput<'a> { + /// Complete transformed HTML; ownership stays with the driver. + pub html: &'a [u8], + /// Separate bounded sink for an atomic replacement candidate. + pub replacement: &'a mut HtmlReplacementWriter, + /// Request-time services, including outbound HTTP and optional KV. + pub services: &'a RuntimeServices, + /// The absolute deadline shared by every finalizer in this document. + pub deadline: &'a HtmlFinalizationDeadline, +} + +/// Selects whether to retain input or commit the bounded replacement candidate. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum HtmlFinalization { + /// Retain the current input without copying it. + Unchanged, + /// Commit bytes appended to [`HtmlReplacementWriter`]. + Replace, +} + +/// Controlled failure categories for optional document finalization. +#[derive(Debug, derive_more::Display)] +pub enum HtmlFinalizationError { + /// The required outside service is unavailable. + #[display("HTML finalization service is unavailable")] + Unavailable, + /// Outside work cannot produce an acceptable result. + #[display("HTML finalization result is rejected")] + Rejected, + /// The bounded candidate cannot accept more bytes. + #[display("HTML replacement exceeds the {limit}-byte limit")] + OutputLimitExceeded { + /// Maximum candidate size in bytes. + limit: usize, + }, +} + +impl core::error::Error for HtmlFinalizationError {} + +/// Separate bounded storage for a finalizer's replacement candidate. +/// +/// A failed append poisons the candidate, preventing partial commitment even +/// when an integration ignores the returned error. +pub struct HtmlReplacementWriter { + bytes: Vec, + limit: usize, + poisoned: bool, +} + +impl HtmlReplacementWriter { + pub(crate) fn new(limit: usize) -> Self { + Self { + bytes: Vec::new(), + limit, + poisoned: false, + } + } + + /// Append bytes without growing the candidate beyond its configured limit. + /// + /// # Errors + /// + /// Returns [`HtmlFinalizationError::OutputLimitExceeded`] when the candidate + /// is poisoned, length arithmetic overflows, or the append exceeds the limit. + /// + /// # Examples + /// + /// ```ignore + /// input.replacement.append(b"

Example article

")?; + /// ``` + pub fn append(&mut self, chunk: &[u8]) -> Result<(), Report> { + let next = self + .bytes + .len() + .checked_add(chunk.len()) + .filter(|length| *length <= self.limit); + if self.poisoned || next.is_none() { + self.poisoned = true; + return Err(Report::new(HtmlFinalizationError::OutputLimitExceeded { + limit: self.limit, + })); + } + let Some(next) = next else { + return Err(Report::new(HtmlFinalizationError::OutputLimitExceeded { + limit: self.limit, + })); + }; + if next > self.bytes.capacity() { + let capacity = self + .bytes + .capacity() + .saturating_mul(2) + .max(next) + .min(self.limit); + self.bytes.reserve_exact(capacity - self.bytes.len()); + } + self.bytes.extend_from_slice(chunk); + Ok(()) + } + + pub(crate) fn is_poisoned(&self) -> bool { + self.poisoned + } + + pub(crate) fn into_bytes(self) -> Vec { + self.bytes + } +} + +/// Absolute finish budget shared by all finalizers for one HTML document. +/// +/// A cooperative timer can interrupt yielding work, but cannot preempt CPU work +/// or blocking host calls. Consumers must bound their own calls on Fastly. +pub struct HtmlFinalizationDeadline { + started_at: Instant, + timeout: Duration, + cooperative_timer: bool, +} + +impl HtmlFinalizationDeadline { + pub(crate) fn new(timeout: Duration, cooperative_timer: bool) -> Self { + Self { + started_at: Instant::now(), + timeout, + cooperative_timer, + } + } + + /// Return the remaining whole-chain budget, saturating at zero. + #[must_use] + pub fn remaining(&self) -> Duration { + self.remaining_at(Instant::now()) + } + + /// Report whether the request has a cooperative timer for yielding work. + #[must_use] + pub fn has_cooperative_timer(&self) -> bool { + self.cooperative_timer + } + + pub(crate) fn remaining_at(&self, now: Instant) -> Duration { + self.timeout + .saturating_sub(now.saturating_duration_since(self.started_at)) + } + + pub(crate) fn is_expired_at(&self, now: Instant) -> bool { + self.remaining_at(now).is_zero() + } +} + +#[cfg(test)] +mod tests { + use std::time::Duration; + + use super::*; + + #[test] + fn replacement_limit_poison_preserves_the_candidate() { + let mut writer = HtmlReplacementWriter::new(4); + writer.append(b"page").expect("should accept the exact cap"); + let _ = writer + .append(b"!") + .expect_err("should reject output beyond the cap"); + assert!(writer.is_poisoned(), "should poison an oversized candidate"); + assert_eq!(writer.bytes, b"page", "should preserve the accepted prefix"); + let _ = writer + .append(b"") + .expect_err("should reject writes after poisoning"); + } + + #[test] + fn replacement_capacity_stays_under_the_limit() { + let mut writer = HtmlReplacementWriter::new(17); + assert_eq!(writer.bytes.capacity(), 0, "should allocate lazily"); + for _ in 0..17 { + writer.append(b"x").expect("should append within the limit"); + assert!(writer.bytes.capacity() <= 17, "should bound reserve growth"); + } + } + + #[test] + fn empty_replacement_is_distinct_from_unchanged() { + let writer = HtmlReplacementWriter::new(4); + assert!( + writer.into_bytes().is_empty(), + "should allow an empty replacement" + ); + assert_ne!( + HtmlFinalization::Unchanged, + HtmlFinalization::Replace, + "should distinguish outcomes" + ); + } + + #[test] + fn deadline_expires_at_the_boundary() { + let deadline = HtmlFinalizationDeadline::new(Duration::from_millis(5), false); + assert_eq!( + deadline.remaining_at(deadline.started_at), + Duration::from_millis(5), + "should start with the complete budget" + ); + let expired = deadline.started_at + Duration::from_millis(5); + assert!( + deadline.is_expired_at(expired), + "should expire at the exact boundary" + ); + assert_eq!( + deadline.remaining_at(expired + Duration::from_millis(1)), + Duration::ZERO, + "should saturate elapsed time" + ); + assert!( + !deadline.has_cooperative_timer(), + "should expose the absence of preemption" + ); + } +} diff --git a/crates/trusted-server-core/src/integrations/mod.rs b/crates/trusted-server-core/src/integrations/mod.rs index 820e2c24b..2419681f3 100644 --- a/crates/trusted-server-core/src/integrations/mod.rs +++ b/crates/trusted-server-core/src/integrations/mod.rs @@ -18,6 +18,7 @@ pub mod didomi; pub mod google_tag_manager; pub mod gpt; pub mod gpt_diagnostics; +mod html_finalization; pub mod js_asset_proxy; pub mod lockr; pub mod nextjs; @@ -28,6 +29,11 @@ mod registry; pub mod sourcepoint; pub mod testlight; +pub use html_finalization::{ + HtmlFinalization, HtmlFinalizationDeadline, HtmlFinalizationError, HtmlFinalizationInput, + HtmlReplacementWriter, IntegrationHtmlFinalizer, IntegrationHtmlFinalizerFactory, +}; + pub use registry::{ AttributeRewriteAction, AttributeRewriteOutcome, HeaderMutation, HeaderMutationMode, IntegrationAttributeContext, IntegrationAttributeRewriter, IntegrationDocumentState, @@ -296,6 +302,11 @@ pub(crate) fn builders() -> &'static [IntegrationBuilder] { id: js_asset_proxy::JS_ASSET_PROXY_INTEGRATION_ID, build: js_asset_proxy::register, }, + #[cfg(feature = "test-utils")] + IntegrationBuilder { + id: crate::test_support::async_html_finalization::ID, + build: crate::test_support::async_html_finalization::register, + }, IntegrationBuilder { id: "testlight", build: testlight::register, diff --git a/crates/trusted-server-core/src/integrations/registry.rs b/crates/trusted-server-core/src/integrations/registry.rs index d3c09cd7d..ab27cad80 100644 --- a/crates/trusted-server-core/src/integrations/registry.rs +++ b/crates/trusted-server-core/src/integrations/registry.rs @@ -8,6 +8,8 @@ use error_stack::Report; use http::{Method, Request, Response}; use matchit::Router; +use super::IntegrationHtmlFinalizerFactory; + use crate::auction::AuctionPlan; use crate::constants::HEADER_X_TS_EC; use crate::ec::EcContext; @@ -627,6 +629,8 @@ pub struct IntegrationRegistration { pub attribute_rewriters: Vec>, pub script_rewriters: Vec>, pub html_stream_processors: Vec>, + /// Terminal finalizers in registration order. + pub html_finalizers: Vec>, pub head_injectors: Vec>, pub request_filters: Vec>, } @@ -653,6 +657,7 @@ impl IntegrationRegistrationBuilder { attribute_rewriters: Vec::new(), script_rewriters: Vec::new(), html_stream_processors: Vec::new(), + html_finalizers: Vec::new(), head_injectors: Vec::new(), request_filters: Vec::new(), }, @@ -689,6 +694,16 @@ impl IntegrationRegistrationBuilder { self } + /// Register an opt-in asynchronous HTML terminal phase. + #[must_use] + pub fn with_html_finalizer( + mut self, + factory: Arc, + ) -> Self { + self.registration.html_finalizers.push(factory); + self + } + #[must_use] pub fn with_head_injector(mut self, injector: Arc) -> Self { self.registration.head_injectors.push(injector); @@ -743,6 +758,7 @@ struct IntegrationRegistryInner { html_rewriters: Vec>, script_rewriters: Vec>, html_stream_processors: Vec>, + html_finalizers: Vec>, head_injectors: Vec>, request_filters: Vec>, } @@ -764,6 +780,7 @@ impl Default for IntegrationRegistryInner { html_rewriters: Vec::new(), script_rewriters: Vec::new(), html_stream_processors: Vec::new(), + html_finalizers: Vec::new(), head_injectors: Vec::new(), request_filters: Vec::new(), } @@ -922,6 +939,7 @@ impl IntegrationRegistry { inner .html_stream_processors .extend(registration.html_stream_processors); + inner.html_finalizers.extend(registration.html_finalizers); inner.head_injectors.extend(registration.head_injectors); inner.request_filters.extend(registration.request_filters); if registration.js_disabled { @@ -1111,6 +1129,31 @@ impl IntegrationRegistry { self.inner.html_stream_processors.clone() } + /// Report finalizer presence before any shared-template lookup. + #[must_use] + pub fn has_html_finalizers(&self) -> bool { + !self.inner.html_finalizers.is_empty() + } + + /// Return terminal factories in registration order. + #[must_use] + pub fn html_finalizer_factories(&self) -> Vec> { + self.inner.html_finalizers.clone() + } + + /// Construct an isolated registry for terminal-phase tests. + #[cfg(any(test, feature = "test-utils"))] + #[must_use] + pub fn from_html_finalizers(factories: Vec>) -> Self { + Self { + inner: Arc::new(IntegrationRegistryInner { + html_finalizers: factories, + ..Default::default() + }), + plan: None, + } + } + /// Collect HTML snippets for insertion at the start of ``. #[must_use] pub fn head_inserts(&self, ctx: &IntegrationHtmlContext<'_>) -> Vec { @@ -1282,6 +1325,7 @@ impl IntegrationRegistry { html_rewriters: attribute_rewriters, script_rewriters, html_stream_processors: Vec::new(), + html_finalizers: Vec::new(), head_injectors: Vec::new(), request_filters: Vec::new(), deferred_js_ids: Vec::new(), @@ -1312,6 +1356,7 @@ impl IntegrationRegistry { html_rewriters: attribute_rewriters, script_rewriters, html_stream_processors: Vec::new(), + html_finalizers: Vec::new(), head_injectors, request_filters: Vec::new(), deferred_js_ids: Vec::new(), @@ -1338,6 +1383,7 @@ impl IntegrationRegistry { html_rewriters: Vec::new(), script_rewriters: Vec::new(), html_stream_processors: Vec::new(), + html_finalizers: Vec::new(), head_injectors: Vec::new(), request_filters, deferred_js_ids: Vec::new(), @@ -1404,6 +1450,7 @@ impl IntegrationRegistry { html_rewriters: Vec::new(), script_rewriters: Vec::new(), html_stream_processors: Vec::new(), + html_finalizers: Vec::new(), head_injectors: Vec::new(), request_filters: Vec::new(), deferred_js_ids: Vec::new(), @@ -1416,6 +1463,24 @@ impl IntegrationRegistry { #[cfg(test)] mod tests { + #[test] + fn html_finalizers_are_empty_by_default() { + let registry = IntegrationRegistry::from_request_filters(Vec::new()); + assert!( + !registry.has_html_finalizers(), + "should retain the ordinary streaming path" + ); + assert!( + registry.html_finalizer_factories().is_empty(), + "should register no default finalizers" + ); + let registration = IntegrationRegistration::builder("example_finalizer").build(); + assert!( + registration.html_finalizers.is_empty(), + "should make finalization opt-in" + ); + } + use super::*; use crate::constants::COOKIE_TS_EC; use crate::platform::test_support::noop_services; diff --git a/crates/trusted-server-core/src/platform/mod.rs b/crates/trusted-server-core/src/platform/mod.rs index 63f5c2a94..99f9f5d9a 100644 --- a/crates/trusted-server-core/src/platform/mod.rs +++ b/crates/trusted-server-core/src/platform/mod.rs @@ -43,6 +43,7 @@ mod template_assembly; mod template_cache; #[cfg(test)] pub(crate) mod test_support; +mod timer; mod traits; mod types; @@ -72,6 +73,7 @@ pub use template_cache::{ TemplateCookieValue, TemplateEntry, TemplateMetadata, TemplateMetadataEncodeError, UnavailableTemplateCache, VaryHeaderValues, VarySpec, reader_url_surrogate_key, }; +pub use timer::PlatformTimer; pub use traits::{PlatformBackend, PlatformConfigStore, PlatformGeo, PlatformSecretStore}; pub use types::{ ClientInfo, GeoInfo, PlatformBackendSpec, RuntimeServices, RuntimeServicesBuilder, StoreId, @@ -83,6 +85,15 @@ pub(crate) const DEFAULT_FIRST_BYTE_TIMEOUT: Duration = Duration::from_secs(15); #[cfg(test)] mod tests { + #[test] + fn runtime_timer_is_optional() { + let services = noop_services(); + assert!( + services.timer().is_none(), + "should allow platforms without cooperative timers" + ); + } + use std::net::{IpAddr, Ipv4Addr}; use std::sync::Arc; use std::time::Duration; @@ -163,6 +174,43 @@ mod tests { ); } + struct MarkerTimer; + impl PlatformTimer for MarkerTimer { + fn delay(&self, _: Duration) -> futures::future::LocalBoxFuture<'static, ()> { + Box::pin(futures::future::ready(())) + } + } + + #[test] + fn runtime_timer_survives_cloning_and_service_replacement() { + let mut services = noop_services(); + services.timer = Some(Arc::new(MarkerTimer)); + let replaced = services + .clone() + .with_kv_store(Arc::new(MarkerKvStore("example"))) + .with_template_cache(Arc::new(UnavailableTemplateCache)) + .with_template_assembler(Arc::new(UnavailableTemplateAssembler)); + assert!( + Arc::ptr_eq( + services + .timer + .as_ref() + .expect("should retain the original timer"), + replaced + .timer + .as_ref() + .expect("should preserve the timer through service replacement") + ), + "should preserve the same timer capability" + ); + futures::executor::block_on( + replaced + .timer() + .expect("should expose the installed timer") + .delay(Duration::ZERO), + ); + } + #[test] fn runtime_services_geo_lookup_returns_none_for_no_ip() { let services = noop_services(); diff --git a/crates/trusted-server-core/src/platform/timer.rs b/crates/trusted-server-core/src/platform/timer.rs new file mode 100644 index 000000000..4bb9834a2 --- /dev/null +++ b/crates/trusted-server-core/src/platform/timer.rs @@ -0,0 +1,12 @@ +//! Cooperative timer capability supplied by supporting adapters. + +use futures::future::LocalBoxFuture; +use std::time::Duration; + +/// Creates local delay futures without requiring a platform executor in core. +/// +/// A delay cannot preempt blocking calls or work that never yields. +pub trait PlatformTimer: Send + Sync { + /// Wait for the duration on the adapter's request executor. + fn delay(&self, duration: Duration) -> LocalBoxFuture<'static, ()>; +} diff --git a/crates/trusted-server-core/src/platform/types.rs b/crates/trusted-server-core/src/platform/types.rs index 3570bdc10..760dba0e5 100644 --- a/crates/trusted-server-core/src/platform/types.rs +++ b/crates/trusted-server-core/src/platform/types.rs @@ -7,7 +7,7 @@ use crate::auction::telemetry::{AuctionTelemetrySink, NoopAuctionTelemetrySink}; use super::{ PlatformBackend, PlatformConfigStore, PlatformGeo, PlatformHttpClient, PlatformKvStore, - PlatformSecretStore, + PlatformSecretStore, PlatformTimer, }; /// Geographic information extracted from a request. @@ -188,6 +188,8 @@ pub struct RuntimeServices { pub(crate) auction_telemetry_sink: Arc, /// Per-request client metadata extracted at the entry point. pub(crate) client_info: ClientInfo, + /// Optional cooperative timer; absent on blocking host runtimes. + pub(crate) timer: Option>, } impl RuntimeServices { @@ -215,6 +217,12 @@ impl RuntimeServices { RuntimeServicesBuilder::new() } + /// Return the request's optional cooperative timer. + #[must_use] + pub fn timer(&self) -> Option<&dyn PlatformTimer> { + self.timer.as_deref() + } + /// Returns the config store service. #[must_use] pub fn config_store(&self) -> &dyn PlatformConfigStore { @@ -340,6 +348,7 @@ pub struct RuntimeServicesBuilder { geo: Option>, auction_telemetry_sink: Option>, client_info: Option, + timer: Option>, } impl RuntimeServicesBuilder { @@ -355,9 +364,17 @@ impl RuntimeServicesBuilder { geo: None, auction_telemetry_sink: None, client_info: None, + timer: None, } } + /// Set the cooperative timer for request-time yielding work. + #[must_use] + pub fn timer(mut self, timer: Arc) -> Self { + self.timer = Some(timer); + self + } + /// Set the config store implementation. #[must_use] pub fn config_store(mut self, config_store: Arc) -> Self { @@ -442,6 +459,7 @@ impl RuntimeServicesBuilder { #[must_use] pub fn build(self) -> RuntimeServices { RuntimeServices { + timer: self.timer, config_store: self .config_store .expect("should set config_store before building RuntimeServices"), diff --git a/crates/trusted-server-core/src/publisher.rs b/crates/trusted-server-core/src/publisher.rs index d8c217076..18389ef1e 100644 --- a/crates/trusted-server-core/src/publisher.rs +++ b/crates/trusted-server-core/src/publisher.rs @@ -19,6 +19,7 @@ //! the streaming route). It is not a content-rewriting concern. use std::borrow::Cow; +mod html_finalization; use std::io::Write; use std::sync::atomic::{AtomicBool, Ordering}; use std::sync::{Arc, Mutex}; @@ -67,9 +68,11 @@ use crate::ec::kv::KvIdentityGraph; use crate::ec::registry::PartnerRegistry; use crate::ec::{EcContext, EidSyncSource}; use crate::error::TrustedServerError; -use crate::html_processor::BodyCloseInjection; +use crate::html_processor::{ + BodyCloseInjection, HtmlProcessorConfig, create_html_processor_with_document_state, +}; use crate::http_util::{RequestInfo, is_navigation_request, serve_static_with_etag}; -use crate::integrations::IntegrationRegistry; +use crate::integrations::{IntegrationDocumentState, IntegrationRegistry}; use crate::platform::{ GeoInfo, PlatformBackendSpec, PlatformHttpRequest, RuntimeServices, TEMPLATE_CACHE_PURGE_ALL_SURROGATE_KEY, VarySpec, contains_publisher_esi_directive, @@ -90,6 +93,7 @@ use crate::streaming_processor::{ STREAM_CHUNK_SIZE, StreamProcessor, StreamingPipeline, }; use crate::streaming_replacer::create_url_replacer; +use html_finalization::HtmlFinalizationDriver; include!(concat!(env!("OUT_DIR"), "/template_build_digest.rs")); @@ -1078,7 +1082,6 @@ async fn hold_step_decoded_chunk( state: &mut AuctionHoldState, collect_refs: &AuctionCollectDeps<'_>, ) -> Result> { - let mut ready = Vec::new(); let processed = match processor .process_chunk(chunk, false) @@ -1091,21 +1094,41 @@ async fn hold_step_decoded_chunk( return Err(err); } }; - let bytes: Cow<'_, [u8]> = match state.hold.as_mut() { - None => Cow::Borrowed(&processed), - Some(seam) => Cow::Owned(seam.push(&processed)), - }; - match encoder.encode_chunk(bytes.into_owned()) { - Ok(encoded) if !encoded.is_empty() => ready.push(bytes::Bytes::from(encoded)), - Ok(_) => {} - Err(err) => { + match encode_processed_html(encoder, processed, Some(state)) { + Ok(step) => Ok(step), + Err(error) => { abandon_hold_auction(state, collect_refs.services, "stream_encode_error").await; - return Err(err.change_context(TrustedServerError::Proxy { - message: "Failed to encode processed chunk".to_string(), - })); + Err(error) } } - let close_found = state.hold.as_ref().is_some_and(InlineBodyCloseSeam::found); +} + +fn encode_processed_html( + encoder: &mut BodyStreamEncoder, + processed: Vec, + state: Option<&mut AuctionHoldState>, +) -> Result> { + let mut close_found = false; + let bytes = if let Some(state) = state { + let bytes = match state.hold.as_mut() { + Some(seam) => seam.push(&processed), + None => processed, + }; + close_found = state.hold.as_ref().is_some_and(InlineBodyCloseSeam::found); + bytes + } else { + processed + }; + let encoded = encoder + .encode_chunk(bytes) + .change_context(TrustedServerError::Proxy { + message: "Failed to encode processed chunk".to_string(), + })?; + let ready = if encoded.is_empty() { + Vec::new() + } else { + vec![bytes::Bytes::from(encoded)] + }; Ok(HoldStepSegments { ready, close_found }) } @@ -1495,8 +1518,13 @@ fn deferred_inline_seam_token( fn create_html_stream_processor( params: HtmlStreamProcessorParams<'_>, ) -> Result, Report> { - use crate::html_processor::{HtmlProcessorConfig, create_html_processor}; + create_html_stream_processor_with_document_state(params, &IntegrationDocumentState::default()) +} +fn create_html_stream_processor_with_document_state( + params: HtmlStreamProcessorParams<'_>, + document_state: &IntegrationDocumentState, +) -> Result, Report> { let config = HtmlProcessorConfig::from_settings( params.settings, params.integration_registry, @@ -1529,7 +1557,10 @@ fn create_html_stream_processor( .with_csp_nonce_observer(csp_nonce_observed) .with_datadome_client_tag_suppression(params.suppress_datadome_client_side_tag); - Ok(create_html_processor(config)) + Ok(create_html_processor_with_document_state( + config, + document_state, + )) } /// Result of publisher request handling, indicating whether the response body @@ -1822,6 +1853,11 @@ pub async fn buffer_publisher_response_async( } return Ok(response); } + if is_html_content_type(¶ms.content_type) + && integration_registry.has_html_finalizers() + { + enforce_synthesized_html_cache_privacy(&mut response); + } let mut output = BoundedWriter::new(settings.publisher.max_buffered_body_bytes); stream_publisher_body_async( body, @@ -2422,14 +2458,17 @@ pub async fn publisher_response_into_streaming_response( &publisher_response, PublisherResponse::Stream { params, .. } if params.template_cache_key.is_some() ) { - return buffer_publisher_response_async( + // This already-buffered cache miss can nest large codec futures. + // Keep its future off the WASI call stack; ordinary inline streaming + // never takes this branch. + return Box::pin(buffer_publisher_response_async( publisher_response, method, &settings, integration_registry, &orchestrator, &services, - ) + )) .await; } @@ -2568,6 +2607,35 @@ pub async fn publisher_response_into_streaming_response( response.headers_mut().remove(header::CONTENT_LENGTH); let mut params = *params; + if is_html_content_type(¶ms.content_type) + && integration_registry.has_html_finalizers() + { + enforce_synthesized_html_cache_privacy(&mut response); + let mut driver = Box::pin(build_html_terminal_driver( + body, + &mut params, + &settings, + integration_registry, + &services, + )) + .await?; + let stream = async_stream::try_stream! { + let deps = AuctionCollectDeps { + price_granularity: params.price_granularity, ad_bids_state: ¶ms.ad_bids_state, + orchestrator: &orchestrator, services: &services, settings: &settings, + request_origin: request_origin(¶ms.request_scheme, ¶ms.request_host), + }; + while let Some(step) = driver.next_step(&deps).await.map_err(publisher_stream_error)? { + for segment in step.ready { yield segment; } + if step.close_found { + for segment in driver.collect_close(&deps).await.map_err(publisher_stream_error)? { yield segment; } + } + } + }; + *response.body_mut() = EdgeBody::from_stream::<_, std::io::Error>(stream); + return Ok(response); + } + let (mut processor, inline_seam_token) = match PublisherBodyProcessor::new(¶ms, &settings, integration_registry) { Ok(built) => built, @@ -2891,6 +2959,12 @@ impl Write for BoundedWriter { /// I/O happened upstream in [`handle_publisher_request`]. Writes processed /// chunks directly to `output`. /// +/// This helper performs synchronous transformations only. Registered HTML +/// finalizers require [`stream_publisher_body_async`], +/// [`buffer_publisher_response_async`], or +/// [`publisher_response_into_streaming_response`]. Adapters use those async +/// entry points when terminal integrations are enabled. +/// /// # Errors /// /// Returns an error if processing fails mid-stream. Since headers are already @@ -2928,6 +3002,87 @@ pub fn stream_publisher_body( process_response_streaming(body, output, &borrowed, output_compression) } +async fn build_html_terminal_driver( + body: EdgeBody, + params: &mut OwnedProcessResponseParams, + settings: &Settings, + registry: &IntegrationRegistry, + services: &RuntimeServices, +) -> Result, Report> { + match HtmlFinalizationDriver::new(body, params, settings, registry) { + Ok(driver) => Ok(Box::new(driver)), + Err(error) => { + if let Some(dispatched) = params.dispatched_auction.take() { + emit_abandoned_auction( + services, + params.auction_observation.take(), + dispatched, + "processor_init_error", + ) + .await; + } + Err(error) + } + } +} + +async fn stream_html_terminal( + body: EdgeBody, + output: &mut W, + params: &mut OwnedProcessResponseParams, + settings: &Settings, + registry: &IntegrationRegistry, + orchestrator: &AuctionOrchestrator, + services: &RuntimeServices, +) -> Result<(), Report> { + let mut driver = build_html_terminal_driver(body, params, settings, registry, services).await?; + let deps = AuctionCollectDeps { + price_granularity: params.price_granularity, + ad_bids_state: ¶ms.ad_bids_state, + orchestrator, + services, + settings, + request_origin: request_origin(¶ms.request_scheme, ¶ms.request_host), + }; + while let Some(step) = driver.next_step(&deps).await? { + for segment in step.ready { + if let Err(error) = output.write_all(&segment) { + driver.abandon(&deps, "stream_write_error").await; + return Err( + Report::new(error).change_context(TrustedServerError::Proxy { + message: "Failed to write finalized HTML".to_string(), + }), + ); + } + } + if (step.close_found || driver.input_finished()) + && let Err(error) = output.flush() + { + driver.abandon(&deps, "stream_write_error").await; + return Err( + Report::new(error).change_context(TrustedServerError::Proxy { + message: "Failed to flush finalized HTML".to_string(), + }), + ); + } + if step.close_found { + for segment in driver.collect_close(&deps).await? { + if let Err(error) = output.write_all(&segment) { + driver.abandon(&deps, "stream_write_error").await; + return Err( + Report::new(error).change_context(TrustedServerError::Proxy { + message: "Failed to write HTML auction tail".to_string(), + }), + ); + } + } + } + } + output.flush().change_context(TrustedServerError::Proxy { + message: "Failed to flush finalized HTML".to_string(), + }) +} + /// Stream publisher body with parser-confirmed live bid injection. /// /// Drives the origin body through the HTML pipeline one chunk at a time. The @@ -2957,6 +3112,18 @@ pub async fn stream_publisher_body_async( orchestrator: &AuctionOrchestrator, services: &RuntimeServices, ) -> Result<(), Report> { + if is_html_content_type(¶ms.content_type) && integration_registry.has_html_finalizers() { + return Box::pin(stream_html_terminal( + body, + output, + params, + settings, + integration_registry, + orchestrator, + services, + )) + .await; + } let Some(dispatched) = params.dispatched_auction.take() else { if body.is_stream() { return process_response_streaming_async( @@ -4310,6 +4477,11 @@ pub struct AuctionDispatch<'a> { pub slots: &'a [crate::creative_opportunities::CreativeOpportunitySlot], /// Partner registry for KV-backed EID resolution. `None` skips KV enrichment. pub registry: Option<&'a PartnerRegistry>, + /// Enabled integration registry used before origin and shared-cache decisions. + /// + /// Pass the same registry to response finalization so request admission and + /// document processing use the same enabled integration set. + pub integration_registry: &'a IntegrationRegistry, } /// Request-side conditions that decide whether this request's origin response may be @@ -4438,6 +4610,7 @@ pub async fn handle_publisher_request( edge_header: EdgeCacheHeader, ) -> Result> { log::debug!("Proxying request to publisher_origin"); + let html_finalizers_present = auction.integration_registry.has_html_finalizers(); // Adapter fallbacks prepare this before EC/cookie handling. Keep this // idempotent call as a direct-handler safety net and for focused tests. @@ -4679,7 +4852,12 @@ pub async fn handle_publisher_request( // a panic-prone invariant in the public request handler. let reader_compression = reader_compression.unwrap_or(Compression::None); - if should_run_ad_stack || datadome_suppression_requires_full_body { + if should_run_ad_stack + || datadome_suppression_requires_full_body + || (html_finalizers_present + && req.method() == Method::GET + && is_html_document_request(&req)) + { // HTML document contexts whose output may be synthesized must not // receive a cached 304 or partial 206. Non-document subresources contain // no executable injected tag, so retain their validators and ranges. @@ -4726,12 +4904,13 @@ pub async fn handle_publisher_request( .creative_opportunities .as_ref() .is_some_and(CreativeOpportunitiesConfig::origin_readthrough_enabled); - let request_can_use_shared_template = request_can_use_shared_template( - shared_request_inputs, - matches!(assembly_mode, AssemblyMode::Esi), - reader_supports_assembly, - reader_requires_origin, - ); + let request_can_use_shared_template = !html_finalizers_present + && request_can_use_shared_template( + shared_request_inputs, + matches!(assembly_mode, AssemblyMode::Esi), + reader_supports_assembly, + reader_requires_origin, + ); // Only advertise encodings the rewrite pipeline can decode and re-encode. This // remains unconditional when template cache negotiation fails: that request bypasses shared @@ -5387,6 +5566,13 @@ pub async fn handle_publisher_request( .trim() .to_ascii_lowercase(); let route = classify_response_route(status, &content_type, &content_encoding, request_host); + if html_finalizers_present + && route == ResponseRoute::Stream + && is_html_content_type(&content_type) + && response_carries_body(&request_method, status) + { + enforce_synthesized_html_cache_privacy(&mut response); + } if template_cache_key.is_some() { set_response_compression(&mut response, reader_compression); } @@ -7756,6 +7942,7 @@ mod tests { Some(&graph), &mut ec_context, AuctionDispatch { + integration_registry: &IntegrationRegistry::empty_for_tests(), orchestrator: &orchestrator, slots: &slots, registry: None, @@ -7966,6 +8153,7 @@ mod tests { Some(&graph), &mut ec_context, AuctionDispatch { + integration_registry: &IntegrationRegistry::empty_for_tests(), orchestrator: &orchestrator, slots: &slots, registry: Some(®istry), @@ -9209,6 +9397,7 @@ mod tests { None, &mut ec_context, AuctionDispatch { + integration_registry: &IntegrationRegistry::empty_for_tests(), orchestrator: &orchestrator, slots: &[], registry: None, @@ -10485,138 +10674,480 @@ mod tests { .await } - async fn run_with_orchestrator( - settings: &Arc, - services: &RuntimeServices, - request: Request, - orchestrator: AuctionOrchestrator, - finalizer: Finalizer, - ) -> Response { + #[tokio::test] + async fn html_finalization_preserves_a_winning_bid_before_the_structural_body_end() { + let settings = Arc::new(settings_with_bidder("inline")); + let stub = Arc::new(StubHttpClient::new()); + let cache = Arc::new(MemoryTemplateCache::default()); + let services = services(Arc::clone(&stub), cache); + let registry = IntegrationRegistry::from_html_finalizers(vec![Arc::new( + super::TerminalTestFactory, + )]); + let mut orchestrator = AuctionOrchestrator::new(settings.auction.clone()); + orchestrator.register_provider(Arc::new(WinningBidProvider { + price: STUB_BID_CPM, + })); let orchestrator = Arc::new(orchestrator); - let registry = - IntegrationRegistry::new(settings).expect("should create integration registry"); + queue_bid_response(&stub); + queue_shareable_html(&stub); let consent = crate::consent::ConsentContext { jurisdiction: crate::consent::jurisdiction::Jurisdiction::NonRegulated, ..Default::default() }; let mut ec_context = EcContext::new_for_test(None, consent); - let publisher_response = handle_publisher_request( - settings, - services, + let response = handle_publisher_request( + &settings, + &services, None, &mut ec_context, AuctionDispatch { + integration_registry: ®istry, orchestrator: &orchestrator, slots: &[article_slot()], registry: None, }, - request, + navigation_request(), EdgeCacheHeader::SMaxageFallback, ) .await - .expect("should proxy publisher request"); - - finalize_test_publisher_response( - publisher_response, + .expect("should launch the winning auction"); + let response = publisher_response_into_streaming_response( + response, + &Method::GET, settings, - services, ®istry, orchestrator, - finalizer, + services, ) .await + .expect("should retain terminal and auction phases"); + let html = + String::from_utf8(body_of(response).await).expect("should emit example UTF-8 HTML"); + let bids = seam_bids(&html); + assert_eq!( + bids.get("test-slot") + .and_then(|bid| bid.get("hb_pb")) + .and_then(serde_json::Value::as_str), + Some("3.50"), + "should retain the winning bid after terminal I/O" + ); + assert!(html.contains("origin"), "should preserve the article bytes"); + assert!( + html.ends_with(""), + "should commit the terminal output" + ); + assert_eq!( + html.matches("var b=JSON.parse(").count(), + 1, + "should inject bids once" + ); + assert!( + html.find("var b=JSON.parse(") + .expect("should find the bids") + < html + .find("") + .expect("should retain the structural body end"), + "should assemble bids before the parsed closing tag" + ); + assert!( + !html.contains("ts-inline-body-close-"), + "should consume the parser seam token" + ); } - async fn finalize_test_publisher_response( - publisher_response: PublisherResponse, - settings: &Arc, - services: &RuntimeServices, - registry: &IntegrationRegistry, - orchestrator: Arc, - finalizer: Finalizer, - ) -> Response { - match finalizer { - Finalizer::Streaming => publisher_response_into_streaming_response( - publisher_response, - &Method::GET, - Arc::clone(settings), - registry, - orchestrator, - services.clone(), - ) - .await - .expect("should finalize publisher response"), - Finalizer::Buffered => buffer_publisher_response_async( - publisher_response, - &Method::GET, - settings, - registry, - &orchestrator, - services, - ) - .await - .expect("should finalize publisher response"), - } + #[derive(Default)] + struct RawOriginCacheClient { + cached: Mutex>>, + misses: AtomicUsize, + hits: AtomicUsize, } - /// Collects a response body whether it was buffered or streamed. - /// - /// Both occur here by design: a response the gate authorized is buffered so it - /// can be stored, and one it refused keeps streaming exactly like `inline`. A - /// helper that assumed either would silently only ever test one of them. - async fn body_of(response: Response) -> Vec { - match response.into_body() { - EdgeBody::Stream(mut stream) => { - let mut out = Vec::new(); - while let Some(chunk) = stream.next().await { - out.extend_from_slice(&chunk.expect("stream chunk should read")); - } - out - } - other => other - .into_bytes() - .expect("a non-stream body has its bytes in hand") - .to_vec(), + #[async_trait::async_trait(?Send)] + impl crate::platform::PlatformHttpClient for RawOriginCacheClient { + async fn send( + &self, + request: crate::platform::PlatformHttpRequest, + ) -> Result> + { + assert!( + matches!( + request.cache_intent, + crate::platform::PlatformCacheIntent::Shared { .. } + ), + "should retain raw-origin cache eligibility" + ); + let mut cached = self.cached.lock().expect("should lock raw origin cache"); + let bytes = if let Some(bytes) = cached.as_ref() { + self.hits.fetch_add(1, Ordering::SeqCst); + bytes.clone() + } else { + self.misses.fetch_add(1, Ordering::SeqCst); + let bytes = + b"example raw article".to_vec(); + *cached = Some(bytes.clone()); + bytes + }; + Ok(crate::platform::PlatformResponse::new( + Response::builder() + .header(header::CONTENT_TYPE, "text/html") + .header(header::CACHE_CONTROL, "public, max-age=60") + .body(EdgeBody::from(bytes)) + .expect("should construct raw origin response"), + )) } - } - - #[tokio::test] - async fn harness_emits_a_summary_row_for_an_ad_serving_navigation() { - let stub = Arc::new(StubHttpClient::new()); - let sink = Arc::new(RecordingTelemetrySink::default()); - let services = services_with_cache_and_telemetry( - Arc::clone(&stub), - Arc::new(MemoryTemplateCache::default()), - Arc::clone(&sink), - ); - let settings = Arc::new(settings_with_mode("esi")); - queue_shareable_html(&stub); - let _ = run(&settings, &services, navigation_request()).await; + async fn send_async( + &self, + _: crate::platform::PlatformHttpRequest, + ) -> Result< + crate::platform::PlatformPendingRequest, + Report, + > { + panic!("should not dispatch an auction in the raw-origin cache fixture") + } - assert!( - last_summary_row(&sink).is_some(), - "the harness must emit a summary row, or every assertion built on it is vacuous" - ); + async fn select( + &self, + _: Vec, + ) -> Result> + { + panic!("should not collect an auction in the raw-origin cache fixture") + } } - /// Drive `handle_publisher_request` through the **EC-preload** fetch path. - /// - /// `run_with_orchestrator` passes `kv: None` and no EC id, so - /// `should_preload_ec_snapshot` is false for every other test in this file and the - /// first of the two origin-fetch call sites is never exercised. Supplying both - /// reaches it. - async fn run_through_ec_preload( - settings: &Arc, - services: &RuntimeServices, - request: Request, - ) { + #[tokio::test] + async fn html_finalization_runs_again_on_a_raw_origin_cache_hit() { + let client = Arc::new(RawOriginCacheClient::default()); + let template_cache = Arc::new(MemoryTemplateCache::default()); + let mut settings = settings_with_readthrough_enabled("esi"); + settings.auction.enabled = false; + let settings = Arc::new(settings); + let services = RuntimeServices::builder() + .config_store(Arc::new(NoopConfigStore)) + .secret_store(Arc::new(NoopSecretStore)) + .kv_store(Arc::new(edgezero_core::key_value_store::NoopKvStore)) + .backend(Arc::new(StubBackend)) + .http_client(client.clone()) + .geo(Arc::new(NoopGeo)) + .client_info(ClientInfo::default()) + .template_cache(template_cache.clone()) + .build(); + let registry = IntegrationRegistry::from_html_finalizers(vec![Arc::new( + super::TerminalTestFactory, + )]); let orchestrator = Arc::new(AuctionOrchestrator::new(settings.auction.clone())); - let kv = crate::ec::kv::KvIdentityGraph::in_memory("test-store"); - let consent = crate::consent::ConsentContext { - jurisdiction: crate::consent::jurisdiction::Jurisdiction::NonRegulated, - ..Default::default() - }; + for _ in 0..2 { + let response = handle_publisher_request( + &settings, + &services, + None, + &mut EcContext::default(), + AuctionDispatch { + integration_registry: ®istry, + orchestrator: &orchestrator, + slots: &[], + registry: None, + }, + navigation_request(), + EdgeCacheHeader::SMaxageFallback, + ) + .await + .expect("should obtain raw origin HTML"); + let response = publisher_response_into_streaming_response( + response, + &Method::GET, + Arc::clone(&settings), + ®istry, + Arc::clone(&orchestrator), + services.clone(), + ) + .await + .expect("should finalize cached or uncached raw HTML"); + assert_eq!( + header_of(&response, header::CACHE_CONTROL), + Some(crate::cache_policy::NO_STORE_PRIVATE_CACHE_CONTROL), + "should keep both synthesized responses private" + ); + let body = body_of(response).await; + assert_eq!( + body.windows(b"".len()) + .filter(|bytes| *bytes == b"") + .count(), + 1, + "should run finalization once for each raw response" + ); + } + assert_eq!( + client.misses.load(Ordering::SeqCst), + 1, + "should fetch origin once" + ); + assert_eq!( + client.hits.load(Ordering::SeqCst), + 1, + "should serve a raw cache hit" + ); + assert!( + !client + .cached + .lock() + .expect("should lock raw origin cache") + .as_ref() + .expect("should retain raw cached HTML") + .windows(b"example-finalized".len()) + .any(|bytes| bytes == b"example-finalized"), + "should cache raw bytes without terminal replacements" + ); + assert!( + template_cache + .lookups + .lock() + .expect("should lock template lookups") + .is_empty(), + "should skip template lookup on both requests" + ); + assert!( + template_cache + .stored_keys + .lock() + .expect("should lock template stores") + .is_empty(), + "should not store finalized HTML as a shared template" + ); + } + + #[tokio::test] + async fn html_finalizers_bypass_warm_and_cold_templates_but_keep_origin_readthrough() { + let stub = Arc::new(StubHttpClient::new()); + let cache = Arc::new(MemoryTemplateCache::default()); + let settings = Arc::new(settings_with_readthrough_enabled("esi")); + let services = services(Arc::clone(&stub), Arc::clone(&cache)); + queue_shareable_html(&stub); + let _ = body_of(run(&settings, &services, navigation_request()).await).await; + let lookups_before = cache.lookups.lock().expect("should lock lookups").len(); + let stores_before = cache + .stored_keys + .lock() + .expect("should lock stored keys") + .len(); + assert!(stores_before > 0, "should seed a warm shared template"); + let registry = IntegrationRegistry::from_html_finalizers(vec![Arc::new( + super::TerminalTestFactory, + )]); + let orchestrator = Arc::new(AuctionOrchestrator::new(settings.auction.clone())); + for path in ["/article", "/article-cold"] { + queue_shareable_html(&stub); + let mut request = navigation_request(); + *request.uri_mut() = format!("https://ts.example.com{path}") + .parse() + .expect("should parse an example URL"); + let mut ec_context = EcContext::default(); + let response = handle_publisher_request( + &settings, + &services, + None, + &mut ec_context, + AuctionDispatch { + integration_registry: ®istry, + orchestrator: &orchestrator, + slots: &[], + registry: None, + }, + request, + EdgeCacheHeader::SMaxageFallback, + ) + .await + .expect("should request raw origin HTML"); + let PublisherResponse::Stream { params, .. } = &response else { + panic!("should bypass shared templates"); + }; + assert!( + params.template_cache_key.is_none(), + "should carry no shared-store authorization" + ); + let response = publisher_response_into_streaming_response( + response, + &Method::GET, + Arc::clone(&settings), + ®istry, + Arc::clone(&orchestrator), + services.clone(), + ) + .await + .expect("should finalize origin HTML"); + assert_eq!( + header_of(&response, header::CACHE_CONTROL), + Some(crate::cache_policy::NO_STORE_PRIVATE_CACHE_CONTROL), + "should make terminal output private" + ); + assert!( + body_of(response) + .await + .ends_with(b""), + "should finish both warm and cold documents" + ); + } + assert_eq!( + cache.lookups.lock().expect("should lock lookups").len(), + lookups_before, + "should skip lookup and reservation entirely" + ); + assert_eq!( + cache + .stored_keys + .lock() + .expect("should lock stored keys") + .len(), + stores_before, + "should never store terminal output" + ); + assert!( + stub.recorded_cache_intents().iter().all(|intent| matches!( + intent, + crate::platform::PlatformCacheIntent::Shared { .. } + )), + "should retain configured raw-origin readthrough" + ); + let _ = body_of(run(&settings, &services, navigation_request()).await).await; + assert!( + cache.lookups.lock().expect("should lock lookups").len() > lookups_before, + "should restore cache eligibility when finalizers are disabled" + ); + } + + async fn run_with_orchestrator( + settings: &Arc, + services: &RuntimeServices, + request: Request, + orchestrator: AuctionOrchestrator, + finalizer: Finalizer, + ) -> Response { + let orchestrator = Arc::new(orchestrator); + let registry = + IntegrationRegistry::new(settings).expect("should create integration registry"); + let consent = crate::consent::ConsentContext { + jurisdiction: crate::consent::jurisdiction::Jurisdiction::NonRegulated, + ..Default::default() + }; + let mut ec_context = EcContext::new_for_test(None, consent); + let publisher_response = handle_publisher_request( + settings, + services, + None, + &mut ec_context, + AuctionDispatch { + integration_registry: ®istry, + orchestrator: &orchestrator, + slots: &[article_slot()], + registry: None, + }, + request, + EdgeCacheHeader::SMaxageFallback, + ) + .await + .expect("should proxy publisher request"); + + finalize_test_publisher_response( + publisher_response, + settings, + services, + ®istry, + orchestrator, + finalizer, + ) + .await + } + + async fn finalize_test_publisher_response( + publisher_response: PublisherResponse, + settings: &Arc, + services: &RuntimeServices, + registry: &IntegrationRegistry, + orchestrator: Arc, + finalizer: Finalizer, + ) -> Response { + match finalizer { + Finalizer::Streaming => publisher_response_into_streaming_response( + publisher_response, + &Method::GET, + Arc::clone(settings), + registry, + orchestrator, + services.clone(), + ) + .await + .expect("should finalize publisher response"), + Finalizer::Buffered => buffer_publisher_response_async( + publisher_response, + &Method::GET, + settings, + registry, + &orchestrator, + services, + ) + .await + .expect("should finalize publisher response"), + } + } + + /// Collects a response body whether it was buffered or streamed. + /// + /// Both occur here by design: a response the gate authorized is buffered so it + /// can be stored, and one it refused keeps streaming exactly like `inline`. A + /// helper that assumed either would silently only ever test one of them. + async fn body_of(response: Response) -> Vec { + match response.into_body() { + EdgeBody::Stream(mut stream) => { + let mut out = Vec::new(); + while let Some(chunk) = stream.next().await { + out.extend_from_slice(&chunk.expect("stream chunk should read")); + } + out + } + other => other + .into_bytes() + .expect("a non-stream body has its bytes in hand") + .to_vec(), + } + } + + #[tokio::test] + async fn harness_emits_a_summary_row_for_an_ad_serving_navigation() { + let stub = Arc::new(StubHttpClient::new()); + let sink = Arc::new(RecordingTelemetrySink::default()); + let services = services_with_cache_and_telemetry( + Arc::clone(&stub), + Arc::new(MemoryTemplateCache::default()), + Arc::clone(&sink), + ); + let settings = Arc::new(settings_with_mode("esi")); + queue_shareable_html(&stub); + + let _ = run(&settings, &services, navigation_request()).await; + + assert!( + last_summary_row(&sink).is_some(), + "the harness must emit a summary row, or every assertion built on it is vacuous" + ); + } + + /// Drive `handle_publisher_request` through the **EC-preload** fetch path. + /// + /// `run_with_orchestrator` passes `kv: None` and no EC id, so + /// `should_preload_ec_snapshot` is false for every other test in this file and the + /// first of the two origin-fetch call sites is never exercised. Supplying both + /// reaches it. + async fn run_through_ec_preload( + settings: &Arc, + services: &RuntimeServices, + request: Request, + ) { + let orchestrator = Arc::new(AuctionOrchestrator::new(settings.auction.clone())); + let kv = crate::ec::kv::KvIdentityGraph::in_memory("test-store"); + let consent = crate::consent::ConsentContext { + jurisdiction: crate::consent::jurisdiction::Jurisdiction::NonRegulated, + ..Default::default() + }; let mut ec_context = EcContext::new_for_test(Some("test-ec-id".to_owned()), consent); let _ = handle_publisher_request( @@ -10625,6 +11156,7 @@ mod tests { Some(&kv), &mut ec_context, AuctionDispatch { + integration_registry: &IntegrationRegistry::empty_for_tests(), orchestrator: &orchestrator, slots: &[article_slot()], registry: None, @@ -11390,6 +11922,7 @@ mod tests { Some(&graph), &mut ec_context, AuctionDispatch { + integration_registry: &IntegrationRegistry::empty_for_tests(), orchestrator: &orchestrator, slots: &[article_slot()], registry: None, @@ -12704,6 +13237,7 @@ mod tests { None, &mut ec_context, AuctionDispatch { + integration_registry: &IntegrationRegistry::empty_for_tests(), orchestrator: &orchestrator, slots: &[article_slot()], registry: None, @@ -12872,6 +13406,7 @@ mod tests { None, &mut ec_context, AuctionDispatch { + integration_registry: &IntegrationRegistry::empty_for_tests(), orchestrator: &orchestrator, slots: &[article_slot()], registry: None, @@ -13694,6 +14229,7 @@ mod tests { None, &mut ec_context, AuctionDispatch { + integration_registry: &IntegrationRegistry::empty_for_tests(), orchestrator: &orchestrator, slots: &[article_slot()], registry: None, @@ -14095,6 +14631,7 @@ mod tests { Some(&graph), &mut ec_context, AuctionDispatch { + integration_registry: &IntegrationRegistry::empty_for_tests(), orchestrator: &orchestrator, slots: &[article_slot()], registry: Some(&partners), @@ -15074,6 +15611,7 @@ mod tests { None, &mut ec_context, AuctionDispatch { + integration_registry: &IntegrationRegistry::empty_for_tests(), orchestrator: &orchestrator, slots: &[article_slot()], registry: None, @@ -16625,6 +17163,7 @@ mod tests { None, &mut ec_context, AuctionDispatch { + integration_registry: &IntegrationRegistry::empty_for_tests(), orchestrator, slots, registry: None, @@ -16686,6 +17225,7 @@ mod tests { Some(&graph), &mut ec_context, AuctionDispatch { + integration_registry: &IntegrationRegistry::empty_for_tests(), orchestrator: &orchestrator, slots: &slots, registry: None, @@ -16757,6 +17297,7 @@ mod tests { Some(&graph), &mut ec_context, AuctionDispatch { + integration_registry: &IntegrationRegistry::empty_for_tests(), orchestrator: &orchestrator, slots: &slots, registry: None, @@ -17913,6 +18454,7 @@ mod tests { None, &mut ec_context, AuctionDispatch { + integration_registry: &IntegrationRegistry::empty_for_tests(), orchestrator: &orchestrator, slots: &[], registry: None, @@ -20583,6 +21125,730 @@ mod tests { } } + #[test] + fn html_finalization_applies_privacy_and_full_body_policy_without_an_auction() { + let settings = create_test_settings(); + let registry = + IntegrationRegistry::from_html_finalizers(vec![Arc::new(TerminalTestFactory)]); + let http = Arc::new(StubHttpClient::new()); + http.push_response_with_headers( + 404, + b"Example missing article".to_vec(), + vec![ + ("content-type", "text/html"), + ("cache-control", "public, max-age=60"), + ("etag", "example-tag"), + ], + ); + let services = build_services_with_http_client( + Arc::clone(&http) as Arc + ); + let orchestrator = AuctionOrchestrator::new(settings.auction.clone()); + let request = Request::builder() + .method(Method::GET) + .uri("https://proxy.example.com/article") + .header(header::HOST, "proxy.example.com") + .header(header::ACCEPT, "text/html") + .header(header::IF_NONE_MATCH, "example-tag") + .header(header::IF_MODIFIED_SINCE, "Mon, 01 Jan 2024 00:00:00 GMT") + .header(header::RANGE, "bytes=0-10") + .header(header::IF_RANGE, "example-tag") + .header(header::IF_MATCH, "example-precondition") + .body(EdgeBody::empty()) + .expect("should build a navigation"); + let result = futures::executor::block_on(handle_publisher_request( + &settings, + &services, + None, + &mut EcContext::default(), + AuctionDispatch { + orchestrator: &orchestrator, + slots: &[], + registry: None, + integration_registry: ®istry, + }, + request, + EdgeCacheHeader::SurrogateControl, + )) + .expect("should fetch the origin"); + let PublisherResponse::Stream { response, .. } = result else { + panic!("should select processable HTML"); + }; + assert_eq!( + response.status(), + StatusCode::NOT_FOUND, + "should preserve non-2xx status" + ); + assert_eq!( + response + .headers() + .get(header::CACHE_CONTROL) + .and_then(|value| value.to_str().ok()), + Some(crate::cache_policy::NO_STORE_PRIVATE_CACHE_CONTROL), + "should make finalized HTML private even without ads" + ); + assert!( + !response.headers().contains_key(header::ETAG), + "should strip origin validators" + ); + let requests = http.recorded_request_headers(); + let forwarded = &requests[0]; + for name in ["if-none-match", "if-modified-since", "range", "if-range"] { + assert!( + !forwarded.iter().any(|(header, _)| header == name), + "should request complete HTML" + ); + } + assert!( + forwarded.iter().any(|(header, _)| header == "if-match"), + "should preserve mutation preconditions" + ); + } + + struct CountingTerminalFactory(Arc); + struct CountingTerminalSession(Arc); + + impl crate::integrations::IntegrationHtmlFinalizerFactory for CountingTerminalFactory { + fn integration_id(&self) -> &'static str { + "example_counter" + } + fn create( + &self, + _: crate::integrations::IntegrationHtmlStreamContext, + ) -> Box { + Box::new(CountingTerminalSession(Arc::clone(&self.0))) + } + } + impl crate::integrations::IntegrationHtmlFinalizer for CountingTerminalSession { + fn finalize<'a>( + &'a mut self, + input: crate::integrations::HtmlFinalizationInput<'a>, + ) -> futures::future::LocalBoxFuture< + 'a, + Result< + crate::integrations::HtmlFinalization, + Report, + >, + > { + Box::pin(async move { + self.0.fetch_add(1, Ordering::SeqCst); + input.replacement.append(b"discarded candidate")?; + Ok(crate::integrations::HtmlFinalization::Unchanged) + }) + } + } + + #[test] + fn html_finalization_handles_eligible_and_excluded_responses() { + for (method, status, content_type, encoding) in [ + (Method::GET, 200, "text/html", ""), + (Method::HEAD, 200, "text/html", ""), + (Method::GET, 204, "text/html", ""), + (Method::GET, 205, "text/html", ""), + (Method::GET, 304, "text/html", ""), + (Method::GET, 200, "text/plain", ""), + (Method::GET, 200, "text/html", "example-unsupported"), + ] { + let settings = Arc::new(create_test_settings()); + let calls = Arc::new(AtomicUsize::new(0)); + let registry = IntegrationRegistry::from_html_finalizers(vec![Arc::new( + CountingTerminalFactory(Arc::clone(&calls)), + )]); + let http = Arc::new(StubHttpClient::new()); + http.push_response_with_headers( + status, + b"Example body".to_vec(), + vec![ + ("content-type", content_type), + ("content-encoding", encoding), + ], + ); + let services = build_services_with_http_client(http); + let orchestrator = Arc::new(AuctionOrchestrator::new(settings.auction.clone())); + let request = Request::builder() + .method(method.clone()) + .uri("https://proxy.example.com/article") + .header(header::HOST, "proxy.example.com") + .header(header::ACCEPT, "text/html") + .body(EdgeBody::empty()) + .expect("should build a fixture request"); + let response = futures::executor::block_on(handle_publisher_request( + &settings, + &services, + None, + &mut EcContext::default(), + AuctionDispatch { + integration_registry: ®istry, + orchestrator: &orchestrator, + slots: &[], + registry: None, + }, + request, + EdgeCacheHeader::SurrogateControl, + )) + .expect("should classify the origin response"); + let response = futures::executor::block_on(publisher_response_into_streaming_response( + response, + &method, + Arc::clone(&settings), + ®istry, + orchestrator, + services, + )) + .expect("should preserve excluded response behavior"); + let _ = + futures::executor::block_on(response.into_body().into_bytes_bounded(1024 * 1024)) + .expect("should drain excluded output"); + let eligible = method == Method::GET + && status == 200 + && content_type == "text/html" + && encoding.is_empty(); + assert_eq!( + calls.load(Ordering::SeqCst), + usize::from(eligible), + "should invoke only eligible callbacks: method={method}, status={status}, type={content_type}, encoding={encoding}" + ); + } + } + + #[test] + fn html_finalization_collection_overflow_replays_rewritten_expansion_and_stays_private() { + let mut settings = create_test_settings(); + settings.publisher.max_buffered_body_bytes = 128; + let settings = Arc::new(settings); + let calls = Arc::new(AtomicUsize::new(0)); + let registry = IntegrationRegistry::from_html_finalizers(vec![Arc::new( + CountingTerminalFactory(Arc::clone(&calls)), + )]); + let source = b"Example"; + let response = PublisherResponse::Stream { + response: Response::builder() + .status(200) + .header(header::CONTENT_TYPE, "text/html") + .body(EdgeBody::empty()) + .expect("should build overflow output"), + body: EdgeBody::stream(futures::stream::iter( + source + .chunks(7) + .map(bytes::Bytes::copy_from_slice) + .collect::>(), + )), + params: Box::new(html_stream_params("", None)), + }; + let orchestrator = Arc::new(AuctionOrchestrator::new(settings.auction.clone())); + let response = futures::executor::block_on(publisher_response_into_streaming_response( + response, + &Method::GET, + Arc::clone(&settings), + ®istry, + orchestrator, + noop_services(), + )) + .expect("should create lazy overflow output"); + assert_eq!( + response + .headers() + .get(header::CACHE_CONTROL) + .expect("should retain privacy"), + crate::cache_policy::NO_STORE_PRIVATE_CACHE_CONTROL, + "should prevent caching even when collection bypasses I/O" + ); + let body = + futures::executor::block_on(response.into_body().into_bytes_bounded(1024 * 1024)) + .expect("should replay overflow output"); + let mut processor = create_html_stream_processor(HtmlStreamProcessorParams { + origin_host: "origin.example.com", + request_host: "proxy.example.com", + request_scheme: "https", + settings: &settings, + integration_registry: ®istry, + ad_slots_script: html_stream_params("", None).ad_slots_script, + ad_bids_state: Arc::new(Mutex::new(None)), + suppress_datadome_client_side_tag: false, + gpt_diagnostics: None, + shared_template_authorized: false, + csp_nonce_observed: None, + deferred_inline_marker: None, + }) + .expect("should build a synchronous control"); + let mut expected = Vec::new(); + for chunk in source.chunks(7) { + expected.extend( + processor + .process_chunk(chunk, false) + .expect("should process the control chunk"), + ); + } + expected.extend( + processor + .process_chunk(&[], true) + .expect("should finish the control"), + ); + assert_eq!( + body.as_ref(), + expected, + "should emit every transformed byte exactly once after bypass" + ); + assert_eq!( + calls.load(Ordering::SeqCst), + 0, + "should skip terminal I/O after retention overflow" + ); + } + + #[test] + fn html_finalization_rejects_corrupt_codecs_without_starting_optional_io() { + let page = b"Example"; + for encoding in ["gzip", "deflate", "br"] { + let mut encoded = match encoding { + "gzip" => gzip_encode(page), + "deflate" => deflate_encode(page), + _ => brotli_encode(page), + }; + encoded.truncate(encoded.len() / 2); + for streamed in [false, true] { + let settings = Arc::new(create_test_settings()); + let calls = Arc::new(AtomicUsize::new(0)); + let registry = IntegrationRegistry::from_html_finalizers(vec![Arc::new( + CountingTerminalFactory(Arc::clone(&calls)), + )]); + let body = if streamed { + EdgeBody::stream(futures::stream::iter(vec![bytes::Bytes::from( + encoded.clone(), + )])) + } else { + EdgeBody::from(encoded.clone()) + }; + let response = PublisherResponse::Stream { + response: Response::builder() + .status(200) + .header(header::CONTENT_TYPE, "text/html") + .body(EdgeBody::empty()) + .expect("should build corrupt-codec response"), + body, + params: Box::new(html_stream_params(encoding, None)), + }; + let orchestrator = Arc::new(AuctionOrchestrator::new(settings.auction.clone())); + let response = + futures::executor::block_on(publisher_response_into_streaming_response( + response, + &Method::GET, + settings, + ®istry, + orchestrator, + noop_services(), + )) + .expect("should create a lazy decode path"); + let _ = futures::executor::block_on( + response.into_body().into_bytes_bounded(1024 * 1024), + ) + .expect_err("should fail corrupt input before terminal EOF"); + assert_eq!( + calls.load(Ordering::SeqCst), + 0, + "should validate {encoding} before outside work: streamed={streamed}" + ); + } + } + } + + std::thread_local! { + static CANCELLATION_WARNINGS: std::cell::Cell> = const { std::cell::Cell::new(None) }; + } + + struct CancellationLogger; + impl log::Log for CancellationLogger { + fn enabled(&self, metadata: &log::Metadata<'_>) -> bool { + metadata.level() <= log::Level::Warn + } + fn log(&self, record: &log::Record<'_>) { + if record + .args() + .to_string() + .starts_with("Dispatched server-side auction dropped without collection") + { + CANCELLATION_WARNINGS.with(|count| { + if let Some(previous) = count.get() { + count.set(Some(previous + 1)); + } + }); + } + } + fn flush(&self) {} + } + + struct HungTerminalFactory { + started: Arc, + dropped: Arc, + } + struct HungTerminalSession { + started: Arc, + dropped: Arc, + } + struct TerminalDropProbe(Arc); + impl Drop for TerminalDropProbe { + fn drop(&mut self) { + self.0.fetch_add(1, Ordering::SeqCst); + } + } + impl crate::integrations::IntegrationHtmlFinalizerFactory for HungTerminalFactory { + fn integration_id(&self) -> &'static str { + "example_hung" + } + fn create( + &self, + _: crate::integrations::IntegrationHtmlStreamContext, + ) -> Box { + Box::new(HungTerminalSession { + started: Arc::clone(&self.started), + dropped: Arc::clone(&self.dropped), + }) + } + } + impl crate::integrations::IntegrationHtmlFinalizer for HungTerminalSession { + fn finalize<'a>( + &'a mut self, + input: crate::integrations::HtmlFinalizationInput<'a>, + ) -> futures::future::LocalBoxFuture< + 'a, + Result< + crate::integrations::HtmlFinalization, + Report, + >, + > { + Box::pin(async move { + self.started.fetch_add(1, Ordering::SeqCst); + let _probe = TerminalDropProbe(Arc::clone(&self.dropped)); + input.replacement.append(b"partial example candidate")?; + futures::future::pending().await + }) + } + } + + #[test] + fn html_finalization_client_cancellation_drops_work_and_warns_for_auction_once() { + static LOGGER: CancellationLogger = CancellationLogger; + let _ = log::set_logger(&LOGGER); + log::set_max_level(log::LevelFilter::Warn); + CANCELLATION_WARNINGS.with(|count| count.set(Some(0))); + let settings = Arc::new(create_test_settings()); + let started = Arc::new(AtomicUsize::new(0)); + let dropped = Arc::new(AtomicUsize::new(0)); + let registry = + IntegrationRegistry::from_html_finalizers(vec![Arc::new(HungTerminalFactory { + started: Arc::clone(&started), + dropped: Arc::clone(&dropped), + })]); + let response = PublisherResponse::Stream { + response: Response::builder() + .status(200) + .header(header::CONTENT_TYPE, "text/html") + .body(EdgeBody::empty()) + .expect("should build the cancellation response"), + body: EdgeBody::from(&b"Example"[..]), + params: Box::new(html_stream_params( + "", + Some(DispatchedAuction::empty_for_test( + test_auction_request(), + 500, + )), + )), + }; + let orchestrator = Arc::new(AuctionOrchestrator::new(settings.auction.clone())); + let response = futures::executor::block_on(publisher_response_into_streaming_response( + response, + &Method::GET, + settings, + ®istry, + orchestrator, + noop_services(), + )) + .expect("should create a lazy terminal body"); + let mut stream = response + .into_body() + .into_stream() + .expect("should retain lazy output"); + { + let work = futures::StreamExt::next(&mut stream); + futures::pin_mut!(work); + let waker = futures::task::noop_waker(); + let mut context = std::task::Context::from_waker(&waker); + assert!( + work.as_mut().poll(&mut context).is_pending(), + "should await terminal work with the auction still owned" + ); + } + assert_eq!( + started.load(Ordering::SeqCst), + 1, + "should start the callback once" + ); + assert_eq!( + dropped.load(Ordering::SeqCst), + 0, + "should keep its candidate alive while pending" + ); + drop(stream); + assert_eq!( + dropped.load(Ordering::SeqCst), + 1, + "should drop the callback when the reader disconnects" + ); + let warnings = CANCELLATION_WARNINGS.with(std::cell::Cell::take); + assert_eq!( + warnings, + Some(1), + "should preserve exactly one existing auction cancellation warning" + ); + } + + #[test] + fn html_finalization_fatal_read_decode_and_sink_failures_abandon_auction_once() { + struct TestWriter { + write_fails: bool, + flush_fails: bool, + } + impl Write for TestWriter { + fn write(&mut self, bytes: &[u8]) -> io::Result { + if self.write_fails { + Err(io::Error::other("example write failure")) + } else { + Ok(bytes.len()) + } + } + fn flush(&mut self) -> io::Result<()> { + if self.flush_fails { + Err(io::Error::other("example flush failure")) + } else { + Ok(()) + } + } + } + for failure in ["read", "decode", "write", "flush"] { + let settings = create_test_settings(); + let calls = Arc::new(AtomicUsize::new(0)); + let registry = IntegrationRegistry::from_html_finalizers(vec![Arc::new( + CountingTerminalFactory(Arc::clone(&calls)), + )]); + let sink = Arc::new(RecordingTelemetrySink::default()); + let services = noop_services_with_telemetry_sink(Arc::clone(&sink) as _); + let orchestrator = AuctionOrchestrator::new(settings.auction.clone()); + let encoding = if failure == "decode" { "gzip" } else { "" }; + let mut params = html_stream_params( + encoding, + Some(DispatchedAuction::empty_for_test( + test_auction_request(), + 500, + )), + ); + params.auction_observation = Some(AuctionObservationContext::from_parts( + AuctionSource::InitialNavigation, + "proxy.example.com", + "/article", + 1, + None, + &EcContext::default(), + )); + let source = b"Example"; + let body = match failure { + "read" => { + EdgeBody::from_stream(futures::stream::iter(vec![Err::( + io::Error::other("example origin read failure"), + )])) + } + "decode" => { + let mut encoded = gzip_encode(source); + encoded.truncate(encoded.len() / 2); + EdgeBody::from(encoded) + } + _ => EdgeBody::from(&source[..]), + }; + let mut writer = TestWriter { + write_fails: failure == "write", + flush_fails: failure == "flush", + }; + let _ = futures::executor::block_on(stream_publisher_body_async( + body, + &mut writer, + &mut params, + &settings, + ®istry, + &orchestrator, + &services, + )) + .expect_err("should preserve fatal pipeline errors"); + let batches = sink.batches.lock().expect("should lock terminal telemetry"); + let summaries: Vec<_> = batches + .iter() + .flat_map(crate::auction::telemetry::AuctionEventBatch::rows) + .filter(|row| row.event_kind == "summary") + .collect(); + assert_eq!( + summaries.len(), + 1, + "should abandon exactly once after {failure} failure" + ); + assert_eq!( + summaries[0].terminal_status.as_deref(), + Some("abandoned"), + "should retain fatal-error abandonment semantics" + ); + assert_eq!( + calls.load(Ordering::SeqCst), + usize::from(matches!(failure, "write" | "flush")), + "should start I/O only after valid input EOF" + ); + } + } + + struct TerminalTestFactory; + + struct TerminalTestSession; + + impl crate::integrations::IntegrationHtmlFinalizerFactory for TerminalTestFactory { + fn integration_id(&self) -> &'static str { + "example_terminal" + } + fn create( + &self, + _context: crate::integrations::IntegrationHtmlStreamContext, + ) -> Box { + Box::new(TerminalTestSession) + } + } + + impl crate::integrations::IntegrationHtmlFinalizer for TerminalTestSession { + fn finalize<'a>( + &'a mut self, + input: crate::integrations::HtmlFinalizationInput<'a>, + ) -> futures::future::LocalBoxFuture< + 'a, + Result< + crate::integrations::HtmlFinalization, + Report, + >, + > { + Box::pin(async move { + let mut yielded = false; + futures::future::poll_fn(|context| { + if yielded { + std::task::Poll::Ready(()) + } else { + yielded = true; + context.waker().wake_by_ref(); + std::task::Poll::Pending + } + }) + .await; + input.replacement.append(input.html)?; + input.replacement.append(b"")?; + Ok(crate::integrations::HtmlFinalization::Replace) + }) + } + } + + #[test] + fn html_finalization_runs_on_materialized_and_streamed_publishers() { + for source in [ + &b"

Example article

"[..], + &b"

Example article

"[..], + "Example café".as_bytes(), + &b""[..], + ] { + for encoding in ["", "gzip", "deflate", "br"] { + let encoded = match encoding { + "gzip" => { + let middle = source.len() / 2; + let mut members = gzip_encode(&source[..middle]); + members.extend(gzip_encode(&source[middle..])); + members + } + "deflate" => deflate_encode(source), + "br" => brotli_encode(source), + _ => source.to_vec(), + }; + for auction in [false, true] { + for lazy in [false, true] { + for streamed in [false, true] { + let settings = Arc::new(create_test_settings()); + let registry = + IntegrationRegistry::from_html_finalizers(vec![Arc::new( + TerminalTestFactory, + )]); + let orchestrator = + Arc::new(AuctionOrchestrator::new(settings.auction.clone())); + let body = if streamed { + let chunks: Vec<_> = encoded + .chunks(7) + .map(|chunk| { + Ok::<_, io::Error>(bytes::Bytes::copy_from_slice(chunk)) + }) + .collect(); + EdgeBody::from_stream(futures::stream::iter(chunks)) + } else { + EdgeBody::from(encoded.clone()) + }; + let dispatched = auction.then(|| { + DispatchedAuction::empty_for_test(test_auction_request(), 10) + }); + let response = PublisherResponse::Stream { + response: Response::builder() + .status(StatusCode::OK) + .header(header::CONTENT_TYPE, "text/html") + .body(EdgeBody::empty()) + .expect("should build response"), + body, + params: Box::new(html_stream_params(encoding, dispatched)), + }; + let response = if lazy { + futures::executor::block_on( + publisher_response_into_streaming_response( + response, + &Method::GET, + Arc::clone(&settings), + ®istry, + Arc::clone(&orchestrator), + noop_services(), + ), + ) + .expect("should build lazy response") + } else { + futures::executor::block_on(buffer_publisher_response_async( + response, + &Method::GET, + &settings, + ®istry, + &orchestrator, + &noop_services(), + )) + .expect("should build buffered response") + }; + let body = futures::executor::block_on( + response.into_body().into_bytes_bounded(1024 * 1024), + ) + .expect("should drain body"); + let decoded = match encoding { + "gzip" => gzip_decode(&body), + "deflate" => deflate_decode(&body), + "br" => brotli_decode(&body), + _ => body.to_vec(), + }; + assert!( + decoded.ends_with(b""), + "should await finalization: encoding={encoding}, auction={auction}, lazy={lazy}, streamed={streamed}" + ); + assert!( + !String::from_utf8_lossy(&decoded) + .contains("ts-inline-body-close-"), + "should remove the parser seam after finalization" + ); + } + } + } + } + } + } + fn streaming_finalize_response(params: OwnedProcessResponseParams, body: EdgeBody) -> EdgeBody { streaming_finalize_response_with_settings(params, body, create_test_settings()) } @@ -23783,6 +25049,7 @@ mod tests { &services, None, AuctionDispatch { + integration_registry: &IntegrationRegistry::empty_for_tests(), orchestrator, slots, registry: None, @@ -23838,6 +25105,7 @@ mod tests { &winning_services, None, AuctionDispatch { + integration_registry: &IntegrationRegistry::empty_for_tests(), orchestrator: &winning_orchestrator, slots: &slots, registry: None, @@ -23893,6 +25161,7 @@ mod tests { &no_winner_services, None, AuctionDispatch { + integration_registry: &IntegrationRegistry::empty_for_tests(), orchestrator: &no_winner_orchestrator, slots: &slots, registry: None, @@ -23945,6 +25214,7 @@ mod tests { &services, None, AuctionDispatch { + integration_registry: &IntegrationRegistry::empty_for_tests(), orchestrator: &orchestrator, slots: &slots, registry: None, @@ -24801,6 +26071,7 @@ mod tests { None, &mut ec_context, AuctionDispatch { + integration_registry: &IntegrationRegistry::empty_for_tests(), orchestrator: &orchestrator, slots: &article_slot(), registry: None, @@ -24843,6 +26114,7 @@ mod tests { &services, None, AuctionDispatch { + integration_registry: &IntegrationRegistry::empty_for_tests(), orchestrator: &orchestrator, slots: &article_slot(), registry: None, @@ -24885,6 +26157,7 @@ mod tests { None, &mut ec_context, AuctionDispatch { + integration_registry: &IntegrationRegistry::empty_for_tests(), orchestrator: &orchestrator, slots: &slots, registry: None, @@ -24929,6 +26202,7 @@ mod tests { &services, None, AuctionDispatch { + integration_registry: &IntegrationRegistry::empty_for_tests(), orchestrator: &orchestrator, slots: &slots, registry: None, diff --git a/crates/trusted-server-core/src/publisher/html_finalization.rs b/crates/trusted-server-core/src/publisher/html_finalization.rs new file mode 100644 index 000000000..872a9062c --- /dev/null +++ b/crates/trusted-server-core/src/publisher/html_finalization.rs @@ -0,0 +1,1090 @@ +//! Optional terminal HTML work; current bytes stay owned outside callback futures. + +use std::collections::VecDeque; +use std::io::{self, Read}; +use std::sync::Arc; +use std::time::Duration; + +use brotli::Decompressor; +use bytes::Bytes; +use edgezero_core::body::Body as EdgeBody; +use error_stack::{Report, ResultExt as _}; +use flate2::{Decompress, FlushDecompress, Status}; +use futures::FutureExt as _; +use web_time::Instant; + +use super::{ + AuctionCollectDeps, AuctionHoldState, AuctionTelemetryCarry, BodyChunkSource, + DispatchedAuctionGuard, HoldStepSegments, HtmlStreamProcessorParams, InlineBodyCloseSeam, + OwnedProcessResponseParams, abandon_hold_auction, body_as_reader, + create_html_stream_processor_with_document_state, deferred_inline_seam_token, + encode_processed_html, hold_collect_close_tail, hold_finish_tail_segments, +}; +use crate::error::TrustedServerError; +use crate::integrations::{ + HtmlFinalization, HtmlFinalizationDeadline, HtmlFinalizationError, HtmlFinalizationInput, + HtmlReplacementWriter, IntegrationDocumentState, IntegrationHtmlFinalizer, + IntegrationHtmlStreamContext, IntegrationRegistry, +}; +use crate::platform::RuntimeServices; +use crate::settings::Settings; +use crate::streaming_processor::{ + BodyStreamDecoder, BodyStreamEncoder, Compression, GzipDecodeReader, STREAM_CHUNK_SIZE, + StreamProcessor, +}; + +struct HtmlFinalizerSession { + integration_id: &'static str, + finalizer: Box, +} + +fn log_timeout(integration_id: &str, started_at: Instant, bytes: usize) { + log::warn!( + "HTML finalizer {integration_id} times out after {} ms; retains {bytes} bytes", + started_at.elapsed().as_millis() + ); +} + +async fn finalize_document( + mut current: Vec, + sessions: &mut [HtmlFinalizerSession], + marker: Option<&[u8]>, + services: &RuntimeServices, + limit: usize, + timeout: Duration, +) -> Vec { + let deadline = HtmlFinalizationDeadline::new(timeout, services.timer().is_some()); + let started_at = Instant::now(); + let mut timer = services.timer().map(|timer| timer.delay(timeout).fuse()); + for session in sessions { + if deadline.is_expired_at(Instant::now()) { + break; + } + let mut replacement = HtmlReplacementWriter::new(limit); + let result = { + let work = session + .finalizer + .finalize(HtmlFinalizationInput { + html: ¤t, + replacement: &mut replacement, + services, + deadline: &deadline, + }) + .fuse(); + futures::pin_mut!(work); + if let Some(mut timer) = timer.as_mut() { + futures::select_biased! { + _ = timer => None, + result = work => Some(result), + } + } else { + Some(work.await) + } + }; + // The callback future has dropped before either borrowed buffer is used. + let expired = result.is_none() || deadline.is_expired_at(Instant::now()); + if expired { + log_timeout(session.integration_id, started_at, current.len()); + break; + } + let result = result.expect("should have a result before the deadline"); + let outcome = match result { + Ok(HtmlFinalization::Unchanged) => "unchanged", + Ok(HtmlFinalization::Replace) if !replacement.is_poisoned() => { + let candidate = replacement.into_bytes(); + if marker.is_none_or(|token| { + marker_count(&candidate, token) == marker_count(¤t, token) + }) { + if deadline.is_expired_at(Instant::now()) { + log_timeout(session.integration_id, started_at, current.len()); + break; + } + current = candidate; + "replaced" + } else { + "invalid_marker" + } + } + Ok(HtmlFinalization::Replace) => "output_limit", + Err(error) => match error.current_context() { + HtmlFinalizationError::Unavailable => "unavailable", + HtmlFinalizationError::Rejected => "rejected", + HtmlFinalizationError::OutputLimitExceeded { .. } => "output_limit", + }, + }; + if matches!(outcome, "unchanged" | "replaced") { + log::debug!( + "HTML finalizer {} completes: {outcome}, {} bytes, {} ms", + session.integration_id, + current.len(), + started_at.elapsed().as_millis() + ); + } else { + log::warn!( + "HTML finalizer {} retains input: {outcome}, {} bytes, {} ms", + session.integration_id, + current.len(), + started_at.elapsed().as_millis() + ); + } + } + current +} + +fn marker_count(bytes: &[u8], token: &[u8]) -> usize { + if token.is_empty() { + return 0; + } + bytes + .windows(token.len()) + .filter(|window| *window == token) + .count() +} + +struct HtmlCollector { + retained: Option, +} + +impl HtmlCollector { + fn new(limit: usize) -> Self { + Self { + retained: Some(HtmlReplacementWriter::new(limit)), + } + } + + fn push(&mut self, output: Vec) -> Vec { + let Some(retained) = self.retained.as_mut() else { + return vec![bytes::Bytes::from(output)]; + }; + if retained.append(&output).is_ok() { + return Vec::new(); + } + let retained = self + .retained + .take() + .expect("should own the collected prefix"); + log::warn!("HTML finalization bypasses document: collection exceeds its byte limit"); + vec![ + bytes::Bytes::from(retained.into_bytes()), + bytes::Bytes::from(output), + ] + } + + fn finish(&mut self) -> Option> { + self.retained.take().map(HtmlReplacementWriter::into_bytes) + } +} + +// The reader codec must prove StreamEnd before terminal work starts. The +// ordinary flate2 Read wrapper also returns EOF for an incomplete zlib stream. +struct ValidatedZlibReader { + reader: Box, + decoder: Decompress, + input: [u8; STREAM_CHUNK_SIZE], + offset: usize, + available: usize, + eof: bool, + complete: bool, +} + +impl ValidatedZlibReader { + fn new(reader: Box) -> Self { + Self { + reader, + decoder: Decompress::new(true), + input: [0; STREAM_CHUNK_SIZE], + offset: 0, + available: 0, + eof: false, + complete: false, + } + } +} + +impl Read for ValidatedZlibReader { + fn read(&mut self, output: &mut [u8]) -> io::Result { + if output.is_empty() || self.complete { + return Ok(0); + } + loop { + if self.offset == self.available && !self.eof { + self.available = self.reader.read(&mut self.input)?; + self.offset = 0; + self.eof = self.available == 0; + } + let before_in = self.decoder.total_in(); + let before_out = self.decoder.total_out(); + let status = self + .decoder + .decompress( + &self.input[self.offset..self.available], + output, + if self.eof { + FlushDecompress::Finish + } else { + FlushDecompress::None + }, + ) + .map_err(io::Error::other)?; + let consumed = + usize::try_from(self.decoder.total_in() - before_in).map_err(io::Error::other)?; + let produced = + usize::try_from(self.decoder.total_out() - before_out).map_err(io::Error::other)?; + self.offset += consumed; + self.complete = status == Status::StreamEnd; + if produced > 0 || self.complete { + return Ok(produced); + } + if self.eof || (consumed == 0 && self.offset < self.available) { + return Err(io::Error::new( + io::ErrorKind::UnexpectedEof, + "incomplete publisher zlib stream", + )); + } + } + } +} + +enum TerminalInput { + Stream { + source: BodyChunkSource, + decoder: Box, + }, + Materialized(Box), +} + +impl TerminalInput { + fn new( + body: EdgeBody, + compression: Compression, + limit: usize, + ) -> Result> { + if body.is_stream() { + return Ok(Self::Stream { + source: BodyChunkSource::new(body, STREAM_CHUNK_SIZE).with_max_bytes(limit), + decoder: Box::new(BodyStreamDecoder::new(compression, limit)), + }); + } + // Preserve the existing reader codecs and per-step gzip decode budget. + let reader = body_as_reader(body)?; + let reader: Box = match compression { + Compression::None => Box::new(reader), + Compression::Gzip => Box::new(GzipDecodeReader::new(reader, limit)), + Compression::Deflate => Box::new(ValidatedZlibReader::new(Box::new(reader))), + Compression::Brotli => Box::new(Decompressor::new(reader, STREAM_CHUNK_SIZE)), + }; + Ok(Self::Materialized(reader)) + } + + async fn next( + &mut self, + failure_reason: &mut &'static str, + ) -> Result<(Bytes, bool), Report> { + match self { + Self::Stream { source, decoder } => { + *failure_reason = "stream_read_error"; + let raw = source.next_chunk().await?; + *failure_reason = "stream_decode_error"; + match raw { + Some(raw) => Ok((decoder.decode_chunk(raw)?, false)), + None => Ok((Bytes::from(decoder.finish()?), true)), + } + } + Self::Materialized(reader) => { + *failure_reason = "stream_read_error"; + let mut bytes = vec![0; STREAM_CHUNK_SIZE]; + let size = reader + .read(&mut bytes) + .change_context(TrustedServerError::Proxy { + message: "Failed to read decoded HTML terminal input".to_string(), + })?; + bytes.truncate(size); + Ok((Bytes::from(bytes), size == 0)) + } + } + } +} + +pub(super) struct HtmlFinalizationDriver { + input: TerminalInput, + encoder: BodyStreamEncoder, + processor: Box, + collector: HtmlCollector, + replay: VecDeque, + sessions: Vec, + marker: Option>, + hold: Option, + limit: usize, + timeout: Duration, + source_finished: bool, + finished: bool, + seam_flushed: bool, + failure_reason: &'static str, +} + +impl HtmlFinalizationDriver { + pub(super) fn new( + body: EdgeBody, + params: &mut OwnedProcessResponseParams, + settings: &Settings, + registry: &IntegrationRegistry, + ) -> Result> { + let document_state = IntegrationDocumentState::default(); + let marker = deferred_inline_seam_token( + settings, + false, + params.ad_slots_script.is_some(), + params.dispatched_auction.is_some(), + ); + let processor = create_html_stream_processor_with_document_state( + HtmlStreamProcessorParams { + origin_host: ¶ms.origin_host, + request_host: ¶ms.request_host, + request_scheme: ¶ms.request_scheme, + settings, + integration_registry: registry, + ad_slots_script: params.ad_slots_script.clone(), + ad_bids_state: Arc::clone(params.ad_bids_state.script_cell()), + suppress_datadome_client_side_tag: params.suppress_datadome_client_side_tag, + gpt_diagnostics: params.gpt_diagnostics.clone(), + shared_template_authorized: false, + csp_nonce_observed: None, + deferred_inline_marker: marker + .as_ref() + .map(|marker| String::from_utf8_lossy(marker).into_owned()), + }, + &document_state, + )?; + let context = IntegrationHtmlStreamContext { + request_host: params.request_host.clone(), + request_scheme: params.request_scheme.clone(), + origin_host: params.origin_host.clone(), + document_state, + }; + let sessions = registry + .html_finalizer_factories() + .iter() + .map(|factory| HtmlFinalizerSession { + integration_id: factory.integration_id(), + finalizer: factory.create(context.clone()), + }) + .collect(); + let hold = params.dispatched_auction.take().map(|dispatched| { + AuctionHoldState::new( + DispatchedAuctionGuard::new(dispatched), + AuctionTelemetryCarry { + observation: params.auction_observation.take(), + auction_request: params.auction_request.take(), + }, + marker.clone(), + ) + }); + let compression = Compression::from_content_encoding(¶ms.content_encoding); + let limit = settings.publisher.max_buffered_body_bytes; + Ok(Self { + input: TerminalInput::new(body, compression, limit)?, + encoder: BodyStreamEncoder::new(compression), + processor: Box::new(processor), + collector: HtmlCollector::new(limit), + replay: VecDeque::new(), + sessions, + marker, + hold, + limit, + timeout: Duration::from_millis(u64::from( + settings.publisher.html_finalization_timeout_ms, + )), + source_finished: false, + finished: false, + seam_flushed: false, + failure_reason: "stream_process_error", + }) + } + + pub(super) async fn next_step( + &mut self, + deps: &AuctionCollectDeps<'_>, + ) -> Result, Report> { + let result = self.next_inner(deps).await; + if result.is_err() { + self.abandon(deps, self.failure_reason).await; + } + result + } + + async fn next_inner( + &mut self, + deps: &AuctionCollectDeps<'_>, + ) -> Result, Report> { + loop { + if let Some(bytes) = self.replay.pop_front() { + if bytes.is_empty() { + continue; + } + let end = bytes.len().min(STREAM_CHUNK_SIZE); + let chunk = bytes.slice(..end); + if end < bytes.len() { + self.replay.push_front(bytes.slice(end..)); + } + self.failure_reason = "stream_encode_error"; + return encode_processed_html( + &mut self.encoder, + chunk.to_vec(), + self.hold.as_mut(), + ) + .map(Some); + } + if self.finished { + return Ok(None); + } + if self.source_finished && !self.seam_flushed { + self.seam_flushed = true; + self.failure_reason = "stream_encode_error"; + let suffix = self.hold.as_mut().and_then(|hold| { + if hold.hold.as_ref().is_some_and(|seam| !seam.found()) { + hold.hold.take().map(InlineBodyCloseSeam::finish) + } else { + None + } + }); + let encoded = self.encoder.encode_chunk(suffix.unwrap_or_default())?; + return Ok(Some(HoldStepSegments { + ready: if encoded.is_empty() { + Vec::new() + } else { + vec![Bytes::from(encoded)] + }, + close_found: false, + })); + } + if self.source_finished { + self.finished = true; + self.failure_reason = "stream_encode_error"; + let ready = if let Some(hold) = self.hold.as_mut() { + hold_finish_tail_segments(&mut self.encoder, hold, deps).await? + } else { + let trailer = self.encoder.finish()?; + if trailer.is_empty() { + Vec::new() + } else { + vec![Bytes::from(trailer)] + } + }; + return Ok(Some(HoldStepSegments { + ready, + close_found: false, + })); + } + let (decoded, ended) = self.input.next(&mut self.failure_reason).await?; + if !decoded.is_empty() { + self.accept(&decoded, false)?; + } + if ended { + self.accept(&[], true)?; + self.source_finished = true; + if let Some(current) = self.collector.finish() { + let current = finalize_document( + current, + &mut self.sessions, + self.marker.as_deref(), + deps.services, + self.limit, + self.timeout, + ) + .await; + self.replay.push_back(Bytes::from(current)); + } + } + } + } + + fn accept(&mut self, bytes: &[u8], is_last: bool) -> Result<(), Report> { + self.failure_reason = "stream_process_error"; + let output = self + .processor + .process_chunk(bytes, is_last) + .change_context(TrustedServerError::Proxy { + message: "Failed to process HTML terminal input".to_string(), + })?; + self.replay.extend(self.collector.push(output)); + Ok(()) + } + + pub(super) fn input_finished(&self) -> bool { + self.source_finished + } + + pub(super) async fn collect_close( + &mut self, + deps: &AuctionCollectDeps<'_>, + ) -> Result, Report> { + if let Some(hold) = self.hold.as_mut() { + hold_collect_close_tail(&mut self.encoder, hold, deps).await + } else { + Ok(Vec::new()) + } + } + + pub(super) async fn abandon(&mut self, deps: &AuctionCollectDeps<'_>, reason: &'static str) { + if let Some(hold) = self.hold.as_mut() { + abandon_hold_auction(hold, deps.services, reason).await; + } + } +} + +#[cfg(test)] +mod tests { + #[test] + fn collection_overflow_replays_prefix_and_crossing_chunk_once() { + let mut collector = HtmlCollector::new(4); + assert!( + collector.push(b"page".to_vec()).is_empty(), + "should retain input at the cap" + ); + let mut emitted = collector.push(b"!".to_vec()).concat(); + emitted.extend(collector.push(b"tail".to_vec()).concat()); + assert_eq!( + emitted, b"page!tail", + "should release overflow without duplication" + ); + assert!( + collector.finish().is_none(), + "should skip finalizer I/O after bypass" + ); + } + + use std::cell::{Cell, RefCell}; + use std::rc::Rc; + use std::time::Duration; + + use error_stack::Report; + use futures::executor::block_on; + use futures::future::{LocalBoxFuture, poll_fn}; + use std::task::Poll; + + use crate::integrations::{ + HtmlFinalization, HtmlFinalizationError, HtmlFinalizationInput, IntegrationHtmlFinalizer, + }; + use crate::platform::test_support::{ + NoopBackend, NoopConfigStore, NoopGeo, NoopHttpClient, NoopSecretStore, noop_services, + }; + use crate::platform::{PlatformTimer, RuntimeServices}; + + use super::*; + + struct AppendFinalizer { + suffix: &'static [u8], + inputs: Rc>>>, + fail: bool, + } + + impl IntegrationHtmlFinalizer for AppendFinalizer { + fn finalize<'a>( + &'a mut self, + input: HtmlFinalizationInput<'a>, + ) -> LocalBoxFuture<'a, Result>> { + Box::pin(async move { + let mut yielded = false; + poll_fn(|context| { + if yielded { + Poll::Ready(()) + } else { + yielded = true; + context.waker().wake_by_ref(); + Poll::Pending + } + }) + .await; + self.inputs.borrow_mut().push(input.html.to_vec()); + input.replacement.append(input.html)?; + input.replacement.append(self.suffix)?; + if self.fail { + Err(Report::new(HtmlFinalizationError::Rejected)) + } else { + Ok(HtmlFinalization::Replace) + } + }) + } + } + + #[test] + fn yielding_finalizers_commit_in_order_and_discard_failed_candidates() { + let inputs = Rc::new(RefCell::new(Vec::new())); + let mut sessions = [ + HtmlFinalizerSession { + integration_id: "example_a", + finalizer: Box::new(AppendFinalizer { + suffix: b"a", + inputs: Rc::clone(&inputs), + fail: false, + }), + }, + HtmlFinalizerSession { + integration_id: "example_b", + finalizer: Box::new(AppendFinalizer { + suffix: b"bad", + inputs: Rc::clone(&inputs), + fail: true, + }), + }, + HtmlFinalizerSession { + integration_id: "example_c", + finalizer: Box::new(AppendFinalizer { + suffix: b"c", + inputs: Rc::clone(&inputs), + fail: false, + }), + }, + ]; + let services = noop_services(); + let output = block_on(finalize_document( + b"page".to_vec(), + &mut sessions, + None, + &services, + 32, + Duration::from_secs(1), + )); + assert_eq!(output, b"pageac", "should retain only accepted candidates"); + assert_eq!( + *inputs.borrow(), + [b"page".to_vec(), b"pagea".to_vec(), b"pagea".to_vec()], + "should propagate committed input in order" + ); + } + + #[test] + fn poisoned_replacement_preserves_the_original_allocation() { + let inputs = Rc::new(RefCell::new(Vec::new())); + let mut sessions = [HtmlFinalizerSession { + integration_id: "example", + finalizer: Box::new(AppendFinalizer { + suffix: b"oversize", + inputs, + fail: false, + }), + }]; + let input = b"page".to_vec(); + let allocation = input.as_ptr(); + let output = block_on(finalize_document( + input, + &mut sessions, + None, + &noop_services(), + 4, + Duration::from_secs(1), + )); + assert_eq!(output, b"page", "should discard oversized output"); + assert_eq!( + output.as_ptr(), + allocation, + "should move fallback without copying it" + ); + } + struct ControlledTimer { + pending_polls: usize, + } + + impl PlatformTimer for ControlledTimer { + fn delay(&self, _: Duration) -> LocalBoxFuture<'static, ()> { + let mut remaining = self.pending_polls; + Box::pin(poll_fn(move |context| { + if remaining == 0 { + Poll::Ready(()) + } else { + remaining -= 1; + context.waker().wake_by_ref(); + Poll::Pending + } + })) + } + } + + struct DropProbe(Rc>); + + impl Drop for DropProbe { + fn drop(&mut self) { + self.0.set(true); + } + } + + struct HungFinalizer { + dropped: Rc>, + started: Rc>, + } + + impl IntegrationHtmlFinalizer for HungFinalizer { + fn finalize<'a>( + &'a mut self, + input: HtmlFinalizationInput<'a>, + ) -> LocalBoxFuture<'a, Result>> { + let probe = DropProbe(Rc::clone(&self.dropped)); + Box::pin(async move { + let _probe = probe; + self.started.set(true); + input.replacement.append(b"partial")?; + futures::future::pending().await + }) + } + } + + fn timer_services(pending_polls: usize) -> RuntimeServices { + RuntimeServices::builder() + .config_store(Arc::new(NoopConfigStore)) + .secret_store(Arc::new(NoopSecretStore)) + .kv_store(Arc::new(edgezero_core::key_value_store::NoopKvStore)) + .backend(Arc::new(NoopBackend)) + .http_client(Arc::new(NoopHttpClient)) + .geo(Arc::new(NoopGeo)) + .client_info(crate::platform::ClientInfo::default()) + .timer(Arc::new(ControlledTimer { pending_polls })) + .build() + } + + #[test] + fn timeout_drops_borrowing_future_preserves_allocation_and_skips_later_work() { + let dropped = Rc::new(Cell::new(false)); + let started = Rc::new(Cell::new(false)); + let later_inputs = Rc::new(RefCell::new(Vec::new())); + let mut sessions = [ + HtmlFinalizerSession { + integration_id: "example_hung", + finalizer: Box::new(HungFinalizer { + dropped: Rc::clone(&dropped), + started: Rc::clone(&started), + }), + }, + HtmlFinalizerSession { + integration_id: "example_later", + finalizer: Box::new(AppendFinalizer { + suffix: b"later", + inputs: Rc::clone(&later_inputs), + fail: false, + }), + }, + ]; + let input = b"page".to_vec(); + let allocation = input.as_ptr(); + let output = block_on(finalize_document( + input, + &mut sessions, + None, + &timer_services(1), + 32, + Duration::from_secs(1), + )); + assert!( + started.get(), + "should poll outside work before timer expiry" + ); + assert!( + dropped.get(), + "should drop the borrowing future before returning" + ); + assert!( + later_inputs.borrow().is_empty(), + "should skip remaining finalizers" + ); + assert_eq!( + output, b"page", + "should discard a timed-out partial candidate" + ); + assert_eq!( + output.as_ptr(), + allocation, + "should retain the original allocation" + ); + } + + #[test] + fn expired_timer_wins_before_polling_ready_work() { + let inputs = Rc::new(RefCell::new(Vec::new())); + let mut sessions = [HtmlFinalizerSession { + integration_id: "example", + finalizer: Box::new(AppendFinalizer { + suffix: b"replacement", + inputs: Rc::clone(&inputs), + fail: false, + }), + }]; + let output = block_on(finalize_document( + b"page".to_vec(), + &mut sessions, + None, + &timer_services(0), + 32, + Duration::from_secs(1), + )); + assert_eq!(output, b"page", "should give timer expiration priority"); + assert!( + inputs.borrow().is_empty(), + "should not poll work after expiry" + ); + } + + #[test] + fn marker_multiplicity_rejects_added_markers() { + let mut sessions = [HtmlFinalizerSession { + integration_id: "example", + finalizer: Box::new(AppendFinalizer { + suffix: b"TOKEN", + inputs: Rc::new(RefCell::new(Vec::new())), + fail: false, + }), + }]; + let output = block_on(finalize_document( + b"pageTOKEN".to_vec(), + &mut sessions, + Some(b"TOKEN"), + &noop_services(), + 32, + Duration::from_secs(1), + )); + assert_eq!( + output, b"pageTOKEN", + "should reject a duplicated assembly marker" + ); + let output = block_on(finalize_document( + b"page".to_vec(), + &mut sessions, + Some(b"TOKEN"), + &noop_services(), + 32, + Duration::from_secs(1), + )); + assert_eq!( + output, b"page", + "should reject adding a marker to a document without one" + ); + } + struct ImmediateFinalizer { + ignore_overflow: bool, + } + + impl IntegrationHtmlFinalizer for ImmediateFinalizer { + fn finalize<'a>( + &'a mut self, + input: HtmlFinalizationInput<'a>, + ) -> LocalBoxFuture<'a, Result>> { + Box::pin(async move { + input.replacement.append(input.html)?; + if self.ignore_overflow { + let _ = input.replacement.append(b"oversize"); + } else { + input.replacement.append(b"a")?; + } + Ok(HtmlFinalization::Replace) + }) + } + } + + #[test] + fn ignored_output_limit_cannot_commit_a_partial_candidate() { + let mut sessions = [HtmlFinalizerSession { + integration_id: "example", + finalizer: Box::new(ImmediateFinalizer { + ignore_overflow: true, + }), + }]; + let output = block_on(finalize_document( + b"page".to_vec(), + &mut sessions, + None, + &noop_services(), + 4, + Duration::from_secs(1), + )); + assert_eq!( + output, b"page", + "should reject Replace after an ignored writer error" + ); + } + + #[test] + fn one_timer_covers_the_chain_and_preserves_an_earlier_commit() { + let dropped = Rc::new(Cell::new(false)); + let mut sessions = [ + HtmlFinalizerSession { + integration_id: "example_ready", + finalizer: Box::new(ImmediateFinalizer { + ignore_overflow: false, + }), + }, + HtmlFinalizerSession { + integration_id: "example_hung", + finalizer: Box::new(HungFinalizer { + dropped: Rc::clone(&dropped), + started: Rc::new(Cell::new(false)), + }), + }, + ]; + let output = block_on(finalize_document( + b"page".to_vec(), + &mut sessions, + None, + &timer_services(1), + 32, + Duration::from_secs(1), + )); + assert_eq!( + output, b"pagea", + "should retain the earlier commit when the shared timer expires" + ); + assert!( + dropped.get(), + "should drop the second callback at the shared deadline" + ); + } + + #[test] + fn cancellation_drops_active_callback_and_its_candidate() { + let dropped = Rc::new(Cell::new(false)); + let mut sessions = [HtmlFinalizerSession { + integration_id: "example", + finalizer: Box::new(HungFinalizer { + dropped: Rc::clone(&dropped), + started: Rc::new(Cell::new(false)), + }), + }]; + let services = noop_services(); + let work = finalize_document( + b"page".to_vec(), + &mut sessions, + None, + &services, + 32, + Duration::from_secs(1), + ); + let mut work = Box::pin(work); + let waker = futures::task::noop_waker(); + let mut context = std::task::Context::from_waker(&waker); + assert!( + std::future::Future::poll(work.as_mut(), &mut context).is_pending(), + "should yield while outside work is pending" + ); + drop(work); + assert!( + dropped.get(), + "should cancel the active future with the document" + ); + } + #[test] + fn materialized_deflate_requires_validated_end_before_terminal_eof() { + let mut encoder = + flate2::write::ZlibEncoder::new(Vec::new(), flate2::Compression::default()); + std::io::Write::write_all(&mut encoder, b"Example") + .expect("should encode an example page"); + let compressed = encoder.finish().expect("should finish zlib input"); + let mut valid = + ValidatedZlibReader::new(Box::new(std::io::Cursor::new(compressed.clone()))); + let mut output = Vec::new(); + valid + .read_to_end(&mut output) + .expect("should accept a validated zlib trailer"); + for missing in [1, 4, 8] { + let mut reader = ValidatedZlibReader::new(Box::new(std::io::Cursor::new( + compressed[..compressed.len() - missing].to_vec(), + ))); + let _ = reader + .read_to_end(&mut Vec::new()) + .expect_err("should reject a truncated trailer before terminal EOF"); + } + } + struct FixedCandidate { + bytes: &'static [u8], + decision: HtmlFinalization, + } + impl IntegrationHtmlFinalizer for FixedCandidate { + fn finalize<'a>( + &'a mut self, + input: HtmlFinalizationInput<'a>, + ) -> LocalBoxFuture<'a, Result>> { + Box::pin(async move { + input.replacement.append(self.bytes)?; + Ok(self.decision) + }) + } + } + + #[test] + fn unchanged_discards_candidate_and_missing_markers_reject_replacement() { + for (decision, marker) in [ + (HtmlFinalization::Unchanged, None), + (HtmlFinalization::Replace, Some(&b"TOKEN"[..])), + ] { + let mut sessions = [HtmlFinalizerSession { + integration_id: "example", + finalizer: Box::new(FixedCandidate { + bytes: b"candidate", + decision, + }), + }]; + let output = block_on(finalize_document( + b"pageTOKEN".to_vec(), + &mut sessions, + marker, + &noop_services(), + 32, + Duration::from_secs(1), + )); + assert_eq!( + output, b"pageTOKEN", + "should retain input when unchanged or a marker is removed" + ); + } + let mut sessions = [HtmlFinalizerSession { + integration_id: "example", + finalizer: Box::new(FixedCandidate { + bytes: b"", + decision: HtmlFinalization::Replace, + }), + }]; + let output = block_on(finalize_document( + b"page".to_vec(), + &mut sessions, + None, + &noop_services(), + 32, + Duration::from_secs(1), + )); + assert!( + output.is_empty(), + "should allow an intentional empty replacement without a marker" + ); + } + + #[cfg(not(target_arch = "wasm32"))] + #[test] + fn no_timer_rejects_completion_after_the_absolute_deadline() { + struct LateFinalizer; + impl IntegrationHtmlFinalizer for LateFinalizer { + fn finalize<'a>( + &'a mut self, + input: HtmlFinalizationInput<'a>, + ) -> LocalBoxFuture<'a, Result>> + { + Box::pin(async move { + std::thread::sleep(input.deadline.remaining() + Duration::from_millis(2)); + input.replacement.append(b"late")?; + Ok(HtmlFinalization::Replace) + }) + } + } + let mut sessions = [HtmlFinalizerSession { + integration_id: "example", + finalizer: Box::new(LateFinalizer), + }]; + let output = block_on(finalize_document( + b"page".to_vec(), + &mut sessions, + None, + &noop_services(), + 32, + Duration::from_millis(1), + )); + assert_eq!( + output, b"page", + "should reject late results from a blocking consumer" + ); + } +} diff --git a/crates/trusted-server-core/src/settings.rs b/crates/trusted-server-core/src/settings.rs index bf4ff7999..def14a676 100644 --- a/crates/trusted-server-core/src/settings.rs +++ b/crates/trusted-server-core/src/settings.rs @@ -78,6 +78,14 @@ pub struct Publisher { #[serde(default = "default_max_buffered_body_bytes")] #[validate(range(min = 1, message = "must be at least 1 byte"))] pub max_buffered_body_bytes: usize, + /// Whole-chain HTML terminal budget in milliseconds; inert without a finalizer. + #[serde(default = "default_html_finalization_timeout_ms")] + #[validate(range(min = 1, max = 60000))] + pub html_finalization_timeout_ms: u32, +} + +fn default_html_finalization_timeout_ms() -> u32 { + 5000 } fn default_max_buffered_body_bytes() -> usize { @@ -98,6 +106,7 @@ impl Default for Publisher { origin_host_header_override: None, proxy_secret: Redacted::default(), max_buffered_body_bytes: default_max_buffered_body_bytes(), + html_finalization_timeout_ms: default_html_finalization_timeout_ms(), } } } @@ -189,6 +198,7 @@ impl Publisher { /// origin_host_header_override: None, /// proxy_secret: Redacted::new("proxy-secret".to_string()), /// max_buffered_body_bytes: 16 * 1024 * 1024, + /// html_finalization_timeout_ms: 5000, /// }; /// assert_eq!(publisher.origin_host(), "origin.example.com:8080"); /// ``` @@ -3718,6 +3728,44 @@ where #[cfg(test)] mod tests { + #[test] + fn html_finalization_timeout_defaults_and_boundaries() { + let parsed = Settings::from_toml(&crate_test_settings_str()) + .expect("should parse an omitted terminal timeout"); + assert_eq!( + parsed.publisher.html_finalization_timeout_ms, + Publisher::default().html_finalization_timeout_ms, + "should match TOML omission and the hand-written default" + ); + assert_eq!( + Publisher::default().html_finalization_timeout_ms, + 5000, + "should default the chain budget to five seconds" + ); + for (milliseconds, invalid) in [ + (0, true), + (1, false), + (5000, false), + (60000, false), + (60001, true), + ] { + let publisher = Publisher { + html_finalization_timeout_ms: milliseconds, + ..Publisher::default() + }; + let errors = publisher + .validate() + .expect_err("should reject the other empty publisher fields"); + assert_eq!( + errors + .field_errors() + .contains_key("html_finalization_timeout_ms"), + invalid, + "should enforce finish timeout boundaries" + ); + } + } + use super::*; use regex::Regex; use serde_json::json; @@ -6201,6 +6249,7 @@ source_domain = "partner.example.com" origin_host_header_override: None, proxy_secret: Redacted::new("test-secret".to_string()), max_buffered_body_bytes: 16 * 1024 * 1024, + html_finalization_timeout_ms: default_html_finalization_timeout_ms(), }; assert_eq!(publisher.origin_host(), "origin.example.com:8080"); @@ -6212,6 +6261,7 @@ source_domain = "partner.example.com" origin_host_header_override: None, proxy_secret: Redacted::new("test-secret".to_string()), max_buffered_body_bytes: 16 * 1024 * 1024, + html_finalization_timeout_ms: default_html_finalization_timeout_ms(), }; assert_eq!(publisher.origin_host(), "origin.example.com"); @@ -6223,6 +6273,7 @@ source_domain = "partner.example.com" origin_host_header_override: None, proxy_secret: Redacted::new("test-secret".to_string()), max_buffered_body_bytes: 16 * 1024 * 1024, + html_finalization_timeout_ms: default_html_finalization_timeout_ms(), }; assert_eq!(publisher.origin_host(), "localhost:9090"); @@ -6234,6 +6285,7 @@ source_domain = "partner.example.com" origin_host_header_override: None, proxy_secret: Redacted::new("test-secret".to_string()), max_buffered_body_bytes: 16 * 1024 * 1024, + html_finalization_timeout_ms: default_html_finalization_timeout_ms(), }; assert_eq!(publisher.origin_host(), "localhost:9090"); @@ -6245,6 +6297,7 @@ source_domain = "partner.example.com" origin_host_header_override: None, proxy_secret: Redacted::new("test-secret".to_string()), max_buffered_body_bytes: 16 * 1024 * 1024, + html_finalization_timeout_ms: default_html_finalization_timeout_ms(), }; assert_eq!(publisher.origin_host(), "192.168.1.1:8080"); @@ -6256,6 +6309,7 @@ source_domain = "partner.example.com" origin_host_header_override: None, proxy_secret: Redacted::new("test-secret".to_string()), max_buffered_body_bytes: 16 * 1024 * 1024, + html_finalization_timeout_ms: default_html_finalization_timeout_ms(), }; assert_eq!(publisher.origin_host(), "[::1]:8080"); } @@ -6269,6 +6323,7 @@ source_domain = "partner.example.com" origin_host_header_override: None, proxy_secret: Redacted::new("test-secret".to_string()), max_buffered_body_bytes: 16 * 1024 * 1024, + html_finalization_timeout_ms: default_html_finalization_timeout_ms(), }; assert_eq!(publisher.origin_host_header(), "origin.example.com:8443"); @@ -6283,6 +6338,7 @@ source_domain = "partner.example.com" origin_host_header_override: Some("www.example.com".to_string()), proxy_secret: Redacted::new("test-secret".to_string()), max_buffered_body_bytes: 16 * 1024 * 1024, + html_finalization_timeout_ms: default_html_finalization_timeout_ms(), }; assert_eq!(publisher.origin_host_header(), "www.example.com"); diff --git a/crates/trusted-server-core/src/test_support.rs b/crates/trusted-server-core/src/test_support.rs index d6e85a167..102855e52 100644 --- a/crates/trusted-server-core/src/test_support.rs +++ b/crates/trusted-server-core/src/test_support.rs @@ -364,6 +364,12 @@ pub mod nextjs_auction { /// stubbed out. #[must_use] pub fn services(client: Arc) -> RuntimeServices { + services_with_http_client(client) + } + + /// Build fixture runtime services with a custom asynchronous upstream. + #[must_use] + pub fn services_with_http_client(client: Arc) -> RuntimeServices { RuntimeServices::builder() .config_store(Arc::new(FixtureStore)) .secret_store(Arc::new(FixtureStore)) @@ -375,3 +381,6 @@ pub mod nextjs_auction { .build() } } + +#[cfg(feature = "test-utils")] +pub(crate) mod async_html_finalization; diff --git a/crates/trusted-server-core/src/test_support/async_html_finalization.rs b/crates/trusted-server-core/src/test_support/async_html_finalization.rs new file mode 100644 index 000000000..9cf27c527 --- /dev/null +++ b/crates/trusted-server-core/src/test_support/async_html_finalization.rs @@ -0,0 +1,193 @@ +//! Fictional terminal HTTP consumer, available only with `test-utils`. + +use std::sync::Arc; +use std::time::Duration; + +use edgezero_core::body::Body; +use edgezero_core::http::{Uri, request_builder}; +use error_stack::{Report, ResultExt as _}; +use futures::future::LocalBoxFuture; +use serde::Deserialize; +use serde_json::json; +use sha2::{Digest as _, Sha256}; +use url::Url; +use validator::Validate; + +use crate::error::TrustedServerError; +use crate::integrations::{ + HtmlFinalization, HtmlFinalizationError, HtmlFinalizationInput, IntegrationHtmlFinalizer, + IntegrationHtmlFinalizerFactory, IntegrationHtmlStreamContext, IntegrationRegistration, +}; +use crate::platform::{PlatformBackendSpec, PlatformHttpRequest}; +use crate::settings::{IntegrationConfig, Settings}; + +pub(crate) const ID: &str = "html_finalization_fixture"; +const EOF_SENTINEL: &[u8] = b""; +const ORIGINAL: &[u8] = b"EXAMPLE_UNSIGNED"; +const REPLACEMENT: &[u8] = b"EXAMPLE_FINALIZED"; + +#[derive(Clone, Deserialize, Validate)] +pub(crate) struct FixtureConfig { + enabled: bool, + #[validate(url)] + lookup_url: String, + #[validate(range(min = 1, max = 60000))] + transport_timeout_ms: u32, +} + +impl IntegrationConfig for FixtureConfig { + fn is_enabled(&self) -> bool { + self.enabled + } +} + +struct FixtureFactory(FixtureConfig); +struct FixtureSession(FixtureConfig); + +pub(crate) fn register( + settings: &Settings, +) -> Result, Report> { + let Some(config) = settings.integration_config::(ID)? else { + return Ok(None); + }; + Ok(Some( + IntegrationRegistration::builder(ID) + .without_js() + .with_html_finalizer(Arc::new(FixtureFactory(config))) + .build(), + )) +} + +impl IntegrationHtmlFinalizerFactory for FixtureFactory { + fn integration_id(&self) -> &'static str { + ID + } + + fn create(&self, _: IntegrationHtmlStreamContext) -> Box { + Box::new(FixtureSession(self.0.clone())) + } +} + +impl IntegrationHtmlFinalizer for FixtureSession { + fn finalize<'a>( + &'a mut self, + input: HtmlFinalizationInput<'a>, + ) -> LocalBoxFuture<'a, Result>> { + Box::pin(async move { + if !input + .html + .windows(EOF_SENTINEL.len()) + .any(|bytes| bytes == EOF_SENTINEL) + { + return Err(Report::new(HtmlFinalizationError::Rejected)); + } + let url = + Url::parse(&self.0.lookup_url).change_context(HtmlFinalizationError::Rejected)?; + // Fastly cannot interrupt a blocking host call. Bound its transport + // by the remaining chain budget; timer adapters deliberately test + // cancellation against a longer transport timeout. + let timeout = if input.deadline.has_cooperative_timer() { + Duration::from_millis(u64::from(self.0.transport_timeout_ms)) + } else { + input + .deadline + .remaining() + .min(Duration::from_millis(u64::from( + self.0.transport_timeout_ms, + ))) + }; + let backend = input + .services + .backend() + .ensure(&PlatformBackendSpec { + scheme: url.scheme().to_string(), + host: url + .host_str() + .ok_or_else(|| Report::new(HtmlFinalizationError::Rejected))? + .to_string(), + port: url.port(), + host_header_override: None, + certificate_check: true, + first_byte_timeout: timeout, + between_bytes_timeout: timeout, + discriminator: Some(ID.to_string()), + }) + .change_context(HtmlFinalizationError::Unavailable)?; + let payload = serde_json::to_vec(&json!({ + "complete": true, + "sha256": hex::encode(Sha256::digest(input.html)), + "bytes": input.html.len(), + "transport_timeout_ms": timeout.as_millis(), + })) + .change_context(HtmlFinalizationError::Rejected)?; + let request = request_builder() + .method(http::Method::POST) + .uri( + self.0 + .lookup_url + .parse::() + .change_context(HtmlFinalizationError::Rejected)?, + ) + .header(http::header::CONTENT_TYPE, "application/json") + .body(Body::from(payload)) + .change_context(HtmlFinalizationError::Rejected)?; + let response = input + .services + .http_client() + .send(PlatformHttpRequest::new(request, backend).with_cache_bypass()) + .await + .change_context(HtmlFinalizationError::Unavailable)?; + if !response.response.status().is_success() { + return Err(Report::new(HtmlFinalizationError::Unavailable)); + } + let reply = response + .response + .into_body() + .into_bytes_bounded(64) + .await + .change_context(HtmlFinalizationError::Rejected)?; + if reply.as_ref() != b"EXAMPLE_APPROVED" { + return Err(Report::new(HtmlFinalizationError::Rejected)); + } + let offset = input + .html + .windows(ORIGINAL.len()) + .position(|bytes| bytes == ORIGINAL) + .ok_or_else(|| Report::new(HtmlFinalizationError::Rejected))?; + input.replacement.append(&input.html[..offset])?; + input.replacement.append(REPLACEMENT)?; + input + .replacement + .append(&input.html[offset + ORIGINAL.len()..])?; + Ok(HtmlFinalization::Replace) + }) + } +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::integrations::IntegrationRegistry; + use crate::test_support::tests::create_test_settings; + + #[test] + fn fictional_consumer_registers_only_when_enabled_and_adds_no_browser_bundle() { + for enabled in [false, true] { + let mut settings = create_test_settings(); + settings.integrations.insert_config(ID, &json!({ + "enabled": enabled, "lookup_url": "https://lookup.example.com/approve", "transport_timeout_ms": 1000, + })).expect("should configure the fictional consumer"); + let registry = + IntegrationRegistry::new(&settings).expect("should build fixture metadata"); + assert_eq!( + registry.has_html_finalizers(), + enabled, + "should exclude disabled terminal capabilities" + ); + assert!( + !registry.js_module_ids().contains(&ID), + "should never add a fictional browser bundle" + ); + } + } +} diff --git a/crates/trusted-server-integration-tests/tests/parity.rs b/crates/trusted-server-integration-tests/tests/parity.rs index 2428d7799..3688160ef 100644 --- a/crates/trusted-server-integration-tests/tests/parity.rs +++ b/crates/trusted-server-integration-tests/tests/parity.rs @@ -8,7 +8,13 @@ // Both adapters define `TrustedServerApp` — alias both to avoid name collision. // axum::http re-exports from the `http` crate, so HeaderMap types are identical. +use error_stack::Report; use std::sync::Arc; +use std::sync::atomic::{AtomicUsize, Ordering}; +use trusted_server_core::platform::{ + PlatformError, PlatformHttpClient, PlatformHttpRequest, PlatformPendingRequest, + PlatformResponse, PlatformSelectResult, +}; use axum::body::Body as AxumBody; use axum::http::Request as AxumRequest; @@ -1119,3 +1125,143 @@ async fn adapter_buffers_nextjs_auction_output() { } } } + +#[derive(Default)] +struct TerminalFixtureOrigin { + lookup_calls: AtomicUsize, +} + +#[async_trait::async_trait(?Send)] +impl PlatformHttpClient for TerminalFixtureOrigin { + async fn send( + &self, + request: PlatformHttpRequest, + ) -> Result> { + let (content_type, body) = if request.request.uri().host() == Some("lookup.example.com") { + tokio::task::yield_now().await; + let payload = request + .request + .into_body() + .into_bytes() + .expect("should send a bounded fixture payload"); + let payload: serde_json::Value = serde_json::from_slice(&payload) + .expect("should encode completed-document metadata"); + assert_eq!( + payload["complete"], true, + "should observe the EOF sentinel before I/O" + ); + assert_eq!( + payload["sha256"] + .as_str() + .expect("should hash transformed HTML") + .len(), + 64, + "should send a complete digest" + ); + self.lookup_calls.fetch_add(1, Ordering::SeqCst); + ("text/plain", "EXAMPLE_APPROVED") + } else { + ( + "text/html", + "

EXAMPLE_UNSIGNED

", + ) + }; + Ok(PlatformResponse::new( + http::Response::builder() + .status(200) + .header("content-type", content_type) + .header("cache-control", "public, max-age=60") + .body(edgezero_core::body::Body::from(body)) + .expect("should build an example upstream response"), + )) + } + async fn send_async( + &self, + _: PlatformHttpRequest, + ) -> Result> { + Err(Report::new(PlatformError::Unsupported)) + } + async fn select( + &self, + _: Vec, + ) -> Result> { + Err(Report::new(PlatformError::Unsupported)) + } +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn asynchronous_html_finalization_and_privacy_match_host_adapters() { + let mut settings = test_settings(); + settings.integrations.insert_config("html_finalization_fixture", &serde_json::json!({ + "enabled": true, "lookup_url": "https://lookup.example.com/success", "transport_timeout_ms": 1000, + })).expect("should enable the fictional terminal consumer"); + let client = Arc::new(TerminalFixtureOrigin::default()); + let services = nextjs_auction::services_with_http_client( + Arc::clone(&client) as Arc + ); + let routers = [ + ( + "Axum", + AxumApp::routes_with_settings_and_services(settings.clone(), services.clone()), + ), + ( + "Cloudflare", + CloudflareApp::routes_with_settings_and_services(settings.clone(), services.clone()), + ), + ( + "Spin", + SpinApp::routes_with_settings_and_services(settings, services), + ), + ]; + let mut expected = None; + for (adapter, router) in routers { + client.lookup_calls.store(0, Ordering::SeqCst); + let request = request_builder() + .method("GET") + .uri("https://test-publisher.example.com/article") + .header("host", "test-publisher.example.com") + .header("accept", "text/html") + .body(edgezero_core::body::Body::empty()) + .expect("should build an example navigation"); + let response = router + .expect("should build fixture routes") + .oneshot(request) + .await + .expect("should serve finalized HTML"); + assert_eq!( + response.status(), + 200, + "{adapter} should preserve the origin status" + ); + assert_eq!( + response + .headers() + .get("cache-control") + .expect("should set terminal privacy"), + "no-store, private", + "{adapter} should prevent downstream caching" + ); + let body = response + .into_body() + .into_bytes_bounded(1024 * 1024) + .await + .expect("should drain finalized HTML"); + assert!( + String::from_utf8_lossy(&body).contains("EXAMPLE_FINALIZED"), + "{adapter} should await the yielding consumer" + ); + assert_eq!( + client.lookup_calls.load(Ordering::SeqCst), + 1, + "{adapter} should call the terminal lookup exactly once" + ); + if let Some(expected) = &expected { + assert_eq!( + &body, expected, + "{adapter} should produce identical terminal HTML" + ); + } else { + expected = Some(body); + } + } +} diff --git a/docs/guide/configuration.md b/docs/guide/configuration.md index 98e375627..6a25ce8b2 100644 --- a/docs/guide/configuration.md +++ b/docs/guide/configuration.md @@ -309,14 +309,15 @@ Core publisher settings for domain, origin, and proxy configuration. ### `[publisher]` -| Field | Type | Required | Description | -| ----------------------------- | ------- | -------- | --------------------------------------------------------------------------- | -| `domain` | String | Yes | Publisher's apex domain name | -| `cookie_domain` | String | Yes | Domain for non-EC cookies (typically with leading dot) | -| `origin_url` | String | Yes | Full URL of publisher origin server | -| `origin_host_header_override` | String | No | Outbound Host header to send while connecting to `origin_url` | -| `proxy_secret` | String | Yes | Secret-store key name for the proxy URL secret | -| `max_buffered_body_bytes` | Integer | No | Buffered-body cap / Fastly stream raw+decoded byte ceiling (default 16 MiB) | +| Field | Type | Required | Description | +| ------------------------------ | ------- | -------- | ---------------------------------------------------------------------------------- | +| `domain` | String | Yes | Publisher's apex domain name | +| `cookie_domain` | String | Yes | Domain for non-EC cookies (typically with leading dot) | +| `origin_url` | String | Yes | Full URL of publisher origin server | +| `origin_host_header_override` | String | No | Outbound Host header to send while connecting to `origin_url` | +| `proxy_secret` | String | Yes | Secret-store key name for the proxy URL secret | +| `max_buffered_body_bytes` | Integer | No | Buffered-body cap / Fastly stream raw+decoded byte ceiling (default 16 MiB) | +| `html_finalization_timeout_ms` | Integer | No | Whole terminal-chain budget after synchronous EOF (default 5000 ms; range 1–60000) | > **Note:** EC cookies (`ts-ec`) derive their domain automatically as `.{domain}` and > do not use `cookie_domain`. The `cookie_domain` field is used by other cookie helpers. @@ -437,6 +438,15 @@ intentionally, and never put it in the TOML file or pushed app-config blob. Changing `proxy_secret` invalidates all existing signed URLs. Plan rotations carefully and use graceful transition periods. ::: +#### `html_finalization_timeout_ms` + +Optional whole-chain budget for registered asynchronous HTML finalizers, in +milliseconds. The default is `5000`; accepted values are `1` through `60000`. +The budget starts after decoding and synchronous HTML EOF and does not change the +auction deadline. It has no effect without enabled finalizers. Axum, Workers, and +Spin can cancel yielding work; Fastly consumers must bound their own blocking host +calls. Timeout keeps the current transformed document and skips later finalizers. + #### `max_buffered_body_bytes` **Purpose**: Upper bound on how much of a publisher origin body the rewrite diff --git a/docs/guide/integration-guide.md b/docs/guide/integration-guide.md index f127d4207..3abef24e8 100644 --- a/docs/guide/integration-guide.md +++ b/docs/guide/integration-guide.md @@ -203,6 +203,93 @@ Streaming support is an adapter capability. Request checked `supports_streaming_responses()`; unsupported adapters must reject the request instead of silently buffering it. +## Asynchronous HTML finalization + +An integration that needs complete transformed HTML plus an outside lookup can +register an `IntegrationHtmlFinalizerFactory` with `.with_html_finalizer(...)`. +The factory creates one request-local `IntegrationHtmlFinalizer` session and +receives the same `IntegrationDocumentState` as the synchronous HTML rewriters. +Factory construction performs setup only; start I/O inside the returned future. +The session and its future may be `?Send`. + +Core completes decoding and every synchronous processor, then awaits terminal +sessions in registration order. Each session sees the previous committed HTML; +its replacement does not pass through the parser again. The auction seam and +encoder complete after the terminal phase. Existing `StreamProcessor` callbacks +remain synchronous, and pages with no registered finalizers keep their existing +streaming path. + +`HtmlFinalizationInput` supplies borrowed `html`, `services`, `deadline`, and a +separate bounded `replacement` writer. Append a complete candidate and return +`HtmlFinalization::Replace`, or return `Unchanged`. An error, timeout, invalid +assembly marker count, or output limit violation discards the candidate and +retains the current input. Ignoring an append error cannot commit partial output. +An empty `Replace` intentionally emits an empty document. Callback panics propagate +through the request and are not converted into optional-work failures. Preserve HTML structure, +CSP, Next.js/RSC semantics, and the opaque parser markers; core checks marker +multiplicity but does not parse or validate replacement structure. + +The publisher's `html_finalization_timeout_ms` is one budget for the whole chain, +starting after synchronous EOF. Use `deadline.remaining()` to bound outside calls. +Axum, Workers, and Spin provide cooperative timers. Fastly provides no timer +because host calls block; the consumer must set transport timeouts itself. Dropping +a future releases its borrows and discards its candidate, but cannot undo a remote +write or preempt work that does not yield. The deadline wins over a simultaneously +ready callback. Avoid logging HTML, lookup URLs, or external response contents. + +Enabled finalizers retain transformed HTML up to `max_buffered_body_bytes` and use +a separate candidate with the same cap. Collection overflow skips terminal I/O +and resumes normal delivery with the retained prefix and remaining bytes. Existing +input, decoder, and final output errors remain fatal. Registration therefore changes +first-byte latency only for enabled pages that fit the retention limit. + +Use `stream_publisher_body_async`, `buffer_publisher_response_async`, or +`publisher_response_into_streaming_response` to execute terminal callbacks. +`stream_publisher_body` and the generic synchronous pipeline perform synchronous +transformations only. Client cancellation drops the callback and retains the existing +auction guard's log warning; error paths that can await emit abandonment telemetry. + +Finalizer-enabled HTML bypasses shared-template reads, reservations, and stores and +uses inline auction assembly. Raw-origin readthrough remains independent. Processed +HTML receives `Cache-Control: no-store, private` even when a finalizer leaves it +unchanged, fails, or is skipped after collection overflow. Document GETs strip the +four conditional/range headers used by synthesized HTML. HEAD, bodiless responses, +unsupported encodings, and non-HTML responses run no terminal callbacks. + +Callers of `handle_publisher_request` provide the enabled integration registry in +`AuctionDispatch::integration_registry` and use that same registry for response +finalization. This metadata is required before a shared-template lookup can happen. + +A fictional HTTP consumer is available only through the existing `test-utils` +feature. To verify real asynchronous I/O and timeout fallback on local runtimes: + +```bash +./scripts/async-html-finalization-local-test.sh axum +./scripts/async-html-finalization-local-test.sh cloudflare +./scripts/async-html-finalization-local-test.sh spin +./scripts/async-html-finalization-local-test.sh fastly +``` + +Each run builds a fresh feature-enabled artifact, generates temporary configuration +and example secrets, and retains diagnostics in the printed temporary directory. +It uses the installed runtime CLIs and, for Workers, the `worker-build` version +matching the locked `worker` SDK and a compatible `wasm-bindgen` CLI. The default +Cargo output is isolated in that directory; an explicit `CARGO_TARGET_DIR` permits +sharing the local build cache. Every run invokes Cargo and builds its fixture Worker +bundle into a fresh directory. Workers and Spin use opt-in adapter `test-utils` +features for local diagnostic sinks. + +The repository pins Viceroy 0.17 for its normal test gate. Backend transport timeout +simulation requires Viceroy 0.19 or later. Use an isolated newer executable in `PATH` +for the Fastly hung-call case; do not interpret a hung test on 0.17 as evidence that +Fastly production ignores the configured timeout. + +Add `--release` for production-profile artifacts. Add `--auction` to run a separate +enabled-auction control against a closed loopback +bid endpoint. It verifies slot/bid fallback assembly and marker removal without +external traffic. The existing Workers clock panic reported in #1240 blocks that +control; the ordinary terminal-phase tests still run independently. + ## Browser and script guards Add a browser module only when browser state is required. Immediate modules diff --git a/docs/superpowers/plans/2026-10-12-1241-async-html-finalization.md b/docs/superpowers/plans/2026-10-12-1241-async-html-finalization.md index 4617c5cd2..9c0675b63 100644 --- a/docs/superpowers/plans/2026-10-12-1241-async-html-finalization.md +++ b/docs/superpowers/plans/2026-10-12-1241-async-html-finalization.md @@ -10,16 +10,23 @@ --- -**Status:** Proposed implementation plan; no implementation started +**Status:** Implemented; see the execution record for verification and the existing Workers auction limitation **Date:** 2026-10-12 -**Branch:** `spec/1241-async-html-finalization` +**Branch:** `feat/1241-async-html-finalization` (planning originated on `spec/1241-async-html-finalization`) **Baseline:** `2a0cb0444dc335bd09105abe5952329bbbdfb621` **Design:** [Optional asynchronous HTML document finalization](../specs/2026-10-12-1241-async-html-finalization-design.md) +**Execution record:** [Implementation and bidirectional coverage review](../reviews/2026-10-12-1241-async-html-finalization-implementation.md) + +The task checklists below preserve the original execution instructions, including +proposed intermediate commit boundaries. The execution record is the authoritative +completion record: implementation was reviewed together before committing, and it +records actual commands, deviations, and the blocked Workers auction control. + ## 1. Execution boundaries This plan implements the selected terminal-phase design, not the original proposal to add @@ -27,8 +34,8 @@ This plan implements the selected terminal-phase design, not the original propos @superpowers:test-driven-development for runtime changes and @superpowers:verification-before-completion before recording results or committing. -The tasks below describe future implementation work. Their commands and expected outcomes -are not claims that the feature exists or has already passed tests. Keep intermediate +The tasks below were the implementation instructions. Their expected outcomes alone +are not verification evidence; consult the execution record for observed results. Keep intermediate commits in a draft branch; do not release an API that registrations can enable before all publisher entry points support it. @@ -666,5 +673,5 @@ The review corrected these points before handoff: - Workers fixture output stays in the temporary workspace; runtime timeout tests verify timer expiry rather than accidentally passing on an early transport error. -This record covers document review only. The unchecked tasks, runtime requirements, and -implementation acceptance criteria remain outstanding. +This historical record covers planning review only. The linked execution record now +maps the implemented contracts and observed verification to both documents. diff --git a/docs/superpowers/reviews/2026-10-12-1241-async-html-finalization-implementation.md b/docs/superpowers/reviews/2026-10-12-1241-async-html-finalization-implementation.md new file mode 100644 index 000000000..194ea5e77 --- /dev/null +++ b/docs/superpowers/reviews/2026-10-12-1241-async-html-finalization-implementation.md @@ -0,0 +1,178 @@ +# Asynchronous HTML finalization implementation review + +**Date:** 2026-10-12 + +**Branch:** `feat/1241-async-html-finalization` + +**Planning commit:** `b750c879a` + +**Status:** Implemented and reviewed. Repository regression gates pass. The actual +Workers enabled-auction control remains blocked by the existing clock panic in #1240. + +**Design:** [Optional asynchronous HTML document finalization](../specs/2026-10-12-1241-async-html-finalization-design.md) + +**Plan:** [Implementation and verification tasks](../plans/2026-10-12-1241-async-html-finalization.md) + +## Result and scope + +Trusted Server now offers an explicitly registered HTML terminal callback after +synchronous processing reaches EOF. It supports borrowing, non-`Send` futures, +ordered atomic replacements, bounded core-owned output, one chain deadline, and +unchanged-input fallback. The generic streaming traits remain unchanged. + +The callback belongs in Trusted Server because its document state, auction assembly, +cache admission, privacy policy, and runtime service container belong here. No +EdgeZero revision, production platform manifest, JS behavior, KV implementation, +signing consumer, or existing clock behavior changes in this delivery. + +Enabled finalizers deliberately retain the completed transformed document. They +bypass shared-template caching before lookup or reservation, and eligible HTML +responses use `private, no-store`. Disabled registrations keep the existing route. +Raw origin readthrough caching remains independent. + +## Spec and plan coverage + +The original granular checklists describe instructions, rather than claiming each +suggested intermediate commit exists. All thirteen tasks have implementation or +verification evidence below. Task 11's Workers auction control has an explicit +external blocker; passing native parity is not substituted for that live check. + +| Plan tasks | Spec sections | Implementation and observed evidence | +| ---------- | ------------------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ | +| 1 | 6.2, 8, 10 | `integrations/html_finalization.rs` defines the factory, session, borrowed input, outcomes, errors, deadline, and bounded replacement writer. Tests cover exact limits, poisoned candidates, empty replacement, unchanged output, and allocation bounds. | +| 2 | 9.1–9.3 | Settings default to 5000 ms and validate 1–60000 inclusive. Runtime services default to no timer and preserve configured timers when cloned or replacing services. Explicit TOML omission and boundary tests pass. | +| 3 | 6.1, 7.1 | Registry metadata preserves registration order and disabled eligibility. HTML parser, synchronous processors, and finalizer factories share one request-local document state. State-sharing and request-isolation tests pass. | +| 4 | 9.2 | Axum installs Tokio sleep, Workers installs `worker::Delay`, and Spin installs SDK sleep. Fastly has no cooperative timer. Target-matched native/WASM compilation, fixture-enabled WASM lint, and live runtime checks pass. | +| 5 | 7.2, 7.4, 9.3, 10, 13 | The private coordinator executes yielding callbacks in order. Tests verify earlier commits survive later failure, one timer covers the chain, expiry wins ties, late results are rejected, and cancellation drops active borrows before owned-byte fallback. | +| 6 | 8 | Collection and candidate buffers use the configured body cap with checked length arithmetic. Overflow replays prefix, crossing chunk, and suffix once, without starting optional I/O. Exact byte-order and allocation-identity tests pass. | +| 7 | 11 | Registry metadata gates shared-template lookup before origin fetch. Tests verify warm/cold bypass, no reservation/store, preserved raw-origin readthrough intent, a working raw-byte cache double hit with per-request finalization, disabled eligibility, four conditional/range headers stripped on eligible GETs, preserved preconditions, and private headers without an auction. | +| 8 | 7.2–7.4, 12, 13 | One terminal driver covers materialized/streamed input and lazy/buffered output, with and without an auction. Decoder and synchronous EOF precede callbacks; seam handling and encoding follow them. Client cancellation verifies exactly one existing guard warning; fatal read/decode/write/flush errors verify exactly one abandonment telemetry summary. | +| 9 | 7.1–7.3, 10–12, 14.3–14.5 | The 128-case route matrix covers four document shapes, four codecs, auction presence, sink kind, and body kind. Corrupt/truncated codec tests prevent premature I/O. Tests cover unsupported/bodiless/non-HTML cases and eligible non-2xx HTML. A positive winning-bid test preserves the article, places one bid injection before the structural body end, and removes the seam. Native and WASI runs pass. | +| 10 | 6.1, 12, 14 | The fictional consumer is behind `test-utils`, contributes no JS, waits for completed HTML, and uses request HTTP services. Axum/Workers/Spin host parity passes. Fixture IDs are absent from fresh ordinary artifacts for all four adapters. | +| 11 | 9.2–9.3, 14.6 | The local harness builds fresh artifacts and verifies disabled, delayed-success, and hung-lookup controls on all four actual runtimes. Workers/Spin/Fastly release-profile runs pass. Actual auction fallback controls pass on Axum, Spin, and Fastly; Workers reproduces #1240. | +| 12 | 9–12, 15 | Canonical configuration and integration guides document registration, timeout semantics, buffer limits, cache/privacy effects, synchronous helper limitations, required registry metadata, and local runtime commands. | +| 13 | 14–16 | Full target-matched gates, actual-runtime controls, existing integration/browser checks, feature-off artifact inspection, and independent architecture/final reviews are recorded below. | + +Every production addition maps to an existing spec contract. The implementation +does not add an interleaved asynchronous streaming API or cacheable finalizer mode. +The timeout starts after synchronous EOF, and optional failure preserves bytes +entering the failing stage. Existing transport, parser, codec, and sink failures +retain fatal handling. Marker-count checks protect opaque seams; they do not claim +to validate arbitrary HTML structure or integration-specific semantics. + +## Necessary implementation refinements + +- `AuctionDispatch` requires the request integration registry. Cache admission + needs finalizer metadata before lookup, and an optional registry could incorrectly + admit a cached template. Callers must pass the same registry to dispatch and + finalization; all production adapters do so. +- Materialized deflate input uses a validated zlib reader. The prior convenience + decoder could accept a truncated trailer as EOF; the opted-in route must verify + completion before starting external I/O. Tests reject truncated trailers while + preserving the existing materialized-body limit semantics. +- The opt-in driver and terminal coordinator futures are boxed. The existing + authorized cold-template buffered fallback is also boxed to limit nested WASI + future stack size. An existing winning-bid cold-miss test trapped on the new + branch, passed on the clean planning baseline, and passed after this narrow fix. + It costs one allocation on already-buffered authorized cache misses; inline + delivery and warm-template delivery do not take that fallback. +- The runtime harness uses a Python process-group driver with generated temporary + manifests and a restricted environment. It handles chunked Spin HTTP requests, + validates transport bounds, and retains body/header/event/timing diagnostics. + Fixture-only Workers/Spin loggers provide deadline diagnostics without altering + ordinary production logging. The Workers fixture also installs a diagnostic + panic hook under the same test feature. +- Verification used one reviewed implementation change instead of manufacturing + the plan's suggested per-task commits. No implementation API was published + between contract creation and complete routing. + +## Repository verification + +All commands ran against the implementation worktree, with Rust 1.95.0 and Node +24.12.0. Target-matched checks used the workspace aliases; shared Cargo cache use +does not substitute an old artifact because fresh builds and artifact hashes were +recorded. The normal Fastly gate used the pinned Viceroy 0.17. + +| Check | Observed result | +| ------------------------------- | --------------------------------------------------------------------------------------------------------------------------------------------------------- | +| Eight Clippy aliases | Pass: Fastly, Axum, Workers native/WASM, Spin native/WASM, CLI, and codegen. | +| Core primary-package Clippy | All targets/all features and the native build-digest lint pass. | +| Fastly tests | 219 adapter tests, 2924 core tests, 3 JS crate tests, 21 OpenRTB tests, and 4 core doc tests pass; existing ignored tests remain ignored. | +| Fastly reusable sandbox | 219 adapter tests pass. | +| Native adapter tests | Axum 44, Workers 53, Spin 87 pass. | +| Core all-feature tests | 2931 unit tests, 10 digest tests, and 4 doc tests pass; 6 doc tests remain ignored. | +| Host parity | 18 tests pass. | +| Exact CI Rust runtime selection | 63 tests pass across configuration generation, documentation fixture, integration tests, and parity. | +| CLI and codegen | 875 CLI tests and 7 codegen tests pass; existing ignored CLI tests remain ignored. | +| JS build, Vitest, type checks | 50 test files / 1208 tests pass. | +| Browser controls | Next.js: 21 pass / 1 expected skip. WordPress: 11 pass / 11 expected skips. Initial-render ownership: 10 scenarios pass. Escaping fixtures: 2 tests pass. | +| Documentation | Lint, build, formatting, and Markdown checks pass. Core rustdoc builds with existing warnings, including private intra-doc links. | +| Ordinary artifacts | Fresh Fastly, Workers, and Spin release builds and an Axum build pass; none contains the fictional finalizer registration ID. | + +The browser controls ran before the final narrow WASI future-layout correction; +that correction changed no JS behavior. Full Rust gates and the exact CI runtime +selection were rerun after the correction. A subsequent test-only raw-origin cache +hit regression passed natively and on WASI with primary-package Clippy; it changes +no production behavior or artifact. No source edits were made by reviewers +or the build validator. + +The existing template-cache controls pass on pinned Viceroy 0.17: ESI has 22 +top-level assertions plus 17 cookie/session cases; inline has 9 top-level +assertions plus 17 cookie/session cases. Both exercise real winning bids; ESI +also verifies the GPT scheduler contract. The ESI warm-hit socket probe delivers its first +body bytes in 16 ms while the delayed auction completes near 3020 ms. These +controls verify existing behavior with the feature disabled. + +The enabled-finalizer raw-origin cache-hit regression uses a working core test +double, rather than claiming a live platform cache hit. Local verification logs +and generated artifacts remain outside version control; they contain exact +command output and runtime diagnostics. + +## Actual-runtime verification and limits + +The fixture uses a 750 ms chain budget, delays document EOF by 50 ms, and delays a +successful lookup by 150 ms. On timer adapters the hung transport timeout is +30000 ms, so returning near 750 ms proves the terminal timer released the page +rather than relying on the transport timeout. Each hung case returns the exact +249-byte disabled-control body. Successful cases return the exact expected +replacement, with one completed-document lookup. + +| Adapter | Profile/runtime | Success request | Hung request | Terminal evidence | +| ------- | ------------------------------------ | --------------- | ------------ | ---------------------------------------------------------------- | +| Axum | Native dev binary | 0.225 s | 0.825 s | 753 ms terminal time; timer present. | +| Workers | Release WASM, `wrangler dev --local` | 0.244 s | 0.826 s | 752 ms terminal time; timer present. | +| Spin | Release WASM, `spin up` | 0.226 s | 0.819 s | 753 ms terminal time; timer present. | +| Fastly | Release WASM, isolated Viceroy 0.19 | 0.213 s | 0.804 s | 749 ms from lookup to response; timer absent, bounded host call. | + +Development-profile baseline/success/hung runs also passed on all four runtimes. +The optional actual-auction fallback control passed on Axum, Spin, and Fastly. +Positive winning-bid assembly is additionally covered by the native/WASI core +regression and the existing template-cache controls. + +Two runtime qualifications remain explicit: + +1. Pinned Viceroy 0.17 does not simulate backend timeouts, so it cannot prove a + Fastly hung-call bound. An isolated official 0.19 executable was used for that + live check; repository pins remain unchanged. Backend timeout support is + documented in the [official 0.19 release](https://github.com/fastly/Viceroy/releases/tag/v0.19.0). +2. The Workers actual-auction control returns HTTP 500 with + `time not implemented on this platform` from `std::time` and then a WASM + trap. This reproduces the separate #1240 clock problem. Ordinary Workers + finalization controls pass. Native parity and mocked auction tests do not + establish that the combined actual-Workers auction route works. No clock + workaround is bundled, and that live combination remains unverified until + #1240 is resolved. + +## Independent review + +Architecture and staff-level final reviews accepted the implementation with no +remaining blocking findings. Review requested a real lazy-body client-cancellation +test and clearer documentation of the synchronous helper's terminal-phase limits; +both were added and re-reviewed. Architecture review requested a positive winning +bid with an enabled finalizer; that regression now passes natively and on WASI. + +The spec, plan, code, and tests agree on ownership, ordering, deadline scope, +cache/privacy policy, optional fallback, and excluded response types. The remaining +Workers runtime limitation is documented rather than counted as a passing test. +The implementation branch and isolated worktree are preserved for maintainer +review; no merge, push, deployment, or public issue comment is part of this work. diff --git a/docs/superpowers/specs/2026-10-12-1241-async-html-finalization-design.md b/docs/superpowers/specs/2026-10-12-1241-async-html-finalization-design.md index 99ba38687..7419d5dc9 100644 --- a/docs/superpowers/specs/2026-10-12-1241-async-html-finalization-design.md +++ b/docs/superpowers/specs/2026-10-12-1241-async-html-finalization-design.md @@ -2,7 +2,7 @@ **Date:** 2026-10-12 -**Status:** Proposed; design only, awaiting maintainer review +**Status:** Implemented on `feat/1241-async-html-finalization`; verification and remaining runtime limitation recorded below **Issue:** trusted-server #1241 @@ -10,6 +10,8 @@ **Implementation plan:** [Task-by-task implementation and verification](../plans/2026-10-12-1241-async-html-finalization.md) +**Implementation review:** [Requirement coverage, verification, and runtime limitations](../reviews/2026-10-12-1241-async-html-finalization-implementation.md) + ## 1. Decision Add an opt-in asynchronous final-document phase to Trusted Server's HTML integration diff --git a/scripts/async-html-finalization-local-test.sh b/scripts/async-html-finalization-local-test.sh new file mode 100755 index 000000000..9b0906af5 --- /dev/null +++ b/scripts/async-html-finalization-local-test.sh @@ -0,0 +1,5 @@ +#!/usr/bin/env bash +# Build a fresh test-utils artifact and exercise terminal I/O on a local runtime. +set -euo pipefail +REPO_ROOT=$(cd "$(dirname "$0")/.." && pwd) +exec python3 "$REPO_ROOT/scripts/fixtures/async-html-finalization-runtime.py" "$@" diff --git a/scripts/fixtures/async-html-finalization-origin.py b/scripts/fixtures/async-html-finalization-origin.py new file mode 100644 index 000000000..01a0fe76f --- /dev/null +++ b/scripts/fixtures/async-html-finalization-origin.py @@ -0,0 +1,84 @@ +#!/usr/bin/env python3 +"""Loopback origin and delayed lookup for the fictional terminal consumer.""" + +import argparse +import json +import threading +import time +from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer +from pathlib import Path + +DOCUMENT = ( + b"

EXAMPLE_UNSIGNED

" + b"" +) + + +def serve(port, event_path): + lock = threading.Lock() + + def event(kind, **fields): + with lock: + with event_path.open("a") as output: + output.write(json.dumps({"event": kind, "time": time.monotonic(), **fields}) + "\n") + + class Handler(BaseHTTPRequestHandler): + def log_message(self, *_): + pass + + def do_HEAD(self): + self.send_response(200) + self.send_header("Content-Type", "text/html") + self.send_header("Content-Length", str(len(DOCUMENT))) + self.end_headers() + + def do_GET(self): + self.send_response(200) + self.send_header("Content-Type", "text/html") + self.send_header("Cache-Control", "public, max-age=60") + self.send_header("Content-Length", str(len(DOCUMENT))) + self.end_headers() + self.wfile.write(DOCUMENT[:-len(b"")]) + self.wfile.flush() + time.sleep(0.05) + event("origin_eof") + self.wfile.write(b"") + self.wfile.flush() + + def do_POST(self): + if self.headers.get("Transfer-Encoding", "").lower() == "chunked": + body = bytearray() + while True: + size = int(self.rfile.readline().split(b";", 1)[0], 16) + if size == 0: + while self.rfile.readline() != b"\r\n": + pass + break + body.extend(self.rfile.read(size)) + assert len(body) <= 2048, "fixture request must remain bounded" + assert self.rfile.read(2) == b"\r\n" + else: + size = int(self.headers.get("Content-Length", "0")) + assert size <= 2048, "fixture request must remain bounded" + body = self.rfile.read(size) + payload = json.loads(body) + event("lookup_start", path=self.path, **payload) + time.sleep(30 if self.path == "/hung" else 0.15) + self.send_response(200) + self.send_header("Content-Length", str(len(b"EXAMPLE_APPROVED"))) + self.end_headers() + try: + self.wfile.write(b"EXAMPLE_APPROVED") + event("lookup_complete", path=self.path) + except (BrokenPipeError, ConnectionResetError): + event("lookup_cancelled", path=self.path) + + ThreadingHTTPServer(("127.0.0.1", port), Handler).serve_forever() + + +if __name__ == "__main__": + parser = argparse.ArgumentParser() + parser.add_argument("--port", type=int, required=True) + parser.add_argument("--events", type=Path, required=True) + args = parser.parse_args() + serve(args.port, args.events) diff --git a/scripts/fixtures/async-html-finalization-runtime.py b/scripts/fixtures/async-html-finalization-runtime.py new file mode 100644 index 000000000..227284dcc --- /dev/null +++ b/scripts/fixtures/async-html-finalization-runtime.py @@ -0,0 +1,275 @@ +#!/usr/bin/env python3 +"""Fresh-build local runtime matrix with isolated config and process cleanup.""" + +import argparse +import base64 +import re +import json +import os +import shutil +import tomllib +import signal +import socket +import subprocess +import tempfile +import time +import urllib.error +import urllib.request +from pathlib import Path + +ROOT = Path(__file__).resolve().parents[2] +SECRETS = { + "handler_password": "example-handler-password-32-bytes-ok", + "publisher_proxy_secret": "example-publisher-secret-32-bytes-ok", + "ec_passphrase": "example-cookie-passphrase-32-bytes-ok", +} + + +def free_port(): + with socket.socket() as listener: + listener.bind(("127.0.0.1", 0)) + return listener.getsockname()[1] + + +def stop(process): + if process is None: + return + try: + os.killpg(process.pid, signal.SIGTERM) + process.wait(timeout=5) + except (ProcessLookupError, subprocess.TimeoutExpired): + pass + try: + os.killpg(process.pid, signal.SIGKILL) + except ProcessLookupError: + pass + + +def run(adapter, auction=False, release=False): + workspace = Path(tempfile.mkdtemp(prefix=f"trusted-server-finalize-{adapter}-")) + print(f"Artifacts: {workspace}", flush=True) + allowed = {"PATH", "RUSTUP_HOME", "CARGO_HOME", "CARGO_TARGET_DIR", "TMPDIR", "WASM_BINDGEN_BIN", "WASM_OPT_BIN", "ESBUILD_BIN", "CARGO_NET_OFFLINE"} + env = {key: value for key, value in os.environ.items() if key in allowed} + target = Path(env.get("CARGO_TARGET_DIR", workspace / "target")).resolve() + env["CARGO_TARGET_DIR"] = str(target) + env["XDG_CONFIG_HOME"] = str(workspace / "cli-config") + env["WRANGLER_SEND_METRICS"] = "false" + env["WRANGLER_LOG_PATH"] = str(workspace / "wrangler.log") + origin_port, app_port = free_port(), free_port() + env.update({ + "TRUSTED_SERVER__PUBLISHER__DOMAIN": "127.0.0.1", + "TRUSTED_SERVER__PUBLISHER__COOKIE_DOMAIN": "127.0.0.1", + "TRUSTED_SERVER__PUBLISHER__ORIGIN_URL": f"http://127.0.0.1:{origin_port}", + }) + required = ["cargo", "python3"] + ({"fastly": ["viceroy"], "cloudflare": ["worker-build", "wrangler"], "spin": ["spin"]}.get(adapter, [])) + for executable in required: + if shutil.which(executable, path=env["PATH"]) is None: + raise RuntimeError(f"required executable missing: {executable}") + if adapter == "cloudflare": + lock = tomllib.loads((ROOT / "Cargo.lock").read_text()) + expected = next(package["version"] for package in lock["package"] if package["name"] == "worker") + actual = subprocess.check_output(["worker-build", "--version"], env=env, text=True).strip() + if actual != expected: + raise RuntimeError(f"worker-build {expected} required for the locked worker SDK; found {actual}") + processes = [] + + def command(args, cwd=ROOT, overrides=None): + subprocess.run(args, cwd=cwd, env=env | (overrides or {}), check=True) + + def launch(args, log, cwd=workspace, runtime_env=None): + output = log.open("w") + process = subprocess.Popen(args, cwd=cwd, env=runtime_env or env, + stdout=output, stderr=subprocess.STDOUT, + start_new_session=True) + output.close() + processes.append(process) + return process + + def get(path, auction_case=False): + headers = {"Accept": "text/html", "User-Agent": "Mozilla/5.0 (Example browser)"} + if auction_case: + # A fictional core TCF v2 segment grants Purpose 1. No identifiers + # or vendor-specific data are included in this local control. + bits = bytearray(29) + def field(offset, width, value): + for index in range(width): + if value & (1 << (width - index - 1)): + position = offset + index + bits[position // 8] |= 1 << (7 - position % 8) + field(0, 6, 2) + field(6, 36, int(time.time() * 10)) + field(42, 36, int(time.time() * 10)) + field(108, 6, 4) + field(114, 6, 13) + field(152, 1, 1) + headers["Cookie"] = "euconsent-v2=" + base64.urlsafe_b64encode(bits).decode().rstrip("=") + request = urllib.request.Request(f"http://127.0.0.1:{app_port}{path}", headers=headers) + with urllib.request.urlopen(request, timeout=10) as response: + return response.read(), response.headers + + try: + command(["cargo", "build", "-p", "trusted-server-cli", "--bin", "ts", "--locked"]) + cli = str(target / "debug/ts") + crate = ROOT / f"crates/trusted-server-adapter-{adapter}" + if adapter == "cloudflare": + worker_args = ["worker-build", "--release" if release else "--dev", "--out-dir", str(workspace / "build"), "--features", "cloudflare,test-utils", "--locked"] + if not release: + worker_args += ["--no-opt"] + command(worker_args, cwd=crate) + artifact = workspace / "build/index.js" + else: + args = ["cargo", "build", "-p", f"trusted-server-adapter-{adapter}", "--locked", + "--features", "trusted-server-core/test-utils" + (",spin,test-utils" if adapter == "spin" else "")] + if adapter != "axum": + args += ["--target", "wasm32-wasip1"] + if release: + args += ["--release"] + command(args) + profile = "release" if release else "debug" + artifact = target / (f"{profile}/trusted-server-axum" if adapter == "axum" else + f"wasm32-wasip1/{profile}/{'trusted_server_adapter_spin' if adapter == 'spin' else 'trusted-server-adapter-fastly'}.wasm") + assert artifact.exists(), f"fresh artifact missing: {artifact}" + launch(["python3", str(ROOT / "scripts/fixtures/async-html-finalization-origin.py"), + "--port", str(origin_port), "--events", str(workspace / "events.jsonl")], workspace / "origin.log") + config = workspace / "trusted-server.toml" + command([cli, "config", "init", "--app-config", str(config)]) + base_config = config.read_text().replace("[publisher]\n", "[publisher]\nhtml_finalization_timeout_ms = 750\n", 1) + manifest = (ROOT / "edgezero.toml").read_text().replace('= "crates/', f'= "{ROOT}/crates/') + manifest = manifest.replace('manifest = "fastly.toml"', f'manifest = "{workspace}/fastly.toml"') + manifest = manifest.replace(str(crate / "spin.toml"), str(workspace / "spin.toml")) + (workspace / "edgezero.toml").write_text(manifest) + fastly_base = 'manifest_version = 3\nname = "example-finalization"\nlanguage = "rust"\n[local_server]\n' + (workspace / "fastly.toml").write_text(fastly_base) + if adapter == "spin": + (workspace / "spin.toml").write_text((crate / "spin.toml").read_text().replace( + '../../target/wasm32-wasip1/release/trusted_server_adapter_spin.wasm', str(artifact))) + + baseline = None + cases = ["disabled", "success", "hung"] + (["auction"] if auction else []) + for case in cases: + case_config = base_config + if case == "auction": + # A closed loopback bid endpoint exercises transport failure and + # empty-bid assembly without sending traffic outside this host. + bid_port = free_port() + table = case_config.index("[auction]\n") + enabled = case_config.index("enabled = false", table) + case_config = case_config[:enabled] + case_config[enabled:].replace("enabled = false", "enabled = true", 1) + case_config = case_config.replace('endpoint = "https://prebid.example.com/openrtb2/auction"', f'endpoint = "https://127.0.0.1:{bid_port}/bid"', 1) + case_config += '\n[[creative_opportunities.slot]]\nid = "example-slot"\npage_patterns = ["/article"]\nformats = [{ width = 300, height = 250 }]\n' + lookup_case = "success" if case == "auction" else case + config.write_text(case_config + f'\n[integrations.html_finalization_fixture]\nenabled = {str(case != "disabled").lower()}\nlookup_url = "http://127.0.0.1:{origin_port}/{lookup_case}"\ntransport_timeout_ms = 30000\n') + # Axum's local JSON provides the same neutral app-config envelope + # used by the other runtime bindings; no operator config is read. + command([cli, "config", "push", "--adapter", "axum", "--local", "--manifest", str(workspace / "edgezero.toml"), + "--app-config", str(config), "--yes", "--no-diff"], cwd=workspace) + envelope = json.loads((workspace / ".edgezero/local-config-trusted_server_config.json").read_text())["trusted_server_config"] + runtime_env = env.copy() + if adapter == "axum": + runtime_env["TRUSTED_SERVER_CONFIG_TRUSTED_SERVER_CONFIG_TRUSTED_SERVER_CONFIG"] = envelope + for key, value in SECRETS.items(): + runtime_env[f"TRUSTED_SERVER_SECRET_TRUSTED_SERVER_SECRETS_{key.upper()}"] = value + runtime_env["PORT"] = str(app_port) + args = [str(artifact)] + elif adapter == "cloudflare": + text = (crate / "wrangler.ci.toml").read_text().replace('main = "build/index.js"', f'main = "{artifact}"') + text = text.replace('TRUSTED_SERVER_CONFIG = "{}"', "TRUSTED_SERVER_CONFIG = '''" + json.dumps({"app_config": envelope}) + "'''") + text += "\n" + "\n".join(f'{key} = "{value}"' for key, value in SECRETS.items()) + "\n" + (workspace / "wrangler.toml").write_text(text) + args = ["wrangler", "dev", "--local", "--config", str(workspace / "wrangler.toml"), + "--ip", "127.0.0.1", "--port", str(app_port), "--persist-to", str(workspace / "worker-state")] + else: + if adapter == "fastly": + (workspace / "fastly.toml").write_text(fastly_base) + command([cli, "config", "push", "--adapter", adapter, "--local", "--manifest", str(workspace / "edgezero.toml"), + "--app-config", str(config), "--yes", "--no-diff"], cwd=workspace, overrides={"EDGEZERO__STORES__CONFIG__TRUSTED_SERVER_CONFIG__NAME": "default"} if adapter == "spin" else None) + if adapter == "fastly": + with (workspace / "fastly.toml").open("a") as output: + for key, value in SECRETS.items(): + output.write(f'\n[[local_server.secret_stores.trusted_server_secrets]]\nkey = "{key}"\ndata = "{value}"\n') + args = ["viceroy", "--config", str(workspace / "fastly.toml"), "--addr", f"127.0.0.1:{app_port}", str(artifact)] + else: + args = ["spin", "up", "--from", str(workspace / "spin.toml"), "--listen", f"127.0.0.1:{app_port}", + "--state-dir", str(workspace / ".spin"), "--cache-dir", str(workspace / "spin-cache"), "--log-dir", str(workspace / "spin-logs")] + for key, value in SECRETS.items(): + encoded = "v_trusted_x5fserver_x5fsecrets_v_" + key.replace("_", "_x5f") + args += ["--variable", f"{encoded}={value}"] + log = workspace / f"{case}.log" + process = launch(args, log, runtime_env=runtime_env) + for _ in range(80): + if process.poll() is not None: + raise RuntimeError(f"{adapter} exited: {log.read_text()}") + try: + with urllib.request.urlopen(urllib.request.Request(f"http://127.0.0.1:{app_port}/health", method="HEAD"), timeout=1) as ready: + if ready.status == 200: + break + except (OSError, urllib.error.URLError): + time.sleep(0.1) + else: + raise RuntimeError(f"runtime did not become ready: {log.read_text()}") + started = time.monotonic() + try: + body, headers = get(f"/article?case={case}", case == "auction") + except urllib.error.HTTPError as error: + (workspace / f"{case}.error.body").write_bytes(error.read()) + (workspace / f"{case}.headers").write_text(str(error.headers)) + time.sleep(0.3) + raise RuntimeError(f"{adapter} {case} returned HTTP {error.code}; see {log}") from error + completed = time.monotonic() + elapsed = time.monotonic() - started + (workspace / f"{case}.body").write_bytes(body) + (workspace / f"{case}.headers").write_text(str(headers)) + if case == "disabled": + assert b"EXAMPLE_UNSIGNED" in body and b"EXAMPLE_FINALIZED" not in body + baseline = body + elif case == "success": + assert body == baseline.replace(b"EXAMPLE_UNSIGNED", b"EXAMPLE_FINALIZED"), "successful terminal output must match the exact expected rewrite" + assert elapsed >= 0.15, "success must actually await delayed lookup" + elif case == "auction": + assert b"EXAMPLE_FINALIZED" in body, "auction-enabled output must await finalization" + assert b"adSlots" in body and b"bids" in body, "auction control must reach slot and bid assembly" + assert b"ts-inline-body-close-" not in body, "auction assembly must consume the opaque parser marker" + else: + assert body == baseline, "timeout must emit byte-identical processed input" + assert 0.6 <= elapsed < 3, f"timeout escaped the 750ms budget: {elapsed}" + diagnostic = log.read_text() + if adapter == "spin": + diagnostic += "\n".join(path.read_text() for path in (workspace / "spin-logs").glob("*stderr*")) + assert "HTML finalizer html_finalization_fixture times out" in diagnostic or (adapter == "fastly" and "retains input: unavailable" in diagnostic), "missing controlled fallback diagnostic" + if case != "disabled": + assert headers["Cache-Control"] == "no-store, private" + current_events = [json.loads(line) for line in (workspace / "events.jsonl").read_text().splitlines()] + lookups = [event for event in current_events if event["event"] == "lookup_start"] + lookup_elapsed = completed - lookups[-1]["time"] if case != "disabled" else None + terminal_ms = None + if case == "hung": + measured = re.search(r"times out after (\d+) ms", diagnostic) + if measured: + terminal_ms = int(measured.group(1)) + assert 700 <= terminal_ms < 2500, "terminal budget must exclude origin transfer" + (workspace / f"{case}.timing.json").write_text(json.dumps({"request_seconds": elapsed, "terminal_ms": terminal_ms, "lookup_to_response_seconds": lookup_elapsed, "configured_budget_ms": 750, "cooperative_timer": adapter != "fastly"})) + print(f"PASS {adapter} {case}: request={elapsed:.3f}s, {len(body)} bytes, chain_budget=750ms, terminal_ms={terminal_ms}, lookup_to_response={lookup_elapsed}, cooperative_timer={adapter != 'fastly'}", flush=True) + stop(process) + events = [json.loads(line) for line in (workspace / "events.jsonl").read_text().splitlines()] + starts = [event for event in events if event["event"] == "lookup_start"] + assert len(starts) == 2 + int(auction) and all(event["complete"] and len(event["sha256"]) == 64 for event in starts) + for event in starts: + assert any(previous["event"] == "origin_eof" and previous["time"] <= event["time"] for previous in events) + if adapter == "fastly": + assert 0 < event["transport_timeout_ms"] <= 750, "blocking host calls must use the remaining budget" + else: + assert event["transport_timeout_ms"] == 30000, "SDK timer must release before the transport deadline" + assert any(event["event"] == "lookup_complete" and event["path"] == "/success" for event in events) + finally: + for process in reversed(processes): + stop(process) + + + +if __name__ == "__main__": + parser = argparse.ArgumentParser() + parser.add_argument("adapter", choices=["fastly", "cloudflare", "spin", "axum"]) + parser.add_argument("--auction", action="store_true", help="also exercise enabled auction transport-failure assembly") + parser.add_argument("--release", action="store_true", help="build production-profile fixture artifacts") + args = parser.parse_args() + run(args.adapter, args.auction, args.release) diff --git a/trusted-server.example.toml b/trusted-server.example.toml index e841515ae..498d6081a 100644 --- a/trusted-server.example.toml +++ b/trusted-server.example.toml @@ -67,6 +67,8 @@ proxy_secret = "publisher_proxy_secret" # rewriting/injection) instead of streamed. Default 16 MiB; larger responses # return 502. Raise for deployments serving very large publisher pages. # max_buffered_body_bytes = 16777216 # 16 MiB +# Optional asynchronous HTML finalizers share this budget after synchronous EOF. +# html_finalization_timeout_ms = 5000 # Valid range: 1..60000 ms # ----------------------------------------------------------------------------- From 853f293a46b772b166d114ba56c4f65b2bb22a84 Mon Sep 17 00:00:00 2001 From: prk-Jr Date: Mon, 12 Oct 2026 19:26:03 +0530 Subject: [PATCH 03/11] Use a platform clock for auction telemetry --- crates/trusted-server-core/src/auction/telemetry.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/crates/trusted-server-core/src/auction/telemetry.rs b/crates/trusted-server-core/src/auction/telemetry.rs index c6e37d02e..ab7de3c32 100644 --- a/crates/trusted-server-core/src/auction/telemetry.rs +++ b/crates/trusted-server-core/src/auction/telemetry.rs @@ -4,12 +4,12 @@ //! adapters provide the concrete sink implementation. use std::collections::HashSet; -use std::time::Instant; use chrono::Utc; use error_stack::Report; use serde::Serialize; use uuid::Uuid; +use web_time::Instant; use crate::auction::orchestrator::OrchestrationResult; use crate::auction::types::{AuctionRequest, AuctionResponse, Bid, BidStatus, MediaType}; From ee6f082fd7beabb1c2e2f24509e50258ad45d6a5 Mon Sep 17 00:00:00 2001 From: prk-Jr Date: Mon, 12 Oct 2026 19:29:29 +0530 Subject: [PATCH 04/11] Preserve partial HTML and harden finalization --- .github/workflows/test.yml | 25 + .../src/integrations/html_finalization.rs | 21 +- .../src/integrations/registry.rs | 10 +- .../trusted-server-core/src/platform/mod.rs | 18 +- crates/trusted-server-core/src/publisher.rs | 852 +++++++++++++++++- .../src/publisher/html_finalization.rs | 67 +- crates/trusted-server-core/src/settings.rs | 34 +- docs/guide/configuration.md | 8 + docs/guide/integration-guide.md | 42 +- ...2026-10-12-1241-async-html-finalization.md | 11 +- .../plans/2026-10-12-1241-review-hardening.md | 179 ++++ ...-async-html-finalization-implementation.md | 38 +- ...-12-1241-async-html-finalization-design.md | 34 +- .../async-html-finalization-runtime.py | 3 + .../test_async_html_finalization_runtime.py | 65 ++ 15 files changed, 1302 insertions(+), 105 deletions(-) create mode 100644 docs/superpowers/plans/2026-10-12-1241-review-hardening.md create mode 100644 scripts/fixtures/test_async_html_finalization_runtime.py diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 95654ecdd..d5eb8af67 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -106,6 +106,31 @@ jobs: - name: Run Axum adapter tests run: cargo test-axum + - name: Set up Python for actual-runtime finalization checks + uses: actions/setup-python@v5 + with: + python-version: '3.12' + + - name: Verify finalization harness process cleanup + env: + PYTHONDONTWRITEBYTECODE: '1' + run: python3 -m unittest discover -s scripts/fixtures -p 'test_async_html_finalization_runtime.py' -v + + - name: Verify actual Axum HTML finalization and auction assembly + timeout-minutes: 10 + env: + CARGO_TARGET_DIR: ${{ github.workspace }}/target + TMPDIR: ${{ runner.temp }} + run: ./scripts/async-html-finalization-local-test.sh axum --auction + + - name: Upload finalization runtime diagnostics on failure + if: failure() + uses: actions/upload-artifact@v4 + with: + name: axum-html-finalization-diagnostics + path: ${{ runner.temp }}/trusted-server-finalize-axum-*/ + if-no-files-found: ignore + - name: Test template build digest inputs (native) run: cargo test -p trusted-server-core --test template_build_digest diff --git a/crates/trusted-server-core/src/integrations/html_finalization.rs b/crates/trusted-server-core/src/integrations/html_finalization.rs index 6b2dab09e..f2296e290 100644 --- a/crates/trusted-server-core/src/integrations/html_finalization.rs +++ b/crates/trusted-server-core/src/integrations/html_finalization.rs @@ -4,6 +4,7 @@ use std::time::Duration; use error_stack::Report; use futures::future::LocalBoxFuture; +use http::StatusCode; use web_time::Instant; use super::IntegrationHtmlStreamContext; @@ -50,6 +51,8 @@ pub trait IntegrationHtmlFinalizer { /// Borrows the current document, separate candidate storage, and request services. pub struct HtmlFinalizationInput<'a> { + /// Origin status of this complete response, including eligible error HTML. + pub response_status: StatusCode, /// Complete transformed HTML; ownership stays with the driver. pub html: &'a [u8], /// Separate bounded sink for an atomic replacement candidate. @@ -125,16 +128,14 @@ impl HtmlReplacementWriter { .len() .checked_add(chunk.len()) .filter(|length| *length <= self.limit); - if self.poisoned || next.is_none() { - self.poisoned = true; - return Err(Report::new(HtmlFinalizationError::OutputLimitExceeded { - limit: self.limit, - })); - } - let Some(next) = next else { - return Err(Report::new(HtmlFinalizationError::OutputLimitExceeded { - limit: self.limit, - })); + let next = match next { + Some(next) if !self.poisoned => next, + _ => { + self.poisoned = true; + return Err(Report::new(HtmlFinalizationError::OutputLimitExceeded { + limit: self.limit, + })); + } }; if next > self.bytes.capacity() { let capacity = self diff --git a/crates/trusted-server-core/src/integrations/registry.rs b/crates/trusted-server-core/src/integrations/registry.rs index ab27cad80..af177c0f1 100644 --- a/crates/trusted-server-core/src/integrations/registry.rs +++ b/crates/trusted-server-core/src/integrations/registry.rs @@ -1463,6 +1463,11 @@ impl IntegrationRegistry { #[cfg(test)] mod tests { + use super::*; + use crate::constants::COOKIE_TS_EC; + use crate::platform::test_support::noop_services; + use http::{HeaderValue, StatusCode, header}; + #[test] fn html_finalizers_are_empty_by_default() { let registry = IntegrationRegistry::from_request_filters(Vec::new()); @@ -1481,11 +1486,6 @@ mod tests { ); } - use super::*; - use crate::constants::COOKIE_TS_EC; - use crate::platform::test_support::noop_services; - use http::{HeaderValue, StatusCode, header}; - struct DefaultMetadataHeadInjector; impl IntegrationHeadInjector for DefaultMetadataHeadInjector { diff --git a/crates/trusted-server-core/src/platform/mod.rs b/crates/trusted-server-core/src/platform/mod.rs index 99f9f5d9a..71258e9c5 100644 --- a/crates/trusted-server-core/src/platform/mod.rs +++ b/crates/trusted-server-core/src/platform/mod.rs @@ -85,15 +85,6 @@ pub(crate) const DEFAULT_FIRST_BYTE_TIMEOUT: Duration = Duration::from_secs(15); #[cfg(test)] mod tests { - #[test] - fn runtime_timer_is_optional() { - let services = noop_services(); - assert!( - services.timer().is_none(), - "should allow platforms without cooperative timers" - ); - } - use std::net::{IpAddr, Ipv4Addr}; use std::sync::Arc; use std::time::Duration; @@ -105,6 +96,15 @@ mod tests { use super::test_support::{noop_services, noop_services_with_client_ip}; use super::*; + #[test] + fn runtime_timer_is_optional() { + let services = noop_services(); + assert!( + services.timer().is_none(), + "should allow platforms without cooperative timers" + ); + } + struct MarkerKvStore(&'static str); #[async_trait(?Send)] diff --git a/crates/trusted-server-core/src/publisher.rs b/crates/trusted-server-core/src/publisher.rs index 18389ef1e..b43b38df0 100644 --- a/crates/trusted-server-core/src/publisher.rs +++ b/crates/trusted-server-core/src/publisher.rs @@ -1737,6 +1737,10 @@ pub struct OwnedProcessResponseParams { pub(crate) request_host: String, pub(crate) request_scheme: String, pub(crate) content_type: String, + /// Authoritative origin status exposed to terminal integrations. + pub(crate) response_status: StatusCode, + /// Whether status or headers identify a range-associated representation. + pub(crate) range_associated: bool, pub(crate) ad_slots_script: Option, pub(crate) ad_bids_state: AdBidsState, /// Observation context for the in-flight auction. @@ -1762,6 +1766,12 @@ pub struct OwnedProcessResponseParams { pub(crate) csp_nonce_observed: Option>, } +impl OwnedProcessResponseParams { + fn is_range_associated(&self) -> bool { + self.range_associated || self.response_status == StatusCode::PARTIAL_CONTENT + } +} + /// Response-authorized template cache insert inputs. The key is built before origin lookup; the /// lifetime is only known after the origin proves this representation is shareable. pub(crate) struct AuthorizedTemplateStore { @@ -1853,6 +1863,16 @@ pub async fn buffer_publisher_response_async( } return Ok(response); } + params.response_status = response.status(); + params.range_associated = response_is_range_associated(&response); + if params.is_range_associated() + && is_html_content_type(¶ms.content_type) + && integration_registry.has_html_finalizers() + { + abandon_response_auction(&mut params, services, "partial_response").await; + *response.body_mut() = body; + return Ok(response); + } if is_html_content_type(¶ms.content_type) && integration_registry.has_html_finalizers() { @@ -2176,6 +2196,8 @@ fn build_template_assembly_params( ad_bids_state: AdBidsState, ) -> OwnedProcessResponseParams { OwnedProcessResponseParams { + response_status: StatusCode::OK, + range_associated: false, csp_nonce_observed: None, // Already stored; storing again on a hit would be pointless work. template_cache_key: None, @@ -2605,6 +2627,16 @@ pub async fn publisher_response_into_streaming_response( return Ok(response); } + params.response_status = response.status(); + params.range_associated = response_is_range_associated(&response); + if params.is_range_associated() + && is_html_content_type(¶ms.content_type) + && integration_registry.has_html_finalizers() + { + abandon_response_auction(&mut params, &services, "partial_response").await; + *response.body_mut() = body; + return Ok(response); + } response.headers_mut().remove(header::CONTENT_LENGTH); let mut params = *params; if is_html_content_type(¶ms.content_type) @@ -2851,6 +2883,11 @@ fn response_carries_body(method: &Method, status: StatusCode) -> bool { && status != StatusCode::NOT_MODIFIED } +fn response_is_range_associated(response: &Response) -> bool { + response.status() == StatusCode::PARTIAL_CONTENT + || response.headers().contains_key(header::CONTENT_RANGE) +} + /// Prevent shared caches from replaying tag-suppressed HTML to other clients. fn apply_datadome_client_tag_cache_privacy( response: &mut Response, @@ -2977,6 +3014,18 @@ pub fn stream_publisher_body( settings: &Settings, integration_registry: &IntegrationRegistry, ) -> Result<(), Report> { + if params.is_range_associated() + && is_html_content_type(¶ms.content_type) + && integration_registry.has_html_finalizers() + { + let mut input = body_as_reader(body)?; + std::io::copy(&mut input, output).change_context(TrustedServerError::Proxy { + message: "Failed to copy range-associated HTML".to_string(), + })?; + return output.flush().change_context(TrustedServerError::Proxy { + message: "Failed to flush range-associated HTML".to_string(), + }); + } let borrowed = ProcessResponseParams { content_encoding: ¶ms.content_encoding, origin_host: ¶ms.origin_host, @@ -3002,6 +3051,22 @@ pub fn stream_publisher_body( process_response_streaming(body, output, &borrowed, output_compression) } +async fn abandon_response_auction( + params: &mut OwnedProcessResponseParams, + services: &RuntimeServices, + reason: &'static str, +) { + if let Some(dispatched) = params.dispatched_auction.take() { + emit_abandoned_auction( + services, + params.auction_observation.take(), + dispatched, + reason, + ) + .await; + } +} + async fn build_html_terminal_driver( body: EdgeBody, params: &mut OwnedProcessResponseParams, @@ -3055,7 +3120,7 @@ async fn stream_html_terminal( ); } } - if (step.close_found || driver.input_finished()) + if (step.close_found || driver.at_finish_barrier()) && let Err(error) = output.flush() { driver.abandon(&deps, "stream_write_error").await; @@ -3112,6 +3177,24 @@ pub async fn stream_publisher_body_async( orchestrator: &AuctionOrchestrator, services: &RuntimeServices, ) -> Result<(), Report> { + if params.is_range_associated() + && is_html_content_type(¶ms.content_type) + && integration_registry.has_html_finalizers() + { + abandon_response_auction(params, services, "partial_response").await; + let mut source = BodyChunkSource::new(body, STREAM_CHUNK_SIZE) + .with_max_bytes(settings.publisher.max_buffered_body_bytes); + while let Some(bytes) = source.next_chunk().await? { + output + .write_all(&bytes) + .change_context(TrustedServerError::Proxy { + message: "Failed to copy range-associated HTML".to_string(), + })?; + } + return output.flush().change_context(TrustedServerError::Proxy { + message: "Failed to flush range-associated HTML".to_string(), + }); + } if is_html_content_type(¶ms.content_type) && integration_registry.has_html_finalizers() { return Box::pin(stream_html_terminal( body, @@ -5512,7 +5595,12 @@ pub async fn handle_publisher_request( // caching directives, letting a downstream cache serve it to a later eligible reader. let assembled_response_must_be_private = template_cache_key.is_some(); let is_not_modified = response.status() == StatusCode::NOT_MODIFIED; - if is_html_content_type(&origin_content_type) || is_not_modified { + let range_associated_html_bypass = html_finalizers_present + && is_html_content_type(&origin_content_type) + && response_is_range_associated(&response); + if !range_associated_html_bypass + && (is_html_content_type(&origin_content_type) || is_not_modified) + { if should_run_ad_stack || assembled_response_must_be_private { enforce_synthesized_html_cache_privacy(&mut response); } else if is_server_side_ad_eligible_navigation( @@ -5533,12 +5621,14 @@ pub async fn handle_publisher_request( } } } - apply_datadome_client_tag_cache_privacy( - &mut response, - &request_method, - suppress_datadome_client_side_tag, - &origin_content_type, - ); + if !range_associated_html_bypass { + apply_datadome_client_tag_cache_privacy( + &mut response, + &request_method, + suppress_datadome_client_side_tag, + &origin_content_type, + ); + } apply_publisher_asset_cache_policy( settings, &request_path, @@ -5565,7 +5655,11 @@ pub async fn handle_publisher_request( .unwrap_or_default() .trim() .to_ascii_lowercase(); - let route = classify_response_route(status, &content_type, &content_encoding, request_host); + let route = if range_associated_html_bypass { + ResponseRoute::BufferedUnmodified + } else { + classify_response_route(status, &content_type, &content_encoding, request_host) + }; if html_finalizers_present && route == ResponseRoute::Stream && is_html_content_type(&content_type) @@ -5657,6 +5751,8 @@ pub async fn handle_publisher_request( response, body, params: Box::new(OwnedProcessResponseParams { + response_status: status, + range_associated: false, // The transform writes here; the post-transform gate reads it. Always // present on the live path so no future caller has to remember to // supply it — the handlers themselves stay gated on authorization. @@ -9167,6 +9263,8 @@ mod tests { content_encoding: &str, ) -> OwnedProcessResponseParams { OwnedProcessResponseParams { + response_status: StatusCode::OK, + range_associated: false, csp_nonce_observed: None, template_cache_key: None, seam_ad_slots: None, @@ -10755,6 +10853,135 @@ mod tests { ); } + #[tokio::test] + async fn html_finalization_range_bypass_preserves_origin_policy_with_live_or_suppressed_ads() + { + for (live, suppressed) in [(true, false), (false, false), (false, true)] { + for (status, content_range) in [(206, false), (200, true), (416, true)] { + let mut settings = settings_with_bidder("inline"); + settings.auction.enabled = live; + let settings = Arc::new(settings); + let stub = Arc::new(StubHttpClient::new()); + let sink = Arc::new(RecordingTelemetrySink::default()); + let services = services_with_cache_and_telemetry( + Arc::clone(&stub), + Arc::new(MemoryTemplateCache::default()), + Arc::clone(&sink), + ); + let calls = Arc::new(AtomicUsize::new(0)); + let registry = IntegrationRegistry::from_html_finalizers(vec![Arc::new( + super::CountingTerminalFactory(Arc::clone(&calls)), + )]); + let mut orchestrator = AuctionOrchestrator::new(settings.auction.clone()); + orchestrator.register_provider(Arc::new(WinningBidProvider { + price: STUB_BID_CPM, + })); + if live { + queue_bid_response(&stub); + } + let raw = b"

Example partial article

"; + let mut headers = vec![ + ("content-type", "text/html"), + ("content-length", "30"), + ("cache-control", "public, max-age=60"), + ("etag", "example-origin-tag"), + ("last-modified", "Mon, 01 Jan 2024 00:00:00 GMT"), + ("surrogate-control", "max-age=60"), + ]; + if content_range { + headers.push(("content-range", "bytes 0-29/100")); + } + stub.push_response_with_headers(status, raw.to_vec(), headers); + let consent = crate::consent::ConsentContext { + jurisdiction: crate::consent::jurisdiction::Jurisdiction::NonRegulated, + ..Default::default() + }; + let mut ec_context = EcContext::new_for_test(None, consent); + let mut request = navigation_request(); + if suppressed { + request + .extensions_mut() + .insert(crate::integrations::datadome::DataDomeClientTagSuppressed); + } + let publisher = handle_publisher_request( + &settings, + &services, + None, + &mut ec_context, + AuctionDispatch { + integration_registry: ®istry, + orchestrator: &orchestrator, + slots: &[article_slot()], + registry: None, + }, + request, + EdgeCacheHeader::SMaxageFallback, + ) + .await + .expect("should classify partial HTML before synthesized policy"); + let response = publisher_response_into_streaming_response( + publisher, + &Method::GET, + settings, + ®istry, + Arc::new(orchestrator), + services, + ) + .await + .expect("should preserve partial origin delivery"); + for (name, expected) in [ + (header::CONTENT_LENGTH, "30"), + (header::CACHE_CONTROL, "public, max-age=60"), + (header::ETAG, "example-origin-tag"), + (header::LAST_MODIFIED, "Mon, 01 Jan 2024 00:00:00 GMT"), + ] { + assert_eq!( + response + .headers() + .get(name) + .and_then(|value| value.to_str().ok()), + Some(expected), + "should preserve origin policy for live={live}, suppressed={suppressed}, status={status}" + ); + } + assert_eq!( + response + .headers() + .get("surrogate-control") + .and_then(|value| value.to_str().ok()), + Some("max-age=60"), + "should preserve origin edge cache policy" + ); + assert_eq!( + body_of(response).await, + raw, + "should preserve partial article bytes" + ); + assert_eq!( + calls.load(Ordering::SeqCst), + 0, + "should perform no terminal I/O" + ); + let batches = sink.batches.lock().expect("should inspect range telemetry"); + let summaries: Vec<_> = batches + .iter() + .flat_map(crate::auction::telemetry::AuctionEventBatch::rows) + .filter(|row| row.event_kind == "summary") + .collect(); + assert_eq!( + summaries.len(), + 1, + "should record one terminal auction outcome" + ); + assert_eq!( + summaries[0].terminal_status.as_deref(), + Some(if live { "abandoned" } else { "skipped" }), + "should abandon dispatched bids once and preserve ordinary skipped telemetry" + ); + } + } + } + #[derive(Default)] struct RawOriginCacheClient { cached: Mutex>>, @@ -20034,6 +20261,8 @@ mod tests { let body = EdgeBody::from(compressed); let params = OwnedProcessResponseParams { + response_status: StatusCode::OK, + range_associated: false, csp_nonce_observed: None, template_cache_key: None, seam_ad_slots: None, @@ -20093,6 +20322,8 @@ mod tests { .expect("should create integration registry"); let params = OwnedProcessResponseParams { + response_status: StatusCode::OK, + range_associated: false, csp_nonce_observed: None, template_cache_key: None, seam_ad_slots: None, @@ -20141,6 +20372,8 @@ mod tests { ) .expect("should create integration registry"); let params = OwnedProcessResponseParams { + response_status: StatusCode::OK, + range_associated: false, csp_nonce_observed: None, template_cache_key: None, seam_ad_slots: None, @@ -20267,6 +20500,8 @@ mod tests { let orchestrator = AuctionOrchestrator::new(settings.auction.clone()); let services = noop_services(); let mut params = OwnedProcessResponseParams { + response_status: StatusCode::OK, + range_associated: false, csp_nonce_observed: None, template_cache_key: None, seam_ad_slots: None, @@ -20331,6 +20566,8 @@ mod tests { let orchestrator = AuctionOrchestrator::new(settings.auction.clone()); let services = noop_services(); let mut params = OwnedProcessResponseParams { + response_status: StatusCode::OK, + range_associated: false, csp_nonce_observed: None, template_cache_key: None, seam_ad_slots: None, @@ -20398,6 +20635,8 @@ mod tests { let orchestrator = AuctionOrchestrator::new(settings.auction.clone()); let services = noop_services(); let mut params = OwnedProcessResponseParams { + response_status: StatusCode::OK, + range_associated: false, csp_nonce_observed: None, template_cache_key: None, seam_ad_slots: None, @@ -20465,6 +20704,8 @@ mod tests { let orchestrator = AuctionOrchestrator::new(settings.auction.clone()); let services = noop_services(); let mut params = OwnedProcessResponseParams { + response_status: StatusCode::OK, + range_associated: false, csp_nonce_observed: None, template_cache_key: None, seam_ad_slots: None, @@ -20532,6 +20773,8 @@ mod tests { let orchestrator = AuctionOrchestrator::new(settings.auction.clone()); let services = noop_services(); let mut params = OwnedProcessResponseParams { + response_status: StatusCode::OK, + range_associated: false, csp_nonce_observed: None, template_cache_key: None, seam_ad_slots: None, @@ -20581,6 +20824,8 @@ mod tests { fn non_html_stream_params(content_encoding: &str) -> OwnedProcessResponseParams { OwnedProcessResponseParams { + response_status: StatusCode::OK, + range_associated: false, csp_nonce_observed: None, template_cache_key: None, seam_ad_slots: None, @@ -20798,6 +21043,8 @@ mod tests { let services = noop_services(); let state = AdBidsState::default(); let mut params = OwnedProcessResponseParams { + response_status: StatusCode::OK, + range_associated: false, csp_nonce_observed: None, template_cache_key: None, seam_ad_slots: None, @@ -20873,6 +21120,8 @@ mod tests { let services = noop_services(); let state = AdBidsState::default(); let mut params = OwnedProcessResponseParams { + response_status: StatusCode::OK, + range_associated: false, csp_nonce_observed: None, template_cache_key: None, seam_ad_slots: None, @@ -20950,6 +21199,8 @@ mod tests { let orchestrator = AuctionOrchestrator::new(settings.auction.clone()); let services = noop_services(); let mut params = OwnedProcessResponseParams { + response_status: StatusCode::OK, + range_associated: false, csp_nonce_observed: None, template_cache_key: None, seam_ad_slots: None, @@ -21021,6 +21272,8 @@ mod tests { .body(EdgeBody::empty()) .expect("should build response"); let params = OwnedProcessResponseParams { + response_status: StatusCode::OK, + range_associated: false, csp_nonce_observed: None, template_cache_key: None, seam_ad_slots: None, @@ -21238,6 +21491,573 @@ mod tests { } } + #[test] + fn html_finalization_preserves_range_associated_responses_on_both_sinks() { + for (status, content_range) in [(206, true), (206, false), (200, true), (416, true)] { + for streamed in [false, true] { + for buffered in [false, true] { + for encoding in ["", "gzip"] { + for auction in [false, true] { + let settings = Arc::new(create_test_settings()); + let calls = Arc::new(AtomicUsize::new(0)); + let registry = + IntegrationRegistry::from_html_finalizers(vec![Arc::new( + CountingTerminalFactory(Arc::clone(&calls)), + )]); + let raw = b"

Example range fragment

"; + let encoded = if encoding == "gzip" { + gzip_encode(raw) + } else { + raw.to_vec() + }; + let body = if streamed { + EdgeBody::stream(futures::stream::iter(vec![bytes::Bytes::from( + encoded.clone(), + )])) + } else { + EdgeBody::from(encoded.clone()) + }; + let mut builder = Response::builder() + .status(status) + .header(header::CONTENT_TYPE, "text/html") + .header(header::CONTENT_ENCODING, encoding) + .header(header::CONTENT_LENGTH, encoded.len()) + .header(header::ETAG, "example-range-tag") + .header(header::CACHE_CONTROL, "public, max-age=60"); + if content_range { + builder = builder.header(header::CONTENT_RANGE, "bytes 0-29/100"); + } + let response = builder + .body(EdgeBody::empty()) + .expect("should build range metadata"); + let headers = response.headers().clone(); + let mut params = html_stream_params( + encoding, + auction.then(|| { + DispatchedAuction::empty_for_test(test_auction_request(), 500) + }), + ); + let sink = Arc::new(RecordingTelemetrySink::default()); + let services = + noop_services_with_telemetry_sink(Arc::clone(&sink) as _); + if auction { + params.auction_observation = + Some(AuctionObservationContext::from_parts( + AuctionSource::InitialNavigation, + "proxy.example.com", + "/article", + 1, + None, + &EcContext::default(), + )); + } + let publisher = PublisherResponse::Stream { + response, + body, + params: Box::new(params), + }; + let orchestrator = + Arc::new(AuctionOrchestrator::new(settings.auction.clone())); + let response = if buffered { + futures::executor::block_on(buffer_publisher_response_async( + publisher, + &Method::GET, + &settings, + ®istry, + &orchestrator, + &services, + )) + } else { + futures::executor::block_on( + publisher_response_into_streaming_response( + publisher, + &Method::GET, + settings, + ®istry, + orchestrator, + services, + ), + ) + } + .expect("should preserve range-associated HTML"); + assert_eq!( + response.status().as_u16(), + status, + "should preserve range status" + ); + assert_eq!( + response.headers(), + &headers, + "should preserve range framing and policy" + ); + let actual = futures::executor::block_on( + response.into_body().into_bytes_bounded(1024), + ) + .expect("should drain the original range bytes"); + assert_eq!( + actual.as_ref(), + encoded.as_slice(), + "should preserve encoded range bytes" + ); + assert_eq!( + calls.load(Ordering::SeqCst), + 0, + "should never finalize a range-associated response" + ); + let batches = sink.batches.lock().expect("should lock range telemetry"); + let summaries: Vec<_> = batches + .iter() + .flat_map(crate::auction::telemetry::AuctionEventBatch::rows) + .filter(|row| row.event_kind == "summary") + .collect(); + assert_eq!( + summaries.len(), + usize::from(auction), + "should abandon a dispatched auction exactly once" + ); + if auction { + assert_eq!( + summaries[0].terminal_status.as_deref(), + Some("abandoned"), + "should report range bypass abandonment" + ); + } + } + } + } + } + } + } + + #[test] + fn html_finalization_bypasses_ranged_fetches_and_unsolicited_partial_origins() { + for destination in ["empty", "document"] { + for (status, content_range) in [(206, false), (206, true), (200, true), (416, true)] { + let settings = Arc::new(create_test_settings()); + let calls = Arc::new(AtomicUsize::new(0)); + let registry = IntegrationRegistry::from_html_finalizers(vec![Arc::new( + CountingTerminalFactory(Arc::clone(&calls)), + )]); + let http = Arc::new(StubHttpClient::new()); + let raw = b"

Example fragment

"; + let mut headers = vec![("content-type", "text/html")]; + if content_range { + headers.push(("content-range", "bytes 0-22/100")); + } + http.push_response_with_headers(status, raw.to_vec(), headers); + let services = build_services_with_http_client(http.clone()); + let orchestrator = Arc::new(AuctionOrchestrator::new(settings.auction.clone())); + let request = Request::builder() + .method(Method::GET) + .uri("https://proxy.example.com/article") + .header(header::HOST, "proxy.example.com") + .header(header::ACCEPT, "text/html") + .header("sec-fetch-dest", destination) + .header(header::RANGE, "bytes=0-22") + .body(EdgeBody::empty()) + .expect("should build a ranged fetch"); + let publisher = futures::executor::block_on(handle_publisher_request( + &settings, + &services, + None, + &mut EcContext::default(), + AuctionDispatch { + integration_registry: ®istry, + orchestrator: &orchestrator, + slots: &[], + registry: None, + }, + request, + EdgeCacheHeader::SurrogateControl, + )) + .expect("should fetch a range-associated response"); + let response = + futures::executor::block_on(publisher_response_into_streaming_response( + publisher, + &Method::GET, + settings, + ®istry, + orchestrator, + services, + )) + .expect("should bypass terminal work on partial origin HTML"); + let actual = + futures::executor::block_on(response.into_body().into_bytes_bounded(1024)) + .expect("should drain the range fragment"); + assert_eq!( + calls.load(Ordering::SeqCst), + 0, + "should skip optional I/O on partial origin HTML" + ); + assert_eq!(actual.as_ref(), raw, "should preserve the origin fragment"); + assert_eq!( + http.recorded_request_headers()[0] + .iter() + .any(|(name, _)| name == "range"), + destination == "empty", + "should preserve non-document range semantics" + ); + } + } + } + + #[test] + fn html_finalization_direct_helpers_copy_encoded_ranges_without_optional_work() { + for streamed in [false, true] { + let settings = create_test_settings(); + let calls = Arc::new(AtomicUsize::new(0)); + let registry = IntegrationRegistry::from_html_finalizers(vec![Arc::new( + CountingTerminalFactory(Arc::clone(&calls)), + )]); + let encoded = gzip_encode(b"

Example partial response

"); + let body = if streamed { + EdgeBody::stream(futures::stream::iter(vec![bytes::Bytes::from( + encoded.clone(), + )])) + } else { + EdgeBody::from(encoded.clone()) + }; + let mut params = html_stream_params("gzip", None); + params.response_status = StatusCode::PARTIAL_CONTENT; + params.range_associated = streamed; + let orchestrator = AuctionOrchestrator::new(settings.auction.clone()); + let mut output = Vec::new(); + futures::executor::block_on(stream_publisher_body_async( + body, + &mut output, + &mut params, + &settings, + ®istry, + &orchestrator, + &noop_services(), + )) + .expect("should copy an encoded partial representation"); + assert_eq!( + output, encoded, + "should avoid decoding or rewriting range bytes" + ); + assert_eq!( + calls.load(Ordering::SeqCst), + 0, + "should skip terminal I/O on direct range delivery" + ); + if !streamed { + output.clear(); + stream_publisher_body( + EdgeBody::from(encoded.clone()), + &mut output, + ¶ms, + &settings, + ®istry, + ) + .expect("should copy an encoded range on the synchronous helper"); + assert_eq!( + output, encoded, + "should preserve the direct synchronous range bytes" + ); + } + } + } + + #[test] + fn html_finalization_flushes_at_the_terminal_barrier_instead_of_each_replay_chunk() { + struct FlushCounter { + bytes: Vec, + flushes: usize, + } + impl Write for FlushCounter { + fn write(&mut self, bytes: &[u8]) -> io::Result { + self.bytes.extend_from_slice(bytes); + Ok(bytes.len()) + } + fn flush(&mut self) -> io::Result<()> { + self.flushes += 1; + Ok(()) + } + } + let settings = create_test_settings(); + let calls = Arc::new(AtomicUsize::new(0)); + let registry = IntegrationRegistry::from_html_finalizers(vec![Arc::new( + CountingTerminalFactory(Arc::clone(&calls)), + )]); + let mut source = b"

".to_vec(); + source.extend(vec![b'x'; 5 * STREAM_CHUNK_SIZE]); + source.extend_from_slice(b"

"); + let orchestrator = AuctionOrchestrator::new(settings.auction.clone()); + let mut output = FlushCounter { + bytes: Vec::new(), + flushes: 0, + }; + futures::executor::block_on(stream_publisher_body_async( + EdgeBody::from(source), + &mut output, + &mut html_stream_params("", None), + &settings, + ®istry, + &orchestrator, + &noop_services(), + )) + .expect("should replay a complete large HTML document"); + assert!( + output.bytes.len() > 5 * STREAM_CHUNK_SIZE, + "should replay multiple bounded chunks" + ); + assert_eq!( + output.flushes, 2, + "should flush the EOF barrier and final output, not every replay chunk" + ); + assert_eq!( + calls.load(Ordering::SeqCst), + 1, + "should retain one terminal callback" + ); + } + + #[test] + fn html_finalization_flushes_all_ready_prefix_before_a_pending_auction() { + struct FlushProbe { + bytes: Arc>>, + flushed_bytes: Arc, + flushes: Arc, + } + impl Write for FlushProbe { + fn write(&mut self, bytes: &[u8]) -> io::Result { + self.bytes + .lock() + .expect("should record ready bytes") + .extend_from_slice(bytes); + Ok(bytes.len()) + } + fn flush(&mut self) -> io::Result<()> { + self.flushed_bytes.store( + self.bytes + .lock() + .expect("should inspect flushed bytes") + .len(), + Ordering::SeqCst, + ); + self.flushes.fetch_add(1, Ordering::SeqCst); + Ok(()) + } + } + for closed in [false, true] { + let mut settings = create_test_settings(); + settings.auction.enabled = true; + settings.auction.providers = + crate::auction::AuctionConfig::legacy_provider_map(&["seam-test"]); + settings.auction.timeout_ms = 60_000; + settings.auction.mediator = None; + let client = Arc::new(GatedAuctionHttpClient { + inner: StubHttpClient::new(), + released: AtomicBool::new(false), + collections: AtomicUsize::new(0), + }); + client.inner.push_response(200, Vec::new()); + let services = build_services_with_http_client(client.clone()); + let mut orchestrator = AuctionOrchestrator::new(settings.auction.clone()); + orchestrator.register_provider(Arc::new(GatedAuctionProvider)); + let request = Request::new(EdgeBody::empty()); + let dispatched = futures::executor::block_on(orchestrator.dispatch_auction( + &test_auction_request(), + &AuctionContext { + settings: &settings, + request: &request, + timeout_ms: 60_000, + transport_timeout_ms: 60_000, + provider_responses: None, + services: &services, + }, + )); + let DispatchAuctionOutcome::Dispatched(dispatched) = dispatched else { + panic!("should dispatch a controlled pending auction"); + }; + let calls = Arc::new(AtomicUsize::new(0)); + let registry = IntegrationRegistry::from_html_finalizers(vec![Arc::new( + CountingTerminalFactory(Arc::clone(&calls)), + )]); + let mut source = b"".to_vec(); + source.extend(vec![b'x'; 5 * STREAM_CHUNK_SIZE]); + source.extend_from_slice(b"Example barrier end"); + if closed { + source.extend_from_slice(b""); + } + let bytes = Arc::new(Mutex::new(Vec::new())); + let flushed = Arc::new(AtomicUsize::new(0)); + let flushes = Arc::new(AtomicUsize::new(0)); + let mut writer = FlushProbe { + bytes: Arc::clone(&bytes), + flushed_bytes: Arc::clone(&flushed), + flushes: Arc::clone(&flushes), + }; + let mut params = html_stream_params("", Some(dispatched)); + let mut work = Box::pin(stream_publisher_body_async( + EdgeBody::from(source), + &mut writer, + &mut params, + &settings, + ®istry, + &orchestrator, + &services, + )); + let waker = futures::task::noop_waker(); + let mut context = std::task::Context::from_waker(&waker); + assert!( + work.as_mut().poll(&mut context).is_pending(), + "should await the controlled auction" + ); + let prefix = bytes + .lock() + .expect("should inspect the delivered prefix") + .clone(); + assert!( + prefix + .windows(b"Example barrier end".len()) + .any(|window| window == b"Example barrier end"), + "should deliver the whole ready prefix before collection" + ); + assert_eq!( + flushed.load(Ordering::SeqCst), + prefix.len(), + "should flush every ready prefix byte before collection" + ); + assert_eq!( + flushes.load(Ordering::SeqCst), + 1, + "should use one pre-auction flush barrier" + ); + assert_eq!( + client.collections.load(Ordering::SeqCst), + 1, + "should poll the pending auction once" + ); + client.released.store(true, Ordering::SeqCst); + futures::executor::block_on(work) + .expect("should complete once the auction is released"); + assert_eq!( + calls.load(Ordering::SeqCst), + 1, + "should finish HTML before auction collection" + ); + } + } + + struct StatusTerminalFactory(Arc>>); + struct StatusTerminalSession(Arc>>); + + impl crate::integrations::IntegrationHtmlFinalizerFactory for StatusTerminalFactory { + fn integration_id(&self) -> &'static str { + "example_status" + } + + fn create( + &self, + _: crate::integrations::IntegrationHtmlStreamContext, + ) -> Box { + Box::new(StatusTerminalSession(Arc::clone(&self.0))) + } + } + + impl crate::integrations::IntegrationHtmlFinalizer for StatusTerminalSession { + fn finalize<'a>( + &'a mut self, + input: crate::integrations::HtmlFinalizationInput<'a>, + ) -> futures::future::LocalBoxFuture< + 'a, + Result< + crate::integrations::HtmlFinalization, + Report, + >, + > { + Box::pin(async move { + self.0 + .lock() + .expect("should record the response status") + .push(input.response_status); + Ok(crate::integrations::HtmlFinalization::Unchanged) + }) + } + } + + #[test] + fn html_finalization_exposes_actual_success_and_error_status_on_both_sinks() { + for buffered in [false, true] { + let statuses = Arc::new(Mutex::new(Vec::new())); + let registry = IntegrationRegistry::from_html_finalizers(vec![Arc::new( + StatusTerminalFactory(Arc::clone(&statuses)), + )]); + for status in [200, 404, 500] { + let settings = Arc::new(create_test_settings()); + let http = Arc::new(StubHttpClient::new()); + http.push_response_with_headers( + status, + b"Example complete response".to_vec(), + vec![("content-type", "text/html")], + ); + let services = build_services_with_http_client(http); + let orchestrator = Arc::new(AuctionOrchestrator::new(settings.auction.clone())); + let request = Request::builder() + .uri("https://proxy.example.com/article") + .header(header::HOST, "proxy.example.com") + .header(header::ACCEPT, "text/html") + .body(EdgeBody::empty()) + .expect("should build a complete document request"); + let publisher = futures::executor::block_on(handle_publisher_request( + &settings, + &services, + None, + &mut EcContext::default(), + AuctionDispatch { + integration_registry: ®istry, + orchestrator: &orchestrator, + slots: &[], + registry: None, + }, + request, + EdgeCacheHeader::SurrogateControl, + )) + .expect("should fetch complete success or error HTML"); + let response = if buffered { + futures::executor::block_on(buffer_publisher_response_async( + publisher, + &Method::GET, + &settings, + ®istry, + &orchestrator, + &services, + )) + } else { + futures::executor::block_on(publisher_response_into_streaming_response( + publisher, + &Method::GET, + settings, + ®istry, + orchestrator, + services, + )) + } + .expect("should finalize complete success or error HTML"); + assert_eq!( + response.status().as_u16(), + status, + "should retain the origin status" + ); + let _ = futures::executor::block_on( + response.into_body().into_bytes_bounded(1024 * 1024), + ) + .expect("should drive the complete terminal response"); + } + assert_eq!( + *statuses.lock().expect("should inspect observed statuses"), + [ + StatusCode::OK, + StatusCode::NOT_FOUND, + StatusCode::INTERNAL_SERVER_ERROR + ], + "should expose status without excluding complete error documents" + ); + } + } + #[test] fn html_finalization_handles_eligible_and_excluded_responses() { for (method, status, content_type, encoding) in [ @@ -21898,6 +22718,8 @@ mod tests { dispatched_auction: Option, ) -> OwnedProcessResponseParams { OwnedProcessResponseParams { + response_status: StatusCode::OK, + range_associated: false, csp_nonce_observed: None, template_cache_key: None, seam_ad_slots: None, @@ -22579,6 +23401,8 @@ mod tests { let ec_context = EcContext::new_for_test(None, crate::consent::types::ConsentContext::default()); OwnedProcessResponseParams { + response_status: StatusCode::OK, + range_associated: false, csp_nonce_observed: None, template_cache_key: None, seam_ad_slots: None, @@ -22774,6 +23598,8 @@ mod tests { .map(bytes::Bytes::copy_from_slice) .collect(); let params = OwnedProcessResponseParams { + response_status: StatusCode::OK, + range_associated: false, csp_nonce_observed: None, template_cache_key: None, seam_ad_slots: None, @@ -22855,6 +23681,8 @@ mod tests { r#""#; let state = AdBidsState::with_script(bids_script); let params = OwnedProcessResponseParams { + response_status: StatusCode::OK, + range_associated: false, csp_nonce_observed: None, template_cache_key: None, seam_ad_slots: None, @@ -22919,6 +23747,8 @@ mod tests { // Claim gzip encoding but feed non-gzip bytes. The GzDecoder will // error as soon as it tries to read the gzip header. let params = OwnedProcessResponseParams { + response_status: StatusCode::OK, + range_associated: false, csp_nonce_observed: None, template_cache_key: None, seam_ad_slots: None, @@ -23037,6 +23867,8 @@ mod tests { let body = EdgeBody::from(html.to_vec()); let params = OwnedProcessResponseParams { + response_status: StatusCode::OK, + range_associated: false, csp_nonce_observed: None, template_cache_key: None, seam_ad_slots: None, @@ -23104,6 +23936,8 @@ mod tests { // Small, single-fragment RSC script — placeholder path (not fallback). let html = br#""#; let params = OwnedProcessResponseParams { + response_status: StatusCode::OK, + range_associated: false, csp_nonce_observed: None, template_cache_key: None, seam_ad_slots: None, diff --git a/crates/trusted-server-core/src/publisher/html_finalization.rs b/crates/trusted-server-core/src/publisher/html_finalization.rs index 872a9062c..dca4bcacb 100644 --- a/crates/trusted-server-core/src/publisher/html_finalization.rs +++ b/crates/trusted-server-core/src/publisher/html_finalization.rs @@ -11,6 +11,7 @@ use edgezero_core::body::Body as EdgeBody; use error_stack::{Report, ResultExt as _}; use flate2::{Decompress, FlushDecompress, Status}; use futures::FutureExt as _; +use http::StatusCode; use web_time::Instant; use super::{ @@ -46,6 +47,7 @@ fn log_timeout(integration_id: &str, started_at: Instant, bytes: usize) { } async fn finalize_document( + response_status: StatusCode, mut current: Vec, sessions: &mut [HtmlFinalizerSession], marker: Option<&[u8]>, @@ -65,6 +67,7 @@ async fn finalize_document( let work = session .finalizer .finalize(HtmlFinalizationInput { + response_status, html: ¤t, replacement: &mut replacement, services, @@ -306,6 +309,7 @@ impl TerminalInput { } pub(super) struct HtmlFinalizationDriver { + response_status: StatusCode, input: TerminalInput, encoder: BodyStreamEncoder, processor: Box, @@ -369,6 +373,9 @@ impl HtmlFinalizationDriver { finalizer: factory.create(context.clone()), }) .collect(); + let compression = Compression::from_content_encoding(¶ms.content_encoding); + let limit = settings.publisher.max_buffered_body_bytes; + let input = TerminalInput::new(body, compression, limit)?; let hold = params.dispatched_auction.take().map(|dispatched| { AuctionHoldState::new( DispatchedAuctionGuard::new(dispatched), @@ -379,10 +386,9 @@ impl HtmlFinalizationDriver { marker.clone(), ) }); - let compression = Compression::from_content_encoding(¶ms.content_encoding); - let limit = settings.publisher.max_buffered_body_bytes; Ok(Self { - input: TerminalInput::new(body, compression, limit)?, + response_status: params.response_status, + input, encoder: BodyStreamEncoder::new(compression), processor: Box::new(processor), collector: HtmlCollector::new(limit), @@ -484,6 +490,7 @@ impl HtmlFinalizationDriver { self.source_finished = true; if let Some(current) = self.collector.finish() { let current = finalize_document( + self.response_status, current, &mut self.sessions, self.marker.as_deref(), @@ -510,8 +517,8 @@ impl HtmlFinalizationDriver { Ok(()) } - pub(super) fn input_finished(&self) -> bool { - self.source_finished + pub(super) fn at_finish_barrier(&self) -> bool { + self.source_finished && self.seam_flushed && !self.finished } pub(super) async fn collect_close( @@ -534,25 +541,6 @@ impl HtmlFinalizationDriver { #[cfg(test)] mod tests { - #[test] - fn collection_overflow_replays_prefix_and_crossing_chunk_once() { - let mut collector = HtmlCollector::new(4); - assert!( - collector.push(b"page".to_vec()).is_empty(), - "should retain input at the cap" - ); - let mut emitted = collector.push(b"!".to_vec()).concat(); - emitted.extend(collector.push(b"tail".to_vec()).concat()); - assert_eq!( - emitted, b"page!tail", - "should release overflow without duplication" - ); - assert!( - collector.finish().is_none(), - "should skip finalizer I/O after bypass" - ); - } - use std::cell::{Cell, RefCell}; use std::rc::Rc; use std::time::Duration; @@ -572,6 +560,25 @@ mod tests { use super::*; + #[test] + fn collection_overflow_replays_prefix_and_crossing_chunk_once() { + let mut collector = HtmlCollector::new(4); + assert!( + collector.push(b"page".to_vec()).is_empty(), + "should retain input at the cap" + ); + let mut emitted = collector.push(b"!".to_vec()).concat(); + emitted.extend(collector.push(b"tail".to_vec()).concat()); + assert_eq!( + emitted, b"page!tail", + "should release overflow without duplication" + ); + assert!( + collector.finish().is_none(), + "should skip finalizer I/O after bypass" + ); + } + struct AppendFinalizer { suffix: &'static [u8], inputs: Rc>>>, @@ -638,6 +645,7 @@ mod tests { ]; let services = noop_services(); let output = block_on(finalize_document( + StatusCode::OK, b"page".to_vec(), &mut sessions, None, @@ -667,6 +675,7 @@ mod tests { let input = b"page".to_vec(); let allocation = input.as_ptr(); let output = block_on(finalize_document( + StatusCode::OK, input, &mut sessions, None, @@ -766,6 +775,7 @@ mod tests { let input = b"page".to_vec(); let allocation = input.as_ptr(); let output = block_on(finalize_document( + StatusCode::OK, input, &mut sessions, None, @@ -808,6 +818,7 @@ mod tests { }), }]; let output = block_on(finalize_document( + StatusCode::OK, b"page".to_vec(), &mut sessions, None, @@ -833,6 +844,7 @@ mod tests { }), }]; let output = block_on(finalize_document( + StatusCode::OK, b"pageTOKEN".to_vec(), &mut sessions, Some(b"TOKEN"), @@ -845,6 +857,7 @@ mod tests { "should reject a duplicated assembly marker" ); let output = block_on(finalize_document( + StatusCode::OK, b"page".to_vec(), &mut sessions, Some(b"TOKEN"), @@ -887,6 +900,7 @@ mod tests { }), }]; let output = block_on(finalize_document( + StatusCode::OK, b"page".to_vec(), &mut sessions, None, @@ -919,6 +933,7 @@ mod tests { }, ]; let output = block_on(finalize_document( + StatusCode::OK, b"page".to_vec(), &mut sessions, None, @@ -948,6 +963,7 @@ mod tests { }]; let services = noop_services(); let work = finalize_document( + StatusCode::OK, b"page".to_vec(), &mut sessions, None, @@ -1020,6 +1036,7 @@ mod tests { }), }]; let output = block_on(finalize_document( + StatusCode::OK, b"pageTOKEN".to_vec(), &mut sessions, marker, @@ -1040,6 +1057,7 @@ mod tests { }), }]; let output = block_on(finalize_document( + StatusCode::OK, b"page".to_vec(), &mut sessions, None, @@ -1075,6 +1093,7 @@ mod tests { finalizer: Box::new(LateFinalizer), }]; let output = block_on(finalize_document( + StatusCode::OK, b"page".to_vec(), &mut sessions, None, diff --git a/crates/trusted-server-core/src/settings.rs b/crates/trusted-server-core/src/settings.rs index def14a676..d0b379693 100644 --- a/crates/trusted-server-core/src/settings.rs +++ b/crates/trusted-server-core/src/settings.rs @@ -80,7 +80,11 @@ pub struct Publisher { pub max_buffered_body_bytes: usize, /// Whole-chain HTML terminal budget in milliseconds; inert without a finalizer. #[serde(default = "default_html_finalization_timeout_ms")] - #[validate(range(min = 1, max = 60000))] + #[validate(range( + min = 1, + max = 60000, + message = "must be between 1 and 60000 milliseconds" + ))] pub html_finalization_timeout_ms: u32, } @@ -3728,6 +3732,19 @@ where #[cfg(test)] mod tests { + use super::*; + use crate::auction::build_orchestrator; + use crate::integrations::{ + IntegrationRegistry, gpt::GptConfig, nextjs::NextJsIntegrationConfig, + prebid::PrebidIntegrationConfig, + }; + use crate::redacted::Redacted; + use crate::test_support::tests::{crate_test_settings_str, create_test_settings}; + use regex::Regex; + use serde_json::json; + use std::collections::BTreeSet; + use std::sync::Arc; + #[test] fn html_finalization_timeout_defaults_and_boundaries() { let parsed = Settings::from_toml(&crate_test_settings_str()) @@ -3765,13 +3782,6 @@ mod tests { ); } } - - use super::*; - use regex::Regex; - use serde_json::json; - use std::collections::BTreeSet; - use std::sync::Arc; - /// `DebugConfig` denies unknown fields, so a binary built before /// `sandbox_metrics_enabled` existed must still accept a default blob. /// That only holds while the flag is skipped during serialization. @@ -3823,14 +3833,6 @@ mod tests { ); } - use crate::auction::build_orchestrator; - use crate::integrations::{ - IntegrationRegistry, gpt::GptConfig, nextjs::NextJsIntegrationConfig, - prebid::PrebidIntegrationConfig, - }; - use crate::redacted::Redacted; - use crate::test_support::tests::{crate_test_settings_str, create_test_settings}; - fn trusted_client_ip_toml(ip_header: &str, auth_header: &str, shared_secret: &str) -> String { format!( "{}\n[trusted_client_ip]\nip_header = \"{ip_header}\"\nauth_header = \"{auth_header}\"\nshared_secret = \"{shared_secret}\"\n", diff --git a/docs/guide/configuration.md b/docs/guide/configuration.md index 6a25ce8b2..539c0c5b8 100644 --- a/docs/guide/configuration.md +++ b/docs/guide/configuration.md @@ -447,6 +447,14 @@ auction deadline. It has no effect without enabled finalizers. Axum, Workers, an Spin can cancel yielding work; Fastly consumers must bound their own blocking host calls. Timeout keeps the current transformed document and skips later finalizers. +Enabled finalizers disable shared-template and downstream caching for processed +HTML, including unchanged/error outcomes; raw-origin readthrough remains separate. +HTML with status 206 or any `Content-Range` bypasses rewriting and terminal I/O, +preserving its encoded bytes and range/framing metadata. Full error pages remain +eligible and expose their status to the integration. See +[Asynchronous HTML finalization](/guide/integration-guide#asynchronous-html-finalization) +for consumer requirements and the cacheability contract needed for a future mode. + #### `max_buffered_body_bytes` **Purpose**: Upper bound on how much of a publisher origin body the rewrite diff --git a/docs/guide/integration-guide.md b/docs/guide/integration-guide.md index 3abef24e8..16504f331 100644 --- a/docs/guide/integration-guide.md +++ b/docs/guide/integration-guide.md @@ -219,7 +219,7 @@ encoder complete after the terminal phase. Existing `StreamProcessor` callbacks remain synchronous, and pages with no registered finalizers keep their existing streaming path. -`HtmlFinalizationInput` supplies borrowed `html`, `services`, `deadline`, and a +`HtmlFinalizationInput` supplies the actual `response_status`, borrowed `html`, `services`, `deadline`, and a separate bounded `replacement` writer. Append a complete candidate and return `HtmlFinalization::Replace`, or return `Unchanged`. An error, timeout, invalid assembly marker count, or output limit violation discards the candidate and @@ -229,6 +229,18 @@ through the request and are not converted into optional-work failures. Preserve CSP, Next.js/RSC semantics, and the opaque parser markers; core checks marker multiplicity but does not parse or validate replacement structure. +Full 404/500 HTML remains eligible. A signing integration should inspect +`response_status` and return `Unchanged` for statuses it does not sign. HTML with +status 206 or any `Content-Range` bypasses rewriting and terminal I/O while +preserving encoded body bytes and range/framing metadata. Non-document fetches +keep their range request semantics. Independent cookie and diagnostics privacy +rules still apply. + +Each real consumer must test a winning auction after replacement: the marker is +consumed once, and the bids script remains before the structural body end, outside +scripts, comments, and attributes. Unchanged bytes beside a marker cannot establish +its parsing context, so adjacent-byte checks are not a structural guarantee. + The publisher's `html_finalization_timeout_ms` is one budget for the whole chain, starting after synchronous EOF. Use `deadline.remaining()` to bound outside calls. Axum, Workers, and Spin provide cooperative timers. Fastly provides no timer @@ -236,12 +248,16 @@ because host calls block; the consumer must set transport timeouts itself. Dropp a future releases its borrows and discards its candidate, but cannot undo a remote write or preempt work that does not yield. The deadline wins over a simultaneously ready callback. Avoid logging HTML, lookup URLs, or external response contents. +The default backend first-byte timeout is 15 seconds; it is not a total-call +deadline. Use the remaining budget for supported transport limits and verify the +consumer's blocking-call behavior on Fastly before enabling it. Enabled finalizers retain transformed HTML up to `max_buffered_body_bytes` and use a separate candidate with the same cap. Collection overflow skips terminal I/O and resumes normal delivery with the retained prefix and remaining bytes. Existing -input, decoder, and final output errors remain fatal. Registration therefore changes -first-byte latency only for enabled pages that fit the retention limit. +input, decoder, and final output errors remain fatal. Enabled pages wait for EOF +within the cap; oversized pages also delay first-byte delivery until the cap is +crossed and ordinary streaming resumes. Use `stream_publisher_body_async`, `buffer_publisher_response_async`, or `publisher_response_into_streaming_response` to execute terminal callbacks. @@ -256,9 +272,19 @@ unchanged, fails, or is skipped after collection overflow. Document GETs strip t four conditional/range headers used by synthesized HTML. HEAD, bodiless responses, unsupported encodings, and non-HTML responses run no terminal callbacks. +This first version therefore removes transformed-template and downstream HTML +caching for enabled pages, including page-level signing. Raw-origin caching and +an integration's own KV result cache can still reduce origin and lookup work. +Cacheable finalization needs a separate contract for reader-neutral output, +external-data/signing-key identity, expiry, invalidation, optional failure caching, +and whether callbacks execute on hits; a user-independent flag alone is insufficient. + Callers of `handle_publisher_request` provide the enabled integration registry in `AuctionDispatch::integration_registry` and use that same registry for response finalization. This metadata is required before a shared-template lookup can happen. +It is a source-compatibility change for external `AuctionDispatch` struct literals: +add the required field using the same request registry. An omitted or optional +registry could incorrectly admit a cached document before finalization is known. A fictional HTTP consumer is available only through the existing `test-utils` feature. To verify real asynchronous I/O and timeout fallback on local runtimes: @@ -287,8 +313,14 @@ Fastly production ignores the configured timeout. Add `--release` for production-profile artifacts. Add `--auction` to run a separate enabled-auction control against a closed loopback bid endpoint. It verifies slot/bid fallback assembly and marker removal without -external traffic. The existing Workers clock panic reported in #1240 blocks that -control; the ordinary terminal-phase tests still run independently. +external traffic. The auction telemetry clock now uses the platform-compatible +`web_time::Instant`; the combined Workers control passes after this targeted +correction. This does not verify every other clock path covered by #1240. + +CI runs the actual Axum harness with `--auction`, including delayed success, +hung-call fallback, privacy, and auction assembly. It also checks process-group +cleanup and retains runtime diagnostics on failure. The other runtime harnesses +remain explicit local verification commands. ## Browser and script guards diff --git a/docs/superpowers/plans/2026-10-12-1241-async-html-finalization.md b/docs/superpowers/plans/2026-10-12-1241-async-html-finalization.md index 9c0675b63..37c13f6be 100644 --- a/docs/superpowers/plans/2026-10-12-1241-async-html-finalization.md +++ b/docs/superpowers/plans/2026-10-12-1241-async-html-finalization.md @@ -10,7 +10,7 @@ --- -**Status:** Implemented; see the execution record for verification and the existing Workers auction limitation +**Status:** Implemented; see the execution record and review hardening for current verification **Date:** 2026-10-12 @@ -25,7 +25,12 @@ The task checklists below preserve the original execution instructions, including proposed intermediate commit boundaries. The execution record is the authoritative completion record: implementation was reviewed together before committing, and it -records actual commands, deviations, and the blocked Workers auction control. +records actual commands, deviations, and the initially blocked Workers auction control. + +The [review hardening addendum](2026-10-12-1241-review-hardening.md) records the +subsequently authorized response-status/range refinements, telemetry clock +correction, runtime CI coverage, and renewed verification. It supplements the +original task instructions below. ## 1. Execution boundaries @@ -40,7 +45,7 @@ commits in a draft branch; do not release an API that registrations can enable b publisher entry points support it. Keep the EdgeZero revision, generic streaming traits, JS runtime, and production platform -manifests unchanged. KV wiring under #1139, clock fixes under #1240, and a real signing +manifests unchanged. KV wiring under #1139, broader clock fixes under #1240, and a real signing integration are separate work. Do not mask a live-runtime failure by replacing actual SDK services with stubs or by reporting a native test as a runtime smoke. diff --git a/docs/superpowers/plans/2026-10-12-1241-review-hardening.md b/docs/superpowers/plans/2026-10-12-1241-review-hardening.md new file mode 100644 index 000000000..c529d6bf0 --- /dev/null +++ b/docs/superpowers/plans/2026-10-12-1241-review-hardening.md @@ -0,0 +1,179 @@ +# HTML finalization review hardening implementation plan + +> **For agentic workers:** Use superpowers:executing-plans for central implementation and independent review. Checkboxes record observed completion. + +**Goal:** Close the confirmed response-eligibility and maintenance gaps from the independent review of PR #1290. + +**Architecture:** Preserve the explicit HTML terminal phase and conservative cache policy. Carry authoritative response metadata to terminal callbacks, bypass all body transformations for opted-in range-associated HTML, and preserve exactly-once auction abandonment. Use the existing platform-compatible clock for auction telemetry and exercise the real Axum harness in CI. + +**Tech Stack:** Existing Rust/local futures, HTTP status/header types, `web_time`, Python standard library, and GitHub Actions. + +## Reviewed design refinements + +The user authorized fixes after the read-only review. These refinements supplement +the [existing design](../specs/2026-10-12-1241-async-html-finalization-design.md). + +- Terminal callbacks receive the actual response status. Full 404/500 HTML remains + eligible; a signing consumer may return `Unchanged` for unwanted statuses. +- When terminal integrations are enabled, HTML with status 206 or any + `Content-Range` bypasses rewriting and terminal I/O. The latter includes + range-associated error responses such as 416. Preserve encoded bytes, + `Content-Range`, framing, and ordinary origin policy. Do not strip ranges from + all GETs: non-document fetch/subresource semantics remain intact. +- Both response wrappers check actual metadata before removing framing or applying + terminal privacy. Direct body helpers use private carried metadata and also + preserve partial bytes. Dispatched auctions are abandoned once on bypass. +- Construct terminal input before moving auction ownership. This closes a latent + error-path hazard without manufacturing an impossible body variant for a test. +- Reduce redundant replay flushes while keeping the pre-auction flush barrier. +- The live Workers failure names `AuctionObservationContext::from_parts` calling + unsupported `std::time::Instant`. Change only that telemetry clock to + `web_time::Instant`, already used by the orchestrator, and verify the actual + Workers auction control. Broader clock work under #1240 remains separate. +- Keep cache bypass explicit. A user-independent flag alone does not define + external freshness, signing-key identity, invalidation, failure caching, or + callback behavior on hits. Document the product cost and future cache contract. +- Preserve the marker-placement integration contract. Neighbor checks do not + establish parsing context. Document required real-consumer placement tests and + avoid a second parser or misleading structural guarantees. +- Document the required registry field's source-compatibility migration. Keep + registry metadata mandatory rather than restoring an unsafe optional fallback. + +## Tasks and evidence + +### 1. Response metadata and partial-response regressions + +Files: `publisher.rs`, `publisher/html_finalization.rs`, +`integrations/html_finalization.rs`, and existing host parity fixtures. + +- [x] Add tests for 206 and 200/416 with `Content-Range`, materialized/streamed + bodies, lazy/buffered/direct helpers, compressed byte identity, preserved + range/framing/validators, no callback/timer, and exactly-once abandonment. +- [x] Run the focused native tests before implementation and record the failure. +- [x] Add private status/range metadata and public borrowed status; wire every + constructor and authoritative response wrapper. Bypass before mutation. +- [x] Verify callbacks observe 200/404/500; verify non-document range forwarding, + unsolicited partial-response bypass, and feature-off behavior unchanged. +- [x] Run focused native and WASI tests and direct core lint. + +### 2. Ownership and inexpensive cleanup + +Files: `publisher/html_finalization.rs`, `publisher.rs`, +`integrations/html_finalization.rs`, `registry.rs`, `platform/mod.rs`, +`settings.rs`, and `scripts/fixtures/async-html-finalization-runtime.py`. + +- [x] Initialize terminal input before moving auction ownership. +- [x] Simplify the unreachable writer branch while retaining poison behavior. +- [x] Put touched tests below imports and add a descriptive timeout validation + message. Preserve existing output-limit and range tests. +- [x] Add a meaningful flush-count regression before reducing replay flushes; + retain prefix delivery before auction collection, including missing body end. +- [x] Explain intentionally ignored cleanup exceptions and reap killed children. + Verify cleanup against real short-lived/stubborn local process groups. + +### 3. Actual-runtime evidence and CI + +Files: `auction/telemetry.rs`, `.github/workflows/test.yml`, runtime harness. + +- [x] Preserve the existing actual Workers panic as red evidence, change the + telemetry clock only, and rerun the fresh release Workers auction fixture. +- [x] Add actual Axum disabled/success/hung/auction harness to existing native CI, + Python 3.12, bounded timeout, and failure artifact retention. +- [x] Run Axum, Workers, Spin, and Fastly runtime controls after source freezes. + Use isolated Viceroy 0.19 for blocking-host timeout proof; retain normal pins. + +### 4. Documentation, verification, and draft PR + +Files: canonical guides, existing spec/plan/review, this execution record. + +- [x] Update status/partial contracts, marker obligations, cache costs, migration, + and actual Workers evidence consistently across documents. +- [x] Run target-matched tests/lints, parity, build-digest checks, JS/docs gates, + ordinary artifact fixture exclusion, and relevant integration/cache controls. +- [x] Obtain independent architecture and final review; resolve findings centrally. +- [ ] Commit and push the verified changes to the existing draft PR. Update its + description with exact evidence and remaining maintainer decisions. + +## Execution record + +The independent architecture review approved the addendum. Staff review found a +remaining handler-ordering gap: active/inactive ads and tag suppression could +change partial-response cache policy before the routing bypass. A new nine-case +handler regression reproduced it, and the range decision now precedes those +body-dependent policy branches. Re-review found no remaining blocking findings. +Independent cookie and diagnostics privacy remains intact. + +### Red/green evidence + +- Partial-response tests failed on the original implementation: a ranged fetch + invoked a finalizer and the wrappers removed framing/validators and changed + cache policy. The new 64-case sink matrix and eight handler cases pass. +- The status contract test initially failed to compile because the input had no + status. Both sinks now expose 200/404/500 accurately without excluding full + error pages. +- A direct 206 helper test with only status marked partial caught decode/re-encode + of the original gzip bytes. The guard now derives partial eligibility from + status as well as carried range metadata. +- Large-document replay initially flushed nine times. It now flushes twice on + the no-auction fixture. A controlled pending-auction test confirms every ready + prefix byte is flushed before collection with and without a structural body end. +- The cleanup test against the exact committed pre-fix harness failed because a + forcibly killed child was not reaped. All three real-process cleanup cases pass + after the bounded final wait. Tests do not replace process handling with mocks. +- The captured Workers stack named the unsupported standard clock in auction + telemetry. A fresh release Workers fixture now passes the combined route after + changing only that clock to `web_time::Instant`. + +The response regressions verify no callback/outside work; independent path review +confirms bypass precedes creation of the terminal coordinator and its timer. No +impossible body variant was introduced to test the latent ownership reorder. + +### Final verification + +| Check | Observed result | +| -------------------------------------- | ----------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| Focused native finalization tests | 36 pass, including the 64-case sink matrix, eight ranged-origin cases, nine live/inactive/suppressed policy cases, actual status, direct helpers, and both flush regressions. | +| Pinned Fastly/WASI gate | 219 adapter, 2932 core, 3 JS crate, 21 OpenRTB, and 4 doc tests pass; existing ignored tests remain ignored. | +| Fastly reusable sandbox | 219 adapter tests pass. | +| Target-matched Clippy | All eight aliases pass on final source. Primary core all-target/all-feature and digest lint pass. | +| Native adapter/core tests | Axum 44, Workers 53, Spin 87; core 2939 unit, 10 digest, 4 doc tests pass. Six core doc tests remain ignored. | +| CLI/codegen/parity | 875 CLI tests pass with 18 existing ignored; codegen 7 and parity 18 pass. CLI tests preceded the final range-policy adjustment; CLI source did not change. | +| Exact CI runtime integration selection | 63 pass, plus the separate Fastly EC lifecycle control. Ordinary artifacts were rebuilt and isolated first. | +| Existing template-cache controls | ESI 22, inline 9, purge 34 checks pass on pinned Viceroy 0.17, including real winning bids and cookie variants. | +| JS | Build, 50 Vitest files / 1208 tests, type checks, and formatting pass. | +| Docs/format/syntax | Docs lint/build/format, repository Markdown, Rust formatting, diff whitespace, Bash and Python syntax checks pass. | +| Cleanup | Three real-process tests pass: already exited, graceful TERM, and TERM ignored then KILL/reap. | +| Ordinary artifacts | All four adapters build; the fictional fixture registration ID is absent from every ordinary artifact. | + +### Actual runtimes + +All four actual runtime controls pass: disabled baseline, delayed success, hung +lookup with exact fallback bytes, and enabled-auction fallback assembly. Workers, +Spin, and Fastly use release-profile fixture artifacts; Axum runs the same command +now used by CI. The fixture budget is 750 ms, and timer adapters retain a 30000 ms +transport bound to distinguish timer release from an early HTTP timeout. + +| Adapter | Success request | Hung request | Terminal evidence | Auction control | +| ------- | --------------- | ------------ | -------------------------------- | --------------- | +| Axum | 0.215 s | 0.811 s | 750 ms terminal time | Pass, 0.214 s. | +| Workers | 0.242 s | 0.828 s | 753 ms terminal time | Pass, 0.292 s. | +| Spin | 0.213 s | 0.809 s | 751 ms terminal time | Pass, 0.211 s. | +| Fastly | 0.209 s | 0.806 s | 750 ms bounded host-call release | Pass, 0.210 s. | + +Workers and Axum were rerun after the final range-policy correction. Spin/Fastly +actual fixture runs preceded that last correction, which only changes opted-in +range-associated response handling; final native/WASI tests and fresh ordinary +builds verify the correction. Existing browser controls were not repeated because +no JS behavior changes in this hardening. + +Fastly transport-bound proof uses an isolated Viceroy 0.19; normal tool pins and +CI controls retain 0.17. Passing Workers auction telemetry does not declare all +clock paths under #1240 fixed. Cacheability and API-shape approval remain +maintainer decisions, explicitly documented in the draft PR. Marker cardinality +still does not establish HTML parsing context; real integrations must preserve +structure and test winning-bid placement. The required registry field remains a +source migration for Git/path consumers. + +Verification logs, generated manifests, body/header/timing diagnostics, and +artifact hashes are retained outside version control. No operator configuration, +credentials, or external lookup services were used. diff --git a/docs/superpowers/reviews/2026-10-12-1241-async-html-finalization-implementation.md b/docs/superpowers/reviews/2026-10-12-1241-async-html-finalization-implementation.md index 194ea5e77..fa3aca8cc 100644 --- a/docs/superpowers/reviews/2026-10-12-1241-async-html-finalization-implementation.md +++ b/docs/superpowers/reviews/2026-10-12-1241-async-html-finalization-implementation.md @@ -6,13 +6,17 @@ **Planning commit:** `b750c879a` -**Status:** Implemented and reviewed. Repository regression gates pass. The actual -Workers enabled-auction control remains blocked by the existing clock panic in #1240. +**Status:** Implemented and reviewed. Review hardening adds range safety, response +status metadata, a targeted telemetry clock correction, and actual-runtime CI. The +Workers enabled-auction control now passes. See the linked hardening execution +record for the latest verification; the original evidence below is historical. **Design:** [Optional asynchronous HTML document finalization](../specs/2026-10-12-1241-async-html-finalization-design.md) **Plan:** [Implementation and verification tasks](../plans/2026-10-12-1241-async-html-finalization.md) +**Review hardening:** [Fixes, retained design decisions, and renewed verification](../plans/2026-10-12-1241-review-hardening.md) + ## Result and scope Trusted Server now offers an explicitly registered HTML terminal callback after @@ -23,7 +27,8 @@ unchanged-input fallback. The generic streaming traits remain unchanged. The callback belongs in Trusted Server because its document state, auction assembly, cache admission, privacy policy, and runtime service container belong here. No EdgeZero revision, production platform manifest, JS behavior, KV implementation, -signing consumer, or existing clock behavior changes in this delivery. +signing consumer changes in this delivery. Review hardening corrects only the +traced unsupported auction telemetry clock; broader clock work remains separate. Enabled finalizers deliberately retain the completed transformed document. They bypass shared-template caching before lookup or reservation, and eligible HTML @@ -34,8 +39,9 @@ Raw origin readthrough caching remains independent. The original granular checklists describe instructions, rather than claiming each suggested intermediate commit exists. All thirteen tasks have implementation or -verification evidence below. Task 11's Workers auction control has an explicit -external blocker; passing native parity is not substituted for that live check. +verification evidence below. Initially Task 11's Workers auction control was +blocked; the review hardening record replaces that gap with actual runtime +evidence after the targeted clock correction. | Plan tasks | Spec sections | Implementation and observed evidence | | ---------- | ------------------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ | @@ -86,7 +92,7 @@ to validate arbitrary HTML structure or integration-specific semantics. the plan's suggested per-task commits. No implementation API was published between contract creation and complete routing. -## Repository verification +## Initial repository verification All commands ran against the implementation worktree, with Rust 1.95.0 and Node 24.12.0. Target-matched checks used the workspace aliases; shared Cargo cache use @@ -128,7 +134,7 @@ double, rather than claiming a live platform cache hit. Local verification logs and generated artifacts remain outside version control; they contain exact command output and runtime diagnostics. -## Actual-runtime verification and limits +## Initial actual-runtime verification and limits The fixture uses a 750 ms chain budget, delays document EOF by 50 ms, and delays a successful lookup by 150 ms. On timer adapters the hung transport timeout is @@ -149,19 +155,18 @@ The optional actual-auction fallback control passed on Axum, Spin, and Fastly. Positive winning-bid assembly is additionally covered by the native/WASI core regression and the existing template-cache controls. -Two runtime qualifications remain explicit: +The initial runtime qualifications were: 1. Pinned Viceroy 0.17 does not simulate backend timeouts, so it cannot prove a Fastly hung-call bound. An isolated official 0.19 executable was used for that live check; repository pins remain unchanged. Backend timeout support is documented in the [official 0.19 release](https://github.com/fastly/Viceroy/releases/tag/v0.19.0). -2. The Workers actual-auction control returns HTTP 500 with +2. The initial Workers actual-auction control returned HTTP 500 with `time not implemented on this platform` from `std::time` and then a WASM - trap. This reproduces the separate #1240 clock problem. Ordinary Workers - finalization controls pass. Native parity and mocked auction tests do not - establish that the combined actual-Workers auction route works. No clock - workaround is bundled, and that live combination remains unverified until - #1240 is resolved. + trap. The named WASM stack traced it to auction telemetry using the standard + clock. Review hardening changes that clock to the already-used platform + abstraction and verifies a fresh release Workers combined control. The + broader #1240 issue is not declared resolved by that one path. ## Independent review @@ -172,7 +177,8 @@ both were added and re-reviewed. Architecture review requested a positive winnin bid with an enabled finalizer; that regression now passes natively and on WASI. The spec, plan, code, and tests agree on ownership, ordering, deadline scope, -cache/privacy policy, optional fallback, and excluded response types. The remaining -Workers runtime limitation is documented rather than counted as a passing test. +cache/privacy policy, optional fallback, and excluded response types. The initial +Workers runtime limitation was documented rather than counted as a passing test; +its subsequently verified correction is recorded in the hardening addendum. The implementation branch and isolated worktree are preserved for maintainer review; no merge, push, deployment, or public issue comment is part of this work. diff --git a/docs/superpowers/specs/2026-10-12-1241-async-html-finalization-design.md b/docs/superpowers/specs/2026-10-12-1241-async-html-finalization-design.md index 7419d5dc9..adef34dbf 100644 --- a/docs/superpowers/specs/2026-10-12-1241-async-html-finalization-design.md +++ b/docs/superpowers/specs/2026-10-12-1241-async-html-finalization-design.md @@ -2,7 +2,7 @@ **Date:** 2026-10-12 -**Status:** Implemented on `feat/1241-async-html-finalization`; verification and remaining runtime limitation recorded below +**Status:** Implemented on `feat/1241-async-html-finalization`; review hardening and verification recorded below **Issue:** trusted-server #1241 @@ -12,6 +12,8 @@ **Implementation review:** [Requirement coverage, verification, and runtime limitations](../reviews/2026-10-12-1241-async-html-finalization-implementation.md) +**Review hardening:** [Response metadata, range safety, telemetry clock, and CI refinements](../plans/2026-10-12-1241-review-hardening.md) + ## 1. Decision Add an opt-in asynchronous final-document phase to Trusted Server's HTML integration @@ -173,6 +175,7 @@ pub trait IntegrationHtmlFinalizer { } pub struct HtmlFinalizationInput<'a> { + pub response_status: StatusCode, pub html: &'a [u8], pub replacement: &'a mut HtmlReplacementWriter, pub services: &'a RuntimeServices, @@ -346,8 +349,9 @@ repin EdgeZero to obtain a timer. Use `web_time::Instant` for absolute budget accounting and elapsed observations in core. Do not introduce `std::time::Instant::now()` or `SystemTime::now()` on the Cloudflare path. -The existing clock sites reported in #1240 are separate work and may block a real-runtime -auction smoke test until corrected; fixing them is not part of this design. +The review hardening exception corrects the traced auction telemetry clock using +that existing abstraction. Broader clock sites reported in #1240 remain separate +work; one successful auction smoke does not verify all of them. ### 9.3 Race and fallback rules @@ -451,8 +455,18 @@ guard when the ad stack is active. This feature does not turn an existing 502 in ### 11.3 Response selection and framing -Run finalizers only on HTML that the existing routing rules already process, including -processable non-2xx HTML. Do not rewrite unsupported encodings, unmodified pass-through +Run finalizers only on complete HTML that the existing routing rules already process, including +processable non-2xx HTML. Expose the actual response status so consumers can choose +which complete documents to process. With enabled finalizers, HTML with status 206 +or any `Content-Range` bypasses all body transformations and terminal I/O. Preserve +encoded bytes, range/framing metadata, and origin policy; independent cookie and +diagnostics privacy still applies. This covers range-associated error responses +such as 416 as well as partial representations. The handler and both response +wrappers must make this decision before body-dependent policy or framing mutation; +direct body helpers carry the same private metadata. Abandon dispatched bids once. +Keep non-document range forwarding intact and feature-off behavior unchanged. + +Do not rewrite unsupported encodings, unmodified pass-through responses, assets, or standalone `text/x-component` bodies. HEAD and `204`, `205`, or `304` responses invoke no finalizer or finalizer I/O. Preserve @@ -586,8 +600,11 @@ fixtures and run a bounded delayed/hung lookup on Viceroy, local Workers, Spin, Record the exact setup, SDK/runtime versions, observed behavior, and limitations. Do not claim a universal exact wall-clock release; cooperative scheduling adds tolerance. -The Cloudflare clock issue #1240 must be resolved separately, or its affected auction smoke -must be explicitly reported as blocked rather than marked passing. Keep KV wiring under +Review hardening permits one targeted correction under #1240: auction telemetry +uses `web_time::Instant`, consistent with the orchestrator, because the captured +Workers stack names that unsupported standard-clock call. Verify the actual +Workers auction control after the correction. Broader clock paths remain separate; +do not claim all of #1240 resolved from one successful fixture. Keep KV wiring under #1139 separate; HTTP-backed and unavailable-KV fixtures can verify this lifecycle without requiring new mandatory store bindings. @@ -612,7 +629,8 @@ Expected production touch points: Keep the generic streaming trait, synchronous pipeline, JS runtime, platform manifests, and EdgeZero pin unchanged. Do not bundle the real signing consumer, KV-store wiring, -Cloudflare clock fixes, or unrelated publisher refactoring into this feature. +broader Cloudflare clock fixes, or unrelated publisher refactoring into this feature. +The narrowly traced auction telemetry clock correction above is the reviewed exception. The implementation plan should separate contract/registration work, terminal coordination, adapter timers, cache/privacy behavior, and verification into small reviewable changes, diff --git a/scripts/fixtures/async-html-finalization-runtime.py b/scripts/fixtures/async-html-finalization-runtime.py index 227284dcc..87f9c19fb 100644 --- a/scripts/fixtures/async-html-finalization-runtime.py +++ b/scripts/fixtures/async-html-finalization-runtime.py @@ -38,11 +38,14 @@ def stop(process): os.killpg(process.pid, signal.SIGTERM) process.wait(timeout=5) except (ProcessLookupError, subprocess.TimeoutExpired): + # An exited group needs no TERM; a graceful-wait timeout proceeds to KILL. pass try: os.killpg(process.pid, signal.SIGKILL) except ProcessLookupError: + # The process group may already have exited after TERM. pass + process.wait(timeout=5) def run(adapter, auction=False, release=False): diff --git a/scripts/fixtures/test_async_html_finalization_runtime.py b/scripts/fixtures/test_async_html_finalization_runtime.py new file mode 100644 index 000000000..d97999b2f --- /dev/null +++ b/scripts/fixtures/test_async_html_finalization_runtime.py @@ -0,0 +1,65 @@ +"""Verify process cleanup used by the actual-runtime finalization harness.""" + +import importlib.util +import os +from pathlib import Path +import signal +import subprocess +import sys +import unittest + +SPEC = importlib.util.spec_from_file_location( + "example_finalization_runtime", + Path(__file__).with_name("async-html-finalization-runtime.py"), +) +RUNTIME = importlib.util.module_from_spec(SPEC) +SPEC.loader.exec_module(RUNTIME) + + +class ProcessCleanupTests(unittest.TestCase): + def test_already_exited_process_is_safe(self): + process = subprocess.Popen([sys.executable, "-c", "pass"], start_new_session=True) + process.wait(timeout=5) + RUNTIME.stop(process) + self.assertEqual(process.returncode, 0, "should tolerate an already exited group") + + def test_graceful_termination_reaps_process(self): + process = subprocess.Popen( + [sys.executable, "-c", "import time; time.sleep(300)"], + start_new_session=True, + ) + try: + RUNTIME.stop(process) + self.assertIsNotNone(process.returncode, "should reap a terminated process") + finally: + try: + process.wait(timeout=5) + except subprocess.TimeoutExpired: + os.killpg(process.pid, signal.SIGKILL) + process.wait(timeout=5) + + def test_stubborn_termination_reaps_process(self): + process = subprocess.Popen( + [sys.executable, "-c", + "import signal, time; signal.signal(signal.SIGTERM, signal.SIG_IGN); " + "print('example-ready', flush=True); time.sleep(300)"], + stdout=subprocess.PIPE, + text=True, + start_new_session=True, + ) + try: + self.assertEqual(process.stdout.readline(), "example-ready\n") + RUNTIME.stop(process) + self.assertIsNotNone(process.returncode, "should reap a forcibly killed process") + self.assertEqual(process.returncode, -signal.SIGKILL) + finally: + try: + process.wait(timeout=5) + except subprocess.TimeoutExpired: + os.killpg(process.pid, signal.SIGKILL) + process.wait(timeout=5) + process.stdout.close() + + +if __name__ == "__main__": + unittest.main() From 5ef15911a6ae9142f660fee45ed9b8b49c8d2d9d Mon Sep 17 00:00:00 2001 From: prk-Jr Date: Mon, 12 Oct 2026 20:00:48 +0530 Subject: [PATCH 05/11] Clarify partial HTML and pending auction handling --- crates/trusted-server-core/src/publisher.rs | 5 +++++ docs/guide/integration-guide.md | 19 ++++++++++++------- 2 files changed, 17 insertions(+), 7 deletions(-) diff --git a/crates/trusted-server-core/src/publisher.rs b/crates/trusted-server-core/src/publisher.rs index b43b38df0..9036aee9b 100644 --- a/crates/trusted-server-core/src/publisher.rs +++ b/crates/trusted-server-core/src/publisher.rs @@ -3002,6 +3002,11 @@ impl Write for BoundedWriter { /// [`publisher_response_into_streaming_response`]. Adapters use those async /// entry points when terminal integrations are enabled. /// +/// Callers with a pending auction must use an async entry point so it can be +/// collected or abandoned. This helper borrows `params` and leaves auction +/// ownership with the caller, including when range-associated HTML bypasses +/// processing. +/// /// # Errors /// /// Returns an error if processing fails mid-stream. Since headers are already diff --git a/docs/guide/integration-guide.md b/docs/guide/integration-guide.md index 16504f331..ff1b2629c 100644 --- a/docs/guide/integration-guide.md +++ b/docs/guide/integration-guide.md @@ -230,11 +230,12 @@ CSP, Next.js/RSC semantics, and the opaque parser markers; core checks marker multiplicity but does not parse or validate replacement structure. Full 404/500 HTML remains eligible. A signing integration should inspect -`response_status` and return `Unchanged` for statuses it does not sign. HTML with -status 206 or any `Content-Range` bypasses rewriting and terminal I/O while -preserving encoded body bytes and range/framing metadata. Non-document fetches -keep their range request semantics. Independent cookie and diagnostics privacy -rules still apply. +`response_status` and return `Unchanged` for statuses it does not sign. When +finalizers are registered, HTML with status 206 or any `Content-Range` bypasses +rewriting and terminal I/O while preserving encoded body bytes and range/framing +metadata. Without registered finalizers, existing partial-response processing +continues. Non-document fetches keep their range request semantics. Independent +cookie and diagnostics privacy rules still apply. Each real consumer must test a winning auction after replacement: the marker is consumed once, and the bids script remains before the structural body end, outside @@ -262,8 +263,12 @@ crossed and ordinary streaming resumes. Use `stream_publisher_body_async`, `buffer_publisher_response_async`, or `publisher_response_into_streaming_response` to execute terminal callbacks. `stream_publisher_body` and the generic synchronous pipeline perform synchronous -transformations only. Client cancellation drops the callback and retains the existing -auction guard's log warning; error paths that can await emit abandonment telemetry. +transformations only. Callers with a pending auction must use an async entry point +to collect or abandon it; `stream_publisher_body` borrows the response parameters +and leaves auction ownership with the caller, including on a partial-HTML bypass. +On async entry points, client cancellation drops the callback and retains the +existing auction guard's log warning; error paths that can await emit abandonment +telemetry. Finalizer-enabled HTML bypasses shared-template reads, reservations, and stores and uses inline auction assembly. Raw-origin readthrough remains independent. Processed From a533df267c149bb8295b23e5d5a1fb700c1b32eb Mon Sep 17 00:00:00 2001 From: prk-Jr Date: Tue, 13 Oct 2026 11:10:23 +0530 Subject: [PATCH 06/11] Use ordered asynchronous HTML processor completion --- .../trusted-server-core/src/html_processor.rs | 86 +- .../src/html_processor/finish.rs | 1404 +++++++++++++++++ .../src/integrations/html_finalization.rs | 277 +--- .../src/integrations/mod.rs | 6 +- .../src/integrations/registry.rs | 197 ++- crates/trusted-server-core/src/publisher.rs | 526 +++++- .../src/publisher/html_finalization.rs | 861 ++-------- crates/trusted-server-core/src/settings.rs | 45 +- .../src/streaming_processor.rs | 167 ++ .../src/streaming_processor/finish.rs | 246 +++ .../test_support/async_html_finalization.rs | 130 +- .../tests/parity.rs | 1 + docs/guide/configuration.md | 58 +- docs/guide/integration-guide.md | 236 +-- ...026-10-12-1241-buffered-terminal-design.md | 691 ++++++++ .../2026-10-12-1241-buffered-terminal-plan.md | 686 ++++++++ ...2026-10-12-1241-async-html-finalization.md | 782 ++------- .../plans/2026-10-12-1241-review-hardening.md | 6 +- ...-async-html-finalization-implementation.md | 8 +- ...026-10-13-1241-ordered-eof-verification.md | 131 ++ ...-12-1241-async-html-finalization-design.md | 904 +++-------- .../async-html-finalization-runtime.py | 26 +- trusted-server.example.toml | 5 +- 23 files changed, 4849 insertions(+), 2630 deletions(-) create mode 100644 crates/trusted-server-core/src/html_processor/finish.rs create mode 100644 crates/trusted-server-core/src/streaming_processor/finish.rs create mode 100644 docs/superpowers/archive/2026-10-12-1241-buffered-terminal-design.md create mode 100644 docs/superpowers/archive/2026-10-12-1241-buffered-terminal-plan.md create mode 100644 docs/superpowers/reviews/2026-10-13-1241-ordered-eof-verification.md diff --git a/crates/trusted-server-core/src/html_processor.rs b/crates/trusted-server-core/src/html_processor.rs index 2f6bc5b48..a6e4855de 100644 --- a/crates/trusted-server-core/src/html_processor.rs +++ b/crates/trusted-server-core/src/html_processor.rs @@ -24,25 +24,44 @@ use crate::settings::Settings; use crate::streaming_processor::{HtmlRewriterAdapter, StreamProcessor}; use crate::tsjs; -struct HtmlWithStreamingProcessors { +pub(crate) mod finish; + +use finish::HtmlProcessorStage; + +pub(crate) struct HtmlWithStreamingProcessors { inner: Box, - processors: Vec>, + stages: Vec, + finish_index: Option, + marker_length: usize, } impl StreamProcessor for HtmlWithStreamingProcessors { fn process_chunk(&mut self, chunk: &[u8], is_last: bool) -> Result, io::Error> { + self.validate_capabilities()?; + if is_last && self.requires_async_finish() { + return Err(io::Error::other( + "HTML processors require asynchronous EOF completion", + )); + } let mut output = self.inner.process_chunk(chunk, is_last)?; - for processor in &mut self.processors { - output = processor.process_chunk(&output, is_last)?; + for stage in &mut self.stages { + output = stage.process(&output, is_last, self.marker_length)?; } Ok(output) } fn reset(&mut self) { self.inner.reset(); - for processor in &mut self.processors { - processor.reset(); + for stage in &mut self.stages { + stage.reset(); } + self.finish_index = None; + } + + fn requires_async_finish(&self) -> bool { + self.stages + .iter() + .any(HtmlProcessorStage::requires_async_finish) } } @@ -204,7 +223,7 @@ pub fn create_html_processor(config: HtmlProcessorConfig) -> impl StreamProcesso pub(crate) fn create_html_processor_with_document_state( config: HtmlProcessorConfig, document_state: &IntegrationDocumentState, -) -> impl StreamProcessor + use<> { +) -> HtmlWithStreamingProcessors { let stream_processor_factories = config.integrations.html_stream_processor_factories(); if config.suppress_datadome_client_side_tag { @@ -719,14 +738,19 @@ pub(crate) fn create_html_processor_with_document_state( origin_host: config.origin_host.clone(), document_state: document_state.clone(), }; - let processors = stream_processor_factories + let stages = stream_processor_factories .into_iter() - .map(|factory| factory.create(stream_context.clone())) + .map(|factory| { + HtmlProcessorStage::new( + factory.integration_id(), + factory.create(stream_context.clone()), + factory.requires_async_finish(), + factory.buffer_output_until_eof(), + config.max_buffered_body_bytes, + ) + }) .collect(); - HtmlWithStreamingProcessors { - inner: Box::new(inner), - processors, - } + HtmlWithStreamingProcessors::from_stages(Box::new(inner), stages) } #[cfg(test)] @@ -1560,22 +1584,34 @@ mod tests { let inner_final_calls = Arc::new(std::sync::atomic::AtomicUsize::new(0)); let first_final_calls = Arc::new(std::sync::atomic::AtomicUsize::new(0)); let second_final_calls = Arc::new(std::sync::atomic::AtomicUsize::new(0)); - let mut processor = HtmlWithStreamingProcessors { - inner: Box::new(DecoratingProcessor { + let mut processor = HtmlWithStreamingProcessors::from_stages( + Box::new(DecoratingProcessor { prefix: b'I', final_calls: Arc::clone(&inner_final_calls), }), - processors: vec![ - Box::new(DecoratingProcessor { - prefix: b'A', - final_calls: Arc::clone(&first_final_calls), - }), - Box::new(DecoratingProcessor { - prefix: b'B', - final_calls: Arc::clone(&second_final_calls), - }), + vec![ + HtmlProcessorStage::new( + "A", + Box::new(DecoratingProcessor { + prefix: b'A', + final_calls: Arc::clone(&first_final_calls), + }), + false, + false, + 64, + ), + HtmlProcessorStage::new( + "B", + Box::new(DecoratingProcessor { + prefix: b'B', + final_calls: Arc::clone(&second_final_calls), + }), + false, + false, + 64, + ), ], - }; + ); assert_eq!( processor diff --git a/crates/trusted-server-core/src/html_processor/finish.rs b/crates/trusted-server-core/src/html_processor/finish.rs new file mode 100644 index 000000000..33407153e --- /dev/null +++ b/crates/trusted-server-core/src/html_processor/finish.rs @@ -0,0 +1,1404 @@ +//! Ordered HTML processor completion and optional bounded output retention. + +use std::io; + +use futures::FutureExt as _; +use futures::future::{Fuse, LocalBoxFuture}; +use http::StatusCode; +use web_time::Instant; + +use super::HtmlWithStreamingProcessors; +use crate::platform::RuntimeServices; +use crate::streaming_processor::StreamProcessor; +use crate::streaming_processor::finish::{ + StreamFinish, StreamFinishDeadline, StreamFinishInput, StreamOutputWriter, +}; + +pub(crate) struct HtmlProcessorStage { + integration_id: &'static str, + processor: Box, + async_capable: bool, + retained: Option, + skip_async: bool, + buffer_output: bool, + limit: usize, + previous_suffix: Vec, +} + +impl HtmlProcessorStage { + pub(super) fn new( + integration_id: &'static str, + processor: Box, + async_capable: bool, + buffered: bool, + limit: usize, + ) -> Self { + Self { + integration_id, + processor, + async_capable, + retained: buffered.then(|| StreamOutputWriter::new(limit)), + skip_async: false, + buffer_output: buffered, + limit, + previous_suffix: Vec::new(), + } + } + + pub(super) fn reset(&mut self) { + self.processor.reset(); + self.retained = self + .buffer_output + .then(|| StreamOutputWriter::new(self.limit)); + self.skip_async = false; + self.previous_suffix.clear(); + } + + pub(super) fn validate(&self) -> io::Result<()> { + if self.async_capable != self.processor.requires_async_finish() + || (self.retained.is_some() && !self.async_capable) + { + return Err(io::Error::other(format!( + "Integration {} declares inconsistent asynchronous EOF capabilities", + self.integration_id, + ))); + } + Ok(()) + } + + pub(super) fn requires_async_finish(&self) -> bool { + self.async_capable || self.processor.requires_async_finish() + } + + fn output(&mut self, chunk: &[u8], last: bool) -> io::Result> { + let output = self.processor.process_chunk(chunk, last)?; + let Some(retained) = self.retained.as_mut() else { + return Ok(output); + }; + if retained.append(&output).is_ok() { + return if last { + Ok(self + .retained + .take() + .expect("should own retained stage output") + .into_bytes()) + } else { + Ok(Vec::new()) + }; + } + self.skip_async = true; + let mut ready = self + .retained + .take() + .expect("should own overflow prefix") + .into_bytes(); + ready.extend_from_slice(&output); + log::warn!( + "HTML EOF processor {} bypasses optional I/O after output collection overflow", + self.integration_id + ); + Ok(ready) + } + + fn remember_suffix(&mut self, output: &[u8], marker_length: usize) { + let keep = marker_length.saturating_sub(1); + if keep == 0 || output.is_empty() { + return; + } + if output.len() >= keep { + self.previous_suffix.clear(); + self.previous_suffix + .extend_from_slice(&output[output.len() - keep..]); + } else { + self.previous_suffix.extend_from_slice(output); + if self.previous_suffix.len() > keep { + self.previous_suffix + .drain(..self.previous_suffix.len() - keep); + } + } + } + + pub(super) fn process( + &mut self, + bytes: &[u8], + last: bool, + marker_length: usize, + ) -> io::Result> { + let output = self.output(bytes, last)?; + self.remember_suffix(&output, marker_length); + Ok(output) + } +} + +pub(crate) struct HtmlFinishContext<'a> { + pub response_status: StatusCode, + pub services: &'a RuntimeServices, + pub deadline: &'a StreamFinishDeadline, + pub timer: &'a mut Option>>, + pub limit: usize, + pub marker: Option<&'a [u8]>, +} + +fn marker_count(prefix: &[u8], bytes: &[u8], marker: &[u8]) -> usize { + if marker.is_empty() { + return 0; + } + let ordinary = bytes + .windows(marker.len()) + .filter(|window| *window == marker) + .count(); + let mut boundary = prefix.to_vec(); + boundary.extend_from_slice(&bytes[..bytes.len().min(marker.len().saturating_sub(1))]); + ordinary + + boundary + .windows(marker.len()) + .enumerate() + .filter(|(start, window)| *start < prefix.len() && *window == marker) + .count() +} + +async fn finish_output( + stage: &mut HtmlProcessorStage, + fallback: Vec, + context: &mut HtmlFinishContext<'_>, +) -> Vec { + if stage.skip_async + || !stage.async_capable + || fallback.len() > context.limit + || context.deadline.is_expired_at(Instant::now()) + { + return fallback; + } + let mut candidate = StreamOutputWriter::new(context.limit); + let result = { + let Some(work) = stage.processor.async_finish(StreamFinishInput { + response_status: context.response_status, + output: &fallback, + replacement: &mut candidate, + services: context.services, + deadline: context.deadline, + }) else { + return fallback; + }; + let work = work.fuse(); + futures::pin_mut!(work); + if context.deadline.is_expired_at(Instant::now()) { + None + } else { + if context.timer.is_none() + && let Some(timer) = context.services.timer() + { + *context.timer = Some(timer.delay(context.deadline.remaining()).fuse()); + } + if let Some(mut timer) = context.timer.as_mut() { + futures::select_biased! { + _ = timer => None, + result = work => Some(result), + } + } else { + Some(work.await) + } + } + }; + if result.is_none() { + context.deadline.mark_expired(); + } + if result.is_none() || context.deadline.is_expired_at(Instant::now()) { + log::warn!( + "HTML EOF processor {} times out after {} ms; retains {} bytes", + stage.integration_id, + context.deadline.elapsed().as_millis(), + fallback.len() + ); + return fallback; + } + if let Some(Err(error)) = &result { + let outcome = match error.current_context() { + crate::streaming_processor::finish::StreamFinishError::Unavailable => "unavailable", + crate::streaming_processor::finish::StreamFinishError::Rejected => "rejected", + crate::streaming_processor::finish::StreamFinishError::OutputLimitExceeded { + .. + } => "output_limit", + }; + log::warn!( + "HTML EOF processor {} retains input: {outcome}", + stage.integration_id + ); + } + if !matches!(result, Some(Ok(StreamFinish::Replace))) || candidate.is_poisoned() { + return fallback; + } + let replacement = candidate.into_bytes(); + if context.marker.is_some_and(|marker| { + marker_count(&stage.previous_suffix, &fallback, marker) + != marker_count(&stage.previous_suffix, &replacement, marker) + }) { + log::warn!( + "HTML EOF processor {} retains input after invalid assembly markers", + stage.integration_id + ); + return fallback; + } + if context.deadline.is_expired_at(Instant::now()) { + fallback + } else { + replacement + } +} + +impl HtmlWithStreamingProcessors { + pub(crate) fn from_stages( + inner: Box, + stages: Vec, + ) -> Self { + Self { + inner, + stages, + finish_index: None, + marker_length: 0, + } + } + + pub(crate) fn validate_capabilities(&self) -> io::Result<()> { + for stage in &self.stages { + stage.validate()?; + } + Ok(()) + } + + pub(crate) fn set_finish_marker(&mut self, marker: Option<&[u8]>) { + self.marker_length = marker.map_or(0, <[u8]>::len); + } + + pub(crate) fn next_finish_requires_async(&self) -> bool { + self.finish_index + .and_then(|index| self.stages.get(index)) + .is_some_and(|stage| stage.async_capable && !stage.skip_async) + } + + pub(crate) fn begin_finish(&mut self) -> io::Result> { + self.validate_capabilities()?; + if self.finish_index.is_some() { + return Err(io::Error::other("HTML parser already receives EOF")); + } + let mut output = self.inner.process_chunk(&[], true)?; + for stage in &mut self.stages { + output = stage.process(&output, false, self.marker_length)?; + } + self.finish_index = Some(0); + Ok(output) + } + + pub(crate) async fn finish_next( + &mut self, + mut context: HtmlFinishContext<'_>, + ) -> io::Result>> { + let index = self + .finish_index + .ok_or_else(|| io::Error::other("HTML parser must finish before processor EOF"))?; + let Some(stage) = self.stages.get_mut(index) else { + return Ok(None); + }; + let fallback = stage.output(&[], true)?; + let mut output = finish_output(stage, fallback, &mut context).await; + stage.remember_suffix(&output, self.marker_length); + for downstream in &mut self.stages[index + 1..] { + output = downstream.process(&output, false, self.marker_length)?; + } + self.finish_index = Some(index + 1); + Ok(Some(output)) + } +} + +#[cfg(test)] +mod tests { + use std::cell::{Cell, RefCell}; + use std::io; + use std::rc::Rc; + #[cfg(not(target_arch = "wasm32"))] + use std::sync::{Arc, Mutex}; + use std::time::Duration; + + use error_stack::Report; + use futures::executor::block_on; + use futures::future::{LocalBoxFuture, poll_fn}; + use http::StatusCode; + use std::task::Poll; + + use super::*; + use crate::html_processor::HtmlWithStreamingProcessors; + use crate::platform::test_support::noop_services; + #[cfg(not(target_arch = "wasm32"))] + use crate::platform::test_support::{ + NoopBackend, NoopConfigStore, NoopGeo, NoopHttpClient, NoopSecretStore, + }; + #[cfg(not(target_arch = "wasm32"))] + use crate::platform::{PlatformTimer, RuntimeServices}; + use crate::streaming_processor::StreamProcessor; + use crate::streaming_processor::finish::{ + StreamFinish, StreamFinishDeadline, StreamFinishError, StreamFinishInput, + }; + + struct Stage { + name: &'static str, + tail: &'static [u8], + replacement: Option<&'static [u8]>, + events: Rc>>, + calls: Rc>, + } + + impl StreamProcessor for Stage { + fn process_chunk(&mut self, chunk: &[u8], last: bool) -> io::Result> { + if last { + self.events.borrow_mut().push(format!("{}:eof", self.name)); + Ok(self.tail.to_vec()) + } else { + if !chunk.is_empty() { + self.events.borrow_mut().push(format!( + "{}:chunk:{}", + self.name, + String::from_utf8_lossy(chunk) + )); + } + if self.name == "C" { + Ok(chunk.to_ascii_uppercase()) + } else { + Ok(chunk.to_vec()) + } + } + } + + fn requires_async_finish(&self) -> bool { + self.replacement.is_some() + } + + fn async_finish<'a>( + &'a mut self, + input: StreamFinishInput<'a>, + ) -> Option>>> { + let replacement = self.replacement?; + Some(Box::pin(async move { + self.calls.set(self.calls.get() + 1); + self.events + .borrow_mut() + .push(format!("{}:async", self.name)); + input.replacement.append(replacement)?; + Ok(StreamFinish::Replace) + })) + } + } + + fn stage( + name: &'static str, + tail: &'static [u8], + replacement: Option<&'static [u8]>, + events: &Rc>>, + calls: &Rc>, + ) -> Box { + Box::new(Stage { + name, + tail, + replacement, + events: Rc::clone(events), + calls: Rc::clone(calls), + }) + } + + #[test] + fn async_eof_output_reaches_downstream_before_its_eof() { + let events = Rc::new(RefCell::new(Vec::new())); + let calls = Rc::new(Cell::new(0)); + let mut chain = HtmlWithStreamingProcessors::from_stages( + stage("parser", b"parser-tail", None, &events, &calls), + vec![ + HtmlProcessorStage::new( + "B", + stage("B", b"old", Some(b"late"), &events, &calls), + true, + false, + 64, + ), + HtmlProcessorStage::new( + "C", + stage("C", b"end", None, &events, &calls), + false, + false, + 64, + ), + ], + ); + assert_eq!( + chain + .process_chunk(b"page", false) + .expect("should stream ordinary bytes"), + b"PAGE" + ); + assert_eq!( + chain.begin_finish().expect("should release parser tail"), + b"PARSER-TAIL" + ); + assert_eq!( + calls.get(), + 0, + "should expose ready prefix before optional I/O" + ); + let services = noop_services(); + let deadline = StreamFinishDeadline::new(Duration::from_secs(1), false); + let mut timer = None; + let first = block_on(chain.finish_next(HtmlFinishContext { + response_status: StatusCode::OK, + services: &services, + deadline: &deadline, + timer: &mut timer, + limit: 64, + marker: None, + })) + .expect("should finish upstream stage") + .expect("should produce a stage result"); + assert_eq!(first, b"LATE", "should transform async output downstream"); + assert!( + !events.borrow().iter().any(|event| event == "C:eof"), + "should not close downstream early" + ); + let second = block_on(chain.finish_next(HtmlFinishContext { + response_status: StatusCode::OK, + services: &services, + deadline: &deadline, + timer: &mut timer, + limit: 64, + marker: None, + })) + .expect("should finish downstream stage") + .expect("should return its tail"); + assert_eq!(second, b"end"); + for name in ["parser", "B", "C"] { + assert_eq!( + events + .borrow() + .iter() + .filter(|event| **event == format!("{name}:eof")) + .count(), + 1, + "should finish every stage once" + ); + } + } + + #[test] + fn async_eof_buffer_overflow_replays_once_and_skips_io() { + let events = Rc::new(RefCell::new(Vec::new())); + let calls = Rc::new(Cell::new(0)); + let mut chain = HtmlWithStreamingProcessors::from_stages( + stage("parser", b"", None, &events, &calls), + vec![HtmlProcessorStage::new( + "B", + stage("B", b"end", Some(b"wrong"), &events, &calls), + true, + true, + 4, + )], + ); + assert!( + chain + .process_chunk(b"page", false) + .expect("should hold under cap") + .is_empty() + ); + assert_eq!( + chain + .process_chunk(b"!", false) + .expect("should replay cap crossing"), + b"page!" + ); + assert_eq!( + chain + .process_chunk(b"next", false) + .expect("should continue delivery"), + b"next" + ); + assert!( + chain + .begin_finish() + .expect("should finish parser") + .is_empty() + ); + let services = noop_services(); + let deadline = StreamFinishDeadline::new(Duration::from_secs(1), false); + let mut timer = None; + let tail = block_on(chain.finish_next(HtmlFinishContext { + response_status: StatusCode::OK, + services: &services, + deadline: &deadline, + timer: &mut timer, + limit: 4, + marker: None, + })) + .expect("should finish bypassed stage") + .expect("should emit sync tail"); + assert_eq!(tail, b"end"); + assert_eq!( + calls.get(), + 0, + "should skip optional I/O after collection overflow" + ); + } + + #[test] + fn async_eof_rejects_marker_removal_across_streamed_boundary() { + let events = Rc::new(RefCell::new(Vec::new())); + let calls = Rc::new(Cell::new(0)); + let mut chain = HtmlWithStreamingProcessors::from_stages( + stage("parser", b"", None, &events, &calls), + vec![HtmlProcessorStage::new( + "B", + stage("B", b"EN", Some(b"wrong"), &events, &calls), + true, + false, + 64, + )], + ); + chain.set_finish_marker(Some(b"TOKEN")); + assert_eq!( + chain + .process_chunk(b"TOK", false) + .expect("should stream prefix"), + b"TOK" + ); + chain.begin_finish().expect("should finish parser"); + let services = noop_services(); + let deadline = StreamFinishDeadline::new(Duration::from_secs(1), false); + let mut timer = None; + let tail = block_on(chain.finish_next(HtmlFinishContext { + response_status: StatusCode::OK, + services: &services, + deadline: &deadline, + timer: &mut timer, + limit: 64, + marker: Some(b"TOKEN"), + })) + .expect("should retain marker fallback") + .expect("should return EOF output"); + assert_eq!( + tail, b"EN", + "should retain a marker completed across the earlier prefix" + ); + } + + fn finish_marker_stage(chain: &mut HtmlWithStreamingProcessors, limit: usize) -> Vec { + let services = noop_services(); + let deadline = StreamFinishDeadline::new(Duration::from_secs(1), false); + let mut timer = None; + block_on(chain.finish_next(HtmlFinishContext { + response_status: StatusCode::OK, + services: &services, + deadline: &deadline, + timer: &mut timer, + limit, + marker: Some(b"TOKEN"), + })) + .expect("should finish the marker-aware stage") + .expect("should return stage EOF output") + } + + #[test] + fn async_eof_rejects_marker_creation_across_streamed_boundary() { + let events = Rc::new(RefCell::new(Vec::new())); + let calls = Rc::new(Cell::new(0)); + let mut chain = HtmlWithStreamingProcessors::from_stages( + stage("parser", b"", None, &events, &calls), + vec![HtmlProcessorStage::new( + "B", + stage("B", b"ordinary", Some(b"EN"), &events, &calls), + true, + false, + 64, + )], + ); + chain.set_finish_marker(Some(b"TOKEN")); + assert_eq!( + chain + .process_chunk(b"TOK", false) + .expect("should stream the prefix"), + b"TOK", + "should release ordinary output before EOF" + ); + chain.begin_finish().expect("should finish the parser"); + assert_eq!( + finish_marker_stage(&mut chain, 64), + b"ordinary", + "should reject a candidate that creates a token across emitted output" + ); + assert_eq!(calls.get(), 1, "should evaluate the candidate once"); + } + + #[test] + fn async_eof_retained_output_preserves_a_marker_completed_at_eof() { + let events = Rc::new(RefCell::new(Vec::new())); + let calls = Rc::new(Cell::new(0)); + let mut chain = HtmlWithStreamingProcessors::from_stages( + stage("parser", b"", None, &events, &calls), + vec![HtmlProcessorStage::new( + "B", + stage("B", b"EN", Some(b"wrong"), &events, &calls), + true, + true, + 64, + )], + ); + chain.set_finish_marker(Some(b"TOKEN")); + assert!( + chain + .process_chunk(b"TOK", false) + .expect("should retain the prefix") + .is_empty(), + "should withhold a declaring stage's output" + ); + chain.begin_finish().expect("should finish the parser"); + assert_eq!( + finish_marker_stage(&mut chain, 64), + b"TOKEN", + "should retain the complete collected fallback when replacement removes its marker" + ); + assert_eq!(calls.get(), 1, "should evaluate retained output once"); + } + + #[test] + fn async_eof_overflow_replays_a_split_marker_and_skips_invalid_replacement() { + let events = Rc::new(RefCell::new(Vec::new())); + let calls = Rc::new(Cell::new(0)); + let mut chain = HtmlWithStreamingProcessors::from_stages( + stage("parser", b"", None, &events, &calls), + vec![HtmlProcessorStage::new( + "B", + stage("B", b"!", Some(b"wrong"), &events, &calls), + true, + true, + 4, + )], + ); + chain.set_finish_marker(Some(b"TOKEN")); + assert!( + chain + .process_chunk(b"TOK", false) + .expect("should retain the prefix") + .is_empty() + ); + assert_eq!( + chain + .process_chunk(b"EN", false) + .expect("should replay cap crossing"), + b"TOKEN", + "should replay the split marker exactly once" + ); + chain.begin_finish().expect("should finish the parser"); + assert_eq!( + finish_marker_stage(&mut chain, 4), + b"!", + "should emit synchronous EOF after overflow" + ); + assert_eq!( + calls.get(), + 0, + "should never start the invalid optional replacement after overflow" + ); + assert_eq!( + events + .borrow() + .iter() + .filter(|event| event.as_str() == "B:eof") + .count(), + 1, + "should finish the bypassed stage exactly once" + ); + } + + #[test] + fn async_eof_upstream_replacement_updates_downstream_marker_boundary() { + let events = Rc::new(RefCell::new(Vec::new())); + let calls = Rc::new(Cell::new(0)); + let mut chain = HtmlWithStreamingProcessors::from_stages( + stage("parser", b"", None, &events, &calls), + vec![ + HtmlProcessorStage::new( + "A", + stage("A", b"old", Some(b"TOK"), &events, &calls), + true, + false, + 64, + ), + HtmlProcessorStage::new( + "B", + stage("B", b"EN", Some(b"wrong"), &events, &calls), + true, + false, + 64, + ), + ], + ); + chain.set_finish_marker(Some(b"TOKEN")); + chain.begin_finish().expect("should finish the parser"); + assert_eq!( + finish_marker_stage(&mut chain, 64), + b"TOK", + "should accept upstream ordinary output" + ); + assert_eq!( + finish_marker_stage(&mut chain, 64), + b"EN", + "should preserve the downstream token completed from accepted upstream output" + ); + assert_eq!(calls.get(), 2, "should evaluate both stage candidates once"); + } + + struct ResettableStage(usize); + impl StreamProcessor for ResettableStage { + fn process_chunk(&mut self, _: &[u8], _: bool) -> io::Result> { + self.0 += 1; + Ok(vec![ + u8::try_from(self.0).expect("should keep test counts in a byte"), + ]) + } + fn reset(&mut self) { + self.0 = 0; + } + } + + #[test] + fn async_eof_revision_preserves_synchronous_reset_forwarding() { + let mut chain = HtmlWithStreamingProcessors::from_stages( + Box::new(ResettableStage(0)), + vec![HtmlProcessorStage::new( + "reset", + Box::new(ResettableStage(0)), + false, + false, + 8, + )], + ); + assert_eq!( + chain + .process_chunk(b"page", false) + .expect("should process the first document"), + vec![1], + "should start with fresh state" + ); + chain.reset(); + assert_eq!( + chain + .process_chunk(b"next", false) + .expect("should process after reset"), + vec![1], + "should reset parser and processor state" + ); + } + + #[cfg(not(target_arch = "wasm32"))] + struct RecordingTimer(Arc>>); + #[cfg(not(target_arch = "wasm32"))] + impl PlatformTimer for RecordingTimer { + fn delay(&self, duration: Duration) -> LocalBoxFuture<'static, ()> { + self.0 + .lock() + .expect("should lock recorded timer requests") + .push(duration); + Box::pin(futures::future::pending()) + } + } + + #[cfg(not(target_arch = "wasm32"))] + #[test] + fn async_eof_timer_uses_remaining_budget_when_first_callback_starts() { + let requested = Arc::new(Mutex::new(Vec::new())); + let services = RuntimeServices::builder() + .config_store(Arc::new(NoopConfigStore)) + .secret_store(Arc::new(NoopSecretStore)) + .kv_store(Arc::new(edgezero_core::key_value_store::NoopKvStore)) + .backend(Arc::new(NoopBackend)) + .http_client(Arc::new(NoopHttpClient)) + .geo(Arc::new(NoopGeo)) + .client_info(crate::platform::ClientInfo::default()) + .timer(Arc::new(RecordingTimer(Arc::clone(&requested)))) + .build(); + let events = Rc::new(RefCell::new(Vec::new())); + let calls = Rc::new(Cell::new(0)); + let mut chain = HtmlWithStreamingProcessors::from_stages( + stage("parser", b"ready", None, &events, &calls), + vec![HtmlProcessorStage::new( + "B", + stage("B", b"tail", Some(b"late"), &events, &calls), + true, + false, + 64, + )], + ); + let deadline = StreamFinishDeadline::new(Duration::from_secs(5), true); + chain + .begin_finish() + .expect("should make parser output ready"); + std::thread::sleep(Duration::from_millis(20)); + let mut timer = None; + block_on(chain.finish_next(HtmlFinishContext { + response_status: StatusCode::OK, + services: &services, + deadline: &deadline, + timer: &mut timer, + limit: 64, + marker: None, + })) + .expect("should finish a callback after ready-prefix delivery"); + let requested = requested + .lock() + .expect("should read recorded timer request"); + assert_eq!( + requested.len(), + 1, + "should initialize one cooperative timer at first callback" + ); + assert!( + requested[0] <= Duration::from_millis(4980), + "should subtract time already spent before callback polling" + ); + } + + struct PendingFinish { + tail: Vec, + dropped: Rc>, + polls: Rc>, + fallback_pointer: Rc>, + } + struct BorrowProbe<'a> { + _bytes: &'a [u8], + dropped: Rc>, + } + impl Drop for BorrowProbe<'_> { + fn drop(&mut self) { + self.dropped.set(true); + } + } + impl StreamProcessor for PendingFinish { + fn process_chunk(&mut self, chunk: &[u8], last: bool) -> io::Result> { + if last { + Ok(std::mem::take(&mut self.tail)) + } else { + Ok(chunk.to_vec()) + } + } + fn requires_async_finish(&self) -> bool { + true + } + fn async_finish<'a>( + &'a mut self, + input: StreamFinishInput<'a>, + ) -> Option>>> { + Some(Box::pin(async move { + let _probe = BorrowProbe { + _bytes: input.output, + dropped: Rc::clone(&self.dropped), + }; + self.fallback_pointer.set(input.output.as_ptr() as usize); + self.polls.set(self.polls.get() + 1); + input.replacement.append(b"partial")?; + futures::future::pending().await + })) + } + } + + fn timer_after_callback_poll() -> Option>> { + let mut polled = false; + Some( + Box::pin(poll_fn(move |context| { + if polled { + Poll::Ready(()) + } else { + polled = true; + context.waker().wake_by_ref(); + Poll::Pending + } + })) + .boxed_local() + .fuse(), + ) + } + + #[test] + fn async_eof_timeout_drops_borrow_and_preserves_fallback_allocation() { + let events = Rc::new(RefCell::new(Vec::new())); + let calls = Rc::new(Cell::new(0)); + let dropped = Rc::new(Cell::new(false)); + let pointer = Rc::new(Cell::new(0)); + let mut chain = HtmlWithStreamingProcessors::from_stages( + stage("parser", b"", None, &events, &calls), + vec![HtmlProcessorStage::new( + "pending", + Box::new(PendingFinish { + tail: b"held".to_vec(), + dropped: Rc::clone(&dropped), + polls: Rc::clone(&calls), + fallback_pointer: Rc::clone(&pointer), + }), + true, + false, + 64, + )], + ); + chain.begin_finish().expect("should finish parser"); + let services = noop_services(); + let deadline = StreamFinishDeadline::new(Duration::from_secs(1), true); + let mut timer = timer_after_callback_poll(); + let output = block_on(chain.finish_next(HtmlFinishContext { + response_status: StatusCode::OK, + services: &services, + deadline: &deadline, + timer: &mut timer, + limit: 64, + marker: None, + })) + .expect("should retain output on timeout") + .expect("should return held output"); + assert_eq!(output, b"held", "should discard partial candidate"); + assert_eq!( + output.as_ptr() as usize, + pointer.get(), + "should keep the original allocation outside the dropped future" + ); + assert!( + dropped.get(), + "should drop callback borrows before selecting fallback" + ); + assert_eq!(calls.get(), 1, "should poll the pending callback"); + } + + #[test] + fn async_eof_timer_expiry_skips_later_io_but_completes_downstream_eof() { + let events = Rc::new(RefCell::new(Vec::new())); + let calls = Rc::new(Cell::new(0)); + let mut chain = HtmlWithStreamingProcessors::from_stages( + stage("parser", b"", None, &events, &calls), + vec![ + HtmlProcessorStage::new( + "pending", + Box::new(PendingFinish { + tail: b"held".to_vec(), + dropped: Rc::new(Cell::new(false)), + polls: Rc::clone(&calls), + fallback_pointer: Rc::new(Cell::new(0)), + }), + true, + false, + 64, + ), + HtmlProcessorStage::new( + "C", + stage("C", b"ctail", Some(b"wrong"), &events, &calls), + true, + false, + 64, + ), + ], + ); + chain.begin_finish().expect("should finish parser"); + let services = noop_services(); + let deadline = StreamFinishDeadline::new(Duration::from_secs(1), true); + let mut timer = timer_after_callback_poll(); + let first = block_on(chain.finish_next(HtmlFinishContext { + response_status: StatusCode::OK, + services: &services, + deadline: &deadline, + timer: &mut timer, + limit: 64, + marker: None, + })) + .expect("should finish timed-out upstream") + .expect("should return transformed fallback"); + assert_eq!( + first, b"HELD", + "should send fallback downstream as normal input" + ); + let second = block_on(chain.finish_next(HtmlFinishContext { + response_status: StatusCode::OK, + services: &services, + deadline: &deadline, + timer: &mut timer, + limit: 64, + marker: None, + })) + .expect("should complete downstream EOF") + .expect("should return its synchronous tail"); + assert_eq!( + second, b"ctail", + "should skip later optional I/O after timer expiry" + ); + assert_eq!(calls.get(), 1, "should not poll a later optional callback"); + assert_eq!( + events + .borrow() + .iter() + .filter(|event| event.as_str() == "C:eof") + .count(), + 1, + "should complete downstream synchronous EOF once" + ); + } + + #[derive(Clone, Copy)] + enum LeafOutcome { + Error, + PoisonedReplacement, + LateReplacement, + EmptyReplacement, + Replacement, + } + + #[derive(Clone, Default)] + struct LeafObservation { + calls: Rc>, + eof_calls: Rc>, + fallback_pointer: Rc>, + dropped: Rc>, + } + + struct LeafProbe { + tail: Vec, + outcome: LeafOutcome, + observation: LeafObservation, + } + + impl StreamProcessor for LeafProbe { + fn process_chunk(&mut self, chunk: &[u8], last: bool) -> io::Result> { + if last { + self.observation + .eof_calls + .set(self.observation.eof_calls.get() + 1); + Ok(std::mem::take(&mut self.tail)) + } else { + Ok(chunk.to_vec()) + } + } + + fn requires_async_finish(&self) -> bool { + true + } + + fn async_finish<'a>( + &'a mut self, + input: StreamFinishInput<'a>, + ) -> Option>>> { + Some(Box::pin(async move { + let _probe = BorrowProbe { + _bytes: input.output, + dropped: Rc::clone(&self.observation.dropped), + }; + self.observation.calls.set(self.observation.calls.get() + 1); + self.observation + .fallback_pointer + .set(input.output.as_ptr() as usize); + match self.outcome { + LeafOutcome::Error => { + input.replacement.append(b"partial")?; + Err(Report::new(StreamFinishError::Unavailable)) + } + LeafOutcome::PoisonedReplacement => { + input.replacement.append(b"partial")?; + let _ = input + .replacement + .append(&[b'x'; 65]) + .expect_err("should poison output beyond the 64-byte cap"); + Ok(StreamFinish::Replace) + } + LeafOutcome::LateReplacement => { + input.replacement.append(b"candidate")?; + input.deadline.mark_expired(); + Ok(StreamFinish::Replace) + } + LeafOutcome::EmptyReplacement => Ok(StreamFinish::Replace), + LeafOutcome::Replacement => { + input.replacement.append(b"candidate")?; + Ok(StreamFinish::Replace) + } + } + })) + } + } + + fn complete_leaf_probe( + outcome: LeafOutcome, + mut timer: Option>>, + ) -> (Vec, LeafObservation, usize) { + let events = Rc::new(RefCell::new(Vec::new())); + let calls = Rc::new(Cell::new(0)); + let observation = LeafObservation::default(); + let fallback = b"fallback".to_vec(); + let fallback_pointer = fallback.as_ptr() as usize; + let mut chain = HtmlWithStreamingProcessors::from_stages( + stage("parser", b"", None, &events, &calls), + vec![HtmlProcessorStage::new( + "probe", + Box::new(LeafProbe { + tail: fallback, + outcome, + observation: observation.clone(), + }), + true, + false, + 64, + )], + ); + chain.begin_finish().expect("should finish the parser"); + let services = noop_services(); + let deadline = StreamFinishDeadline::new(Duration::from_secs(30), timer.is_some()); + let output = block_on(chain.finish_next(HtmlFinishContext { + response_status: StatusCode::OK, + services: &services, + deadline: &deadline, + timer: &mut timer, + limit: 64, + marker: None, + })) + .expect("should complete synchronous EOF despite optional failure") + .expect("should return the selected stage output"); + assert_eq!( + observation.eof_calls.get(), + 1, + "should complete the processor EOF exactly once" + ); + (output, observation, fallback_pointer) + } + + #[test] + fn async_eof_callback_error_discards_partial_candidate_without_copying_fallback() { + let (output, observation, fallback_pointer) = complete_leaf_probe(LeafOutcome::Error, None); + assert_eq!(output, b"fallback", "should discard the partial candidate"); + assert_eq!( + output.as_ptr() as usize, + fallback_pointer, + "should retain the driver-owned fallback allocation" + ); + assert_eq!( + observation.fallback_pointer.get(), + fallback_pointer, + "should lend the original fallback to optional work" + ); + assert!( + observation.dropped.get(), + "should drop callback borrows before returning fallback" + ); + assert_eq!(observation.calls.get(), 1, "should run optional work once"); + } + + #[test] + fn async_eof_poisoned_candidate_cannot_commit_even_when_callback_returns_replace() { + let (output, observation, fallback_pointer) = + complete_leaf_probe(LeafOutcome::PoisonedReplacement, None); + assert_eq!( + output, b"fallback", + "should reject partial output after an ignored append error" + ); + assert_eq!( + output.as_ptr() as usize, + fallback_pointer, + "should retain the original fallback allocation after poisoning" + ); + assert!(observation.dropped.get(), "should release callback borrows"); + assert_eq!(observation.calls.get(), 1, "should run optional work once"); + } + + #[test] + fn async_eof_timer_wins_when_timer_and_replacement_are_both_ready() { + let timer = Some(futures::future::ready(()).boxed_local().fuse()); + let (output, observation, fallback_pointer) = + complete_leaf_probe(LeafOutcome::Replacement, timer); + assert_eq!( + output, b"fallback", + "should prefer expiry over ready output" + ); + assert_eq!( + output.as_ptr() as usize, + fallback_pointer, + "should preserve fallback allocation when the timer wins" + ); + assert_eq!( + observation.calls.get(), + 0, + "should not poll callback work when the timer is already ready" + ); + } + + #[test] + fn async_eof_replacement_completed_after_expiry_retains_original_fallback() { + let (output, observation, fallback_pointer) = + complete_leaf_probe(LeafOutcome::LateReplacement, None); + assert_eq!( + output, b"fallback", + "should discard a late successful result" + ); + assert_eq!( + output.as_ptr() as usize, + fallback_pointer, + "should preserve the original fallback after late completion" + ); + assert!( + observation.dropped.get(), + "should drop the completed callback before selecting fallback" + ); + assert_eq!( + observation.calls.get(), + 1, + "should exercise expiry during callback execution" + ); + } + + #[test] + fn async_eof_empty_replace_intentionally_removes_nonempty_fallback() { + let (output, observation, _) = complete_leaf_probe(LeafOutcome::EmptyReplacement, None); + assert!( + output.is_empty(), + "should accept an empty replacement instead of retaining fallback" + ); + assert_eq!(observation.calls.get(), 1, "should run optional work once"); + assert!(observation.dropped.get(), "should release callback borrows"); + } + + #[test] + fn async_eof_failed_later_stage_preserves_accepted_upstream_replacement() { + let events = Rc::new(RefCell::new(Vec::new())); + let upstream_calls = Rc::new(Cell::new(0)); + let observation = LeafObservation::default(); + let mut chain = HtmlWithStreamingProcessors::from_stages( + stage("parser", b"", None, &events, &upstream_calls), + vec![ + HtmlProcessorStage::new( + "upstream", + stage("A", b"old", Some(b"accepted"), &events, &upstream_calls), + true, + false, + 64, + ), + HtmlProcessorStage::new( + "later", + Box::new(LeafProbe { + tail: b"tail".to_vec(), + outcome: LeafOutcome::Error, + observation: observation.clone(), + }), + true, + true, + 64, + ), + ], + ); + assert!( + chain + .process_chunk(b"page", false) + .expect("should process ordinary input") + .is_empty(), + "should retain only the explicitly buffered downstream stage" + ); + chain.begin_finish().expect("should finish the parser"); + let services = noop_services(); + let deadline = StreamFinishDeadline::new(Duration::from_secs(30), false); + let mut timer = None; + let first = block_on(chain.finish_next(HtmlFinishContext { + response_status: StatusCode::OK, + services: &services, + deadline: &deadline, + timer: &mut timer, + limit: 64, + marker: None, + })) + .expect("should commit the upstream replacement") + .expect("should return the downstream stage's ready output"); + assert!( + first.is_empty(), + "should deliver accepted upstream bytes into downstream retention" + ); + assert_eq!(upstream_calls.get(), 1, "should finish upstream I/O once"); + assert_eq!( + observation.eof_calls.get(), + 0, + "should preserve downstream EOF ordering" + ); + let output = block_on(chain.finish_next(HtmlFinishContext { + response_status: StatusCode::OK, + services: &services, + deadline: &deadline, + timer: &mut timer, + limit: 64, + marker: None, + })) + .expect("should retain downstream fallback on optional failure") + .expect("should release downstream retained output"); + assert_eq!( + output, b"pageacceptedtail", + "should retain accepted upstream output while discarding the later candidate" + ); + assert_eq!( + output.as_ptr() as usize, + observation.fallback_pointer.get(), + "should move the retained fallback without a rollback copy" + ); + assert_eq!(observation.calls.get(), 1, "should run later I/O once"); + assert_eq!( + observation.eof_calls.get(), + 1, + "should finish later EOF once" + ); + assert!(observation.dropped.get(), "should release callback borrows"); + } + + #[cfg(not(target_arch = "wasm32"))] + struct SlowConstructor(Rc>); + #[cfg(not(target_arch = "wasm32"))] + impl StreamProcessor for SlowConstructor { + fn process_chunk(&mut self, chunk: &[u8], _: bool) -> io::Result> { + Ok(chunk.to_vec()) + } + fn requires_async_finish(&self) -> bool { + true + } + fn async_finish<'a>( + &'a mut self, + _: StreamFinishInput<'a>, + ) -> Option>>> { + std::thread::sleep(Duration::from_millis(5)); + Some(Box::pin(async move { + self.0.set(self.0.get() + 1); + Ok(StreamFinish::Unchanged) + })) + } + } + + #[cfg(not(target_arch = "wasm32"))] + #[test] + fn async_eof_does_not_poll_work_after_constructor_consumes_budget() { + let events = Rc::new(RefCell::new(Vec::new())); + let calls = Rc::new(Cell::new(0)); + let mut chain = HtmlWithStreamingProcessors::from_stages( + stage("parser", b"", None, &events, &calls), + vec![HtmlProcessorStage::new( + "slow", + Box::new(SlowConstructor(Rc::clone(&calls))), + true, + false, + 64, + )], + ); + chain.begin_finish().expect("should finish the parser"); + let services = noop_services(); + let deadline = StreamFinishDeadline::new(Duration::from_millis(1), false); + let mut timer = None; + block_on(chain.finish_next(HtmlFinishContext { + response_status: StatusCode::OK, + services: &services, + deadline: &deadline, + timer: &mut timer, + limit: 64, + marker: None, + })) + .expect("should retain fallback after constructor consumes budget"); + assert_eq!( + calls.get(), + 0, + "should check deadline again before polling optional work" + ); + } +} diff --git a/crates/trusted-server-core/src/integrations/html_finalization.rs b/crates/trusted-server-core/src/integrations/html_finalization.rs index f2296e290..bf32db48a 100644 --- a/crates/trusted-server-core/src/integrations/html_finalization.rs +++ b/crates/trusted-server-core/src/integrations/html_finalization.rs @@ -1,270 +1,75 @@ -//! Contracts for optional, bounded asynchronous HTML document finalization. +//! Test-only adapter preserving existing buffered-output regression scenarios. -use std::time::Duration; +use std::io; +use std::sync::Arc; use error_stack::Report; use futures::future::LocalBoxFuture; -use http::StatusCode; -use web_time::Instant; -use super::IntegrationHtmlStreamContext; -use crate::platform::RuntimeServices; +use super::{IntegrationHtmlStreamContext, IntegrationHtmlStreamProcessorFactory}; +use crate::streaming_processor::StreamProcessor; +use crate::streaming_processor::finish::{StreamFinishDeadline, StreamFinishInput}; + +pub use crate::streaming_processor::finish::{ + StreamFinish as HtmlFinalization, StreamFinishError as HtmlFinalizationError, + StreamOutputWriter as HtmlReplacementWriter, +}; -/// Creates a single-use finalizer for each processed HTML document. -/// -/// Factories may be shared across requests; the returned session is request-local. -/// Creation performs synchronous setup only and must not start outside I/O. pub trait IntegrationHtmlFinalizerFactory: Send + Sync { - /// Return the integration identifier used for controlled diagnostic messages. fn integration_id(&self) -> &'static str; - - /// Create a session using the parser's shared document state. fn create(&self, context: IntegrationHtmlStreamContext) -> Box; } -/// Finalizes complete, synchronously transformed HTML before auction assembly. -/// -/// Sessions and their futures need not be `Send`. Output is committed atomically. -/// The method returns its future promptly, with I/O inside the future. Replacements -/// preserve document structure, CSP/RSC semantics, and opaque parser seam markers. pub trait IntegrationHtmlFinalizer { - /// Await optional external work and select unchanged or replacement output. - /// - /// # Errors - /// - /// Return [`HtmlFinalizationError`] when optional work cannot complete. Core - /// discards partial replacement output and retains the input on failure. - /// - /// # Examples - /// - /// ```ignore - /// Box::pin(async move { - /// input.replacement.append(b"

Example article

")?; - /// Ok(HtmlFinalization::Replace) - /// }) - /// ``` fn finalize<'a>( &'a mut self, input: HtmlFinalizationInput<'a>, ) -> LocalBoxFuture<'a, Result>>; } -/// Borrows the current document, separate candidate storage, and request services. pub struct HtmlFinalizationInput<'a> { - /// Origin status of this complete response, including eligible error HTML. - pub response_status: StatusCode, - /// Complete transformed HTML; ownership stays with the driver. + pub response_status: http::StatusCode, pub html: &'a [u8], - /// Separate bounded sink for an atomic replacement candidate. pub replacement: &'a mut HtmlReplacementWriter, - /// Request-time services, including outbound HTTP and optional KV. - pub services: &'a RuntimeServices, - /// The absolute deadline shared by every finalizer in this document. - pub deadline: &'a HtmlFinalizationDeadline, -} - -/// Selects whether to retain input or commit the bounded replacement candidate. -#[derive(Debug, Clone, Copy, PartialEq, Eq)] -pub enum HtmlFinalization { - /// Retain the current input without copying it. - Unchanged, - /// Commit bytes appended to [`HtmlReplacementWriter`]. - Replace, + pub services: &'a crate::platform::RuntimeServices, + pub deadline: &'a StreamFinishDeadline, } -/// Controlled failure categories for optional document finalization. -#[derive(Debug, derive_more::Display)] -pub enum HtmlFinalizationError { - /// The required outside service is unavailable. - #[display("HTML finalization service is unavailable")] - Unavailable, - /// Outside work cannot produce an acceptable result. - #[display("HTML finalization result is rejected")] - Rejected, - /// The bounded candidate cannot accept more bytes. - #[display("HTML replacement exceeds the {limit}-byte limit")] - OutputLimitExceeded { - /// Maximum candidate size in bytes. - limit: usize, - }, -} - -impl core::error::Error for HtmlFinalizationError {} - -/// Separate bounded storage for a finalizer's replacement candidate. -/// -/// A failed append poisons the candidate, preventing partial commitment even -/// when an integration ignores the returned error. -pub struct HtmlReplacementWriter { - bytes: Vec, - limit: usize, - poisoned: bool, -} +pub(super) struct TestFinishFactory(pub Arc); +struct TestFinishProcessor(Box); -impl HtmlReplacementWriter { - pub(crate) fn new(limit: usize) -> Self { - Self { - bytes: Vec::new(), - limit, - poisoned: false, - } +impl IntegrationHtmlStreamProcessorFactory for TestFinishFactory { + fn integration_id(&self) -> &'static str { + self.0.integration_id() } - - /// Append bytes without growing the candidate beyond its configured limit. - /// - /// # Errors - /// - /// Returns [`HtmlFinalizationError::OutputLimitExceeded`] when the candidate - /// is poisoned, length arithmetic overflows, or the append exceeds the limit. - /// - /// # Examples - /// - /// ```ignore - /// input.replacement.append(b"

Example article

")?; - /// ``` - pub fn append(&mut self, chunk: &[u8]) -> Result<(), Report> { - let next = self - .bytes - .len() - .checked_add(chunk.len()) - .filter(|length| *length <= self.limit); - let next = match next { - Some(next) if !self.poisoned => next, - _ => { - self.poisoned = true; - return Err(Report::new(HtmlFinalizationError::OutputLimitExceeded { - limit: self.limit, - })); - } - }; - if next > self.bytes.capacity() { - let capacity = self - .bytes - .capacity() - .saturating_mul(2) - .max(next) - .min(self.limit); - self.bytes.reserve_exact(capacity - self.bytes.len()); - } - self.bytes.extend_from_slice(chunk); - Ok(()) + fn requires_async_finish(&self) -> bool { + true } - - pub(crate) fn is_poisoned(&self) -> bool { - self.poisoned + fn buffer_output_until_eof(&self) -> bool { + true } - - pub(crate) fn into_bytes(self) -> Vec { - self.bytes + fn create(&self, context: IntegrationHtmlStreamContext) -> Box { + Box::new(TestFinishProcessor(self.0.create(context))) } } -/// Absolute finish budget shared by all finalizers for one HTML document. -/// -/// A cooperative timer can interrupt yielding work, but cannot preempt CPU work -/// or blocking host calls. Consumers must bound their own calls on Fastly. -pub struct HtmlFinalizationDeadline { - started_at: Instant, - timeout: Duration, - cooperative_timer: bool, -} - -impl HtmlFinalizationDeadline { - pub(crate) fn new(timeout: Duration, cooperative_timer: bool) -> Self { - Self { - started_at: Instant::now(), - timeout, - cooperative_timer, - } - } - - /// Return the remaining whole-chain budget, saturating at zero. - #[must_use] - pub fn remaining(&self) -> Duration { - self.remaining_at(Instant::now()) +impl StreamProcessor for TestFinishProcessor { + fn process_chunk(&mut self, chunk: &[u8], _: bool) -> io::Result> { + Ok(chunk.to_vec()) } - - /// Report whether the request has a cooperative timer for yielding work. - #[must_use] - pub fn has_cooperative_timer(&self) -> bool { - self.cooperative_timer + fn requires_async_finish(&self) -> bool { + true } - - pub(crate) fn remaining_at(&self, now: Instant) -> Duration { - self.timeout - .saturating_sub(now.saturating_duration_since(self.started_at)) - } - - pub(crate) fn is_expired_at(&self, now: Instant) -> bool { - self.remaining_at(now).is_zero() - } -} - -#[cfg(test)] -mod tests { - use std::time::Duration; - - use super::*; - - #[test] - fn replacement_limit_poison_preserves_the_candidate() { - let mut writer = HtmlReplacementWriter::new(4); - writer.append(b"page").expect("should accept the exact cap"); - let _ = writer - .append(b"!") - .expect_err("should reject output beyond the cap"); - assert!(writer.is_poisoned(), "should poison an oversized candidate"); - assert_eq!(writer.bytes, b"page", "should preserve the accepted prefix"); - let _ = writer - .append(b"") - .expect_err("should reject writes after poisoning"); - } - - #[test] - fn replacement_capacity_stays_under_the_limit() { - let mut writer = HtmlReplacementWriter::new(17); - assert_eq!(writer.bytes.capacity(), 0, "should allocate lazily"); - for _ in 0..17 { - writer.append(b"x").expect("should append within the limit"); - assert!(writer.bytes.capacity() <= 17, "should bound reserve growth"); - } - } - - #[test] - fn empty_replacement_is_distinct_from_unchanged() { - let writer = HtmlReplacementWriter::new(4); - assert!( - writer.into_bytes().is_empty(), - "should allow an empty replacement" - ); - assert_ne!( - HtmlFinalization::Unchanged, - HtmlFinalization::Replace, - "should distinguish outcomes" - ); - } - - #[test] - fn deadline_expires_at_the_boundary() { - let deadline = HtmlFinalizationDeadline::new(Duration::from_millis(5), false); - assert_eq!( - deadline.remaining_at(deadline.started_at), - Duration::from_millis(5), - "should start with the complete budget" - ); - let expired = deadline.started_at + Duration::from_millis(5); - assert!( - deadline.is_expired_at(expired), - "should expire at the exact boundary" - ); - assert_eq!( - deadline.remaining_at(expired + Duration::from_millis(1)), - Duration::ZERO, - "should saturate elapsed time" - ); - assert!( - !deadline.has_cooperative_timer(), - "should expose the absence of preemption" - ); + fn async_finish<'a>( + &'a mut self, + input: StreamFinishInput<'a>, + ) -> Option>>> { + Some(self.0.finalize(HtmlFinalizationInput { + response_status: input.response_status, + html: input.output, + replacement: input.replacement, + services: input.services, + deadline: input.deadline, + })) } } diff --git a/crates/trusted-server-core/src/integrations/mod.rs b/crates/trusted-server-core/src/integrations/mod.rs index 2419681f3..1d33b8436 100644 --- a/crates/trusted-server-core/src/integrations/mod.rs +++ b/crates/trusted-server-core/src/integrations/mod.rs @@ -18,6 +18,7 @@ pub mod didomi; pub mod google_tag_manager; pub mod gpt; pub mod gpt_diagnostics; +#[cfg(test)] mod html_finalization; pub mod js_asset_proxy; pub mod lockr; @@ -29,9 +30,10 @@ mod registry; pub mod sourcepoint; pub mod testlight; +#[cfg(test)] pub use html_finalization::{ - HtmlFinalization, HtmlFinalizationDeadline, HtmlFinalizationError, HtmlFinalizationInput, - HtmlReplacementWriter, IntegrationHtmlFinalizer, IntegrationHtmlFinalizerFactory, + HtmlFinalization, HtmlFinalizationError, HtmlFinalizationInput, HtmlReplacementWriter, + IntegrationHtmlFinalizer, IntegrationHtmlFinalizerFactory, }; pub use registry::{ diff --git a/crates/trusted-server-core/src/integrations/registry.rs b/crates/trusted-server-core/src/integrations/registry.rs index af177c0f1..8fdde55ea 100644 --- a/crates/trusted-server-core/src/integrations/registry.rs +++ b/crates/trusted-server-core/src/integrations/registry.rs @@ -8,6 +8,7 @@ use error_stack::Report; use http::{Method, Request, Response}; use matchit::Router; +#[cfg(test)] use super::IntegrationHtmlFinalizerFactory; use crate::auction::AuctionPlan; @@ -603,6 +604,18 @@ pub trait IntegrationHtmlStreamProcessorFactory: Send + Sync { /// Identifier for logging and diagnostics. fn integration_id(&self) -> &'static str; + /// Declare optional asynchronous EOF capability before cache admission. + fn requires_async_finish(&self) -> bool { + false + } + + /// Declare that core must retain this stage's synchronous output until EOF. + /// + /// Requires asynchronous completion and explicit operator permission. + fn buffer_output_until_eof(&self) -> bool { + false + } + /// Create a request-local streaming processor. fn create(&self, context: IntegrationHtmlStreamContext) -> Box; } @@ -629,8 +642,6 @@ pub struct IntegrationRegistration { pub attribute_rewriters: Vec>, pub script_rewriters: Vec>, pub html_stream_processors: Vec>, - /// Terminal finalizers in registration order. - pub html_finalizers: Vec>, pub head_injectors: Vec>, pub request_filters: Vec>, } @@ -657,7 +668,6 @@ impl IntegrationRegistrationBuilder { attribute_rewriters: Vec::new(), script_rewriters: Vec::new(), html_stream_processors: Vec::new(), - html_finalizers: Vec::new(), head_injectors: Vec::new(), request_filters: Vec::new(), }, @@ -694,16 +704,6 @@ impl IntegrationRegistrationBuilder { self } - /// Register an opt-in asynchronous HTML terminal phase. - #[must_use] - pub fn with_html_finalizer( - mut self, - factory: Arc, - ) -> Self { - self.registration.html_finalizers.push(factory); - self - } - #[must_use] pub fn with_head_injector(mut self, injector: Arc) -> Self { self.registration.head_injectors.push(injector); @@ -758,7 +758,6 @@ struct IntegrationRegistryInner { html_rewriters: Vec>, script_rewriters: Vec>, html_stream_processors: Vec>, - html_finalizers: Vec>, head_injectors: Vec>, request_filters: Vec>, } @@ -780,7 +779,6 @@ impl Default for IntegrationRegistryInner { html_rewriters: Vec::new(), script_rewriters: Vec::new(), html_stream_processors: Vec::new(), - html_finalizers: Vec::new(), head_injectors: Vec::new(), request_filters: Vec::new(), } @@ -939,7 +937,6 @@ impl IntegrationRegistry { inner .html_stream_processors .extend(registration.html_stream_processors); - inner.html_finalizers.extend(registration.html_finalizers); inner.head_injectors.extend(registration.head_injectors); inner.request_filters.extend(registration.request_filters); if registration.js_disabled { @@ -949,10 +946,12 @@ impl IntegrationRegistry { } } - Ok(Self { + let registry = Self { inner: Arc::new(inner), plan: Some(plan), - }) + }; + registry.ensure_async_html_finish_allowed(settings)?; + Ok(registry) } /// Return whether this registry and another consumer share the same plan allocation. @@ -1129,25 +1128,70 @@ impl IntegrationRegistry { self.inner.html_stream_processors.clone() } - /// Report finalizer presence before any shared-template lookup. + /// Report async EOF capability before any processed-template lookup. #[must_use] - pub fn has_html_finalizers(&self) -> bool { - !self.inner.html_finalizers.is_empty() + pub fn has_async_html_finish(&self) -> bool { + self.inner + .html_stream_processors + .iter() + .any(|factory| factory.requires_async_finish()) + } + + /// Validate operator permission and factory retention declarations. + /// + /// # Errors + /// + /// Returns a configuration error for missing permission or inconsistent + /// factory metadata. Requested integration work is never silently omitted. + pub fn ensure_async_html_finish_allowed( + &self, + settings: &Settings, + ) -> Result<(), Report> { + for factory in &self.inner.html_stream_processors { + if factory.buffer_output_until_eof() && !factory.requires_async_finish() { + return Err(Report::new(TrustedServerError::Configuration { + message: format!( + "Integration {} declares EOF buffering without async completion", + factory.integration_id() + ), + })); + } + if factory.requires_async_finish() && !settings.publisher.allow_async_html_finish { + return Err(Report::new(TrustedServerError::Configuration { + message: format!( + "Integration {} requires publisher.allow_async_html_finish = true; EOF I/O can delay HTML delivery", + factory.integration_id() + ), + })); + } + } + Ok(()) } - /// Return terminal factories in registration order. + /// Construct an isolated processor registry for regression tests. + #[cfg(test)] #[must_use] - pub fn html_finalizer_factories(&self) -> Vec> { - self.inner.html_finalizers.clone() + pub fn from_html_finalizers(factories: Vec>) -> Self { + Self::from_html_stream_processors( + factories + .into_iter() + .map(|factory| { + Arc::new(super::html_finalization::TestFinishFactory(factory)) + as Arc + }) + .collect(), + ) } - /// Construct an isolated registry for terminal-phase tests. + /// Construct an isolated ordered processor registry for fixture controls. #[cfg(any(test, feature = "test-utils"))] #[must_use] - pub fn from_html_finalizers(factories: Vec>) -> Self { + pub fn from_html_stream_processors( + factories: Vec>, + ) -> Self { Self { inner: Arc::new(IntegrationRegistryInner { - html_finalizers: factories, + html_stream_processors: factories, ..Default::default() }), plan: None, @@ -1325,7 +1369,6 @@ impl IntegrationRegistry { html_rewriters: attribute_rewriters, script_rewriters, html_stream_processors: Vec::new(), - html_finalizers: Vec::new(), head_injectors: Vec::new(), request_filters: Vec::new(), deferred_js_ids: Vec::new(), @@ -1356,7 +1399,6 @@ impl IntegrationRegistry { html_rewriters: attribute_rewriters, script_rewriters, html_stream_processors: Vec::new(), - html_finalizers: Vec::new(), head_injectors, request_filters: Vec::new(), deferred_js_ids: Vec::new(), @@ -1383,7 +1425,6 @@ impl IntegrationRegistry { html_rewriters: Vec::new(), script_rewriters: Vec::new(), html_stream_processors: Vec::new(), - html_finalizers: Vec::new(), head_injectors: Vec::new(), request_filters, deferred_js_ids: Vec::new(), @@ -1450,7 +1491,6 @@ impl IntegrationRegistry { html_rewriters: Vec::new(), script_rewriters: Vec::new(), html_stream_processors: Vec::new(), - html_finalizers: Vec::new(), head_injectors: Vec::new(), request_filters: Vec::new(), deferred_js_ids: Vec::new(), @@ -1468,20 +1508,109 @@ mod tests { use crate::platform::test_support::noop_services; use http::{HeaderValue, StatusCode, header}; + #[cfg(feature = "test-utils")] + #[test] + fn async_eof_requires_explicit_operator_permission() { + let mut settings = crate::test_support::tests::create_test_settings(); + assert!( + !settings.publisher.allow_async_html_finish, + "should default to no EOF I/O permission" + ); + settings + .integrations + .insert_config( + "html_finalization_fixture", + &serde_json::json!({ + "enabled": true, + "lookup_url": "https://lookup.example.com/approve", + "transport_timeout_ms": 1000, + }), + ) + .expect("should configure the fictional EOF integration"); + let error = IntegrationRegistry::new(&settings) + .err() + .expect("should reject enabled EOF I/O without operator permission"); + let message = format!("{error:?}"); + assert!( + message.contains("allow_async_html_finish"), + "should name the required setting" + ); + assert!( + message.contains("html_finalization_fixture"), + "should identify the enabled integration" + ); + settings.publisher.allow_async_html_finish = true; + let registry = IntegrationRegistry::new(&settings) + .expect("should register EOF I/O after explicit permission"); + assert!( + registry.has_async_html_finish(), + "should expose capability before body processing" + ); + } + + #[test] + fn async_eof_permission_alone_adds_no_processing_path() { + let mut settings = crate::test_support::tests::create_test_settings(); + settings.publisher.allow_async_html_finish = true; + let registry = IntegrationRegistry::new(&settings).expect("should build ordinary registry"); + assert!( + !registry.has_async_html_finish(), + "should retain the normal path without EOF factories" + ); + } + + struct RetentionWithoutAsyncFactory; + + impl IntegrationHtmlStreamProcessorFactory for RetentionWithoutAsyncFactory { + fn integration_id(&self) -> &'static str { + "example_inconsistent_retention" + } + fn buffer_output_until_eof(&self) -> bool { + true + } + fn create( + &self, + _: IntegrationHtmlStreamContext, + ) -> Box { + Box::new(crate::streaming_replacer::StreamingReplacer::new(Vec::new())) + } + } + + #[test] + fn async_eof_rejects_retention_without_async_capability() { + let mut settings = crate::test_support::tests::create_test_settings(); + settings.publisher.allow_async_html_finish = true; + let registry = IntegrationRegistry::from_html_stream_processors(vec![Arc::new( + RetentionWithoutAsyncFactory, + )]); + let error = registry + .ensure_async_html_finish_allowed(&settings) + .expect_err("should reject retention without declared EOF capability"); + let message = format!("{error:?}"); + assert!( + message.contains("example_inconsistent_retention"), + "should identify the invalid factory" + ); + assert!( + message.contains("buffering without async completion"), + "should explain the invalid metadata even with permission" + ); + } + #[test] fn html_finalizers_are_empty_by_default() { let registry = IntegrationRegistry::from_request_filters(Vec::new()); assert!( - !registry.has_html_finalizers(), + !registry.has_async_html_finish(), "should retain the ordinary streaming path" ); assert!( - registry.html_finalizer_factories().is_empty(), + registry.html_stream_processor_factories().is_empty(), "should register no default finalizers" ); let registration = IntegrationRegistration::builder("example_finalizer").build(); assert!( - registration.html_finalizers.is_empty(), + registration.html_stream_processors.is_empty(), "should make finalization opt-in" ); } diff --git a/crates/trusted-server-core/src/publisher.rs b/crates/trusted-server-core/src/publisher.rs index 9036aee9b..ee2ef94de 100644 --- a/crates/trusted-server-core/src/publisher.rs +++ b/crates/trusted-server-core/src/publisher.rs @@ -93,7 +93,7 @@ use crate::streaming_processor::{ STREAM_CHUNK_SIZE, StreamProcessor, StreamingPipeline, }; use crate::streaming_replacer::create_url_replacer; -use html_finalization::HtmlFinalizationDriver; +use html_finalization::HtmlEofDriver; include!(concat!(env!("OUT_DIR"), "/template_build_digest.rs")); @@ -709,6 +709,9 @@ impl PublisherBodyProcessor { } impl StreamProcessor for PublisherBodyProcessor { + fn requires_async_finish(&self) -> bool { + self.inner.requires_async_finish() + } fn process_chunk(&mut self, chunk: &[u8], is_last: bool) -> Result, std::io::Error> { self.inner.process_chunk(chunk, is_last) } @@ -1524,7 +1527,7 @@ fn create_html_stream_processor( fn create_html_stream_processor_with_document_state( params: HtmlStreamProcessorParams<'_>, document_state: &IntegrationDocumentState, -) -> Result, Report> { +) -> Result> { let config = HtmlProcessorConfig::from_settings( params.settings, params.integration_registry, @@ -1557,10 +1560,13 @@ fn create_html_stream_processor_with_document_state( .with_csp_nonce_observer(csp_nonce_observed) .with_datadome_client_tag_suppression(params.suppress_datadome_client_side_tag); - Ok(create_html_processor_with_document_state( - config, - document_state, - )) + let processor = create_html_processor_with_document_state(config, document_state); + processor + .validate_capabilities() + .change_context(TrustedServerError::Configuration { + message: "HTML processor EOF capabilities are inconsistent".to_string(), + })?; + Ok(processor) } /// Result of publisher request handling, indicating whether the response body @@ -1817,13 +1823,19 @@ enum TemplateStoreOutcome { /// Panics if the `ad_bids_state` mutex is poisoned, which requires a prior panic while /// it was held. Every holder is a short, infallible read or write of an `Option`. pub async fn buffer_publisher_response_async( - publisher_response: PublisherResponse, + mut publisher_response: PublisherResponse, method: &Method, settings: &Settings, integration_registry: &IntegrationRegistry, orchestrator: &AuctionOrchestrator, services: &RuntimeServices, ) -> Result, Report> { + if let Err(error) = integration_registry.ensure_async_html_finish_allowed(settings) { + if let PublisherResponse::Stream { params, .. } = &mut publisher_response { + abandon_response_auction(params, services, "processor_init_error").await; + } + return Err(error); + } match publisher_response { PublisherResponse::Buffered(mut response) => { // A buffered-unmodified response can carry an origin body (a stream @@ -1867,14 +1879,14 @@ pub async fn buffer_publisher_response_async( params.range_associated = response_is_range_associated(&response); if params.is_range_associated() && is_html_content_type(¶ms.content_type) - && integration_registry.has_html_finalizers() + && integration_registry.has_async_html_finish() { abandon_response_auction(&mut params, services, "partial_response").await; *response.body_mut() = body; return Ok(response); } if is_html_content_type(¶ms.content_type) - && integration_registry.has_html_finalizers() + && integration_registry.has_async_html_finish() { enforce_synthesized_html_cache_privacy(&mut response); } @@ -2456,13 +2468,19 @@ async fn store_template_if_authorized( /// Panics if the `ad_bids_state` mutex is poisoned, which requires a prior panic while /// it was held. Every holder is a short, infallible read or write of an `Option`. pub async fn publisher_response_into_streaming_response( - publisher_response: PublisherResponse, + mut publisher_response: PublisherResponse, method: &Method, settings: Arc, integration_registry: &IntegrationRegistry, orchestrator: Arc, services: RuntimeServices, ) -> Result, Report> { + if let Err(error) = integration_registry.ensure_async_html_finish_allowed(&settings) { + if let PublisherResponse::Stream { params, .. } = &mut publisher_response { + abandon_response_auction(params, &services, "processor_init_error").await; + } + return Err(error); + } // A template can only be stored once the transform has produced every byte, and // streaming hands bytes to the client as they are produced rather than collecting // them. Shared modes therefore take the buffered finalizer, which already @@ -2631,7 +2649,7 @@ pub async fn publisher_response_into_streaming_response( params.range_associated = response_is_range_associated(&response); if params.is_range_associated() && is_html_content_type(¶ms.content_type) - && integration_registry.has_html_finalizers() + && integration_registry.has_async_html_finish() { abandon_response_auction(&mut params, &services, "partial_response").await; *response.body_mut() = body; @@ -2640,10 +2658,10 @@ pub async fn publisher_response_into_streaming_response( response.headers_mut().remove(header::CONTENT_LENGTH); let mut params = *params; if is_html_content_type(¶ms.content_type) - && integration_registry.has_html_finalizers() + && integration_registry.has_async_html_finish() { enforce_synthesized_html_cache_privacy(&mut response); - let mut driver = Box::pin(build_html_terminal_driver( + let mut driver = Box::pin(build_html_eof_driver( body, &mut params, &settings, @@ -3021,7 +3039,7 @@ pub fn stream_publisher_body( ) -> Result<(), Report> { if params.is_range_associated() && is_html_content_type(¶ms.content_type) - && integration_registry.has_html_finalizers() + && integration_registry.has_async_html_finish() { let mut input = body_as_reader(body)?; std::io::copy(&mut input, output).change_context(TrustedServerError::Proxy { @@ -3031,6 +3049,12 @@ pub fn stream_publisher_body( message: "Failed to flush range-associated HTML".to_string(), }); } + if is_html_content_type(¶ms.content_type) && integration_registry.has_async_html_finish() { + return Err(Report::new(TrustedServerError::Configuration { + message: "Registered HTML processors require an asynchronous publisher entry point" + .to_string(), + })); + } let borrowed = ProcessResponseParams { content_encoding: ¶ms.content_encoding, origin_host: ¶ms.origin_host, @@ -3072,14 +3096,14 @@ async fn abandon_response_auction( } } -async fn build_html_terminal_driver( +async fn build_html_eof_driver( body: EdgeBody, params: &mut OwnedProcessResponseParams, settings: &Settings, registry: &IntegrationRegistry, services: &RuntimeServices, -) -> Result, Report> { - match HtmlFinalizationDriver::new(body, params, settings, registry) { +) -> Result, Report> { + match HtmlEofDriver::new(body, params, settings, registry) { Ok(driver) => Ok(Box::new(driver)), Err(error) => { if let Some(dispatched) = params.dispatched_auction.take() { @@ -3096,7 +3120,7 @@ async fn build_html_terminal_driver( } } -async fn stream_html_terminal( +async fn stream_html_eof( body: EdgeBody, output: &mut W, params: &mut OwnedProcessResponseParams, @@ -3105,7 +3129,7 @@ async fn stream_html_terminal( orchestrator: &AuctionOrchestrator, services: &RuntimeServices, ) -> Result<(), Report> { - let mut driver = build_html_terminal_driver(body, params, settings, registry, services).await?; + let mut driver = build_html_eof_driver(body, params, settings, registry, services).await?; let deps = AuctionCollectDeps { price_granularity: params.price_granularity, ad_bids_state: ¶ms.ad_bids_state, @@ -3182,9 +3206,13 @@ pub async fn stream_publisher_body_async( orchestrator: &AuctionOrchestrator, services: &RuntimeServices, ) -> Result<(), Report> { + if let Err(error) = integration_registry.ensure_async_html_finish_allowed(settings) { + abandon_response_auction(params, services, "processor_init_error").await; + return Err(error); + } if params.is_range_associated() && is_html_content_type(¶ms.content_type) - && integration_registry.has_html_finalizers() + && integration_registry.has_async_html_finish() { abandon_response_auction(params, services, "partial_response").await; let mut source = BodyChunkSource::new(body, STREAM_CHUNK_SIZE) @@ -3200,8 +3228,8 @@ pub async fn stream_publisher_body_async( message: "Failed to flush range-associated HTML".to_string(), }); } - if is_html_content_type(¶ms.content_type) && integration_registry.has_html_finalizers() { - return Box::pin(stream_html_terminal( + if is_html_content_type(¶ms.content_type) && integration_registry.has_async_html_finish() { + return Box::pin(stream_html_eof( body, output, params, @@ -4158,6 +4186,72 @@ impl InlineBodyCloseSeam { self.buffered.drain(..split_at).collect() } + fn push_bounded(&mut self, chunk: &[u8], limit: usize) -> std::io::Result> { + let overflow = || std::io::Error::other("HTML auction suffix exceeds retained-byte limit"); + if self.found { + let next = self + .buffered + .len() + .checked_add(chunk.len()) + .filter(|size| *size <= limit) + .ok_or_else(overflow)?; + self.buffered.reserve_exact(next - self.buffered.len()); + self.buffered.extend_from_slice(chunk); + return Ok(Vec::new()); + } + let mut boundary = self.buffered.clone(); + boundary.extend_from_slice(&chunk[..chunk.len().min(self.token.len().saturating_sub(1))]); + let crossing = + find_bytes(&boundary, &self.token).filter(|position| *position < self.buffered.len()); + let inside = find_bytes(chunk, &self.token); + if let Some((old_prefix, chunk_prefix, suffix_start)) = crossing + .map(|position| { + ( + position, + 0, + position + self.token.len() - self.buffered.len(), + ) + }) + .or_else(|| { + inside.map(|position| (self.buffered.len(), position, position + self.token.len())) + }) + { + let suffix = &chunk[suffix_start..]; + if suffix.len() > limit { + return Err(overflow()); + } + let mut ready = self.buffered[..old_prefix].to_vec(); + ready.extend_from_slice(&chunk[..chunk_prefix]); + self.buffered = Vec::new(); + self.buffered.reserve_exact(suffix.len()); + self.buffered.extend_from_slice(suffix); + self.found = true; + return Ok(ready); + } + let tail = if chunk.len() >= self.token.len().saturating_sub(1) { + chunk[chunk.len() - self.token.len().saturating_sub(1)..].to_vec() + } else { + let mut tail = self.buffered.clone(); + tail.extend_from_slice(chunk); + tail + }; + let keep = longest_suffix_prefix(&tail, &self.token); + if keep > limit { + return Err(overflow()); + } + let old_ready = self + .buffered + .len() + .saturating_sub(keep.saturating_sub(chunk.len())); + let chunk_ready = chunk.len().saturating_sub(keep); + let mut ready = self.buffered[..old_ready].to_vec(); + ready.extend_from_slice(&chunk[..chunk_ready]); + self.buffered = Vec::new(); + self.buffered.reserve_exact(keep); + self.buffered.extend_from_slice(&tail[tail.len() - keep..]); + Ok(ready) + } + fn found(&self) -> bool { self.found } @@ -4697,8 +4791,11 @@ pub async fn handle_publisher_request( mut req: Request, edge_header: EdgeCacheHeader, ) -> Result> { + auction + .integration_registry + .ensure_async_html_finish_allowed(settings)?; log::debug!("Proxying request to publisher_origin"); - let html_finalizers_present = auction.integration_registry.has_html_finalizers(); + let async_html_finish_present = auction.integration_registry.has_async_html_finish(); // Adapter fallbacks prepare this before EC/cookie handling. Keep this // idempotent call as a direct-handler safety net and for focused tests. @@ -4942,7 +5039,7 @@ pub async fn handle_publisher_request( if should_run_ad_stack || datadome_suppression_requires_full_body - || (html_finalizers_present + || (async_html_finish_present && req.method() == Method::GET && is_html_document_request(&req)) { @@ -4992,7 +5089,7 @@ pub async fn handle_publisher_request( .creative_opportunities .as_ref() .is_some_and(CreativeOpportunitiesConfig::origin_readthrough_enabled); - let request_can_use_shared_template = !html_finalizers_present + let request_can_use_shared_template = !async_html_finish_present && request_can_use_shared_template( shared_request_inputs, matches!(assembly_mode, AssemblyMode::Esi), @@ -5600,7 +5697,7 @@ pub async fn handle_publisher_request( // caching directives, letting a downstream cache serve it to a later eligible reader. let assembled_response_must_be_private = template_cache_key.is_some(); let is_not_modified = response.status() == StatusCode::NOT_MODIFIED; - let range_associated_html_bypass = html_finalizers_present + let range_associated_html_bypass = async_html_finish_present && is_html_content_type(&origin_content_type) && response_is_range_associated(&response); if !range_associated_html_bypass @@ -5665,7 +5762,7 @@ pub async fn handle_publisher_request( } else { classify_response_route(status, &content_type, &content_encoding, request_host) }; - if html_finalizers_present + if async_html_finish_present && route == ResponseRoute::Stream && is_html_content_type(&content_type) && response_carries_body(&request_method, status) @@ -10779,7 +10876,9 @@ mod tests { #[tokio::test] async fn html_finalization_preserves_a_winning_bid_before_the_structural_body_end() { - let settings = Arc::new(settings_with_bidder("inline")); + let settings = Arc::new(crate::publisher::allow_eof_test_settings( + settings_with_bidder("inline"), + )); let stub = Arc::new(StubHttpClient::new()); let cache = Arc::new(MemoryTemplateCache::default()); let services = services(Arc::clone(&stub), cache); @@ -10863,7 +10962,8 @@ mod tests { { for (live, suppressed) in [(true, false), (false, false), (false, true)] { for (status, content_range) in [(206, false), (200, true), (416, true)] { - let mut settings = settings_with_bidder("inline"); + let mut settings = + crate::publisher::allow_eof_test_settings(settings_with_bidder("inline")); settings.auction.enabled = live; let settings = Arc::new(settings); let stub = Arc::new(StubHttpClient::new()); @@ -11051,7 +11151,8 @@ mod tests { async fn html_finalization_runs_again_on_a_raw_origin_cache_hit() { let client = Arc::new(RawOriginCacheClient::default()); let template_cache = Arc::new(MemoryTemplateCache::default()); - let mut settings = settings_with_readthrough_enabled("esi"); + let mut settings = + crate::publisher::allow_eof_test_settings(settings_with_readthrough_enabled("esi")); settings.auction.enabled = false; let settings = Arc::new(settings); let services = RuntimeServices::builder() @@ -11152,7 +11253,9 @@ mod tests { async fn html_finalizers_bypass_warm_and_cold_templates_but_keep_origin_readthrough() { let stub = Arc::new(StubHttpClient::new()); let cache = Arc::new(MemoryTemplateCache::default()); - let settings = Arc::new(settings_with_readthrough_enabled("esi")); + let settings = Arc::new(crate::publisher::allow_eof_test_settings( + settings_with_readthrough_enabled("esi"), + )); let services = services(Arc::clone(&stub), Arc::clone(&cache)); queue_shareable_html(&stub); let _ = body_of(run(&settings, &services, navigation_request()).await).await; @@ -21383,9 +21486,177 @@ mod tests { } } + struct AsyncTailFactory(Arc); + struct AsyncTailProcessor(Arc); + + impl crate::integrations::IntegrationHtmlStreamProcessorFactory for AsyncTailFactory { + fn integration_id(&self) -> &'static str { + "example_async_tail" + } + fn requires_async_finish(&self) -> bool { + true + } + fn create( + &self, + _: crate::integrations::IntegrationHtmlStreamContext, + ) -> Box { + Box::new(AsyncTailProcessor(Arc::clone(&self.0))) + } + } + + impl StreamProcessor for AsyncTailProcessor { + fn process_chunk(&mut self, chunk: &[u8], _: bool) -> std::io::Result> { + Ok(chunk.to_vec()) + } + fn requires_async_finish(&self) -> bool { + true + } + fn async_finish<'a>( + &'a mut self, + _: crate::streaming_processor::finish::StreamFinishInput<'a>, + ) -> Option< + futures::future::LocalBoxFuture< + 'a, + Result< + crate::streaming_processor::finish::StreamFinish, + Report, + >, + >, + > { + Some(Box::pin(async move { + self.0.fetch_add(1, Ordering::SeqCst); + futures::future::pending().await + })) + } + } + + struct UndeclaredAsyncFactory(Arc); + impl crate::integrations::IntegrationHtmlStreamProcessorFactory for UndeclaredAsyncFactory { + fn integration_id(&self) -> &'static str { + "example_inconsistent_factory" + } + fn create( + &self, + _: crate::integrations::IntegrationHtmlStreamContext, + ) -> Box { + Box::new(AsyncTailProcessor(Arc::clone(&self.0))) + } + } + #[test] - fn html_finalization_applies_privacy_and_full_body_policy_without_an_auction() { + fn async_eof_factory_mismatch_rejects_before_stream_reads() { let settings = create_test_settings(); + let reads = Arc::new(AtomicUsize::new(0)); + let calls = Arc::new(AtomicUsize::new(0)); + let registry = IntegrationRegistry::from_html_stream_processors(vec![Arc::new( + UndeclaredAsyncFactory(Arc::clone(&calls)), + )]); + let observed_reads = Arc::clone(&reads); + let body = EdgeBody::stream(futures::stream::poll_fn(move |_| { + observed_reads.fetch_add(1, Ordering::SeqCst); + std::task::Poll::Ready(Some(bytes::Bytes::from_static(b"

Example

"))) + })); + let mut output = Vec::new(); + let mut params = html_stream_params("", None); + let _ = futures::executor::block_on(stream_publisher_body_async( + body, + &mut output, + &mut params, + &settings, + ®istry, + &AuctionOrchestrator::new(settings.auction.clone()), + &noop_services(), + )) + .expect_err("should reject inconsistent EOF declarations before input effects"); + assert_eq!( + reads.load(Ordering::SeqCst), + 0, + "should not consume a stream before capability validation" + ); + assert!( + output.is_empty(), + "should not commit output for inconsistent declarations" + ); + assert_eq!( + calls.load(Ordering::SeqCst), + 0, + "should not start undeclared EOF I/O" + ); + } + + #[test] + fn async_eof_lazy_delivery_releases_page_before_pending_io() { + for streamed in [false, true] { + let settings = Arc::new(crate::publisher::allow_eof_test_settings( + create_test_settings(), + )); + let calls = Arc::new(AtomicUsize::new(0)); + let registry = IntegrationRegistry::from_html_stream_processors(vec![Arc::new( + AsyncTailFactory(Arc::clone(&calls)), + )]); + let html = b"

Example renderable page

"; + let body = if streamed { + EdgeBody::stream(futures::stream::iter(vec![bytes::Bytes::copy_from_slice( + html, + )])) + } else { + EdgeBody::from(html.to_vec()) + }; + let publisher = PublisherResponse::Stream { + response: Response::builder() + .status(200) + .header(header::CONTENT_TYPE, "text/html") + .body(EdgeBody::empty()) + .expect("should build example response"), + body, + params: Box::new(html_stream_params("", None)), + }; + let response = futures::executor::block_on(publisher_response_into_streaming_response( + publisher, + &Method::GET, + Arc::clone(&settings), + ®istry, + Arc::new(AuctionOrchestrator::new(settings.auction.clone())), + noop_services(), + )) + .expect("should prepare lazy EOF output"); + let EdgeBody::Stream(mut stream) = response.into_body() else { + panic!("should return a lazy body"); + }; + let first = futures::executor::block_on(stream.next()) + .expect("should yield ready HTML") + .expect("should deliver prefix"); + assert!( + String::from_utf8_lossy(&first).contains("Example renderable page"), + "should release the page before optional EOF I/O" + ); + assert_eq!( + calls.load(Ordering::SeqCst), + 0, + "should not start the callback before ready bytes are yielded" + ); + let waker = futures::task::noop_waker(); + let mut context = std::task::Context::from_waker(&waker); + let mut pending = false; + for _ in 0..16 { + let mut next = Box::pin(stream.next()); + match core::future::Future::poll(next.as_mut(), &mut context) { + std::task::Poll::Pending => { + pending = true; + break; + } + std::task::Poll::Ready(Some(Ok(_))) => {} + _ => panic!("should reach pending optional EOF work after delivering the page"), + } + } + assert!(pending, "should wait only after ready output is delivered"); + assert_eq!(calls.load(Ordering::SeqCst), 1, "should start EOF I/O once"); + } + } + + #[test] + fn html_finalization_applies_privacy_and_full_body_policy_without_an_auction() { + let settings = crate::publisher::allow_eof_test_settings(create_test_settings()); let registry = IntegrationRegistry::from_html_finalizers(vec![Arc::new(TerminalTestFactory)]); let http = Arc::new(StubHttpClient::new()); @@ -21503,7 +21774,9 @@ mod tests { for buffered in [false, true] { for encoding in ["", "gzip"] { for auction in [false, true] { - let settings = Arc::new(create_test_settings()); + let settings = Arc::new(crate::publisher::allow_eof_test_settings( + create_test_settings(), + )); let calls = Arc::new(AtomicUsize::new(0)); let registry = IntegrationRegistry::from_html_finalizers(vec![Arc::new( @@ -21638,7 +21911,9 @@ mod tests { fn html_finalization_bypasses_ranged_fetches_and_unsolicited_partial_origins() { for destination in ["empty", "document"] { for (status, content_range) in [(206, false), (206, true), (200, true), (416, true)] { - let settings = Arc::new(create_test_settings()); + let settings = Arc::new(crate::publisher::allow_eof_test_settings( + create_test_settings(), + )); let calls = Arc::new(AtomicUsize::new(0)); let registry = IntegrationRegistry::from_html_finalizers(vec![Arc::new( CountingTerminalFactory(Arc::clone(&calls)), @@ -21709,7 +21984,7 @@ mod tests { #[test] fn html_finalization_direct_helpers_copy_encoded_ranges_without_optional_work() { for streamed in [false, true] { - let settings = create_test_settings(); + let settings = crate::publisher::allow_eof_test_settings(create_test_settings()); let calls = Arc::new(AtomicUsize::new(0)); let registry = IntegrationRegistry::from_html_finalizers(vec![Arc::new( CountingTerminalFactory(Arc::clone(&calls)), @@ -21780,7 +22055,7 @@ mod tests { Ok(()) } } - let settings = create_test_settings(); + let settings = crate::publisher::allow_eof_test_settings(create_test_settings()); let calls = Arc::new(AtomicUsize::new(0)); let registry = IntegrationRegistry::from_html_finalizers(vec![Arc::new( CountingTerminalFactory(Arc::clone(&calls)), @@ -21846,7 +22121,7 @@ mod tests { } } for closed in [false, true] { - let mut settings = create_test_settings(); + let mut settings = crate::publisher::allow_eof_test_settings(create_test_settings()); settings.auction.enabled = true; settings.auction.providers = crate::auction::AuctionConfig::legacy_provider_map(&["seam-test"]); @@ -21991,7 +22266,9 @@ mod tests { StatusTerminalFactory(Arc::clone(&statuses)), )]); for status in [200, 404, 500] { - let settings = Arc::new(create_test_settings()); + let settings = Arc::new(crate::publisher::allow_eof_test_settings( + create_test_settings(), + )); let http = Arc::new(StubHttpClient::new()); http.push_response_with_headers( status, @@ -22074,7 +22351,9 @@ mod tests { (Method::GET, 200, "text/plain", ""), (Method::GET, 200, "text/html", "example-unsupported"), ] { - let settings = Arc::new(create_test_settings()); + let settings = Arc::new(crate::publisher::allow_eof_test_settings( + create_test_settings(), + )); let calls = Arc::new(AtomicUsize::new(0)); let registry = IntegrationRegistry::from_html_finalizers(vec![Arc::new( CountingTerminalFactory(Arc::clone(&calls)), @@ -22138,7 +22417,7 @@ mod tests { #[test] fn html_finalization_collection_overflow_replays_rewritten_expansion_and_stays_private() { - let mut settings = create_test_settings(); + let mut settings = crate::publisher::allow_eof_test_settings(create_test_settings()); settings.publisher.max_buffered_body_bytes = 128; let settings = Arc::new(settings); let calls = Arc::new(AtomicUsize::new(0)); @@ -22181,12 +22460,13 @@ mod tests { let body = futures::executor::block_on(response.into_body().into_bytes_bounded(1024 * 1024)) .expect("should replay overflow output"); + let control_registry = IntegrationRegistry::from_html_stream_processors(Vec::new()); let mut processor = create_html_stream_processor(HtmlStreamProcessorParams { origin_host: "origin.example.com", request_host: "proxy.example.com", request_scheme: "https", settings: &settings, - integration_registry: ®istry, + integration_registry: &control_registry, ad_slots_script: html_stream_params("", None).ad_slots_script, ad_bids_state: Arc::new(Mutex::new(None)), suppress_datadome_client_side_tag: false, @@ -22232,7 +22512,9 @@ mod tests { }; encoded.truncate(encoded.len() / 2); for streamed in [false, true] { - let settings = Arc::new(create_test_settings()); + let settings = Arc::new(crate::publisher::allow_eof_test_settings( + create_test_settings(), + )); let calls = Arc::new(AtomicUsize::new(0)); let registry = IntegrationRegistry::from_html_finalizers(vec![Arc::new( CountingTerminalFactory(Arc::clone(&calls)), @@ -22356,7 +22638,9 @@ mod tests { let _ = log::set_logger(&LOGGER); log::set_max_level(log::LevelFilter::Warn); CANCELLATION_WARNINGS.with(|count| count.set(Some(0))); - let settings = Arc::new(create_test_settings()); + let settings = Arc::new(crate::publisher::allow_eof_test_settings( + create_test_settings(), + )); let started = Arc::new(AtomicUsize::new(0)); let dropped = Arc::new(AtomicUsize::new(0)); let registry = @@ -22450,7 +22734,7 @@ mod tests { } } for failure in ["read", "decode", "write", "flush"] { - let settings = create_test_settings(); + let settings = crate::publisher::allow_eof_test_settings(create_test_settings()); let calls = Arc::new(AtomicUsize::new(0)); let registry = IntegrationRegistry::from_html_finalizers(vec![Arc::new( CountingTerminalFactory(Arc::clone(&calls)), @@ -22526,6 +22810,81 @@ mod tests { } } + #[test] + fn async_eof_fatal_auction_suffix_overflow_abandons_once() { + let mut settings = crate::publisher::allow_eof_test_settings(create_test_settings()); + settings.publisher.max_buffered_body_bytes = 256; + let calls = Arc::new(AtomicUsize::new(0)); + let registry = IntegrationRegistry::from_html_stream_processors(vec![Arc::new( + AsyncTailFactory(Arc::clone(&calls)), + )]); + let sink = Arc::new(RecordingTelemetrySink::default()); + let services = noop_services_with_telemetry_sink(Arc::clone(&sink) as _); + let orchestrator = AuctionOrchestrator::new(settings.auction.clone()); + let mut params = html_stream_params( + "", + Some(DispatchedAuction::empty_for_test( + test_auction_request(), + 500, + )), + ); + params.auction_observation = Some(AuctionObservationContext::from_parts( + AuctionSource::InitialNavigation, + "proxy.example.com", + "/article", + 1, + None, + &EcContext::default(), + )); + let source = format!( + "Example{}", + "x".repeat(settings.publisher.max_buffered_body_bytes + 1) + ); + let mut output = Vec::new(); + let error = futures::executor::block_on(stream_publisher_body_async( + EdgeBody::from(source.into_bytes()), + &mut output, + &mut params, + &settings, + ®istry, + &orchestrator, + &services, + )) + .expect_err("should preserve fatal suffix overflow"); + assert!( + format!("{error:?}").contains("HTML auction suffix exceeds its retained-byte limit"), + "should fail at the auction holder rather than optional replacement" + ); + assert_eq!( + calls.load(Ordering::SeqCst), + 0, + "should not start EOF I/O after fatal suffix overflow" + ); + let batches = sink + .batches + .lock() + .expect("should inspect abandonment telemetry"); + let summaries: Vec<_> = batches + .iter() + .flat_map(crate::auction::telemetry::AuctionEventBatch::rows) + .filter(|row| row.event_kind == "summary") + .collect(); + assert_eq!( + summaries.len(), + 1, + "should abandon the dispatched auction exactly once" + ); + assert_eq!( + summaries[0].terminal_status.as_deref(), + Some("abandoned"), + "should record fatal body abandonment" + ); + assert!( + params.dispatched_auction.is_none(), + "should transfer auction ownership to the failing driver" + ); + } + struct TerminalTestFactory; struct TerminalTestSession; @@ -22595,7 +22954,9 @@ mod tests { for auction in [false, true] { for lazy in [false, true] { for streamed in [false, true] { - let settings = Arc::new(create_test_settings()); + let settings = Arc::new(crate::publisher::allow_eof_test_settings( + create_test_settings(), + )); let registry = IntegrationRegistry::from_html_finalizers(vec![Arc::new( TerminalTestFactory, @@ -22675,7 +23036,11 @@ mod tests { } fn streaming_finalize_response(params: OwnedProcessResponseParams, body: EdgeBody) -> EdgeBody { - streaming_finalize_response_with_settings(params, body, create_test_settings()) + streaming_finalize_response_with_settings( + params, + body, + crate::publisher::allow_eof_test_settings(create_test_settings()), + ) } fn streaming_finalize_response_with_settings( @@ -27056,3 +27421,72 @@ mod tests { } } } + +#[cfg(test)] +mod async_eof_seam_tests { + use super::*; + + #[test] + fn async_eof_seam_caps_suffix_before_growth() { + let mut seam = InlineBodyCloseSeam::new(b"TOKEN".to_vec()); + assert_eq!( + seam.push_bounded(b"prefixTOKEN1234", 4) + .expect("should release the prefix and hold exact cap"), + b"prefix", + "should release ordinary HTML" + ); + assert_eq!( + seam.buffered, b"1234", + "should retain only the closing suffix" + ); + seam.push_bounded(b"!", 4) + .expect_err("should reject held suffix cap crossing"); + assert_eq!( + seam.buffered, b"1234", + "should reject before growing retained bytes" + ); + } + + #[test] + fn async_eof_seam_splits_large_ready_prefix_before_retention() { + let mut seam = InlineBodyCloseSeam::new(b"TOKEN".to_vec()); + let mut input = vec![b'x'; 100_000]; + input.extend_from_slice(b"TOKENtail"); + let ready = seam + .push_bounded(&input, 4) + .expect("should not count ready prefix against held cap"); + assert_eq!(ready.len(), 100_000, "should release the full ready prefix"); + assert_eq!( + seam.buffered, b"tail", + "should retain only bounded tail bytes" + ); + assert!( + seam.buffered.capacity() <= 4, + "should not allocate the whole ready prefix in the holder" + ); + } + + #[test] + fn async_eof_seam_handles_split_tokens_under_the_cap() { + let mut seam = InlineBodyCloseSeam::new(b"TOKEN".to_vec()); + assert_eq!( + seam.push_bounded(b"pageTOK", 4) + .expect("should retain a possible split marker"), + b"page", + "should stream the prefix" + ); + assert!( + seam.push_bounded(b"ENtail", 4) + .expect("should finish split marker") + .is_empty(), + "should hold the closing suffix" + ); + assert_eq!(seam.finish(), b"tail", "should remove the marker once"); + } +} + +#[cfg(test)] +fn allow_eof_test_settings(mut settings: Settings) -> Settings { + settings.publisher.allow_async_html_finish = true; + settings +} diff --git a/crates/trusted-server-core/src/publisher/html_finalization.rs b/crates/trusted-server-core/src/publisher/html_finalization.rs index dca4bcacb..1a6bd01c2 100644 --- a/crates/trusted-server-core/src/publisher/html_finalization.rs +++ b/crates/trusted-server-core/src/publisher/html_finalization.rs @@ -1,4 +1,4 @@ -//! Optional terminal HTML work; current bytes stay owned outside callback futures. +//! Ordered optional HTML EOF work with driver-owned fallback and streaming output. use std::collections::VecDeque; use std::io::{self, Read}; @@ -10,174 +10,27 @@ use bytes::Bytes; use edgezero_core::body::Body as EdgeBody; use error_stack::{Report, ResultExt as _}; use flate2::{Decompress, FlushDecompress, Status}; -use futures::FutureExt as _; +use futures::future::{Fuse, LocalBoxFuture}; use http::StatusCode; -use web_time::Instant; use super::{ AuctionCollectDeps, AuctionHoldState, AuctionTelemetryCarry, BodyChunkSource, DispatchedAuctionGuard, HoldStepSegments, HtmlStreamProcessorParams, InlineBodyCloseSeam, OwnedProcessResponseParams, abandon_hold_auction, body_as_reader, create_html_stream_processor_with_document_state, deferred_inline_seam_token, - encode_processed_html, hold_collect_close_tail, hold_finish_tail_segments, + hold_collect_close_tail, hold_finish_tail_segments, }; use crate::error::TrustedServerError; -use crate::integrations::{ - HtmlFinalization, HtmlFinalizationDeadline, HtmlFinalizationError, HtmlFinalizationInput, - HtmlReplacementWriter, IntegrationDocumentState, IntegrationHtmlFinalizer, - IntegrationHtmlStreamContext, IntegrationRegistry, -}; -use crate::platform::RuntimeServices; +use crate::html_processor::HtmlWithStreamingProcessors; +use crate::html_processor::finish::HtmlFinishContext; +use crate::integrations::{IntegrationDocumentState, IntegrationRegistry}; use crate::settings::Settings; +use crate::streaming_processor::finish::StreamFinishDeadline; use crate::streaming_processor::{ BodyStreamDecoder, BodyStreamEncoder, Compression, GzipDecodeReader, STREAM_CHUNK_SIZE, StreamProcessor, }; -struct HtmlFinalizerSession { - integration_id: &'static str, - finalizer: Box, -} - -fn log_timeout(integration_id: &str, started_at: Instant, bytes: usize) { - log::warn!( - "HTML finalizer {integration_id} times out after {} ms; retains {bytes} bytes", - started_at.elapsed().as_millis() - ); -} - -async fn finalize_document( - response_status: StatusCode, - mut current: Vec, - sessions: &mut [HtmlFinalizerSession], - marker: Option<&[u8]>, - services: &RuntimeServices, - limit: usize, - timeout: Duration, -) -> Vec { - let deadline = HtmlFinalizationDeadline::new(timeout, services.timer().is_some()); - let started_at = Instant::now(); - let mut timer = services.timer().map(|timer| timer.delay(timeout).fuse()); - for session in sessions { - if deadline.is_expired_at(Instant::now()) { - break; - } - let mut replacement = HtmlReplacementWriter::new(limit); - let result = { - let work = session - .finalizer - .finalize(HtmlFinalizationInput { - response_status, - html: ¤t, - replacement: &mut replacement, - services, - deadline: &deadline, - }) - .fuse(); - futures::pin_mut!(work); - if let Some(mut timer) = timer.as_mut() { - futures::select_biased! { - _ = timer => None, - result = work => Some(result), - } - } else { - Some(work.await) - } - }; - // The callback future has dropped before either borrowed buffer is used. - let expired = result.is_none() || deadline.is_expired_at(Instant::now()); - if expired { - log_timeout(session.integration_id, started_at, current.len()); - break; - } - let result = result.expect("should have a result before the deadline"); - let outcome = match result { - Ok(HtmlFinalization::Unchanged) => "unchanged", - Ok(HtmlFinalization::Replace) if !replacement.is_poisoned() => { - let candidate = replacement.into_bytes(); - if marker.is_none_or(|token| { - marker_count(&candidate, token) == marker_count(¤t, token) - }) { - if deadline.is_expired_at(Instant::now()) { - log_timeout(session.integration_id, started_at, current.len()); - break; - } - current = candidate; - "replaced" - } else { - "invalid_marker" - } - } - Ok(HtmlFinalization::Replace) => "output_limit", - Err(error) => match error.current_context() { - HtmlFinalizationError::Unavailable => "unavailable", - HtmlFinalizationError::Rejected => "rejected", - HtmlFinalizationError::OutputLimitExceeded { .. } => "output_limit", - }, - }; - if matches!(outcome, "unchanged" | "replaced") { - log::debug!( - "HTML finalizer {} completes: {outcome}, {} bytes, {} ms", - session.integration_id, - current.len(), - started_at.elapsed().as_millis() - ); - } else { - log::warn!( - "HTML finalizer {} retains input: {outcome}, {} bytes, {} ms", - session.integration_id, - current.len(), - started_at.elapsed().as_millis() - ); - } - } - current -} - -fn marker_count(bytes: &[u8], token: &[u8]) -> usize { - if token.is_empty() { - return 0; - } - bytes - .windows(token.len()) - .filter(|window| *window == token) - .count() -} - -struct HtmlCollector { - retained: Option, -} - -impl HtmlCollector { - fn new(limit: usize) -> Self { - Self { - retained: Some(HtmlReplacementWriter::new(limit)), - } - } - - fn push(&mut self, output: Vec) -> Vec { - let Some(retained) = self.retained.as_mut() else { - return vec![bytes::Bytes::from(output)]; - }; - if retained.append(&output).is_ok() { - return Vec::new(); - } - let retained = self - .retained - .take() - .expect("should own the collected prefix"); - log::warn!("HTML finalization bypasses document: collection exceeds its byte limit"); - vec![ - bytes::Bytes::from(retained.into_bytes()), - bytes::Bytes::from(output), - ] - } - - fn finish(&mut self) -> Option> { - self.retained.take().map(HtmlReplacementWriter::into_bytes) - } -} - // The reader codec must prove StreamEnd before terminal work starts. The // ordinary flate2 Read wrapper also returns EOF for an incomplete zlib stream. struct ValidatedZlibReader { @@ -308,31 +161,36 @@ impl TerminalInput { } } -pub(super) struct HtmlFinalizationDriver { +pub(super) struct HtmlEofDriver { response_status: StatusCode, input: TerminalInput, encoder: BodyStreamEncoder, - processor: Box, - collector: HtmlCollector, + processor: HtmlWithStreamingProcessors, replay: VecDeque, - sessions: Vec, marker: Option>, hold: Option, limit: usize, timeout: Duration, source_finished: bool, + stages_finished: bool, + finish_deadline: Option, + finish_timer: Option>>, + had_ready_bytes: bool, + callback_barrier_issued: bool, + flush_barrier: bool, finished: bool, seam_flushed: bool, failure_reason: &'static str, } -impl HtmlFinalizationDriver { +impl HtmlEofDriver { pub(super) fn new( body: EdgeBody, params: &mut OwnedProcessResponseParams, settings: &Settings, registry: &IntegrationRegistry, ) -> Result> { + registry.ensure_async_html_finish_allowed(settings)?; let document_state = IntegrationDocumentState::default(); let marker = deferred_inline_seam_token( settings, @@ -340,7 +198,7 @@ impl HtmlFinalizationDriver { params.ad_slots_script.is_some(), params.dispatched_auction.is_some(), ); - let processor = create_html_stream_processor_with_document_state( + let mut processor = create_html_stream_processor_with_document_state( HtmlStreamProcessorParams { origin_host: ¶ms.origin_host, request_host: ¶ms.request_host, @@ -359,20 +217,12 @@ impl HtmlFinalizationDriver { }, &document_state, )?; - let context = IntegrationHtmlStreamContext { - request_host: params.request_host.clone(), - request_scheme: params.request_scheme.clone(), - origin_host: params.origin_host.clone(), - document_state, - }; - let sessions = registry - .html_finalizer_factories() - .iter() - .map(|factory| HtmlFinalizerSession { - integration_id: factory.integration_id(), - finalizer: factory.create(context.clone()), - }) - .collect(); + processor + .validate_capabilities() + .change_context(TrustedServerError::Configuration { + message: "HTML processor EOF capabilities are inconsistent".to_string(), + })?; + processor.set_finish_marker(marker.as_deref()); let compression = Compression::from_content_encoding(¶ms.content_encoding); let limit = settings.publisher.max_buffered_body_bytes; let input = TerminalInput::new(body, compression, limit)?; @@ -390,10 +240,8 @@ impl HtmlFinalizationDriver { response_status: params.response_status, input, encoder: BodyStreamEncoder::new(compression), - processor: Box::new(processor), - collector: HtmlCollector::new(limit), + processor, replay: VecDeque::new(), - sessions, marker, hold, limit, @@ -401,6 +249,12 @@ impl HtmlFinalizationDriver { settings.publisher.html_finalization_timeout_ms, )), source_finished: false, + stages_finished: false, + finish_deadline: None, + finish_timer: None, + had_ready_bytes: false, + callback_barrier_issued: false, + flush_barrier: false, finished: false, seam_flushed: false, failure_reason: "stream_process_error", @@ -411,6 +265,7 @@ impl HtmlFinalizationDriver { &mut self, deps: &AuctionCollectDeps<'_>, ) -> Result, Report> { + self.flush_barrier = false; let result = self.next_inner(deps).await; if result.is_err() { self.abandon(deps, self.failure_reason).await; @@ -433,19 +288,83 @@ impl HtmlFinalizationDriver { self.replay.push_front(bytes.slice(end..)); } self.failure_reason = "stream_encode_error"; - return encode_processed_html( - &mut self.encoder, - chunk.to_vec(), - self.hold.as_mut(), - ) - .map(Some); + let processed = + if let Some(seam) = self.hold.as_mut().and_then(|hold| hold.hold.as_mut()) { + seam.push_bounded(&chunk, self.limit).change_context( + TrustedServerError::Proxy { + message: "HTML auction suffix exceeds its retained-byte limit" + .to_string(), + }, + )? + } else { + chunk.to_vec() + }; + let encoded = self.encoder.encode_chunk(processed)?; + self.had_ready_bytes |= !encoded.is_empty(); + return Ok(Some(HoldStepSegments { + ready: if encoded.is_empty() { + Vec::new() + } else { + vec![Bytes::from(encoded)] + }, + close_found: false, + })); } if self.finished { return Ok(None); } - if self.source_finished && !self.seam_flushed { + if self.source_finished && !self.stages_finished { + if self.had_ready_bytes + && !self.callback_barrier_issued + && self.processor.next_finish_requires_async() + { + self.callback_barrier_issued = true; + self.flush_barrier = true; + return Ok(Some(HoldStepSegments { + ready: Vec::new(), + close_found: false, + })); + } + self.failure_reason = "stream_process_error"; + let output = self + .processor + .finish_next(HtmlFinishContext { + response_status: self.response_status, + services: deps.services, + deadline: self + .finish_deadline + .as_ref() + .expect("should start EOF deadline after parser completion"), + timer: &mut self.finish_timer, + limit: self.limit, + marker: self.marker.as_deref(), + }) + .await + .change_context(TrustedServerError::Proxy { + message: "Failed to finish HTML processor".to_string(), + })?; + self.callback_barrier_issued = false; + if let Some(output) = output { + self.replay.push_back(Bytes::from(output)); + } else { + self.stages_finished = true; + } + continue; + } + if self.stages_finished && !self.seam_flushed { self.seam_flushed = true; self.failure_reason = "stream_encode_error"; + if self + .hold + .as_ref() + .and_then(|hold| hold.hold.as_ref()) + .is_some_and(InlineBodyCloseSeam::found) + { + return Ok(Some(HoldStepSegments { + ready: Vec::new(), + close_found: true, + })); + } let suffix = self.hold.as_mut().and_then(|hold| { if hold.hold.as_ref().is_some_and(|seam| !seam.found()) { hold.hold.take().map(InlineBodyCloseSeam::finish) @@ -463,7 +382,7 @@ impl HtmlFinalizationDriver { close_found: false, })); } - if self.source_finished { + if self.stages_finished { self.finished = true; self.failure_reason = "stream_encode_error"; let ready = if let Some(hold) = self.hold.as_mut() { @@ -486,21 +405,20 @@ impl HtmlFinalizationDriver { self.accept(&decoded, false)?; } if ended { - self.accept(&[], true)?; + self.failure_reason = "stream_process_error"; + let output = + self.processor + .begin_finish() + .change_context(TrustedServerError::Proxy { + message: "Failed to finish HTML parser".to_string(), + })?; self.source_finished = true; - if let Some(current) = self.collector.finish() { - let current = finalize_document( - self.response_status, - current, - &mut self.sessions, - self.marker.as_deref(), - deps.services, - self.limit, - self.timeout, - ) - .await; - self.replay.push_back(Bytes::from(current)); - } + self.finish_deadline = Some(StreamFinishDeadline::new( + self.timeout, + deps.services.timer().is_some(), + )); + + self.replay.push_back(Bytes::from(output)); } } } @@ -513,12 +431,12 @@ impl HtmlFinalizationDriver { .change_context(TrustedServerError::Proxy { message: "Failed to process HTML terminal input".to_string(), })?; - self.replay.extend(self.collector.push(output)); + self.replay.push_back(Bytes::from(output)); Ok(()) } pub(super) fn at_finish_barrier(&self) -> bool { - self.source_finished && self.seam_flushed && !self.finished + self.flush_barrier || (self.stages_finished && self.seam_flushed && !self.finished) } pub(super) async fn collect_close( @@ -541,449 +459,8 @@ impl HtmlFinalizationDriver { #[cfg(test)] mod tests { - use std::cell::{Cell, RefCell}; - use std::rc::Rc; - use std::time::Duration; - - use error_stack::Report; - use futures::executor::block_on; - use futures::future::{LocalBoxFuture, poll_fn}; - use std::task::Poll; - - use crate::integrations::{ - HtmlFinalization, HtmlFinalizationError, HtmlFinalizationInput, IntegrationHtmlFinalizer, - }; - use crate::platform::test_support::{ - NoopBackend, NoopConfigStore, NoopGeo, NoopHttpClient, NoopSecretStore, noop_services, - }; - use crate::platform::{PlatformTimer, RuntimeServices}; - use super::*; - #[test] - fn collection_overflow_replays_prefix_and_crossing_chunk_once() { - let mut collector = HtmlCollector::new(4); - assert!( - collector.push(b"page".to_vec()).is_empty(), - "should retain input at the cap" - ); - let mut emitted = collector.push(b"!".to_vec()).concat(); - emitted.extend(collector.push(b"tail".to_vec()).concat()); - assert_eq!( - emitted, b"page!tail", - "should release overflow without duplication" - ); - assert!( - collector.finish().is_none(), - "should skip finalizer I/O after bypass" - ); - } - - struct AppendFinalizer { - suffix: &'static [u8], - inputs: Rc>>>, - fail: bool, - } - - impl IntegrationHtmlFinalizer for AppendFinalizer { - fn finalize<'a>( - &'a mut self, - input: HtmlFinalizationInput<'a>, - ) -> LocalBoxFuture<'a, Result>> { - Box::pin(async move { - let mut yielded = false; - poll_fn(|context| { - if yielded { - Poll::Ready(()) - } else { - yielded = true; - context.waker().wake_by_ref(); - Poll::Pending - } - }) - .await; - self.inputs.borrow_mut().push(input.html.to_vec()); - input.replacement.append(input.html)?; - input.replacement.append(self.suffix)?; - if self.fail { - Err(Report::new(HtmlFinalizationError::Rejected)) - } else { - Ok(HtmlFinalization::Replace) - } - }) - } - } - - #[test] - fn yielding_finalizers_commit_in_order_and_discard_failed_candidates() { - let inputs = Rc::new(RefCell::new(Vec::new())); - let mut sessions = [ - HtmlFinalizerSession { - integration_id: "example_a", - finalizer: Box::new(AppendFinalizer { - suffix: b"a", - inputs: Rc::clone(&inputs), - fail: false, - }), - }, - HtmlFinalizerSession { - integration_id: "example_b", - finalizer: Box::new(AppendFinalizer { - suffix: b"bad", - inputs: Rc::clone(&inputs), - fail: true, - }), - }, - HtmlFinalizerSession { - integration_id: "example_c", - finalizer: Box::new(AppendFinalizer { - suffix: b"c", - inputs: Rc::clone(&inputs), - fail: false, - }), - }, - ]; - let services = noop_services(); - let output = block_on(finalize_document( - StatusCode::OK, - b"page".to_vec(), - &mut sessions, - None, - &services, - 32, - Duration::from_secs(1), - )); - assert_eq!(output, b"pageac", "should retain only accepted candidates"); - assert_eq!( - *inputs.borrow(), - [b"page".to_vec(), b"pagea".to_vec(), b"pagea".to_vec()], - "should propagate committed input in order" - ); - } - - #[test] - fn poisoned_replacement_preserves_the_original_allocation() { - let inputs = Rc::new(RefCell::new(Vec::new())); - let mut sessions = [HtmlFinalizerSession { - integration_id: "example", - finalizer: Box::new(AppendFinalizer { - suffix: b"oversize", - inputs, - fail: false, - }), - }]; - let input = b"page".to_vec(); - let allocation = input.as_ptr(); - let output = block_on(finalize_document( - StatusCode::OK, - input, - &mut sessions, - None, - &noop_services(), - 4, - Duration::from_secs(1), - )); - assert_eq!(output, b"page", "should discard oversized output"); - assert_eq!( - output.as_ptr(), - allocation, - "should move fallback without copying it" - ); - } - struct ControlledTimer { - pending_polls: usize, - } - - impl PlatformTimer for ControlledTimer { - fn delay(&self, _: Duration) -> LocalBoxFuture<'static, ()> { - let mut remaining = self.pending_polls; - Box::pin(poll_fn(move |context| { - if remaining == 0 { - Poll::Ready(()) - } else { - remaining -= 1; - context.waker().wake_by_ref(); - Poll::Pending - } - })) - } - } - - struct DropProbe(Rc>); - - impl Drop for DropProbe { - fn drop(&mut self) { - self.0.set(true); - } - } - - struct HungFinalizer { - dropped: Rc>, - started: Rc>, - } - - impl IntegrationHtmlFinalizer for HungFinalizer { - fn finalize<'a>( - &'a mut self, - input: HtmlFinalizationInput<'a>, - ) -> LocalBoxFuture<'a, Result>> { - let probe = DropProbe(Rc::clone(&self.dropped)); - Box::pin(async move { - let _probe = probe; - self.started.set(true); - input.replacement.append(b"partial")?; - futures::future::pending().await - }) - } - } - - fn timer_services(pending_polls: usize) -> RuntimeServices { - RuntimeServices::builder() - .config_store(Arc::new(NoopConfigStore)) - .secret_store(Arc::new(NoopSecretStore)) - .kv_store(Arc::new(edgezero_core::key_value_store::NoopKvStore)) - .backend(Arc::new(NoopBackend)) - .http_client(Arc::new(NoopHttpClient)) - .geo(Arc::new(NoopGeo)) - .client_info(crate::platform::ClientInfo::default()) - .timer(Arc::new(ControlledTimer { pending_polls })) - .build() - } - - #[test] - fn timeout_drops_borrowing_future_preserves_allocation_and_skips_later_work() { - let dropped = Rc::new(Cell::new(false)); - let started = Rc::new(Cell::new(false)); - let later_inputs = Rc::new(RefCell::new(Vec::new())); - let mut sessions = [ - HtmlFinalizerSession { - integration_id: "example_hung", - finalizer: Box::new(HungFinalizer { - dropped: Rc::clone(&dropped), - started: Rc::clone(&started), - }), - }, - HtmlFinalizerSession { - integration_id: "example_later", - finalizer: Box::new(AppendFinalizer { - suffix: b"later", - inputs: Rc::clone(&later_inputs), - fail: false, - }), - }, - ]; - let input = b"page".to_vec(); - let allocation = input.as_ptr(); - let output = block_on(finalize_document( - StatusCode::OK, - input, - &mut sessions, - None, - &timer_services(1), - 32, - Duration::from_secs(1), - )); - assert!( - started.get(), - "should poll outside work before timer expiry" - ); - assert!( - dropped.get(), - "should drop the borrowing future before returning" - ); - assert!( - later_inputs.borrow().is_empty(), - "should skip remaining finalizers" - ); - assert_eq!( - output, b"page", - "should discard a timed-out partial candidate" - ); - assert_eq!( - output.as_ptr(), - allocation, - "should retain the original allocation" - ); - } - - #[test] - fn expired_timer_wins_before_polling_ready_work() { - let inputs = Rc::new(RefCell::new(Vec::new())); - let mut sessions = [HtmlFinalizerSession { - integration_id: "example", - finalizer: Box::new(AppendFinalizer { - suffix: b"replacement", - inputs: Rc::clone(&inputs), - fail: false, - }), - }]; - let output = block_on(finalize_document( - StatusCode::OK, - b"page".to_vec(), - &mut sessions, - None, - &timer_services(0), - 32, - Duration::from_secs(1), - )); - assert_eq!(output, b"page", "should give timer expiration priority"); - assert!( - inputs.borrow().is_empty(), - "should not poll work after expiry" - ); - } - - #[test] - fn marker_multiplicity_rejects_added_markers() { - let mut sessions = [HtmlFinalizerSession { - integration_id: "example", - finalizer: Box::new(AppendFinalizer { - suffix: b"TOKEN", - inputs: Rc::new(RefCell::new(Vec::new())), - fail: false, - }), - }]; - let output = block_on(finalize_document( - StatusCode::OK, - b"pageTOKEN".to_vec(), - &mut sessions, - Some(b"TOKEN"), - &noop_services(), - 32, - Duration::from_secs(1), - )); - assert_eq!( - output, b"pageTOKEN", - "should reject a duplicated assembly marker" - ); - let output = block_on(finalize_document( - StatusCode::OK, - b"page".to_vec(), - &mut sessions, - Some(b"TOKEN"), - &noop_services(), - 32, - Duration::from_secs(1), - )); - assert_eq!( - output, b"page", - "should reject adding a marker to a document without one" - ); - } - struct ImmediateFinalizer { - ignore_overflow: bool, - } - - impl IntegrationHtmlFinalizer for ImmediateFinalizer { - fn finalize<'a>( - &'a mut self, - input: HtmlFinalizationInput<'a>, - ) -> LocalBoxFuture<'a, Result>> { - Box::pin(async move { - input.replacement.append(input.html)?; - if self.ignore_overflow { - let _ = input.replacement.append(b"oversize"); - } else { - input.replacement.append(b"a")?; - } - Ok(HtmlFinalization::Replace) - }) - } - } - - #[test] - fn ignored_output_limit_cannot_commit_a_partial_candidate() { - let mut sessions = [HtmlFinalizerSession { - integration_id: "example", - finalizer: Box::new(ImmediateFinalizer { - ignore_overflow: true, - }), - }]; - let output = block_on(finalize_document( - StatusCode::OK, - b"page".to_vec(), - &mut sessions, - None, - &noop_services(), - 4, - Duration::from_secs(1), - )); - assert_eq!( - output, b"page", - "should reject Replace after an ignored writer error" - ); - } - - #[test] - fn one_timer_covers_the_chain_and_preserves_an_earlier_commit() { - let dropped = Rc::new(Cell::new(false)); - let mut sessions = [ - HtmlFinalizerSession { - integration_id: "example_ready", - finalizer: Box::new(ImmediateFinalizer { - ignore_overflow: false, - }), - }, - HtmlFinalizerSession { - integration_id: "example_hung", - finalizer: Box::new(HungFinalizer { - dropped: Rc::clone(&dropped), - started: Rc::new(Cell::new(false)), - }), - }, - ]; - let output = block_on(finalize_document( - StatusCode::OK, - b"page".to_vec(), - &mut sessions, - None, - &timer_services(1), - 32, - Duration::from_secs(1), - )); - assert_eq!( - output, b"pagea", - "should retain the earlier commit when the shared timer expires" - ); - assert!( - dropped.get(), - "should drop the second callback at the shared deadline" - ); - } - - #[test] - fn cancellation_drops_active_callback_and_its_candidate() { - let dropped = Rc::new(Cell::new(false)); - let mut sessions = [HtmlFinalizerSession { - integration_id: "example", - finalizer: Box::new(HungFinalizer { - dropped: Rc::clone(&dropped), - started: Rc::new(Cell::new(false)), - }), - }]; - let services = noop_services(); - let work = finalize_document( - StatusCode::OK, - b"page".to_vec(), - &mut sessions, - None, - &services, - 32, - Duration::from_secs(1), - ); - let mut work = Box::pin(work); - let waker = futures::task::noop_waker(); - let mut context = std::task::Context::from_waker(&waker); - assert!( - std::future::Future::poll(work.as_mut(), &mut context).is_pending(), - "should yield while outside work is pending" - ); - drop(work); - assert!( - dropped.get(), - "should cancel the active future with the document" - ); - } #[test] fn materialized_deflate_requires_validated_end_before_terminal_eof() { let mut encoder = @@ -1006,104 +483,4 @@ mod tests { .expect_err("should reject a truncated trailer before terminal EOF"); } } - struct FixedCandidate { - bytes: &'static [u8], - decision: HtmlFinalization, - } - impl IntegrationHtmlFinalizer for FixedCandidate { - fn finalize<'a>( - &'a mut self, - input: HtmlFinalizationInput<'a>, - ) -> LocalBoxFuture<'a, Result>> { - Box::pin(async move { - input.replacement.append(self.bytes)?; - Ok(self.decision) - }) - } - } - - #[test] - fn unchanged_discards_candidate_and_missing_markers_reject_replacement() { - for (decision, marker) in [ - (HtmlFinalization::Unchanged, None), - (HtmlFinalization::Replace, Some(&b"TOKEN"[..])), - ] { - let mut sessions = [HtmlFinalizerSession { - integration_id: "example", - finalizer: Box::new(FixedCandidate { - bytes: b"candidate", - decision, - }), - }]; - let output = block_on(finalize_document( - StatusCode::OK, - b"pageTOKEN".to_vec(), - &mut sessions, - marker, - &noop_services(), - 32, - Duration::from_secs(1), - )); - assert_eq!( - output, b"pageTOKEN", - "should retain input when unchanged or a marker is removed" - ); - } - let mut sessions = [HtmlFinalizerSession { - integration_id: "example", - finalizer: Box::new(FixedCandidate { - bytes: b"", - decision: HtmlFinalization::Replace, - }), - }]; - let output = block_on(finalize_document( - StatusCode::OK, - b"page".to_vec(), - &mut sessions, - None, - &noop_services(), - 32, - Duration::from_secs(1), - )); - assert!( - output.is_empty(), - "should allow an intentional empty replacement without a marker" - ); - } - - #[cfg(not(target_arch = "wasm32"))] - #[test] - fn no_timer_rejects_completion_after_the_absolute_deadline() { - struct LateFinalizer; - impl IntegrationHtmlFinalizer for LateFinalizer { - fn finalize<'a>( - &'a mut self, - input: HtmlFinalizationInput<'a>, - ) -> LocalBoxFuture<'a, Result>> - { - Box::pin(async move { - std::thread::sleep(input.deadline.remaining() + Duration::from_millis(2)); - input.replacement.append(b"late")?; - Ok(HtmlFinalization::Replace) - }) - } - } - let mut sessions = [HtmlFinalizerSession { - integration_id: "example", - finalizer: Box::new(LateFinalizer), - }]; - let output = block_on(finalize_document( - StatusCode::OK, - b"page".to_vec(), - &mut sessions, - None, - &noop_services(), - 32, - Duration::from_millis(1), - )); - assert_eq!( - output, b"page", - "should reject late results from a blocking consumer" - ); - } } diff --git a/crates/trusted-server-core/src/settings.rs b/crates/trusted-server-core/src/settings.rs index 315259aaf..ba06c807d 100644 --- a/crates/trusted-server-core/src/settings.rs +++ b/crates/trusted-server-core/src/settings.rs @@ -78,7 +78,13 @@ pub struct Publisher { #[serde(default = "default_max_buffered_body_bytes")] #[validate(range(min = 1, message = "must be at least 1 byte"))] pub max_buffered_body_bytes: usize, - /// Whole-chain HTML terminal budget in milliseconds; inert without a finalizer. + /// Explicit operator permission for asynchronous HTML EOF work. + /// + /// Defaults to false. Enabled work can delay response delivery, especially + /// on buffered adapters and processors retaining output until EOF. + #[serde(default)] + pub allow_async_html_finish: bool, + /// Whole-chain HTML EOF budget in milliseconds; inert without an async processor. #[serde(default = "default_html_finalization_timeout_ms")] #[validate(range( min = 1, @@ -111,6 +117,7 @@ impl Default for Publisher { proxy_secret: Redacted::default(), max_buffered_body_bytes: default_max_buffered_body_bytes(), html_finalization_timeout_ms: default_html_finalization_timeout_ms(), + allow_async_html_finish: false, } } } @@ -203,6 +210,7 @@ impl Publisher { /// proxy_secret: Redacted::new("proxy-secret".to_string()), /// max_buffered_body_bytes: 16 * 1024 * 1024, /// html_finalization_timeout_ms: 5000, + /// allow_async_html_finish: false, /// }; /// assert_eq!(publisher.origin_host(), "origin.example.com:8080"); /// ``` @@ -3065,9 +3073,14 @@ impl Settings { /// - [`TrustedServerError::Configuration`] if the TOML is invalid or missing required fields #[cfg(test)] pub fn from_toml_and_env(toml_str: &str) -> Result> { + // Cargo supplies build metadata to test processes; it is not app config. + let settings_environment = std::env::vars() + .filter(|(name, _)| !name.eq_ignore_ascii_case("TRUSTED_SERVER__GIT_VERSION")) + .collect(); let environment = Environment::default() .prefix(ENVIRONMENT_VARIABLE_PREFIX) - .separator(ENVIRONMENT_VARIABLE_SEPARATOR); + .separator(ENVIRONMENT_VARIABLE_SEPARATOR) + .source(Some(settings_environment)); let toml = File::from_str(toml_str, FileFormat::Toml); let config = Config::builder() @@ -6330,6 +6343,26 @@ source_domain = "partner.example.com" ); } + #[test] + fn legacy_settings_loader_ignores_build_version_metadata() { + temp_env::with_var( + "TRUSTED_SERVER__GIT_VERSION", + Some("example-revision"), + || { + let settings = Settings::from_toml_and_env(&crate_test_settings_str()) + .expect("should keep build metadata outside application settings"); + assert_eq!( + settings.publisher.domain, + Settings::from_toml(&crate_test_settings_str()) + .expect("should parse the control configuration") + .publisher + .domain, + "should preserve configured publisher settings" + ); + }, + ); + } + #[test] fn test_override_env() { let toml_str = crate_test_settings_str(); @@ -6405,6 +6438,7 @@ source_domain = "partner.example.com" proxy_secret: Redacted::new("test-secret".to_string()), max_buffered_body_bytes: 16 * 1024 * 1024, html_finalization_timeout_ms: default_html_finalization_timeout_ms(), + allow_async_html_finish: false, }; assert_eq!(publisher.origin_host(), "origin.example.com:8080"); @@ -6417,6 +6451,7 @@ source_domain = "partner.example.com" proxy_secret: Redacted::new("test-secret".to_string()), max_buffered_body_bytes: 16 * 1024 * 1024, html_finalization_timeout_ms: default_html_finalization_timeout_ms(), + allow_async_html_finish: false, }; assert_eq!(publisher.origin_host(), "origin.example.com"); @@ -6429,6 +6464,7 @@ source_domain = "partner.example.com" proxy_secret: Redacted::new("test-secret".to_string()), max_buffered_body_bytes: 16 * 1024 * 1024, html_finalization_timeout_ms: default_html_finalization_timeout_ms(), + allow_async_html_finish: false, }; assert_eq!(publisher.origin_host(), "localhost:9090"); @@ -6441,6 +6477,7 @@ source_domain = "partner.example.com" proxy_secret: Redacted::new("test-secret".to_string()), max_buffered_body_bytes: 16 * 1024 * 1024, html_finalization_timeout_ms: default_html_finalization_timeout_ms(), + allow_async_html_finish: false, }; assert_eq!(publisher.origin_host(), "localhost:9090"); @@ -6453,6 +6490,7 @@ source_domain = "partner.example.com" proxy_secret: Redacted::new("test-secret".to_string()), max_buffered_body_bytes: 16 * 1024 * 1024, html_finalization_timeout_ms: default_html_finalization_timeout_ms(), + allow_async_html_finish: false, }; assert_eq!(publisher.origin_host(), "192.168.1.1:8080"); @@ -6465,6 +6503,7 @@ source_domain = "partner.example.com" proxy_secret: Redacted::new("test-secret".to_string()), max_buffered_body_bytes: 16 * 1024 * 1024, html_finalization_timeout_ms: default_html_finalization_timeout_ms(), + allow_async_html_finish: false, }; assert_eq!(publisher.origin_host(), "[::1]:8080"); } @@ -6479,6 +6518,7 @@ source_domain = "partner.example.com" proxy_secret: Redacted::new("test-secret".to_string()), max_buffered_body_bytes: 16 * 1024 * 1024, html_finalization_timeout_ms: default_html_finalization_timeout_ms(), + allow_async_html_finish: false, }; assert_eq!(publisher.origin_host_header(), "origin.example.com:8443"); @@ -6494,6 +6534,7 @@ source_domain = "partner.example.com" proxy_secret: Redacted::new("test-secret".to_string()), max_buffered_body_bytes: 16 * 1024 * 1024, html_finalization_timeout_ms: default_html_finalization_timeout_ms(), + allow_async_html_finish: false, }; assert_eq!(publisher.origin_host_header(), "www.example.com"); diff --git a/crates/trusted-server-core/src/streaming_processor.rs b/crates/trusted-server-core/src/streaming_processor.rs index d95e6ebf8..05e8da763 100644 --- a/crates/trusted-server-core/src/streaming_processor.rs +++ b/crates/trusted-server-core/src/streaming_processor.rs @@ -27,6 +27,11 @@ use brotli::Decompressor; use brotli::enc::BrotliEncoderParams; use brotli::enc::writer::CompressorWriter; use error_stack::{Report, ResultExt as _}; +use futures::future::LocalBoxFuture; + +pub mod finish; + +use finish::{StreamFinish, StreamFinishError, StreamFinishInput}; use flate2::read::ZlibDecoder; use flate2::write::{GzEncoder, ZlibEncoder}; @@ -48,6 +53,29 @@ pub trait StreamProcessor { /// Returns an error if processing fails (e.g., I/O errors, encoding issues). fn process_chunk(&mut self, chunk: &[u8], is_last: bool) -> Result, io::Error>; + /// Report whether this processor requires an asynchronous EOF driver. + /// + /// Synchronous drivers reject this capability before consuming input. + fn requires_async_finish(&self) -> bool { + false + } + + /// Return optional borrowing EOF work after synchronous stage completion. + /// + /// The driver retains fallback output outside the future. Implementations + /// return promptly and perform I/O inside the future. The default creates + /// no future. Returned output reaches downstream stages before their EOF. + /// + /// # Errors + /// + /// Callback errors discard the candidate and preserve synchronous output. + fn async_finish<'a>( + &'a mut self, + _input: StreamFinishInput<'a>, + ) -> Option>>> { + None + } + /// Reset the processor state (useful for reuse) fn reset(&mut self) {} } @@ -164,6 +192,11 @@ impl StreamingPipeline

{ input: R, output: W, ) -> Result<(), Report> { + if self.processor.requires_async_finish() { + return Err(Report::new(TrustedServerError::Proxy { + message: "Processor requires asynchronous EOF completion".to_string(), + })); + } match ( self.config.input_compression, self.config.output_compression, @@ -2321,3 +2354,137 @@ mod tests { ); } } + +#[cfg(test)] +mod async_eof_contract_tests { + use std::cell::Cell; + use std::rc::Rc; + use std::time::Duration; + + use futures::executor::block_on; + use futures::future::LocalBoxFuture; + use http::StatusCode; + + use super::*; + use crate::platform::test_support::noop_services; + use crate::streaming_processor::finish::{ + StreamFinish, StreamFinishDeadline, StreamFinishError, StreamFinishInput, + StreamOutputWriter, + }; + + struct OptionalProcessor { + calls: Rc>, + } + + impl StreamProcessor for OptionalProcessor { + fn process_chunk(&mut self, chunk: &[u8], _: bool) -> io::Result> { + Ok(chunk.to_vec()) + } + + fn requires_async_finish(&self) -> bool { + true + } + + fn async_finish<'a>( + &'a mut self, + input: StreamFinishInput<'a>, + ) -> Option>>> { + Some(Box::pin(async move { + self.calls.set(self.calls.get() + 1); + input.replacement.append(input.output)?; + input.replacement.append(b"!")?; + Ok(StreamFinish::Replace) + })) + } + } + + struct ObservedReader(Rc>); + + impl Read for ObservedReader { + fn read(&mut self, _: &mut [u8]) -> io::Result { + self.0.set(self.0.get() + 1); + Ok(0) + } + } + + #[test] + fn async_eof_synchronous_pipeline_rejects_before_reading() { + let reads = Rc::new(Cell::new(0)); + let calls = Rc::new(Cell::new(0)); + let mut pipeline = StreamingPipeline::new( + PipelineConfig::default(), + OptionalProcessor { + calls: Rc::clone(&calls), + }, + ); + let mut output = Vec::new(); + let error = pipeline + .process(ObservedReader(Rc::clone(&reads)), &mut output) + .expect_err("should reject an async-capable processor before input effects"); + assert!( + format!("{error:?}").contains("async"), + "should explain async requirement" + ); + assert_eq!(reads.get(), 0, "should not consume the reader"); + assert_eq!(calls.get(), 0, "should not start the callback"); + assert!(output.is_empty(), "should not write output"); + } + + #[test] + fn async_eof_callback_borrows_non_send_state_and_driver_output() { + let calls = Rc::new(Cell::new(0)); + let mut processor = OptionalProcessor { + calls: Rc::clone(&calls), + }; + let services = noop_services(); + let deadline = StreamFinishDeadline::new(Duration::from_secs(1), false); + let mut replacement = StreamOutputWriter::new(8); + let outcome = block_on( + processor + .async_finish(StreamFinishInput { + response_status: StatusCode::OK, + output: b"tail", + replacement: &mut replacement, + services: &services, + deadline: &deadline, + }) + .expect("should provide optional EOF work"), + ) + .expect("should complete borrowed callback"); + assert_eq!( + outcome, + StreamFinish::Replace, + "should accept the candidate" + ); + assert_eq!( + replacement.into_bytes(), + b"tail!", + "should append to un-emitted output" + ); + assert_eq!(calls.get(), 1, "should call the finish once"); + } + + #[test] + fn async_eof_ordinary_processor_creates_no_future() { + let mut processor = StreamingReplacer::new(Vec::new()); + let services = noop_services(); + let deadline = StreamFinishDeadline::new(Duration::from_secs(1), false); + let mut replacement = StreamOutputWriter::new(8); + assert!( + !processor.requires_async_finish(), + "should default to synchronous processing" + ); + assert!( + processor + .async_finish(StreamFinishInput { + response_status: StatusCode::OK, + output: b"tail", + replacement: &mut replacement, + services: &services, + deadline: &deadline, + }) + .is_none(), + "should create no future by default" + ); + } +} diff --git a/crates/trusted-server-core/src/streaming_processor/finish.rs b/crates/trusted-server-core/src/streaming_processor/finish.rs new file mode 100644 index 000000000..2b4c98a3a --- /dev/null +++ b/crates/trusted-server-core/src/streaming_processor/finish.rs @@ -0,0 +1,246 @@ +//! Optional asynchronous processor completion contracts. + +use std::cell::Cell; +use std::time::Duration; + +use error_stack::Report; +use http::StatusCode; +use web_time::Instant; + +use crate::platform::RuntimeServices; + +/// Borrows un-emitted stage output and request services for optional EOF work. +/// +/// `output` is a streaming stage's tail or a deliberately buffered stage's +/// retained output. It is not universally the complete document. +pub struct StreamFinishInput<'a> { + /// Actual response status of the processed HTML. + pub response_status: StatusCode, + /// Driver-owned synchronous output still eligible for replacement. + pub output: &'a [u8], + /// Separate bounded storage for an atomic replacement candidate. + pub replacement: &'a mut StreamOutputWriter, + /// Request-scoped HTTP, KV and other runtime capabilities. + pub services: &'a RuntimeServices, + /// Absolute deadline shared across optional callbacks. + pub deadline: &'a StreamFinishDeadline, +} + +/// Selects whether to retain input or commit the bounded replacement candidate. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum StreamFinish { + /// Retain the current input without copying it. + Unchanged, + /// Commit bytes appended to [`StreamOutputWriter`]. + Replace, +} + +/// Controlled failure categories for optional document finalization. +#[derive(Debug, derive_more::Display)] +pub enum StreamFinishError { + /// The required outside service is unavailable. + #[display("Stream completion service is unavailable")] + Unavailable, + /// Outside work cannot produce an acceptable result. + #[display("Stream completion result is rejected")] + Rejected, + /// The bounded candidate cannot accept more bytes. + #[display("Stream replacement exceeds the {limit}-byte limit")] + OutputLimitExceeded { + /// Maximum candidate size in bytes. + limit: usize, + }, +} + +impl core::error::Error for StreamFinishError {} + +/// Separate bounded storage for a finalizer's replacement candidate. +/// +/// A failed append poisons the candidate, preventing partial commitment even +/// when an integration ignores the returned error. +pub struct StreamOutputWriter { + bytes: Vec, + limit: usize, + poisoned: bool, +} + +impl StreamOutputWriter { + pub(crate) fn new(limit: usize) -> Self { + Self { + bytes: Vec::new(), + limit, + poisoned: false, + } + } + + /// Append bytes without growing the candidate beyond its configured limit. + /// + /// # Errors + /// + /// Returns [`StreamFinishError::OutputLimitExceeded`] when the candidate + /// is poisoned, length arithmetic overflows, or the append exceeds the limit. + /// + /// # Examples + /// + /// ```ignore + /// input.replacement.append(b"

Example article

")?; + /// ``` + pub fn append(&mut self, chunk: &[u8]) -> Result<(), Report> { + let next = self + .bytes + .len() + .checked_add(chunk.len()) + .filter(|length| *length <= self.limit); + let next = match next { + Some(next) if !self.poisoned => next, + _ => { + self.poisoned = true; + return Err(Report::new(StreamFinishError::OutputLimitExceeded { + limit: self.limit, + })); + } + }; + if next > self.bytes.capacity() { + let capacity = self + .bytes + .capacity() + .saturating_mul(2) + .max(next) + .min(self.limit); + self.bytes.reserve_exact(capacity - self.bytes.len()); + } + self.bytes.extend_from_slice(chunk); + Ok(()) + } + + pub(crate) fn is_poisoned(&self) -> bool { + self.poisoned + } + + pub(crate) fn into_bytes(self) -> Vec { + self.bytes + } +} + +/// Absolute finish budget shared by all finalizers for one HTML document. +/// +/// A cooperative timer can interrupt yielding work, but cannot preempt CPU work +/// or blocking host calls. Consumers must bound their own calls on Fastly. +pub struct StreamFinishDeadline { + started_at: Instant, + timeout: Duration, + cooperative_timer: bool, + timer_expired: Cell, +} + +impl StreamFinishDeadline { + pub(crate) fn new(timeout: Duration, cooperative_timer: bool) -> Self { + Self { + started_at: Instant::now(), + timeout, + cooperative_timer, + timer_expired: Cell::new(false), + } + } + + /// Return the remaining whole-chain budget, saturating at zero. + #[must_use] + pub fn remaining(&self) -> Duration { + self.remaining_at(Instant::now()) + } + + pub(crate) fn elapsed(&self) -> Duration { + self.started_at.elapsed() + } + + /// Report whether the request has a cooperative timer for yielding work. + #[must_use] + pub fn has_cooperative_timer(&self) -> bool { + self.cooperative_timer + } + + pub(crate) fn mark_expired(&self) { + self.timer_expired.set(true); + } + + pub(crate) fn remaining_at(&self, now: Instant) -> Duration { + if self.timer_expired.get() { + return Duration::ZERO; + } + self.timeout + .saturating_sub(now.saturating_duration_since(self.started_at)) + } + + pub(crate) fn is_expired_at(&self, now: Instant) -> bool { + self.remaining_at(now).is_zero() + } +} + +#[cfg(test)] +mod tests { + use std::time::Duration; + + use super::*; + + #[test] + fn replacement_limit_poison_preserves_the_candidate() { + let mut writer = StreamOutputWriter::new(4); + writer.append(b"page").expect("should accept the exact cap"); + let _ = writer + .append(b"!") + .expect_err("should reject output beyond the cap"); + assert!(writer.is_poisoned(), "should poison an oversized candidate"); + assert_eq!(writer.bytes, b"page", "should preserve the accepted prefix"); + let _ = writer + .append(b"") + .expect_err("should reject writes after poisoning"); + } + + #[test] + fn replacement_capacity_stays_under_the_limit() { + let mut writer = StreamOutputWriter::new(17); + assert_eq!(writer.bytes.capacity(), 0, "should allocate lazily"); + for _ in 0..17 { + writer.append(b"x").expect("should append within the limit"); + assert!(writer.bytes.capacity() <= 17, "should bound reserve growth"); + } + } + + #[test] + fn empty_replacement_is_distinct_from_unchanged() { + let writer = StreamOutputWriter::new(4); + assert!( + writer.into_bytes().is_empty(), + "should allow an empty replacement" + ); + assert_ne!( + StreamFinish::Unchanged, + StreamFinish::Replace, + "should distinguish outcomes" + ); + } + + #[test] + fn deadline_expires_at_the_boundary() { + let deadline = StreamFinishDeadline::new(Duration::from_millis(5), false); + assert_eq!( + deadline.remaining_at(deadline.started_at), + Duration::from_millis(5), + "should start with the complete budget" + ); + let expired = deadline.started_at + Duration::from_millis(5); + assert!( + deadline.is_expired_at(expired), + "should expire at the exact boundary" + ); + assert_eq!( + deadline.remaining_at(expired + Duration::from_millis(1)), + Duration::ZERO, + "should saturate elapsed time" + ); + assert!( + !deadline.has_cooperative_timer(), + "should expose the absence of preemption" + ); + } +} diff --git a/crates/trusted-server-core/src/test_support/async_html_finalization.rs b/crates/trusted-server-core/src/test_support/async_html_finalization.rs index 9cf27c527..9e488c4ef 100644 --- a/crates/trusted-server-core/src/test_support/async_html_finalization.rs +++ b/crates/trusted-server-core/src/test_support/async_html_finalization.rs @@ -1,5 +1,6 @@ //! Fictional terminal HTTP consumer, available only with `test-utils`. +use std::io; use std::sync::Arc; use std::time::Duration; @@ -15,11 +16,12 @@ use validator::Validate; use crate::error::TrustedServerError; use crate::integrations::{ - HtmlFinalization, HtmlFinalizationError, HtmlFinalizationInput, IntegrationHtmlFinalizer, - IntegrationHtmlFinalizerFactory, IntegrationHtmlStreamContext, IntegrationRegistration, + IntegrationHtmlStreamContext, IntegrationHtmlStreamProcessorFactory, IntegrationRegistration, }; use crate::platform::{PlatformBackendSpec, PlatformHttpRequest}; use crate::settings::{IntegrationConfig, Settings}; +use crate::streaming_processor::StreamProcessor; +use crate::streaming_processor::finish::{StreamFinish, StreamFinishError, StreamFinishInput}; pub(crate) const ID: &str = "html_finalization_fixture"; const EOF_SENTINEL: &[u8] = b""; @@ -33,6 +35,8 @@ pub(crate) struct FixtureConfig { lookup_url: String, #[validate(range(min = 1, max = 60000))] transport_timeout_ms: u32, + #[serde(default)] + streaming: bool, } impl IntegrationConfig for FixtureConfig { @@ -42,7 +46,14 @@ impl IntegrationConfig for FixtureConfig { } struct FixtureFactory(FixtureConfig); -struct FixtureSession(FixtureConfig); +struct FixtureSession { + config: FixtureConfig, + digest: Sha256, + bytes: usize, + tail: Vec, + saw_eof_sentinel: bool, + synchronous_eof: bool, +} pub(crate) fn register( settings: &Settings, @@ -53,47 +64,78 @@ pub(crate) fn register( Ok(Some( IntegrationRegistration::builder(ID) .without_js() - .with_html_finalizer(Arc::new(FixtureFactory(config))) + .with_html_stream_processor(Arc::new(FixtureFactory(config))) .build(), )) } -impl IntegrationHtmlFinalizerFactory for FixtureFactory { +impl IntegrationHtmlStreamProcessorFactory for FixtureFactory { fn integration_id(&self) -> &'static str { ID } - fn create(&self, _: IntegrationHtmlStreamContext) -> Box { - Box::new(FixtureSession(self.0.clone())) + fn requires_async_finish(&self) -> bool { + true + } + fn buffer_output_until_eof(&self) -> bool { + !self.0.streaming + } + + fn create(&self, _: IntegrationHtmlStreamContext) -> Box { + Box::new(FixtureSession { + config: self.0.clone(), + digest: Sha256::new(), + bytes: 0, + tail: Vec::new(), + saw_eof_sentinel: false, + synchronous_eof: false, + }) } } -impl IntegrationHtmlFinalizer for FixtureSession { - fn finalize<'a>( +impl StreamProcessor for FixtureSession { + fn process_chunk(&mut self, chunk: &[u8], last: bool) -> io::Result> { + self.digest.update(chunk); + self.bytes = self + .bytes + .checked_add(chunk.len()) + .ok_or_else(|| io::Error::other("Fixture byte count exceeds its range"))?; + let mut observed = std::mem::take(&mut self.tail); + observed.extend_from_slice(chunk); + self.saw_eof_sentinel |= observed + .windows(EOF_SENTINEL.len()) + .any(|bytes| bytes == EOF_SENTINEL); + let keep = observed.len().min(EOF_SENTINEL.len() - 1); + self.tail + .extend_from_slice(&observed[observed.len() - keep..]); + self.synchronous_eof = last; + Ok(chunk.to_vec()) + } + fn requires_async_finish(&self) -> bool { + true + } + + fn async_finish<'a>( &'a mut self, - input: HtmlFinalizationInput<'a>, - ) -> LocalBoxFuture<'a, Result>> { - Box::pin(async move { - if !input - .html - .windows(EOF_SENTINEL.len()) - .any(|bytes| bytes == EOF_SENTINEL) - { - return Err(Report::new(HtmlFinalizationError::Rejected)); + input: StreamFinishInput<'a>, + ) -> Option>>> { + Some(Box::pin(async move { + if !self.saw_eof_sentinel || !self.synchronous_eof { + return Err(Report::new(StreamFinishError::Rejected)); } let url = - Url::parse(&self.0.lookup_url).change_context(HtmlFinalizationError::Rejected)?; + Url::parse(&self.config.lookup_url).change_context(StreamFinishError::Rejected)?; // Fastly cannot interrupt a blocking host call. Bound its transport // by the remaining chain budget; timer adapters deliberately test // cancellation against a longer transport timeout. let timeout = if input.deadline.has_cooperative_timer() { - Duration::from_millis(u64::from(self.0.transport_timeout_ms)) + Duration::from_millis(u64::from(self.config.transport_timeout_ms)) } else { input .deadline .remaining() .min(Duration::from_millis(u64::from( - self.0.transport_timeout_ms, + self.config.transport_timeout_ms, ))) }; let backend = input @@ -103,7 +145,7 @@ impl IntegrationHtmlFinalizer for FixtureSession { scheme: url.scheme().to_string(), host: url .host_str() - .ok_or_else(|| Report::new(HtmlFinalizationError::Rejected))? + .ok_or_else(|| Report::new(StreamFinishError::Rejected))? .to_string(), port: url.port(), host_header_override: None, @@ -112,55 +154,62 @@ impl IntegrationHtmlFinalizer for FixtureSession { between_bytes_timeout: timeout, discriminator: Some(ID.to_string()), }) - .change_context(HtmlFinalizationError::Unavailable)?; + .change_context(StreamFinishError::Unavailable)?; let payload = serde_json::to_vec(&json!({ "complete": true, - "sha256": hex::encode(Sha256::digest(input.html)), - "bytes": input.html.len(), + "sha256": hex::encode(self.digest.clone().finalize()), + "bytes": self.bytes, "transport_timeout_ms": timeout.as_millis(), })) - .change_context(HtmlFinalizationError::Rejected)?; + .change_context(StreamFinishError::Rejected)?; let request = request_builder() .method(http::Method::POST) .uri( - self.0 + self.config .lookup_url .parse::() - .change_context(HtmlFinalizationError::Rejected)?, + .change_context(StreamFinishError::Rejected)?, ) .header(http::header::CONTENT_TYPE, "application/json") .body(Body::from(payload)) - .change_context(HtmlFinalizationError::Rejected)?; + .change_context(StreamFinishError::Rejected)?; let response = input .services .http_client() .send(PlatformHttpRequest::new(request, backend).with_cache_bypass()) .await - .change_context(HtmlFinalizationError::Unavailable)?; + .change_context(StreamFinishError::Unavailable)?; if !response.response.status().is_success() { - return Err(Report::new(HtmlFinalizationError::Unavailable)); + return Err(Report::new(StreamFinishError::Unavailable)); } let reply = response .response .into_body() .into_bytes_bounded(64) .await - .change_context(HtmlFinalizationError::Rejected)?; + .change_context(StreamFinishError::Rejected)?; if reply.as_ref() != b"EXAMPLE_APPROVED" { - return Err(Report::new(HtmlFinalizationError::Rejected)); + return Err(Report::new(StreamFinishError::Rejected)); + } + if self.config.streaming { + input.replacement.append(input.output)?; + input + .replacement + .append(b"")?; + return Ok(StreamFinish::Replace); } let offset = input - .html + .output .windows(ORIGINAL.len()) .position(|bytes| bytes == ORIGINAL) - .ok_or_else(|| Report::new(HtmlFinalizationError::Rejected))?; - input.replacement.append(&input.html[..offset])?; + .ok_or_else(|| Report::new(StreamFinishError::Rejected))?; + input.replacement.append(&input.output[..offset])?; input.replacement.append(REPLACEMENT)?; input .replacement - .append(&input.html[offset + ORIGINAL.len()..])?; - Ok(HtmlFinalization::Replace) - }) + .append(&input.output[offset + ORIGINAL.len()..])?; + Ok(StreamFinish::Replace) + })) } } @@ -174,13 +223,14 @@ mod tests { fn fictional_consumer_registers_only_when_enabled_and_adds_no_browser_bundle() { for enabled in [false, true] { let mut settings = create_test_settings(); + settings.publisher.allow_async_html_finish = true; settings.integrations.insert_config(ID, &json!({ "enabled": enabled, "lookup_url": "https://lookup.example.com/approve", "transport_timeout_ms": 1000, })).expect("should configure the fictional consumer"); let registry = IntegrationRegistry::new(&settings).expect("should build fixture metadata"); assert_eq!( - registry.has_html_finalizers(), + registry.has_async_html_finish(), enabled, "should exclude disabled terminal capabilities" ); diff --git a/crates/trusted-server-integration-tests/tests/parity.rs b/crates/trusted-server-integration-tests/tests/parity.rs index 3688160ef..9196e71ba 100644 --- a/crates/trusted-server-integration-tests/tests/parity.rs +++ b/crates/trusted-server-integration-tests/tests/parity.rs @@ -1192,6 +1192,7 @@ impl PlatformHttpClient for TerminalFixtureOrigin { #[tokio::test(flavor = "multi_thread", worker_threads = 2)] async fn asynchronous_html_finalization_and_privacy_match_host_adapters() { let mut settings = test_settings(); + settings.publisher.allow_async_html_finish = true; settings.integrations.insert_config("html_finalization_fixture", &serde_json::json!({ "enabled": true, "lookup_url": "https://lookup.example.com/success", "transport_timeout_ms": 1000, })).expect("should enable the fictional terminal consumer"); diff --git a/docs/guide/configuration.md b/docs/guide/configuration.md index f97450dbf..0c634d025 100644 --- a/docs/guide/configuration.md +++ b/docs/guide/configuration.md @@ -309,15 +309,16 @@ Core publisher settings for domain, origin, and proxy configuration. ### `[publisher]` -| Field | Type | Required | Description | -| ------------------------------ | ------- | -------- | ---------------------------------------------------------------------------------- | -| `domain` | String | Yes | Publisher's apex domain name | -| `cookie_domain` | String | Yes | Domain for non-EC cookies (typically with leading dot) | -| `origin_url` | String | Yes | Full URL of publisher origin server | -| `origin_host_header_override` | String | No | Outbound Host header to send while connecting to `origin_url` | -| `proxy_secret` | String | Yes | Secret-store key name for the proxy URL secret | -| `max_buffered_body_bytes` | Integer | No | Buffered-body cap / Fastly stream raw+decoded byte ceiling (default 16 MiB) | -| `html_finalization_timeout_ms` | Integer | No | Whole terminal-chain budget after synchronous EOF (default 5000 ms; range 1–60000) | +| Field | Type | Required | Description | +| ------------------------------ | ------- | -------- | ------------------------------------------------------------------------------------ | +| `domain` | String | Yes | Publisher's apex domain name | +| `cookie_domain` | String | Yes | Domain for non-EC cookies (typically with leading dot) | +| `origin_url` | String | Yes | Full URL of publisher origin server | +| `origin_host_header_override` | String | No | Outbound Host header to send while connecting to `origin_url` | +| `proxy_secret` | String | Yes | Secret-store key name for the proxy URL secret | +| `max_buffered_body_bytes` | Integer | No | Buffered-body cap / Fastly stream raw+decoded byte ceiling (default 16 MiB) | +| `allow_async_html_finish` | Boolean | No | Explicit permission for async HTML EOF work and its delivery cost (default false) | +| `html_finalization_timeout_ms` | Integer | No | Shared optional EOF budget after decoder/parser EOF (default 5000 ms; range 1–60000) | > **Note:** EC cookies (`ts-ec`) derive their domain automatically as `.{domain}` and > do not use `cookie_domain`. The `cookie_domain` field is used by other cookie helpers. @@ -438,22 +439,33 @@ intentionally, and never put it in the TOML file or pushed app-config blob. Changing `proxy_secret` invalidates all existing signed URLs. Plan rotations carefully and use graceful transition periods. ::: +#### `allow_async_html_finish` + +Explicit permission for integrations declaring asynchronous HTML processor completion. +Defaults to `false`. An enabled integration requiring EOF I/O without permission is +rejected with a configuration error; its requested work is not silently skipped. +Permission alone does not change processing when no async factories are registered. + +Non-retaining processors can release ready HTML before EOF I/O on Fastly. A factory +requesting output retention delays initial delivery until EOF or overflow. Axum, +Workers and Spin currently buffer publisher responses, so enabled EOF I/O can add to +TTFB there even without stage retention. Enable only when accepting that delivery cost. + #### `html_finalization_timeout_ms` -Optional whole-chain budget for registered asynchronous HTML finalizers, in -milliseconds. The default is `5000`; accepted values are `1` through `60000`. -The budget starts after decoding and synchronous HTML EOF and does not change the -auction deadline. It has no effect without enabled finalizers. Axum, Workers, and -Spin can cancel yielding work; Fastly consumers must bound their own blocking host -calls. Timeout keeps the current transformed document and skips later finalizers. - -Enabled finalizers disable shared-template and downstream caching for processed -HTML, including unchanged/error outcomes; raw-origin readthrough remains separate. -HTML with status 206 or any `Content-Range` bypasses rewriting and terminal I/O, -preserving its encoded bytes and range/framing metadata. Full error pages remain -eligible and expose their status to the integration. See -[Asynchronous HTML finalization](/guide/integration-guide#asynchronous-html-finalization) -for consumer requirements and the cacheability contract needed for a future mode. +Shared budget for optional asynchronous processor EOF work in milliseconds. The default +is `5000`; accepted values are `1` through `60000`. It begins after decoder/parser EOF +and does not change the auction deadline. Timer expiry skips later optional I/O while +still completing downstream synchronous EOF. Timeout retains the current un-emitted +stage output; it cannot undo bytes already delivered or remote side effects. + +The setting is inert without async processors. Axum, Workers and Spin can cancel +yielding callbacks. Fastly consumers must independently bound blocking host calls. +Opted-in processed HTML bypasses shared-template caching and uses private/no-store +policy; raw-origin readthrough remains separate. Opted-in HTML with 206 or any +`Content-Range` bypasses rewriting and EOF work while preserving encoded bytes and +range/framing metadata. Full error HTML remains eligible and exposes actual status. +See [Asynchronous HTML processor completion](/guide/integration-guide#asynchronous-html-processor-completion). #### `max_buffered_body_bytes` diff --git a/docs/guide/integration-guide.md b/docs/guide/integration-guide.md index ff1b2629c..c8172d56b 100644 --- a/docs/guide/integration-guide.md +++ b/docs/guide/integration-guide.md @@ -203,129 +203,131 @@ Streaming support is an adapter capability. Request checked `supports_streaming_responses()`; unsupported adapters must reject the request instead of silently buffering it. -## Asynchronous HTML finalization - -An integration that needs complete transformed HTML plus an outside lookup can -register an `IntegrationHtmlFinalizerFactory` with `.with_html_finalizer(...)`. -The factory creates one request-local `IntegrationHtmlFinalizer` session and -receives the same `IntegrationDocumentState` as the synchronous HTML rewriters. -Factory construction performs setup only; start I/O inside the returned future. -The session and its future may be `?Send`. - -Core completes decoding and every synchronous processor, then awaits terminal -sessions in registration order. Each session sees the previous committed HTML; -its replacement does not pass through the parser again. The auction seam and -encoder complete after the terminal phase. Existing `StreamProcessor` callbacks -remain synchronous, and pages with no registered finalizers keep their existing -streaming path. - -`HtmlFinalizationInput` supplies the actual `response_status`, borrowed `html`, `services`, `deadline`, and a -separate bounded `replacement` writer. Append a complete candidate and return -`HtmlFinalization::Replace`, or return `Unchanged`. An error, timeout, invalid -assembly marker count, or output limit violation discards the candidate and -retains the current input. Ignoring an append error cannot commit partial output. -An empty `Replace` intentionally emits an empty document. Callback panics propagate -through the request and are not converted into optional-work failures. Preserve HTML structure, -CSP, Next.js/RSC semantics, and the opaque parser markers; core checks marker -multiplicity but does not parse or validate replacement structure. - -Full 404/500 HTML remains eligible. A signing integration should inspect -`response_status` and return `Unchanged` for statuses it does not sign. When -finalizers are registered, HTML with status 206 or any `Content-Range` bypasses -rewriting and terminal I/O while preserving encoded body bytes and range/framing -metadata. Without registered finalizers, existing partial-response processing -continues. Non-document fetches keep their range request semantics. Independent -cookie and diagnostics privacy rules still apply. - -Each real consumer must test a winning auction after replacement: the marker is -consumed once, and the bids script remains before the structural body end, outside -scripts, comments, and attributes. Unchanged bytes beside a marker cannot establish -its parsing context, so adjacent-byte checks are not a structural guarantee. - -The publisher's `html_finalization_timeout_ms` is one budget for the whole chain, -starting after synchronous EOF. Use `deadline.remaining()` to bound outside calls. -Axum, Workers, and Spin provide cooperative timers. Fastly provides no timer -because host calls block; the consumer must set transport timeouts itself. Dropping -a future releases its borrows and discards its candidate, but cannot undo a remote -write or preempt work that does not yield. The deadline wins over a simultaneously -ready callback. Avoid logging HTML, lookup URLs, or external response contents. -The default backend first-byte timeout is 15 seconds; it is not a total-call -deadline. Use the remaining budget for supported transport limits and verify the -consumer's blocking-call behavior on Fastly before enabling it. - -Enabled finalizers retain transformed HTML up to `max_buffered_body_bytes` and use -a separate candidate with the same cap. Collection overflow skips terminal I/O -and resumes normal delivery with the retained prefix and remaining bytes. Existing -input, decoder, and final output errors remain fatal. Enabled pages wait for EOF -within the cap; oversized pages also delay first-byte delivery until the cap is -crossed and ordinary streaming resumes. +## Asynchronous HTML processor completion + +An integration can register an HTML stream processor that optionally awaits HTTP, +KV or other request services after its synchronous EOF call. Use the existing +`IntegrationHtmlStreamProcessorFactory` and `.with_html_stream_processor(...)`. +Article signing is only one possible consumer; core implements the processor +lifecycle, not a signing protocol or background job system. + +Factories declare `requires_async_finish()`. Sessions implement the matching +`StreamProcessor::requires_async_finish()` and optional `async_finish(...)`. +The default callback returns `None`, creating no future. Factory metadata is +available before template lookup, so requested EOF work cannot be skipped by a +processed cache hit. The factory and session share request-local +`IntegrationDocumentState` with the existing parser hooks. + +Operators must explicitly accept the possible delivery cost: + +```toml +[publisher] +allow_async_html_finish = true +html_finalization_timeout_ms = 5000 +``` + +Permission defaults to false. Enabling an integration requiring async completion +without permission produces a configuration error naming the integration and the +setting. Work is not silently omitted. Permission alone, with no async factories, +keeps ordinary processing unchanged. + +Chunk callbacks remain synchronous. At EOF, core validates/drains the decoder and +finishes the HTML parser, sending ready bytes through downstream processors as +ordinary chunks. It then completes each processor in registration order: synchronous +EOF, optional awaited finish, accepted output passed downstream, then downstream EOF. +The auction and encoder finish afterward. Ready bytes are released before optional +I/O; a callback is never run inside a parser handler. + +`StreamFinishInput` supplies actual `response_status`, borrowed `output`, `services`, +`deadline` and a separate bounded `replacement` writer. The output is the un-emitted +synchronous tail of this stage. It is the complete retained output only when the +factory explicitly requests `buffer_output_until_eof()`. Return `StreamFinish::Unchanged` +to retain it, or write a complete candidate and return `Replace`. To append a tail, +write the existing `output` followed by the new bytes. Empty replacement is intentional. +The session and future may be non-`Send`; start I/O inside the returned future. + +Core buffers a stage's output only when its factory explicitly requests retention. +The holder and replacement candidate each use `max_buffered_body_bytes` as their cap. +On collection overflow, core releases prefix/crossing/later output exactly once and +skips that stage's optional I/O, while still completing its synchronous EOF. It cannot +bound arbitrary private allocations inside an integration. Streaming stages cannot +replace bytes already delivered; EOF append does not imply safe insertion into an +already emitted head or before an earlier body close. + +Callback errors, poisoned candidates, invalid assembly-marker multiplicity and timeout +discard the candidate and retain current stage output. Previously accepted stages stay +accepted. One absolute deadline covers every callback, starting after decoder/parser +EOF. On expiry, core drops the callback, skips later optional I/O, and still completes +all downstream synchronous EOF calls. Origin, parser, synchronous processing, codec and +sink errors remain fatal. Callback panics propagate. Cancellation does not undo remote +writes or guarantee that transport work is cancelled. + +Axum, Workers and Spin provide cooperative timers. Fastly has none because its host +calls block; bound those calls with `deadline.remaining()` and transport timeouts. +A timer cannot preempt CPU work or blocking host calls. Avoid logging HTML, lookup URLs +or external response contents. + +The latency contract depends on the adapter and the stage: + +- Without async factories, delivery and auction timing keep their existing path. +- On Fastly, a non-retaining stage releases ready HTML before optional EOF I/O; the + response can finish later. +- A retaining stage delays first HTML delivery until EOF or overflow release. +- Axum, Workers and Spin currently buffer publisher responses. Enabled EOF I/O can + add to TTFB there even without stage retention. This feature does not add streaming + support to those adapters. Use `stream_publisher_body_async`, `buffer_publisher_response_async`, or -`publisher_response_into_streaming_response` to execute terminal callbacks. -`stream_publisher_body` and the generic synchronous pipeline perform synchronous -transformations only. Callers with a pending auction must use an async entry point -to collect or abandon it; `stream_publisher_body` borrows the response parameters -and leaves auction ownership with the caller, including on a partial-HTML bypass. -On async entry points, client cancellation drops the callback and retains the -existing auction guard's log warning; error paths that can await emit abandonment -telemetry. - -Finalizer-enabled HTML bypasses shared-template reads, reservations, and stores and -uses inline auction assembly. Raw-origin readthrough remains independent. Processed -HTML receives `Cache-Control: no-store, private` even when a finalizer leaves it -unchanged, fails, or is skipped after collection overflow. Document GETs strip the -four conditional/range headers used by synthesized HTML. HEAD, bodiless responses, -unsupported encodings, and non-HTML responses run no terminal callbacks. - -This first version therefore removes transformed-template and downstream HTML -caching for enabled pages, including page-level signing. Raw-origin caching and -an integration's own KV result cache can still reduce origin and lookup work. -Cacheable finalization needs a separate contract for reader-neutral output, -external-data/signing-key identity, expiry, invalidation, optional failure caching, -and whether callbacks execute on hits; a user-independent flag alone is insufficient. - -Callers of `handle_publisher_request` provide the enabled integration registry in -`AuctionDispatch::integration_registry` and use that same registry for response -finalization. This metadata is required before a shared-template lookup can happen. -It is a source-compatibility change for external `AuctionDispatch` struct literals: -add the required field using the same request registry. An omitted or optional -registry could incorrectly admit a cached document before finalization is known. - -A fictional HTTP consumer is available only through the existing `test-utils` -feature. To verify real asynchronous I/O and timeout fallback on local runtimes: +`publisher_response_into_streaming_response` for opted-in HTML. The synchronous +pipeline and `stream_publisher_body` reject async processing before input/output +effects. The raw partial-HTML copy path performs no callback; the synchronous helper +leaves pending auction ownership with its caller, so pending auctions still require +an async entry point. Async cancellation retains the auction guard's warning; error +paths that can await emit abandonment telemetry. + +Opted-in HTML bypasses shared-template reads/reservations/stores and receives +`Cache-Control: no-store, private`, including unchanged, failed and overflow-bypassed +work. Raw-origin readthrough remains independent. Cacheable asynchronous output needs +separate rules for output identity, external-data freshness, expiry, invalidation, +failure caching and callback execution on hits. + +When async factories are registered, HTML with 206 or any `Content-Range` bypasses +rewriting and EOF I/O while preserving encoded bytes, framing, validators and origin +cache policy. Independent cookie/diagnostics privacy still applies. HEAD, bodiless, +non-HTML and unsupported encodings run no callbacks. Complete error HTML is eligible; +inspect `response_status` to decide whether the integration applies. Without async +factories, existing partial-response handling continues. + +Callers of `handle_publisher_request` supply the same enabled registry in +`AuctionDispatch::integration_registry` and response processing. This required field +allows cache admission to see async capabilities before lookup. + +Replacement output passes through later stream processors, but is not parsed again. +Preserve CSP, Next.js/RSC semantics and opaque assembly markers. Core checks marker +multiplicity including split boundaries, not HTML structure. Each real consumer must +verify winning-auction placement after its changes and preserve the structural body +end; neighboring bytes alone cannot establish a marker's parsing context. + +A fictional HTTP processor is available only with `test-utils`: ```bash -./scripts/async-html-finalization-local-test.sh axum -./scripts/async-html-finalization-local-test.sh cloudflare -./scripts/async-html-finalization-local-test.sh spin -./scripts/async-html-finalization-local-test.sh fastly +./scripts/async-html-finalization-local-test.sh axum --auction +./scripts/async-html-finalization-local-test.sh cloudflare --auction +./scripts/async-html-finalization-local-test.sh spin --auction +./scripts/async-html-finalization-local-test.sh fastly --auction ``` -Each run builds a fresh feature-enabled artifact, generates temporary configuration -and example secrets, and retains diagnostics in the printed temporary directory. -It uses the installed runtime CLIs and, for Workers, the `worker-build` version -matching the locked `worker` SDK and a compatible `wasm-bindgen` CLI. The default -Cargo output is isolated in that directory; an explicit `CARGO_TARGET_DIR` permits -sharing the local build cache. Every run invokes Cargo and builds its fixture Worker -bundle into a fresh directory. Workers and Spin use opt-in adapter `test-utils` -features for local diagnostic sinks. - -The repository pins Viceroy 0.17 for its normal test gate. Backend transport timeout -simulation requires Viceroy 0.19 or later. Use an isolated newer executable in `PATH` -for the Fastly hung-call case; do not interpret a hung test on 0.17 as evidence that -Fastly production ignores the configured timeout. - -Add `--release` for production-profile artifacts. Add `--auction` to run a separate -enabled-auction control against a closed loopback -bid endpoint. It verifies slot/bid fallback assembly and marker removal without -external traffic. The auction telemetry clock now uses the platform-compatible -`web_time::Instant`; the combined Workers control passes after this targeted -correction. This does not verify every other clock path covered by #1240. - -CI runs the actual Axum harness with `--auction`, including delayed success, -hung-call fallback, privacy, and auction assembly. It also checks process-group -cleanup and retains runtime diagnostics on failure. The other runtime harnesses -remain explicit local verification commands. +The harness builds fresh enabled artifacts, generates isolated example configuration, +and checks disabled, successful, hung and auction controls. Fastly also runs a +non-retaining processor and proves initial body bytes arrive while its delayed EOF +lookup is pending. `--release` selects production-profile fixture artifacts. Diagnostics +are retained in the printed temporary directory. An explicit `CARGO_TARGET_DIR` permits +sharing the build cache. Workers requires tooling matching the locked SDK. + +Repository-pinned Viceroy 0.17 remains the normal WASI test runner. The Fastly hung-call +transport simulation requires an isolated Viceroy 0.19 or newer; repository pins stay +unchanged. CI runs actual Axum controls and real-process cleanup tests. The targeted +auction telemetry clock correction remains separate from broader #1240 clock work. ## Browser and script guards diff --git a/docs/superpowers/archive/2026-10-12-1241-buffered-terminal-design.md b/docs/superpowers/archive/2026-10-12-1241-buffered-terminal-design.md new file mode 100644 index 000000000..d63c6e721 --- /dev/null +++ b/docs/superpowers/archive/2026-10-12-1241-buffered-terminal-design.md @@ -0,0 +1,691 @@ +# Optional asynchronous HTML document finalization + +> Historical record: this specification describes the buffered terminal API. +> The [revised specification](../specs/2026-10-12-1241-async-html-finalization-design.md) +> governs the ordered EOF revision authorized on 2026-10-13. + +**Date:** 2026-10-12 + +**Status:** Implemented on `feat/1241-async-html-finalization`; review hardening and verification recorded below + +**Issue:** trusted-server #1241 + +**Inspected baseline:** `2a0cb0444dc335bd09105abe5952329bbbdfb621` + +**Implementation plan:** [Task-by-task implementation and verification](2026-10-12-1241-buffered-terminal-plan.md) + +**Implementation review:** [Requirement coverage, verification, and runtime limitations](../reviews/2026-10-12-1241-async-html-finalization-implementation.md) + +**Review hardening:** [Response metadata, range safety, telemetry clock, and CI refinements](../plans/2026-10-12-1241-review-hardening.md) + +## 1. Decision + +Add an opt-in asynchronous final-document phase to Trusted Server's HTML integration +pipeline. An enabled finalizer receives the completed, synchronously transformed HTML +document and may await request-time HTTP or KV services before producing replacement +HTML. Core owns both the retained document and a separate bounded replacement writer. + +Keep `StreamProcessor::process_chunk` synchronous and leave `StreamProcessor` and +`StreamingPipeline::process` unchanged. The new capability is registered explicitly as +an HTML finalizer, rather than being an optional method that some generic synchronous +drivers cannot execute. + +Only processable HTML documents with an enabled finalizer take this path. Existing +integrations retain their streaming behavior. Finalizers execute in their own registration +order, after all existing synchronous HTML processors. This is a terminal phase, not an +interleaved asynchronous extension to every streaming stage. + +One absolute finish deadline covers the entire finalizer chain. On optional-finalizer +failure, retain the bytes entering that finalizer; on deadline expiry, also skip all +remaining finalizers. Preserve existing fatal handling for origin, parser, synchronous +processor, compression, and downstream-write failures. + +For the first version, enabled HTML finalizers bypass shared-template cache reads and +writes before lookup. Their processed HTML responses use the existing synthesized HTML +`private, no-store` policy. Supporting cacheable asynchronous finalizers requires a +separate design for output identity, freshness, and invalidation. + +No EdgeZero revision change is required. Trusted Server owns this document lifecycle and +its request service container. A reusable EdgeZero timer can be considered separately. + +## 2. Verified problem and limits of the evidence + +At the inspected baseline: + +- `streaming_processor.rs::StreamProcessor` provides synchronous `process_chunk` and + `reset` methods. It offers no future-returning operation. +- `html_processor.rs::HtmlWithStreamingProcessors` forwards each chunk and the same + `is_last` value through the entire synchronous chain immediately. +- `IntegrationHtmlStreamProcessorFactory` creates per-document processors. Its context + contains hosts, scheme, and document-local state, but no request services. +- `publisher.rs` awaits auction collection outside synchronous processing callbacks. + The auction already has an application-specific asynchronous seam. +- Buffered materialized responses still use `StreamingPipeline::process` or the separate + reader-based auction hold loop. An asynchronous stream-helper change alone would miss + those paths. + +A local probe using the actual `StreamingPipeline` verified that `now_or_never()` completes +an immediately ready lookup but discards a lookup that returns `Pending`. Awaiting the same +yielding lookup completes it. This establishes the API gap; it does not establish that +every operation yields on every adapter. A local or already available KV result can be +immediately ready. + +No shipped integration at this baseline uses the problematic workaround. The reported +article-signing consumer and the contributor's unsubmitted patch, runtime timings, and +test results have not been independently validated. This is an extension capability, +not a claim that current synchronous integrations are broken. + +The first demonstrated requirement is whole-document text signing: obtain a signature +after reading the article and annotate the page. Such a consumer already withholds the +document. Making its buffering an explicit, bounded core responsibility adds no new +whole-document waiting requirement to that consumer. + +## 3. Goals + +1. Allow a registered HTML integration to await I/O after synchronous document processing + ends, on all four adapters. +2. Make buffer ownership, limits, cancellation, and unchanged-byte fallback enforceable + by core rather than implicit integration conventions. +3. Preserve early streaming, chunk boundaries, processor EOF counts, and auction-seam + timing when no finalizer is enabled. +4. Support both `EdgeBody::Once` and `EdgeBody::Stream`, with and without an auction. +5. Keep decoder validation ahead of finalization, and auction-seam replacement and encoder + finalization after it on the opted-in path. +6. Prevent warm shared-cache hits or downstream response caches from bypassing required + per-request finalization. +7. Specify observable, deterministic failure behavior and tests before implementation. + +## 4. Non-goals + +- Implementing an article-signing integration, an external API protocol, signing-key + management, or a new KV binding. +- Making `lol_html` callbacks await, making every chunk method asynchronous, or inserting + asynchronous processors between arbitrary synchronous processors. +- Adding another framework HTML parser or reparsing the finalizer's output. +- Changing the standalone RSC Flight processor, asset rewriting, EC behavior, consent + gates, or auction admission and transport guarantees. +- Guaranteeing interruption of synchronous Fastly host calls or CPU-heavy callback work. +- Guaranteeing cancellation or rollback of remote HTTP/KV side effects. +- Allowing finalizers to edit HTTP headers or alter the assembly protocol. +- Supporting shared-template storage of asynchronous output in the first version. +- Adding deployment manifests, a general task scheduler, background retries, persistent + queues, or a new platform clock abstraction. + +## 5. Alternatives and tradeoffs + +| Approach | Benefit | Cost or limitation | Decision | +| ------------------------------------------------ | --------------------------------------------------------------------------------------------------------------------------- | ---------------------------------------------------------------------------------------------------------------------------------------------- | ---------------------------------------------------- | +| Optional async finish on every `StreamProcessor` | Supports asynchronous completion inside a mixed streaming chain | Generic synchronous drivers cannot await it; downstream EOF propagation, wrappers, holding behavior, and services become broader contracts | Defer until an actual consumer requires interleaving | +| Make `process_chunk` asynchronous | Supports waits at every chunk boundary | Changes every processor and driver for an EOF-only need; boxed async trait methods can allocate per chunk; parser callbacks still cannot await | Reject | +| Explicit asynchronous HTML terminal phase | Matches the known whole-document consumer; leaves existing streaming traits intact; core can own and bound fallback buffers | Enabled pages wait for EOF; cannot interleave with synchronous stages | Select | +| Perform integration I/O before HTML rewriting | Produces metadata before transformations | Still requires buffering enabled pages and delays parsing; lacks the completed transformed document | Reject for this requirement | + +Opt-in buffering does not mean buffering every page. A registry without finalizers keeps +the current path. Within an opted-in page, all existing synchronous processors run first, +in their existing order, and receive exactly one EOF notification. Finalizers then run in +their own order. Registration order does not imply interleaving between these two phases. + +The framework can provide completed HTML and existing document-local capture state without +adding a parser. The consumer must separately explain how it extracts text and preserves +HTML semantics; this specification does not validate its extraction implementation. + +## 6. Ownership and registration + +### 6.1 Integration registration + +Add `IntegrationHtmlFinalizerFactory` and `IntegrationHtmlFinalizer` contracts alongside +the existing integration factory contracts. Add an empty-by-default finalizer collection +to `IntegrationRegistration` and `IntegrationRegistry`, plus the builder method +`with_html_finalizer`. Preserve factory registration order; do not sort finalizers by ID +or deduplicate registrations silently. + +A factory is `Send + Sync` and can live in the retained application registry. A factory +creates a mutable, single-use finalizer instance for each processed document using the +existing owned `IntegrationHtmlStreamContext`, including its `IntegrationDocumentState`. +The finalizer instance and its returned future need not be `Send`. + +The same document-state instance must be available to parser hooks, synchronous streaming +processors, and finalizers for that document. It must not be recreated between phases or +retained across requests, including reusable Fastly sandboxes. + +Currently `create_html_processor` constructs that state internally. The implementation +must expose the same state through a request-local construction helper or an explicit +construction input; creating a second state for finalizers would disconnect their capture +data from the parser. Preserve the current public helper's synchronous behavior. + +Finalizer presence must be discoverable from enabled registration metadata before shared +template lookup. It must not depend on polling a future, inspecting completed HTML, or +creating a finalizer instance. A factory may later return `Unchanged` when no relevant +article is present, but that cannot retroactively authorize a shared-cache lookup. + +Disabled integrations contribute no finalizers and cause no finalizer-related buffering, +privacy changes, timer creation, or cache bypass. No global setting enables this feature +without an integration explicitly registering a finalizer. + +### 6.2 Callback contract + +Use a regular, object-safe method returning `LocalBoxFuture`, with an explicit lifetime +covering mutable session state and borrowed inputs. Do not add native `async fn` to a trait +used behind `dyn` without an object-safety strategy, and do not require `Send` futures. + +The following is the intended API shape, not implementation code: + +```rust +pub trait IntegrationHtmlFinalizer { + fn finalize<'a>( + &'a mut self, + input: HtmlFinalizationInput<'a>, + ) -> LocalBoxFuture<'a, Result>>; +} + +pub struct HtmlFinalizationInput<'a> { + pub response_status: StatusCode, + pub html: &'a [u8], + pub replacement: &'a mut HtmlReplacementWriter, + pub services: &'a RuntimeServices, + pub deadline: &'a HtmlFinalizationDeadline, +} + +pub enum HtmlFinalization { + Unchanged, + Replace, +} +``` + +`Unchanged` emits the driver-owned input without cloning it. `Replace` commits only bytes +successfully written into `HtmlReplacementWriter`. Neither the original buffer nor an +unbounded replacement `Vec` is transferred into the future. + +The method must return its future promptly and perform I/O inside that future. Factory +creation is synchronous, request-local setup; it must not perform an outside lookup or +hide whole-document buffering. Public implementations must follow repository documentation +and error conventions; `HtmlFinalizationError` is a concrete error type with `Report`. + +The replacement writer is a core-owned sink, separate from `html`. Its append operations +are fallible and enforce the byte cap before extending storage. A failed append poisons +that attempt: a later `Replace` cannot commit truncated output even if the callback ignored +the writer error. Empty replacement output is valid only when no required seam token would +be lost; `Unchanged` is the explicit no-replacement result. + +The framework does not make arbitrary integration allocations safe. The cap covers its +retained HTML and replacement storage. Consumer extraction buffers and any external +response buffering must have their own bounds and be reviewed. + +## 7. Processing lifecycle + +### 7.1 No finalizers + +```text +origin -> existing decoder -> existing HTML parser and synchronous processors + -> existing auction seam controller -> existing encoder -> client +``` + +Do not collect a whole document, allocate a replacement writer or boxed finish future, +create a timer, or move auction collection to EOF. Keep existing synchronous helper +semantics. A cheap registration-presence check is acceptable. + +### 7.2 Enabled finalizers + +```text +origin -> existing decoder -> existing HTML parser and synchronous processors + -> bounded retained HTML + -> ordered asynchronous HTML finalizers + -> existing auction seam controller -> existing encoder -> client +``` + +The driver performs these steps: + +1. Create normal document-local parser/processor state and finalizer sessions. +2. Pull and decode origin chunks, run synchronous transformations, and retain the resulting + HTML under the cap. No finalizer-visible HTML is emitted before this phase completes. +3. Drain and validate decoder tails, then finish the existing synchronous chain exactly + once. Include its final output in the retained HTML. +4. Start one absolute finalization deadline and, on supporting adapters, one timer. +5. For each finalizer, lend the current retained HTML immutably and a fresh bounded + replacement writer mutably. Race its future against the same remaining deadline. +6. For an accepted `Replace`, validate the candidate and move it into current retained + storage. Drop the previous stage's input. For `Unchanged` or a soft failure, keep the + current input and discard the candidate. Finalizer B sees the accepted output of A. +7. On deadline expiry, drop the active future and candidate, skip the remaining finalizers, + and retain the current input. Earlier accepted replacements remain accepted. +8. Feed the accepted HTML into the existing auction seam controller in bounded chunks. + Emit its available prefix before collecting bids. Preserve no-seam fallback and terminal + auction telemetry. Then emit the encoder trailer and finish the response. + +Already completed synchronous processors are never re-entered with replacement output. +This prevents duplicate parsing, script injection, Next.js processing, and EOF calls. + +Callbacks observe post-rewrite HTML before reader-specific auction insertion. They cannot +claim to sign the final assembled response or content added later by the auction. A text +signing consumer must document its article scope, exclusions, and canonicalization so +framework scripts, placeholders, and later ad content do not invalidate its claims. + +### 7.3 Auction behavior + +Auction dispatch and its existing deadline remain unchanged. They are not restarted or +extended for finalization. Full-document withholding can make an opted-in page's auction +older by the time its seam reaches collection; an expired auction follows today's +empty-bid behavior. This is an explicit latency/monetization tradeoff for enabled pages. + +When no finalizer is enabled, retain the existing ability to collect at the structural +body seam before origin EOF. The terminal phase must not become a universal EOF barrier. +Missing structural body ends, bot/prefetch/consent suppression, non-HTML bodies, and +abandonment remain governed by existing auction policy. + +### 7.4 Cancellation + +The driver must own current HTML outside the pending finalizer future. Keep a losing +future's lifetime separate from the retained buffer's ownership; explicitly drop it before +moving that buffer or inspecting the borrowed replacement writer. + +Reusing a single deadline across finalizers must not put the only fallback buffer inside +a chain future that is destroyed on timeout. The driver remains the owner at every await. + +A client disconnect drops active work and request-local buffers. It does not invoke another +finalizer, persist partial output, or initiate a background retry. Preserve the existing +dispatched-auction guard so terminal abandonment is recorded exactly once on supported +drop paths. Dropping local work is not a guarantee that a remote write did not occur. + +## 8. Memory and output limits + +Use `publisher.max_buffered_body_bytes` as the cap for both retained transformed HTML and +each replacement writer. Its existing default is 16 MiB. Do not add a second independently +configured page-size limit in this change. + +Check lengths with overflow-safe arithmetic before append or reserve. Avoid capacity +growth beyond the configured cap, including `Vec` geometric reserve growth. Allocate +incrementally; do not reserve the entire cap for every small page. + +At a finalizer await, core may hold one current HTML buffer and one replacement buffer, +each capped at that limit. It must not retain previous successful candidates or clone the +input for rollback. Codec state, pending chunks, consumer-owned state, and platform response +buffers are additional memory; this is not a claim that total request heap use is 16 MiB. + +If collection for the optional terminal phase would exceed the cap: + +- Disable all finalizers for that document before starting I/O. +- On a streaming sink, release the retained prefix, the chunk that crossed the limit, and + subsequent synchronous output through the normal auction/encoding path in order. Avoid + duplicate bytes and a fallback copy of the retained prefix. +- On a buffered sink, preserve the existing outer response-size cap. Optional-finalizer + bypass does not authorize an oversized buffered response. +- Existing raw-input, decoder, parser, or response-output limit failures remain fatal. + This fallback applies only when the new optional collection layer reaches its own cap. + +A replacement-writer overflow is a soft failure of that finalizer. Discard the candidate +and continue with its input, subject to the same remaining finish budget. Never raise the +cap to accept an integration's output. + +## 9. Deadline and adapter services + +### 9.1 Configuration + +Add `publisher.html_finalization_timeout_ms`, defaulting to `5000`. It covers only the +terminal phase after decoded/synchronously processed EOF, across all finalizers together. +It is not an origin-body timeout, auction timeout, or end-to-end response deadline. + +Accept values from `1` through `60000` inclusive and reject invalid settings at startup. +The upper bound keeps the adapter timer duration comfortably within its representable +range and gives the new optional phase a finite operator-controlled budget. Match TOML +deserialization and programmatic `Publisher::default()` behavior. The field has no runtime +effect when no finalizer is registered. + +The five-second default is a proposed product default based on the issue's example, not a +claim that all adapter execution budgets permit or guarantee five seconds. Operators must +also respect their platform's invocation limits. + +### 9.2 Timer contract + +Add an optional `PlatformTimer` service to `RuntimeServices`, defaulting to absent. The +service object is `Send + Sync`; its delay future is local and need not be `Send`. A regular +object-safe method can return `LocalBoxFuture<'static, ()>` for a `Duration`. + +| Adapter | Timer implementation | Finish deadline semantics | +| ---------- | ----------------------- | ----------------------------------------------------------------------------------- | +| Cloudflare | `worker::Delay` | Cooperative race against JS-backed I/O | +| Spin | `spin_sdk::time::sleep` | Cooperative race against runtime-backed I/O | +| Axum | `tokio::time::sleep` | Cooperative race on the existing Tokio executor | +| Fastly | No timer | Check elapsed budget at control boundaries; bound host calls within the integration | + +Keep SDK imports and concrete timer implementations in Trusted Server adapters. These +primitives exist in the pinned dependencies; do not add a Tokio dependency to core or +repin EdgeZero to obtain a timer. + +Use `web_time::Instant` for absolute budget accounting and elapsed observations in core. +Do not introduce `std::time::Instant::now()` or `SystemTime::now()` on the Cloudflare path. +The review hardening exception corrects the traced auction telemetry clock using +that existing abstraction. Broader clock sites reported in #1240 remain separate +work; one successful auction smoke does not verify all of them. + +### 9.3 Race and fallback rules + +Create one absolute deadline and retain one pinned timer across the entire chain. Poll +the deadline before active work at decision points. Check absolute expiry before a stage +starts and after it resolves; if completion and expiry coincide, deadline expiry wins. +Do not restart a full-duration sleep for each finalizer. + +Callbacks receive a deadline handle exposing remaining time and whether a cooperative +timer is available. Bound individual network requests and external response reads to the +remaining budget where the service permits it. Successful but late candidates are +discarded even if a timer could not run while the callback was polling. + +Timers cannot preempt a blocking host call or a callback that never yields. On Fastly, +consumer acceptance requires evidence that its calls are bounded independently, or that +it returns `Unchanged` without performing unsupported unbounded operations. Core cannot +promise a hard five-second release for arbitrary Fastly HTTP/KV operations. + +Cancellation frees borrowed buffers and releases the publisher's wait. It does not +universally abort transport work or reverse external writes. Consumers must tolerate +duplicate requests/retries and use idempotent external operations where required. + +## 10. Failure and validation contract + +Fallback bytes are the synchronously transformed input of the current finalizer, including +accepted earlier finalizers. They are not necessarily the origin's original bytes. No +rollback of prior accepted HTML or remote side effects is promised. + +| Event | Document behavior | Remaining finalizers | +| ----------------------------------------------------------------------------- | ---------------------------------------------------------------- | ------------------------------- | +| `Unchanged` | Keep current input; discard any candidate writes | Continue | +| Valid `Replace` within budget | Commit bounded candidate | Continue with replacement | +| Callback reports unavailable/rejected external work | Keep current input; discard candidate | Continue while budget remains | +| Replacement writer overflow or invalid required marker | Keep current input; discard candidate | Continue while budget remains | +| Overall deadline expires | Drop active future and candidate; keep current input | Skip all remaining | +| Optional collection limit reached | Release retained synchronous output through ordinary delivery | Skip all finalizers | +| Origin read, decoder, parser, synchronous processor, encoder, or sink failure | Existing fatal error/truncation behavior and auction abandonment | Stop | +| Callback panics | Existing panic behavior | No new panic-catching mechanism | + +Define controlled error categories for external failure and candidate validation. Keep +`Report` at orchestration boundaries and existing `io::Error` where +streaming I/O uses it. Do not turn unrelated processor errors into silent passthrough. + +Finalizers must preserve the parser-generated control marker, when present, as opaque +bytes. They must not remove, duplicate, or relocate it into script/text content. Core +checks the active token's required cardinality before committing a replacement; missing +or duplicated tokens reject that candidate. A count check cannot prove structural +placement. This remaining invariant is a reviewed integration contract, not a claim of +automatic HTML validation without parsing. + +Core does not inspect arbitrary replacement HTML for semantic validity or reconstruct +parser observations. Finalizers must preserve valid document structure, existing CSP +requirements, URL/RSC encodings, and the meaning of upstream rewrite decisions. Header +changes and transformations needing new parser-derived security observations require a +separate capability. The first consumer should add signature metadata without changing +executable markup or article text after canonicalization. + +## 11. Cache, privacy, conditional requests, and framing + +### 11.1 Shared-template cache + +When enabled finalizer metadata is present for the request's document pipeline, disallow +shared-template lookup/reservation before the first cache read. The same decision must +exclude later storage. A factory deciding `Unchanged` at EOF does not restore eligibility. + +Use the existing non-shared inline fallback consistently across slot scripts, body-seam +selection, transformation, and assembly. Do not generate a shared-template marker for a +response that has no shared-template consumer. Record the existing request-bypass outcome +where cache response-state reporting is enabled. + +Warm entries may remain stored but must not be read while the finalizer is enabled. No +purge is required merely to activate it. Disabling an integration can restore ordinary +eligibility, subject to existing settings/build fingerprints and template validation. + +There is no first-version `shared_safe` escape hatch. Later cache support must specify at +least reader-neutral output, external-data/signing-key identity, expiration, invalidation, +and whether failures can be cached. Cold-miss ordering alone does not settle these rules. + +### 11.2 Downstream privacy and origin readthrough + +Apply the existing synthesized HTML `private, no-store` policy to body-carrying HTML +responses that enter the enabled finalizer pipeline, regardless of auction eligibility or +the finalizer outcome. Strip validators and stale downstream cache directives using the +same existing helper as other synthesized HTML. Apply this before streaming headers are +committed. An unchanged or size-bypassed result remains private for that response. + +This prevents a browser or downstream cache from treating a per-request result as the +origin's public representation. Origin readthrough remains a separate raw-byte cache: it +may still provide a complete shareable origin body under existing policy, because every +request subsequently runs transformation/finalization. Do not cache assembled output as +an origin body or modify EC-cookie privacy rules. + +For document GETs requiring finalization, reuse +`strip_conditional_and_range_headers` so client revalidation or range requests cannot +make the origin return a bodyless `304` or partial `206` instead of the full document. +That helper removes `If-None-Match`, `If-Modified-Since`, `Range`, and `If-Range`. +Do not discard mutation preconditions such as `If-Match` or `If-Unmodified-Since`, +or broaden GET behavior to other methods. Unexpected bodyless responses retain existing +status/error handling and invoke no finalizer, including the existing invalid-origin-304 +guard when the ad stack is active. This feature does not turn an existing 502 into a 304. + +### 11.3 Response selection and framing + +Run finalizers only on complete HTML that the existing routing rules already process, including +processable non-2xx HTML. Expose the actual response status so consumers can choose +which complete documents to process. With enabled finalizers, HTML with status 206 +or any `Content-Range` bypasses all body transformations and terminal I/O. Preserve +encoded bytes, range/framing metadata, and origin policy; independent cookie and +diagnostics privacy still applies. This covers range-associated error responses +such as 416 as well as partial representations. The handler and both response +wrappers must make this decision before body-dependent policy or framing mutation; +direct body helpers carry the same private metadata. Abandon dispatched bids once. +Keep non-document range forwarding intact and feature-off behavior unchanged. + +Do not rewrite unsupported encodings, unmodified pass-through +responses, assets, or standalone `text/x-component` bodies. + +HEAD and `204`, `205`, or `304` responses invoke no finalizer or finalizer I/O. Preserve +their existing framing and any dispatched-auction abandonment. Preserve method/status +handling for other requests instead of introducing a GET-only body transformer silently. + +Finalizers do not change the selected content encoding. Strip stale `Content-Length` as +the existing transformed-body path requires; stream encoders finalize only after accepted +HTML and auction output are emitted. Preserve repeated headers, cookies, and CSP policy. + +## 12. Integration into publisher paths + +The first implementation must deliberately cover these existing branches: + +| Source and sink | Auction | Required integration | +| ----------------------------------------------- | ------- | ------------------------------------------------------------------------------------- | +| Lazy response stream, `Once` or `Stream` source | None | Retain opted-in synchronous HTML output, await finalizers, then encode | +| Lazy response stream, `Once` or `Stream` source | Pending | Same phase before existing seam collection; retain auction guard | +| Buffered response, streamed source | None | Same terminal phase before completing bounded response output | +| Buffered response, streamed source | Pending | Same phase before seam/encoder finish | +| Buffered response, materialized source | None | Opted-in path cannot fall through `stream_publisher_body` and skip finalization | +| Buffered response, materialized source | Pending | Reader-based hold path must route opted-in HTML through the same terminal coordinator | + +Introduce one private HTML terminal coordinator shared by the enabled paths. Feed it the +same synchronous transformation semantics on materialized and streamed inputs. Keep +ordinary no-finalizer branches intact; avoid replacing the entire generic compression +architecture to obtain this capability. + +Do not add finalizers to a generic synchronous helper that cannot await them. Public +synchronous helpers continue to implement synchronous transformations only; document that +HTML finalizers require the supported publisher finalization entry points. Adapters must +never select a synchronous-only path for a registered finalizer silently. + +Thread document state and finalizer metadata explicitly through the relevant publisher +parameters. Do not reconstruct a separate registry/session inside a lazy stream or copy +request services into global state. Reuse existing services ownership for request-lifetime +and lazy response futures. + +## 13. Observability + +Use existing `log` instrumentation with integration ID, controlled outcome, elapsed time, +and bounded byte counts. Log ordinary completion/unchanged results at debug level and soft +failure, timeout, or optional-collection bypass at warn level. Emit at most one outcome +per attempted finalizer and one terminal collection/deadline bypass for the document. + +Never log HTML, article text, signatures, external response bodies, control tokens, client +identifiers, secrets, URLs containing request data, or arbitrary error-report attachments. +Log controlled categories instead of formatting an integration-supplied report wholesale. + +No new public response headers, persistent telemetry store, or metrics subsystem is needed. +Existing auction terminal telemetry retains its meanings and exactly-once behavior. + +## 14. Verification requirements + +The checks below are acceptance requirements for the implementation, not claims that the +unsubmitted feature has already passed them. + +### 14.1 API, registration, and state + +- Default registration has no finalizers; a disabled integration contributes none. +- Factory and finalizer ordering is deterministic, including multiple factories for one ID. +- Sessions and document capture state are isolated between requests and reused sandboxes. +- A non-`Send` future borrowing input, replacement writer, and session state compiles and + executes through each supported publisher entry point. +- Existing processors retain chunk behavior and receive EOF once before any finalizer. +- Synchronous helpers retain their documented scope; no adapter silently skips a finalizer. + +### 14.2 Real yielding, fallback, and deadlines + +- A stub that returns `Pending` at least once completes when awaited and changes output. +- Finalizer A's accepted output is exactly B's input; B never receives partial candidates. +- A timeout after partial candidate writes drops the future, emits exact input bytes, and + invokes no later finalizer. Use a drop observer and allocation/buffer identity assertions + to prove no fallback document clone. +- `Unchanged`, callback error, invalid marker, replacement overflow, and a late result + preserve exact stage input and discard partial candidate state. +- Two or more delayed finalizers share one deadline; the budget does not multiply. +- Deadline/completion ties follow the documented timeout-wins rule. Use an injectable + deterministic timer/clock in tests rather than flaky wall-clock sleeps. +- Client cancellation drops active work and retains exactly-once auction abandonment. +- Fastly tests demonstrate no timer dependency, budget checks, and bounded-call consumer + behavior; they do not pretend a synchronous call can be preempted. + +### 14.3 Bounds and byte ordering + +- Test cap minus one, exact cap, cap plus one, and length-overflow rejection. +- Test multiple chunks and one large chunk crossing the optional collection cap. Prefix, + crossing chunk, and suffix emerge once in order on the streaming bypass path. +- Buffered output caps and existing decoder/input caps retain their existing failures. +- Replacement sink overflow cannot commit even when a callback ignores its write error. +- Input and replacement capacities remain bounded; no full-cap allocation for tiny pages. +- Accepted prior stages are released as ownership advances; timeout does not restore the + origin or undo an earlier successful finalizer. + +### 14.4 Publisher and codec matrix + +Exercise `Once` and `Stream` sources, lazy and buffered sinks, auction present/absent, +and identity, gzip, deflate, and Brotli encoding. Include concatenated gzip members, +decoder tails released only at EOF, truncated/corrupt streams, empty HTML, split UTF-8, +missing body ends, and malformed-but-renderable HTML. + +Verify decoded/synchronous EOF precedes finalizer I/O, candidates preserve private marker +cardinality, ready prefixes precede auction collection, and encoder trailers follow all +accepted bytes. Finalizer failure must not leave a raw private marker in client output. + +Retain the current tests for early compressed output, parser-confirmed Next.js seams, +placeholder substitution, body-source cancellation, and auction abandonment. Those tests +must continue to use the no-finalizer path and protect its existing streaming behavior. + +### 14.5 Cache, headers, and eligibility + +- Populate a template entry with finalization disabled, enable a finalizer, and prove the + request bypasses the warm entry and runs its finalizer. Assert no lookup reservation or + store is made; cover ESI configuration falling back to inline. +- Disable the finalizer and verify ordinary cache miss/store/hit behavior is restored. +- Responses remain `private, no-store` for success, `Unchanged`, timeout, external failure, + and collection bypass, including HTML without an auction or matched ad slot. +- Raw origin readthrough hits still execute finalizers per request; no assembled output + enters the raw origin cache. +- Test synthesized GET revalidation/range handling and preserved mutation preconditions, + including unexpected origin 304 behavior with and without an active ad stack. +- Verify non-2xx HTML is processed, while HEAD, bodiless statuses, non-HTML, unsupported + encodings, and unmodified responses make no finalizer call. Preserve framing and cookies. +- Existing shared-template validation, assembly, CSP nonce, and cache-privacy tests pass + unchanged with no finalizer enabled. + +### 14.6 Actual adapters and repository gates + +Native Cloudflare/Spin tests cannot establish JS or WASI timer behavior. Add fictional +fixtures and run a bounded delayed/hung lookup on Viceroy, local Workers, Spin, and Axum. +Record the exact setup, SDK/runtime versions, observed behavior, and limitations. Do not +claim a universal exact wall-clock release; cooperative scheduling adds tolerance. + +Review hardening permits one targeted correction under #1240: auction telemetry +uses `web_time::Instant`, consistent with the orchestrator, because the captured +Workers stack names that unsupported standard-clock call. Verify the actual +Workers auction control after the correction. Broader clock paths remain separate; +do not claim all of #1240 resolved from one successful fixture. Keep KV wiring under +#1139 separate; HTTP-backed and unavailable-KV fixtures can verify this lifecycle without +requiring new mandatory store bindings. + +Before implementation PR handoff, run every CI gate in [AGENTS.md](../../../AGENTS.md#ci-gates), +including target-matched adapter checks/tests, parity, JS/docs formatting and tests, and +native build-digest checks. Run existing template-cache ESI/inline harnesses as appropriate. +Use only fictional fixture hosts, text, configuration, and credentials. + +## 15. Implementation scope and delivery + +Expected production touch points: + +- Core integration registration: terminal factory/session contracts and presence metadata. +- Core HTML/publisher processing: shared document state, bounded collection/replacement, + terminal coordinator, path selection, cache/conditional/privacy integration, and errors. +- Core platform services: optional timer and deadline helper using existing clock support. +- Core settings: the finish timeout default/validation and canonical configuration docs. +- Adapter service construction: Cloudflare, Spin, and Axum timer implementations; Fastly + explicitly retains no cooperative timer. +- Tests, integration fixtures, example configuration, and canonical integration/adapter + guides documenting opt-in behavior and limitations. + +Keep the generic streaming trait, synchronous pipeline, JS runtime, platform manifests, +and EdgeZero pin unchanged. Do not bundle the real signing consumer, KV-store wiring, +broader Cloudflare clock fixes, or unrelated publisher refactoring into this feature. +The narrowly traced auction telemetry clock correction above is the reviewed exception. + +The implementation plan should separate contract/registration work, terminal coordination, +adapter timers, cache/privacy behavior, and verification into small reviewable changes, +while not shipping an intermediate state in which enabled finalizers can be skipped. + +This document authorizes no deployment, external API use, or publication. After maintainer +review, write the implementation plan separately. The current branch contains design work +only. + +## 16. Acceptance criteria + +The feature is ready only when: + +1. A yielding finalizer works through every supported publisher response path. +2. No-finalizer output, streaming timing, auction sequencing, and error handling retain + existing behavior under their regression tests. +3. Fallback is exact at the failed stage, candidate writes are bounded and isolated, and + timeout drops the active future without cloning the retained document. +4. The deadline covers the entire terminal chain, with Fastly's non-preemption limitation + explicitly tested and documented. +5. Shared-template reads and writes cannot bypass or persist the new per-request phase; + downstream privacy and origin readthrough remain correctly separated. +6. No unsupported body, method/status, encoding, or synchronous-only adapter path silently + invokes or skips the new capability. +7. Real runtime evidence and all required repository gates are recorded with honest limits. + +## 17. References and investigation record + +- Issue #1241: requested asynchronous completion for HTML stream processors. +- PR #1135 and the [parser-aware body hold design](2026-09-07-850-parser-aware-body-hold-nextjs-streaming-design.md): + synchronous stream processors and parser-generated auction seam. +- Issues #1240 and #1139: related clock and runtime KV findings; independent scope. +- [Streaming processor](../../../crates/trusted-server-core/src/streaming_processor.rs), + [HTML processor](../../../crates/trusted-server-core/src/html_processor.rs), and + [integration registry](../../../crates/trusted-server-core/src/integrations/registry.rs). +- [Publisher](../../../crates/trusted-server-core/src/publisher.rs), + [runtime services](../../../crates/trusted-server-core/src/platform/types.rs), + [template cache contract](../../../crates/trusted-server-core/src/platform/template_cache.rs), + and [publisher settings](../../../crates/trusted-server-core/src/settings.rs). + +During investigation at the baseline, the following native core test filters passed: + +```text +cargo test -p trusted-server-core --lib streaming_processor::tests --offline +34 passed +cargo test -p trusted-server-core --lib html_processor::tests --offline +32 passed +cargo test -p trusted-server-core --lib publisher::tests --offline +369 passed +``` + +Two temporary probe tests also passed: the current synchronous pipeline discards a +yielding one-poll lookup, and awaiting the same lookup completes it. These are investigation +evidence, not implementation regression tests or proof of the contributor's runtime claims. diff --git a/docs/superpowers/archive/2026-10-12-1241-buffered-terminal-plan.md b/docs/superpowers/archive/2026-10-12-1241-buffered-terminal-plan.md new file mode 100644 index 000000000..97e82cfb0 --- /dev/null +++ b/docs/superpowers/archive/2026-10-12-1241-buffered-terminal-plan.md @@ -0,0 +1,686 @@ +# Optional asynchronous HTML finalization implementation plan + +> Historical record: this plan describes the buffered terminal API implemented on +> 2026-10-12. The [revised plan](../plans/2026-10-12-1241-async-html-finalization.md) +> governs the ordered EOF revision. Its test counts do not verify the revision. + +> **For agentic workers:** REQUIRED SUB-SKILL: Use superpowers:subagent-driven-development (recommended) or superpowers:executing-plans to implement this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking. + +**Goal:** Allow explicitly registered HTML integrations to await I/O after synchronous document processing, with bounded core-owned buffers and exact fallback on optional failure. + +**Architecture:** Add an HTML-only terminal phase after existing synchronous processors and before auction-seam handling and encoding. One coordinator owns retained HTML, bounded replacement storage, and the chain deadline; the no-finalizer route remains unchanged. Enabled finalizers bypass shared-template caching before lookup and use synthesized HTML privacy rules. + +**Tech Stack:** Rust 2024, existing `futures` local futures, `error-stack`, `derive_more`, `web_time`, `lol_html`, existing compression codecs, and adapter-local Workers/Spin/Tokio timers. + +--- + +**Status:** Implemented; see the execution record and review hardening for current verification + +**Date:** 2026-10-12 + +**Branch:** `feat/1241-async-html-finalization` (planning originated on `spec/1241-async-html-finalization`) + +**Baseline:** `2a0cb0444dc335bd09105abe5952329bbbdfb621` + +**Design:** [Optional asynchronous HTML document finalization](2026-10-12-1241-buffered-terminal-design.md) + +**Execution record:** [Implementation and bidirectional coverage review](../reviews/2026-10-12-1241-async-html-finalization-implementation.md) + +The task checklists below preserve the original execution instructions, including +proposed intermediate commit boundaries. The execution record is the authoritative +completion record: implementation was reviewed together before committing, and it +records actual commands, deviations, and the initially blocked Workers auction control. + +The [review hardening addendum](../plans/2026-10-12-1241-review-hardening.md) records the +subsequently authorized response-status/range refinements, telemetry clock +correction, runtime CI coverage, and renewed verification. It supplements the +original task instructions below. + +## 1. Execution boundaries + +This plan implements the selected terminal-phase design, not the original proposal to add +`async_finish` to `StreamProcessor`. Read the spec before implementing. Use +@superpowers:test-driven-development for runtime changes and +@superpowers:verification-before-completion before recording results or committing. + +The tasks below were the implementation instructions. Their expected outcomes alone +are not verification evidence; consult the execution record for observed results. Keep intermediate +commits in a draft branch; do not release an API that registrations can enable before all +publisher entry points support it. + +Keep the EdgeZero revision, generic streaming traits, JS runtime, and production platform +manifests unchanged. KV wiring under #1139, broader clock fixes under #1240, and a real signing +integration are separate work. Do not mask a live-runtime failure by replacing actual +SDK services with stubs or by reporting a native test as a runtime smoke. + +## 2. File responsibilities + +| File | Responsibility | +| --------------------------------------------------------------------------------------------------------------- | -------------------------------------------------------------------------------------------------------------- | +| Create `crates/trusted-server-core/src/integrations/html_finalization.rs` | Public factory/session/input/outcome/error contracts, bounded replacement writer, and finish deadline handle | +| Modify `crates/trusted-server-core/src/integrations/mod.rs` | Export the new contracts; later register a feature-gated fictional fixture | +| Modify `crates/trusted-server-core/src/integrations/registry.rs` | Empty-by-default finalizer metadata, builder method, ordered factory access, and test construction seam | +| Modify `crates/trusted-server-core/src/html_processor.rs` | Share one document-state instance through parser, synchronous processors, and finalizer construction | +| Create `crates/trusted-server-core/src/platform/timer.rs` | Optional platform-neutral timer contract | +| Modify `crates/trusted-server-core/src/platform/mod.rs` and `types.rs` | Timer exports, default-absent runtime service, builder/accessor support | +| Modify `crates/trusted-server-core/src/settings.rs` | Global terminal-phase timeout, matching defaults, and validation | +| Create `crates/trusted-server-core/src/publisher/html_finalization.rs` | Private retained-document collector, ordered async driver, candidate validation, and bounded replay | +| Modify `crates/trusted-server-core/src/publisher.rs` | Route opted-in HTML through the coordinator on every body/sink path; early cache/privacy/conditional decisions | +| Modify adapter `src/platform.rs` files and Fastly `src/app.rs` tests | Concrete timers and explicit Fastly timer absence | +| Create `crates/trusted-server-core/src/test_support/async_html_finalization.rs`; modify `src/test_support.rs` | Fictional integration compiled only under existing `test-utils` | +| Modify `crates/trusted-server-integration-tests/tests/parity.rs` | Host-adapter finalizer behavior and privacy parity | +| Create `scripts/fixtures/async-html-finalization-origin.py` and `scripts/async-html-finalization-local-test.sh` | Delayed/hung local HTTP lookup and actual-runtime smoke driver | +| Modify `trusted-server.example.toml` and canonical guide files | Explain the timeout, registration API, buffer/cache effects, and adapter limitations | + +Do not split or reorganize unrelated sections of `publisher.rs`. The focused child module +contains the new state machine; existing decoding, auction collection, telemetry guards, +and framing helpers remain authoritative. + +## 3. Dependency order and verification rhythm + +Tasks 1–4 establish dormant contracts and services. Task 5 implements asynchronous execution; +Task 6 implements bounded collection/replay. Task 7 establishes request-time policy before +Task 8 enables production routing. Tasks 9–11 verify codecs, adapter paths, and real runtimes. +Tasks 12–13 finish documentation and the complete gate run. + +Use native core filters for fast feedback, then the target-matched gates associated with +the touched adapters. Before each commit, run formatting, the task's tests, and its applicable +lint gate. Before PR handoff, run the entire gate list in Task 13. An unavailable runtime +is a reported verification gap, not permission to mark the check passing. + +For core-only native checkpoints, run +`cargo clippy -p trusted-server-core --all-targets --all-features -- -D warnings`. +An adapter lint alias can compile core as a dependency but does not directly lint it as +a primary package. Retain target-matched Fastly/core and adapter gates where listed. + +### Task 1: Define the terminal callback and bounded replacement writer + +**Spec coverage:** Sections 6.2, 8, 9.3, 10, and 16.3. + +**Files:** Create `crates/trusted-server-core/src/integrations/html_finalization.rs`; +modify `crates/trusted-server-core/src/integrations/mod.rs`. + +- [ ] Write failing unit tests in the new module for exact-cap append, cap overflow, + overflow-safe length checking, bounded capacity growth, poisoned-writer rejection, + and explicit `Unchanged` versus empty `Replace` outcomes. +- [ ] Define/export the spec's object-safe factory/session/input/outcome/error types. + The factory exposes `integration_id()` and `create(IntegrationHtmlStreamContext)`; + it is `Send + Sync`. A session is single-use and returns a local borrowing future. + Define `HtmlFinalizationDeadline` in this same task so the input contract compiles: + use `web_time::Instant`, a timeout `Duration`, and cooperative-timer availability; + expose remaining time and keep expiry-at-explicit-time helpers private for tests. + Cover before/exactly-at/after expiry and saturating zero remaining time. + +The callback signature is: + +```rust +pub trait IntegrationHtmlFinalizer { + fn finalize<'a>( + &'a mut self, + input: HtmlFinalizationInput<'a>, + ) -> LocalBoxFuture<'a, Result>>; +} +``` + +The input fields and result variants must match spec section 6.2 exactly. Keep the retained +HTML immutable. The writer exposes fallible `append(&[u8])`; only core can consume it to +commit a candidate. Use a concrete `derive_more::Display` error and `core::error::Error`. + +- [ ] Implement capped append: compute the next length with `checked_add`; poison and + return a controlled limit error before mutation on overflow or excess. For growth, + use an amortized capacity target bounded by the cap and reserve exactly that target. + Do not allocate the full cap up front or accept an arbitrary replacement `Vec`. +- [ ] Run `cargo test -p trusted-server-core --lib integrations::html_finalization`. + Confirm the new tests actually run, reproduce the limit failure before the fix, then pass. + Also run `cargo fmt --all -- --check`, the focused native core clippy command above, + and `cargo clippy-axum`. +- [ ] Commit the contract/writer change as `Add bounded HTML finalizer contracts`. + +The essential overflow regression has this behavior: + +```rust +let mut writer = HtmlReplacementWriter::new(4); +writer.append(b"page").expect("should accept the exact cap"); +writer.append(b"!").expect_err("should reject output beyond the cap"); +assert!(writer.is_poisoned(), "should reject later replacement commitment"); +``` + +Keep writer construction/poison inspection crate-private where only core needs them. Public +items require summaries, intra-doc links, fallible-function error documentation, and a +fictional usage example. + +### Task 2: Add the shared finish budget and optional timer contract + +**Spec coverage:** Sections 9 and 16.4. + +**Files:** Modify `integrations/html_finalization.rs`, `settings.rs`, +`platform/types.rs`, and `platform/mod.rs`; create +`crates/trusted-server-core/src/platform/timer.rs`. + +- [ ] Write failing tests for settings omission/default parity, `1`/`5000`/`60000`, + rejected `0`/`60001`, default-absent timer construction, and services clone/replacement + methods preserving an installed timer. +- [ ] Add the publisher setting with this exact default and validation: + +```rust +#[serde(default = "default_html_finalization_timeout_ms")] +#[validate(range(min = 1, max = 60000))] +pub html_finalization_timeout_ms: u32, + +fn default_html_finalization_timeout_ms() -> u32 { + 5000 +} +``` + +Update the hand-written `Publisher::default()` and only compiler-required publisher +settings literals. Do not modify unrelated OpenRTB types also named `Publisher`. + +- [ ] Add a `Send + Sync` timer trait with `delay(Duration) -> LocalBoxFuture<'static, ()>`. + Add optional `Arc` storage, a builder method, and an accessor to + `RuntimeServices`; missing timer configuration must never cause builder failure. +- [ ] Connect the timer-presence flag to Task 1's deadline type. Keep any deterministic + clock injection private to tests; do not introduce a public platform clock service. + Add settings-to-deadline tests proving the whole chain gets the configured duration, + rather than a duration independently supplied by each finalizer. +- [ ] Run `cargo test -p trusted-server-core --lib html_finalization`, + `cargo test -p trusted-server-core --lib settings::tests`, and + `cargo test -p trusted-server-core --lib platform::tests`. + Expected: new default/validation/deadline tests pass and existing service tests remain green. + Run `cargo fmt --all -- --check`, the focused native core clippy command above, + and `cargo clippy-axum`. +- [ ] Commit as `Add HTML finalization budgets and optional timers`. + +### Task 3: Register finalizers and share the parser's document state + +**Spec coverage:** Sections 6.1, 7.1, 7.2, and 14.1. + +**Files:** Modify `integrations/registry.rs`, `integrations/mod.rs`, and +`html_processor.rs`. + +- [ ] Write failing registry tests for empty defaults, disabled integration exclusion, + multiple finalizers preserving registration order, and presence checks that never + instantiate sessions. Extend the existing single-EOF processor test. +- [ ] Add `.with_html_finalizer(...)`, `has_html_finalizers()`, and ordered factory access. + Update existing hand-written registration/registry literals with empty collections. + Follow the current registry's integration enablement/build order; do not sort by ID. +- [ ] Add a `#[cfg(any(test, feature = "test-utils"))]` registry constructor from finalizer + factories for isolated core tests. Keep it out of ordinary production API builds. +- [ ] Introduce a crate-private construction helper taking an explicit document-state + instance, and keep `create_html_processor(config)` as a synchronous wrapper that + constructs state and delegates. Publisher assembly creates one state and passes clones + of its shared handle to parser hooks, streaming factories, and finalizer factories. +- [ ] Test parser-written state reaching the finalizer, two independent requests, and + retained-registry/reusable-sandbox isolation. Assert synchronous EOF is delivered once + before the first finalizer. No replacement is ever reprocessed by that chain. +- [ ] Run the registry and HTML filters: + `cargo test -p trusted-server-core --lib integrations::registry::tests` and + `cargo test -p trusted-server-core --lib html_processor::tests`. + Run `cargo test-fastly-reuse`, `cargo fmt --all -- --check`, and `cargo clippy-fastly`. +- [ ] Commit as `Register HTML finalizers with shared document state`. + +### Task 4: Wire concrete adapter timers + +**Spec coverage:** Sections 9.2, 9.3, and 14.6. + +**Files:** Modify `crates/trusted-server-adapter-axum/src/platform.rs`, +`crates/trusted-server-adapter-cloudflare/src/platform.rs`, +`crates/trusted-server-adapter-spin/src/platform.rs`, and +`crates/trusted-server-adapter-fastly/src/app.rs` tests. + +- [ ] Add service-construction tests: Axum installs a timer; production Workers and Spin + install their SDK timer under the existing runtime cfg guards; native Workers/Spin + construction remains safe without those SDKs; Fastly installs no timer. +- [ ] Implement stateless timer service objects locally. Return a boxed local future from + `worker::Delay::from(duration)`, `spin_sdk::time::sleep(duration)`, or + `tokio::time::sleep(duration)` as applicable. Delay construction is inside the returned + async future where required to avoid needing a runtime merely to build services. +- [ ] Add the timer to each supporting adapter's runtime-services builder. Preserve config, + secret, KV, HTTP, telemetry, and template service wiring. No manifest or EdgeZero changes. +- [ ] Run `cargo test-axum`, `cargo test-cloudflare`, `cargo test-spin`, and + `cargo test-fastly-reuse`. Native tests prove construction contracts, not SDK behavior. +- [ ] Run all touched adapter lint targets: `cargo clippy-axum`, + `cargo clippy-cloudflare`, `cargo clippy-cloudflare-wasm`, + `cargo clippy-spin-native`, `cargo clippy-spin-wasm`, and `cargo clippy-fastly`. + Expected: runtime cfg separation compiles on every relevant target. +- [ ] Commit as `Wire adapter timers for HTML finalization`. + +### Task 5: Implement ordered, cancellation-safe terminal execution + +**Spec coverage:** Sections 7.2–7.4, 9.3, 10, 13, and 14.2. + +**Files:** Create `crates/trusted-server-core/src/publisher/html_finalization.rs`; +declare the child module in `publisher.rs`. + +- [ ] Write failing coordinator tests using sessions that yield at least once, write partial + candidates, return unchanged/error/replacement, and record their input and drop events. + Use a local `Rc` inside a session/future to prove no accidental `Send` requirement. +- [ ] Give the coordinator ownership of current HTML outside every awaited future. Each + stage borrows current bytes and a separate fresh writer. A valid replacement moves + writer storage into current HTML; an unchanged/error result discards the writer. +- [ ] Construct one deadline and retain one pinned timer across the entire loop. Poll expiry + first, check absolute expiry before each stage and after completion, and make expiry win + ties. With no timer, await work and check elapsed budget afterward; no preemption claim. +- [ ] On timeout, explicitly drop the active future before accessing its borrowed writer or + moving current HTML; discard the candidate and stop the chain. Do not put current HTML + exclusively inside an async chain future that is dropped by a race. + +The driver-level ownership invariant is: + +```text +driver owns current HTML and current replacement writer + -> callback borrows both + -> result/timeout + -> callback future is dropped, releasing both borrows + -> driver either commits replacement or emits current HTML +``` + +- [ ] Validate poisoned writers and required marker cardinality before commitment. Preserve + the marker as an opaque integration contract; do not add a parser or pretend cardinality + proves structural placement. Callback errors remain soft; upstream/codec/sink errors + remain fatal and retain existing `Report` mappings. +- [ ] Test A-to-B output propagation, B timeout preserving A's accepted bytes, ignored writer + errors, invalid markers, one chain budget, late Fastly results, exact tie handling, and + input-buffer identity on fallback. Capture pointer identity at stage input and compare + before sink encoding, since writing to a sink necessarily copies into that sink. +- [ ] Emit controlled outcome/elapsed/byte-count logs; never format callback attachments, + HTML, signatures, tokens, or request data. Test callback drop on cancellation and ensure + no finalizer starts afterward. Panics retain existing runtime behavior. +- [ ] Run `cargo test -p trusted-server-core --lib publisher::html_finalization`, + `cargo test-fastly html_finalization`, `cargo fmt --all -- --check`, and + `cargo clippy-fastly`. Expected: all deterministic coordinator cases pass on native + and Fastly test targets. +- [ ] Commit as `Coordinate ordered HTML finalization and fallback`. + +### Task 6: Collect bounded transformed HTML and replay optional bypasses + +**Spec coverage:** Sections 7.2, 8, 12, and 14.3. + +**Files:** Modify `publisher/html_finalization.rs` and minimal collector call sites in +`publisher.rs`. + +- [ ] Add collector tests for cap minus one/exact/plus one, many chunks, a large crossing + chunk, synchronous EOF output crossing the cap, and integer-overflow rejection. +- [ ] Model collection states explicitly: collecting, bypassing, and finished. Append + transformed bytes under the existing `publisher.max_buffered_body_bytes` cap; enforce + bounded capacity growth without cloning the retained document. +- [ ] On optional collection overflow, move out the retained prefix and forward it, then the + unretained crossing chunk, then later synchronous output to ordinary seam/encoding + delivery. Skip every finalizer and create no timer or outside lookup for that document. +- [ ] Reuse existing auction hold state during bypass; a private seam token can straddle + the retained/crossing boundary. Test exact order, no duplicated EOF, no raw-token leak, + and prefix-before-auction collection. +- [ ] Retain the outer buffered writer cap and all existing raw/decoded/parser limits. A + decoder cap error is not an optional-finalizer bypass. Test both failure categories. + In route tests, reach the optional cap through rewritten-output expansion while raw + and decoded input remain within their limits; an oversized origin is an old fatal case. +- [ ] Run `cargo test -p trusted-server-core --lib publisher::html_finalization` and + `cargo test-fastly html_finalization`; run `cargo fmt --all -- --check` and + `cargo clippy-fastly`. Expected: collector/replay cases preserve bytes and old fatal limits. +- [ ] Commit as `Bound retained HTML and preserve streaming bypass`. + +### Task 7: Gate cache lookup and preserve synthesized-document policy + +**Spec coverage:** Sections 11, 14.5, and 16.5. + +**Files:** Modify `publisher.rs` and its existing cache/privacy test modules. Modify +`cache_policy.rs` only if reusing its current public helpers is insufficient. + +- [ ] Write failing request tests showing an enabled finalizer bypasses a prepopulated warm + template before lookup and excludes reservation/storage, while disabling it restores + existing eligibility. Include ESI configuration, no matched slots, consent denial, + bots/prefetch, and ordinary HTML without an auction. +- [ ] Derive the early finalizer-presence decision from enabled registry metadata. Combine + it with shared-template admission before key creation/lookup. Keep origin readthrough + eligibility separate; do not accidentally force every raw origin cache hit to miss. +- [ ] Use the existing non-shared inline assembly fallback consistently: no authorized + shared-template key, no shared-template token, and matching head/body script choices. + Do not add a first-version `shared_safe` override or a new public cache header. +- [ ] Apply `enforce_synthesized_html_cache_privacy` before response headers are committed for + HTML entering the opted-in path, including unchanged/failure/collection bypass. Preserve + duplicate cookies/CSP and existing EC/privacy finalization. +- [ ] For document GETs, extend the existing full-body predicate so + `strip_conditional_and_range_headers` removes its same four headers. Preserve `If-Match` + and `If-Unmodified-Since`. Do not broaden this stripping to assets or other methods. +- [ ] Test raw origin readthrough eligibility with finalizer metadata; validator/header + stripping; unexpected origin 304 behavior with/without the current ad-stack guard; + bodyless, unsupported-encoding, and non-HTML route selection. This task proves policy + admission and header decisions only. Assertions requiring callback execution, callback + outcome, or callback invocation counts belong to Task 8 after routing is wired. +- [ ] Run `cargo test -p trusted-server-core --lib publisher::tests`, + `cargo test-fastly publisher::tests`, `cargo fmt --all -- --check`, and + `cargo clippy-fastly`. Expected: new policy tests and all existing publisher tests pass. +- [ ] Commit as `Gate shared caching for asynchronous HTML finalizers`. + +### Task 8: Route every enabled publisher body through the terminal coordinator + +**Spec coverage:** Sections 7, 12, 14.4, and 16.1–16.2. + +**Files:** Modify `publisher.rs`, `publisher/html_finalization.rs`, and +`html_processor.rs` only where construction data must be threaded. + +- [ ] Build failing route tests across `Once`/`Stream`, lazy/buffered output, and auction + present/absent. Assert every enabled body calls its finalizer once after decoder and + synchronous EOF, and no disabled/bodiless path calls it. + Explicitly include processable 404/500 HTML and preserved method/status behavior. +- [ ] Extend relevant publisher parameters with request-local finalizer factories/session + construction data and shared document state. Keep ownership compatible with the existing + lazy response stream and borrowed buffered driver. +- [ ] Wire enabled HTML in both `publisher_response_into_streaming_response` and + `stream_publisher_body_async`/`buffer_publisher_response_async` through one coordinator. + In particular, prevent opted-in `Once` bodies without an auction from falling through + the synchronous `stream_publisher_body` path, and route enabled reader-based auction + bodies through the same terminal sequence. +- [ ] Reuse the existing decoder semantics on each body representation: drain/validate tail, + finish the synchronous chain once, run finalizers, replay accepted bytes in bounded chunks + through the existing seam controller, emit ready prefix, collect auction, finish encoder. + Do not feed candidates through `lol_html` or Next.js a second time. +- [ ] Keep current no-finalizer branches and public synchronous helper behavior intact. Add + API documentation that finalizers require supported publisher entry points; synchronous + helpers provide synchronous transformations only. +- [ ] Preserve dispatched-auction guards through collection and every new await. Test + cancellation during finalizer I/O and each fatal error category for exactly-once terminal + telemetry. Auction dispatch/deadlines must not restart or expand. +- [ ] Complete Task 7's execution-dependent assertions: raw origin readthrough hits run + finalizers on every request; success, unchanged, callback error, timeout, and collection + bypass retain private/no-store headers; HEAD/204/205/304 and unsupported routes invoke + no callback. Include enabled pages with no auction or matched slot. +- [ ] Run `cargo test-fastly`, `cargo test-fastly-reuse`, `cargo test-axum`, + `cargo test-cloudflare`, and `cargo test-spin`; then touched adapter lint aliases. + Expected: all enabled route combinations work and existing early-streaming tests pass. +- [ ] Commit as `Route opted-in HTML through terminal finalization`. + +Tasks 1–8 are one coherent feature. No release/PR readiness claim is allowed until this task +passes on all applicable paths, even if earlier dormant API commits compile. + +### Task 9: Complete the codec and legacy-behavior regression matrix + +**Spec coverage:** Sections 14.3–14.5 and 16.2–16.3. + +**Files:** Modify core tests in `publisher.rs`, `publisher/html_finalization.rs`, and +`html_processor.rs`. Keep `streaming_processor.rs` implementation unchanged. + +- [ ] Parameterize the new route tests over identity, gzip, deflate, and Brotli; test + concatenated gzip members and output released only in decoder/processor tails. +- [ ] Add malformed/truncated origin, empty HTML, split UTF-8, absent body end, marker + loss/duplication, oversized candidate, and optional collection bypass cases. +- [ ] Assert decoder validation precedes I/O, synchronous EOF precedes callback, finalizer + output precedes auction token removal, and encoder trailers follow accepted output. +- [ ] Retain and run these existing regressions unchanged: + +```text +html_stream_processors_compose_in_order_and_receive_final_once +streaming_html_with_stream_processors_rewrites_body +document_state_placeholders_substitute_through_streaming_path +streaming_finalize_emits_compressed_html_before_origin_eof +streaming_finalize_auction_hold_emits_prefix_before_origin_eof +streaming_finalize_auction_hold_emits_small_compressed_page_before_collection +parser_confirmed_auction_seam_streams_nextjs_for_every_encoding +stream_publisher_body_async_auction_hold_decodes_multi_member_gzip_buffered +body_chunk_source_resumes_after_cancelled_poll +finalizers_emit_abandoned_auction_for_bodiless_dispatched_response +``` + +- [ ] Compare no-finalizer results against the baseline using equivalent deterministic + fixtures: status, critical/repeated headers, decoded body, emission before source EOF, + and collection/telemetry counts. Do not compare request-random cookies or IDs literally. + Preserve exact encoded output where fixtures and chunking are deterministic. +- [ ] Run the three focused native filters from spec section 17 and `cargo test-fastly`. + Run `cargo fmt --all -- --check` and `cargo clippy-fastly`. +- [ ] Commit as `Verify HTML finalization across body and codec paths`. + +### Task 10: Add a feature-gated fictional consumer and host parity tests + +**Spec coverage:** Sections 4, 14.1, 14.5, and 14.6. + +**Files:** Create `crates/trusted-server-core/src/test_support/async_html_finalization.rs`; +modify `src/test_support.rs`, `integrations/mod.rs`, and +`crates/trusted-server-integration-tests/tests/parity.rs`. + +- [ ] Add a fictional `html_finalization_fixture` builder only under the existing + `test-utils` feature. Keep its Rust module and builder entry absent from ordinary builds. + Use `.without_js()` so no nonexistent JS module enters the bundle. +- [ ] Configure only fixture `enabled`, `lookup_url`, and `transport_timeout_ms` values in + the existing integration settings map. The finish budget comes exclusively from + `[publisher].html_finalization_timeout_ms`; do not introduce a second fixture deadline. +- [ ] Implement a terminal finalizer, not a buffering `StreamProcessor`. It receives + `input.html`, confirms a fictional EOF sentinel, sends a digest/EOF observation using + `json!` through `RuntimeServices::http_client`, then appends prefix/replacement/suffix + to the bounded writer. It performs no signing and needs no KV handle. +- [ ] Ensure its backend through `PlatformBackendSpec`, deriving TLS policy from the URL + scheme. Existing integration convenience helpers force TLS; do not use them for the + harness's generated loopback HTTP endpoint. Derive deterministic backend names from the + same spec where Viceroy needs a local backend mapping. +- [ ] Cap response collection for the fixture lookup. Honor remaining finish budget when + bounding Fastly first-byte/between-byte transport calls. When a transport cannot offer + the required bound without a timer, return `Unchanged` instead of an unbounded call. +- [ ] Use the normal enabled registry builder in host parity tests and inject deterministic + services through existing adapter testing seams. Verify yielding completion, unchanged + error/timeout bytes, no-finalizer output, privacy, and no mandatory KV. Do not add public + adapter constructors only to inject a custom registry. +- [ ] Run `cargo test --manifest-path crates/trusted-server-integration-tests/Cargo.toml --test parity`, + plus `cargo test-axum`, `cargo test-cloudflare`, and `cargo test-spin`. + Run `cargo clippy-axum`, `cargo clippy-cloudflare`, and `cargo clippy-spin-native`. +- [ ] Build/check normal artifacts without `test-utils` and prove the fixture builder ID is + absent using feature-off registry tests. Commit as `Add fictional HTML finalization parity fixtures`. + +### Task 11: Verify finalization on the four actual local runtimes + +**Spec coverage:** Sections 9.2–9.3 and 14.6. + +**Files:** Create `scripts/fixtures/async-html-finalization-origin.py` and +`scripts/async-html-finalization-local-test.sh`; use existing `scripts/smoke-common.sh` +helpers and provisioning examples. Do not alter production manifests. + +- [ ] Write the threaded local fixture origin with `/article/success`, `/article/hang`, and + `/lookup`. Generate fictional HTML containing `ASYNC_FINALIZATION_PENDING` and + `ARTICLE_EOF_SENTINEL`; serve the EOF sentinel in a later chunk. A lookup records receipt + of a complete-document digest/EOF observation before replying after 150 ms or remaining + pending for 30 seconds. Record start/completion counts without request-sensitive data. +- [ ] Add a driver accepting `fastly|cloudflare|spin|axum`. Generate all configs, fake secrets, + endpoints, runtime manifests, response captures, and logs in a temporary workspace. + Reuse each adapter's smoke-script provisioning pattern and `smoke-common.sh` cleanup. + Do not read operator-owned `trusted-server.toml`, `.env`, or cloud credentials. +- [ ] Build a fresh fixture artifact with the feature explicitly enabled and isolated output + paths. Do not reuse an ordinary cached Worker bundle, and do not call the existing + Cloudflare `build.sh`, which sources operator env files and rebuilds without fixture flags. + +Build commands used by the driver, with its own `CARGO_TARGET_DIR`, are: + +```bash +cargo build -p trusted-server-adapter-fastly --target wasm32-wasip1 --release --features trusted-server-core/test-utils +cargo build -p trusted-server-adapter-axum --features trusted-server-core/test-utils +cargo build -p trusted-server-adapter-spin --target wasm32-wasip1 --release --features spin,trusted-server-core/test-utils +``` + +For Workers, resolve the `worker-build` version from `Cargo.lock`, use that installed +version, and invoke from `crates/trusted-server-adapter-cloudflare`. The driver sets +`FINALIZATION_WORKER_OUTPUT` to an absolute directory inside its temporary workspace: + +```bash +worker-build --release --out-dir "$FINALIZATION_WORKER_OUTPUT" . --features trusted-server-core/test-utils +``` + +Point the temporary Wrangler manifest directly at that fixture bundle and disable implicit +rebuild. Use temporary manifest copies for Viceroy and Spin; point them to the fresh +fixture artifacts. Provision only existing application config/secrets infrastructure. +The new consumer must not require an integration KV namespace or label. + +- [ ] Run disabled/enabled cases on the same artifact, host, and port with auctions and + request-random body content disabled. Capture baseline transformed bodies. Success changes + only the expected sentinel; a hung lookup returns exact baseline body bytes. +- [ ] Set the fixture publisher finish timeout to 750 ms. On Axum/Workers/Spin, keep the + transport hang much longer and verify release within a documented cooperative scheduling + tolerance, well before 30 seconds. Record elapsed values and lookup invocation counts; + do not make native tests stand in for SDK timers. + Require a controlled timeout outcome so an early transport error cannot pass the test. + Measure terminal-phase elapsed time separately from origin transfer and server readiness. +- [ ] For Fastly, use an independently bounded transport timeout within the remaining finish + budget and verify unchanged fallback after the host timeout. Report the host bound and + elapsed checks; never describe this as timer-driven interruption. +- [ ] Run the actual runtime entry points through the driver: + +```bash +./scripts/async-html-finalization-local-test.sh fastly +./scripts/async-html-finalization-local-test.sh cloudflare +./scripts/async-html-finalization-local-test.sh spin +./scripts/async-html-finalization-local-test.sh axum +``` + +The driver launches `viceroy serve`, `wrangler dev --local`, `spin up`, or the freshly built +Axum binary with its generated manifests/environment, waits for readiness, captures +headers/body/timing, and always terminates child process groups. Missing tools fail with +an explicit prerequisite message; a script cannot silently skip a selected adapter. + +- [ ] Exercise auction-enabled runtime cases separately where supported. If the existing + Cloudflare clock panic prevents that case, record the exact failure and #1240 dependency; + no change to this feature's scope and no passing claim for the blocked case. +- [ ] Check script syntax and fixture server tests; run + `bash -n scripts/async-html-finalization-local-test.sh` and + `python3 -m py_compile scripts/fixtures/async-html-finalization-origin.py`. + Commit as `Add local runtime checks for HTML finalization`. + +### Task 12: Document the operator and integration contracts + +**Spec coverage:** Sections 6–13 and 15. + +**Files:** Modify `trusted-server.example.toml`, `docs/guide/configuration.md`, +`docs/guide/integration-guide.md`, `docs/guide/cloudflare.md`, +`docs/guide/spin.md`, `docs/guide/axum-dev.md`, and +`crates/trusted-server-integration-tests/README.md` only in relevant sections. + +- [ ] Add the optional publisher setting to the example configuration and canonical settings + table: default 5000, range 1–60000, chain scope after EOF, no effect without registration. + Put comments above configuration values, following repository conventions. +- [ ] Document `.with_html_finalizer`, object-safe local borrowing futures, terminal ordering, + shared document capture state, bounded replacement writes, unchanged fallback, and the + distinction between stage-input bytes and origin bytes. Include a fictional example. +- [ ] Explain opted-in whole-page latency, unchanged no-finalizer streaming, auction budget + interactions, shared-template bypass, `private, no-store`, raw readthrough separation, + and preservation of internal markers/CSP/RSC semantics. +- [ ] Describe timer availability, cooperative scheduling, Fastly host bounds, remote-side-effect + limitations, and independent KV/clock work without copying volatile adapter matrices into + several guides. Link to the canonical integration contract from each affected guide. +- [ ] Document the feature-gated fixture and exact runtime commands, prerequisite versions, + generated temporary workspace behavior, and how to record blocked smoke cases. +- [ ] Run `cargo doc -p trusted-server-core --no-deps --all-features`, `npm run lint` and + `npm run format` from `docs`, plus the repository Markdown check from Task 13. + Validate new local links. Commit as `Document asynchronous HTML finalizer behavior`. + +### Task 13: Run complete gates and review the implemented change against both documents + +**Spec coverage:** Sections 14, 16, and every coverage row below. + +**Files:** No additional production changes unless a check exposes a defect in this feature. + +- [ ] Inspect the diff for scope: no generic streaming trait change, EdgeZero repin, mandatory + store, real signing consumer, JS behavior change, or unrelated refactor. Check feature-off + fixture exclusion and request-state isolation. +- [ ] Run these Rust gates from the workspace root, using the target-matched aliases: + +```bash +cargo fmt --all -- --check +cargo clippy-fastly +cargo clippy-axum +cargo clippy-cloudflare +cargo clippy-cloudflare-wasm +cargo clippy-spin-native +cargo clippy-spin-wasm +cargo clippy-cli +cargo clippy-codegen +cargo test-fastly +cargo test-fastly-reuse +cargo test-axum +cargo test-cloudflare +cargo test-spin +cargo test --manifest-path crates/trusted-server-integration-tests/Cargo.toml --test parity +cargo test -p trusted-server-core --test template_build_digest +cargo clippy -p trusted-server-core --test template_build_digest -- -D warnings +``` + +- [ ] From `crates/trusted-server-js/lib`, run `node build-all.mjs`, `npx vitest run`, and + `npm run format`. From `docs`, run `npm run lint` and `npm run format`. + Install locked docs dependencies with `npm ci` if absent; then from root run: + +```bash +docs/node_modules/.bin/prettier --config docs/.prettierrc --check "*.md" ".claude/**/*.md" ".github/**/*.md" "crates/**/*.md" "scripts/**/*.md" "tinybird/**/*.md" +``` + +- [ ] Run the existing shared-template controls: + +```bash +./scripts/template-cache-local-test.sh +./scripts/template-cache-local-test.sh inline +``` + +Also run the feature's four runtime commands from Task 11 and the existing integration +jobs described in `scripts/integration-tests.sh` and `scripts/integration-tests-browser.sh` +where the feature changes their exercised routes. Record setup and exact outcomes; do not +substitute an empty-auction fixture for the existing winning-bid checks. + +- [ ] Map implementation and tests to every coverage row below. Have a reviewer inspect + the spec and plan together with the final diff; resolve behavioral contradictions centrally. + Do not weaken a regression assertion merely to match a new implementation. +- [ ] Record passed gates and all unavailable/blocked runtime cases in the PR handoff. If any + required gate is unresolved, keep the PR draft. No merge/deploy authorization is implied. + +## 4. Bidirectional spec/plan coverage + +Every requirement must have a task and evidence, and every task must have a requirement. +Use this table during review, not just as a task checklist. + +| Spec contract | Implementation tasks | Required evidence | +| -------------------------------------------------------------------------------------- | -------------------- | ------------------------------------------------------------------------------- | +| Sections 1, 4, 5: HTML terminal phase; generic streaming unchanged; no unrelated work | 1, 3, 8, 13 | API/diff inspection; unchanged legacy streaming tests | +| Section 6.1: ordered registration and one request-local document state | 3, 10 | factory-order and capture-state tests; request/reuse isolation | +| Section 6.2: local borrowing future and bounded replacement sink | 1, 5 | non-`Send` execution; poisoned writer tests | +| Section 7.1: no-finalizer behavior unchanged | 3, 8, 9 | prefix before EOF; one synchronous EOF; no timer/finalizer I/O | +| Section 7.2: drain decoder, synchronous EOF, terminal chain, seam, encoder | 5, 8, 9 | all body/sink/auction/codec combinations; exact event ordering | +| Section 7.3: auction deadline unchanged; opted-in delay explicit | 8, 9, 11, 12 | unchanged dispatch deadline; prefix-before-collection; expired-auction behavior | +| Section 7.4: cancellation releases borrows and does not lose owned input | 5, 8 | drop observer, buffer identity, exactly-once auction abandonment | +| Section 8: capped collection/candidate capacity; safe optional overflow bypass | 1, 6, 9 | boundary/overflow/capacity tests; exact replay; old fatal caps | +| Section 9.1: default 5000 and validated 1–60000 chain budget | 2, 12 | defaults/TOML/range tests; canonical configuration docs | +| Sections 9.2–9.3: optional SDK timers, one absolute budget, Fastly limitations | 1, 2, 4, 5, 11 | deterministic deadline tests; actual-runtime timings; bounded host calls | +| Section 10: stage-local fallback, error categories, markers, panic behavior | 1, 5, 9, 12 | each outcome/candidate failure; late/tie behavior; reviewed callback contract | +| Section 11.1: template bypass before lookup/storage and consistent inline fallback | 7, 9 | warm-hit bypass; no store/reservation; disabled cache behavior restored | +| Section 11.2: downstream privacy, raw-origin cache separation, conditional/range rules | 7, 8, 10 | private headers with no auction; raw-hit execution; four-header stripping | +| Section 11.3: existing body eligibility, status/method/framing/encoding | 7, 8, 9 | bodiless/no-HTML/unsupported cases; non-2xx HTML; header preservation | +| Section 12: every materialized/streamed and lazy/buffered branch | 8, 9, 10 | route matrix and host parity; no synchronous-only skip | +| Section 13: controlled logging and preserved auction telemetry | 5, 8, 11 | no sensitive log fields; outcome counts and abandonment tests | +| Section 14: deterministic, real-runtime, and full repository verification | 9, 10, 11, 13 | named regressions, runtime reports, complete gates | +| Sections 15–16: scoped delivery and readiness conditions | 12, 13 | guides, diff review, requirements/evidence mapping | + +## 5. Review and handoff record + +Before implementation planning is considered ready, review both documents for: + +- Spec requirements omitted from the plan or tasks that contradict the spec. +- Plan additions without a spec requirement, especially fixtures becoming production + consumers, private buffering reappearing, or deadlines multiplying per callback. +- Unrealistic runtime commands, unsupported cfg combinations, and existing behavior that + the plan mistakenly assumes instead of checking. +- Ownership of current HTML during timeout, shared state during factory construction, and + early cache admission that an EOF-only decision could not influence. + +Document review results describe planning readiness only. They do not prove the feature +is implemented, regression-free, or ready for deployment. Execution begins only after +the user requests implementation. + +### Review completed on 2026-10-12 + +The standalone spec review and both directions of the independent spec/plan alignment +review approved planning readiness with no unresolved substantive findings. + +The review corrected these points before handoff: + +- Policy-admission tests in Task 7 no longer require callback execution first wired in + Task 8; execution-dependent assertions now follow the routing implementation. +- The deadline type referenced by the callback API is defined in Task 1, avoiding an + unbuildable intermediate contract commit. +- Core edits have a direct primary-package lint checkpoint; non-2xx HTML is an explicit + acceptance test rather than implicit coverage. +- Overflow route tests distinguish rewrite expansion from preexisting fatal decoder caps. +- Workers fixture output stays in the temporary workspace; runtime timeout tests verify + timer expiry rather than accidentally passing on an early transport error. + +This historical record covers planning review only. The linked execution record now +maps the implemented contracts and observed verification to both documents. diff --git a/docs/superpowers/plans/2026-10-12-1241-async-html-finalization.md b/docs/superpowers/plans/2026-10-12-1241-async-html-finalization.md index 37c13f6be..407d109cb 100644 --- a/docs/superpowers/plans/2026-10-12-1241-async-html-finalization.md +++ b/docs/superpowers/plans/2026-10-12-1241-async-html-finalization.md @@ -1,682 +1,138 @@ -# Optional asynchronous HTML finalization implementation plan +# Ordered HTML EOF Completion Implementation Plan -> **For agentic workers:** REQUIRED SUB-SKILL: Use superpowers:subagent-driven-development (recommended) or superpowers:executing-plans to implement this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking. +> **For agentic workers:** REQUIRED SUB-SKILL: Use superpowers:subagent-driven-development or superpowers:executing-plans to implement this plan task-by-task. Follow repository instructions for centralized implementation and independent review. Steps use checkbox syntax for tracking. -**Goal:** Allow explicitly registered HTML integrations to await I/O after synchronous document processing, with bounded core-owned buffers and exact fallback on optional failure. +**Goal:** Let registered HTML processors optionally await EOF I/O in stage order without mandatory whole-page buffering. -**Architecture:** Add an HTML-only terminal phase after existing synchronous processors and before auction-seam handling and encoding. One coordinator owns retained HTML, bounded replacement storage, and the chain deadline; the no-finalizer route remains unchanged. Enabled finalizers bypass shared-template caching before lookup and use synthesized HTML privacy rules. +**Architecture:** Keep synchronous chunk processing and add one optional processor EOF callback. A stepped concrete HTML chain lends driver-owned stage output to a bounded callback and sends the accepted result downstream before downstream EOF. Operator permission is default off; only explicit retention stages buffer output. -**Tech Stack:** Rust 2024, existing `futures` local futures, `error-stack`, `derive_more`, `web_time`, `lol_html`, existing compression codecs, and adapter-local Workers/Spin/Tokio timers. +**Tech Stack:** Rust 2024, futures LocalBoxFuture, error-stack, lol_html, existing adapter timers and publisher codec/auction drivers. ---- +**Revised:** 2026-10-13. User authorizes specification, plan and implementation updates. +The [specification](../specs/2026-10-12-1241-async-html-finalization-design.md) governs this revision. +The [verification report](../reviews/2026-10-13-1241-ordered-eof-verification.md) records fresh revised-source evidence. +The [old plan](../archive/2026-10-12-1241-buffered-terminal-plan.md) and historical implementation records describe the superseded terminal phase. -**Status:** Implemented; see the execution record and review hardening for current verification +## File responsibilities -**Date:** 2026-10-12 +| File | Responsibility | +| ----------------------------------------------------- | --------------------------------------------------------------------------------- | +| `streaming_processor/finish.rs` | Optional finish input/outcome/error, bounded writer and absolute deadline | +| `streaming_processor.rs` | Object-safe default-absent callback and synchronous rejection | +| `integrations/registry.rs` | One processor registration sequence, early capabilities and permission validation | +| `html_processor.rs` and `html_processor/finish.rs` | Concrete stepped chain, stage retention and leaf callback completion | +| `publisher/html_finalization.rs` | Existing body/codec/auction driver adapted to stepped EOF | +| `publisher.rs` | Early capability routing, direct-helper guards and cache/privacy policy | +| `settings.rs` | Default-off operator permission; existing finish timeout retained | +| `test_support/async_html_finalization.rs` | Fictional EOF HTTP processor using the production lifecycle | +| `scripts/fixtures/async-html-finalization-runtime.py` | Explicit permission in enabled runtime configuration | +| Integration/config guides and example TOML | Capability semantics, operator acceptance and runtime limits | -**Branch:** `feat/1241-async-html-finalization` (planning originated on `spec/1241-async-html-finalization`) +Paths above are relative to `crates/trusted-server-core/src/` unless prefixed with `scripts/` or documentation paths. -**Baseline:** `2a0cb0444dc335bd09105abe5952329bbbdfb621` +## Task 1: Review the revised specification and plan -**Design:** [Optional asynchronous HTML document finalization](../specs/2026-10-12-1241-async-html-finalization-design.md) +- [x] Independently review both documents against issue1241 and the accepted latency policy. +- [x] Resolve lifecycle, ownership, cancellation, cache and runtime contradictions before implementation. +- [x] Archive the old documents and link historical reports to the archived design. -**Execution record:** [Implementation and bidirectional coverage review](../reviews/2026-10-12-1241-async-html-finalization-implementation.md) +## Task 2: Optional callback contract and default behavior -The task checklists below preserve the original execution instructions, including -proposed intermediate commit boundaries. The execution record is the authoritative -completion record: implementation was reviewed together before committing, and it -records actual commands, deviations, and the initially blocked Workers auction control. +**Files:** `crates/trusted-server-core/src/streaming_processor.rs`, new `streaming_processor/finish.rs`, remove production finalizer API from `integrations/html_finalization.rs` and `integrations/mod.rs`. -The [review hardening addendum](2026-10-12-1241-review-hardening.md) records the -subsequently authorized response-status/range refinements, telemetry clock -correction, runtime CI coverage, and renewed verification. It supplements the -original task instructions below. +- [x] Add failing tests for default `None`, a borrowing non-Send callback, bounded/poisoned replacement and synchronous rejection before reader/writer effects. +- [x] Run `cargo test -p trusted-server-core --all-features async_eof` and record expected missing-capability failures. +- [x] Add `requires_async_finish` and `async_finish` defaults to `StreamProcessor`; adapt existing writer/deadline implementations into generic finish contracts. +- [x] Keep `process_chunk` unchanged. Document that callback input is un-emitted stage output, not necessarily the whole document. +- [x] Make `StreamingPipeline::process` reject async-capable processors before consuming input. +- [x] Re-run targeted tests; format and inspect the diff. -## 1. Execution boundaries +## Task 3: Registration and operator permission -This plan implements the selected terminal-phase design, not the original proposal to add -`async_finish` to `StreamProcessor`. Read the spec before implementing. Use -@superpowers:test-driven-development for runtime changes and -@superpowers:verification-before-completion before recording results or committing. - -The tasks below were the implementation instructions. Their expected outcomes alone -are not verification evidence; consult the execution record for observed results. Keep intermediate -commits in a draft branch; do not release an API that registrations can enable before all -publisher entry points support it. - -Keep the EdgeZero revision, generic streaming traits, JS runtime, and production platform -manifests unchanged. KV wiring under #1139, broader clock fixes under #1240, and a real signing -integration are separate work. Do not mask a live-runtime failure by replacing actual -SDK services with stubs or by reporting a native test as a runtime smoke. +**Files:** `integrations/registry.rs`, `settings.rs`, `trusted-server.example.toml`, fixture configuration tests. -## 2. File responsibilities +- [x] Write failing tests: permission defaults false; enabled async factory requires explicit acceptance; disabled integration needs no acceptance; permission alone creates no capability; buffered declaration without async capability is invalid. +- [x] Add `publisher.allow_async_html_finish` with serde/default false. +- [x] Add factory capability metadata and preserve the existing single factory sequence. +- [x] Reject missing permission with integration ID and setting in the error. Do not filter or silently omit requested stages. +- [x] Remove separate public finalizer registration and metadata. Test adapters may wrap old fake actions into ordinary production processors. +- [x] Run settings/registry tests and target-matched Axum checks. -| File | Responsibility | -| --------------------------------------------------------------------------------------------------------------- | -------------------------------------------------------------------------------------------------------------- | -| Create `crates/trusted-server-core/src/integrations/html_finalization.rs` | Public factory/session/input/outcome/error contracts, bounded replacement writer, and finish deadline handle | -| Modify `crates/trusted-server-core/src/integrations/mod.rs` | Export the new contracts; later register a feature-gated fictional fixture | -| Modify `crates/trusted-server-core/src/integrations/registry.rs` | Empty-by-default finalizer metadata, builder method, ordered factory access, and test construction seam | -| Modify `crates/trusted-server-core/src/html_processor.rs` | Share one document-state instance through parser, synchronous processors, and finalizer construction | -| Create `crates/trusted-server-core/src/platform/timer.rs` | Optional platform-neutral timer contract | -| Modify `crates/trusted-server-core/src/platform/mod.rs` and `types.rs` | Timer exports, default-absent runtime service, builder/accessor support | -| Modify `crates/trusted-server-core/src/settings.rs` | Global terminal-phase timeout, matching defaults, and validation | -| Create `crates/trusted-server-core/src/publisher/html_finalization.rs` | Private retained-document collector, ordered async driver, candidate validation, and bounded replay | -| Modify `crates/trusted-server-core/src/publisher.rs` | Route opted-in HTML through the coordinator on every body/sink path; early cache/privacy/conditional decisions | -| Modify adapter `src/platform.rs` files and Fastly `src/app.rs` tests | Concrete timers and explicit Fastly timer absence | -| Create `crates/trusted-server-core/src/test_support/async_html_finalization.rs`; modify `src/test_support.rs` | Fictional integration compiled only under existing `test-utils` | -| Modify `crates/trusted-server-integration-tests/tests/parity.rs` | Host-adapter finalizer behavior and privacy parity | -| Create `scripts/fixtures/async-html-finalization-origin.py` and `scripts/async-html-finalization-local-test.sh` | Delayed/hung local HTTP lookup and actual-runtime smoke driver | -| Modify `trusted-server.example.toml` and canonical guide files | Explain the timeout, registration API, buffer/cache effects, and adapter limitations | +## Task 4: Ordered, stepped processor EOF -Do not split or reorganize unrelated sections of `publisher.rs`. The focused child module -contains the new state machine; existing decoding, auction collection, telemetry guards, -and framing helpers remain authoritative. +**Files:** `html_processor.rs`, new `html_processor/finish.rs`, `publisher.rs` concrete async construction helper. -## 3. Dependency order and verification rhythm - -Tasks 1–4 establish dormant contracts and services. Task 5 implements asynchronous execution; -Task 6 implements bounded collection/replay. Task 7 establishes request-time policy before -Task 8 enables production routing. Tasks 9–11 verify codecs, adapter paths, and real runtimes. -Tasks 12–13 finish documentation and the complete gate run. - -Use native core filters for fast feedback, then the target-matched gates associated with -the touched adapters. Before each commit, run formatting, the task's tests, and its applicable -lint gate. Before PR handoff, run the entire gate list in Task 13. An unavailable runtime -is a reported verification gap, not permission to mark the check passing. - -For core-only native checkpoints, run -`cargo clippy -p trusted-server-core --all-targets --all-features -- -D warnings`. -An adapter lint alias can compile core as a dependency but does not directly lint it as -a primary package. Retain target-matched Fastly/core and adapter gates where listed. - -### Task 1: Define the terminal callback and bounded replacement writer - -**Spec coverage:** Sections 6.2, 8, 9.3, 10, and 16.3. - -**Files:** Create `crates/trusted-server-core/src/integrations/html_finalization.rs`; -modify `crates/trusted-server-core/src/integrations/mod.rs`. - -- [ ] Write failing unit tests in the new module for exact-cap append, cap overflow, - overflow-safe length checking, bounded capacity growth, poisoned-writer rejection, - and explicit `Unchanged` versus empty `Replace` outcomes. -- [ ] Define/export the spec's object-safe factory/session/input/outcome/error types. - The factory exposes `integration_id()` and `create(IntegrationHtmlStreamContext)`; - it is `Send + Sync`. A session is single-use and returns a local borrowing future. - Define `HtmlFinalizationDeadline` in this same task so the input contract compiles: - use `web_time::Instant`, a timeout `Duration`, and cooperative-timer availability; - expose remaining time and keep expiry-at-explicit-time helpers private for tests. - Cover before/exactly-at/after expiry and saturating zero remaining time. - -The callback signature is: - -```rust -pub trait IntegrationHtmlFinalizer { - fn finalize<'a>( - &'a mut self, - input: HtmlFinalizationInput<'a>, - ) -> LocalBoxFuture<'a, Result>>; -} -``` - -The input fields and result variants must match spec section 6.2 exactly. Keep the retained -HTML immutable. The writer exposes fallible `append(&[u8])`; only core can consume it to -commit a candidate. Use a concrete `derive_more::Display` error and `core::error::Error`. - -- [ ] Implement capped append: compute the next length with `checked_add`; poison and - return a controlled limit error before mutation on overflow or excess. For growth, - use an amortized capacity target bounded by the cap and reserve exactly that target. - Do not allocate the full cap up front or accept an arbitrary replacement `Vec`. -- [ ] Run `cargo test -p trusted-server-core --lib integrations::html_finalization`. - Confirm the new tests actually run, reproduce the limit failure before the fix, then pass. - Also run `cargo fmt --all -- --check`, the focused native core clippy command above, - and `cargo clippy-axum`. -- [ ] Commit the contract/writer change as `Add bounded HTML finalizer contracts`. - -The essential overflow regression has this behavior: - -```rust -let mut writer = HtmlReplacementWriter::new(4); -writer.append(b"page").expect("should accept the exact cap"); -writer.append(b"!").expect_err("should reject output beyond the cap"); -assert!(writer.is_poisoned(), "should reject later replacement commitment"); -``` - -Keep writer construction/poison inspection crate-private where only core needs them. Public -items require summaries, intra-doc links, fallible-function error documentation, and a -fictional usage example. - -### Task 2: Add the shared finish budget and optional timer contract - -**Spec coverage:** Sections 9 and 16.4. - -**Files:** Modify `integrations/html_finalization.rs`, `settings.rs`, -`platform/types.rs`, and `platform/mod.rs`; create -`crates/trusted-server-core/src/platform/timer.rs`. - -- [ ] Write failing tests for settings omission/default parity, `1`/`5000`/`60000`, - rejected `0`/`60001`, default-absent timer construction, and services clone/replacement - methods preserving an installed timer. -- [ ] Add the publisher setting with this exact default and validation: - -```rust -#[serde(default = "default_html_finalization_timeout_ms")] -#[validate(range(min = 1, max = 60000))] -pub html_finalization_timeout_ms: u32, - -fn default_html_finalization_timeout_ms() -> u32 { - 5000 -} -``` - -Update the hand-written `Publisher::default()` and only compiler-required publisher -settings literals. Do not modify unrelated OpenRTB types also named `Publisher`. - -- [ ] Add a `Send + Sync` timer trait with `delay(Duration) -> LocalBoxFuture<'static, ()>`. - Add optional `Arc` storage, a builder method, and an accessor to - `RuntimeServices`; missing timer configuration must never cause builder failure. -- [ ] Connect the timer-presence flag to Task 1's deadline type. Keep any deterministic - clock injection private to tests; do not introduce a public platform clock service. - Add settings-to-deadline tests proving the whole chain gets the configured duration, - rather than a duration independently supplied by each finalizer. -- [ ] Run `cargo test -p trusted-server-core --lib html_finalization`, - `cargo test -p trusted-server-core --lib settings::tests`, and - `cargo test -p trusted-server-core --lib platform::tests`. - Expected: new default/validation/deadline tests pass and existing service tests remain green. - Run `cargo fmt --all -- --check`, the focused native core clippy command above, - and `cargo clippy-axum`. -- [ ] Commit as `Add HTML finalization budgets and optional timers`. - -### Task 3: Register finalizers and share the parser's document state - -**Spec coverage:** Sections 6.1, 7.1, 7.2, and 14.1. - -**Files:** Modify `integrations/registry.rs`, `integrations/mod.rs`, and -`html_processor.rs`. - -- [ ] Write failing registry tests for empty defaults, disabled integration exclusion, - multiple finalizers preserving registration order, and presence checks that never - instantiate sessions. Extend the existing single-EOF processor test. -- [ ] Add `.with_html_finalizer(...)`, `has_html_finalizers()`, and ordered factory access. - Update existing hand-written registration/registry literals with empty collections. - Follow the current registry's integration enablement/build order; do not sort by ID. -- [ ] Add a `#[cfg(any(test, feature = "test-utils"))]` registry constructor from finalizer - factories for isolated core tests. Keep it out of ordinary production API builds. -- [ ] Introduce a crate-private construction helper taking an explicit document-state - instance, and keep `create_html_processor(config)` as a synchronous wrapper that - constructs state and delegates. Publisher assembly creates one state and passes clones - of its shared handle to parser hooks, streaming factories, and finalizer factories. -- [ ] Test parser-written state reaching the finalizer, two independent requests, and - retained-registry/reusable-sandbox isolation. Assert synchronous EOF is delivered once - before the first finalizer. No replacement is ever reprocessed by that chain. -- [ ] Run the registry and HTML filters: - `cargo test -p trusted-server-core --lib integrations::registry::tests` and - `cargo test -p trusted-server-core --lib html_processor::tests`. - Run `cargo test-fastly-reuse`, `cargo fmt --all -- --check`, and `cargo clippy-fastly`. -- [ ] Commit as `Register HTML finalizers with shared document state`. - -### Task 4: Wire concrete adapter timers - -**Spec coverage:** Sections 9.2, 9.3, and 14.6. - -**Files:** Modify `crates/trusted-server-adapter-axum/src/platform.rs`, -`crates/trusted-server-adapter-cloudflare/src/platform.rs`, -`crates/trusted-server-adapter-spin/src/platform.rs`, and -`crates/trusted-server-adapter-fastly/src/app.rs` tests. - -- [ ] Add service-construction tests: Axum installs a timer; production Workers and Spin - install their SDK timer under the existing runtime cfg guards; native Workers/Spin - construction remains safe without those SDKs; Fastly installs no timer. -- [ ] Implement stateless timer service objects locally. Return a boxed local future from - `worker::Delay::from(duration)`, `spin_sdk::time::sleep(duration)`, or - `tokio::time::sleep(duration)` as applicable. Delay construction is inside the returned - async future where required to avoid needing a runtime merely to build services. -- [ ] Add the timer to each supporting adapter's runtime-services builder. Preserve config, - secret, KV, HTTP, telemetry, and template service wiring. No manifest or EdgeZero changes. -- [ ] Run `cargo test-axum`, `cargo test-cloudflare`, `cargo test-spin`, and - `cargo test-fastly-reuse`. Native tests prove construction contracts, not SDK behavior. -- [ ] Run all touched adapter lint targets: `cargo clippy-axum`, - `cargo clippy-cloudflare`, `cargo clippy-cloudflare-wasm`, - `cargo clippy-spin-native`, `cargo clippy-spin-wasm`, and `cargo clippy-fastly`. - Expected: runtime cfg separation compiles on every relevant target. -- [ ] Commit as `Wire adapter timers for HTML finalization`. - -### Task 5: Implement ordered, cancellation-safe terminal execution - -**Spec coverage:** Sections 7.2–7.4, 9.3, 10, 13, and 14.2. - -**Files:** Create `crates/trusted-server-core/src/publisher/html_finalization.rs`; -declare the child module in `publisher.rs`. - -- [ ] Write failing coordinator tests using sessions that yield at least once, write partial - candidates, return unchanged/error/replacement, and record their input and drop events. - Use a local `Rc` inside a session/future to prove no accidental `Send` requirement. -- [ ] Give the coordinator ownership of current HTML outside every awaited future. Each - stage borrows current bytes and a separate fresh writer. A valid replacement moves - writer storage into current HTML; an unchanged/error result discards the writer. -- [ ] Construct one deadline and retain one pinned timer across the entire loop. Poll expiry - first, check absolute expiry before each stage and after completion, and make expiry win - ties. With no timer, await work and check elapsed budget afterward; no preemption claim. -- [ ] On timeout, explicitly drop the active future before accessing its borrowed writer or - moving current HTML; discard the candidate and stop the chain. Do not put current HTML - exclusively inside an async chain future that is dropped by a race. - -The driver-level ownership invariant is: - -```text -driver owns current HTML and current replacement writer - -> callback borrows both - -> result/timeout - -> callback future is dropped, releasing both borrows - -> driver either commits replacement or emits current HTML -``` - -- [ ] Validate poisoned writers and required marker cardinality before commitment. Preserve - the marker as an opaque integration contract; do not add a parser or pretend cardinality - proves structural placement. Callback errors remain soft; upstream/codec/sink errors - remain fatal and retain existing `Report` mappings. -- [ ] Test A-to-B output propagation, B timeout preserving A's accepted bytes, ignored writer - errors, invalid markers, one chain budget, late Fastly results, exact tie handling, and - input-buffer identity on fallback. Capture pointer identity at stage input and compare - before sink encoding, since writing to a sink necessarily copies into that sink. -- [ ] Emit controlled outcome/elapsed/byte-count logs; never format callback attachments, - HTML, signatures, tokens, or request data. Test callback drop on cancellation and ensure - no finalizer starts afterward. Panics retain existing runtime behavior. -- [ ] Run `cargo test -p trusted-server-core --lib publisher::html_finalization`, - `cargo test-fastly html_finalization`, `cargo fmt --all -- --check`, and - `cargo clippy-fastly`. Expected: all deterministic coordinator cases pass on native - and Fastly test targets. -- [ ] Commit as `Coordinate ordered HTML finalization and fallback`. - -### Task 6: Collect bounded transformed HTML and replay optional bypasses - -**Spec coverage:** Sections 7.2, 8, 12, and 14.3. - -**Files:** Modify `publisher/html_finalization.rs` and minimal collector call sites in -`publisher.rs`. - -- [ ] Add collector tests for cap minus one/exact/plus one, many chunks, a large crossing - chunk, synchronous EOF output crossing the cap, and integer-overflow rejection. -- [ ] Model collection states explicitly: collecting, bypassing, and finished. Append - transformed bytes under the existing `publisher.max_buffered_body_bytes` cap; enforce - bounded capacity growth without cloning the retained document. -- [ ] On optional collection overflow, move out the retained prefix and forward it, then the - unretained crossing chunk, then later synchronous output to ordinary seam/encoding - delivery. Skip every finalizer and create no timer or outside lookup for that document. -- [ ] Reuse existing auction hold state during bypass; a private seam token can straddle - the retained/crossing boundary. Test exact order, no duplicated EOF, no raw-token leak, - and prefix-before-auction collection. -- [ ] Retain the outer buffered writer cap and all existing raw/decoded/parser limits. A - decoder cap error is not an optional-finalizer bypass. Test both failure categories. - In route tests, reach the optional cap through rewritten-output expansion while raw - and decoded input remain within their limits; an oversized origin is an old fatal case. -- [ ] Run `cargo test -p trusted-server-core --lib publisher::html_finalization` and - `cargo test-fastly html_finalization`; run `cargo fmt --all -- --check` and - `cargo clippy-fastly`. Expected: collector/replay cases preserve bytes and old fatal limits. -- [ ] Commit as `Bound retained HTML and preserve streaming bypass`. - -### Task 7: Gate cache lookup and preserve synthesized-document policy - -**Spec coverage:** Sections 11, 14.5, and 16.5. - -**Files:** Modify `publisher.rs` and its existing cache/privacy test modules. Modify -`cache_policy.rs` only if reusing its current public helpers is insufficient. - -- [ ] Write failing request tests showing an enabled finalizer bypasses a prepopulated warm - template before lookup and excludes reservation/storage, while disabling it restores - existing eligibility. Include ESI configuration, no matched slots, consent denial, - bots/prefetch, and ordinary HTML without an auction. -- [ ] Derive the early finalizer-presence decision from enabled registry metadata. Combine - it with shared-template admission before key creation/lookup. Keep origin readthrough - eligibility separate; do not accidentally force every raw origin cache hit to miss. -- [ ] Use the existing non-shared inline assembly fallback consistently: no authorized - shared-template key, no shared-template token, and matching head/body script choices. - Do not add a first-version `shared_safe` override or a new public cache header. -- [ ] Apply `enforce_synthesized_html_cache_privacy` before response headers are committed for - HTML entering the opted-in path, including unchanged/failure/collection bypass. Preserve - duplicate cookies/CSP and existing EC/privacy finalization. -- [ ] For document GETs, extend the existing full-body predicate so - `strip_conditional_and_range_headers` removes its same four headers. Preserve `If-Match` - and `If-Unmodified-Since`. Do not broaden this stripping to assets or other methods. -- [ ] Test raw origin readthrough eligibility with finalizer metadata; validator/header - stripping; unexpected origin 304 behavior with/without the current ad-stack guard; - bodyless, unsupported-encoding, and non-HTML route selection. This task proves policy - admission and header decisions only. Assertions requiring callback execution, callback - outcome, or callback invocation counts belong to Task 8 after routing is wired. -- [ ] Run `cargo test -p trusted-server-core --lib publisher::tests`, - `cargo test-fastly publisher::tests`, `cargo fmt --all -- --check`, and - `cargo clippy-fastly`. Expected: new policy tests and all existing publisher tests pass. -- [ ] Commit as `Gate shared caching for asynchronous HTML finalizers`. - -### Task 8: Route every enabled publisher body through the terminal coordinator - -**Spec coverage:** Sections 7, 12, 14.4, and 16.1–16.2. - -**Files:** Modify `publisher.rs`, `publisher/html_finalization.rs`, and -`html_processor.rs` only where construction data must be threaded. - -- [ ] Build failing route tests across `Once`/`Stream`, lazy/buffered output, and auction - present/absent. Assert every enabled body calls its finalizer once after decoder and - synchronous EOF, and no disabled/bodiless path calls it. - Explicitly include processable 404/500 HTML and preserved method/status behavior. -- [ ] Extend relevant publisher parameters with request-local finalizer factories/session - construction data and shared document state. Keep ownership compatible with the existing - lazy response stream and borrowed buffered driver. -- [ ] Wire enabled HTML in both `publisher_response_into_streaming_response` and - `stream_publisher_body_async`/`buffer_publisher_response_async` through one coordinator. - In particular, prevent opted-in `Once` bodies without an auction from falling through - the synchronous `stream_publisher_body` path, and route enabled reader-based auction - bodies through the same terminal sequence. -- [ ] Reuse the existing decoder semantics on each body representation: drain/validate tail, - finish the synchronous chain once, run finalizers, replay accepted bytes in bounded chunks - through the existing seam controller, emit ready prefix, collect auction, finish encoder. - Do not feed candidates through `lol_html` or Next.js a second time. -- [ ] Keep current no-finalizer branches and public synchronous helper behavior intact. Add - API documentation that finalizers require supported publisher entry points; synchronous - helpers provide synchronous transformations only. -- [ ] Preserve dispatched-auction guards through collection and every new await. Test - cancellation during finalizer I/O and each fatal error category for exactly-once terminal - telemetry. Auction dispatch/deadlines must not restart or expand. -- [ ] Complete Task 7's execution-dependent assertions: raw origin readthrough hits run - finalizers on every request; success, unchanged, callback error, timeout, and collection - bypass retain private/no-store headers; HEAD/204/205/304 and unsupported routes invoke - no callback. Include enabled pages with no auction or matched slot. -- [ ] Run `cargo test-fastly`, `cargo test-fastly-reuse`, `cargo test-axum`, - `cargo test-cloudflare`, and `cargo test-spin`; then touched adapter lint aliases. - Expected: all enabled route combinations work and existing early-streaming tests pass. -- [ ] Commit as `Route opted-in HTML through terminal finalization`. - -Tasks 1–8 are one coherent feature. No release/PR readiness claim is allowed until this task -passes on all applicable paths, even if earlier dormant API commits compile. - -### Task 9: Complete the codec and legacy-behavior regression matrix - -**Spec coverage:** Sections 14.3–14.5 and 16.2–16.3. - -**Files:** Modify core tests in `publisher.rs`, `publisher/html_finalization.rs`, and -`html_processor.rs`. Keep `streaming_processor.rs` implementation unchanged. - -- [ ] Parameterize the new route tests over identity, gzip, deflate, and Brotli; test - concatenated gzip members and output released only in decoder/processor tails. -- [ ] Add malformed/truncated origin, empty HTML, split UTF-8, absent body end, marker - loss/duplication, oversized candidate, and optional collection bypass cases. -- [ ] Assert decoder validation precedes I/O, synchronous EOF precedes callback, finalizer - output precedes auction token removal, and encoder trailers follow accepted output. -- [ ] Retain and run these existing regressions unchanged: - -```text -html_stream_processors_compose_in_order_and_receive_final_once -streaming_html_with_stream_processors_rewrites_body -document_state_placeholders_substitute_through_streaming_path -streaming_finalize_emits_compressed_html_before_origin_eof -streaming_finalize_auction_hold_emits_prefix_before_origin_eof -streaming_finalize_auction_hold_emits_small_compressed_page_before_collection -parser_confirmed_auction_seam_streams_nextjs_for_every_encoding -stream_publisher_body_async_auction_hold_decodes_multi_member_gzip_buffered -body_chunk_source_resumes_after_cancelled_poll -finalizers_emit_abandoned_auction_for_bodiless_dispatched_response -``` - -- [ ] Compare no-finalizer results against the baseline using equivalent deterministic - fixtures: status, critical/repeated headers, decoded body, emission before source EOF, - and collection/telemetry counts. Do not compare request-random cookies or IDs literally. - Preserve exact encoded output where fixtures and chunking are deterministic. -- [ ] Run the three focused native filters from spec section 17 and `cargo test-fastly`. - Run `cargo fmt --all -- --check` and `cargo clippy-fastly`. -- [ ] Commit as `Verify HTML finalization across body and codec paths`. - -### Task 10: Add a feature-gated fictional consumer and host parity tests - -**Spec coverage:** Sections 4, 14.1, 14.5, and 14.6. - -**Files:** Create `crates/trusted-server-core/src/test_support/async_html_finalization.rs`; -modify `src/test_support.rs`, `integrations/mod.rs`, and -`crates/trusted-server-integration-tests/tests/parity.rs`. - -- [ ] Add a fictional `html_finalization_fixture` builder only under the existing - `test-utils` feature. Keep its Rust module and builder entry absent from ordinary builds. - Use `.without_js()` so no nonexistent JS module enters the bundle. -- [ ] Configure only fixture `enabled`, `lookup_url`, and `transport_timeout_ms` values in - the existing integration settings map. The finish budget comes exclusively from - `[publisher].html_finalization_timeout_ms`; do not introduce a second fixture deadline. -- [ ] Implement a terminal finalizer, not a buffering `StreamProcessor`. It receives - `input.html`, confirms a fictional EOF sentinel, sends a digest/EOF observation using - `json!` through `RuntimeServices::http_client`, then appends prefix/replacement/suffix - to the bounded writer. It performs no signing and needs no KV handle. -- [ ] Ensure its backend through `PlatformBackendSpec`, deriving TLS policy from the URL - scheme. Existing integration convenience helpers force TLS; do not use them for the - harness's generated loopback HTTP endpoint. Derive deterministic backend names from the - same spec where Viceroy needs a local backend mapping. -- [ ] Cap response collection for the fixture lookup. Honor remaining finish budget when - bounding Fastly first-byte/between-byte transport calls. When a transport cannot offer - the required bound without a timer, return `Unchanged` instead of an unbounded call. -- [ ] Use the normal enabled registry builder in host parity tests and inject deterministic - services through existing adapter testing seams. Verify yielding completion, unchanged - error/timeout bytes, no-finalizer output, privacy, and no mandatory KV. Do not add public - adapter constructors only to inject a custom registry. -- [ ] Run `cargo test --manifest-path crates/trusted-server-integration-tests/Cargo.toml --test parity`, - plus `cargo test-axum`, `cargo test-cloudflare`, and `cargo test-spin`. - Run `cargo clippy-axum`, `cargo clippy-cloudflare`, and `cargo clippy-spin-native`. -- [ ] Build/check normal artifacts without `test-utils` and prove the fixture builder ID is - absent using feature-off registry tests. Commit as `Add fictional HTML finalization parity fixtures`. - -### Task 11: Verify finalization on the four actual local runtimes - -**Spec coverage:** Sections 9.2–9.3 and 14.6. - -**Files:** Create `scripts/fixtures/async-html-finalization-origin.py` and -`scripts/async-html-finalization-local-test.sh`; use existing `scripts/smoke-common.sh` -helpers and provisioning examples. Do not alter production manifests. - -- [ ] Write the threaded local fixture origin with `/article/success`, `/article/hang`, and - `/lookup`. Generate fictional HTML containing `ASYNC_FINALIZATION_PENDING` and - `ARTICLE_EOF_SENTINEL`; serve the EOF sentinel in a later chunk. A lookup records receipt - of a complete-document digest/EOF observation before replying after 150 ms or remaining - pending for 30 seconds. Record start/completion counts without request-sensitive data. -- [ ] Add a driver accepting `fastly|cloudflare|spin|axum`. Generate all configs, fake secrets, - endpoints, runtime manifests, response captures, and logs in a temporary workspace. - Reuse each adapter's smoke-script provisioning pattern and `smoke-common.sh` cleanup. - Do not read operator-owned `trusted-server.toml`, `.env`, or cloud credentials. -- [ ] Build a fresh fixture artifact with the feature explicitly enabled and isolated output - paths. Do not reuse an ordinary cached Worker bundle, and do not call the existing - Cloudflare `build.sh`, which sources operator env files and rebuilds without fixture flags. - -Build commands used by the driver, with its own `CARGO_TARGET_DIR`, are: - -```bash -cargo build -p trusted-server-adapter-fastly --target wasm32-wasip1 --release --features trusted-server-core/test-utils -cargo build -p trusted-server-adapter-axum --features trusted-server-core/test-utils -cargo build -p trusted-server-adapter-spin --target wasm32-wasip1 --release --features spin,trusted-server-core/test-utils -``` - -For Workers, resolve the `worker-build` version from `Cargo.lock`, use that installed -version, and invoke from `crates/trusted-server-adapter-cloudflare`. The driver sets -`FINALIZATION_WORKER_OUTPUT` to an absolute directory inside its temporary workspace: - -```bash -worker-build --release --out-dir "$FINALIZATION_WORKER_OUTPUT" . --features trusted-server-core/test-utils -``` - -Point the temporary Wrangler manifest directly at that fixture bundle and disable implicit -rebuild. Use temporary manifest copies for Viceroy and Spin; point them to the fresh -fixture artifacts. Provision only existing application config/secrets infrastructure. -The new consumer must not require an integration KV namespace or label. - -- [ ] Run disabled/enabled cases on the same artifact, host, and port with auctions and - request-random body content disabled. Capture baseline transformed bodies. Success changes - only the expected sentinel; a hung lookup returns exact baseline body bytes. -- [ ] Set the fixture publisher finish timeout to 750 ms. On Axum/Workers/Spin, keep the - transport hang much longer and verify release within a documented cooperative scheduling - tolerance, well before 30 seconds. Record elapsed values and lookup invocation counts; - do not make native tests stand in for SDK timers. - Require a controlled timeout outcome so an early transport error cannot pass the test. - Measure terminal-phase elapsed time separately from origin transfer and server readiness. -- [ ] For Fastly, use an independently bounded transport timeout within the remaining finish - budget and verify unchanged fallback after the host timeout. Report the host bound and - elapsed checks; never describe this as timer-driven interruption. -- [ ] Run the actual runtime entry points through the driver: - -```bash -./scripts/async-html-finalization-local-test.sh fastly -./scripts/async-html-finalization-local-test.sh cloudflare -./scripts/async-html-finalization-local-test.sh spin -./scripts/async-html-finalization-local-test.sh axum -``` - -The driver launches `viceroy serve`, `wrangler dev --local`, `spin up`, or the freshly built -Axum binary with its generated manifests/environment, waits for readiness, captures -headers/body/timing, and always terminates child process groups. Missing tools fail with -an explicit prerequisite message; a script cannot silently skip a selected adapter. - -- [ ] Exercise auction-enabled runtime cases separately where supported. If the existing - Cloudflare clock panic prevents that case, record the exact failure and #1240 dependency; - no change to this feature's scope and no passing claim for the blocked case. -- [ ] Check script syntax and fixture server tests; run - `bash -n scripts/async-html-finalization-local-test.sh` and - `python3 -m py_compile scripts/fixtures/async-html-finalization-origin.py`. - Commit as `Add local runtime checks for HTML finalization`. - -### Task 12: Document the operator and integration contracts - -**Spec coverage:** Sections 6–13 and 15. - -**Files:** Modify `trusted-server.example.toml`, `docs/guide/configuration.md`, -`docs/guide/integration-guide.md`, `docs/guide/cloudflare.md`, -`docs/guide/spin.md`, `docs/guide/axum-dev.md`, and -`crates/trusted-server-integration-tests/README.md` only in relevant sections. - -- [ ] Add the optional publisher setting to the example configuration and canonical settings - table: default 5000, range 1–60000, chain scope after EOF, no effect without registration. - Put comments above configuration values, following repository conventions. -- [ ] Document `.with_html_finalizer`, object-safe local borrowing futures, terminal ordering, - shared document capture state, bounded replacement writes, unchanged fallback, and the - distinction between stage-input bytes and origin bytes. Include a fictional example. -- [ ] Explain opted-in whole-page latency, unchanged no-finalizer streaming, auction budget - interactions, shared-template bypass, `private, no-store`, raw readthrough separation, - and preservation of internal markers/CSP/RSC semantics. -- [ ] Describe timer availability, cooperative scheduling, Fastly host bounds, remote-side-effect - limitations, and independent KV/clock work without copying volatile adapter matrices into - several guides. Link to the canonical integration contract from each affected guide. -- [ ] Document the feature-gated fixture and exact runtime commands, prerequisite versions, - generated temporary workspace behavior, and how to record blocked smoke cases. -- [ ] Run `cargo doc -p trusted-server-core --no-deps --all-features`, `npm run lint` and - `npm run format` from `docs`, plus the repository Markdown check from Task 13. - Validate new local links. Commit as `Document asynchronous HTML finalizer behavior`. - -### Task 13: Run complete gates and review the implemented change against both documents - -**Spec coverage:** Sections 14, 16, and every coverage row below. - -**Files:** No additional production changes unless a check exposes a defect in this feature. - -- [ ] Inspect the diff for scope: no generic streaming trait change, EdgeZero repin, mandatory - store, real signing consumer, JS behavior change, or unrelated refactor. Check feature-off - fixture exclusion and request-state isolation. -- [ ] Run these Rust gates from the workspace root, using the target-matched aliases: - -```bash -cargo fmt --all -- --check -cargo clippy-fastly -cargo clippy-axum -cargo clippy-cloudflare -cargo clippy-cloudflare-wasm -cargo clippy-spin-native -cargo clippy-spin-wasm -cargo clippy-cli -cargo clippy-codegen -cargo test-fastly -cargo test-fastly-reuse -cargo test-axum -cargo test-cloudflare -cargo test-spin -cargo test --manifest-path crates/trusted-server-integration-tests/Cargo.toml --test parity -cargo test -p trusted-server-core --test template_build_digest -cargo clippy -p trusted-server-core --test template_build_digest -- -D warnings -``` - -- [ ] From `crates/trusted-server-js/lib`, run `node build-all.mjs`, `npx vitest run`, and - `npm run format`. From `docs`, run `npm run lint` and `npm run format`. - Install locked docs dependencies with `npm ci` if absent; then from root run: - -```bash -docs/node_modules/.bin/prettier --config docs/.prettierrc --check "*.md" ".claude/**/*.md" ".github/**/*.md" "crates/**/*.md" "scripts/**/*.md" "tinybird/**/*.md" -``` - -- [ ] Run the existing shared-template controls: - -```bash -./scripts/template-cache-local-test.sh -./scripts/template-cache-local-test.sh inline -``` - -Also run the feature's four runtime commands from Task 11 and the existing integration -jobs described in `scripts/integration-tests.sh` and `scripts/integration-tests-browser.sh` -where the feature changes their exercised routes. Record setup and exact outcomes; do not -substitute an empty-auction fixture for the existing winning-bid checks. - -- [ ] Map implementation and tests to every coverage row below. Have a reviewer inspect - the spec and plan together with the final diff; resolve behavioral contradictions centrally. - Do not weaken a regression assertion merely to match a new implementation. -- [ ] Record passed gates and all unavailable/blocked runtime cases in the PR handoff. If any - required gate is unresolved, keep the PR draft. No merge/deploy authorization is implied. - -## 4. Bidirectional spec/plan coverage - -Every requirement must have a task and evidence, and every task must have a requirement. -Use this table during review, not just as a task checklist. - -| Spec contract | Implementation tasks | Required evidence | -| -------------------------------------------------------------------------------------- | -------------------- | ------------------------------------------------------------------------------- | -| Sections 1, 4, 5: HTML terminal phase; generic streaming unchanged; no unrelated work | 1, 3, 8, 13 | API/diff inspection; unchanged legacy streaming tests | -| Section 6.1: ordered registration and one request-local document state | 3, 10 | factory-order and capture-state tests; request/reuse isolation | -| Section 6.2: local borrowing future and bounded replacement sink | 1, 5 | non-`Send` execution; poisoned writer tests | -| Section 7.1: no-finalizer behavior unchanged | 3, 8, 9 | prefix before EOF; one synchronous EOF; no timer/finalizer I/O | -| Section 7.2: drain decoder, synchronous EOF, terminal chain, seam, encoder | 5, 8, 9 | all body/sink/auction/codec combinations; exact event ordering | -| Section 7.3: auction deadline unchanged; opted-in delay explicit | 8, 9, 11, 12 | unchanged dispatch deadline; prefix-before-collection; expired-auction behavior | -| Section 7.4: cancellation releases borrows and does not lose owned input | 5, 8 | drop observer, buffer identity, exactly-once auction abandonment | -| Section 8: capped collection/candidate capacity; safe optional overflow bypass | 1, 6, 9 | boundary/overflow/capacity tests; exact replay; old fatal caps | -| Section 9.1: default 5000 and validated 1–60000 chain budget | 2, 12 | defaults/TOML/range tests; canonical configuration docs | -| Sections 9.2–9.3: optional SDK timers, one absolute budget, Fastly limitations | 1, 2, 4, 5, 11 | deterministic deadline tests; actual-runtime timings; bounded host calls | -| Section 10: stage-local fallback, error categories, markers, panic behavior | 1, 5, 9, 12 | each outcome/candidate failure; late/tie behavior; reviewed callback contract | -| Section 11.1: template bypass before lookup/storage and consistent inline fallback | 7, 9 | warm-hit bypass; no store/reservation; disabled cache behavior restored | -| Section 11.2: downstream privacy, raw-origin cache separation, conditional/range rules | 7, 8, 10 | private headers with no auction; raw-hit execution; four-header stripping | -| Section 11.3: existing body eligibility, status/method/framing/encoding | 7, 8, 9 | bodiless/no-HTML/unsupported cases; non-2xx HTML; header preservation | -| Section 12: every materialized/streamed and lazy/buffered branch | 8, 9, 10 | route matrix and host parity; no synchronous-only skip | -| Section 13: controlled logging and preserved auction telemetry | 5, 8, 11 | no sensitive log fields; outcome counts and abandonment tests | -| Section 14: deterministic, real-runtime, and full repository verification | 9, 10, 11, 13 | named regressions, runtime reports, complete gates | -| Sections 15–16: scoped delivery and readiness conditions | 12, 13 | guides, diff review, requirements/evidence mapping | - -## 5. Review and handoff record - -Before implementation planning is considered ready, review both documents for: - -- Spec requirements omitted from the plan or tasks that contradict the spec. -- Plan additions without a spec requirement, especially fixtures becoming production - consumers, private buffering reappearing, or deadlines multiplying per callback. -- Unrealistic runtime commands, unsupported cfg combinations, and existing behavior that - the plan mistakenly assumes instead of checking. -- Ownership of current HTML during timeout, shared state during factory construction, and - early cache admission that an EOF-only decision could not influence. - -Document review results describe planning readiness only. They do not prove the feature -is implemented, regression-free, or ready for deployment. Execution begins only after -the user requests implementation. - -### Review completed on 2026-10-12 - -The standalone spec review and both directions of the independent spec/plan alignment -review approved planning readiness with no unresolved substantive findings. - -The review corrected these points before handoff: - -- Policy-admission tests in Task 7 no longer require callback execution first wired in - Task 8; execution-dependent assertions now follow the routing implementation. -- The deadline type referenced by the callback API is defined in Task 1, avoiding an - unbuildable intermediate contract commit. -- Core edits have a direct primary-package lint checkpoint; non-2xx HTML is an explicit - acceptance test rather than implicit coverage. -- Overflow route tests distinguish rewrite expansion from preexisting fatal decoder caps. -- Workers fixture output stays in the temporary workspace; runtime timeout tests verify - timer expiry rather than accidentally passing on an early transport error. - -This historical record covers planning review only. The linked execution record now -maps the implemented contracts and observed verification to both documents. +- [x] Write a failing chain test: streaming A, yielding async B, downstream C; assert C transforms B's async output before C receives EOF and every stage receives EOF once. +- [x] Write pending-callback tests proving parser and prior-stage ready bytes are available first. +- [x] Write timeout tests proving downstream synchronous EOF still executes; never race the whole composite future. +- [x] Add boundary-aware marker tests for ordinary-output/EOF splits, buffered/overflow stages, and upstream replacements feeding downstream stages. +- [x] Expose crate-private concrete HTML composition for the enabled driver while preserving public opaque helpers for synchronous callers. +- [x] Implement parser finish, stage-index advancement, driver-owned fallback, bounded candidate and downstream ordinary-chunk propagation. +- [x] Reuse one absolute deadline/timer. Drop each losing callback before selecting output. +- [x] Check factory/session capability consistency before parser effects; reject synchronous composite use with async stages. +- [x] Run `cargo test -p trusted-server-core --all-features async_eof` plus existing HTML/Next.js tests. + +## Task 5: Explicit bounded buffering + +**Files:** `html_processor/finish.rs` and its tests. + +- [x] Write failing tests for full retained input at EOF, cap crossing and one-time replay; verify callbacks do not start after overflow. +- [x] Add framework-owned stage-output retention only for declaring factories; default stages stream through. +- [x] Move retained bytes into fallback at EOF. Skip optional work for oversized streaming EOF fallback without truncation. +- [x] Preserve exact allocation identity on unchanged/error/timeout; preserve fatal outer buffered-output limits. +- [x] Run holder/finish tests and memory-boundary cases. + +## Task 6: Publisher, auction and codec migration + +**Files:** `publisher/html_finalization.rs`, `publisher.rs`, publisher test helpers. + +- [x] Add failing tests for early output while EOF work is pending across materialized/streamed sources and write/lazy sinks. +- [x] Remove universal collection and separate sessions from the driver. Use the concrete chain's stepped EOF operations. +- [x] Drain decoder/parser ready output before starting stage callbacks. Finish auction/encoder after all stage EOF. +- [x] Cap the held auction suffix with `publisher.max_buffered_body_bytes`, check before growth, split releasable prefixes before copying, and test exact-cap/crossing plus fatal abandonment for large post-body content. +- [x] Keep exact flush-before-await barriers; avoid flushing every replay chunk. +- [x] Guard direct async entry points for operator permission and synchronous helpers before effects. +- [x] Preserve range/bodiless/unsupported bypass, actual status, early template-cache admission and independent privacy. +- [x] Migrate existing full-document matrix tests via a test-only buffered processor adapter, not another terminal list. +- [x] Run `cargo test -p trusted-server-core --all-features` and Fastly/WASI regression tests. + +## Task 7: Runtime fixture and guides + +**Files:** `test_support/async_html_finalization.rs`, registry fixture tests, runtime Python configuration, guides, example TOML. + +- [x] Migrate the fixture to an identity processor requesting bounded output retention and optional HTTP finish. +- [x] Add explicit operator permission to enabled harness configurations; disabled controls retain default permission. +- [x] Require a non-retaining EOF fixture/control proving a prefix reaches the Fastly client before a pending callback completes; keep retained controls for fallback and other runtimes. +- [x] Update public examples and guides to one callback API, stage order, fallback scope, explicit buffering and actual adapter TTFB limits. +- [x] Preserve existing cleanup, deadline, exact fallback, privacy and auction harness assertions. +- [x] Run Bash/Python validation and real-process cleanup tests. + +## Task 8: Full verification and review + +- [x] Run cargo formatting and all eight target-matched Clippy aliases. +- [x] Run all five adapter test aliases, native core all-feature/digest tests and lint, CLI/codegen tests, parity and exact integration-test CI selections. +- [x] Run existing ESI/inline/purge and winning-bid controls; preserve no-hook cache behavior. +- [x] Run JS build/tests/format, docs lint/build/format and repository Markdown format. +- [x] Run actual Axum, Workers, Spin and Fastly enabled/disabled/success/hung/auction controls on final source; qualify installed-tool/runtime limits honestly. +- [x] Obtain independent architecture and implementation review; fix findings and repeat only affected checks. +- [x] Record source and evidence in a new verification report; do not repurpose historical counts as revised-design evidence. + +## Task 9: Publish the reviewed revision + +- [ ] Verify clean intended diff, archive links, spec/plan/implementation alignment and fresh passing checks. +- [ ] Commit using repository sentence-case conventions and push the existing feature branch. +- [ ] Rewrite draft PR description around generic ordered EOF completion and operator permission; keep issue1241 as the target and keep draft status. +- [ ] Confirm remote head/body and report any remaining maintainer decisions or pending hosted checks. + +## Acceptance mapping + +| Specification | Plan coverage | +| ------------------------------------------------- | -------------- | +| Generic callback and single lifecycle | Tasks 2–4 | +| Operator acceptance and early metadata | Task 3, Task 6 | +| EOF order and release before waits | Tasks 4, 6 | +| Optional bounded retention and fallback ownership | Tasks 2, 4, 5 | +| Deadline/cancellation and downstream EOF | Tasks 2, 4, 6 | +| Auction/codec/partial/cache/privacy safeguards | Task 6, Task 8 | +| Honest adapter TTFB contract and consumer scope | Task 7, Task 8 | +| Reproducible verification and draft publication | Tasks 8–9 | diff --git a/docs/superpowers/plans/2026-10-12-1241-review-hardening.md b/docs/superpowers/plans/2026-10-12-1241-review-hardening.md index c529d6bf0..3836a0dcf 100644 --- a/docs/superpowers/plans/2026-10-12-1241-review-hardening.md +++ b/docs/superpowers/plans/2026-10-12-1241-review-hardening.md @@ -1,5 +1,9 @@ # HTML finalization review hardening implementation plan +**Historical record:** This describes the buffered terminal implementation before the +2026-10-13 ordered EOF revision. Its verification counts do not verify the revised +implementation. See the current spec and plan for the active architecture. + > **For agentic workers:** Use superpowers:executing-plans for central implementation and independent review. Checkboxes record observed completion. **Goal:** Close the confirmed response-eligibility and maintenance gaps from the independent review of PR #1290. @@ -11,7 +15,7 @@ ## Reviewed design refinements The user authorized fixes after the read-only review. These refinements supplement -the [existing design](../specs/2026-10-12-1241-async-html-finalization-design.md). +the [existing design](../archive/2026-10-12-1241-buffered-terminal-design.md). - Terminal callbacks receive the actual response status. Full 404/500 HTML remains eligible; a signing consumer may return `Unchanged` for unwanted statuses. diff --git a/docs/superpowers/reviews/2026-10-12-1241-async-html-finalization-implementation.md b/docs/superpowers/reviews/2026-10-12-1241-async-html-finalization-implementation.md index fa3aca8cc..2f0a4f6da 100644 --- a/docs/superpowers/reviews/2026-10-12-1241-async-html-finalization-implementation.md +++ b/docs/superpowers/reviews/2026-10-12-1241-async-html-finalization-implementation.md @@ -1,5 +1,9 @@ # Asynchronous HTML finalization implementation review +**Historical record:** This describes the buffered terminal implementation before the +2026-10-13 ordered EOF revision. Its verification counts do not verify the revised +implementation. See the current spec and plan for the active architecture. + **Date:** 2026-10-12 **Branch:** `feat/1241-async-html-finalization` @@ -11,9 +15,9 @@ status metadata, a targeted telemetry clock correction, and actual-runtime CI. T Workers enabled-auction control now passes. See the linked hardening execution record for the latest verification; the original evidence below is historical. -**Design:** [Optional asynchronous HTML document finalization](../specs/2026-10-12-1241-async-html-finalization-design.md) +**Design:** [Optional asynchronous HTML document finalization](../archive/2026-10-12-1241-buffered-terminal-design.md) -**Plan:** [Implementation and verification tasks](../plans/2026-10-12-1241-async-html-finalization.md) +**Plan:** [Implementation and verification tasks](../archive/2026-10-12-1241-buffered-terminal-plan.md) **Review hardening:** [Fixes, retained design decisions, and renewed verification](../plans/2026-10-12-1241-review-hardening.md) diff --git a/docs/superpowers/reviews/2026-10-13-1241-ordered-eof-verification.md b/docs/superpowers/reviews/2026-10-13-1241-ordered-eof-verification.md new file mode 100644 index 000000000..d435c9e3d --- /dev/null +++ b/docs/superpowers/reviews/2026-10-13-1241-ordered-eof-verification.md @@ -0,0 +1,131 @@ +# Ordered HTML EOF completion: implementation and verification + +**Date:** 2026-10-13 + +**Branch:** `feat/1241-async-html-finalization`; draft PR #1290; issue #1241. + +**Review baseline:** `3981d0222bf0767ba3c8de767d633018b0df4b58`. +The newer remote merge from `main` was preserved before final verification. + +This report covers the revised [specification](../specs/2026-10-12-1241-async-html-finalization-design.md) +and [plan](../plans/2026-10-12-1241-async-html-finalization.md). +The October 12 reports describe the superseded buffered terminal API. + +## Resulting design + +Existing processors keep synchronous `process_chunk` calls. An optional borrowing +`async_finish` callback finishes a stage before downstream EOF. Accepted async output +passes through downstream ordinary processing. Factories declare capability and optional +output retention in the existing registration list; there is no separate production +finalizer lifecycle. `publisher.allow_async_html_finish` defaults to false and enabled +capability without permission is a configuration error. + +Only declaring stages withhold output. Core owns the fallback and bounded candidate +outside each callback. Timeout, optional errors, poisoned candidates, late completion +and invalid assembly-marker counts retain the fallback allocation. One absolute budget +is shared; expiry skips later optional work while still finishing synchronous EOF. +Auction collection and encoding finish after the chain. Auction suffix storage is capped +before growth. Range/bodiless/unsupported response handling, actual status, template-cache +bypass, raw-origin readthrough and independent privacy policies remain covered. + +The retention cap bounds withheld output and candidates. After overflow, the collector +becomes ordinary ready output; the existing `Vec` chunk interface can coalesce it with +the crossing chunk above that optional cap. Outer buffered-response limits still apply. + +## Coverage and review + +| Requirement | Implementation and evidence | +| --------------------------------------------------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| Optional local future; unchanged synchronous entry points | `streaming_processor.rs`, `streaming_processor/finish.rs`: default absence, borrowing non-Send work and pre-effect synchronous rejection | +| One ordered registration list and explicit permission | `integrations/registry.rs`, `settings.rs`: default false, missing permission, permission alone and inconsistent retention metadata | +| Stage-by-stage EOF and ready output | `html_processor/finish.rs`, publisher driver: downstream transforms async output before its own EOF; lazy materialized/streamed input releases prefix before pending work | +| Cancellation and replacement acceptance | Leaf tests cover timeout borrowing/allocation identity, errors after partial writes, ignored poisoned writes, timer ties, late results, empty replacement and earlier accepted output surviving later failure | +| Retention and assembly boundaries | Exact-cap/crossing replay, creation/removal across streamed boundaries, retained markers, overflow bypass and accepted upstream replacement feeding downstream boundary validation | +| Auction, codec, cache and eligibility | Existing migrated publisher matrix, winning-bid/raw-origin controls, partial/status/flush regressions and fatal suffix overflow with exactly one abandonment summary | +| Actual runtime behavior | Fresh local Axum, Workers, Spin and Fastly disabled/success/hung/auction controls; required non-retaining Fastly early-byte control | + +Independent architecture and implementation reviews found and resolved constructor-time +deadline expiry, session validation before lazy reads, parser EOF failure classification, +remaining-budget timer initialization, timer-expiry propagation, reset forwarding and +missing acceptance tests. The final bidirectional document/source review approved the +expanded coverage. Review is separate from executed verification. + +## Executed evidence + +Observed red/green failures included missing callback/metadata/chain contracts, reset +state leakage, timer initialization and expiry propagation, constructor budget exhaustion, +and an undeclared async session consuming a body read before rejection. The final focused +`cargo test -p trusted-server-core --all-features async_eof` run passed 30 tests. + +The newer main merge exposed a separate test-only compatibility failure: Cargo exports +`TRUSTED_SERVER__GIT_VERSION` to test processes, and the legacy configuration loader treated +it as application configuration. An explicit regression failed before filtering only that +metadata from the test-only environment source. Production configuration loading and +version-header behavior are unchanged. The final native all-feature run passed 2,986 unit +tests, 10 git-version filesystem tests, 10 template build-digest tests and 4 documentation +tests (5 existing ignored documentation examples). + +Fresh JS build, formatting and type checks passed; Vitest passed 1,211 tests in 50 files. +Docs lint, formatting and build passed. Python fixture syntax, Bash syntax and all three +real-process cleanup tests passed. + +### Runtime observations + +All runs build fresh fixture artifacts, create isolated configuration, use loopback +services and reap their child processes. Disabled controls make no lookup; success runs +check exact output/privacy; hung runs retain exact fallback bytes; auction runs preserve +assembly. The fixture budget is 750 ms, distinct from the production default of 5,000 ms. + +| Runtime | Success request | Hung request | Observed timeout | Auction request | +| ------- | --------------- | ------------ | ---------------- | --------------- | +| Axum | 0.260 s | 0.819 s | 754 ms | 0.232 s | +| Workers | 0.230 s | 0.822 s | 750 ms | 0.251 s | +| Spin | 0.224 s | 0.805 s | 751 ms | 0.216 s | +| Fastly | 0.224 s | 0.815 s | 751 ms | 0.213 s | + +Fastly's additional non-retaining control received its first body byte at **0.0053 s**, +before the 150 ms lookup completed; total request time was 0.217 s. The harness asserts +actual body-byte timing against the lookup-completion event, not response headers. +This proves the supported Fastly streaming path can release ready bytes before EOF I/O. +It does not promise arbitrary processors or buffered adapters have zero TTFB cost. + +Commands: `scripts/async-html-finalization-local-test.sh axum --auction`, and the +`cloudflare`, `spin`, `fastly` variants with `--release --auction`. +Workers uses isolated `worker-build` 0.8.5 and `wasm-bindgen` 0.2.126 matching the lockfile. +Fastly timeout/streaming controls use isolated Viceroy 0.19 for backend timeout simulation; +ordinary WASI tests and template controls use the repository-pinned Viceroy 0.17. +Spin 4.0.0 passed the actual local controls; EdgeZero's CLI still warns that its verified +SQLite writer range is Spin 2–3. Repository runtime/tool pins were not changed. + +## Evidence boundary + +The final revised source snapshot (changed Rust/fixture/example files, sorted paths and +contents separated by NUL bytes) has SHA-256 +`766c30f633421b259436a5d220ac8e52bf73308cedc81462cd4f8ece9269cdf8`. +Earlier pre-merge runs are not substituted for final evidence. + +| Final local check | Result | +| ------------------------------------------ | -------------------------------------------------------------------------------------------------------------------------------- | +| All eight target-matched Clippy aliases | Passed; Fastly repeated after the test-only metadata fix | +| Core all-target/all-feature Clippy | Passed | +| `cargo test-fastly` on pinned Viceroy 0.17 | Fastly 222, core 2978 (6 existing ignored), JS 10, git-version 10, OpenRTB 21, docs 4 (5 existing ignored) passed | +| `cargo test-fastly-reuse` | 222 passed | +| Native adapter test aliases | Axum 49, Workers 54, Spin 88 passed | +| Native core/digest gates | 2986 core, 10 git-version, 10 digest, 4 docs passed; digest lint passed | +| CLI helper and codegen | 892 executed CLI tests including selected ignored browser controls; codegen 7 passed | +| Integration parity and lint | 18 parity tests and all-target integration lint passed | +| Exact CI integration selection | Generator 8, docs fixture 1, integration 36, parity 18 passed; separate focused Fastly EC also passed | +| Existing template-cache controls | ESI 22, inline 9 and purge 34 passed | +| JS, docs and formatting | Fresh build, 1211 tests/type checks, JS format, docs lint/build/format, outside-doc Markdown, Rust format and diff checks passed | +| Runtime/cleanup | All four actual runtimes plus Fastly early-byte proof passed; three cleanup tests and syntax checks passed | + +Native checks used fresh JS bundles through the supported `TSJS_PREBUILT_DIR` setting. +The merged main change intentionally rejects the old `TSJS_SKIP_BUILD` setting. Real +runtime harnesses built their own private bundles. Template controls ran an external +copy of the unchanged harness with only worktree/shared-target artifact paths adjusted; +the first ESI attempt failed to locate its GPT artifact, and the final attempt corrected +that harness environment path. No assertions were relaxed or source tooling changed. +Hosted CI is separate from local verification. Maintainers still need to accept the API, +conservative processed-template cache policy and consumer-owned marker placement before +the PR leaves draft. No background signing service, external queue or publisher workflow +is required or introduced. diff --git a/docs/superpowers/specs/2026-10-12-1241-async-html-finalization-design.md b/docs/superpowers/specs/2026-10-12-1241-async-html-finalization-design.md index adef34dbf..b607ce13c 100644 --- a/docs/superpowers/specs/2026-10-12-1241-async-html-finalization-design.md +++ b/docs/superpowers/specs/2026-10-12-1241-async-html-finalization-design.md @@ -1,687 +1,217 @@ -# Optional asynchronous HTML document finalization - -**Date:** 2026-10-12 - -**Status:** Implemented on `feat/1241-async-html-finalization`; review hardening and verification recorded below - -**Issue:** trusted-server #1241 - -**Inspected baseline:** `2a0cb0444dc335bd09105abe5952329bbbdfb621` - -**Implementation plan:** [Task-by-task implementation and verification](../plans/2026-10-12-1241-async-html-finalization.md) - -**Implementation review:** [Requirement coverage, verification, and runtime limitations](../reviews/2026-10-12-1241-async-html-finalization-implementation.md) - -**Review hardening:** [Response metadata, range safety, telemetry clock, and CI refinements](../plans/2026-10-12-1241-review-hardening.md) - -## 1. Decision - -Add an opt-in asynchronous final-document phase to Trusted Server's HTML integration -pipeline. An enabled finalizer receives the completed, synchronously transformed HTML -document and may await request-time HTTP or KV services before producing replacement -HTML. Core owns both the retained document and a separate bounded replacement writer. - -Keep `StreamProcessor::process_chunk` synchronous and leave `StreamProcessor` and -`StreamingPipeline::process` unchanged. The new capability is registered explicitly as -an HTML finalizer, rather than being an optional method that some generic synchronous -drivers cannot execute. - -Only processable HTML documents with an enabled finalizer take this path. Existing -integrations retain their streaming behavior. Finalizers execute in their own registration -order, after all existing synchronous HTML processors. This is a terminal phase, not an -interleaved asynchronous extension to every streaming stage. - -One absolute finish deadline covers the entire finalizer chain. On optional-finalizer -failure, retain the bytes entering that finalizer; on deadline expiry, also skip all -remaining finalizers. Preserve existing fatal handling for origin, parser, synchronous -processor, compression, and downstream-write failures. - -For the first version, enabled HTML finalizers bypass shared-template cache reads and -writes before lookup. Their processed HTML responses use the existing synthesized HTML -`private, no-store` policy. Supporting cacheable asynchronous finalizers requires a -separate design for output identity, freshness, and invalidation. - -No EdgeZero revision change is required. Trusted Server owns this document lifecycle and -its request service container. A reusable EdgeZero timer can be considered separately. - -## 2. Verified problem and limits of the evidence - -At the inspected baseline: - -- `streaming_processor.rs::StreamProcessor` provides synchronous `process_chunk` and - `reset` methods. It offers no future-returning operation. -- `html_processor.rs::HtmlWithStreamingProcessors` forwards each chunk and the same - `is_last` value through the entire synchronous chain immediately. -- `IntegrationHtmlStreamProcessorFactory` creates per-document processors. Its context - contains hosts, scheme, and document-local state, but no request services. -- `publisher.rs` awaits auction collection outside synchronous processing callbacks. - The auction already has an application-specific asynchronous seam. -- Buffered materialized responses still use `StreamingPipeline::process` or the separate - reader-based auction hold loop. An asynchronous stream-helper change alone would miss - those paths. - -A local probe using the actual `StreamingPipeline` verified that `now_or_never()` completes -an immediately ready lookup but discards a lookup that returns `Pending`. Awaiting the same -yielding lookup completes it. This establishes the API gap; it does not establish that -every operation yields on every adapter. A local or already available KV result can be -immediately ready. - -No shipped integration at this baseline uses the problematic workaround. The reported -article-signing consumer and the contributor's unsubmitted patch, runtime timings, and -test results have not been independently validated. This is an extension capability, -not a claim that current synchronous integrations are broken. - -The first demonstrated requirement is whole-document text signing: obtain a signature -after reading the article and annotate the page. Such a consumer already withholds the -document. Making its buffering an explicit, bounded core responsibility adds no new -whole-document waiting requirement to that consumer. - -## 3. Goals - -1. Allow a registered HTML integration to await I/O after synchronous document processing - ends, on all four adapters. -2. Make buffer ownership, limits, cancellation, and unchanged-byte fallback enforceable - by core rather than implicit integration conventions. -3. Preserve early streaming, chunk boundaries, processor EOF counts, and auction-seam - timing when no finalizer is enabled. -4. Support both `EdgeBody::Once` and `EdgeBody::Stream`, with and without an auction. -5. Keep decoder validation ahead of finalization, and auction-seam replacement and encoder - finalization after it on the opted-in path. -6. Prevent warm shared-cache hits or downstream response caches from bypassing required - per-request finalization. -7. Specify observable, deterministic failure behavior and tests before implementation. - -## 4. Non-goals - -- Implementing an article-signing integration, an external API protocol, signing-key - management, or a new KV binding. -- Making `lol_html` callbacks await, making every chunk method asynchronous, or inserting - asynchronous processors between arbitrary synchronous processors. -- Adding another framework HTML parser or reparsing the finalizer's output. -- Changing the standalone RSC Flight processor, asset rewriting, EC behavior, consent - gates, or auction admission and transport guarantees. -- Guaranteeing interruption of synchronous Fastly host calls or CPU-heavy callback work. -- Guaranteeing cancellation or rollback of remote HTTP/KV side effects. -- Allowing finalizers to edit HTTP headers or alter the assembly protocol. -- Supporting shared-template storage of asynchronous output in the first version. -- Adding deployment manifests, a general task scheduler, background retries, persistent - queues, or a new platform clock abstraction. - -## 5. Alternatives and tradeoffs - -| Approach | Benefit | Cost or limitation | Decision | -| ------------------------------------------------ | --------------------------------------------------------------------------------------------------------------------------- | ---------------------------------------------------------------------------------------------------------------------------------------------- | ---------------------------------------------------- | -| Optional async finish on every `StreamProcessor` | Supports asynchronous completion inside a mixed streaming chain | Generic synchronous drivers cannot await it; downstream EOF propagation, wrappers, holding behavior, and services become broader contracts | Defer until an actual consumer requires interleaving | -| Make `process_chunk` asynchronous | Supports waits at every chunk boundary | Changes every processor and driver for an EOF-only need; boxed async trait methods can allocate per chunk; parser callbacks still cannot await | Reject | -| Explicit asynchronous HTML terminal phase | Matches the known whole-document consumer; leaves existing streaming traits intact; core can own and bound fallback buffers | Enabled pages wait for EOF; cannot interleave with synchronous stages | Select | -| Perform integration I/O before HTML rewriting | Produces metadata before transformations | Still requires buffering enabled pages and delays parsing; lacks the completed transformed document | Reject for this requirement | - -Opt-in buffering does not mean buffering every page. A registry without finalizers keeps -the current path. Within an opted-in page, all existing synchronous processors run first, -in their existing order, and receive exactly one EOF notification. Finalizers then run in -their own order. Registration order does not imply interleaving between these two phases. - -The framework can provide completed HTML and existing document-local capture state without -adding a parser. The consumer must separately explain how it extracts text and preserves -HTML semantics; this specification does not validate its extraction implementation. - -## 6. Ownership and registration - -### 6.1 Integration registration - -Add `IntegrationHtmlFinalizerFactory` and `IntegrationHtmlFinalizer` contracts alongside -the existing integration factory contracts. Add an empty-by-default finalizer collection -to `IntegrationRegistration` and `IntegrationRegistry`, plus the builder method -`with_html_finalizer`. Preserve factory registration order; do not sort finalizers by ID -or deduplicate registrations silently. - -A factory is `Send + Sync` and can live in the retained application registry. A factory -creates a mutable, single-use finalizer instance for each processed document using the -existing owned `IntegrationHtmlStreamContext`, including its `IntegrationDocumentState`. -The finalizer instance and its returned future need not be `Send`. - -The same document-state instance must be available to parser hooks, synchronous streaming -processors, and finalizers for that document. It must not be recreated between phases or -retained across requests, including reusable Fastly sandboxes. - -Currently `create_html_processor` constructs that state internally. The implementation -must expose the same state through a request-local construction helper or an explicit -construction input; creating a second state for finalizers would disconnect their capture -data from the parser. Preserve the current public helper's synchronous behavior. - -Finalizer presence must be discoverable from enabled registration metadata before shared -template lookup. It must not depend on polling a future, inspecting completed HTML, or -creating a finalizer instance. A factory may later return `Unchanged` when no relevant -article is present, but that cannot retroactively authorize a shared-cache lookup. - -Disabled integrations contribute no finalizers and cause no finalizer-related buffering, -privacy changes, timer creation, or cache bypass. No global setting enables this feature -without an integration explicitly registering a finalizer. - -### 6.2 Callback contract - -Use a regular, object-safe method returning `LocalBoxFuture`, with an explicit lifetime -covering mutable session state and borrowed inputs. Do not add native `async fn` to a trait -used behind `dyn` without an object-safety strategy, and do not require `Send` futures. - -The following is the intended API shape, not implementation code: - -```rust -pub trait IntegrationHtmlFinalizer { - fn finalize<'a>( - &'a mut self, - input: HtmlFinalizationInput<'a>, - ) -> LocalBoxFuture<'a, Result>>; -} - -pub struct HtmlFinalizationInput<'a> { - pub response_status: StatusCode, - pub html: &'a [u8], - pub replacement: &'a mut HtmlReplacementWriter, - pub services: &'a RuntimeServices, - pub deadline: &'a HtmlFinalizationDeadline, -} - -pub enum HtmlFinalization { - Unchanged, - Replace, -} -``` - -`Unchanged` emits the driver-owned input without cloning it. `Replace` commits only bytes -successfully written into `HtmlReplacementWriter`. Neither the original buffer nor an -unbounded replacement `Vec` is transferred into the future. - -The method must return its future promptly and perform I/O inside that future. Factory -creation is synchronous, request-local setup; it must not perform an outside lookup or -hide whole-document buffering. Public implementations must follow repository documentation -and error conventions; `HtmlFinalizationError` is a concrete error type with `Report`. - -The replacement writer is a core-owned sink, separate from `html`. Its append operations -are fallible and enforce the byte cap before extending storage. A failed append poisons -that attempt: a later `Replace` cannot commit truncated output even if the callback ignored -the writer error. Empty replacement output is valid only when no required seam token would -be lost; `Unchanged` is the explicit no-replacement result. - -The framework does not make arbitrary integration allocations safe. The cap covers its -retained HTML and replacement storage. Consumer extraction buffers and any external -response buffering must have their own bounds and be reviewed. - -## 7. Processing lifecycle - -### 7.1 No finalizers - -```text -origin -> existing decoder -> existing HTML parser and synchronous processors - -> existing auction seam controller -> existing encoder -> client -``` - -Do not collect a whole document, allocate a replacement writer or boxed finish future, -create a timer, or move auction collection to EOF. Keep existing synchronous helper -semantics. A cheap registration-presence check is acceptable. - -### 7.2 Enabled finalizers - -```text -origin -> existing decoder -> existing HTML parser and synchronous processors - -> bounded retained HTML - -> ordered asynchronous HTML finalizers - -> existing auction seam controller -> existing encoder -> client -``` - -The driver performs these steps: - -1. Create normal document-local parser/processor state and finalizer sessions. -2. Pull and decode origin chunks, run synchronous transformations, and retain the resulting - HTML under the cap. No finalizer-visible HTML is emitted before this phase completes. -3. Drain and validate decoder tails, then finish the existing synchronous chain exactly - once. Include its final output in the retained HTML. -4. Start one absolute finalization deadline and, on supporting adapters, one timer. -5. For each finalizer, lend the current retained HTML immutably and a fresh bounded - replacement writer mutably. Race its future against the same remaining deadline. -6. For an accepted `Replace`, validate the candidate and move it into current retained - storage. Drop the previous stage's input. For `Unchanged` or a soft failure, keep the - current input and discard the candidate. Finalizer B sees the accepted output of A. -7. On deadline expiry, drop the active future and candidate, skip the remaining finalizers, - and retain the current input. Earlier accepted replacements remain accepted. -8. Feed the accepted HTML into the existing auction seam controller in bounded chunks. - Emit its available prefix before collecting bids. Preserve no-seam fallback and terminal - auction telemetry. Then emit the encoder trailer and finish the response. - -Already completed synchronous processors are never re-entered with replacement output. -This prevents duplicate parsing, script injection, Next.js processing, and EOF calls. - -Callbacks observe post-rewrite HTML before reader-specific auction insertion. They cannot -claim to sign the final assembled response or content added later by the auction. A text -signing consumer must document its article scope, exclusions, and canonicalization so -framework scripts, placeholders, and later ad content do not invalidate its claims. - -### 7.3 Auction behavior - -Auction dispatch and its existing deadline remain unchanged. They are not restarted or -extended for finalization. Full-document withholding can make an opted-in page's auction -older by the time its seam reaches collection; an expired auction follows today's -empty-bid behavior. This is an explicit latency/monetization tradeoff for enabled pages. - -When no finalizer is enabled, retain the existing ability to collect at the structural -body seam before origin EOF. The terminal phase must not become a universal EOF barrier. -Missing structural body ends, bot/prefetch/consent suppression, non-HTML bodies, and -abandonment remain governed by existing auction policy. - -### 7.4 Cancellation - -The driver must own current HTML outside the pending finalizer future. Keep a losing -future's lifetime separate from the retained buffer's ownership; explicitly drop it before -moving that buffer or inspecting the borrowed replacement writer. - -Reusing a single deadline across finalizers must not put the only fallback buffer inside -a chain future that is destroyed on timeout. The driver remains the owner at every await. - -A client disconnect drops active work and request-local buffers. It does not invoke another -finalizer, persist partial output, or initiate a background retry. Preserve the existing -dispatched-auction guard so terminal abandonment is recorded exactly once on supported -drop paths. Dropping local work is not a guarantee that a remote write did not occur. - -## 8. Memory and output limits - -Use `publisher.max_buffered_body_bytes` as the cap for both retained transformed HTML and -each replacement writer. Its existing default is 16 MiB. Do not add a second independently -configured page-size limit in this change. - -Check lengths with overflow-safe arithmetic before append or reserve. Avoid capacity -growth beyond the configured cap, including `Vec` geometric reserve growth. Allocate -incrementally; do not reserve the entire cap for every small page. - -At a finalizer await, core may hold one current HTML buffer and one replacement buffer, -each capped at that limit. It must not retain previous successful candidates or clone the -input for rollback. Codec state, pending chunks, consumer-owned state, and platform response -buffers are additional memory; this is not a claim that total request heap use is 16 MiB. - -If collection for the optional terminal phase would exceed the cap: - -- Disable all finalizers for that document before starting I/O. -- On a streaming sink, release the retained prefix, the chunk that crossed the limit, and - subsequent synchronous output through the normal auction/encoding path in order. Avoid - duplicate bytes and a fallback copy of the retained prefix. -- On a buffered sink, preserve the existing outer response-size cap. Optional-finalizer - bypass does not authorize an oversized buffered response. -- Existing raw-input, decoder, parser, or response-output limit failures remain fatal. - This fallback applies only when the new optional collection layer reaches its own cap. - -A replacement-writer overflow is a soft failure of that finalizer. Discard the candidate -and continue with its input, subject to the same remaining finish budget. Never raise the -cap to accept an integration's output. - -## 9. Deadline and adapter services - -### 9.1 Configuration - -Add `publisher.html_finalization_timeout_ms`, defaulting to `5000`. It covers only the -terminal phase after decoded/synchronously processed EOF, across all finalizers together. -It is not an origin-body timeout, auction timeout, or end-to-end response deadline. - -Accept values from `1` through `60000` inclusive and reject invalid settings at startup. -The upper bound keeps the adapter timer duration comfortably within its representable -range and gives the new optional phase a finite operator-controlled budget. Match TOML -deserialization and programmatic `Publisher::default()` behavior. The field has no runtime -effect when no finalizer is registered. - -The five-second default is a proposed product default based on the issue's example, not a -claim that all adapter execution budgets permit or guarantee five seconds. Operators must -also respect their platform's invocation limits. - -### 9.2 Timer contract - -Add an optional `PlatformTimer` service to `RuntimeServices`, defaulting to absent. The -service object is `Send + Sync`; its delay future is local and need not be `Send`. A regular -object-safe method can return `LocalBoxFuture<'static, ()>` for a `Duration`. - -| Adapter | Timer implementation | Finish deadline semantics | -| ---------- | ----------------------- | ----------------------------------------------------------------------------------- | -| Cloudflare | `worker::Delay` | Cooperative race against JS-backed I/O | -| Spin | `spin_sdk::time::sleep` | Cooperative race against runtime-backed I/O | -| Axum | `tokio::time::sleep` | Cooperative race on the existing Tokio executor | -| Fastly | No timer | Check elapsed budget at control boundaries; bound host calls within the integration | - -Keep SDK imports and concrete timer implementations in Trusted Server adapters. These -primitives exist in the pinned dependencies; do not add a Tokio dependency to core or -repin EdgeZero to obtain a timer. - -Use `web_time::Instant` for absolute budget accounting and elapsed observations in core. -Do not introduce `std::time::Instant::now()` or `SystemTime::now()` on the Cloudflare path. -The review hardening exception corrects the traced auction telemetry clock using -that existing abstraction. Broader clock sites reported in #1240 remain separate -work; one successful auction smoke does not verify all of them. - -### 9.3 Race and fallback rules - -Create one absolute deadline and retain one pinned timer across the entire chain. Poll -the deadline before active work at decision points. Check absolute expiry before a stage -starts and after it resolves; if completion and expiry coincide, deadline expiry wins. -Do not restart a full-duration sleep for each finalizer. - -Callbacks receive a deadline handle exposing remaining time and whether a cooperative -timer is available. Bound individual network requests and external response reads to the -remaining budget where the service permits it. Successful but late candidates are -discarded even if a timer could not run while the callback was polling. - -Timers cannot preempt a blocking host call or a callback that never yields. On Fastly, -consumer acceptance requires evidence that its calls are bounded independently, or that -it returns `Unchanged` without performing unsupported unbounded operations. Core cannot -promise a hard five-second release for arbitrary Fastly HTTP/KV operations. - -Cancellation frees borrowed buffers and releases the publisher's wait. It does not -universally abort transport work or reverse external writes. Consumers must tolerate -duplicate requests/retries and use idempotent external operations where required. - -## 10. Failure and validation contract - -Fallback bytes are the synchronously transformed input of the current finalizer, including -accepted earlier finalizers. They are not necessarily the origin's original bytes. No -rollback of prior accepted HTML or remote side effects is promised. - -| Event | Document behavior | Remaining finalizers | -| ----------------------------------------------------------------------------- | ---------------------------------------------------------------- | ------------------------------- | -| `Unchanged` | Keep current input; discard any candidate writes | Continue | -| Valid `Replace` within budget | Commit bounded candidate | Continue with replacement | -| Callback reports unavailable/rejected external work | Keep current input; discard candidate | Continue while budget remains | -| Replacement writer overflow or invalid required marker | Keep current input; discard candidate | Continue while budget remains | -| Overall deadline expires | Drop active future and candidate; keep current input | Skip all remaining | -| Optional collection limit reached | Release retained synchronous output through ordinary delivery | Skip all finalizers | -| Origin read, decoder, parser, synchronous processor, encoder, or sink failure | Existing fatal error/truncation behavior and auction abandonment | Stop | -| Callback panics | Existing panic behavior | No new panic-catching mechanism | - -Define controlled error categories for external failure and candidate validation. Keep -`Report` at orchestration boundaries and existing `io::Error` where -streaming I/O uses it. Do not turn unrelated processor errors into silent passthrough. - -Finalizers must preserve the parser-generated control marker, when present, as opaque -bytes. They must not remove, duplicate, or relocate it into script/text content. Core -checks the active token's required cardinality before committing a replacement; missing -or duplicated tokens reject that candidate. A count check cannot prove structural -placement. This remaining invariant is a reviewed integration contract, not a claim of -automatic HTML validation without parsing. - -Core does not inspect arbitrary replacement HTML for semantic validity or reconstruct -parser observations. Finalizers must preserve valid document structure, existing CSP -requirements, URL/RSC encodings, and the meaning of upstream rewrite decisions. Header -changes and transformations needing new parser-derived security observations require a -separate capability. The first consumer should add signature metadata without changing -executable markup or article text after canonicalization. - -## 11. Cache, privacy, conditional requests, and framing - -### 11.1 Shared-template cache - -When enabled finalizer metadata is present for the request's document pipeline, disallow -shared-template lookup/reservation before the first cache read. The same decision must -exclude later storage. A factory deciding `Unchanged` at EOF does not restore eligibility. - -Use the existing non-shared inline fallback consistently across slot scripts, body-seam -selection, transformation, and assembly. Do not generate a shared-template marker for a -response that has no shared-template consumer. Record the existing request-bypass outcome -where cache response-state reporting is enabled. - -Warm entries may remain stored but must not be read while the finalizer is enabled. No -purge is required merely to activate it. Disabling an integration can restore ordinary -eligibility, subject to existing settings/build fingerprints and template validation. - -There is no first-version `shared_safe` escape hatch. Later cache support must specify at -least reader-neutral output, external-data/signing-key identity, expiration, invalidation, -and whether failures can be cached. Cold-miss ordering alone does not settle these rules. - -### 11.2 Downstream privacy and origin readthrough - -Apply the existing synthesized HTML `private, no-store` policy to body-carrying HTML -responses that enter the enabled finalizer pipeline, regardless of auction eligibility or -the finalizer outcome. Strip validators and stale downstream cache directives using the -same existing helper as other synthesized HTML. Apply this before streaming headers are -committed. An unchanged or size-bypassed result remains private for that response. - -This prevents a browser or downstream cache from treating a per-request result as the -origin's public representation. Origin readthrough remains a separate raw-byte cache: it -may still provide a complete shareable origin body under existing policy, because every -request subsequently runs transformation/finalization. Do not cache assembled output as -an origin body or modify EC-cookie privacy rules. - -For document GETs requiring finalization, reuse -`strip_conditional_and_range_headers` so client revalidation or range requests cannot -make the origin return a bodyless `304` or partial `206` instead of the full document. -That helper removes `If-None-Match`, `If-Modified-Since`, `Range`, and `If-Range`. -Do not discard mutation preconditions such as `If-Match` or `If-Unmodified-Since`, -or broaden GET behavior to other methods. Unexpected bodyless responses retain existing -status/error handling and invoke no finalizer, including the existing invalid-origin-304 -guard when the ad stack is active. This feature does not turn an existing 502 into a 304. - -### 11.3 Response selection and framing - -Run finalizers only on complete HTML that the existing routing rules already process, including -processable non-2xx HTML. Expose the actual response status so consumers can choose -which complete documents to process. With enabled finalizers, HTML with status 206 -or any `Content-Range` bypasses all body transformations and terminal I/O. Preserve -encoded bytes, range/framing metadata, and origin policy; independent cookie and -diagnostics privacy still applies. This covers range-associated error responses -such as 416 as well as partial representations. The handler and both response -wrappers must make this decision before body-dependent policy or framing mutation; -direct body helpers carry the same private metadata. Abandon dispatched bids once. -Keep non-document range forwarding intact and feature-off behavior unchanged. - -Do not rewrite unsupported encodings, unmodified pass-through -responses, assets, or standalone `text/x-component` bodies. - -HEAD and `204`, `205`, or `304` responses invoke no finalizer or finalizer I/O. Preserve -their existing framing and any dispatched-auction abandonment. Preserve method/status -handling for other requests instead of introducing a GET-only body transformer silently. - -Finalizers do not change the selected content encoding. Strip stale `Content-Length` as -the existing transformed-body path requires; stream encoders finalize only after accepted -HTML and auction output are emitted. Preserve repeated headers, cookies, and CSP policy. - -## 12. Integration into publisher paths - -The first implementation must deliberately cover these existing branches: - -| Source and sink | Auction | Required integration | -| ----------------------------------------------- | ------- | ------------------------------------------------------------------------------------- | -| Lazy response stream, `Once` or `Stream` source | None | Retain opted-in synchronous HTML output, await finalizers, then encode | -| Lazy response stream, `Once` or `Stream` source | Pending | Same phase before existing seam collection; retain auction guard | -| Buffered response, streamed source | None | Same terminal phase before completing bounded response output | -| Buffered response, streamed source | Pending | Same phase before seam/encoder finish | -| Buffered response, materialized source | None | Opted-in path cannot fall through `stream_publisher_body` and skip finalization | -| Buffered response, materialized source | Pending | Reader-based hold path must route opted-in HTML through the same terminal coordinator | - -Introduce one private HTML terminal coordinator shared by the enabled paths. Feed it the -same synchronous transformation semantics on materialized and streamed inputs. Keep -ordinary no-finalizer branches intact; avoid replacing the entire generic compression -architecture to obtain this capability. - -Do not add finalizers to a generic synchronous helper that cannot await them. Public -synchronous helpers continue to implement synchronous transformations only; document that -HTML finalizers require the supported publisher finalization entry points. Adapters must -never select a synchronous-only path for a registered finalizer silently. - -Thread document state and finalizer metadata explicitly through the relevant publisher -parameters. Do not reconstruct a separate registry/session inside a lazy stream or copy -request services into global state. Reuse existing services ownership for request-lifetime -and lazy response futures. - -## 13. Observability - -Use existing `log` instrumentation with integration ID, controlled outcome, elapsed time, -and bounded byte counts. Log ordinary completion/unchanged results at debug level and soft -failure, timeout, or optional-collection bypass at warn level. Emit at most one outcome -per attempted finalizer and one terminal collection/deadline bypass for the document. - -Never log HTML, article text, signatures, external response bodies, control tokens, client -identifiers, secrets, URLs containing request data, or arbitrary error-report attachments. -Log controlled categories instead of formatting an integration-supplied report wholesale. - -No new public response headers, persistent telemetry store, or metrics subsystem is needed. -Existing auction terminal telemetry retains its meanings and exactly-once behavior. - -## 14. Verification requirements - -The checks below are acceptance requirements for the implementation, not claims that the -unsubmitted feature has already passed them. - -### 14.1 API, registration, and state - -- Default registration has no finalizers; a disabled integration contributes none. -- Factory and finalizer ordering is deterministic, including multiple factories for one ID. -- Sessions and document capture state are isolated between requests and reused sandboxes. -- A non-`Send` future borrowing input, replacement writer, and session state compiles and - executes through each supported publisher entry point. -- Existing processors retain chunk behavior and receive EOF once before any finalizer. -- Synchronous helpers retain their documented scope; no adapter silently skips a finalizer. - -### 14.2 Real yielding, fallback, and deadlines - -- A stub that returns `Pending` at least once completes when awaited and changes output. -- Finalizer A's accepted output is exactly B's input; B never receives partial candidates. -- A timeout after partial candidate writes drops the future, emits exact input bytes, and - invokes no later finalizer. Use a drop observer and allocation/buffer identity assertions - to prove no fallback document clone. -- `Unchanged`, callback error, invalid marker, replacement overflow, and a late result - preserve exact stage input and discard partial candidate state. -- Two or more delayed finalizers share one deadline; the budget does not multiply. -- Deadline/completion ties follow the documented timeout-wins rule. Use an injectable - deterministic timer/clock in tests rather than flaky wall-clock sleeps. -- Client cancellation drops active work and retains exactly-once auction abandonment. -- Fastly tests demonstrate no timer dependency, budget checks, and bounded-call consumer - behavior; they do not pretend a synchronous call can be preempted. - -### 14.3 Bounds and byte ordering - -- Test cap minus one, exact cap, cap plus one, and length-overflow rejection. -- Test multiple chunks and one large chunk crossing the optional collection cap. Prefix, - crossing chunk, and suffix emerge once in order on the streaming bypass path. -- Buffered output caps and existing decoder/input caps retain their existing failures. -- Replacement sink overflow cannot commit even when a callback ignores its write error. -- Input and replacement capacities remain bounded; no full-cap allocation for tiny pages. -- Accepted prior stages are released as ownership advances; timeout does not restore the - origin or undo an earlier successful finalizer. - -### 14.4 Publisher and codec matrix - -Exercise `Once` and `Stream` sources, lazy and buffered sinks, auction present/absent, -and identity, gzip, deflate, and Brotli encoding. Include concatenated gzip members, -decoder tails released only at EOF, truncated/corrupt streams, empty HTML, split UTF-8, -missing body ends, and malformed-but-renderable HTML. - -Verify decoded/synchronous EOF precedes finalizer I/O, candidates preserve private marker -cardinality, ready prefixes precede auction collection, and encoder trailers follow all -accepted bytes. Finalizer failure must not leave a raw private marker in client output. - -Retain the current tests for early compressed output, parser-confirmed Next.js seams, -placeholder substitution, body-source cancellation, and auction abandonment. Those tests -must continue to use the no-finalizer path and protect its existing streaming behavior. - -### 14.5 Cache, headers, and eligibility - -- Populate a template entry with finalization disabled, enable a finalizer, and prove the - request bypasses the warm entry and runs its finalizer. Assert no lookup reservation or - store is made; cover ESI configuration falling back to inline. -- Disable the finalizer and verify ordinary cache miss/store/hit behavior is restored. -- Responses remain `private, no-store` for success, `Unchanged`, timeout, external failure, - and collection bypass, including HTML without an auction or matched ad slot. -- Raw origin readthrough hits still execute finalizers per request; no assembled output - enters the raw origin cache. -- Test synthesized GET revalidation/range handling and preserved mutation preconditions, - including unexpected origin 304 behavior with and without an active ad stack. -- Verify non-2xx HTML is processed, while HEAD, bodiless statuses, non-HTML, unsupported - encodings, and unmodified responses make no finalizer call. Preserve framing and cookies. -- Existing shared-template validation, assembly, CSP nonce, and cache-privacy tests pass - unchanged with no finalizer enabled. - -### 14.6 Actual adapters and repository gates - -Native Cloudflare/Spin tests cannot establish JS or WASI timer behavior. Add fictional -fixtures and run a bounded delayed/hung lookup on Viceroy, local Workers, Spin, and Axum. -Record the exact setup, SDK/runtime versions, observed behavior, and limitations. Do not -claim a universal exact wall-clock release; cooperative scheduling adds tolerance. - -Review hardening permits one targeted correction under #1240: auction telemetry -uses `web_time::Instant`, consistent with the orchestrator, because the captured -Workers stack names that unsupported standard-clock call. Verify the actual -Workers auction control after the correction. Broader clock paths remain separate; -do not claim all of #1240 resolved from one successful fixture. Keep KV wiring under -#1139 separate; HTTP-backed and unavailable-KV fixtures can verify this lifecycle without -requiring new mandatory store bindings. - -Before implementation PR handoff, run every CI gate in [AGENTS.md](../../../AGENTS.md#ci-gates), -including target-matched adapter checks/tests, parity, JS/docs formatting and tests, and -native build-digest checks. Run existing template-cache ESI/inline harnesses as appropriate. -Use only fictional fixture hosts, text, configuration, and credentials. - -## 15. Implementation scope and delivery - -Expected production touch points: - -- Core integration registration: terminal factory/session contracts and presence metadata. -- Core HTML/publisher processing: shared document state, bounded collection/replacement, - terminal coordinator, path selection, cache/conditional/privacy integration, and errors. -- Core platform services: optional timer and deadline helper using existing clock support. -- Core settings: the finish timeout default/validation and canonical configuration docs. -- Adapter service construction: Cloudflare, Spin, and Axum timer implementations; Fastly - explicitly retains no cooperative timer. -- Tests, integration fixtures, example configuration, and canonical integration/adapter - guides documenting opt-in behavior and limitations. - -Keep the generic streaming trait, synchronous pipeline, JS runtime, platform manifests, -and EdgeZero pin unchanged. Do not bundle the real signing consumer, KV-store wiring, -broader Cloudflare clock fixes, or unrelated publisher refactoring into this feature. -The narrowly traced auction telemetry clock correction above is the reviewed exception. - -The implementation plan should separate contract/registration work, terminal coordination, -adapter timers, cache/privacy behavior, and verification into small reviewable changes, -while not shipping an intermediate state in which enabled finalizers can be skipped. - -This document authorizes no deployment, external API use, or publication. After maintainer -review, write the implementation plan separately. The current branch contains design work -only. - -## 16. Acceptance criteria - -The feature is ready only when: - -1. A yielding finalizer works through every supported publisher response path. -2. No-finalizer output, streaming timing, auction sequencing, and error handling retain - existing behavior under their regression tests. -3. Fallback is exact at the failed stage, candidate writes are bounded and isolated, and - timeout drops the active future without cloning the retained document. -4. The deadline covers the entire terminal chain, with Fastly's non-preemption limitation - explicitly tested and documented. -5. Shared-template reads and writes cannot bypass or persist the new per-request phase; - downstream privacy and origin readthrough remain correctly separated. -6. No unsupported body, method/status, encoding, or synchronous-only adapter path silently - invokes or skips the new capability. -7. Real runtime evidence and all required repository gates are recorded with honest limits. - -## 17. References and investigation record - -- Issue #1241: requested asynchronous completion for HTML stream processors. -- PR #1135 and the [parser-aware body hold design](2026-09-07-850-parser-aware-body-hold-nextjs-streaming-design.md): - synchronous stream processors and parser-generated auction seam. -- Issues #1240 and #1139: related clock and runtime KV findings; independent scope. -- [Streaming processor](../../../crates/trusted-server-core/src/streaming_processor.rs), - [HTML processor](../../../crates/trusted-server-core/src/html_processor.rs), and - [integration registry](../../../crates/trusted-server-core/src/integrations/registry.rs). -- [Publisher](../../../crates/trusted-server-core/src/publisher.rs), - [runtime services](../../../crates/trusted-server-core/src/platform/types.rs), - [template cache contract](../../../crates/trusted-server-core/src/platform/template_cache.rs), - and [publisher settings](../../../crates/trusted-server-core/src/settings.rs). - -During investigation at the baseline, the following native core test filters passed: - -```text -cargo test -p trusted-server-core --lib streaming_processor::tests --offline -34 passed -cargo test -p trusted-server-core --lib html_processor::tests --offline -32 passed -cargo test -p trusted-server-core --lib publisher::tests --offline -369 passed -``` - -Two temporary probe tests also passed: the current synchronous pipeline discards a -yielding one-poll lookup, and awaiting the same lookup completes it. These are investigation -evidence, not implementation regression tests or proof of the contributor's runtime claims. +# Ordered asynchronous HTML processor completion + +**Issue:** #1241. **Revised:** 2026-10-13. **Status:** revised implementation and local verification complete; maintainer review pending. +Verification is recorded in the [fresh execution report](../reviews/2026-10-13-1241-ordered-eof-verification.md). + +This supersedes the [buffered terminal design](../archive/2026-10-12-1241-buffered-terminal-design.md). +The previous implementation's evidence belongs to that archived design. The current +[implementation plan](../plans/2026-10-12-1241-async-html-finalization.md) covers this revision. + +## 1. Requirement and scope + +HTML integrations receive synchronous `process_chunk(bytes, is_last)` calls but have +no supported opportunity to await HTTP or KV after document EOF. Provide optional +asynchronous processor completion. Keep chunk processing synchronous. A processor's +completed output must reach downstream processors before their own EOF. Article +signing is an example, not an API requirement or a framework responsibility. + +Trusted Server owns processor composition, publisher driving, auction assembly, +cache admission and request services. No EdgeZero repin is needed for EOF completion. +Changing buffered adapters into streaming adapters is separate work. + +This revision does not implement signing, verification endpoints, queues, key +management, background jobs, publisher CMS changes, async parser callbacks, +per-chunk I/O, a second parser, or insertion into already emitted HTML positions. + +## 2. One processor lifecycle + +`StreamProcessor` gains an object-safe optional `async_finish` returning a +`LocalBoxFuture`. The default returns `None` without allocating a future. Its input +borrows driver-owned synchronous EOF output, a separate bounded replacement writer, +actual response status, request `RuntimeServices`, and a shared absolute deadline. +It may borrow processor state and need not be `Send`. A capability method lets +synchronous drivers reject unsupported async processing before effects. + +`StreamFinishInput.output` is un-emitted output eligible for replacement. It is a +streaming processor's EOF tail or an explicitly buffered processor's retained +output; it is not universally the complete document. Outcomes are `Unchanged` and +`Replace`. Empty replacement is intentional. Return the future promptly and perform +I/O inside it. Bytes already delivered cannot be replaced or rolled back. + +Use the existing `IntegrationHtmlStreamProcessorFactory` and one registration +sequence. Factory metadata declares async EOF capability and optional retention of +its synchronous output, both false by default. Reject retention without async +capability. Metadata and the created session's capability must agree; undeclared +async work must never silently disappear through a synchronous driver. + +Remove the unmerged separate finalizer factory, session trait and registration list. +Existing tests can use a test-only adapter to the one production processor lifecycle. +No production compatibility API for the previous draft is required. + +Factory metadata is available before template lookup, without creating a session, +reading HTML or polling a future. Every session uses the existing request-local +`IntegrationDocumentState`; state must not survive requests or reusable sandboxes. + +## 3. Operator acceptance of latency + +Add `publisher.allow_async_html_finish`, default `false`. An enabled integration +requiring async EOF without this permission produces an actionable configuration +error identifying the integration and setting. Do not silently omit required work. +A true setting with no async factories leaves the old processing path intact. + +Check permission at registry construction and defend direct publisher entry points +using manually constructed registries before consuming a body or committing +processed output. Factory retention metadata specifies whether core withholds output; +permission accepts the possible delivery cost of async EOF, including on buffered +adapters. It is not a guarantee of background execution after request completion. + +## 4. Ordered EOF driver + +For no-hook pages, preserve existing composition, chunk boundaries, streaming and +auction seam timing. Create no future, collector or timer. + +For opted-in HTML: + +1. Decode and validate origin bytes under existing limits. Run parser and processor + chunks synchronously in their existing order. +2. Complete decoder/parser EOF. Feed their remaining output downstream with + `is_last = false` and release ready bytes before optional I/O. +3. Finish the next integration processor exactly once with `is_last = true`. + Keep its synchronous EOF output outside its optional borrowing future. +4. Await optional finish under the remaining shared deadline. Commit an accepted + bounded candidate or retain the synchronous fallback. +5. Feed the selected output through later processors as ordinary input. Release + ready bytes before finishing the next stage. Repeat steps 3–5. +6. After all processors complete, assemble the auction and finish the encoder. + +Use a stepped concrete HTML chain in the async publisher driver. Do not await one +composite future that holds all EOF output. Race only a leaf callback, never the +whole chain: timeout must still complete downstream synchronous EOF. + +The generic `StreamingPipeline` remains synchronous. Its entry point, synchronous +publisher helpers and direct composite EOF reject async-capable use before reader, +parser or sink effects. Both materialized and streamed bodies reach the ordered driver. +Raw partial-response copying remains a documented exception because it executes no +processor or callback; pending auctions must still use an async entry point. + +## 5. Optional bounded retention + +For factories requesting retention, core owns a collector for that stage's synchronous +output. Use `publisher.max_buffered_body_bytes` (existing 16 MiB default), checked +arithmetic and incremental allocation. At EOF, move the retained output into the +fallback and lend a slice to the callback. Candidate storage has the same cap. + +Overflow permanently disables that stage's optional EOF I/O. Replay retained prefix, +crossing chunk and later output exactly once through downstream stages, then finish +its synchronous EOF normally. It does not authorize an oversized outer buffered +response. Core bounds its holder and candidate, not arbitrary private allocations +inside integration implementations. + +After overflow, the collector is consumed into ordinary ready output. The existing +`Vec` chunk interface may coalesce the prefix and crossing chunk above this +optional retention cap; ordinary output and outer buffered-response limits still +apply. The cap bounds withheld output, not every transient streaming chunk. + +Without retention, emit ordinary stage output immediately and keep only its EOF +fallback while awaiting. If fallback exceeds the optional-work cap, deliver it under +the ordinary output policy and skip that stage's optional I/O. Do not truncate it. + +## 6. Deadlines, failures and cancellation + +Retain `publisher.html_finalization_timeout_ms`: default 5000, valid 1–60000. +One absolute budget begins after decoder/parser EOF and covers every optional finish. +Share one timer across stages, initializing its delay with the remaining budget +immediately before the first callback is polled. The deadline wins readiness ties. Check before polling +callbacks and before accepting output. + +Callback errors, poisoned candidates, invalid seam-marker multiplicity and late results +discard only that candidate. For each stage, retain at most marker-length minus one +bytes of its previously emitted output and validate token counts across that boundary +plus the EOF fallback/candidate. This catches split-token creation/removal without +retaining the page. Buffered and overflow-bypassed stages use the same boundary rule. +Preserve the current fallback allocation and earlier +accepted output. On expiry, drop the leaf future, skip later optional I/O, and still +finish all later synchronous stages. Access borrowed buffers only after dropping +that future. Callback panics propagate. + +Origin, parser, synchronous processor, codec and sink failures remain fatal. Client +cancellation drops local work; preserve existing auction diagnostics. No detached +futures, retries or background execution are implied. + +Axum, Workers and Spin supply cooperative timers. Fastly supplies none: consumers +must bound blocking host calls independently. Timers cannot preempt blocking calls +or CPU work that never yields; do not promise universal hard deadline release. + +## 7. Auction and encoding + +Keep auction dispatch and its transport deadline unchanged. On enabled EOF paths, +complete processor finishes before auction collection/assembly. Release available +prefix bytes before waiting. Preserve exact-token seam handling, missing-body-end +fallback, winning bids, encoder ordering and abandonment telemetry. + +A structural body-close may precede substantial trailing content. Bound the held +auction suffix using `publisher.max_buffered_body_bytes`; count every retained byte +after the marker (and an incomplete marker prefix before it). Check before allocation. +Split and release a large chunk's ready prefix before retaining its bounded suffix; +do not first copy the entire chunk into the holder. Exact-cap retention succeeds; +cap crossing is a fatal body error with abandonment, not an unbounded +allocation. No-hook auction seam timing remains unchanged. + +Marker multiplicity protects cardinality, not HTML structure. Async output passes +through later stream processors but does not pass through the HTML parser again. +Consumers preserve markers, CSP and application semantics. EOF append does not +promise insertion into an earlier head or before an already emitted body end. + +## 8. Cache, privacy and response eligibility + +Use active async factory metadata before the first processed-template lookup to skip +shared-template reads, reservations and stores. Raw-origin readthrough is independent. +Processed opted-in HTML uses `Cache-Control: no-store, private` even when work fails, +returns unchanged, or is bypassed after overflow. Cacheable async output needs a +separate identity, freshness, invalidation and hit-execution contract. + +Preserve `AuctionDispatch.integration_registry` and same-registry dispatch/finish. +Keep early synthesized document-GET stripping of conditional/range headers; preserve +non-document fetch semantics. + +Opted-in HTML with status 206 or any `Content-Range` bypasses all rewriting and EOF +I/O. Preserve encoded bytes, framing, validators and ordinary origin policy. +Independent cookie/diagnostics privacy remains. Async helpers abandon unneeded +auctions. HEAD, bodiless, non-HTML and unsupported encodings run no callbacks. +Complete error HTML remains eligible and exposes its actual status. + +## 9. TTFB contract + +| Path | Contract | +| ----------------------------------------- | -------------------------------------------------------- | +| No async factories | Existing delivery and auction timing; no new EOF wait | +| Permission enabled, no async factories | Same as default | +| Async stage without retention on Fastly | Ready bytes released before EOF I/O; completion can wait | +| Explicit retention | Initial delivery waits until EOF or overflow release | +| Async stage on buffered Axum/Workers/Spin | EOF wait can add to TTFB before response return | + +Do not claim zero CPU overhead or enabled-hook TTFB neutrality on buffered adapters. +Their streaming support, or reliable post-response work, is separate runtime work. + +## 10. Acceptance criteria + +- Default permission is false; disabled integrations need no permission; enabled + async integrations fail configuration without explicit permission. +- Absent callbacks allocate no future and preserve the no-hook paths. +- Yielding non-`Send` callbacks work with materialized and streamed input. +- Upstream async output reaches downstream normal chunks before downstream EOF. +- Every parser/processor receives EOF once, including after timeout. +- Ready decoder/parser/stage prefixes are released while later EOF I/O is pending. +- Buffered overflow replays prefix/crossing/suffix once and starts no optional I/O. +- Cancellation, ties, callback errors, poisoned candidates and late results preserve + driver-owned fallback without a rollback copy. +- Synchronous helpers reject async use before effects. +- Codecs, ranges, actual statuses, privacy and early cache admission retain previous + regression protections; corrupt input starts no optional work. +- Auction assembly follows processor finishes; held suffix overflow abandons safely. +- All target-matched CI gates and actual runtime fixture controls pass on final source. + +The fictional fixture demonstrates generic EOF HTTP I/O, not a production signing +integration. Verification records identify checked source, real runtimes, hosted CI, +expected skips and tooling limits separately. Historical evidence does not prove this +revised implementation. diff --git a/scripts/fixtures/async-html-finalization-runtime.py b/scripts/fixtures/async-html-finalization-runtime.py index 87f9c19fb..1493dc697 100644 --- a/scripts/fixtures/async-html-finalization-runtime.py +++ b/scripts/fixtures/async-html-finalization-runtime.py @@ -108,7 +108,9 @@ def field(offset, width, value): headers["Cookie"] = "euconsent-v2=" + base64.urlsafe_b64encode(bits).decode().rstrip("=") request = urllib.request.Request(f"http://127.0.0.1:{app_port}{path}", headers=headers) with urllib.request.urlopen(request, timeout=10) as response: - return response.read(), response.headers + first = response.read(1) + first_byte_at = time.monotonic() + return first + response.read(), response.headers, first_byte_at try: command(["cargo", "build", "-p", "trusted-server-cli", "--bin", "ts", "--locked"]) @@ -148,9 +150,9 @@ def field(offset, width, value): '../../target/wasm32-wasip1/release/trusted_server_adapter_spin.wasm', str(artifact))) baseline = None - cases = ["disabled", "success", "hung"] + (["auction"] if auction else []) + cases = ["disabled", "success", "hung"] + (["auction"] if auction else []) + (["streaming"] if adapter == "fastly" else []) for case in cases: - case_config = base_config + case_config = base_config.replace("[publisher]\n", f"[publisher]\nallow_async_html_finish = {str(case != 'disabled').lower()}\n", 1) if case == "auction": # A closed loopback bid endpoint exercises transport failure and # empty-bid assembly without sending traffic outside this host. @@ -160,8 +162,8 @@ def field(offset, width, value): case_config = case_config[:enabled] + case_config[enabled:].replace("enabled = false", "enabled = true", 1) case_config = case_config.replace('endpoint = "https://prebid.example.com/openrtb2/auction"', f'endpoint = "https://127.0.0.1:{bid_port}/bid"', 1) case_config += '\n[[creative_opportunities.slot]]\nid = "example-slot"\npage_patterns = ["/article"]\nformats = [{ width = 300, height = 250 }]\n' - lookup_case = "success" if case == "auction" else case - config.write_text(case_config + f'\n[integrations.html_finalization_fixture]\nenabled = {str(case != "disabled").lower()}\nlookup_url = "http://127.0.0.1:{origin_port}/{lookup_case}"\ntransport_timeout_ms = 30000\n') + lookup_case = "success" if case in ("auction", "streaming") else case + config.write_text(case_config + f'\n[integrations.html_finalization_fixture]\nenabled = {str(case != "disabled").lower()}\nlookup_url = "http://127.0.0.1:{origin_port}/{lookup_case}"\ntransport_timeout_ms = 30000\nstreaming = {str(case == "streaming").lower()}\n') # Axum's local JSON provides the same neutral app-config envelope # used by the other runtime bindings; no operator config is read. command([cli, "config", "push", "--adapter", "axum", "--local", "--manifest", str(workspace / "edgezero.toml"), @@ -212,7 +214,7 @@ def field(offset, width, value): raise RuntimeError(f"runtime did not become ready: {log.read_text()}") started = time.monotonic() try: - body, headers = get(f"/article?case={case}", case == "auction") + body, headers, first_byte_at = get(f"/article?case={case}", case == "auction") except urllib.error.HTTPError as error: (workspace / f"{case}.error.body").write_bytes(error.read()) (workspace / f"{case}.headers").write_text(str(error.headers)) @@ -228,6 +230,8 @@ def field(offset, width, value): elif case == "success": assert body == baseline.replace(b"EXAMPLE_UNSIGNED", b"EXAMPLE_FINALIZED"), "successful terminal output must match the exact expected rewrite" assert elapsed >= 0.15, "success must actually await delayed lookup" + elif case == "streaming": + assert body == baseline + b"", "streaming EOF must append only after already delivered HTML" elif case == "auction": assert b"EXAMPLE_FINALIZED" in body, "auction-enabled output must await finalization" assert b"adSlots" in body and b"bids" in body, "auction control must reach slot and bid assembly" @@ -238,24 +242,28 @@ def field(offset, width, value): diagnostic = log.read_text() if adapter == "spin": diagnostic += "\n".join(path.read_text() for path in (workspace / "spin-logs").glob("*stderr*")) - assert "HTML finalizer html_finalization_fixture times out" in diagnostic or (adapter == "fastly" and "retains input: unavailable" in diagnostic), "missing controlled fallback diagnostic" + assert "HTML EOF processor html_finalization_fixture times out" in diagnostic or (adapter == "fastly" and "retains input: unavailable" in diagnostic), "missing controlled fallback diagnostic" if case != "disabled": assert headers["Cache-Control"] == "no-store, private" current_events = [json.loads(line) for line in (workspace / "events.jsonl").read_text().splitlines()] lookups = [event for event in current_events if event["event"] == "lookup_start"] lookup_elapsed = completed - lookups[-1]["time"] if case != "disabled" else None + if case == "streaming": + completions = [event for event in current_events if event["event"] == "lookup_complete"] + assert completions, "streaming finish must complete real delayed I/O" + assert first_byte_at < completions[-1]["time"] - 0.05, "initial HTML bytes must reach the client while EOF I/O remains pending" terminal_ms = None if case == "hung": measured = re.search(r"times out after (\d+) ms", diagnostic) if measured: terminal_ms = int(measured.group(1)) assert 700 <= terminal_ms < 2500, "terminal budget must exclude origin transfer" - (workspace / f"{case}.timing.json").write_text(json.dumps({"request_seconds": elapsed, "terminal_ms": terminal_ms, "lookup_to_response_seconds": lookup_elapsed, "configured_budget_ms": 750, "cooperative_timer": adapter != "fastly"})) + (workspace / f"{case}.timing.json").write_text(json.dumps({"request_seconds": elapsed, "first_body_byte_seconds": first_byte_at - started, "terminal_ms": terminal_ms, "lookup_to_response_seconds": lookup_elapsed, "configured_budget_ms": 750, "cooperative_timer": adapter != "fastly"})) print(f"PASS {adapter} {case}: request={elapsed:.3f}s, {len(body)} bytes, chain_budget=750ms, terminal_ms={terminal_ms}, lookup_to_response={lookup_elapsed}, cooperative_timer={adapter != 'fastly'}", flush=True) stop(process) events = [json.loads(line) for line in (workspace / "events.jsonl").read_text().splitlines()] starts = [event for event in events if event["event"] == "lookup_start"] - assert len(starts) == 2 + int(auction) and all(event["complete"] and len(event["sha256"]) == 64 for event in starts) + assert len(starts) == 2 + int(auction) + int(adapter == "fastly") and all(event["complete"] and len(event["sha256"]) == 64 for event in starts) for event in starts: assert any(previous["event"] == "origin_eof" and previous["time"] <= event["time"] for previous in events) if adapter == "fastly": diff --git a/trusted-server.example.toml b/trusted-server.example.toml index e38f9cbbe..18cc26310 100644 --- a/trusted-server.example.toml +++ b/trusted-server.example.toml @@ -67,7 +67,10 @@ proxy_secret = "publisher_proxy_secret" # rewriting/injection) instead of streamed. Default 16 MiB; larger responses # return 502. Raise for deployments serving very large publisher pages. # max_buffered_body_bytes = 16777216 # 16 MiB -# Optional asynchronous HTML finalizers share this budget after synchronous EOF. +# Explicit permission for integration EOF I/O that can delay HTML delivery. +# Defaults to false; enabled async processors require true. +# allow_async_html_finish = false +# Optional asynchronous HTML processors share this budget after decoder/parser EOF. # html_finalization_timeout_ms = 5000 # Valid range: 1..60000 ms From 597a5281288300c206184725f208617222c9ed8d Mon Sep 17 00:00:00 2001 From: prk-Jr Date: Tue, 13 Oct 2026 11:12:09 +0530 Subject: [PATCH 07/11] Record ordered EOF revision publication --- .../2026-10-12-1241-async-html-finalization.md | 15 +++++++++++---- .../2026-10-13-1241-ordered-eof-verification.md | 2 ++ 2 files changed, 13 insertions(+), 4 deletions(-) diff --git a/docs/superpowers/plans/2026-10-12-1241-async-html-finalization.md b/docs/superpowers/plans/2026-10-12-1241-async-html-finalization.md index 407d109cb..de87382c1 100644 --- a/docs/superpowers/plans/2026-10-12-1241-async-html-finalization.md +++ b/docs/superpowers/plans/2026-10-12-1241-async-html-finalization.md @@ -119,10 +119,10 @@ Paths above are relative to `crates/trusted-server-core/src/` unless prefixed wi ## Task 9: Publish the reviewed revision -- [ ] Verify clean intended diff, archive links, spec/plan/implementation alignment and fresh passing checks. -- [ ] Commit using repository sentence-case conventions and push the existing feature branch. -- [ ] Rewrite draft PR description around generic ordered EOF completion and operator permission; keep issue1241 as the target and keep draft status. -- [ ] Confirm remote head/body and report any remaining maintainer decisions or pending hosted checks. +- [x] Verify clean intended diff, archive links, spec/plan/implementation alignment and fresh passing checks. +- [x] Commit using repository sentence-case conventions and push the existing feature branch. +- [x] Rewrite draft PR description around generic ordered EOF completion and operator permission; keep issue1241 as the target and keep draft status. +- [x] Confirm remote head/body and report any remaining maintainer decisions or pending hosted checks. ## Acceptance mapping @@ -136,3 +136,10 @@ Paths above are relative to `crates/trusted-server-core/src/` unless prefixed wi | Auction/codec/partial/cache/privacy safeguards | Task 6, Task 8 | | Honest adapter TTFB contract and consumer scope | Task 7, Task 8 | | Reproducible verification and draft publication | Tasks 8–9 | + +## Execution completion + +Implementation commit `a533df267c149bb8295b23e5d5a1fb700c1b32eb` was pushed and +draft PR #1290 was updated and verified. The newer remote merge was preserved. +All local verification passed; hosted CI results remain separate. This completion +record changes documentation only. See the [verification report](../reviews/2026-10-13-1241-ordered-eof-verification.md). diff --git a/docs/superpowers/reviews/2026-10-13-1241-ordered-eof-verification.md b/docs/superpowers/reviews/2026-10-13-1241-ordered-eof-verification.md index d435c9e3d..94cf377bc 100644 --- a/docs/superpowers/reviews/2026-10-13-1241-ordered-eof-verification.md +++ b/docs/superpowers/reviews/2026-10-13-1241-ordered-eof-verification.md @@ -4,6 +4,8 @@ **Branch:** `feat/1241-async-html-finalization`; draft PR #1290; issue #1241. +**Published implementation:** `a533df267c149bb8295b23e5d5a1fb700c1b32eb`; PR remains draft. + **Review baseline:** `3981d0222bf0767ba3c8de767d633018b0df4b58`. The newer remote merge from `main` was preserved before final verification. From de735bc44f5825f86dd61d2ea51d3a4d2024f86b Mon Sep 17 00:00:00 2001 From: prk-Jr Date: Tue, 13 Oct 2026 13:04:59 +0530 Subject: [PATCH 08/11] Harden HTML EOF buffering and runtime verification --- .github/workflows/test.yml | 88 ++- AGENTS.md | 3 + .../src/platform.rs | 8 +- .../trusted-server-adapter-fastly/src/app.rs | 2 + .../src/html_processor/finish.rs | 298 +++++++- .../src/integrations/html_finalization.rs | 14 +- .../src/integrations/mod.rs | 6 +- .../src/integrations/registry.rs | 4 +- .../trusted-server-core/src/platform/timer.rs | 4 +- crates/trusted-server-core/src/publisher.rs | 47 +- .../src/publisher/html_finalization.rs | 3 +- .../src/streaming_processor.rs | 4 +- .../tests/documentation_snippets.rs | 21 +- .../tests/parity.rs | 155 ++-- docs/guide/configuration.md | 14 +- docs/guide/integration-guide.md | 83 ++- docs/guide/testing.md | 34 + ...026-10-12-1241-buffered-terminal-design.md | 691 ------------------ .../2026-10-12-1241-buffered-terminal-plan.md | 686 ----------------- ...2026-10-12-1241-async-html-finalization.md | 169 ++--- .../plans/2026-10-12-1241-review-hardening.md | 183 ----- ...-async-html-finalization-implementation.md | 188 ----- ...026-10-13-1241-ordered-eof-verification.md | 133 ---- ...-12-1241-async-html-finalization-design.md | 19 +- scripts/README.md | 59 +- .../async-html-finalization-runtime.py | 204 +++++- .../test_async_html_finalization_runtime.py | 172 +++++ 27 files changed, 1090 insertions(+), 2202 deletions(-) delete mode 100644 docs/superpowers/archive/2026-10-12-1241-buffered-terminal-design.md delete mode 100644 docs/superpowers/archive/2026-10-12-1241-buffered-terminal-plan.md delete mode 100644 docs/superpowers/plans/2026-10-12-1241-review-hardening.md delete mode 100644 docs/superpowers/reviews/2026-10-12-1241-async-html-finalization-implementation.md delete mode 100644 docs/superpowers/reviews/2026-10-13-1241-ordered-eof-verification.md diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index d5eb8af67..8b8e1772b 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -1,4 +1,4 @@ -name: "Run Tests" +name: 'Run Tests' permissions: contents: read @@ -18,19 +18,19 @@ jobs: - name: Retrieve Rust version id: rust-version - run: echo "rust-version=$(grep '^rust ' .tool-versions | awk '{print $2}')" >> $GITHUB_OUTPUT + run: echo "rust-version=$(grep '^rust ' .tool-versions | awk '{print $2}')" >> "$GITHUB_OUTPUT" shell: bash - name: Retrieve Viceroy version id: viceroy-version # `.tool-versions` is the single source of truth so this workflow and # `.github/actions/setup-integration-test-env/action.yml` can't drift. - run: echo "viceroy-version=$(grep '^viceroy ' .tool-versions | awk '{print $2}')" >> $GITHUB_OUTPUT + run: echo "viceroy-version=$(grep '^viceroy ' .tool-versions | awk '{print $2}')" >> "$GITHUB_OUTPUT" shell: bash - name: Retrieve Node.js version id: node-version - run: echo "node-version=$(grep '^nodejs ' .tool-versions | awk '{print $2}')" >> $GITHUB_OUTPUT + run: echo "node-version=$(grep '^nodejs ' .tool-versions | awk '{print $2}')" >> "$GITHUB_OUTPUT" shell: bash - name: Set up Rust toolchain @@ -71,6 +71,68 @@ jobs: - name: Run cache purge harness run: BID_DELAY=3 ./scripts/template-cache-local-test.sh purge + test-fastly-eof: + name: HTML EOF runtime (Fastly) + runs-on: ubuntu-latest + timeout-minutes: 25 + steps: + - uses: actions/checkout@v4 + + - name: Read pinned build tool versions + id: versions + shell: bash + run: | + echo "rust=$(awk '$1 == "rust" { print $2 }' .tool-versions)" >> "$GITHUB_OUTPUT" + echo "node=$(awk '$1 == "nodejs" { print $2 }' .tool-versions)" >> "$GITHUB_OUTPUT" + + - uses: actions-rust-lang/setup-rust-toolchain@v1 + with: + toolchain: ${{ steps.versions.outputs.rust }} + target: wasm32-wasip1 + cache-shared-key: cargo-${{ runner.os }} + + - uses: actions/setup-node@v4 + with: + node-version: ${{ steps.versions.outputs.node }} + + - uses: actions/setup-python@v5 + with: + python-version: '3.12' + + - name: Cache isolated EOF fixture Viceroy + id: cache-eof-viceroy + uses: actions/cache@v4 + with: + path: ${{ runner.temp }}/viceroy-eof/bin/viceroy + key: viceroy-eof-${{ runner.os }}-amd64-v0.19.0 + + - name: Install isolated Viceroy 0.19 for backend timeout simulation + if: steps.cache-eof-viceroy.outputs.cache-hit != 'true' + shell: bash + run: | + curl -fsSL --retry 3 https://github.com/fastly/Viceroy/releases/download/v0.19.0/viceroy_v0.19.0_linux-amd64.tar.gz -o "$RUNNER_TEMP/viceroy-eof.tar.gz" + mkdir -p "$RUNNER_TEMP/viceroy-eof/bin" + tar -xzf "$RUNNER_TEMP/viceroy-eof.tar.gz" -C "$RUNNER_TEMP/viceroy-eof/bin" viceroy + + - name: Select isolated fixture runtime + shell: bash + run: echo "$RUNNER_TEMP/viceroy-eof/bin" >> "$GITHUB_PATH" + + - name: Verify actual Fastly HTML EOF completion and auction assembly + timeout-minutes: 15 + env: + CARGO_TARGET_DIR: ${{ github.workspace }}/target + TMPDIR: ${{ runner.temp }} + run: ./scripts/async-html-finalization-local-test.sh fastly --release --auction + + - name: Upload EOF runtime diagnostics on failure + if: failure() + uses: actions/upload-artifact@v4 + with: + name: fastly-html-eof-diagnostics + path: ${{ runner.temp }}/trusted-server-finalize-fastly-*/ + if-no-files-found: ignore + test-axum: name: cargo test (axum native) runs-on: ubuntu-latest @@ -79,7 +141,7 @@ jobs: - name: Retrieve Rust version id: rust-version - run: echo "rust-version=$(grep '^rust ' .tool-versions | awk '{print $2}')" >> $GITHUB_OUTPUT + run: echo "rust-version=$(grep '^rust ' .tool-versions | awk '{print $2}')" >> "$GITHUB_OUTPUT" shell: bash - name: Set up Rust toolchain @@ -158,7 +220,7 @@ jobs: - name: Retrieve Rust version id: rust-version - run: echo "rust-version=$(grep '^rust ' .tool-versions | awk '{print $2}')" >> $GITHUB_OUTPUT + run: echo "rust-version=$(grep '^rust ' .tool-versions | awk '{print $2}')" >> "$GITHUB_OUTPUT" shell: bash - name: Set up Rust toolchain (native + wasm32-unknown-unknown) @@ -185,7 +247,7 @@ jobs: - name: Retrieve Rust version id: rust-version - run: echo "rust-version=$(grep '^rust ' .tool-versions | awk '{print $2}')" >> $GITHUB_OUTPUT + run: echo "rust-version=$(grep '^rust ' .tool-versions | awk '{print $2}')" >> "$GITHUB_OUTPUT" shell: bash - name: Set up Rust toolchain (native + wasm32-wasip1) @@ -210,7 +272,7 @@ jobs: TRUSTED_SERVER__PUBLISHER__ORIGIN_URL: http://127.0.0.1:8080 TRUSTED_SERVER__PUBLISHER__PROXY_SECRET: integration-test-proxy-secret TRUSTED_SERVER__EC__PASSPHRASE: integration-test-ec-secret-padded-32 - TRUSTED_SERVER__PROXY__CERTIFICATE_CHECK: "false" + TRUSTED_SERVER__PROXY__CERTIFICATE_CHECK: 'false' run: cargo build --package trusted-server-adapter-spin --target wasm32-wasip1 --features spin --release - name: Run Spin adapter tests (native host) @@ -224,7 +286,7 @@ jobs: - name: Retrieve Rust version id: rust-version - run: echo "rust-version=$(grep '^rust ' .tool-versions | awk '{print $2}')" >> $GITHUB_OUTPUT + run: echo "rust-version=$(grep '^rust ' .tool-versions | awk '{print $2}')" >> "$GITHUB_OUTPUT" shell: bash - name: Set up Rust toolchain @@ -255,14 +317,14 @@ jobs: - name: Retrieve Rust version id: rust-version - run: echo "rust-version=$(grep '^rust ' .tool-versions | awk '{print $2}')" >> $GITHUB_OUTPUT + run: echo "rust-version=$(grep '^rust ' .tool-versions | awk '{print $2}')" >> "$GITHUB_OUTPUT" shell: bash - name: Set up Rust toolchain uses: actions-rust-lang/setup-rust-toolchain@v1 with: toolchain: ${{ steps.rust-version.outputs.rust-version }} - components: "clippy, rustfmt" + components: 'clippy, rustfmt' cache-shared-key: cargo-cli-${{ runner.os }} - name: Read pinned Node.js version for CLI fixtures @@ -329,14 +391,14 @@ jobs: - name: Retrieve Node.js version id: node-version working-directory: . - run: echo "node-version=$(grep '^nodejs ' .tool-versions | awk '{print $2}')" >> $GITHUB_OUTPUT + run: echo "node-version=$(grep '^nodejs ' .tool-versions | awk '{print $2}')" >> "$GITHUB_OUTPUT" shell: bash - name: Use Node.js uses: actions/setup-node@v4 with: node-version: ${{ steps.node-version.outputs.node-version }} - cache: "npm" + cache: 'npm' cache-dependency-path: crates/trusted-server-js/lib/package.json - name: Install dependencies diff --git a/AGENTS.md b/AGENTS.md index 2685d164e..188c1b100 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -362,6 +362,9 @@ Every PR must pass: 7. Docs format (`cd docs && npm run format`) 8. Markdown format outside `docs/` (requires `cd docs && npm ci` first): `docs/node_modules/.bin/prettier --config docs/.prettierrc --check "*.md" ".claude/**/*.md" ".github/**/*.md" "crates/**/*.md" "scripts/**/*.md" "tinybird/**/*.md"`; fix with `--write` in place of `--check` 9. Native core build-digest test and lint (`cargo test -p trusted-server-core --test template_build_digest` and `cargo clippy -p trusted-server-core --test template_build_digest -- -D warnings`) +10. HTML EOF harness cleanup (Python 3.11+): `python3 -m unittest discover -s scripts/fixtures -p 'test_async_html_finalization_runtime.py' -v` +11. Actual Axum EOF/auction controls: `./scripts/async-html-finalization-local-test.sh axum --auction` +12. Actual Fastly EOF/auction controls with isolated Viceroy 0.19+: `PATH="/tmp/example-eof-viceroy/bin:$PATH" ./scripts/async-html-finalization-local-test.sh fastly --release --auction`; install with `cargo install viceroy --version 0.19.0 --locked --root /tmp/example-eof-viceroy`. Ordinary tests retain pinned Viceroy 0.17. See `scripts/README.md` for retained diagnostics and other runtimes. --- diff --git a/crates/trusted-server-adapter-axum/src/platform.rs b/crates/trusted-server-adapter-axum/src/platform.rs index 402ff2753..c91a422d1 100644 --- a/crates/trusted-server-adapter-axum/src/platform.rs +++ b/crates/trusted-server-adapter-axum/src/platform.rs @@ -582,10 +582,6 @@ pub fn build_runtime_services(ctx: &edgezero_core::context::RequestContext) -> R .build() } -// --------------------------------------------------------------------------- -// Tests -// --------------------------------------------------------------------------- - struct AxumTimer; impl trusted_server_core::platform::PlatformTimer for AxumTimer { @@ -596,6 +592,10 @@ impl trusted_server_core::platform::PlatformTimer for AxumTimer { } } +// --------------------------------------------------------------------------- +// Tests +// --------------------------------------------------------------------------- + #[cfg(test)] mod tests { #[test] diff --git a/crates/trusted-server-adapter-fastly/src/app.rs b/crates/trusted-server-adapter-fastly/src/app.rs index f269e5a7f..b96bcccbf 100644 --- a/crates/trusted-server-adapter-fastly/src/app.rs +++ b/crates/trusted-server-adapter-fastly/src/app.rs @@ -255,6 +255,8 @@ fn warn_if_certificate_check_disabled(settings: &Settings) { /// continues to rely on the trusted `fastly-ssl` header injected by /// `edgezero_main` after sanitization. fn build_per_request_services(state: &AppState, ctx: &RequestContext) -> RuntimeServices { + // No cooperative timer is installed: Fastly host calls block. EOF consumers + // must bound their own transport calls using the remaining finish budget. let client_info = ctx .request() .extensions() diff --git a/crates/trusted-server-core/src/html_processor/finish.rs b/crates/trusted-server-core/src/html_processor/finish.rs index 33407153e..2615c0148 100644 --- a/crates/trusted-server-core/src/html_processor/finish.rs +++ b/crates/trusted-server-core/src/html_processor/finish.rs @@ -11,7 +11,7 @@ use super::HtmlWithStreamingProcessors; use crate::platform::RuntimeServices; use crate::streaming_processor::StreamProcessor; use crate::streaming_processor::finish::{ - StreamFinish, StreamFinishDeadline, StreamFinishInput, StreamOutputWriter, + StreamFinish, StreamFinishDeadline, StreamFinishError, StreamFinishInput, StreamOutputWriter, }; pub(crate) struct HtmlProcessorStage { @@ -162,11 +162,23 @@ async fn finish_output( fallback: Vec, context: &mut HtmlFinishContext<'_>, ) -> Vec { - if stage.skip_async - || !stage.async_capable - || fallback.len() > context.limit - || context.deadline.is_expired_at(Instant::now()) - { + if stage.skip_async || !stage.async_capable { + return fallback; + } + if fallback.len() > context.limit { + log::warn!( + "HTML EOF processor {} skips optional I/O for {} bytes exceeding the {}-byte limit", + stage.integration_id, + fallback.len(), + context.limit + ); + return fallback; + } + if context.deadline.is_expired_at(Instant::now()) { + log::debug!( + "HTML EOF processor {} skips optional I/O after deadline expiry", + stage.integration_id + ); return fallback; } let mut candidate = StreamOutputWriter::new(context.limit); @@ -214,11 +226,9 @@ async fn finish_output( } if let Some(Err(error)) = &result { let outcome = match error.current_context() { - crate::streaming_processor::finish::StreamFinishError::Unavailable => "unavailable", - crate::streaming_processor::finish::StreamFinishError::Rejected => "rejected", - crate::streaming_processor::finish::StreamFinishError::OutputLimitExceeded { - .. - } => "output_limit", + StreamFinishError::Unavailable => "unavailable", + StreamFinishError::Rejected => "rejected", + StreamFinishError::OutputLimitExceeded { .. } => "output_limit", }; log::warn!( "HTML EOF processor {} retains input: {outcome}", @@ -1045,6 +1055,9 @@ mod tests { Error, PoisonedReplacement, LateReplacement, + #[cfg(not(target_arch = "wasm32"))] + ElapsedReplacement, + EchoReplacement, EmptyReplacement, Replacement, } @@ -1110,6 +1123,16 @@ mod tests { input.deadline.mark_expired(); Ok(StreamFinish::Replace) } + #[cfg(not(target_arch = "wasm32"))] + LeafOutcome::ElapsedReplacement => { + std::thread::sleep(input.deadline.remaining() + Duration::from_millis(5)); + input.replacement.append(b"candidate")?; + Ok(StreamFinish::Replace) + } + LeafOutcome::EchoReplacement => { + input.replacement.append(input.output)?; + Ok(StreamFinish::Replace) + } LeafOutcome::EmptyReplacement => Ok(StreamFinish::Replace), LeafOutcome::Replacement => { input.replacement.append(b"candidate")?; @@ -1345,6 +1368,259 @@ mod tests { assert!(observation.dropped.get(), "should release callback borrows"); } + #[cfg(not(target_arch = "wasm32"))] + #[test] + fn async_eof_timerless_callback_elapsed_past_deadline_preserves_fallback() { + let events = Rc::new(RefCell::new(Vec::new())); + let calls = Rc::new(Cell::new(0)); + let observation = LeafObservation::default(); + let fallback = b"fallback".to_vec(); + let fallback_pointer = fallback.as_ptr() as usize; + let mut chain = HtmlWithStreamingProcessors::from_stages( + stage("parser", b"", None, &events, &calls), + vec![HtmlProcessorStage::new( + "elapsed", + Box::new(LeafProbe { + tail: fallback, + outcome: LeafOutcome::ElapsedReplacement, + observation: observation.clone(), + }), + true, + false, + 64, + )], + ); + chain.begin_finish().expect("should finish the parser"); + let services = noop_services(); + let deadline = StreamFinishDeadline::new(Duration::from_millis(100), false); + let mut timer = None; + let output = block_on(chain.finish_next(HtmlFinishContext { + response_status: StatusCode::OK, + services: &services, + deadline: &deadline, + timer: &mut timer, + limit: 64, + marker: None, + })) + .expect("should finish timerless work that completes late") + .expect("should retain synchronous output"); + assert_eq!( + observation.calls.get(), + 1, + "should exercise callback execution rather than pre-poll expiry" + ); + assert!( + deadline.elapsed() >= Duration::from_millis(100), + "should cross the actual wall-clock deadline" + ); + assert_eq!(output, b"fallback", "should reject the late replacement"); + assert_eq!( + output.as_ptr() as usize, + fallback_pointer, + "should preserve the original fallback allocation" + ); + assert!(observation.dropped.get(), "should release callback borrows"); + } + + #[test] + fn async_eof_oversized_streaming_fallback_skips_io_without_truncation_or_copy() { + let events = Rc::new(RefCell::new(Vec::new())); + let calls = Rc::new(Cell::new(0)); + let observation = LeafObservation::default(); + let fallback = vec![b'x'; 65]; + let fallback_pointer = fallback.as_ptr() as usize; + let mut chain = HtmlWithStreamingProcessors::from_stages( + stage("parser", b"", None, &events, &calls), + vec![HtmlProcessorStage::new( + "oversized", + Box::new(LeafProbe { + tail: fallback, + outcome: LeafOutcome::Replacement, + observation: observation.clone(), + }), + true, + false, + 64, + )], + ); + chain.begin_finish().expect("should finish the parser"); + let services = noop_services(); + let deadline = StreamFinishDeadline::new(Duration::from_secs(30), false); + let mut timer = None; + let output = block_on(chain.finish_next(HtmlFinishContext { + response_status: StatusCode::OK, + services: &services, + deadline: &deadline, + timer: &mut timer, + limit: 64, + marker: None, + })) + .expect("should deliver an oversized synchronous tail") + .expect("should return the complete tail"); + assert_eq!( + output.as_slice(), + &[b'x'; 65], + "should not truncate fallback" + ); + assert_eq!( + output.as_ptr() as usize, + fallback_pointer, + "should deliver the original allocation" + ); + assert_eq!(observation.calls.get(), 0, "should skip optional I/O"); + assert_eq!( + observation.eof_calls.get(), + 1, + "should finish synchronous EOF once" + ); + } + + #[test] + fn async_eof_single_byte_marker_rejects_creation_and_removal() { + for (tail, replacement) in [ + (b"|".as_slice(), b"wrong".as_slice()), + (b"tail".as_slice(), b"|".as_slice()), + ] { + let events = Rc::new(RefCell::new(Vec::new())); + let calls = Rc::new(Cell::new(0)); + let mut chain = HtmlWithStreamingProcessors::from_stages( + stage("parser", b"", None, &events, &calls), + vec![HtmlProcessorStage::new( + "marker", + stage("B", tail, Some(replacement), &events, &calls), + true, + false, + 64, + )], + ); + chain.set_finish_marker(Some(b"|")); + assert_eq!( + chain + .process_chunk(b"page|", false) + .expect("should release ordinary output"), + b"page|", + "should preserve the already emitted marker" + ); + chain.begin_finish().expect("should finish the parser"); + let services = noop_services(); + let deadline = StreamFinishDeadline::new(Duration::from_secs(30), false); + let mut timer = None; + let output = block_on(chain.finish_next(HtmlFinishContext { + response_status: StatusCode::OK, + services: &services, + deadline: &deadline, + timer: &mut timer, + limit: 64, + marker: Some(b"|"), + })) + .expect("should retain output after invalid marker replacement") + .expect("should return the fallback tail"); + assert_eq!( + output, tail, + "should reject changing single-byte marker count" + ); + assert_eq!(calls.get(), 1, "should evaluate optional output once"); + } + } + + #[test] + fn async_eof_duplicate_parser_finish_rejects_before_repeating_effects() { + let events = Rc::new(RefCell::new(Vec::new())); + let calls = Rc::new(Cell::new(0)); + let mut chain = HtmlWithStreamingProcessors::from_stages( + stage("parser", b"parser-tail", None, &events, &calls), + vec![HtmlProcessorStage::new( + "B", + stage("B", b"tail", Some(b"candidate"), &events, &calls), + true, + false, + 64, + )], + ); + assert_eq!( + chain.begin_finish().expect("should finish the parser once"), + b"parser-tail", + "should release its tail once" + ); + let original_events = events.borrow().clone(); + chain + .begin_finish() + .expect_err("should reject a repeated parser EOF"); + assert_eq!( + *events.borrow(), + original_events, + "should not repeat parser or downstream chunk effects" + ); + assert_eq!( + calls.get(), + 0, + "should not start optional work at parser EOF" + ); + } + + #[test] + fn async_eof_exact_retention_and_replacement_cap_succeed_at_eof() { + let events = Rc::new(RefCell::new(Vec::new())); + let calls = Rc::new(Cell::new(0)); + let observation = LeafObservation::default(); + let mut chain = HtmlWithStreamingProcessors::from_stages( + stage("parser", b"", None, &events, &calls), + vec![HtmlProcessorStage::new( + "exact", + Box::new(LeafProbe { + tail: b"tail".to_vec(), + outcome: LeafOutcome::EchoReplacement, + observation: observation.clone(), + }), + true, + true, + 12, + )], + ); + for chunk in [b"page", b"more"] { + assert!( + chain + .process_chunk(chunk, false) + .expect("should retain ordinary output") + .is_empty(), + "should withhold output until EOF" + ); + } + chain.begin_finish().expect("should finish the parser"); + let services = noop_services(); + let deadline = StreamFinishDeadline::new(Duration::from_secs(30), false); + let mut timer = None; + let output = block_on(chain.finish_next(HtmlFinishContext { + response_status: StatusCode::OK, + services: &services, + deadline: &deadline, + timer: &mut timer, + limit: 12, + marker: None, + })) + .expect("should accept exact-cap EOF completion") + .expect("should return retained output through its replacement"); + assert_eq!( + output, b"pagemoretail", + "should include the whole retained output" + ); + assert_eq!( + observation.calls.get(), + 1, + "should not bypass exact-cap I/O" + ); + assert_ne!( + output.as_ptr() as usize, + observation.fallback_pointer.get(), + "should commit the exact-cap replacement instead of retaining fallback" + ); + assert_eq!( + observation.eof_calls.get(), + 1, + "should finish synchronous EOF once" + ); + } + #[cfg(not(target_arch = "wasm32"))] struct SlowConstructor(Rc>); #[cfg(not(target_arch = "wasm32"))] diff --git a/crates/trusted-server-core/src/integrations/html_finalization.rs b/crates/trusted-server-core/src/integrations/html_finalization.rs index bf32db48a..460770197 100644 --- a/crates/trusted-server-core/src/integrations/html_finalization.rs +++ b/crates/trusted-server-core/src/integrations/html_finalization.rs @@ -8,31 +8,29 @@ use futures::future::LocalBoxFuture; use super::{IntegrationHtmlStreamContext, IntegrationHtmlStreamProcessorFactory}; use crate::streaming_processor::StreamProcessor; -use crate::streaming_processor::finish::{StreamFinishDeadline, StreamFinishInput}; +use crate::streaming_processor::finish::StreamFinishInput; -pub use crate::streaming_processor::finish::{ +pub(crate) use crate::streaming_processor::finish::{ StreamFinish as HtmlFinalization, StreamFinishError as HtmlFinalizationError, StreamOutputWriter as HtmlReplacementWriter, }; -pub trait IntegrationHtmlFinalizerFactory: Send + Sync { +pub(crate) trait IntegrationHtmlFinalizerFactory: Send + Sync { fn integration_id(&self) -> &'static str; fn create(&self, context: IntegrationHtmlStreamContext) -> Box; } -pub trait IntegrationHtmlFinalizer { +pub(crate) trait IntegrationHtmlFinalizer { fn finalize<'a>( &'a mut self, input: HtmlFinalizationInput<'a>, ) -> LocalBoxFuture<'a, Result>>; } -pub struct HtmlFinalizationInput<'a> { +pub(crate) struct HtmlFinalizationInput<'a> { pub response_status: http::StatusCode, pub html: &'a [u8], pub replacement: &'a mut HtmlReplacementWriter, - pub services: &'a crate::platform::RuntimeServices, - pub deadline: &'a StreamFinishDeadline, } pub(super) struct TestFinishFactory(pub Arc); @@ -68,8 +66,6 @@ impl StreamProcessor for TestFinishProcessor { response_status: input.response_status, html: input.output, replacement: input.replacement, - services: input.services, - deadline: input.deadline, })) } } diff --git a/crates/trusted-server-core/src/integrations/mod.rs b/crates/trusted-server-core/src/integrations/mod.rs index 1d33b8436..2ff464f5a 100644 --- a/crates/trusted-server-core/src/integrations/mod.rs +++ b/crates/trusted-server-core/src/integrations/mod.rs @@ -31,9 +31,9 @@ pub mod sourcepoint; pub mod testlight; #[cfg(test)] -pub use html_finalization::{ - HtmlFinalization, HtmlFinalizationError, HtmlFinalizationInput, HtmlReplacementWriter, - IntegrationHtmlFinalizer, IntegrationHtmlFinalizerFactory, +pub(crate) use html_finalization::{ + HtmlFinalization, HtmlFinalizationError, HtmlFinalizationInput, IntegrationHtmlFinalizer, + IntegrationHtmlFinalizerFactory, }; pub use registry::{ diff --git a/crates/trusted-server-core/src/integrations/registry.rs b/crates/trusted-server-core/src/integrations/registry.rs index 8fdde55ea..5bfdd5450 100644 --- a/crates/trusted-server-core/src/integrations/registry.rs +++ b/crates/trusted-server-core/src/integrations/registry.rs @@ -1171,7 +1171,9 @@ impl IntegrationRegistry { /// Construct an isolated processor registry for regression tests. #[cfg(test)] #[must_use] - pub fn from_html_finalizers(factories: Vec>) -> Self { + pub(crate) fn from_html_finalizers( + factories: Vec>, + ) -> Self { Self::from_html_stream_processors( factories .into_iter() diff --git a/crates/trusted-server-core/src/platform/timer.rs b/crates/trusted-server-core/src/platform/timer.rs index 4bb9834a2..fcb78bc05 100644 --- a/crates/trusted-server-core/src/platform/timer.rs +++ b/crates/trusted-server-core/src/platform/timer.rs @@ -5,7 +5,9 @@ use std::time::Duration; /// Creates local delay futures without requiring a platform executor in core. /// -/// A delay cannot preempt blocking calls or work that never yields. +/// A delay cannot preempt blocking calls or work that never yields. Fastly does +/// not supply this service because its host calls block. Consumers on Fastly +/// must independently bound their calls using the remaining EOF budget. pub trait PlatformTimer: Send + Sync { /// Wait for the duration on the adapter's request executor. fn delay(&self, duration: Duration) -> LocalBoxFuture<'static, ()>; diff --git a/crates/trusted-server-core/src/publisher.rs b/crates/trusted-server-core/src/publisher.rs index ee2ef94de..a6f4ea833 100644 --- a/crates/trusted-server-core/src/publisher.rs +++ b/crates/trusted-server-core/src/publisher.rs @@ -4195,7 +4195,15 @@ impl InlineBodyCloseSeam { .checked_add(chunk.len()) .filter(|size| *size <= limit) .ok_or_else(overflow)?; - self.buffered.reserve_exact(next - self.buffered.len()); + if next > self.buffered.capacity() { + let capacity = self + .buffered + .capacity() + .saturating_mul(2) + .max(next) + .min(limit); + self.buffered.reserve_exact(capacity - self.buffered.len()); + } self.buffered.extend_from_slice(chunk); return Ok(Vec::new()); } @@ -22879,6 +22887,11 @@ mod tests { Some("abandoned"), "should record fatal body abandonment" ); + assert_eq!( + summaries[0].terminal_reason.as_deref(), + Some("auction_suffix_overflow"), + "should distinguish suffix bounds from encoder failures" + ); assert!( params.dispatched_auction.is_none(), "should transfer auction ownership to the failing driver" @@ -27466,6 +27479,38 @@ mod async_eof_seam_tests { ); } + #[test] + fn async_eof_seam_grows_suffix_amortized_without_exceeding_non_power_of_two_cap() { + let limit = 1031; + let mut seam = InlineBodyCloseSeam::new(b"TOKEN".to_vec()); + seam.push_bounded(b"TOKEN", limit) + .expect("should locate the marker before collecting its suffix"); + let mut capacity_changes = 0; + for _ in 0..limit { + let previous_capacity = seam.buffered.capacity(); + assert!( + seam.push_bounded(b"x", limit) + .expect("should retain the suffix within its cap") + .is_empty(), + "should withhold suffix output until auction assembly" + ); + capacity_changes += usize::from(seam.buffered.capacity() != previous_capacity); + assert!( + seam.buffered.capacity() <= limit, + "should cap allocation even when the byte cap is not a power of two" + ); + } + assert!( + capacity_changes < 32, + "should avoid a capacity increase for every small suffix chunk" + ); + assert_eq!( + seam.finish(), + vec![b'x'; limit], + "should retain the exact-cap suffix without losing bytes" + ); + } + #[test] fn async_eof_seam_handles_split_tokens_under_the_cap() { let mut seam = InlineBodyCloseSeam::new(b"TOKEN".to_vec()); diff --git a/crates/trusted-server-core/src/publisher/html_finalization.rs b/crates/trusted-server-core/src/publisher/html_finalization.rs index 1a6bd01c2..699042619 100644 --- a/crates/trusted-server-core/src/publisher/html_finalization.rs +++ b/crates/trusted-server-core/src/publisher/html_finalization.rs @@ -287,7 +287,7 @@ impl HtmlEofDriver { if end < bytes.len() { self.replay.push_front(bytes.slice(end..)); } - self.failure_reason = "stream_encode_error"; + self.failure_reason = "auction_suffix_overflow"; let processed = if let Some(seam) = self.hold.as_mut().and_then(|hold| hold.hold.as_mut()) { seam.push_bounded(&chunk, self.limit).change_context( @@ -299,6 +299,7 @@ impl HtmlEofDriver { } else { chunk.to_vec() }; + self.failure_reason = "stream_encode_error"; let encoded = self.encoder.encode_chunk(processed)?; self.had_ready_bytes |= !encoded.is_empty(); return Ok(Some(HoldStepSegments { diff --git a/crates/trusted-server-core/src/streaming_processor.rs b/crates/trusted-server-core/src/streaming_processor.rs index 05e8da763..dbb5d0821 100644 --- a/crates/trusted-server-core/src/streaming_processor.rs +++ b/crates/trusted-server-core/src/streaming_processor.rs @@ -19,6 +19,8 @@ //! streaming interface. See `crate::platform` module doc for the //! authoritative note. +pub mod finish; + use std::cell::{Cell, RefCell}; use std::io::{self, Read, Write}; use std::rc::Rc; @@ -29,8 +31,6 @@ use brotli::enc::writer::CompressorWriter; use error_stack::{Report, ResultExt as _}; use futures::future::LocalBoxFuture; -pub mod finish; - use finish::{StreamFinish, StreamFinishError, StreamFinishInput}; use flate2::read::ZlibDecoder; use flate2::write::{GzEncoder, ZlibEncoder}; diff --git a/crates/trusted-server-integration-tests/tests/documentation_snippets.rs b/crates/trusted-server-integration-tests/tests/documentation_snippets.rs index d9c36d233..bac7d0b4a 100644 --- a/crates/trusted-server-integration-tests/tests/documentation_snippets.rs +++ b/crates/trusted-server-integration-tests/tests/documentation_snippets.rs @@ -3,15 +3,24 @@ use std::process::Command; use std::time::{SystemTime, UNIX_EPOCH}; const GUIDE: &str = include_str!("../../../docs/guide/integration-guide.md"); -const START: &str = ""; -const END: &str = ""; #[test] fn integration_guide_runtime_services_fixture_compiles() { + check_guide_snippet("runtime-services"); +} + +#[test] +fn integration_guide_html_eof_fixture_compiles() { + check_guide_snippet("html-eof"); +} + +fn check_guide_snippet(name: &str) { + let start = format!(""); + let end = format!(""); let marked = GUIDE - .split_once(START) - .and_then(|(_, tail)| tail.split_once(END).map(|(body, _)| body)) - .expect("should contain one bounded runtime-services snippet"); + .split_once(&start) + .and_then(|(_, tail)| tail.split_once(&end).map(|(body, _)| body)) + .expect("should contain one bounded integration snippet"); let source = marked .trim() .strip_prefix("```rust\n") @@ -49,7 +58,7 @@ fn integration_guide_runtime_services_fixture_compiles() { }) .expect("should find the workspace error-stack requirement"); let manifest = format!( - "[package]\nname = \"documentation-snippet\"\nversion = \"0.0.0\"\nedition = \"2024\"\n\n[workspace]\n\n[dependencies]\nerror-stack = \"{error_stack_requirement}\"\ntrusted-server-core = {{ path = {:?} }}\n", + "[package]\nname = \"documentation-snippet\"\nversion = \"0.0.0\"\nedition = \"2024\"\n\n[workspace]\n\n[dependencies]\nerror-stack = \"{error_stack_requirement}\"\nfutures = \"0.3\"\ntrusted-server-core = {{ path = {:?} }}\n", core_path ); fs::write(fixture_root.join("Cargo.toml"), manifest) diff --git a/crates/trusted-server-integration-tests/tests/parity.rs b/crates/trusted-server-integration-tests/tests/parity.rs index 9196e71ba..878753963 100644 --- a/crates/trusted-server-integration-tests/tests/parity.rs +++ b/crates/trusted-server-integration-tests/tests/parity.rs @@ -1129,6 +1129,7 @@ async fn adapter_buffers_nextjs_auction_output() { #[derive(Default)] struct TerminalFixtureOrigin { lookup_calls: AtomicUsize, + fail_lookup: bool, } #[async_trait::async_trait(?Send)] @@ -1159,6 +1160,9 @@ impl PlatformHttpClient for TerminalFixtureOrigin { "should send a complete digest" ); self.lookup_calls.fetch_add(1, Ordering::SeqCst); + if self.fail_lookup { + return Err(Report::new(PlatformError::HttpClient)); + } ("text/plain", "EXAMPLE_APPROVED") } else { ( @@ -1191,78 +1195,93 @@ impl PlatformHttpClient for TerminalFixtureOrigin { #[tokio::test(flavor = "multi_thread", worker_threads = 2)] async fn asynchronous_html_finalization_and_privacy_match_host_adapters() { - let mut settings = test_settings(); - settings.publisher.allow_async_html_finish = true; - settings.integrations.insert_config("html_finalization_fixture", &serde_json::json!({ - "enabled": true, "lookup_url": "https://lookup.example.com/success", "transport_timeout_ms": 1000, + for case in ["disabled", "failed", "success"] { + let enabled = case != "disabled"; + let mut settings = test_settings(); + settings.publisher.allow_async_html_finish = enabled; + settings.integrations.insert_config("html_finalization_fixture", &serde_json::json!({ + "enabled": enabled, "lookup_url": format!("https://lookup.example.com/{case}"), "transport_timeout_ms": 1000, })).expect("should enable the fictional terminal consumer"); - let client = Arc::new(TerminalFixtureOrigin::default()); - let services = nextjs_auction::services_with_http_client( - Arc::clone(&client) as Arc - ); - let routers = [ - ( - "Axum", - AxumApp::routes_with_settings_and_services(settings.clone(), services.clone()), - ), - ( - "Cloudflare", - CloudflareApp::routes_with_settings_and_services(settings.clone(), services.clone()), - ), - ( - "Spin", - SpinApp::routes_with_settings_and_services(settings, services), - ), - ]; - let mut expected = None; - for (adapter, router) in routers { - client.lookup_calls.store(0, Ordering::SeqCst); - let request = request_builder() - .method("GET") - .uri("https://test-publisher.example.com/article") - .header("host", "test-publisher.example.com") - .header("accept", "text/html") - .body(edgezero_core::body::Body::empty()) - .expect("should build an example navigation"); - let response = router - .expect("should build fixture routes") - .oneshot(request) - .await - .expect("should serve finalized HTML"); - assert_eq!( - response.status(), - 200, - "{adapter} should preserve the origin status" - ); - assert_eq!( - response - .headers() - .get("cache-control") - .expect("should set terminal privacy"), - "no-store, private", - "{adapter} should prevent downstream caching" + let client = Arc::new(TerminalFixtureOrigin { + lookup_calls: AtomicUsize::new(0), + fail_lookup: case == "failed", + }); + let services = nextjs_auction::services_with_http_client( + Arc::clone(&client) as Arc ); - let body = response - .into_body() - .into_bytes_bounded(1024 * 1024) - .await - .expect("should drain finalized HTML"); - assert!( - String::from_utf8_lossy(&body).contains("EXAMPLE_FINALIZED"), - "{adapter} should await the yielding consumer" - ); - assert_eq!( - client.lookup_calls.load(Ordering::SeqCst), - 1, - "{adapter} should call the terminal lookup exactly once" - ); - if let Some(expected) = &expected { + let routers = [ + ( + "Axum", + AxumApp::routes_with_settings_and_services(settings.clone(), services.clone()), + ), + ( + "Cloudflare", + CloudflareApp::routes_with_settings_and_services( + settings.clone(), + services.clone(), + ), + ), + ( + "Spin", + SpinApp::routes_with_settings_and_services(settings, services), + ), + ]; + let mut expected = None; + for (adapter, router) in routers { + client.lookup_calls.store(0, Ordering::SeqCst); + let request = request_builder() + .method("GET") + .uri("https://test-publisher.example.com/article") + .header("host", "test-publisher.example.com") + .header("accept", "text/html") + .body(edgezero_core::body::Body::empty()) + .expect("should build an example navigation"); + let response = router + .expect("should build fixture routes") + .oneshot(request) + .await + .expect("should serve finalized HTML"); assert_eq!( - &body, expected, - "{adapter} should produce identical terminal HTML" + response.status(), + 200, + "{adapter} should preserve the origin status" ); - } else { - expected = Some(body); + if enabled { + assert_eq!( + response + .headers() + .get("cache-control") + .expect("should set terminal privacy"), + "no-store, private", + "{adapter} should prevent downstream caching" + ); + } + let body = response + .into_body() + .into_bytes_bounded(1024 * 1024) + .await + .expect("should drain finalized HTML"); + assert!( + String::from_utf8_lossy(&body).contains(if case == "success" { + "EXAMPLE_FINALIZED" + } else { + "EXAMPLE_UNSIGNED" + }), + "{adapter} should select the correct EOF result for {case}" + ); + assert_eq!( + client.lookup_calls.load(Ordering::SeqCst), + usize::from(enabled), + "{adapter} should call EOF lookup only for enabled work in {case}" + ); + if let Some(expected) = &expected { + assert_eq!( + &body, expected, + "{adapter} should produce identical terminal HTML" + ); + } else { + expected = Some(body); + } } } } diff --git a/docs/guide/configuration.md b/docs/guide/configuration.md index 0c634d025..961ecfba4 100644 --- a/docs/guide/configuration.md +++ b/docs/guide/configuration.md @@ -344,6 +344,8 @@ TRUSTED_SERVER__PUBLISHER__ORIGIN_URL=https://origin.publisher.com TRUSTED_SERVER__PUBLISHER__ORIGIN_HOST_HEADER_OVERRIDE=www.publisher.com TRUSTED_SERVER__PUBLISHER__PROXY_SECRET=publisher_proxy_secret TRUSTED_SERVER__PUBLISHER__MAX_BUFFERED_BODY_BYTES=16777216 +TRUSTED_SERVER__PUBLISHER__ALLOW_ASYNC_HTML_FINISH=true +TRUSTED_SERVER__PUBLISHER__HTML_FINALIZATION_TIMEOUT_MS=5000 ``` ### Field Details @@ -455,7 +457,9 @@ TTFB there even without stage retention. Enable only when accepting that deliver Shared budget for optional asynchronous processor EOF work in milliseconds. The default is `5000`; accepted values are `1` through `60000`. It begins after decoder/parser EOF -and does not change the auction deadline. Timer expiry skips later optional I/O while +and does not change the auction deadline. The initial 5-second default follows the +issue example; 60 seconds is a configuration ceiling, not a recommended delivery +latency or a measured optimum. Timer expiry skips later optional I/O while still completing downstream synchronous EOF. Timeout retains the current un-emitted stage output; it cannot undo bytes already delivered or remote side effects. @@ -469,9 +473,11 @@ See [Asynchronous HTML processor completion](/guide/integration-guide#asynchrono #### `max_buffered_body_bytes` -**Purpose**: Upper bound on how much of a publisher origin body the rewrite -pipeline holds in memory — the post-rewrite output buffer on buffered adapters, -and the per-stream raw/decoded byte ceiling on the Fastly streaming path. +**Purpose**: Limit individual publisher buffers and per-stream raw/decoded byte +totals. This is not an aggregate request-memory budget. Async retention collectors, +replacement candidates, auction suffixes, codec state and adapter response buffers +can coexist; total memory may exceed this setting. Integration-owned allocations +are outside the framework's bound. **Usage**: diff --git a/docs/guide/integration-guide.md b/docs/guide/integration-guide.md index c8172d56b..d40c26161 100644 --- a/docs/guide/integration-guide.md +++ b/docs/guide/integration-guide.md @@ -218,6 +218,58 @@ available before template lookup, so requested EOF work cannot be skipped by a processed cache hit. The factory and session share request-local `IntegrationDocumentState` with the existing parser hooks. +A non-retaining session can make a KV lookup at EOF and append a fixed footer: + + + +```rust +use std::io; +use error_stack::Report; +use futures::future::LocalBoxFuture; +use trusted_server_core::streaming_processor::StreamProcessor; +use trusted_server_core::streaming_processor::finish::{ + StreamFinish, StreamFinishError, StreamFinishInput, +}; + +/// Append a fixed footer when an example KV key is present. +pub struct ExampleFooterProcessor; + +impl StreamProcessor for ExampleFooterProcessor { + fn process_chunk(&mut self, bytes: &[u8], _is_last: bool) -> io::Result> { + Ok(bytes.to_vec()) + } + + fn requires_async_finish(&self) -> bool { + true + } + + fn async_finish<'a>( + &'a mut self, + input: StreamFinishInput<'a>, + ) -> Option>>> { + Some(Box::pin(async move { + let enabled = input.services.kv_store().get_bytes("example:footer").await + .map_err(|_| Report::new(StreamFinishError::Unavailable))? + .is_some(); + if !enabled { + return Ok(StreamFinish::Unchanged); + } + input.replacement.append(input.output)?; + input.replacement.append(b"")?; + Ok(StreamFinish::Replace) + })) + } +} +``` + + + +The factory also returns `true` from `requires_async_finish()` and creates this +session. Leave `buffer_output_until_eof()` at its default `false` for this append +operation. KV must be available on the runtime; unavailable lookup falls back to +the synchronous tail. Fastly consumers must use host operations with independently +bounded latency; this generic KV interface does not provide a hard timeout. + Operators must explicitly accept the possible delivery cost: ```toml @@ -248,7 +300,8 @@ The session and future may be non-`Send`; start I/O inside the returned future. Core buffers a stage's output only when its factory explicitly requests retention. The holder and replacement candidate each use `max_buffered_body_bytes` as their cap. -On collection overflow, core releases prefix/crossing/later output exactly once and +These caps apply per holder; simultaneous holders and codec/adapter buffers can +use more memory in total. On collection overflow, core releases prefix/crossing/later output exactly once and skips that stage's optional I/O, while still completing its synchronous EOF. It cannot bound arbitrary private allocations inside an integration. Streaming stages cannot replace bytes already delivered; EOF append does not imply safe insertion into an @@ -270,8 +323,9 @@ or external response contents. The latency contract depends on the adapter and the stage: - Without async factories, delivery and auction timing keep their existing path. -- On Fastly, a non-retaining stage releases ready HTML before optional EOF I/O; the - response can finish later. +- On Fastly, a non-retaining stage releases ready HTML before optional EOF I/O. + With an auction seam, the body-close and following tail wait for all hooks before + auction assembly; initial bytes can arrive promptly while page completion waits. - A retaining stage delays first HTML delivery until EOF or overflow release. - Axum, Workers and Spin currently buffer publisher responses. Enabled EOF I/O can add to TTFB there even without stage retention. This feature does not add streaming @@ -308,26 +362,9 @@ multiplicity including split boundaries, not HTML structure. Each real consumer verify winning-auction placement after its changes and preserve the structural body end; neighboring bytes alone cannot establish a marker's parsing context. -A fictional HTTP processor is available only with `test-utils`: - -```bash -./scripts/async-html-finalization-local-test.sh axum --auction -./scripts/async-html-finalization-local-test.sh cloudflare --auction -./scripts/async-html-finalization-local-test.sh spin --auction -./scripts/async-html-finalization-local-test.sh fastly --auction -``` - -The harness builds fresh enabled artifacts, generates isolated example configuration, -and checks disabled, successful, hung and auction controls. Fastly also runs a -non-retaining processor and proves initial body bytes arrive while its delayed EOF -lookup is pending. `--release` selects production-profile fixture artifacts. Diagnostics -are retained in the printed temporary directory. An explicit `CARGO_TARGET_DIR` permits -sharing the build cache. Workers requires tooling matching the locked SDK. - -Repository-pinned Viceroy 0.17 remains the normal WASI test runner. The Fastly hung-call -transport simulation requires an isolated Viceroy 0.19 or newer; repository pins stay -unchanged. CI runs actual Axum controls and real-process cleanup tests. The targeted -auction telemetry clock correction remains separate from broader #1240 clock work. +A fictional HTTP processor is available only with `test-utils`. See +[HTML EOF runtime checks](/guide/testing#html-eof-runtime-checks) for the runtime +harness, Python/tooling prerequisites and CI coverage. ## Browser and script guards diff --git a/docs/guide/testing.md b/docs/guide/testing.md index 86c025908..8c924c6cc 100644 --- a/docs/guide/testing.md +++ b/docs/guide/testing.md @@ -42,6 +42,40 @@ tests are in `crates/trusted-server-core/src/edge_cookie.rs`, creative rewrite tests are in `crates/trusted-server-core/src/creative.rs`, and proxy tests are in `crates/trusted-server-core/src/proxy.rs`. +## HTML EOF runtime checks + +The EOF harness builds fresh `test-utils` artifacts, creates isolated configuration +and runs disabled, successful, hung and enabled-auction controls against a loopback +origin. It requires Python **3.11 or newer**, Rust and pinned Node. Compiler/profile +settings and an explicit `CARGO_TARGET_DIR` are preserved; operator configuration and `CARGO_BUILD_TARGET` are +not inherited because the harness selects native and adapter targets. Logs, bodies, headers and timing JSON remain in its printed temporary +directory. Owned process groups are stopped on normal exit, failure and SIGTERM. + +```bash +python3 -m unittest discover -s scripts/fixtures -p 'test_async_html_finalization_runtime.py' -v +./scripts/async-html-finalization-local-test.sh axum --auction +./scripts/async-html-finalization-local-test.sh cloudflare --release --auction +./scripts/async-html-finalization-local-test.sh spin --release --auction +``` + +Workers requires `worker-build` and `wasm-bindgen` matching the locked SDK plus Wrangler. +Spin requires the Spin CLI. `--release` selects production-profile fixture artifacts. + +The Fastly EOF timeout fixture requires **Viceroy 0.19 or newer** for backend timeout +simulation. Install it in an isolated location; ordinary WASI tests continue to use +repository-pinned Viceroy 0.17: + +```bash +cargo install viceroy --version 0.19.0 --locked --root /tmp/example-eof-viceroy +PATH="/tmp/example-eof-viceroy/bin:$PATH" ./scripts/async-html-finalization-local-test.sh fastly --release --auction +``` + +The harness checks this prerequisite before building. Fastly additionally proves +that an actual body byte arrives before a non-retaining EOF lookup completes. +CI runs Axum, process-cleanup checks and an isolated Fastly EOF job. Workers/Spin +actual EOF timeout controls remain local commands; their target builds, native tests +and shared parity controls run in CI. Hosted results are separate from local runs. + ## Documentation checks Check the documentation surfaces from the repository root: diff --git a/docs/superpowers/archive/2026-10-12-1241-buffered-terminal-design.md b/docs/superpowers/archive/2026-10-12-1241-buffered-terminal-design.md deleted file mode 100644 index d63c6e721..000000000 --- a/docs/superpowers/archive/2026-10-12-1241-buffered-terminal-design.md +++ /dev/null @@ -1,691 +0,0 @@ -# Optional asynchronous HTML document finalization - -> Historical record: this specification describes the buffered terminal API. -> The [revised specification](../specs/2026-10-12-1241-async-html-finalization-design.md) -> governs the ordered EOF revision authorized on 2026-10-13. - -**Date:** 2026-10-12 - -**Status:** Implemented on `feat/1241-async-html-finalization`; review hardening and verification recorded below - -**Issue:** trusted-server #1241 - -**Inspected baseline:** `2a0cb0444dc335bd09105abe5952329bbbdfb621` - -**Implementation plan:** [Task-by-task implementation and verification](2026-10-12-1241-buffered-terminal-plan.md) - -**Implementation review:** [Requirement coverage, verification, and runtime limitations](../reviews/2026-10-12-1241-async-html-finalization-implementation.md) - -**Review hardening:** [Response metadata, range safety, telemetry clock, and CI refinements](../plans/2026-10-12-1241-review-hardening.md) - -## 1. Decision - -Add an opt-in asynchronous final-document phase to Trusted Server's HTML integration -pipeline. An enabled finalizer receives the completed, synchronously transformed HTML -document and may await request-time HTTP or KV services before producing replacement -HTML. Core owns both the retained document and a separate bounded replacement writer. - -Keep `StreamProcessor::process_chunk` synchronous and leave `StreamProcessor` and -`StreamingPipeline::process` unchanged. The new capability is registered explicitly as -an HTML finalizer, rather than being an optional method that some generic synchronous -drivers cannot execute. - -Only processable HTML documents with an enabled finalizer take this path. Existing -integrations retain their streaming behavior. Finalizers execute in their own registration -order, after all existing synchronous HTML processors. This is a terminal phase, not an -interleaved asynchronous extension to every streaming stage. - -One absolute finish deadline covers the entire finalizer chain. On optional-finalizer -failure, retain the bytes entering that finalizer; on deadline expiry, also skip all -remaining finalizers. Preserve existing fatal handling for origin, parser, synchronous -processor, compression, and downstream-write failures. - -For the first version, enabled HTML finalizers bypass shared-template cache reads and -writes before lookup. Their processed HTML responses use the existing synthesized HTML -`private, no-store` policy. Supporting cacheable asynchronous finalizers requires a -separate design for output identity, freshness, and invalidation. - -No EdgeZero revision change is required. Trusted Server owns this document lifecycle and -its request service container. A reusable EdgeZero timer can be considered separately. - -## 2. Verified problem and limits of the evidence - -At the inspected baseline: - -- `streaming_processor.rs::StreamProcessor` provides synchronous `process_chunk` and - `reset` methods. It offers no future-returning operation. -- `html_processor.rs::HtmlWithStreamingProcessors` forwards each chunk and the same - `is_last` value through the entire synchronous chain immediately. -- `IntegrationHtmlStreamProcessorFactory` creates per-document processors. Its context - contains hosts, scheme, and document-local state, but no request services. -- `publisher.rs` awaits auction collection outside synchronous processing callbacks. - The auction already has an application-specific asynchronous seam. -- Buffered materialized responses still use `StreamingPipeline::process` or the separate - reader-based auction hold loop. An asynchronous stream-helper change alone would miss - those paths. - -A local probe using the actual `StreamingPipeline` verified that `now_or_never()` completes -an immediately ready lookup but discards a lookup that returns `Pending`. Awaiting the same -yielding lookup completes it. This establishes the API gap; it does not establish that -every operation yields on every adapter. A local or already available KV result can be -immediately ready. - -No shipped integration at this baseline uses the problematic workaround. The reported -article-signing consumer and the contributor's unsubmitted patch, runtime timings, and -test results have not been independently validated. This is an extension capability, -not a claim that current synchronous integrations are broken. - -The first demonstrated requirement is whole-document text signing: obtain a signature -after reading the article and annotate the page. Such a consumer already withholds the -document. Making its buffering an explicit, bounded core responsibility adds no new -whole-document waiting requirement to that consumer. - -## 3. Goals - -1. Allow a registered HTML integration to await I/O after synchronous document processing - ends, on all four adapters. -2. Make buffer ownership, limits, cancellation, and unchanged-byte fallback enforceable - by core rather than implicit integration conventions. -3. Preserve early streaming, chunk boundaries, processor EOF counts, and auction-seam - timing when no finalizer is enabled. -4. Support both `EdgeBody::Once` and `EdgeBody::Stream`, with and without an auction. -5. Keep decoder validation ahead of finalization, and auction-seam replacement and encoder - finalization after it on the opted-in path. -6. Prevent warm shared-cache hits or downstream response caches from bypassing required - per-request finalization. -7. Specify observable, deterministic failure behavior and tests before implementation. - -## 4. Non-goals - -- Implementing an article-signing integration, an external API protocol, signing-key - management, or a new KV binding. -- Making `lol_html` callbacks await, making every chunk method asynchronous, or inserting - asynchronous processors between arbitrary synchronous processors. -- Adding another framework HTML parser or reparsing the finalizer's output. -- Changing the standalone RSC Flight processor, asset rewriting, EC behavior, consent - gates, or auction admission and transport guarantees. -- Guaranteeing interruption of synchronous Fastly host calls or CPU-heavy callback work. -- Guaranteeing cancellation or rollback of remote HTTP/KV side effects. -- Allowing finalizers to edit HTTP headers or alter the assembly protocol. -- Supporting shared-template storage of asynchronous output in the first version. -- Adding deployment manifests, a general task scheduler, background retries, persistent - queues, or a new platform clock abstraction. - -## 5. Alternatives and tradeoffs - -| Approach | Benefit | Cost or limitation | Decision | -| ------------------------------------------------ | --------------------------------------------------------------------------------------------------------------------------- | ---------------------------------------------------------------------------------------------------------------------------------------------- | ---------------------------------------------------- | -| Optional async finish on every `StreamProcessor` | Supports asynchronous completion inside a mixed streaming chain | Generic synchronous drivers cannot await it; downstream EOF propagation, wrappers, holding behavior, and services become broader contracts | Defer until an actual consumer requires interleaving | -| Make `process_chunk` asynchronous | Supports waits at every chunk boundary | Changes every processor and driver for an EOF-only need; boxed async trait methods can allocate per chunk; parser callbacks still cannot await | Reject | -| Explicit asynchronous HTML terminal phase | Matches the known whole-document consumer; leaves existing streaming traits intact; core can own and bound fallback buffers | Enabled pages wait for EOF; cannot interleave with synchronous stages | Select | -| Perform integration I/O before HTML rewriting | Produces metadata before transformations | Still requires buffering enabled pages and delays parsing; lacks the completed transformed document | Reject for this requirement | - -Opt-in buffering does not mean buffering every page. A registry without finalizers keeps -the current path. Within an opted-in page, all existing synchronous processors run first, -in their existing order, and receive exactly one EOF notification. Finalizers then run in -their own order. Registration order does not imply interleaving between these two phases. - -The framework can provide completed HTML and existing document-local capture state without -adding a parser. The consumer must separately explain how it extracts text and preserves -HTML semantics; this specification does not validate its extraction implementation. - -## 6. Ownership and registration - -### 6.1 Integration registration - -Add `IntegrationHtmlFinalizerFactory` and `IntegrationHtmlFinalizer` contracts alongside -the existing integration factory contracts. Add an empty-by-default finalizer collection -to `IntegrationRegistration` and `IntegrationRegistry`, plus the builder method -`with_html_finalizer`. Preserve factory registration order; do not sort finalizers by ID -or deduplicate registrations silently. - -A factory is `Send + Sync` and can live in the retained application registry. A factory -creates a mutable, single-use finalizer instance for each processed document using the -existing owned `IntegrationHtmlStreamContext`, including its `IntegrationDocumentState`. -The finalizer instance and its returned future need not be `Send`. - -The same document-state instance must be available to parser hooks, synchronous streaming -processors, and finalizers for that document. It must not be recreated between phases or -retained across requests, including reusable Fastly sandboxes. - -Currently `create_html_processor` constructs that state internally. The implementation -must expose the same state through a request-local construction helper or an explicit -construction input; creating a second state for finalizers would disconnect their capture -data from the parser. Preserve the current public helper's synchronous behavior. - -Finalizer presence must be discoverable from enabled registration metadata before shared -template lookup. It must not depend on polling a future, inspecting completed HTML, or -creating a finalizer instance. A factory may later return `Unchanged` when no relevant -article is present, but that cannot retroactively authorize a shared-cache lookup. - -Disabled integrations contribute no finalizers and cause no finalizer-related buffering, -privacy changes, timer creation, or cache bypass. No global setting enables this feature -without an integration explicitly registering a finalizer. - -### 6.2 Callback contract - -Use a regular, object-safe method returning `LocalBoxFuture`, with an explicit lifetime -covering mutable session state and borrowed inputs. Do not add native `async fn` to a trait -used behind `dyn` without an object-safety strategy, and do not require `Send` futures. - -The following is the intended API shape, not implementation code: - -```rust -pub trait IntegrationHtmlFinalizer { - fn finalize<'a>( - &'a mut self, - input: HtmlFinalizationInput<'a>, - ) -> LocalBoxFuture<'a, Result>>; -} - -pub struct HtmlFinalizationInput<'a> { - pub response_status: StatusCode, - pub html: &'a [u8], - pub replacement: &'a mut HtmlReplacementWriter, - pub services: &'a RuntimeServices, - pub deadline: &'a HtmlFinalizationDeadline, -} - -pub enum HtmlFinalization { - Unchanged, - Replace, -} -``` - -`Unchanged` emits the driver-owned input without cloning it. `Replace` commits only bytes -successfully written into `HtmlReplacementWriter`. Neither the original buffer nor an -unbounded replacement `Vec` is transferred into the future. - -The method must return its future promptly and perform I/O inside that future. Factory -creation is synchronous, request-local setup; it must not perform an outside lookup or -hide whole-document buffering. Public implementations must follow repository documentation -and error conventions; `HtmlFinalizationError` is a concrete error type with `Report`. - -The replacement writer is a core-owned sink, separate from `html`. Its append operations -are fallible and enforce the byte cap before extending storage. A failed append poisons -that attempt: a later `Replace` cannot commit truncated output even if the callback ignored -the writer error. Empty replacement output is valid only when no required seam token would -be lost; `Unchanged` is the explicit no-replacement result. - -The framework does not make arbitrary integration allocations safe. The cap covers its -retained HTML and replacement storage. Consumer extraction buffers and any external -response buffering must have their own bounds and be reviewed. - -## 7. Processing lifecycle - -### 7.1 No finalizers - -```text -origin -> existing decoder -> existing HTML parser and synchronous processors - -> existing auction seam controller -> existing encoder -> client -``` - -Do not collect a whole document, allocate a replacement writer or boxed finish future, -create a timer, or move auction collection to EOF. Keep existing synchronous helper -semantics. A cheap registration-presence check is acceptable. - -### 7.2 Enabled finalizers - -```text -origin -> existing decoder -> existing HTML parser and synchronous processors - -> bounded retained HTML - -> ordered asynchronous HTML finalizers - -> existing auction seam controller -> existing encoder -> client -``` - -The driver performs these steps: - -1. Create normal document-local parser/processor state and finalizer sessions. -2. Pull and decode origin chunks, run synchronous transformations, and retain the resulting - HTML under the cap. No finalizer-visible HTML is emitted before this phase completes. -3. Drain and validate decoder tails, then finish the existing synchronous chain exactly - once. Include its final output in the retained HTML. -4. Start one absolute finalization deadline and, on supporting adapters, one timer. -5. For each finalizer, lend the current retained HTML immutably and a fresh bounded - replacement writer mutably. Race its future against the same remaining deadline. -6. For an accepted `Replace`, validate the candidate and move it into current retained - storage. Drop the previous stage's input. For `Unchanged` or a soft failure, keep the - current input and discard the candidate. Finalizer B sees the accepted output of A. -7. On deadline expiry, drop the active future and candidate, skip the remaining finalizers, - and retain the current input. Earlier accepted replacements remain accepted. -8. Feed the accepted HTML into the existing auction seam controller in bounded chunks. - Emit its available prefix before collecting bids. Preserve no-seam fallback and terminal - auction telemetry. Then emit the encoder trailer and finish the response. - -Already completed synchronous processors are never re-entered with replacement output. -This prevents duplicate parsing, script injection, Next.js processing, and EOF calls. - -Callbacks observe post-rewrite HTML before reader-specific auction insertion. They cannot -claim to sign the final assembled response or content added later by the auction. A text -signing consumer must document its article scope, exclusions, and canonicalization so -framework scripts, placeholders, and later ad content do not invalidate its claims. - -### 7.3 Auction behavior - -Auction dispatch and its existing deadline remain unchanged. They are not restarted or -extended for finalization. Full-document withholding can make an opted-in page's auction -older by the time its seam reaches collection; an expired auction follows today's -empty-bid behavior. This is an explicit latency/monetization tradeoff for enabled pages. - -When no finalizer is enabled, retain the existing ability to collect at the structural -body seam before origin EOF. The terminal phase must not become a universal EOF barrier. -Missing structural body ends, bot/prefetch/consent suppression, non-HTML bodies, and -abandonment remain governed by existing auction policy. - -### 7.4 Cancellation - -The driver must own current HTML outside the pending finalizer future. Keep a losing -future's lifetime separate from the retained buffer's ownership; explicitly drop it before -moving that buffer or inspecting the borrowed replacement writer. - -Reusing a single deadline across finalizers must not put the only fallback buffer inside -a chain future that is destroyed on timeout. The driver remains the owner at every await. - -A client disconnect drops active work and request-local buffers. It does not invoke another -finalizer, persist partial output, or initiate a background retry. Preserve the existing -dispatched-auction guard so terminal abandonment is recorded exactly once on supported -drop paths. Dropping local work is not a guarantee that a remote write did not occur. - -## 8. Memory and output limits - -Use `publisher.max_buffered_body_bytes` as the cap for both retained transformed HTML and -each replacement writer. Its existing default is 16 MiB. Do not add a second independently -configured page-size limit in this change. - -Check lengths with overflow-safe arithmetic before append or reserve. Avoid capacity -growth beyond the configured cap, including `Vec` geometric reserve growth. Allocate -incrementally; do not reserve the entire cap for every small page. - -At a finalizer await, core may hold one current HTML buffer and one replacement buffer, -each capped at that limit. It must not retain previous successful candidates or clone the -input for rollback. Codec state, pending chunks, consumer-owned state, and platform response -buffers are additional memory; this is not a claim that total request heap use is 16 MiB. - -If collection for the optional terminal phase would exceed the cap: - -- Disable all finalizers for that document before starting I/O. -- On a streaming sink, release the retained prefix, the chunk that crossed the limit, and - subsequent synchronous output through the normal auction/encoding path in order. Avoid - duplicate bytes and a fallback copy of the retained prefix. -- On a buffered sink, preserve the existing outer response-size cap. Optional-finalizer - bypass does not authorize an oversized buffered response. -- Existing raw-input, decoder, parser, or response-output limit failures remain fatal. - This fallback applies only when the new optional collection layer reaches its own cap. - -A replacement-writer overflow is a soft failure of that finalizer. Discard the candidate -and continue with its input, subject to the same remaining finish budget. Never raise the -cap to accept an integration's output. - -## 9. Deadline and adapter services - -### 9.1 Configuration - -Add `publisher.html_finalization_timeout_ms`, defaulting to `5000`. It covers only the -terminal phase after decoded/synchronously processed EOF, across all finalizers together. -It is not an origin-body timeout, auction timeout, or end-to-end response deadline. - -Accept values from `1` through `60000` inclusive and reject invalid settings at startup. -The upper bound keeps the adapter timer duration comfortably within its representable -range and gives the new optional phase a finite operator-controlled budget. Match TOML -deserialization and programmatic `Publisher::default()` behavior. The field has no runtime -effect when no finalizer is registered. - -The five-second default is a proposed product default based on the issue's example, not a -claim that all adapter execution budgets permit or guarantee five seconds. Operators must -also respect their platform's invocation limits. - -### 9.2 Timer contract - -Add an optional `PlatformTimer` service to `RuntimeServices`, defaulting to absent. The -service object is `Send + Sync`; its delay future is local and need not be `Send`. A regular -object-safe method can return `LocalBoxFuture<'static, ()>` for a `Duration`. - -| Adapter | Timer implementation | Finish deadline semantics | -| ---------- | ----------------------- | ----------------------------------------------------------------------------------- | -| Cloudflare | `worker::Delay` | Cooperative race against JS-backed I/O | -| Spin | `spin_sdk::time::sleep` | Cooperative race against runtime-backed I/O | -| Axum | `tokio::time::sleep` | Cooperative race on the existing Tokio executor | -| Fastly | No timer | Check elapsed budget at control boundaries; bound host calls within the integration | - -Keep SDK imports and concrete timer implementations in Trusted Server adapters. These -primitives exist in the pinned dependencies; do not add a Tokio dependency to core or -repin EdgeZero to obtain a timer. - -Use `web_time::Instant` for absolute budget accounting and elapsed observations in core. -Do not introduce `std::time::Instant::now()` or `SystemTime::now()` on the Cloudflare path. -The review hardening exception corrects the traced auction telemetry clock using -that existing abstraction. Broader clock sites reported in #1240 remain separate -work; one successful auction smoke does not verify all of them. - -### 9.3 Race and fallback rules - -Create one absolute deadline and retain one pinned timer across the entire chain. Poll -the deadline before active work at decision points. Check absolute expiry before a stage -starts and after it resolves; if completion and expiry coincide, deadline expiry wins. -Do not restart a full-duration sleep for each finalizer. - -Callbacks receive a deadline handle exposing remaining time and whether a cooperative -timer is available. Bound individual network requests and external response reads to the -remaining budget where the service permits it. Successful but late candidates are -discarded even if a timer could not run while the callback was polling. - -Timers cannot preempt a blocking host call or a callback that never yields. On Fastly, -consumer acceptance requires evidence that its calls are bounded independently, or that -it returns `Unchanged` without performing unsupported unbounded operations. Core cannot -promise a hard five-second release for arbitrary Fastly HTTP/KV operations. - -Cancellation frees borrowed buffers and releases the publisher's wait. It does not -universally abort transport work or reverse external writes. Consumers must tolerate -duplicate requests/retries and use idempotent external operations where required. - -## 10. Failure and validation contract - -Fallback bytes are the synchronously transformed input of the current finalizer, including -accepted earlier finalizers. They are not necessarily the origin's original bytes. No -rollback of prior accepted HTML or remote side effects is promised. - -| Event | Document behavior | Remaining finalizers | -| ----------------------------------------------------------------------------- | ---------------------------------------------------------------- | ------------------------------- | -| `Unchanged` | Keep current input; discard any candidate writes | Continue | -| Valid `Replace` within budget | Commit bounded candidate | Continue with replacement | -| Callback reports unavailable/rejected external work | Keep current input; discard candidate | Continue while budget remains | -| Replacement writer overflow or invalid required marker | Keep current input; discard candidate | Continue while budget remains | -| Overall deadline expires | Drop active future and candidate; keep current input | Skip all remaining | -| Optional collection limit reached | Release retained synchronous output through ordinary delivery | Skip all finalizers | -| Origin read, decoder, parser, synchronous processor, encoder, or sink failure | Existing fatal error/truncation behavior and auction abandonment | Stop | -| Callback panics | Existing panic behavior | No new panic-catching mechanism | - -Define controlled error categories for external failure and candidate validation. Keep -`Report` at orchestration boundaries and existing `io::Error` where -streaming I/O uses it. Do not turn unrelated processor errors into silent passthrough. - -Finalizers must preserve the parser-generated control marker, when present, as opaque -bytes. They must not remove, duplicate, or relocate it into script/text content. Core -checks the active token's required cardinality before committing a replacement; missing -or duplicated tokens reject that candidate. A count check cannot prove structural -placement. This remaining invariant is a reviewed integration contract, not a claim of -automatic HTML validation without parsing. - -Core does not inspect arbitrary replacement HTML for semantic validity or reconstruct -parser observations. Finalizers must preserve valid document structure, existing CSP -requirements, URL/RSC encodings, and the meaning of upstream rewrite decisions. Header -changes and transformations needing new parser-derived security observations require a -separate capability. The first consumer should add signature metadata without changing -executable markup or article text after canonicalization. - -## 11. Cache, privacy, conditional requests, and framing - -### 11.1 Shared-template cache - -When enabled finalizer metadata is present for the request's document pipeline, disallow -shared-template lookup/reservation before the first cache read. The same decision must -exclude later storage. A factory deciding `Unchanged` at EOF does not restore eligibility. - -Use the existing non-shared inline fallback consistently across slot scripts, body-seam -selection, transformation, and assembly. Do not generate a shared-template marker for a -response that has no shared-template consumer. Record the existing request-bypass outcome -where cache response-state reporting is enabled. - -Warm entries may remain stored but must not be read while the finalizer is enabled. No -purge is required merely to activate it. Disabling an integration can restore ordinary -eligibility, subject to existing settings/build fingerprints and template validation. - -There is no first-version `shared_safe` escape hatch. Later cache support must specify at -least reader-neutral output, external-data/signing-key identity, expiration, invalidation, -and whether failures can be cached. Cold-miss ordering alone does not settle these rules. - -### 11.2 Downstream privacy and origin readthrough - -Apply the existing synthesized HTML `private, no-store` policy to body-carrying HTML -responses that enter the enabled finalizer pipeline, regardless of auction eligibility or -the finalizer outcome. Strip validators and stale downstream cache directives using the -same existing helper as other synthesized HTML. Apply this before streaming headers are -committed. An unchanged or size-bypassed result remains private for that response. - -This prevents a browser or downstream cache from treating a per-request result as the -origin's public representation. Origin readthrough remains a separate raw-byte cache: it -may still provide a complete shareable origin body under existing policy, because every -request subsequently runs transformation/finalization. Do not cache assembled output as -an origin body or modify EC-cookie privacy rules. - -For document GETs requiring finalization, reuse -`strip_conditional_and_range_headers` so client revalidation or range requests cannot -make the origin return a bodyless `304` or partial `206` instead of the full document. -That helper removes `If-None-Match`, `If-Modified-Since`, `Range`, and `If-Range`. -Do not discard mutation preconditions such as `If-Match` or `If-Unmodified-Since`, -or broaden GET behavior to other methods. Unexpected bodyless responses retain existing -status/error handling and invoke no finalizer, including the existing invalid-origin-304 -guard when the ad stack is active. This feature does not turn an existing 502 into a 304. - -### 11.3 Response selection and framing - -Run finalizers only on complete HTML that the existing routing rules already process, including -processable non-2xx HTML. Expose the actual response status so consumers can choose -which complete documents to process. With enabled finalizers, HTML with status 206 -or any `Content-Range` bypasses all body transformations and terminal I/O. Preserve -encoded bytes, range/framing metadata, and origin policy; independent cookie and -diagnostics privacy still applies. This covers range-associated error responses -such as 416 as well as partial representations. The handler and both response -wrappers must make this decision before body-dependent policy or framing mutation; -direct body helpers carry the same private metadata. Abandon dispatched bids once. -Keep non-document range forwarding intact and feature-off behavior unchanged. - -Do not rewrite unsupported encodings, unmodified pass-through -responses, assets, or standalone `text/x-component` bodies. - -HEAD and `204`, `205`, or `304` responses invoke no finalizer or finalizer I/O. Preserve -their existing framing and any dispatched-auction abandonment. Preserve method/status -handling for other requests instead of introducing a GET-only body transformer silently. - -Finalizers do not change the selected content encoding. Strip stale `Content-Length` as -the existing transformed-body path requires; stream encoders finalize only after accepted -HTML and auction output are emitted. Preserve repeated headers, cookies, and CSP policy. - -## 12. Integration into publisher paths - -The first implementation must deliberately cover these existing branches: - -| Source and sink | Auction | Required integration | -| ----------------------------------------------- | ------- | ------------------------------------------------------------------------------------- | -| Lazy response stream, `Once` or `Stream` source | None | Retain opted-in synchronous HTML output, await finalizers, then encode | -| Lazy response stream, `Once` or `Stream` source | Pending | Same phase before existing seam collection; retain auction guard | -| Buffered response, streamed source | None | Same terminal phase before completing bounded response output | -| Buffered response, streamed source | Pending | Same phase before seam/encoder finish | -| Buffered response, materialized source | None | Opted-in path cannot fall through `stream_publisher_body` and skip finalization | -| Buffered response, materialized source | Pending | Reader-based hold path must route opted-in HTML through the same terminal coordinator | - -Introduce one private HTML terminal coordinator shared by the enabled paths. Feed it the -same synchronous transformation semantics on materialized and streamed inputs. Keep -ordinary no-finalizer branches intact; avoid replacing the entire generic compression -architecture to obtain this capability. - -Do not add finalizers to a generic synchronous helper that cannot await them. Public -synchronous helpers continue to implement synchronous transformations only; document that -HTML finalizers require the supported publisher finalization entry points. Adapters must -never select a synchronous-only path for a registered finalizer silently. - -Thread document state and finalizer metadata explicitly through the relevant publisher -parameters. Do not reconstruct a separate registry/session inside a lazy stream or copy -request services into global state. Reuse existing services ownership for request-lifetime -and lazy response futures. - -## 13. Observability - -Use existing `log` instrumentation with integration ID, controlled outcome, elapsed time, -and bounded byte counts. Log ordinary completion/unchanged results at debug level and soft -failure, timeout, or optional-collection bypass at warn level. Emit at most one outcome -per attempted finalizer and one terminal collection/deadline bypass for the document. - -Never log HTML, article text, signatures, external response bodies, control tokens, client -identifiers, secrets, URLs containing request data, or arbitrary error-report attachments. -Log controlled categories instead of formatting an integration-supplied report wholesale. - -No new public response headers, persistent telemetry store, or metrics subsystem is needed. -Existing auction terminal telemetry retains its meanings and exactly-once behavior. - -## 14. Verification requirements - -The checks below are acceptance requirements for the implementation, not claims that the -unsubmitted feature has already passed them. - -### 14.1 API, registration, and state - -- Default registration has no finalizers; a disabled integration contributes none. -- Factory and finalizer ordering is deterministic, including multiple factories for one ID. -- Sessions and document capture state are isolated between requests and reused sandboxes. -- A non-`Send` future borrowing input, replacement writer, and session state compiles and - executes through each supported publisher entry point. -- Existing processors retain chunk behavior and receive EOF once before any finalizer. -- Synchronous helpers retain their documented scope; no adapter silently skips a finalizer. - -### 14.2 Real yielding, fallback, and deadlines - -- A stub that returns `Pending` at least once completes when awaited and changes output. -- Finalizer A's accepted output is exactly B's input; B never receives partial candidates. -- A timeout after partial candidate writes drops the future, emits exact input bytes, and - invokes no later finalizer. Use a drop observer and allocation/buffer identity assertions - to prove no fallback document clone. -- `Unchanged`, callback error, invalid marker, replacement overflow, and a late result - preserve exact stage input and discard partial candidate state. -- Two or more delayed finalizers share one deadline; the budget does not multiply. -- Deadline/completion ties follow the documented timeout-wins rule. Use an injectable - deterministic timer/clock in tests rather than flaky wall-clock sleeps. -- Client cancellation drops active work and retains exactly-once auction abandonment. -- Fastly tests demonstrate no timer dependency, budget checks, and bounded-call consumer - behavior; they do not pretend a synchronous call can be preempted. - -### 14.3 Bounds and byte ordering - -- Test cap minus one, exact cap, cap plus one, and length-overflow rejection. -- Test multiple chunks and one large chunk crossing the optional collection cap. Prefix, - crossing chunk, and suffix emerge once in order on the streaming bypass path. -- Buffered output caps and existing decoder/input caps retain their existing failures. -- Replacement sink overflow cannot commit even when a callback ignores its write error. -- Input and replacement capacities remain bounded; no full-cap allocation for tiny pages. -- Accepted prior stages are released as ownership advances; timeout does not restore the - origin or undo an earlier successful finalizer. - -### 14.4 Publisher and codec matrix - -Exercise `Once` and `Stream` sources, lazy and buffered sinks, auction present/absent, -and identity, gzip, deflate, and Brotli encoding. Include concatenated gzip members, -decoder tails released only at EOF, truncated/corrupt streams, empty HTML, split UTF-8, -missing body ends, and malformed-but-renderable HTML. - -Verify decoded/synchronous EOF precedes finalizer I/O, candidates preserve private marker -cardinality, ready prefixes precede auction collection, and encoder trailers follow all -accepted bytes. Finalizer failure must not leave a raw private marker in client output. - -Retain the current tests for early compressed output, parser-confirmed Next.js seams, -placeholder substitution, body-source cancellation, and auction abandonment. Those tests -must continue to use the no-finalizer path and protect its existing streaming behavior. - -### 14.5 Cache, headers, and eligibility - -- Populate a template entry with finalization disabled, enable a finalizer, and prove the - request bypasses the warm entry and runs its finalizer. Assert no lookup reservation or - store is made; cover ESI configuration falling back to inline. -- Disable the finalizer and verify ordinary cache miss/store/hit behavior is restored. -- Responses remain `private, no-store` for success, `Unchanged`, timeout, external failure, - and collection bypass, including HTML without an auction or matched ad slot. -- Raw origin readthrough hits still execute finalizers per request; no assembled output - enters the raw origin cache. -- Test synthesized GET revalidation/range handling and preserved mutation preconditions, - including unexpected origin 304 behavior with and without an active ad stack. -- Verify non-2xx HTML is processed, while HEAD, bodiless statuses, non-HTML, unsupported - encodings, and unmodified responses make no finalizer call. Preserve framing and cookies. -- Existing shared-template validation, assembly, CSP nonce, and cache-privacy tests pass - unchanged with no finalizer enabled. - -### 14.6 Actual adapters and repository gates - -Native Cloudflare/Spin tests cannot establish JS or WASI timer behavior. Add fictional -fixtures and run a bounded delayed/hung lookup on Viceroy, local Workers, Spin, and Axum. -Record the exact setup, SDK/runtime versions, observed behavior, and limitations. Do not -claim a universal exact wall-clock release; cooperative scheduling adds tolerance. - -Review hardening permits one targeted correction under #1240: auction telemetry -uses `web_time::Instant`, consistent with the orchestrator, because the captured -Workers stack names that unsupported standard-clock call. Verify the actual -Workers auction control after the correction. Broader clock paths remain separate; -do not claim all of #1240 resolved from one successful fixture. Keep KV wiring under -#1139 separate; HTTP-backed and unavailable-KV fixtures can verify this lifecycle without -requiring new mandatory store bindings. - -Before implementation PR handoff, run every CI gate in [AGENTS.md](../../../AGENTS.md#ci-gates), -including target-matched adapter checks/tests, parity, JS/docs formatting and tests, and -native build-digest checks. Run existing template-cache ESI/inline harnesses as appropriate. -Use only fictional fixture hosts, text, configuration, and credentials. - -## 15. Implementation scope and delivery - -Expected production touch points: - -- Core integration registration: terminal factory/session contracts and presence metadata. -- Core HTML/publisher processing: shared document state, bounded collection/replacement, - terminal coordinator, path selection, cache/conditional/privacy integration, and errors. -- Core platform services: optional timer and deadline helper using existing clock support. -- Core settings: the finish timeout default/validation and canonical configuration docs. -- Adapter service construction: Cloudflare, Spin, and Axum timer implementations; Fastly - explicitly retains no cooperative timer. -- Tests, integration fixtures, example configuration, and canonical integration/adapter - guides documenting opt-in behavior and limitations. - -Keep the generic streaming trait, synchronous pipeline, JS runtime, platform manifests, -and EdgeZero pin unchanged. Do not bundle the real signing consumer, KV-store wiring, -broader Cloudflare clock fixes, or unrelated publisher refactoring into this feature. -The narrowly traced auction telemetry clock correction above is the reviewed exception. - -The implementation plan should separate contract/registration work, terminal coordination, -adapter timers, cache/privacy behavior, and verification into small reviewable changes, -while not shipping an intermediate state in which enabled finalizers can be skipped. - -This document authorizes no deployment, external API use, or publication. After maintainer -review, write the implementation plan separately. The current branch contains design work -only. - -## 16. Acceptance criteria - -The feature is ready only when: - -1. A yielding finalizer works through every supported publisher response path. -2. No-finalizer output, streaming timing, auction sequencing, and error handling retain - existing behavior under their regression tests. -3. Fallback is exact at the failed stage, candidate writes are bounded and isolated, and - timeout drops the active future without cloning the retained document. -4. The deadline covers the entire terminal chain, with Fastly's non-preemption limitation - explicitly tested and documented. -5. Shared-template reads and writes cannot bypass or persist the new per-request phase; - downstream privacy and origin readthrough remain correctly separated. -6. No unsupported body, method/status, encoding, or synchronous-only adapter path silently - invokes or skips the new capability. -7. Real runtime evidence and all required repository gates are recorded with honest limits. - -## 17. References and investigation record - -- Issue #1241: requested asynchronous completion for HTML stream processors. -- PR #1135 and the [parser-aware body hold design](2026-09-07-850-parser-aware-body-hold-nextjs-streaming-design.md): - synchronous stream processors and parser-generated auction seam. -- Issues #1240 and #1139: related clock and runtime KV findings; independent scope. -- [Streaming processor](../../../crates/trusted-server-core/src/streaming_processor.rs), - [HTML processor](../../../crates/trusted-server-core/src/html_processor.rs), and - [integration registry](../../../crates/trusted-server-core/src/integrations/registry.rs). -- [Publisher](../../../crates/trusted-server-core/src/publisher.rs), - [runtime services](../../../crates/trusted-server-core/src/platform/types.rs), - [template cache contract](../../../crates/trusted-server-core/src/platform/template_cache.rs), - and [publisher settings](../../../crates/trusted-server-core/src/settings.rs). - -During investigation at the baseline, the following native core test filters passed: - -```text -cargo test -p trusted-server-core --lib streaming_processor::tests --offline -34 passed -cargo test -p trusted-server-core --lib html_processor::tests --offline -32 passed -cargo test -p trusted-server-core --lib publisher::tests --offline -369 passed -``` - -Two temporary probe tests also passed: the current synchronous pipeline discards a -yielding one-poll lookup, and awaiting the same lookup completes it. These are investigation -evidence, not implementation regression tests or proof of the contributor's runtime claims. diff --git a/docs/superpowers/archive/2026-10-12-1241-buffered-terminal-plan.md b/docs/superpowers/archive/2026-10-12-1241-buffered-terminal-plan.md deleted file mode 100644 index 97e82cfb0..000000000 --- a/docs/superpowers/archive/2026-10-12-1241-buffered-terminal-plan.md +++ /dev/null @@ -1,686 +0,0 @@ -# Optional asynchronous HTML finalization implementation plan - -> Historical record: this plan describes the buffered terminal API implemented on -> 2026-10-12. The [revised plan](../plans/2026-10-12-1241-async-html-finalization.md) -> governs the ordered EOF revision. Its test counts do not verify the revision. - -> **For agentic workers:** REQUIRED SUB-SKILL: Use superpowers:subagent-driven-development (recommended) or superpowers:executing-plans to implement this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking. - -**Goal:** Allow explicitly registered HTML integrations to await I/O after synchronous document processing, with bounded core-owned buffers and exact fallback on optional failure. - -**Architecture:** Add an HTML-only terminal phase after existing synchronous processors and before auction-seam handling and encoding. One coordinator owns retained HTML, bounded replacement storage, and the chain deadline; the no-finalizer route remains unchanged. Enabled finalizers bypass shared-template caching before lookup and use synthesized HTML privacy rules. - -**Tech Stack:** Rust 2024, existing `futures` local futures, `error-stack`, `derive_more`, `web_time`, `lol_html`, existing compression codecs, and adapter-local Workers/Spin/Tokio timers. - ---- - -**Status:** Implemented; see the execution record and review hardening for current verification - -**Date:** 2026-10-12 - -**Branch:** `feat/1241-async-html-finalization` (planning originated on `spec/1241-async-html-finalization`) - -**Baseline:** `2a0cb0444dc335bd09105abe5952329bbbdfb621` - -**Design:** [Optional asynchronous HTML document finalization](2026-10-12-1241-buffered-terminal-design.md) - -**Execution record:** [Implementation and bidirectional coverage review](../reviews/2026-10-12-1241-async-html-finalization-implementation.md) - -The task checklists below preserve the original execution instructions, including -proposed intermediate commit boundaries. The execution record is the authoritative -completion record: implementation was reviewed together before committing, and it -records actual commands, deviations, and the initially blocked Workers auction control. - -The [review hardening addendum](../plans/2026-10-12-1241-review-hardening.md) records the -subsequently authorized response-status/range refinements, telemetry clock -correction, runtime CI coverage, and renewed verification. It supplements the -original task instructions below. - -## 1. Execution boundaries - -This plan implements the selected terminal-phase design, not the original proposal to add -`async_finish` to `StreamProcessor`. Read the spec before implementing. Use -@superpowers:test-driven-development for runtime changes and -@superpowers:verification-before-completion before recording results or committing. - -The tasks below were the implementation instructions. Their expected outcomes alone -are not verification evidence; consult the execution record for observed results. Keep intermediate -commits in a draft branch; do not release an API that registrations can enable before all -publisher entry points support it. - -Keep the EdgeZero revision, generic streaming traits, JS runtime, and production platform -manifests unchanged. KV wiring under #1139, broader clock fixes under #1240, and a real signing -integration are separate work. Do not mask a live-runtime failure by replacing actual -SDK services with stubs or by reporting a native test as a runtime smoke. - -## 2. File responsibilities - -| File | Responsibility | -| --------------------------------------------------------------------------------------------------------------- | -------------------------------------------------------------------------------------------------------------- | -| Create `crates/trusted-server-core/src/integrations/html_finalization.rs` | Public factory/session/input/outcome/error contracts, bounded replacement writer, and finish deadline handle | -| Modify `crates/trusted-server-core/src/integrations/mod.rs` | Export the new contracts; later register a feature-gated fictional fixture | -| Modify `crates/trusted-server-core/src/integrations/registry.rs` | Empty-by-default finalizer metadata, builder method, ordered factory access, and test construction seam | -| Modify `crates/trusted-server-core/src/html_processor.rs` | Share one document-state instance through parser, synchronous processors, and finalizer construction | -| Create `crates/trusted-server-core/src/platform/timer.rs` | Optional platform-neutral timer contract | -| Modify `crates/trusted-server-core/src/platform/mod.rs` and `types.rs` | Timer exports, default-absent runtime service, builder/accessor support | -| Modify `crates/trusted-server-core/src/settings.rs` | Global terminal-phase timeout, matching defaults, and validation | -| Create `crates/trusted-server-core/src/publisher/html_finalization.rs` | Private retained-document collector, ordered async driver, candidate validation, and bounded replay | -| Modify `crates/trusted-server-core/src/publisher.rs` | Route opted-in HTML through the coordinator on every body/sink path; early cache/privacy/conditional decisions | -| Modify adapter `src/platform.rs` files and Fastly `src/app.rs` tests | Concrete timers and explicit Fastly timer absence | -| Create `crates/trusted-server-core/src/test_support/async_html_finalization.rs`; modify `src/test_support.rs` | Fictional integration compiled only under existing `test-utils` | -| Modify `crates/trusted-server-integration-tests/tests/parity.rs` | Host-adapter finalizer behavior and privacy parity | -| Create `scripts/fixtures/async-html-finalization-origin.py` and `scripts/async-html-finalization-local-test.sh` | Delayed/hung local HTTP lookup and actual-runtime smoke driver | -| Modify `trusted-server.example.toml` and canonical guide files | Explain the timeout, registration API, buffer/cache effects, and adapter limitations | - -Do not split or reorganize unrelated sections of `publisher.rs`. The focused child module -contains the new state machine; existing decoding, auction collection, telemetry guards, -and framing helpers remain authoritative. - -## 3. Dependency order and verification rhythm - -Tasks 1–4 establish dormant contracts and services. Task 5 implements asynchronous execution; -Task 6 implements bounded collection/replay. Task 7 establishes request-time policy before -Task 8 enables production routing. Tasks 9–11 verify codecs, adapter paths, and real runtimes. -Tasks 12–13 finish documentation and the complete gate run. - -Use native core filters for fast feedback, then the target-matched gates associated with -the touched adapters. Before each commit, run formatting, the task's tests, and its applicable -lint gate. Before PR handoff, run the entire gate list in Task 13. An unavailable runtime -is a reported verification gap, not permission to mark the check passing. - -For core-only native checkpoints, run -`cargo clippy -p trusted-server-core --all-targets --all-features -- -D warnings`. -An adapter lint alias can compile core as a dependency but does not directly lint it as -a primary package. Retain target-matched Fastly/core and adapter gates where listed. - -### Task 1: Define the terminal callback and bounded replacement writer - -**Spec coverage:** Sections 6.2, 8, 9.3, 10, and 16.3. - -**Files:** Create `crates/trusted-server-core/src/integrations/html_finalization.rs`; -modify `crates/trusted-server-core/src/integrations/mod.rs`. - -- [ ] Write failing unit tests in the new module for exact-cap append, cap overflow, - overflow-safe length checking, bounded capacity growth, poisoned-writer rejection, - and explicit `Unchanged` versus empty `Replace` outcomes. -- [ ] Define/export the spec's object-safe factory/session/input/outcome/error types. - The factory exposes `integration_id()` and `create(IntegrationHtmlStreamContext)`; - it is `Send + Sync`. A session is single-use and returns a local borrowing future. - Define `HtmlFinalizationDeadline` in this same task so the input contract compiles: - use `web_time::Instant`, a timeout `Duration`, and cooperative-timer availability; - expose remaining time and keep expiry-at-explicit-time helpers private for tests. - Cover before/exactly-at/after expiry and saturating zero remaining time. - -The callback signature is: - -```rust -pub trait IntegrationHtmlFinalizer { - fn finalize<'a>( - &'a mut self, - input: HtmlFinalizationInput<'a>, - ) -> LocalBoxFuture<'a, Result>>; -} -``` - -The input fields and result variants must match spec section 6.2 exactly. Keep the retained -HTML immutable. The writer exposes fallible `append(&[u8])`; only core can consume it to -commit a candidate. Use a concrete `derive_more::Display` error and `core::error::Error`. - -- [ ] Implement capped append: compute the next length with `checked_add`; poison and - return a controlled limit error before mutation on overflow or excess. For growth, - use an amortized capacity target bounded by the cap and reserve exactly that target. - Do not allocate the full cap up front or accept an arbitrary replacement `Vec`. -- [ ] Run `cargo test -p trusted-server-core --lib integrations::html_finalization`. - Confirm the new tests actually run, reproduce the limit failure before the fix, then pass. - Also run `cargo fmt --all -- --check`, the focused native core clippy command above, - and `cargo clippy-axum`. -- [ ] Commit the contract/writer change as `Add bounded HTML finalizer contracts`. - -The essential overflow regression has this behavior: - -```rust -let mut writer = HtmlReplacementWriter::new(4); -writer.append(b"page").expect("should accept the exact cap"); -writer.append(b"!").expect_err("should reject output beyond the cap"); -assert!(writer.is_poisoned(), "should reject later replacement commitment"); -``` - -Keep writer construction/poison inspection crate-private where only core needs them. Public -items require summaries, intra-doc links, fallible-function error documentation, and a -fictional usage example. - -### Task 2: Add the shared finish budget and optional timer contract - -**Spec coverage:** Sections 9 and 16.4. - -**Files:** Modify `integrations/html_finalization.rs`, `settings.rs`, -`platform/types.rs`, and `platform/mod.rs`; create -`crates/trusted-server-core/src/platform/timer.rs`. - -- [ ] Write failing tests for settings omission/default parity, `1`/`5000`/`60000`, - rejected `0`/`60001`, default-absent timer construction, and services clone/replacement - methods preserving an installed timer. -- [ ] Add the publisher setting with this exact default and validation: - -```rust -#[serde(default = "default_html_finalization_timeout_ms")] -#[validate(range(min = 1, max = 60000))] -pub html_finalization_timeout_ms: u32, - -fn default_html_finalization_timeout_ms() -> u32 { - 5000 -} -``` - -Update the hand-written `Publisher::default()` and only compiler-required publisher -settings literals. Do not modify unrelated OpenRTB types also named `Publisher`. - -- [ ] Add a `Send + Sync` timer trait with `delay(Duration) -> LocalBoxFuture<'static, ()>`. - Add optional `Arc` storage, a builder method, and an accessor to - `RuntimeServices`; missing timer configuration must never cause builder failure. -- [ ] Connect the timer-presence flag to Task 1's deadline type. Keep any deterministic - clock injection private to tests; do not introduce a public platform clock service. - Add settings-to-deadline tests proving the whole chain gets the configured duration, - rather than a duration independently supplied by each finalizer. -- [ ] Run `cargo test -p trusted-server-core --lib html_finalization`, - `cargo test -p trusted-server-core --lib settings::tests`, and - `cargo test -p trusted-server-core --lib platform::tests`. - Expected: new default/validation/deadline tests pass and existing service tests remain green. - Run `cargo fmt --all -- --check`, the focused native core clippy command above, - and `cargo clippy-axum`. -- [ ] Commit as `Add HTML finalization budgets and optional timers`. - -### Task 3: Register finalizers and share the parser's document state - -**Spec coverage:** Sections 6.1, 7.1, 7.2, and 14.1. - -**Files:** Modify `integrations/registry.rs`, `integrations/mod.rs`, and -`html_processor.rs`. - -- [ ] Write failing registry tests for empty defaults, disabled integration exclusion, - multiple finalizers preserving registration order, and presence checks that never - instantiate sessions. Extend the existing single-EOF processor test. -- [ ] Add `.with_html_finalizer(...)`, `has_html_finalizers()`, and ordered factory access. - Update existing hand-written registration/registry literals with empty collections. - Follow the current registry's integration enablement/build order; do not sort by ID. -- [ ] Add a `#[cfg(any(test, feature = "test-utils"))]` registry constructor from finalizer - factories for isolated core tests. Keep it out of ordinary production API builds. -- [ ] Introduce a crate-private construction helper taking an explicit document-state - instance, and keep `create_html_processor(config)` as a synchronous wrapper that - constructs state and delegates. Publisher assembly creates one state and passes clones - of its shared handle to parser hooks, streaming factories, and finalizer factories. -- [ ] Test parser-written state reaching the finalizer, two independent requests, and - retained-registry/reusable-sandbox isolation. Assert synchronous EOF is delivered once - before the first finalizer. No replacement is ever reprocessed by that chain. -- [ ] Run the registry and HTML filters: - `cargo test -p trusted-server-core --lib integrations::registry::tests` and - `cargo test -p trusted-server-core --lib html_processor::tests`. - Run `cargo test-fastly-reuse`, `cargo fmt --all -- --check`, and `cargo clippy-fastly`. -- [ ] Commit as `Register HTML finalizers with shared document state`. - -### Task 4: Wire concrete adapter timers - -**Spec coverage:** Sections 9.2, 9.3, and 14.6. - -**Files:** Modify `crates/trusted-server-adapter-axum/src/platform.rs`, -`crates/trusted-server-adapter-cloudflare/src/platform.rs`, -`crates/trusted-server-adapter-spin/src/platform.rs`, and -`crates/trusted-server-adapter-fastly/src/app.rs` tests. - -- [ ] Add service-construction tests: Axum installs a timer; production Workers and Spin - install their SDK timer under the existing runtime cfg guards; native Workers/Spin - construction remains safe without those SDKs; Fastly installs no timer. -- [ ] Implement stateless timer service objects locally. Return a boxed local future from - `worker::Delay::from(duration)`, `spin_sdk::time::sleep(duration)`, or - `tokio::time::sleep(duration)` as applicable. Delay construction is inside the returned - async future where required to avoid needing a runtime merely to build services. -- [ ] Add the timer to each supporting adapter's runtime-services builder. Preserve config, - secret, KV, HTTP, telemetry, and template service wiring. No manifest or EdgeZero changes. -- [ ] Run `cargo test-axum`, `cargo test-cloudflare`, `cargo test-spin`, and - `cargo test-fastly-reuse`. Native tests prove construction contracts, not SDK behavior. -- [ ] Run all touched adapter lint targets: `cargo clippy-axum`, - `cargo clippy-cloudflare`, `cargo clippy-cloudflare-wasm`, - `cargo clippy-spin-native`, `cargo clippy-spin-wasm`, and `cargo clippy-fastly`. - Expected: runtime cfg separation compiles on every relevant target. -- [ ] Commit as `Wire adapter timers for HTML finalization`. - -### Task 5: Implement ordered, cancellation-safe terminal execution - -**Spec coverage:** Sections 7.2–7.4, 9.3, 10, 13, and 14.2. - -**Files:** Create `crates/trusted-server-core/src/publisher/html_finalization.rs`; -declare the child module in `publisher.rs`. - -- [ ] Write failing coordinator tests using sessions that yield at least once, write partial - candidates, return unchanged/error/replacement, and record their input and drop events. - Use a local `Rc` inside a session/future to prove no accidental `Send` requirement. -- [ ] Give the coordinator ownership of current HTML outside every awaited future. Each - stage borrows current bytes and a separate fresh writer. A valid replacement moves - writer storage into current HTML; an unchanged/error result discards the writer. -- [ ] Construct one deadline and retain one pinned timer across the entire loop. Poll expiry - first, check absolute expiry before each stage and after completion, and make expiry win - ties. With no timer, await work and check elapsed budget afterward; no preemption claim. -- [ ] On timeout, explicitly drop the active future before accessing its borrowed writer or - moving current HTML; discard the candidate and stop the chain. Do not put current HTML - exclusively inside an async chain future that is dropped by a race. - -The driver-level ownership invariant is: - -```text -driver owns current HTML and current replacement writer - -> callback borrows both - -> result/timeout - -> callback future is dropped, releasing both borrows - -> driver either commits replacement or emits current HTML -``` - -- [ ] Validate poisoned writers and required marker cardinality before commitment. Preserve - the marker as an opaque integration contract; do not add a parser or pretend cardinality - proves structural placement. Callback errors remain soft; upstream/codec/sink errors - remain fatal and retain existing `Report` mappings. -- [ ] Test A-to-B output propagation, B timeout preserving A's accepted bytes, ignored writer - errors, invalid markers, one chain budget, late Fastly results, exact tie handling, and - input-buffer identity on fallback. Capture pointer identity at stage input and compare - before sink encoding, since writing to a sink necessarily copies into that sink. -- [ ] Emit controlled outcome/elapsed/byte-count logs; never format callback attachments, - HTML, signatures, tokens, or request data. Test callback drop on cancellation and ensure - no finalizer starts afterward. Panics retain existing runtime behavior. -- [ ] Run `cargo test -p trusted-server-core --lib publisher::html_finalization`, - `cargo test-fastly html_finalization`, `cargo fmt --all -- --check`, and - `cargo clippy-fastly`. Expected: all deterministic coordinator cases pass on native - and Fastly test targets. -- [ ] Commit as `Coordinate ordered HTML finalization and fallback`. - -### Task 6: Collect bounded transformed HTML and replay optional bypasses - -**Spec coverage:** Sections 7.2, 8, 12, and 14.3. - -**Files:** Modify `publisher/html_finalization.rs` and minimal collector call sites in -`publisher.rs`. - -- [ ] Add collector tests for cap minus one/exact/plus one, many chunks, a large crossing - chunk, synchronous EOF output crossing the cap, and integer-overflow rejection. -- [ ] Model collection states explicitly: collecting, bypassing, and finished. Append - transformed bytes under the existing `publisher.max_buffered_body_bytes` cap; enforce - bounded capacity growth without cloning the retained document. -- [ ] On optional collection overflow, move out the retained prefix and forward it, then the - unretained crossing chunk, then later synchronous output to ordinary seam/encoding - delivery. Skip every finalizer and create no timer or outside lookup for that document. -- [ ] Reuse existing auction hold state during bypass; a private seam token can straddle - the retained/crossing boundary. Test exact order, no duplicated EOF, no raw-token leak, - and prefix-before-auction collection. -- [ ] Retain the outer buffered writer cap and all existing raw/decoded/parser limits. A - decoder cap error is not an optional-finalizer bypass. Test both failure categories. - In route tests, reach the optional cap through rewritten-output expansion while raw - and decoded input remain within their limits; an oversized origin is an old fatal case. -- [ ] Run `cargo test -p trusted-server-core --lib publisher::html_finalization` and - `cargo test-fastly html_finalization`; run `cargo fmt --all -- --check` and - `cargo clippy-fastly`. Expected: collector/replay cases preserve bytes and old fatal limits. -- [ ] Commit as `Bound retained HTML and preserve streaming bypass`. - -### Task 7: Gate cache lookup and preserve synthesized-document policy - -**Spec coverage:** Sections 11, 14.5, and 16.5. - -**Files:** Modify `publisher.rs` and its existing cache/privacy test modules. Modify -`cache_policy.rs` only if reusing its current public helpers is insufficient. - -- [ ] Write failing request tests showing an enabled finalizer bypasses a prepopulated warm - template before lookup and excludes reservation/storage, while disabling it restores - existing eligibility. Include ESI configuration, no matched slots, consent denial, - bots/prefetch, and ordinary HTML without an auction. -- [ ] Derive the early finalizer-presence decision from enabled registry metadata. Combine - it with shared-template admission before key creation/lookup. Keep origin readthrough - eligibility separate; do not accidentally force every raw origin cache hit to miss. -- [ ] Use the existing non-shared inline assembly fallback consistently: no authorized - shared-template key, no shared-template token, and matching head/body script choices. - Do not add a first-version `shared_safe` override or a new public cache header. -- [ ] Apply `enforce_synthesized_html_cache_privacy` before response headers are committed for - HTML entering the opted-in path, including unchanged/failure/collection bypass. Preserve - duplicate cookies/CSP and existing EC/privacy finalization. -- [ ] For document GETs, extend the existing full-body predicate so - `strip_conditional_and_range_headers` removes its same four headers. Preserve `If-Match` - and `If-Unmodified-Since`. Do not broaden this stripping to assets or other methods. -- [ ] Test raw origin readthrough eligibility with finalizer metadata; validator/header - stripping; unexpected origin 304 behavior with/without the current ad-stack guard; - bodyless, unsupported-encoding, and non-HTML route selection. This task proves policy - admission and header decisions only. Assertions requiring callback execution, callback - outcome, or callback invocation counts belong to Task 8 after routing is wired. -- [ ] Run `cargo test -p trusted-server-core --lib publisher::tests`, - `cargo test-fastly publisher::tests`, `cargo fmt --all -- --check`, and - `cargo clippy-fastly`. Expected: new policy tests and all existing publisher tests pass. -- [ ] Commit as `Gate shared caching for asynchronous HTML finalizers`. - -### Task 8: Route every enabled publisher body through the terminal coordinator - -**Spec coverage:** Sections 7, 12, 14.4, and 16.1–16.2. - -**Files:** Modify `publisher.rs`, `publisher/html_finalization.rs`, and -`html_processor.rs` only where construction data must be threaded. - -- [ ] Build failing route tests across `Once`/`Stream`, lazy/buffered output, and auction - present/absent. Assert every enabled body calls its finalizer once after decoder and - synchronous EOF, and no disabled/bodiless path calls it. - Explicitly include processable 404/500 HTML and preserved method/status behavior. -- [ ] Extend relevant publisher parameters with request-local finalizer factories/session - construction data and shared document state. Keep ownership compatible with the existing - lazy response stream and borrowed buffered driver. -- [ ] Wire enabled HTML in both `publisher_response_into_streaming_response` and - `stream_publisher_body_async`/`buffer_publisher_response_async` through one coordinator. - In particular, prevent opted-in `Once` bodies without an auction from falling through - the synchronous `stream_publisher_body` path, and route enabled reader-based auction - bodies through the same terminal sequence. -- [ ] Reuse the existing decoder semantics on each body representation: drain/validate tail, - finish the synchronous chain once, run finalizers, replay accepted bytes in bounded chunks - through the existing seam controller, emit ready prefix, collect auction, finish encoder. - Do not feed candidates through `lol_html` or Next.js a second time. -- [ ] Keep current no-finalizer branches and public synchronous helper behavior intact. Add - API documentation that finalizers require supported publisher entry points; synchronous - helpers provide synchronous transformations only. -- [ ] Preserve dispatched-auction guards through collection and every new await. Test - cancellation during finalizer I/O and each fatal error category for exactly-once terminal - telemetry. Auction dispatch/deadlines must not restart or expand. -- [ ] Complete Task 7's execution-dependent assertions: raw origin readthrough hits run - finalizers on every request; success, unchanged, callback error, timeout, and collection - bypass retain private/no-store headers; HEAD/204/205/304 and unsupported routes invoke - no callback. Include enabled pages with no auction or matched slot. -- [ ] Run `cargo test-fastly`, `cargo test-fastly-reuse`, `cargo test-axum`, - `cargo test-cloudflare`, and `cargo test-spin`; then touched adapter lint aliases. - Expected: all enabled route combinations work and existing early-streaming tests pass. -- [ ] Commit as `Route opted-in HTML through terminal finalization`. - -Tasks 1–8 are one coherent feature. No release/PR readiness claim is allowed until this task -passes on all applicable paths, even if earlier dormant API commits compile. - -### Task 9: Complete the codec and legacy-behavior regression matrix - -**Spec coverage:** Sections 14.3–14.5 and 16.2–16.3. - -**Files:** Modify core tests in `publisher.rs`, `publisher/html_finalization.rs`, and -`html_processor.rs`. Keep `streaming_processor.rs` implementation unchanged. - -- [ ] Parameterize the new route tests over identity, gzip, deflate, and Brotli; test - concatenated gzip members and output released only in decoder/processor tails. -- [ ] Add malformed/truncated origin, empty HTML, split UTF-8, absent body end, marker - loss/duplication, oversized candidate, and optional collection bypass cases. -- [ ] Assert decoder validation precedes I/O, synchronous EOF precedes callback, finalizer - output precedes auction token removal, and encoder trailers follow accepted output. -- [ ] Retain and run these existing regressions unchanged: - -```text -html_stream_processors_compose_in_order_and_receive_final_once -streaming_html_with_stream_processors_rewrites_body -document_state_placeholders_substitute_through_streaming_path -streaming_finalize_emits_compressed_html_before_origin_eof -streaming_finalize_auction_hold_emits_prefix_before_origin_eof -streaming_finalize_auction_hold_emits_small_compressed_page_before_collection -parser_confirmed_auction_seam_streams_nextjs_for_every_encoding -stream_publisher_body_async_auction_hold_decodes_multi_member_gzip_buffered -body_chunk_source_resumes_after_cancelled_poll -finalizers_emit_abandoned_auction_for_bodiless_dispatched_response -``` - -- [ ] Compare no-finalizer results against the baseline using equivalent deterministic - fixtures: status, critical/repeated headers, decoded body, emission before source EOF, - and collection/telemetry counts. Do not compare request-random cookies or IDs literally. - Preserve exact encoded output where fixtures and chunking are deterministic. -- [ ] Run the three focused native filters from spec section 17 and `cargo test-fastly`. - Run `cargo fmt --all -- --check` and `cargo clippy-fastly`. -- [ ] Commit as `Verify HTML finalization across body and codec paths`. - -### Task 10: Add a feature-gated fictional consumer and host parity tests - -**Spec coverage:** Sections 4, 14.1, 14.5, and 14.6. - -**Files:** Create `crates/trusted-server-core/src/test_support/async_html_finalization.rs`; -modify `src/test_support.rs`, `integrations/mod.rs`, and -`crates/trusted-server-integration-tests/tests/parity.rs`. - -- [ ] Add a fictional `html_finalization_fixture` builder only under the existing - `test-utils` feature. Keep its Rust module and builder entry absent from ordinary builds. - Use `.without_js()` so no nonexistent JS module enters the bundle. -- [ ] Configure only fixture `enabled`, `lookup_url`, and `transport_timeout_ms` values in - the existing integration settings map. The finish budget comes exclusively from - `[publisher].html_finalization_timeout_ms`; do not introduce a second fixture deadline. -- [ ] Implement a terminal finalizer, not a buffering `StreamProcessor`. It receives - `input.html`, confirms a fictional EOF sentinel, sends a digest/EOF observation using - `json!` through `RuntimeServices::http_client`, then appends prefix/replacement/suffix - to the bounded writer. It performs no signing and needs no KV handle. -- [ ] Ensure its backend through `PlatformBackendSpec`, deriving TLS policy from the URL - scheme. Existing integration convenience helpers force TLS; do not use them for the - harness's generated loopback HTTP endpoint. Derive deterministic backend names from the - same spec where Viceroy needs a local backend mapping. -- [ ] Cap response collection for the fixture lookup. Honor remaining finish budget when - bounding Fastly first-byte/between-byte transport calls. When a transport cannot offer - the required bound without a timer, return `Unchanged` instead of an unbounded call. -- [ ] Use the normal enabled registry builder in host parity tests and inject deterministic - services through existing adapter testing seams. Verify yielding completion, unchanged - error/timeout bytes, no-finalizer output, privacy, and no mandatory KV. Do not add public - adapter constructors only to inject a custom registry. -- [ ] Run `cargo test --manifest-path crates/trusted-server-integration-tests/Cargo.toml --test parity`, - plus `cargo test-axum`, `cargo test-cloudflare`, and `cargo test-spin`. - Run `cargo clippy-axum`, `cargo clippy-cloudflare`, and `cargo clippy-spin-native`. -- [ ] Build/check normal artifacts without `test-utils` and prove the fixture builder ID is - absent using feature-off registry tests. Commit as `Add fictional HTML finalization parity fixtures`. - -### Task 11: Verify finalization on the four actual local runtimes - -**Spec coverage:** Sections 9.2–9.3 and 14.6. - -**Files:** Create `scripts/fixtures/async-html-finalization-origin.py` and -`scripts/async-html-finalization-local-test.sh`; use existing `scripts/smoke-common.sh` -helpers and provisioning examples. Do not alter production manifests. - -- [ ] Write the threaded local fixture origin with `/article/success`, `/article/hang`, and - `/lookup`. Generate fictional HTML containing `ASYNC_FINALIZATION_PENDING` and - `ARTICLE_EOF_SENTINEL`; serve the EOF sentinel in a later chunk. A lookup records receipt - of a complete-document digest/EOF observation before replying after 150 ms or remaining - pending for 30 seconds. Record start/completion counts without request-sensitive data. -- [ ] Add a driver accepting `fastly|cloudflare|spin|axum`. Generate all configs, fake secrets, - endpoints, runtime manifests, response captures, and logs in a temporary workspace. - Reuse each adapter's smoke-script provisioning pattern and `smoke-common.sh` cleanup. - Do not read operator-owned `trusted-server.toml`, `.env`, or cloud credentials. -- [ ] Build a fresh fixture artifact with the feature explicitly enabled and isolated output - paths. Do not reuse an ordinary cached Worker bundle, and do not call the existing - Cloudflare `build.sh`, which sources operator env files and rebuilds without fixture flags. - -Build commands used by the driver, with its own `CARGO_TARGET_DIR`, are: - -```bash -cargo build -p trusted-server-adapter-fastly --target wasm32-wasip1 --release --features trusted-server-core/test-utils -cargo build -p trusted-server-adapter-axum --features trusted-server-core/test-utils -cargo build -p trusted-server-adapter-spin --target wasm32-wasip1 --release --features spin,trusted-server-core/test-utils -``` - -For Workers, resolve the `worker-build` version from `Cargo.lock`, use that installed -version, and invoke from `crates/trusted-server-adapter-cloudflare`. The driver sets -`FINALIZATION_WORKER_OUTPUT` to an absolute directory inside its temporary workspace: - -```bash -worker-build --release --out-dir "$FINALIZATION_WORKER_OUTPUT" . --features trusted-server-core/test-utils -``` - -Point the temporary Wrangler manifest directly at that fixture bundle and disable implicit -rebuild. Use temporary manifest copies for Viceroy and Spin; point them to the fresh -fixture artifacts. Provision only existing application config/secrets infrastructure. -The new consumer must not require an integration KV namespace or label. - -- [ ] Run disabled/enabled cases on the same artifact, host, and port with auctions and - request-random body content disabled. Capture baseline transformed bodies. Success changes - only the expected sentinel; a hung lookup returns exact baseline body bytes. -- [ ] Set the fixture publisher finish timeout to 750 ms. On Axum/Workers/Spin, keep the - transport hang much longer and verify release within a documented cooperative scheduling - tolerance, well before 30 seconds. Record elapsed values and lookup invocation counts; - do not make native tests stand in for SDK timers. - Require a controlled timeout outcome so an early transport error cannot pass the test. - Measure terminal-phase elapsed time separately from origin transfer and server readiness. -- [ ] For Fastly, use an independently bounded transport timeout within the remaining finish - budget and verify unchanged fallback after the host timeout. Report the host bound and - elapsed checks; never describe this as timer-driven interruption. -- [ ] Run the actual runtime entry points through the driver: - -```bash -./scripts/async-html-finalization-local-test.sh fastly -./scripts/async-html-finalization-local-test.sh cloudflare -./scripts/async-html-finalization-local-test.sh spin -./scripts/async-html-finalization-local-test.sh axum -``` - -The driver launches `viceroy serve`, `wrangler dev --local`, `spin up`, or the freshly built -Axum binary with its generated manifests/environment, waits for readiness, captures -headers/body/timing, and always terminates child process groups. Missing tools fail with -an explicit prerequisite message; a script cannot silently skip a selected adapter. - -- [ ] Exercise auction-enabled runtime cases separately where supported. If the existing - Cloudflare clock panic prevents that case, record the exact failure and #1240 dependency; - no change to this feature's scope and no passing claim for the blocked case. -- [ ] Check script syntax and fixture server tests; run - `bash -n scripts/async-html-finalization-local-test.sh` and - `python3 -m py_compile scripts/fixtures/async-html-finalization-origin.py`. - Commit as `Add local runtime checks for HTML finalization`. - -### Task 12: Document the operator and integration contracts - -**Spec coverage:** Sections 6–13 and 15. - -**Files:** Modify `trusted-server.example.toml`, `docs/guide/configuration.md`, -`docs/guide/integration-guide.md`, `docs/guide/cloudflare.md`, -`docs/guide/spin.md`, `docs/guide/axum-dev.md`, and -`crates/trusted-server-integration-tests/README.md` only in relevant sections. - -- [ ] Add the optional publisher setting to the example configuration and canonical settings - table: default 5000, range 1–60000, chain scope after EOF, no effect without registration. - Put comments above configuration values, following repository conventions. -- [ ] Document `.with_html_finalizer`, object-safe local borrowing futures, terminal ordering, - shared document capture state, bounded replacement writes, unchanged fallback, and the - distinction between stage-input bytes and origin bytes. Include a fictional example. -- [ ] Explain opted-in whole-page latency, unchanged no-finalizer streaming, auction budget - interactions, shared-template bypass, `private, no-store`, raw readthrough separation, - and preservation of internal markers/CSP/RSC semantics. -- [ ] Describe timer availability, cooperative scheduling, Fastly host bounds, remote-side-effect - limitations, and independent KV/clock work without copying volatile adapter matrices into - several guides. Link to the canonical integration contract from each affected guide. -- [ ] Document the feature-gated fixture and exact runtime commands, prerequisite versions, - generated temporary workspace behavior, and how to record blocked smoke cases. -- [ ] Run `cargo doc -p trusted-server-core --no-deps --all-features`, `npm run lint` and - `npm run format` from `docs`, plus the repository Markdown check from Task 13. - Validate new local links. Commit as `Document asynchronous HTML finalizer behavior`. - -### Task 13: Run complete gates and review the implemented change against both documents - -**Spec coverage:** Sections 14, 16, and every coverage row below. - -**Files:** No additional production changes unless a check exposes a defect in this feature. - -- [ ] Inspect the diff for scope: no generic streaming trait change, EdgeZero repin, mandatory - store, real signing consumer, JS behavior change, or unrelated refactor. Check feature-off - fixture exclusion and request-state isolation. -- [ ] Run these Rust gates from the workspace root, using the target-matched aliases: - -```bash -cargo fmt --all -- --check -cargo clippy-fastly -cargo clippy-axum -cargo clippy-cloudflare -cargo clippy-cloudflare-wasm -cargo clippy-spin-native -cargo clippy-spin-wasm -cargo clippy-cli -cargo clippy-codegen -cargo test-fastly -cargo test-fastly-reuse -cargo test-axum -cargo test-cloudflare -cargo test-spin -cargo test --manifest-path crates/trusted-server-integration-tests/Cargo.toml --test parity -cargo test -p trusted-server-core --test template_build_digest -cargo clippy -p trusted-server-core --test template_build_digest -- -D warnings -``` - -- [ ] From `crates/trusted-server-js/lib`, run `node build-all.mjs`, `npx vitest run`, and - `npm run format`. From `docs`, run `npm run lint` and `npm run format`. - Install locked docs dependencies with `npm ci` if absent; then from root run: - -```bash -docs/node_modules/.bin/prettier --config docs/.prettierrc --check "*.md" ".claude/**/*.md" ".github/**/*.md" "crates/**/*.md" "scripts/**/*.md" "tinybird/**/*.md" -``` - -- [ ] Run the existing shared-template controls: - -```bash -./scripts/template-cache-local-test.sh -./scripts/template-cache-local-test.sh inline -``` - -Also run the feature's four runtime commands from Task 11 and the existing integration -jobs described in `scripts/integration-tests.sh` and `scripts/integration-tests-browser.sh` -where the feature changes their exercised routes. Record setup and exact outcomes; do not -substitute an empty-auction fixture for the existing winning-bid checks. - -- [ ] Map implementation and tests to every coverage row below. Have a reviewer inspect - the spec and plan together with the final diff; resolve behavioral contradictions centrally. - Do not weaken a regression assertion merely to match a new implementation. -- [ ] Record passed gates and all unavailable/blocked runtime cases in the PR handoff. If any - required gate is unresolved, keep the PR draft. No merge/deploy authorization is implied. - -## 4. Bidirectional spec/plan coverage - -Every requirement must have a task and evidence, and every task must have a requirement. -Use this table during review, not just as a task checklist. - -| Spec contract | Implementation tasks | Required evidence | -| -------------------------------------------------------------------------------------- | -------------------- | ------------------------------------------------------------------------------- | -| Sections 1, 4, 5: HTML terminal phase; generic streaming unchanged; no unrelated work | 1, 3, 8, 13 | API/diff inspection; unchanged legacy streaming tests | -| Section 6.1: ordered registration and one request-local document state | 3, 10 | factory-order and capture-state tests; request/reuse isolation | -| Section 6.2: local borrowing future and bounded replacement sink | 1, 5 | non-`Send` execution; poisoned writer tests | -| Section 7.1: no-finalizer behavior unchanged | 3, 8, 9 | prefix before EOF; one synchronous EOF; no timer/finalizer I/O | -| Section 7.2: drain decoder, synchronous EOF, terminal chain, seam, encoder | 5, 8, 9 | all body/sink/auction/codec combinations; exact event ordering | -| Section 7.3: auction deadline unchanged; opted-in delay explicit | 8, 9, 11, 12 | unchanged dispatch deadline; prefix-before-collection; expired-auction behavior | -| Section 7.4: cancellation releases borrows and does not lose owned input | 5, 8 | drop observer, buffer identity, exactly-once auction abandonment | -| Section 8: capped collection/candidate capacity; safe optional overflow bypass | 1, 6, 9 | boundary/overflow/capacity tests; exact replay; old fatal caps | -| Section 9.1: default 5000 and validated 1–60000 chain budget | 2, 12 | defaults/TOML/range tests; canonical configuration docs | -| Sections 9.2–9.3: optional SDK timers, one absolute budget, Fastly limitations | 1, 2, 4, 5, 11 | deterministic deadline tests; actual-runtime timings; bounded host calls | -| Section 10: stage-local fallback, error categories, markers, panic behavior | 1, 5, 9, 12 | each outcome/candidate failure; late/tie behavior; reviewed callback contract | -| Section 11.1: template bypass before lookup/storage and consistent inline fallback | 7, 9 | warm-hit bypass; no store/reservation; disabled cache behavior restored | -| Section 11.2: downstream privacy, raw-origin cache separation, conditional/range rules | 7, 8, 10 | private headers with no auction; raw-hit execution; four-header stripping | -| Section 11.3: existing body eligibility, status/method/framing/encoding | 7, 8, 9 | bodiless/no-HTML/unsupported cases; non-2xx HTML; header preservation | -| Section 12: every materialized/streamed and lazy/buffered branch | 8, 9, 10 | route matrix and host parity; no synchronous-only skip | -| Section 13: controlled logging and preserved auction telemetry | 5, 8, 11 | no sensitive log fields; outcome counts and abandonment tests | -| Section 14: deterministic, real-runtime, and full repository verification | 9, 10, 11, 13 | named regressions, runtime reports, complete gates | -| Sections 15–16: scoped delivery and readiness conditions | 12, 13 | guides, diff review, requirements/evidence mapping | - -## 5. Review and handoff record - -Before implementation planning is considered ready, review both documents for: - -- Spec requirements omitted from the plan or tasks that contradict the spec. -- Plan additions without a spec requirement, especially fixtures becoming production - consumers, private buffering reappearing, or deadlines multiplying per callback. -- Unrealistic runtime commands, unsupported cfg combinations, and existing behavior that - the plan mistakenly assumes instead of checking. -- Ownership of current HTML during timeout, shared state during factory construction, and - early cache admission that an EOF-only decision could not influence. - -Document review results describe planning readiness only. They do not prove the feature -is implemented, regression-free, or ready for deployment. Execution begins only after -the user requests implementation. - -### Review completed on 2026-10-12 - -The standalone spec review and both directions of the independent spec/plan alignment -review approved planning readiness with no unresolved substantive findings. - -The review corrected these points before handoff: - -- Policy-admission tests in Task 7 no longer require callback execution first wired in - Task 8; execution-dependent assertions now follow the routing implementation. -- The deadline type referenced by the callback API is defined in Task 1, avoiding an - unbuildable intermediate contract commit. -- Core edits have a direct primary-package lint checkpoint; non-2xx HTML is an explicit - acceptance test rather than implicit coverage. -- Overflow route tests distinguish rewrite expansion from preexisting fatal decoder caps. -- Workers fixture output stays in the temporary workspace; runtime timeout tests verify - timer expiry rather than accidentally passing on an early transport error. - -This historical record covers planning review only. The linked execution record now -maps the implemented contracts and observed verification to both documents. diff --git a/docs/superpowers/plans/2026-10-12-1241-async-html-finalization.md b/docs/superpowers/plans/2026-10-12-1241-async-html-finalization.md index de87382c1..4aba58584 100644 --- a/docs/superpowers/plans/2026-10-12-1241-async-html-finalization.md +++ b/docs/superpowers/plans/2026-10-12-1241-async-html-finalization.md @@ -1,145 +1,76 @@ # Ordered HTML EOF Completion Implementation Plan -> **For agentic workers:** REQUIRED SUB-SKILL: Use superpowers:subagent-driven-development or superpowers:executing-plans to implement this plan task-by-task. Follow repository instructions for centralized implementation and independent review. Steps use checkbox syntax for tracking. +> **For agentic workers:** REQUIRED SUB-SKILL: Use superpowers:executing-plans or superpowers:subagent-driven-development. Follow repository instructions for centralized implementation and independent review. -**Goal:** Let registered HTML processors optionally await EOF I/O in stage order without mandatory whole-page buffering. +**Goal:** Support optional ordered EOF I/O with explicit operator permission, bounded held output and predictable failure behavior. -**Architecture:** Keep synchronous chunk processing and add one optional processor EOF callback. A stepped concrete HTML chain lends driver-owned stage output to a bounded callback and sends the accepted result downstream before downstream EOF. Operator permission is default off; only explicit retention stages buffer output. +**Architecture:** Keep synchronous chunk processing and one optional `StreamProcessor::async_finish` callback. Complete each stage before downstream EOF, stream ordinary output, retain output only on explicit declaration, and finish auction/encoding afterward. Limits apply per holder; this is not a total request-memory guarantee. -**Tech Stack:** Rust 2024, futures LocalBoxFuture, error-stack, lol_html, existing adapter timers and publisher codec/auction drivers. +**Tech Stack:** Rust 2024, futures local futures, error-stack, lol_html, adapter timers, Python 3.11+ and GitHub Actions. -**Revised:** 2026-10-13. User authorizes specification, plan and implementation updates. -The [specification](../specs/2026-10-12-1241-async-html-finalization-design.md) governs this revision. -The [verification report](../reviews/2026-10-13-1241-ordered-eof-verification.md) records fresh revised-source evidence. -The [old plan](../archive/2026-10-12-1241-buffered-terminal-plan.md) and historical implementation records describe the superseded terminal phase. +**Design:** [Ordered asynchronous HTML processor completion](../specs/2026-10-12-1241-async-html-finalization-design.md). -## File responsibilities +## Implemented lifecycle -| File | Responsibility | -| ----------------------------------------------------- | --------------------------------------------------------------------------------- | -| `streaming_processor/finish.rs` | Optional finish input/outcome/error, bounded writer and absolute deadline | -| `streaming_processor.rs` | Object-safe default-absent callback and synchronous rejection | -| `integrations/registry.rs` | One processor registration sequence, early capabilities and permission validation | -| `html_processor.rs` and `html_processor/finish.rs` | Concrete stepped chain, stage retention and leaf callback completion | -| `publisher/html_finalization.rs` | Existing body/codec/auction driver adapted to stepped EOF | -| `publisher.rs` | Early capability routing, direct-helper guards and cache/privacy policy | -| `settings.rs` | Default-off operator permission; existing finish timeout retained | -| `test_support/async_html_finalization.rs` | Fictional EOF HTTP processor using the production lifecycle | -| `scripts/fixtures/async-html-finalization-runtime.py` | Explicit permission in enabled runtime configuration | -| Integration/config guides and example TOML | Capability semantics, operator acceptance and runtime limits | +- [x] Add default-absent borrowing callback contracts, bounded candidate writer and shared absolute deadline in `streaming_processor.rs` and `streaming_processor/finish.rs`. +- [x] Declare async/retention capabilities in the existing registry sequence; reject missing operator permission and inconsistent metadata before input effects. +- [x] Implement stepped parser/stage EOF, optional retention, overflow replay, downstream propagation and cancellation in `html_processor.rs` and `html_processor/finish.rs`. +- [x] Migrate the publisher driver without universal buffering; preserve auction/codec ordering, cache/privacy and partial-response safeguards. +- [x] Add production-lifecycle fictional fixtures, native/WASI regression coverage and actual four-runtime controls. -Paths above are relative to `crates/trusted-server-core/src/` unless prefixed with `scripts/` or documentation paths. +## Review hardening -## Task 1: Review the revised specification and plan +### 1. Auction suffix storage and diagnostics -- [x] Independently review both documents against issue1241 and the accepted latency policy. -- [x] Resolve lifecycle, ownership, cancellation, cache and runtime contradictions before implementation. -- [x] Archive the old documents and link historical reports to the archived design. +**Files:** `crates/trusted-server-core/src/publisher.rs`, `crates/trusted-server-core/src/publisher/html_finalization.rs`. -## Task 2: Optional callback contract and default behavior +- [x] Extend the fatal-overflow regression to require `auction_suffix_overflow` telemetry. Add repeated-chunk growth coverage with a non-power-of-two cap. +- [x] Run `cargo test -p trusted-server-core --all-features async_eof` and observe the reason/growth failures. +- [x] Reserve only when needed, doubling capacity up to the checked cap. Classify suffix overflow before scanning and encoding errors immediately before encoding. +- [x] Repeat the focused tests and relevant adapter checks. -**Files:** `crates/trusted-server-core/src/streaming_processor.rs`, new `streaming_processor/finish.rs`, remove production finalizer API from `integrations/html_finalization.rs` and `integrations/mod.rs`. +### 2. Harness configuration, build settings and termination -- [x] Add failing tests for default `None`, a borrowing non-Send callback, bounded/poisoned replacement and synchronous rejection before reader/writer effects. -- [x] Run `cargo test -p trusted-server-core --all-features async_eof` and record expected missing-capability failures. -- [x] Add `requires_async_finish` and `async_finish` defaults to `StreamProcessor`; adapt existing writer/deadline implementations into generic finish contracts. -- [x] Keep `process_chunk` unchanged. Document that callback input is un-emitted stage output, not necessarily the whole document. -- [x] Make `StreamingPipeline::process` reject async-capable processors before consuming input. -- [x] Re-run targeted tests; format and inspect the diff. +**Files:** `scripts/fixtures/async-html-finalization-runtime.py`, `scripts/fixtures/test_async_html_finalization_runtime.py`. -## Task 3: Registration and operator permission +- [x] Add failing tests for retained compiler/profile settings, missing template matches, invalid generated configuration, unsupported Viceroy and parent SIGTERM cleanup. +- [x] Run `python3 -m unittest discover -s scripts/fixtures -p 'test_async_html_finalization_runtime.py' -v` and record failures before fixes. +- [x] Preserve explicit Rust build settings without inheriting operator configuration; check template replacements and parsed timeout/permission/loopback targets before configuration push. +- [x] Safely quote TOML paths, validate Fastly tooling before builds, unwind on SIGTERM, and require cooperative timeout diagnostics. +- [x] Repeat unit/real-process tests and all four actual-runtime commands in `scripts/README.md`. -**Files:** `integrations/registry.rs`, `settings.rs`, `trusted-server.example.toml`, fixture configuration tests. +### 3. Remaining EOF acceptance boundaries -- [x] Write failing tests: permission defaults false; enabled async factory requires explicit acceptance; disabled integration needs no acceptance; permission alone creates no capability; buffered declaration without async capability is invalid. -- [x] Add `publisher.allow_async_html_finish` with serde/default false. -- [x] Add factory capability metadata and preserve the existing single factory sequence. -- [x] Reject missing permission with integration ID and setting in the error. Do not filter or silently omit requested stages. -- [x] Remove separate public finalizer registration and metadata. Test adapters may wrap old fake actions into ordinary production processors. -- [x] Run settings/registry tests and target-matched Axum checks. +**File:** `crates/trusted-server-core/src/html_processor/finish.rs`. -## Task 4: Ordered, stepped processor EOF +- [x] Cover elapsed timerless completion after polling, oversized non-retained fallback, one-byte markers, duplicate parser EOF and successful EOF at the retention cap. +- [x] Verify fallback ownership, no late commitment, no optional polling after overflow and exactly-once EOF. +- [x] Keep existing cancellation, poisoning, ties, split markers and accepted-upstream-output tests; verify disabled, failed and successful EOF parity across all host adapters. -**Files:** `html_processor.rs`, new `html_processor/finish.rs`, `publisher.rs` concrete async construction helper. +### 4. CI and maintainer documentation -- [x] Write a failing chain test: streaming A, yielding async B, downstream C; assert C transforms B's async output before C receives EOF and every stage receives EOF once. -- [x] Write pending-callback tests proving parser and prior-stage ready bytes are available first. -- [x] Write timeout tests proving downstream synchronous EOF still executes; never race the whole composite future. -- [x] Add boundary-aware marker tests for ordinary-output/EOF splits, buffered/overflow stages, and upstream replacements feeding downstream stages. -- [x] Expose crate-private concrete HTML composition for the enabled driver while preserving public opaque helpers for synchronous callers. -- [x] Implement parser finish, stage-index advancement, driver-owned fallback, bounded candidate and downstream ordinary-chunk propagation. -- [x] Reuse one absolute deadline/timer. Drop each losing callback before selecting output. -- [x] Check factory/session capability consistency before parser effects; reject synchronous composite use with async stages. -- [x] Run `cargo test -p trusted-server-core --all-features async_eof` plus existing HTML/Next.js tests. +**Files:** `.github/workflows/test.yml`, `AGENTS.md`, `scripts/README.md`, `docs/guide/testing.md`, configuration/integration guides, canonical spec/plan. -## Task 5: Explicit bounded buffering +- [x] Add isolated Viceroy 0.19 Fastly EOF controls to CI, preserving pinned 0.17 ordinary tests. Cache the isolated tool and retain failure artifacts. +- [x] Document Python/runtime prerequisites, exact cleanup/Axum/Fastly commands and compiler setting preservation. Validate workflow syntax and the actual commands locally. +- [x] Explain per-holder memory limits, tail latency, Fastly consumer transport bounds and the configured timeout policy. Add a minimal Rust callback example and environment override documentation. +- [x] Keep the current spec and plan; remove superseded archived designs, hardening logs and execution histories. Preserve executed evidence in the PR description. +- [x] State the deferred processed-template cache requirement explicitly in the draft PR, and document new required public fields. -**Files:** `html_processor/finish.rs` and its tests. +### 5. Verification and publication -- [x] Write failing tests for full retained input at EOF, cap crossing and one-time replay; verify callbacks do not start after overflow. -- [x] Add framework-owned stage-output retention only for declaring factories; default stages stream through. -- [x] Move retained bytes into fallback at EOF. Skip optional work for oversized streaming EOF fallback without truncation. -- [x] Preserve exact allocation identity on unchanged/error/timeout; preserve fatal outer buffered-output limits. -- [x] Run holder/finish tests and memory-boundary cases. - -## Task 6: Publisher, auction and codec migration - -**Files:** `publisher/html_finalization.rs`, `publisher.rs`, publisher test helpers. - -- [x] Add failing tests for early output while EOF work is pending across materialized/streamed sources and write/lazy sinks. -- [x] Remove universal collection and separate sessions from the driver. Use the concrete chain's stepped EOF operations. -- [x] Drain decoder/parser ready output before starting stage callbacks. Finish auction/encoder after all stage EOF. -- [x] Cap the held auction suffix with `publisher.max_buffered_body_bytes`, check before growth, split releasable prefixes before copying, and test exact-cap/crossing plus fatal abandonment for large post-body content. -- [x] Keep exact flush-before-await barriers; avoid flushing every replay chunk. -- [x] Guard direct async entry points for operator permission and synchronous helpers before effects. -- [x] Preserve range/bodiless/unsupported bypass, actual status, early template-cache admission and independent privacy. -- [x] Migrate existing full-document matrix tests via a test-only buffered processor adapter, not another terminal list. -- [x] Run `cargo test -p trusted-server-core --all-features` and Fastly/WASI regression tests. - -## Task 7: Runtime fixture and guides - -**Files:** `test_support/async_html_finalization.rs`, registry fixture tests, runtime Python configuration, guides, example TOML. - -- [x] Migrate the fixture to an identity processor requesting bounded output retention and optional HTTP finish. -- [x] Add explicit operator permission to enabled harness configurations; disabled controls retain default permission. -- [x] Require a non-retaining EOF fixture/control proving a prefix reaches the Fastly client before a pending callback completes; keep retained controls for fallback and other runtimes. -- [x] Update public examples and guides to one callback API, stage order, fallback scope, explicit buffering and actual adapter TTFB limits. -- [x] Preserve existing cleanup, deadline, exact fallback, privacy and auction harness assertions. -- [x] Run Bash/Python validation and real-process cleanup tests. - -## Task 8: Full verification and review - -- [x] Run cargo formatting and all eight target-matched Clippy aliases. -- [x] Run all five adapter test aliases, native core all-feature/digest tests and lint, CLI/codegen tests, parity and exact integration-test CI selections. -- [x] Run existing ESI/inline/purge and winning-bid controls; preserve no-hook cache behavior. -- [x] Run JS build/tests/format, docs lint/build/format and repository Markdown format. -- [x] Run actual Axum, Workers, Spin and Fastly enabled/disabled/success/hung/auction controls on final source; qualify installed-tool/runtime limits honestly. -- [x] Obtain independent architecture and implementation review; fix findings and repeat only affected checks. -- [x] Record source and evidence in a new verification report; do not repurpose historical counts as revised-design evidence. - -## Task 9: Publish the reviewed revision - -- [x] Verify clean intended diff, archive links, spec/plan/implementation alignment and fresh passing checks. -- [x] Commit using repository sentence-case conventions and push the existing feature branch. -- [x] Rewrite draft PR description around generic ordered EOF completion and operator permission; keep issue1241 as the target and keep draft status. -- [x] Confirm remote head/body and report any remaining maintainer decisions or pending hosted checks. +- [x] Obtain independent source/spec/plan review and resolve concrete findings. +- [x] Run `cargo fmt --all -- --check` and the integration crate format check, all eight target-matched Clippy aliases, all adapter test aliases and Fastly reuse, native core/digest gates, CLI/codegen, parity and exact integration CI selections. +- [x] Run existing ESI/inline/purge controls, JS build/tests/format, docs lint/build/format and repository Markdown formatting. +- [x] Record fresh four-runtime success, disabled, hung and auction results, including Fastly early-body-byte proof. + **Publication procedure:** Commit and push the existing branch, update draft PR #1290, and confirm remote head/status. Hosted CI results remain separate from local checks. ## Acceptance mapping -| Specification | Plan coverage | -| ------------------------------------------------- | -------------- | -| Generic callback and single lifecycle | Tasks 2–4 | -| Operator acceptance and early metadata | Task 3, Task 6 | -| EOF order and release before waits | Tasks 4, 6 | -| Optional bounded retention and fallback ownership | Tasks 2, 4, 5 | -| Deadline/cancellation and downstream EOF | Tasks 2, 4, 6 | -| Auction/codec/partial/cache/privacy safeguards | Task 6, Task 8 | -| Honest adapter TTFB contract and consumer scope | Task 7, Task 8 | -| Reproducible verification and draft publication | Tasks 8–9 | - -## Execution completion - -Implementation commit `a533df267c149bb8295b23e5d5a1fb700c1b32eb` was pushed and -draft PR #1290 was updated and verified. The newer remote merge was preserved. -All local verification passed; hosted CI results remain separate. This completion -record changes documentation only. See the [verification report](../reviews/2026-10-13-1241-ordered-eof-verification.md). +| Requirement | Coverage | +| ------------------------------------------------- | ------------------------------------------------------------------- | +| Ordered generic lifecycle and explicit permission | Implemented lifecycle plus EOF boundary tests | +| Per-holder limits, ownership and cancellation | Suffix growth, fallback and deadline tests | +| No-hook delivery/cache behavior | Existing publisher and template-cache controls | +| Honest opt-in latency and cache scope | Canonical spec, configuration/integration guides and PR description | +| Reliable runtime evidence and repeatability | Harness regressions, actual runtimes and isolated Fastly CI job | diff --git a/docs/superpowers/plans/2026-10-12-1241-review-hardening.md b/docs/superpowers/plans/2026-10-12-1241-review-hardening.md deleted file mode 100644 index 3836a0dcf..000000000 --- a/docs/superpowers/plans/2026-10-12-1241-review-hardening.md +++ /dev/null @@ -1,183 +0,0 @@ -# HTML finalization review hardening implementation plan - -**Historical record:** This describes the buffered terminal implementation before the -2026-10-13 ordered EOF revision. Its verification counts do not verify the revised -implementation. See the current spec and plan for the active architecture. - -> **For agentic workers:** Use superpowers:executing-plans for central implementation and independent review. Checkboxes record observed completion. - -**Goal:** Close the confirmed response-eligibility and maintenance gaps from the independent review of PR #1290. - -**Architecture:** Preserve the explicit HTML terminal phase and conservative cache policy. Carry authoritative response metadata to terminal callbacks, bypass all body transformations for opted-in range-associated HTML, and preserve exactly-once auction abandonment. Use the existing platform-compatible clock for auction telemetry and exercise the real Axum harness in CI. - -**Tech Stack:** Existing Rust/local futures, HTTP status/header types, `web_time`, Python standard library, and GitHub Actions. - -## Reviewed design refinements - -The user authorized fixes after the read-only review. These refinements supplement -the [existing design](../archive/2026-10-12-1241-buffered-terminal-design.md). - -- Terminal callbacks receive the actual response status. Full 404/500 HTML remains - eligible; a signing consumer may return `Unchanged` for unwanted statuses. -- When terminal integrations are enabled, HTML with status 206 or any - `Content-Range` bypasses rewriting and terminal I/O. The latter includes - range-associated error responses such as 416. Preserve encoded bytes, - `Content-Range`, framing, and ordinary origin policy. Do not strip ranges from - all GETs: non-document fetch/subresource semantics remain intact. -- Both response wrappers check actual metadata before removing framing or applying - terminal privacy. Direct body helpers use private carried metadata and also - preserve partial bytes. Dispatched auctions are abandoned once on bypass. -- Construct terminal input before moving auction ownership. This closes a latent - error-path hazard without manufacturing an impossible body variant for a test. -- Reduce redundant replay flushes while keeping the pre-auction flush barrier. -- The live Workers failure names `AuctionObservationContext::from_parts` calling - unsupported `std::time::Instant`. Change only that telemetry clock to - `web_time::Instant`, already used by the orchestrator, and verify the actual - Workers auction control. Broader clock work under #1240 remains separate. -- Keep cache bypass explicit. A user-independent flag alone does not define - external freshness, signing-key identity, invalidation, failure caching, or - callback behavior on hits. Document the product cost and future cache contract. -- Preserve the marker-placement integration contract. Neighbor checks do not - establish parsing context. Document required real-consumer placement tests and - avoid a second parser or misleading structural guarantees. -- Document the required registry field's source-compatibility migration. Keep - registry metadata mandatory rather than restoring an unsafe optional fallback. - -## Tasks and evidence - -### 1. Response metadata and partial-response regressions - -Files: `publisher.rs`, `publisher/html_finalization.rs`, -`integrations/html_finalization.rs`, and existing host parity fixtures. - -- [x] Add tests for 206 and 200/416 with `Content-Range`, materialized/streamed - bodies, lazy/buffered/direct helpers, compressed byte identity, preserved - range/framing/validators, no callback/timer, and exactly-once abandonment. -- [x] Run the focused native tests before implementation and record the failure. -- [x] Add private status/range metadata and public borrowed status; wire every - constructor and authoritative response wrapper. Bypass before mutation. -- [x] Verify callbacks observe 200/404/500; verify non-document range forwarding, - unsolicited partial-response bypass, and feature-off behavior unchanged. -- [x] Run focused native and WASI tests and direct core lint. - -### 2. Ownership and inexpensive cleanup - -Files: `publisher/html_finalization.rs`, `publisher.rs`, -`integrations/html_finalization.rs`, `registry.rs`, `platform/mod.rs`, -`settings.rs`, and `scripts/fixtures/async-html-finalization-runtime.py`. - -- [x] Initialize terminal input before moving auction ownership. -- [x] Simplify the unreachable writer branch while retaining poison behavior. -- [x] Put touched tests below imports and add a descriptive timeout validation - message. Preserve existing output-limit and range tests. -- [x] Add a meaningful flush-count regression before reducing replay flushes; - retain prefix delivery before auction collection, including missing body end. -- [x] Explain intentionally ignored cleanup exceptions and reap killed children. - Verify cleanup against real short-lived/stubborn local process groups. - -### 3. Actual-runtime evidence and CI - -Files: `auction/telemetry.rs`, `.github/workflows/test.yml`, runtime harness. - -- [x] Preserve the existing actual Workers panic as red evidence, change the - telemetry clock only, and rerun the fresh release Workers auction fixture. -- [x] Add actual Axum disabled/success/hung/auction harness to existing native CI, - Python 3.12, bounded timeout, and failure artifact retention. -- [x] Run Axum, Workers, Spin, and Fastly runtime controls after source freezes. - Use isolated Viceroy 0.19 for blocking-host timeout proof; retain normal pins. - -### 4. Documentation, verification, and draft PR - -Files: canonical guides, existing spec/plan/review, this execution record. - -- [x] Update status/partial contracts, marker obligations, cache costs, migration, - and actual Workers evidence consistently across documents. -- [x] Run target-matched tests/lints, parity, build-digest checks, JS/docs gates, - ordinary artifact fixture exclusion, and relevant integration/cache controls. -- [x] Obtain independent architecture and final review; resolve findings centrally. -- [ ] Commit and push the verified changes to the existing draft PR. Update its - description with exact evidence and remaining maintainer decisions. - -## Execution record - -The independent architecture review approved the addendum. Staff review found a -remaining handler-ordering gap: active/inactive ads and tag suppression could -change partial-response cache policy before the routing bypass. A new nine-case -handler regression reproduced it, and the range decision now precedes those -body-dependent policy branches. Re-review found no remaining blocking findings. -Independent cookie and diagnostics privacy remains intact. - -### Red/green evidence - -- Partial-response tests failed on the original implementation: a ranged fetch - invoked a finalizer and the wrappers removed framing/validators and changed - cache policy. The new 64-case sink matrix and eight handler cases pass. -- The status contract test initially failed to compile because the input had no - status. Both sinks now expose 200/404/500 accurately without excluding full - error pages. -- A direct 206 helper test with only status marked partial caught decode/re-encode - of the original gzip bytes. The guard now derives partial eligibility from - status as well as carried range metadata. -- Large-document replay initially flushed nine times. It now flushes twice on - the no-auction fixture. A controlled pending-auction test confirms every ready - prefix byte is flushed before collection with and without a structural body end. -- The cleanup test against the exact committed pre-fix harness failed because a - forcibly killed child was not reaped. All three real-process cleanup cases pass - after the bounded final wait. Tests do not replace process handling with mocks. -- The captured Workers stack named the unsupported standard clock in auction - telemetry. A fresh release Workers fixture now passes the combined route after - changing only that clock to `web_time::Instant`. - -The response regressions verify no callback/outside work; independent path review -confirms bypass precedes creation of the terminal coordinator and its timer. No -impossible body variant was introduced to test the latent ownership reorder. - -### Final verification - -| Check | Observed result | -| -------------------------------------- | ----------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | -| Focused native finalization tests | 36 pass, including the 64-case sink matrix, eight ranged-origin cases, nine live/inactive/suppressed policy cases, actual status, direct helpers, and both flush regressions. | -| Pinned Fastly/WASI gate | 219 adapter, 2932 core, 3 JS crate, 21 OpenRTB, and 4 doc tests pass; existing ignored tests remain ignored. | -| Fastly reusable sandbox | 219 adapter tests pass. | -| Target-matched Clippy | All eight aliases pass on final source. Primary core all-target/all-feature and digest lint pass. | -| Native adapter/core tests | Axum 44, Workers 53, Spin 87; core 2939 unit, 10 digest, 4 doc tests pass. Six core doc tests remain ignored. | -| CLI/codegen/parity | 875 CLI tests pass with 18 existing ignored; codegen 7 and parity 18 pass. CLI tests preceded the final range-policy adjustment; CLI source did not change. | -| Exact CI runtime integration selection | 63 pass, plus the separate Fastly EC lifecycle control. Ordinary artifacts were rebuilt and isolated first. | -| Existing template-cache controls | ESI 22, inline 9, purge 34 checks pass on pinned Viceroy 0.17, including real winning bids and cookie variants. | -| JS | Build, 50 Vitest files / 1208 tests, type checks, and formatting pass. | -| Docs/format/syntax | Docs lint/build/format, repository Markdown, Rust formatting, diff whitespace, Bash and Python syntax checks pass. | -| Cleanup | Three real-process tests pass: already exited, graceful TERM, and TERM ignored then KILL/reap. | -| Ordinary artifacts | All four adapters build; the fictional fixture registration ID is absent from every ordinary artifact. | - -### Actual runtimes - -All four actual runtime controls pass: disabled baseline, delayed success, hung -lookup with exact fallback bytes, and enabled-auction fallback assembly. Workers, -Spin, and Fastly use release-profile fixture artifacts; Axum runs the same command -now used by CI. The fixture budget is 750 ms, and timer adapters retain a 30000 ms -transport bound to distinguish timer release from an early HTTP timeout. - -| Adapter | Success request | Hung request | Terminal evidence | Auction control | -| ------- | --------------- | ------------ | -------------------------------- | --------------- | -| Axum | 0.215 s | 0.811 s | 750 ms terminal time | Pass, 0.214 s. | -| Workers | 0.242 s | 0.828 s | 753 ms terminal time | Pass, 0.292 s. | -| Spin | 0.213 s | 0.809 s | 751 ms terminal time | Pass, 0.211 s. | -| Fastly | 0.209 s | 0.806 s | 750 ms bounded host-call release | Pass, 0.210 s. | - -Workers and Axum were rerun after the final range-policy correction. Spin/Fastly -actual fixture runs preceded that last correction, which only changes opted-in -range-associated response handling; final native/WASI tests and fresh ordinary -builds verify the correction. Existing browser controls were not repeated because -no JS behavior changes in this hardening. - -Fastly transport-bound proof uses an isolated Viceroy 0.19; normal tool pins and -CI controls retain 0.17. Passing Workers auction telemetry does not declare all -clock paths under #1240 fixed. Cacheability and API-shape approval remain -maintainer decisions, explicitly documented in the draft PR. Marker cardinality -still does not establish HTML parsing context; real integrations must preserve -structure and test winning-bid placement. The required registry field remains a -source migration for Git/path consumers. - -Verification logs, generated manifests, body/header/timing diagnostics, and -artifact hashes are retained outside version control. No operator configuration, -credentials, or external lookup services were used. diff --git a/docs/superpowers/reviews/2026-10-12-1241-async-html-finalization-implementation.md b/docs/superpowers/reviews/2026-10-12-1241-async-html-finalization-implementation.md deleted file mode 100644 index 2f0a4f6da..000000000 --- a/docs/superpowers/reviews/2026-10-12-1241-async-html-finalization-implementation.md +++ /dev/null @@ -1,188 +0,0 @@ -# Asynchronous HTML finalization implementation review - -**Historical record:** This describes the buffered terminal implementation before the -2026-10-13 ordered EOF revision. Its verification counts do not verify the revised -implementation. See the current spec and plan for the active architecture. - -**Date:** 2026-10-12 - -**Branch:** `feat/1241-async-html-finalization` - -**Planning commit:** `b750c879a` - -**Status:** Implemented and reviewed. Review hardening adds range safety, response -status metadata, a targeted telemetry clock correction, and actual-runtime CI. The -Workers enabled-auction control now passes. See the linked hardening execution -record for the latest verification; the original evidence below is historical. - -**Design:** [Optional asynchronous HTML document finalization](../archive/2026-10-12-1241-buffered-terminal-design.md) - -**Plan:** [Implementation and verification tasks](../archive/2026-10-12-1241-buffered-terminal-plan.md) - -**Review hardening:** [Fixes, retained design decisions, and renewed verification](../plans/2026-10-12-1241-review-hardening.md) - -## Result and scope - -Trusted Server now offers an explicitly registered HTML terminal callback after -synchronous processing reaches EOF. It supports borrowing, non-`Send` futures, -ordered atomic replacements, bounded core-owned output, one chain deadline, and -unchanged-input fallback. The generic streaming traits remain unchanged. - -The callback belongs in Trusted Server because its document state, auction assembly, -cache admission, privacy policy, and runtime service container belong here. No -EdgeZero revision, production platform manifest, JS behavior, KV implementation, -signing consumer changes in this delivery. Review hardening corrects only the -traced unsupported auction telemetry clock; broader clock work remains separate. - -Enabled finalizers deliberately retain the completed transformed document. They -bypass shared-template caching before lookup or reservation, and eligible HTML -responses use `private, no-store`. Disabled registrations keep the existing route. -Raw origin readthrough caching remains independent. - -## Spec and plan coverage - -The original granular checklists describe instructions, rather than claiming each -suggested intermediate commit exists. All thirteen tasks have implementation or -verification evidence below. Initially Task 11's Workers auction control was -blocked; the review hardening record replaces that gap with actual runtime -evidence after the targeted clock correction. - -| Plan tasks | Spec sections | Implementation and observed evidence | -| ---------- | ------------------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ | -| 1 | 6.2, 8, 10 | `integrations/html_finalization.rs` defines the factory, session, borrowed input, outcomes, errors, deadline, and bounded replacement writer. Tests cover exact limits, poisoned candidates, empty replacement, unchanged output, and allocation bounds. | -| 2 | 9.1–9.3 | Settings default to 5000 ms and validate 1–60000 inclusive. Runtime services default to no timer and preserve configured timers when cloned or replacing services. Explicit TOML omission and boundary tests pass. | -| 3 | 6.1, 7.1 | Registry metadata preserves registration order and disabled eligibility. HTML parser, synchronous processors, and finalizer factories share one request-local document state. State-sharing and request-isolation tests pass. | -| 4 | 9.2 | Axum installs Tokio sleep, Workers installs `worker::Delay`, and Spin installs SDK sleep. Fastly has no cooperative timer. Target-matched native/WASM compilation, fixture-enabled WASM lint, and live runtime checks pass. | -| 5 | 7.2, 7.4, 9.3, 10, 13 | The private coordinator executes yielding callbacks in order. Tests verify earlier commits survive later failure, one timer covers the chain, expiry wins ties, late results are rejected, and cancellation drops active borrows before owned-byte fallback. | -| 6 | 8 | Collection and candidate buffers use the configured body cap with checked length arithmetic. Overflow replays prefix, crossing chunk, and suffix once, without starting optional I/O. Exact byte-order and allocation-identity tests pass. | -| 7 | 11 | Registry metadata gates shared-template lookup before origin fetch. Tests verify warm/cold bypass, no reservation/store, preserved raw-origin readthrough intent, a working raw-byte cache double hit with per-request finalization, disabled eligibility, four conditional/range headers stripped on eligible GETs, preserved preconditions, and private headers without an auction. | -| 8 | 7.2–7.4, 12, 13 | One terminal driver covers materialized/streamed input and lazy/buffered output, with and without an auction. Decoder and synchronous EOF precede callbacks; seam handling and encoding follow them. Client cancellation verifies exactly one existing guard warning; fatal read/decode/write/flush errors verify exactly one abandonment telemetry summary. | -| 9 | 7.1–7.3, 10–12, 14.3–14.5 | The 128-case route matrix covers four document shapes, four codecs, auction presence, sink kind, and body kind. Corrupt/truncated codec tests prevent premature I/O. Tests cover unsupported/bodiless/non-HTML cases and eligible non-2xx HTML. A positive winning-bid test preserves the article, places one bid injection before the structural body end, and removes the seam. Native and WASI runs pass. | -| 10 | 6.1, 12, 14 | The fictional consumer is behind `test-utils`, contributes no JS, waits for completed HTML, and uses request HTTP services. Axum/Workers/Spin host parity passes. Fixture IDs are absent from fresh ordinary artifacts for all four adapters. | -| 11 | 9.2–9.3, 14.6 | The local harness builds fresh artifacts and verifies disabled, delayed-success, and hung-lookup controls on all four actual runtimes. Workers/Spin/Fastly release-profile runs pass. Actual auction fallback controls pass on Axum, Spin, and Fastly; Workers reproduces #1240. | -| 12 | 9–12, 15 | Canonical configuration and integration guides document registration, timeout semantics, buffer limits, cache/privacy effects, synchronous helper limitations, required registry metadata, and local runtime commands. | -| 13 | 14–16 | Full target-matched gates, actual-runtime controls, existing integration/browser checks, feature-off artifact inspection, and independent architecture/final reviews are recorded below. | - -Every production addition maps to an existing spec contract. The implementation -does not add an interleaved asynchronous streaming API or cacheable finalizer mode. -The timeout starts after synchronous EOF, and optional failure preserves bytes -entering the failing stage. Existing transport, parser, codec, and sink failures -retain fatal handling. Marker-count checks protect opaque seams; they do not claim -to validate arbitrary HTML structure or integration-specific semantics. - -## Necessary implementation refinements - -- `AuctionDispatch` requires the request integration registry. Cache admission - needs finalizer metadata before lookup, and an optional registry could incorrectly - admit a cached template. Callers must pass the same registry to dispatch and - finalization; all production adapters do so. -- Materialized deflate input uses a validated zlib reader. The prior convenience - decoder could accept a truncated trailer as EOF; the opted-in route must verify - completion before starting external I/O. Tests reject truncated trailers while - preserving the existing materialized-body limit semantics. -- The opt-in driver and terminal coordinator futures are boxed. The existing - authorized cold-template buffered fallback is also boxed to limit nested WASI - future stack size. An existing winning-bid cold-miss test trapped on the new - branch, passed on the clean planning baseline, and passed after this narrow fix. - It costs one allocation on already-buffered authorized cache misses; inline - delivery and warm-template delivery do not take that fallback. -- The runtime harness uses a Python process-group driver with generated temporary - manifests and a restricted environment. It handles chunked Spin HTTP requests, - validates transport bounds, and retains body/header/event/timing diagnostics. - Fixture-only Workers/Spin loggers provide deadline diagnostics without altering - ordinary production logging. The Workers fixture also installs a diagnostic - panic hook under the same test feature. -- Verification used one reviewed implementation change instead of manufacturing - the plan's suggested per-task commits. No implementation API was published - between contract creation and complete routing. - -## Initial repository verification - -All commands ran against the implementation worktree, with Rust 1.95.0 and Node -24.12.0. Target-matched checks used the workspace aliases; shared Cargo cache use -does not substitute an old artifact because fresh builds and artifact hashes were -recorded. The normal Fastly gate used the pinned Viceroy 0.17. - -| Check | Observed result | -| ------------------------------- | --------------------------------------------------------------------------------------------------------------------------------------------------------- | -| Eight Clippy aliases | Pass: Fastly, Axum, Workers native/WASM, Spin native/WASM, CLI, and codegen. | -| Core primary-package Clippy | All targets/all features and the native build-digest lint pass. | -| Fastly tests | 219 adapter tests, 2924 core tests, 3 JS crate tests, 21 OpenRTB tests, and 4 core doc tests pass; existing ignored tests remain ignored. | -| Fastly reusable sandbox | 219 adapter tests pass. | -| Native adapter tests | Axum 44, Workers 53, Spin 87 pass. | -| Core all-feature tests | 2931 unit tests, 10 digest tests, and 4 doc tests pass; 6 doc tests remain ignored. | -| Host parity | 18 tests pass. | -| Exact CI Rust runtime selection | 63 tests pass across configuration generation, documentation fixture, integration tests, and parity. | -| CLI and codegen | 875 CLI tests and 7 codegen tests pass; existing ignored CLI tests remain ignored. | -| JS build, Vitest, type checks | 50 test files / 1208 tests pass. | -| Browser controls | Next.js: 21 pass / 1 expected skip. WordPress: 11 pass / 11 expected skips. Initial-render ownership: 10 scenarios pass. Escaping fixtures: 2 tests pass. | -| Documentation | Lint, build, formatting, and Markdown checks pass. Core rustdoc builds with existing warnings, including private intra-doc links. | -| Ordinary artifacts | Fresh Fastly, Workers, and Spin release builds and an Axum build pass; none contains the fictional finalizer registration ID. | - -The browser controls ran before the final narrow WASI future-layout correction; -that correction changed no JS behavior. Full Rust gates and the exact CI runtime -selection were rerun after the correction. A subsequent test-only raw-origin cache -hit regression passed natively and on WASI with primary-package Clippy; it changes -no production behavior or artifact. No source edits were made by reviewers -or the build validator. - -The existing template-cache controls pass on pinned Viceroy 0.17: ESI has 22 -top-level assertions plus 17 cookie/session cases; inline has 9 top-level -assertions plus 17 cookie/session cases. Both exercise real winning bids; ESI -also verifies the GPT scheduler contract. The ESI warm-hit socket probe delivers its first -body bytes in 16 ms while the delayed auction completes near 3020 ms. These -controls verify existing behavior with the feature disabled. - -The enabled-finalizer raw-origin cache-hit regression uses a working core test -double, rather than claiming a live platform cache hit. Local verification logs -and generated artifacts remain outside version control; they contain exact -command output and runtime diagnostics. - -## Initial actual-runtime verification and limits - -The fixture uses a 750 ms chain budget, delays document EOF by 50 ms, and delays a -successful lookup by 150 ms. On timer adapters the hung transport timeout is -30000 ms, so returning near 750 ms proves the terminal timer released the page -rather than relying on the transport timeout. Each hung case returns the exact -249-byte disabled-control body. Successful cases return the exact expected -replacement, with one completed-document lookup. - -| Adapter | Profile/runtime | Success request | Hung request | Terminal evidence | -| ------- | ------------------------------------ | --------------- | ------------ | ---------------------------------------------------------------- | -| Axum | Native dev binary | 0.225 s | 0.825 s | 753 ms terminal time; timer present. | -| Workers | Release WASM, `wrangler dev --local` | 0.244 s | 0.826 s | 752 ms terminal time; timer present. | -| Spin | Release WASM, `spin up` | 0.226 s | 0.819 s | 753 ms terminal time; timer present. | -| Fastly | Release WASM, isolated Viceroy 0.19 | 0.213 s | 0.804 s | 749 ms from lookup to response; timer absent, bounded host call. | - -Development-profile baseline/success/hung runs also passed on all four runtimes. -The optional actual-auction fallback control passed on Axum, Spin, and Fastly. -Positive winning-bid assembly is additionally covered by the native/WASI core -regression and the existing template-cache controls. - -The initial runtime qualifications were: - -1. Pinned Viceroy 0.17 does not simulate backend timeouts, so it cannot prove a - Fastly hung-call bound. An isolated official 0.19 executable was used for that - live check; repository pins remain unchanged. Backend timeout support is - documented in the [official 0.19 release](https://github.com/fastly/Viceroy/releases/tag/v0.19.0). -2. The initial Workers actual-auction control returned HTTP 500 with - `time not implemented on this platform` from `std::time` and then a WASM - trap. The named WASM stack traced it to auction telemetry using the standard - clock. Review hardening changes that clock to the already-used platform - abstraction and verifies a fresh release Workers combined control. The - broader #1240 issue is not declared resolved by that one path. - -## Independent review - -Architecture and staff-level final reviews accepted the implementation with no -remaining blocking findings. Review requested a real lazy-body client-cancellation -test and clearer documentation of the synchronous helper's terminal-phase limits; -both were added and re-reviewed. Architecture review requested a positive winning -bid with an enabled finalizer; that regression now passes natively and on WASI. - -The spec, plan, code, and tests agree on ownership, ordering, deadline scope, -cache/privacy policy, optional fallback, and excluded response types. The initial -Workers runtime limitation was documented rather than counted as a passing test; -its subsequently verified correction is recorded in the hardening addendum. -The implementation branch and isolated worktree are preserved for maintainer -review; no merge, push, deployment, or public issue comment is part of this work. diff --git a/docs/superpowers/reviews/2026-10-13-1241-ordered-eof-verification.md b/docs/superpowers/reviews/2026-10-13-1241-ordered-eof-verification.md deleted file mode 100644 index 94cf377bc..000000000 --- a/docs/superpowers/reviews/2026-10-13-1241-ordered-eof-verification.md +++ /dev/null @@ -1,133 +0,0 @@ -# Ordered HTML EOF completion: implementation and verification - -**Date:** 2026-10-13 - -**Branch:** `feat/1241-async-html-finalization`; draft PR #1290; issue #1241. - -**Published implementation:** `a533df267c149bb8295b23e5d5a1fb700c1b32eb`; PR remains draft. - -**Review baseline:** `3981d0222bf0767ba3c8de767d633018b0df4b58`. -The newer remote merge from `main` was preserved before final verification. - -This report covers the revised [specification](../specs/2026-10-12-1241-async-html-finalization-design.md) -and [plan](../plans/2026-10-12-1241-async-html-finalization.md). -The October 12 reports describe the superseded buffered terminal API. - -## Resulting design - -Existing processors keep synchronous `process_chunk` calls. An optional borrowing -`async_finish` callback finishes a stage before downstream EOF. Accepted async output -passes through downstream ordinary processing. Factories declare capability and optional -output retention in the existing registration list; there is no separate production -finalizer lifecycle. `publisher.allow_async_html_finish` defaults to false and enabled -capability without permission is a configuration error. - -Only declaring stages withhold output. Core owns the fallback and bounded candidate -outside each callback. Timeout, optional errors, poisoned candidates, late completion -and invalid assembly-marker counts retain the fallback allocation. One absolute budget -is shared; expiry skips later optional work while still finishing synchronous EOF. -Auction collection and encoding finish after the chain. Auction suffix storage is capped -before growth. Range/bodiless/unsupported response handling, actual status, template-cache -bypass, raw-origin readthrough and independent privacy policies remain covered. - -The retention cap bounds withheld output and candidates. After overflow, the collector -becomes ordinary ready output; the existing `Vec` chunk interface can coalesce it with -the crossing chunk above that optional cap. Outer buffered-response limits still apply. - -## Coverage and review - -| Requirement | Implementation and evidence | -| --------------------------------------------------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | -| Optional local future; unchanged synchronous entry points | `streaming_processor.rs`, `streaming_processor/finish.rs`: default absence, borrowing non-Send work and pre-effect synchronous rejection | -| One ordered registration list and explicit permission | `integrations/registry.rs`, `settings.rs`: default false, missing permission, permission alone and inconsistent retention metadata | -| Stage-by-stage EOF and ready output | `html_processor/finish.rs`, publisher driver: downstream transforms async output before its own EOF; lazy materialized/streamed input releases prefix before pending work | -| Cancellation and replacement acceptance | Leaf tests cover timeout borrowing/allocation identity, errors after partial writes, ignored poisoned writes, timer ties, late results, empty replacement and earlier accepted output surviving later failure | -| Retention and assembly boundaries | Exact-cap/crossing replay, creation/removal across streamed boundaries, retained markers, overflow bypass and accepted upstream replacement feeding downstream boundary validation | -| Auction, codec, cache and eligibility | Existing migrated publisher matrix, winning-bid/raw-origin controls, partial/status/flush regressions and fatal suffix overflow with exactly one abandonment summary | -| Actual runtime behavior | Fresh local Axum, Workers, Spin and Fastly disabled/success/hung/auction controls; required non-retaining Fastly early-byte control | - -Independent architecture and implementation reviews found and resolved constructor-time -deadline expiry, session validation before lazy reads, parser EOF failure classification, -remaining-budget timer initialization, timer-expiry propagation, reset forwarding and -missing acceptance tests. The final bidirectional document/source review approved the -expanded coverage. Review is separate from executed verification. - -## Executed evidence - -Observed red/green failures included missing callback/metadata/chain contracts, reset -state leakage, timer initialization and expiry propagation, constructor budget exhaustion, -and an undeclared async session consuming a body read before rejection. The final focused -`cargo test -p trusted-server-core --all-features async_eof` run passed 30 tests. - -The newer main merge exposed a separate test-only compatibility failure: Cargo exports -`TRUSTED_SERVER__GIT_VERSION` to test processes, and the legacy configuration loader treated -it as application configuration. An explicit regression failed before filtering only that -metadata from the test-only environment source. Production configuration loading and -version-header behavior are unchanged. The final native all-feature run passed 2,986 unit -tests, 10 git-version filesystem tests, 10 template build-digest tests and 4 documentation -tests (5 existing ignored documentation examples). - -Fresh JS build, formatting and type checks passed; Vitest passed 1,211 tests in 50 files. -Docs lint, formatting and build passed. Python fixture syntax, Bash syntax and all three -real-process cleanup tests passed. - -### Runtime observations - -All runs build fresh fixture artifacts, create isolated configuration, use loopback -services and reap their child processes. Disabled controls make no lookup; success runs -check exact output/privacy; hung runs retain exact fallback bytes; auction runs preserve -assembly. The fixture budget is 750 ms, distinct from the production default of 5,000 ms. - -| Runtime | Success request | Hung request | Observed timeout | Auction request | -| ------- | --------------- | ------------ | ---------------- | --------------- | -| Axum | 0.260 s | 0.819 s | 754 ms | 0.232 s | -| Workers | 0.230 s | 0.822 s | 750 ms | 0.251 s | -| Spin | 0.224 s | 0.805 s | 751 ms | 0.216 s | -| Fastly | 0.224 s | 0.815 s | 751 ms | 0.213 s | - -Fastly's additional non-retaining control received its first body byte at **0.0053 s**, -before the 150 ms lookup completed; total request time was 0.217 s. The harness asserts -actual body-byte timing against the lookup-completion event, not response headers. -This proves the supported Fastly streaming path can release ready bytes before EOF I/O. -It does not promise arbitrary processors or buffered adapters have zero TTFB cost. - -Commands: `scripts/async-html-finalization-local-test.sh axum --auction`, and the -`cloudflare`, `spin`, `fastly` variants with `--release --auction`. -Workers uses isolated `worker-build` 0.8.5 and `wasm-bindgen` 0.2.126 matching the lockfile. -Fastly timeout/streaming controls use isolated Viceroy 0.19 for backend timeout simulation; -ordinary WASI tests and template controls use the repository-pinned Viceroy 0.17. -Spin 4.0.0 passed the actual local controls; EdgeZero's CLI still warns that its verified -SQLite writer range is Spin 2–3. Repository runtime/tool pins were not changed. - -## Evidence boundary - -The final revised source snapshot (changed Rust/fixture/example files, sorted paths and -contents separated by NUL bytes) has SHA-256 -`766c30f633421b259436a5d220ac8e52bf73308cedc81462cd4f8ece9269cdf8`. -Earlier pre-merge runs are not substituted for final evidence. - -| Final local check | Result | -| ------------------------------------------ | -------------------------------------------------------------------------------------------------------------------------------- | -| All eight target-matched Clippy aliases | Passed; Fastly repeated after the test-only metadata fix | -| Core all-target/all-feature Clippy | Passed | -| `cargo test-fastly` on pinned Viceroy 0.17 | Fastly 222, core 2978 (6 existing ignored), JS 10, git-version 10, OpenRTB 21, docs 4 (5 existing ignored) passed | -| `cargo test-fastly-reuse` | 222 passed | -| Native adapter test aliases | Axum 49, Workers 54, Spin 88 passed | -| Native core/digest gates | 2986 core, 10 git-version, 10 digest, 4 docs passed; digest lint passed | -| CLI helper and codegen | 892 executed CLI tests including selected ignored browser controls; codegen 7 passed | -| Integration parity and lint | 18 parity tests and all-target integration lint passed | -| Exact CI integration selection | Generator 8, docs fixture 1, integration 36, parity 18 passed; separate focused Fastly EC also passed | -| Existing template-cache controls | ESI 22, inline 9 and purge 34 passed | -| JS, docs and formatting | Fresh build, 1211 tests/type checks, JS format, docs lint/build/format, outside-doc Markdown, Rust format and diff checks passed | -| Runtime/cleanup | All four actual runtimes plus Fastly early-byte proof passed; three cleanup tests and syntax checks passed | - -Native checks used fresh JS bundles through the supported `TSJS_PREBUILT_DIR` setting. -The merged main change intentionally rejects the old `TSJS_SKIP_BUILD` setting. Real -runtime harnesses built their own private bundles. Template controls ran an external -copy of the unchanged harness with only worktree/shared-target artifact paths adjusted; -the first ESI attempt failed to locate its GPT artifact, and the final attempt corrected -that harness environment path. No assertions were relaxed or source tooling changed. -Hosted CI is separate from local verification. Maintainers still need to accept the API, -conservative processed-template cache policy and consumer-owned marker placement before -the PR leaves draft. No background signing service, external queue or publisher workflow -is required or introduced. diff --git a/docs/superpowers/specs/2026-10-12-1241-async-html-finalization-design.md b/docs/superpowers/specs/2026-10-12-1241-async-html-finalization-design.md index b607ce13c..f54e9403e 100644 --- a/docs/superpowers/specs/2026-10-12-1241-async-html-finalization-design.md +++ b/docs/superpowers/specs/2026-10-12-1241-async-html-finalization-design.md @@ -1,11 +1,8 @@ # Ordered asynchronous HTML processor completion -**Issue:** #1241. **Revised:** 2026-10-13. **Status:** revised implementation and local verification complete; maintainer review pending. -Verification is recorded in the [fresh execution report](../reviews/2026-10-13-1241-ordered-eof-verification.md). +**Issue:** #1241. **Status:** draft implementation; review hardening and local verification complete. -This supersedes the [buffered terminal design](../archive/2026-10-12-1241-buffered-terminal-design.md). -The previous implementation's evidence belongs to that archived design. The current -[implementation plan](../plans/2026-10-12-1241-async-html-finalization.md) covers this revision. +The [implementation plan](../plans/2026-10-12-1241-async-html-finalization.md) maps this contract to implementation and verification. Fresh executed evidence is recorded in the draft PR. ## 1. Requirement and scope @@ -101,6 +98,12 @@ output. Use `publisher.max_buffered_body_bytes` (existing 16 MiB default), check arithmetic and incremental allocation. At EOF, move the retained output into the fallback and lend a slice to the callback. Candidate storage has the same cap. +These are per-holder limits, not an aggregate request-memory budget. Retained stage output, +fallback/candidate, auction suffix, codec state and buffered adapter responses can coexist. +Total memory may exceed this setting; integrations also own allocations core cannot bound. +Candidates run one at a time and replay moves/slices selected output. Do not infer a fixed +peak-memory multiplier or a total heap bound from the body-byte limit. + Overflow permanently disables that stage's optional EOF I/O. Replay retained prefix, crossing chunk and later output exactly once through downstream stages, then finish its synchronous EOF normally. It does not authorize an oversized outer buffered @@ -119,6 +122,9 @@ the ordinary output policy and skip that stage's optional I/O. Do not truncate i ## 6. Deadlines, failures and cancellation Retain `publisher.html_finalization_timeout_ms`: default 5000, valid 1–60000. +The initial default follows the 5-second budget in the issue example; 60 seconds is +a configuration ceiling. Neither value is a measured optimum or a response latency +guarantee. Operators explicitly accept and tune this budget when enabling EOF work. One absolute budget begins after decoder/parser EOF and covers every optional finish. Share one timer across stages, initializing its delay with the remaining budget immediately before the first callback is polled. The deadline wins readiness ties. Check before polling @@ -166,6 +172,9 @@ promise insertion into an earlier head or before an already emitted body end. Use active async factory metadata before the first processed-template lookup to skip shared-template reads, reservations and stores. Raw-origin readthrough is independent. +This intentionally defers the issue's shared-template cold-miss storage behavior until +maintainers accept an output identity, freshness and hit-execution policy. Keep the +PR in draft and describe it as partial coverage of #1241 until that scope is accepted. Processed opted-in HTML uses `Cache-Control: no-store, private` even when work fails, returns unchanged, or is bypassed after overflow. Cacheable async output needs a separate identity, freshness, invalidation and hit-execution contract. diff --git a/scripts/README.md b/scripts/README.md index ad4ecf926..9b224202e 100644 --- a/scripts/README.md +++ b/scripts/README.md @@ -3,21 +3,50 @@ Run these scripts from the repository root. They fail on command errors unless their documented orchestration handles a probe deliberately. -| Script | Inputs and prerequisites | Side effects and cleanup | -| ----------------------------------------- | ------------------------------------------------------------------ | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | -| `batch-sync.sh` | Endpoint, bearer token, EC ID, partner UID; `curl` and Python | Sends one authenticated batch-sync request. Its temporary response file is removed on exit. | -| `benchmark.sh` | A running server; `curl`, `bc`, and optionally `hey` | May install `hey` through Homebrew. `--save` writes under `benchmark-results/`; it does not stop the server. | -| `profile.sh` | Fastly CLI, Rust WASM target, `curl`; endpoint and request options | Builds and starts the Fastly app, stops the owned process, and retains a profile under `benchmark-results/profiles/`. `--open` launches the local viewer. | -| `generate-integration-viceroy-configs.sh` | Rust toolchain; optional origin port and artifact directory | Builds the native generator and writes Viceroy config under `target/integration-test-artifacts/`; generated files persist. | -| `integration-tests.sh` | Docker, Viceroy, Rust WASM target, pinned Node | Builds WASM/native artifacts and two Docker images, generates Viceroy config, and runs native integration tests serially. Build products and images persist. | -| `integration-tests-browser.sh` | The integration prerequisites plus npm and Playwright | Installs package/browser dependencies, builds fixtures and images, runs both browser suites, and stops matching test containers on exit. Build and npm artifacts persist. | -| `smoke-axum.sh` | `cargo`, `curl`, Python | Uses an isolated temporary config/store and stub origin; stops owned processes and removes its workspace. | -| `smoke-fastly.sh` | Fastly CLI, `cargo`, `curl`, Python | Uses an isolated Fastly project and application config; stops owned processes and removes its workspace. | -| `smoke-cloudflare.sh` | Pinned Wrangler, `cargo`, `curl`, `jq`, and Python | Uses isolated Wrangler manifests, KV state, ports, and logs; stops owned processes and removes its workspace. | -| `smoke-spin.sh` | Spin CLI, `cargo`, `curl`, and Python | Uses an isolated Spin manifest and SQLite KV store; stops owned processes and removes its workspace. | -| `smoke-common.sh` | Sourced by the four smoke scripts | Defines bounded port, process, config, secret, and response assertions. Do not invoke it as a standalone smoke. | -| `template-cache-local-test.sh` | Viceroy, Node, OpenSSL, `curl`, `lsof`; optional ports and mode | Builds local artifacts, generates certificates and fixtures in a temporary directory, stops owned servers, and removes the directory. | -| `test-cli.sh` | Rustup and the host toolchain; optional host triple | May install the selected Rust target, then runs native CLI and browser-audit tests. Cargo artifacts persist. | +| Script | Inputs and prerequisites | Side effects and cleanup | +| ----------------------------------------- | ------------------------------------------------------------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| `batch-sync.sh` | Endpoint, bearer token, EC ID, partner UID; `curl` and Python | Sends one authenticated batch-sync request. Its temporary response file is removed on exit. | +| `benchmark.sh` | A running server; `curl`, `bc`, and optionally `hey` | May install `hey` through Homebrew. `--save` writes under `benchmark-results/`; it does not stop the server. | +| `profile.sh` | Fastly CLI, Rust WASM target, `curl`; endpoint and request options | Builds and starts the Fastly app, stops the owned process, and retains a profile under `benchmark-results/profiles/`. `--open` launches the local viewer. | +| `generate-integration-viceroy-configs.sh` | Rust toolchain; optional origin port and artifact directory | Builds the native generator and writes Viceroy config under `target/integration-test-artifacts/`; generated files persist. | +| `integration-tests.sh` | Docker, Viceroy, Rust WASM target, pinned Node | Builds WASM/native artifacts and two Docker images, generates Viceroy config, and runs native integration tests serially. Build products and images persist. | +| `integration-tests-browser.sh` | The integration prerequisites plus npm and Playwright | Installs package/browser dependencies, builds fixtures and images, runs both browser suites, and stops matching test containers on exit. Build and npm artifacts persist. | +| `smoke-axum.sh` | `cargo`, `curl`, Python | Uses an isolated temporary config/store and stub origin; stops owned processes and removes its workspace. | +| `smoke-fastly.sh` | Fastly CLI, `cargo`, `curl`, Python | Uses an isolated Fastly project and application config; stops owned processes and removes its workspace. | +| `smoke-cloudflare.sh` | Pinned Wrangler, `cargo`, `curl`, `jq`, and Python | Uses isolated Wrangler manifests, KV state, ports, and logs; stops owned processes and removes its workspace. | +| `smoke-spin.sh` | Spin CLI, `cargo`, `curl`, and Python | Uses an isolated Spin manifest and SQLite KV store; stops owned processes and removes its workspace. | +| `smoke-common.sh` | Sourced by the four smoke scripts | Defines bounded port, process, config, secret, and response assertions. Do not invoke it as a standalone smoke. | +| `async-html-finalization-local-test.sh` | Python 3.11+, Rust, Node and selected runtime; Fastly Viceroy 0.19+ | Builds feature-gated fixtures, preserves compiler settings, retains diagnostics, and stops owned process groups on failure or SIGTERM. | +| `template-cache-local-test.sh` | Viceroy, Node, OpenSSL, `curl`, `lsof`; optional ports and mode | Builds local artifacts, generates certificates and fixtures in a temporary directory, stops owned servers, and removes the directory. | +| `test-cli.sh` | Rustup and the host toolchain; optional host triple | May install the selected Rust target, then runs native CLI and browser-audit tests. Cargo artifacts persist. | + +## HTML EOF runtime checks + +Requires Python 3.11+, Rust, pinned Node and the selected runtime CLI. Run from the +repository root: + +```bash +python3 -m unittest discover -s scripts/fixtures -p 'test_async_html_finalization_runtime.py' -v +./scripts/async-html-finalization-local-test.sh axum --auction +./scripts/async-html-finalization-local-test.sh cloudflare --release --auction +./scripts/async-html-finalization-local-test.sh spin --release --auction +``` + +**Fastly requires isolated Viceroy 0.19 or newer** for the hung-backend fixture; +ordinary tests retain the repository's 0.17 pin: + +```bash +cargo install viceroy --version 0.19.0 --locked --root /tmp/example-eof-viceroy +PATH="/tmp/example-eof-viceroy/bin:$PATH" ./scripts/async-html-finalization-local-test.sh fastly --release --auction +``` + +Workers requires matching `worker-build`/`wasm-bindgen` versions and Wrangler. +The harness builds fresh feature-gated artifacts, checks generated timeout, +permission and loopback configuration, preserves explicit compiler/profile settings +and `CARGO_TARGET_DIR`, selects build targets itself, and stops owned process groups on failure or SIGTERM. +Artifacts and diagnostics persist in the printed temporary directory for inspection. +CI runs cleanup tests, Axum and isolated Fastly EOF controls. See the +[testing guide](../docs/guide/testing.md#html-eof-runtime-checks) for coverage details. The four adapter smoke contracts are documented in the [deployment guides](../docs/guide/integrations-overview.md#adapter-support). diff --git a/scripts/fixtures/async-html-finalization-runtime.py b/scripts/fixtures/async-html-finalization-runtime.py index 1493dc697..614973fef 100644 --- a/scripts/fixtures/async-html-finalization-runtime.py +++ b/scripts/fixtures/async-html-finalization-runtime.py @@ -48,11 +48,140 @@ def stop(process): process.wait(timeout=5) +def build_environment(source): + allowed = { + "PATH", "RUSTUP_HOME", "CARGO_HOME", "CARGO_TARGET_DIR", "TMPDIR", + "WASM_BINDGEN_BIN", "WASM_OPT_BIN", "ESBUILD_BIN", "CARGO_NET_OFFLINE", + "RUSTFLAGS", "CARGO_ENCODED_RUSTFLAGS", "RUSTDOCFLAGS", "CARGO_INCREMENTAL", + "RUSTC", "RUSTDOC", "RUSTC_WRAPPER", "RUSTC_WORKSPACE_WRAPPER", + "TSJS_PREBUILT_DIR", + } + return {key: value for key, value in source.items() + if key != "CARGO_BUILD_TARGET" + and (key in allowed or key.startswith(("CARGO_PROFILE_", "CARGO_BUILD_", "CARGO_TARGET_")))} + + +def checked_replace(text, old, new, description): + matches = text.count(old) + if matches != 1: + raise RuntimeError(f"expected one {description} template field, found {matches}") + return text.replace(old, new, 1) + + +def toml_string(value): + return json.dumps(str(value), ensure_ascii=False) + + +def fixture_manifest(text, workspace): + manifest = tomllib.loads(text) + entry = manifest["app"]["entry"] + text = checked_replace(text, f"entry = {toml_string(entry)}", + f"entry = {toml_string(ROOT / entry)}", "app entry") + for adapter, settings in manifest["adapters"].items(): + settings = settings["adapter"] + crate = settings["crate"] + text = checked_replace(text, f"crate = {toml_string(crate)}", + f"crate = {toml_string(ROOT / crate)}", f"{adapter} crate") + old = settings["manifest"] + path = workspace / f"{adapter}.toml" if adapter in ("fastly", "spin") else ROOT / old + text = checked_replace(text, f"manifest = {toml_string(old)}", + f"manifest = {toml_string(path)}", f"{adapter} manifest") + return text + + +def fixture_config(template, case, origin_port, bid_port=None): + permission = str(case != "disabled").lower() + publisher = tomllib.loads(template)["publisher"] + text = checked_replace(template, "[publisher]\n", + "[publisher]\nhtml_finalization_timeout_ms = 750\n" + f"allow_async_html_finish = {permission}\n", "publisher table") + table = text.index("[publisher]\n") + end = text.find("\n[", table + len("[publisher]\n")) + if end == -1: + end = len(text) + block = text[table:end] + for key, value in { + "domain": "127.0.0.1", + "cookie_domain": "127.0.0.1", + "origin_url": f"http://127.0.0.1:{origin_port}", + }.items(): + block = checked_replace(block, f"\n{key} = {toml_string(publisher[key])}", + f"\n{key} = {toml_string(value)}", f"publisher {key}") + text = text[:table] + block + text[end:] + if case == "auction": + table = text.index("[auction]\n") + end = text.find("\n[", table + len("[auction]\n")) + if end == -1: + end = len(text) + block = checked_replace(text[table:end], "enabled = false", "enabled = true", "auction enablement") + text = text[:table] + block + text[end:] + text = checked_replace(text, 'endpoint = "https://prebid.example.com/openrtb2/auction"', + f'endpoint = "https://127.0.0.1:{bid_port}/bid"', "auction endpoint") + text += '\n[[creative_opportunities.slot]]\nid = "example-slot"\npage_patterns = ["/article"]\nformats = [{ width = 300, height = 250 }]\n' + lookup_case = "success" if case in ("auction", "streaming") else case + text += (f'\n[integrations.html_finalization_fixture]\nenabled = {permission}\n' + f'lookup_url = "http://127.0.0.1:{origin_port}/{lookup_case}"\n' + f'transport_timeout_ms = 30000\nstreaming = {str(case == "streaming").lower()}\n') + validate_fixture_config(text, case, origin_port, bid_port) + return text + + +def validate_fixture_config(text, case, origin_port, bid_port=None): + config = tomllib.loads(text) + publisher = config["publisher"] + fixture = config["integrations"]["html_finalization_fixture"] + permission = case != "disabled" + lookup_case = "success" if case in ("auction", "streaming") else case + expected = [ + (publisher.get("domain"), "127.0.0.1", "loopback publisher domain"), + (publisher.get("cookie_domain"), "127.0.0.1", "loopback cookie domain"), + (publisher.get("origin_url"), f"http://127.0.0.1:{origin_port}", "loopback origin"), + (publisher.get("html_finalization_timeout_ms"), 750, "EOF timeout"), + (publisher.get("allow_async_html_finish"), permission, "EOF permission"), + (fixture.get("enabled"), permission, "fixture enablement"), + (fixture.get("lookup_url"), f"http://127.0.0.1:{origin_port}/{lookup_case}", "loopback lookup"), + (fixture.get("transport_timeout_ms"), 30000, "fixture transport timeout"), + (fixture.get("streaming"), case == "streaming", "streaming control"), + (config["auction"].get("enabled"), case == "auction", "auction enablement"), + ] + for actual, required, description in expected: + if actual != required: + raise RuntimeError(f"invalid {description}: expected {required!r}, found {actual!r}") + if case == "auction": + providers = config["auction"].get("providers", {}) + if not providers: + raise RuntimeError("auction control requires a loopback provider") + for provider in providers.values(): + if provider.get("endpoint") != f"https://127.0.0.1:{bid_port}/bid": + raise RuntimeError("auction provider must use the controlled loopback endpoint") + + +def preflight_viceroy(env): + output = subprocess.check_output(["viceroy", "--version"], env=env, text=True).strip() + match = re.search(r"\bviceroy (\d+)\.(\d+)\.(\d+)(?:\s|$)", output) + if not match or tuple(map(int, match.groups())) < (0, 19, 0): + raise RuntimeError(f"Viceroy >= 0.19.0 required for the Fastly hung-call control; found {output!r}") + + +def measured_terminal_ms(adapter, diagnostic): + match = re.search(r"times out after (\d+) ms", diagnostic) + if match is None: + if adapter != "fastly": + raise RuntimeError(f"missing measured cooperative timeout diagnostic for {adapter}") + return None + terminal_ms = int(match.group(1)) + assert 700 <= terminal_ms < 2500, "terminal budget must exclude origin transfer" + return terminal_ms + + +def terminate_run(signum, _frame): + raise SystemExit(128 + signum) + + def run(adapter, auction=False, release=False): workspace = Path(tempfile.mkdtemp(prefix=f"trusted-server-finalize-{adapter}-")) print(f"Artifacts: {workspace}", flush=True) - allowed = {"PATH", "RUSTUP_HOME", "CARGO_HOME", "CARGO_TARGET_DIR", "TMPDIR", "WASM_BINDGEN_BIN", "WASM_OPT_BIN", "ESBUILD_BIN", "CARGO_NET_OFFLINE"} - env = {key: value for key, value in os.environ.items() if key in allowed} + env = build_environment(os.environ) target = Path(env.get("CARGO_TARGET_DIR", workspace / "target")).resolve() env["CARGO_TARGET_DIR"] = str(target) env["XDG_CONFIG_HOME"] = str(workspace / "cli-config") @@ -68,6 +197,8 @@ def run(adapter, auction=False, release=False): for executable in required: if shutil.which(executable, path=env["PATH"]) is None: raise RuntimeError(f"required executable missing: {executable}") + if adapter == "fastly": + preflight_viceroy(env) if adapter == "cloudflare": lock = tomllib.loads((ROOT / "Cargo.lock").read_text()) expected = next(package["version"] for package in lock["package"] if package["name"] == "worker") @@ -75,9 +206,14 @@ def run(adapter, auction=False, release=False): if actual != expected: raise RuntimeError(f"worker-build {expected} required for the locked worker SDK; found {actual}") processes = [] + previous_sigterm = signal.signal(signal.SIGTERM, terminate_run) def command(args, cwd=ROOT, overrides=None): - subprocess.run(args, cwd=cwd, env=env | (overrides or {}), check=True) + process = subprocess.Popen(args, cwd=cwd, env=env | (overrides or {}), start_new_session=True) + processes.append(process) + status = process.wait() + if status != 0: + raise subprocess.CalledProcessError(status, args) def launch(args, log, cwd=workspace, runtime_env=None): output = log.open("w") @@ -138,32 +274,24 @@ def field(offset, width, value): "--port", str(origin_port), "--events", str(workspace / "events.jsonl")], workspace / "origin.log") config = workspace / "trusted-server.toml" command([cli, "config", "init", "--app-config", str(config)]) - base_config = config.read_text().replace("[publisher]\n", "[publisher]\nhtml_finalization_timeout_ms = 750\n", 1) - manifest = (ROOT / "edgezero.toml").read_text().replace('= "crates/', f'= "{ROOT}/crates/') - manifest = manifest.replace('manifest = "fastly.toml"', f'manifest = "{workspace}/fastly.toml"') - manifest = manifest.replace(str(crate / "spin.toml"), str(workspace / "spin.toml")) - (workspace / "edgezero.toml").write_text(manifest) + base_config = config.read_text() + (workspace / "edgezero.toml").write_text(fixture_manifest((ROOT / "edgezero.toml").read_text(), workspace)) fastly_base = 'manifest_version = 3\nname = "example-finalization"\nlanguage = "rust"\n[local_server]\n' (workspace / "fastly.toml").write_text(fastly_base) if adapter == "spin": - (workspace / "spin.toml").write_text((crate / "spin.toml").read_text().replace( - '../../target/wasm32-wasip1/release/trusted_server_adapter_spin.wasm', str(artifact))) + spin_config = checked_replace((crate / "spin.toml").read_text(), + 'source = "../../target/wasm32-wasip1/release/trusted_server_adapter_spin.wasm"', + f"source = {toml_string(artifact)}", "Spin artifact") + (workspace / "spin.toml").write_text(spin_config) baseline = None cases = ["disabled", "success", "hung"] + (["auction"] if auction else []) + (["streaming"] if adapter == "fastly" else []) for case in cases: - case_config = base_config.replace("[publisher]\n", f"[publisher]\nallow_async_html_finish = {str(case != 'disabled').lower()}\n", 1) - if case == "auction": - # A closed loopback bid endpoint exercises transport failure and - # empty-bid assembly without sending traffic outside this host. - bid_port = free_port() - table = case_config.index("[auction]\n") - enabled = case_config.index("enabled = false", table) - case_config = case_config[:enabled] + case_config[enabled:].replace("enabled = false", "enabled = true", 1) - case_config = case_config.replace('endpoint = "https://prebid.example.com/openrtb2/auction"', f'endpoint = "https://127.0.0.1:{bid_port}/bid"', 1) - case_config += '\n[[creative_opportunities.slot]]\nid = "example-slot"\npage_patterns = ["/article"]\nformats = [{ width = 300, height = 250 }]\n' - lookup_case = "success" if case in ("auction", "streaming") else case - config.write_text(case_config + f'\n[integrations.html_finalization_fixture]\nenabled = {str(case != "disabled").lower()}\nlookup_url = "http://127.0.0.1:{origin_port}/{lookup_case}"\ntransport_timeout_ms = 30000\nstreaming = {str(case == "streaming").lower()}\n') + # A closed loopback bid endpoint exercises transport failure without + # sending auction requests outside this host. + bid_port = free_port() if case == "auction" else None + case_config = fixture_config(base_config, case, origin_port, bid_port) + config.write_text(case_config) # Axum's local JSON provides the same neutral app-config envelope # used by the other runtime bindings; no operator config is read. command([cli, "config", "push", "--adapter", "axum", "--local", "--manifest", str(workspace / "edgezero.toml"), @@ -177,8 +305,11 @@ def field(offset, width, value): runtime_env["PORT"] = str(app_port) args = [str(artifact)] elif adapter == "cloudflare": - text = (crate / "wrangler.ci.toml").read_text().replace('main = "build/index.js"', f'main = "{artifact}"') - text = text.replace('TRUSTED_SERVER_CONFIG = "{}"', "TRUSTED_SERVER_CONFIG = '''" + json.dumps({"app_config": envelope}) + "'''") + text = checked_replace((crate / "wrangler.ci.toml").read_text(), 'main = "build/index.js"', + f"main = {toml_string(artifact)}", "Workers artifact") + text = checked_replace(text, 'TRUSTED_SERVER_CONFIG = "{}"', + "TRUSTED_SERVER_CONFIG = '''" + json.dumps({"app_config": envelope}) + "'''", + "Workers config binding") text += "\n" + "\n".join(f'{key} = "{value}"' for key, value in SECRETS.items()) + "\n" (workspace / "wrangler.toml").write_text(text) args = ["wrangler", "dev", "--local", "--config", str(workspace / "wrangler.toml"), @@ -225,7 +356,7 @@ def field(offset, width, value): (workspace / f"{case}.body").write_bytes(body) (workspace / f"{case}.headers").write_text(str(headers)) if case == "disabled": - assert b"EXAMPLE_UNSIGNED" in body and b"EXAMPLE_FINALIZED" not in body + assert b"EXAMPLE_UNSIGNED" in body and b"EXAMPLE_FINALIZED" not in body, "disabled fixture must leave the original marker unchanged" baseline = body elif case == "success": assert body == baseline.replace(b"EXAMPLE_UNSIGNED", b"EXAMPLE_FINALIZED"), "successful terminal output must match the exact expected rewrite" @@ -244,7 +375,7 @@ def field(offset, width, value): diagnostic += "\n".join(path.read_text() for path in (workspace / "spin-logs").glob("*stderr*")) assert "HTML EOF processor html_finalization_fixture times out" in diagnostic or (adapter == "fastly" and "retains input: unavailable" in diagnostic), "missing controlled fallback diagnostic" if case != "disabled": - assert headers["Cache-Control"] == "no-store, private" + assert headers["Cache-Control"] == "no-store, private", "active EOF processing must prohibit shared caching" current_events = [json.loads(line) for line in (workspace / "events.jsonl").read_text().splitlines()] lookups = [event for event in current_events if event["event"] == "lookup_start"] lookup_elapsed = completed - lookups[-1]["time"] if case != "disabled" else None @@ -254,26 +385,29 @@ def field(offset, width, value): assert first_byte_at < completions[-1]["time"] - 0.05, "initial HTML bytes must reach the client while EOF I/O remains pending" terminal_ms = None if case == "hung": - measured = re.search(r"times out after (\d+) ms", diagnostic) - if measured: - terminal_ms = int(measured.group(1)) - assert 700 <= terminal_ms < 2500, "terminal budget must exclude origin transfer" + terminal_ms = measured_terminal_ms(adapter, diagnostic) (workspace / f"{case}.timing.json").write_text(json.dumps({"request_seconds": elapsed, "first_body_byte_seconds": first_byte_at - started, "terminal_ms": terminal_ms, "lookup_to_response_seconds": lookup_elapsed, "configured_budget_ms": 750, "cooperative_timer": adapter != "fastly"})) print(f"PASS {adapter} {case}: request={elapsed:.3f}s, {len(body)} bytes, chain_budget=750ms, terminal_ms={terminal_ms}, lookup_to_response={lookup_elapsed}, cooperative_timer={adapter != 'fastly'}", flush=True) stop(process) events = [json.loads(line) for line in (workspace / "events.jsonl").read_text().splitlines()] starts = [event for event in events if event["event"] == "lookup_start"] - assert len(starts) == 2 + int(auction) + int(adapter == "fastly") and all(event["complete"] and len(event["sha256"]) == 64 for event in starts) + assert len(starts) == 2 + int(auction) + int(adapter == "fastly"), "each enabled control must issue exactly one EOF lookup" + assert all(event["complete"] and len(event["sha256"]) == 64 for event in starts), "every lookup must observe complete-document metadata" for event in starts: - assert any(previous["event"] == "origin_eof" and previous["time"] <= event["time"] for previous in events) + assert any(previous["event"] == "origin_eof" and previous["time"] <= event["time"] for previous in events), "EOF lookup must start after the origin finishes" if adapter == "fastly": assert 0 < event["transport_timeout_ms"] <= 750, "blocking host calls must use the remaining budget" else: assert event["transport_timeout_ms"] == 30000, "SDK timer must release before the transport deadline" - assert any(event["event"] == "lookup_complete" and event["path"] == "/success" for event in events) + assert any(event["event"] == "lookup_complete" and event["path"] == "/success" for event in events), "success control must complete real lookup I/O" finally: - for process in reversed(processes): - stop(process) + # Finish owned-process cleanup even if another termination signal arrives. + signal.signal(signal.SIGTERM, signal.SIG_IGN) + try: + for process in reversed(processes): + stop(process) + finally: + signal.signal(signal.SIGTERM, previous_sigterm) diff --git a/scripts/fixtures/test_async_html_finalization_runtime.py b/scripts/fixtures/test_async_html_finalization_runtime.py index d97999b2f..5cc65a967 100644 --- a/scripts/fixtures/test_async_html_finalization_runtime.py +++ b/scripts/fixtures/test_async_html_finalization_runtime.py @@ -2,11 +2,17 @@ import importlib.util import os +import select from pathlib import Path import signal import subprocess import sys +import tempfile +import textwrap +import time +import tomllib import unittest +from unittest import mock SPEC = importlib.util.spec_from_file_location( "example_finalization_runtime", @@ -61,5 +67,171 @@ def test_stubborn_termination_reaps_process(self): process.stdout.close() +class HarnessConfigurationTests(unittest.TestCase): + def test_build_environment_preserves_compiler_settings_without_operator_config(self): + source = { + "PATH": "/private/tmp/example-bin", "RUSTFLAGS": "-D warnings", + "CARGO_ENCODED_RUSTFLAGS": "-D\x1fwarnings", "CARGO_INCREMENTAL": "0", + "CARGO_PROFILE_DEV_DEBUG": "0", "CARGO_PROFILE_RELEASE_LTO": "thin", + "TSJS_PREBUILT_DIR": "/private/tmp/example-bundles", + "TRUSTED_SERVER__PUBLISHER__ORIGIN_URL": "https://origin.example.com", + "TRUSTED_SERVER_SECRET_EXAMPLE": "example-secret", + "TSJS_SKIP_BUILD": "1", + "CARGO_BUILD_TARGET": "wasm32-wasip1", + } + preserved = RUNTIME.build_environment(source) + for key in ["RUSTFLAGS", "CARGO_ENCODED_RUSTFLAGS", "CARGO_INCREMENTAL", + "CARGO_PROFILE_DEV_DEBUG", "CARGO_PROFILE_RELEASE_LTO", "TSJS_PREBUILT_DIR"]: + self.assertEqual(preserved.get(key), source[key], f"should preserve explicit {key}") + self.assertNotIn("TRUSTED_SERVER__PUBLISHER__ORIGIN_URL", preserved) + self.assertNotIn("TRUSTED_SERVER_SECRET_EXAMPLE", preserved) + self.assertNotIn("TSJS_SKIP_BUILD", preserved) + self.assertNotIn("CARGO_BUILD_TARGET", preserved, "harness must select native/adapter targets itself") + + def test_checked_replacement_rejects_missing_or_ambiguous_template_fields(self): + for text in ["missing", "token token"]: + with self.subTest(text=text), self.assertRaisesRegex(RuntimeError, "example field"): + RUNTIME.checked_replace(text, "token", "value", "example field") + self.assertEqual(RUNTIME.checked_replace("token", "token", "value", "example field"), "value") + + def test_fixture_config_checks_loopback_budget_and_permission(self): + template = (RUNTIME.ROOT / "trusted-server.example.toml").read_text() + for case in ["disabled", "success", "hung", "auction", "streaming"]: + with self.subTest(case=case): + text = RUNTIME.fixture_config(template, case, 12345, 12346 if case == "auction" else None) + RUNTIME.validate_fixture_config(text, case, 12345, 12346 if case == "auction" else None) + if case == "success": + for old, new in [("html_finalization_timeout_ms = 750", "html_finalization_timeout_ms = 5000"), + ("allow_async_html_finish = true", "allow_async_html_finish = false"), + ("http://127.0.0.1:12345/success", "https://lookup.example.com/success")]: + with self.subTest(mutation=old), self.assertRaises(RuntimeError): + RUNTIME.validate_fixture_config(text.replace(old, new), case, 12345) + drifted = template.replace('endpoint = "https://prebid.example.com/openrtb2/auction"', + 'endpoint="https://prebid.example.com/openrtb2/auction"', 1) + with self.assertRaisesRegex(RuntimeError, "auction endpoint"): + RUNTIME.fixture_config(drifted, "auction", 12345, 12346) + + def test_toml_paths_round_trip_quotes_backslashes_and_newlines(self): + value = Path('/private/tmp/example 🚀 "quoted"\\path\nnext') + parsed = tomllib.loads("path = " + RUNTIME.toml_string(value)) + self.assertEqual(parsed["path"], str(value), "should preserve path characters in TOML") + workspace = value / "workspace" + template = (RUNTIME.ROOT / "edgezero.toml").read_text() + with mock.patch.object(RUNTIME, "ROOT", value): + parsed = tomllib.loads(RUNTIME.fixture_manifest(template, workspace)) + self.assertEqual(parsed["app"]["entry"], str(value / "crates/trusted-server-core")) + self.assertEqual(parsed["adapters"]["fastly"]["adapter"]["manifest"], str(workspace / "fastly.toml")) + self.assertEqual(parsed["adapters"]["spin"]["adapter"]["manifest"], str(workspace / "spin.toml")) + + def test_fastly_preflight_rejects_old_viceroy_before_building(self): + with tempfile.TemporaryDirectory() as directory, \ + mock.patch.object(RUNTIME.tempfile, "mkdtemp", return_value=directory), \ + mock.patch.object(RUNTIME, "free_port", return_value=12345), \ + mock.patch.object(RUNTIME.shutil, "which", return_value="/private/tmp/example-bin"), \ + mock.patch.object(RUNTIME.subprocess, "check_output", return_value="viceroy 0.17.0\n"), \ + mock.patch.object(RUNTIME.subprocess, "Popen", side_effect=RuntimeError("unexpected build before Viceroy preflight")) as build: + with self.assertRaisesRegex(RuntimeError, "Viceroy.*0.19"): + RUNTIME.run("fastly") + build.assert_not_called() + for version in ["viceroy 0.19.0\n", "viceroy 0.20.1\n", "viceroy 1.0.0\n"]: + with self.subTest(version=version), mock.patch.object(RUNTIME.subprocess, "check_output", return_value=version): + RUNTIME.preflight_viceroy({}) + + def test_cooperative_timeout_requires_a_measured_diagnostic(self): + for adapter in ["axum", "cloudflare", "spin"]: + with self.subTest(adapter=adapter): + with self.assertRaisesRegex(RuntimeError, "timeout diagnostic"): + RUNTIME.measured_terminal_ms(adapter, "times out without a duration") + self.assertEqual(RUNTIME.measured_terminal_ms(adapter, "times out after 751 ms"), 751) + self.assertIsNone(RUNTIME.measured_terminal_ms("fastly", "retains input: unavailable")) + + +class CancellationTests(unittest.TestCase): + def test_sigterm_unwinds_actual_run_and_stops_owned_origin(self): + with tempfile.TemporaryDirectory() as directory: + workspace = Path(directory) + target = workspace / "target/debug" + target.mkdir(parents=True) + (target / "trusted-server-axum").touch() + marker = workspace / "origin-stopped" + runner = workspace / "runner.py" + runner.write_text(textwrap.dedent(''' + import importlib.util, os, subprocess, sys, time + spec = importlib.util.spec_from_file_location("runtime", sys.argv[1]) + runtime = importlib.util.module_from_spec(spec) + spec.loader.exec_module(runtime) + runtime.free_port = lambda: 12345 + runtime.tempfile.mkdtemp = lambda **kwargs: sys.argv[2] + real_popen = subprocess.Popen + origin = None + origin_program = "import os, pathlib, signal, sys, time; pathlib.Path(sys.argv[2]).write_text(str(os.getpid())); signal.signal(signal.SIGTERM, lambda *_: (pathlib.Path(sys.argv[1]).write_text('stopped'), sys.exit(0))); pathlib.Path(sys.argv[2] + '.ready').write_text('ready'); time.sleep(300)" + def launch_origin(kwargs): + global origin + origin = real_popen([sys.executable, "-c", origin_program, sys.argv[3], os.path.join(sys.argv[2], "origin.pid")], **kwargs) + ready = os.path.join(sys.argv[2], "origin.pid.ready") + deadline = time.monotonic() + 5 + while not os.path.exists(ready): + if time.monotonic() >= deadline: + raise RuntimeError("origin probe failed to become ready") + time.sleep(0.01) + return origin + def fake_popen(args, **kwargs): + if args[0] == "python3": + return launch_origin(kwargs) + if args[1:3] == ["config", "init"]: + print("READY", flush=True) + return real_popen([sys.executable, "-c", "import os, pathlib, sys, time; pathlib.Path(sys.argv[1]).write_text(str(os.getpid())); time.sleep(300)", os.path.join(sys.argv[2], "command.pid")], **kwargs) + return real_popen([sys.executable, "-c", "pass"], **kwargs) + def fake_run(args, **kwargs): + if args[1:3] == ["config", "init"]: + print("READY", flush=True) + time.sleep(300) + runtime.subprocess.run = fake_run + runtime.subprocess.Popen = fake_popen + runtime.run("axum") + ''')) + environment = dict(os.environ) + environment.update(CARGO_TARGET_DIR=str(workspace / "target"), PYTHONDONTWRITEBYTECODE="1") + process = subprocess.Popen([sys.executable, str(runner), str(Path(RUNTIME.__file__)), str(workspace), str(marker)], + env=environment, stdout=subprocess.PIPE, stderr=subprocess.PIPE, + start_new_session=True) + try: + ready = False + until = time.monotonic() + 5 + transcript = b"" + while time.monotonic() < until: + readable, _, _ = select.select([process.stdout], [], [], 0.1) + if readable: + transcript += os.read(process.stdout.fileno(), 65536) + if b"READY\n" in transcript: + ready = True + break + if process.poll() is not None: + break + self.assertTrue(ready, "should reach the owned-origin cancellation probe") + time.sleep(0.1) + process.terminate() + process.wait(timeout=10) + self.assertTrue(marker.exists(), "SIGTERM should unwind run() and stop its owned origin") + self.assertEqual(process.returncode, 128 + signal.SIGTERM, "should preserve cancellation exit status") + for filename in ["origin.pid", "command.pid"]: + pid_file = workspace / filename + self.assertTrue(pid_file.exists(), "should launch the owned process probe") + with self.assertRaises(ProcessLookupError): + os.kill(int(pid_file.read_text()), 0) + finally: + if process.poll() is None: + os.killpg(process.pid, signal.SIGKILL) + process.wait(timeout=5) + # The child writes its own PID so failures can also clean the detached group. + for path in workspace.glob("*.pid"): + try: + os.killpg(int(path.read_text()), signal.SIGKILL) + except ProcessLookupError: + pass + process.stdout.close() + process.stderr.close() + + if __name__ == "__main__": unittest.main() From 0ac4b60ebdaf5774e8ab0ad1acb814925a8860ac Mon Sep 17 00:00:00 2001 From: prk-Jr Date: Tue, 13 Oct 2026 13:57:43 +0530 Subject: [PATCH 09/11] Tighten HTML EOF runtime evidence and regression checks --- .github/workflows/test.yml | 3 +- .../src/html_processor/finish.rs | 41 +++++++++++++++++++ .../tests/parity.rs | 28 +++++++++++++ docs/guide/configuration.md | 2 +- docs/guide/integration-guide.md | 12 ++++-- ...2026-10-12-1241-async-html-finalization.md | 12 +++++- ...-12-1241-async-html-finalization-design.md | 6 +-- .../async-html-finalization-runtime.py | 24 ++++++++--- .../test_async_html_finalization_runtime.py | 31 +++++++++++++- 9 files changed, 144 insertions(+), 15 deletions(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 8b8e1772b..b2f9a6d4f 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -104,13 +104,14 @@ jobs: uses: actions/cache@v4 with: path: ${{ runner.temp }}/viceroy-eof/bin/viceroy - key: viceroy-eof-${{ runner.os }}-amd64-v0.19.0 + key: viceroy-eof-${{ runner.os }}-amd64-v0.19.0-sha256-23b2af7d - name: Install isolated Viceroy 0.19 for backend timeout simulation if: steps.cache-eof-viceroy.outputs.cache-hit != 'true' shell: bash run: | curl -fsSL --retry 3 https://github.com/fastly/Viceroy/releases/download/v0.19.0/viceroy_v0.19.0_linux-amd64.tar.gz -o "$RUNNER_TEMP/viceroy-eof.tar.gz" + echo "23b2af7d8dce8f4a217716d6bd8b15c4f41494316a0d4e80f179ffb4b6c74744 $RUNNER_TEMP/viceroy-eof.tar.gz" | sha256sum --check --strict mkdir -p "$RUNNER_TEMP/viceroy-eof/bin" tar -xzf "$RUNNER_TEMP/viceroy-eof.tar.gz" -C "$RUNNER_TEMP/viceroy-eof/bin" viceroy diff --git a/crates/trusted-server-core/src/html_processor/finish.rs b/crates/trusted-server-core/src/html_processor/finish.rs index 2615c0148..05b1705a9 100644 --- a/crates/trusted-server-core/src/html_processor/finish.rs +++ b/crates/trusted-server-core/src/html_processor/finish.rs @@ -1475,6 +1475,47 @@ mod tests { ); } + #[test] + fn async_eof_small_chunks_keep_a_bounded_marker_suffix() { + let events = Rc::new(RefCell::new(Vec::new())); + let calls = Rc::new(Cell::new(0)); + let mut chain = HtmlWithStreamingProcessors::from_stages( + stage("parser", b"", None, &events, &calls), + vec![HtmlProcessorStage::new( + "B", + stage("B", b"EN", Some(b"wrong"), &events, &calls), + true, + false, + 64, + )], + ); + chain.set_finish_marker(Some(b"TOKEN")); + for chunk in [b"xxxx".as_slice(), b"TO", b"K"] { + assert_eq!( + chain + .process_chunk(chunk, false) + .expect("should stream each small chunk"), + chunk, + "should preserve ordinary output" + ); + assert!( + chain.stages[0].previous_suffix.len() <= 4, + "should bound marker history after every chunk" + ); + } + assert_eq!( + chain.stages[0].previous_suffix, b"xTOK", + "should retain the trailing bytes across small chunks" + ); + chain.begin_finish().expect("should finish the parser"); + assert_eq!( + finish_marker_stage(&mut chain, 64), + b"EN", + "should reject removal of a marker completed after trimmed history" + ); + assert_eq!(calls.get(), 1, "should evaluate the candidate once"); + } + #[test] fn async_eof_single_byte_marker_rejects_creation_and_removal() { for (tail, replacement) in [ diff --git a/crates/trusted-server-integration-tests/tests/parity.rs b/crates/trusted-server-integration-tests/tests/parity.rs index 878753963..dc26666ef 100644 --- a/crates/trusted-server-integration-tests/tests/parity.rs +++ b/crates/trusted-server-integration-tests/tests/parity.rs @@ -1195,6 +1195,7 @@ impl PlatformHttpClient for TerminalFixtureOrigin { #[tokio::test(flavor = "multi_thread", worker_threads = 2)] async fn asynchronous_html_finalization_and_privacy_match_host_adapters() { + let mut disabled_output = None; for case in ["disabled", "failed", "success"] { let enabled = case != "disabled"; let mut settings = test_settings(); @@ -1255,6 +1256,15 @@ async fn asynchronous_html_finalization_and_privacy_match_host_adapters() { "no-store, private", "{adapter} should prevent downstream caching" ); + } else { + assert_ne!( + response + .headers() + .get("cache-control") + .expect("should preserve the ordinary cache policy"), + "no-store, private", + "{adapter} should not apply EOF privacy to the inactive fixture" + ); } let body = response .into_body() @@ -1269,6 +1279,21 @@ async fn asynchronous_html_finalization_and_privacy_match_host_adapters() { }), "{adapter} should select the correct EOF result for {case}" ); + if case != "success" { + assert!( + !String::from_utf8_lossy(&body).contains("EXAMPLE_FINALIZED"), + "{adapter} should not emit a finalized marker for {case}" + ); + } + if case == "failed" { + assert_eq!( + &body, + disabled_output + .as_ref() + .expect("should have the disabled baseline"), + "{adapter} should preserve the exact baseline after failed EOF I/O" + ); + } assert_eq!( client.lookup_calls.load(Ordering::SeqCst), usize::from(enabled), @@ -1283,5 +1308,8 @@ async fn asynchronous_html_finalization_and_privacy_match_host_adapters() { expected = Some(body); } } + if case == "disabled" { + disabled_output = expected; + } } } diff --git a/docs/guide/configuration.md b/docs/guide/configuration.md index 961ecfba4..70eafda92 100644 --- a/docs/guide/configuration.md +++ b/docs/guide/configuration.md @@ -344,7 +344,7 @@ TRUSTED_SERVER__PUBLISHER__ORIGIN_URL=https://origin.publisher.com TRUSTED_SERVER__PUBLISHER__ORIGIN_HOST_HEADER_OVERRIDE=www.publisher.com TRUSTED_SERVER__PUBLISHER__PROXY_SECRET=publisher_proxy_secret TRUSTED_SERVER__PUBLISHER__MAX_BUFFERED_BODY_BYTES=16777216 -TRUSTED_SERVER__PUBLISHER__ALLOW_ASYNC_HTML_FINISH=true +TRUSTED_SERVER__PUBLISHER__ALLOW_ASYNC_HTML_FINISH=false TRUSTED_SERVER__PUBLISHER__HTML_FINALIZATION_TIMEOUT_MS=5000 ``` diff --git a/docs/guide/integration-guide.md b/docs/guide/integration-guide.md index d40c26161..f0f317522 100644 --- a/docs/guide/integration-guide.md +++ b/docs/guide/integration-guide.md @@ -223,9 +223,9 @@ A non-retaining session can make a KV lookup at EOF and append a fixed footer: ```rust -use std::io; use error_stack::Report; use futures::future::LocalBoxFuture; +use std::io; use trusted_server_core::streaming_processor::StreamProcessor; use trusted_server_core::streaming_processor::finish::{ StreamFinish, StreamFinishError, StreamFinishInput, @@ -248,7 +248,11 @@ impl StreamProcessor for ExampleFooterProcessor { input: StreamFinishInput<'a>, ) -> Option>>> { Some(Box::pin(async move { - let enabled = input.services.kv_store().get_bytes("example:footer").await + let enabled = input + .services + .kv_store() + .get_bytes("example:footer") + .await .map_err(|_| Report::new(StreamFinishError::Unavailable))? .is_some(); if !enabled { @@ -268,7 +272,9 @@ The factory also returns `true` from `requires_async_finish()` and creates this session. Leave `buffer_output_until_eof()` at its default `false` for this append operation. KV must be available on the runtime; unavailable lookup falls back to the synchronous tail. Fastly consumers must use host operations with independently -bounded latency; this generic KV interface does not provide a hard timeout. +bounded latency and cap configurable transport timeouts with `input.deadline.remaining()`; +this generic KV interface does not provide a hard timeout. Checking the remaining +budget alone cannot interrupt a blocking KV call. Operators must explicitly accept the possible delivery cost: diff --git a/docs/superpowers/plans/2026-10-12-1241-async-html-finalization.md b/docs/superpowers/plans/2026-10-12-1241-async-html-finalization.md index 4aba58584..09a20611b 100644 --- a/docs/superpowers/plans/2026-10-12-1241-async-html-finalization.md +++ b/docs/superpowers/plans/2026-10-12-1241-async-html-finalization.md @@ -63,7 +63,17 @@ - [x] Run `cargo fmt --all -- --check` and the integration crate format check, all eight target-matched Clippy aliases, all adapter test aliases and Fastly reuse, native core/digest gates, CLI/codegen, parity and exact integration CI selections. - [x] Run existing ESI/inline/purge controls, JS build/tests/format, docs lint/build/format and repository Markdown formatting. - [x] Record fresh four-runtime success, disabled, hung and auction results, including Fastly early-body-byte proof. - **Publication procedure:** Commit and push the existing branch, update draft PR #1290, and confirm remote head/status. Hosted CI results remain separate from local checks. + +### 6. Runtime evidence and regression assertions + +- [x] Correlate EOF and lookup events within each request; prove an earlier request's EOF cannot satisfy the check. +- [x] Require measured lookup duration for Fastly transport fallback without assuming a cooperative timeout log. +- [x] Assert disabled/failed parity has no finalized marker and failed output equals the disabled baseline; preserve the fixture's ordinary cache policy. +- [x] Verify bounded marker history across multiple small chunks and EOF marker protection. +- [x] Check the isolated Viceroy archive against its pinned SHA-256 before extraction; invalidate the earlier unchecked tool cache. +- [x] Preserve the default-disabled permission example, format the Rust example and keep the spec independent of superseded drafts. + +**Publication:** After validation, commit and push to the existing draft PR. Confirm its remote head and report hosted CI separately from local results. ## Acceptance mapping diff --git a/docs/superpowers/specs/2026-10-12-1241-async-html-finalization-design.md b/docs/superpowers/specs/2026-10-12-1241-async-html-finalization-design.md index f54e9403e..98dbb659c 100644 --- a/docs/superpowers/specs/2026-10-12-1241-async-html-finalization-design.md +++ b/docs/superpowers/specs/2026-10-12-1241-async-html-finalization-design.md @@ -41,9 +41,9 @@ its synchronous output, both false by default. Reject retention without async capability. Metadata and the created session's capability must agree; undeclared async work must never silently disappear through a synchronous driver. -Remove the unmerged separate finalizer factory, session trait and registration list. -Existing tests can use a test-only adapter to the one production processor lifecycle. -No production compatibility API for the previous draft is required. +Use one production processor lifecycle without a separate finalizer factory, +session trait or registration list. Tests can use a test-only adapter to this +lifecycle. Factory metadata is available before template lookup, without creating a session, reading HTML or polling a future. Every session uses the existing request-local diff --git a/scripts/fixtures/async-html-finalization-runtime.py b/scripts/fixtures/async-html-finalization-runtime.py index 614973fef..fe9862ec7 100644 --- a/scripts/fixtures/async-html-finalization-runtime.py +++ b/scripts/fixtures/async-html-finalization-runtime.py @@ -163,11 +163,26 @@ def preflight_viceroy(env): raise RuntimeError(f"Viceroy >= 0.19.0 required for the Fastly hung-call control; found {output!r}") -def measured_terminal_ms(adapter, diagnostic): +def lookup_for_request(events, started, completed, enabled): + lookups = [event for event in events if event["event"] == "lookup_start" + and started <= event["time"] <= completed] + assert len(lookups) == int(enabled), "each request must issue exactly one enabled EOF lookup" + if not enabled: + return None + lookup = lookups[0] + assert any(event["event"] == "origin_eof" and started <= event["time"] <= lookup["time"] + for event in events), "EOF lookup must start after the origin finishes" + return lookup + + +def measured_terminal_ms(adapter, diagnostic, lookup_elapsed=None): match = re.search(r"times out after (\d+) ms", diagnostic) if match is None: if adapter != "fastly": raise RuntimeError(f"missing measured cooperative timeout diagnostic for {adapter}") + if lookup_elapsed is None: + raise RuntimeError("missing measured lookup duration for the Fastly transport fallback") + assert 0.7 <= lookup_elapsed < 2.5, "blocking transport must finish within the terminal budget tolerance" return None terminal_ms = int(match.group(1)) assert 700 <= terminal_ms < 2500, "terminal budget must exclude origin transfer" @@ -377,15 +392,15 @@ def field(offset, width, value): if case != "disabled": assert headers["Cache-Control"] == "no-store, private", "active EOF processing must prohibit shared caching" current_events = [json.loads(line) for line in (workspace / "events.jsonl").read_text().splitlines()] - lookups = [event for event in current_events if event["event"] == "lookup_start"] - lookup_elapsed = completed - lookups[-1]["time"] if case != "disabled" else None + lookup = lookup_for_request(current_events, started, completed, case != "disabled") + lookup_elapsed = completed - lookup["time"] if lookup is not None else None if case == "streaming": completions = [event for event in current_events if event["event"] == "lookup_complete"] assert completions, "streaming finish must complete real delayed I/O" assert first_byte_at < completions[-1]["time"] - 0.05, "initial HTML bytes must reach the client while EOF I/O remains pending" terminal_ms = None if case == "hung": - terminal_ms = measured_terminal_ms(adapter, diagnostic) + terminal_ms = measured_terminal_ms(adapter, diagnostic, lookup_elapsed) (workspace / f"{case}.timing.json").write_text(json.dumps({"request_seconds": elapsed, "first_body_byte_seconds": first_byte_at - started, "terminal_ms": terminal_ms, "lookup_to_response_seconds": lookup_elapsed, "configured_budget_ms": 750, "cooperative_timer": adapter != "fastly"})) print(f"PASS {adapter} {case}: request={elapsed:.3f}s, {len(body)} bytes, chain_budget=750ms, terminal_ms={terminal_ms}, lookup_to_response={lookup_elapsed}, cooperative_timer={adapter != 'fastly'}", flush=True) stop(process) @@ -394,7 +409,6 @@ def field(offset, width, value): assert len(starts) == 2 + int(auction) + int(adapter == "fastly"), "each enabled control must issue exactly one EOF lookup" assert all(event["complete"] and len(event["sha256"]) == 64 for event in starts), "every lookup must observe complete-document metadata" for event in starts: - assert any(previous["event"] == "origin_eof" and previous["time"] <= event["time"] for previous in events), "EOF lookup must start after the origin finishes" if adapter == "fastly": assert 0 < event["transport_timeout_ms"] <= 750, "blocking host calls must use the remaining budget" else: diff --git a/scripts/fixtures/test_async_html_finalization_runtime.py b/scripts/fixtures/test_async_html_finalization_runtime.py index 5cc65a967..2a42e6100 100644 --- a/scripts/fixtures/test_async_html_finalization_runtime.py +++ b/scripts/fixtures/test_async_html_finalization_runtime.py @@ -143,7 +143,36 @@ def test_cooperative_timeout_requires_a_measured_diagnostic(self): with self.assertRaisesRegex(RuntimeError, "timeout diagnostic"): RUNTIME.measured_terminal_ms(adapter, "times out without a duration") self.assertEqual(RUNTIME.measured_terminal_ms(adapter, "times out after 751 ms"), 751) - self.assertIsNone(RUNTIME.measured_terminal_ms("fastly", "retains input: unavailable")) + self.assertIsNone(RUNTIME.measured_terminal_ms("fastly", "retains input: unavailable", 0.749)) + + def test_fastly_transport_fallback_requires_measured_lookup_duration(self): + with self.assertRaisesRegex(RuntimeError, "lookup duration"): + RUNTIME.measured_terminal_ms("fastly", "retains input: unavailable") + for elapsed in [0.1, 3.0]: + with self.subTest(elapsed=elapsed), self.assertRaises(AssertionError): + RUNTIME.measured_terminal_ms("fastly", "retains input: unavailable", elapsed) + + def test_eof_lookup_rejects_an_earlier_requests_eof(self): + lookup = {"event": "lookup_start", "time": 2.2} + for current_eof in [[], [{"event": "origin_eof", "time": 2.3}]]: + events = [{"event": "origin_eof", "time": 1.0}, lookup, *current_eof] + with self.subTest(events=events), self.assertRaisesRegex(AssertionError, "origin finishes"): + RUNTIME.lookup_for_request(events, 2.0, 3.0, True) + + def test_eof_lookup_uses_only_the_current_request(self): + lookup = {"event": "lookup_start", "time": 2.2} + events = [ + {"event": "origin_eof", "time": 1.0}, + {"event": "lookup_start", "time": 1.1}, + {"event": "origin_eof", "time": 2.1}, + lookup, + ] + self.assertIs(RUNTIME.lookup_for_request(events, 2.0, 3.0, True), lookup) + self.assertIsNone(RUNTIME.lookup_for_request(events, 4.0, 5.0, False)) + with self.assertRaisesRegex(AssertionError, "exactly one"): + RUNTIME.lookup_for_request([*events, lookup], 2.0, 3.0, True) + with self.assertRaisesRegex(AssertionError, "exactly one"): + RUNTIME.lookup_for_request(events, 2.0, 3.0, False) class CancellationTests(unittest.TestCase): From 74184094632d26fd08db871bf498c71d40b2308d Mon Sep 17 00:00:00 2001 From: prk-Jr Date: Tue, 13 Oct 2026 16:45:54 +0530 Subject: [PATCH 10/11] Replace HTML EOF Python harness with Rust tests --- .github/workflows/test.yml | 16 +- AGENTS.md | 2 +- Cargo.lock | 3 + .../Cargo.toml | 13 +- .../tests/html_eof_runtime.rs | 871 ++++++++++++++++++ .../tests/html_eof_runtime/config.rs | 325 +++++++ .../tests/html_eof_runtime/origin.rs | 257 ++++++ .../tests/html_eof_runtime/process.rs | 445 +++++++++ docs/guide/integration-guide.md | 2 +- docs/guide/testing.md | 8 +- ...2026-10-12-1241-async-html-finalization.md | 22 +- ...-12-1241-async-html-finalization-design.md | 14 + scripts/README.md | 38 +- scripts/async-html-finalization-local-test.sh | 18 +- .../async-html-finalization-origin.py | 84 -- .../async-html-finalization-runtime.py | 434 --------- .../test_async_html_finalization_runtime.py | 266 ------ 17 files changed, 1991 insertions(+), 827 deletions(-) create mode 100644 crates/trusted-server-integration-tests/tests/html_eof_runtime.rs create mode 100644 crates/trusted-server-integration-tests/tests/html_eof_runtime/config.rs create mode 100644 crates/trusted-server-integration-tests/tests/html_eof_runtime/origin.rs create mode 100644 crates/trusted-server-integration-tests/tests/html_eof_runtime/process.rs delete mode 100644 scripts/fixtures/async-html-finalization-origin.py delete mode 100644 scripts/fixtures/async-html-finalization-runtime.py delete mode 100644 scripts/fixtures/test_async_html_finalization_runtime.py diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index b2f9a6d4f..510d03be8 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -95,10 +95,6 @@ jobs: with: node-version: ${{ steps.versions.outputs.node }} - - uses: actions/setup-python@v5 - with: - python-version: '3.12' - - name: Cache isolated EOF fixture Viceroy id: cache-eof-viceroy uses: actions/cache@v4 @@ -169,15 +165,11 @@ jobs: - name: Run Axum adapter tests run: cargo test-axum - - name: Set up Python for actual-runtime finalization checks - uses: actions/setup-python@v5 - with: - python-version: '3.12' - - name: Verify finalization harness process cleanup - env: - PYTHONDONTWRITEBYTECODE: '1' - run: python3 -m unittest discover -s scripts/fixtures -p 'test_async_html_finalization_runtime.py' -v + run: cargo test --manifest-path crates/trusted-server-integration-tests/Cargo.toml --features html-eof-runtime --test html_eof_runtime + + - name: Lint native finalization harness + run: cargo clippy --manifest-path crates/trusted-server-integration-tests/Cargo.toml --features html-eof-runtime --test html_eof_runtime -- -D warnings - name: Verify actual Axum HTML finalization and auction assembly timeout-minutes: 10 diff --git a/AGENTS.md b/AGENTS.md index 188c1b100..e1204bbeb 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -362,7 +362,7 @@ Every PR must pass: 7. Docs format (`cd docs && npm run format`) 8. Markdown format outside `docs/` (requires `cd docs && npm ci` first): `docs/node_modules/.bin/prettier --config docs/.prettierrc --check "*.md" ".claude/**/*.md" ".github/**/*.md" "crates/**/*.md" "scripts/**/*.md" "tinybird/**/*.md"`; fix with `--write` in place of `--check` 9. Native core build-digest test and lint (`cargo test -p trusted-server-core --test template_build_digest` and `cargo clippy -p trusted-server-core --test template_build_digest -- -D warnings`) -10. HTML EOF harness cleanup (Python 3.11+): `python3 -m unittest discover -s scripts/fixtures -p 'test_async_html_finalization_runtime.py' -v` +10. Native Rust HTML EOF harness configuration, timing and cleanup: `cargo test --manifest-path crates/trusted-server-integration-tests/Cargo.toml --features html-eof-runtime --test html_eof_runtime` and `cargo clippy --manifest-path crates/trusted-server-integration-tests/Cargo.toml --features html-eof-runtime --test html_eof_runtime -- -D warnings` 11. Actual Axum EOF/auction controls: `./scripts/async-html-finalization-local-test.sh axum --auction` 12. Actual Fastly EOF/auction controls with isolated Viceroy 0.19+: `PATH="/tmp/example-eof-viceroy/bin:$PATH" ./scripts/async-html-finalization-local-test.sh fastly --release --auction`; install with `cargo install viceroy --version 0.19.0 --locked --root /tmp/example-eof-viceroy`. Ordinary tests retain pinned Viceroy 0.17. See `scripts/README.md` for retained diagnostics and other runtimes. diff --git a/Cargo.lock b/Cargo.lock index e915a0ea8..6097394bf 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -6686,12 +6686,14 @@ version = "0.1.0" dependencies = [ "async-trait", "axum", + "base64", "bytes", "derive_more", "edgezero-adapter-axum", "edgezero-core", "env_logger", "error-stack", + "futures", "http", "http-body-util", "libc", @@ -6699,6 +6701,7 @@ dependencies = [ "reqwest 0.12.28", "scraper", "serde_json", + "tempfile", "testcontainers", "tokio", "toml", diff --git a/crates/trusted-server-integration-tests/Cargo.toml b/crates/trusted-server-integration-tests/Cargo.toml index 39f985860..b2369d9a6 100644 --- a/crates/trusted-server-integration-tests/Cargo.toml +++ b/crates/trusted-server-integration-tests/Cargo.toml @@ -7,6 +7,14 @@ license = { workspace = true } publish = { workspace = true } version = { workspace = true } +[features] +html-eof-runtime = [] + +[[test]] +name = "html_eof_runtime" +path = "tests/html_eof_runtime.rs" +required-features = ["html-eof-runtime"] + [[test]] name = "integration" path = "tests/integration.rs" @@ -29,11 +37,13 @@ trusted-server-core = { workspace = true, features = ["test-utils"] } [dev-dependencies] async-trait = { workspace = true } axum = { workspace = true } +base64 = { workspace = true } bytes = { workspace = true } derive_more = { workspace = true } edgezero-adapter-axum = { workspace = true, features = ["axum"] } env_logger = { workspace = true } error-stack = { workspace = true } +futures = { workspace = true } http = { workspace = true } http-body-util = { workspace = true } libc = { workspace = true } @@ -41,7 +51,8 @@ log = { workspace = true } reqwest = { workspace = true, features = ["blocking", "cookies"] } scraper = { workspace = true } testcontainers = { workspace = true } -tokio = { workspace = true, features = ["rt-multi-thread"] } +tempfile = { workspace = true } +tokio = { workspace = true, features = ["net", "rt-multi-thread", "signal"] } tower = { workspace = true, features = ["util"] } trusted-server-adapter-axum = { path = "../trusted-server-adapter-axum" } trusted-server-adapter-cloudflare = { path = "../trusted-server-adapter-cloudflare" } diff --git a/crates/trusted-server-integration-tests/tests/html_eof_runtime.rs b/crates/trusted-server-integration-tests/tests/html_eof_runtime.rs new file mode 100644 index 000000000..32146b127 --- /dev/null +++ b/crates/trusted-server-integration-tests/tests/html_eof_runtime.rs @@ -0,0 +1,871 @@ +//! Native verification of ordered HTML EOF work on actual local runtimes. + +#[path = "html_eof_runtime/config.rs"] +mod config; +#[path = "html_eof_runtime/origin.rs"] +mod origin; +#[path = "html_eof_runtime/process.rs"] +mod process; + +use std::collections::BTreeMap; +use std::ffi::OsString; +use std::fs::{self, File}; +use std::net::{Ipv4Addr, TcpListener}; +use std::path::{Path, PathBuf}; +use std::process::{Command, Stdio}; +use std::thread; +use std::time::{Duration, Instant, SystemTime, UNIX_EPOCH}; + +use base64::Engine as _; +use base64::engine::general_purpose::URL_SAFE_NO_PAD; +use error_stack::{Report, ResultExt as _}; +use reqwest::blocking::Client; +use serde_json::{Value, json}; + +use process::{Cancellation, ManagedChild}; + +#[derive(Debug, derive_more::Display)] +enum HarnessError { + #[display("Invalid isolated runtime configuration")] + Configuration, + #[display("Runtime command fails")] + Runtime, + #[display("Runtime HTTP request fails")] + Http, + #[display("Runtime fixture I/O fails")] + Io, + #[display("Runtime verification is cancelled")] + Cancelled, +} + +impl core::error::Error for HarnessError {} + +type HarnessResult = Result>; + +struct Harness<'a> { + origin: &'a origin::Origin, + root: PathBuf, + workspace: PathBuf, + target: PathBuf, + environment: BTreeMap, + cancellation: Cancellation, +} + +impl<'a> Harness<'a> { + fn new(adapter: &str, origin: &'a origin::Origin) -> HarnessResult { + let root = PathBuf::from(env!("CARGO_MANIFEST_DIR")) + .join("../..") + .canonicalize() + .change_context(HarnessError::Io)?; + let workspace = tempfile::Builder::new() + .prefix(&format!("trusted-server-finalize-{adapter}-")) + .tempdir() + .change_context(HarnessError::Io)? + .keep(); + log::info!("Artifacts: {}", workspace.display()); + let mut environment = process::build_environment(std::env::vars_os()); + let target = environment + .get(&OsString::from("CARGO_TARGET_DIR")) + .map(PathBuf::from) + .unwrap_or_else(|| root.join("target")); + let target = if target.is_absolute() { + target + } else { + root.join(target) + }; + for (name, value) in [ + ("CARGO_TARGET_DIR", target.as_os_str().to_owned()), + ( + "XDG_CONFIG_HOME", + workspace.join("cli-config").into_os_string(), + ), + ("WRANGLER_SEND_METRICS", OsString::from("false")), + ( + "WRANGLER_LOG_PATH", + workspace.join("wrangler.log").into_os_string(), + ), + ( + "TRUSTED_SERVER__PUBLISHER__DOMAIN", + OsString::from("127.0.0.1"), + ), + ( + "TRUSTED_SERVER__PUBLISHER__COOKIE_DOMAIN", + OsString::from("127.0.0.1"), + ), + ( + "TRUSTED_SERVER__PUBLISHER__ORIGIN_URL", + OsString::from(format!("http://127.0.0.1:{}", origin.port)), + ), + ] { + environment.insert(OsString::from(name), value); + } + let cancellation = Cancellation::default(); + cancellation.watch(origin.runtime.handle())?; + Ok(Self { + origin, + root, + workspace, + target, + environment, + cancellation, + }) + } + + fn command(&self, program: impl AsRef) -> Command { + let mut command = Command::new(program); + command + .current_dir(&self.root) + .env_clear() + .envs(&self.environment); + command + } + + fn run(&self, command: &mut Command, name: &str) -> HarnessResult<()> { + process::logged_command( + command, + &self.workspace.join(format!("{name}.log")), + &self.cancellation, + ) + } + + fn preflight(&self, adapter: &str) -> HarnessResult<()> { + if adapter == "fastly" { + self.run(self.command("viceroy").arg("--version"), "viceroy-version")?; + let version = fs::read_to_string(self.workspace.join("viceroy-version.log")) + .change_context(HarnessError::Io)?; + validate_viceroy(&version)?; + } + if adapter == "cloudflare" { + self.run( + self.command("worker-build").arg("--version"), + "worker-version", + )?; + let actual = fs::read_to_string(self.workspace.join("worker-version.log")) + .change_context(HarnessError::Io)?; + let lock = config::parse( + &fs::read_to_string(self.root.join("Cargo.lock")) + .change_context(HarnessError::Io)?, + )?; + let expected = lock["package"] + .as_array() + .and_then(|packages| { + packages + .iter() + .find(|package| package["name"].as_str() == Some("worker")) + }) + .and_then(|package| package["version"].as_str()) + .ok_or_else(|| Report::new(HarnessError::Configuration))?; + if actual.trim() != expected { + return Err(Report::new(HarnessError::Runtime).attach(format!( + "worker-build {expected} required; found {}", + actual.trim() + ))); + } + } + Ok(()) + } + + fn build(&self, adapter: &str, release: bool) -> HarnessResult { + self.run( + self.command("cargo").args([ + "build", + "-p", + "trusted-server-cli", + "--bin", + "ts", + "--locked", + ]), + "build-cli", + )?; + let crate_root = self + .root + .join(format!("crates/trusted-server-adapter-{adapter}")); + let profile = if release { "release" } else { "debug" }; + let artifact = if adapter == "cloudflare" { + let mut command = self.command("worker-build"); + command + .current_dir(crate_root) + .arg(if release { "--release" } else { "--dev" }) + .arg("--out-dir") + .arg(self.workspace.join("build")) + .args(["--features", "cloudflare,test-utils", "--locked"]); + if !release { + command.arg("--no-opt"); + } + self.run(&mut command, "build-runtime")?; + self.workspace.join("build/index.js") + } else { + let mut command = self.command("cargo"); + command.args([ + "build", + "-p", + &format!("trusted-server-adapter-{adapter}"), + "--locked", + "--features", + if adapter == "spin" { + "trusted-server-core/test-utils,spin,test-utils" + } else { + "trusted-server-core/test-utils" + }, + ]); + if adapter != "axum" { + command.args(["--target", "wasm32-wasip1"]); + } + if release { + command.arg("--release"); + } + self.run(&mut command, "build-runtime")?; + match adapter { + "axum" => self.target.join(format!("{profile}/trusted-server-axum")), + "spin" => self.target.join(format!( + "wasm32-wasip1/{profile}/trusted_server_adapter_spin.wasm" + )), + _ => self.target.join(format!( + "wasm32-wasip1/{profile}/trusted-server-adapter-fastly.wasm" + )), + } + }; + if !artifact.is_file() { + return Err(Report::new(HarnessError::Runtime) + .attach(format!("fresh artifact missing: {}", artifact.display()))); + } + Ok(artifact) + } + + fn push(&self, adapter: &str) -> HarnessResult<()> { + let mut command = self.command(self.target.join("debug/ts")); + command + .current_dir(&self.workspace) + .args([ + "config", + "push", + "--adapter", + adapter, + "--local", + "--manifest", + ]) + .arg(self.workspace.join("edgezero.toml")) + .arg("--app-config") + .arg(self.workspace.join("trusted-server.toml")) + .args(["--yes", "--no-diff"]); + if adapter == "spin" { + command.env( + "EDGEZERO__STORES__CONFIG__TRUSTED_SERVER_CONFIG__NAME", + "default", + ); + } + self.run(&mut command, &format!("push-{adapter}")) + } + + fn runtime_command(&self, adapter: &str, artifact: &Path, port: u16) -> HarnessResult { + let mut command; + let raw = fs::read_to_string( + self.workspace + .join(".edgezero/local-config-trusted_server_config.json"), + ) + .change_context(HarnessError::Io)?; + let value: Value = + serde_json::from_str(&raw).change_context(HarnessError::Configuration)?; + let envelope = value["trusted_server_config"] + .as_str() + .ok_or_else(|| Report::new(HarnessError::Configuration))?; + match adapter { + "axum" => { + command = self.command(artifact); + command.env("PORT", port.to_string()).env( + "TRUSTED_SERVER_CONFIG_TRUSTED_SERVER_CONFIG_TRUSTED_SERVER_CONFIG", + envelope, + ); + for (key, value) in config::SECRETS { + command.env( + format!( + "TRUSTED_SERVER_SECRET_TRUSTED_SERVER_SECRETS_{}", + key.to_uppercase() + ), + value, + ); + } + } + "cloudflare" => { + let mut manifest = config::parse( + &fs::read_to_string( + self.root + .join("crates/trusted-server-adapter-cloudflare/wrangler.ci.toml"), + ) + .change_context(HarnessError::Io)?, + )?; + config::replace( + &mut manifest, + &["main"], + toml::Value::String(artifact.to_string_lossy().into_owned()), + )?; + config::replace( + &mut manifest, + &["vars", "TRUSTED_SERVER_CONFIG"], + toml::Value::String(json!({"app_config": envelope}).to_string()), + )?; + let vars = manifest["vars"] + .as_table_mut() + .ok_or_else(|| Report::new(HarnessError::Configuration))?; + for (key, value) in config::SECRETS { + vars.insert((*key).to_owned(), toml::Value::String((*value).to_owned())); + } + fs::write( + self.workspace.join("wrangler.toml"), + config::serialize(&manifest)?, + ) + .change_context(HarnessError::Io)?; + command = self.command("wrangler"); + command + .args(["dev", "--local", "--config"]) + .arg(self.workspace.join("wrangler.toml")) + .args([ + "--ip", + "127.0.0.1", + "--port", + &port.to_string(), + "--persist-to", + ]) + .arg(self.workspace.join("worker-state")); + } + "fastly" => { + let mut manifest = config::parse( + &fs::read_to_string(self.workspace.join("fastly.toml")) + .change_context(HarnessError::Io)?, + )?; + let secrets: Vec<_> = config::SECRETS + .iter() + .map(|(key, value)| { + let mut entry = toml::map::Map::new(); + entry.insert("key".to_owned(), toml::Value::String((*key).to_owned())); + entry.insert("data".to_owned(), toml::Value::String((*value).to_owned())); + toml::Value::Table(entry) + }) + .collect(); + let local = manifest["local_server"] + .as_table_mut() + .ok_or_else(|| Report::new(HarnessError::Configuration))?; + let mut stores = toml::map::Map::new(); + stores.insert( + "trusted_server_secrets".to_owned(), + toml::Value::Array(secrets), + ); + local.insert("secret_stores".to_owned(), toml::Value::Table(stores)); + fs::write( + self.workspace.join("fastly.toml"), + config::serialize(&manifest)?, + ) + .change_context(HarnessError::Io)?; + command = self.command("viceroy"); + command + .arg("--config") + .arg(self.workspace.join("fastly.toml")) + .args(["--addr", &format!("127.0.0.1:{port}")]) + .arg(artifact); + } + "spin" => { + command = self.command("spin"); + command + .args(["up", "--from"]) + .arg(self.workspace.join("spin.toml")) + .args(["--listen", &format!("127.0.0.1:{port}"), "--state-dir"]) + .arg(self.workspace.join(".spin")) + .arg("--cache-dir") + .arg(self.workspace.join("spin-cache")) + .arg("--log-dir") + .arg(self.workspace.join("spin-logs")); + for (key, value) in config::SECRETS { + command.arg("--variable").arg(format!( + "v_trusted_x5fserver_x5fsecrets_v_{}={value}", + key.replace('_', "_x5f") + )); + } + } + _ => { + return Err( + Report::new(HarnessError::Configuration).attach("unknown EOF runtime adapter") + ); + } + } + command.current_dir(&self.workspace); + Ok(command) + } +} + +impl Drop for Harness<'_> { + fn drop(&mut self) { + if let Err(error) = self + .origin + .write_events(&self.workspace.join("events.json")) + { + log::warn!("EOF harness cannot retain event diagnostics: {error:?}"); + } + } +} + +fn available_port() -> HarnessResult { + TcpListener::bind((Ipv4Addr::LOCALHOST, 0)) + .and_then(|listener| listener.local_addr()) + .map(|address| address.port()) + .change_context(HarnessError::Io) +} + +fn validate_viceroy(output: &str) -> HarnessResult<()> { + let version = output + .trim() + .strip_prefix("viceroy ") + .and_then(|version| version.split_whitespace().next()) + .and_then(|version| { + version + .split('.') + .map(str::parse::) + .collect::, _>>() + .ok() + }); + if !version + .is_some_and(|version| version.len() == 3 && version.as_slice() >= [0, 19, 0].as_slice()) + { + return Err(Report::new(HarnessError::Runtime) + .attach(format!("Viceroy >= 0.19.0 required; found {output:?}"))); + } + Ok(()) +} + +fn measured_terminal_ms( + adapter: &str, + diagnostic: &str, + lookup_elapsed: Duration, +) -> HarnessResult> { + let measured = diagnostic + .split("times out after ") + .nth(1) + .and_then(|tail| tail.split_once(" ms")) + .and_then(|(millis, _)| millis.parse::().ok()); + if let Some(millis) = measured { + if !(700..2500).contains(&millis) { + return Err( + Report::new(HarnessError::Runtime).attach("terminal budget escaped its tolerance") + ); + } + return Ok(Some(millis)); + } + if adapter != "fastly" + || !diagnostic.contains("retains input: unavailable") + || !(Duration::from_millis(700)..Duration::from_millis(2500)).contains(&lookup_elapsed) + { + return Err( + Report::new(HarnessError::Runtime).attach("missing measured bounded timeout evidence") + ); + } + Ok(None) +} + +fn consent_cookie() -> String { + let mut bits = [0_u8; 29]; + let now = SystemTime::now() + .duration_since(UNIX_EPOCH) + .expect("should have a current fixture clock") + .as_millis() + / 100; + for (offset, width, value) in [ + (0, 6, 2), + (6, 36, now), + (42, 36, now), + (108, 6, 4), + (114, 6, 13), + (152, 1, 1), + ] { + for index in 0..width { + if value & (1 << (width - index - 1)) != 0 { + bits[(offset + index) / 8] |= 1 << (7 - (offset + index) % 8); + } + } + } + format!("euconsent-v2={}", URL_SAFE_NO_PAD.encode(bits)) +} + +struct CapturedBody { + bytes: Vec, + first_byte_at: Instant, + result: std::io::Result<()>, +} + +fn capture_body(reader: &mut impl std::io::Read) -> CapturedBody { + let mut first = [0_u8; 1]; + let result = reader.read_exact(&mut first); + let first_byte_at = Instant::now(); + let mut bytes = if result.is_ok() { + Vec::from(first) + } else { + Vec::new() + }; + let result = result.and_then(|()| reader.read_to_end(&mut bytes).map(|_| ())); + CapturedBody { + bytes, + first_byte_at, + result, + } +} + +#[test] +fn truncated_response_preserves_received_bytes() { + struct TruncatedResponse(usize); + impl std::io::Read for TruncatedResponse { + fn read(&mut self, bytes: &mut [u8]) -> std::io::Result { + if bytes.is_empty() { + return Ok(0); + } + if self.0 >= 2 { + return Err(std::io::Error::from(std::io::ErrorKind::UnexpectedEof)); + } + bytes[0] = b"ab"[self.0]; + self.0 += 1; + Ok(1) + } + } + let captured = capture_body(&mut TruncatedResponse(0)); + assert!( + captured.result.is_err(), + "should report the truncated transfer" + ); + assert_eq!( + captured.bytes, b"ab", + "should retain bytes for failure diagnostics" + ); +} + +#[test] +#[ignore = "requires the selected local runtime and fresh adapter builds"] +fn actual_runtime() -> HarnessResult<()> { + let _ = env_logger::Builder::new() + .filter_level(log::LevelFilter::Info) + .try_init(); + let adapter = std::env::var("HTML_EOF_ADAPTER").change_context(HarnessError::Configuration)?; + if !["axum", "fastly", "cloudflare", "spin"].contains(&adapter.as_str()) { + return Err(Report::new(HarnessError::Configuration)); + } + let release = std::env::var("HTML_EOF_RELEASE").as_deref() == Ok("true"); + let auction = std::env::var("HTML_EOF_AUCTION").as_deref() == Ok("true"); + let origin = origin::Origin::start()?; + let harness = Harness::new(&adapter, &origin)?; + harness.preflight(&adapter)?; + let artifact = harness.build(&adapter, release)?; + harness.run( + harness + .command(harness.target.join("debug/ts")) + .args(["config", "init", "--app-config"]) + .arg(harness.workspace.join("trusted-server.toml")), + "config-init", + )?; + let template = fs::read_to_string(harness.workspace.join("trusted-server.toml")) + .change_context(HarnessError::Io)?; + fs::write( + harness.workspace.join("edgezero.toml"), + config::fixture_manifest( + include_str!("../../../edgezero.toml"), + &harness.root, + &harness.workspace, + )?, + ) + .change_context(HarnessError::Io)?; + if adapter == "spin" { + let mut manifest = + config::parse(include_str!("../../trusted-server-adapter-spin/spin.toml"))?; + config::replace( + &mut manifest, + &["component", "trusted-server", "source"], + toml::Value::String(artifact.to_string_lossy().into_owned()), + )?; + fs::write( + harness.workspace.join("spin.toml"), + config::serialize(&manifest)?, + ) + .change_context(HarnessError::Io)?; + } + let client = Client::builder() + .timeout(Duration::from_secs(10)) + .build() + .change_context(HarnessError::Http)?; + let mut cases = vec!["disabled", "success", "hung"]; + if auction { + cases.push("auction"); + } + if adapter == "fastly" { + cases.push("streaming"); + } + let mut baseline = Vec::new(); + for case in cases { + harness.cancellation.check()?; + let config = config::fixture_config(&template, case, origin.port, available_port()?)?; + fs::write(harness.workspace.join("trusted-server.toml"), config) + .change_context(HarnessError::Io)?; + fs::write(harness.workspace.join("fastly.toml"), "manifest_version = 3\nname = 'example-finalization'\nlanguage = 'rust'\n[local_server]\n").change_context(HarnessError::Io)?; + harness.push("axum")?; + if matches!(adapter.as_str(), "fastly" | "spin") { + harness.push(&adapter)?; + } + let port = available_port()?; + let log = harness.workspace.join(format!("{case}.log")); + let output = File::create(&log).change_context(HarnessError::Io)?; + let mut command = harness.runtime_command(&adapter, &artifact, port)?; + command + .stdout(Stdio::from( + output.try_clone().change_context(HarnessError::Io)?, + )) + .stderr(Stdio::from(output)); + let mut process = ManagedChild::spawn(&mut command, &harness.cancellation)?; + let base_url = format!("http://127.0.0.1:{port}"); + let ready_client = Client::builder() + .timeout(Duration::from_secs(1)) + .build() + .change_context(HarnessError::Http)?; + let mut ready = false; + for _ in 0..80 { + if let Some(status) = process.status()? { + return Err(Report::new(HarnessError::Runtime) + .attach(format!("runtime exits {status}; see {}", log.display()))); + } + if ready_client + .head(format!("{base_url}/health")) + .send() + .is_ok_and(|response| response.status().is_success()) + { + ready = true; + break; + } + thread::sleep(Duration::from_millis(100)); + } + if !ready { + return Err(Report::new(HarnessError::Runtime) + .attach(format!("runtime not ready; see {}", log.display()))); + } + let started = Instant::now(); + let mut request = client + .get(format!("{base_url}/article?case={case}")) + .header("accept", "text/html") + .header("accept-encoding", "identity") + .header("sec-fetch-dest", "document") + .header("sec-fetch-mode", "navigate"); + if case == "auction" { + request = request.header("cookie", consent_cookie()); + } + let mut response = request.send().change_context(HarnessError::Http)?; + let headers = response.headers().clone(); + let status = response.status(); + fs::write( + harness.workspace.join(format!("{case}.headers")), + format!("status={status}\n{headers:?}"), + ) + .change_context(HarnessError::Io)?; + let captured = capture_body(&mut response); + let first_byte_at = captured.first_byte_at; + let body = captured.bytes; + let completed = Instant::now(); + fs::write(harness.workspace.join(format!("{case}.body")), &body) + .change_context(HarnessError::Io)?; + captured + .result + .change_context(HarnessError::Http) + .attach(format!( + "HTTP {status} during {case}; response artifacts in {}", + harness.workspace.display() + ))?; + assert!( + status.is_success(), + "should serve {case} HTML; see {}", + log.display() + ); + match case { + "disabled" => { + assert!( + String::from_utf8_lossy(&body).contains("EXAMPLE_UNSIGNED") + && !String::from_utf8_lossy(&body).contains("EXAMPLE_FINALIZED"), + "should preserve disabled output" + ); + baseline = body.clone(); + } + "success" => { + assert_eq!( + body, + String::from_utf8_lossy(&baseline) + .replace("EXAMPLE_UNSIGNED", "EXAMPLE_FINALIZED") + .as_bytes(), + "should emit the exact approved rewrite" + ); + assert!( + completed.duration_since(started) >= Duration::from_millis(150), + "should perform delayed I/O" + ); + } + "streaming" => { + let mut expected = baseline.clone(); + expected.extend_from_slice(b""); + assert_eq!(body, expected, "should append only the EOF output"); + } + "auction" => { + let body = String::from_utf8_lossy(&body); + assert!( + body.contains("EXAMPLE_FINALIZED") + && body.contains("adSlots") + && body.contains("bids") + && !body.contains("ts-inline-body-close-"), + "should finish EOF work and auction assembly" + ); + } + _ => { + assert_eq!( + body, baseline, + "should emit byte-identical timeout fallback" + ); + assert!( + (Duration::from_millis(600)..Duration::from_secs(3)) + .contains(&completed.duration_since(started)), + "should bound the hung control" + ); + } + } + if case != "disabled" { + assert_eq!( + headers["cache-control"], "no-store, private", + "should prohibit shared caching" + ); + } + let events = origin.events(); + let lookup = origin::lookup_for_request(&events, started, completed, case != "disabled")?; + if let Some(lookup) = lookup { + assert_eq!( + lookup.data["complete"], true, + "should observe complete HTML" + ); + assert_eq!( + lookup.data["sha256"].as_str().map(str::len), + Some(64), + "should hash complete output" + ); + let timeout = lookup.data["transport_timeout_ms"] + .as_u64() + .expect("should report the transport budget"); + if adapter == "fastly" { + assert!( + (1..=750).contains(&timeout), + "should bound blocking host calls" + ); + } else { + assert_eq!( + timeout, 30000, + "should time out before the SDK transport deadline" + ); + } + } + if matches!(case, "success" | "auction" | "streaming") { + let finish = events + .iter() + .find(|event| { + event.kind == "lookup_complete" && event.at >= started && event.at <= completed + }) + .expect("should complete the current lookup"); + if case == "streaming" { + assert!( + first_byte_at + Duration::from_millis(50) < finish.at, + "should deliver actual HTML while EOF I/O remains pending" + ); + } + } + let lookup_elapsed = lookup.map(|event| completed.duration_since(event.at)); + let measured = if case == "hung" { + let mut diagnostic = fs::read_to_string(&log).change_context(HarnessError::Io)?; + if adapter == "spin" { + for path in fs::read_dir(harness.workspace.join("spin-logs")) + .change_context(HarnessError::Io)? + { + let path = path.change_context(HarnessError::Io)?.path(); + if path + .file_name() + .is_some_and(|name| name.to_string_lossy().contains("stderr")) + { + diagnostic + .push_str(&fs::read_to_string(path).change_context(HarnessError::Io)?); + } + } + } + measured_terminal_ms( + &adapter, + &diagnostic, + lookup_elapsed.expect("should measure the hung lookup"), + )? + } else { + None + }; + let timing = json!({"request_started_seconds": started.duration_since(origin.started).as_secs_f64(), "request_seconds": completed.duration_since(started).as_secs_f64(), "first_body_byte_seconds": first_byte_at.duration_since(started).as_secs_f64(), "terminal_ms": measured, "lookup_to_response_seconds": lookup_elapsed.map(|elapsed| elapsed.as_secs_f64()), "configured_budget_ms": 750, "cooperative_timer": adapter != "fastly"}); + fs::write( + harness.workspace.join(format!("{case}.timing.json")), + timing.to_string(), + ) + .change_context(HarnessError::Io)?; + origin.write_events(&harness.workspace.join("events.json"))?; + log::info!("PASS {adapter} {case}: {timing}"); + drop(process); + } + Ok(()) +} + +#[test] +fn viceroy_preflight_rejects_old_or_invalid_versions() { + for version in ["viceroy 0.17.0", "unknown"] { + assert!( + validate_viceroy(version).is_err(), + "should reject {version}" + ); + } + for version in ["viceroy 0.19.0", "viceroy 0.20.1", "viceroy 1.0.0"] { + assert!(validate_viceroy(version).is_ok(), "should accept {version}"); + } +} + +#[test] +fn timeout_proof_requires_measured_evidence() { + for adapter in ["axum", "cloudflare", "spin"] { + assert!( + measured_terminal_ms(adapter, "missing", Duration::from_millis(750)).is_err(), + "should require a timer diagnostic" + ); + assert_eq!( + measured_terminal_ms( + adapter, + "times out after 751 ms", + Duration::from_millis(750) + ) + .expect("should parse measured expiry"), + Some(751), + "should report actual expiry" + ); + } + assert!( + measured_terminal_ms( + "fastly", + "retains input: unavailable", + Duration::from_millis(100) + ) + .is_err(), + "should reject early unmeasured failure" + ); + assert!( + measured_terminal_ms( + "fastly", + "retains input: unavailable", + Duration::from_secs(3) + ) + .is_err(), + "should reject escaped budget" + ); + assert_eq!( + measured_terminal_ms( + "fastly", + "retains input: unavailable", + Duration::from_millis(749) + ) + .expect("should accept measured transport fallback"), + None, + "should not invent a cooperative timer measurement" + ); +} diff --git a/crates/trusted-server-integration-tests/tests/html_eof_runtime/config.rs b/crates/trusted-server-integration-tests/tests/html_eof_runtime/config.rs new file mode 100644 index 000000000..0b848c96b --- /dev/null +++ b/crates/trusted-server-integration-tests/tests/html_eof_runtime/config.rs @@ -0,0 +1,325 @@ +use std::path::Path; + +use error_stack::{Report, ResultExt as _}; +use toml::Value; + +use super::{HarnessError, HarnessResult}; + +pub(super) const SECRETS: &[(&str, &str)] = &[ + ("handler_password", "example-handler-password-32-bytes-ok"), + ( + "publisher_proxy_secret", + "example-publisher-secret-32-bytes-ok", + ), + ("ec_passphrase", "example-cookie-passphrase-32-bytes-ok"), +]; + +fn invalid(field: &str) -> Report { + Report::new(HarnessError::Configuration) + .attach(format!("missing or invalid template field: {field}")) +} + +pub(super) fn parse(template: &str) -> HarnessResult { + toml::from_str(template).change_context(HarnessError::Configuration) +} + +pub(super) fn serialize(value: &Value) -> HarnessResult { + toml::to_string(value).change_context(HarnessError::Configuration) +} + +pub(super) fn replace(value: &mut Value, path: &[&str], replacement: Value) -> HarnessResult<()> { + let mut field = value; + for key in path { + field = field + .get_mut(*key) + .ok_or_else(|| invalid(&path.join(".")))?; + } + if field.type_str() != replacement.type_str() { + return Err(invalid(&path.join("."))); + } + *field = replacement; + Ok(()) +} + +fn table<'a>( + value: &'a mut Value, + path: &[&str], +) -> HarnessResult<&'a mut toml::map::Map> { + let mut field = value; + for key in path { + field = field + .get_mut(*key) + .ok_or_else(|| invalid(&path.join(".")))?; + } + field.as_table_mut().ok_or_else(|| invalid(&path.join("."))) +} + +pub(super) fn fixture_config( + template: &str, + case: &str, + origin: u16, + bid: u16, +) -> HarnessResult { + let mut value = parse(template)?; + for (key, replacement) in [ + ("domain", "127.0.0.1".to_owned()), + ("cookie_domain", "127.0.0.1".to_owned()), + ("origin_url", format!("http://127.0.0.1:{origin}")), + ] { + replace(&mut value, &["publisher", key], Value::String(replacement))?; + } + let publisher = table(&mut value, &["publisher"])?; + publisher.insert( + "allow_async_html_finish".to_owned(), + Value::Boolean(case != "disabled"), + ); + publisher.insert( + "html_finalization_timeout_ms".to_owned(), + Value::Integer(750), + ); + replace( + &mut value, + &["auction", "enabled"], + Value::Boolean(case == "auction"), + )?; + if case == "auction" { + let providers = table(&mut value, &["auction", "providers"])?; + if providers.is_empty() { + return Err(invalid("auction.providers")); + } + for (_, provider) in providers.iter_mut() { + replace( + provider, + &["endpoint"], + Value::String(format!("https://127.0.0.1:{bid}/bid")), + )?; + } + let slot = parse( + "id = 'example-slot'\npage_patterns = ['/article']\nformats = [{ width = 300, height = 250 }]\n", + )?; + table(&mut value, &["creative_opportunities"])? + .insert("slot".to_owned(), Value::Array(vec![slot])); + } + let lookup = if matches!(case, "auction" | "streaming") { + "success" + } else { + case + }; + let fixture = parse(&format!( + "enabled = {}\nlookup_url = 'http://127.0.0.1:{origin}/{lookup}'\ntransport_timeout_ms = 30000\nstreaming = {}\n", + case != "disabled", + case == "streaming" + ))?; + table(&mut value, &["integrations"])?.insert("html_finalization_fixture".to_owned(), fixture); + validate_fixture(&value, case, origin, bid)?; + serialize(&value) +} + +pub(super) fn validate_fixture( + value: &Value, + case: &str, + origin: u16, + bid: u16, +) -> HarnessResult<()> { + let lookup = if matches!(case, "auction" | "streaming") { + "success" + } else { + case + }; + let required = [ + ( + vec!["publisher", "domain"], + Value::String("127.0.0.1".to_owned()), + ), + ( + vec!["publisher", "cookie_domain"], + Value::String("127.0.0.1".to_owned()), + ), + ( + vec!["publisher", "origin_url"], + Value::String(format!("http://127.0.0.1:{origin}")), + ), + ( + vec!["publisher", "allow_async_html_finish"], + Value::Boolean(case != "disabled"), + ), + ( + vec!["publisher", "html_finalization_timeout_ms"], + Value::Integer(750), + ), + ( + vec!["integrations", "html_finalization_fixture", "enabled"], + Value::Boolean(case != "disabled"), + ), + ( + vec!["integrations", "html_finalization_fixture", "lookup_url"], + Value::String(format!("http://127.0.0.1:{origin}/{lookup}")), + ), + ( + vec![ + "integrations", + "html_finalization_fixture", + "transport_timeout_ms", + ], + Value::Integer(30000), + ), + ( + vec!["integrations", "html_finalization_fixture", "streaming"], + Value::Boolean(case == "streaming"), + ), + ( + vec!["auction", "enabled"], + Value::Boolean(case == "auction"), + ), + ]; + for (path, expected) in required { + let actual = path.iter().try_fold(value, |field, key| field.get(*key)); + if actual != Some(&expected) { + return Err(invalid(&path.join("."))); + } + } + if case == "auction" { + let providers = value + .get("auction") + .and_then(|v| v.get("providers")) + .and_then(Value::as_table) + .ok_or_else(|| invalid("auction.providers"))?; + if providers.is_empty() + || providers.values().any(|provider| { + provider.get("endpoint").and_then(Value::as_str) + != Some(format!("https://127.0.0.1:{bid}/bid").as_str()) + }) + { + return Err(invalid("auction endpoint")); + } + } + Ok(()) +} + +pub(super) fn fixture_manifest( + template: &str, + root: &Path, + workspace: &Path, +) -> HarnessResult { + let mut value = parse(template)?; + let entry = value + .get("app") + .and_then(|v| v.get("entry")) + .and_then(Value::as_str) + .ok_or_else(|| invalid("app.entry"))?; + let entry = root.join(entry); + replace( + &mut value, + &["app", "entry"], + Value::String(entry.to_string_lossy().into_owned()), + )?; + for (name, adapter) in table(&mut value, &["adapters"])? { + let adapter = table(adapter, &["adapter"])?; + for key in ["crate", "manifest"] { + let old = adapter + .get(key) + .and_then(Value::as_str) + .ok_or_else(|| invalid(key))?; + let path = if key == "manifest" && matches!(name.as_str(), "fastly" | "spin") { + workspace.join(format!("{name}.toml")) + } else { + root.join(old) + }; + adapter.insert( + key.to_owned(), + Value::String(path.to_string_lossy().into_owned()), + ); + } + } + serialize(&value) +} + +#[cfg(test)] +mod tests { + use super::*; + + const CONFIG: &str = include_str!("../../../../trusted-server.example.toml"); + const MANIFEST: &str = include_str!("../../../../edgezero.toml"); + + #[test] + fn fixture_config_is_loopback_only_with_explicit_permission() { + for case in ["disabled", "success", "hung", "auction", "streaming"] { + let config = fixture_config(CONFIG, case, 12345, 12346) + .expect("should generate isolated configuration"); + let parsed: toml::Value = toml::from_str(&config).expect("should parse configuration"); + assert_eq!( + parsed["publisher"]["domain"].as_str(), + Some("127.0.0.1"), + "should isolate the publisher" + ); + assert_eq!( + parsed["publisher"]["origin_url"].as_str(), + Some("http://127.0.0.1:12345"), + "should isolate the origin" + ); + assert_eq!( + parsed["publisher"]["allow_async_html_finish"].as_bool(), + Some(case != "disabled"), + "should set explicit permission" + ); + assert_eq!( + parsed["publisher"]["html_finalization_timeout_ms"].as_integer(), + Some(750), + "should configure the fixture budget" + ); + } + } + + #[test] + fn fixture_config_rejects_missing_required_template_fields() { + let result = fixture_config( + "[publisher]\ndomain = 'example.com'", + "success", + 12345, + 12346, + ); + assert!(result.is_err(), "should reject incomplete templates"); + } + + #[test] + fn fixture_validation_rejects_external_endpoints() { + let config = fixture_config(CONFIG, "auction", 12345, 12346) + .expect("should generate auction configuration"); + let mut value = parse(&config).expect("should parse generated configuration"); + let provider = table(&mut value, &["auction", "providers"]) + .expect("should contain a provider") + .iter_mut() + .next() + .expect("should have a provider") + .1; + replace( + provider, + &["endpoint"], + Value::String("https://auction.example.com/bid".to_owned()), + ) + .expect("should change the controlled endpoint"); + assert!( + validate_fixture(&value, "auction", 12345, 12346).is_err(), + "should reject non-loopback auction endpoints" + ); + } + + #[test] + fn fixture_manifest_round_trips_paths() { + let root = Path::new("/tmp/example 🚀 \"quoted\"\\path\nnext"); + let workspace = root.join("workspace"); + let manifest = + fixture_manifest(MANIFEST, root, &workspace).expect("should isolate manifests"); + let parsed: toml::Value = toml::from_str(&manifest).expect("should parse quoted paths"); + assert_eq!( + parsed["app"]["entry"].as_str(), + root.join("crates/trusted-server-core").to_str(), + "should preserve all path characters" + ); + assert_eq!( + parsed["adapters"]["spin"]["adapter"]["manifest"].as_str(), + workspace.join("spin.toml").to_str(), + "should use an isolated runtime manifest" + ); + } +} diff --git a/crates/trusted-server-integration-tests/tests/html_eof_runtime/origin.rs b/crates/trusted-server-integration-tests/tests/html_eof_runtime/origin.rs new file mode 100644 index 000000000..828a6e8fd --- /dev/null +++ b/crates/trusted-server-integration-tests/tests/html_eof_runtime/origin.rs @@ -0,0 +1,257 @@ +use std::fs; +use std::io; +use std::net::{Ipv4Addr, TcpListener}; +use std::path::Path; +use std::sync::{Arc, Mutex}; +use std::time::{Duration, Instant}; + +use axum::body::Body; +use axum::extract::{DefaultBodyLimit, OriginalUri, State}; +use axum::http::StatusCode; +use axum::response::Response; +use axum::routing::{get, post}; +use axum::{Json, Router}; +use bytes::Bytes; +use error_stack::{Report, ResultExt as _}; +use futures::stream; +use serde_json::{Value, json}; +use tokio::runtime::Runtime; +use tokio::task::JoinHandle; + +use super::{HarnessError, HarnessResult}; + +const PREFIX: &[u8] = + b"

EXAMPLE_UNSIGNED

"; +const EOF: &[u8] = b""; + +#[derive(Clone)] +pub(super) struct Event { + pub kind: &'static str, + pub at: Instant, + pub data: Value, +} + +#[derive(Clone, Default)] +struct Events(Arc>>); + +impl Events { + fn record(&self, kind: &'static str, data: Value) { + self.0 + .lock() + .expect("should lock fixture events") + .push(Event { + kind, + at: Instant::now(), + data, + }); + } +} + +pub(super) struct Origin { + pub port: u16, + pub started: Instant, + pub runtime: Runtime, + events: Events, + server: JoinHandle<()>, +} + +impl Origin { + pub(super) fn start() -> HarnessResult { + let started = Instant::now(); + let listener = + TcpListener::bind((Ipv4Addr::LOCALHOST, 0)).change_context(HarnessError::Io)?; + let port = listener + .local_addr() + .change_context(HarnessError::Io)? + .port(); + listener + .set_nonblocking(true) + .change_context(HarnessError::Io)?; + let runtime = Runtime::new().change_context(HarnessError::Runtime)?; + let _entered = runtime.enter(); + let listener = + tokio::net::TcpListener::from_std(listener).change_context(HarnessError::Io)?; + let events = Events::default(); + let app = Router::new() + .route("/health", axum::routing::head(|| async { StatusCode::OK })) + .route("/article", get(article)) + .route("/success", post(lookup)) + .route("/hung", post(lookup)) + .layer(DefaultBodyLimit::max(2048)) + .with_state(events.clone()); + let server = runtime.spawn(async move { + if let Err(error) = axum::serve(listener, app).await { + log::error!("EOF origin fails: {error}"); + } + }); + drop(_entered); + Ok(Self { + port, + started, + runtime, + events, + server, + }) + } + + pub(super) fn events(&self) -> Vec { + self.events + .0 + .lock() + .expect("should lock fixture events") + .clone() + } + + pub(super) fn write_events(&self, path: &Path) -> HarnessResult<()> { + let events = self.events().iter().map(|event| json!({ + "event": event.kind, + "seconds_from_origin_start": event.at.duration_since(self.started).as_secs_f64(), + "data": event.data, + })).collect::>(); + fs::write(path, json!(events).to_string()).change_context(HarnessError::Io) + } +} + +impl Drop for Origin { + fn drop(&mut self) { + // Runtime teardown cancels pending lookup handlers, including the hung control. + self.server.abort(); + } +} + +async fn article(State(events): State) -> Response { + let body = stream::unfold(0, move |index| { + let events = events.clone(); + async move { + let bytes = match index { + 0 => Bytes::from_static(PREFIX), + 1 => { + tokio::time::sleep(Duration::from_millis(50)).await; + events.record("origin_eof", json!({})); + Bytes::from_static(EOF) + } + _ => return None, + }; + Some((Ok::<_, io::Error>(bytes), index + 1)) + } + }); + Response::builder() + .header("content-type", "text/html") + .header("cache-control", "public, max-age=60") + .header("content-length", PREFIX.len() + EOF.len()) + .body(Body::from_stream(body)) + .expect("should build the controlled origin response") +} + +async fn lookup( + State(events): State, + OriginalUri(uri): OriginalUri, + Json(mut payload): Json, +) -> &'static str { + payload["path"] = Value::String(uri.path().to_owned()); + events.record("lookup_start", payload); + let delay = if uri.path() == "/hung" { + Duration::from_secs(30) + } else { + Duration::from_millis(150) + }; + tokio::time::sleep(delay).await; + events.record("lookup_complete", json!({"path": uri.path()})); + "EXAMPLE_APPROVED" +} + +pub(super) fn lookup_for_request( + events: &[Event], + started: Instant, + completed: Instant, + enabled: bool, +) -> HarnessResult> { + let lookups: Vec<_> = events + .iter() + .filter(|event| { + event.kind == "lookup_start" && started <= event.at && event.at <= completed + }) + .collect(); + if lookups.len() != usize::from(enabled) { + return Err(Report::new(HarnessError::Runtime) + .attach("each request must issue exactly one enabled EOF lookup")); + } + let Some(lookup) = lookups.first().copied() else { + return Ok(None); + }; + if !events + .iter() + .any(|event| event.kind == "origin_eof" && started <= event.at && event.at <= lookup.at) + { + return Err(Report::new(HarnessError::Runtime) + .attach("EOF lookup starts before its own origin EOF")); + } + Ok(Some(lookup)) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn earlier_request_eof_cannot_satisfy_lookup_ordering() { + let start = Instant::now(); + let event = |kind, millis| Event { + kind, + at: start + Duration::from_millis(millis), + data: json!({}), + }; + let events = [ + event("origin_eof", 1), + event("lookup_start", 20), + event("origin_eof", 25), + ]; + assert!( + lookup_for_request( + &events, + start + Duration::from_millis(10), + start + Duration::from_millis(30), + true + ) + .is_err(), + "should reject an EOF from an earlier request" + ); + } + + #[test] + fn request_scoped_lookup_requires_exactly_one_enabled_call() { + let start = Instant::now(); + let events = [ + Event { + kind: "origin_eof", + at: start, + data: json!({}), + }, + Event { + kind: "lookup_start", + at: start, + data: json!({}), + }, + ]; + assert!( + lookup_for_request(&events, start, start, true) + .expect("should match the current EOF") + .is_some(), + "should return the current lookup" + ); + assert!( + lookup_for_request(&events, start, start, false).is_err(), + "should reject lookup with disabled capability" + ); + assert!( + lookup_for_request(&[], start, start, true).is_err(), + "should reject missing lookup" + ); + assert!( + lookup_for_request(&[], start, start, false) + .expect("should accept disabled work") + .is_none(), + "should accept no disabled lookup" + ); + } +} diff --git a/crates/trusted-server-integration-tests/tests/html_eof_runtime/process.rs b/crates/trusted-server-integration-tests/tests/html_eof_runtime/process.rs new file mode 100644 index 000000000..642db0f86 --- /dev/null +++ b/crates/trusted-server-integration-tests/tests/html_eof_runtime/process.rs @@ -0,0 +1,445 @@ +use std::collections::{BTreeMap, BTreeSet}; +use std::ffi::OsString; +use std::fs::File; +use std::path::Path; +use std::process::{Child, Command, ExitStatus, Stdio}; +use std::sync::atomic::{AtomicBool, Ordering}; +use std::sync::{Arc, Mutex}; +use std::thread; +use std::time::{Duration, Instant}; + +#[cfg(unix)] +use std::os::unix::process::CommandExt as _; + +use error_stack::{Report, ResultExt as _}; +use tokio::runtime::Handle; + +use super::{HarnessError, HarnessResult}; + +pub(super) fn build_environment( + source: impl IntoIterator, +) -> BTreeMap { + source + .into_iter() + .filter(|(name, _)| { + let Some(name) = name.to_str() else { + return false; + }; + name != "CARGO_BUILD_TARGET" + && (matches!( + name, + "PATH" + | "RUSTUP_HOME" + | "CARGO_HOME" + | "CARGO_TARGET_DIR" + | "TMPDIR" + | "WASM_BINDGEN_BIN" + | "WASM_OPT_BIN" + | "ESBUILD_BIN" + | "CARGO_NET_OFFLINE" + | "RUSTFLAGS" + | "CARGO_ENCODED_RUSTFLAGS" + | "RUSTDOCFLAGS" + | "CARGO_INCREMENTAL" + | "RUSTC" + | "RUSTDOC" + | "RUSTC_WRAPPER" + | "RUSTC_WORKSPACE_WRAPPER" + | "TSJS_PREBUILT_DIR" + ) || name.starts_with("CARGO_PROFILE_") + || name.starts_with("CARGO_BUILD_") + || name.starts_with("CARGO_TARGET_")) + }) + .collect() +} + +#[derive(Clone, Default)] +pub(super) struct Cancellation { + cancelled: Arc, + groups: Arc>>, +} + +impl Cancellation { + pub(super) fn check(&self) -> HarnessResult<()> { + if self.cancelled.load(Ordering::SeqCst) { + Err(Report::new(HarnessError::Cancelled)) + } else { + Ok(()) + } + } + + fn cancel(&self) { + self.cancelled.store(true, Ordering::SeqCst); + for id in self + .groups + .lock() + .expect("should lock owned process groups") + .iter() + { + signal_group(*id, libc::SIGTERM); + } + } + + pub(super) fn watch(&self, handle: &Handle) -> HarnessResult<()> { + #[cfg(unix)] + { + let _entered = handle.enter(); + let mut term = + tokio::signal::unix::signal(tokio::signal::unix::SignalKind::terminate()) + .change_context(HarnessError::Runtime)?; + let mut interrupt = + tokio::signal::unix::signal(tokio::signal::unix::SignalKind::interrupt()) + .change_context(HarnessError::Runtime)?; + // Cargo owns the test process. Detect its disappearance as well as + // direct signals so terminating the shell entry point leaves no runtimes. + let parent = unsafe { libc::getppid() }; + let cancellation = self.clone(); + handle.spawn(async move { + loop { + tokio::select! { + _ = term.recv() => break, + _ = interrupt.recv() => break, + _ = tokio::time::sleep(Duration::from_millis(100)) => { + if unsafe { libc::getppid() } != parent { break; } + } + } + } + cancellation.cancel(); + tokio::time::sleep(Duration::from_secs(5)).await; + for id in cancellation + .groups + .lock() + .expect("should lock owned process groups") + .iter() + { + signal_group(*id, libc::SIGKILL); + } + }); + } + #[cfg(not(unix))] + { + let _ = handle; + return Err(Report::new(HarnessError::Runtime) + .attach("EOF runtime controls require Unix process groups")); + } + Ok(()) + } +} + +fn signal_group(id: u32, signal: i32) { + #[cfg(unix)] + if let Ok(id) = libc::pid_t::try_from(id) { + // Every registered ID belongs to a child started in its own process group. + unsafe { + libc::killpg(id, signal); + } + } + #[cfg(not(unix))] + let _ = (id, signal); +} + +pub(super) struct ManagedChild { + child: Child, + cancellation: Cancellation, +} + +impl ManagedChild { + pub(super) fn spawn(command: &mut Command, cancellation: &Cancellation) -> HarnessResult { + cancellation.check()?; + #[cfg(unix)] + command.process_group(0); + let child = command.spawn().change_context(HarnessError::Runtime)?; + cancellation + .groups + .lock() + .expect("should lock owned process groups") + .insert(child.id()); + Ok(Self { + child, + cancellation: cancellation.clone(), + }) + } + + pub(super) fn status(&mut self) -> HarnessResult> { + self.cancellation.check()?; + self.child.try_wait().change_context(HarnessError::Runtime) + } + + pub(super) fn wait(&mut self) -> HarnessResult { + loop { + if let Some(status) = self.status()? { + return Ok(status); + } + thread::sleep(Duration::from_millis(50)); + } + } +} + +impl Drop for ManagedChild { + fn drop(&mut self) { + let id = self.child.id(); + signal_group(id, libc::SIGTERM); + let until = Instant::now() + Duration::from_secs(5); + while matches!(self.child.try_wait(), Ok(None)) && Instant::now() < until { + thread::sleep(Duration::from_millis(20)); + } + signal_group(id, libc::SIGKILL); + #[cfg(not(unix))] + let _ = self.child.kill(); + let _ = self.child.wait(); + self.cancellation + .groups + .lock() + .expect("should lock owned process groups") + .remove(&id); + } +} + +pub(super) fn logged_command( + command: &mut Command, + log: &Path, + cancellation: &Cancellation, +) -> HarnessResult<()> { + let output = File::create(log).change_context(HarnessError::Io)?; + command + .stdout(Stdio::from( + output.try_clone().change_context(HarnessError::Io)?, + )) + .stderr(Stdio::from(output)); + let status = ManagedChild::spawn(command, cancellation)?.wait()?; + if !status.success() { + return Err(Report::new(HarnessError::Runtime).attach(format!( + "command exits with {status}; see {}", + log.display() + ))); + } + Ok(()) +} + +#[cfg(test)] +mod tests { + use super::*; + use std::fs; + use std::path::PathBuf; + + #[test] + fn compiler_settings_are_preserved_without_operator_settings() { + let source = [ + ("PATH", "/tmp/example-bin"), + ("RUSTFLAGS", "-D warnings"), + ("CARGO_PROFILE_DEV_DEBUG", "0"), + ("CARGO_INCREMENTAL", "0"), + ("CARGO_TARGET_DIR", "/tmp/example-target"), + ("CARGO_BUILD_TARGET", "wasm32-wasip1"), + ("TRUSTED_SERVER__PUBLISHER__DOMAIN", "example.com"), + ("TSJS_SKIP_BUILD", "1"), + ]; + let selected = build_environment( + source.map(|(name, value)| (OsString::from(name), OsString::from(value))), + ); + for name in [ + "PATH", + "RUSTFLAGS", + "CARGO_PROFILE_DEV_DEBUG", + "CARGO_INCREMENTAL", + "CARGO_TARGET_DIR", + ] { + assert!( + selected.contains_key(&OsString::from(name)), + "should preserve {name}" + ); + } + for name in [ + "CARGO_BUILD_TARGET", + "TRUSTED_SERVER__PUBLISHER__DOMAIN", + "TSJS_SKIP_BUILD", + ] { + assert!( + !selected.contains_key(&OsString::from(name)), + "should isolate {name}" + ); + } + } + + #[test] + #[cfg(unix)] + fn owned_process_is_reaped_on_drop() { + let cancellation = Cancellation::default(); + let mut command = Command::new("sleep"); + command.arg("300"); + let child = + ManagedChild::spawn(&mut command, &cancellation).expect("should launch an owned child"); + let id = child.child.id(); + drop(child); + assert_eq!( + unsafe { libc::kill(libc::pid_t::try_from(id).expect("should fit pid"), 0) }, + -1, + "should reap the terminated process" + ); + assert!( + cancellation + .groups + .lock() + .expect("should lock groups") + .is_empty(), + "should release ownership" + ); + } + + #[test] + #[cfg(unix)] + fn completed_process_is_safe_to_drop() { + let cancellation = Cancellation::default(); + let child = ManagedChild::spawn(&mut Command::new("true"), &cancellation) + .expect("should launch a completed command"); + let mut child = child; + assert!( + child.wait().expect("should wait for completion").success(), + "should complete normally" + ); + drop(child); + assert!( + cancellation + .groups + .lock() + .expect("should lock groups") + .is_empty(), + "should release completed ownership" + ); + } + + #[test] + #[cfg(unix)] + fn sigterm_cleans_owned_descendants() { + signal_cleanup(libc::SIGTERM); + } + + #[test] + #[cfg(unix)] + fn sigint_cleans_owned_descendants() { + signal_cleanup(libc::SIGINT); + } + + #[cfg(unix)] + fn signal_cleanup(signal: i32) { + let directory = tempfile::tempdir().expect("should create probe workspace"); + let cancellation = Cancellation::default(); + let executable = std::env::current_exe().expect("should locate Rust probe executable"); + let mut command = Command::new(executable); + command + .args([ + "--exact", + "process::tests::cleanup_owner_probe", + "--ignored", + ]) + .env("HTML_EOF_PROBE_DIR", directory.path()) + .stdout(Stdio::null()) + .stderr(Stdio::null()); + let mut parent = ManagedChild::spawn(&mut command, &cancellation) + .expect("should launch the owner probe"); + let ready = directory.path().join("owner.ready"); + let until = Instant::now() + Duration::from_secs(5); + while !ready.exists() { + assert!( + parent + .status() + .expect("should inspect the owner probe") + .is_none(), + "should keep the probe alive until ready" + ); + assert!(Instant::now() < until, "should start the cleanup probe"); + thread::sleep(Duration::from_millis(20)); + } + let leaf: libc::pid_t = fs::read_to_string(&ready) + .expect("should read owned PID") + .parse() + .expect("should parse owned PID"); + assert_eq!( + unsafe { + libc::kill( + libc::pid_t::try_from(parent.child.id()).expect("should fit pid"), + signal, + ) + }, + 0, + "should signal the owner" + ); + let until = Instant::now() + Duration::from_secs(15); + let status = loop { + if let Some(status) = parent.status().expect("should inspect cleanup progress") { + break status; + } + assert!( + Instant::now() < until, + "should finish cancellation cleanup within its deadline" + ); + thread::sleep(Duration::from_millis(20)); + }; + assert!(status.success(), "should complete cancellation cleanup"); + assert!( + directory.path().join("cleaned").exists(), + "should unwind owned-process guards" + ); + assert_eq!( + unsafe { libc::kill(leaf, 0) }, + -1, + "should reap the stubborn descendant" + ); + } + + #[test] + #[ignore = "internal subprocess probe, selected only by cleanup tests"] + #[cfg(unix)] + fn cleanup_leaf_probe() { + let directory = PathBuf::from( + std::env::var_os("HTML_EOF_PROBE_DIR").expect("should supply probe workspace"), + ); + unsafe { + libc::signal(libc::SIGTERM, libc::SIG_IGN); + } + fs::write(directory.join("leaf.ready"), "ready") + .expect("should report installed signal handler"); + thread::sleep(Duration::from_secs(300)); + } + + #[test] + #[ignore = "internal subprocess probe, selected only by cleanup tests"] + #[cfg(unix)] + fn cleanup_owner_probe() { + let directory = PathBuf::from( + std::env::var_os("HTML_EOF_PROBE_DIR").expect("should supply probe workspace"), + ); + let runtime = tokio::runtime::Runtime::new().expect("should create signal runtime"); + let cancellation = Cancellation::default(); + cancellation + .watch(runtime.handle()) + .expect("should watch cancellation"); + let mut command = Command::new(std::env::current_exe().expect("should locate leaf probe")); + command + .args(["--exact", "process::tests::cleanup_leaf_probe", "--ignored"]) + .env("HTML_EOF_PROBE_DIR", &directory) + .stdout(Stdio::null()) + .stderr(Stdio::null()); + let mut leaf = ManagedChild::spawn(&mut command, &cancellation) + .expect("should spawn the stubborn descendant"); + let until = Instant::now() + Duration::from_secs(5); + while !directory.join("leaf.ready").exists() { + assert!( + Instant::now() < until, + "should start the stubborn descendant" + ); + thread::sleep(Duration::from_millis(20)); + } + fs::write(directory.join("owner.ready"), leaf.child.id().to_string()) + .expect("should report owned descendant"); + let result = leaf + .wait() + .expect_err("should interrupt an active owned command"); + assert!( + matches!(result.current_context(), HarnessError::Cancelled), + "should report cancellation" + ); + drop(leaf); + fs::write(directory.join("cleaned"), "cleaned").expect("should report completed cleanup"); + } +} diff --git a/docs/guide/integration-guide.md b/docs/guide/integration-guide.md index f0f317522..5ece3878a 100644 --- a/docs/guide/integration-guide.md +++ b/docs/guide/integration-guide.md @@ -370,7 +370,7 @@ end; neighboring bytes alone cannot establish a marker's parsing context. A fictional HTTP processor is available only with `test-utils`. See [HTML EOF runtime checks](/guide/testing#html-eof-runtime-checks) for the runtime -harness, Python/tooling prerequisites and CI coverage. +harness, runtime/tooling prerequisites and CI coverage. ## Browser and script guards diff --git a/docs/guide/testing.md b/docs/guide/testing.md index 8c924c6cc..61e2802eb 100644 --- a/docs/guide/testing.md +++ b/docs/guide/testing.md @@ -46,13 +46,13 @@ in `crates/trusted-server-core/src/proxy.rs`. The EOF harness builds fresh `test-utils` artifacts, creates isolated configuration and runs disabled, successful, hung and enabled-auction controls against a loopback -origin. It requires Python **3.11 or newer**, Rust and pinned Node. Compiler/profile +origin. It uses the native Rust integration-test target and requires Rust and pinned Node. Compiler/profile settings and an explicit `CARGO_TARGET_DIR` are preserved; operator configuration and `CARGO_BUILD_TARGET` are not inherited because the harness selects native and adapter targets. Logs, bodies, headers and timing JSON remain in its printed temporary directory. Owned process groups are stopped on normal exit, failure and SIGTERM. ```bash -python3 -m unittest discover -s scripts/fixtures -p 'test_async_html_finalization_runtime.py' -v +cargo test --manifest-path crates/trusted-server-integration-tests/Cargo.toml --features html-eof-runtime --test html_eof_runtime ./scripts/async-html-finalization-local-test.sh axum --auction ./scripts/async-html-finalization-local-test.sh cloudflare --release --auction ./scripts/async-html-finalization-local-test.sh spin --release --auction @@ -70,9 +70,9 @@ cargo install viceroy --version 0.19.0 --locked --root /tmp/example-eof-viceroy PATH="/tmp/example-eof-viceroy/bin:$PATH" ./scripts/async-html-finalization-local-test.sh fastly --release --auction ``` -The harness checks this prerequisite before building. Fastly additionally proves +The harness checks this prerequisite before building adapter fixtures. Fastly additionally proves that an actual body byte arrives before a non-retaining EOF lookup completes. -CI runs Axum, process-cleanup checks and an isolated Fastly EOF job. Workers/Spin +CI runs Axum, native Rust helper tests and lint, and an isolated Fastly EOF job. Workers/Spin actual EOF timeout controls remain local commands; their target builds, native tests and shared parity controls run in CI. Hosted results are separate from local runs. diff --git a/docs/superpowers/plans/2026-10-12-1241-async-html-finalization.md b/docs/superpowers/plans/2026-10-12-1241-async-html-finalization.md index 09a20611b..4c5da824e 100644 --- a/docs/superpowers/plans/2026-10-12-1241-async-html-finalization.md +++ b/docs/superpowers/plans/2026-10-12-1241-async-html-finalization.md @@ -6,7 +6,7 @@ **Architecture:** Keep synchronous chunk processing and one optional `StreamProcessor::async_finish` callback. Complete each stage before downstream EOF, stream ordinary output, retain output only on explicit declaration, and finish auction/encoding afterward. Limits apply per holder; this is not a total request-memory guarantee. -**Tech Stack:** Rust 2024, futures local futures, error-stack, lol_html, adapter timers, Python 3.11+ and GitHub Actions. +**Tech Stack:** Rust 2024, futures local futures, error-stack, lol_html, adapter timers, native Rust integration tests and GitHub Actions. **Design:** [Ordered asynchronous HTML processor completion](../specs/2026-10-12-1241-async-html-finalization-design.md). @@ -31,10 +31,10 @@ ### 2. Harness configuration, build settings and termination -**Files:** `scripts/fixtures/async-html-finalization-runtime.py`, `scripts/fixtures/test_async_html_finalization_runtime.py`. +**Files:** `crates/trusted-server-integration-tests/tests/html_eof_runtime.rs` and `tests/html_eof_runtime/{config,origin,process}.rs`. - [x] Add failing tests for retained compiler/profile settings, missing template matches, invalid generated configuration, unsupported Viceroy and parent SIGTERM cleanup. -- [x] Run `python3 -m unittest discover -s scripts/fixtures -p 'test_async_html_finalization_runtime.py' -v` and record failures before fixes. +- [x] Run `cargo test --manifest-path crates/trusted-server-integration-tests/Cargo.toml --features html-eof-runtime --test html_eof_runtime` and record failures before fixes. - [x] Preserve explicit Rust build settings without inheriting operator configuration; check template replacements and parsed timeout/permission/loopback targets before configuration push. - [x] Safely quote TOML paths, validate Fastly tooling before builds, unwind on SIGTERM, and require cooperative timeout diagnostics. - [x] Repeat unit/real-process tests and all four actual-runtime commands in `scripts/README.md`. @@ -52,7 +52,7 @@ **Files:** `.github/workflows/test.yml`, `AGENTS.md`, `scripts/README.md`, `docs/guide/testing.md`, configuration/integration guides, canonical spec/plan. - [x] Add isolated Viceroy 0.19 Fastly EOF controls to CI, preserving pinned 0.17 ordinary tests. Cache the isolated tool and retain failure artifacts. -- [x] Document Python/runtime prerequisites, exact cleanup/Axum/Fastly commands and compiler setting preservation. Validate workflow syntax and the actual commands locally. +- [x] Document Rust/runtime prerequisites, exact cleanup/Axum/Fastly commands and compiler setting preservation. Validate workflow syntax and the actual commands locally. - [x] Explain per-holder memory limits, tail latency, Fastly consumer transport bounds and the configured timeout policy. Add a minimal Rust callback example and environment override documentation. - [x] Keep the current spec and plan; remove superseded archived designs, hardening logs and execution histories. Preserve executed evidence in the PR description. - [x] State the deferred processed-template cache requirement explicitly in the draft PR, and document new required public fields. @@ -75,6 +75,20 @@ **Publication:** After validation, commit and push to the existing draft PR. Confirm its remote head and report hosted CI separately from local results. +### 7. Native Rust runtime harness + +**Files:** `crates/trusted-server-integration-tests/tests/html_eof_runtime.rs`, +`tests/html_eof_runtime/{config,origin,process}.rs`, integration-test manifest, +shell entry point, workflow, verification guides and this plan. + +- [x] Add a feature-gated Rust test target so the existing integration CI selection cannot accidentally launch external runtime controls. +- [x] Port configuration, compiler environment, event ordering, timing and real process cleanup regressions; observe failures before completing helpers. +- [x] Use typed TOML edits and validated loopback configuration; an Axum origin records monotonic events in process and streams delayed EOF. +- [x] Own spawned process groups with bounded TERM/KILL/wait cleanup and cancellation handling, including active build commands and runtime descendants. +- [x] Preserve all four adapter controls and Fastly early-body proof through the existing shell command, selecting the host target explicitly. +- [x] Remove the three added Python files and their CI/tooling prerequisites; use Rust helper tests and lint gates instead. +- [x] Review this replacement independently, run the full gate matrix and all four actual runtime controls, then update the existing draft PR. + ## Acceptance mapping | Requirement | Coverage | diff --git a/docs/superpowers/specs/2026-10-12-1241-async-html-finalization-design.md b/docs/superpowers/specs/2026-10-12-1241-async-html-finalization-design.md index 98dbb659c..7fcd4f1ef 100644 --- a/docs/superpowers/specs/2026-10-12-1241-async-html-finalization-design.md +++ b/docs/superpowers/specs/2026-10-12-1241-async-html-finalization-design.md @@ -224,3 +224,17 @@ The fictional fixture demonstrates generic EOF HTTP I/O, not a production signin integration. Verification records identify checked source, real runtimes, hosted CI, expected skips and tooling limits separately. Historical evidence does not prove this revised implementation. + +## 11. Native Rust verification tooling + +Keep EOF runtime verification in the existing Rust integration-test crate. A +feature-gated native test target owns its loopback origin, request-scoped monotonic +events and external runtime process groups. An ignored test runs the selected +adapter; normal helper tests need no adapter tools. The shell entry point selects +the adapter/profile and invokes Cargo with the explicit native target. + +Preserve disabled/success/hung/auction controls on all four adapters and actual +early-body delivery on Fastly. Validate typed, isolated configuration before use, +retain diagnostics, preserve compiler/profile settings, and clean process groups +on normal return, failure, SIGTERM and SIGINT. Use existing workspace dependencies +only for tests. No added verification path requires Python. diff --git a/scripts/README.md b/scripts/README.md index 9b224202e..89d37e5df 100644 --- a/scripts/README.md +++ b/scripts/README.md @@ -3,30 +3,30 @@ Run these scripts from the repository root. They fail on command errors unless their documented orchestration handles a probe deliberately. -| Script | Inputs and prerequisites | Side effects and cleanup | -| ----------------------------------------- | ------------------------------------------------------------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | -| `batch-sync.sh` | Endpoint, bearer token, EC ID, partner UID; `curl` and Python | Sends one authenticated batch-sync request. Its temporary response file is removed on exit. | -| `benchmark.sh` | A running server; `curl`, `bc`, and optionally `hey` | May install `hey` through Homebrew. `--save` writes under `benchmark-results/`; it does not stop the server. | -| `profile.sh` | Fastly CLI, Rust WASM target, `curl`; endpoint and request options | Builds and starts the Fastly app, stops the owned process, and retains a profile under `benchmark-results/profiles/`. `--open` launches the local viewer. | -| `generate-integration-viceroy-configs.sh` | Rust toolchain; optional origin port and artifact directory | Builds the native generator and writes Viceroy config under `target/integration-test-artifacts/`; generated files persist. | -| `integration-tests.sh` | Docker, Viceroy, Rust WASM target, pinned Node | Builds WASM/native artifacts and two Docker images, generates Viceroy config, and runs native integration tests serially. Build products and images persist. | -| `integration-tests-browser.sh` | The integration prerequisites plus npm and Playwright | Installs package/browser dependencies, builds fixtures and images, runs both browser suites, and stops matching test containers on exit. Build and npm artifacts persist. | -| `smoke-axum.sh` | `cargo`, `curl`, Python | Uses an isolated temporary config/store and stub origin; stops owned processes and removes its workspace. | -| `smoke-fastly.sh` | Fastly CLI, `cargo`, `curl`, Python | Uses an isolated Fastly project and application config; stops owned processes and removes its workspace. | -| `smoke-cloudflare.sh` | Pinned Wrangler, `cargo`, `curl`, `jq`, and Python | Uses isolated Wrangler manifests, KV state, ports, and logs; stops owned processes and removes its workspace. | -| `smoke-spin.sh` | Spin CLI, `cargo`, `curl`, and Python | Uses an isolated Spin manifest and SQLite KV store; stops owned processes and removes its workspace. | -| `smoke-common.sh` | Sourced by the four smoke scripts | Defines bounded port, process, config, secret, and response assertions. Do not invoke it as a standalone smoke. | -| `async-html-finalization-local-test.sh` | Python 3.11+, Rust, Node and selected runtime; Fastly Viceroy 0.19+ | Builds feature-gated fixtures, preserves compiler settings, retains diagnostics, and stops owned process groups on failure or SIGTERM. | -| `template-cache-local-test.sh` | Viceroy, Node, OpenSSL, `curl`, `lsof`; optional ports and mode | Builds local artifacts, generates certificates and fixtures in a temporary directory, stops owned servers, and removes the directory. | -| `test-cli.sh` | Rustup and the host toolchain; optional host triple | May install the selected Rust target, then runs native CLI and browser-audit tests. Cargo artifacts persist. | +| Script | Inputs and prerequisites | Side effects and cleanup | +| ----------------------------------------- | ------------------------------------------------------------------ | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| `batch-sync.sh` | Endpoint, bearer token, EC ID, partner UID; `curl` and Python | Sends one authenticated batch-sync request. Its temporary response file is removed on exit. | +| `benchmark.sh` | A running server; `curl`, `bc`, and optionally `hey` | May install `hey` through Homebrew. `--save` writes under `benchmark-results/`; it does not stop the server. | +| `profile.sh` | Fastly CLI, Rust WASM target, `curl`; endpoint and request options | Builds and starts the Fastly app, stops the owned process, and retains a profile under `benchmark-results/profiles/`. `--open` launches the local viewer. | +| `generate-integration-viceroy-configs.sh` | Rust toolchain; optional origin port and artifact directory | Builds the native generator and writes Viceroy config under `target/integration-test-artifacts/`; generated files persist. | +| `integration-tests.sh` | Docker, Viceroy, Rust WASM target, pinned Node | Builds WASM/native artifacts and two Docker images, generates Viceroy config, and runs native integration tests serially. Build products and images persist. | +| `integration-tests-browser.sh` | The integration prerequisites plus npm and Playwright | Installs package/browser dependencies, builds fixtures and images, runs both browser suites, and stops matching test containers on exit. Build and npm artifacts persist. | +| `smoke-axum.sh` | `cargo`, `curl`, Python | Uses an isolated temporary config/store and stub origin; stops owned processes and removes its workspace. | +| `smoke-fastly.sh` | Fastly CLI, `cargo`, `curl`, Python | Uses an isolated Fastly project and application config; stops owned processes and removes its workspace. | +| `smoke-cloudflare.sh` | Pinned Wrangler, `cargo`, `curl`, `jq`, and Python | Uses isolated Wrangler manifests, KV state, ports, and logs; stops owned processes and removes its workspace. | +| `smoke-spin.sh` | Spin CLI, `cargo`, `curl`, and Python | Uses an isolated Spin manifest and SQLite KV store; stops owned processes and removes its workspace. | +| `smoke-common.sh` | Sourced by the four smoke scripts | Defines bounded port, process, config, secret, and response assertions. Do not invoke it as a standalone smoke. | +| `async-html-finalization-local-test.sh` | Rust, pinned Node and selected runtime; Fastly Viceroy 0.19+ | Builds feature-gated fixtures, preserves compiler settings, retains diagnostics, and stops owned process groups on failure or SIGTERM. | +| `template-cache-local-test.sh` | Viceroy, Node, OpenSSL, `curl`, `lsof`; optional ports and mode | Builds local artifacts, generates certificates and fixtures in a temporary directory, stops owned servers, and removes the directory. | +| `test-cli.sh` | Rustup and the host toolchain; optional host triple | May install the selected Rust target, then runs native CLI and browser-audit tests. Cargo artifacts persist. | ## HTML EOF runtime checks -Requires Python 3.11+, Rust, pinned Node and the selected runtime CLI. Run from the +Requires Rust, pinned Node and the selected runtime CLI. Run from the repository root: ```bash -python3 -m unittest discover -s scripts/fixtures -p 'test_async_html_finalization_runtime.py' -v +cargo test --manifest-path crates/trusted-server-integration-tests/Cargo.toml --features html-eof-runtime --test html_eof_runtime ./scripts/async-html-finalization-local-test.sh axum --auction ./scripts/async-html-finalization-local-test.sh cloudflare --release --auction ./scripts/async-html-finalization-local-test.sh spin --release --auction @@ -45,7 +45,7 @@ The harness builds fresh feature-gated artifacts, checks generated timeout, permission and loopback configuration, preserves explicit compiler/profile settings and `CARGO_TARGET_DIR`, selects build targets itself, and stops owned process groups on failure or SIGTERM. Artifacts and diagnostics persist in the printed temporary directory for inspection. -CI runs cleanup tests, Axum and isolated Fastly EOF controls. See the +CI runs native Rust helper tests and lint, Axum and isolated Fastly EOF controls. See the [testing guide](../docs/guide/testing.md#html-eof-runtime-checks) for coverage details. The four adapter smoke contracts are documented in the diff --git a/scripts/async-html-finalization-local-test.sh b/scripts/async-html-finalization-local-test.sh index 9b0906af5..f941ab017 100755 --- a/scripts/async-html-finalization-local-test.sh +++ b/scripts/async-html-finalization-local-test.sh @@ -2,4 +2,20 @@ # Build a fresh test-utils artifact and exercise terminal I/O on a local runtime. set -euo pipefail REPO_ROOT=$(cd "$(dirname "$0")/.." && pwd) -exec python3 "$REPO_ROOT/scripts/fixtures/async-html-finalization-runtime.py" "$@" +case "${1:-}" in + axum|fastly|cloudflare|spin) export HTML_EOF_ADAPTER="$1"; shift ;; + *) echo "Usage: $0 {axum|fastly|cloudflare|spin} [--release] [--auction]" >&2; exit 2 ;; +esac +export HTML_EOF_RELEASE=false HTML_EOF_AUCTION=false +for argument in "$@"; do + case "$argument" in + --release) export HTML_EOF_RELEASE=true ;; + --auction) export HTML_EOF_AUCTION=true ;; + *) echo "Unknown argument: $argument" >&2; exit 2 ;; + esac +done +cd "$REPO_ROOT" +EOF_HOST_TARGET=$(rustc -vV | sed -n 's/host: //p') +exec cargo test --manifest-path crates/trusted-server-integration-tests/Cargo.toml \ + --target "$EOF_HOST_TARGET" --features html-eof-runtime --test html_eof_runtime \ + actual_runtime -- --ignored --exact --nocapture --test-threads=1 diff --git a/scripts/fixtures/async-html-finalization-origin.py b/scripts/fixtures/async-html-finalization-origin.py deleted file mode 100644 index 01a0fe76f..000000000 --- a/scripts/fixtures/async-html-finalization-origin.py +++ /dev/null @@ -1,84 +0,0 @@ -#!/usr/bin/env python3 -"""Loopback origin and delayed lookup for the fictional terminal consumer.""" - -import argparse -import json -import threading -import time -from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer -from pathlib import Path - -DOCUMENT = ( - b"

EXAMPLE_UNSIGNED

" - b"" -) - - -def serve(port, event_path): - lock = threading.Lock() - - def event(kind, **fields): - with lock: - with event_path.open("a") as output: - output.write(json.dumps({"event": kind, "time": time.monotonic(), **fields}) + "\n") - - class Handler(BaseHTTPRequestHandler): - def log_message(self, *_): - pass - - def do_HEAD(self): - self.send_response(200) - self.send_header("Content-Type", "text/html") - self.send_header("Content-Length", str(len(DOCUMENT))) - self.end_headers() - - def do_GET(self): - self.send_response(200) - self.send_header("Content-Type", "text/html") - self.send_header("Cache-Control", "public, max-age=60") - self.send_header("Content-Length", str(len(DOCUMENT))) - self.end_headers() - self.wfile.write(DOCUMENT[:-len(b"")]) - self.wfile.flush() - time.sleep(0.05) - event("origin_eof") - self.wfile.write(b"") - self.wfile.flush() - - def do_POST(self): - if self.headers.get("Transfer-Encoding", "").lower() == "chunked": - body = bytearray() - while True: - size = int(self.rfile.readline().split(b";", 1)[0], 16) - if size == 0: - while self.rfile.readline() != b"\r\n": - pass - break - body.extend(self.rfile.read(size)) - assert len(body) <= 2048, "fixture request must remain bounded" - assert self.rfile.read(2) == b"\r\n" - else: - size = int(self.headers.get("Content-Length", "0")) - assert size <= 2048, "fixture request must remain bounded" - body = self.rfile.read(size) - payload = json.loads(body) - event("lookup_start", path=self.path, **payload) - time.sleep(30 if self.path == "/hung" else 0.15) - self.send_response(200) - self.send_header("Content-Length", str(len(b"EXAMPLE_APPROVED"))) - self.end_headers() - try: - self.wfile.write(b"EXAMPLE_APPROVED") - event("lookup_complete", path=self.path) - except (BrokenPipeError, ConnectionResetError): - event("lookup_cancelled", path=self.path) - - ThreadingHTTPServer(("127.0.0.1", port), Handler).serve_forever() - - -if __name__ == "__main__": - parser = argparse.ArgumentParser() - parser.add_argument("--port", type=int, required=True) - parser.add_argument("--events", type=Path, required=True) - args = parser.parse_args() - serve(args.port, args.events) diff --git a/scripts/fixtures/async-html-finalization-runtime.py b/scripts/fixtures/async-html-finalization-runtime.py deleted file mode 100644 index fe9862ec7..000000000 --- a/scripts/fixtures/async-html-finalization-runtime.py +++ /dev/null @@ -1,434 +0,0 @@ -#!/usr/bin/env python3 -"""Fresh-build local runtime matrix with isolated config and process cleanup.""" - -import argparse -import base64 -import re -import json -import os -import shutil -import tomllib -import signal -import socket -import subprocess -import tempfile -import time -import urllib.error -import urllib.request -from pathlib import Path - -ROOT = Path(__file__).resolve().parents[2] -SECRETS = { - "handler_password": "example-handler-password-32-bytes-ok", - "publisher_proxy_secret": "example-publisher-secret-32-bytes-ok", - "ec_passphrase": "example-cookie-passphrase-32-bytes-ok", -} - - -def free_port(): - with socket.socket() as listener: - listener.bind(("127.0.0.1", 0)) - return listener.getsockname()[1] - - -def stop(process): - if process is None: - return - try: - os.killpg(process.pid, signal.SIGTERM) - process.wait(timeout=5) - except (ProcessLookupError, subprocess.TimeoutExpired): - # An exited group needs no TERM; a graceful-wait timeout proceeds to KILL. - pass - try: - os.killpg(process.pid, signal.SIGKILL) - except ProcessLookupError: - # The process group may already have exited after TERM. - pass - process.wait(timeout=5) - - -def build_environment(source): - allowed = { - "PATH", "RUSTUP_HOME", "CARGO_HOME", "CARGO_TARGET_DIR", "TMPDIR", - "WASM_BINDGEN_BIN", "WASM_OPT_BIN", "ESBUILD_BIN", "CARGO_NET_OFFLINE", - "RUSTFLAGS", "CARGO_ENCODED_RUSTFLAGS", "RUSTDOCFLAGS", "CARGO_INCREMENTAL", - "RUSTC", "RUSTDOC", "RUSTC_WRAPPER", "RUSTC_WORKSPACE_WRAPPER", - "TSJS_PREBUILT_DIR", - } - return {key: value for key, value in source.items() - if key != "CARGO_BUILD_TARGET" - and (key in allowed or key.startswith(("CARGO_PROFILE_", "CARGO_BUILD_", "CARGO_TARGET_")))} - - -def checked_replace(text, old, new, description): - matches = text.count(old) - if matches != 1: - raise RuntimeError(f"expected one {description} template field, found {matches}") - return text.replace(old, new, 1) - - -def toml_string(value): - return json.dumps(str(value), ensure_ascii=False) - - -def fixture_manifest(text, workspace): - manifest = tomllib.loads(text) - entry = manifest["app"]["entry"] - text = checked_replace(text, f"entry = {toml_string(entry)}", - f"entry = {toml_string(ROOT / entry)}", "app entry") - for adapter, settings in manifest["adapters"].items(): - settings = settings["adapter"] - crate = settings["crate"] - text = checked_replace(text, f"crate = {toml_string(crate)}", - f"crate = {toml_string(ROOT / crate)}", f"{adapter} crate") - old = settings["manifest"] - path = workspace / f"{adapter}.toml" if adapter in ("fastly", "spin") else ROOT / old - text = checked_replace(text, f"manifest = {toml_string(old)}", - f"manifest = {toml_string(path)}", f"{adapter} manifest") - return text - - -def fixture_config(template, case, origin_port, bid_port=None): - permission = str(case != "disabled").lower() - publisher = tomllib.loads(template)["publisher"] - text = checked_replace(template, "[publisher]\n", - "[publisher]\nhtml_finalization_timeout_ms = 750\n" - f"allow_async_html_finish = {permission}\n", "publisher table") - table = text.index("[publisher]\n") - end = text.find("\n[", table + len("[publisher]\n")) - if end == -1: - end = len(text) - block = text[table:end] - for key, value in { - "domain": "127.0.0.1", - "cookie_domain": "127.0.0.1", - "origin_url": f"http://127.0.0.1:{origin_port}", - }.items(): - block = checked_replace(block, f"\n{key} = {toml_string(publisher[key])}", - f"\n{key} = {toml_string(value)}", f"publisher {key}") - text = text[:table] + block + text[end:] - if case == "auction": - table = text.index("[auction]\n") - end = text.find("\n[", table + len("[auction]\n")) - if end == -1: - end = len(text) - block = checked_replace(text[table:end], "enabled = false", "enabled = true", "auction enablement") - text = text[:table] + block + text[end:] - text = checked_replace(text, 'endpoint = "https://prebid.example.com/openrtb2/auction"', - f'endpoint = "https://127.0.0.1:{bid_port}/bid"', "auction endpoint") - text += '\n[[creative_opportunities.slot]]\nid = "example-slot"\npage_patterns = ["/article"]\nformats = [{ width = 300, height = 250 }]\n' - lookup_case = "success" if case in ("auction", "streaming") else case - text += (f'\n[integrations.html_finalization_fixture]\nenabled = {permission}\n' - f'lookup_url = "http://127.0.0.1:{origin_port}/{lookup_case}"\n' - f'transport_timeout_ms = 30000\nstreaming = {str(case == "streaming").lower()}\n') - validate_fixture_config(text, case, origin_port, bid_port) - return text - - -def validate_fixture_config(text, case, origin_port, bid_port=None): - config = tomllib.loads(text) - publisher = config["publisher"] - fixture = config["integrations"]["html_finalization_fixture"] - permission = case != "disabled" - lookup_case = "success" if case in ("auction", "streaming") else case - expected = [ - (publisher.get("domain"), "127.0.0.1", "loopback publisher domain"), - (publisher.get("cookie_domain"), "127.0.0.1", "loopback cookie domain"), - (publisher.get("origin_url"), f"http://127.0.0.1:{origin_port}", "loopback origin"), - (publisher.get("html_finalization_timeout_ms"), 750, "EOF timeout"), - (publisher.get("allow_async_html_finish"), permission, "EOF permission"), - (fixture.get("enabled"), permission, "fixture enablement"), - (fixture.get("lookup_url"), f"http://127.0.0.1:{origin_port}/{lookup_case}", "loopback lookup"), - (fixture.get("transport_timeout_ms"), 30000, "fixture transport timeout"), - (fixture.get("streaming"), case == "streaming", "streaming control"), - (config["auction"].get("enabled"), case == "auction", "auction enablement"), - ] - for actual, required, description in expected: - if actual != required: - raise RuntimeError(f"invalid {description}: expected {required!r}, found {actual!r}") - if case == "auction": - providers = config["auction"].get("providers", {}) - if not providers: - raise RuntimeError("auction control requires a loopback provider") - for provider in providers.values(): - if provider.get("endpoint") != f"https://127.0.0.1:{bid_port}/bid": - raise RuntimeError("auction provider must use the controlled loopback endpoint") - - -def preflight_viceroy(env): - output = subprocess.check_output(["viceroy", "--version"], env=env, text=True).strip() - match = re.search(r"\bviceroy (\d+)\.(\d+)\.(\d+)(?:\s|$)", output) - if not match or tuple(map(int, match.groups())) < (0, 19, 0): - raise RuntimeError(f"Viceroy >= 0.19.0 required for the Fastly hung-call control; found {output!r}") - - -def lookup_for_request(events, started, completed, enabled): - lookups = [event for event in events if event["event"] == "lookup_start" - and started <= event["time"] <= completed] - assert len(lookups) == int(enabled), "each request must issue exactly one enabled EOF lookup" - if not enabled: - return None - lookup = lookups[0] - assert any(event["event"] == "origin_eof" and started <= event["time"] <= lookup["time"] - for event in events), "EOF lookup must start after the origin finishes" - return lookup - - -def measured_terminal_ms(adapter, diagnostic, lookup_elapsed=None): - match = re.search(r"times out after (\d+) ms", diagnostic) - if match is None: - if adapter != "fastly": - raise RuntimeError(f"missing measured cooperative timeout diagnostic for {adapter}") - if lookup_elapsed is None: - raise RuntimeError("missing measured lookup duration for the Fastly transport fallback") - assert 0.7 <= lookup_elapsed < 2.5, "blocking transport must finish within the terminal budget tolerance" - return None - terminal_ms = int(match.group(1)) - assert 700 <= terminal_ms < 2500, "terminal budget must exclude origin transfer" - return terminal_ms - - -def terminate_run(signum, _frame): - raise SystemExit(128 + signum) - - -def run(adapter, auction=False, release=False): - workspace = Path(tempfile.mkdtemp(prefix=f"trusted-server-finalize-{adapter}-")) - print(f"Artifacts: {workspace}", flush=True) - env = build_environment(os.environ) - target = Path(env.get("CARGO_TARGET_DIR", workspace / "target")).resolve() - env["CARGO_TARGET_DIR"] = str(target) - env["XDG_CONFIG_HOME"] = str(workspace / "cli-config") - env["WRANGLER_SEND_METRICS"] = "false" - env["WRANGLER_LOG_PATH"] = str(workspace / "wrangler.log") - origin_port, app_port = free_port(), free_port() - env.update({ - "TRUSTED_SERVER__PUBLISHER__DOMAIN": "127.0.0.1", - "TRUSTED_SERVER__PUBLISHER__COOKIE_DOMAIN": "127.0.0.1", - "TRUSTED_SERVER__PUBLISHER__ORIGIN_URL": f"http://127.0.0.1:{origin_port}", - }) - required = ["cargo", "python3"] + ({"fastly": ["viceroy"], "cloudflare": ["worker-build", "wrangler"], "spin": ["spin"]}.get(adapter, [])) - for executable in required: - if shutil.which(executable, path=env["PATH"]) is None: - raise RuntimeError(f"required executable missing: {executable}") - if adapter == "fastly": - preflight_viceroy(env) - if adapter == "cloudflare": - lock = tomllib.loads((ROOT / "Cargo.lock").read_text()) - expected = next(package["version"] for package in lock["package"] if package["name"] == "worker") - actual = subprocess.check_output(["worker-build", "--version"], env=env, text=True).strip() - if actual != expected: - raise RuntimeError(f"worker-build {expected} required for the locked worker SDK; found {actual}") - processes = [] - previous_sigterm = signal.signal(signal.SIGTERM, terminate_run) - - def command(args, cwd=ROOT, overrides=None): - process = subprocess.Popen(args, cwd=cwd, env=env | (overrides or {}), start_new_session=True) - processes.append(process) - status = process.wait() - if status != 0: - raise subprocess.CalledProcessError(status, args) - - def launch(args, log, cwd=workspace, runtime_env=None): - output = log.open("w") - process = subprocess.Popen(args, cwd=cwd, env=runtime_env or env, - stdout=output, stderr=subprocess.STDOUT, - start_new_session=True) - output.close() - processes.append(process) - return process - - def get(path, auction_case=False): - headers = {"Accept": "text/html", "User-Agent": "Mozilla/5.0 (Example browser)"} - if auction_case: - # A fictional core TCF v2 segment grants Purpose 1. No identifiers - # or vendor-specific data are included in this local control. - bits = bytearray(29) - def field(offset, width, value): - for index in range(width): - if value & (1 << (width - index - 1)): - position = offset + index - bits[position // 8] |= 1 << (7 - position % 8) - field(0, 6, 2) - field(6, 36, int(time.time() * 10)) - field(42, 36, int(time.time() * 10)) - field(108, 6, 4) - field(114, 6, 13) - field(152, 1, 1) - headers["Cookie"] = "euconsent-v2=" + base64.urlsafe_b64encode(bits).decode().rstrip("=") - request = urllib.request.Request(f"http://127.0.0.1:{app_port}{path}", headers=headers) - with urllib.request.urlopen(request, timeout=10) as response: - first = response.read(1) - first_byte_at = time.monotonic() - return first + response.read(), response.headers, first_byte_at - - try: - command(["cargo", "build", "-p", "trusted-server-cli", "--bin", "ts", "--locked"]) - cli = str(target / "debug/ts") - crate = ROOT / f"crates/trusted-server-adapter-{adapter}" - if adapter == "cloudflare": - worker_args = ["worker-build", "--release" if release else "--dev", "--out-dir", str(workspace / "build"), "--features", "cloudflare,test-utils", "--locked"] - if not release: - worker_args += ["--no-opt"] - command(worker_args, cwd=crate) - artifact = workspace / "build/index.js" - else: - args = ["cargo", "build", "-p", f"trusted-server-adapter-{adapter}", "--locked", - "--features", "trusted-server-core/test-utils" + (",spin,test-utils" if adapter == "spin" else "")] - if adapter != "axum": - args += ["--target", "wasm32-wasip1"] - if release: - args += ["--release"] - command(args) - profile = "release" if release else "debug" - artifact = target / (f"{profile}/trusted-server-axum" if adapter == "axum" else - f"wasm32-wasip1/{profile}/{'trusted_server_adapter_spin' if adapter == 'spin' else 'trusted-server-adapter-fastly'}.wasm") - assert artifact.exists(), f"fresh artifact missing: {artifact}" - launch(["python3", str(ROOT / "scripts/fixtures/async-html-finalization-origin.py"), - "--port", str(origin_port), "--events", str(workspace / "events.jsonl")], workspace / "origin.log") - config = workspace / "trusted-server.toml" - command([cli, "config", "init", "--app-config", str(config)]) - base_config = config.read_text() - (workspace / "edgezero.toml").write_text(fixture_manifest((ROOT / "edgezero.toml").read_text(), workspace)) - fastly_base = 'manifest_version = 3\nname = "example-finalization"\nlanguage = "rust"\n[local_server]\n' - (workspace / "fastly.toml").write_text(fastly_base) - if adapter == "spin": - spin_config = checked_replace((crate / "spin.toml").read_text(), - 'source = "../../target/wasm32-wasip1/release/trusted_server_adapter_spin.wasm"', - f"source = {toml_string(artifact)}", "Spin artifact") - (workspace / "spin.toml").write_text(spin_config) - - baseline = None - cases = ["disabled", "success", "hung"] + (["auction"] if auction else []) + (["streaming"] if adapter == "fastly" else []) - for case in cases: - # A closed loopback bid endpoint exercises transport failure without - # sending auction requests outside this host. - bid_port = free_port() if case == "auction" else None - case_config = fixture_config(base_config, case, origin_port, bid_port) - config.write_text(case_config) - # Axum's local JSON provides the same neutral app-config envelope - # used by the other runtime bindings; no operator config is read. - command([cli, "config", "push", "--adapter", "axum", "--local", "--manifest", str(workspace / "edgezero.toml"), - "--app-config", str(config), "--yes", "--no-diff"], cwd=workspace) - envelope = json.loads((workspace / ".edgezero/local-config-trusted_server_config.json").read_text())["trusted_server_config"] - runtime_env = env.copy() - if adapter == "axum": - runtime_env["TRUSTED_SERVER_CONFIG_TRUSTED_SERVER_CONFIG_TRUSTED_SERVER_CONFIG"] = envelope - for key, value in SECRETS.items(): - runtime_env[f"TRUSTED_SERVER_SECRET_TRUSTED_SERVER_SECRETS_{key.upper()}"] = value - runtime_env["PORT"] = str(app_port) - args = [str(artifact)] - elif adapter == "cloudflare": - text = checked_replace((crate / "wrangler.ci.toml").read_text(), 'main = "build/index.js"', - f"main = {toml_string(artifact)}", "Workers artifact") - text = checked_replace(text, 'TRUSTED_SERVER_CONFIG = "{}"', - "TRUSTED_SERVER_CONFIG = '''" + json.dumps({"app_config": envelope}) + "'''", - "Workers config binding") - text += "\n" + "\n".join(f'{key} = "{value}"' for key, value in SECRETS.items()) + "\n" - (workspace / "wrangler.toml").write_text(text) - args = ["wrangler", "dev", "--local", "--config", str(workspace / "wrangler.toml"), - "--ip", "127.0.0.1", "--port", str(app_port), "--persist-to", str(workspace / "worker-state")] - else: - if adapter == "fastly": - (workspace / "fastly.toml").write_text(fastly_base) - command([cli, "config", "push", "--adapter", adapter, "--local", "--manifest", str(workspace / "edgezero.toml"), - "--app-config", str(config), "--yes", "--no-diff"], cwd=workspace, overrides={"EDGEZERO__STORES__CONFIG__TRUSTED_SERVER_CONFIG__NAME": "default"} if adapter == "spin" else None) - if adapter == "fastly": - with (workspace / "fastly.toml").open("a") as output: - for key, value in SECRETS.items(): - output.write(f'\n[[local_server.secret_stores.trusted_server_secrets]]\nkey = "{key}"\ndata = "{value}"\n') - args = ["viceroy", "--config", str(workspace / "fastly.toml"), "--addr", f"127.0.0.1:{app_port}", str(artifact)] - else: - args = ["spin", "up", "--from", str(workspace / "spin.toml"), "--listen", f"127.0.0.1:{app_port}", - "--state-dir", str(workspace / ".spin"), "--cache-dir", str(workspace / "spin-cache"), "--log-dir", str(workspace / "spin-logs")] - for key, value in SECRETS.items(): - encoded = "v_trusted_x5fserver_x5fsecrets_v_" + key.replace("_", "_x5f") - args += ["--variable", f"{encoded}={value}"] - log = workspace / f"{case}.log" - process = launch(args, log, runtime_env=runtime_env) - for _ in range(80): - if process.poll() is not None: - raise RuntimeError(f"{adapter} exited: {log.read_text()}") - try: - with urllib.request.urlopen(urllib.request.Request(f"http://127.0.0.1:{app_port}/health", method="HEAD"), timeout=1) as ready: - if ready.status == 200: - break - except (OSError, urllib.error.URLError): - time.sleep(0.1) - else: - raise RuntimeError(f"runtime did not become ready: {log.read_text()}") - started = time.monotonic() - try: - body, headers, first_byte_at = get(f"/article?case={case}", case == "auction") - except urllib.error.HTTPError as error: - (workspace / f"{case}.error.body").write_bytes(error.read()) - (workspace / f"{case}.headers").write_text(str(error.headers)) - time.sleep(0.3) - raise RuntimeError(f"{adapter} {case} returned HTTP {error.code}; see {log}") from error - completed = time.monotonic() - elapsed = time.monotonic() - started - (workspace / f"{case}.body").write_bytes(body) - (workspace / f"{case}.headers").write_text(str(headers)) - if case == "disabled": - assert b"EXAMPLE_UNSIGNED" in body and b"EXAMPLE_FINALIZED" not in body, "disabled fixture must leave the original marker unchanged" - baseline = body - elif case == "success": - assert body == baseline.replace(b"EXAMPLE_UNSIGNED", b"EXAMPLE_FINALIZED"), "successful terminal output must match the exact expected rewrite" - assert elapsed >= 0.15, "success must actually await delayed lookup" - elif case == "streaming": - assert body == baseline + b"", "streaming EOF must append only after already delivered HTML" - elif case == "auction": - assert b"EXAMPLE_FINALIZED" in body, "auction-enabled output must await finalization" - assert b"adSlots" in body and b"bids" in body, "auction control must reach slot and bid assembly" - assert b"ts-inline-body-close-" not in body, "auction assembly must consume the opaque parser marker" - else: - assert body == baseline, "timeout must emit byte-identical processed input" - assert 0.6 <= elapsed < 3, f"timeout escaped the 750ms budget: {elapsed}" - diagnostic = log.read_text() - if adapter == "spin": - diagnostic += "\n".join(path.read_text() for path in (workspace / "spin-logs").glob("*stderr*")) - assert "HTML EOF processor html_finalization_fixture times out" in diagnostic or (adapter == "fastly" and "retains input: unavailable" in diagnostic), "missing controlled fallback diagnostic" - if case != "disabled": - assert headers["Cache-Control"] == "no-store, private", "active EOF processing must prohibit shared caching" - current_events = [json.loads(line) for line in (workspace / "events.jsonl").read_text().splitlines()] - lookup = lookup_for_request(current_events, started, completed, case != "disabled") - lookup_elapsed = completed - lookup["time"] if lookup is not None else None - if case == "streaming": - completions = [event for event in current_events if event["event"] == "lookup_complete"] - assert completions, "streaming finish must complete real delayed I/O" - assert first_byte_at < completions[-1]["time"] - 0.05, "initial HTML bytes must reach the client while EOF I/O remains pending" - terminal_ms = None - if case == "hung": - terminal_ms = measured_terminal_ms(adapter, diagnostic, lookup_elapsed) - (workspace / f"{case}.timing.json").write_text(json.dumps({"request_seconds": elapsed, "first_body_byte_seconds": first_byte_at - started, "terminal_ms": terminal_ms, "lookup_to_response_seconds": lookup_elapsed, "configured_budget_ms": 750, "cooperative_timer": adapter != "fastly"})) - print(f"PASS {adapter} {case}: request={elapsed:.3f}s, {len(body)} bytes, chain_budget=750ms, terminal_ms={terminal_ms}, lookup_to_response={lookup_elapsed}, cooperative_timer={adapter != 'fastly'}", flush=True) - stop(process) - events = [json.loads(line) for line in (workspace / "events.jsonl").read_text().splitlines()] - starts = [event for event in events if event["event"] == "lookup_start"] - assert len(starts) == 2 + int(auction) + int(adapter == "fastly"), "each enabled control must issue exactly one EOF lookup" - assert all(event["complete"] and len(event["sha256"]) == 64 for event in starts), "every lookup must observe complete-document metadata" - for event in starts: - if adapter == "fastly": - assert 0 < event["transport_timeout_ms"] <= 750, "blocking host calls must use the remaining budget" - else: - assert event["transport_timeout_ms"] == 30000, "SDK timer must release before the transport deadline" - assert any(event["event"] == "lookup_complete" and event["path"] == "/success" for event in events), "success control must complete real lookup I/O" - finally: - # Finish owned-process cleanup even if another termination signal arrives. - signal.signal(signal.SIGTERM, signal.SIG_IGN) - try: - for process in reversed(processes): - stop(process) - finally: - signal.signal(signal.SIGTERM, previous_sigterm) - - - -if __name__ == "__main__": - parser = argparse.ArgumentParser() - parser.add_argument("adapter", choices=["fastly", "cloudflare", "spin", "axum"]) - parser.add_argument("--auction", action="store_true", help="also exercise enabled auction transport-failure assembly") - parser.add_argument("--release", action="store_true", help="build production-profile fixture artifacts") - args = parser.parse_args() - run(args.adapter, args.auction, args.release) diff --git a/scripts/fixtures/test_async_html_finalization_runtime.py b/scripts/fixtures/test_async_html_finalization_runtime.py deleted file mode 100644 index 2a42e6100..000000000 --- a/scripts/fixtures/test_async_html_finalization_runtime.py +++ /dev/null @@ -1,266 +0,0 @@ -"""Verify process cleanup used by the actual-runtime finalization harness.""" - -import importlib.util -import os -import select -from pathlib import Path -import signal -import subprocess -import sys -import tempfile -import textwrap -import time -import tomllib -import unittest -from unittest import mock - -SPEC = importlib.util.spec_from_file_location( - "example_finalization_runtime", - Path(__file__).with_name("async-html-finalization-runtime.py"), -) -RUNTIME = importlib.util.module_from_spec(SPEC) -SPEC.loader.exec_module(RUNTIME) - - -class ProcessCleanupTests(unittest.TestCase): - def test_already_exited_process_is_safe(self): - process = subprocess.Popen([sys.executable, "-c", "pass"], start_new_session=True) - process.wait(timeout=5) - RUNTIME.stop(process) - self.assertEqual(process.returncode, 0, "should tolerate an already exited group") - - def test_graceful_termination_reaps_process(self): - process = subprocess.Popen( - [sys.executable, "-c", "import time; time.sleep(300)"], - start_new_session=True, - ) - try: - RUNTIME.stop(process) - self.assertIsNotNone(process.returncode, "should reap a terminated process") - finally: - try: - process.wait(timeout=5) - except subprocess.TimeoutExpired: - os.killpg(process.pid, signal.SIGKILL) - process.wait(timeout=5) - - def test_stubborn_termination_reaps_process(self): - process = subprocess.Popen( - [sys.executable, "-c", - "import signal, time; signal.signal(signal.SIGTERM, signal.SIG_IGN); " - "print('example-ready', flush=True); time.sleep(300)"], - stdout=subprocess.PIPE, - text=True, - start_new_session=True, - ) - try: - self.assertEqual(process.stdout.readline(), "example-ready\n") - RUNTIME.stop(process) - self.assertIsNotNone(process.returncode, "should reap a forcibly killed process") - self.assertEqual(process.returncode, -signal.SIGKILL) - finally: - try: - process.wait(timeout=5) - except subprocess.TimeoutExpired: - os.killpg(process.pid, signal.SIGKILL) - process.wait(timeout=5) - process.stdout.close() - - -class HarnessConfigurationTests(unittest.TestCase): - def test_build_environment_preserves_compiler_settings_without_operator_config(self): - source = { - "PATH": "/private/tmp/example-bin", "RUSTFLAGS": "-D warnings", - "CARGO_ENCODED_RUSTFLAGS": "-D\x1fwarnings", "CARGO_INCREMENTAL": "0", - "CARGO_PROFILE_DEV_DEBUG": "0", "CARGO_PROFILE_RELEASE_LTO": "thin", - "TSJS_PREBUILT_DIR": "/private/tmp/example-bundles", - "TRUSTED_SERVER__PUBLISHER__ORIGIN_URL": "https://origin.example.com", - "TRUSTED_SERVER_SECRET_EXAMPLE": "example-secret", - "TSJS_SKIP_BUILD": "1", - "CARGO_BUILD_TARGET": "wasm32-wasip1", - } - preserved = RUNTIME.build_environment(source) - for key in ["RUSTFLAGS", "CARGO_ENCODED_RUSTFLAGS", "CARGO_INCREMENTAL", - "CARGO_PROFILE_DEV_DEBUG", "CARGO_PROFILE_RELEASE_LTO", "TSJS_PREBUILT_DIR"]: - self.assertEqual(preserved.get(key), source[key], f"should preserve explicit {key}") - self.assertNotIn("TRUSTED_SERVER__PUBLISHER__ORIGIN_URL", preserved) - self.assertNotIn("TRUSTED_SERVER_SECRET_EXAMPLE", preserved) - self.assertNotIn("TSJS_SKIP_BUILD", preserved) - self.assertNotIn("CARGO_BUILD_TARGET", preserved, "harness must select native/adapter targets itself") - - def test_checked_replacement_rejects_missing_or_ambiguous_template_fields(self): - for text in ["missing", "token token"]: - with self.subTest(text=text), self.assertRaisesRegex(RuntimeError, "example field"): - RUNTIME.checked_replace(text, "token", "value", "example field") - self.assertEqual(RUNTIME.checked_replace("token", "token", "value", "example field"), "value") - - def test_fixture_config_checks_loopback_budget_and_permission(self): - template = (RUNTIME.ROOT / "trusted-server.example.toml").read_text() - for case in ["disabled", "success", "hung", "auction", "streaming"]: - with self.subTest(case=case): - text = RUNTIME.fixture_config(template, case, 12345, 12346 if case == "auction" else None) - RUNTIME.validate_fixture_config(text, case, 12345, 12346 if case == "auction" else None) - if case == "success": - for old, new in [("html_finalization_timeout_ms = 750", "html_finalization_timeout_ms = 5000"), - ("allow_async_html_finish = true", "allow_async_html_finish = false"), - ("http://127.0.0.1:12345/success", "https://lookup.example.com/success")]: - with self.subTest(mutation=old), self.assertRaises(RuntimeError): - RUNTIME.validate_fixture_config(text.replace(old, new), case, 12345) - drifted = template.replace('endpoint = "https://prebid.example.com/openrtb2/auction"', - 'endpoint="https://prebid.example.com/openrtb2/auction"', 1) - with self.assertRaisesRegex(RuntimeError, "auction endpoint"): - RUNTIME.fixture_config(drifted, "auction", 12345, 12346) - - def test_toml_paths_round_trip_quotes_backslashes_and_newlines(self): - value = Path('/private/tmp/example 🚀 "quoted"\\path\nnext') - parsed = tomllib.loads("path = " + RUNTIME.toml_string(value)) - self.assertEqual(parsed["path"], str(value), "should preserve path characters in TOML") - workspace = value / "workspace" - template = (RUNTIME.ROOT / "edgezero.toml").read_text() - with mock.patch.object(RUNTIME, "ROOT", value): - parsed = tomllib.loads(RUNTIME.fixture_manifest(template, workspace)) - self.assertEqual(parsed["app"]["entry"], str(value / "crates/trusted-server-core")) - self.assertEqual(parsed["adapters"]["fastly"]["adapter"]["manifest"], str(workspace / "fastly.toml")) - self.assertEqual(parsed["adapters"]["spin"]["adapter"]["manifest"], str(workspace / "spin.toml")) - - def test_fastly_preflight_rejects_old_viceroy_before_building(self): - with tempfile.TemporaryDirectory() as directory, \ - mock.patch.object(RUNTIME.tempfile, "mkdtemp", return_value=directory), \ - mock.patch.object(RUNTIME, "free_port", return_value=12345), \ - mock.patch.object(RUNTIME.shutil, "which", return_value="/private/tmp/example-bin"), \ - mock.patch.object(RUNTIME.subprocess, "check_output", return_value="viceroy 0.17.0\n"), \ - mock.patch.object(RUNTIME.subprocess, "Popen", side_effect=RuntimeError("unexpected build before Viceroy preflight")) as build: - with self.assertRaisesRegex(RuntimeError, "Viceroy.*0.19"): - RUNTIME.run("fastly") - build.assert_not_called() - for version in ["viceroy 0.19.0\n", "viceroy 0.20.1\n", "viceroy 1.0.0\n"]: - with self.subTest(version=version), mock.patch.object(RUNTIME.subprocess, "check_output", return_value=version): - RUNTIME.preflight_viceroy({}) - - def test_cooperative_timeout_requires_a_measured_diagnostic(self): - for adapter in ["axum", "cloudflare", "spin"]: - with self.subTest(adapter=adapter): - with self.assertRaisesRegex(RuntimeError, "timeout diagnostic"): - RUNTIME.measured_terminal_ms(adapter, "times out without a duration") - self.assertEqual(RUNTIME.measured_terminal_ms(adapter, "times out after 751 ms"), 751) - self.assertIsNone(RUNTIME.measured_terminal_ms("fastly", "retains input: unavailable", 0.749)) - - def test_fastly_transport_fallback_requires_measured_lookup_duration(self): - with self.assertRaisesRegex(RuntimeError, "lookup duration"): - RUNTIME.measured_terminal_ms("fastly", "retains input: unavailable") - for elapsed in [0.1, 3.0]: - with self.subTest(elapsed=elapsed), self.assertRaises(AssertionError): - RUNTIME.measured_terminal_ms("fastly", "retains input: unavailable", elapsed) - - def test_eof_lookup_rejects_an_earlier_requests_eof(self): - lookup = {"event": "lookup_start", "time": 2.2} - for current_eof in [[], [{"event": "origin_eof", "time": 2.3}]]: - events = [{"event": "origin_eof", "time": 1.0}, lookup, *current_eof] - with self.subTest(events=events), self.assertRaisesRegex(AssertionError, "origin finishes"): - RUNTIME.lookup_for_request(events, 2.0, 3.0, True) - - def test_eof_lookup_uses_only_the_current_request(self): - lookup = {"event": "lookup_start", "time": 2.2} - events = [ - {"event": "origin_eof", "time": 1.0}, - {"event": "lookup_start", "time": 1.1}, - {"event": "origin_eof", "time": 2.1}, - lookup, - ] - self.assertIs(RUNTIME.lookup_for_request(events, 2.0, 3.0, True), lookup) - self.assertIsNone(RUNTIME.lookup_for_request(events, 4.0, 5.0, False)) - with self.assertRaisesRegex(AssertionError, "exactly one"): - RUNTIME.lookup_for_request([*events, lookup], 2.0, 3.0, True) - with self.assertRaisesRegex(AssertionError, "exactly one"): - RUNTIME.lookup_for_request(events, 2.0, 3.0, False) - - -class CancellationTests(unittest.TestCase): - def test_sigterm_unwinds_actual_run_and_stops_owned_origin(self): - with tempfile.TemporaryDirectory() as directory: - workspace = Path(directory) - target = workspace / "target/debug" - target.mkdir(parents=True) - (target / "trusted-server-axum").touch() - marker = workspace / "origin-stopped" - runner = workspace / "runner.py" - runner.write_text(textwrap.dedent(''' - import importlib.util, os, subprocess, sys, time - spec = importlib.util.spec_from_file_location("runtime", sys.argv[1]) - runtime = importlib.util.module_from_spec(spec) - spec.loader.exec_module(runtime) - runtime.free_port = lambda: 12345 - runtime.tempfile.mkdtemp = lambda **kwargs: sys.argv[2] - real_popen = subprocess.Popen - origin = None - origin_program = "import os, pathlib, signal, sys, time; pathlib.Path(sys.argv[2]).write_text(str(os.getpid())); signal.signal(signal.SIGTERM, lambda *_: (pathlib.Path(sys.argv[1]).write_text('stopped'), sys.exit(0))); pathlib.Path(sys.argv[2] + '.ready').write_text('ready'); time.sleep(300)" - def launch_origin(kwargs): - global origin - origin = real_popen([sys.executable, "-c", origin_program, sys.argv[3], os.path.join(sys.argv[2], "origin.pid")], **kwargs) - ready = os.path.join(sys.argv[2], "origin.pid.ready") - deadline = time.monotonic() + 5 - while not os.path.exists(ready): - if time.monotonic() >= deadline: - raise RuntimeError("origin probe failed to become ready") - time.sleep(0.01) - return origin - def fake_popen(args, **kwargs): - if args[0] == "python3": - return launch_origin(kwargs) - if args[1:3] == ["config", "init"]: - print("READY", flush=True) - return real_popen([sys.executable, "-c", "import os, pathlib, sys, time; pathlib.Path(sys.argv[1]).write_text(str(os.getpid())); time.sleep(300)", os.path.join(sys.argv[2], "command.pid")], **kwargs) - return real_popen([sys.executable, "-c", "pass"], **kwargs) - def fake_run(args, **kwargs): - if args[1:3] == ["config", "init"]: - print("READY", flush=True) - time.sleep(300) - runtime.subprocess.run = fake_run - runtime.subprocess.Popen = fake_popen - runtime.run("axum") - ''')) - environment = dict(os.environ) - environment.update(CARGO_TARGET_DIR=str(workspace / "target"), PYTHONDONTWRITEBYTECODE="1") - process = subprocess.Popen([sys.executable, str(runner), str(Path(RUNTIME.__file__)), str(workspace), str(marker)], - env=environment, stdout=subprocess.PIPE, stderr=subprocess.PIPE, - start_new_session=True) - try: - ready = False - until = time.monotonic() + 5 - transcript = b"" - while time.monotonic() < until: - readable, _, _ = select.select([process.stdout], [], [], 0.1) - if readable: - transcript += os.read(process.stdout.fileno(), 65536) - if b"READY\n" in transcript: - ready = True - break - if process.poll() is not None: - break - self.assertTrue(ready, "should reach the owned-origin cancellation probe") - time.sleep(0.1) - process.terminate() - process.wait(timeout=10) - self.assertTrue(marker.exists(), "SIGTERM should unwind run() and stop its owned origin") - self.assertEqual(process.returncode, 128 + signal.SIGTERM, "should preserve cancellation exit status") - for filename in ["origin.pid", "command.pid"]: - pid_file = workspace / filename - self.assertTrue(pid_file.exists(), "should launch the owned process probe") - with self.assertRaises(ProcessLookupError): - os.kill(int(pid_file.read_text()), 0) - finally: - if process.poll() is None: - os.killpg(process.pid, signal.SIGKILL) - process.wait(timeout=5) - # The child writes its own PID so failures can also clean the detached group. - for path in workspace.glob("*.pid"): - try: - os.killpg(int(path.read_text()), signal.SIGKILL) - except ProcessLookupError: - pass - process.stdout.close() - process.stderr.close() - - -if __name__ == "__main__": - unittest.main() From 463e0cb67b393ae01fb4262e76ab76489e146cc4 Mon Sep 17 00:00:00 2001 From: prk-Jr Date: Wed, 14 Oct 2026 22:55:21 +0530 Subject: [PATCH 11/11] Install clippy in the Axum native test job --- .github/workflows/test.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 510d03be8..7b374bcff 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -148,6 +148,8 @@ jobs: # wasm32-wasip1 is required by the "Verify Fastly WASM release # build" step below; the axum build and tests are native. target: wasm32-wasip1 + # clippy is required by the "Lint native finalization harness" step. + components: clippy cache-shared-key: cargo-${{ runner.os }} - name: Read pinned Node.js version for CLI fixtures