From f84a4d221140d504096d7445bad0392a0deea8c4 Mon Sep 17 00:00:00 2001 From: Brian O'Kelley Date: Thu, 17 Sep 2026 04:20:28 +0000 Subject: [PATCH 1/8] feat(reporting): materialize revisions with durable fenced work B2.1 of 4 within B2 of B1/B2. Preserve the coherent reserve, verified I/O, captured finish and logical enqueue transaction unit; keep production activation gated on the remaining seller slices. Refs #1167 --- .github/workflows/ci.yml | 75 +- docs/reporting-destination-writer.md | 5 + docs/reporting-durable-materializer.md | 270 +++++ examples/reporting_durable_materializer.py | 61 ++ pyproject.toml | 1 + src/adcp/reporting/ledger/delivery.py | 59 +- src/adcp/reporting/ledger/delivery_pg.py | 81 +- .../ledger/reporting_materializer.sql | 420 ++++++++ src/adcp/reporting/ledger/store.py | 23 +- src/adcp/reporting/materializer/__init__.py | 37 +- src/adcp/reporting/materializer/_errors.py | 36 + src/adcp/reporting/materializer/capture.py | 158 +++ src/adcp/reporting/materializer/memory.py | 670 ++++++++++++ src/adcp/reporting/materializer/pg.py | 955 ++++++++++++++++++ .../materializer/required_schema.json | 750 ++++++++++++++ src/adcp/reporting/materializer/schema.py | 43 + src/adcp/reporting/materializer/service.py | 154 +++ .../reporting/materializer/verification.py | 62 +- src/adcp/reporting/materializer/work.py | 284 ++++++ .../_durable_materializer_support.py | 319 ++++++ .../reporting/_materializer_frozen.py | 295 ++++++ .../reporting/_materializer_installed.py | 48 +- .../reporting/_materializer_process.py | 179 ++++ .../test_reporting_materializer_contracts.py | 12 +- .../test_reporting_materializer_durable.py | 335 ++++++ .../test_reporting_materializer_history.py | 272 +++++ ...est_reporting_materializer_installed_pg.py | 114 +++ .../test_reporting_materializer_migration.py | 274 +++++ .../test_reporting_materializer_packaging.py | 19 +- .../test_reporting_materializer_process.py | 151 +++ .../test_reporting_materializer_rolling.py | 258 +++++ .../test_reporting_materializer_service.py | 285 ++++++ ...est_reporting_materializer_transactions.py | 312 ++++++ .../test_reporting_notification_packaging.py | 121 ++- .../test_reporting_status_process_matrix.py | 3 +- .../reporting_durable_materializer.py | 32 + 36 files changed, 7065 insertions(+), 108 deletions(-) create mode 100644 docs/reporting-durable-materializer.md create mode 100644 examples/reporting_durable_materializer.py create mode 100644 src/adcp/reporting/ledger/reporting_materializer.sql create mode 100644 src/adcp/reporting/materializer/_errors.py create mode 100644 src/adcp/reporting/materializer/capture.py create mode 100644 src/adcp/reporting/materializer/memory.py create mode 100644 src/adcp/reporting/materializer/pg.py create mode 100644 src/adcp/reporting/materializer/required_schema.json create mode 100644 src/adcp/reporting/materializer/schema.py create mode 100644 src/adcp/reporting/materializer/service.py create mode 100644 src/adcp/reporting/materializer/work.py create mode 100644 tests/conformance/reporting/_durable_materializer_support.py create mode 100644 tests/conformance/reporting/_materializer_frozen.py create mode 100644 tests/conformance/reporting/_materializer_process.py create mode 100644 tests/conformance/reporting/test_reporting_materializer_durable.py create mode 100644 tests/conformance/reporting/test_reporting_materializer_history.py create mode 100644 tests/conformance/reporting/test_reporting_materializer_installed_pg.py create mode 100644 tests/conformance/reporting/test_reporting_materializer_migration.py create mode 100644 tests/conformance/reporting/test_reporting_materializer_process.py create mode 100644 tests/conformance/reporting/test_reporting_materializer_rolling.py create mode 100644 tests/conformance/reporting/test_reporting_materializer_service.py create mode 100644 tests/conformance/reporting/test_reporting_materializer_transactions.py create mode 100644 tests/type_checks/reporting_durable_materializer.py diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 958ecf34c..a8055e152 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -4,7 +4,11 @@ on: push: branches: [main, python-adcp-sdk-setup] pull_request: - branches: [main, conductor/reporting-webhook-activity-1168b, conductor/reporting-status-notifications-1168c] + branches: + - main + - conductor/reporting-webhook-activity-1168b + - conductor/reporting-status-notifications-1168c + - conductor/1167b1-materializer-contracts # Default @adcp/sdk runner alias for storyboard jobs. Tracks the current # stable @adcp/sdk release via the ``latest`` npm dist-tag. @@ -99,7 +103,7 @@ jobs: - name: Run adopter type-check suite if: matrix.python-version == '3.12' - run: mypy --strict tests/type_checks/ examples/reporting_webhook_activity.py examples/reporting_status_notifications.py examples/reporting_destination_writer.py + run: mypy --strict tests/type_checks/ examples/reporting_webhook_activity.py examples/reporting_status_notifications.py examples/reporting_destination_writer.py examples/reporting_durable_materializer.py - name: Enforce adopter type-check fixture contract if: matrix.python-version == '3.12' @@ -187,6 +191,10 @@ jobs: tests/conformance/decisioning/test_pg_reference_workflow_queue.py \ tests/conformance/reporting/ \ --ignore-glob='tests/conformance/reporting/test_reporting_status_*.py' \ + --ignore=tests/conformance/reporting/test_reporting_materializer_rolling.py \ + --ignore=tests/conformance/reporting/test_reporting_materializer_process.py \ + --ignore=tests/conformance/reporting/test_reporting_materializer_migration.py \ + --ignore=tests/conformance/reporting/test_reporting_materializer_installed_pg.py \ -v pg-reporting-status: @@ -243,6 +251,69 @@ jobs: ADCP_PG_TEST_URL: postgresql://postgres@localhost:5432/adcp_status_test run: pytest tests/conformance/reporting/test_reporting_status_*.py -v + pg-reporting-materializer: + name: Durable materializer and frozen artifacts (Postgres 16) + runs-on: ubuntu-latest + # Actual wheel builds/installations plus isolated worker processes are a + # separate bounded job; do not consume the existing conformance headroom. + timeout-minutes: 25 + services: + postgres: + image: postgres:16 + env: + POSTGRES_HOST_AUTH_METHOD: trust + POSTGRES_DB: adcp_materializer_test + POSTGRES_INITDB_ARGS: "--encoding=UTF8 --lc-collate=C --lc-ctype=C" + ports: + - 5432:5432 + options: >- + --health-cmd pg_isready + --health-interval 5s + --health-timeout 5s + --health-retries 10 + steps: + - uses: actions/checkout@v6 + - name: Fetch exact frozen reporting artifacts + timeout-minutes: 2 + run: | + git fetch --no-tags --depth=1 origin \ + 3e76aa54623529a3dda01cd690b8a5c287c75641 \ + 3c405a21f978ed9d3208611bb4a7a8434a056933 \ + 037de4ac822ecefb2f95d32c15c297fb4c45d683 \ + 21bf443e7d850d1800ec8a6f2e4abec1c8f85541 \ + 198d50e61c74fb82aedbf2c77e06a0e200b91db6 \ + ea150fabd5ad90e3abf93f89729d2919f1c61798 \ + 1c91311ec28d25506d5db43f59d0c34936ecb8f7 + - uses: actions/setup-python@v6 + id: materializer-python310 + with: + python-version: "3.10" + - uses: actions/setup-python@v6 + with: + python-version: "3.12" + cache: pip + cache-dependency-path: pyproject.toml + - name: Install test dependencies + run: pip install -e ".[dev,pg]" + - name: Run installed frozen binaries and materializer migrations + timeout-minutes: 20 + env: + ADCP_PG_TEST_URL: postgresql://postgres@localhost:5432/adcp_materializer_test + ADCP_PYTHON310: ${{ steps.materializer-python310.outputs.python-path }} + run: | + pytest tests/conformance/reporting/test_reporting_materializer_rolling.py \ + tests/conformance/reporting/test_reporting_materializer_migration.py \ + tests/conformance/reporting/test_reporting_materializer_process.py \ + tests/conformance/reporting/test_reporting_materializer_installed_pg.py \ + -v -s | tee materializer-evidence.log + - name: Preserve installed module and worker evidence + if: always() + uses: actions/upload-artifact@v7 + with: + name: materializer-evidence-${{ github.run_attempt }} + path: materializer-evidence.log + if-no-files-found: error + conventional-commits: name: Validate conventional commit format runs-on: ubuntu-latest diff --git a/docs/reporting-destination-writer.md b/docs/reporting-destination-writer.md index 895aa9ed2..0fe8b9b02 100644 --- a/docs/reporting-destination-writer.md +++ b/docs/reporting-destination-writer.md @@ -20,6 +20,11 @@ binding never advertises `managed_delivery`, `reconciled_billing`, or claim until B2 can prove a complete durable materializer. The producer's `extra` argument rejects SDK-owned task, tier and notification keys. +The additive [B2.1 durable materializer](reporting-durable-materializer.md) now +implements reservation, recovery and verified atomic finish. Production tier +and delivery activation remain gated on the later B2 slices; the development +writer and the B1 contracts above are unchanged. + ## Trusted contracts and lifecycle `ReportingDestinationRequest` includes the exact account and canonical consumer diff --git a/docs/reporting-durable-materializer.md b/docs/reporting-durable-materializer.md new file mode 100644 index 000000000..9b8fb16cf --- /dev/null +++ b/docs/reporting-durable-materializer.md @@ -0,0 +1,270 @@ +# Durable materializer — #1167B2.1 + +**B2.1 of 4 within B2 of B1/B2. Refs #1167.** This unit builds on the +independently approved B1 commit +`1c91311ec28d25506d5db43f59d0c34936ecb8f7`. It supplies durable reservation, +verified destination I/O, recovery, and one atomic finish transaction. It does +not activate a complete Managed Delivery or Reconciled Billing offering. + +The remaining, separately reviewed dependencies are B2.2 (seller revision and +adjustment receipt ingress), B2.3 (persisted combined feed and revision +ownership), and B2.4 (versioned reconciliation projection, offering readiness, +and production notification activation). They remain required inside B2. +Full Python buyer adjustment automation and `client.reporting` remain the +explicit Wave 6 prerequisite owned by the coordinator. The later #1172 +cross-language process matrix is separate. + +## Composition + +Use the [strict production-oriented example](../examples/reporting_durable_materializer.py). +Construct `PgReportingMaterializerStore(pool=pool, notifications=False)` for an +explicit polling-only deployment, or set `notifications=True` for the atomic +notification path. Install schema during deployment, then call +`compose_materializer` with the application's installed verifier registry and +trusted resolver/writer. Run `service.run_once()` repeatedly; the adopter never +enumerates accounts. The optional HTTP delivery worker is not a materializer +dependency. + +`materializer_ready()` validates storage prerequisites; it is **not** a tier or +mount readiness certificate. There is no caller-supplied production-ready +switch. B1's development writer remains ineligible, and the current capability +helpers retain C's Managed/Reconciled veto. This unit offers a production +orchestration primitive, not an activated seller offering. Core-only deployments +keep their existing stores and do not need destination or receipt components. + +Import the optional `ReportingMaterializerStore` protocol, service, lease, +turn and boundary types from `adcp.reporting.materializer`. The PostgreSQL store +is lazy; importing the public surface without `[pg]` works. Older required +store protocols, constructors, enums, and closed record decoders are unchanged. + +## Transactions and discovery + +`reporting_materializer.sql` installs only objects named `reporting_materializer_*`: +account scheduling heads, binding discovery cursors, obligation candidates, +work, captured status heads/boundaries, and isolated notification events and +expansion work. Its own `materializer/required_schema.json` validates the exact +objects. No B2 object is appended to A, B, C, or B1's mandatory manifest, and no +old guard is replaced. Old, partial, and structurally mismatched installations +fail closed. The migration is transactional, repeatable and serialized per +schema; an interrupted installation leaves no partial work schema. + +Installation seeds retained destination bindings once. A durable keyset cursor +discovers at most 32 existing obligations per turn. New binding and obligation +triggers cover either arrival order; restart never rewinds a completed cursor. +Publication, restatement, readability, configuration and materialization-check +mutations dirty indexed candidates. A consumer receipt does not schedule a +materializer retry. Exact no-op writes do not invalidate a generation. + +Claims read a bounded page of at most 16 due accounts without row locks, try the +account advisory lock, and only then lock candidate/work rows in that account. +There is no global `SKIP LOCKED` followed by an account lock. A bounded sampling +cursor advances past busy account pages and wraps; persistent served positions +keep other accounts eligible across restarts. Workers do not periodically scan +the complete ledger. PostgreSQL time controls due dates and 3–300 second leases; +tokens are random UUIDs. Account advisory locks are shared across schemas in +one database, so concurrent test groups must use different databases or run +sequentially when fixture account IDs collide. + +Reservation creates any missing immutable obligation delivery record, the next +immutable attempt, and its durable work on the same connection. Binding may +arrive after publication. Its retention floor is the later of reservation time +and period end, plus the binding's retention duration. An inactive or +deactivated configuration parks work; a future activation is scheduled, and +configuration changes wake the existing history. A binding itself is immutable; +replacement uses a new configuration generation, not an in-place destination +rewrite. Exact trusted principal/binding authorization is still checked at I/O. + +Source preparation, destination write, and verifier-controlled paginated +readback occur **outside** the final account/work transaction. A service-owned, +cancellation-safe heartbeat renews the lease while I/O runs, including with a +size-one connection pool. Each I/O phase has a deadline and joins cleanup. The +service reselects authority before preparation, immediately before write, and +before readback. Write and readback open separate freshly authorized credential +sessions. Preserve the SDK session manager and put every credential and partial +acquisition inside its redacted, non-persistable context; close happens exactly +once, including interrupted opens. + +Finish takes the account lock on one connection, checks exact schema, the +unexpired token, frozen generation/binding/request, SDK-sealed verification, and +strict current/readable revision selection from complete history. Only the SDK +readback verifier can mint the process-local immutable verification seal; +it is bound to that reservation's fencing token and revalidated under the +finish lock. Restart or a new fencing token requires a fresh readback. Private +provenance stays outside B1's three-field constructor, dataclass serialization +and Pydantic wire shape. The transaction commits all of: + +1. The immutable success or compatible safe failure. +2. Required old status dirty work and an isolated immutable boundary containing + actual captured Core and caller-private reconciliation inputs, DB `as_of`, + and monotonic caller/account sequence heads. +3. The logical readiness enqueue when enabled and this is a new verified success. +4. The fenced work ACK and next candidate schedule. + +There is no reacquired connection or external I/O in that unit. An enqueue, +capture, fence, or commit failure rolls it all back. Exact completed replay +does not enqueue, capture, or allocate again. The memory conformance store +unconditionally restores all changed collections and initialized sequence +heads on failure, with notifications either off or on. + +## Notification activation boundary + +The finish uses the existing SDK logical event builder, identity and enqueue +implementation through a closed table adapter on that same connection. The +queue retains caller namespace, deterministic logical cause/idempotency, and +account-ordered captured provenance. Its expansion-work row points to the +logical event; recipient selection/fanout is the separately crash-safe phase +from #1168A, not a synchronous finish requirement. + +**Every B2.1 reservation has immutable `admission_epoch=0`. Every event from +that work is permanently quarantined.** SQL rejects promoting its expansion +work or changing the event/work provenance. No legacy worker reads these +tables, and B2.1 installs no materializer HTTP dispatch route. Creating this +quarantined internal intent does not advertise or emit `reporting.delivery_ready`. +Failures, stale work, corruption, exact replay, and Core mode enqueue no intent. +Explicitly disabled notifications enqueue nothing. Enabled work cannot be +resumed by a replacement process configured with notifications disabled. + +B2.4 must extend the single SDK composition with real installed component and +projection readiness, an isolated activation/fence, and a compatible read path. +Only a **newly admitted reservation** after that activation can enqueue a +deliverable logical event, in its original verified-finish transaction. No +later best-effort copy/publish transaction may become the only actual enqueue. +Pending epoch-zero work retains its epoch and external identity after restart +or upgrade, even if it finishes after activation. Completed epoch-zero events +are never released, promoted, copied, or replayed as current readiness. + +This policy avoids historical delivery under a changed offering, principal, +binding, configuration generation, expired/revoked evidence, or newly selected +official revision. It also avoids inventing retries for previously successful +work just to generate an event. Captured records remain available for polling +and B2.4's explicit baseline/activation procedure. B2.1's minimal capture +primitive preserves the original finish boundary; it does not replace C's +projector or reconstruct historical boundaries from today's state. Full +versioned projection, drain/fence of incompatible C workers, and delivery +activation belong to B2.4. + +## Recovery and retention + +| Durable situation | Autonomous behavior | Operator action | +| --- | --- | --- | +| Pending; timeout, cancellation, worker/connection/lease loss, or unknown external effect | Resume the same attempt and external identity after the lease/backoff | Repair availability; do not reset the sequence | +| Own known terminal failure allowing a new attempt | Allocate N+1 only after that failure; never while N is pending | None for a retryable failure | +| Own terminal failure with no retry | Park as `operator_required` | Correct the cause and explicitly recover through supported persistence | +| Unowned legacy pending attempt | Park as `legacy_pending`, including when a later official revision exists | Drain legacy writers and prove/import the original external identity | +| Legacy terminal outcome | Preserve it; do not silently own its retry policy | Explicit recovery; ordinary public N+1 persistence remains supported | +| Current revision/readability changes during I/O | Immutable safe failure, no readiness and no artifact reuse | A new selected revision uses its own existing history; same-revision recovery increments that history | +| Fork, cycle, multiple official leaves, broken attempt history | Park as `history_corrupt`; no hot loop | Repair/import under an audited operator procedure | +| Successful artifact later becomes unreadable, revoked, unhealthy or expired | Preserve the immutable successful outcome/count; park or project degraded health | Repair authority/health or explicitly persist recovery; no automatic new attempt after success | + +Public persistence still permits attempt N+1 after **any** immutable terminal +outcome. The autonomous allocator intentionally owns a narrower retry policy. +Attempt numbers are revision-specific. Selecting a different revision never +deletes another revision's work/history; selecting the same unreadable revision +never resets its sequence. Official-required selection never falls back to a +snapshot when official evidence is missing or unreadable. Rich internal reasons +map to the existing public `MaterializationFailure` variants; the public enum +is unchanged. + +Before enabling this service, **drain every legacy materialization writer**. +For a known compatible pending effect, call +`import_pending_materialization(scope=..., reporting_materialization_id=..., +original_external_id=..., keys=...)` only after independently verifying its +original destination identity. The primitive requires the exact SDK identity, +current immutable binding/key and sole pending history. It cannot infer an +unknown or different legacy provider identity. Unknown legacy effects require +operator resolution and an explicit terminal record; do not manufacture proof +or have the service adopt them automatically. Restore the same installed key +and explicit import to wake parked owned pending work without changing its ID. + +The destination must durably enforce its advertised idempotent/conditional +write identity. No database transaction can eliminate the external-write / +outcome-commit crash window. Resume must not overwrite a different artifact. +The verifier rechecks all pages, exact digests, totals, rows, immutable locator, +retention and principal. Finish checks retention again against DB time. + +Keep attempts, outcomes, bindings, work, captured inputs and event provenance +conservatively; this unit installs no automatic garbage collector. Never purge +pending work or the external identity needed to recover it. External cleanup +is best effort and cannot authorize success, a retry, or an ACK. A cleanup +failure must not change correctness. Observe only closed `ReportingMaterializerTurn` +state/reason and opaque IDs. Never log sessions, credentials, signed URLs, +provider bodies, raw driver exceptions, or secret destination configuration. + +## Rolling compatibility envelope + +The executable gate is +`tests/conformance/reporting/test_reporting_materializer_rolling.py`. It builds +and installs actual wheels from these exact commits, verifies installed module +origins and source SHA-256 hashes, and runs them with `python -I` outside the +checkout. It first installs each native schema, then the **actual approved B1 +wheel** as comparison baseline. The same frozen binary reads populated records +and performs permitted ordinary writes both before and after B2 installation. + +| Frozen binary | Exact commit | Notification readiness on C/B1 baseline and after B2 | +| --- | --- | --- | +| beta.15 | `3e76aa54623529a3dda01cd690b8a5c287c75641` | No notification API | +| #1167A records | `3c405a21f978ed9d3208611bb4a7a8434a056933` | No notification API | +| Foundation integration | `037de4ac822ecefb2f95d32c15c297fb4c45d683` | No notification API | +| #1168A outbox | `21bf443e7d850d1800ec8a6f2e4abec1c8f85541` | Aggregate readiness closed on both | +| #1168B activity | `198d50e61c74fb82aedbf2c77e06a0e200b91db6` | Required-object readiness valid on both | +| #1168C status | `ea150fabd5ad90e3abf93f89729d2919f1c61798` | Required-object readiness valid on both | +| #1167B1 | `1c91311ec28d25506d5db43f59d0c34936ecb8f7` | Required-object readiness valid on both | + +The historical positive-readiness fixture requires PostgreSQL 16, **UTF8 with +`LC_COLLATE=C` and `LC_CTYPE=C`**. Frozen A is ready on its native schema under +that precondition. Reviewed C's additive triggers already close A's aggregate +digest on the approved C/B1 baseline. B2 must preserve that exact classification; +it is not a new waiver. A's permitted default-off Core reads/writes and existing +ordinary notification worker remain functional. The newer B/C required-object +manifests stay ready. The gate checks old catalog objects unchanged and that +every newly introduced object has the isolated prefix. + +Frozen beta.15 retains its original definition shape; monetary-unit declarations +are an A prerequisite. URL reconciliation principals became readable in C. +Earlier binary probes use their supported opaque principal with a second URL +consumer populated in the same account; C/B1 and current probes exercise URL +principals directly. This preserves the actual historical contract rather than +loosening an installed decoder assertion. Old readers do not see another +caller's materializations. Actual old ordinary/status workers consume positive +control events but cannot claim/mutate B2 event/expansion work, materializer work, +or captured boundaries/heads. + +Non-C historical aggregate portability is a separate probe with a different +precondition; it must not be presented as this positive readiness gate or as +shared-database lock contention. New B2 failures on supported old operations +are regressions, regardless of the inherited frozen-A aggregate limitation. + +Run the historical comparison with its provenance output preserved: + +```sh +ADCP_PG_TEST_URL="$REPORTING_TEST_DSN" python -m pytest -q -s \ + tests/conformance/reporting/test_reporting_materializer_rolling.py +``` + +Run the materializer PostgreSQL groups sequentially per database. The focused +shared vectors, process kills, migrations and installed-artifact gate are in +`test_reporting_materializer_{durable,transactions,history,service,process,migration,installed_pg}.py`. +Set `ADCP_PYTHON310` to a real 3.10 interpreter to run VCS and sdist-built wheels +on that interpreter; base wheel tests explicitly prohibit PostgreSQL extras. +CI includes the B1 stacked target, all Python 3.10–3.13 jobs, existing PostgreSQL +and status jobs, and a bounded dedicated UTF8/C materializer artifact job. See +the PR's exact head evidence for commands, pass/skip counts and module origins; +a skipped or absent job is not passing evidence. + +Artifact tests retain exact commit, wheel/module hashes, installed module +origins, database preconditions, and event identities in their output. Frozen +builds use compressed archives, remove extracted build-only sources after +hashing, and clean their own installed trees after dependent processes finish. +The common wheel/sdist fixture likewise removes its owned build/install tree +at teardown. Its optional `ADCP_REPORTING_DISTRIBUTION` cache is reusable only +when the complete source fingerprint matches; a changed input fails instead +of silently using an old wheel. A build failure reports a bounded allowlist of +stderr signatures and its stage/exit/digest, never arbitrary provider prose. + +For repeated local matrices, preserve evidence logs outside disposable fixture +trees, run artifact-heavy groups sequentially, and budget for the repository's +2.2 GiB schema cache while a build is active. Clear completed task-owned pytest +scratch and obsolete local `adcp` package cache entries only after checking +process ownership. Recreate contaminated temporary databases with the same +UTF8/C precondition. Disk-full or source-straddling runs are exploratory and +cannot certify the frozen candidate. diff --git a/examples/reporting_durable_materializer.py b/examples/reporting_durable_materializer.py new file mode 100644 index 000000000..c429b85f8 --- /dev/null +++ b/examples/reporting_durable_materializer.py @@ -0,0 +1,61 @@ +"""Durable seller composition; install migrations and drain legacy writers first. + +Supply the application's installed verifier registry and trusted destination +resolver/writer pair. Sessions fetch credentials afresh for each I/O phase. +This B2.1 unit preserves private polling inputs and atomic notification intents; +it does not activate Managed/Reconciled capability advertising or HTTP delivery. +""" + +from __future__ import annotations + +import asyncio + +from adcp.reporting.materializer import ( + PgReportingMaterializerStore, + ReportingDestinationIO, + ReportingDestinationResolver, + ReportingDestinationWriter, + ReportingMaterializerService, + ReportingRevisionVerifierRegistry, + ReportingWriterError, + ReportingWriterFailure, +) + + +async def compose_materializer( + store: PgReportingMaterializerStore, + *, + registry: ReportingRevisionVerifierRegistry, + resolver: ReportingDestinationResolver, + writer: ReportingDestinationWriter, +) -> ReportingMaterializerService: + """Validate installed storage, then construct the single SDK orchestration path. + + The store's notifications option is an explicit deployment choice. It is + frozen per reservation, so a replacement process cannot silently downgrade + enabled work. Readiness of the complete seller offering is proved separately + by its installed components, never by an adopter-maintained ready boolean. + """ + if not writer.production_eligible: + raise ReportingWriterError(ReportingWriterFailure("UNSUPPORTED_VERIFICATION")) + await store.materializer_ready() + return ReportingMaterializerService( + store, + ReportingDestinationIO(registry, resolver), + writer, + lease_seconds=30, + io_timeout_seconds=300, + ) + + +async def run_materializer(service: ReportingMaterializerService, stop: asyncio.Event) -> None: + """No account inventory: each turn claims bounded, fair durable work.""" + while not stop.is_set(): + turn = await service.run_once() + # Observe only these closed fields in application metrics. Destination + # credentials, signed locations and provider error text stay in sessions. + if turn.state in {"idle", "parked", "pending"}: + try: + await asyncio.wait_for(stop.wait(), timeout=1) + except asyncio.TimeoutError: + pass diff --git a/pyproject.toml b/pyproject.toml index 8913ecfc1..64997d47c 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -192,6 +192,7 @@ adcp = [ "reporting/ledger/*.sql", "reporting/outbox/*.json", "reporting/materializer/assets/*.json", + "reporting/materializer/*.json", # PREVIEW: vendored sync_reporting_status schemas. They are the runtime # validator for the wire conditionals codegen cannot express, so the wheel # must carry them. Removed with the rest of _preview/ at rc.2. diff --git a/src/adcp/reporting/ledger/delivery.py b/src/adcp/reporting/ledger/delivery.py index 8f854e211..c5dab5a45 100644 --- a/src/adcp/reporting/ledger/delivery.py +++ b/src/adcp/reporting/ledger/delivery.py @@ -404,35 +404,42 @@ class InMemoryReportingReconciliationStore(InMemoryReportingLedgerStore, _Reconc _delivery_records: list[tuple[int, ReportingDeliveryPrincipal, ReportingDeliveryRecord]] async def _commit(self, record: RecordT) -> tuple[RecordT, bool]: + candidate = cast(RecordT, decode_record(payload(record))) + async with self._mutation(): + return self._commit_record_unlocked(candidate) + + def _commit_record_unlocked( + self, record: RecordT, *, notify: bool = True + ) -> tuple[RecordT, bool]: + """Caller owns the memory mutation; no lock or callback is acquired here.""" candidate = decode_record(payload(record)) who = principal(candidate) - async with self._mutation(): - records = tuple(item.record for item in self._caller_changes(who)) - existing = replay(candidate, records) - if existing is not None: - return cast(RecordT, existing), False - context = self._delivery_context(candidate) - stored = validate_transition(candidate, records, context, self._clock()) - self._append_reconciliation_change(stored) - if self._notification_state is not None: - from adcp.reporting.ledger.notification_events import ( - delivery_dirty, - materialization_event, - ) + records = tuple(item.record for item in self._caller_changes(who)) + existing = replay(candidate, records) + if existing is not None: + return cast(RecordT, existing), False + context = self._delivery_context(candidate) + stored = validate_transition(candidate, records, context, self._clock()) + self._append_reconciliation_change(stored) + if notify and self._notification_state is not None: + from adcp.reporting.ledger.notification_events import ( + delivery_dirty, + materialization_event, + ) - event = materialization_event( - stored, - records, - context.obligation, - context.revision, - context.configuration, - self._clock(), - ) - if event is not None: - self._record_notification(event) - scope, reason, evidence = delivery_dirty(stored, context.obligation) - self._dirty_status(scope, reason, after=evidence) - return cast(RecordT, stored), True + event = materialization_event( + stored, + records, + context.obligation, + context.revision, + context.configuration, + self._clock(), + ) + if event is not None: + self._record_notification(event) + scope, reason, evidence = delivery_dirty(stored, context.obligation) + self._dirty_status(scope, reason, after=evidence) + return cast(RecordT, stored), True def _append_reconciliation_change(self, record: ReportingDeliveryRecord) -> None: who = principal(record) diff --git a/src/adcp/reporting/ledger/delivery_pg.py b/src/adcp/reporting/ledger/delivery_pg.py index a3fc3ec5b..7c977a74d 100644 --- a/src/adcp/reporting/ledger/delivery_pg.py +++ b/src/adcp/reporting/ledger/delivery_pg.py @@ -110,46 +110,57 @@ async def _commit(self, record: RecordT) -> tuple[RecordT, bool]: unavailable() async def _commit_record(self, record: RecordT) -> tuple[RecordT, bool]: - candidate = decode_record(payload(record)) + candidate = cast(RecordT, decode_record(payload(record))) who = principal(candidate) async with self._connection() as connection: async with connection.transaction(): await self._lock_account(connection, who.account_id) - records = await self._records(connection, who) - existing = replay(candidate, records) - if existing is not None: - return cast(RecordT, existing), False - context = await self._delivery_context(connection, candidate) - if self._clock is not None: - now = self._clock() - else: - time_row = await ( - await connection.execute("SELECT clock_timestamp()") - ).fetchone() - assert time_row is not None - now = time_row[0] - stored = validate_transition(candidate, records, context, now) - await self._insert(connection, stored) - await self._append_reconciliation_change(connection, stored) - if self._notifications_enabled: - from adcp.reporting.ledger.notification_events import ( - delivery_dirty, - materialization_event, - ) + return await self._commit_record_on(connection, candidate) - event = materialization_event( - stored, - records, - context.obligation, - context.revision, - context.configuration, - now, - ) - if event is not None: - await self._record_notification(connection, event) - scope, reason, evidence = delivery_dirty(stored, context.obligation) - await self._dirty_status(connection, scope, reason, after=evidence) - return cast(RecordT, stored), True + async def _commit_record_on( + self, connection: Any, record: RecordT, *, notify: bool = True + ) -> tuple[RecordT, bool]: + """Connection-bound primitive. Caller holds the account transaction lock. + + Materializer finish uses this exact connection for evidence, caller + feed, projection dirty work and its acknowledgment. + No connection is acquired and no external code runs here. + """ + candidate = decode_record(payload(record)) + who = principal(candidate) + records = await self._records(connection, who) + existing = replay(candidate, records) + if existing is not None: + return cast(RecordT, existing), False + context = await self._delivery_context(connection, candidate) + if self._clock is not None: + now = self._clock() + else: + time_row = await (await connection.execute("SELECT clock_timestamp()")).fetchone() + assert time_row is not None + now = time_row[0] + stored = validate_transition(candidate, records, context, now) + await self._insert(connection, stored) + await self._append_reconciliation_change(connection, stored) + if notify and self._notifications_enabled: + from adcp.reporting.ledger.notification_events import ( + delivery_dirty, + materialization_event, + ) + + event = materialization_event( + stored, + records, + context.obligation, + context.revision, + context.configuration, + now, + ) + if event is not None: + await self._record_notification(connection, event) + scope, reason, evidence = delivery_dirty(stored, context.obligation) + await self._dirty_status(connection, scope, reason, after=evidence) + return cast(RecordT, stored), True async def _append_reconciliation_change( self, connection: Any, record: ReportingDeliveryRecord diff --git a/src/adcp/reporting/ledger/reporting_materializer.sql b/src/adcp/reporting/ledger/reporting_materializer.sql new file mode 100644 index 000000000..d27331117 --- /dev/null +++ b/src/adcp/reporting/ledger/reporting_materializer.sql @@ -0,0 +1,420 @@ +-- #1167B2. Isolated, additive work objects; no A/B/C/B1 guard is replaced. +-- A single statement also makes autocommit/repeated/concurrent installation atomic. +DO $materializer$ +BEGIN + PERFORM pg_advisory_xact_lock(hashtext('adcp.reporting.schema'), hashtext(current_schema())); + PERFORM account_id, consumer_id FROM reporting_reconciliation_records LIMIT 0; + + CREATE TABLE IF NOT EXISTS reporting_materializer_accounts ( + account_id TEXT COLLATE "C" PRIMARY KEY, + due_at TIMESTAMPTZ, + served_at TIMESTAMPTZ NOT NULL DEFAULT '-infinity', + captured_sequence BIGINT NOT NULL DEFAULT 0 CHECK (captured_sequence >= 0) + ); + CREATE INDEX IF NOT EXISTS reporting_materializer_account_due + ON reporting_materializer_accounts (served_at, account_id) WHERE due_at IS NOT NULL; + CREATE INDEX IF NOT EXISTS reporting_materializer_account_wakeup + ON reporting_materializer_accounts (due_at, served_at, account_id) WHERE due_at IS NOT NULL; + + CREATE TABLE IF NOT EXISTS reporting_materializer_discovery ( + account_id TEXT COLLATE "C" NOT NULL, + consumer_id TEXT COLLATE "C" NOT NULL, + delivery_config_id TEXT COLLATE "C" NOT NULL, + delivery_config_version BIGINT NOT NULL CHECK (delivery_config_version > 0), + after_obligation_id TEXT COLLATE "C" NOT NULL DEFAULT '', + complete BOOLEAN NOT NULL DEFAULT FALSE, + PRIMARY KEY (account_id, consumer_id, delivery_config_id, delivery_config_version), + FOREIGN KEY (account_id, delivery_config_id, delivery_config_version) + REFERENCES reporting_configurations(account_id, delivery_config_id, delivery_config_version) + ); + CREATE INDEX IF NOT EXISTS reporting_materializer_discovery_pending + ON reporting_materializer_discovery (account_id, consumer_id, delivery_config_id, + delivery_config_version) WHERE NOT complete; + CREATE INDEX IF NOT EXISTS reporting_materializer_obligation_discovery + ON reporting_obligations (account_id, delivery_config_id, delivery_config_version, + reporting_obligation_id); + CREATE INDEX IF NOT EXISTS reporting_materializer_binding_discovery + ON reporting_reconciliation_records (account_id, consumer_id, delivery_config_id, + delivery_config_version) WHERE record_kind='destination_binding'; + + CREATE TABLE IF NOT EXISTS reporting_materializer_candidates ( + account_id TEXT COLLATE "C" NOT NULL REFERENCES reporting_materializer_accounts, + consumer_id TEXT COLLATE "C" NOT NULL, + delivery_config_id TEXT COLLATE "C" NOT NULL, + delivery_config_version BIGINT NOT NULL CHECK (delivery_config_version > 0), + reporting_obligation_id TEXT COLLATE "C" NOT NULL, + generation BIGINT NOT NULL DEFAULT 1 CHECK (generation > 0), + due_at TIMESTAMPTZ, + reason TEXT COLLATE "C" NOT NULL DEFAULT 'ready' CHECK (reason IN ( + 'ready','verified','retry','inactive','revision_not_ready','revision_unreadable', + 'target_changed','history_corrupt','legacy_pending','legacy_terminal', + 'component_unavailable','binding_changed','effect_unknown','operator_required')), + served_at TIMESTAMPTZ NOT NULL DEFAULT '-infinity', + PRIMARY KEY (account_id, consumer_id, delivery_config_id, delivery_config_version, + reporting_obligation_id), + FOREIGN KEY (account_id, delivery_config_id, delivery_config_version, reporting_obligation_id) + REFERENCES reporting_obligations(account_id, delivery_config_id, + delivery_config_version, reporting_obligation_id) + ); + CREATE INDEX IF NOT EXISTS reporting_materializer_candidate_due + ON reporting_materializer_candidates (account_id, due_at, served_at, + consumer_id, reporting_obligation_id) WHERE due_at IS NOT NULL; + CREATE INDEX IF NOT EXISTS reporting_materializer_candidate_publication + ON reporting_materializer_candidates (account_id, reporting_obligation_id); + + CREATE TABLE IF NOT EXISTS reporting_materializer_work ( + account_id TEXT COLLATE "C" NOT NULL, + consumer_id TEXT COLLATE "C" NOT NULL, + delivery_config_id TEXT COLLATE "C" NOT NULL, + delivery_config_version BIGINT NOT NULL, + reporting_obligation_id TEXT COLLATE "C" NOT NULL, + reporting_revision_id TEXT COLLATE "C" NOT NULL, + reporting_materialization_id TEXT COLLATE "C" NOT NULL, + attempt_namespace TEXT COLLATE "C" NOT NULL DEFAULT 'materialization_attempt' + CHECK (attempt_namespace = 'materialization_attempt'), + generation BIGINT NOT NULL CHECK (generation > 0), + binding_sha256 TEXT COLLATE "C" NOT NULL CHECK (binding_sha256 ~ '^[0-9a-f]{64}$'), + verification_key_sha256 TEXT COLLATE "C" NOT NULL + CHECK (verification_key_sha256 ~ '^[0-9a-f]{64}$'), + external_id TEXT COLLATE "C" NOT NULL CHECK (external_id ~ '^rwm_[0-9a-f]{64}$'), + state TEXT COLLATE "C" NOT NULL DEFAULT 'pending' CHECK (state IN ('pending','acked')), + retry_allowed BOOLEAN NOT NULL DEFAULT FALSE, + reason TEXT COLLATE "C" NOT NULL DEFAULT 'ready' CHECK (reason IN ( + 'ready','verified','retry','inactive','revision_not_ready','revision_unreadable', + 'target_changed','history_corrupt','legacy_pending','legacy_terminal', + 'component_unavailable','binding_changed','effect_unknown','operator_required')), + created_at TIMESTAMPTZ NOT NULL DEFAULT clock_timestamp(), + acknowledged_at TIMESTAMPTZ, + completion_token UUID, + lease_token UUID, + lease_until TIMESTAMPTZ, + due_at TIMESTAMPTZ NOT NULL DEFAULT clock_timestamp(), + imported BOOLEAN NOT NULL DEFAULT FALSE, + notifications_enabled BOOLEAN NOT NULL, + -- No B2.1 reservation is production-admitted. Future activation must + -- preserve this epoch on retained work, including pending resumes. + admission_epoch BIGINT NOT NULL DEFAULT 0 CHECK (admission_epoch = 0), + PRIMARY KEY (account_id, consumer_id, reporting_materialization_id), + UNIQUE (account_id, consumer_id, external_id), + FOREIGN KEY (account_id, consumer_id, delivery_config_id, delivery_config_version, + reporting_obligation_id) REFERENCES reporting_materializer_candidates, + FOREIGN KEY (account_id, consumer_id, attempt_namespace, reporting_materialization_id) + REFERENCES reporting_reconciliation_records(account_id, consumer_id, namespace, record_id), + FOREIGN KEY (account_id, reporting_obligation_id, reporting_revision_id) + REFERENCES reporting_revisions(account_id, reporting_obligation_id, reporting_revision_id), + CHECK ((lease_token IS NULL) = (lease_until IS NULL)), + CHECK ((state = 'acked') = (acknowledged_at IS NOT NULL)), + CHECK ((state = 'acked') = (completion_token IS NOT NULL)), + CHECK (state <> 'acked' OR lease_token IS NULL), + CHECK (NOT retry_allowed OR state = 'acked') + ); + CREATE UNIQUE INDEX IF NOT EXISTS reporting_materializer_one_pending + ON reporting_materializer_work (account_id, consumer_id, delivery_config_id, + delivery_config_version, reporting_obligation_id) WHERE state = 'pending'; + CREATE INDEX IF NOT EXISTS reporting_materializer_work_due + ON reporting_materializer_work (account_id, due_at, reporting_materialization_id) + WHERE state = 'pending'; + + CREATE TABLE IF NOT EXISTS reporting_materializer_status_heads ( + account_id TEXT COLLATE "C" NOT NULL, + consumer_id TEXT COLLATE "C" NOT NULL, + max_sequence BIGINT NOT NULL CHECK (max_sequence > 0), + PRIMARY KEY (account_id, consumer_id) + ); + CREATE TABLE IF NOT EXISTS reporting_materializer_status_boundaries ( + account_id TEXT COLLATE "C" NOT NULL, + consumer_id TEXT COLLATE "C" NOT NULL, + sequence BIGINT NOT NULL CHECK (sequence > 0), + account_sequence BIGINT NOT NULL CHECK (account_sequence > 0), + reporting_materialization_id TEXT COLLATE "C" NOT NULL, + outcome_namespace TEXT COLLATE "C" NOT NULL DEFAULT 'materialization' + CHECK (outcome_namespace = 'materialization'), + as_of TIMESTAMPTZ NOT NULL, + input JSONB NOT NULL, + content_sha256 TEXT COLLATE "C" NOT NULL CHECK (content_sha256 ~ '^[0-9a-f]{64}$'), + PRIMARY KEY (account_id, consumer_id, sequence), + UNIQUE (account_id, consumer_id, reporting_materialization_id), + UNIQUE (account_id, account_sequence), + FOREIGN KEY (account_id, consumer_id, reporting_materialization_id) + REFERENCES reporting_materializer_work, + FOREIGN KEY (account_id, consumer_id, outcome_namespace, reporting_materialization_id) + REFERENCES reporting_reconciliation_records(account_id, consumer_id, namespace, record_id), + CHECK ((input->>'version')::integer IS NOT DISTINCT FROM 1), + CHECK ((input->>'account_id') IS NOT DISTINCT FROM account_id), + CHECK ((input->>'consumer_id') IS NOT DISTINCT FROM consumer_id), + CHECK ((input->>'reporting_materialization_id') IS NOT DISTINCT FROM reporting_materialization_id), + CHECK ((input->>'sequence')::bigint IS NOT DISTINCT FROM sequence), + CHECK ((input->>'account_sequence')::bigint IS NOT DISTINCT FROM account_sequence), + CHECK ((input->>'as_of')::timestamptz IS NOT DISTINCT FROM as_of), + CHECK (content_sha256 = reporting_payload_sha256(input)), + CHECK (input - ARRAY['version','account_id','consumer_id','reporting_materialization_id', + 'sequence','account_sequence','as_of','core','reconciliation'] = '{}'::jsonb) + ); + CREATE TABLE IF NOT EXISTS reporting_materializer_notification_events ( + account_id TEXT COLLATE "C" NOT NULL, + notification_id TEXT COLLATE "C" NOT NULL, + notification_type TEXT COLLATE "C" NOT NULL CHECK (notification_type='reporting.delivery_ready'), + cause_kind TEXT COLLATE "C" NOT NULL CHECK (cause_kind='materialization_ready'), + cause_id TEXT COLLATE "C" NOT NULL, + cause_generation BIGINT NOT NULL CHECK (cause_generation=1), + consumer_namespace TEXT COLLATE "C" NOT NULL CHECK (length(consumer_namespace)>0), + admission_epoch BIGINT NOT NULL DEFAULT 0 CHECK (admission_epoch = 0), + fired_at TIMESTAMPTZ NOT NULL, + snapshot JSONB NOT NULL, + reporting_materialization_id TEXT COLLATE "C" GENERATED ALWAYS AS + (snapshot #>> '{cause,reporting_materialization_id}') STORED, + PRIMARY KEY (account_id, consumer_namespace, notification_id), + UNIQUE (account_id, consumer_namespace, notification_type, cause_kind, cause_id, cause_generation), + FOREIGN KEY (account_id, consumer_namespace, reporting_materialization_id) + REFERENCES reporting_materializer_status_boundaries + (account_id, consumer_id, reporting_materialization_id), + CHECK ((snapshot->>'account_id') IS NOT DISTINCT FROM account_id), + CHECK ((snapshot->>'notification_id') IS NOT DISTINCT FROM notification_id), + CHECK (cause_id::jsonb IS NOT DISTINCT FROM + jsonb_build_array(consumer_namespace, reporting_materialization_id)), + CHECK ((snapshot #>> '{cause,consumer_id}') IS NOT DISTINCT FROM consumer_namespace) + ); + CREATE TABLE IF NOT EXISTS reporting_materializer_notification_expansions ( + account_id TEXT COLLATE "C" NOT NULL, + consumer_namespace TEXT COLLATE "C" NOT NULL, + notification_id TEXT COLLATE "C" NOT NULL, + emission_generation BIGINT NOT NULL CHECK (emission_generation>0), + state TEXT NOT NULL DEFAULT 'quarantined' CHECK + (state IN ('pending','leased','complete','suppressed','quarantined')), + due_at TIMESTAMPTZ NOT NULL, + lease_token TEXT, + lease_expires_at TIMESTAMPTZ, + claim_count BIGINT NOT NULL DEFAULT 0, + error_code TEXT CHECK (error_code IN ( + 'network','retryable_http','permanent_http','signing_unavailable', + 'permanent_scope','subscription_unavailable','subscription_changed', + 'invalid_configuration','invalid_payload','integrity_failure','lease_expired')), + PRIMARY KEY (account_id, consumer_namespace, notification_id, emission_generation), + FOREIGN KEY (account_id, consumer_namespace, notification_id) + REFERENCES reporting_materializer_notification_events + ); + CREATE INDEX IF NOT EXISTS reporting_materializer_notification_due + ON reporting_materializer_notification_expansions (account_id, due_at) + WHERE state IN ('pending','leased'); + + EXECUTE $function$ + CREATE OR REPLACE FUNCTION reporting_materializer_expansion_guard() RETURNS trigger + LANGUAGE plpgsql AS $body$ + DECLARE epoch BIGINT; + BEGIN + SELECT admission_epoch INTO epoch FROM reporting_materializer_notification_events + WHERE account_id=NEW.account_id AND consumer_namespace=NEW.consumer_namespace + AND notification_id=NEW.notification_id; + IF epoch=0 AND NEW.state <> 'quarantined' THEN + RAISE EXCEPTION 'reporting_materializer_pre_activation_event' + USING ERRCODE='23514'; + END IF; + RETURN NEW; + END + $body$ $function$; + IF NOT EXISTS (SELECT 1 FROM pg_trigger + WHERE tgrelid='reporting_materializer_notification_expansions'::regclass + AND tgname='reporting_materializer_pre_activation_guard') THEN + CREATE TRIGGER reporting_materializer_pre_activation_guard BEFORE INSERT OR UPDATE + ON reporting_materializer_notification_expansions FOR EACH ROW + EXECUTE FUNCTION reporting_materializer_expansion_guard(); + END IF; + + EXECUTE $function$ + CREATE OR REPLACE FUNCTION reporting_materializer_retained_guard() RETURNS trigger + LANGUAGE plpgsql AS $body$ + BEGIN + RAISE EXCEPTION 'reporting_materializer_evidence_immutable' USING ERRCODE='23514'; + END + $body$ $function$; + IF NOT EXISTS (SELECT 1 FROM pg_trigger WHERE tgrelid='reporting_materializer_status_boundaries'::regclass + AND tgname='reporting_materializer_boundary_immutable') THEN + CREATE TRIGGER reporting_materializer_boundary_immutable BEFORE UPDATE OR DELETE + ON reporting_materializer_status_boundaries FOR EACH ROW + EXECUTE FUNCTION reporting_materializer_retained_guard(); + END IF; + IF NOT EXISTS (SELECT 1 FROM pg_trigger WHERE tgrelid='reporting_materializer_notification_events'::regclass + AND tgname='reporting_materializer_event_immutable') THEN + CREATE TRIGGER reporting_materializer_event_immutable BEFORE UPDATE OR DELETE + ON reporting_materializer_notification_events FOR EACH ROW + EXECUTE FUNCTION reporting_materializer_retained_guard(); + END IF; + + EXECUTE $function$ + CREATE OR REPLACE FUNCTION reporting_materializer_wake(a TEXT) RETURNS void + LANGUAGE sql AS $body$ + INSERT INTO reporting_materializer_accounts (account_id, due_at) + VALUES (a, clock_timestamp()) ON CONFLICT (account_id) DO UPDATE + SET due_at = LEAST(reporting_materializer_accounts.due_at, EXCLUDED.due_at) + $body$ $function$; + + EXECUTE $function$ + CREATE OR REPLACE FUNCTION reporting_materializer_dirty( + a TEXT, c TEXT, d TEXT, v BIGINT, o TEXT + ) RETURNS void LANGUAGE plpgsql AS $body$ + BEGIN + PERFORM pg_advisory_xact_lock(hashtext('adcp.reporting:' || a)); + PERFORM reporting_materializer_wake(a); + INSERT INTO reporting_materializer_candidates + (account_id, consumer_id, delivery_config_id, delivery_config_version, + reporting_obligation_id, due_at) + VALUES (a,c,d,v,o,clock_timestamp()) + ON CONFLICT (account_id, consumer_id, delivery_config_id, delivery_config_version, + reporting_obligation_id) DO UPDATE + SET generation = reporting_materializer_candidates.generation + 1, + due_at = clock_timestamp(), reason = 'ready'; + END + $body$ $function$; + + EXECUTE $function$ + CREATE OR REPLACE FUNCTION reporting_materializer_source_dirty() RETURNS trigger + LANGUAGE plpgsql AS $body$ + DECLARE r RECORD; + BEGIN + IF TG_OP = 'UPDATE' AND NEW IS NOT DISTINCT FROM OLD THEN + RETURN NEW; + END IF; + PERFORM pg_advisory_xact_lock(hashtext('adcp.reporting:' || NEW.account_id)); + IF TG_TABLE_NAME = 'reporting_configurations' THEN + UPDATE reporting_materializer_candidates SET generation = generation + 1, + due_at = clock_timestamp(), reason = 'ready' + WHERE account_id = NEW.account_id AND delivery_config_id = NEW.delivery_config_id + AND delivery_config_version = NEW.delivery_config_version; + IF FOUND THEN PERFORM reporting_materializer_wake(NEW.account_id); END IF; + ELSIF TG_TABLE_NAME = 'reporting_revisions' THEN + UPDATE reporting_materializer_candidates SET generation = generation + 1, + due_at = clock_timestamp(), reason = 'ready' + WHERE account_id = NEW.account_id AND reporting_obligation_id = NEW.reporting_obligation_id; + IF FOUND THEN PERFORM reporting_materializer_wake(NEW.account_id); END IF; + ELSE + FOR r IN SELECT consumer_id FROM reporting_materializer_discovery + WHERE account_id = NEW.account_id AND delivery_config_id = NEW.delivery_config_id + AND delivery_config_version = NEW.delivery_config_version + ORDER BY consumer_id + LOOP + PERFORM reporting_materializer_dirty(NEW.account_id, r.consumer_id, + NEW.delivery_config_id, NEW.delivery_config_version, NEW.reporting_obligation_id); + END LOOP; + END IF; + RETURN NEW; + END + $body$ $function$; + + EXECUTE $function$ + CREATE OR REPLACE FUNCTION reporting_materializer_binding_dirty() RETURNS trigger + LANGUAGE plpgsql AS $body$ + BEGIN + IF NEW.record_kind = 'destination_binding' THEN + PERFORM pg_advisory_xact_lock(hashtext('adcp.reporting:' || NEW.account_id)); + INSERT INTO reporting_materializer_discovery + (account_id, consumer_id, delivery_config_id, delivery_config_version) + VALUES (NEW.account_id, NEW.consumer_id, NEW.delivery_config_id, NEW.delivery_config_version) + ON CONFLICT DO NOTHING; + PERFORM reporting_materializer_wake(NEW.account_id); + ELSIF NEW.record_kind = 'materialization_check' THEN + PERFORM reporting_materializer_dirty(NEW.account_id, NEW.consumer_id, + NEW.delivery_config_id, NEW.delivery_config_version, NEW.reporting_obligation_id); + END IF; + -- Consumer rejection is never a materializer retry signal. + RETURN NEW; + END + $body$ $function$; + + EXECUTE $function$ + CREATE OR REPLACE FUNCTION reporting_materializer_work_guard() RETURNS trigger + LANGUAGE plpgsql AS $body$ + DECLARE a JSONB; result RECORD; + BEGIN + IF TG_OP = 'DELETE' THEN + RAISE EXCEPTION 'reporting_materializer_history_immutable' USING ERRCODE='23514'; + END IF; + IF TG_OP = 'UPDATE' AND ( + (to_jsonb(NEW) - ARRAY['lease_token','lease_until','due_at','state','retry_allowed', + 'reason','acknowledged_at','completion_token']) IS DISTINCT FROM + (to_jsonb(OLD) - ARRAY['lease_token','lease_until','due_at','state','retry_allowed', + 'reason','acknowledged_at','completion_token']) OR OLD.state = 'acked' + ) THEN + RAISE EXCEPTION 'reporting_materializer_identity_immutable' USING ERRCODE='23514'; + END IF; + SELECT payload INTO a FROM reporting_reconciliation_records + WHERE account_id = NEW.account_id AND consumer_id = NEW.consumer_id + AND namespace = 'materialization_attempt' AND record_id = NEW.reporting_materialization_id; + IF a IS NULL OR a->>'reporting_revision_id' IS DISTINCT FROM NEW.reporting_revision_id + OR a #>> '{scope,reporting_obligation_id}' IS DISTINCT FROM NEW.reporting_obligation_id + OR a #>> '{scope,generation_key,delivery_config_id}' IS DISTINCT FROM NEW.delivery_config_id + OR (a #>> '{scope,generation_key,delivery_config_version}')::bigint + IS DISTINCT FROM NEW.delivery_config_version THEN + RAISE EXCEPTION 'reporting_materializer_attempt_mismatch' USING ERRCODE='23514'; + END IF; + IF NEW.state = 'acked' THEN + IF TG_OP <> 'UPDATE' OR OLD.state <> 'pending' OR OLD.lease_token IS NULL + OR OLD.lease_until <= clock_timestamp() + OR NEW.completion_token IS DISTINCT FROM OLD.lease_token THEN + RAISE EXCEPTION 'reporting_materializer_fence_required' USING ERRCODE='23514'; + END IF; + SELECT payload INTO result FROM reporting_reconciliation_records + WHERE account_id = NEW.account_id AND consumer_id = NEW.consumer_id + AND namespace = 'materialization' AND record_id = NEW.reporting_materialization_id; + IF NOT FOUND OR (NEW.retry_allowed AND result.payload->>'status' <> 'failed') THEN + RAISE EXCEPTION 'reporting_materializer_outcome_required' USING ERRCODE='23514'; + END IF; + IF NOT EXISTS (SELECT 1 FROM reporting_materializer_status_boundaries b + WHERE b.account_id=NEW.account_id AND b.consumer_id=NEW.consumer_id + AND b.reporting_materialization_id=NEW.reporting_materialization_id + AND b.as_of=NEW.acknowledged_at + AND b.as_of=(result.payload->>'completed_at')::timestamptz) THEN + RAISE EXCEPTION 'reporting_materializer_capture_required' USING ERRCODE='23514'; + END IF; + IF NEW.notifications_enabled AND result.payload->>'status' <> 'failed' + AND NOT EXISTS (SELECT 1 FROM reporting_materializer_notification_events e + WHERE e.account_id=NEW.account_id AND e.consumer_namespace=NEW.consumer_id + AND e.reporting_materialization_id=NEW.reporting_materialization_id + AND e.admission_epoch=NEW.admission_epoch) THEN + RAISE EXCEPTION 'reporting_materializer_event_required' USING ERRCODE='23514'; + END IF; + END IF; + RETURN NEW; + END + $body$ $function$; + + IF NOT EXISTS (SELECT 1 FROM pg_trigger WHERE tgrelid='reporting_reconciliation_records'::regclass + AND tgname='reporting_materializer_binding') THEN + CREATE TRIGGER reporting_materializer_binding AFTER INSERT ON reporting_reconciliation_records + FOR EACH ROW EXECUTE FUNCTION reporting_materializer_binding_dirty(); + END IF; + IF NOT EXISTS (SELECT 1 FROM pg_trigger WHERE tgrelid='reporting_obligations'::regclass + AND tgname='reporting_materializer_obligation') THEN + CREATE TRIGGER reporting_materializer_obligation AFTER INSERT ON reporting_obligations + FOR EACH ROW EXECUTE FUNCTION reporting_materializer_source_dirty(); + END IF; + IF NOT EXISTS (SELECT 1 FROM pg_trigger WHERE tgrelid='reporting_revisions'::regclass + AND tgname='reporting_materializer_publication') THEN + CREATE TRIGGER reporting_materializer_publication AFTER INSERT OR UPDATE ON reporting_revisions + FOR EACH ROW EXECUTE FUNCTION reporting_materializer_source_dirty(); + END IF; + IF NOT EXISTS (SELECT 1 FROM pg_trigger WHERE tgrelid='reporting_configurations'::regclass + AND tgname='reporting_materializer_configuration') THEN + CREATE TRIGGER reporting_materializer_configuration AFTER UPDATE ON reporting_configurations + FOR EACH ROW EXECUTE FUNCTION reporting_materializer_source_dirty(); + END IF; + IF NOT EXISTS (SELECT 1 FROM pg_trigger WHERE tgrelid='reporting_materializer_work'::regclass + AND tgname='reporting_materializer_guard') THEN + CREATE TRIGGER reporting_materializer_guard BEFORE INSERT OR UPDATE OR DELETE + ON reporting_materializer_work FOR EACH ROW + EXECUTE FUNCTION reporting_materializer_work_guard(); + END IF; + + -- One-time indexed backfill seeds bindings, not account enumeration or + -- recurring whole-ledger scans. Reinstall never rewinds a discovery cursor. + INSERT INTO reporting_materializer_discovery + (account_id, consumer_id, delivery_config_id, delivery_config_version) + SELECT account_id, consumer_id, delivery_config_id, delivery_config_version + FROM reporting_reconciliation_records WHERE record_kind='destination_binding' + ON CONFLICT DO NOTHING; + INSERT INTO reporting_materializer_accounts (account_id, due_at) + SELECT account_id, clock_timestamp() FROM reporting_materializer_discovery WHERE NOT complete + GROUP BY account_id + ON CONFLICT (account_id) DO UPDATE + SET due_at=LEAST(reporting_materializer_accounts.due_at, EXCLUDED.due_at); +END +$materializer$; diff --git a/src/adcp/reporting/ledger/store.py b/src/adcp/reporting/ledger/store.py index 19949bc45..3c12e2041 100644 --- a/src/adcp/reporting/ledger/store.py +++ b/src/adcp/reporting/ledger/store.py @@ -679,26 +679,18 @@ def __init__( async def _mutation(self) -> AsyncIterator[None]: """Publish domain changes and notifications under one rollback boundary. - Default-off stores retain their original lock cost. The reference - in-memory transaction copies retained values only when opted in. + Rollback also applies with notifications disabled. Newly initialized + collections and sequence heads belong to the transaction too. """ owner = (id(self), asyncio.current_task()) if _MEMORY_TRANSACTION.get() == owner: yield return async with self._lock: - token = _MEMORY_TRANSACTION.set(owner) - before = ( - deepcopy( - { - key: value - for key, value in vars(self).items() - if key not in {"_lock", "_clock"} - } - ) - if self._notification_state is not None - else None + before = deepcopy( + {key: value for key, value in vars(self).items() if key not in {"_lock", "_clock"}} ) + token = _MEMORY_TRANSACTION.set(owner) try: dirty_start = len(self._notification_state.dirty) if self._notification_state else 0 yield @@ -720,8 +712,9 @@ async def _mutation(self) -> AsyncIterator[None]: ) ) except BaseException: - if before is not None: - vars(self).update(before) + for key in set(vars(self)) - {"_lock", "_clock"} - set(before): + del vars(self)[key] + vars(self).update(before) raise finally: _MEMORY_TRANSACTION.reset(token) diff --git a/src/adcp/reporting/materializer/__init__.py b/src/adcp/reporting/materializer/__init__.py index 165d6d3d8..28c7edf26 100644 --- a/src/adcp/reporting/materializer/__init__.py +++ b/src/adcp/reporting/materializer/__init__.py @@ -1,11 +1,14 @@ -"""B1 public destination contracts and verification; no durable Managed service. +"""Destination contracts, verification and optional durable materialization. Use the immutable registry to prepare all frozen source rows, then invoke write and verify explicitly with separate authorization sessions. The reference -writer is exclusively for tests/development. B2 owns durable work, fencing, -retry allocation, final target reselection, and readiness transactions. +writer is exclusively for tests/development. The durable service owns fencing, +retry allocation, target reselection and atomic finish. Production tier and +notification activation additionally require the complete seller projection. """ +from typing import TYPE_CHECKING + from adcp.reporting.ledger.delivery_models import ( ReportingDeliveryPrincipal, ReportingDestinationBinding, @@ -17,6 +20,7 @@ parse_reporting_json, strict_reporting_json, ) +from adcp.reporting.materializer.capture import ReportingMaterializerBoundary from adcp.reporting.materializer.contracts import ( ReportingCanonicalization, ReportingDestinationLocator, @@ -38,12 +42,14 @@ ReportingWriterFailureCode, ReportingWriterRetry, ) +from adcp.reporting.materializer.memory import InMemoryReportingMaterializerStore from adcp.reporting.materializer.reference import ( ReferenceReportingDestinationWriter, ReferenceReportingResolver, reference_digest, reference_verifier, ) +from adcp.reporting.materializer.service import ReportingMaterializerService from adcp.reporting.materializer.verification import ( ReportingDestinationIO, ReportingRevisionRowReader, @@ -52,8 +58,20 @@ ReportingVerifiedDestination, validate_materialization_target, ) +from adcp.reporting.materializer.work import ( + MaterializerReason, + ReportingMaterializerLease, + ReportingMaterializerStore, + ReportingMaterializerTurn, +) + +if TYPE_CHECKING: + from adcp.reporting.materializer.pg import PgReportingMaterializerStore __all__ = [ + "InMemoryReportingMaterializerStore", + "MaterializerReason", + "PgReportingMaterializerStore", "ReferenceReportingDestinationWriter", "ReferenceReportingResolver", "ReportingCanonicalization", @@ -71,6 +89,11 @@ "ReportingIOContext", "ReportingIOPhase", "ReportingMaterializationAttempt", + "ReportingMaterializerBoundary", + "ReportingMaterializerLease", + "ReportingMaterializerService", + "ReportingMaterializerStore", + "ReportingMaterializerTurn", "ReportingNativeObservation", "ReportingObligationDeliveryRecord", "ReportingPreparedRevision", @@ -91,3 +114,11 @@ "strict_reporting_json", "validate_materialization_target", ] + + +def __getattr__(name: str) -> object: + if name == "PgReportingMaterializerStore": + from adcp.reporting.materializer.pg import PgReportingMaterializerStore + + return PgReportingMaterializerStore + raise AttributeError(f"module {__name__!r} has no attribute {name!r}") diff --git a/src/adcp/reporting/materializer/_errors.py b/src/adcp/reporting/materializer/_errors.py new file mode 100644 index 000000000..d4f090d2d --- /dev/null +++ b/src/adcp/reporting/materializer/_errors.py @@ -0,0 +1,36 @@ +"""Do not let a driver, hook or cancellation message become a diagnostic.""" + +from __future__ import annotations + +import asyncio +from collections.abc import Awaitable, Callable, Coroutine +from functools import wraps +from typing import Any, ParamSpec, TypeVar + +from adcp.reporting.ledger.store import LedgerConflictError +from adcp.reporting.materializer.contracts import ReportingWriterError, ReportingWriterFailure + +P = ParamSpec("P") +T = TypeVar("T") + + +def materializer_errors(method: Callable[P, Awaitable[T]]) -> Callable[P, Coroutine[Any, Any, T]]: + @wraps(method) + async def guarded(*args: P.args, **kwargs: P.kwargs) -> T: + try: + return await method(*args, **kwargs) + except (ReportingWriterError, LedgerConflictError): + raise + except asyncio.CancelledError: + canceled = True + except Exception: + canceled = False + # Outside the handler: neither a provider's message nor exception chain + # survives. Commit uncertainty is always resumed with the same identity. + if canceled: + raise asyncio.CancelledError + raise ReportingWriterError( + ReportingWriterFailure("RESOURCE_UNAVAILABLE", "same_identity", "unknown") + ) + + return guarded diff --git a/src/adcp/reporting/materializer/capture.py b/src/adcp/reporting/materializer/capture.py new file mode 100644 index 000000000..e36012310 --- /dev/null +++ b/src/adcp/reporting/materializer/capture.py @@ -0,0 +1,158 @@ +"""Immutable finish boundaries and a real, isolated notification enqueue path. + +B2.1 does not expose a delivery worker or an activation certificate. A/B/C +workers cannot see this queue. Pre-activation events stay quarantined forever. +B2.4 must prove the mounted projection and tier components before admitting new +events in the original verified-finish transaction; it cannot release old ones. +""" + +from __future__ import annotations + +from dataclasses import dataclass, field, replace +from datetime import datetime +from typing import Any + +from pydantic import TypeAdapter, ValidationError + +from adcp.reporting.evidence import aware_utc, reporting_identifier +from adcp.reporting.ledger._delivery_state import decode_record, payload, principal +from adcp.reporting.ledger.delivery_models import ( + ReportingDeliveryPrincipal, + ReportingDeliveryRecord, + ReportingMaterializationRecord, +) +from adcp.reporting.ledger.notification_models import ( + ReportingDomainEvent, + ReportingNotificationError, +) +from adcp.reporting.ledger.status_projection import ReportingStatusSnapshot +from adcp.reporting.outbox.memory import NotificationState + +_CORE = TypeAdapter(ReportingStatusSnapshot) + + +@dataclass(frozen=True) +class ReportingMaterializerBoundary: + """Private frozen projection inputs; never returned as a public wire blob.""" + + caller: ReportingDeliveryPrincipal + sequence: int + account_sequence: int + reporting_materialization_id: str + as_of: datetime + core: ReportingStatusSnapshot = field(repr=False) + reconciliation: tuple[ReportingDeliveryRecord, ...] = field(repr=False) + + def __post_init__(self) -> None: + outcomes = tuple( + record + for record in self.reconciliation + if isinstance(record, ReportingMaterializationRecord) + and record.reporting_materialization_id == self.reporting_materialization_id + ) + if ( + type(self.sequence) is not int + or self.sequence < 1 + or type(self.account_sequence) is not int + or self.account_sequence < self.sequence + or self.core.account_id != self.caller.account_id + or self.core.as_of != self.as_of + or self.core.consumer_ids != (self.caller.consumer_id,) + or any(s.consumer_id != self.caller.consumer_id for s in self.core.statuses) + or any( + i.consumer_id not in {None, self.caller.consumer_id} for i in self.core.lifecycles + ) + or any(principal(r) != self.caller for r in self.reconciliation) + or len(outcomes) != 1 + or outcomes[0].completed_at != self.as_of + ): + raise ReportingNotificationError("materializer_boundary_invalid") + reporting_identifier(self.reporting_materialization_id, maximum=255) + object.__setattr__(self, "as_of", aware_utc(self.as_of)) + + def to_storage(self) -> dict[str, Any]: + return { + "version": 1, + "account_id": self.caller.account_id, + "consumer_id": self.caller.consumer_id, + "sequence": self.sequence, + "account_sequence": self.account_sequence, + "reporting_materialization_id": self.reporting_materialization_id, + "as_of": self.as_of.isoformat(), + "core": _CORE.dump_python(self.core, mode="json"), + "reconciliation": [payload(r) for r in self.reconciliation], + } + + +def decode_materializer_boundary(value: dict[str, Any]) -> ReportingMaterializerBoundary: + result = None + try: + if type(value) is dict and type(value.get("version")) is int and value["version"] == 1: + result = ReportingMaterializerBoundary( + ReportingDeliveryPrincipal(value["account_id"], value["consumer_id"]), + value["sequence"], + value["account_sequence"], + value["reporting_materialization_id"], + datetime.fromisoformat(value["as_of"]), + _CORE.validate_python(value["core"]), + tuple(decode_record(r) for r in value["reconciliation"]), + ) + except (ValueError, TypeError, KeyError, ValidationError): + pass + if result is None or result.to_storage() != value: + raise ReportingNotificationError("materializer_boundary_invalid") + return result + + +class _MaterializerQueueConnection: + """Closed table substitution, preserving the finish transaction's connection.""" + + def __init__(self, connection: Any) -> None: + self.connection = connection + + async def execute(self, query: str, params: Any = None) -> Any: + return await self.connection.execute( + query.replace("reporting_notification_", "reporting_materializer_notification_"), params + ) + + +async def enqueue_materializer_event_on(connection: Any, event: ReportingDomainEvent) -> None: + from adcp.reporting.outbox.pg import enqueue_event + + if event.notification_type != "reporting.delivery_ready": + raise ReportingNotificationError("materializer_event_invalid") + await enqueue_event(_MaterializerQueueConnection(connection), event) + + +def private_snapshot( + snapshot: ReportingStatusSnapshot, caller: ReportingDeliveryPrincipal +) -> ReportingStatusSnapshot: + """Internal boundaries also exclude other consumers' private statements.""" + statuses = tuple(s for s in snapshot.statuses if s.consumer_id == caller.consumer_id) + status_ids = {s.reporting_status_id for s in statuses} + issues = tuple(i for i in snapshot.lifecycles if i.consumer_id in {None, caller.consumer_id}) + issue_ids = {i.issue_id for i in issues} + return replace( + snapshot, + statuses=statuses, + lifecycles=issues, + consumer_ids=(caller.consumer_id,), + issue_scopes=tuple((i, scope) for i, scope in snapshot.issue_scopes if i in issue_ids), + changes=tuple( + c for c in snapshot.changes if c[1] != "consumer_status" or c[2] in status_ids + ), + ) + + +class MaterializerNotificationState(NotificationState): + """Pre-activation events remain permanently quarantined, including on restart.""" + + def enqueue(self, event: ReportingDomainEvent) -> None: + super().enqueue(event) + for (account, consumer, notification_id, _), work in self.expansions.items(): + if (account, consumer, notification_id) == ( + event.account_id, + event.consumer_namespace, + event.notification_id, + ): + work.state = "quarantined" diff --git a/src/adcp/reporting/materializer/memory.py b/src/adcp/reporting/materializer/memory.py new file mode 100644 index 000000000..424e33eba --- /dev/null +++ b/src/adcp/reporting/materializer/memory.py @@ -0,0 +1,670 @@ +"""Deterministic in-memory model of the durable materializer state machine. + +This implementation is for conformance and never establishes production tier +readiness. Mutations, including disabled-notification turns, roll back together. +""" + +from __future__ import annotations + +from dataclasses import dataclass, replace +from datetime import datetime, timedelta +from typing import Any +from uuid import uuid4 + +from adcp.reporting.ledger._delivery_state import RecordT +from adcp.reporting.ledger.delivery import InMemoryReportingReconciliationStore +from adcp.reporting.ledger.delivery_models import ( + ReportingDeliveryPrincipal, + ReportingDeliveryScope, + ReportingDestinationBinding, + ReportingMaterializationAttempt, + ReportingMaterializationCheck, + ReportingMaterializationRecord, + ReportingObligationDeliveryRecord, +) +from adcp.reporting.ledger.models import ReportingConfiguration +from adcp.reporting.ledger.notification_events import delivery_dirty, materialization_event +from adcp.reporting.ledger.store import LedgerConflictError +from adcp.reporting.materializer._errors import materializer_errors +from adcp.reporting.materializer.capture import ( + MaterializerNotificationState, + ReportingMaterializerBoundary, + private_snapshot, +) +from adcp.reporting.materializer.contracts import ( + ReportingDestinationRequest, + ReportingPreparedRevision, + ReportingVerificationKey, + ReportingWriterError, + ReportingWriterFailure, + binding_fingerprint, + failure, +) +from adcp.reporting.materializer.verification import ( + ReportingVerifiedDestination, + validate_materialization_target, + validate_verified_destination, +) +from adcp.reporting.materializer.work import ( + MaterializerContext, + MaterializerReason, + ReportingMaterializerLease, + ReportingMaterializerTurn, + key_for, + public_failure, + validate_lease_seconds, + verification_key_id, +) + + +@dataclass +class _Candidate: + scope: ReportingDeliveryScope + generation: int = 1 + due_at: datetime | None = None + reason: MaterializerReason = "ready" + turn: int = 0 + + +@dataclass +class _Work: + scope: ReportingDeliveryScope + generation: int + attempt: ReportingMaterializationAttempt + request: ReportingDestinationRequest + due_at: datetime | None + notifications_enabled: bool + # This reservation can never produce a production-admitted readiness event, + # including when a later installation resumes an uncertain external effect. + admission_epoch: int = 0 + token: str | None = None + lease_until: datetime | None = None + completion_token: str | None = None + acked: bool = False + retry_allowed: bool = False + reason: MaterializerReason = "ready" + + +class InMemoryReportingMaterializerStore(InMemoryReportingReconciliationStore): + def __init__(self, **kwargs: Any) -> None: + super().__init__(**kwargs) + self._materializer_candidates: dict[ReportingDeliveryScope, _Candidate] = {} + self._materializer_work: dict[tuple[str, str, str], _Work] = {} + self._materializer_account_turns: dict[str, int] = {} + self._materializer_turn = 0 + self._materializer_outbox = ( + MaterializerNotificationState() if self._notification_state is not None else None + ) + self._materializer_boundaries: list[ReportingMaterializerBoundary] = [] + self._materializer_status_heads: dict[ReportingDeliveryPrincipal, int] = {} + self._materializer_account_heads: dict[str, int] = {} + + def _wake(self, scope: ReportingDeliveryScope) -> None: + candidate = self._materializer_candidates.get(scope) + if candidate is None: + candidate = _Candidate(scope) + self._materializer_candidates[scope] = candidate + else: + candidate.generation += 1 + candidate.due_at, candidate.reason = self._clock(), "ready" + + def _wake_obligation(self, account_id: str, obligation_id: str) -> None: + obligation = self._obligations.get(obligation_id) + if obligation is None or obligation.account_id != account_id: + return + for _, _, record in self._retained_delivery_records(): + if isinstance(record, ReportingDestinationBinding) and ( + record.generation_key == obligation.generation_key + ): + self._wake( + ReportingDeliveryScope( + obligation.generation_key, record.consumer_id, obligation_id + ) + ) + + def _append(self, account_id: str, kind: Any, record_id: str) -> None: + super()._append(account_id, kind, record_id) + if kind == "obligation": + self._wake_obligation(account_id, record_id) + elif kind == "revision": + self._wake_obligation(account_id, self._revisions[record_id].reporting_obligation_id) + + async def put_configuration(self, configuration: ReportingConfiguration) -> None: + async with self._mutation(): + before = self._configurations.get(configuration.generation_key) + await super().put_configuration(configuration) + if before != configuration: + for scope in self._materializer_candidates: + if scope.generation_key == configuration.generation_key: + self._wake(scope) + + async def set_revision_readable( + self, + *, + account_id: str, + reporting_revision_id: str, + readable: bool, + ) -> None: + async with self._mutation(): + before = self._revisions.get(reporting_revision_id) + await super().set_revision_readable( + account_id=account_id, + reporting_revision_id=reporting_revision_id, + readable=readable, + ) + revision = self._revisions.get(reporting_revision_id) + if revision is not None and revision != before: + self._wake_obligation(account_id, revision.reporting_obligation_id) + + def _commit_record_unlocked( + self, record: RecordT, *, notify: bool = True + ) -> tuple[RecordT, bool]: + stored, added = super()._commit_record_unlocked( + record, notify=notify and not isinstance(record, ReportingMaterializationRecord) + ) + if added: + if isinstance(record, ReportingDestinationBinding): + for obligation in self._obligations.values(): + if obligation.generation_key == record.generation_key: + self._wake( + ReportingDeliveryScope( + obligation.generation_key, + record.consumer_id, + obligation.reporting_obligation_id, + ) + ) + elif isinstance(record, ReportingMaterializationCheck): + self._wake(record.scope) + return stored, added + + def _context(self, scope: ReportingDeliveryScope) -> MaterializerContext: + records = tuple(c.record for c in self._caller_changes(scope.principal)) + binding = next( + ( + r + for r in records + if isinstance(r, ReportingDestinationBinding) + and r.generation_key == scope.generation_key + ), + None, + ) + obligation = self._obligations.get(scope.reporting_obligation_id) + configuration = self._configurations.get(scope.generation_key) + if ( + binding is None + or obligation is None + or configuration is None + or obligation.generation_key != scope.generation_key + ): + raise failure("BINDING_MISMATCH") + delivery = next( + ( + r + for r in records + if isinstance(r, ReportingObligationDeliveryRecord) and r.scope == scope + ), + None, + ) + return MaterializerContext( + configuration, + obligation, + binding, + delivery, + tuple( + r + for r in self._revisions.values() + if r.account_id == scope.principal.account_id + and r.reporting_obligation_id == scope.reporting_obligation_id + ), + records, + ) + + def _work_key(self, attempt: ReportingMaterializationAttempt) -> tuple[str, str, str]: + return ( + attempt.scope.principal.account_id, + attempt.scope.consumer_id, + attempt.reporting_materialization_id, + ) + + def _park( + self, + candidate: _Candidate, + reason: MaterializerReason, + due: datetime | None = None, + ) -> ReportingMaterializerTurn: + candidate.reason, candidate.due_at = reason, due + return ReportingMaterializerTurn("parked", reason) + + @materializer_errors + async def claim_materialization( + self, + *, + keys: tuple[ReportingVerificationKey, ...], + lease_seconds: int = 30, + ) -> ReportingMaterializerLease | ReportingMaterializerTurn: + validate_lease_seconds(lease_seconds) + async with self._mutation(): + now = self._clock() + pending = {w.scope for w in self._materializer_work.values() if not w.acked} + works = [ + w + for w in self._materializer_work.values() + if not w.acked + and w.due_at is not None + and w.due_at <= now + and (w.lease_until is None or w.lease_until <= now) + ] + candidates = [ + c + for c in self._materializer_candidates.values() + if c.due_at is not None and c.due_at <= now and c.scope not in pending + ] + accounts = {w.scope.principal.account_id for w in works} | { + c.scope.principal.account_id for c in candidates + } + if not accounts: + return ReportingMaterializerTurn("idle") + account_id = min( + accounts, key=lambda a: (self._materializer_account_turns.get(a, 0), a) + ) + self._materializer_turn += 1 + self._materializer_account_turns[account_id] = self._materializer_turn + account_work = [w for w in works if w.scope.principal.account_id == account_id] + if account_work: + work = min(account_work, key=lambda w: (w.due_at or now, w.request.external_id)) + return self._lease(work, keys, lease_seconds) + candidate = min( + (c for c in candidates if c.scope.principal.account_id == account_id), + key=lambda c: (c.turn, c.scope.consumer_id, c.scope.reporting_obligation_id), + ) + candidate.turn = self._materializer_turn + try: + context = self._context(candidate.scope) + key = key_for(context.binding, context.obligation, keys) + except LedgerConflictError: + return self._park(candidate, "history_corrupt") + except ReportingWriterError: + return self._park(candidate, "component_unavailable") + revision, reason = context.selection(now) + if revision is None or reason != "ready": + at = context.configuration.activated_at + return self._park(candidate, reason, at if at is not None and at > now else None) + attempts = context.attempts(revision.reporting_revision_id) + if tuple(a.attempt for a in attempts) != tuple(range(1, len(attempts) + 1)): + return self._park(candidate, "history_corrupt") + if context.pending_attempts(): + return self._park(candidate, "legacy_pending") + if attempts: + outcome = context.outcome(attempts[-1]) + assert outcome is not None + if outcome.status != "failed": + reason, expires = context.retained_success(outcome, now) + return self._park(candidate, reason, expires) + owned = self._materializer_work.get(self._work_key(attempts[-1])) + if owned is None or not owned.retry_allowed: + return self._park( + candidate, "legacy_terminal" if owned is None else "operator_required" + ) + if context.delivery is None: + if context.obligation.currency is None: + return self._park(candidate, "operator_required") + delivery = ReportingObligationDeliveryRecord( + candidate.scope, + context.obligation.currency, + max(now, context.obligation.period.end) + + timedelta(days=context.binding.resource_retention_days), + now, + ) + self._commit_record_unlocked(delivery, notify=False) + attempt = ReportingMaterializationAttempt( + candidate.scope, + revision.reporting_revision_id, + "rpm_" + uuid4().hex, + len(attempts) + 1, + now, + ) + self._commit_record_unlocked(attempt, notify=False) + request = ReportingDestinationRequest.from_binding(context.binding, attempt, key) + work = _Work( + candidate.scope, + candidate.generation, + attempt, + request, + now, + self._notification_state is not None, + ) + self._materializer_work[self._work_key(attempt)] = work + self._park(candidate, "ready") + return self._lease(work, keys, lease_seconds) + + def _lease( + self, + work: _Work, + keys: tuple[ReportingVerificationKey, ...], + seconds: int, + ) -> ReportingMaterializerLease | ReportingMaterializerTurn: + if work.notifications_enabled != (self._notification_state is not None): + return self._park_work(work, "component_unavailable") + try: + context = self._context(work.scope) + if not context.resumable(work.attempt): + return self._park_work(work, "history_corrupt") + key = key_for( + context.binding, + context.obligation, + keys, + required=verification_key_id(work.request.verification_key), + ) + if ( + ReportingDestinationRequest.from_binding(context.binding, work.attempt, key) + != work.request + ): + raise failure("BINDING_MISMATCH") + except (ReportingWriterError, LedgerConflictError): + return self._park_work(work, "component_unavailable") + work.token = str(uuid4()) + work.lease_until = self._clock() + timedelta(seconds=seconds) + work.due_at = work.lease_until + return ReportingMaterializerLease( + work.scope, + work.generation, + work.token, + work.lease_until, + work.attempt, + work.request, + context, + work.notifications_enabled, + ) + + def _park_work(self, work: _Work, reason: MaterializerReason) -> ReportingMaterializerTurn: + work.due_at, work.reason = None, reason + work.token, work.lease_until = None, None + return self._park(self._materializer_candidates[work.scope], reason) + + def _held(self, lease: ReportingMaterializerLease) -> _Work | None: + work = self._materializer_work.get(self._work_key(lease.attempt)) + if ( + work is None + or work.acked + or work.token != lease.token + or work.lease_until is None + or work.lease_until <= self._clock() + ): + return None + if ( + work.request != lease.request + or work.generation != lease.generation + or work.notifications_enabled != lease.notifications_enabled + or work.notifications_enabled != (self._notification_state is not None) + ): + raise failure("BINDING_MISMATCH") + return work + + def _target( + self, lease: ReportingMaterializerLease + ) -> tuple[MaterializerContext, MaterializerReason]: + context = self._context(lease.scope) + selected, reason = context.selection(self._clock()) + if reason != "ready": + return context, reason + candidate = self._materializer_candidates[lease.scope] + if ( + candidate.generation != lease.generation + or selected is None + or selected.reporting_revision_id != lease.attempt.reporting_revision_id + ): + return context, "target_changed" + if binding_fingerprint(context.binding) != lease.request.binding_fingerprint: + return context, "binding_changed" + return context, "ready" + + @materializer_errors + async def renew_materialization( + self, lease: ReportingMaterializerLease, *, lease_seconds: int + ) -> bool: + validate_lease_seconds(lease_seconds) + async with self._mutation(): + work = self._held(lease) + if work is None: + return False + work.lease_until = self._clock() + timedelta(seconds=lease_seconds) + work.due_at = work.lease_until + return True + + @materializer_errors + async def authorize_materialization(self, lease: ReportingMaterializerLease) -> None: + async with self._mutation(): + if self._held(lease) is None: + raise failure("LEASE_LOST") + _, reason = self._target(lease) + if reason != "ready": + raise failure( + "HISTORY_CORRUPT" if reason == "history_corrupt" else "CURRENT_REVISION_CHANGED" + ) + + @materializer_errors + async def finish_materialization( + self, + lease: ReportingMaterializerLease, + *, + prepared: ReportingPreparedRevision | None = None, + verified: ReportingVerifiedDestination | None = None, + error: ReportingWriterFailure | None = None, + ) -> ReportingMaterializerTurn: + if verified is not None: + validate_verified_destination(verified, lease.request, token=lease.token) + if prepared is None or prepared.request != lease.request or error is not None: + raise failure("BINDING_MISMATCH") + elif error is None: + raise failure("BINDING_MISMATCH") + async with self._mutation(): + work = self._held(lease) + if work is None: + previous = self._materializer_work.get(self._work_key(lease.attempt)) + if ( + previous is not None + and previous.acked + and previous.completion_token == lease.token + and previous.request == lease.request + ): + return ReportingMaterializerTurn( + "verified" if previous.reason == "verified" else "failed", + previous.reason, + lease.attempt.reporting_materialization_id, + ) + return ReportingMaterializerTurn( + "pending", "effect_unknown", lease.attempt.reporting_materialization_id + ) + context, reason = self._target(lease) + if reason != "ready": + error = ReportingWriterFailure("CURRENT_REVISION_CHANGED", "new_attempt", "applied") + verified = None + elif verified is not None and prepared is not None: + validate_materialization_target( + prepared, binding=context.binding, revisions=context.revisions + ) + if error is not None and ( + error.effect == "unknown" + or error.retry == "same_identity" + or error.code in {"DEADLINE_EXCEEDED", "LEASE_LOST"} + ): + work.token, work.lease_until, work.reason = None, None, "effect_unknown" + work.due_at = self._clock() + timedelta( + seconds=max(1, min(error.retry_after_seconds or 5, 300)) + ) + return ReportingMaterializerTurn( + "pending", work.reason, work.attempt.reporting_materialization_id + ) + now = self._clock() + if verified is not None and ( + context.delivery is None + or verified.resource.expires_at + < max( + context.delivery.resource_retained_until, + now + timedelta(days=context.binding.resource_retention_days), + ) + ): + error = ReportingWriterFailure("RESOURCE_UNAVAILABLE", "new_attempt", "applied") + verified = None + if verified is not None: + # Validate again under the finish lock, immediately before + # copying observations into the immutable terminal record. + validate_verified_destination(verified, lease.request, token=lease.token) + outcome = ReportingMaterializationRecord( + lease.scope, + lease.attempt.reporting_revision_id, + lease.attempt.reporting_materialization_id, + context.binding.success_status if verified is not None else "failed", + now, + verified.resource if verified is not None else None, + replace(verified.verification, verified_at=now) if verified is not None else None, + public_failure(error) if error is not None else None, + ) + stored, inserted = self._commit_record_unlocked(outcome, notify=False) + self._materializer_dirty(stored, context) + if inserted and verified is not None and self._notification_state is not None: + revision = next( + r + for r in context.revisions + if r.reporting_revision_id == stored.reporting_revision_id + ) + event = materialization_event( + stored, + context.records, + context.obligation, + revision, + context.configuration, + now, + ) + if event is None: + raise failure("BINDING_MISMATCH") + assert self._materializer_outbox is not None + self._materializer_outbox.enqueue(event) + if ( + self._materializer_outbox.events.get( + (event.account_id, event.consumer_namespace, event.notification_id) + ) + != event + ): + raise failure("BINDING_MISMATCH") + if self._held(lease) is None: + raise failure("LEASE_LOST") + work.completion_token = work.token + work.acked, work.token, work.lease_until = True, None, None + work.retry_allowed = error is not None and error.retry == "new_attempt" + if reason == "ready": + reason = ( + "verified" + if verified is not None + else ("retry" if work.retry_allowed else "operator_required") + ) + work.reason = reason + due = ( + now + timedelta(seconds=max(1, error.retry_after_seconds or 5)) + if work.retry_allowed and error + else None + ) + if reason == "target_changed": + due = now + elif reason not in {"retry", "verified"}: + due = None + if ( + reason == "inactive" + and context.configuration.activated_at is not None + and context.configuration.activated_at > now + ): + due = context.configuration.activated_at + if verified is not None: + due = verified.resource.expires_at + self._park(self._materializer_candidates[lease.scope], reason, due) + return ReportingMaterializerTurn( + "verified" if verified is not None else "failed", + reason, + stored.reporting_materialization_id, + ) + + def _materializer_dirty( + self, outcome: ReportingMaterializationRecord, context: MaterializerContext + ) -> None: + scope, reason, evidence = delivery_dirty(outcome, context.obligation) + self._dirty_status(scope, reason, after=evidence) + + from adcp.reporting.ledger.status_snapshot import settle_memory_snapshot + + core = settle_memory_snapshot(self, scope.account_id) + core = replace(core, as_of=outcome.completed_at) + sequence = self._materializer_status_heads.get(outcome.scope.principal, 0) + 1 + self._materializer_status_heads[outcome.scope.principal] = sequence + account_sequence = self._materializer_account_heads.get(scope.account_id, 0) + 1 + self._materializer_account_heads[scope.account_id] = account_sequence + self._materializer_boundaries.append( + ReportingMaterializerBoundary( + outcome.scope.principal, + sequence, + account_sequence, + outcome.reporting_materialization_id, + outcome.completed_at, + private_snapshot(core, outcome.scope.principal), + tuple(c.record for c in self._caller_changes(outcome.scope.principal)), + ) + ) + + @materializer_errors + async def read_materializer_boundaries( + self, *, caller: ReportingDeliveryPrincipal, after: int = 0, limit: int = 100 + ) -> tuple[ReportingMaterializerBoundary, ...]: + if type(after) is not int or after < 0 or type(limit) is not int or not 1 <= limit <= 100: + raise ValueError("materializer boundary reads require bounded positions") + async with self._lock: + return tuple( + b + for b in self._materializer_boundaries + if b.caller == caller and b.sequence > after + )[:limit] + + @materializer_errors + async def import_pending_materialization( + self, + *, + scope: ReportingDeliveryScope, + reporting_materialization_id: str, + original_external_id: str, + keys: tuple[ReportingVerificationKey, ...], + ) -> None: + async with self._mutation(): + context = self._context(scope) + attempt = next( + ( + r + for r in context.records + if isinstance(r, ReportingMaterializationAttempt) + and r.reporting_materialization_id == reporting_materialization_id + and r.scope == scope + ), + None, + ) + if attempt is None or context.outcome(attempt) is not None: + raise failure("BINDING_MISMATCH") + if not context.resumable(attempt): + raise failure("HISTORY_CORRUPT") + key = key_for(context.binding, context.obligation, keys) + request = ReportingDestinationRequest.from_binding(context.binding, attempt, key) + if original_external_id != request.external_id: + raise failure("BINDING_MISMATCH") + if scope not in self._materializer_candidates: + self._wake(scope) + existing = self._materializer_work.get(self._work_key(attempt)) + if existing is not None: + if existing.token is None and not existing.acked: + existing.due_at = self._clock() + return + if any(w.scope == scope and not w.acked for w in self._materializer_work.values()): + raise failure("BINDING_MISMATCH") + self._materializer_work[self._work_key(attempt)] = _Work( + scope, + self._materializer_candidates[scope].generation, + attempt, + request, + self._clock(), + self._notification_state is not None, + ) diff --git a/src/adcp/reporting/materializer/pg.py b/src/adcp/reporting/materializer/pg.py new file mode 100644 index 000000000..0adab053c --- /dev/null +++ b/src/adcp/reporting/materializer/pg.py @@ -0,0 +1,955 @@ +"""PostgreSQL account-first reservation, fenced resume and verified atomic finish.""" + +from __future__ import annotations + +from dataclasses import replace +from datetime import datetime, timedelta +from importlib.resources import files +from typing import Any, cast +from uuid import uuid4 + +from adcp.reporting.ledger._delivery_state import RecordT +from adcp.reporting.ledger.delivery_models import ( + ReportingDeliveryPrincipal, + ReportingDeliveryScope, + ReportingDestinationBinding, + ReportingMaterializationAttempt, + ReportingMaterializationRecord, + ReportingObligationDeliveryRecord, +) +from adcp.reporting.ledger.delivery_pg import PgReportingReconciliationStore +from adcp.reporting.ledger.models import ReportingConfigurationGenerationKey +from adcp.reporting.ledger.notification_events import materialization_event +from adcp.reporting.ledger.pg import ( + _OBLIGATION_COLUMNS, + _REVISION_COLUMNS, + _configuration_from_row, + _obligation_from_row, + _revision_from_row, +) +from adcp.reporting.ledger.store import LedgerConflictError +from adcp.reporting.materializer._errors import materializer_errors +from adcp.reporting.materializer.capture import ( + ReportingMaterializerBoundary, + decode_materializer_boundary, + enqueue_materializer_event_on, + private_snapshot, +) +from adcp.reporting.materializer.contracts import ( + ReportingDestinationRequest, + ReportingPreparedRevision, + ReportingVerificationKey, + ReportingWriterError, + ReportingWriterFailure, + binding_fingerprint, + failure, +) +from adcp.reporting.materializer.schema import validate_materializer_schema +from adcp.reporting.materializer.verification import ( + ReportingVerifiedDestination, + validate_materialization_target, + validate_verified_destination, +) +from adcp.reporting.materializer.work import ( + MaterializerContext, + MaterializerReason, + ReportingMaterializerLease, + ReportingMaterializerTurn, + key_for, + public_failure, + validate_lease_seconds, + verification_key_id, +) + +_SCOPE_WHERE = ( + "account_id=%s AND consumer_id=%s AND delivery_config_id=%s" + " AND delivery_config_version=%s AND reporting_obligation_id=%s" +) + + +def _scope_args(scope: ReportingDeliveryScope) -> tuple[str | int, ...]: + generation = scope.generation_key + return ( + generation.account_id, + scope.consumer_id, + generation.delivery_config_id, + generation.delivery_config_version, + scope.reporting_obligation_id, + ) + + +def _scope(row: dict[str, Any]) -> ReportingDeliveryScope: + return ReportingDeliveryScope( + ReportingConfigurationGenerationKey( + row["account_id"], row["delivery_config_id"], row["delivery_config_version"] + ), + row["consumer_id"], + row["reporting_obligation_id"], + ) + + +async def _now(connection: Any) -> datetime: + row = await (await connection.execute("SELECT clock_timestamp()")).fetchone() + assert row is not None + return cast(datetime, row[0]) + + +class PgReportingMaterializerStore(PgReportingReconciliationStore): + """Additive durable extension. No account enumeration or destination I/O. + + Each turn samples at most sixteen due accounts without row locks, tries the + Core advisory lock first, then claims within that account. Long work returns + its connection before any source/destination call; even a size-one pool can + renew its lease. Clock overrides affect old conformance APIs only: work time + is always PostgreSQL time. + """ + + # Only a read-only sampling position, never a lease or durable correctness + # boundary. Walk past busy accounts in bounded pages, wrapping after the end. + # Restart may repeat a page; it cannot lose or acknowledge work. + _materializer_sample_after: tuple[str, str] | None = None + + async def _commit_record_on( + self, connection: Any, record: RecordT, *, notify: bool = True + ) -> tuple[RecordT, bool]: + # Only the fenced verified finish may enqueue a readiness intent. + return await super()._commit_record_on( + connection, + record, + notify=notify and not isinstance(record, ReportingMaterializationRecord), + ) + + @materializer_errors + async def create_schema(self) -> None: + async with self._connection() as connection, connection.transaction(): + await self._create_schema_on(connection) + await connection.execute( + files("adcp.reporting.ledger").joinpath("reporting_materializer.sql").read_text() + ) + + @materializer_errors + async def materializer_ready(self) -> bool: + async with self._connection() as connection: + await validate_materializer_schema( + connection, notifications=self._notifications_enabled + ) + return True + + async def _materializer_context_on( + self, connection: Any, scope: ReportingDeliveryScope + ) -> MaterializerContext: + records = await self._records(connection, scope.principal) + binding = next( + ( + r + for r in records + if isinstance(r, ReportingDestinationBinding) + and r.generation_key == scope.generation_key + ), + None, + ) + obligation_row = await ( + await connection.execute( + f"SELECT {_OBLIGATION_COLUMNS} FROM reporting_obligations" # nosec B608 + " WHERE account_id=%s AND reporting_obligation_id=%s", + (scope.principal.account_id, scope.reporting_obligation_id), + ) + ).fetchone() + config_row = await ( + await connection.execute( + "SELECT delivery_config_id, delivery_config_version, account_id," + " report_definition_id, reporting_profile, feed_purpose, required_finality," + " account_timezone, schedule, media_buy_ids, activated_at, deactivated_at," + " automated_recovery_seconds, status_retention_days, definition," + " authoritative_party" + " FROM reporting_configurations WHERE account_id=%s" + " AND delivery_config_id=%s AND delivery_config_version=%s", + ( + scope.principal.account_id, + scope.generation_key.delivery_config_id, + scope.generation_key.delivery_config_version, + ), + ) + ).fetchone() + revision_rows = await ( + await connection.execute( + f"SELECT {_REVISION_COLUMNS} FROM reporting_revisions" # nosec B608 + " WHERE account_id=%s AND reporting_obligation_id=%s", + (scope.principal.account_id, scope.reporting_obligation_id), + ) + ).fetchall() + if binding is None or obligation_row is None or config_row is None: + raise failure("BINDING_MISMATCH") + obligation = _obligation_from_row(obligation_row) + configuration = _configuration_from_row(config_row) + if obligation.generation_key != scope.generation_key: + raise failure("BINDING_MISMATCH") + delivery = next( + ( + r + for r in records + if isinstance(r, ReportingObligationDeliveryRecord) and r.scope == scope + ), + None, + ) + return MaterializerContext( + configuration, + obligation, + binding, + delivery, + tuple(_revision_from_row(r) for r in revision_rows), + records, + ) + + async def _schedule_account_on(self, connection: Any, account_id: str) -> None: + await connection.execute( + "UPDATE reporting_materializer_accounts SET due_at=(SELECT min(due) FROM (" + " SELECT due_at AS due FROM reporting_materializer_work" + " WHERE account_id=%s AND state='pending'" + " UNION ALL SELECT c.due_at FROM reporting_materializer_candidates c" + " WHERE c.account_id=%s AND c.due_at IS NOT NULL AND NOT EXISTS (" + " SELECT 1 FROM reporting_materializer_work w WHERE w.account_id=c.account_id" + " AND w.consumer_id=c.consumer_id AND w.delivery_config_id=c.delivery_config_id" + " AND w.delivery_config_version=c.delivery_config_version" + " AND w.reporting_obligation_id=c.reporting_obligation_id AND w.state='pending')" + " UNION ALL SELECT clock_timestamp() FROM reporting_materializer_discovery" + " WHERE account_id=%s AND NOT complete) ready) WHERE account_id=%s", + (account_id,) * 4, + ) + + async def _discover_on(self, connection: Any, account_id: str) -> bool: + row = await ( + await connection.execute( + "SELECT consumer_id, delivery_config_id, delivery_config_version," + " after_obligation_id" + " FROM reporting_materializer_discovery WHERE account_id=%s AND NOT complete" + " ORDER BY consumer_id, delivery_config_id, delivery_config_version" + " LIMIT 1 FOR UPDATE", + (account_id,), + ) + ).fetchone() + if row is None: + return False + consumer, config, version, after = row + obligations = await ( + await connection.execute( + "SELECT reporting_obligation_id FROM reporting_obligations WHERE account_id=%s" + " AND delivery_config_id=%s AND delivery_config_version=%s" + " AND reporting_obligation_id>%s ORDER BY reporting_obligation_id LIMIT 32", + (account_id, config, version, after), + ) + ).fetchall() + for (obligation_id,) in obligations: + # Only missing candidates: a later backfill batch never invalidates + # a worker already reserved through the publication trigger. + await connection.execute( + "INSERT INTO reporting_materializer_candidates" + " (account_id,consumer_id,delivery_config_id,delivery_config_version," + " reporting_obligation_id,due_at) VALUES (%s,%s,%s,%s,%s,clock_timestamp())" + " ON CONFLICT DO NOTHING", + (account_id, consumer, config, version, obligation_id), + ) + await connection.execute( + "UPDATE reporting_materializer_discovery SET after_obligation_id=%s,complete=%s" + " WHERE account_id=%s AND consumer_id=%s AND delivery_config_id=%s" + " AND delivery_config_version=%s", + ( + obligations[-1][0] if obligations else after, + len(obligations) < 32, + account_id, + consumer, + config, + version, + ), + ) + return True + + async def _park_on( + self, + connection: Any, + scope: ReportingDeliveryScope, + reason: MaterializerReason, + *, + due_at: datetime | None = None, + ) -> ReportingMaterializerTurn: + await connection.execute( + "UPDATE reporting_materializer_candidates SET reason=%s,due_at=%s," + f" served_at=clock_timestamp() WHERE {_SCOPE_WHERE}", # nosec B608 + (reason, due_at, *_scope_args(scope)), + ) + return ReportingMaterializerTurn("parked", reason) + + @materializer_errors + async def claim_materialization( + self, *, keys: tuple[ReportingVerificationKey, ...], lease_seconds: int = 30 + ) -> ReportingMaterializerLease | ReportingMaterializerTurn: + validate_lease_seconds(lease_seconds) + await self.materializer_ready() + async with self._connection() as connection: + # Read-only sampling. Never lock global candidate rows before the + # account advisory lock, including when a competing worker is busy. + for _ in range(2): + after = self._materializer_sample_after + if after: + query = ( + "SELECT account_id,served_at::text FROM reporting_materializer_accounts" + " WHERE due_at<=%s AND (served_at,account_id)>(%s::timestamptz,%s)" + " ORDER BY served_at,account_id LIMIT 16" + ) + else: + query = ( + "SELECT account_id,served_at::text FROM reporting_materializer_accounts" + " WHERE due_at<=%s ORDER BY served_at,account_id LIMIT 16" + ) + accounts = await ( + await connection.execute( + query, + (await _now(connection), *(after or ())), + ) + ).fetchall() + if accounts: + break + self._materializer_sample_after = None + if after is None: + break + for account_id, served_at in accounts: + self._materializer_sample_after = (served_at, account_id) + async with self._connection() as connection, connection.transaction(): + held = await ( + await connection.execute( + "SELECT pg_try_advisory_xact_lock(hashtext(%s))", + (f"adcp.reporting:{account_id}",), + ) + ).fetchone() + if held is None or not held[0]: + continue + await self._lock_account(connection, account_id) + await validate_materializer_schema( + connection, notifications=self._notifications_enabled + ) + await connection.execute( + "UPDATE reporting_materializer_accounts SET served_at=clock_timestamp()" + " WHERE account_id=%s", + (account_id,), + ) + result = await self._claim_account_on(connection, account_id, keys, lease_seconds) + await self._schedule_account_on(connection, account_id) + return result + return ReportingMaterializerTurn("idle") + + async def _claim_account_on( + self, + connection: Any, + account_id: str, + keys: tuple[ReportingVerificationKey, ...], + lease_seconds: int, + ) -> ReportingMaterializerLease | ReportingMaterializerTurn: + # A bound DB instant is an indexable range predicate. clock_timestamp() + # is volatile and cannot be used as a PostgreSQL index scan boundary. + now = await _now(connection) + pending = await ( + await connection.execute( + "SELECT to_jsonb(w) FROM reporting_materializer_work w WHERE account_id=%s" + " AND state='pending' AND due_at<=%s" + " AND (lease_until IS NULL OR lease_until<=%s)" + " ORDER BY due_at,reporting_materialization_id LIMIT 1 FOR UPDATE", + (account_id, now, now), + ) + ).fetchone() + if pending is not None: + return await self._lease_on(connection, pending[0], keys, lease_seconds) + discovered = await self._discover_on(connection, account_id) + row = await ( + await connection.execute( + "SELECT to_jsonb(c) FROM reporting_materializer_candidates c WHERE account_id=%s" + " AND due_at<=%s AND NOT EXISTS (" + " SELECT 1 FROM reporting_materializer_work w WHERE w.account_id=c.account_id" + " AND w.consumer_id=c.consumer_id AND w.delivery_config_id=c.delivery_config_id" + " AND w.delivery_config_version=c.delivery_config_version" + " AND w.reporting_obligation_id=c.reporting_obligation_id AND w.state='pending')" + " ORDER BY served_at,consumer_id,reporting_obligation_id LIMIT 1 FOR UPDATE", + (account_id, await _now(connection)), + ) + ).fetchone() + if row is None: + return ReportingMaterializerTurn("discovered" if discovered else "idle") + scope = _scope(row[0]) + try: + context = await self._materializer_context_on(connection, scope) + key = key_for(context.binding, context.obligation, keys) + except LedgerConflictError: + return await self._park_on(connection, scope, "history_corrupt") + except ReportingWriterError: + return await self._park_on(connection, scope, "component_unavailable") + now = await _now(connection) + revision, reason = context.selection(now) + if reason != "ready" or revision is None: + due = context.configuration.activated_at + return await self._park_on( + connection, scope, reason, due_at=due if due is not None and due > now else None + ) + attempts = context.attempts(revision.reporting_revision_id) + if tuple(a.attempt for a in attempts) != tuple(range(1, len(attempts) + 1)): + return await self._park_on(connection, scope, "history_corrupt") + if context.pending_attempts(): + # Includes legacy pending effects on an older selected revision. + # Publication cannot make their unknown external history disappear. + return await self._park_on(connection, scope, "legacy_pending") + if attempts: + outcome = context.outcome(attempts[-1]) + assert outcome is not None + if outcome.status != "failed": + reason, expires = context.retained_success(outcome, now) + return await self._park_on(connection, scope, reason, due_at=expires) + owned = await ( + await connection.execute( + "SELECT retry_allowed FROM reporting_materializer_work WHERE account_id=%s" + " AND consumer_id=%s AND reporting_materialization_id=%s AND state='acked'", + (account_id, scope.consumer_id, attempts[-1].reporting_materialization_id), + ) + ).fetchone() + if owned is None or not owned[0]: + return await self._park_on( + connection, scope, "legacy_terminal" if owned is None else "operator_required" + ) + delivery = context.delivery + if delivery is None: + if context.obligation.currency is None: + return await self._park_on(connection, scope, "operator_required") + delivery = ReportingObligationDeliveryRecord( + scope, + context.obligation.currency, + max(now, context.obligation.period.end) + + timedelta(days=context.binding.resource_retention_days), + now, + ) + await self._commit_record_on(connection, delivery, notify=False) + attempt = ReportingMaterializationAttempt( + scope, revision.reporting_revision_id, "rpm_" + uuid4().hex, len(attempts) + 1, now + ) + await self._commit_record_on(connection, attempt, notify=False) + request = ReportingDestinationRequest.from_binding(context.binding, attempt, key) + inserted = await ( + await connection.execute( + "INSERT INTO reporting_materializer_work" + " (account_id,consumer_id,delivery_config_id,delivery_config_version," + " reporting_obligation_id,reporting_revision_id,reporting_materialization_id," + " generation,binding_sha256,verification_key_sha256,external_id," + " notifications_enabled)" + " VALUES (%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s)" + " RETURNING to_jsonb(reporting_materializer_work)", + ( + *_scope_args(scope), + attempt.reporting_revision_id, + attempt.reporting_materialization_id, + row[0]["generation"], + request.binding_fingerprint, + verification_key_id(key), + request.external_id, + self._notifications_enabled, + ), + ) + ).fetchone() + assert inserted is not None + await self._park_on(connection, scope, "ready") + return await self._lease_on(connection, inserted[0], keys, lease_seconds) + + async def _lease_on( + self, + connection: Any, + work: dict[str, Any], + keys: tuple[ReportingVerificationKey, ...], + lease_seconds: int, + ) -> ReportingMaterializerLease | ReportingMaterializerTurn: + scope = _scope(work) + try: + if work["notifications_enabled"] != self._notifications_enabled: + raise failure("UNSUPPORTED_VERIFICATION") + context = await self._materializer_context_on(connection, scope) + key = key_for( + context.binding, context.obligation, keys, required=work["verification_key_sha256"] + ) + attempt = next( + r + for r in context.records + if isinstance(r, ReportingMaterializationAttempt) + and r.reporting_materialization_id == work["reporting_materialization_id"] + ) + if not context.resumable(attempt): + return await self._park_work_on(connection, work, "history_corrupt") + request = ReportingDestinationRequest.from_binding(context.binding, attempt, key) + if ( + request.external_id != work["external_id"] + or request.binding_fingerprint != work["binding_sha256"] + ): + raise failure("BINDING_MISMATCH") + except (LedgerConflictError, ReportingWriterError, StopIteration): + # Park pending unknown effects until an operator restores the exact + # installed contract. No hot loop and no replacement identity. + return await self._park_work_on(connection, work, "component_unavailable") + token = uuid4() + updated = await ( + await connection.execute( + "UPDATE reporting_materializer_work SET lease_token=%s," + " lease_until=clock_timestamp()+make_interval(secs=>%s)," + " due_at=clock_timestamp()+make_interval(secs=>%s)" + " WHERE account_id=%s AND consumer_id=%s AND reporting_materialization_id=%s" + " AND state='pending' AND (lease_until IS NULL OR lease_until<=clock_timestamp())" + " RETURNING lease_until", + ( + token, + lease_seconds, + lease_seconds, + scope.principal.account_id, + scope.consumer_id, + work["reporting_materialization_id"], + ), + ) + ).fetchone() + if updated is None: + return ReportingMaterializerTurn("idle") + return ReportingMaterializerLease( + scope, + work["generation"], + str(token), + updated[0], + attempt, + request, + context, + work["notifications_enabled"], + ) + + async def _park_work_on( + self, connection: Any, work: dict[str, Any], reason: MaterializerReason + ) -> ReportingMaterializerTurn: + scope = _scope(work) + await connection.execute( + "UPDATE reporting_materializer_work SET due_at='infinity'," + " lease_token=NULL,lease_until=NULL,reason=%s" + " WHERE account_id=%s AND consumer_id=%s AND reporting_materialization_id=%s", + ( + reason, + scope.principal.account_id, + scope.consumer_id, + work["reporting_materialization_id"], + ), + ) + return await self._park_on(connection, scope, reason) + + async def _held_on( + self, connection: Any, lease: ReportingMaterializerLease + ) -> dict[str, Any] | None: + row = await ( + await connection.execute( + "SELECT to_jsonb(w) FROM reporting_materializer_work w WHERE account_id=%s" + " AND consumer_id=%s AND reporting_materialization_id=%s AND state='pending'" + " AND lease_token=%s::uuid AND lease_until>clock_timestamp() FOR UPDATE", + ( + lease.scope.principal.account_id, + lease.scope.consumer_id, + lease.attempt.reporting_materialization_id, + lease.token, + ), + ) + ).fetchone() + if row is None: + return None + work = cast(dict[str, Any], row[0]) + if ( + work["external_id"] != lease.request.external_id + or work["generation"] != lease.generation + or work["binding_sha256"] != lease.request.binding_fingerprint + or work["notifications_enabled"] != lease.notifications_enabled + or work["notifications_enabled"] != self._notifications_enabled + ): + raise failure("BINDING_MISMATCH") + return work + + @materializer_errors + async def renew_materialization( + self, lease: ReportingMaterializerLease, *, lease_seconds: int + ) -> bool: + validate_lease_seconds(lease_seconds) + async with self._connection() as connection, connection.transaction(): + await self._lock_account(connection, lease.scope.principal.account_id) + if await self._held_on(connection, lease) is None: + return False + await connection.execute( + "UPDATE reporting_materializer_work SET lease_until=clock_timestamp()" + " +make_interval(secs=>%s),due_at=clock_timestamp()+make_interval(secs=>%s)" + " WHERE account_id=%s AND consumer_id=%s AND reporting_materialization_id=%s", + ( + lease_seconds, + lease_seconds, + lease.scope.principal.account_id, + lease.scope.consumer_id, + lease.attempt.reporting_materialization_id, + ), + ) + await self._schedule_account_on(connection, lease.scope.principal.account_id) + return True + + async def _target_on( + self, + connection: Any, + lease: ReportingMaterializerLease, + ) -> tuple[MaterializerContext, MaterializerReason]: + context = await self._materializer_context_on(connection, lease.scope) + selected, reason = context.selection(await _now(connection)) + if reason != "ready": + return context, reason + row = await ( + await connection.execute( + f"SELECT generation FROM reporting_materializer_candidates WHERE {_SCOPE_WHERE}", # nosec B608 + _scope_args(lease.scope), + ) + ).fetchone() + if ( + row is None + or row[0] != lease.generation + or selected is None + or selected.reporting_revision_id != lease.attempt.reporting_revision_id + ): + return context, "target_changed" + if binding_fingerprint(context.binding) != lease.request.binding_fingerprint: + return context, "binding_changed" + return context, "ready" + + @materializer_errors + async def authorize_materialization(self, lease: ReportingMaterializerLease) -> None: + async with self._connection() as connection, connection.transaction(): + await self._lock_account(connection, lease.scope.principal.account_id) + await validate_materializer_schema( + connection, notifications=self._notifications_enabled + ) + if await self._held_on(connection, lease) is None: + raise failure("LEASE_LOST") + _, reason = await self._target_on(connection, lease) + if reason != "ready": + raise failure( + "HISTORY_CORRUPT" if reason == "history_corrupt" else "CURRENT_REVISION_CHANGED" + ) + + @materializer_errors + async def finish_materialization( + self, + lease: ReportingMaterializerLease, + *, + prepared: ReportingPreparedRevision | None = None, + verified: ReportingVerifiedDestination | None = None, + error: ReportingWriterFailure | None = None, + ) -> ReportingMaterializerTurn: + if verified is not None: + validate_verified_destination(verified, lease.request, token=lease.token) + if prepared is None or prepared.request != lease.request or error is not None: + raise failure("BINDING_MISMATCH") + elif error is None: + raise failure("BINDING_MISMATCH") + async with self._connection() as connection, connection.transaction(): + await self._lock_account(connection, lease.scope.principal.account_id) + await validate_materializer_schema( + connection, notifications=self._notifications_enabled + ) + held = await self._held_on(connection, lease) + if held is None: + replay = await ( + await connection.execute( + "SELECT reason FROM reporting_materializer_work WHERE account_id=%s" + " AND consumer_id=%s AND reporting_materialization_id=%s AND state='acked'" + " AND completion_token=%s::uuid AND external_id=%s", + ( + lease.scope.principal.account_id, + lease.scope.consumer_id, + lease.attempt.reporting_materialization_id, + lease.token, + lease.request.external_id, + ), + ) + ).fetchone() + if replay is not None: + return ReportingMaterializerTurn( + "verified" if replay[0] == "verified" else "failed", + replay[0], + lease.attempt.reporting_materialization_id, + ) + return ReportingMaterializerTurn( + "pending", "effect_unknown", lease.attempt.reporting_materialization_id + ) + context, reason = await self._target_on(connection, lease) + if reason != "ready": + error = ReportingWriterFailure("CURRENT_REVISION_CHANGED", "new_attempt", "applied") + verified = None + elif verified is not None and prepared is not None: + validate_materialization_target( + prepared, binding=context.binding, revisions=context.revisions + ) + if error is not None and ( + error.effect == "unknown" + or error.retry == "same_identity" + or error.code in {"DEADLINE_EXCEEDED", "LEASE_LOST"} + ): + delay = max(1, min(error.retry_after_seconds or 5, 300)) + await connection.execute( + "UPDATE reporting_materializer_work SET lease_token=NULL,lease_until=NULL," + " due_at=clock_timestamp()+make_interval(secs=>%s),reason='effect_unknown'" + " WHERE account_id=%s AND consumer_id=%s AND reporting_materialization_id=%s", + ( + delay, + lease.scope.principal.account_id, + lease.scope.consumer_id, + lease.attempt.reporting_materialization_id, + ), + ) + await self._schedule_account_on(connection, lease.scope.principal.account_id) + return ReportingMaterializerTurn( + "pending", "effect_unknown", lease.attempt.reporting_materialization_id + ) + now = await _now(connection) + if verified is not None and ( + context.delivery is None + or verified.resource.expires_at + < max( + context.delivery.resource_retained_until, + now + timedelta(days=context.binding.resource_retention_days), + ) + ): + error = ReportingWriterFailure("RESOURCE_UNAVAILABLE", "new_attempt", "applied") + verified = None + if verified is not None: + # Same-connection finish validation after all lock acquisition + # and DB-time reads; no I/O separates this from the copy below. + validate_verified_destination(verified, lease.request, token=lease.token) + outcome = ReportingMaterializationRecord( + lease.scope, + lease.attempt.reporting_revision_id, + lease.attempt.reporting_materialization_id, + context.binding.success_status if verified is not None else "failed", + now, + verified.resource if verified is not None else None, + replace(verified.verification, verified_at=now) if verified is not None else None, + public_failure(error) if error is not None else None, + ) + stored, inserted = await self._commit_record_on(connection, outcome, notify=False) + await self._materializer_dirty_on(connection, stored, context) + if inserted and verified is not None and self._notifications_enabled: + revision = next( + r + for r in context.revisions + if r.reporting_revision_id == stored.reporting_revision_id + ) + event = materialization_event( + stored, + context.records, + context.obligation, + revision, + context.configuration, + now, + ) + if event is None: + raise failure("BINDING_MISMATCH") + await enqueue_materializer_event_on(connection, event) + retry = error is not None and error.retry == "new_attempt" + if reason == "ready": + reason = ( + "verified" + if verified is not None + else ("retry" if retry else "operator_required") + ) + ack = await ( + await connection.execute( + "UPDATE reporting_materializer_work SET state='acked',acknowledged_at=%s," + " completion_token=lease_token,lease_token=NULL,lease_until=NULL," + " retry_allowed=%s,reason=%s" + " WHERE account_id=%s AND consumer_id=%s AND reporting_materialization_id=%s" + " AND lease_token=%s::uuid AND lease_until>clock_timestamp() RETURNING 1", + ( + now, + retry, + reason, + lease.scope.principal.account_id, + lease.scope.consumer_id, + lease.attempt.reporting_materialization_id, + lease.token, + ), + ) + ).fetchone() + if ack is None: + raise failure("LEASE_LOST") # Rolls back evidence, dirty, ACK and outbox together. + due = ( + now + timedelta(seconds=max(1, error.retry_after_seconds or 5)) + if retry and error + else None + ) + if reason == "target_changed": + due = now + elif reason not in {"retry", "verified"}: + due = None + if ( + reason == "inactive" + and context.configuration.activated_at is not None + and context.configuration.activated_at > now + ): + due = context.configuration.activated_at + if verified is not None: + due = verified.resource.expires_at + await self._park_on(connection, lease.scope, reason, due_at=due) + await self._schedule_account_on(connection, lease.scope.principal.account_id) + return ReportingMaterializerTurn( + "verified" if verified is not None else "failed", + reason, + stored.reporting_materialization_id, + ) + + async def _materializer_dirty_on( + self, connection: Any, outcome: ReportingMaterializationRecord, context: MaterializerContext + ) -> None: + from adcp.reporting.ledger.notification_events import delivery_dirty + + scope, reason, evidence = delivery_dirty(outcome, context.obligation) + await self._dirty_status(connection, scope, reason, after=evidence) + + from adcp.reporting.canonical_json import canonical_json_sha256_v1 + from adcp.reporting.ledger.pg import _json + from adcp.reporting.ledger.status_snapshot import settle_snapshot_on + + core = await settle_snapshot_on( + self, connection, account_id=scope.account_id, as_of=outcome.completed_at + ) + row = await ( + await connection.execute( + "INSERT INTO reporting_materializer_status_heads" + " (account_id,consumer_id,max_sequence)" + " VALUES (%s,%s,1) ON CONFLICT (account_id,consumer_id) DO UPDATE" + " SET max_sequence=reporting_materializer_status_heads.max_sequence+1" + " RETURNING max_sequence", + (scope.account_id, outcome.scope.consumer_id), + ) + ).fetchone() + assert row is not None + account = await ( + await connection.execute( + "UPDATE reporting_materializer_accounts SET captured_sequence=captured_sequence+1" + " WHERE account_id=%s RETURNING captured_sequence", + (scope.account_id,), + ) + ).fetchone() + assert account is not None + boundary = ReportingMaterializerBoundary( + outcome.scope.principal, + row[0], + account[0], + outcome.reporting_materialization_id, + outcome.completed_at, + private_snapshot(core, outcome.scope.principal), + await self._records(connection, outcome.scope.principal), + ).to_storage() + await connection.execute( + "INSERT INTO reporting_materializer_status_boundaries" + " (account_id,consumer_id,sequence,account_sequence,reporting_materialization_id," + " as_of,input,content_sha256)" + " VALUES (%s,%s,%s,%s,%s,%s,%s::jsonb,%s)", + ( + scope.account_id, + outcome.scope.consumer_id, + row[0], + account[0], + outcome.reporting_materialization_id, + outcome.completed_at, + _json(boundary), + canonical_json_sha256_v1(boundary), + ), + ) + + @materializer_errors + async def read_materializer_boundaries( + self, *, caller: ReportingDeliveryPrincipal, after: int = 0, limit: int = 100 + ) -> tuple[ReportingMaterializerBoundary, ...]: + if type(after) is not int or after < 0 or type(limit) is not int or not 1 <= limit <= 100: + raise ValueError("materializer boundary reads require bounded positions") + async with self._connection() as connection, connection.transaction(): + await self._lock_account(connection, caller.account_id) + rows = await ( + await connection.execute( + "SELECT input, content_sha256=reporting_payload_sha256(input)" + " FROM reporting_materializer_status_boundaries" + " WHERE account_id=%s AND consumer_id=%s AND sequence>%s" + " ORDER BY sequence LIMIT %s", + (caller.account_id, caller.consumer_id, after, limit), + ) + ).fetchall() + if any(not row[1] for row in rows): + raise failure("HISTORY_CORRUPT") + return tuple(decode_materializer_boundary(row[0]) for row in rows) + + @materializer_errors + async def import_pending_materialization( + self, + *, + scope: ReportingDeliveryScope, + reporting_materialization_id: str, + original_external_id: str, + keys: tuple[ReportingVerificationKey, ...], + ) -> None: + """Explicit operator recovery after verifying the original external identity. + + Never infer legacy effect history. An unknown/non-SDK external identity + requires operator cleanup and a public terminal outcome before recovery. + """ + await self.materializer_ready() + async with self._connection() as connection, connection.transaction(): + await self._lock_account(connection, scope.principal.account_id) + context = await self._materializer_context_on(connection, scope) + attempt = next( + ( + r + for r in context.records + if isinstance(r, ReportingMaterializationAttempt) + and r.reporting_materialization_id == reporting_materialization_id + and r.scope == scope + ), + None, + ) + if attempt is None or context.outcome(attempt) is not None: + raise failure("BINDING_MISMATCH") + if not context.resumable(attempt): + raise failure("HISTORY_CORRUPT") + key = key_for(context.binding, context.obligation, keys) + request = ReportingDestinationRequest.from_binding(context.binding, attempt, key) + if original_external_id != request.external_id: + raise failure("BINDING_MISMATCH") + await connection.execute( + "SELECT reporting_materializer_wake(%s)", (scope.principal.account_id,) + ) + await connection.execute( + "INSERT INTO reporting_materializer_candidates" + " (account_id,consumer_id,delivery_config_id,delivery_config_version," + " reporting_obligation_id)" + " VALUES (%s,%s,%s,%s,%s) ON CONFLICT DO NOTHING", + _scope_args(scope), + ) + await connection.execute( + "INSERT INTO reporting_materializer_work" + " (account_id,consumer_id,delivery_config_id,delivery_config_version," + " reporting_obligation_id,reporting_revision_id,reporting_materialization_id," + " generation,binding_sha256,verification_key_sha256,external_id,imported," + " notifications_enabled)" + " SELECT account_id,consumer_id,delivery_config_id,delivery_config_version," + " reporting_obligation_id,%s,%s,generation,%s,%s,%s,TRUE,%s" + f" FROM reporting_materializer_candidates WHERE {_SCOPE_WHERE}" # nosec B608 + " ON CONFLICT (account_id,consumer_id,reporting_materialization_id) DO NOTHING", + ( + attempt.reporting_revision_id, + reporting_materialization_id, + request.binding_fingerprint, + verification_key_id(key), + request.external_id, + self._notifications_enabled, + *_scope_args(scope), + ), + ) + await connection.execute( + "UPDATE reporting_materializer_work SET due_at=clock_timestamp()" + " WHERE account_id=%s AND consumer_id=%s AND reporting_materialization_id=%s" + " AND state='pending' AND lease_token IS NULL", + (scope.principal.account_id, scope.consumer_id, reporting_materialization_id), + ) diff --git a/src/adcp/reporting/materializer/required_schema.json b/src/adcp/reporting/materializer/required_schema.json new file mode 100644 index 000000000..483a548bd --- /dev/null +++ b/src/adcp/reporting/materializer/required_schema.json @@ -0,0 +1,750 @@ +{ + "column:reporting_materializer_accounts.account_id": { + "enabled": true, + "fingerprint": "fdc4b549138bf90a5af5ed00b87a86ff9a0cbea553e7f017b4d95510708f6f88" + }, + "column:reporting_materializer_accounts.captured_sequence": { + "enabled": true, + "fingerprint": "42202005517b72e082eb22c9eceb2ac0252815e5df700c83eb50c54cfeb46297" + }, + "column:reporting_materializer_accounts.due_at": { + "enabled": true, + "fingerprint": "6f1466ce5d0aaac8471e39834b9c4b1f85d6f7169d9238a245ac035ff518e0fc" + }, + "column:reporting_materializer_accounts.served_at": { + "enabled": true, + "fingerprint": "ff6b592f18aa2fe4a2d7393bd4976409093f2650300d7073f5f485bed633309c" + }, + "column:reporting_materializer_candidates.account_id": { + "enabled": true, + "fingerprint": "fdc4b549138bf90a5af5ed00b87a86ff9a0cbea553e7f017b4d95510708f6f88" + }, + "column:reporting_materializer_candidates.consumer_id": { + "enabled": true, + "fingerprint": "fdc4b549138bf90a5af5ed00b87a86ff9a0cbea553e7f017b4d95510708f6f88" + }, + "column:reporting_materializer_candidates.delivery_config_id": { + "enabled": true, + "fingerprint": "fdc4b549138bf90a5af5ed00b87a86ff9a0cbea553e7f017b4d95510708f6f88" + }, + "column:reporting_materializer_candidates.delivery_config_version": { + "enabled": true, + "fingerprint": "54a5f9962598caa8723435f8b55bd5e15823053af5dd63d531663997537dc6ba" + }, + "column:reporting_materializer_candidates.due_at": { + "enabled": true, + "fingerprint": "6f1466ce5d0aaac8471e39834b9c4b1f85d6f7169d9238a245ac035ff518e0fc" + }, + "column:reporting_materializer_candidates.generation": { + "enabled": true, + "fingerprint": "69ade56844c40f1606daf51b6a1de9e07d8b838db8f35a89e7ef1576035794be" + }, + "column:reporting_materializer_candidates.reason": { + "enabled": true, + "fingerprint": "806b02ec486fb1a02f57be76cf01aba1287baf2d669233d70849bcb0df7c558c" + }, + "column:reporting_materializer_candidates.reporting_obligation_id": { + "enabled": true, + "fingerprint": "fdc4b549138bf90a5af5ed00b87a86ff9a0cbea553e7f017b4d95510708f6f88" + }, + "column:reporting_materializer_candidates.served_at": { + "enabled": true, + "fingerprint": "ff6b592f18aa2fe4a2d7393bd4976409093f2650300d7073f5f485bed633309c" + }, + "column:reporting_materializer_discovery.account_id": { + "enabled": true, + "fingerprint": "fdc4b549138bf90a5af5ed00b87a86ff9a0cbea553e7f017b4d95510708f6f88" + }, + "column:reporting_materializer_discovery.after_obligation_id": { + "enabled": true, + "fingerprint": "128b66e02f14946100273f112c144af7605d626124d20fe16c1ba30c6b19ae3a" + }, + "column:reporting_materializer_discovery.complete": { + "enabled": true, + "fingerprint": "981e469ff869d932309f9e6aab9b07ff394c7439fe2431281320e25b41c4198d" + }, + "column:reporting_materializer_discovery.consumer_id": { + "enabled": true, + "fingerprint": "fdc4b549138bf90a5af5ed00b87a86ff9a0cbea553e7f017b4d95510708f6f88" + }, + "column:reporting_materializer_discovery.delivery_config_id": { + "enabled": true, + "fingerprint": "fdc4b549138bf90a5af5ed00b87a86ff9a0cbea553e7f017b4d95510708f6f88" + }, + "column:reporting_materializer_discovery.delivery_config_version": { + "enabled": true, + "fingerprint": "54a5f9962598caa8723435f8b55bd5e15823053af5dd63d531663997537dc6ba" + }, + "column:reporting_materializer_notification_events.account_id": { + "enabled": true, + "fingerprint": "fdc4b549138bf90a5af5ed00b87a86ff9a0cbea553e7f017b4d95510708f6f88" + }, + "column:reporting_materializer_notification_events.admission_epoch": { + "enabled": true, + "fingerprint": "42202005517b72e082eb22c9eceb2ac0252815e5df700c83eb50c54cfeb46297" + }, + "column:reporting_materializer_notification_events.cause_generation": { + "enabled": true, + "fingerprint": "54a5f9962598caa8723435f8b55bd5e15823053af5dd63d531663997537dc6ba" + }, + "column:reporting_materializer_notification_events.cause_id": { + "enabled": true, + "fingerprint": "fdc4b549138bf90a5af5ed00b87a86ff9a0cbea553e7f017b4d95510708f6f88" + }, + "column:reporting_materializer_notification_events.cause_kind": { + "enabled": true, + "fingerprint": "fdc4b549138bf90a5af5ed00b87a86ff9a0cbea553e7f017b4d95510708f6f88" + }, + "column:reporting_materializer_notification_events.consumer_namespace": { + "enabled": true, + "fingerprint": "fdc4b549138bf90a5af5ed00b87a86ff9a0cbea553e7f017b4d95510708f6f88" + }, + "column:reporting_materializer_notification_events.fired_at": { + "enabled": true, + "fingerprint": "1cac4e73af11a8ecafd646ef6a0ff6ecb087d46f150408dcfebc630fe1bf5e1e" + }, + "column:reporting_materializer_notification_events.notification_id": { + "enabled": true, + "fingerprint": "fdc4b549138bf90a5af5ed00b87a86ff9a0cbea553e7f017b4d95510708f6f88" + }, + "column:reporting_materializer_notification_events.notification_type": { + "enabled": true, + "fingerprint": "fdc4b549138bf90a5af5ed00b87a86ff9a0cbea553e7f017b4d95510708f6f88" + }, + "column:reporting_materializer_notification_events.reporting_materialization_id": { + "enabled": true, + "fingerprint": "a0d17631e6e95ba976e4615a034020bc089fa63601fb91205ebb35798a39c0de" + }, + "column:reporting_materializer_notification_events.snapshot": { + "enabled": true, + "fingerprint": "ac355fc16c02b70cb0a24afee8214cdce5f5cbfdc7fd1630786d5101932ecfa4" + }, + "column:reporting_materializer_notification_expansions.account_id": { + "enabled": true, + "fingerprint": "fdc4b549138bf90a5af5ed00b87a86ff9a0cbea553e7f017b4d95510708f6f88" + }, + "column:reporting_materializer_notification_expansions.claim_count": { + "enabled": true, + "fingerprint": "42202005517b72e082eb22c9eceb2ac0252815e5df700c83eb50c54cfeb46297" + }, + "column:reporting_materializer_notification_expansions.consumer_namespace": { + "enabled": true, + "fingerprint": "fdc4b549138bf90a5af5ed00b87a86ff9a0cbea553e7f017b4d95510708f6f88" + }, + "column:reporting_materializer_notification_expansions.due_at": { + "enabled": true, + "fingerprint": "1cac4e73af11a8ecafd646ef6a0ff6ecb087d46f150408dcfebc630fe1bf5e1e" + }, + "column:reporting_materializer_notification_expansions.emission_generation": { + "enabled": true, + "fingerprint": "54a5f9962598caa8723435f8b55bd5e15823053af5dd63d531663997537dc6ba" + }, + "column:reporting_materializer_notification_expansions.error_code": { + "enabled": true, + "fingerprint": "5b92595d0b54d473a3a3818455845f1fe0dc20b48cfe0faf63061a82d1a3cb02" + }, + "column:reporting_materializer_notification_expansions.lease_expires_at": { + "enabled": true, + "fingerprint": "6f1466ce5d0aaac8471e39834b9c4b1f85d6f7169d9238a245ac035ff518e0fc" + }, + "column:reporting_materializer_notification_expansions.lease_token": { + "enabled": true, + "fingerprint": "5b92595d0b54d473a3a3818455845f1fe0dc20b48cfe0faf63061a82d1a3cb02" + }, + "column:reporting_materializer_notification_expansions.notification_id": { + "enabled": true, + "fingerprint": "fdc4b549138bf90a5af5ed00b87a86ff9a0cbea553e7f017b4d95510708f6f88" + }, + "column:reporting_materializer_notification_expansions.state": { + "enabled": true, + "fingerprint": "a1ea96aa09ee40a7801ee346ca59f35c6ceccd207397a2b7bfd70d130d159091" + }, + "column:reporting_materializer_status_boundaries.account_id": { + "enabled": true, + "fingerprint": "fdc4b549138bf90a5af5ed00b87a86ff9a0cbea553e7f017b4d95510708f6f88" + }, + "column:reporting_materializer_status_boundaries.account_sequence": { + "enabled": true, + "fingerprint": "54a5f9962598caa8723435f8b55bd5e15823053af5dd63d531663997537dc6ba" + }, + "column:reporting_materializer_status_boundaries.as_of": { + "enabled": true, + "fingerprint": "1cac4e73af11a8ecafd646ef6a0ff6ecb087d46f150408dcfebc630fe1bf5e1e" + }, + "column:reporting_materializer_status_boundaries.consumer_id": { + "enabled": true, + "fingerprint": "fdc4b549138bf90a5af5ed00b87a86ff9a0cbea553e7f017b4d95510708f6f88" + }, + "column:reporting_materializer_status_boundaries.content_sha256": { + "enabled": true, + "fingerprint": "fdc4b549138bf90a5af5ed00b87a86ff9a0cbea553e7f017b4d95510708f6f88" + }, + "column:reporting_materializer_status_boundaries.input": { + "enabled": true, + "fingerprint": "ac355fc16c02b70cb0a24afee8214cdce5f5cbfdc7fd1630786d5101932ecfa4" + }, + "column:reporting_materializer_status_boundaries.outcome_namespace": { + "enabled": true, + "fingerprint": "37f064bd049ffa20c99bccfcb2ddd77b4e7b65fc6471d7683087ed0beec2098e" + }, + "column:reporting_materializer_status_boundaries.reporting_materialization_id": { + "enabled": true, + "fingerprint": "fdc4b549138bf90a5af5ed00b87a86ff9a0cbea553e7f017b4d95510708f6f88" + }, + "column:reporting_materializer_status_boundaries.sequence": { + "enabled": true, + "fingerprint": "54a5f9962598caa8723435f8b55bd5e15823053af5dd63d531663997537dc6ba" + }, + "column:reporting_materializer_status_heads.account_id": { + "enabled": true, + "fingerprint": "fdc4b549138bf90a5af5ed00b87a86ff9a0cbea553e7f017b4d95510708f6f88" + }, + "column:reporting_materializer_status_heads.consumer_id": { + "enabled": true, + "fingerprint": "fdc4b549138bf90a5af5ed00b87a86ff9a0cbea553e7f017b4d95510708f6f88" + }, + "column:reporting_materializer_status_heads.max_sequence": { + "enabled": true, + "fingerprint": "54a5f9962598caa8723435f8b55bd5e15823053af5dd63d531663997537dc6ba" + }, + "column:reporting_materializer_work.account_id": { + "enabled": true, + "fingerprint": "fdc4b549138bf90a5af5ed00b87a86ff9a0cbea553e7f017b4d95510708f6f88" + }, + "column:reporting_materializer_work.acknowledged_at": { + "enabled": true, + "fingerprint": "6f1466ce5d0aaac8471e39834b9c4b1f85d6f7169d9238a245ac035ff518e0fc" + }, + "column:reporting_materializer_work.admission_epoch": { + "enabled": true, + "fingerprint": "42202005517b72e082eb22c9eceb2ac0252815e5df700c83eb50c54cfeb46297" + }, + "column:reporting_materializer_work.attempt_namespace": { + "enabled": true, + "fingerprint": "48602ba37bcbee2d9c8104042cc7868262df4d288eaf1543e387b9a059a08117" + }, + "column:reporting_materializer_work.binding_sha256": { + "enabled": true, + "fingerprint": "fdc4b549138bf90a5af5ed00b87a86ff9a0cbea553e7f017b4d95510708f6f88" + }, + "column:reporting_materializer_work.completion_token": { + "enabled": true, + "fingerprint": "88223bccb5aae7ddfe4b3feacd193586f6699cbfaa3507ae6c21250775392e25" + }, + "column:reporting_materializer_work.consumer_id": { + "enabled": true, + "fingerprint": "fdc4b549138bf90a5af5ed00b87a86ff9a0cbea553e7f017b4d95510708f6f88" + }, + "column:reporting_materializer_work.created_at": { + "enabled": true, + "fingerprint": "336df3243b293695d8321965e92f26d3f132e11514ed9678fb5e64e0015fa580" + }, + "column:reporting_materializer_work.delivery_config_id": { + "enabled": true, + "fingerprint": "fdc4b549138bf90a5af5ed00b87a86ff9a0cbea553e7f017b4d95510708f6f88" + }, + "column:reporting_materializer_work.delivery_config_version": { + "enabled": true, + "fingerprint": "54a5f9962598caa8723435f8b55bd5e15823053af5dd63d531663997537dc6ba" + }, + "column:reporting_materializer_work.due_at": { + "enabled": true, + "fingerprint": "336df3243b293695d8321965e92f26d3f132e11514ed9678fb5e64e0015fa580" + }, + "column:reporting_materializer_work.external_id": { + "enabled": true, + "fingerprint": "fdc4b549138bf90a5af5ed00b87a86ff9a0cbea553e7f017b4d95510708f6f88" + }, + "column:reporting_materializer_work.generation": { + "enabled": true, + "fingerprint": "54a5f9962598caa8723435f8b55bd5e15823053af5dd63d531663997537dc6ba" + }, + "column:reporting_materializer_work.imported": { + "enabled": true, + "fingerprint": "981e469ff869d932309f9e6aab9b07ff394c7439fe2431281320e25b41c4198d" + }, + "column:reporting_materializer_work.lease_token": { + "enabled": true, + "fingerprint": "88223bccb5aae7ddfe4b3feacd193586f6699cbfaa3507ae6c21250775392e25" + }, + "column:reporting_materializer_work.lease_until": { + "enabled": true, + "fingerprint": "6f1466ce5d0aaac8471e39834b9c4b1f85d6f7169d9238a245ac035ff518e0fc" + }, + "column:reporting_materializer_work.notifications_enabled": { + "enabled": true, + "fingerprint": "1abe3a1c570fbe885784dab5d979307c373d50f567ff4481ce3996869bf58fed" + }, + "column:reporting_materializer_work.reason": { + "enabled": true, + "fingerprint": "806b02ec486fb1a02f57be76cf01aba1287baf2d669233d70849bcb0df7c558c" + }, + "column:reporting_materializer_work.reporting_materialization_id": { + "enabled": true, + "fingerprint": "fdc4b549138bf90a5af5ed00b87a86ff9a0cbea553e7f017b4d95510708f6f88" + }, + "column:reporting_materializer_work.reporting_obligation_id": { + "enabled": true, + "fingerprint": "fdc4b549138bf90a5af5ed00b87a86ff9a0cbea553e7f017b4d95510708f6f88" + }, + "column:reporting_materializer_work.reporting_revision_id": { + "enabled": true, + "fingerprint": "fdc4b549138bf90a5af5ed00b87a86ff9a0cbea553e7f017b4d95510708f6f88" + }, + "column:reporting_materializer_work.retry_allowed": { + "enabled": true, + "fingerprint": "981e469ff869d932309f9e6aab9b07ff394c7439fe2431281320e25b41c4198d" + }, + "column:reporting_materializer_work.state": { + "enabled": true, + "fingerprint": "675d9766cc6787c63f91ee16167a1de38069be7ad44f6d8362bd498885ae1355" + }, + "column:reporting_materializer_work.verification_key_sha256": { + "enabled": true, + "fingerprint": "fdc4b549138bf90a5af5ed00b87a86ff9a0cbea553e7f017b4d95510708f6f88" + }, + "constraint:reporting_materializer_accounts.reporting_materializer_accounts_captured_sequence_check": { + "enabled": true, + "fingerprint": "1d2836d91f73454d02a4299e43a4ed52d0df6f60d4a1c62358b213b1e2f184b5" + }, + "constraint:reporting_materializer_accounts.reporting_materializer_accounts_pkey": { + "enabled": true, + "fingerprint": "e65d70e61c89a93d66c4c4f4c59ef755d0ff526b0fcdd1d9ae6d830d2abea913" + }, + "constraint:reporting_materializer_candidates.reporting_materializer_candid_account_id_delivery_config_i_fkey": { + "enabled": true, + "fingerprint": "f2fd7665f778fae1bec63184c8468e0212256b25009a44659abd7284bc54b8a7" + }, + "constraint:reporting_materializer_candidates.reporting_materializer_candidates_account_id_fkey": { + "enabled": true, + "fingerprint": "56b5c2865d458269356ade49fde975a0ebaf0a1338a80846132d5ec172dcddf5" + }, + "constraint:reporting_materializer_candidates.reporting_materializer_candidates_delivery_config_version_check": { + "enabled": true, + "fingerprint": "aa64b81d15585376b0e6a0904344e39bc5010257097cee5c8118967efdac4d34" + }, + "constraint:reporting_materializer_candidates.reporting_materializer_candidates_generation_check": { + "enabled": true, + "fingerprint": "04ce269eb45cc51febf8cbf44b760185b1eb2fd4b1935d6b53ecae063496539e" + }, + "constraint:reporting_materializer_candidates.reporting_materializer_candidates_pkey": { + "enabled": true, + "fingerprint": "eeabaa92e67458b97f1288ad0ca744a7a6b2bdcf24bed3511258141cf0635498" + }, + "constraint:reporting_materializer_candidates.reporting_materializer_candidates_reason_check": { + "enabled": true, + "fingerprint": "d2dc21e781a468c735dd8b305b5405e779c85c27657683c19570cb1ba4b94aa7" + }, + "constraint:reporting_materializer_discovery.reporting_materializer_discov_account_id_delivery_config_i_fkey": { + "enabled": true, + "fingerprint": "4761471309c35947c8d3928dce4595b4d455e7de6166aeb78a5de220465cb64a" + }, + "constraint:reporting_materializer_discovery.reporting_materializer_discovery_delivery_config_version_check": { + "enabled": true, + "fingerprint": "aa64b81d15585376b0e6a0904344e39bc5010257097cee5c8118967efdac4d34" + }, + "constraint:reporting_materializer_discovery.reporting_materializer_discovery_pkey": { + "enabled": true, + "fingerprint": "cb0658b07aa5249a6934cca6248293967db4ab1a848d706c3d5310fa98f08785" + }, + "constraint:reporting_materializer_notification_events.reporting_materializer_notifi_account_id_consumer_namespac_fkey": { + "enabled": true, + "fingerprint": "9274f47b408013aa1d44c57310026f2178212e0e3ec823f2a61d21cc08efd2f1" + }, + "constraint:reporting_materializer_notification_events.reporting_materializer_notifi_account_id_consumer_namespace_key": { + "enabled": true, + "fingerprint": "7156cd6623f45f181906d2886b53dfba82bfdfc6cdcbdd0017c1f1cd7ceae407" + }, + "constraint:reporting_materializer_notification_events.reporting_materializer_notification_ev_consumer_namespace_check": { + "enabled": true, + "fingerprint": "69951e411156e1739417ca750e81d5d806b2fcccea7e1914674f730ca6433b58" + }, + "constraint:reporting_materializer_notification_events.reporting_materializer_notification_eve_notification_type_check": { + "enabled": true, + "fingerprint": "4dc2b2e5d5fe003a3b98e1832dd453c9115cc3d03d59d3cb23258e90605548e3" + }, + "constraint:reporting_materializer_notification_events.reporting_materializer_notification_even_cause_generation_check": { + "enabled": true, + "fingerprint": "b43ce8b2f74254027bcc0d382aa0bda21801bd6539f4ea3365679718c5cd80ba" + }, + "constraint:reporting_materializer_notification_events.reporting_materializer_notification_event_admission_epoch_check": { + "enabled": true, + "fingerprint": "abd3e698b241244e9e751e940838ef5bcecdd79d21aab371f4b595818b5b1f95" + }, + "constraint:reporting_materializer_notification_events.reporting_materializer_notification_events_cause_kind_check": { + "enabled": true, + "fingerprint": "4d213a62ba48d8d428c2811ddc5039b1349d3a41943509a85062b43235ab669d" + }, + "constraint:reporting_materializer_notification_events.reporting_materializer_notification_events_check": { + "enabled": true, + "fingerprint": "bc5fdbb84edccb22f49869598e550f0f72ee45323f0852be3438c8c5d5a387fc" + }, + "constraint:reporting_materializer_notification_events.reporting_materializer_notification_events_check1": { + "enabled": true, + "fingerprint": "3d95e12acbdfe40cfd7c36ea0c4cceddc8106395e3165b6c8034dd14783ba596" + }, + "constraint:reporting_materializer_notification_events.reporting_materializer_notification_events_check2": { + "enabled": true, + "fingerprint": "abcd49a900cec31d6a71175389de38cf44ffe6d16fa9083e45fe82967313db78" + }, + "constraint:reporting_materializer_notification_events.reporting_materializer_notification_events_check3": { + "enabled": true, + "fingerprint": "66bffc213553425f6a34852e7394536bdd550304c09cdb472a70223bcadb9f7b" + }, + "constraint:reporting_materializer_notification_events.reporting_materializer_notification_events_pkey": { + "enabled": true, + "fingerprint": "e8d63f44b25700915bcb3cb69e2e24b98cf21fd304f36a2e9c2992236309309d" + }, + "constraint:reporting_materializer_notification_expansions.reporting_materializer_notif_account_id_consumer_namespac_fkey1": { + "enabled": true, + "fingerprint": "6819542fa33be0e7a42bead23f6729b2dfb82d447b59543aaa96071054d56289" + }, + "constraint:reporting_materializer_notification_expansions.reporting_materializer_notification_e_emission_generation_check": { + "enabled": true, + "fingerprint": "5b7b2793bd710c7eef8c02003fd8adb1df91d7c10659364fe37d7f6351d4bf5e" + }, + "constraint:reporting_materializer_notification_expansions.reporting_materializer_notification_expansions_error_code_check": { + "enabled": true, + "fingerprint": "502085808cda7e7ec1e45e3d88257a660bdc65b59a71727b473303a362b9cf9d" + }, + "constraint:reporting_materializer_notification_expansions.reporting_materializer_notification_expansions_pkey": { + "enabled": true, + "fingerprint": "8ac920e8e1ed5a59417d5d1ffb9bea608bbeb64dbcd1a431240370fe555fe6f0" + }, + "constraint:reporting_materializer_notification_expansions.reporting_materializer_notification_expansions_state_check": { + "enabled": true, + "fingerprint": "12b7a569e49f4bf374d5dade9ec0a0d812e4dea251366467c95bcd58bee90b9d" + }, + "constraint:reporting_materializer_status_boundaries.reporting_materializer_status_account_id_consumer_id_outco_fkey": { + "enabled": true, + "fingerprint": "e3e10c3f3d574511ee5944b8bade5d927099809b0a45539ce8502caa9ebb05bf" + }, + "constraint:reporting_materializer_status_boundaries.reporting_materializer_status_account_id_consumer_id_repor_fkey": { + "enabled": true, + "fingerprint": "837c9aa80214bc7b0b25cd299aec9cad71de4bb616eefd0960ae5ca57efb1683" + }, + "constraint:reporting_materializer_status_boundaries.reporting_materializer_status_account_id_consumer_id_report_key": { + "enabled": true, + "fingerprint": "72945097549e77da2940c9cc5e54db645c12c0d5ef3bed0d4d7e7ba5cc93ffc1" + }, + "constraint:reporting_materializer_status_boundaries.reporting_materializer_status_b_account_id_account_sequence_key": { + "enabled": true, + "fingerprint": "460b11d8840751c3e894245e33c24e704407f4b3ca42bbd9e832cb48a0c3b24d" + }, + "constraint:reporting_materializer_status_boundaries.reporting_materializer_status_boundarie_outcome_namespace_check": { + "enabled": true, + "fingerprint": "60e879d7d85bbf368a4f30d08481dd59f5aa1d92eb6943b0e60a8e8e219c2ff2" + }, + "constraint:reporting_materializer_status_boundaries.reporting_materializer_status_boundaries_account_sequence_check": { + "enabled": true, + "fingerprint": "1a64697a81e3b39433d851f9916df1964cbbc260aa54dd6fee4076c7e805c70e" + }, + "constraint:reporting_materializer_status_boundaries.reporting_materializer_status_boundaries_check": { + "enabled": true, + "fingerprint": "6db9e761fce16f6deaabe04581b3813f3495382b5edf905c2c01a772151eed89" + }, + "constraint:reporting_materializer_status_boundaries.reporting_materializer_status_boundaries_check1": { + "enabled": true, + "fingerprint": "6c5865ad0c28d00ec74194418def9d764ddf85b230f1dfb6d14c6e505eda3737" + }, + "constraint:reporting_materializer_status_boundaries.reporting_materializer_status_boundaries_check2": { + "enabled": true, + "fingerprint": "307105bea1a76e214540a207ab8dddcffb26853845a89236454a6e9ecb0508d3" + }, + "constraint:reporting_materializer_status_boundaries.reporting_materializer_status_boundaries_check3": { + "enabled": true, + "fingerprint": "65d0dca50d7ca308123252fcc784fb427fd857d8bf3092bd85fc8fff5787d7ec" + }, + "constraint:reporting_materializer_status_boundaries.reporting_materializer_status_boundaries_check4": { + "enabled": true, + "fingerprint": "0d806a2b956a1e8947dcc43c634352ed52c1c3f158235145b2fe5a96db5fb379" + }, + "constraint:reporting_materializer_status_boundaries.reporting_materializer_status_boundaries_check5": { + "enabled": true, + "fingerprint": "b330c171bc0bb87ac12aa97aca4d4ae7167e0c07d7b61b3b688c63df9588d927" + }, + "constraint:reporting_materializer_status_boundaries.reporting_materializer_status_boundaries_check6": { + "enabled": true, + "fingerprint": "9b7f5755a4d4093ab3618e85ff78ea92e931d1895d601ee48f781dcbca07155d" + }, + "constraint:reporting_materializer_status_boundaries.reporting_materializer_status_boundaries_content_sha256_check": { + "enabled": true, + "fingerprint": "2afe58d90df96e397cb8e6a941e52fcf8e01b7835024dd3b4ba67141a3578505" + }, + "constraint:reporting_materializer_status_boundaries.reporting_materializer_status_boundaries_input_check": { + "enabled": true, + "fingerprint": "1403f246fdde17015118d212847ed5d8169df3adc0364598cbd5e82be066e1bf" + }, + "constraint:reporting_materializer_status_boundaries.reporting_materializer_status_boundaries_input_check1": { + "enabled": true, + "fingerprint": "fadd88bcfddae6b78d4fc68c90018f5c21ba800b5beb4ddfb2d5805da7703a2b" + }, + "constraint:reporting_materializer_status_boundaries.reporting_materializer_status_boundaries_pkey": { + "enabled": true, + "fingerprint": "00dd5f9ba4e0d0face7ab27fe5e52ffa9c4101943c95afb8ec1badb401c7010d" + }, + "constraint:reporting_materializer_status_boundaries.reporting_materializer_status_boundaries_sequence_check": { + "enabled": true, + "fingerprint": "554d491362648e795a6567528a6244977d5df276216297f35e26e4750f869dcf" + }, + "constraint:reporting_materializer_status_heads.reporting_materializer_status_heads_max_sequence_check": { + "enabled": true, + "fingerprint": "ee47acec8d450eed7f99b637596847d21bc449a12317a605fd6b6cb0fad6aa79" + }, + "constraint:reporting_materializer_status_heads.reporting_materializer_status_heads_pkey": { + "enabled": true, + "fingerprint": "4d853add149814edf9eadd3f752bf43f1f7164cba2bd079f0349d4d540a7f20c" + }, + "constraint:reporting_materializer_work.reporting_materializer_work_account_id_consumer_id_attempt_fkey": { + "enabled": true, + "fingerprint": "e3a1d24fef8da0fbe5587a7f7887a4cb9adf0133071070a09c950db1474304c8" + }, + "constraint:reporting_materializer_work.reporting_materializer_work_account_id_consumer_id_deliver_fkey": { + "enabled": true, + "fingerprint": "9390df97c6b965e9f16f700a4bd4e90392e60f89a15b6a9840d05682cdcc532f" + }, + "constraint:reporting_materializer_work.reporting_materializer_work_account_id_consumer_id_external_key": { + "enabled": true, + "fingerprint": "78641c1ca086a76ec1a81540664d98811ffb2adb51d5dc27db4d307f428bd8cd" + }, + "constraint:reporting_materializer_work.reporting_materializer_work_account_id_reporting_obligatio_fkey": { + "enabled": true, + "fingerprint": "a0dca2cc4a27c9d4380eadb1513d30fe6383849657fc5e0d0f93c4b4c4ed4c08" + }, + "constraint:reporting_materializer_work.reporting_materializer_work_admission_epoch_check": { + "enabled": true, + "fingerprint": "abd3e698b241244e9e751e940838ef5bcecdd79d21aab371f4b595818b5b1f95" + }, + "constraint:reporting_materializer_work.reporting_materializer_work_attempt_namespace_check": { + "enabled": true, + "fingerprint": "639b8bfe70df1945046d65f693e67c45a4015e0e318356d77a97f5e4b3a7b629" + }, + "constraint:reporting_materializer_work.reporting_materializer_work_binding_sha256_check": { + "enabled": true, + "fingerprint": "c9db9bf2eca41364cec2fc19983d43e23fedf6711e5c4deb5f1d80aed317fd2b" + }, + "constraint:reporting_materializer_work.reporting_materializer_work_check": { + "enabled": true, + "fingerprint": "a6826ba06594c4200bda46be2704e03cb9d234d36538780211c04c2b2392e61b" + }, + "constraint:reporting_materializer_work.reporting_materializer_work_check1": { + "enabled": true, + "fingerprint": "311f8f3d7b44fd93e0b4ed5945b86997d4c6fd3beea28eca693b1e506466676d" + }, + "constraint:reporting_materializer_work.reporting_materializer_work_check2": { + "enabled": true, + "fingerprint": "3b94ebc10b0f1c882069a518c6496ff87db17e6b3405aecdb2f9aca7b158c644" + }, + "constraint:reporting_materializer_work.reporting_materializer_work_check3": { + "enabled": true, + "fingerprint": "fc30826fbb5c547751d5b430e90d54631bc537ca3af856785d68bb4a350cf9cb" + }, + "constraint:reporting_materializer_work.reporting_materializer_work_check4": { + "enabled": true, + "fingerprint": "50525da46b6ed71b60656e35fbba42e4fcbb45719c4d2032c2eccf35b971fafc" + }, + "constraint:reporting_materializer_work.reporting_materializer_work_external_id_check": { + "enabled": true, + "fingerprint": "df053aff1b660dec5635dbcc1043c90a9b338c3d871b72d4ed26ec60d465d278" + }, + "constraint:reporting_materializer_work.reporting_materializer_work_generation_check": { + "enabled": true, + "fingerprint": "04ce269eb45cc51febf8cbf44b760185b1eb2fd4b1935d6b53ecae063496539e" + }, + "constraint:reporting_materializer_work.reporting_materializer_work_pkey": { + "enabled": true, + "fingerprint": "fb2ee6241427fec8a9ae91d2940abe9e20b25f47554bd2d0a7c4046eb7704c41" + }, + "constraint:reporting_materializer_work.reporting_materializer_work_reason_check": { + "enabled": true, + "fingerprint": "d2dc21e781a468c735dd8b305b5405e779c85c27657683c19570cb1ba4b94aa7" + }, + "constraint:reporting_materializer_work.reporting_materializer_work_state_check": { + "enabled": true, + "fingerprint": "2e00e4b40728462003ac3d7f891ca8354aaeee6e19d808f7809030ff4db7758a" + }, + "constraint:reporting_materializer_work.reporting_materializer_work_verification_key_sha256_check": { + "enabled": true, + "fingerprint": "5442191b821cf162c8dfe50da9181b02acbf254eeb477c27b98e69d1fd26134b" + }, + "function:reporting_materializer_binding_dirty()": { + "enabled": true, + "fingerprint": "eea2454f1ded0e1bc966fac4cb567e43520182595e9961cd6052c6b7f5cf94e3" + }, + "function:reporting_materializer_dirty(a text, c text, d text, v bigint, o text)": { + "enabled": true, + "fingerprint": "bbcf34da712e0fae850148f133223786b751f6b03bc71c4c6d4f3f55bb78586b" + }, + "function:reporting_materializer_expansion_guard()": { + "enabled": true, + "fingerprint": "cb98dadaf9c1932f1ff1671b4cd2a7509488c6f00eacbeacb6b7cf069ea72356" + }, + "function:reporting_materializer_retained_guard()": { + "enabled": true, + "fingerprint": "f3a494b6d28a4e5340a3f4707da937a765122016eb68d93572ca05c982bae49b" + }, + "function:reporting_materializer_source_dirty()": { + "enabled": true, + "fingerprint": "49bb0040dd74ba1af0f4f0ecf7b308cff4d8f6cad76be064dbe7ea46e71aee3e" + }, + "function:reporting_materializer_wake(a text)": { + "enabled": true, + "fingerprint": "c0487a973b7faba9ee0766ebd845b67f44486f5e31cdb2cacb85fd7805a636e7" + }, + "function:reporting_materializer_work_guard()": { + "enabled": true, + "fingerprint": "53beb6d2d06c61acbdbd6963c9f4869421d98af3842ee4b574c885061ddcd955" + }, + "index:reporting_materializer_accounts.reporting_materializer_account_due": { + "enabled": true, + "fingerprint": "aa5282c87c241a1fb4231aedf5c2729ed8b7e4d72c5f6fddd93001fba5edf4b3" + }, + "index:reporting_materializer_accounts.reporting_materializer_account_wakeup": { + "enabled": true, + "fingerprint": "fd3be523100f148ee3d372fe749107f426ea21c1c5e22062c77329c8b3e5404d" + }, + "index:reporting_materializer_accounts.reporting_materializer_accounts_pkey": { + "enabled": true, + "fingerprint": "8b2d5d80f629b4f7a049604a72b947d79d365ab67ba4caaebc48d0ea8163167b" + }, + "index:reporting_materializer_candidates.reporting_materializer_candidate_due": { + "enabled": true, + "fingerprint": "3be6530dbb55b15eaada3576100fac5fc4d2b9852e848b3163cef3e2bc66ba48" + }, + "index:reporting_materializer_candidates.reporting_materializer_candidate_publication": { + "enabled": true, + "fingerprint": "999e962294bc6d17632cf79252bfa9be4a1cff222238b55ed920cad9a0eb24c3" + }, + "index:reporting_materializer_candidates.reporting_materializer_candidates_pkey": { + "enabled": true, + "fingerprint": "851a791d7161fb93f6d3e0001fb59d46ecf58b96c3df1f9cfad8983b59b7699b" + }, + "index:reporting_materializer_discovery.reporting_materializer_discovery_pending": { + "enabled": true, + "fingerprint": "be5e169a272b75f8ff83ccffb7ef1af3b629456a405c4170f5f4f40611a14860" + }, + "index:reporting_materializer_discovery.reporting_materializer_discovery_pkey": { + "enabled": true, + "fingerprint": "b6b8787db0e681f9bee61e99514097d8d0c150fcb78102e439a92c0483521f62" + }, + "index:reporting_materializer_notification_events.reporting_materializer_notifi_account_id_consumer_namespace_key": { + "enabled": true, + "fingerprint": "8d588106a4322296872f083deffc88263341ba7b62217919ce1cb64ba5c7175b" + }, + "index:reporting_materializer_notification_events.reporting_materializer_notification_events_pkey": { + "enabled": true, + "fingerprint": "d9a07bc15c257dc9c5f0832be98dca66e6fd908f12c4f739ff294a8a95ceb736" + }, + "index:reporting_materializer_notification_expansions.reporting_materializer_notification_due": { + "enabled": true, + "fingerprint": "b5ce1d459ed7406b956d3c9144b95693791ce956229e62aa419ca9dc92d91b03" + }, + "index:reporting_materializer_notification_expansions.reporting_materializer_notification_expansions_pkey": { + "enabled": true, + "fingerprint": "d58d479e1ce11640e782f92435570967ae778f19e3ae38e5c1d7d1c819123920" + }, + "index:reporting_materializer_status_boundaries.reporting_materializer_status_account_id_consumer_id_report_key": { + "enabled": true, + "fingerprint": "9608f9409e9efd9d58e70c1d3e6bb814b3e61bdd50c51dc8680fd4376520ca1c" + }, + "index:reporting_materializer_status_boundaries.reporting_materializer_status_b_account_id_account_sequence_key": { + "enabled": true, + "fingerprint": "da68c1f5003f093b5cf98c7ea057638c0c4690884972298b529433c3c457633d" + }, + "index:reporting_materializer_status_boundaries.reporting_materializer_status_boundaries_pkey": { + "enabled": true, + "fingerprint": "1862208bb1b96147d96b83f2f772208eb62dc39470d6eca03431b76f5ec07c6b" + }, + "index:reporting_materializer_status_heads.reporting_materializer_status_heads_pkey": { + "enabled": true, + "fingerprint": "6f3fd3013e95dc10388625b673ae88b6aac381066f81981825039dfaf913f412" + }, + "index:reporting_materializer_work.reporting_materializer_one_pending": { + "enabled": true, + "fingerprint": "c8fca3e3da79cc0c5488f1b7003b93169c727337b0fbf5e4b472878d1002aa14" + }, + "index:reporting_materializer_work.reporting_materializer_work_account_id_consumer_id_external_key": { + "enabled": true, + "fingerprint": "ab5fd65a6f6e47fff0b4d35473a7e6d05973d83721ba412722ae2d5dea03f4c6" + }, + "index:reporting_materializer_work.reporting_materializer_work_due": { + "enabled": true, + "fingerprint": "86ca449c8a2ab3161dad3dc6cfcca8e7fe45e01d3ac73472be058917463f865b" + }, + "index:reporting_materializer_work.reporting_materializer_work_pkey": { + "enabled": true, + "fingerprint": "a36b975739037fdb8f2eefa757f3265be6c06afe19ec8340fd8fa32669cf6220" + }, + "index:reporting_obligations.reporting_materializer_obligation_discovery": { + "enabled": true, + "fingerprint": "e715e2c28cbb6c25f9bb78d7c53422884ef9fa83c3740b4d1859bee1800c6855" + }, + "index:reporting_reconciliation_records.reporting_materializer_binding_discovery": { + "enabled": true, + "fingerprint": "c09880d92551983d9aec30e6a7cb34058db7f473849d8cc6d8a194baee1707a0" + }, + "table:reporting_materializer_accounts": { + "enabled": true, + "fingerprint": "1f824779ff80f110344420b019786663d8c9beaad230da90e0439795e734ccda" + }, + "table:reporting_materializer_candidates": { + "enabled": true, + "fingerprint": "1f824779ff80f110344420b019786663d8c9beaad230da90e0439795e734ccda" + }, + "table:reporting_materializer_discovery": { + "enabled": true, + "fingerprint": "1f824779ff80f110344420b019786663d8c9beaad230da90e0439795e734ccda" + }, + "table:reporting_materializer_notification_events": { + "enabled": true, + "fingerprint": "1f824779ff80f110344420b019786663d8c9beaad230da90e0439795e734ccda" + }, + "table:reporting_materializer_notification_expansions": { + "enabled": true, + "fingerprint": "1f824779ff80f110344420b019786663d8c9beaad230da90e0439795e734ccda" + }, + "table:reporting_materializer_status_boundaries": { + "enabled": true, + "fingerprint": "1f824779ff80f110344420b019786663d8c9beaad230da90e0439795e734ccda" + }, + "table:reporting_materializer_status_heads": { + "enabled": true, + "fingerprint": "1f824779ff80f110344420b019786663d8c9beaad230da90e0439795e734ccda" + }, + "table:reporting_materializer_work": { + "enabled": true, + "fingerprint": "1f824779ff80f110344420b019786663d8c9beaad230da90e0439795e734ccda" + }, + "trigger:reporting_configurations.reporting_materializer_configuration": { + "enabled": true, + "fingerprint": "68c28dd89618041f28c1e1fb5e833d1af17b14224cbe54cde4f0b3a62dc13407" + }, + "trigger:reporting_materializer_notification_events.reporting_materializer_event_immutable": { + "enabled": true, + "fingerprint": "faf323248f4100b2d93324789dcf1435eddfc55aa970b04e08cb1ef8705e1062" + }, + "trigger:reporting_materializer_notification_expansions.reporting_materializer_pre_activation_guard": { + "enabled": true, + "fingerprint": "f3132d843dd232dca85cb174aaf3d398a959d26bcfbcfd622e6e93be932fc23e" + }, + "trigger:reporting_materializer_status_boundaries.reporting_materializer_boundary_immutable": { + "enabled": true, + "fingerprint": "54cabb176a7b94c44c16c1e420d948dd8cf8462807ae41ac9d1f8fd3cfa391da" + }, + "trigger:reporting_materializer_work.reporting_materializer_guard": { + "enabled": true, + "fingerprint": "c157e0d74d41e71eaf8b1c5c391ff40231a6c909ddcdc891f3e046c76152d219" + }, + "trigger:reporting_obligations.reporting_materializer_obligation": { + "enabled": true, + "fingerprint": "cbea44a53e0542a797454a9af681915c5933dd92e3e00520b64a8f2849937234" + }, + "trigger:reporting_reconciliation_records.reporting_materializer_binding": { + "enabled": true, + "fingerprint": "45497bd65d22b5709f9ed2e3f863e3585ffc1d54a211e1fa76306237824612da" + }, + "trigger:reporting_revisions.reporting_materializer_publication": { + "enabled": true, + "fingerprint": "8afe04a8f60e5f863981505f688b55295a42694e6835951c30750470b1162b5b" + } +} diff --git a/src/adcp/reporting/materializer/schema.py b/src/adcp/reporting/materializer/schema.py new file mode 100644 index 000000000..c6c42ff25 --- /dev/null +++ b/src/adcp/reporting/materializer/schema.py @@ -0,0 +1,43 @@ +"""Materializer readiness is independent of every prior mandatory manifest.""" + +from __future__ import annotations + +import json +from importlib.resources import files +from typing import Any + +from adcp.reporting.ledger.store import LedgerConflictError +from adcp.reporting.outbox._schema import REQUIRED_OBJECTS, schema_objects, validate_schema + + +async def validate_materializer_schema(connection: Any, *, notifications: bool = False) -> None: + required = json.loads( + files("adcp.reporting.materializer").joinpath("required_schema.json").read_text() + ) + if not required: + raise LedgerConflictError("MATERIALIZER_SCHEMA_UNREADY", "install the materializer schema") + installed = None + try: + installed = await schema_objects(connection) + except Exception: + installed = None + if installed is None: + raise LedgerConflictError( + "MATERIALIZER_SCHEMA_UNREADY", "materializer schema catalog is unavailable" + ) + if any(installed.get(key) != expected for key, expected in required.items()): + raise LedgerConflictError("MATERIALIZER_SCHEMA_UNREADY", "install the materializer schema") + # Inherit the reviewed foundation prerequisites without appending B2 objects + # to A's manifest or requiring optional notification delivery infrastructure. + for key, expected in REQUIRED_OBJECTS.items(): + if any( + word in key + for word in ("notification_", "webhook_", "status_dirty", "status_checkpoints") + ): + continue + if installed.get(key) != expected: + raise LedgerConflictError( + "MATERIALIZER_SCHEMA_UNREADY", "install the retained evidence schema" + ) + if notifications: + await validate_schema(connection) diff --git a/src/adcp/reporting/materializer/service.py b/src/adcp/reporting/materializer/service.py new file mode 100644 index 000000000..fe29a14fd --- /dev/null +++ b/src/adcp/reporting/materializer/service.py @@ -0,0 +1,154 @@ +"""Service-owned heartbeat around destination I/O outside database transactions.""" + +from __future__ import annotations + +import asyncio +from dataclasses import dataclass, field +from datetime import datetime, timedelta, timezone + +from adcp.reporting.materializer.contracts import ( + ReportingDestinationWriter, + ReportingIOContext, + ReportingWriterError, + ReportingWriterFailure, + _join_tasks, + failure, +) +from adcp.reporting.materializer.verification import ReportingDestinationIO +from adcp.reporting.materializer.work import ( + ReportingMaterializerLease, + ReportingMaterializerStore, + ReportingMaterializerTurn, + validate_lease_seconds, +) + + +class _LeaseHeartbeat: + def __init__( + self, + store: ReportingMaterializerStore, + lease: ReportingMaterializerLease, + seconds: int, + cancel: asyncio.Event, + ) -> None: + self.store, self.lease, self.seconds, self.cancel = store, lease, seconds, cancel + self.stopped = asyncio.Event() + self.lost = False + + async def checkpoint(self) -> None: + if self.lost: + raise failure("LEASE_LOST") + + async def run(self) -> None: + while not self.stopped.is_set(): + try: + await asyncio.wait_for(self.stopped.wait(), self.seconds / 3) + return + except asyncio.TimeoutError: + pass + held = False + try: + held = await self.store.renew_materialization( + self.lease, lease_seconds=self.seconds + ) + except asyncio.CancelledError: + raise + except Exception: + held = False # Driver/provider bodies never escape service diagnostics. + if not held: + self.lost = True + self.cancel.set() + return + + +@dataclass(frozen=True) +class ReportingMaterializerService: + """One autonomous turn; schedule repeatedly, including after process restart. + + The destination must implement its advertised conditional/idempotent write + semantics durably. A timeout or interrupted outcome commit retains the same + immutable attempt. The SDK opens fresh authorization sessions for write and + paginated readback; the service checks current ledger authority before each. + """ + + store: ReportingMaterializerStore + io: ReportingDestinationIO = field(repr=False) + writer: ReportingDestinationWriter = field(repr=False) + lease_seconds: int = 30 + io_timeout_seconds: int = 300 + + def __post_init__(self) -> None: + validate_lease_seconds(self.lease_seconds) + if type(self.io_timeout_seconds) is not int or not 1 <= self.io_timeout_seconds <= 3600: + raise ValueError("materializer I/O deadline requires 1..3600 seconds") + if type(self.io) is not ReportingDestinationIO: + raise failure("UNSUPPORTED_VERIFICATION") + + async def run_once(self) -> ReportingMaterializerTurn: + keys = tuple( + v.key + for v in self.io.registry.verifiers + if v.key.capability in self.writer.capabilities + ) + reserved = await self.store.claim_materialization( + keys=keys, lease_seconds=self.lease_seconds + ) + if isinstance(reserved, ReportingMaterializerTurn): + return reserved + cancel = asyncio.Event() + heartbeat = _LeaseHeartbeat(self.store, reserved, self.lease_seconds, cancel) + task = asyncio.create_task(heartbeat.run()) + context = ReportingIOContext( + datetime.now(timezone.utc) + timedelta(seconds=self.io_timeout_seconds), + cancel, + heartbeat, + ) + canceled = False + turn = ReportingMaterializerTurn( + "pending", "effect_unknown", reserved.attempt.reporting_materialization_id + ) + try: + try: + await self.store.authorize_materialization(reserved) + target = reserved.context + if target.delivery is None: + raise failure("BINDING_MISMATCH") + prepared = await self.io.registry.prepare( + key=reserved.request.verification_key, + binding=target.binding, + delivery=target.delivery, + obligation=target.obligation, + revisions=target.revisions, + attempt=reserved.attempt, + reader=self.store, + context=context, + ) + await self.store.authorize_materialization(reserved) + locator = await self.io.write(prepared, context=context) + await self.store.authorize_materialization(reserved) + verified = await self.io.verify(prepared, locator, context=context) + turn = await self.store.finish_materialization( + reserved, prepared=prepared, verified=verified + ) + except ReportingWriterError as exc: + record = exc.failure + # The writer owns whether a failure is known and terminal. All + # uncertain I/O and lost leases keep the external identity. + if record.code in {"DEADLINE_EXCEEDED", "LEASE_LOST"}: + record = ReportingWriterFailure(record.code, "same_identity", "unknown") + turn = await self.store.finish_materialization(reserved, error=record) + except asyncio.CancelledError: + canceled = not heartbeat.lost + except Exception: + # Includes commit/connection loss and failed atomic outbox enqueue. + # The lease expires; a restart reuses the original identity. + turn = ReportingMaterializerTurn( + "pending", "effect_unknown", reserved.attempt.reporting_materialization_id + ) + finally: + heartbeat.stopped.set() + task.cancel() + canceled = await _join_tasks(task) or canceled + if canceled: + raise asyncio.CancelledError + return turn diff --git a/src/adcp/reporting/materializer/verification.py b/src/adcp/reporting/materializer/verification.py index 308b7880d..3fd36dcc7 100644 --- a/src/adcp/reporting/materializer/verification.py +++ b/src/adcp/reporting/materializer/verification.py @@ -10,10 +10,13 @@ import asyncio import base64 import hashlib +import hmac +import json import re +import secrets from collections.abc import AsyncIterator, Sequence from contextlib import asynccontextmanager -from dataclasses import dataclass, field, replace +from dataclasses import asdict, dataclass, field, replace from datetime import datetime, timedelta, timezone from decimal import Decimal, localcontext from typing import Any, Protocol, cast @@ -63,6 +66,8 @@ ) from adcp.reporting.revision_selection import select_reporting_revision +_VERIFICATION_SEAL_KEY = secrets.token_bytes(32) + class ReportingRevisionRowReader(Protocol): async def read_revision_rows( @@ -75,8 +80,14 @@ async def read_revision_rows( ) -> ReportingRowPage: ... +class _VerifiedProvenance(_ClosedValue): + # Non-dataclass slots deliberately stay out of B1's three-field constructor, + # dataclasses.fields/asdict, Pydantic schemas and serialized observations. + __slots__ = ("_sdk_seal", "_materializer_fence") + + @dataclass(frozen=True, slots=True) -class ReportingVerifiedDestination(_ClosedValue): +class ReportingVerifiedDestination(_VerifiedProvenance): """Immutable SDK observations. B2 alone owns atomic publication/readiness.""" request: ReportingDestinationRequest @@ -85,6 +96,43 @@ class ReportingVerifiedDestination(_ClosedValue): def __post_init__(self) -> None: _freeze_fields(self) + object.__setattr__(self, "_sdk_seal", b"") + object.__setattr__(self, "_materializer_fence", None) + + +def _verification_seal(value: ReportingVerifiedDestination) -> bytes: + from adcp.reporting.canonical_json import canonical_json_utf8_v1 + + content = json.loads( + json.dumps( + [ + asdict(value.request), + asdict(value.resource), + asdict(value.verification), + getattr(value, "_materializer_fence", None), + ], + default=lambda at: at.isoformat(), + ) + ) + return hmac.digest(_VERIFICATION_SEAL_KEY, canonical_json_utf8_v1(content), "sha256") + + +def validate_verified_destination( + value: ReportingVerifiedDestination, request: ReportingDestinationRequest, *, token: str +) -> None: + """Require unmodified evidence minted by this process's SDK readback. + + A public value constructor remains source compatible with B1. Constructed, + copied or restored claims cannot authorize B2 finish: restart repeats the + actual readback. The seal is never a persisted destination credential. + """ + if ( + type(value) is not ReportingVerifiedDestination + or value.request != request + or getattr(value, "_materializer_fence", None) != token + or not hmac.compare_digest(getattr(value, "_sdk_seal", b""), _verification_seal(value)) + ): + raise failure("DESTINATION_CORRUPT") def validate_materialization_target( @@ -925,9 +973,17 @@ async def _verify_destination( native_observed_through=native.verification_path if native else None, verified_format=cap.format, ) - return ReportingVerifiedDestination( + result = ReportingVerifiedDestination( prepared.request, replace(resource, manifest_sha256=manifest_digest), verification ) + # Only the service-owned heartbeat binds readback to its reservation. B1 + # standalone verification remains available without conferring B2 authority. + from adcp.reporting.materializer.service import _LeaseHeartbeat + + if type(context.heartbeat) is _LeaseHeartbeat: + object.__setattr__(result, "_materializer_fence", context.heartbeat.lease.token) + object.__setattr__(result, "_sdk_seal", _verification_seal(result)) + return result def _native( diff --git a/src/adcp/reporting/materializer/work.py b/src/adcp/reporting/materializer/work.py new file mode 100644 index 000000000..10593eb37 --- /dev/null +++ b/src/adcp/reporting/materializer/work.py @@ -0,0 +1,284 @@ +"""Optional durable work contracts, independent of the foundation store protocols.""" + +from __future__ import annotations + +import hashlib +from dataclasses import asdict, dataclass, field +from datetime import datetime +from typing import Literal, Protocol, get_args, runtime_checkable +from uuid import UUID + +from adcp.reporting.canonical_json import canonical_json_utf8_v1 +from adcp.reporting.evidence import aware_utc, reporting_identifier +from adcp.reporting.ledger._delivery_state import latest_check +from adcp.reporting.ledger.delivery_models import ( + MaterializationFailure, + ReportingDeliveryRecord, + ReportingDeliveryScope, + ReportingDestinationBinding, + ReportingMaterializationAttempt, + ReportingMaterializationRecord, + ReportingObligationDeliveryRecord, +) +from adcp.reporting.ledger.models import ( + ReportingConfiguration, + ReportingObligationRecord, + ReportingRevisionRecord, +) +from adcp.reporting.materializer.contracts import ( + ReportingDestinationRequest, + ReportingPreparedRevision, + ReportingVerificationKey, + ReportingWriterFailure, + failure, +) +from adcp.reporting.materializer.verification import ( + ReportingRevisionRowReader, + ReportingVerifiedDestination, + _same_definition, +) +from adcp.reporting.revision_selection import select_reporting_revision + +MaterializerReason = Literal[ + "ready", + "verified", + "retry", + "inactive", + "revision_not_ready", + "revision_unreadable", + "target_changed", + "history_corrupt", + "legacy_pending", + "legacy_terminal", + "component_unavailable", + "binding_changed", + "effect_unknown", + "operator_required", +] + + +def scope_id(scope: ReportingDeliveryScope) -> str: + return hashlib.sha256(canonical_json_utf8_v1(asdict(scope))).hexdigest() + + +def verification_key_id(key: ReportingVerificationKey) -> str: + return hashlib.sha256(canonical_json_utf8_v1(asdict(key))).hexdigest() + + +def key_for( + binding: ReportingDestinationBinding, + obligation: ReportingObligationRecord, + keys: tuple[ReportingVerificationKey, ...], + *, + required: str | None = None, +) -> ReportingVerificationKey: + matches = tuple( + key + for key in keys + if (required is None or verification_key_id(key) == required) + and _same_definition(key, obligation.definition) + and (key.report_definition_id, key.reporting_profile) + == (obligation.report_definition_id, obligation.reporting_profile) + and ( + key.capability.method, + key.capability.transport, + key.capability.format, + key.capability.verification_profile, + ) + == (binding.method, binding.transport, binding.format, binding.verification_profile) + ) + if len(matches) != 1: + raise failure("UNSUPPORTED_VERIFICATION") + return matches[0] + + +@dataclass(frozen=True) +class MaterializerContext: + configuration: ReportingConfiguration + obligation: ReportingObligationRecord + binding: ReportingDestinationBinding + delivery: ReportingObligationDeliveryRecord | None + revisions: tuple[ReportingRevisionRecord, ...] + records: tuple[ReportingDeliveryRecord, ...] + + @property + def scope(self) -> ReportingDeliveryScope: + return ReportingDeliveryScope( + self.obligation.generation_key, + self.binding.consumer_id, + self.obligation.reporting_obligation_id, + ) + + def selection(self, now: datetime) -> tuple[ReportingRevisionRecord | None, MaterializerReason]: + config = self.configuration + if ( + config.activated_at is None + or config.activated_at > now + or (config.deactivated_at is not None and config.deactivated_at <= now) + ): + return None, "inactive" + selected = select_reporting_revision( + self.revisions, + account_id=self.scope.principal.account_id, + reporting_obligation_id=self.scope.reporting_obligation_id, + required_finality=self.obligation.required_finality, + ) + if selected.kind == "corrupt": + return None, "history_corrupt" + if selected.kind != "selected": + return None, "revision_not_ready" + if not selected.revision.readable: + return selected.revision, "revision_unreadable" + return selected.revision, "ready" + + def attempts(self, revision_id: str) -> tuple[ReportingMaterializationAttempt, ...]: + return tuple( + sorted( + ( + r + for r in self.records + if isinstance(r, ReportingMaterializationAttempt) + and r.scope == self.scope + and r.reporting_revision_id == revision_id + ), + key=lambda r: r.attempt, + ) + ) + + def outcome( + self, attempt: ReportingMaterializationAttempt + ) -> ReportingMaterializationRecord | None: + return next( + ( + r + for r in self.records + if isinstance(r, ReportingMaterializationRecord) and r.key == attempt.key + ), + None, + ) + + def pending_attempts(self) -> tuple[ReportingMaterializationAttempt, ...]: + return tuple( + r + for r in self.records + if isinstance(r, ReportingMaterializationAttempt) + and r.scope == self.scope + and self.outcome(r) is None + ) + + def resumable(self, attempt: ReportingMaterializationAttempt) -> bool: + history = self.attempts(attempt.reporting_revision_id) + return ( + bool(history) + and history[-1] == attempt + and tuple(a.attempt for a in history) == tuple(range(1, len(history) + 1)) + and self.pending_attempts() == (attempt,) + ) + + def retained_success( + self, outcome: ReportingMaterializationRecord, now: datetime + ) -> tuple[MaterializerReason, datetime | None]: + check = latest_check(self.records, outcome.reporting_materialization_id, at=now) + if ( + outcome.resource is None + or outcome.resource.expires_at <= now + or (check is not None and check.state != "readable") + ): + return "operator_required", None + return "verified", outcome.resource.expires_at + + +@dataclass(frozen=True) +class ReportingMaterializerLease: + """A short fenced reservation. The token grants no destination authorization.""" + + scope: ReportingDeliveryScope + generation: int + token: str = field(repr=False) + expires_at: datetime + attempt: ReportingMaterializationAttempt + request: ReportingDestinationRequest + context: MaterializerContext + notifications_enabled: bool + + def __post_init__(self) -> None: + invalid = False + try: + invalid = ( + type(self.generation) is not int + or self.generation < 1 + or type(self.notifications_enabled) is not bool + or UUID(self.token).version != 4 + or self.attempt.scope != self.scope + or self.context.scope != self.scope + or self.request + != ReportingDestinationRequest.from_binding( + self.context.binding, self.attempt, self.request.verification_key + ) + ) + object.__setattr__(self, "expires_at", aware_utc(self.expires_at)) + except (ValueError, TypeError, AttributeError): + invalid = True + if invalid: + raise failure("BINDING_MISMATCH") + + +@dataclass(frozen=True) +class ReportingMaterializerTurn: + state: Literal["idle", "discovered", "parked", "pending", "verified", "failed"] + reason: MaterializerReason | None = None + reporting_materialization_id: str | None = None + + def __post_init__(self) -> None: + if self.state not in {"idle", "discovered", "parked", "pending", "verified", "failed"} or ( + self.reason is not None and self.reason not in get_args(MaterializerReason) + ): + raise ValueError("invalid materializer turn") + if self.reporting_materialization_id is not None: + reporting_identifier(self.reporting_materialization_id, maximum=255) + + +@runtime_checkable +class ReportingMaterializerStore(ReportingRevisionRowReader, Protocol): + """Opt-in service primitives. Old structural store implementations stay valid.""" + + async def claim_materialization( + self, *, keys: tuple[ReportingVerificationKey, ...], lease_seconds: int = 30 + ) -> ReportingMaterializerLease | ReportingMaterializerTurn: ... + + async def renew_materialization( + self, lease: ReportingMaterializerLease, *, lease_seconds: int + ) -> bool: ... + + async def authorize_materialization(self, lease: ReportingMaterializerLease) -> None: ... + + async def finish_materialization( + self, + lease: ReportingMaterializerLease, + *, + prepared: ReportingPreparedRevision | None = None, + verified: ReportingVerifiedDestination | None = None, + error: ReportingWriterFailure | None = None, + ) -> ReportingMaterializerTurn: ... + + +def validate_lease_seconds(value: int) -> None: + if type(value) is not int or not 3 <= value <= 300: + raise ValueError("materializer leases require 3..300 seconds") + + +def public_failure(error: ReportingWriterFailure) -> MaterializationFailure: + if error.code in {"DESTINATION_CORRUPT", "SOURCE_INVALID"}: + return "CONTENT_CORRUPT" + if error.code == "WRITE_FAILED": + return "WRITE_FAILED" + if error.code in { + "RESOURCE_UNAVAILABLE", + "CURRENT_REVISION_CHANGED", + "REVISION_NOT_READY", + "AUTHORIZATION_DENIED", + "BINDING_MISMATCH", + "HISTORY_CORRUPT", + }: + return "RESOURCE_UNAVAILABLE" + return "VERIFICATION_FAILED" diff --git a/tests/conformance/reporting/_durable_materializer_support.py b/tests/conformance/reporting/_durable_materializer_support.py new file mode 100644 index 000000000..3ea9861b5 --- /dev/null +++ b/tests/conformance/reporting/_durable_materializer_support.py @@ -0,0 +1,319 @@ +"""Shared deterministic vectors; PostgreSQL work always uses real database time.""" + +from contextlib import asynccontextmanager +from copy import deepcopy +from dataclasses import dataclass, replace +from datetime import datetime, timedelta, timezone + +from adcp.reporting.ledger import ( + ReportingDeliveryScope, + ReportingDestinationBinding, + ReportingMaterializationRecord, + ReportingRevisionRecord, + revision_content_sha256, +) +from adcp.reporting.materializer import ( + ReferenceReportingDestinationWriter, + ReferenceReportingResolver, + ReportingDestinationIO, + ReportingRevisionVerifierRegistry, + reference_digest, + reference_verifier, +) +from adcp.reporting.materializer.memory import InMemoryReportingMaterializerStore +from adcp.reporting.materializer.service import ReportingMaterializerService +from adcp.reporting.materializer.work import ReportingMaterializerLease + +from ._generation_support import END, START, configuration, isolated_reporting_pool, obligation_for +from ._materializer_support import io_context, reference_rows +from ._reliable_support import ManualClock + + +@dataclass +class DurableHarness: + store: object + clock: object + pool: object = None + + async def image(self): + """Every table/collection changed by an SDK transaction, including heads.""" + if self.pool is None: + return deepcopy( + {k: v for k, v in vars(self.store).items() if k not in {"_clock", "_lock"}} + ) + from psycopg import sql + + result = {} + async with self.pool.connection() as c: + tables = await ( + await c.execute( + "SELECT tablename FROM pg_tables WHERE schemaname=current_schema()" + " AND starts_with(tablename,'reporting_') ORDER BY tablename" + ) + ).fetchall() + for (table,) in tables: + result[table] = await ( + await c.execute( + sql.SQL("SELECT to_jsonb(t) FROM {} t ORDER BY to_jsonb(t)::text").format( + sql.Identifier(table) + ) + ) + ).fetchall() + return result + + async def expire(self): + """Persist expiry, then let the production fence observe it; no timing sleeps.""" + if self.pool is None: + self.clock.advance(timedelta(seconds=1)) + for work in self.store._materializer_work.values(): + if not work.acked: + work.lease_until = self.clock() if work.token is not None else None + work.due_at = self.clock() + for candidate in self.store._materializer_candidates.values(): + if candidate.due_at is not None: + candidate.due_at = self.clock() + return + async with self.pool.connection() as c, c.transaction(): + await self.store._lock_account(c, "acct_a") + await c.execute( + "UPDATE reporting_materializer_work SET lease_until=CASE" + " WHEN lease_token IS NOT NULL THEN clock_timestamp() END," + " due_at=clock_timestamp() WHERE state='pending'" + ) + await c.execute( + "UPDATE reporting_materializer_candidates SET due_at=clock_timestamp()" + " WHERE due_at IS NOT NULL" + ) + await c.execute("UPDATE reporting_materializer_accounts SET due_at=clock_timestamp()") + + async def queue(self): + if self.pool is None: + state = self.store._materializer_outbox + if state is None: + return (), () + return tuple(state.events.values()), tuple(w.state for w in state.expansions.values()) + async with self.pool.connection() as c: + events = await ( + await c.execute("SELECT snapshot FROM reporting_materializer_notification_events") + ).fetchall() + expansions = await ( + await c.execute("SELECT state FROM reporting_materializer_notification_expansions") + ).fetchall() + return tuple(r[0] for r in events), tuple(r[0] for r in expansions) + + async def works(self): + if self.pool is None: + return tuple( + (w.request.external_id, "acked" if w.acked else "pending", w.generation) + for w in self.store._materializer_work.values() + ) + async with self.pool.connection() as c: + return tuple( + await ( + await c.execute( + "SELECT external_id,state,generation FROM reporting_materializer_work" + " ORDER BY created_at,reporting_materialization_id" + ) + ).fetchall() + ) + + +@asynccontextmanager +async def durable_harness(backend, *, notifications=False): + clock = ManualClock(datetime.now(timezone.utc)) + if backend == "memory": + yield DurableHarness( + InMemoryReportingMaterializerStore(clock=clock, notifications=notifications), clock + ) + else: + from adcp.reporting.materializer.pg import PgReportingMaterializerStore + + async with isolated_reporting_pool(autocommit=True) as pool: + store = PgReportingMaterializerStore(pool=pool, notifications=notifications) + await store.create_schema() + yield DurableHarness(store, clock, pool) + + +@dataclass +class DurableCase: + store: object + config: object + obligation: object + binding: object + revision: object + rows: object + verifier: object + registry: object + writer: object + resolver: object + io: object + + @property + def scope(self): + return ReportingDeliveryScope( + self.config.generation_key, + self.binding.consumer_id, + self.obligation.reporting_obligation_id, + ) + + @property + def keys(self): + return (self.verifier.key,) + + def service(self, **kwargs): + return ReportingMaterializerService(self.store, self.io, self.writer, **kwargs) + + async def claim(self, **kwargs): + for _ in range(4): + result = await self.store.claim_materialization(keys=self.keys, **kwargs) + if getattr(result, "state", None) != "discovered": + return result + raise AssertionError("discovery did not converge") + + async def verified(self, lease): + from adcp.reporting.materializer.service import _LeaseHeartbeat + + assert isinstance(lease, ReportingMaterializerLease) + target = lease.context + context = io_context() + # The same SDK-owned I/O component used by the service; the test holds + # its explicit lease while injecting individual finish boundaries. + context = replace(context, heartbeat=_LeaseHeartbeat(self.store, lease, 30, context.cancel)) + prepared = await self.registry.prepare( + key=self.verifier.key, + binding=target.binding, + delivery=target.delivery, + obligation=target.obligation, + revisions=target.revisions, + attempt=lease.attempt, + reader=self.store, + context=context, + ) + locator = await self.io.write(prepared, context=context) + verified = await self.io.verify(prepared, locator, context=context) + return prepared, verified + + async def outcomes(self): + snapshot = await self.store.read_reconciliation_snapshot(caller=self.scope.principal) + return tuple(r for r in snapshot.records if isinstance(r, ReportingMaterializationRecord)) + + async def publish( + self, revision_id="revision-official", *, finality="official", supersedes=None + ): + pairs = self.revision.control_totals + revision = replace( + self.revision, + reporting_revision_id=revision_id, + finality=finality, + revision_content_sha256=revision_content_sha256( + reporting_revision_id=revision_id, + row_count=len(self.rows), + control_totals=pairs, + reporting_rows=self.rows, + control_total_evidence=self.revision.managed_control_totals, + ), + finality_basis="source_final" if finality == "official" else None, + finality_policy_id="reference-final" if finality == "official" else None, + finalized_at=END if finality == "official" else None, + supersedes_reporting_revision_id=supersedes, + ) + return await self.store.commit_revision(revision, self.rows) + + +async def durable_case( + store, + *, + count=1, + account="acct_a", + consumer="https://buyer.example.test/agent", + finality="snapshot", + required="snapshot", + active=True, + binding=True, + legacy_definition=False, + reconciliation_mode="delivery_only", +): + verifier = reference_verifier() + if legacy_definition: + verifier = replace( + verifier, + key=replace( + verifier.key, + definition=replace( + verifier.key.definition, + monetary_metric_units=(), + monetary_control_total_units=(), + ), + ), + ) + registry = ReportingRevisionVerifierRegistry((verifier,)) + config = replace( + configuration(account), + deactivated_at=None if active else END, + definition=verifier.key.definition, + report_definition_id=verifier.key.report_definition_id, + required_finality=required, + ) + await store.put_configuration(config) + obligation = await store.commit_obligation(obligation_for(config)) + cap = verifier.key.capability + destination = ReportingDestinationBinding( + config.generation_key, + consumer, + "destination", + "trusted-reference-binding", + cap.method, + cap.transport, + cap.verification_profile, + reconciliation_mode, + "analytics", + 400, + START, + cap.format, + ("reference-v1",), + "available", + ) + if binding: + await store.put_destination_binding(destination) + rows = reference_rows(count) + _, totals = verifier.canonicalize(rows) + pairs = tuple((t.name, t.value) for t in totals) + revision = ReportingRevisionRecord( + f"revision-{account}", + account, + obligation.reporting_obligation_id, + finality, + revision_content_sha256( + reporting_revision_id=f"revision-{account}", + row_count=count, + control_totals=pairs, + reporting_rows=rows, + control_total_evidence=totals, + ), + count, + pairs, + END, + END, + END, + finality_basis="source_final" if finality == "official" else None, + finality_policy_id="reference-final" if finality == "official" else None, + finalized_at=END if finality == "official" else None, + canonical_content_digest=reference_digest(verifier, rows), + managed_control_totals=totals, + ) + await store.commit_revision(revision, rows) + writer = ReferenceReportingDestinationWriter((cap,)) + resolver = ReferenceReportingResolver(writer, registry, (destination,)) + return DurableCase( + store, + config, + obligation, + destination, + revision, + rows, + verifier, + registry, + writer, + resolver, + ReportingDestinationIO(registry, resolver), + ) diff --git a/tests/conformance/reporting/_materializer_frozen.py b/tests/conformance/reporting/_materializer_frozen.py new file mode 100644 index 000000000..65b76454a --- /dev/null +++ b/tests/conformance/reporting/_materializer_frozen.py @@ -0,0 +1,295 @@ +"""Copied out of the checkout and run with -I in an installed frozen wheel.""" + +import asyncio +import hashlib +import importlib +import json +import sys +import traceback +from dataclasses import replace +from pathlib import Path + + +async def main(settings): + from psycopg_pool import AsyncConnectionPool + + import adcp.reporting.ledger as ledger_module + from adcp.reporting.ledger import PgReportingLedgerStore + + origins = {} + for name, expected in settings["modules"].items(): + module = importlib.import_module(name) + assert hashlib.sha256(Path(module.__file__).read_bytes()).hexdigest() == expected + origins[name] = str(Path(module.__file__).resolve()) + workspace = Path(settings["workspace"]).resolve() + assert not any(Path(p).resolve().is_relative_to(workspace) for p in sys.path) + for name, module in tuple(sys.modules.items()): + if name == "adcp" or name.startswith("adcp."): + if getattr(module, "__file__", None): + assert not Path(module.__file__).resolve().is_relative_to(workspace) + assert "site-packages" in module.__file__ + has_records = hasattr(ledger_module, "PgReportingReconciliationStore") + store_type = ( + ledger_module.PgReportingReconciliationStore if has_records else PgReportingLedgerStore + ) + async with AsyncConnectionPool( + settings["conninfo"], kwargs=settings["kwargs"], min_size=2, max_size=4, open=False + ) as pool: + async with pool.connection() as connection: + database = await ( + await connection.execute( + "SELECT current_setting('server_version_num')::int," + " current_setting('server_encoding'),datcollate,datctype" + " FROM pg_database WHERE datname=current_database()" + ) + ).fetchone() + assert 160000 <= database[0] < 170000 and database[1:] == ("UTF8", "C", "C") + store = store_type(pool=pool) + if settings["action"] == "install": + await store.create_schema() + if settings["artifact"] in {"c", "b1"}: + from adcp.reporting.outbox import PgStatusNotificationStore + + await PgStatusNotificationStore( + store_type(pool=pool, notifications=True) + ).create_schema() + if settings["artifact"] in {"a", "b", "c", "b1"}: + from adcp.reporting.outbox._schema import validate_schema + + async with pool.connection() as connection: + await validate_schema(connection) + return { + "artifact": settings["artifact"], + "sha": settings["sha"], + "installed": True, + "origins": origins, + "module_hashes": settings["modules"], + "database": database, + "notifications_ready": ( + True if settings["artifact"] in {"a", "b", "c", "b1"} else None + ), + } + + notifications_ready = None + if settings["artifact"] in {"a", "b", "c", "b1"}: + from adcp.reporting.ledger.notification_models import ReportingNotificationError + from adcp.reporting.outbox._schema import validate_schema + + async with pool.connection() as connection: + try: + await validate_schema(connection) + notifications_ready = True + except ReportingNotificationError: + notifications_ready = False + # Before AND after B2: exactly the reviewed C/B1 envelope. A's + # aggregate closes; B/C/B1 per-object required manifests stay ready. + assert notifications_ready == (settings["artifact"] != "a") + + configs = await store.list_configurations(account_id="acct_a") + assert len(configs) == 1 + obligation = await store.get_obligation( + account_id="acct_a", reporting_obligation_id="rpo_acct_a" + ) + assert obligation is not None + revisions = await store.list_revisions( + account_id="acct_a", reporting_obligation_id=obligation.reporting_obligation_id + ) + assert len(revisions) == 1 and revisions[0].row_count == 3 + snapshot = await store.open_snapshot(account_id="acct_a", filters_fingerprint="frozen-b2") + page = await store.read_page( + snapshot=snapshot, + consumer_id=settings["consumer"], + delivery_config_ids=None, + media_buy_ids=None, + offset=0, + limit=100, + changes_after_sequence=None, + ) + assert len(page.obligations) == len(page.revisions) == 1 + assert not page.has_more + records = 0 + if has_records: + principal = ledger_module.ReportingDeliveryPrincipal("acct_a", settings["consumer"]) + retained = await store.read_reconciliation_snapshot(caller=principal) + assert len(retained.records) == (1 if settings["action"] == "baseline" else 4) + assert [r.status for r in retained.records if r.kind == "materialization"] == [ + "available" + ] * (settings["action"] != "baseline") + records = len(retained.records) + # Permitted ordinary old writes remain functional; legacy materializer + # writers are deliberately drained, not run beside the new worker. + status = None + projector_turns = 0 + if settings["artifact"] in {"c", "b1"} and settings["action"] != "baseline": + from adcp.reporting.outbox import PgStatusNotificationStore + + store = store_type(pool=pool, notifications=True) + status = PgStatusNotificationStore(store) + await status.baseline(account_id="acct_a") + await store.put_configuration( + replace(configs[0], status_retention_days=configs[0].status_retention_days + 1) + ) + for readable in (False, True): + await store.set_revision_readable( + account_id="acct_a", + reporting_revision_id=revisions[0].reporting_revision_id, + readable=readable, + ) + if status is not None: + while (await status.project_one(account_id="acct_a")).did_work: + projector_turns += 1 + assert projector_turns < 32 + assert ( + await store.get_revision( + account_id="acct_a", reporting_revision_id=revisions[0].reporting_revision_id + ) + ).readable + workers = {} + event_identities = {} + if settings["artifact"] in {"a", "b", "c", "b1"} and settings["action"] != "baseline": + from adcp.reporting.ledger.notification_models import decode_event + from adcp.reporting.outbox import ( + PgReportingOutbox, + ReportingEnvelopeCipher, + ReportingNotificationWorker, + ) + + class EmptySubscriptions: + async def list_active(self, **kwargs): + assert kwargs["notification_type"] != "reporting.delivery_ready" + return () + + async def get_active(self, **kwargs): + raise AssertionError("empty recipient membership has no HTTP delivery") + + outboxes = {"ordinary": PgReportingOutbox(pool=pool)} + if settings["artifact"] in {"c", "b1"}: + from adcp.reporting.outbox.status_pg import PgReportingStatusOutbox + + outboxes["status"] = PgReportingStatusOutbox(pool=pool) + for name, outbox in outboxes.items(): + events = await outbox.list_events(account_id="acct_a") + if name == "ordinary": + assert len(events) == 1 + event = events[0] + assert event.account_id == "acct_a" and event.consumer_namespace == "" + assert event.notification_type == "reporting.ledger_changed" + assert event.cause.kind == "revision_published" + assert event.cause.reporting_revision_id == "revision-acct_a" + assert event.cause.finality == "snapshot" and event.cause_generation == 1 + assert event.cause.supersedes_reporting_revision_id is None + else: + # Six known Core views (configuration + obligation for the + # seller and two callers), each complete -> action -> complete + # for this closed period with snapshot-required finality. + # Materialization captures add no C reconciliation semantics. + expected = { + ( + ("acct_a", consumer, "daily", 1, kind, obligation_id), + generation, + previous, + health, + ) + for consumer in ( + "", + settings["consumer"], + "https://buyer.example.test/isolated", + ) + for kind, obligation_id in ( + ("configuration", ""), + ("obligation", "rpo_acct_a"), + ) + for generation, previous, health in ( + (1, "complete", "action_required"), + (2, "action_required", "complete"), + ) + } + assert len(events) == len(expected) == 12 + assert all( + e.notification_type == "reporting.status_changed" + and e.cause.kind == "status_changed" + and e.cause_generation == e.cause.checkpoint_generation + and len(e.cause.issue_ids) == int(e.cause.health == "action_required") + for e in events + ) + assert { + ( + e.cause.scope.checkpoint_key, + e.cause_generation, + e.cause.previous_health, + e.cause.health, + ) + for e in events + } == expected + assert len({e.causal_key for e in events}) == len(events) + identities = { + (e.account_id, e.consumer_namespace, e.notification_id): e for e in events + } + assert len(identities) == len(events) + seen = set() + + class ObservedOutbox: + def __getattr__(self, attribute): + return getattr(outbox, attribute) + + async def claim_expansion(self, **kwargs): + lease = await outbox.claim_expansion(**kwargs) + if lease is not None: + identity = ( + lease.account_id, + lease.consumer_namespace, + lease.notification_id, + ) + assert identity in identities and identity not in seen + assert decode_event(lease.event) == identities[identity] + seen.add(identity) + return lease + + worker = ReportingNotificationWorker( + outbox=ObservedOutbox(), + subscriptions=EmptySubscriptions(), + cipher=ReportingEnvelopeCipher(b"e" * 32), + ) + for _ in events: + assert await worker.expand_one(account_id="acct_a") + assert seen == identities.keys() + assert not await worker.deliver_one(account_id="acct_a") + assert not await worker.expand_one(account_id="acct_a") + assert await outbox.list_events(account_id="acct_a") == events + workers[name] = len(seen) + event_identities[name] = [ + {"notification_id": e.notification_id, "causal_key": e.causal_key} + for e in events + ] + assert workers["ordinary"] == 1 # Positive control: an actual Core event was claimed. + if status is not None: + assert projector_turns > 0 and workers["status"] > 0 + return { + "artifact": settings["artifact"], + "sha": settings["sha"], + "installed": True, + "core_records": 2, + "managed_records": records, + "ordinary_writes": True, + "workers": workers, + "event_identities": event_identities, + "projector_turns": projector_turns, + "notifications_ready": notifications_ready, + "origins": origins, + "module_hashes": settings["modules"], + "database": database, + } + + +if __name__ == "__main__": + try: + result = asyncio.run(main(json.load(sys.stdin))) + except Exception as error: + result = { + "failure": type(error).__name__, + "frames": [ + [Path(frame.filename).name, frame.lineno] + for frame in traceback.extract_tb(error.__traceback__) + ], + } + print(json.dumps(result)) diff --git a/tests/conformance/reporting/_materializer_installed.py b/tests/conformance/reporting/_materializer_installed.py index dc5f5c4f2..a7c7d62ed 100644 --- a/tests/conformance/reporting/_materializer_installed.py +++ b/tests/conformance/reporting/_materializer_installed.py @@ -6,10 +6,13 @@ import importlib.util import json import sys +from dataclasses import fields from datetime import datetime, timedelta, timezone from importlib.resources import files from pathlib import Path +from pydantic import TypeAdapter + async def main(): config = json.load(sys.stdin) @@ -39,9 +42,13 @@ async def main(): revision_content_sha256, ) from adcp.reporting.materializer import ( + InMemoryReportingMaterializerStore, ReportingDestinationBinding, + ReportingDestinationIO, ReportingMaterializationAttempt, + ReportingMaterializerService, ReportingObligationDeliveryRecord, + ReportingVerifiedDestination, reference_digest, reference_verifier, ) @@ -63,6 +70,8 @@ async def main(): files("adcp.reporting.ledger").joinpath("reporting_status_selector_version.sql").is_file() ) assert files("adcp.reporting.outbox").joinpath("required_status_selector_schema.json").is_file() + assert files("adcp.reporting.ledger").joinpath("reporting_materializer.sql").is_file() + assert files("adcp.reporting.materializer").joinpath("required_schema.json").is_file() start = datetime(2026, 9, 1, tzinfo=timezone.utc) schedule = ReportingScheduleSpec("PT1H", "PT1H", period_anchor=start) period = derive_period(schedule, account_timezone="UTC", ordinal=0) @@ -170,9 +179,36 @@ async def main(): ) ) assert all(result.verification.row_count == count for result in results) + codec = TypeAdapter(ReportingVerifiedDestination) + assert {item.name for item in fields(results[0])} == { + "request", + "resource", + "verification", + } + assert set(json.loads(codec.dump_json(results[0]))) == { + "request", + "resource", + "verification", + } + assert codec.validate_json(codec.dump_json(results[0])) == results[0] assert results[0].request.external_id == results[1].request.external_id assert destination.writer.write_effects == 1 and not destination.writer.production_eligible assert destination.writer.open_count == destination.writer.close_count == 4 + durable = InMemoryReportingMaterializerStore(notifications=False) + await durable.put_configuration(configuration) + await durable.commit_obligation(obligation) + await durable.put_destination_binding(binding) + await durable.commit_revision(revision, rows) + destination = example.development_destination(binding) + service = ReportingMaterializerService( + durable, + ReportingDestinationIO(destination.registry, destination.resolver), + destination.writer, + ) + assert (await service.run_once()).state == "verified" + boundaries = await durable.read_materializer_boundaries(caller=scope.principal) + assert len(boundaries) == boundaries[0].sequence == boundaries[0].account_sequence == 1 + assert durable._materializer_outbox is None workspace = Path(config["workspace"]).resolve() assert all(not Path(path).resolve().is_relative_to(workspace) for path in sys.path) assert all( @@ -182,7 +218,17 @@ async def main(): if getattr(module, "__file__", None) ) assert not any(name.startswith("psycopg") for name in sys.modules) - print(json.dumps({"python": "3.10", "rows": [0, 501], "installed": True, "assets": actual})) + print( + json.dumps( + { + "python": "3.10", + "rows": [0, 501], + "installed": True, + "assets": actual, + "durable": True, + } + ) + ) asyncio.run(main()) diff --git a/tests/conformance/reporting/_materializer_process.py b/tests/conformance/reporting/_materializer_process.py new file mode 100644 index 000000000..cc62fd3bf --- /dev/null +++ b/tests/conformance/reporting/_materializer_process.py @@ -0,0 +1,179 @@ +"""Real worker process and conditional file destination, solely for crash tests.""" + +import asyncio +import hashlib +import importlib +import json +import os +import sys +from pathlib import Path +from uuid import uuid4 + + +def emit(point, **values): + print(json.dumps({"point": point, **values}), flush=True) + + +async def read(): + return json.loads(await asyncio.to_thread(sys.stdin.readline)) + + +async def main(): + settings = await read() + from psycopg import AsyncConnection + from psycopg_pool import AsyncConnectionPool + from pydantic import TypeAdapter + + from adcp.reporting.ledger._delivery_state import decode_record + from adcp.reporting.materializer import ( + PgReportingMaterializerStore, + ReferenceReportingDestinationWriter, + ReferenceReportingResolver, + ReportingDestinationIO, + ReportingMaterializerService, + ReportingRevisionVerifierRegistry, + reference_verifier, + ) + from adcp.reporting.materializer.reference import _Artifact, _Session + + origins = {} + if settings.get("installed"): + installed = settings["installed"] + workspace = Path(installed["workspace"]).resolve() + for name, expected in installed["modules"].items(): + module = importlib.import_module(name) + path = Path(module.__file__).resolve() + assert hashlib.sha256(path.read_bytes()).hexdigest() == expected + origins[name] = str(path) + assert not any(Path(p).resolve().is_relative_to(workspace) for p in sys.path) + for name, module in tuple(sys.modules.items()): + if name == "adcp" or name.startswith("adcp."): + if getattr(module, "__file__", None): + assert not Path(module.__file__).resolve().is_relative_to(workspace) + assert "site-packages" in module.__file__ + assert list(sys.version_info[:2]) == installed["python"] + + if settings.get("action") == "install": + from adcp.reporting.ledger import LedgerConflictError + + async with AsyncConnectionPool( + settings["conninfo"], kwargs=settings["kwargs"], min_size=1, max_size=1, open=False + ) as pool: + store = PgReportingMaterializerStore(pool=pool, notifications=settings["notifications"]) + try: + await store.materializer_ready() + except LedgerConflictError: + pass + else: + raise AssertionError("empty schema must be unready") + await store.create_schema() + await store.create_schema() + assert await store.materializer_ready() + emit("done", state="installed", origins=origins) + return + + async def hit(point): + if point == settings.get("pause"): + emit(point) + assert (await read())["continue"] + + class Connection(AsyncConnection): + async def execute(self, query, params=None, **kwargs): + result = await super().execute(query, params, **kwargs) + if isinstance(query, str) and query.startswith( + "INSERT INTO reporting_materializer_notification_events" + ): + await hit("after_event") + return result + + class Store(PgReportingMaterializerStore): + async def claim_materialization(self, **kwargs): + result = await super().claim_materialization(**kwargs) + if hasattr(result, "attempt"): + await hit("reserved") + return result + + async def _commit_record_on(self, connection, record, **kwargs): + result = await super()._commit_record_on(connection, record, **kwargs) + if record.kind == "materialization": + await hit("after_outcome") + return result + + async def _materializer_dirty_on(self, *args): + await super()._materializer_dirty_on(*args) + await hit("after_capture") + + async def finish_materialization(self, *args, **kwargs): + if kwargs.get("verified") is not None: + await hit("after_readback") + result = await super().finish_materialization(*args, **kwargs) + if result.state == "verified": + await hit("after_commit") + return result + + verifier = reference_verifier() + registry = ReportingRevisionVerifierRegistry((verifier,)) + writer = ReferenceReportingDestinationWriter((verifier.key.capability,)) + reference = ReferenceReportingResolver(writer, registry, (decode_record(settings["binding"]),)) + artifact_codec = TypeAdapter(_Artifact) + directory = Path(settings["destination"]) + + class Session(_Session): + async def write(self, content): + await hit("before_write") + path = directory / self.request.external_id + if path.exists(): + writer._artifacts[self.request.external_id] = artifact_codec.validate_json( + path.read_bytes() + ) + locator = await super().write(content) + if not path.exists(): + temporary = directory / ("staged-" + uuid4().hex) + temporary.write_bytes( + artifact_codec.dump_json(writer._artifacts[self.request.external_id]) + ) + with temporary.open("rb") as stream: + os.fsync(stream.fileno()) + try: + os.link(temporary, path) # Conditional atomic publication, no overwrite. + except FileExistsError: + writer._artifacts[self.request.external_id] = artifact_codec.validate_json( + path.read_bytes() + ) + locator = await super().write(content) + finally: + temporary.unlink() + descriptor = os.open(directory, os.O_RDONLY) + try: + os.fsync(descriptor) + finally: + os.close(descriptor) + await hit("after_write") + return locator + + class Resolver: + def resolve(self, request, *, phase, context): + return Session(reference, request, phase, context) + + async with AsyncConnectionPool( + settings["conninfo"], + kwargs=settings["kwargs"], + connection_class=Connection, + min_size=1, + max_size=1, + open=False, + ) as pool: + store = Store(pool=pool, notifications=settings["notifications"]) + service = ReportingMaterializerService( + store, ReportingDestinationIO(registry, Resolver()), writer, lease_seconds=90 + ) + result = await service.run_once() + emit("done", state=result.state, reason=result.reason, origins=origins) + + +if __name__ == "__main__": + try: + asyncio.run(main()) + except BaseException as error: + emit("failed", classification=type(error).__name__) + sys.exit(1) diff --git a/tests/conformance/reporting/test_reporting_materializer_contracts.py b/tests/conformance/reporting/test_reporting_materializer_contracts.py index e0f56586c..370ad7ea8 100644 --- a/tests/conformance/reporting/test_reporting_materializer_contracts.py +++ b/tests/conformance/reporting/test_reporting_materializer_contracts.py @@ -226,7 +226,7 @@ async def checkpoint(self): assert calls == 1 and case.writer.open_count == 0 -def test_curated_all_exports_resolve_and_do_not_add_materializer_sql(): +def test_curated_all_exports_resolve_with_the_isolated_optional_materializer(): for name in ( "adcp.reporting.materializer", "adcp.reporting.revision_selection", @@ -238,9 +238,13 @@ def test_curated_all_exports_resolve_and_do_not_add_materializer_sql(): for public in module.__all__: assert getattr(module, public) is not None module = importlib.import_module("adcp.reporting.materializer") - assert not any( - name.endswith(("Coordinator", "Service", "Lease", "WorkQueue")) for name in module.__all__ - ) + # B1 deliberately stopped before orchestration. B2.1 adds only this optional + # service/lease surface; it does not promote the reference writer or tiers. + assert { + name + for name in module.__all__ + if name.endswith(("Coordinator", "Service", "Lease", "WorkQueue")) + } == {"ReportingMaterializerService", "ReportingMaterializerLease"} assert inspect.isclass(module.ReportingDestinationSession) assert get_type_hints(module.ReportingDestinationSession.write)["content"] is ( module.ReportingPreparedRevision diff --git a/tests/conformance/reporting/test_reporting_materializer_durable.py b/tests/conformance/reporting/test_reporting_materializer_durable.py new file mode 100644 index 000000000..899ced453 --- /dev/null +++ b/tests/conformance/reporting/test_reporting_materializer_durable.py @@ -0,0 +1,335 @@ +"""Shared durable state machine: real PostgreSQL and deterministic memory.""" + +from dataclasses import fields, replace + +import pytest +from pydantic import TypeAdapter + +from adcp.reporting.ledger import ReportingMaterializationAttempt +from adcp.reporting.materializer import ( + ReportingVerifiedDestination, + ReportingWriterError, + ReportingWriterFailure, +) +from adcp.reporting.materializer.work import ReportingMaterializerLease + +from ._durable_materializer_support import durable_case, durable_harness +from ._generation_support import configuration, obligation_for, revision_for + +pytestmark = pytest.mark.parametrize("backend", ["memory", "postgres"]) + + +async def test_core_records_do_not_create_managed_work_capture_or_readiness(backend): + async with durable_harness(backend, notifications=True) as h: + config = configuration() + obligation = obligation_for(config) + revision, rows = revision_for(obligation) + await h.store.put_configuration(config) + await h.store.commit_obligation(obligation) + await h.store.commit_revision(revision, rows) + assert (await h.store.claim_materialization(keys=())).state == "idle" + assert not await h.works() and await h.queue() == ((), ()) + + +async def test_durable_verified_success_still_cannot_advertise_unactivated_readiness(backend): + from adcp.reporting.ledger.notification_models import ReportingNotificationError + from adcp.reporting.outbox import ( + InMemoryReportingOutbox, + PgReportingOutbox, + ReportingEnvelopeCipher, + ReportingNotificationWorker, + ) + + class NoSubscriptions: + async def list_active(self, **kwargs): + raise AssertionError("capability inspection must not enumerate subscriptions") + + async def get_active(self, **kwargs): + raise AssertionError("capability inspection must not authorize an HTTP delivery") + + async with durable_harness(backend, notifications=True) as h: + case = await durable_case(h.store) + assert (await case.service().run_once()).state == "verified" + outbox = ( + PgReportingOutbox(pool=h.pool) + if h.pool is not None + else InMemoryReportingOutbox(h.store) + ) + worker = ReportingNotificationWorker( + outbox=outbox, + subscriptions=NoSubscriptions(), + cipher=ReportingEnvelopeCipher(b"e" * 32), + ) + # The new store is deliberately not in the production composition's + # approved identity set until B2.4 proves projection/mount readiness. + with pytest.raises(ReportingNotificationError, match="notification_chain_unready"): + await worker.advertised_notifications( + h.store, account_id="acct_a", ready_scope=case.scope + ) + assert len((await h.queue())[0]) == 1 and (await h.queue())[1] == ("quarantined",) + + +@pytest.mark.parametrize("notifications", [False, True]) +@pytest.mark.parametrize("count", [0, 501]) +async def test_verified_finish_captures_private_inputs_acks_and_quarantines_exact_event( + backend, notifications, count +): + async with durable_harness(backend, notifications=notifications) as h: + case = await durable_case(h.store, count=count) + lease = await case.claim() + assert lease.attempt.attempt == 1 + assert lease.context.delivery.created_at == lease.attempt.created_at + prepared, evidence = await case.verified(lease) + result = await h.store.finish_materialization(lease, prepared=prepared, verified=evidence) + assert result.state == "verified" + assert case.writer.write_effects == 1 + assert case.writer.open_count == case.writer.close_count == 2 + outcomes = await case.outcomes() + assert len(outcomes) == 1 and outcomes[0].verification.row_count == count + boundaries = await h.store.read_materializer_boundaries(caller=case.scope.principal) + assert len(boundaries) == 1 + assert boundaries[0].as_of == outcomes[0].completed_at + assert boundaries[0].core.consumer_ids == (case.binding.consumer_id,) + assert outcomes[0] in boundaries[0].reconciliation + assert ( + await h.store.read_materializer_boundaries( + caller=replace(case.scope.principal, consumer_id="another-buyer") + ) + == () + ) + events, expansions = await h.queue() + assert len(events) == int(notifications) + assert expansions == (("quarantined",) if notifications else ()) + assert (await h.works())[0][1] == "acked" + assert ( + await h.store.finish_materialization(lease, prepared=prepared, verified=evidence) + == result + ) + assert await h.queue() == (events, expansions) + assert await h.store.read_materializer_boundaries(caller=case.scope.principal) == boundaries + + +async def test_service_owns_reservation_verification_and_finish(backend): + async with durable_harness(backend) as h: + case = await durable_case(h.store) + assert (await case.service().run_once()).state == "verified" + assert (await case.service().run_once()).state in {"idle", "discovered"} + assert case.writer.write_effects == 1 + + +async def test_unknown_effect_resumes_same_attempt_external_identity_and_rejects_old_fence(backend): + async with durable_harness(backend) as h: + case = await durable_case(h.store) + first = await case.claim() + prepared, evidence = await case.verified(first) + result = await h.store.finish_materialization( + first, error=ReportingWriterFailure("WRITE_FAILED", "same_identity", "unknown") + ) + assert result.state == "pending" + assert not await case.outcomes() + await h.expire() + second = await case.claim() + assert second.attempt == first.attempt + assert second.request.external_id == first.request.external_id + assert second.token != first.token + assert ( + await h.store.finish_materialization(first, prepared=prepared, verified=evidence) + ).state == "pending" + prepared, evidence = await case.verified(second) + assert case.writer.write_effects == 1 + assert ( + await h.store.finish_materialization(second, prepared=prepared, verified=evidence) + ).state == "verified" + + +@pytest.mark.parametrize("change", ["official", "unreadable", "deactivated"]) +async def test_changed_authority_finishes_safe_failure_without_reusing_artifact(backend, change): + async with durable_harness(backend, notifications=True) as h: + case = await durable_case(h.store) + first = await case.claim() + prepared, evidence = await case.verified(first) + if change == "official": + await case.publish() + elif change == "unreadable": + await h.store.set_revision_readable( + account_id=case.config.account_id, + reporting_revision_id=case.revision.reporting_revision_id, + readable=False, + ) + else: + await h.store.put_configuration( + replace(case.config, deactivated_at=case.revision.created_at) + ) + result = await h.store.finish_materialization(first, prepared=prepared, verified=evidence) + assert result.state == "failed" + assert (await case.outcomes())[0].failure_code == "RESOURCE_UNAVAILABLE" + assert await h.queue() == ((), ()) + next_work = await case.claim() + if change == "official": + assert next_work.attempt.reporting_revision_id == "revision-official" + assert next_work.attempt.attempt == 1 + assert next_work.request.external_id != first.request.external_id + else: + assert not isinstance(next_work, ReportingMaterializerLease) + if change == "unreadable": + await h.store.set_revision_readable( + account_id=case.config.account_id, + reporting_revision_id=case.revision.reporting_revision_id, + readable=True, + ) + next_work = await case.claim() + assert ( + next_work.attempt.reporting_revision_id == first.attempt.reporting_revision_id + ) + assert next_work.attempt.attempt == 2 + + +async def test_known_service_failure_allows_next_attempt_but_pending_does_not(backend): + async with durable_harness(backend) as h: + case = await durable_case(h.store) + first = await case.claim() + assert not isinstance(await case.claim(), ReportingMaterializerLease) + await h.store.finish_materialization( + first, error=ReportingWriterFailure("WRITE_FAILED", "new_attempt", "not_started") + ) + await h.expire() + second = await case.claim() + assert second.attempt.attempt == 2 + assert second.request.external_id != first.request.external_id + + +async def test_sdk_seal_rejects_fabricated_or_modified_readback(backend): + async with durable_harness(backend) as h: + case = await durable_case(h.store) + lease = await case.claim() + prepared, evidence = await case.verified(lease) + forged = ReportingVerifiedDestination( + evidence.request, evidence.resource, evidence.verification + ) + with pytest.raises(ReportingWriterError): + await h.store.finish_materialization(lease, prepared=prepared, verified=forged) + codec = TypeAdapter(ReportingVerifiedDestination) + assert {f.name for f in fields(evidence)} == {"request", "resource", "verification"} + assert set(codec.dump_python(evidence, mode="json")) == { + "request", + "resource", + "verification", + } + restored = codec.validate_json(codec.dump_json(evidence)) + assert restored == evidence # B1's serializable observation contract survives. + with pytest.raises(ReportingWriterError): + await h.store.finish_materialization(lease, prepared=prepared, verified=restored) + object.__setattr__(evidence, "verification", replace(evidence.verification, row_count=7)) + with pytest.raises(ReportingWriterError): + await h.store.finish_materialization(lease, prepared=prepared, verified=evidence) + assert not await case.outcomes() + assert (await h.works())[0][1] == "pending" + + +async def test_prior_lease_readback_cannot_finish_a_new_fence_in_the_same_process(backend): + async with durable_harness(backend, notifications=True) as h: + case = await durable_case(h.store) + old = await case.claim() + prepared, proof = await case.verified(old) + await h.expire() + resumed = await case.claim() + assert resumed.attempt == old.attempt and resumed.token != old.token + with pytest.raises(ReportingWriterError): + await h.store.finish_materialization(resumed, prepared=prepared, verified=proof) + assert not await case.outcomes() and await h.queue() == ((), ()) + prepared, proof = await case.verified(resumed) + assert ( + await h.store.finish_materialization(resumed, prepared=prepared, verified=proof) + ).state == "verified" + assert case.writer.write_effects == 1 + + +async def test_finish_rechecks_sdk_evidence_after_acquiring_transaction_locks(backend, monkeypatch): + async with durable_harness(backend, notifications=True) as h: + case = await durable_case(h.store) + lease = await case.claim() + prepared, proof = await case.verified(lease) + before = await h.image() + cls = type(h.store) + if h.pool is None: + original = cls._held + + def held(self, reserved): + result = original(self, reserved) + object.__setattr__(proof, "verification", replace(proof.verification, row_count=7)) + return result + + monkeypatch.setattr(cls, "_held", held) + else: + original = cls._held_on + + async def held(self, connection, reserved): + result = await original(self, connection, reserved) + object.__setattr__(proof, "verification", replace(proof.verification, row_count=7)) + return result + + monkeypatch.setattr(cls, "_held_on", held) + with pytest.raises(ReportingWriterError): + await h.store.finish_materialization(lease, prepared=prepared, verified=proof) + assert await h.image() == before + assert not await case.outcomes() and await h.queue() == ((), ()) + + +async def test_legacy_pending_requires_explicit_exact_external_history_import(backend): + async with durable_harness(backend) as h: + case = await durable_case(h.store) + # The public API remains independent; this predates any service reservation. + from datetime import timedelta + + from adcp.reporting.ledger import ReportingObligationDeliveryRecord + from adcp.reporting.materializer import ReportingDestinationRequest + + await h.store.bind_obligation_delivery( + ReportingObligationDeliveryRecord( + case.scope, + "USD", + case.revision.created_at + timedelta(days=400), + case.revision.created_at, + ) + ) + attempt = ReportingMaterializationAttempt( + case.scope, case.revision.reporting_revision_id, "legacy", 1, case.revision.created_at + ) + await h.store.commit_materialization_attempt(attempt) + result = await case.claim() + assert result.reason == "legacy_pending" + assert not await h.works() + with pytest.raises(ReportingWriterError): + await h.store.import_pending_materialization( + scope=case.scope, + reporting_materialization_id="legacy", + original_external_id="unknown", + keys=case.keys, + ) + identity = ReportingDestinationRequest.from_binding( + case.binding, attempt, case.verifier.key + ).external_id + await h.store.import_pending_materialization( + scope=case.scope, + reporting_materialization_id="legacy", + original_external_id=identity, + keys=case.keys, + ) + lease = await case.claim() + assert lease.attempt == attempt and lease.request.external_id == identity + + +@pytest.mark.parametrize( + "required,finality,expected", + [ + ("snapshot", "snapshot", "verified"), + ("official", "snapshot", "parked"), + ("official", "official", "verified"), + ], +) +async def test_finality_and_late_delivery_binding(backend, required, finality, expected): + async with durable_harness(backend) as h: + case = await durable_case(h.store, required=required, finality=finality, binding=False) + assert (await case.service().run_once()).state == "idle" + await h.store.put_destination_binding(case.binding) + assert (await case.service().run_once()).state == expected diff --git a/tests/conformance/reporting/test_reporting_materializer_history.py b/tests/conformance/reporting/test_reporting_materializer_history.py new file mode 100644 index 000000000..a6d1b08b8 --- /dev/null +++ b/tests/conformance/reporting/test_reporting_materializer_history.py @@ -0,0 +1,272 @@ +"""History, late binding, private captured ordering and selector corruption.""" + +from dataclasses import replace +from datetime import timedelta + +import pytest + +from adcp.reporting.ledger import ( + ReportingMaterializationAttempt, + ReportingMaterializationRecord, + ReportingObligationDeliveryRecord, + ReportingRevisionReceiptRecord, +) +from adcp.reporting.materializer import ReportingDestinationRequest, ReportingWriterError + +from ._durable_materializer_support import durable_case, durable_harness + +pytestmark = pytest.mark.parametrize("backend", ["memory", "postgres"]) + + +async def test_exact_source_replay_does_not_dirty_an_inflight_generation(backend): + async with durable_harness(backend, notifications=True) as h: + case = await durable_case(h.store) + lease = await case.claim() + await h.store.put_configuration(case.config) + await h.store.commit_revision(case.revision, case.rows) + await h.store.set_revision_readable( + account_id="acct_a", + reporting_revision_id=case.revision.reporting_revision_id, + readable=True, + ) + prepared, evidence = await case.verified(lease) + assert ( + await h.store.finish_materialization(lease, prepared=prepared, verified=evidence) + ).state == "verified" + + +async def test_selected_official_never_uses_snapshot_artifact_and_preserves_both_histories(backend): + async with durable_harness(backend, notifications=True) as h: + case = await durable_case(h.store) + first = await case.claim() + prepared, evidence = await case.verified(first) + await h.store.finish_materialization(first, prepared=prepared, verified=evidence) + original = await case.outcomes() + official = await case.publish() + await h.store.set_revision_readable( + account_id="acct_a", + reporting_revision_id=official.reporting_revision_id, + readable=False, + ) + assert (await case.claim()).reason == "revision_unreadable" + assert await case.outcomes() == original + await h.store.set_revision_readable( + account_id="acct_a", reporting_revision_id=official.reporting_revision_id, readable=True + ) + second = await case.claim() + assert first.attempt.attempt == second.attempt.attempt == 1 + assert first.attempt.reporting_revision_id != second.attempt.reporting_revision_id + assert first.request.external_id != second.request.external_id + prepared, evidence = await case.verified(second) + await h.store.finish_materialization(second, prepared=prepared, verified=evidence) + assert len(await case.outcomes()) == 2 + assert (await case.outcomes())[0] == original[0] + assert case.writer.write_effects == 2 + + +@pytest.mark.parametrize("corruption", ["fork", "cycle", "multiple_official"]) +async def test_corrupt_topology_parks_without_attempt_or_hot_loop(backend, corruption, monkeypatch): + async with durable_harness(backend, notifications=True) as h: + case = await durable_case(h.store) + + def corrupt(context): + first = case.revision + second = replace( + first, + reporting_revision_id="second", + supersedes_reporting_revision_id=first.reporting_revision_id, + ) + if corruption == "fork": + revisions = (first, second, replace(second, reporting_revision_id="third")) + elif corruption == "cycle": + revisions = (replace(first, supersedes_reporting_revision_id="second"), second) + else: + official = replace( + first, + finality="official", + finality_basis="source_final", + finality_policy_id="policy", + finalized_at=first.created_at, + ) + revisions = (official, replace(official, reporting_revision_id="second")) + return replace(context, revisions=revisions) + + cls = type(h.store) + if backend == "memory": + original = cls._context + + def read(self, scope): + return corrupt(original(self, scope)) + + method = "_context" + else: + original = cls._materializer_context_on + + async def read(self, connection, scope): + return corrupt(await original(self, connection, scope)) + + method = "_materializer_context_on" + with monkeypatch.context() as patch: + patch.setattr(cls, method, read) + assert (await case.claim()).reason == "history_corrupt" + assert (await case.claim()).state == "idle" + assert not await h.works() and await h.queue() == ((), ()) + + +async def test_inactive_late_binding_waits_for_activation_without_allocating_history(backend): + async with durable_harness(backend) as h: + case = await durable_case(h.store, active=False, binding=False) + await h.store.put_destination_binding(case.binding) + assert (await case.claim()).reason == "inactive" + assert not await h.works() + await h.store.put_configuration(replace(case.config, deactivated_at=None)) + assert (await case.claim()).attempt.attempt == 1 + + +async def test_private_consumer_boundaries_keep_a_durable_account_order(backend): + from adcp.reporting.ledger.notification_models import ReportingNotificationError + + async with durable_harness(backend, notifications=True) as h: + first = await durable_case(h.store) + second = await durable_case(h.store, consumer="https://buyer.example.test/second") + for case in (first, second): + lease = await case.claim() + # Account-fair discovery chooses the canonical consumer order. + target = first if lease.scope == first.scope else second + prepared, evidence = await target.verified(lease) + await h.store.finish_materialization(lease, prepared=prepared, verified=evidence) + boundaries = [ + (await h.store.read_materializer_boundaries(caller=case.scope.principal))[0] + for case in (first, second) + ] + assert {b.sequence for b in boundaries} == {1} + assert {b.account_sequence for b in boundaries} == {1, 2} + for boundary in boundaries: + assert boundary.core.consumer_ids == (boundary.caller.consumer_id,) + assert all( + getattr(r, "consumer_id", getattr(getattr(r, "scope", None), "consumer_id", None)) + == boundary.caller.consumer_id + for r in boundary.reconciliation + ) + with pytest.raises(ReportingNotificationError): + replace(boundary, core=replace(boundary.core, consumer_ids=("another-consumer",))) + with pytest.raises(ReportingNotificationError): + replace(boundary, reporting_materialization_id="missing-outcome") + with pytest.raises(ReportingNotificationError): + replace(boundary, reconciliation=()) + assert len((await h.queue())[0]) == 2 + + +async def test_rejected_consumer_receipt_does_not_dirty_materializer_or_allocate_retry(backend): + async with durable_harness(backend, notifications=True) as h: + case = await durable_case(h.store, reconciliation_mode="consumer_receipt") + assert (await case.service().run_once()).state == "verified" + outcome = (await case.outcomes())[0] + before = await h.works() + await h.store.record_revision_receipt( + ReportingRevisionReceiptRecord( + case.scope, + "receipt-rejected-0001", + case.revision.reporting_revision_id, + outcome.reporting_materialization_id, + "rejected", + case.binding.verification_profile, + 0, + (), + outcome.completed_at, + rejection_codes=("ROW_COUNT_MISMATCH",), + ) + ) + assert (await case.claim()).state in {"idle", "discovered"} + assert await h.works() == before + + +async def test_tampered_lease_cannot_cross_principal_or_generation(backend): + async with durable_harness(backend) as h: + case = await durable_case(h.store) + lease = await case.claim() + for changes in ( + {"scope": replace(lease.scope, consumer_id="another-consumer")}, + {"attempt": replace(lease.attempt, attempt=2)}, + {"token": "invalid"}, + ): + with pytest.raises(ReportingWriterError): + replace(lease, **changes) + assert not await case.outcomes() + + +async def test_publication_does_not_bypass_legacy_pending_on_an_older_revision(backend): + async with durable_harness(backend, notifications=True) as h: + case = await durable_case(h.store) + await h.store.bind_obligation_delivery( + ReportingObligationDeliveryRecord( + case.scope, + "USD", + case.revision.created_at + timedelta(days=400), + case.revision.created_at, + ) + ) + legacy = ReportingMaterializationAttempt( + case.scope, case.revision.reporting_revision_id, "legacy", 1, case.revision.created_at + ) + await h.store.commit_materialization_attempt(legacy) + official = await case.publish() + assert (await case.claim()).reason == "legacy_pending" + assert not await h.works() + identity = ReportingDestinationRequest.from_binding( + case.binding, legacy, case.verifier.key + ).external_id + await h.store.import_pending_materialization( + scope=case.scope, + reporting_materialization_id="legacy", + original_external_id=identity, + keys=case.keys, + ) + # Exact recovery concludes the obsolete attempt safely before the + # current official gets its own first attempt; no old artifact is reused. + assert (await case.service().run_once()).state == "failed" + assert case.writer.write_effects == 0 + current = await case.claim() + assert current.attempt.reporting_revision_id == official.reporting_revision_id + assert current.attempt.attempt == 1 + assert len(await h.works()) == 2 and await h.queue() == ((), ()) + + +async def test_external_terminal_write_parks_owned_pending_history_without_a_hot_loop(backend): + async with durable_harness(backend, notifications=True) as h: + case = await durable_case(h.store) + lease = await case.claim() + await h.store.commit_materialization( + ReportingMaterializationRecord( + case.scope, + lease.attempt.reporting_revision_id, + lease.attempt.reporting_materialization_id, + "failed", + lease.attempt.created_at, + failure_code="WRITE_FAILED", + ) + ) + await h.expire() + assert (await case.claim()).reason == "history_corrupt" + assert (await case.claim()).state == "idle" + assert len(await h.works()) == 1 + assert not await h.store.read_materializer_boundaries(caller=case.scope.principal) + assert await h.queue() == ((), ()) + + +async def test_restored_exact_component_can_explicitly_resume_parked_identity(backend): + async with durable_harness(backend, notifications=True) as h: + case = await durable_case(h.store) + lease = await case.claim() + await h.expire() + assert (await h.store.claim_materialization(keys=())).reason == "component_unavailable" + assert (await case.claim()).state == "idle" + await h.store.import_pending_materialization( + scope=case.scope, + reporting_materialization_id=lease.attempt.reporting_materialization_id, + original_external_id=lease.request.external_id, + keys=case.keys, + ) + resumed = await case.claim() + assert resumed.attempt == lease.attempt and resumed.request == lease.request + assert resumed.token != lease.token diff --git a/tests/conformance/reporting/test_reporting_materializer_installed_pg.py b/tests/conformance/reporting/test_reporting_materializer_installed_pg.py new file mode 100644 index 000000000..4a4c12211 --- /dev/null +++ b/tests/conformance/reporting/test_reporting_materializer_installed_pg.py @@ -0,0 +1,114 @@ +"""Non-editable VCS/sdist SQL installation and killed/resumed PostgreSQL workers.""" + +import asyncio +import hashlib +import json +import os +import shutil +import sys +from pathlib import Path + +import pytest + +from adcp.reporting.materializer import PgReportingMaterializerStore + +from ._durable_materializer_support import DurableHarness, durable_case +from ._generation_support import assert_c_collated_rolling_database, isolated_reporting_pool +from .test_reporting_materializer_packaging import ROOT, b1_wheels, built_distribution, run_step +from .test_reporting_materializer_process import worker + +# Expose the exact existing fixtures without rebuilding a current module as an +# alleged historical artifact. These are the new head's two distribution paths. +__all__ = ["b1_wheels", "built_distribution"] + + +@pytest.fixture(scope="module", params=["vcs", "sdist"]) +def installed_materializer(request): + assert_c_collated_rolling_database() + b1_wheels = request.getfixturevalue("b1_wheels") + path, wheels, _ = b1_wheels + interpreter = os.environ.get("ADCP_PYTHON310") or sys.executable + environment = path / f"materializer-pg-{request.param}" + run_step( + [interpreter, "-m", "venv", str(environment)], + label=f"materializer-{request.param}-pg-environment", + cwd=path, + ) + python = environment / "bin/python" + installer = ( + [shutil.which("uv"), "pip", "install", "--python", str(python)] + if shutil.which("uv") + else [str(python), "-m", "pip", "install"] + ) + run_step( + [*installer, f"{wheels[request.param]}[pg]"], + label=f"materializer-{request.param}-pg-install", + cwd=path, + timeout=180, + ) + script = path / f"materializer_pg_{request.param}.py" + shutil.copy2(Path(__file__).with_name("_materializer_process.py"), script) + version = json.loads( + run_step( + [str(python), "-I", "-c", "import json,sys;print(json.dumps(sys.version_info[:2]))"], + label="materializer-installed-python-version", + cwd=path, + ) + ) + if os.environ.get("ADCP_PYTHON310"): + assert version == [3, 10] + modules = {} + for name in ("materializer/pg.py", "materializer/service.py", "materializer/verification.py"): + modules["adcp.reporting." + name.removesuffix(".py").replace("/", ".")] = hashlib.sha256( + (ROOT / "src/adcp/reporting" / name).read_bytes() + ).hexdigest() + return path, python, script, {"workspace": str(ROOT), "python": version, "modules": modules} + + +@pytest.mark.parametrize("notifications", [False, True]) +async def test_installed_sql_and_process_restart_preserve_original_effect( + installed_materializer, notifications, tmp_path +): + path, python, script, installed = installed_materializer + async with isolated_reporting_pool(autocommit=True) as pool: + result = json.loads( + await asyncio.to_thread( + run_step, + [str(python), "-I", str(script)], + label="materializer-installed-sql-readiness", + cwd=path, + value={ + "conninfo": pool.conninfo, + "kwargs": pool.kwargs, + "notifications": notifications, + "action": "install", + "installed": installed, + }, + timeout=60, + ) + ) + assert result["point"] == "done" and result["state"] == "installed" + h = DurableHarness( + PgReportingMaterializerStore(pool=pool, notifications=notifications), None, pool + ) + case = await durable_case(h.store, count=501) + options = dict( + python=python, script=script, installed=installed, notifications=notifications + ) + async with worker(h, case, tmp_path, pause="after_write", **options) as child: + await child.event("after_write") + before = await h.works() + await child.kill() + assert not await case.outcomes() + assert await h.queue() == ((), ()) + await h.expire() + async with worker(h, case, tmp_path, **options) as child: + done = await child.event("done") + assert done["state"] == "verified" and done["origins"] == result["origins"] + assert await asyncio.wait_for(child.process.wait(), 5) == 0 + after = await h.works() + assert len(after) == 1 and after[0][0] == before[0][0] and after[0][1] == "acked" + assert len(tuple(tmp_path.glob("rwm_*"))) == 1 + assert len(await h.store.read_materializer_boundaries(caller=case.scope.principal)) == 1 + assert len((await h.queue())[0]) == int(notifications) + print(json.dumps({"installed": installed, "origins": done["origins"]}), flush=True) diff --git a/tests/conformance/reporting/test_reporting_materializer_migration.py b/tests/conformance/reporting/test_reporting_materializer_migration.py new file mode 100644 index 000000000..0c96910ce --- /dev/null +++ b/tests/conformance/reporting/test_reporting_materializer_migration.py @@ -0,0 +1,274 @@ +"""Isolated manifests, atomic migration, bounded discovery and database fences.""" + +import asyncio +import json +from dataclasses import replace +from importlib.resources import files + +import pytest + +from adcp.reporting.ledger import LedgerConflictError, PgReportingReconciliationStore, derive_period +from adcp.reporting.materializer import PgReportingMaterializerStore +from adcp.reporting.outbox._schema import REQUIRED_OBJECTS, schema_objects, validate_schema + +from ._durable_materializer_support import durable_case, durable_harness +from ._generation_support import isolated_reporting_pool, obligation_for + +SQL = files("adcp.reporting.ledger").joinpath("reporting_materializer.sql").read_text() +MANIFEST = json.loads( + files("adcp.reporting.materializer").joinpath("required_schema.json").read_text() +) + + +@pytest.mark.parametrize("autocommit", [False, True]) +async def test_populated_repeated_and_concurrent_install_preserves_all_old_objects_and_rows( + autocommit, +): + async with isolated_reporting_pool(autocommit=autocommit) as pool: + old = PgReportingReconciliationStore(pool=pool) + await old.create_schema() + case = await durable_case(old) + async with pool.connection() as c: + original = await schema_objects(c) + physical = await ( + await c.execute( + "SELECT tableoid::regclass::text,ctid::text,xmin::text,to_jsonb(r)" + " FROM reporting_reconciliation_records r ORDER BY record_id" + ) + ).fetchall() + new = PgReportingMaterializerStore(pool=pool) + with pytest.raises(LedgerConflictError, match="materializer schema"): + await new.materializer_ready() + await asyncio.gather(*(new.create_schema() for _ in range(3))) + for _ in range(2): + await new.create_schema() + assert await new.materializer_ready() + async with pool.connection() as c: + actual = await schema_objects(c) + assert {key: actual[key] for key in original} == original == REQUIRED_OBJECTS + assert { + key: value for key, value in actual.items() if "reporting_materializer_" in key + } == MANIFEST + assert ( + await ( + await c.execute( + "SELECT tableoid::regclass::text,ctid::text,xmin::text,to_jsonb(r)" + " FROM reporting_reconciliation_records r ORDER BY record_id" + ) + ).fetchall() + == physical + ) + await validate_schema(c, activity=True) + case.store = new + assert (await case.service().run_once()).state == "verified" + # Discovery is persisted, and reinstall cannot invalidate leased generations. + before = await new.read_reconciliation_snapshot(caller=case.scope.principal) + await new.create_schema() + assert ( + await new.read_reconciliation_snapshot(caller=case.scope.principal) + ).records == before.records + + +async def test_interrupted_migration_is_invisible_and_restart_converges(): + async with isolated_reporting_pool(autocommit=True) as pool: + await PgReportingReconciliationStore(pool=pool).create_schema() + entered, release = asyncio.Event(), asyncio.Event() + + async def install(): + async with pool.connection() as c, c.transaction(): + await c.execute(SQL) + entered.set() + await release.wait() + + task = asyncio.create_task(install()) + await asyncio.wait_for(entered.wait(), 10) + async with pool.connection() as c: + assert ( + await ( + await c.execute("SELECT to_regclass('reporting_materializer_work')") + ).fetchone() + )[0] is None + task.cancel() + with pytest.raises(asyncio.CancelledError): + await task + store = PgReportingMaterializerStore(pool=pool) + with pytest.raises(LedgerConflictError): + await store.materializer_ready() + await store.create_schema() + assert await store.materializer_ready() + + +@pytest.mark.parametrize( + "damage", + [ + "DROP INDEX reporting_materializer_work_due", + "ALTER TABLE reporting_materializer_work DISABLE TRIGGER reporting_materializer_guard", + "ALTER TABLE reporting_materializer_work ALTER COLUMN generation DROP NOT NULL", + "ALTER TABLE reporting_materializer_notification_expansions" + " ALTER COLUMN state SET DEFAULT 'pending'", + "DROP TABLE reporting_materializer_status_boundaries CASCADE", + "ALTER TABLE reporting_reconciliation_records" + " DISABLE TRIGGER reporting_reconciliation_guard", + ], +) +@pytest.mark.parametrize("notifications", [False, True]) +async def test_partial_mismatched_and_disabled_guard_schemas_fail_closed(damage, notifications): + async with durable_harness("postgres", notifications=notifications) as h: + case = await durable_case(h.store) + async with h.pool.connection() as c: + await c.execute(damage) + with pytest.raises(LedgerConflictError): + await case.claim() + assert not await h.works() + + +async def test_backfill_is_bounded_indexed_restartable_and_fair_between_accounts(): + async with isolated_reporting_pool(autocommit=True) as pool: + old = PgReportingReconciliationStore(pool=pool) + await old.create_schema() + cases = [await durable_case(old, account=name) for name in ("acct_a", "acct_b")] + first = cases[0] + for ordinal in range(1, 70): + period = derive_period(first.config.schedule, account_timezone="UTC", ordinal=ordinal) + await old.commit_obligation( + replace( + obligation_for(first.config), + reporting_obligation_id=f"pending-{ordinal:03d}", + period=period, + scope_resolved_at=period.end, + automated_recovery_deadline_at=period.expected_at + + first.config.automated_recovery_window, + ) + ) + store = PgReportingMaterializerStore(pool=pool) + await store.create_schema() + claimed = [] + for _ in range(4): + result = await store.claim_materialization(keys=first.keys) + if hasattr(result, "attempt"): + claimed.append(result.scope.principal.account_id) + assert "acct_b" in claimed # A's large backlog cannot starve B. + for _ in range(80): + store = PgReportingMaterializerStore(pool=pool) # Restart each turn. + result = await store.claim_materialization(keys=first.keys) + if getattr(result, "state", None) == "idle": + break + async with pool.connection() as c: + assert ( + await ( + await c.execute("SELECT count(*) FROM reporting_materializer_candidates") + ).fetchone() + )[0] == 71 + assert ( + await ( + await c.execute( + "SELECT bool_and(complete) FROM reporting_materializer_discovery" + ) + ).fetchone() + )[0] + await c.execute("SET enable_seqscan=off") + (now,) = await (await c.execute("SELECT clock_timestamp()")).fetchone() + plans = [] + for query, params in ( + ( + "SELECT account_id,consumer_id,delivery_config_id,delivery_config_version" + " FROM reporting_reconciliation_records" + " WHERE record_kind='destination_binding'", + (), + ), + ( + "SELECT account_id FROM reporting_materializer_accounts WHERE due_at<=%s" + " ORDER BY served_at,account_id LIMIT 16", + (now,), + ), + ( + "SELECT reporting_obligation_id FROM reporting_obligations WHERE account_id=%s" + " AND delivery_config_id=%s AND delivery_config_version=%s" + " AND reporting_obligation_id>%s ORDER BY reporting_obligation_id LIMIT 32", + ("acct_a", "daily", 1, ""), + ), + ( + "SELECT reporting_materialization_id FROM reporting_materializer_work" + " WHERE account_id=%s AND state='pending' AND due_at<=%s" + " ORDER BY due_at LIMIT 1", + ("acct_a", now), + ), + ): + plan = await (await c.execute("EXPLAIN (FORMAT JSON) " + query, params)).fetchone() + plans.append(json.dumps(plan)) + assert all("Seq Scan" not in plan and "Index" in plan for plan in plans) + + +async def test_enabled_work_cannot_be_resumed_by_notifications_disabled_store(): + async with durable_harness("postgres", notifications=True) as h: + case = await durable_case(h.store) + lease = await case.claim() + await h.expire() + disabled = PgReportingMaterializerStore(pool=h.pool, notifications=False) + result = await disabled.claim_materialization(keys=case.keys) + assert result.reason == "component_unavailable" + assert (await h.works())[0][0] == lease.request.external_id + assert (await h.works())[0][1] == "pending" + + +async def test_bounded_sampling_walks_past_a_full_page_of_busy_account_locks(): + async with durable_harness("postgres") as h: + busy = [f"busy-{i:02d}" for i in range(16)] + for account in busy: + await durable_case(h.store, account=account) + available = await durable_case(h.store, account="zz-available") + async with h.pool.connection() as connection, connection.transaction(): + for account in busy: + await h.store._lock_account(connection, account) + assert (await available.claim()).state == "idle" + selected = await available.claim() + assert selected.scope.principal.account_id == "zz-available" + assert len(await h.works()) == 1 + + +async def test_pre_activation_event_never_promotes_and_old_outbox_cannot_claim_it(): + from adcp.reporting.ledger.notification_models import decode_event + from adcp.reporting.outbox import PgReportingOutbox + + async with durable_harness("postgres", notifications=True) as h: + case = await durable_case(h.store) + assert (await case.service().run_once()).state == "verified" + before = await h.queue() + # Remove only the ordinary Core event's pending expansion via its own + # worker protocol. Materializer records remain in their isolated queue. + outbox = PgReportingOutbox(pool=h.pool) + from datetime import datetime, timezone + + while lease := await outbox.claim_expansion( + account_id="acct_a", now=datetime.now(timezone.utc), lease_seconds=30 + ): + assert decode_event(lease.event).notification_type != "reporting.delivery_ready" + await outbox.finish_expansion(lease, now=datetime.now(timezone.utc), state="suppressed") + from psycopg import errors + + for mutation in ( + "UPDATE reporting_materializer_notification_events SET admission_epoch=1", + "UPDATE reporting_materializer_notification_expansions SET state='pending'", + "UPDATE reporting_materializer_work SET admission_epoch=1", + ): + with pytest.raises(errors.CheckViolation): + async with h.pool.connection() as c, c.transaction(): + await c.execute(mutation) + await case.publish() + await h.store.put_configuration( + replace(case.config, deactivated_at=case.revision.created_at) + ) + assert await h.queue() == before + + +async def test_schema_loss_after_reservation_fails_authorization_before_external_write(): + async with durable_harness("postgres", notifications=True) as h: + case = await durable_case(h.store) + lease = await case.claim() + async with h.pool.connection() as c: + await c.execute("DROP INDEX reporting_materializer_work_due") + with pytest.raises(LedgerConflictError): + await h.store.authorize_materialization(lease) + assert case.writer.write_effects == 0 + assert not await case.outcomes() + assert len(await h.works()) == 1 and (await h.works())[0][1] == "pending" diff --git a/tests/conformance/reporting/test_reporting_materializer_packaging.py b/tests/conformance/reporting/test_reporting_materializer_packaging.py index 007a279bf..36187037b 100644 --- a/tests/conformance/reporting/test_reporting_materializer_packaging.py +++ b/tests/conformance/reporting/test_reporting_materializer_packaging.py @@ -45,15 +45,14 @@ def b1_wheels(built_distribution): for relative in ( "ledger/reporting_status_selector_version.sql", "outbox/required_status_selector_schema.json", + "ledger/reporting_materializer.sql", + "materializer/required_schema.json", ): assert ( vcs.read(f"adcp/reporting/{relative}") == wheel.read(f"adcp/reporting/{relative}") == (ROOT / "src/adcp/reporting" / relative).read_bytes() ) - assert not any( - "reporting_materializer_" in name and name.endswith(".sql") for name in vcs.namelist() - ) return ( path, {"vcs": vcs_wheel, "sdist": sdist_wheel}, @@ -95,6 +94,10 @@ def test_python310_installed_wheel_exports_verifier_reference_and_strict_adopter shutil.copy2(Path(__file__).with_name("_materializer_installed.py"), smoke) shutil.copy2(ROOT / "examples/reporting_destination_writer.py", example) shutil.copy2(ROOT / "tests/type_checks/reporting_destination_writer.py", adopter) + durable_example = path / f"durable_example_{kind}.py" + durable_adopter = path / f"durable_adopter_{kind}.py" + shutil.copy2(ROOT / "examples/reporting_durable_materializer.py", durable_example) + shutil.copy2(ROOT / "tests/type_checks/reporting_durable_materializer.py", durable_adopter) result = json.loads( run_step( [str(python), "-I", str(smoke)], @@ -104,7 +107,13 @@ def test_python310_installed_wheel_exports_verifier_reference_and_strict_adopter timeout=120, ) ) - assert result == {"python": "3.10", "rows": [0, 501], "installed": True, "assets": hashes} + assert result == { + "python": "3.10", + "rows": [0, 501], + "installed": True, + "assets": hashes, + "durable": True, + } config = path / "mypy.ini" config.write_text( "[mypy]\npython_version = 3.10\nstrict = True\n" @@ -122,6 +131,8 @@ def test_python310_installed_wheel_exports_verifier_reference_and_strict_adopter "--no-incremental", str(adopter), str(example), + str(durable_adopter), + str(durable_example), ], label=f"b1-{kind}-installed-adopter-types", cwd=path, diff --git a/tests/conformance/reporting/test_reporting_materializer_process.py b/tests/conformance/reporting/test_reporting_materializer_process.py new file mode 100644 index 000000000..80ea527b8 --- /dev/null +++ b/tests/conformance/reporting/test_reporting_materializer_process.py @@ -0,0 +1,151 @@ +"""SIGKILL across external-effect and outcome commit boundaries; durable restart.""" + +import asyncio +import json +import sys +from contextlib import asynccontextmanager +from pathlib import Path + +import pytest + +from adcp.reporting.ledger._delivery_state import payload + +from ._durable_materializer_support import durable_case, durable_harness + + +class Child: + def __init__(self, process): + self.process = process + + async def event(self, point): + line = await asyncio.wait_for(self.process.stdout.readline(), 30) + assert line, "materializer worker exited before its boundary" + result = json.loads(line) + assert result["point"] == point, result + return result + + async def send(self, value): + self.process.stdin.write(json.dumps(value).encode() + b"\n") + await self.process.stdin.drain() + + async def kill(self): + if self.process.returncode is None: + self.process.kill() + await asyncio.wait_for(self.process.wait(), 5) + + +async def visible(h): + """MVCC observation while the child deliberately holds the account lock.""" + async with h.pool.connection() as connection: + return await ( + await connection.execute( + "SELECT (SELECT count(*) FROM reporting_reconciliation_records" + " WHERE record_kind='materialization')," + " (SELECT count(*) FROM reporting_materializer_status_boundaries)" + ) + ).fetchone() + + +@asynccontextmanager +async def worker( + h, case, directory, *, pause=None, notifications=True, installed=None, python=None, script=None +): + process = await asyncio.create_subprocess_exec( + str(python or sys.executable), + *(["-I"] if installed else []), + str(script or Path(__file__).with_name("_materializer_process.py")), + stdin=asyncio.subprocess.PIPE, + stdout=asyncio.subprocess.PIPE, + stderr=asyncio.subprocess.DEVNULL, + ) + child = Child(process) + try: + await child.send( + { + "conninfo": h.pool.conninfo, + "kwargs": h.pool.kwargs, + "binding": payload(case.binding), + "destination": str(directory), + "pause": pause, + "notifications": notifications, + "installed": installed, + } + ) + yield child + finally: + await child.kill() + + +@pytest.mark.parametrize("notifications", [False, True]) +@pytest.mark.parametrize( + "boundary", + [ + "reserved", + "before_write", + "after_write", + "after_readback", + "after_outcome", + "after_capture", + "after_commit", + ], +) +async def test_real_process_crash_resume_preserves_external_identity_and_atomic_finish( + boundary, notifications, tmp_path +): + async with durable_harness("postgres", notifications=notifications) as h: + case = await durable_case(h.store, count=501) + async with worker(h, case, tmp_path, pause=boundary, notifications=notifications) as child: + await child.event(boundary) + work_before = await h.works() + assert len(work_before) == 1 + committed = boundary == "after_commit" + assert await visible(h) == (int(committed), int(committed)) + assert len((await h.queue())[0]) == int(committed and notifications) + await child.kill() + await h.expire() + async with worker(h, case, tmp_path, notifications=notifications) as child: + result = await child.event("done") + assert result["state"] in ( + {"idle", "parked", "discovered"} if committed else {"verified"} + ) + assert await asyncio.wait_for(child.process.wait(), 5) == 0 + after = await h.works() + assert len(after) == 1 and after[0][0] == work_before[0][0] and after[0][1] == "acked" + assert len(tuple(tmp_path.glob("rwm_*"))) == 1 + assert len(await case.outcomes()) == 1 + assert len(await h.store.read_materializer_boundaries(caller=case.scope.principal)) == 1 + events, expansions = await h.queue() + assert len(events) == int(notifications) + assert expansions == (("quarantined",) if notifications else ()) + + +async def test_crash_after_actual_enabled_logical_enqueue_rolls_back_before_restart(tmp_path): + async with durable_harness("postgres", notifications=True) as h: + case = await durable_case(h.store) + async with worker(h, case, tmp_path, pause="after_event") as child: + await child.event("after_event") + assert await visible(h) == (0, 0) + assert await h.queue() == ((), ()) + await child.kill() + assert not await h.store.read_materializer_boundaries(caller=case.scope.principal) + await h.expire() + async with worker(h, case, tmp_path) as child: + assert (await child.event("done"))["state"] == "verified" + assert await asyncio.wait_for(child.process.wait(), 5) == 0 + assert len(tuple(tmp_path.glob("rwm_*"))) == 1 + assert len((await h.queue())[0]) == 1 + + +async def test_two_real_workers_reserve_once_without_global_candidate_locks(tmp_path): + async with durable_harness("postgres", notifications=True) as h: + case = await durable_case(h.store) + async with worker(h, case, tmp_path, pause="reserved") as first: + await first.event("reserved") + async with worker(h, case, tmp_path) as second: + assert (await second.event("done"))["state"] in {"idle", "discovered"} + assert await asyncio.wait_for(second.process.wait(), 5) == 0 + assert not tuple(tmp_path.glob("rwm_*")) + await first.send({"continue": True}) + assert (await first.event("done"))["state"] == "verified" + assert await asyncio.wait_for(first.process.wait(), 5) == 0 + assert len(await h.works()) == 1 diff --git a/tests/conformance/reporting/test_reporting_materializer_rolling.py b/tests/conformance/reporting/test_reporting_materializer_rolling.py new file mode 100644 index 000000000..fa97646bd --- /dev/null +++ b/tests/conformance/reporting/test_reporting_materializer_rolling.py @@ -0,0 +1,258 @@ +"""Actual built/installed frozen artifacts, not renamed or path-selected modules.""" + +import asyncio +import hashlib +import json +import shutil +import sys +import tarfile +from pathlib import Path + +import pytest + +from adcp.reporting.ledger import LedgerConflictError, PgReportingReconciliationStore +from adcp.reporting.materializer import PgReportingMaterializerStore +from adcp.reporting.outbox._schema import schema_objects + +from ._durable_materializer_support import DurableHarness, durable_case +from ._generation_support import assert_c_collated_rolling_database, isolated_reporting_pool +from .test_reporting_notification_packaging import ROOT, run_step + +ARTIFACTS = { + "beta15": "3e76aa54623529a3dda01cd690b8a5c287c75641", + "records": "3c405a21f978ed9d3208611bb4a7a8434a056933", + "integration": "037de4ac822ecefb2f95d32c15c297fb4c45d683", + "a": "21bf443e7d850d1800ec8a6f2e4abec1c8f85541", + "b": "198d50e61c74fb82aedbf2c77e06a0e200b91db6", + "c": "ea150fabd5ad90e3abf93f89729d2919f1c61798", + "b1": "1c91311ec28d25506d5db43f59d0c34936ecb8f7", +} + + +def build_frozen(artifact, tmp_path_factory, request): + assert_c_collated_rolling_database() + sha = ARTIFACTS[artifact] + root = tmp_path_factory.mktemp(f"materializer-{artifact}") + request.addfinalizer(lambda: shutil.rmtree(root)) + archive, source, dist, environment = ( + root / n for n in ("source.tar.gz", "source", "dist", "installed") + ) + source.mkdir() + run_step( + ["git", "archive", "--format=tar.gz", f"--output={archive}", sha], + label=f"{artifact}-exact-git-archive", + cwd=ROOT, + ) + with tarfile.open(archive) as tar: + tar.extractall(source, filter="data") + archive.unlink() + run_step( + [sys.executable, "-m", "build", "--wheel", "--outdir", str(dist), str(source)], + label=f"{artifact}-build-frozen-wheel", + cwd=root, + timeout=180, + ) + wheel = next(dist.glob("*.whl")) + run_step( + [sys.executable, "-m", "venv", str(environment)], + label=f"{artifact}-isolated-environment", + cwd=root, + ) + python = environment / "bin/python" + installer = ( + [shutil.which("uv"), "pip", "install", "--python", str(python)] + if shutil.which("uv") + else [str(python), "-m", "pip", "install"] + ) + run_step( + [*installer, f"{wheel}[pg]"], + label=f"{artifact}-install-frozen-wheel", + cwd=root, + timeout=180, + ) + script = root / "frozen.py" + shutil.copy2(Path(__file__).with_name("_materializer_frozen.py"), script) + modules = {} + for relative in ( + "ledger/pg.py", + "ledger/delivery_pg.py", + "outbox/worker.py", + "outbox/status_pg.py", + "materializer/contracts.py", + "materializer/verification.py", + ): + path = source / "src/adcp/reporting" / relative + if path.exists(): + modules["adcp.reporting." + relative.removesuffix(".py").replace("/", ".")] = ( + hashlib.sha256(path.read_bytes()).hexdigest() + ) + settings = { + "artifact": artifact, + "sha": sha, + "modules": modules, + "workspace": str(ROOT), + "wheel_sha256": hashlib.sha256(wheel.read_bytes()).hexdigest(), + } + # The exact source hashes and wheel survive in the evidence log; the build + # tree contains 2.2 GiB of cached schemas and is not an execution dependency. + print(json.dumps({"frozen_build": settings}), flush=True) + shutil.rmtree(source) + return ( + root, + python, + script, + settings, + ) + + +@pytest.fixture(scope="module") +def installed_parent(tmp_path_factory, request): + return build_frozen("b1", tmp_path_factory, request) + + +@pytest.fixture(scope="module", params=tuple(ARTIFACTS)) +def installed_frozen(request, tmp_path_factory, installed_parent): + if request.param == "b1": + return installed_parent + return build_frozen(request.param, tmp_path_factory, request) + + +async def frozen_call(artifact, pool, action, **kwargs): + root, python, script, settings = artifact + settings = { + **settings, + "conninfo": pool.conninfo, + "kwargs": pool.kwargs, + "action": action, + **kwargs, + } + raw = await asyncio.to_thread( + run_step, + [str(python), "-I", str(script)], + label=f"{settings['artifact']}-{action}", + cwd=root, + value=settings, + timeout=60, + ) + result = json.loads(raw) + assert "failure" not in result, result + return result + + +async def test_installed_old_reader_writer_and_workers_on_populated_materializer_schema( + installed_frozen, + installed_parent, +): + async with isolated_reporting_pool(autocommit=True) as pool: + evidence = await frozen_call(installed_frozen, pool, "install") + assert evidence["installed"] + # The approved B1 wheel installs the comparison baseline, including + # reviewed C's additive capture and B1's unactivated selector fence. + parent = await frozen_call(installed_parent, pool, "install") + assert parent["sha"] == ARTIFACTS["b1"] + baseline_store = PgReportingReconciliationStore(pool=pool, notifications=True) + # beta.15 can read its original definition shape. Unit declarations are + # an A prerequisite, not a B2 schema or decoder change. New Managed facts + # stay in A's separate feed and never enter beta.15's closed Core feed. + before_url_principals = evidence["artifact"] in { + "beta15", + "records", + "integration", + "a", + "b", + } + case = await durable_case( + baseline_store, + count=3, + consumer=( + "frozen-buyer" if before_url_principals else "https://buyer.example.test/agent" + ), + legacy_definition=evidence["artifact"] == "beta15", + ) + # URL principals became readable in C. Earlier binaries still read their + # opaque caller's records while a separate URL consumer has populated + # materializations/captures/events in the same account and schema. + sibling = await durable_case( + baseline_store, + count=3, + consumer="https://buyer.example.test/isolated", + legacy_definition=evidence["artifact"] == "beta15", + ) + baseline = await frozen_call( + installed_frozen, pool, "baseline", consumer=case.binding.consumer_id + ) + assert baseline["ordinary_writes"] and baseline["core_records"] == 2 + async with pool.connection() as connection: + old_objects = await schema_objects(connection) + store = PgReportingMaterializerStore(pool=pool, notifications=True) + with pytest.raises(LedgerConflictError): + await store.materializer_ready() + await store.create_schema() + async with pool.connection() as connection: + objects = await schema_objects(connection) + assert {key: objects[key] for key in old_objects} == old_objects + assert all("reporting_materializer_" in key for key in objects.keys() - old_objects.keys()) + case.store = sibling.store = store + assert (await case.service().run_once()).state == "verified" + assert (await sibling.service().run_once()).state == "verified" + h = DurableHarness(store, None, pool) + before = await h.queue() + async with pool.connection() as connection: + immutable_before = [ + await (await connection.execute(f"SELECT * FROM {table} ORDER BY 1,2,3")).fetchall() + for table in ( + "reporting_materializer_work", + "reporting_materializer_status_boundaries", + "reporting_materializer_status_heads", + "reporting_materializer_notification_events", + "reporting_materializer_notification_expansions", + ) + ] + captured_heads = await ( + await connection.execute( + "SELECT account_id,captured_sequence FROM reporting_materializer_accounts" + " ORDER BY account_id" + ) + ).fetchall() + result = await frozen_call( + installed_frozen, pool, "exercise", consumer=case.binding.consumer_id + ) + assert result["core_records"] == 2 and result["ordinary_writes"] + assert result["managed_records"] == (0 if result["artifact"] == "beta15" else 4) + assert result["notifications_ready"] == baseline["notifications_ready"] + assert await h.queue() == before + async with pool.connection() as connection: + immutable_after = [ + await (await connection.execute(f"SELECT * FROM {table} ORDER BY 1,2,3")).fetchall() + for table in ( + "reporting_materializer_work", + "reporting_materializer_status_boundaries", + "reporting_materializer_status_heads", + "reporting_materializer_notification_events", + "reporting_materializer_notification_expansions", + ) + ] + assert ( + await ( + await connection.execute( + "SELECT account_id,captured_sequence FROM reporting_materializer_accounts" + " ORDER BY account_id" + ) + ).fetchall() + ) == captured_heads + assert immutable_after == immutable_before + assert before[1] == ("quarantined", "quarantined") + assert await store.materializer_ready() + print( + json.dumps( + { + **result, + "native": evidence, + "baseline": baseline, + "baseline_installer": parent, + "additive_objects": len(objects.keys() - old_objects.keys()), + "wheel_sha256": installed_frozen[3]["wheel_sha256"], + } + ), + flush=True, + ) diff --git a/tests/conformance/reporting/test_reporting_materializer_service.py b/tests/conformance/reporting/test_reporting_materializer_service.py new file mode 100644 index 000000000..6e4d1de15 --- /dev/null +++ b/tests/conformance/reporting/test_reporting_materializer_service.py @@ -0,0 +1,285 @@ +"""Long I/O, reauthorization, cleanup and uncertain-effect service convergence.""" + +import asyncio +from dataclasses import replace +from datetime import timedelta + +import pytest + +from adcp.reporting.ledger import ( + ReportingMaterializationAttempt, + ReportingMaterializationCheck, + ReportingObligationDeliveryRecord, +) +from adcp.reporting.materializer import ( + ReportingDestinationIO, + ReportingMaterializerService, + ReportingWriterError, + ReportingWriterFailure, +) + +from ._durable_materializer_support import durable_case, durable_harness +from .test_reporting_materializer_lifecycle import SECRET, Phases, safe_exception + + +@pytest.mark.parametrize("backend", ["memory", "postgres"]) +@pytest.mark.parametrize("stage", ["write-before", "write-after", "object", "readback-resolve"]) +async def test_cancellation_and_restart_never_allocate_a_new_external_identity( + backend, stage, tmp_path, caplog +): + async with durable_harness(backend, notifications=True) as h: + case = await durable_case(h.store, count=501) + owner = Phases(case, tmp_path, stage, "cancel") + service = ReportingMaterializerService( + h.store, ReportingDestinationIO(case.registry, owner), case.writer, lease_seconds=3 + ) + before = set(asyncio.all_tasks()) + task = asyncio.create_task(service.run_once()) + await asyncio.wait_for(owner.entered.wait(), 10) + initial = await h.works() + task.cancel(SECRET) + with pytest.raises(asyncio.CancelledError) as error: + await asyncio.wait_for(task, 10) + safe_exception(error.value) + assert all(s._closed and s._credential is None for s in owner.sessions) + assert not await case.outcomes() and await h.queue() == ((), ()) + assert not list(tmp_path.iterdir()) + assert not (set(asyncio.all_tasks()) - before) + await h.expire() + assert (await case.service().run_once()).state == "verified" + works = await h.works() + assert len(works) == 1 and works[0][0] == initial[0][0] + assert case.writer.write_effects == 1 + assert case.writer.open_count == case.writer.close_count + assert SECRET not in caplog.text + + +@pytest.mark.parametrize("stage", ["write-after", "object"]) +async def test_service_heartbeat_keeps_long_io_alive_with_size_one_postgres_pool( + stage, tmp_path, monkeypatch +): + from adcp.reporting.materializer import PgReportingMaterializerStore + + async with durable_harness("postgres", notifications=True) as h: + from psycopg_pool import AsyncConnectionPool + + case = await durable_case(h.store, count=501) + async with AsyncConnectionPool( + h.pool.conninfo, kwargs=h.pool.kwargs, min_size=1, max_size=1, open=False + ) as single: + store = PgReportingMaterializerStore(pool=single, notifications=True) + case.store = store + owner = Phases(case, tmp_path, stage, "wait") + renewed, turns = asyncio.Event(), [] + original = store.renew_materialization + + async def renew(*args, **kwargs): + held = await original(*args, **kwargs) + turns.append(held) + if len(turns) == 4: + renewed.set() + return held + + monkeypatch.setattr(store, "renew_materialization", renew) + service = ReportingMaterializerService( + store, ReportingDestinationIO(case.registry, owner), case.writer, lease_seconds=3 + ) + task = asyncio.create_task(service.run_once()) + await asyncio.wait_for(owner.entered.wait(), 10) + await asyncio.wait_for(renewed.wait(), 10) # Four DB-time heartbeats > one whole lease. + assert all(turns) and len(turns) >= 4 + assert not hasattr( + await store.claim_materialization(keys=case.keys, lease_seconds=3), "attempt" + ) + owner.release.set() + assert (await asyncio.wait_for(task, 10)).state == "verified" + assert case.writer.write_effects == 1 + assert case.writer.open_count == case.writer.close_count == 2 + + +@pytest.mark.parametrize("backend", ["memory", "postgres"]) +@pytest.mark.parametrize("change", ["rotate", "revoke", "official", "readability"]) +async def test_write_to_readback_window_reauthorizes_exact_principal_and_current_generation( + backend, change, tmp_path +): + async with durable_harness(backend, notifications=True) as h: + case = await durable_case(h.store) + owner = Phases(case, tmp_path, "write-after", "wait") + service = ReportingMaterializerService( + h.store, ReportingDestinationIO(case.registry, owner), case.writer + ) + task = asyncio.create_task(service.run_once()) + await asyncio.wait_for(owner.entered.wait(), 10) + if change == "rotate": + case.resolver.rotate() + elif change == "revoke": + case.resolver.revoke(case.scope.principal) + elif change == "official": + await case.publish() + else: + await h.store.set_revision_readable( + account_id="acct_a", + reporting_revision_id=case.revision.reporting_revision_id, + readable=False, + ) + owner.release.set() + turn = await asyncio.wait_for(task, 10) + assert turn.state == ("verified" if change == "rotate" else "failed") + assert len((await h.queue())[0]) == int(change == "rotate") + assert case.writer.open_count == case.writer.close_count + assert all(s._credential is None for s in owner.sessions) + + +@pytest.mark.parametrize("backend", ["memory", "postgres"]) +@pytest.mark.parametrize("state", ["corrupt", "unavailable"]) +async def test_later_health_loss_never_resets_or_retries_successful_history(backend, state): + async with durable_harness(backend) as h: + case = await durable_case(h.store) + assert (await case.service().run_once()).state == "verified" + outcome = (await case.outcomes())[0] + await h.store.record_materialization_check( + ReportingMaterializationCheck( + case.scope, + outcome.reporting_materialization_id, + "health", + state, + outcome.completed_at, + ) + ) + result = await case.claim() + assert result.reason == "operator_required" + assert (await case.claim()).state == "idle" + assert len(await h.works()) == 1 + assert await case.outcomes() == (outcome,) + + +async def test_retention_loss_between_readback_and_finish_is_immutable_safe_failure(): + async with durable_harness("memory", notifications=True) as h: + case = await durable_case(h.store) + lease = await case.claim(lease_seconds=300) + prepared, evidence = await case.verified(lease) + h.clock.advance(timedelta(seconds=60)) + result = await h.store.finish_materialization(lease, prepared=prepared, verified=evidence) + assert result.state == "failed" and await h.queue() == ((), ()) + assert (await case.outcomes())[0].failure_code == "RESOURCE_UNAVAILABLE" + + +@pytest.mark.parametrize("backend", ["memory", "postgres"]) +@pytest.mark.parametrize("terminal", ["verified", "failed"]) +async def test_public_next_attempt_is_still_permitted_after_any_terminal_outcome(backend, terminal): + async with durable_harness(backend) as h: + case = await durable_case(h.store) + lease = await case.claim() + if terminal == "verified": + prepared, evidence = await case.verified(lease) + await h.store.finish_materialization(lease, prepared=prepared, verified=evidence) + else: + await h.store.finish_materialization( + lease, error=ReportingWriterFailure("WRITE_FAILED", "never", "not_started") + ) + outcome = (await case.outcomes())[0] + attempt = replace( + lease.attempt, + attempt=2, + reporting_materialization_id="operator-next", + created_at=outcome.completed_at, + ) + await h.store.commit_materialization_attempt(attempt) + snapshot = await h.store.read_reconciliation_snapshot(caller=case.scope.principal) + assert [ + r.attempt for r in snapshot.records if isinstance(r, ReportingMaterializationAttempt) + ] == [1, 2] + await h.expire() + assert not hasattr(await case.claim(), "attempt") + assert len(await h.works()) == 1 + + +@pytest.mark.parametrize("backend", ["memory", "postgres"]) +async def test_driver_failure_is_a_closed_error_without_provider_context(backend, monkeypatch): + async with durable_harness(backend) as h: + case = await durable_case(h.store) + if backend == "memory": + + def fail(*args, **kwargs): + raise OSError(SECRET) + + monkeypatch.setattr(type(h.store), "_context", fail) + else: + + async def fail(*args, **kwargs): + raise OSError(SECRET) + + monkeypatch.setattr(type(h.store), "_materializer_context_on", fail) + with pytest.raises(ReportingWriterError) as error: + await case.claim() + safe_exception(error.value) + + +@pytest.mark.parametrize("backend", ["memory", "postgres"]) +@pytest.mark.parametrize("cause", ["timeout", "cleanup"]) +async def test_timeout_or_cleanup_failure_after_effect_resumes_original_identity( + backend, cause, tmp_path, caplog +): + async with durable_harness(backend, notifications=True) as h: + case = await durable_case(h.store) + # This fixture's short I/O deadline must not be its artifact lifetime. + # Recovery reuses a conditional object that still meets the full + # required retention window, independently of the timed-out request. + await h.store.bind_obligation_delivery( + ReportingObligationDeliveryRecord( + case.scope, "USD", h.clock() + timedelta(days=401), h.clock() + ) + ) + owner = Phases( + case, + tmp_path, + "write-after" if cause == "timeout" else "close", + "wait" if cause == "timeout" else "error", + ) + service = ReportingMaterializerService( + h.store, + ReportingDestinationIO(case.registry, owner), + case.writer, + io_timeout_seconds=1 if cause == "timeout" else 30, + ) + assert (await asyncio.wait_for(service.run_once(), 10)).state == "pending" + before = await h.works() + assert not await case.outcomes() and await h.queue() == ((), ()) + assert case.writer.write_effects == 1 + assert all(s._closed and s._credential is None for s in owner.sessions) + assert not tuple(tmp_path.iterdir()) and SECRET not in caplog.text + await h.expire() + assert (await case.service().run_once()).state == "verified" + assert (await h.works())[0][0] == before[0][0] + assert case.writer.write_effects == 1 + + +@pytest.mark.parametrize("backend", ["memory", "postgres"]) +async def test_heartbeat_cancels_io_after_expiry_and_competing_worker_steals_fence( + backend, tmp_path +): + async with durable_harness(backend, notifications=True) as h: + case = await durable_case(h.store) + owner = Phases(case, tmp_path, "write-after", "wait") + service = ReportingMaterializerService( + h.store, ReportingDestinationIO(case.registry, owner), case.writer, lease_seconds=3 + ) + task = asyncio.create_task(service.run_once()) + try: + await asyncio.wait_for(owner.entered.wait(), 10) + before = await h.works() + await h.expire() + winner = await case.claim() + assert (await asyncio.wait_for(task, 10)).state == "pending" + assert all(s._closed and s._credential is None for s in owner.sessions) + assert not tuple(tmp_path.iterdir()) + assert not await case.outcomes() and await h.queue() == ((), ()) + prepared, evidence = await case.verified(winner) + assert ( + await h.store.finish_materialization(winner, prepared=prepared, verified=evidence) + ).state == "verified" + assert (await h.works())[0][0] == before[0][0] + assert case.writer.write_effects == 1 + finally: + task.cancel() + await asyncio.gather(task, return_exceptions=True) diff --git a/tests/conformance/reporting/test_reporting_materializer_transactions.py b/tests/conformance/reporting/test_reporting_materializer_transactions.py new file mode 100644 index 000000000..0f6f61f20 --- /dev/null +++ b/tests/conformance/reporting/test_reporting_materializer_transactions.py @@ -0,0 +1,312 @@ +"""Faults at every reserve/finish write boundary with notifications off and on.""" + +from contextlib import contextmanager +from dataclasses import replace +from datetime import timedelta + +import pytest + +from adcp.reporting.ledger import ReportingMaterializationRecord +from adcp.reporting.materializer import ReportingWriterError +from adcp.reporting.materializer.memory import InMemoryReportingMaterializerStore + +from ._durable_materializer_support import durable_case, durable_harness + + +@pytest.mark.parametrize("notifications", [False, True]) +async def test_failed_memory_snapshot_does_not_leak_transaction_ownership(notifications): + class Uncopyable: + def __deepcopy__(self, memo): + raise ValueError("injected snapshot fault") + + async with durable_harness("memory", notifications=notifications) as h: + case = await durable_case(h.store) + changed = replace(case.config, status_retention_days=case.config.status_retention_days + 1) + h.store._injected_snapshot_fault = Uncopyable() + with pytest.raises(ValueError, match="injected snapshot fault"): + await h.store.put_configuration(changed) + del h.store._injected_snapshot_fault + before = await h.image() + with pytest.raises(RuntimeError, match="rollback subsequent transaction"): + async with h.store.transaction(): + await h.store.put_configuration(changed) + raise RuntimeError("rollback subsequent transaction") + assert await h.image() == before + + +@contextmanager +def postgres_failure(monkeypatch, prefix): + from psycopg import AsyncConnection + + original = AsyncConnection.execute + hit = [] + + async def execute(self, query, *args, **kwargs): + result = await original(self, query, *args, **kwargs) + if isinstance(query, str) and query.startswith(prefix): + hit.append(self.pgconn.backend_pid) + raise OSError("injected transaction boundary") + return result + + with monkeypatch.context() as patch: + patch.setattr(AsyncConnection, "execute", execute) + yield hit + + +@pytest.mark.parametrize("backend", ["memory", "postgres"]) +@pytest.mark.parametrize("notifications", [False, True]) +@pytest.mark.parametrize("position", ["delivery", "attempt", "work", "lease"]) +async def test_reservation_rolls_back_attempt_work_delivery_and_sequence_heads( + backend, notifications, position, monkeypatch +): + async with durable_harness(backend, notifications=notifications) as h: + case = await durable_case(h.store) + before = await h.image() + if backend == "postgres": + prefixes = { + "delivery": "INSERT INTO reporting_reconciliation_records", + "attempt": "INSERT INTO reporting_reconciliation_changes", + "work": "INSERT INTO reporting_materializer_work", + "lease": "UPDATE reporting_materializer_work SET lease_token=", + } + # Count explicitly so attempt failure happens after the delivery's + # own complete insertion, not at its earlier feed append. + if position == "attempt": + from adcp.reporting.materializer.pg import PgReportingMaterializerStore + + original = PgReportingMaterializerStore._commit_record_on + + async def fail(self, connection, record, **kwargs): + result = await original(self, connection, record, **kwargs) + if record.kind == "materialization_attempt": + raise OSError("injected transaction boundary") + return result + + with monkeypatch.context() as patch: + patch.setattr(PgReportingMaterializerStore, "_commit_record_on", fail) + with pytest.raises(ReportingWriterError): + await case.claim() + else: + with postgres_failure(monkeypatch, prefixes[position]) as hit: + with pytest.raises(ReportingWriterError): + await case.claim() + assert len(hit) == 1 + else: + cls = InMemoryReportingMaterializerStore + if position in {"delivery", "attempt"}: + original = cls._commit_record_unlocked + + def fail(self, record, **kwargs): + result = original(self, record, **kwargs) + if ( + record.kind + == { + "delivery": "obligation_delivery", + "attempt": "materialization_attempt", + }[position] + ): + raise OSError("injected transaction boundary") + return result + + method = "_commit_record_unlocked" + else: + original = cls._lease + + def fail(self, *args): + if position == "lease": + original(self, *args) + raise OSError("injected transaction boundary") + + method = "_lease" + with monkeypatch.context() as patch: + patch.setattr(cls, method, fail) + with pytest.raises(ReportingWriterError): + await case.claim() + assert await h.image() == before + lease = await case.claim() + assert lease.attempt.attempt == 1 + assert len(await h.works()) == 1 + + +@pytest.mark.parametrize("backend", ["memory", "postgres"]) +@pytest.mark.parametrize("notifications", [False, True]) +@pytest.mark.parametrize("position", ["outcome", "status_head", "account_head", "boundary", "ack"]) +async def test_finish_rolls_back_terminal_capture_ack_and_all_initialized_heads( + backend, notifications, position, monkeypatch +): + async with durable_harness(backend, notifications=notifications) as h: + case = await durable_case(h.store) + lease = await case.claim() + prepared, verified = await case.verified(lease) + before = await h.image() + + async def finish(): + return await h.store.finish_materialization(lease, prepared=prepared, verified=verified) + + if backend == "postgres": + prefixes = { + "outcome": "INSERT INTO reporting_reconciliation_records", + "status_head": "INSERT INTO reporting_materializer_status_heads", + "account_head": "UPDATE reporting_materializer_accounts SET captured_sequence=", + "boundary": "INSERT INTO reporting_materializer_status_boundaries", + "ack": "UPDATE reporting_materializer_work SET state='acked'", + } + with postgres_failure(monkeypatch, prefixes[position]) as hit: + with pytest.raises(ReportingWriterError): + await finish() + assert len(hit) == 1 + else: + import adcp.reporting.materializer.memory as memory + + cls = InMemoryReportingMaterializerStore + if position == "outcome": + original = cls._commit_record_unlocked + + def fail(self, record, **kwargs): + result = original(self, record, **kwargs) + if isinstance(record, ReportingMaterializationRecord): + raise OSError("injected transaction boundary") + return result + + target, method = cls, "_commit_record_unlocked" + elif position in {"status_head", "account_head"}: + + def fail(*args, **kwargs): + raise OSError("injected transaction boundary") + + target, method = memory, "ReportingMaterializerBoundary" + else: + method = "_materializer_dirty" if position == "boundary" else "_park" + target, original = cls, getattr(cls, method) + + def fail(self, *args, **kwargs): + original(self, *args, **kwargs) + raise OSError("injected transaction boundary") + + with monkeypatch.context() as patch: + patch.setattr(target, method, fail) + with pytest.raises(ReportingWriterError): + await finish() + assert await h.image() == before + assert (await finish()).state == "verified" + assert len(await case.outcomes()) == 1 + assert len(await h.store.read_materializer_boundaries(caller=case.scope.principal)) == 1 + + +@pytest.mark.parametrize("backend", ["memory", "postgres"]) +@pytest.mark.parametrize("position", ["event", "expansion_work"]) +async def test_enabled_logical_enqueue_failure_never_downgrades_to_polling( + backend, position, monkeypatch +): + async with durable_harness(backend, notifications=True) as h: + case = await durable_case(h.store) + lease = await case.claim() + prepared, verified = await case.verified(lease) + before = await h.image() + + async def finish(): + return await h.store.finish_materialization(lease, prepared=prepared, verified=verified) + + if backend == "postgres": + table = "events" if position == "event" else "expansions" + with postgres_failure( + monkeypatch, f"INSERT INTO reporting_materializer_notification_{table}" + ) as hit: + with pytest.raises(ReportingWriterError): + await finish() + assert len(hit) == 1 + else: + import adcp.reporting.outbox.memory as memory + from adcp.reporting.materializer.capture import MaterializerNotificationState + + if position == "event": + target, method = memory, "_Work" + + def fail(*args, **kwargs): + raise OSError("injected transaction boundary") + + else: + target, method = MaterializerNotificationState, "enqueue" + original = target.enqueue + + def fail(self, event): + original(self, event) + raise OSError("injected transaction boundary") + + with monkeypatch.context() as patch: + patch.setattr(target, method, fail) + with pytest.raises(ReportingWriterError): + await finish() + assert await h.image() == before + assert await h.queue() == ((), ()) + assert (await finish()).state == "verified" + events, work = await h.queue() + assert len(events) == 1 and work == ("quarantined",) + + +@pytest.mark.parametrize("backend", ["memory", "postgres"]) +async def test_missing_enabled_logical_event_cannot_ack_a_verified_finish(backend, monkeypatch): + async with durable_harness(backend, notifications=True) as h: + case = await durable_case(h.store) + lease = await case.claim() + prepared, evidence = await case.verified(lease) + before = await h.image() + if backend == "memory": + from adcp.reporting.materializer.capture import MaterializerNotificationState + + def empty(self, event): + pass + + monkeypatch.setattr(MaterializerNotificationState, "enqueue", empty) + else: + import adcp.reporting.materializer.pg as pg + + async def empty(connection, event): + pass + + monkeypatch.setattr(pg, "enqueue_materializer_event_on", empty) + with pytest.raises(ReportingWriterError): + await h.store.finish_materialization(lease, prepared=prepared, verified=evidence) + assert await h.image() == before + assert await h.queue() == ((), ()) + + +@pytest.mark.parametrize("backend", ["memory", "postgres"]) +@pytest.mark.parametrize("notifications", [False, True]) +async def test_expiry_inside_finish_rolls_back_outcome_capture_event_and_ack( + backend, notifications, monkeypatch +): + async with durable_harness(backend, notifications=notifications) as h: + case = await durable_case(h.store) + lease = await case.claim(lease_seconds=3) + prepared, evidence = await case.verified(lease) + before = await h.image() + cls = type(h.store) + if backend == "memory": + original = cls._materializer_dirty + + def expire(self, *args): + original(self, *args) + h.clock.advance(timedelta(seconds=4)) + + method = "_materializer_dirty" + else: + original = cls._materializer_dirty_on + + async def expire(self, connection, *args): + await original(self, connection, *args) + await connection.execute( + "UPDATE reporting_materializer_work SET lease_until=clock_timestamp()" + " WHERE state='pending'" + ) + + method = "_materializer_dirty_on" + with monkeypatch.context() as patch: + patch.setattr(cls, method, expire) + with pytest.raises(ReportingWriterError): + await h.store.finish_materialization(lease, prepared=prepared, verified=evidence) + assert await h.image() == before and await h.queue() == ((), ()) + await h.expire() + assert (await case.service().run_once()).state == "verified" + assert case.writer.write_effects == 1 + assert len(await h.works()) == 1 diff --git a/tests/conformance/reporting/test_reporting_notification_packaging.py b/tests/conformance/reporting/test_reporting_notification_packaging.py index e902fb342..d692fc6b2 100644 --- a/tests/conformance/reporting/test_reporting_notification_packaging.py +++ b/tests/conformance/reporting/test_reporting_notification_packaging.py @@ -30,6 +30,45 @@ ROOT = Path(__file__).resolve().parents[3] +def redacted_stage_stderr(stderr): + """Keep known error signatures, never arbitrary build/provider prose. + + Even an unfamiliar credential format cannot escape an allowlist of fixed + labels. Line positions and a digest distinguish otherwise unclassified + traces without logging URLs, environment values or exception messages. + """ + signatures = { + "No space left on device": "disk_full", + "[Errno 28]": "disk_full", + "PermissionError": "permission_denied", + "FileNotFoundError": "file_missing", + "ModuleNotFoundError": "module_missing", + "BackendUnavailable": "backend_unavailable", + "subprocess-exited-with-error": "subprocess_failed", + "No matching distribution found": "distribution_unavailable", + "Temporary failure in name resolution": "dns_failure", + "ConnectionError": "connection_failure", + "ReadTimeout": "read_timeout", + "SyntaxError": "syntax_error", + "AssertionError": "assertion_failed", + } + lines = stderr.splitlines() + trace = [] + for position, line in enumerate(lines): + found = sorted({tag for marker, tag in signatures.items() if marker in line}) + if found: + trace.append([position + 1, found]) + return json.dumps( + { + "lines": len(lines), + "sha256": hashlib.sha256(stderr.encode()).hexdigest(), + "trace": trace[-8:], + "classification": "recognized" if trace else "unclassified", + }, + separators=(",", ":"), + ) + + def run_step(command, *, label, cwd, value=None, timeout=120): started = time.monotonic() process = subprocess.Popen( @@ -44,7 +83,7 @@ def run_step(command, *, label, cwd, value=None, timeout=120): ) print(f"notification_distribution stage={label} pid={process.pid} started", flush=True) try: - stdout, _ = process.communicate( + stdout, stderr = process.communicate( json.dumps(value) if value is not None else None, timeout=timeout, ) @@ -77,9 +116,16 @@ def run_step(command, *, label, cwd, value=None, timeout=120): f" pid={process.pid} exit={process.returncode} cleanup={cleanup}" f" elapsed_ms={int((time.monotonic() - started) * 1000)}" f" stdout_chars={len(stdout)} stderr_chars={len(stderr)}" + f" stderr={redacted_stage_stderr(stderr)}" ) from None # Do not turn package-manager/provider output into test diagnostics. - assert process.returncode == 0, f"notification distribution {label}: exit {process.returncode}" + if process.returncode != 0: + raise AssertionError( + f"notification distribution {label}: exit {process.returncode} pid={process.pid}" + f" elapsed_ms={int((time.monotonic() - started) * 1000)}" + f" stdout_chars={len(stdout)} stderr_chars={len(stderr)}" + f" stderr={redacted_stage_stderr(stderr)}" + ) print(f"notification_distribution stage={label} passed", flush=True) return stdout @@ -94,9 +140,39 @@ def test_distribution_subprocess_deadline_is_bounded_and_sanitized(tmp_path): ) +def test_distribution_failure_diagnostics_classify_without_echoing_prose(tmp_path): + body = ( + "https://index-user:index-password@example.test/simple?signature=private\n" + "Authorization: Bearer opaque-credential\n" + "TOKEN_WITH_UNFAMILIAR_FORMAT=private-configuration\n" + "OSError: [Errno 28] No space left on device\n" + ) + with pytest.raises(AssertionError) as captured: + run_step( + [ + sys.executable, + "-c", + "import sys;sys.stderr.write(sys.stdin.read());raise SystemExit(3)", + ], + label="diagnostic_probe", + cwd=tmp_path, + value=body, + ) + message = str(captured.value) + assert "disk_full" in message and "exit 3" in message + assert len(message) < 1024 + assert all(word not in message for word in ("private", "index-user", "opaque-credential")) + unknown = redacted_stage_stderr("unrecognized provider body and secret configuration") + assert "unclassified" in unknown and "provider" not in unknown and "secret" not in unknown + + @pytest.fixture(scope="module") -def built_distribution(tmp_path_factory): +def built_distribution(tmp_path_factory, request): path = tmp_path_factory.mktemp("reporting-outbox-distribution") + # A module can allocate several installed environments and copied schemas. + # Remove only this fixture's tree after all its dependent fixtures/processes + # finish. Keep the optional fingerprint-checked immutable cache separately. + request.addfinalizer(lambda: shutil.rmtree(path)) cache = os.environ.get("ADCP_REPORTING_DISTRIBUTION") sources = [ ROOT / name @@ -193,8 +269,12 @@ def installed_distribution(built_distribution, request): resolve_reporting_consumer, ) from adcp.reporting.ledger import InMemoryReportingLedgerStore, ReportingProducer -from adcp.reporting.ledger import ReportingStatusSnapshot, StatusProjectionInput, project_status_scope -from adcp.reporting.outbox import ReportingStatusNotificationLifecycle, ReportingStatusService, StatusChanged +from adcp.reporting.ledger import ( + ReportingStatusSnapshot, StatusProjectionInput, project_status_scope, +) +from adcp.reporting.outbox import ( + ReportingStatusNotificationLifecycle, ReportingStatusService, StatusChanged, +) from adcp.validation.schema_loader import get_named_validator import inspect, sys @@ -213,7 +293,9 @@ def installed_distribution(built_distribution, request): assert "adcp[pg]" in str(error), str(error) else: raise AssertionError("PgReportingOutbox must raise the [pg] install hint") -from adcp.reporting.outbox import PgStatusNotificationStore, PgReportingStatusOutbox, PgReportingActivityUnionStore +from adcp.reporting.outbox import ( + PgStatusNotificationStore, PgReportingStatusOutbox, PgReportingActivityUnionStore, +) for constructor in (lambda: PgStatusNotificationStore(None), lambda: PgReportingStatusOutbox(pool=None), lambda: PgReportingActivityUnionStore(None, None)): @@ -230,7 +312,9 @@ def installed_distribution(built_distribution, request): assert files("adcp.reporting.ledger").joinpath("reporting_status_notifications.sql").is_file() assert files("adcp.reporting.ledger").joinpath("reporting_status_selector_version.sql").is_file() assert files("adcp.reporting.outbox").joinpath("required_status_selector_schema.json").is_file() -assert get_named_validator("core/reporting-status-changed-webhook.json", version="3.2.0-rc.3") is not None +assert get_named_validator( + "core/reporting-status-changed-webhook.json", version="3.2.0-rc.3" +) is not None assert get_named_validator("core/webhook-activity-record.json", version="3.2.0-rc.3") is not None assert ( get_named_validator("core/reporting-ledger-changed-webhook.json", version="3.2.0-rc.3") @@ -431,26 +515,35 @@ async def get_active(self, *, account_id, subscriber_id, notification_type): status_subscription = replace(subscription, event_types=("reporting.status_changed",)) class StatusConfigurations: async def list_active(self, *, account_id, notification_type): - return (status_subscription,) if account_id == status_subscription.account_id else () + return ( + (status_subscription,) if account_id == status_subscription.account_id else () + ) async def get_active(self, *, account_id, subscriber_id, notification_type): return status_subscription if account_id == status_subscription.account_id else None status_worker = ReportingNotificationWorker(outbox=status.outbox, - subscriptions=StatusConfigurations(), cipher=cipher, clock=clock, activity=status.outbox) + subscriptions=StatusConfigurations(), cipher=cipher, clock=clock, + activity=status.outbox) assert await status_worker.expand_one(account_id="acct_a") - status_lease = await status.outbox.claim_delivery(account_id="acct_a", now=clock(), lease_seconds=60) + status_lease = await status.outbox.claim_delivery( + account_id="acct_a", now=clock(), lease_seconds=60) status_body = cipher.open(status_lease.delivery).prepared assert json.loads(status_body.body)["notification_type"] == "reporting.status_changed" - assert await status.outbox.finish_delivery(status_lease, now=clock(), state="pending", retry_at=clock()) - async with AsyncConnectionPool(values["conninfo"], kwargs=values["kwargs"], open=False) as c_restart: + assert await status.outbox.finish_delivery( + status_lease, now=clock(), state="pending", retry_at=clock()) + async with AsyncConnectionPool( + values["conninfo"], kwargs=values["kwargs"], open=False + ) as c_restart: ledger = PgReportingReconciliationStore(pool=c_restart, clock=clock, notifications=True) status = PgStatusNotificationStore(ledger) await status.create_schema() assert await status.baseline_ready(account_id="acct_a") assert not (await status.project_one(account_id="acct_a")).did_work assert await status.outbox.list_events(account_id="acct_a") == status_events - status_lease = await status.outbox.claim_delivery(account_id="acct_a", now=clock(), lease_seconds=60) + status_lease = await status.outbox.claim_delivery( + account_id="acct_a", now=clock(), lease_seconds=60) retry = cipher.open(status_lease.delivery).prepared - assert retry.body == status_body.body and retry.idempotency_key == status_body.idempotency_key + assert retry.body == status_body.body + assert retry.idempotency_key == status_body.idempotency_key assert await status.outbox.finish_delivery(status_lease, now=clock(), state="complete") assert await status.outbox.reemit(account_id="acct_a", consumer_namespace="", notification_id=status_events[0].notification_id, now=clock()) == 2 diff --git a/tests/conformance/reporting/test_reporting_status_process_matrix.py b/tests/conformance/reporting/test_reporting_status_process_matrix.py index 99550d01f..02a15adf2 100644 --- a/tests/conformance/reporting/test_reporting_status_process_matrix.py +++ b/tests/conformance/reporting/test_reporting_status_process_matrix.py @@ -146,7 +146,8 @@ async def database_seed(pool, case="expected", *, baseline=True): await c.execute( "UPDATE reporting_status_accounts SET baseline_complete=TRUE," " baseline_highwater=%s, dirty_sequence=%s, baseline_at=%s," - " replay_lifecycles=%s::jsonb, selector_target_version=2, selector_transition='complete' WHERE account_id='acct_a'", + " replay_lifecycles=%s::jsonb, selector_target_version=2," + " selector_transition='complete' WHERE account_id='acct_a'", (through, through, snapshot.as_of, _replay_storage(snapshot)), ) return ledger, status, at, escalation diff --git a/tests/type_checks/reporting_durable_materializer.py b/tests/type_checks/reporting_durable_materializer.py new file mode 100644 index 000000000..7ff9b7f4c --- /dev/null +++ b/tests/type_checks/reporting_durable_materializer.py @@ -0,0 +1,32 @@ +"""The optional durable protocol and public lazy imports remain strict-adopter APIs.""" + +from typing_extensions import assert_type + +from adcp.reporting.materializer import ( + InMemoryReportingMaterializerStore, + PgReportingMaterializerStore, + ReportingDestinationIO, + ReportingDestinationWriter, + ReportingMaterializerLease, + ReportingMaterializerService, + ReportingMaterializerStore, + ReportingMaterializerTurn, + ReportingVerificationKey, +) + + +async def adopter( + postgres: PgReportingMaterializerStore, + memory: InMemoryReportingMaterializerStore, + io: ReportingDestinationIO, + writer: ReportingDestinationWriter, + key: ReportingVerificationKey, +) -> None: + store: ReportingMaterializerStore = postgres + store = memory + service = ReportingMaterializerService(store, io, writer) + assert_type(await service.run_once(), ReportingMaterializerTurn) + lease = await store.claim_materialization(keys=(key,)) + if isinstance(lease, ReportingMaterializerLease): + assert_type(await store.renew_materialization(lease, lease_seconds=30), bool) + await store.authorize_materialization(lease) From 0e22e059639b7ca60bd38b8bc4ac0b9846bf3996 Mon Sep 17 00:00:00 2001 From: Brian O'Kelley Date: Thu, 17 Sep 2026 04:42:59 +0000 Subject: [PATCH 2/8] fix(reporting): close materializer entry-point diagnostics Guard writer metadata and custom-store reservation failures, keep component reprs private, and retain bounded artifact setup and cleanup headroom. Refs #1167. --- .github/workflows/ci.yml | 7 +++-- src/adcp/reporting/materializer/service.py | 4 ++- .../test_reporting_materializer_rolling.py | 2 +- .../test_reporting_materializer_service.py | 26 +++++++++++++++++++ .../test_reporting_notification_packaging.py | 6 ++++- 5 files changed, 40 insertions(+), 5 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index a8055e152..2df2afee4 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -256,7 +256,10 @@ jobs: runs-on: ubuntu-latest # Actual wheel builds/installations plus isolated worker processes are a # separate bounded job; do not consume the existing conformance headroom. - timeout-minutes: 25 + # Seven real installations plus process/crash and Python 3.10 distribution + # cells approach twenty minutes locally; allow runner variance and retain + # five minutes outside the suite for setup and cleanup. + timeout-minutes: 35 services: postgres: image: postgres:16 @@ -296,7 +299,7 @@ jobs: - name: Install test dependencies run: pip install -e ".[dev,pg]" - name: Run installed frozen binaries and materializer migrations - timeout-minutes: 20 + timeout-minutes: 30 env: ADCP_PG_TEST_URL: postgresql://postgres@localhost:5432/adcp_materializer_test ADCP_PYTHON310: ${{ steps.materializer-python310.outputs.python-path }} diff --git a/src/adcp/reporting/materializer/service.py b/src/adcp/reporting/materializer/service.py index fe29a14fd..06c8d9ebb 100644 --- a/src/adcp/reporting/materializer/service.py +++ b/src/adcp/reporting/materializer/service.py @@ -6,6 +6,7 @@ from dataclasses import dataclass, field from datetime import datetime, timedelta, timezone +from adcp.reporting.materializer._errors import materializer_errors from adcp.reporting.materializer.contracts import ( ReportingDestinationWriter, ReportingIOContext, @@ -71,7 +72,7 @@ class ReportingMaterializerService: paginated readback; the service checks current ledger authority before each. """ - store: ReportingMaterializerStore + store: ReportingMaterializerStore = field(repr=False) io: ReportingDestinationIO = field(repr=False) writer: ReportingDestinationWriter = field(repr=False) lease_seconds: int = 30 @@ -84,6 +85,7 @@ def __post_init__(self) -> None: if type(self.io) is not ReportingDestinationIO: raise failure("UNSUPPORTED_VERIFICATION") + @materializer_errors async def run_once(self) -> ReportingMaterializerTurn: keys = tuple( v.key diff --git a/tests/conformance/reporting/test_reporting_materializer_rolling.py b/tests/conformance/reporting/test_reporting_materializer_rolling.py index fa97646bd..6f09b41a3 100644 --- a/tests/conformance/reporting/test_reporting_materializer_rolling.py +++ b/tests/conformance/reporting/test_reporting_materializer_rolling.py @@ -39,7 +39,7 @@ def build_frozen(artifact, tmp_path_factory, request): ) source.mkdir() run_step( - ["git", "archive", "--format=tar.gz", f"--output={archive}", sha], + ["git", "archive", "--format=tar.gz", "-1", f"--output={archive}", sha], label=f"{artifact}-exact-git-archive", cwd=ROOT, ) diff --git a/tests/conformance/reporting/test_reporting_materializer_service.py b/tests/conformance/reporting/test_reporting_materializer_service.py index 6e4d1de15..b078ef4b3 100644 --- a/tests/conformance/reporting/test_reporting_materializer_service.py +++ b/tests/conformance/reporting/test_reporting_materializer_service.py @@ -194,6 +194,32 @@ async def test_public_next_attempt_is_still_permitted_after_any_terminal_outcome assert len(await h.works()) == 1 +@pytest.mark.parametrize("backend", ["memory", "postgres"]) +@pytest.mark.parametrize("source", ["writer_metadata", "store_reservation"]) +async def test_service_entry_point_closes_errors_before_reservation(backend, source, monkeypatch): + async with durable_harness(backend, notifications=True) as h: + case = await durable_case(h.store) + monkeypatch.setattr(type(h.store), "__repr__", lambda self: SECRET) + assert SECRET not in repr(case.service()) + if source == "writer_metadata": + + def broken_metadata(self): + raise RuntimeError(SECRET) + + monkeypatch.setattr(type(case.writer), "capabilities", property(broken_metadata)) + else: + + async def broken_reservation(self, **kwargs): + raise RuntimeError(SECRET) + + monkeypatch.setattr(type(h.store), "claim_materialization", broken_reservation) + with pytest.raises(ReportingWriterError) as error: + await case.service().run_once() + safe_exception(error.value) + assert not await h.works() and await h.queue() == ((), ()) + assert case.writer.write_effects == 0 + + @pytest.mark.parametrize("backend", ["memory", "postgres"]) async def test_driver_failure_is_a_closed_error_without_provider_context(backend, monkeypatch): async with durable_harness(backend) as h: diff --git a/tests/conformance/reporting/test_reporting_notification_packaging.py b/tests/conformance/reporting/test_reporting_notification_packaging.py index d692fc6b2..fdae1b910 100644 --- a/tests/conformance/reporting/test_reporting_notification_packaging.py +++ b/tests/conformance/reporting/test_reporting_notification_packaging.py @@ -126,7 +126,11 @@ def run_step(command, *, label, cwd, value=None, timeout=120): f" stdout_chars={len(stdout)} stderr_chars={len(stderr)}" f" stderr={redacted_stage_stderr(stderr)}" ) - print(f"notification_distribution stage={label} passed", flush=True) + print( + f"notification_distribution stage={label} passed" + f" elapsed_ms={int((time.monotonic() - started) * 1000)}", + flush=True, + ) return stdout From 3637059a49d52e72b9450b10ee360d07a04abf93 Mon Sep 17 00:00:00 2001 From: Brian O'Kelley Date: Thu, 17 Sep 2026 05:15:32 +0000 Subject: [PATCH 3/8] ci(reporting): propagate materializer suite failures --- .github/workflows/ci.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 2df2afee4..bf83c4956 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -299,6 +299,8 @@ jobs: - name: Install test dependencies run: pip install -e ".[dev,pg]" - name: Run installed frozen binaries and materializer migrations + # Propagate pytest failures through tee while retaining evidence for upload. + shell: bash timeout-minutes: 30 env: ADCP_PG_TEST_URL: postgresql://postgres@localhost:5432/adcp_materializer_test From 1b066a5f6bb0dd63d28352a8b6c28437bc1790cc Mon Sep 17 00:00:00 2001 From: Brian O'Kelley Date: Thu, 17 Sep 2026 05:54:07 +0000 Subject: [PATCH 4/8] fix(reporting): keep projection dirty work on public materializations The materializer stores suppress `materialization_event` for every `ReportingMaterializationRecord` so only a fenced verified finish can enqueue a readiness intent. That single `notify` flag also gated `delivery_dirty`, so an ordinary public `commit_materialization` through `InMemoryReportingMaterializerStore` or `PgReportingMaterializerStore` stopped marking the status scope dirty. The projector then never saw the outcome and the projected status stayed stale indefinitely, losing the public persistence compatibility the parent kept. Split the connection-bound and memory commit primitives into independent `notify` and `dirty` flags. Reservation still writes its delivery record and attempt without either, and the finish transaction still owns its own captured dirty work, so the atomic finish boundary is unchanged. `test_public_terminal_outcome_keeps_projection_dirty_work_without_readiness` asserts the restored `materialization` dirty evidence with no readiness in the materializer or the ordinary queue, and `test_reserved_attempt_keeps_the_finish_transaction_as_the_only_dirty_work` pins reservation as the only path that stays clean. Both run on memory and real PostgreSQL and fail on the previous behaviour. Co-Authored-By: Claude Opus 5 (1M context) --- docs/reporting-durable-materializer.md | 6 ++- src/adcp/reporting/ledger/delivery.py | 37 +++++++++------ src/adcp/reporting/ledger/delivery_pg.py | 34 ++++++++------ src/adcp/reporting/materializer/memory.py | 15 ++++-- src/adcp/reporting/materializer/pg.py | 15 ++++-- .../_durable_materializer_support.py | 30 ++++++++++++ .../test_reporting_materializer_history.py | 47 +++++++++++++++++++ 7 files changed, 144 insertions(+), 40 deletions(-) diff --git a/docs/reporting-durable-materializer.md b/docs/reporting-durable-materializer.md index 9b8fb16cf..311d21b0f 100644 --- a/docs/reporting-durable-materializer.md +++ b/docs/reporting-durable-materializer.md @@ -157,7 +157,11 @@ activation belong to B2.4. | Successful artifact later becomes unreadable, revoked, unhealthy or expired | Preserve the immutable successful outcome/count; park or project degraded health | Repair authority/health or explicitly persist recovery; no automatic new attempt after success | Public persistence still permits attempt N+1 after **any** immutable terminal -outcome. The autonomous allocator intentionally owns a narrower retry policy. +outcome, and an ordinary public materialization outcome keeps producing its usual +status projection work. Only the deliverable readiness event is reserved for the +fenced verified finish; the projector must never go stale because an adopter +persisted an outcome itself. The autonomous allocator intentionally owns a +narrower retry policy. Attempt numbers are revision-specific. Selecting a different revision never deletes another revision's work/history; selecting the same unreadable revision never resets its sequence. Official-required selection never falls back to a diff --git a/src/adcp/reporting/ledger/delivery.py b/src/adcp/reporting/ledger/delivery.py index c5dab5a45..d07bad3c4 100644 --- a/src/adcp/reporting/ledger/delivery.py +++ b/src/adcp/reporting/ledger/delivery.py @@ -409,9 +409,14 @@ async def _commit(self, record: RecordT) -> tuple[RecordT, bool]: return self._commit_record_unlocked(candidate) def _commit_record_unlocked( - self, record: RecordT, *, notify: bool = True + self, record: RecordT, *, notify: bool = True, dirty: bool = True ) -> tuple[RecordT, bool]: - """Caller owns the memory mutation; no lock or callback is acquired here.""" + """Caller owns the memory mutation; no lock or callback is acquired here. + + ``notify`` and ``dirty`` are independent: suppressing a readiness event + must never also drop the projection work that an ordinary public write + has always produced. + """ candidate = decode_record(payload(record)) who = principal(candidate) records = tuple(item.record for item in self._caller_changes(who)) @@ -421,24 +426,26 @@ def _commit_record_unlocked( context = self._delivery_context(candidate) stored = validate_transition(candidate, records, context, self._clock()) self._append_reconciliation_change(stored) - if notify and self._notification_state is not None: + if (notify or dirty) and self._notification_state is not None: from adcp.reporting.ledger.notification_events import ( delivery_dirty, materialization_event, ) - event = materialization_event( - stored, - records, - context.obligation, - context.revision, - context.configuration, - self._clock(), - ) - if event is not None: - self._record_notification(event) - scope, reason, evidence = delivery_dirty(stored, context.obligation) - self._dirty_status(scope, reason, after=evidence) + if notify: + event = materialization_event( + stored, + records, + context.obligation, + context.revision, + context.configuration, + self._clock(), + ) + if event is not None: + self._record_notification(event) + if dirty: + scope, reason, evidence = delivery_dirty(stored, context.obligation) + self._dirty_status(scope, reason, after=evidence) return cast(RecordT, stored), True def _append_reconciliation_change(self, record: ReportingDeliveryRecord) -> None: diff --git a/src/adcp/reporting/ledger/delivery_pg.py b/src/adcp/reporting/ledger/delivery_pg.py index 7c977a74d..274a98826 100644 --- a/src/adcp/reporting/ledger/delivery_pg.py +++ b/src/adcp/reporting/ledger/delivery_pg.py @@ -118,13 +118,17 @@ async def _commit_record(self, record: RecordT) -> tuple[RecordT, bool]: return await self._commit_record_on(connection, candidate) async def _commit_record_on( - self, connection: Any, record: RecordT, *, notify: bool = True + self, connection: Any, record: RecordT, *, notify: bool = True, dirty: bool = True ) -> tuple[RecordT, bool]: """Connection-bound primitive. Caller holds the account transaction lock. Materializer finish uses this exact connection for evidence, caller feed, projection dirty work and its acknowledgment. No connection is acquired and no external code runs here. + + ``notify`` and ``dirty`` are independent: suppressing a readiness event + must never also drop the projection work that an ordinary public write + has always produced. """ candidate = decode_record(payload(record)) who = principal(candidate) @@ -142,24 +146,26 @@ async def _commit_record_on( stored = validate_transition(candidate, records, context, now) await self._insert(connection, stored) await self._append_reconciliation_change(connection, stored) - if notify and self._notifications_enabled: + if (notify or dirty) and self._notifications_enabled: from adcp.reporting.ledger.notification_events import ( delivery_dirty, materialization_event, ) - event = materialization_event( - stored, - records, - context.obligation, - context.revision, - context.configuration, - now, - ) - if event is not None: - await self._record_notification(connection, event) - scope, reason, evidence = delivery_dirty(stored, context.obligation) - await self._dirty_status(connection, scope, reason, after=evidence) + if notify: + event = materialization_event( + stored, + records, + context.obligation, + context.revision, + context.configuration, + now, + ) + if event is not None: + await self._record_notification(connection, event) + if dirty: + scope, reason, evidence = delivery_dirty(stored, context.obligation) + await self._dirty_status(connection, scope, reason, after=evidence) return cast(RecordT, stored), True async def _append_reconciliation_change( diff --git a/src/adcp/reporting/materializer/memory.py b/src/adcp/reporting/materializer/memory.py index 424e33eba..0431da862 100644 --- a/src/adcp/reporting/materializer/memory.py +++ b/src/adcp/reporting/materializer/memory.py @@ -157,10 +157,15 @@ async def set_revision_readable( self._wake_obligation(account_id, revision.reporting_obligation_id) def _commit_record_unlocked( - self, record: RecordT, *, notify: bool = True + self, record: RecordT, *, notify: bool = True, dirty: bool = True ) -> tuple[RecordT, bool]: + # Only the fenced verified finish may enqueue a readiness intent. An + # ordinary public outcome keeps the projection dirty work it has always + # produced; suppressing that would silently stall the status projector. stored, added = super()._commit_record_unlocked( - record, notify=notify and not isinstance(record, ReportingMaterializationRecord) + record, + notify=notify and not isinstance(record, ReportingMaterializationRecord), + dirty=dirty, ) if added: if isinstance(record, ReportingDestinationBinding): @@ -315,7 +320,7 @@ async def claim_materialization( + timedelta(days=context.binding.resource_retention_days), now, ) - self._commit_record_unlocked(delivery, notify=False) + self._commit_record_unlocked(delivery, notify=False, dirty=False) attempt = ReportingMaterializationAttempt( candidate.scope, revision.reporting_revision_id, @@ -323,7 +328,7 @@ async def claim_materialization( len(attempts) + 1, now, ) - self._commit_record_unlocked(attempt, notify=False) + self._commit_record_unlocked(attempt, notify=False, dirty=False) request = ReportingDestinationRequest.from_binding(context.binding, attempt, key) work = _Work( candidate.scope, @@ -520,7 +525,7 @@ async def finish_materialization( replace(verified.verification, verified_at=now) if verified is not None else None, public_failure(error) if error is not None else None, ) - stored, inserted = self._commit_record_unlocked(outcome, notify=False) + stored, inserted = self._commit_record_unlocked(outcome, notify=False, dirty=False) self._materializer_dirty(stored, context) if inserted and verified is not None and self._notification_state is not None: revision = next( diff --git a/src/adcp/reporting/materializer/pg.py b/src/adcp/reporting/materializer/pg.py index 0adab053c..762639b45 100644 --- a/src/adcp/reporting/materializer/pg.py +++ b/src/adcp/reporting/materializer/pg.py @@ -110,13 +110,16 @@ class PgReportingMaterializerStore(PgReportingReconciliationStore): _materializer_sample_after: tuple[str, str] | None = None async def _commit_record_on( - self, connection: Any, record: RecordT, *, notify: bool = True + self, connection: Any, record: RecordT, *, notify: bool = True, dirty: bool = True ) -> tuple[RecordT, bool]: - # Only the fenced verified finish may enqueue a readiness intent. + # Only the fenced verified finish may enqueue a readiness intent. An + # ordinary public outcome keeps the projection dirty work it has always + # produced; suppressing that would silently stall the status projector. return await super()._commit_record_on( connection, record, notify=notify and not isinstance(record, ReportingMaterializationRecord), + dirty=dirty, ) @materializer_errors @@ -423,11 +426,11 @@ async def _claim_account_on( + timedelta(days=context.binding.resource_retention_days), now, ) - await self._commit_record_on(connection, delivery, notify=False) + await self._commit_record_on(connection, delivery, notify=False, dirty=False) attempt = ReportingMaterializationAttempt( scope, revision.reporting_revision_id, "rpm_" + uuid4().hex, len(attempts) + 1, now ) - await self._commit_record_on(connection, attempt, notify=False) + await self._commit_record_on(connection, attempt, notify=False, dirty=False) request = ReportingDestinationRequest.from_binding(context.binding, attempt, key) inserted = await ( await connection.execute( @@ -729,7 +732,9 @@ async def finish_materialization( replace(verified.verification, verified_at=now) if verified is not None else None, public_failure(error) if error is not None else None, ) - stored, inserted = await self._commit_record_on(connection, outcome, notify=False) + stored, inserted = await self._commit_record_on( + connection, outcome, notify=False, dirty=False + ) await self._materializer_dirty_on(connection, stored, context) if inserted and verified is not None and self._notifications_enabled: revision = next( diff --git a/tests/conformance/reporting/_durable_materializer_support.py b/tests/conformance/reporting/_durable_materializer_support.py index 3ea9861b5..1f0c52b8d 100644 --- a/tests/conformance/reporting/_durable_materializer_support.py +++ b/tests/conformance/reporting/_durable_materializer_support.py @@ -101,6 +101,36 @@ async def queue(self): ).fetchall() return tuple(r[0] for r in events), tuple(r[0] for r in expansions) + async def dirty(self): + """Ordered status projection work, identical shape on both backends.""" + if self.pool is None: + state = self.store._notification_state + records = () if state is None else tuple(state.dirty) + else: + from adcp.reporting.ledger.notification_models import decode_dirty + + async with self.pool.connection() as c: + rows = await ( + await c.execute( + "SELECT snapshot FROM reporting_status_dirty ORDER BY account_id,sequence" + ) + ).fetchall() + records = tuple(decode_dirty(row[0]) for row in rows) + return tuple((r.reason, r.after) for r in records) + + async def ordinary_events(self): + """The old shared notification queue, which B2 must never publish into.""" + if self.pool is None: + state = self.store._notification_state + return ( + () if state is None else tuple(e.notification_type for e in state.events.values()) + ) + async with self.pool.connection() as c: + rows = await ( + await c.execute("SELECT notification_type FROM reporting_notification_events") + ).fetchall() + return tuple(row[0] for row in rows) + async def works(self): if self.pool is None: return tuple( diff --git a/tests/conformance/reporting/test_reporting_materializer_history.py b/tests/conformance/reporting/test_reporting_materializer_history.py index a6d1b08b8..f59cd4a4b 100644 --- a/tests/conformance/reporting/test_reporting_materializer_history.py +++ b/tests/conformance/reporting/test_reporting_materializer_history.py @@ -254,6 +254,53 @@ async def test_external_terminal_write_parks_owned_pending_history_without_a_hot assert await h.queue() == ((), ()) +async def test_public_terminal_outcome_keeps_projection_dirty_work_without_readiness(backend): + """Suppressing the fenced readiness event must not drop ordinary projection work. + + Adopters keep persisting materialization outcomes directly while the durable + service runs. Before this regression the materializer store silently stopped + marking the status scope dirty for those writes, so the projector never saw + the outcome and `get_reporting_status` stayed stale indefinitely. + """ + from adcp.reporting.ledger._delivery_state import change_id + from adcp.reporting.ledger.notification_models import ReportingStatusEvidence + + async with durable_harness(backend, notifications=True) as h: + case = await durable_case(h.store) + lease = await case.claim() + outcome = ReportingMaterializationRecord( + case.scope, + lease.attempt.reporting_revision_id, + lease.attempt.reporting_materialization_id, + "failed", + lease.attempt.created_at, + failure_code="WRITE_FAILED", + ) + before = await h.dirty() + stored, created = await h.store.commit_materialization(outcome) + assert created + added = (await h.dirty())[len(before) :] + assert [reason for reason, _ in added] == ["materialization"] + assert added[0][1] == ReportingStatusEvidence(stored.kind, change_id(stored)) + # The readiness intent still belongs only to a fenced verified finish. + assert await h.queue() == ((), ()) + assert "reporting.delivery_ready" not in await h.ordinary_events() + + +async def test_reserved_attempt_keeps_the_finish_transaction_as_the_only_dirty_work(backend): + """Reservation stays short: the terminal finish owns the projection work.""" + async with durable_harness(backend, notifications=True) as h: + case = await durable_case(h.store) + before = await h.dirty() + lease = await case.claim() + assert (await h.dirty())[len(before) :] == () + prepared, verified = await case.verified(lease) + assert ( + await h.store.finish_materialization(lease, prepared=prepared, verified=verified) + ).state == "verified" + assert [reason for reason, _ in (await h.dirty())[len(before) :]] == ["materialization"] + + async def test_restored_exact_component_can_explicitly_resume_parked_identity(backend): async with durable_harness(backend, notifications=True) as h: case = await durable_case(h.store) From d1d3b1330f13f777426b65aff46d6a3921fa591b Mon Sep 17 00:00:00 2001 From: Brian O'Kelley Date: Thu, 17 Sep 2026 05:54:57 +0000 Subject: [PATCH 5/8] fix(reporting): surface caller-argument errors instead of unknown effects `materializer_errors` closes every non-writer exception into `ReportingWriterError("RESOURCE_UNAVAILABLE", "same_identity", "unknown")` so a driver, hook or provider message can never become a diagnostic. It also caught the SDK's own argument validators, so `claim_materialization(lease_seconds=1)` and `read_materializer_boundaries(after=-1)` reported an unknown external effect: the adopter is told a destination write may have started, and loses the message naming the bound they violated. Raise those two fixed-message validations as `ReportingMaterializerUsageError`, a `ValueError` subclass, and re-raise it from the guard. Existing `except ValueError` handling is unchanged, arbitrary internal failures stay closed, and `_LeaseHeartbeat` still treats any renewal failure as a lost lease. `test_invalid_caller_arguments_stay_actionable_and_claim_no_external_effect` covers both bounds on memory and real PostgreSQL, asserting the actionable message, the absence of a writer failure record, an unchanged store image and no write effect. Co-Authored-By: Claude Opus 5 (1M context) --- docs/reporting-durable-materializer.md | 5 +++ src/adcp/reporting/materializer/_errors.py | 12 +++++- src/adcp/reporting/materializer/memory.py | 9 ++++- src/adcp/reporting/materializer/pg.py | 9 ++++- src/adcp/reporting/materializer/work.py | 3 +- .../test_reporting_materializer_durable.py | 38 +++++++++++++++++++ 6 files changed, 70 insertions(+), 6 deletions(-) diff --git a/docs/reporting-durable-materializer.md b/docs/reporting-durable-materializer.md index 311d21b0f..47ed9a712 100644 --- a/docs/reporting-durable-materializer.md +++ b/docs/reporting-durable-materializer.md @@ -194,6 +194,11 @@ failure must not change correctness. Observe only closed `ReportingMaterializerT state/reason and opaque IDs. Never log sessions, credentials, signed URLs, provider bodies, raw driver exceptions, or secret destination configuration. +An out-of-range `lease_seconds` or boundary position is a caller-argument +rejection, not a destination outcome: it raises a `ValueError` naming the bound +before any store or destination work. Reserve `ReportingWriterError` for real +failures, where `retry`/`effect` describe an actual external attempt. + ## Rolling compatibility envelope The executable gate is diff --git a/src/adcp/reporting/materializer/_errors.py b/src/adcp/reporting/materializer/_errors.py index d4f090d2d..e00cafb81 100644 --- a/src/adcp/reporting/materializer/_errors.py +++ b/src/adcp/reporting/materializer/_errors.py @@ -14,12 +14,22 @@ T = TypeVar("T") +class ReportingMaterializerUsageError(ValueError): + """A deterministic caller-argument rejection with a fixed SDK message. + + Raised only by the SDK's own argument validators, before any store or + destination work. It stays a ``ValueError`` so existing adopter handling is + unchanged, and the guard re-raises it instead of reporting an unknown + external effect that never happened. + """ + + def materializer_errors(method: Callable[P, Awaitable[T]]) -> Callable[P, Coroutine[Any, Any, T]]: @wraps(method) async def guarded(*args: P.args, **kwargs: P.kwargs) -> T: try: return await method(*args, **kwargs) - except (ReportingWriterError, LedgerConflictError): + except (ReportingWriterError, LedgerConflictError, ReportingMaterializerUsageError): raise except asyncio.CancelledError: canceled = True diff --git a/src/adcp/reporting/materializer/memory.py b/src/adcp/reporting/materializer/memory.py index 0431da862..dd922be0a 100644 --- a/src/adcp/reporting/materializer/memory.py +++ b/src/adcp/reporting/materializer/memory.py @@ -25,7 +25,10 @@ from adcp.reporting.ledger.models import ReportingConfiguration from adcp.reporting.ledger.notification_events import delivery_dirty, materialization_event from adcp.reporting.ledger.store import LedgerConflictError -from adcp.reporting.materializer._errors import materializer_errors +from adcp.reporting.materializer._errors import ( + ReportingMaterializerUsageError, + materializer_errors, +) from adcp.reporting.materializer.capture import ( MaterializerNotificationState, ReportingMaterializerBoundary, @@ -619,7 +622,9 @@ async def read_materializer_boundaries( self, *, caller: ReportingDeliveryPrincipal, after: int = 0, limit: int = 100 ) -> tuple[ReportingMaterializerBoundary, ...]: if type(after) is not int or after < 0 or type(limit) is not int or not 1 <= limit <= 100: - raise ValueError("materializer boundary reads require bounded positions") + raise ReportingMaterializerUsageError( + "materializer boundary reads require bounded positions" + ) async with self._lock: return tuple( b diff --git a/src/adcp/reporting/materializer/pg.py b/src/adcp/reporting/materializer/pg.py index 762639b45..9ee925d6f 100644 --- a/src/adcp/reporting/materializer/pg.py +++ b/src/adcp/reporting/materializer/pg.py @@ -28,7 +28,10 @@ _revision_from_row, ) from adcp.reporting.ledger.store import LedgerConflictError -from adcp.reporting.materializer._errors import materializer_errors +from adcp.reporting.materializer._errors import ( + ReportingMaterializerUsageError, + materializer_errors, +) from adcp.reporting.materializer.capture import ( ReportingMaterializerBoundary, decode_materializer_boundary, @@ -870,7 +873,9 @@ async def read_materializer_boundaries( self, *, caller: ReportingDeliveryPrincipal, after: int = 0, limit: int = 100 ) -> tuple[ReportingMaterializerBoundary, ...]: if type(after) is not int or after < 0 or type(limit) is not int or not 1 <= limit <= 100: - raise ValueError("materializer boundary reads require bounded positions") + raise ReportingMaterializerUsageError( + "materializer boundary reads require bounded positions" + ) async with self._connection() as connection, connection.transaction(): await self._lock_account(connection, caller.account_id) rows = await ( diff --git a/src/adcp/reporting/materializer/work.py b/src/adcp/reporting/materializer/work.py index 10593eb37..e2f306f15 100644 --- a/src/adcp/reporting/materializer/work.py +++ b/src/adcp/reporting/materializer/work.py @@ -25,6 +25,7 @@ ReportingObligationRecord, ReportingRevisionRecord, ) +from adcp.reporting.materializer._errors import ReportingMaterializerUsageError from adcp.reporting.materializer.contracts import ( ReportingDestinationRequest, ReportingPreparedRevision, @@ -264,7 +265,7 @@ async def finish_materialization( def validate_lease_seconds(value: int) -> None: if type(value) is not int or not 3 <= value <= 300: - raise ValueError("materializer leases require 3..300 seconds") + raise ReportingMaterializerUsageError("materializer leases require 3..300 seconds") def public_failure(error: ReportingWriterFailure) -> MaterializationFailure: diff --git a/tests/conformance/reporting/test_reporting_materializer_durable.py b/tests/conformance/reporting/test_reporting_materializer_durable.py index 899ced453..8f9ffae74 100644 --- a/tests/conformance/reporting/test_reporting_materializer_durable.py +++ b/tests/conformance/reporting/test_reporting_materializer_durable.py @@ -117,6 +117,44 @@ async def test_service_owns_reservation_verification_and_finish(backend): assert case.writer.write_effects == 1 +async def test_invalid_caller_arguments_stay_actionable_and_claim_no_external_effect(backend): + """A caller-argument rejection must not masquerade as an unknown external effect. + + The closed-error guard previously rewrote these deterministic ``ValueError``s + into ``RESOURCE_UNAVAILABLE``/``same_identity``/``unknown``, which tells an + adopter a destination write may have started and hides what to correct. + """ + async with durable_harness(backend) as h: + case = await durable_case(h.store) + before = await h.image() + calls = ( + ( + "materializer leases", + lambda: h.store.claim_materialization(keys=case.keys, lease_seconds=1), + ), + ( + "materializer leases", + lambda: h.store.claim_materialization(keys=case.keys, lease_seconds=301), + ), + ( + "materializer boundary reads", + lambda: h.store.read_materializer_boundaries(caller=case.scope.principal, after=-1), + ), + ( + "materializer boundary reads", + lambda: h.store.read_materializer_boundaries(caller=case.scope.principal, limit=0), + ), + ) + for expected, call in calls: + with pytest.raises(ValueError) as caught: + await call() + assert not isinstance(caught.value, ReportingWriterError) + assert str(caught.value).startswith(expected) + assert not hasattr(caught.value, "failure") + assert await h.image() == before + assert case.writer.write_effects == 0 + + async def test_unknown_effect_resumes_same_attempt_external_identity_and_rejects_old_fence(backend): async with durable_harness(backend) as h: case = await durable_case(h.store) From 8e18ca12b9a0c3750f80aa822058c02982ab3e52 Mon Sep 17 00:00:00 2001 From: Brian O'Kelley Date: Thu, 17 Sep 2026 06:13:50 +0000 Subject: [PATCH 6/8] docs(reporting): state the full extent of the closed advertisement MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit `advertised_notifications` matches the ledger by exact `type(...)`, so a materializer store is refused outright rather than only being denied the Managed/Reconciled claims. A deployment advertising Core `reporting.ledger_changed` today through `PgReportingReconciliationStore` gets a closed `notification_chain_unready` after switching to `PgReportingMaterializerStore`. That is correct and fail closed — B2.4 admits these stores only once it proves projection, component and mount readiness — but the rollout doc said only that the helpers "retain C's Managed/Reconciled veto", which reads as if Core advertisement survives. Say what actually happens, give adopters the two supported options, and note that polling is unaffected because this helper gates notification capabilities only. No production identity set is widened. `test_materializer_store_closes_core_advertisement_until_b24_admits_it` pins both halves on memory and real PostgreSQL: the reviewed store still advertises `reporting.ledger_changed`, and the materializer store raises. B2.4 has to change this deliberately. Co-Authored-By: Claude Opus 5 (1M context) --- docs/reporting-durable-materializer.md | 20 ++++++-- .../test_reporting_materializer_durable.py | 51 +++++++++++++++++++ 2 files changed, 67 insertions(+), 4 deletions(-) diff --git a/docs/reporting-durable-materializer.md b/docs/reporting-durable-materializer.md index 47ed9a712..358473c01 100644 --- a/docs/reporting-durable-materializer.md +++ b/docs/reporting-durable-materializer.md @@ -27,10 +27,22 @@ dependency. `materializer_ready()` validates storage prerequisites; it is **not** a tier or mount readiness certificate. There is no caller-supplied production-ready -switch. B1's development writer remains ineligible, and the current capability -helpers retain C's Managed/Reconciled veto. This unit offers a production -orchestration primitive, not an activated seller offering. Core-only deployments -keep their existing stores and do not need destination or receipt components. +switch. B1's development writer remains ineligible. This unit offers a +production orchestration primitive, not an activated seller offering. + +**Adopting a materializer store closes notification advertisement entirely, not +only Managed/Reconciled.** `advertised_notifications` matches the ledger by exact +`type(...)`, and neither `InMemoryReportingMaterializerStore` nor +`PgReportingMaterializerStore` is in that approved production identity set. A +deployment that advertises Core `reporting.ledger_changed` today through +`PgReportingReconciliationStore` will instead get a closed +`notification_chain_unready` after switching classes. That is deliberate and +fail closed: B2.4 admits these stores only once it proves projection, component +and mount readiness. Until then, run the durable service on a materializer store +and keep advertising from the reviewed store, or accept closed advertisement. +Polling is unaffected — this helper gates notification capabilities only. +Core-only deployments keep their existing stores and do not need destination or +receipt components. Import the optional `ReportingMaterializerStore` protocol, service, lease, turn and boundary types from `adcp.reporting.materializer`. The PostgreSQL store diff --git a/tests/conformance/reporting/test_reporting_materializer_durable.py b/tests/conformance/reporting/test_reporting_materializer_durable.py index 8f9ffae74..27fdcc1ef 100644 --- a/tests/conformance/reporting/test_reporting_materializer_durable.py +++ b/tests/conformance/reporting/test_reporting_materializer_durable.py @@ -69,6 +69,57 @@ async def get_active(self, **kwargs): assert len((await h.queue())[0]) == 1 and (await h.queue())[1] == ("quarantined",) +async def test_materializer_store_closes_core_advertisement_until_b24_admits_it(backend): + """Pin the full extent of the closed advertisement so B2.4 must act deliberately. + + `advertised_notifications` matches the ledger by exact `type(...)`, so both + materializer stores lose Core `reporting.ledger_changed` too, not only the + Managed/Reconciled claims. Fail closed is correct here, but it is broader + than a tier veto and an adopter switching store classes must be told. + """ + from adcp.reporting.ledger.notification_models import ReportingNotificationError + from adcp.reporting.outbox import ( + InMemoryReportingOutbox, + PgReportingOutbox, + ReportingEnvelopeCipher, + ReportingNotificationWorker, + ) + + class NoSubscriptions: + async def list_active(self, **kwargs): + return () + + async def get_active(self, **kwargs): + return None + + async def advertise(store, outbox): + return await ReportingNotificationWorker( + outbox=outbox, + subscriptions=NoSubscriptions(), + cipher=ReportingEnvelopeCipher(b"e" * 32), + ).advertised_notifications(store, account_id="acct_a", ready_scope=None) + + async with durable_harness(backend, notifications=True) as h: + if h.pool is None: + from adcp.reporting.ledger.delivery import InMemoryReportingReconciliationStore + + reviewed = InMemoryReportingReconciliationStore(notifications=True) + reviewed_outbox = InMemoryReportingOutbox(reviewed) + outbox = InMemoryReportingOutbox(h.store) + else: + from adcp.reporting.ledger.delivery_pg import PgReportingReconciliationStore + + reviewed = PgReportingReconciliationStore(pool=h.pool, notifications=True) + reviewed_outbox = PgReportingOutbox(pool=h.pool) + outbox = PgReportingOutbox(pool=h.pool) + # The reviewed store still advertises the Core notification. + assert (await advertise(reviewed, reviewed_outbox))["ledger_notification"] == ( + "reporting.ledger_changed" + ) + with pytest.raises(ReportingNotificationError, match="notification_chain_unready"): + await advertise(h.store, outbox) + + @pytest.mark.parametrize("notifications", [False, True]) @pytest.mark.parametrize("count", [0, 501]) async def test_verified_finish_captures_private_inputs_acks_and_quarantines_exact_event( From 2fed6422090ba65bd5dd2a8ed3fde0ac561b8579 Mon Sep 17 00:00:00 2001 From: Brian O'Kelley Date: Thu, 24 Sep 2026 14:30:59 +0000 Subject: [PATCH 7/8] fix(reporting): qualify materializer against integrated baselines --- .github/workflows/ci.yml | 17 +++--- docs/reporting-durable-materializer.md | 28 +++++----- examples/reporting_durable_materializer.py | 1 + src/adcp/reporting/materializer/capture.py | 1 + src/adcp/reporting/materializer/service.py | 1 + .../reporting/_materializer_frozen.py | 11 ++-- .../reporting/_materializer_installed.py | 3 +- .../reporting/_materializer_process.py | 4 +- .../test_reporting_materializer_durable.py | 53 +++++++++++-------- .../test_reporting_materializer_history.py | 53 ++++++++++++------- ...est_reporting_materializer_installed_pg.py | 7 +-- .../test_reporting_materializer_migration.py | 21 +++++--- .../test_reporting_materializer_process.py | 21 +++++--- .../test_reporting_materializer_rolling.py | 18 ++++--- .../test_reporting_materializer_service.py | 36 ++++++++----- ...est_reporting_materializer_transactions.py | 9 ++-- .../test_reporting_notification_packaging.py | 3 +- 17 files changed, 178 insertions(+), 109 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index fd7e48b5a..1cddb242f 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -211,17 +211,19 @@ jobs: runs-on: ubuntu-latest timeout-minutes: 5 permissions: {} - needs: [pg-conformance, pg-reporting-status] + needs: [pg-conformance, pg-reporting-status, pg-reporting-materializer] if: ${{ always() }} steps: - name: Require every Postgres conformance lane env: PG_RESULT: ${{ needs.pg-conformance.result }} STATUS_RESULT: ${{ needs.pg-reporting-status.result }} + MATERIALIZER_RESULT: ${{ needs.pg-reporting-materializer.result }} run: | - if [ "$PG_RESULT" != "success" ] || [ "$STATUS_RESULT" != "success" ]; then + if [ "$PG_RESULT" != "success" ] || [ "$STATUS_RESULT" != "success" ] || [ "$MATERIALIZER_RESULT" != "success" ]; then echo "Postgres conformance matrix result: $PG_RESULT" echo "Reporting status conformance result: $STATUS_RESULT" + echo "Durable materializer conformance result: $MATERIALIZER_RESULT" exit 1 fi echo "All Postgres conformance lanes passed" @@ -282,13 +284,14 @@ jobs: # cells approach twenty minutes locally; allow runner variance and retain # five minutes outside the suite for setup and cleanup. timeout-minutes: 35 + permissions: + contents: read services: postgres: image: postgres:16 env: POSTGRES_HOST_AUTH_METHOD: trust POSTGRES_DB: adcp_materializer_test - POSTGRES_INITDB_ARGS: "--encoding=UTF8 --lc-collate=C --lc-ctype=C" ports: - 5432:5432 options: >- @@ -305,10 +308,10 @@ jobs: 3e76aa54623529a3dda01cd690b8a5c287c75641 \ 3c405a21f978ed9d3208611bb4a7a8434a056933 \ 037de4ac822ecefb2f95d32c15c297fb4c45d683 \ - 21bf443e7d850d1800ec8a6f2e4abec1c8f85541 \ - 198d50e61c74fb82aedbf2c77e06a0e200b91db6 \ - ea150fabd5ad90e3abf93f89729d2919f1c61798 \ - 1c91311ec28d25506d5db43f59d0c34936ecb8f7 + 17ee407ae3978c8a2bb54437287afbf9dafb8130 \ + 0f34c666ac1961e9832fce43ef0ef6937b3c1dde \ + 967b6e286301d7e5d089aea6fdbb90bea8ee5a16 \ + 5487f2bdef23c5102118b305be9e868228f6ce61 - uses: actions/setup-python@v6 id: materializer-python310 with: diff --git a/docs/reporting-durable-materializer.md b/docs/reporting-durable-materializer.md index 358473c01..e7437e36e 100644 --- a/docs/reporting-durable-materializer.md +++ b/docs/reporting-durable-materializer.md @@ -2,7 +2,7 @@ **B2.1 of 4 within B2 of B1/B2. Refs #1167.** This unit builds on the independently approved B1 commit -`1c91311ec28d25506d5db43f59d0c34936ecb8f7`. It supplies durable reservation, +`5487f2bdef23c5102118b305be9e868228f6ce61`. It supplies durable reservation, verified destination I/O, recovery, and one atomic finish transaction. It does not activate a complete Managed Delivery or Reconciled Billing offering. @@ -226,14 +226,14 @@ and performs permitted ordinary writes both before and after B2 installation. | beta.15 | `3e76aa54623529a3dda01cd690b8a5c287c75641` | No notification API | | #1167A records | `3c405a21f978ed9d3208611bb4a7a8434a056933` | No notification API | | Foundation integration | `037de4ac822ecefb2f95d32c15c297fb4c45d683` | No notification API | -| #1168A outbox | `21bf443e7d850d1800ec8a6f2e4abec1c8f85541` | Aggregate readiness closed on both | -| #1168B activity | `198d50e61c74fb82aedbf2c77e06a0e200b91db6` | Required-object readiness valid on both | -| #1168C status | `ea150fabd5ad90e3abf93f89729d2919f1c61798` | Required-object readiness valid on both | -| #1167B1 | `1c91311ec28d25506d5db43f59d0c34936ecb8f7` | Required-object readiness valid on both | - -The historical positive-readiness fixture requires PostgreSQL 16, **UTF8 with -`LC_COLLATE=C` and `LC_CTYPE=C`**. Frozen A is ready on its native schema under -that precondition. Reviewed C's additive triggers already close A's aggregate +| #1168A outbox | `17ee407ae3978c8a2bb54437287afbf9dafb8130` | Aggregate readiness closed on both | +| #1168B activity | `0f34c666ac1961e9832fce43ef0ef6937b3c1dde` | Required-object readiness valid on both | +| #1168C status | `967b6e286301d7e5d089aea6fdbb90bea8ee5a16` | Required-object readiness valid on both | +| #1167B1 | `5487f2bdef23c5102118b305be9e868228f6ce61` | Required-object readiness valid on both | + +The positive-readiness fixture requires PostgreSQL 16 and UTF8. The integrated +A/B/C/B1 pins above preserve the catalog-portability and rc.6 corrections now on +main; no C-only database locale is imposed. Frozen A is ready on its native schema. Reviewed C's additive triggers already close A's aggregate digest on the approved C/B1 baseline. B2 must preserve that exact classification; it is not a new waiver. A's permitted default-off Core reads/writes and existing ordinary notification worker remain functional. The newer B/C required-object @@ -250,10 +250,12 @@ caller's materializations. Actual old ordinary/status workers consume positive control events but cannot claim/mutate B2 event/expansion work, materializer work, or captured boundaries/heads. -Non-C historical aggregate portability is a separate probe with a different -precondition; it must not be presented as this positive readiness gate or as -shared-database lock contention. New B2 failures on supported old operations -are regressions, regardless of the inherited frozen-A aggregate limitation. +These controls do not qualify pre-`17ee407a` A, pre-`0f34c666` B, +pre-`967b6e28` C or pre-`5487f2bd` B1 binaries. In particular, the earlier +`21bf443e` A catalog digests depended on database collation. This rolling scope +limit must accompany release notes; it does not retroactively qualify the older +snapshots. New B2 failures on supported old operations are regressions, regardless +of the inherited frozen-A aggregate limitation. Run the historical comparison with its provenance output preserved: diff --git a/examples/reporting_durable_materializer.py b/examples/reporting_durable_materializer.py index c429b85f8..ebbf1e0bf 100644 --- a/examples/reporting_durable_materializer.py +++ b/examples/reporting_durable_materializer.py @@ -58,4 +58,5 @@ async def run_materializer(service: ReportingMaterializerService, stop: asyncio. try: await asyncio.wait_for(stop.wait(), timeout=1) except asyncio.TimeoutError: + # The poll interval elapsed; check for work or shutdown again. pass diff --git a/src/adcp/reporting/materializer/capture.py b/src/adcp/reporting/materializer/capture.py index e36012310..f2f1aee89 100644 --- a/src/adcp/reporting/materializer/capture.py +++ b/src/adcp/reporting/materializer/capture.py @@ -98,6 +98,7 @@ def decode_materializer_boundary(value: dict[str, Any]) -> ReportingMaterializer tuple(decode_record(r) for r in value["reconciliation"]), ) except (ValueError, TypeError, KeyError, ValidationError): + # Convert malformed persisted input to the closed protocol error below. pass if result is None or result.to_storage() != value: raise ReportingNotificationError("materializer_boundary_invalid") diff --git a/src/adcp/reporting/materializer/service.py b/src/adcp/reporting/materializer/service.py index 06c8d9ebb..11a564f7a 100644 --- a/src/adcp/reporting/materializer/service.py +++ b/src/adcp/reporting/materializer/service.py @@ -46,6 +46,7 @@ async def run(self) -> None: await asyncio.wait_for(self.stopped.wait(), self.seconds / 3) return except asyncio.TimeoutError: + # The heartbeat interval elapsed; renew the owned lease below. pass held = False try: diff --git a/tests/conformance/reporting/_materializer_frozen.py b/tests/conformance/reporting/_materializer_frozen.py index 65b76454a..ce7d0106c 100644 --- a/tests/conformance/reporting/_materializer_frozen.py +++ b/tests/conformance/reporting/_materializer_frozen.py @@ -43,7 +43,7 @@ async def main(settings): " FROM pg_database WHERE datname=current_database()" ) ).fetchone() - assert 160000 <= database[0] < 170000 and database[1:] == ("UTF8", "C", "C") + assert 160000 <= database[0] < 170000 and database[1] == "UTF8" store = store_type(pool=pool) if settings["action"] == "install": await store.create_schema() @@ -251,10 +251,13 @@ async def claim_expansion(self, **kwargs): cipher=ReportingEnvelopeCipher(b"e" * 32), ) for _ in events: - assert await worker.expand_one(account_id="acct_a") + materializer_operation_3 = await worker.expand_one(account_id="acct_a") + assert materializer_operation_3 assert seen == identities.keys() - assert not await worker.deliver_one(account_id="acct_a") - assert not await worker.expand_one(account_id="acct_a") + materializer_operation_1 = await worker.deliver_one(account_id="acct_a") + assert not materializer_operation_1 + materializer_operation_2 = await worker.expand_one(account_id="acct_a") + assert not materializer_operation_2 assert await outbox.list_events(account_id="acct_a") == events workers[name] = len(seen) event_identities[name] = [ diff --git a/tests/conformance/reporting/_materializer_installed.py b/tests/conformance/reporting/_materializer_installed.py index a7c7d62ed..c56349360 100644 --- a/tests/conformance/reporting/_materializer_installed.py +++ b/tests/conformance/reporting/_materializer_installed.py @@ -205,7 +205,8 @@ async def main(): ReportingDestinationIO(destination.registry, destination.resolver), destination.writer, ) - assert (await service.run_once()).state == "verified" + materializer_operation_1 = await service.run_once() + assert (materializer_operation_1).state == "verified" boundaries = await durable.read_materializer_boundaries(caller=scope.principal) assert len(boundaries) == boundaries[0].sequence == boundaries[0].account_sequence == 1 assert durable._materializer_outbox is None diff --git a/tests/conformance/reporting/_materializer_process.py b/tests/conformance/reporting/_materializer_process.py index cc62fd3bf..a6267a225 100644 --- a/tests/conformance/reporting/_materializer_process.py +++ b/tests/conformance/reporting/_materializer_process.py @@ -63,6 +63,7 @@ async def main(): try: await store.materializer_ready() except LedgerConflictError: + # An empty schema must reject readiness before installation. pass else: raise AssertionError("empty schema must be unready") @@ -75,7 +76,8 @@ async def main(): async def hit(point): if point == settings.get("pause"): emit(point) - assert (await read())["continue"] + materializer_operation_1 = await read() + assert (materializer_operation_1)["continue"] class Connection(AsyncConnection): async def execute(self, query, params=None, **kwargs): diff --git a/tests/conformance/reporting/test_reporting_materializer_durable.py b/tests/conformance/reporting/test_reporting_materializer_durable.py index 27fdcc1ef..153c252d8 100644 --- a/tests/conformance/reporting/test_reporting_materializer_durable.py +++ b/tests/conformance/reporting/test_reporting_materializer_durable.py @@ -27,7 +27,8 @@ async def test_core_records_do_not_create_managed_work_capture_or_readiness(back await h.store.put_configuration(config) await h.store.commit_obligation(obligation) await h.store.commit_revision(revision, rows) - assert (await h.store.claim_materialization(keys=())).state == "idle" + materializer_operation_1 = await h.store.claim_materialization(keys=()) + assert (materializer_operation_1).state == "idle" assert not await h.works() and await h.queue() == ((), ()) @@ -49,7 +50,8 @@ async def get_active(self, **kwargs): async with durable_harness(backend, notifications=True) as h: case = await durable_case(h.store) - assert (await case.service().run_once()).state == "verified" + materializer_operation_2 = await case.service().run_once() + assert (materializer_operation_2).state == "verified" outbox = ( PgReportingOutbox(pool=h.pool) if h.pool is not None @@ -113,9 +115,8 @@ async def advertise(store, outbox): reviewed_outbox = PgReportingOutbox(pool=h.pool) outbox = PgReportingOutbox(pool=h.pool) # The reviewed store still advertises the Core notification. - assert (await advertise(reviewed, reviewed_outbox))["ledger_notification"] == ( - "reporting.ledger_changed" - ) + materializer_operation_3 = await advertise(reviewed, reviewed_outbox) + assert (materializer_operation_3)["ledger_notification"] == ("reporting.ledger_changed") with pytest.raises(ReportingNotificationError, match="notification_chain_unready"): await advertise(h.store, outbox) @@ -152,10 +153,10 @@ async def test_verified_finish_captures_private_inputs_acks_and_quarantines_exac assert len(events) == int(notifications) assert expansions == (("quarantined",) if notifications else ()) assert (await h.works())[0][1] == "acked" - assert ( - await h.store.finish_materialization(lease, prepared=prepared, verified=evidence) - == result + materializer_operation_4 = await h.store.finish_materialization( + lease, prepared=prepared, verified=evidence ) + assert materializer_operation_4 == result assert await h.queue() == (events, expansions) assert await h.store.read_materializer_boundaries(caller=case.scope.principal) == boundaries @@ -163,8 +164,10 @@ async def test_verified_finish_captures_private_inputs_acks_and_quarantines_exac async def test_service_owns_reservation_verification_and_finish(backend): async with durable_harness(backend) as h: case = await durable_case(h.store) - assert (await case.service().run_once()).state == "verified" - assert (await case.service().run_once()).state in {"idle", "discovered"} + materializer_operation_5 = await case.service().run_once() + assert (materializer_operation_5).state == "verified" + materializer_operation_6 = await case.service().run_once() + assert (materializer_operation_6).state in {"idle", "discovered"} assert case.writer.write_effects == 1 @@ -221,14 +224,16 @@ async def test_unknown_effect_resumes_same_attempt_external_identity_and_rejects assert second.attempt == first.attempt assert second.request.external_id == first.request.external_id assert second.token != first.token - assert ( - await h.store.finish_materialization(first, prepared=prepared, verified=evidence) - ).state == "pending" + materializer_operation_7 = await h.store.finish_materialization( + first, prepared=prepared, verified=evidence + ) + assert (materializer_operation_7).state == "pending" prepared, evidence = await case.verified(second) assert case.writer.write_effects == 1 - assert ( - await h.store.finish_materialization(second, prepared=prepared, verified=evidence) - ).state == "verified" + materializer_operation_8 = await h.store.finish_materialization( + second, prepared=prepared, verified=evidence + ) + assert (materializer_operation_8).state == "verified" @pytest.mark.parametrize("change", ["official", "unreadable", "deactivated"]) @@ -277,7 +282,8 @@ async def test_known_service_failure_allows_next_attempt_but_pending_does_not(ba async with durable_harness(backend) as h: case = await durable_case(h.store) first = await case.claim() - assert not isinstance(await case.claim(), ReportingMaterializerLease) + materializer_operation_9 = await case.claim() + assert not isinstance(materializer_operation_9, ReportingMaterializerLease) await h.store.finish_materialization( first, error=ReportingWriterFailure("WRITE_FAILED", "new_attempt", "not_started") ) @@ -327,9 +333,10 @@ async def test_prior_lease_readback_cannot_finish_a_new_fence_in_the_same_proces await h.store.finish_materialization(resumed, prepared=prepared, verified=proof) assert not await case.outcomes() and await h.queue() == ((), ()) prepared, proof = await case.verified(resumed) - assert ( - await h.store.finish_materialization(resumed, prepared=prepared, verified=proof) - ).state == "verified" + materializer_operation_10 = await h.store.finish_materialization( + resumed, prepared=prepared, verified=proof + ) + assert (materializer_operation_10).state == "verified" assert case.writer.write_effects == 1 @@ -419,6 +426,8 @@ async def test_legacy_pending_requires_explicit_exact_external_history_import(ba async def test_finality_and_late_delivery_binding(backend, required, finality, expected): async with durable_harness(backend) as h: case = await durable_case(h.store, required=required, finality=finality, binding=False) - assert (await case.service().run_once()).state == "idle" + materializer_operation_11 = await case.service().run_once() + assert (materializer_operation_11).state == "idle" await h.store.put_destination_binding(case.binding) - assert (await case.service().run_once()).state == expected + materializer_operation_12 = await case.service().run_once() + assert (materializer_operation_12).state == expected diff --git a/tests/conformance/reporting/test_reporting_materializer_history.py b/tests/conformance/reporting/test_reporting_materializer_history.py index f59cd4a4b..5c9ae7716 100644 --- a/tests/conformance/reporting/test_reporting_materializer_history.py +++ b/tests/conformance/reporting/test_reporting_materializer_history.py @@ -30,9 +30,10 @@ async def test_exact_source_replay_does_not_dirty_an_inflight_generation(backend readable=True, ) prepared, evidence = await case.verified(lease) - assert ( - await h.store.finish_materialization(lease, prepared=prepared, verified=evidence) - ).state == "verified" + materializer_operation_1 = await h.store.finish_materialization( + lease, prepared=prepared, verified=evidence + ) + assert (materializer_operation_1).state == "verified" async def test_selected_official_never_uses_snapshot_artifact_and_preserves_both_histories(backend): @@ -48,7 +49,8 @@ async def test_selected_official_never_uses_snapshot_artifact_and_preserves_both reporting_revision_id=official.reporting_revision_id, readable=False, ) - assert (await case.claim()).reason == "revision_unreadable" + materializer_operation_2 = await case.claim() + assert (materializer_operation_2).reason == "revision_unreadable" assert await case.outcomes() == original await h.store.set_revision_readable( account_id="acct_a", reporting_revision_id=official.reporting_revision_id, readable=True @@ -108,8 +110,10 @@ async def read(self, connection, scope): method = "_materializer_context_on" with monkeypatch.context() as patch: patch.setattr(cls, method, read) - assert (await case.claim()).reason == "history_corrupt" - assert (await case.claim()).state == "idle" + materializer_operation_14 = await case.claim() + assert (materializer_operation_14).reason == "history_corrupt" + materializer_operation_15 = await case.claim() + assert (materializer_operation_15).state == "idle" assert not await h.works() and await h.queue() == ((), ()) @@ -117,10 +121,12 @@ async def test_inactive_late_binding_waits_for_activation_without_allocating_his async with durable_harness(backend) as h: case = await durable_case(h.store, active=False, binding=False) await h.store.put_destination_binding(case.binding) - assert (await case.claim()).reason == "inactive" + materializer_operation_3 = await case.claim() + assert (materializer_operation_3).reason == "inactive" assert not await h.works() await h.store.put_configuration(replace(case.config, deactivated_at=None)) - assert (await case.claim()).attempt.attempt == 1 + materializer_operation_4 = await case.claim() + assert (materializer_operation_4).attempt.attempt == 1 async def test_private_consumer_boundaries_keep_a_durable_account_order(backend): @@ -160,7 +166,8 @@ async def test_private_consumer_boundaries_keep_a_durable_account_order(backend) async def test_rejected_consumer_receipt_does_not_dirty_materializer_or_allocate_retry(backend): async with durable_harness(backend, notifications=True) as h: case = await durable_case(h.store, reconciliation_mode="consumer_receipt") - assert (await case.service().run_once()).state == "verified" + materializer_operation_5 = await case.service().run_once() + assert (materializer_operation_5).state == "verified" outcome = (await case.outcomes())[0] before = await h.works() await h.store.record_revision_receipt( @@ -177,7 +184,8 @@ async def test_rejected_consumer_receipt_does_not_dirty_materializer_or_allocate rejection_codes=("ROW_COUNT_MISMATCH",), ) ) - assert (await case.claim()).state in {"idle", "discovered"} + materializer_operation_6 = await case.claim() + assert (materializer_operation_6).state in {"idle", "discovered"} assert await h.works() == before @@ -211,7 +219,8 @@ async def test_publication_does_not_bypass_legacy_pending_on_an_older_revision(b ) await h.store.commit_materialization_attempt(legacy) official = await case.publish() - assert (await case.claim()).reason == "legacy_pending" + materializer_operation_7 = await case.claim() + assert (materializer_operation_7).reason == "legacy_pending" assert not await h.works() identity = ReportingDestinationRequest.from_binding( case.binding, legacy, case.verifier.key @@ -224,7 +233,8 @@ async def test_publication_does_not_bypass_legacy_pending_on_an_older_revision(b ) # Exact recovery concludes the obsolete attempt safely before the # current official gets its own first attempt; no old artifact is reused. - assert (await case.service().run_once()).state == "failed" + materializer_operation_8 = await case.service().run_once() + assert (materializer_operation_8).state == "failed" assert case.writer.write_effects == 0 current = await case.claim() assert current.attempt.reporting_revision_id == official.reporting_revision_id @@ -247,8 +257,10 @@ async def test_external_terminal_write_parks_owned_pending_history_without_a_hot ) ) await h.expire() - assert (await case.claim()).reason == "history_corrupt" - assert (await case.claim()).state == "idle" + materializer_operation_9 = await case.claim() + assert (materializer_operation_9).reason == "history_corrupt" + materializer_operation_10 = await case.claim() + assert (materializer_operation_10).state == "idle" assert len(await h.works()) == 1 assert not await h.store.read_materializer_boundaries(caller=case.scope.principal) assert await h.queue() == ((), ()) @@ -295,9 +307,10 @@ async def test_reserved_attempt_keeps_the_finish_transaction_as_the_only_dirty_w lease = await case.claim() assert (await h.dirty())[len(before) :] == () prepared, verified = await case.verified(lease) - assert ( - await h.store.finish_materialization(lease, prepared=prepared, verified=verified) - ).state == "verified" + materializer_operation_11 = await h.store.finish_materialization( + lease, prepared=prepared, verified=verified + ) + assert (materializer_operation_11).state == "verified" assert [reason for reason, _ in (await h.dirty())[len(before) :]] == ["materialization"] @@ -306,8 +319,10 @@ async def test_restored_exact_component_can_explicitly_resume_parked_identity(ba case = await durable_case(h.store) lease = await case.claim() await h.expire() - assert (await h.store.claim_materialization(keys=())).reason == "component_unavailable" - assert (await case.claim()).state == "idle" + materializer_operation_12 = await h.store.claim_materialization(keys=()) + assert (materializer_operation_12).reason == "component_unavailable" + materializer_operation_13 = await case.claim() + assert (materializer_operation_13).state == "idle" await h.store.import_pending_materialization( scope=case.scope, reporting_materialization_id=lease.attempt.reporting_materialization_id, diff --git a/tests/conformance/reporting/test_reporting_materializer_installed_pg.py b/tests/conformance/reporting/test_reporting_materializer_installed_pg.py index 4a4c12211..86e4d7e49 100644 --- a/tests/conformance/reporting/test_reporting_materializer_installed_pg.py +++ b/tests/conformance/reporting/test_reporting_materializer_installed_pg.py @@ -13,7 +13,7 @@ from adcp.reporting.materializer import PgReportingMaterializerStore from ._durable_materializer_support import DurableHarness, durable_case -from ._generation_support import assert_c_collated_rolling_database, isolated_reporting_pool +from ._generation_support import isolated_reporting_pool, require_rolling_database from .test_reporting_materializer_packaging import ROOT, b1_wheels, built_distribution, run_step from .test_reporting_materializer_process import worker @@ -24,7 +24,7 @@ @pytest.fixture(scope="module", params=["vcs", "sdist"]) def installed_materializer(request): - assert_c_collated_rolling_database() + require_rolling_database() b1_wheels = request.getfixturevalue("b1_wheels") path, wheels, _ = b1_wheels interpreter = os.environ.get("ADCP_PYTHON310") or sys.executable @@ -105,7 +105,8 @@ async def test_installed_sql_and_process_restart_preserve_original_effect( async with worker(h, case, tmp_path, **options) as child: done = await child.event("done") assert done["state"] == "verified" and done["origins"] == result["origins"] - assert await asyncio.wait_for(child.process.wait(), 5) == 0 + materializer_operation_1 = await asyncio.wait_for(child.process.wait(), 5) + assert materializer_operation_1 == 0 after = await h.works() assert len(after) == 1 and after[0][0] == before[0][0] and after[0][1] == "acked" assert len(tuple(tmp_path.glob("rwm_*"))) == 1 diff --git a/tests/conformance/reporting/test_reporting_materializer_migration.py b/tests/conformance/reporting/test_reporting_materializer_migration.py index 0c96910ce..a76636d2e 100644 --- a/tests/conformance/reporting/test_reporting_materializer_migration.py +++ b/tests/conformance/reporting/test_reporting_materializer_migration.py @@ -60,7 +60,8 @@ async def test_populated_repeated_and_concurrent_install_preserves_all_old_objec ) await validate_schema(c, activity=True) case.store = new - assert (await case.service().run_once()).state == "verified" + materializer_operation_1 = await case.service().run_once() + assert (materializer_operation_1).state == "verified" # Discovery is persisted, and reinstall cannot invalidate leased generations. before = await new.read_reconciliation_snapshot(caller=case.scope.principal) await new.create_schema() @@ -104,11 +105,15 @@ async def install(): "DROP INDEX reporting_materializer_work_due", "ALTER TABLE reporting_materializer_work DISABLE TRIGGER reporting_materializer_guard", "ALTER TABLE reporting_materializer_work ALTER COLUMN generation DROP NOT NULL", - "ALTER TABLE reporting_materializer_notification_expansions" - " ALTER COLUMN state SET DEFAULT 'pending'", + ( + "ALTER TABLE reporting_materializer_notification_expansions" + " ALTER COLUMN state SET DEFAULT 'pending'" + ), "DROP TABLE reporting_materializer_status_boundaries CASCADE", - "ALTER TABLE reporting_reconciliation_records" - " DISABLE TRIGGER reporting_reconciliation_guard", + ( + "ALTER TABLE reporting_reconciliation_records" + " DISABLE TRIGGER reporting_reconciliation_guard" + ), ], ) @pytest.mark.parametrize("notifications", [False, True]) @@ -220,7 +225,8 @@ async def test_bounded_sampling_walks_past_a_full_page_of_busy_account_locks(): async with h.pool.connection() as connection, connection.transaction(): for account in busy: await h.store._lock_account(connection, account) - assert (await available.claim()).state == "idle" + materializer_operation_3 = await available.claim() + assert (materializer_operation_3).state == "idle" selected = await available.claim() assert selected.scope.principal.account_id == "zz-available" assert len(await h.works()) == 1 @@ -232,7 +238,8 @@ async def test_pre_activation_event_never_promotes_and_old_outbox_cannot_claim_i async with durable_harness("postgres", notifications=True) as h: case = await durable_case(h.store) - assert (await case.service().run_once()).state == "verified" + materializer_operation_2 = await case.service().run_once() + assert (materializer_operation_2).state == "verified" before = await h.queue() # Remove only the ordinary Core event's pending expansion via its own # worker protocol. Materializer records remain in their isolated queue. diff --git a/tests/conformance/reporting/test_reporting_materializer_process.py b/tests/conformance/reporting/test_reporting_materializer_process.py index 80ea527b8..ceb451f94 100644 --- a/tests/conformance/reporting/test_reporting_materializer_process.py +++ b/tests/conformance/reporting/test_reporting_materializer_process.py @@ -108,7 +108,8 @@ async def test_real_process_crash_resume_preserves_external_identity_and_atomic_ assert result["state"] in ( {"idle", "parked", "discovered"} if committed else {"verified"} ) - assert await asyncio.wait_for(child.process.wait(), 5) == 0 + materializer_operation_1 = await asyncio.wait_for(child.process.wait(), 5) + assert materializer_operation_1 == 0 after = await h.works() assert len(after) == 1 and after[0][0] == work_before[0][0] and after[0][1] == "acked" assert len(tuple(tmp_path.glob("rwm_*"))) == 1 @@ -130,8 +131,10 @@ async def test_crash_after_actual_enabled_logical_enqueue_rolls_back_before_rest assert not await h.store.read_materializer_boundaries(caller=case.scope.principal) await h.expire() async with worker(h, case, tmp_path) as child: - assert (await child.event("done"))["state"] == "verified" - assert await asyncio.wait_for(child.process.wait(), 5) == 0 + materializer_operation_2 = await child.event("done") + assert (materializer_operation_2)["state"] == "verified" + materializer_operation_3 = await asyncio.wait_for(child.process.wait(), 5) + assert materializer_operation_3 == 0 assert len(tuple(tmp_path.glob("rwm_*"))) == 1 assert len((await h.queue())[0]) == 1 @@ -142,10 +145,14 @@ async def test_two_real_workers_reserve_once_without_global_candidate_locks(tmp_ async with worker(h, case, tmp_path, pause="reserved") as first: await first.event("reserved") async with worker(h, case, tmp_path) as second: - assert (await second.event("done"))["state"] in {"idle", "discovered"} - assert await asyncio.wait_for(second.process.wait(), 5) == 0 + materializer_operation_6 = await second.event("done") + assert (materializer_operation_6)["state"] in {"idle", "discovered"} + materializer_operation_7 = await asyncio.wait_for(second.process.wait(), 5) + assert materializer_operation_7 == 0 assert not tuple(tmp_path.glob("rwm_*")) await first.send({"continue": True}) - assert (await first.event("done"))["state"] == "verified" - assert await asyncio.wait_for(first.process.wait(), 5) == 0 + materializer_operation_4 = await first.event("done") + assert (materializer_operation_4)["state"] == "verified" + materializer_operation_5 = await asyncio.wait_for(first.process.wait(), 5) + assert materializer_operation_5 == 0 assert len(await h.works()) == 1 diff --git a/tests/conformance/reporting/test_reporting_materializer_rolling.py b/tests/conformance/reporting/test_reporting_materializer_rolling.py index 6f09b41a3..46eaf7956 100644 --- a/tests/conformance/reporting/test_reporting_materializer_rolling.py +++ b/tests/conformance/reporting/test_reporting_materializer_rolling.py @@ -15,22 +15,22 @@ from adcp.reporting.outbox._schema import schema_objects from ._durable_materializer_support import DurableHarness, durable_case -from ._generation_support import assert_c_collated_rolling_database, isolated_reporting_pool +from ._generation_support import isolated_reporting_pool, require_rolling_database from .test_reporting_notification_packaging import ROOT, run_step ARTIFACTS = { "beta15": "3e76aa54623529a3dda01cd690b8a5c287c75641", "records": "3c405a21f978ed9d3208611bb4a7a8434a056933", "integration": "037de4ac822ecefb2f95d32c15c297fb4c45d683", - "a": "21bf443e7d850d1800ec8a6f2e4abec1c8f85541", - "b": "198d50e61c74fb82aedbf2c77e06a0e200b91db6", - "c": "ea150fabd5ad90e3abf93f89729d2919f1c61798", - "b1": "1c91311ec28d25506d5db43f59d0c34936ecb8f7", + "a": "17ee407ae3978c8a2bb54437287afbf9dafb8130", + "b": "0f34c666ac1961e9832fce43ef0ef6937b3c1dde", + "c": "967b6e286301d7e5d089aea6fdbb90bea8ee5a16", + "b1": "5487f2bdef23c5102118b305be9e868228f6ce61", } def build_frozen(artifact, tmp_path_factory, request): - assert_c_collated_rolling_database() + require_rolling_database() sha = ARTIFACTS[artifact] root = tmp_path_factory.mktemp(f"materializer-{artifact}") request.addfinalizer(lambda: shutil.rmtree(root)) @@ -193,8 +193,10 @@ async def test_installed_old_reader_writer_and_workers_on_populated_materializer assert {key: objects[key] for key in old_objects} == old_objects assert all("reporting_materializer_" in key for key in objects.keys() - old_objects.keys()) case.store = sibling.store = store - assert (await case.service().run_once()).state == "verified" - assert (await sibling.service().run_once()).state == "verified" + materializer_operation_1 = await case.service().run_once() + assert (materializer_operation_1).state == "verified" + materializer_operation_2 = await sibling.service().run_once() + assert (materializer_operation_2).state == "verified" h = DurableHarness(store, None, pool) before = await h.queue() async with pool.connection() as connection: diff --git a/tests/conformance/reporting/test_reporting_materializer_service.py b/tests/conformance/reporting/test_reporting_materializer_service.py index b078ef4b3..97292ab4e 100644 --- a/tests/conformance/reporting/test_reporting_materializer_service.py +++ b/tests/conformance/reporting/test_reporting_materializer_service.py @@ -46,7 +46,8 @@ async def test_cancellation_and_restart_never_allocate_a_new_external_identity( assert not list(tmp_path.iterdir()) assert not (set(asyncio.all_tasks()) - before) await h.expire() - assert (await case.service().run_once()).state == "verified" + materializer_operation_1 = await case.service().run_once() + assert (materializer_operation_1).state == "verified" works = await h.works() assert len(works) == 1 and works[0][0] == initial[0][0] assert case.writer.write_effects == 1 @@ -88,11 +89,13 @@ async def renew(*args, **kwargs): await asyncio.wait_for(owner.entered.wait(), 10) await asyncio.wait_for(renewed.wait(), 10) # Four DB-time heartbeats > one whole lease. assert all(turns) and len(turns) >= 4 - assert not hasattr( - await store.claim_materialization(keys=case.keys, lease_seconds=3), "attempt" + materializer_operation_7 = await store.claim_materialization( + keys=case.keys, lease_seconds=3 ) + assert not hasattr(materializer_operation_7, "attempt") owner.release.set() - assert (await asyncio.wait_for(task, 10)).state == "verified" + materializer_operation_8 = await asyncio.wait_for(task, 10) + assert (materializer_operation_8).state == "verified" assert case.writer.write_effects == 1 assert case.writer.open_count == case.writer.close_count == 2 @@ -135,7 +138,8 @@ async def test_write_to_readback_window_reauthorizes_exact_principal_and_current async def test_later_health_loss_never_resets_or_retries_successful_history(backend, state): async with durable_harness(backend) as h: case = await durable_case(h.store) - assert (await case.service().run_once()).state == "verified" + materializer_operation_2 = await case.service().run_once() + assert (materializer_operation_2).state == "verified" outcome = (await case.outcomes())[0] await h.store.record_materialization_check( ReportingMaterializationCheck( @@ -148,7 +152,8 @@ async def test_later_health_loss_never_resets_or_retries_successful_history(back ) result = await case.claim() assert result.reason == "operator_required" - assert (await case.claim()).state == "idle" + materializer_operation_3 = await case.claim() + assert (materializer_operation_3).state == "idle" assert len(await h.works()) == 1 assert await case.outcomes() == (outcome,) @@ -190,7 +195,8 @@ async def test_public_next_attempt_is_still_permitted_after_any_terminal_outcome r.attempt for r in snapshot.records if isinstance(r, ReportingMaterializationAttempt) ] == [1, 2] await h.expire() - assert not hasattr(await case.claim(), "attempt") + materializer_operation_4 = await case.claim() + assert not hasattr(materializer_operation_4, "attempt") assert len(await h.works()) == 1 @@ -268,14 +274,16 @@ async def test_timeout_or_cleanup_failure_after_effect_resumes_original_identity case.writer, io_timeout_seconds=1 if cause == "timeout" else 30, ) - assert (await asyncio.wait_for(service.run_once(), 10)).state == "pending" + materializer_operation_5 = await asyncio.wait_for(service.run_once(), 10) + assert (materializer_operation_5).state == "pending" before = await h.works() assert not await case.outcomes() and await h.queue() == ((), ()) assert case.writer.write_effects == 1 assert all(s._closed and s._credential is None for s in owner.sessions) assert not tuple(tmp_path.iterdir()) and SECRET not in caplog.text await h.expire() - assert (await case.service().run_once()).state == "verified" + materializer_operation_6 = await case.service().run_once() + assert (materializer_operation_6).state == "verified" assert (await h.works())[0][0] == before[0][0] assert case.writer.write_effects == 1 @@ -296,14 +304,16 @@ async def test_heartbeat_cancels_io_after_expiry_and_competing_worker_steals_fen before = await h.works() await h.expire() winner = await case.claim() - assert (await asyncio.wait_for(task, 10)).state == "pending" + materializer_operation_9 = await asyncio.wait_for(task, 10) + assert (materializer_operation_9).state == "pending" assert all(s._closed and s._credential is None for s in owner.sessions) assert not tuple(tmp_path.iterdir()) assert not await case.outcomes() and await h.queue() == ((), ()) prepared, evidence = await case.verified(winner) - assert ( - await h.store.finish_materialization(winner, prepared=prepared, verified=evidence) - ).state == "verified" + materializer_operation_10 = await h.store.finish_materialization( + winner, prepared=prepared, verified=evidence + ) + assert (materializer_operation_10).state == "verified" assert (await h.works())[0][0] == before[0][0] assert case.writer.write_effects == 1 finally: diff --git a/tests/conformance/reporting/test_reporting_materializer_transactions.py b/tests/conformance/reporting/test_reporting_materializer_transactions.py index 0f6f61f20..9788fba5a 100644 --- a/tests/conformance/reporting/test_reporting_materializer_transactions.py +++ b/tests/conformance/reporting/test_reporting_materializer_transactions.py @@ -188,7 +188,8 @@ def fail(self, *args, **kwargs): with pytest.raises(ReportingWriterError): await finish() assert await h.image() == before - assert (await finish()).state == "verified" + materializer_operation_1 = await finish() + assert (materializer_operation_1).state == "verified" assert len(await case.outcomes()) == 1 assert len(await h.store.read_materializer_boundaries(caller=case.scope.principal)) == 1 @@ -239,7 +240,8 @@ def fail(self, event): await finish() assert await h.image() == before assert await h.queue() == ((), ()) - assert (await finish()).state == "verified" + materializer_operation_2 = await finish() + assert (materializer_operation_2).state == "verified" events, work = await h.queue() assert len(events) == 1 and work == ("quarantined",) @@ -307,6 +309,7 @@ async def expire(self, connection, *args): await h.store.finish_materialization(lease, prepared=prepared, verified=evidence) assert await h.image() == before and await h.queue() == ((), ()) await h.expire() - assert (await case.service().run_once()).state == "verified" + materializer_operation_3 = await case.service().run_once() + assert (materializer_operation_3).state == "verified" assert case.writer.write_effects == 1 assert len(await h.works()) == 1 diff --git a/tests/conformance/reporting/test_reporting_notification_packaging.py b/tests/conformance/reporting/test_reporting_notification_packaging.py index 1ec78e5c1..6679a909e 100644 --- a/tests/conformance/reporting/test_reporting_notification_packaging.py +++ b/tests/conformance/reporting/test_reporting_notification_packaging.py @@ -469,7 +469,8 @@ async def get_active(self, *, account_id, subscriber_id, notification_type): clock=clock, ) projector = ReportingActivityProjector(outbox) - assert await ReportingActivitySupport(worker, store, projector).durable() + materializer_operation_1 = await ReportingActivitySupport(worker, store, projector).durable() + assert materializer_operation_1 events = await outbox.list_events(account_id="acct_a") expanded_1 = await worker.expand_one(account_id="acct_a") assert expanded_1 From 8ea0dbe46aafa356a40bb256dbb97316487c2468 Mon Sep 17 00:00:00 2001 From: Brian O'Kelley Date: Thu, 24 Sep 2026 14:43:03 +0000 Subject: [PATCH 8/8] fix(reporting): order materializer accounts by native timestamps --- src/adcp/reporting/materializer/pg.py | 7 +++++-- .../test_reporting_materializer_migration.py | 17 ++++++++++++++++- 2 files changed, 21 insertions(+), 3 deletions(-) diff --git a/src/adcp/reporting/materializer/pg.py b/src/adcp/reporting/materializer/pg.py index 9ee925d6f..990ce2405 100644 --- a/src/adcp/reporting/materializer/pg.py +++ b/src/adcp/reporting/materializer/pg.py @@ -295,17 +295,20 @@ async def claim_materialization( # Read-only sampling. Never lock global candidate rows before the # account advisory lock, including when a competing worker is busy. for _ in range(2): + # Keep the cursor serialized, but order native timestamps: the + # served_at::text output alias otherwise makes fairness locale-dependent. after = self._materializer_sample_after if after: query = ( "SELECT account_id,served_at::text FROM reporting_materializer_accounts" " WHERE due_at<=%s AND (served_at,account_id)>(%s::timestamptz,%s)" - " ORDER BY served_at,account_id LIMIT 16" + " ORDER BY reporting_materializer_accounts.served_at,account_id LIMIT 16" ) else: query = ( "SELECT account_id,served_at::text FROM reporting_materializer_accounts" - " WHERE due_at<=%s ORDER BY served_at,account_id LIMIT 16" + " WHERE due_at<=%s" + " ORDER BY reporting_materializer_accounts.served_at,account_id LIMIT 16" ) accounts = await ( await connection.execute( diff --git a/tests/conformance/reporting/test_reporting_materializer_migration.py b/tests/conformance/reporting/test_reporting_materializer_migration.py index a76636d2e..10deb5a80 100644 --- a/tests/conformance/reporting/test_reporting_materializer_migration.py +++ b/tests/conformance/reporting/test_reporting_materializer_migration.py @@ -10,6 +10,7 @@ from adcp.reporting.ledger import LedgerConflictError, PgReportingReconciliationStore, derive_period from adcp.reporting.materializer import PgReportingMaterializerStore from adcp.reporting.outbox._schema import REQUIRED_OBJECTS, schema_objects, validate_schema +from adcp.reporting.outbox.status_schema import REQUIRED_STATUS_OBJECTS from ._durable_materializer_support import durable_case, durable_harness from ._generation_support import isolated_reporting_pool, obligation_for @@ -45,7 +46,21 @@ async def test_populated_repeated_and_concurrent_install_preserves_all_old_objec assert await new.materializer_ready() async with pool.connection() as c: actual = await schema_objects(c) - assert {key: actual[key] for key in original} == original == REQUIRED_OBJECTS + # The populated rc.6 case also installs its exact waiver-binding objects. + waiver_objects = { + key: value + for key, value in REQUIRED_STATUS_OBJECTS.items() + if "reporting_issue_waiver_bindings" in key + } + assert len(waiver_objects) == 10 + assert ( + {key: actual[key] for key in original} + == original + == { + **REQUIRED_OBJECTS, + **waiver_objects, + } + ) assert { key: value for key, value in actual.items() if "reporting_materializer_" in key } == MANIFEST