diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 11a78e79e..1cddb242f 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -4,7 +4,11 @@ on: push: branches: [main, python-adcp-sdk-setup] pull_request: - branches: [main, conductor/reporting-webhook-activity-1168b, conductor/reporting-status-notifications-1168c] + branches: + - main + - conductor/reporting-webhook-activity-1168b + - conductor/reporting-status-notifications-1168c + - conductor/1167b1-materializer-contracts # Default @adcp/sdk runner alias for storyboard jobs. Tracks the current # stable @adcp/sdk release via the ``latest`` npm dist-tag. @@ -102,7 +106,7 @@ jobs: - name: Run adopter type-check suite if: matrix.python-version == '3.12' - run: mypy --strict tests/type_checks/ examples/reporting_webhook_activity.py examples/reporting_status_notifications.py examples/reporting_destination_writer.py + run: mypy --strict tests/type_checks/ examples/reporting_webhook_activity.py examples/reporting_status_notifications.py examples/reporting_destination_writer.py examples/reporting_durable_materializer.py - name: Enforce adopter type-check fixture contract if: matrix.python-version == '3.12' @@ -187,6 +191,10 @@ jobs: tests/conformance/reporting/ \ --ignore=tests/conformance/reporting/test_reporting_notification_process_matrix.py \ --ignore-glob='tests/conformance/reporting/test_reporting_status_*.py' \ + --ignore=tests/conformance/reporting/test_reporting_materializer_rolling.py \ + --ignore=tests/conformance/reporting/test_reporting_materializer_process.py \ + --ignore=tests/conformance/reporting/test_reporting_materializer_migration.py \ + --ignore=tests/conformance/reporting/test_reporting_materializer_installed_pg.py \ -v ;; process) @@ -203,17 +211,19 @@ jobs: runs-on: ubuntu-latest timeout-minutes: 5 permissions: {} - needs: [pg-conformance, pg-reporting-status] + needs: [pg-conformance, pg-reporting-status, pg-reporting-materializer] if: ${{ always() }} steps: - name: Require every Postgres conformance lane env: PG_RESULT: ${{ needs.pg-conformance.result }} STATUS_RESULT: ${{ needs.pg-reporting-status.result }} + MATERIALIZER_RESULT: ${{ needs.pg-reporting-materializer.result }} run: | - if [ "$PG_RESULT" != "success" ] || [ "$STATUS_RESULT" != "success" ]; then + if [ "$PG_RESULT" != "success" ] || [ "$STATUS_RESULT" != "success" ] || [ "$MATERIALIZER_RESULT" != "success" ]; then echo "Postgres conformance matrix result: $PG_RESULT" echo "Reporting status conformance result: $STATUS_RESULT" + echo "Durable materializer conformance result: $MATERIALIZER_RESULT" exit 1 fi echo "All Postgres conformance lanes passed" @@ -265,6 +275,75 @@ jobs: ADCP_PG_TEST_URL: postgresql://postgres@localhost:5432/adcp_status_test run: pytest tests/conformance/reporting/test_reporting_status_*.py -v + pg-reporting-materializer: + name: Durable materializer and frozen artifacts (Postgres 16) + runs-on: ubuntu-latest + # Actual wheel builds/installations plus isolated worker processes are a + # separate bounded job; do not consume the existing conformance headroom. + # Seven real installations plus process/crash and Python 3.10 distribution + # cells approach twenty minutes locally; allow runner variance and retain + # five minutes outside the suite for setup and cleanup. + timeout-minutes: 35 + permissions: + contents: read + services: + postgres: + image: postgres:16 + env: + POSTGRES_HOST_AUTH_METHOD: trust + POSTGRES_DB: adcp_materializer_test + ports: + - 5432:5432 + options: >- + --health-cmd pg_isready + --health-interval 5s + --health-timeout 5s + --health-retries 10 + steps: + - uses: actions/checkout@v6 + - name: Fetch exact frozen reporting artifacts + timeout-minutes: 2 + run: | + git fetch --no-tags --depth=1 origin \ + 3e76aa54623529a3dda01cd690b8a5c287c75641 \ + 3c405a21f978ed9d3208611bb4a7a8434a056933 \ + 037de4ac822ecefb2f95d32c15c297fb4c45d683 \ + 17ee407ae3978c8a2bb54437287afbf9dafb8130 \ + 0f34c666ac1961e9832fce43ef0ef6937b3c1dde \ + 967b6e286301d7e5d089aea6fdbb90bea8ee5a16 \ + 5487f2bdef23c5102118b305be9e868228f6ce61 + - uses: actions/setup-python@v6 + id: materializer-python310 + with: + python-version: "3.10" + - uses: actions/setup-python@v6 + with: + python-version: "3.12" + cache: pip + cache-dependency-path: pyproject.toml + - name: Install test dependencies + run: pip install -e ".[dev,pg]" + - name: Run installed frozen binaries and materializer migrations + # Propagate pytest failures through tee while retaining evidence for upload. + shell: bash + timeout-minutes: 30 + env: + ADCP_PG_TEST_URL: postgresql://postgres@localhost:5432/adcp_materializer_test + ADCP_PYTHON310: ${{ steps.materializer-python310.outputs.python-path }} + run: | + pytest tests/conformance/reporting/test_reporting_materializer_rolling.py \ + tests/conformance/reporting/test_reporting_materializer_migration.py \ + tests/conformance/reporting/test_reporting_materializer_process.py \ + tests/conformance/reporting/test_reporting_materializer_installed_pg.py \ + -v -s | tee materializer-evidence.log + - name: Preserve installed module and worker evidence + if: always() + uses: actions/upload-artifact@v7 + with: + name: materializer-evidence-${{ github.run_attempt }} + path: materializer-evidence.log + if-no-files-found: error + conventional-commits: name: Validate conventional commit format runs-on: ubuntu-latest diff --git a/docs/reporting-destination-writer.md b/docs/reporting-destination-writer.md index 01a66bfc2..d2c089637 100644 --- a/docs/reporting-destination-writer.md +++ b/docs/reporting-destination-writer.md @@ -20,6 +20,11 @@ binding never advertises `managed_delivery`, `reconciled_billing`, or claim until B2 can prove a complete durable materializer. The producer's `extra` argument rejects SDK-owned task, tier and notification keys. +The additive [B2.1 durable materializer](reporting-durable-materializer.md) now +implements reservation, recovery and verified atomic finish. Production tier +and delivery activation remain gated on the later B2 slices; the development +writer and the B1 contracts above are unchanged. + ## Trusted contracts and lifecycle `ReportingDestinationRequest` includes the exact account and canonical consumer diff --git a/docs/reporting-durable-materializer.md b/docs/reporting-durable-materializer.md new file mode 100644 index 000000000..e7437e36e --- /dev/null +++ b/docs/reporting-durable-materializer.md @@ -0,0 +1,293 @@ +# Durable materializer — #1167B2.1 + +**B2.1 of 4 within B2 of B1/B2. Refs #1167.** This unit builds on the +independently approved B1 commit +`5487f2bdef23c5102118b305be9e868228f6ce61`. It supplies durable reservation, +verified destination I/O, recovery, and one atomic finish transaction. It does +not activate a complete Managed Delivery or Reconciled Billing offering. + +The remaining, separately reviewed dependencies are B2.2 (seller revision and +adjustment receipt ingress), B2.3 (persisted combined feed and revision +ownership), and B2.4 (versioned reconciliation projection, offering readiness, +and production notification activation). They remain required inside B2. +Full Python buyer adjustment automation and `client.reporting` remain the +explicit Wave 6 prerequisite owned by the coordinator. The later #1172 +cross-language process matrix is separate. + +## Composition + +Use the [strict production-oriented example](../examples/reporting_durable_materializer.py). +Construct `PgReportingMaterializerStore(pool=pool, notifications=False)` for an +explicit polling-only deployment, or set `notifications=True` for the atomic +notification path. Install schema during deployment, then call +`compose_materializer` with the application's installed verifier registry and +trusted resolver/writer. Run `service.run_once()` repeatedly; the adopter never +enumerates accounts. The optional HTTP delivery worker is not a materializer +dependency. + +`materializer_ready()` validates storage prerequisites; it is **not** a tier or +mount readiness certificate. There is no caller-supplied production-ready +switch. B1's development writer remains ineligible. This unit offers a +production orchestration primitive, not an activated seller offering. + +**Adopting a materializer store closes notification advertisement entirely, not +only Managed/Reconciled.** `advertised_notifications` matches the ledger by exact +`type(...)`, and neither `InMemoryReportingMaterializerStore` nor +`PgReportingMaterializerStore` is in that approved production identity set. A +deployment that advertises Core `reporting.ledger_changed` today through +`PgReportingReconciliationStore` will instead get a closed +`notification_chain_unready` after switching classes. That is deliberate and +fail closed: B2.4 admits these stores only once it proves projection, component +and mount readiness. Until then, run the durable service on a materializer store +and keep advertising from the reviewed store, or accept closed advertisement. +Polling is unaffected — this helper gates notification capabilities only. +Core-only deployments keep their existing stores and do not need destination or +receipt components. + +Import the optional `ReportingMaterializerStore` protocol, service, lease, +turn and boundary types from `adcp.reporting.materializer`. The PostgreSQL store +is lazy; importing the public surface without `[pg]` works. Older required +store protocols, constructors, enums, and closed record decoders are unchanged. + +## Transactions and discovery + +`reporting_materializer.sql` installs only objects named `reporting_materializer_*`: +account scheduling heads, binding discovery cursors, obligation candidates, +work, captured status heads/boundaries, and isolated notification events and +expansion work. Its own `materializer/required_schema.json` validates the exact +objects. No B2 object is appended to A, B, C, or B1's mandatory manifest, and no +old guard is replaced. Old, partial, and structurally mismatched installations +fail closed. The migration is transactional, repeatable and serialized per +schema; an interrupted installation leaves no partial work schema. + +Installation seeds retained destination bindings once. A durable keyset cursor +discovers at most 32 existing obligations per turn. New binding and obligation +triggers cover either arrival order; restart never rewinds a completed cursor. +Publication, restatement, readability, configuration and materialization-check +mutations dirty indexed candidates. A consumer receipt does not schedule a +materializer retry. Exact no-op writes do not invalidate a generation. + +Claims read a bounded page of at most 16 due accounts without row locks, try the +account advisory lock, and only then lock candidate/work rows in that account. +There is no global `SKIP LOCKED` followed by an account lock. A bounded sampling +cursor advances past busy account pages and wraps; persistent served positions +keep other accounts eligible across restarts. Workers do not periodically scan +the complete ledger. PostgreSQL time controls due dates and 3–300 second leases; +tokens are random UUIDs. Account advisory locks are shared across schemas in +one database, so concurrent test groups must use different databases or run +sequentially when fixture account IDs collide. + +Reservation creates any missing immutable obligation delivery record, the next +immutable attempt, and its durable work on the same connection. Binding may +arrive after publication. Its retention floor is the later of reservation time +and period end, plus the binding's retention duration. An inactive or +deactivated configuration parks work; a future activation is scheduled, and +configuration changes wake the existing history. A binding itself is immutable; +replacement uses a new configuration generation, not an in-place destination +rewrite. Exact trusted principal/binding authorization is still checked at I/O. + +Source preparation, destination write, and verifier-controlled paginated +readback occur **outside** the final account/work transaction. A service-owned, +cancellation-safe heartbeat renews the lease while I/O runs, including with a +size-one connection pool. Each I/O phase has a deadline and joins cleanup. The +service reselects authority before preparation, immediately before write, and +before readback. Write and readback open separate freshly authorized credential +sessions. Preserve the SDK session manager and put every credential and partial +acquisition inside its redacted, non-persistable context; close happens exactly +once, including interrupted opens. + +Finish takes the account lock on one connection, checks exact schema, the +unexpired token, frozen generation/binding/request, SDK-sealed verification, and +strict current/readable revision selection from complete history. Only the SDK +readback verifier can mint the process-local immutable verification seal; +it is bound to that reservation's fencing token and revalidated under the +finish lock. Restart or a new fencing token requires a fresh readback. Private +provenance stays outside B1's three-field constructor, dataclass serialization +and Pydantic wire shape. The transaction commits all of: + +1. The immutable success or compatible safe failure. +2. Required old status dirty work and an isolated immutable boundary containing + actual captured Core and caller-private reconciliation inputs, DB `as_of`, + and monotonic caller/account sequence heads. +3. The logical readiness enqueue when enabled and this is a new verified success. +4. The fenced work ACK and next candidate schedule. + +There is no reacquired connection or external I/O in that unit. An enqueue, +capture, fence, or commit failure rolls it all back. Exact completed replay +does not enqueue, capture, or allocate again. The memory conformance store +unconditionally restores all changed collections and initialized sequence +heads on failure, with notifications either off or on. + +## Notification activation boundary + +The finish uses the existing SDK logical event builder, identity and enqueue +implementation through a closed table adapter on that same connection. The +queue retains caller namespace, deterministic logical cause/idempotency, and +account-ordered captured provenance. Its expansion-work row points to the +logical event; recipient selection/fanout is the separately crash-safe phase +from #1168A, not a synchronous finish requirement. + +**Every B2.1 reservation has immutable `admission_epoch=0`. Every event from +that work is permanently quarantined.** SQL rejects promoting its expansion +work or changing the event/work provenance. No legacy worker reads these +tables, and B2.1 installs no materializer HTTP dispatch route. Creating this +quarantined internal intent does not advertise or emit `reporting.delivery_ready`. +Failures, stale work, corruption, exact replay, and Core mode enqueue no intent. +Explicitly disabled notifications enqueue nothing. Enabled work cannot be +resumed by a replacement process configured with notifications disabled. + +B2.4 must extend the single SDK composition with real installed component and +projection readiness, an isolated activation/fence, and a compatible read path. +Only a **newly admitted reservation** after that activation can enqueue a +deliverable logical event, in its original verified-finish transaction. No +later best-effort copy/publish transaction may become the only actual enqueue. +Pending epoch-zero work retains its epoch and external identity after restart +or upgrade, even if it finishes after activation. Completed epoch-zero events +are never released, promoted, copied, or replayed as current readiness. + +This policy avoids historical delivery under a changed offering, principal, +binding, configuration generation, expired/revoked evidence, or newly selected +official revision. It also avoids inventing retries for previously successful +work just to generate an event. Captured records remain available for polling +and B2.4's explicit baseline/activation procedure. B2.1's minimal capture +primitive preserves the original finish boundary; it does not replace C's +projector or reconstruct historical boundaries from today's state. Full +versioned projection, drain/fence of incompatible C workers, and delivery +activation belong to B2.4. + +## Recovery and retention + +| Durable situation | Autonomous behavior | Operator action | +| --- | --- | --- | +| Pending; timeout, cancellation, worker/connection/lease loss, or unknown external effect | Resume the same attempt and external identity after the lease/backoff | Repair availability; do not reset the sequence | +| Own known terminal failure allowing a new attempt | Allocate N+1 only after that failure; never while N is pending | None for a retryable failure | +| Own terminal failure with no retry | Park as `operator_required` | Correct the cause and explicitly recover through supported persistence | +| Unowned legacy pending attempt | Park as `legacy_pending`, including when a later official revision exists | Drain legacy writers and prove/import the original external identity | +| Legacy terminal outcome | Preserve it; do not silently own its retry policy | Explicit recovery; ordinary public N+1 persistence remains supported | +| Current revision/readability changes during I/O | Immutable safe failure, no readiness and no artifact reuse | A new selected revision uses its own existing history; same-revision recovery increments that history | +| Fork, cycle, multiple official leaves, broken attempt history | Park as `history_corrupt`; no hot loop | Repair/import under an audited operator procedure | +| Successful artifact later becomes unreadable, revoked, unhealthy or expired | Preserve the immutable successful outcome/count; park or project degraded health | Repair authority/health or explicitly persist recovery; no automatic new attempt after success | + +Public persistence still permits attempt N+1 after **any** immutable terminal +outcome, and an ordinary public materialization outcome keeps producing its usual +status projection work. Only the deliverable readiness event is reserved for the +fenced verified finish; the projector must never go stale because an adopter +persisted an outcome itself. The autonomous allocator intentionally owns a +narrower retry policy. +Attempt numbers are revision-specific. Selecting a different revision never +deletes another revision's work/history; selecting the same unreadable revision +never resets its sequence. Official-required selection never falls back to a +snapshot when official evidence is missing or unreadable. Rich internal reasons +map to the existing public `MaterializationFailure` variants; the public enum +is unchanged. + +Before enabling this service, **drain every legacy materialization writer**. +For a known compatible pending effect, call +`import_pending_materialization(scope=..., reporting_materialization_id=..., +original_external_id=..., keys=...)` only after independently verifying its +original destination identity. The primitive requires the exact SDK identity, +current immutable binding/key and sole pending history. It cannot infer an +unknown or different legacy provider identity. Unknown legacy effects require +operator resolution and an explicit terminal record; do not manufacture proof +or have the service adopt them automatically. Restore the same installed key +and explicit import to wake parked owned pending work without changing its ID. + +The destination must durably enforce its advertised idempotent/conditional +write identity. No database transaction can eliminate the external-write / +outcome-commit crash window. Resume must not overwrite a different artifact. +The verifier rechecks all pages, exact digests, totals, rows, immutable locator, +retention and principal. Finish checks retention again against DB time. + +Keep attempts, outcomes, bindings, work, captured inputs and event provenance +conservatively; this unit installs no automatic garbage collector. Never purge +pending work or the external identity needed to recover it. External cleanup +is best effort and cannot authorize success, a retry, or an ACK. A cleanup +failure must not change correctness. Observe only closed `ReportingMaterializerTurn` +state/reason and opaque IDs. Never log sessions, credentials, signed URLs, +provider bodies, raw driver exceptions, or secret destination configuration. + +An out-of-range `lease_seconds` or boundary position is a caller-argument +rejection, not a destination outcome: it raises a `ValueError` naming the bound +before any store or destination work. Reserve `ReportingWriterError` for real +failures, where `retry`/`effect` describe an actual external attempt. + +## Rolling compatibility envelope + +The executable gate is +`tests/conformance/reporting/test_reporting_materializer_rolling.py`. It builds +and installs actual wheels from these exact commits, verifies installed module +origins and source SHA-256 hashes, and runs them with `python -I` outside the +checkout. It first installs each native schema, then the **actual approved B1 +wheel** as comparison baseline. The same frozen binary reads populated records +and performs permitted ordinary writes both before and after B2 installation. + +| Frozen binary | Exact commit | Notification readiness on C/B1 baseline and after B2 | +| --- | --- | --- | +| beta.15 | `3e76aa54623529a3dda01cd690b8a5c287c75641` | No notification API | +| #1167A records | `3c405a21f978ed9d3208611bb4a7a8434a056933` | No notification API | +| Foundation integration | `037de4ac822ecefb2f95d32c15c297fb4c45d683` | No notification API | +| #1168A outbox | `17ee407ae3978c8a2bb54437287afbf9dafb8130` | Aggregate readiness closed on both | +| #1168B activity | `0f34c666ac1961e9832fce43ef0ef6937b3c1dde` | Required-object readiness valid on both | +| #1168C status | `967b6e286301d7e5d089aea6fdbb90bea8ee5a16` | Required-object readiness valid on both | +| #1167B1 | `5487f2bdef23c5102118b305be9e868228f6ce61` | Required-object readiness valid on both | + +The positive-readiness fixture requires PostgreSQL 16 and UTF8. The integrated +A/B/C/B1 pins above preserve the catalog-portability and rc.6 corrections now on +main; no C-only database locale is imposed. Frozen A is ready on its native schema. Reviewed C's additive triggers already close A's aggregate +digest on the approved C/B1 baseline. B2 must preserve that exact classification; +it is not a new waiver. A's permitted default-off Core reads/writes and existing +ordinary notification worker remain functional. The newer B/C required-object +manifests stay ready. The gate checks old catalog objects unchanged and that +every newly introduced object has the isolated prefix. + +Frozen beta.15 retains its original definition shape; monetary-unit declarations +are an A prerequisite. URL reconciliation principals became readable in C. +Earlier binary probes use their supported opaque principal with a second URL +consumer populated in the same account; C/B1 and current probes exercise URL +principals directly. This preserves the actual historical contract rather than +loosening an installed decoder assertion. Old readers do not see another +caller's materializations. Actual old ordinary/status workers consume positive +control events but cannot claim/mutate B2 event/expansion work, materializer work, +or captured boundaries/heads. + +These controls do not qualify pre-`17ee407a` A, pre-`0f34c666` B, +pre-`967b6e28` C or pre-`5487f2bd` B1 binaries. In particular, the earlier +`21bf443e` A catalog digests depended on database collation. This rolling scope +limit must accompany release notes; it does not retroactively qualify the older +snapshots. New B2 failures on supported old operations are regressions, regardless +of the inherited frozen-A aggregate limitation. + +Run the historical comparison with its provenance output preserved: + +```sh +ADCP_PG_TEST_URL="$REPORTING_TEST_DSN" python -m pytest -q -s \ + tests/conformance/reporting/test_reporting_materializer_rolling.py +``` + +Run the materializer PostgreSQL groups sequentially per database. The focused +shared vectors, process kills, migrations and installed-artifact gate are in +`test_reporting_materializer_{durable,transactions,history,service,process,migration,installed_pg}.py`. +Set `ADCP_PYTHON310` to a real 3.10 interpreter to run VCS and sdist-built wheels +on that interpreter; base wheel tests explicitly prohibit PostgreSQL extras. +CI includes the B1 stacked target, all Python 3.10–3.13 jobs, existing PostgreSQL +and status jobs, and a bounded dedicated UTF8/C materializer artifact job. See +the PR's exact head evidence for commands, pass/skip counts and module origins; +a skipped or absent job is not passing evidence. + +Artifact tests retain exact commit, wheel/module hashes, installed module +origins, database preconditions, and event identities in their output. Frozen +builds use compressed archives, remove extracted build-only sources after +hashing, and clean their own installed trees after dependent processes finish. +The common wheel/sdist fixture likewise removes its owned build/install tree +at teardown. Its optional `ADCP_REPORTING_DISTRIBUTION` cache is reusable only +when the complete source fingerprint matches; a changed input fails instead +of silently using an old wheel. A build failure reports a bounded allowlist of +stderr signatures and its stage/exit/digest, never arbitrary provider prose. + +For repeated local matrices, preserve evidence logs outside disposable fixture +trees, run artifact-heavy groups sequentially, and budget for the repository's +2.2 GiB schema cache while a build is active. Clear completed task-owned pytest +scratch and obsolete local `adcp` package cache entries only after checking +process ownership. Recreate contaminated temporary databases with the same +UTF8/C precondition. Disk-full or source-straddling runs are exploratory and +cannot certify the frozen candidate. diff --git a/examples/reporting_durable_materializer.py b/examples/reporting_durable_materializer.py new file mode 100644 index 000000000..ebbf1e0bf --- /dev/null +++ b/examples/reporting_durable_materializer.py @@ -0,0 +1,62 @@ +"""Durable seller composition; install migrations and drain legacy writers first. + +Supply the application's installed verifier registry and trusted destination +resolver/writer pair. Sessions fetch credentials afresh for each I/O phase. +This B2.1 unit preserves private polling inputs and atomic notification intents; +it does not activate Managed/Reconciled capability advertising or HTTP delivery. +""" + +from __future__ import annotations + +import asyncio + +from adcp.reporting.materializer import ( + PgReportingMaterializerStore, + ReportingDestinationIO, + ReportingDestinationResolver, + ReportingDestinationWriter, + ReportingMaterializerService, + ReportingRevisionVerifierRegistry, + ReportingWriterError, + ReportingWriterFailure, +) + + +async def compose_materializer( + store: PgReportingMaterializerStore, + *, + registry: ReportingRevisionVerifierRegistry, + resolver: ReportingDestinationResolver, + writer: ReportingDestinationWriter, +) -> ReportingMaterializerService: + """Validate installed storage, then construct the single SDK orchestration path. + + The store's notifications option is an explicit deployment choice. It is + frozen per reservation, so a replacement process cannot silently downgrade + enabled work. Readiness of the complete seller offering is proved separately + by its installed components, never by an adopter-maintained ready boolean. + """ + if not writer.production_eligible: + raise ReportingWriterError(ReportingWriterFailure("UNSUPPORTED_VERIFICATION")) + await store.materializer_ready() + return ReportingMaterializerService( + store, + ReportingDestinationIO(registry, resolver), + writer, + lease_seconds=30, + io_timeout_seconds=300, + ) + + +async def run_materializer(service: ReportingMaterializerService, stop: asyncio.Event) -> None: + """No account inventory: each turn claims bounded, fair durable work.""" + while not stop.is_set(): + turn = await service.run_once() + # Observe only these closed fields in application metrics. Destination + # credentials, signed locations and provider error text stay in sessions. + if turn.state in {"idle", "parked", "pending"}: + try: + await asyncio.wait_for(stop.wait(), timeout=1) + except asyncio.TimeoutError: + # The poll interval elapsed; check for work or shutdown again. + pass diff --git a/pyproject.toml b/pyproject.toml index 6a2204e1e..45fc80191 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -196,6 +196,7 @@ adcp = [ "reporting/ledger/*.sql", "reporting/outbox/*.json", "reporting/materializer/assets/*.json", + "reporting/materializer/*.json", # PREVIEW: vendored sync_reporting_status schemas. They are the runtime # validator for the wire conditionals codegen cannot express, so the wheel # must carry them. Removed with the rest of _preview/ at rc.2. diff --git a/src/adcp/reporting/ledger/delivery.py b/src/adcp/reporting/ledger/delivery.py index 8f854e211..d07bad3c4 100644 --- a/src/adcp/reporting/ledger/delivery.py +++ b/src/adcp/reporting/ledger/delivery.py @@ -404,22 +404,35 @@ class InMemoryReportingReconciliationStore(InMemoryReportingLedgerStore, _Reconc _delivery_records: list[tuple[int, ReportingDeliveryPrincipal, ReportingDeliveryRecord]] async def _commit(self, record: RecordT) -> tuple[RecordT, bool]: + candidate = cast(RecordT, decode_record(payload(record))) + async with self._mutation(): + return self._commit_record_unlocked(candidate) + + def _commit_record_unlocked( + self, record: RecordT, *, notify: bool = True, dirty: bool = True + ) -> tuple[RecordT, bool]: + """Caller owns the memory mutation; no lock or callback is acquired here. + + ``notify`` and ``dirty`` are independent: suppressing a readiness event + must never also drop the projection work that an ordinary public write + has always produced. + """ candidate = decode_record(payload(record)) who = principal(candidate) - async with self._mutation(): - records = tuple(item.record for item in self._caller_changes(who)) - existing = replay(candidate, records) - if existing is not None: - return cast(RecordT, existing), False - context = self._delivery_context(candidate) - stored = validate_transition(candidate, records, context, self._clock()) - self._append_reconciliation_change(stored) - if self._notification_state is not None: - from adcp.reporting.ledger.notification_events import ( - delivery_dirty, - materialization_event, - ) + records = tuple(item.record for item in self._caller_changes(who)) + existing = replay(candidate, records) + if existing is not None: + return cast(RecordT, existing), False + context = self._delivery_context(candidate) + stored = validate_transition(candidate, records, context, self._clock()) + self._append_reconciliation_change(stored) + if (notify or dirty) and self._notification_state is not None: + from adcp.reporting.ledger.notification_events import ( + delivery_dirty, + materialization_event, + ) + if notify: event = materialization_event( stored, records, @@ -430,9 +443,10 @@ async def _commit(self, record: RecordT) -> tuple[RecordT, bool]: ) if event is not None: self._record_notification(event) + if dirty: scope, reason, evidence = delivery_dirty(stored, context.obligation) self._dirty_status(scope, reason, after=evidence) - return cast(RecordT, stored), True + return cast(RecordT, stored), True def _append_reconciliation_change(self, record: ReportingDeliveryRecord) -> None: who = principal(record) diff --git a/src/adcp/reporting/ledger/delivery_pg.py b/src/adcp/reporting/ledger/delivery_pg.py index a3fc3ec5b..274a98826 100644 --- a/src/adcp/reporting/ledger/delivery_pg.py +++ b/src/adcp/reporting/ledger/delivery_pg.py @@ -110,46 +110,63 @@ async def _commit(self, record: RecordT) -> tuple[RecordT, bool]: unavailable() async def _commit_record(self, record: RecordT) -> tuple[RecordT, bool]: - candidate = decode_record(payload(record)) + candidate = cast(RecordT, decode_record(payload(record))) who = principal(candidate) async with self._connection() as connection: async with connection.transaction(): await self._lock_account(connection, who.account_id) - records = await self._records(connection, who) - existing = replay(candidate, records) - if existing is not None: - return cast(RecordT, existing), False - context = await self._delivery_context(connection, candidate) - if self._clock is not None: - now = self._clock() - else: - time_row = await ( - await connection.execute("SELECT clock_timestamp()") - ).fetchone() - assert time_row is not None - now = time_row[0] - stored = validate_transition(candidate, records, context, now) - await self._insert(connection, stored) - await self._append_reconciliation_change(connection, stored) - if self._notifications_enabled: - from adcp.reporting.ledger.notification_events import ( - delivery_dirty, - materialization_event, - ) + return await self._commit_record_on(connection, candidate) - event = materialization_event( - stored, - records, - context.obligation, - context.revision, - context.configuration, - now, - ) - if event is not None: - await self._record_notification(connection, event) - scope, reason, evidence = delivery_dirty(stored, context.obligation) - await self._dirty_status(connection, scope, reason, after=evidence) - return cast(RecordT, stored), True + async def _commit_record_on( + self, connection: Any, record: RecordT, *, notify: bool = True, dirty: bool = True + ) -> tuple[RecordT, bool]: + """Connection-bound primitive. Caller holds the account transaction lock. + + Materializer finish uses this exact connection for evidence, caller + feed, projection dirty work and its acknowledgment. + No connection is acquired and no external code runs here. + + ``notify`` and ``dirty`` are independent: suppressing a readiness event + must never also drop the projection work that an ordinary public write + has always produced. + """ + candidate = decode_record(payload(record)) + who = principal(candidate) + records = await self._records(connection, who) + existing = replay(candidate, records) + if existing is not None: + return cast(RecordT, existing), False + context = await self._delivery_context(connection, candidate) + if self._clock is not None: + now = self._clock() + else: + time_row = await (await connection.execute("SELECT clock_timestamp()")).fetchone() + assert time_row is not None + now = time_row[0] + stored = validate_transition(candidate, records, context, now) + await self._insert(connection, stored) + await self._append_reconciliation_change(connection, stored) + if (notify or dirty) and self._notifications_enabled: + from adcp.reporting.ledger.notification_events import ( + delivery_dirty, + materialization_event, + ) + + if notify: + event = materialization_event( + stored, + records, + context.obligation, + context.revision, + context.configuration, + now, + ) + if event is not None: + await self._record_notification(connection, event) + if dirty: + scope, reason, evidence = delivery_dirty(stored, context.obligation) + await self._dirty_status(connection, scope, reason, after=evidence) + return cast(RecordT, stored), True async def _append_reconciliation_change( self, connection: Any, record: ReportingDeliveryRecord diff --git a/src/adcp/reporting/ledger/reporting_materializer.sql b/src/adcp/reporting/ledger/reporting_materializer.sql new file mode 100644 index 000000000..d27331117 --- /dev/null +++ b/src/adcp/reporting/ledger/reporting_materializer.sql @@ -0,0 +1,420 @@ +-- #1167B2. Isolated, additive work objects; no A/B/C/B1 guard is replaced. +-- A single statement also makes autocommit/repeated/concurrent installation atomic. +DO $materializer$ +BEGIN + PERFORM pg_advisory_xact_lock(hashtext('adcp.reporting.schema'), hashtext(current_schema())); + PERFORM account_id, consumer_id FROM reporting_reconciliation_records LIMIT 0; + + CREATE TABLE IF NOT EXISTS reporting_materializer_accounts ( + account_id TEXT COLLATE "C" PRIMARY KEY, + due_at TIMESTAMPTZ, + served_at TIMESTAMPTZ NOT NULL DEFAULT '-infinity', + captured_sequence BIGINT NOT NULL DEFAULT 0 CHECK (captured_sequence >= 0) + ); + CREATE INDEX IF NOT EXISTS reporting_materializer_account_due + ON reporting_materializer_accounts (served_at, account_id) WHERE due_at IS NOT NULL; + CREATE INDEX IF NOT EXISTS reporting_materializer_account_wakeup + ON reporting_materializer_accounts (due_at, served_at, account_id) WHERE due_at IS NOT NULL; + + CREATE TABLE IF NOT EXISTS reporting_materializer_discovery ( + account_id TEXT COLLATE "C" NOT NULL, + consumer_id TEXT COLLATE "C" NOT NULL, + delivery_config_id TEXT COLLATE "C" NOT NULL, + delivery_config_version BIGINT NOT NULL CHECK (delivery_config_version > 0), + after_obligation_id TEXT COLLATE "C" NOT NULL DEFAULT '', + complete BOOLEAN NOT NULL DEFAULT FALSE, + PRIMARY KEY (account_id, consumer_id, delivery_config_id, delivery_config_version), + FOREIGN KEY (account_id, delivery_config_id, delivery_config_version) + REFERENCES reporting_configurations(account_id, delivery_config_id, delivery_config_version) + ); + CREATE INDEX IF NOT EXISTS reporting_materializer_discovery_pending + ON reporting_materializer_discovery (account_id, consumer_id, delivery_config_id, + delivery_config_version) WHERE NOT complete; + CREATE INDEX IF NOT EXISTS reporting_materializer_obligation_discovery + ON reporting_obligations (account_id, delivery_config_id, delivery_config_version, + reporting_obligation_id); + CREATE INDEX IF NOT EXISTS reporting_materializer_binding_discovery + ON reporting_reconciliation_records (account_id, consumer_id, delivery_config_id, + delivery_config_version) WHERE record_kind='destination_binding'; + + CREATE TABLE IF NOT EXISTS reporting_materializer_candidates ( + account_id TEXT COLLATE "C" NOT NULL REFERENCES reporting_materializer_accounts, + consumer_id TEXT COLLATE "C" NOT NULL, + delivery_config_id TEXT COLLATE "C" NOT NULL, + delivery_config_version BIGINT NOT NULL CHECK (delivery_config_version > 0), + reporting_obligation_id TEXT COLLATE "C" NOT NULL, + generation BIGINT NOT NULL DEFAULT 1 CHECK (generation > 0), + due_at TIMESTAMPTZ, + reason TEXT COLLATE "C" NOT NULL DEFAULT 'ready' CHECK (reason IN ( + 'ready','verified','retry','inactive','revision_not_ready','revision_unreadable', + 'target_changed','history_corrupt','legacy_pending','legacy_terminal', + 'component_unavailable','binding_changed','effect_unknown','operator_required')), + served_at TIMESTAMPTZ NOT NULL DEFAULT '-infinity', + PRIMARY KEY (account_id, consumer_id, delivery_config_id, delivery_config_version, + reporting_obligation_id), + FOREIGN KEY (account_id, delivery_config_id, delivery_config_version, reporting_obligation_id) + REFERENCES reporting_obligations(account_id, delivery_config_id, + delivery_config_version, reporting_obligation_id) + ); + CREATE INDEX IF NOT EXISTS reporting_materializer_candidate_due + ON reporting_materializer_candidates (account_id, due_at, served_at, + consumer_id, reporting_obligation_id) WHERE due_at IS NOT NULL; + CREATE INDEX IF NOT EXISTS reporting_materializer_candidate_publication + ON reporting_materializer_candidates (account_id, reporting_obligation_id); + + CREATE TABLE IF NOT EXISTS reporting_materializer_work ( + account_id TEXT COLLATE "C" NOT NULL, + consumer_id TEXT COLLATE "C" NOT NULL, + delivery_config_id TEXT COLLATE "C" NOT NULL, + delivery_config_version BIGINT NOT NULL, + reporting_obligation_id TEXT COLLATE "C" NOT NULL, + reporting_revision_id TEXT COLLATE "C" NOT NULL, + reporting_materialization_id TEXT COLLATE "C" NOT NULL, + attempt_namespace TEXT COLLATE "C" NOT NULL DEFAULT 'materialization_attempt' + CHECK (attempt_namespace = 'materialization_attempt'), + generation BIGINT NOT NULL CHECK (generation > 0), + binding_sha256 TEXT COLLATE "C" NOT NULL CHECK (binding_sha256 ~ '^[0-9a-f]{64}$'), + verification_key_sha256 TEXT COLLATE "C" NOT NULL + CHECK (verification_key_sha256 ~ '^[0-9a-f]{64}$'), + external_id TEXT COLLATE "C" NOT NULL CHECK (external_id ~ '^rwm_[0-9a-f]{64}$'), + state TEXT COLLATE "C" NOT NULL DEFAULT 'pending' CHECK (state IN ('pending','acked')), + retry_allowed BOOLEAN NOT NULL DEFAULT FALSE, + reason TEXT COLLATE "C" NOT NULL DEFAULT 'ready' CHECK (reason IN ( + 'ready','verified','retry','inactive','revision_not_ready','revision_unreadable', + 'target_changed','history_corrupt','legacy_pending','legacy_terminal', + 'component_unavailable','binding_changed','effect_unknown','operator_required')), + created_at TIMESTAMPTZ NOT NULL DEFAULT clock_timestamp(), + acknowledged_at TIMESTAMPTZ, + completion_token UUID, + lease_token UUID, + lease_until TIMESTAMPTZ, + due_at TIMESTAMPTZ NOT NULL DEFAULT clock_timestamp(), + imported BOOLEAN NOT NULL DEFAULT FALSE, + notifications_enabled BOOLEAN NOT NULL, + -- No B2.1 reservation is production-admitted. Future activation must + -- preserve this epoch on retained work, including pending resumes. + admission_epoch BIGINT NOT NULL DEFAULT 0 CHECK (admission_epoch = 0), + PRIMARY KEY (account_id, consumer_id, reporting_materialization_id), + UNIQUE (account_id, consumer_id, external_id), + FOREIGN KEY (account_id, consumer_id, delivery_config_id, delivery_config_version, + reporting_obligation_id) REFERENCES reporting_materializer_candidates, + FOREIGN KEY (account_id, consumer_id, attempt_namespace, reporting_materialization_id) + REFERENCES reporting_reconciliation_records(account_id, consumer_id, namespace, record_id), + FOREIGN KEY (account_id, reporting_obligation_id, reporting_revision_id) + REFERENCES reporting_revisions(account_id, reporting_obligation_id, reporting_revision_id), + CHECK ((lease_token IS NULL) = (lease_until IS NULL)), + CHECK ((state = 'acked') = (acknowledged_at IS NOT NULL)), + CHECK ((state = 'acked') = (completion_token IS NOT NULL)), + CHECK (state <> 'acked' OR lease_token IS NULL), + CHECK (NOT retry_allowed OR state = 'acked') + ); + CREATE UNIQUE INDEX IF NOT EXISTS reporting_materializer_one_pending + ON reporting_materializer_work (account_id, consumer_id, delivery_config_id, + delivery_config_version, reporting_obligation_id) WHERE state = 'pending'; + CREATE INDEX IF NOT EXISTS reporting_materializer_work_due + ON reporting_materializer_work (account_id, due_at, reporting_materialization_id) + WHERE state = 'pending'; + + CREATE TABLE IF NOT EXISTS reporting_materializer_status_heads ( + account_id TEXT COLLATE "C" NOT NULL, + consumer_id TEXT COLLATE "C" NOT NULL, + max_sequence BIGINT NOT NULL CHECK (max_sequence > 0), + PRIMARY KEY (account_id, consumer_id) + ); + CREATE TABLE IF NOT EXISTS reporting_materializer_status_boundaries ( + account_id TEXT COLLATE "C" NOT NULL, + consumer_id TEXT COLLATE "C" NOT NULL, + sequence BIGINT NOT NULL CHECK (sequence > 0), + account_sequence BIGINT NOT NULL CHECK (account_sequence > 0), + reporting_materialization_id TEXT COLLATE "C" NOT NULL, + outcome_namespace TEXT COLLATE "C" NOT NULL DEFAULT 'materialization' + CHECK (outcome_namespace = 'materialization'), + as_of TIMESTAMPTZ NOT NULL, + input JSONB NOT NULL, + content_sha256 TEXT COLLATE "C" NOT NULL CHECK (content_sha256 ~ '^[0-9a-f]{64}$'), + PRIMARY KEY (account_id, consumer_id, sequence), + UNIQUE (account_id, consumer_id, reporting_materialization_id), + UNIQUE (account_id, account_sequence), + FOREIGN KEY (account_id, consumer_id, reporting_materialization_id) + REFERENCES reporting_materializer_work, + FOREIGN KEY (account_id, consumer_id, outcome_namespace, reporting_materialization_id) + REFERENCES reporting_reconciliation_records(account_id, consumer_id, namespace, record_id), + CHECK ((input->>'version')::integer IS NOT DISTINCT FROM 1), + CHECK ((input->>'account_id') IS NOT DISTINCT FROM account_id), + CHECK ((input->>'consumer_id') IS NOT DISTINCT FROM consumer_id), + CHECK ((input->>'reporting_materialization_id') IS NOT DISTINCT FROM reporting_materialization_id), + CHECK ((input->>'sequence')::bigint IS NOT DISTINCT FROM sequence), + CHECK ((input->>'account_sequence')::bigint IS NOT DISTINCT FROM account_sequence), + CHECK ((input->>'as_of')::timestamptz IS NOT DISTINCT FROM as_of), + CHECK (content_sha256 = reporting_payload_sha256(input)), + CHECK (input - ARRAY['version','account_id','consumer_id','reporting_materialization_id', + 'sequence','account_sequence','as_of','core','reconciliation'] = '{}'::jsonb) + ); + CREATE TABLE IF NOT EXISTS reporting_materializer_notification_events ( + account_id TEXT COLLATE "C" NOT NULL, + notification_id TEXT COLLATE "C" NOT NULL, + notification_type TEXT COLLATE "C" NOT NULL CHECK (notification_type='reporting.delivery_ready'), + cause_kind TEXT COLLATE "C" NOT NULL CHECK (cause_kind='materialization_ready'), + cause_id TEXT COLLATE "C" NOT NULL, + cause_generation BIGINT NOT NULL CHECK (cause_generation=1), + consumer_namespace TEXT COLLATE "C" NOT NULL CHECK (length(consumer_namespace)>0), + admission_epoch BIGINT NOT NULL DEFAULT 0 CHECK (admission_epoch = 0), + fired_at TIMESTAMPTZ NOT NULL, + snapshot JSONB NOT NULL, + reporting_materialization_id TEXT COLLATE "C" GENERATED ALWAYS AS + (snapshot #>> '{cause,reporting_materialization_id}') STORED, + PRIMARY KEY (account_id, consumer_namespace, notification_id), + UNIQUE (account_id, consumer_namespace, notification_type, cause_kind, cause_id, cause_generation), + FOREIGN KEY (account_id, consumer_namespace, reporting_materialization_id) + REFERENCES reporting_materializer_status_boundaries + (account_id, consumer_id, reporting_materialization_id), + CHECK ((snapshot->>'account_id') IS NOT DISTINCT FROM account_id), + CHECK ((snapshot->>'notification_id') IS NOT DISTINCT FROM notification_id), + CHECK (cause_id::jsonb IS NOT DISTINCT FROM + jsonb_build_array(consumer_namespace, reporting_materialization_id)), + CHECK ((snapshot #>> '{cause,consumer_id}') IS NOT DISTINCT FROM consumer_namespace) + ); + CREATE TABLE IF NOT EXISTS reporting_materializer_notification_expansions ( + account_id TEXT COLLATE "C" NOT NULL, + consumer_namespace TEXT COLLATE "C" NOT NULL, + notification_id TEXT COLLATE "C" NOT NULL, + emission_generation BIGINT NOT NULL CHECK (emission_generation>0), + state TEXT NOT NULL DEFAULT 'quarantined' CHECK + (state IN ('pending','leased','complete','suppressed','quarantined')), + due_at TIMESTAMPTZ NOT NULL, + lease_token TEXT, + lease_expires_at TIMESTAMPTZ, + claim_count BIGINT NOT NULL DEFAULT 0, + error_code TEXT CHECK (error_code IN ( + 'network','retryable_http','permanent_http','signing_unavailable', + 'permanent_scope','subscription_unavailable','subscription_changed', + 'invalid_configuration','invalid_payload','integrity_failure','lease_expired')), + PRIMARY KEY (account_id, consumer_namespace, notification_id, emission_generation), + FOREIGN KEY (account_id, consumer_namespace, notification_id) + REFERENCES reporting_materializer_notification_events + ); + CREATE INDEX IF NOT EXISTS reporting_materializer_notification_due + ON reporting_materializer_notification_expansions (account_id, due_at) + WHERE state IN ('pending','leased'); + + EXECUTE $function$ + CREATE OR REPLACE FUNCTION reporting_materializer_expansion_guard() RETURNS trigger + LANGUAGE plpgsql AS $body$ + DECLARE epoch BIGINT; + BEGIN + SELECT admission_epoch INTO epoch FROM reporting_materializer_notification_events + WHERE account_id=NEW.account_id AND consumer_namespace=NEW.consumer_namespace + AND notification_id=NEW.notification_id; + IF epoch=0 AND NEW.state <> 'quarantined' THEN + RAISE EXCEPTION 'reporting_materializer_pre_activation_event' + USING ERRCODE='23514'; + END IF; + RETURN NEW; + END + $body$ $function$; + IF NOT EXISTS (SELECT 1 FROM pg_trigger + WHERE tgrelid='reporting_materializer_notification_expansions'::regclass + AND tgname='reporting_materializer_pre_activation_guard') THEN + CREATE TRIGGER reporting_materializer_pre_activation_guard BEFORE INSERT OR UPDATE + ON reporting_materializer_notification_expansions FOR EACH ROW + EXECUTE FUNCTION reporting_materializer_expansion_guard(); + END IF; + + EXECUTE $function$ + CREATE OR REPLACE FUNCTION reporting_materializer_retained_guard() RETURNS trigger + LANGUAGE plpgsql AS $body$ + BEGIN + RAISE EXCEPTION 'reporting_materializer_evidence_immutable' USING ERRCODE='23514'; + END + $body$ $function$; + IF NOT EXISTS (SELECT 1 FROM pg_trigger WHERE tgrelid='reporting_materializer_status_boundaries'::regclass + AND tgname='reporting_materializer_boundary_immutable') THEN + CREATE TRIGGER reporting_materializer_boundary_immutable BEFORE UPDATE OR DELETE + ON reporting_materializer_status_boundaries FOR EACH ROW + EXECUTE FUNCTION reporting_materializer_retained_guard(); + END IF; + IF NOT EXISTS (SELECT 1 FROM pg_trigger WHERE tgrelid='reporting_materializer_notification_events'::regclass + AND tgname='reporting_materializer_event_immutable') THEN + CREATE TRIGGER reporting_materializer_event_immutable BEFORE UPDATE OR DELETE + ON reporting_materializer_notification_events FOR EACH ROW + EXECUTE FUNCTION reporting_materializer_retained_guard(); + END IF; + + EXECUTE $function$ + CREATE OR REPLACE FUNCTION reporting_materializer_wake(a TEXT) RETURNS void + LANGUAGE sql AS $body$ + INSERT INTO reporting_materializer_accounts (account_id, due_at) + VALUES (a, clock_timestamp()) ON CONFLICT (account_id) DO UPDATE + SET due_at = LEAST(reporting_materializer_accounts.due_at, EXCLUDED.due_at) + $body$ $function$; + + EXECUTE $function$ + CREATE OR REPLACE FUNCTION reporting_materializer_dirty( + a TEXT, c TEXT, d TEXT, v BIGINT, o TEXT + ) RETURNS void LANGUAGE plpgsql AS $body$ + BEGIN + PERFORM pg_advisory_xact_lock(hashtext('adcp.reporting:' || a)); + PERFORM reporting_materializer_wake(a); + INSERT INTO reporting_materializer_candidates + (account_id, consumer_id, delivery_config_id, delivery_config_version, + reporting_obligation_id, due_at) + VALUES (a,c,d,v,o,clock_timestamp()) + ON CONFLICT (account_id, consumer_id, delivery_config_id, delivery_config_version, + reporting_obligation_id) DO UPDATE + SET generation = reporting_materializer_candidates.generation + 1, + due_at = clock_timestamp(), reason = 'ready'; + END + $body$ $function$; + + EXECUTE $function$ + CREATE OR REPLACE FUNCTION reporting_materializer_source_dirty() RETURNS trigger + LANGUAGE plpgsql AS $body$ + DECLARE r RECORD; + BEGIN + IF TG_OP = 'UPDATE' AND NEW IS NOT DISTINCT FROM OLD THEN + RETURN NEW; + END IF; + PERFORM pg_advisory_xact_lock(hashtext('adcp.reporting:' || NEW.account_id)); + IF TG_TABLE_NAME = 'reporting_configurations' THEN + UPDATE reporting_materializer_candidates SET generation = generation + 1, + due_at = clock_timestamp(), reason = 'ready' + WHERE account_id = NEW.account_id AND delivery_config_id = NEW.delivery_config_id + AND delivery_config_version = NEW.delivery_config_version; + IF FOUND THEN PERFORM reporting_materializer_wake(NEW.account_id); END IF; + ELSIF TG_TABLE_NAME = 'reporting_revisions' THEN + UPDATE reporting_materializer_candidates SET generation = generation + 1, + due_at = clock_timestamp(), reason = 'ready' + WHERE account_id = NEW.account_id AND reporting_obligation_id = NEW.reporting_obligation_id; + IF FOUND THEN PERFORM reporting_materializer_wake(NEW.account_id); END IF; + ELSE + FOR r IN SELECT consumer_id FROM reporting_materializer_discovery + WHERE account_id = NEW.account_id AND delivery_config_id = NEW.delivery_config_id + AND delivery_config_version = NEW.delivery_config_version + ORDER BY consumer_id + LOOP + PERFORM reporting_materializer_dirty(NEW.account_id, r.consumer_id, + NEW.delivery_config_id, NEW.delivery_config_version, NEW.reporting_obligation_id); + END LOOP; + END IF; + RETURN NEW; + END + $body$ $function$; + + EXECUTE $function$ + CREATE OR REPLACE FUNCTION reporting_materializer_binding_dirty() RETURNS trigger + LANGUAGE plpgsql AS $body$ + BEGIN + IF NEW.record_kind = 'destination_binding' THEN + PERFORM pg_advisory_xact_lock(hashtext('adcp.reporting:' || NEW.account_id)); + INSERT INTO reporting_materializer_discovery + (account_id, consumer_id, delivery_config_id, delivery_config_version) + VALUES (NEW.account_id, NEW.consumer_id, NEW.delivery_config_id, NEW.delivery_config_version) + ON CONFLICT DO NOTHING; + PERFORM reporting_materializer_wake(NEW.account_id); + ELSIF NEW.record_kind = 'materialization_check' THEN + PERFORM reporting_materializer_dirty(NEW.account_id, NEW.consumer_id, + NEW.delivery_config_id, NEW.delivery_config_version, NEW.reporting_obligation_id); + END IF; + -- Consumer rejection is never a materializer retry signal. + RETURN NEW; + END + $body$ $function$; + + EXECUTE $function$ + CREATE OR REPLACE FUNCTION reporting_materializer_work_guard() RETURNS trigger + LANGUAGE plpgsql AS $body$ + DECLARE a JSONB; result RECORD; + BEGIN + IF TG_OP = 'DELETE' THEN + RAISE EXCEPTION 'reporting_materializer_history_immutable' USING ERRCODE='23514'; + END IF; + IF TG_OP = 'UPDATE' AND ( + (to_jsonb(NEW) - ARRAY['lease_token','lease_until','due_at','state','retry_allowed', + 'reason','acknowledged_at','completion_token']) IS DISTINCT FROM + (to_jsonb(OLD) - ARRAY['lease_token','lease_until','due_at','state','retry_allowed', + 'reason','acknowledged_at','completion_token']) OR OLD.state = 'acked' + ) THEN + RAISE EXCEPTION 'reporting_materializer_identity_immutable' USING ERRCODE='23514'; + END IF; + SELECT payload INTO a FROM reporting_reconciliation_records + WHERE account_id = NEW.account_id AND consumer_id = NEW.consumer_id + AND namespace = 'materialization_attempt' AND record_id = NEW.reporting_materialization_id; + IF a IS NULL OR a->>'reporting_revision_id' IS DISTINCT FROM NEW.reporting_revision_id + OR a #>> '{scope,reporting_obligation_id}' IS DISTINCT FROM NEW.reporting_obligation_id + OR a #>> '{scope,generation_key,delivery_config_id}' IS DISTINCT FROM NEW.delivery_config_id + OR (a #>> '{scope,generation_key,delivery_config_version}')::bigint + IS DISTINCT FROM NEW.delivery_config_version THEN + RAISE EXCEPTION 'reporting_materializer_attempt_mismatch' USING ERRCODE='23514'; + END IF; + IF NEW.state = 'acked' THEN + IF TG_OP <> 'UPDATE' OR OLD.state <> 'pending' OR OLD.lease_token IS NULL + OR OLD.lease_until <= clock_timestamp() + OR NEW.completion_token IS DISTINCT FROM OLD.lease_token THEN + RAISE EXCEPTION 'reporting_materializer_fence_required' USING ERRCODE='23514'; + END IF; + SELECT payload INTO result FROM reporting_reconciliation_records + WHERE account_id = NEW.account_id AND consumer_id = NEW.consumer_id + AND namespace = 'materialization' AND record_id = NEW.reporting_materialization_id; + IF NOT FOUND OR (NEW.retry_allowed AND result.payload->>'status' <> 'failed') THEN + RAISE EXCEPTION 'reporting_materializer_outcome_required' USING ERRCODE='23514'; + END IF; + IF NOT EXISTS (SELECT 1 FROM reporting_materializer_status_boundaries b + WHERE b.account_id=NEW.account_id AND b.consumer_id=NEW.consumer_id + AND b.reporting_materialization_id=NEW.reporting_materialization_id + AND b.as_of=NEW.acknowledged_at + AND b.as_of=(result.payload->>'completed_at')::timestamptz) THEN + RAISE EXCEPTION 'reporting_materializer_capture_required' USING ERRCODE='23514'; + END IF; + IF NEW.notifications_enabled AND result.payload->>'status' <> 'failed' + AND NOT EXISTS (SELECT 1 FROM reporting_materializer_notification_events e + WHERE e.account_id=NEW.account_id AND e.consumer_namespace=NEW.consumer_id + AND e.reporting_materialization_id=NEW.reporting_materialization_id + AND e.admission_epoch=NEW.admission_epoch) THEN + RAISE EXCEPTION 'reporting_materializer_event_required' USING ERRCODE='23514'; + END IF; + END IF; + RETURN NEW; + END + $body$ $function$; + + IF NOT EXISTS (SELECT 1 FROM pg_trigger WHERE tgrelid='reporting_reconciliation_records'::regclass + AND tgname='reporting_materializer_binding') THEN + CREATE TRIGGER reporting_materializer_binding AFTER INSERT ON reporting_reconciliation_records + FOR EACH ROW EXECUTE FUNCTION reporting_materializer_binding_dirty(); + END IF; + IF NOT EXISTS (SELECT 1 FROM pg_trigger WHERE tgrelid='reporting_obligations'::regclass + AND tgname='reporting_materializer_obligation') THEN + CREATE TRIGGER reporting_materializer_obligation AFTER INSERT ON reporting_obligations + FOR EACH ROW EXECUTE FUNCTION reporting_materializer_source_dirty(); + END IF; + IF NOT EXISTS (SELECT 1 FROM pg_trigger WHERE tgrelid='reporting_revisions'::regclass + AND tgname='reporting_materializer_publication') THEN + CREATE TRIGGER reporting_materializer_publication AFTER INSERT OR UPDATE ON reporting_revisions + FOR EACH ROW EXECUTE FUNCTION reporting_materializer_source_dirty(); + END IF; + IF NOT EXISTS (SELECT 1 FROM pg_trigger WHERE tgrelid='reporting_configurations'::regclass + AND tgname='reporting_materializer_configuration') THEN + CREATE TRIGGER reporting_materializer_configuration AFTER UPDATE ON reporting_configurations + FOR EACH ROW EXECUTE FUNCTION reporting_materializer_source_dirty(); + END IF; + IF NOT EXISTS (SELECT 1 FROM pg_trigger WHERE tgrelid='reporting_materializer_work'::regclass + AND tgname='reporting_materializer_guard') THEN + CREATE TRIGGER reporting_materializer_guard BEFORE INSERT OR UPDATE OR DELETE + ON reporting_materializer_work FOR EACH ROW + EXECUTE FUNCTION reporting_materializer_work_guard(); + END IF; + + -- One-time indexed backfill seeds bindings, not account enumeration or + -- recurring whole-ledger scans. Reinstall never rewinds a discovery cursor. + INSERT INTO reporting_materializer_discovery + (account_id, consumer_id, delivery_config_id, delivery_config_version) + SELECT account_id, consumer_id, delivery_config_id, delivery_config_version + FROM reporting_reconciliation_records WHERE record_kind='destination_binding' + ON CONFLICT DO NOTHING; + INSERT INTO reporting_materializer_accounts (account_id, due_at) + SELECT account_id, clock_timestamp() FROM reporting_materializer_discovery WHERE NOT complete + GROUP BY account_id + ON CONFLICT (account_id) DO UPDATE + SET due_at=LEAST(reporting_materializer_accounts.due_at, EXCLUDED.due_at); +END +$materializer$; diff --git a/src/adcp/reporting/ledger/store.py b/src/adcp/reporting/ledger/store.py index 54bb592c5..805f8c413 100644 --- a/src/adcp/reporting/ledger/store.py +++ b/src/adcp/reporting/ledger/store.py @@ -724,26 +724,18 @@ def __init__( async def _mutation(self) -> AsyncIterator[None]: """Publish domain changes and notifications under one rollback boundary. - Default-off stores retain their original lock cost. The reference - in-memory transaction copies retained values only when opted in. + Rollback also applies with notifications disabled. Newly initialized + collections and sequence heads belong to the transaction too. """ owner = (id(self), asyncio.current_task()) if _MEMORY_TRANSACTION.get() == owner: yield return async with self._lock: - token = _MEMORY_TRANSACTION.set(owner) - before = ( - deepcopy( - { - key: value - for key, value in vars(self).items() - if key not in {"_lock", "_clock"} - } - ) - if self._notification_state is not None - else None + before = deepcopy( + {key: value for key, value in vars(self).items() if key not in {"_lock", "_clock"}} ) + token = _MEMORY_TRANSACTION.set(owner) try: dirty_start = len(self._notification_state.dirty) if self._notification_state else 0 yield @@ -765,8 +757,9 @@ async def _mutation(self) -> AsyncIterator[None]: ) ) except BaseException: - if before is not None: - vars(self).update(before) + for key in set(vars(self)) - {"_lock", "_clock"} - set(before): + del vars(self)[key] + vars(self).update(before) raise finally: _MEMORY_TRANSACTION.reset(token) diff --git a/src/adcp/reporting/materializer/__init__.py b/src/adcp/reporting/materializer/__init__.py index 165d6d3d8..28c7edf26 100644 --- a/src/adcp/reporting/materializer/__init__.py +++ b/src/adcp/reporting/materializer/__init__.py @@ -1,11 +1,14 @@ -"""B1 public destination contracts and verification; no durable Managed service. +"""Destination contracts, verification and optional durable materialization. Use the immutable registry to prepare all frozen source rows, then invoke write and verify explicitly with separate authorization sessions. The reference -writer is exclusively for tests/development. B2 owns durable work, fencing, -retry allocation, final target reselection, and readiness transactions. +writer is exclusively for tests/development. The durable service owns fencing, +retry allocation, target reselection and atomic finish. Production tier and +notification activation additionally require the complete seller projection. """ +from typing import TYPE_CHECKING + from adcp.reporting.ledger.delivery_models import ( ReportingDeliveryPrincipal, ReportingDestinationBinding, @@ -17,6 +20,7 @@ parse_reporting_json, strict_reporting_json, ) +from adcp.reporting.materializer.capture import ReportingMaterializerBoundary from adcp.reporting.materializer.contracts import ( ReportingCanonicalization, ReportingDestinationLocator, @@ -38,12 +42,14 @@ ReportingWriterFailureCode, ReportingWriterRetry, ) +from adcp.reporting.materializer.memory import InMemoryReportingMaterializerStore from adcp.reporting.materializer.reference import ( ReferenceReportingDestinationWriter, ReferenceReportingResolver, reference_digest, reference_verifier, ) +from adcp.reporting.materializer.service import ReportingMaterializerService from adcp.reporting.materializer.verification import ( ReportingDestinationIO, ReportingRevisionRowReader, @@ -52,8 +58,20 @@ ReportingVerifiedDestination, validate_materialization_target, ) +from adcp.reporting.materializer.work import ( + MaterializerReason, + ReportingMaterializerLease, + ReportingMaterializerStore, + ReportingMaterializerTurn, +) + +if TYPE_CHECKING: + from adcp.reporting.materializer.pg import PgReportingMaterializerStore __all__ = [ + "InMemoryReportingMaterializerStore", + "MaterializerReason", + "PgReportingMaterializerStore", "ReferenceReportingDestinationWriter", "ReferenceReportingResolver", "ReportingCanonicalization", @@ -71,6 +89,11 @@ "ReportingIOContext", "ReportingIOPhase", "ReportingMaterializationAttempt", + "ReportingMaterializerBoundary", + "ReportingMaterializerLease", + "ReportingMaterializerService", + "ReportingMaterializerStore", + "ReportingMaterializerTurn", "ReportingNativeObservation", "ReportingObligationDeliveryRecord", "ReportingPreparedRevision", @@ -91,3 +114,11 @@ "strict_reporting_json", "validate_materialization_target", ] + + +def __getattr__(name: str) -> object: + if name == "PgReportingMaterializerStore": + from adcp.reporting.materializer.pg import PgReportingMaterializerStore + + return PgReportingMaterializerStore + raise AttributeError(f"module {__name__!r} has no attribute {name!r}") diff --git a/src/adcp/reporting/materializer/_errors.py b/src/adcp/reporting/materializer/_errors.py new file mode 100644 index 000000000..e00cafb81 --- /dev/null +++ b/src/adcp/reporting/materializer/_errors.py @@ -0,0 +1,46 @@ +"""Do not let a driver, hook or cancellation message become a diagnostic.""" + +from __future__ import annotations + +import asyncio +from collections.abc import Awaitable, Callable, Coroutine +from functools import wraps +from typing import Any, ParamSpec, TypeVar + +from adcp.reporting.ledger.store import LedgerConflictError +from adcp.reporting.materializer.contracts import ReportingWriterError, ReportingWriterFailure + +P = ParamSpec("P") +T = TypeVar("T") + + +class ReportingMaterializerUsageError(ValueError): + """A deterministic caller-argument rejection with a fixed SDK message. + + Raised only by the SDK's own argument validators, before any store or + destination work. It stays a ``ValueError`` so existing adopter handling is + unchanged, and the guard re-raises it instead of reporting an unknown + external effect that never happened. + """ + + +def materializer_errors(method: Callable[P, Awaitable[T]]) -> Callable[P, Coroutine[Any, Any, T]]: + @wraps(method) + async def guarded(*args: P.args, **kwargs: P.kwargs) -> T: + try: + return await method(*args, **kwargs) + except (ReportingWriterError, LedgerConflictError, ReportingMaterializerUsageError): + raise + except asyncio.CancelledError: + canceled = True + except Exception: + canceled = False + # Outside the handler: neither a provider's message nor exception chain + # survives. Commit uncertainty is always resumed with the same identity. + if canceled: + raise asyncio.CancelledError + raise ReportingWriterError( + ReportingWriterFailure("RESOURCE_UNAVAILABLE", "same_identity", "unknown") + ) + + return guarded diff --git a/src/adcp/reporting/materializer/capture.py b/src/adcp/reporting/materializer/capture.py new file mode 100644 index 000000000..f2f1aee89 --- /dev/null +++ b/src/adcp/reporting/materializer/capture.py @@ -0,0 +1,159 @@ +"""Immutable finish boundaries and a real, isolated notification enqueue path. + +B2.1 does not expose a delivery worker or an activation certificate. A/B/C +workers cannot see this queue. Pre-activation events stay quarantined forever. +B2.4 must prove the mounted projection and tier components before admitting new +events in the original verified-finish transaction; it cannot release old ones. +""" + +from __future__ import annotations + +from dataclasses import dataclass, field, replace +from datetime import datetime +from typing import Any + +from pydantic import TypeAdapter, ValidationError + +from adcp.reporting.evidence import aware_utc, reporting_identifier +from adcp.reporting.ledger._delivery_state import decode_record, payload, principal +from adcp.reporting.ledger.delivery_models import ( + ReportingDeliveryPrincipal, + ReportingDeliveryRecord, + ReportingMaterializationRecord, +) +from adcp.reporting.ledger.notification_models import ( + ReportingDomainEvent, + ReportingNotificationError, +) +from adcp.reporting.ledger.status_projection import ReportingStatusSnapshot +from adcp.reporting.outbox.memory import NotificationState + +_CORE = TypeAdapter(ReportingStatusSnapshot) + + +@dataclass(frozen=True) +class ReportingMaterializerBoundary: + """Private frozen projection inputs; never returned as a public wire blob.""" + + caller: ReportingDeliveryPrincipal + sequence: int + account_sequence: int + reporting_materialization_id: str + as_of: datetime + core: ReportingStatusSnapshot = field(repr=False) + reconciliation: tuple[ReportingDeliveryRecord, ...] = field(repr=False) + + def __post_init__(self) -> None: + outcomes = tuple( + record + for record in self.reconciliation + if isinstance(record, ReportingMaterializationRecord) + and record.reporting_materialization_id == self.reporting_materialization_id + ) + if ( + type(self.sequence) is not int + or self.sequence < 1 + or type(self.account_sequence) is not int + or self.account_sequence < self.sequence + or self.core.account_id != self.caller.account_id + or self.core.as_of != self.as_of + or self.core.consumer_ids != (self.caller.consumer_id,) + or any(s.consumer_id != self.caller.consumer_id for s in self.core.statuses) + or any( + i.consumer_id not in {None, self.caller.consumer_id} for i in self.core.lifecycles + ) + or any(principal(r) != self.caller for r in self.reconciliation) + or len(outcomes) != 1 + or outcomes[0].completed_at != self.as_of + ): + raise ReportingNotificationError("materializer_boundary_invalid") + reporting_identifier(self.reporting_materialization_id, maximum=255) + object.__setattr__(self, "as_of", aware_utc(self.as_of)) + + def to_storage(self) -> dict[str, Any]: + return { + "version": 1, + "account_id": self.caller.account_id, + "consumer_id": self.caller.consumer_id, + "sequence": self.sequence, + "account_sequence": self.account_sequence, + "reporting_materialization_id": self.reporting_materialization_id, + "as_of": self.as_of.isoformat(), + "core": _CORE.dump_python(self.core, mode="json"), + "reconciliation": [payload(r) for r in self.reconciliation], + } + + +def decode_materializer_boundary(value: dict[str, Any]) -> ReportingMaterializerBoundary: + result = None + try: + if type(value) is dict and type(value.get("version")) is int and value["version"] == 1: + result = ReportingMaterializerBoundary( + ReportingDeliveryPrincipal(value["account_id"], value["consumer_id"]), + value["sequence"], + value["account_sequence"], + value["reporting_materialization_id"], + datetime.fromisoformat(value["as_of"]), + _CORE.validate_python(value["core"]), + tuple(decode_record(r) for r in value["reconciliation"]), + ) + except (ValueError, TypeError, KeyError, ValidationError): + # Convert malformed persisted input to the closed protocol error below. + pass + if result is None or result.to_storage() != value: + raise ReportingNotificationError("materializer_boundary_invalid") + return result + + +class _MaterializerQueueConnection: + """Closed table substitution, preserving the finish transaction's connection.""" + + def __init__(self, connection: Any) -> None: + self.connection = connection + + async def execute(self, query: str, params: Any = None) -> Any: + return await self.connection.execute( + query.replace("reporting_notification_", "reporting_materializer_notification_"), params + ) + + +async def enqueue_materializer_event_on(connection: Any, event: ReportingDomainEvent) -> None: + from adcp.reporting.outbox.pg import enqueue_event + + if event.notification_type != "reporting.delivery_ready": + raise ReportingNotificationError("materializer_event_invalid") + await enqueue_event(_MaterializerQueueConnection(connection), event) + + +def private_snapshot( + snapshot: ReportingStatusSnapshot, caller: ReportingDeliveryPrincipal +) -> ReportingStatusSnapshot: + """Internal boundaries also exclude other consumers' private statements.""" + statuses = tuple(s for s in snapshot.statuses if s.consumer_id == caller.consumer_id) + status_ids = {s.reporting_status_id for s in statuses} + issues = tuple(i for i in snapshot.lifecycles if i.consumer_id in {None, caller.consumer_id}) + issue_ids = {i.issue_id for i in issues} + return replace( + snapshot, + statuses=statuses, + lifecycles=issues, + consumer_ids=(caller.consumer_id,), + issue_scopes=tuple((i, scope) for i, scope in snapshot.issue_scopes if i in issue_ids), + changes=tuple( + c for c in snapshot.changes if c[1] != "consumer_status" or c[2] in status_ids + ), + ) + + +class MaterializerNotificationState(NotificationState): + """Pre-activation events remain permanently quarantined, including on restart.""" + + def enqueue(self, event: ReportingDomainEvent) -> None: + super().enqueue(event) + for (account, consumer, notification_id, _), work in self.expansions.items(): + if (account, consumer, notification_id) == ( + event.account_id, + event.consumer_namespace, + event.notification_id, + ): + work.state = "quarantined" diff --git a/src/adcp/reporting/materializer/memory.py b/src/adcp/reporting/materializer/memory.py new file mode 100644 index 000000000..dd922be0a --- /dev/null +++ b/src/adcp/reporting/materializer/memory.py @@ -0,0 +1,680 @@ +"""Deterministic in-memory model of the durable materializer state machine. + +This implementation is for conformance and never establishes production tier +readiness. Mutations, including disabled-notification turns, roll back together. +""" + +from __future__ import annotations + +from dataclasses import dataclass, replace +from datetime import datetime, timedelta +from typing import Any +from uuid import uuid4 + +from adcp.reporting.ledger._delivery_state import RecordT +from adcp.reporting.ledger.delivery import InMemoryReportingReconciliationStore +from adcp.reporting.ledger.delivery_models import ( + ReportingDeliveryPrincipal, + ReportingDeliveryScope, + ReportingDestinationBinding, + ReportingMaterializationAttempt, + ReportingMaterializationCheck, + ReportingMaterializationRecord, + ReportingObligationDeliveryRecord, +) +from adcp.reporting.ledger.models import ReportingConfiguration +from adcp.reporting.ledger.notification_events import delivery_dirty, materialization_event +from adcp.reporting.ledger.store import LedgerConflictError +from adcp.reporting.materializer._errors import ( + ReportingMaterializerUsageError, + materializer_errors, +) +from adcp.reporting.materializer.capture import ( + MaterializerNotificationState, + ReportingMaterializerBoundary, + private_snapshot, +) +from adcp.reporting.materializer.contracts import ( + ReportingDestinationRequest, + ReportingPreparedRevision, + ReportingVerificationKey, + ReportingWriterError, + ReportingWriterFailure, + binding_fingerprint, + failure, +) +from adcp.reporting.materializer.verification import ( + ReportingVerifiedDestination, + validate_materialization_target, + validate_verified_destination, +) +from adcp.reporting.materializer.work import ( + MaterializerContext, + MaterializerReason, + ReportingMaterializerLease, + ReportingMaterializerTurn, + key_for, + public_failure, + validate_lease_seconds, + verification_key_id, +) + + +@dataclass +class _Candidate: + scope: ReportingDeliveryScope + generation: int = 1 + due_at: datetime | None = None + reason: MaterializerReason = "ready" + turn: int = 0 + + +@dataclass +class _Work: + scope: ReportingDeliveryScope + generation: int + attempt: ReportingMaterializationAttempt + request: ReportingDestinationRequest + due_at: datetime | None + notifications_enabled: bool + # This reservation can never produce a production-admitted readiness event, + # including when a later installation resumes an uncertain external effect. + admission_epoch: int = 0 + token: str | None = None + lease_until: datetime | None = None + completion_token: str | None = None + acked: bool = False + retry_allowed: bool = False + reason: MaterializerReason = "ready" + + +class InMemoryReportingMaterializerStore(InMemoryReportingReconciliationStore): + def __init__(self, **kwargs: Any) -> None: + super().__init__(**kwargs) + self._materializer_candidates: dict[ReportingDeliveryScope, _Candidate] = {} + self._materializer_work: dict[tuple[str, str, str], _Work] = {} + self._materializer_account_turns: dict[str, int] = {} + self._materializer_turn = 0 + self._materializer_outbox = ( + MaterializerNotificationState() if self._notification_state is not None else None + ) + self._materializer_boundaries: list[ReportingMaterializerBoundary] = [] + self._materializer_status_heads: dict[ReportingDeliveryPrincipal, int] = {} + self._materializer_account_heads: dict[str, int] = {} + + def _wake(self, scope: ReportingDeliveryScope) -> None: + candidate = self._materializer_candidates.get(scope) + if candidate is None: + candidate = _Candidate(scope) + self._materializer_candidates[scope] = candidate + else: + candidate.generation += 1 + candidate.due_at, candidate.reason = self._clock(), "ready" + + def _wake_obligation(self, account_id: str, obligation_id: str) -> None: + obligation = self._obligations.get(obligation_id) + if obligation is None or obligation.account_id != account_id: + return + for _, _, record in self._retained_delivery_records(): + if isinstance(record, ReportingDestinationBinding) and ( + record.generation_key == obligation.generation_key + ): + self._wake( + ReportingDeliveryScope( + obligation.generation_key, record.consumer_id, obligation_id + ) + ) + + def _append(self, account_id: str, kind: Any, record_id: str) -> None: + super()._append(account_id, kind, record_id) + if kind == "obligation": + self._wake_obligation(account_id, record_id) + elif kind == "revision": + self._wake_obligation(account_id, self._revisions[record_id].reporting_obligation_id) + + async def put_configuration(self, configuration: ReportingConfiguration) -> None: + async with self._mutation(): + before = self._configurations.get(configuration.generation_key) + await super().put_configuration(configuration) + if before != configuration: + for scope in self._materializer_candidates: + if scope.generation_key == configuration.generation_key: + self._wake(scope) + + async def set_revision_readable( + self, + *, + account_id: str, + reporting_revision_id: str, + readable: bool, + ) -> None: + async with self._mutation(): + before = self._revisions.get(reporting_revision_id) + await super().set_revision_readable( + account_id=account_id, + reporting_revision_id=reporting_revision_id, + readable=readable, + ) + revision = self._revisions.get(reporting_revision_id) + if revision is not None and revision != before: + self._wake_obligation(account_id, revision.reporting_obligation_id) + + def _commit_record_unlocked( + self, record: RecordT, *, notify: bool = True, dirty: bool = True + ) -> tuple[RecordT, bool]: + # Only the fenced verified finish may enqueue a readiness intent. An + # ordinary public outcome keeps the projection dirty work it has always + # produced; suppressing that would silently stall the status projector. + stored, added = super()._commit_record_unlocked( + record, + notify=notify and not isinstance(record, ReportingMaterializationRecord), + dirty=dirty, + ) + if added: + if isinstance(record, ReportingDestinationBinding): + for obligation in self._obligations.values(): + if obligation.generation_key == record.generation_key: + self._wake( + ReportingDeliveryScope( + obligation.generation_key, + record.consumer_id, + obligation.reporting_obligation_id, + ) + ) + elif isinstance(record, ReportingMaterializationCheck): + self._wake(record.scope) + return stored, added + + def _context(self, scope: ReportingDeliveryScope) -> MaterializerContext: + records = tuple(c.record for c in self._caller_changes(scope.principal)) + binding = next( + ( + r + for r in records + if isinstance(r, ReportingDestinationBinding) + and r.generation_key == scope.generation_key + ), + None, + ) + obligation = self._obligations.get(scope.reporting_obligation_id) + configuration = self._configurations.get(scope.generation_key) + if ( + binding is None + or obligation is None + or configuration is None + or obligation.generation_key != scope.generation_key + ): + raise failure("BINDING_MISMATCH") + delivery = next( + ( + r + for r in records + if isinstance(r, ReportingObligationDeliveryRecord) and r.scope == scope + ), + None, + ) + return MaterializerContext( + configuration, + obligation, + binding, + delivery, + tuple( + r + for r in self._revisions.values() + if r.account_id == scope.principal.account_id + and r.reporting_obligation_id == scope.reporting_obligation_id + ), + records, + ) + + def _work_key(self, attempt: ReportingMaterializationAttempt) -> tuple[str, str, str]: + return ( + attempt.scope.principal.account_id, + attempt.scope.consumer_id, + attempt.reporting_materialization_id, + ) + + def _park( + self, + candidate: _Candidate, + reason: MaterializerReason, + due: datetime | None = None, + ) -> ReportingMaterializerTurn: + candidate.reason, candidate.due_at = reason, due + return ReportingMaterializerTurn("parked", reason) + + @materializer_errors + async def claim_materialization( + self, + *, + keys: tuple[ReportingVerificationKey, ...], + lease_seconds: int = 30, + ) -> ReportingMaterializerLease | ReportingMaterializerTurn: + validate_lease_seconds(lease_seconds) + async with self._mutation(): + now = self._clock() + pending = {w.scope for w in self._materializer_work.values() if not w.acked} + works = [ + w + for w in self._materializer_work.values() + if not w.acked + and w.due_at is not None + and w.due_at <= now + and (w.lease_until is None or w.lease_until <= now) + ] + candidates = [ + c + for c in self._materializer_candidates.values() + if c.due_at is not None and c.due_at <= now and c.scope not in pending + ] + accounts = {w.scope.principal.account_id for w in works} | { + c.scope.principal.account_id for c in candidates + } + if not accounts: + return ReportingMaterializerTurn("idle") + account_id = min( + accounts, key=lambda a: (self._materializer_account_turns.get(a, 0), a) + ) + self._materializer_turn += 1 + self._materializer_account_turns[account_id] = self._materializer_turn + account_work = [w for w in works if w.scope.principal.account_id == account_id] + if account_work: + work = min(account_work, key=lambda w: (w.due_at or now, w.request.external_id)) + return self._lease(work, keys, lease_seconds) + candidate = min( + (c for c in candidates if c.scope.principal.account_id == account_id), + key=lambda c: (c.turn, c.scope.consumer_id, c.scope.reporting_obligation_id), + ) + candidate.turn = self._materializer_turn + try: + context = self._context(candidate.scope) + key = key_for(context.binding, context.obligation, keys) + except LedgerConflictError: + return self._park(candidate, "history_corrupt") + except ReportingWriterError: + return self._park(candidate, "component_unavailable") + revision, reason = context.selection(now) + if revision is None or reason != "ready": + at = context.configuration.activated_at + return self._park(candidate, reason, at if at is not None and at > now else None) + attempts = context.attempts(revision.reporting_revision_id) + if tuple(a.attempt for a in attempts) != tuple(range(1, len(attempts) + 1)): + return self._park(candidate, "history_corrupt") + if context.pending_attempts(): + return self._park(candidate, "legacy_pending") + if attempts: + outcome = context.outcome(attempts[-1]) + assert outcome is not None + if outcome.status != "failed": + reason, expires = context.retained_success(outcome, now) + return self._park(candidate, reason, expires) + owned = self._materializer_work.get(self._work_key(attempts[-1])) + if owned is None or not owned.retry_allowed: + return self._park( + candidate, "legacy_terminal" if owned is None else "operator_required" + ) + if context.delivery is None: + if context.obligation.currency is None: + return self._park(candidate, "operator_required") + delivery = ReportingObligationDeliveryRecord( + candidate.scope, + context.obligation.currency, + max(now, context.obligation.period.end) + + timedelta(days=context.binding.resource_retention_days), + now, + ) + self._commit_record_unlocked(delivery, notify=False, dirty=False) + attempt = ReportingMaterializationAttempt( + candidate.scope, + revision.reporting_revision_id, + "rpm_" + uuid4().hex, + len(attempts) + 1, + now, + ) + self._commit_record_unlocked(attempt, notify=False, dirty=False) + request = ReportingDestinationRequest.from_binding(context.binding, attempt, key) + work = _Work( + candidate.scope, + candidate.generation, + attempt, + request, + now, + self._notification_state is not None, + ) + self._materializer_work[self._work_key(attempt)] = work + self._park(candidate, "ready") + return self._lease(work, keys, lease_seconds) + + def _lease( + self, + work: _Work, + keys: tuple[ReportingVerificationKey, ...], + seconds: int, + ) -> ReportingMaterializerLease | ReportingMaterializerTurn: + if work.notifications_enabled != (self._notification_state is not None): + return self._park_work(work, "component_unavailable") + try: + context = self._context(work.scope) + if not context.resumable(work.attempt): + return self._park_work(work, "history_corrupt") + key = key_for( + context.binding, + context.obligation, + keys, + required=verification_key_id(work.request.verification_key), + ) + if ( + ReportingDestinationRequest.from_binding(context.binding, work.attempt, key) + != work.request + ): + raise failure("BINDING_MISMATCH") + except (ReportingWriterError, LedgerConflictError): + return self._park_work(work, "component_unavailable") + work.token = str(uuid4()) + work.lease_until = self._clock() + timedelta(seconds=seconds) + work.due_at = work.lease_until + return ReportingMaterializerLease( + work.scope, + work.generation, + work.token, + work.lease_until, + work.attempt, + work.request, + context, + work.notifications_enabled, + ) + + def _park_work(self, work: _Work, reason: MaterializerReason) -> ReportingMaterializerTurn: + work.due_at, work.reason = None, reason + work.token, work.lease_until = None, None + return self._park(self._materializer_candidates[work.scope], reason) + + def _held(self, lease: ReportingMaterializerLease) -> _Work | None: + work = self._materializer_work.get(self._work_key(lease.attempt)) + if ( + work is None + or work.acked + or work.token != lease.token + or work.lease_until is None + or work.lease_until <= self._clock() + ): + return None + if ( + work.request != lease.request + or work.generation != lease.generation + or work.notifications_enabled != lease.notifications_enabled + or work.notifications_enabled != (self._notification_state is not None) + ): + raise failure("BINDING_MISMATCH") + return work + + def _target( + self, lease: ReportingMaterializerLease + ) -> tuple[MaterializerContext, MaterializerReason]: + context = self._context(lease.scope) + selected, reason = context.selection(self._clock()) + if reason != "ready": + return context, reason + candidate = self._materializer_candidates[lease.scope] + if ( + candidate.generation != lease.generation + or selected is None + or selected.reporting_revision_id != lease.attempt.reporting_revision_id + ): + return context, "target_changed" + if binding_fingerprint(context.binding) != lease.request.binding_fingerprint: + return context, "binding_changed" + return context, "ready" + + @materializer_errors + async def renew_materialization( + self, lease: ReportingMaterializerLease, *, lease_seconds: int + ) -> bool: + validate_lease_seconds(lease_seconds) + async with self._mutation(): + work = self._held(lease) + if work is None: + return False + work.lease_until = self._clock() + timedelta(seconds=lease_seconds) + work.due_at = work.lease_until + return True + + @materializer_errors + async def authorize_materialization(self, lease: ReportingMaterializerLease) -> None: + async with self._mutation(): + if self._held(lease) is None: + raise failure("LEASE_LOST") + _, reason = self._target(lease) + if reason != "ready": + raise failure( + "HISTORY_CORRUPT" if reason == "history_corrupt" else "CURRENT_REVISION_CHANGED" + ) + + @materializer_errors + async def finish_materialization( + self, + lease: ReportingMaterializerLease, + *, + prepared: ReportingPreparedRevision | None = None, + verified: ReportingVerifiedDestination | None = None, + error: ReportingWriterFailure | None = None, + ) -> ReportingMaterializerTurn: + if verified is not None: + validate_verified_destination(verified, lease.request, token=lease.token) + if prepared is None or prepared.request != lease.request or error is not None: + raise failure("BINDING_MISMATCH") + elif error is None: + raise failure("BINDING_MISMATCH") + async with self._mutation(): + work = self._held(lease) + if work is None: + previous = self._materializer_work.get(self._work_key(lease.attempt)) + if ( + previous is not None + and previous.acked + and previous.completion_token == lease.token + and previous.request == lease.request + ): + return ReportingMaterializerTurn( + "verified" if previous.reason == "verified" else "failed", + previous.reason, + lease.attempt.reporting_materialization_id, + ) + return ReportingMaterializerTurn( + "pending", "effect_unknown", lease.attempt.reporting_materialization_id + ) + context, reason = self._target(lease) + if reason != "ready": + error = ReportingWriterFailure("CURRENT_REVISION_CHANGED", "new_attempt", "applied") + verified = None + elif verified is not None and prepared is not None: + validate_materialization_target( + prepared, binding=context.binding, revisions=context.revisions + ) + if error is not None and ( + error.effect == "unknown" + or error.retry == "same_identity" + or error.code in {"DEADLINE_EXCEEDED", "LEASE_LOST"} + ): + work.token, work.lease_until, work.reason = None, None, "effect_unknown" + work.due_at = self._clock() + timedelta( + seconds=max(1, min(error.retry_after_seconds or 5, 300)) + ) + return ReportingMaterializerTurn( + "pending", work.reason, work.attempt.reporting_materialization_id + ) + now = self._clock() + if verified is not None and ( + context.delivery is None + or verified.resource.expires_at + < max( + context.delivery.resource_retained_until, + now + timedelta(days=context.binding.resource_retention_days), + ) + ): + error = ReportingWriterFailure("RESOURCE_UNAVAILABLE", "new_attempt", "applied") + verified = None + if verified is not None: + # Validate again under the finish lock, immediately before + # copying observations into the immutable terminal record. + validate_verified_destination(verified, lease.request, token=lease.token) + outcome = ReportingMaterializationRecord( + lease.scope, + lease.attempt.reporting_revision_id, + lease.attempt.reporting_materialization_id, + context.binding.success_status if verified is not None else "failed", + now, + verified.resource if verified is not None else None, + replace(verified.verification, verified_at=now) if verified is not None else None, + public_failure(error) if error is not None else None, + ) + stored, inserted = self._commit_record_unlocked(outcome, notify=False, dirty=False) + self._materializer_dirty(stored, context) + if inserted and verified is not None and self._notification_state is not None: + revision = next( + r + for r in context.revisions + if r.reporting_revision_id == stored.reporting_revision_id + ) + event = materialization_event( + stored, + context.records, + context.obligation, + revision, + context.configuration, + now, + ) + if event is None: + raise failure("BINDING_MISMATCH") + assert self._materializer_outbox is not None + self._materializer_outbox.enqueue(event) + if ( + self._materializer_outbox.events.get( + (event.account_id, event.consumer_namespace, event.notification_id) + ) + != event + ): + raise failure("BINDING_MISMATCH") + if self._held(lease) is None: + raise failure("LEASE_LOST") + work.completion_token = work.token + work.acked, work.token, work.lease_until = True, None, None + work.retry_allowed = error is not None and error.retry == "new_attempt" + if reason == "ready": + reason = ( + "verified" + if verified is not None + else ("retry" if work.retry_allowed else "operator_required") + ) + work.reason = reason + due = ( + now + timedelta(seconds=max(1, error.retry_after_seconds or 5)) + if work.retry_allowed and error + else None + ) + if reason == "target_changed": + due = now + elif reason not in {"retry", "verified"}: + due = None + if ( + reason == "inactive" + and context.configuration.activated_at is not None + and context.configuration.activated_at > now + ): + due = context.configuration.activated_at + if verified is not None: + due = verified.resource.expires_at + self._park(self._materializer_candidates[lease.scope], reason, due) + return ReportingMaterializerTurn( + "verified" if verified is not None else "failed", + reason, + stored.reporting_materialization_id, + ) + + def _materializer_dirty( + self, outcome: ReportingMaterializationRecord, context: MaterializerContext + ) -> None: + scope, reason, evidence = delivery_dirty(outcome, context.obligation) + self._dirty_status(scope, reason, after=evidence) + + from adcp.reporting.ledger.status_snapshot import settle_memory_snapshot + + core = settle_memory_snapshot(self, scope.account_id) + core = replace(core, as_of=outcome.completed_at) + sequence = self._materializer_status_heads.get(outcome.scope.principal, 0) + 1 + self._materializer_status_heads[outcome.scope.principal] = sequence + account_sequence = self._materializer_account_heads.get(scope.account_id, 0) + 1 + self._materializer_account_heads[scope.account_id] = account_sequence + self._materializer_boundaries.append( + ReportingMaterializerBoundary( + outcome.scope.principal, + sequence, + account_sequence, + outcome.reporting_materialization_id, + outcome.completed_at, + private_snapshot(core, outcome.scope.principal), + tuple(c.record for c in self._caller_changes(outcome.scope.principal)), + ) + ) + + @materializer_errors + async def read_materializer_boundaries( + self, *, caller: ReportingDeliveryPrincipal, after: int = 0, limit: int = 100 + ) -> tuple[ReportingMaterializerBoundary, ...]: + if type(after) is not int or after < 0 or type(limit) is not int or not 1 <= limit <= 100: + raise ReportingMaterializerUsageError( + "materializer boundary reads require bounded positions" + ) + async with self._lock: + return tuple( + b + for b in self._materializer_boundaries + if b.caller == caller and b.sequence > after + )[:limit] + + @materializer_errors + async def import_pending_materialization( + self, + *, + scope: ReportingDeliveryScope, + reporting_materialization_id: str, + original_external_id: str, + keys: tuple[ReportingVerificationKey, ...], + ) -> None: + async with self._mutation(): + context = self._context(scope) + attempt = next( + ( + r + for r in context.records + if isinstance(r, ReportingMaterializationAttempt) + and r.reporting_materialization_id == reporting_materialization_id + and r.scope == scope + ), + None, + ) + if attempt is None or context.outcome(attempt) is not None: + raise failure("BINDING_MISMATCH") + if not context.resumable(attempt): + raise failure("HISTORY_CORRUPT") + key = key_for(context.binding, context.obligation, keys) + request = ReportingDestinationRequest.from_binding(context.binding, attempt, key) + if original_external_id != request.external_id: + raise failure("BINDING_MISMATCH") + if scope not in self._materializer_candidates: + self._wake(scope) + existing = self._materializer_work.get(self._work_key(attempt)) + if existing is not None: + if existing.token is None and not existing.acked: + existing.due_at = self._clock() + return + if any(w.scope == scope and not w.acked for w in self._materializer_work.values()): + raise failure("BINDING_MISMATCH") + self._materializer_work[self._work_key(attempt)] = _Work( + scope, + self._materializer_candidates[scope].generation, + attempt, + request, + self._clock(), + self._notification_state is not None, + ) diff --git a/src/adcp/reporting/materializer/pg.py b/src/adcp/reporting/materializer/pg.py new file mode 100644 index 000000000..990ce2405 --- /dev/null +++ b/src/adcp/reporting/materializer/pg.py @@ -0,0 +1,968 @@ +"""PostgreSQL account-first reservation, fenced resume and verified atomic finish.""" + +from __future__ import annotations + +from dataclasses import replace +from datetime import datetime, timedelta +from importlib.resources import files +from typing import Any, cast +from uuid import uuid4 + +from adcp.reporting.ledger._delivery_state import RecordT +from adcp.reporting.ledger.delivery_models import ( + ReportingDeliveryPrincipal, + ReportingDeliveryScope, + ReportingDestinationBinding, + ReportingMaterializationAttempt, + ReportingMaterializationRecord, + ReportingObligationDeliveryRecord, +) +from adcp.reporting.ledger.delivery_pg import PgReportingReconciliationStore +from adcp.reporting.ledger.models import ReportingConfigurationGenerationKey +from adcp.reporting.ledger.notification_events import materialization_event +from adcp.reporting.ledger.pg import ( + _OBLIGATION_COLUMNS, + _REVISION_COLUMNS, + _configuration_from_row, + _obligation_from_row, + _revision_from_row, +) +from adcp.reporting.ledger.store import LedgerConflictError +from adcp.reporting.materializer._errors import ( + ReportingMaterializerUsageError, + materializer_errors, +) +from adcp.reporting.materializer.capture import ( + ReportingMaterializerBoundary, + decode_materializer_boundary, + enqueue_materializer_event_on, + private_snapshot, +) +from adcp.reporting.materializer.contracts import ( + ReportingDestinationRequest, + ReportingPreparedRevision, + ReportingVerificationKey, + ReportingWriterError, + ReportingWriterFailure, + binding_fingerprint, + failure, +) +from adcp.reporting.materializer.schema import validate_materializer_schema +from adcp.reporting.materializer.verification import ( + ReportingVerifiedDestination, + validate_materialization_target, + validate_verified_destination, +) +from adcp.reporting.materializer.work import ( + MaterializerContext, + MaterializerReason, + ReportingMaterializerLease, + ReportingMaterializerTurn, + key_for, + public_failure, + validate_lease_seconds, + verification_key_id, +) + +_SCOPE_WHERE = ( + "account_id=%s AND consumer_id=%s AND delivery_config_id=%s" + " AND delivery_config_version=%s AND reporting_obligation_id=%s" +) + + +def _scope_args(scope: ReportingDeliveryScope) -> tuple[str | int, ...]: + generation = scope.generation_key + return ( + generation.account_id, + scope.consumer_id, + generation.delivery_config_id, + generation.delivery_config_version, + scope.reporting_obligation_id, + ) + + +def _scope(row: dict[str, Any]) -> ReportingDeliveryScope: + return ReportingDeliveryScope( + ReportingConfigurationGenerationKey( + row["account_id"], row["delivery_config_id"], row["delivery_config_version"] + ), + row["consumer_id"], + row["reporting_obligation_id"], + ) + + +async def _now(connection: Any) -> datetime: + row = await (await connection.execute("SELECT clock_timestamp()")).fetchone() + assert row is not None + return cast(datetime, row[0]) + + +class PgReportingMaterializerStore(PgReportingReconciliationStore): + """Additive durable extension. No account enumeration or destination I/O. + + Each turn samples at most sixteen due accounts without row locks, tries the + Core advisory lock first, then claims within that account. Long work returns + its connection before any source/destination call; even a size-one pool can + renew its lease. Clock overrides affect old conformance APIs only: work time + is always PostgreSQL time. + """ + + # Only a read-only sampling position, never a lease or durable correctness + # boundary. Walk past busy accounts in bounded pages, wrapping after the end. + # Restart may repeat a page; it cannot lose or acknowledge work. + _materializer_sample_after: tuple[str, str] | None = None + + async def _commit_record_on( + self, connection: Any, record: RecordT, *, notify: bool = True, dirty: bool = True + ) -> tuple[RecordT, bool]: + # Only the fenced verified finish may enqueue a readiness intent. An + # ordinary public outcome keeps the projection dirty work it has always + # produced; suppressing that would silently stall the status projector. + return await super()._commit_record_on( + connection, + record, + notify=notify and not isinstance(record, ReportingMaterializationRecord), + dirty=dirty, + ) + + @materializer_errors + async def create_schema(self) -> None: + async with self._connection() as connection, connection.transaction(): + await self._create_schema_on(connection) + await connection.execute( + files("adcp.reporting.ledger").joinpath("reporting_materializer.sql").read_text() + ) + + @materializer_errors + async def materializer_ready(self) -> bool: + async with self._connection() as connection: + await validate_materializer_schema( + connection, notifications=self._notifications_enabled + ) + return True + + async def _materializer_context_on( + self, connection: Any, scope: ReportingDeliveryScope + ) -> MaterializerContext: + records = await self._records(connection, scope.principal) + binding = next( + ( + r + for r in records + if isinstance(r, ReportingDestinationBinding) + and r.generation_key == scope.generation_key + ), + None, + ) + obligation_row = await ( + await connection.execute( + f"SELECT {_OBLIGATION_COLUMNS} FROM reporting_obligations" # nosec B608 + " WHERE account_id=%s AND reporting_obligation_id=%s", + (scope.principal.account_id, scope.reporting_obligation_id), + ) + ).fetchone() + config_row = await ( + await connection.execute( + "SELECT delivery_config_id, delivery_config_version, account_id," + " report_definition_id, reporting_profile, feed_purpose, required_finality," + " account_timezone, schedule, media_buy_ids, activated_at, deactivated_at," + " automated_recovery_seconds, status_retention_days, definition," + " authoritative_party" + " FROM reporting_configurations WHERE account_id=%s" + " AND delivery_config_id=%s AND delivery_config_version=%s", + ( + scope.principal.account_id, + scope.generation_key.delivery_config_id, + scope.generation_key.delivery_config_version, + ), + ) + ).fetchone() + revision_rows = await ( + await connection.execute( + f"SELECT {_REVISION_COLUMNS} FROM reporting_revisions" # nosec B608 + " WHERE account_id=%s AND reporting_obligation_id=%s", + (scope.principal.account_id, scope.reporting_obligation_id), + ) + ).fetchall() + if binding is None or obligation_row is None or config_row is None: + raise failure("BINDING_MISMATCH") + obligation = _obligation_from_row(obligation_row) + configuration = _configuration_from_row(config_row) + if obligation.generation_key != scope.generation_key: + raise failure("BINDING_MISMATCH") + delivery = next( + ( + r + for r in records + if isinstance(r, ReportingObligationDeliveryRecord) and r.scope == scope + ), + None, + ) + return MaterializerContext( + configuration, + obligation, + binding, + delivery, + tuple(_revision_from_row(r) for r in revision_rows), + records, + ) + + async def _schedule_account_on(self, connection: Any, account_id: str) -> None: + await connection.execute( + "UPDATE reporting_materializer_accounts SET due_at=(SELECT min(due) FROM (" + " SELECT due_at AS due FROM reporting_materializer_work" + " WHERE account_id=%s AND state='pending'" + " UNION ALL SELECT c.due_at FROM reporting_materializer_candidates c" + " WHERE c.account_id=%s AND c.due_at IS NOT NULL AND NOT EXISTS (" + " SELECT 1 FROM reporting_materializer_work w WHERE w.account_id=c.account_id" + " AND w.consumer_id=c.consumer_id AND w.delivery_config_id=c.delivery_config_id" + " AND w.delivery_config_version=c.delivery_config_version" + " AND w.reporting_obligation_id=c.reporting_obligation_id AND w.state='pending')" + " UNION ALL SELECT clock_timestamp() FROM reporting_materializer_discovery" + " WHERE account_id=%s AND NOT complete) ready) WHERE account_id=%s", + (account_id,) * 4, + ) + + async def _discover_on(self, connection: Any, account_id: str) -> bool: + row = await ( + await connection.execute( + "SELECT consumer_id, delivery_config_id, delivery_config_version," + " after_obligation_id" + " FROM reporting_materializer_discovery WHERE account_id=%s AND NOT complete" + " ORDER BY consumer_id, delivery_config_id, delivery_config_version" + " LIMIT 1 FOR UPDATE", + (account_id,), + ) + ).fetchone() + if row is None: + return False + consumer, config, version, after = row + obligations = await ( + await connection.execute( + "SELECT reporting_obligation_id FROM reporting_obligations WHERE account_id=%s" + " AND delivery_config_id=%s AND delivery_config_version=%s" + " AND reporting_obligation_id>%s ORDER BY reporting_obligation_id LIMIT 32", + (account_id, config, version, after), + ) + ).fetchall() + for (obligation_id,) in obligations: + # Only missing candidates: a later backfill batch never invalidates + # a worker already reserved through the publication trigger. + await connection.execute( + "INSERT INTO reporting_materializer_candidates" + " (account_id,consumer_id,delivery_config_id,delivery_config_version," + " reporting_obligation_id,due_at) VALUES (%s,%s,%s,%s,%s,clock_timestamp())" + " ON CONFLICT DO NOTHING", + (account_id, consumer, config, version, obligation_id), + ) + await connection.execute( + "UPDATE reporting_materializer_discovery SET after_obligation_id=%s,complete=%s" + " WHERE account_id=%s AND consumer_id=%s AND delivery_config_id=%s" + " AND delivery_config_version=%s", + ( + obligations[-1][0] if obligations else after, + len(obligations) < 32, + account_id, + consumer, + config, + version, + ), + ) + return True + + async def _park_on( + self, + connection: Any, + scope: ReportingDeliveryScope, + reason: MaterializerReason, + *, + due_at: datetime | None = None, + ) -> ReportingMaterializerTurn: + await connection.execute( + "UPDATE reporting_materializer_candidates SET reason=%s,due_at=%s," + f" served_at=clock_timestamp() WHERE {_SCOPE_WHERE}", # nosec B608 + (reason, due_at, *_scope_args(scope)), + ) + return ReportingMaterializerTurn("parked", reason) + + @materializer_errors + async def claim_materialization( + self, *, keys: tuple[ReportingVerificationKey, ...], lease_seconds: int = 30 + ) -> ReportingMaterializerLease | ReportingMaterializerTurn: + validate_lease_seconds(lease_seconds) + await self.materializer_ready() + async with self._connection() as connection: + # Read-only sampling. Never lock global candidate rows before the + # account advisory lock, including when a competing worker is busy. + for _ in range(2): + # Keep the cursor serialized, but order native timestamps: the + # served_at::text output alias otherwise makes fairness locale-dependent. + after = self._materializer_sample_after + if after: + query = ( + "SELECT account_id,served_at::text FROM reporting_materializer_accounts" + " WHERE due_at<=%s AND (served_at,account_id)>(%s::timestamptz,%s)" + " ORDER BY reporting_materializer_accounts.served_at,account_id LIMIT 16" + ) + else: + query = ( + "SELECT account_id,served_at::text FROM reporting_materializer_accounts" + " WHERE due_at<=%s" + " ORDER BY reporting_materializer_accounts.served_at,account_id LIMIT 16" + ) + accounts = await ( + await connection.execute( + query, + (await _now(connection), *(after or ())), + ) + ).fetchall() + if accounts: + break + self._materializer_sample_after = None + if after is None: + break + for account_id, served_at in accounts: + self._materializer_sample_after = (served_at, account_id) + async with self._connection() as connection, connection.transaction(): + held = await ( + await connection.execute( + "SELECT pg_try_advisory_xact_lock(hashtext(%s))", + (f"adcp.reporting:{account_id}",), + ) + ).fetchone() + if held is None or not held[0]: + continue + await self._lock_account(connection, account_id) + await validate_materializer_schema( + connection, notifications=self._notifications_enabled + ) + await connection.execute( + "UPDATE reporting_materializer_accounts SET served_at=clock_timestamp()" + " WHERE account_id=%s", + (account_id,), + ) + result = await self._claim_account_on(connection, account_id, keys, lease_seconds) + await self._schedule_account_on(connection, account_id) + return result + return ReportingMaterializerTurn("idle") + + async def _claim_account_on( + self, + connection: Any, + account_id: str, + keys: tuple[ReportingVerificationKey, ...], + lease_seconds: int, + ) -> ReportingMaterializerLease | ReportingMaterializerTurn: + # A bound DB instant is an indexable range predicate. clock_timestamp() + # is volatile and cannot be used as a PostgreSQL index scan boundary. + now = await _now(connection) + pending = await ( + await connection.execute( + "SELECT to_jsonb(w) FROM reporting_materializer_work w WHERE account_id=%s" + " AND state='pending' AND due_at<=%s" + " AND (lease_until IS NULL OR lease_until<=%s)" + " ORDER BY due_at,reporting_materialization_id LIMIT 1 FOR UPDATE", + (account_id, now, now), + ) + ).fetchone() + if pending is not None: + return await self._lease_on(connection, pending[0], keys, lease_seconds) + discovered = await self._discover_on(connection, account_id) + row = await ( + await connection.execute( + "SELECT to_jsonb(c) FROM reporting_materializer_candidates c WHERE account_id=%s" + " AND due_at<=%s AND NOT EXISTS (" + " SELECT 1 FROM reporting_materializer_work w WHERE w.account_id=c.account_id" + " AND w.consumer_id=c.consumer_id AND w.delivery_config_id=c.delivery_config_id" + " AND w.delivery_config_version=c.delivery_config_version" + " AND w.reporting_obligation_id=c.reporting_obligation_id AND w.state='pending')" + " ORDER BY served_at,consumer_id,reporting_obligation_id LIMIT 1 FOR UPDATE", + (account_id, await _now(connection)), + ) + ).fetchone() + if row is None: + return ReportingMaterializerTurn("discovered" if discovered else "idle") + scope = _scope(row[0]) + try: + context = await self._materializer_context_on(connection, scope) + key = key_for(context.binding, context.obligation, keys) + except LedgerConflictError: + return await self._park_on(connection, scope, "history_corrupt") + except ReportingWriterError: + return await self._park_on(connection, scope, "component_unavailable") + now = await _now(connection) + revision, reason = context.selection(now) + if reason != "ready" or revision is None: + due = context.configuration.activated_at + return await self._park_on( + connection, scope, reason, due_at=due if due is not None and due > now else None + ) + attempts = context.attempts(revision.reporting_revision_id) + if tuple(a.attempt for a in attempts) != tuple(range(1, len(attempts) + 1)): + return await self._park_on(connection, scope, "history_corrupt") + if context.pending_attempts(): + # Includes legacy pending effects on an older selected revision. + # Publication cannot make their unknown external history disappear. + return await self._park_on(connection, scope, "legacy_pending") + if attempts: + outcome = context.outcome(attempts[-1]) + assert outcome is not None + if outcome.status != "failed": + reason, expires = context.retained_success(outcome, now) + return await self._park_on(connection, scope, reason, due_at=expires) + owned = await ( + await connection.execute( + "SELECT retry_allowed FROM reporting_materializer_work WHERE account_id=%s" + " AND consumer_id=%s AND reporting_materialization_id=%s AND state='acked'", + (account_id, scope.consumer_id, attempts[-1].reporting_materialization_id), + ) + ).fetchone() + if owned is None or not owned[0]: + return await self._park_on( + connection, scope, "legacy_terminal" if owned is None else "operator_required" + ) + delivery = context.delivery + if delivery is None: + if context.obligation.currency is None: + return await self._park_on(connection, scope, "operator_required") + delivery = ReportingObligationDeliveryRecord( + scope, + context.obligation.currency, + max(now, context.obligation.period.end) + + timedelta(days=context.binding.resource_retention_days), + now, + ) + await self._commit_record_on(connection, delivery, notify=False, dirty=False) + attempt = ReportingMaterializationAttempt( + scope, revision.reporting_revision_id, "rpm_" + uuid4().hex, len(attempts) + 1, now + ) + await self._commit_record_on(connection, attempt, notify=False, dirty=False) + request = ReportingDestinationRequest.from_binding(context.binding, attempt, key) + inserted = await ( + await connection.execute( + "INSERT INTO reporting_materializer_work" + " (account_id,consumer_id,delivery_config_id,delivery_config_version," + " reporting_obligation_id,reporting_revision_id,reporting_materialization_id," + " generation,binding_sha256,verification_key_sha256,external_id," + " notifications_enabled)" + " VALUES (%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s)" + " RETURNING to_jsonb(reporting_materializer_work)", + ( + *_scope_args(scope), + attempt.reporting_revision_id, + attempt.reporting_materialization_id, + row[0]["generation"], + request.binding_fingerprint, + verification_key_id(key), + request.external_id, + self._notifications_enabled, + ), + ) + ).fetchone() + assert inserted is not None + await self._park_on(connection, scope, "ready") + return await self._lease_on(connection, inserted[0], keys, lease_seconds) + + async def _lease_on( + self, + connection: Any, + work: dict[str, Any], + keys: tuple[ReportingVerificationKey, ...], + lease_seconds: int, + ) -> ReportingMaterializerLease | ReportingMaterializerTurn: + scope = _scope(work) + try: + if work["notifications_enabled"] != self._notifications_enabled: + raise failure("UNSUPPORTED_VERIFICATION") + context = await self._materializer_context_on(connection, scope) + key = key_for( + context.binding, context.obligation, keys, required=work["verification_key_sha256"] + ) + attempt = next( + r + for r in context.records + if isinstance(r, ReportingMaterializationAttempt) + and r.reporting_materialization_id == work["reporting_materialization_id"] + ) + if not context.resumable(attempt): + return await self._park_work_on(connection, work, "history_corrupt") + request = ReportingDestinationRequest.from_binding(context.binding, attempt, key) + if ( + request.external_id != work["external_id"] + or request.binding_fingerprint != work["binding_sha256"] + ): + raise failure("BINDING_MISMATCH") + except (LedgerConflictError, ReportingWriterError, StopIteration): + # Park pending unknown effects until an operator restores the exact + # installed contract. No hot loop and no replacement identity. + return await self._park_work_on(connection, work, "component_unavailable") + token = uuid4() + updated = await ( + await connection.execute( + "UPDATE reporting_materializer_work SET lease_token=%s," + " lease_until=clock_timestamp()+make_interval(secs=>%s)," + " due_at=clock_timestamp()+make_interval(secs=>%s)" + " WHERE account_id=%s AND consumer_id=%s AND reporting_materialization_id=%s" + " AND state='pending' AND (lease_until IS NULL OR lease_until<=clock_timestamp())" + " RETURNING lease_until", + ( + token, + lease_seconds, + lease_seconds, + scope.principal.account_id, + scope.consumer_id, + work["reporting_materialization_id"], + ), + ) + ).fetchone() + if updated is None: + return ReportingMaterializerTurn("idle") + return ReportingMaterializerLease( + scope, + work["generation"], + str(token), + updated[0], + attempt, + request, + context, + work["notifications_enabled"], + ) + + async def _park_work_on( + self, connection: Any, work: dict[str, Any], reason: MaterializerReason + ) -> ReportingMaterializerTurn: + scope = _scope(work) + await connection.execute( + "UPDATE reporting_materializer_work SET due_at='infinity'," + " lease_token=NULL,lease_until=NULL,reason=%s" + " WHERE account_id=%s AND consumer_id=%s AND reporting_materialization_id=%s", + ( + reason, + scope.principal.account_id, + scope.consumer_id, + work["reporting_materialization_id"], + ), + ) + return await self._park_on(connection, scope, reason) + + async def _held_on( + self, connection: Any, lease: ReportingMaterializerLease + ) -> dict[str, Any] | None: + row = await ( + await connection.execute( + "SELECT to_jsonb(w) FROM reporting_materializer_work w WHERE account_id=%s" + " AND consumer_id=%s AND reporting_materialization_id=%s AND state='pending'" + " AND lease_token=%s::uuid AND lease_until>clock_timestamp() FOR UPDATE", + ( + lease.scope.principal.account_id, + lease.scope.consumer_id, + lease.attempt.reporting_materialization_id, + lease.token, + ), + ) + ).fetchone() + if row is None: + return None + work = cast(dict[str, Any], row[0]) + if ( + work["external_id"] != lease.request.external_id + or work["generation"] != lease.generation + or work["binding_sha256"] != lease.request.binding_fingerprint + or work["notifications_enabled"] != lease.notifications_enabled + or work["notifications_enabled"] != self._notifications_enabled + ): + raise failure("BINDING_MISMATCH") + return work + + @materializer_errors + async def renew_materialization( + self, lease: ReportingMaterializerLease, *, lease_seconds: int + ) -> bool: + validate_lease_seconds(lease_seconds) + async with self._connection() as connection, connection.transaction(): + await self._lock_account(connection, lease.scope.principal.account_id) + if await self._held_on(connection, lease) is None: + return False + await connection.execute( + "UPDATE reporting_materializer_work SET lease_until=clock_timestamp()" + " +make_interval(secs=>%s),due_at=clock_timestamp()+make_interval(secs=>%s)" + " WHERE account_id=%s AND consumer_id=%s AND reporting_materialization_id=%s", + ( + lease_seconds, + lease_seconds, + lease.scope.principal.account_id, + lease.scope.consumer_id, + lease.attempt.reporting_materialization_id, + ), + ) + await self._schedule_account_on(connection, lease.scope.principal.account_id) + return True + + async def _target_on( + self, + connection: Any, + lease: ReportingMaterializerLease, + ) -> tuple[MaterializerContext, MaterializerReason]: + context = await self._materializer_context_on(connection, lease.scope) + selected, reason = context.selection(await _now(connection)) + if reason != "ready": + return context, reason + row = await ( + await connection.execute( + f"SELECT generation FROM reporting_materializer_candidates WHERE {_SCOPE_WHERE}", # nosec B608 + _scope_args(lease.scope), + ) + ).fetchone() + if ( + row is None + or row[0] != lease.generation + or selected is None + or selected.reporting_revision_id != lease.attempt.reporting_revision_id + ): + return context, "target_changed" + if binding_fingerprint(context.binding) != lease.request.binding_fingerprint: + return context, "binding_changed" + return context, "ready" + + @materializer_errors + async def authorize_materialization(self, lease: ReportingMaterializerLease) -> None: + async with self._connection() as connection, connection.transaction(): + await self._lock_account(connection, lease.scope.principal.account_id) + await validate_materializer_schema( + connection, notifications=self._notifications_enabled + ) + if await self._held_on(connection, lease) is None: + raise failure("LEASE_LOST") + _, reason = await self._target_on(connection, lease) + if reason != "ready": + raise failure( + "HISTORY_CORRUPT" if reason == "history_corrupt" else "CURRENT_REVISION_CHANGED" + ) + + @materializer_errors + async def finish_materialization( + self, + lease: ReportingMaterializerLease, + *, + prepared: ReportingPreparedRevision | None = None, + verified: ReportingVerifiedDestination | None = None, + error: ReportingWriterFailure | None = None, + ) -> ReportingMaterializerTurn: + if verified is not None: + validate_verified_destination(verified, lease.request, token=lease.token) + if prepared is None or prepared.request != lease.request or error is not None: + raise failure("BINDING_MISMATCH") + elif error is None: + raise failure("BINDING_MISMATCH") + async with self._connection() as connection, connection.transaction(): + await self._lock_account(connection, lease.scope.principal.account_id) + await validate_materializer_schema( + connection, notifications=self._notifications_enabled + ) + held = await self._held_on(connection, lease) + if held is None: + replay = await ( + await connection.execute( + "SELECT reason FROM reporting_materializer_work WHERE account_id=%s" + " AND consumer_id=%s AND reporting_materialization_id=%s AND state='acked'" + " AND completion_token=%s::uuid AND external_id=%s", + ( + lease.scope.principal.account_id, + lease.scope.consumer_id, + lease.attempt.reporting_materialization_id, + lease.token, + lease.request.external_id, + ), + ) + ).fetchone() + if replay is not None: + return ReportingMaterializerTurn( + "verified" if replay[0] == "verified" else "failed", + replay[0], + lease.attempt.reporting_materialization_id, + ) + return ReportingMaterializerTurn( + "pending", "effect_unknown", lease.attempt.reporting_materialization_id + ) + context, reason = await self._target_on(connection, lease) + if reason != "ready": + error = ReportingWriterFailure("CURRENT_REVISION_CHANGED", "new_attempt", "applied") + verified = None + elif verified is not None and prepared is not None: + validate_materialization_target( + prepared, binding=context.binding, revisions=context.revisions + ) + if error is not None and ( + error.effect == "unknown" + or error.retry == "same_identity" + or error.code in {"DEADLINE_EXCEEDED", "LEASE_LOST"} + ): + delay = max(1, min(error.retry_after_seconds or 5, 300)) + await connection.execute( + "UPDATE reporting_materializer_work SET lease_token=NULL,lease_until=NULL," + " due_at=clock_timestamp()+make_interval(secs=>%s),reason='effect_unknown'" + " WHERE account_id=%s AND consumer_id=%s AND reporting_materialization_id=%s", + ( + delay, + lease.scope.principal.account_id, + lease.scope.consumer_id, + lease.attempt.reporting_materialization_id, + ), + ) + await self._schedule_account_on(connection, lease.scope.principal.account_id) + return ReportingMaterializerTurn( + "pending", "effect_unknown", lease.attempt.reporting_materialization_id + ) + now = await _now(connection) + if verified is not None and ( + context.delivery is None + or verified.resource.expires_at + < max( + context.delivery.resource_retained_until, + now + timedelta(days=context.binding.resource_retention_days), + ) + ): + error = ReportingWriterFailure("RESOURCE_UNAVAILABLE", "new_attempt", "applied") + verified = None + if verified is not None: + # Same-connection finish validation after all lock acquisition + # and DB-time reads; no I/O separates this from the copy below. + validate_verified_destination(verified, lease.request, token=lease.token) + outcome = ReportingMaterializationRecord( + lease.scope, + lease.attempt.reporting_revision_id, + lease.attempt.reporting_materialization_id, + context.binding.success_status if verified is not None else "failed", + now, + verified.resource if verified is not None else None, + replace(verified.verification, verified_at=now) if verified is not None else None, + public_failure(error) if error is not None else None, + ) + stored, inserted = await self._commit_record_on( + connection, outcome, notify=False, dirty=False + ) + await self._materializer_dirty_on(connection, stored, context) + if inserted and verified is not None and self._notifications_enabled: + revision = next( + r + for r in context.revisions + if r.reporting_revision_id == stored.reporting_revision_id + ) + event = materialization_event( + stored, + context.records, + context.obligation, + revision, + context.configuration, + now, + ) + if event is None: + raise failure("BINDING_MISMATCH") + await enqueue_materializer_event_on(connection, event) + retry = error is not None and error.retry == "new_attempt" + if reason == "ready": + reason = ( + "verified" + if verified is not None + else ("retry" if retry else "operator_required") + ) + ack = await ( + await connection.execute( + "UPDATE reporting_materializer_work SET state='acked',acknowledged_at=%s," + " completion_token=lease_token,lease_token=NULL,lease_until=NULL," + " retry_allowed=%s,reason=%s" + " WHERE account_id=%s AND consumer_id=%s AND reporting_materialization_id=%s" + " AND lease_token=%s::uuid AND lease_until>clock_timestamp() RETURNING 1", + ( + now, + retry, + reason, + lease.scope.principal.account_id, + lease.scope.consumer_id, + lease.attempt.reporting_materialization_id, + lease.token, + ), + ) + ).fetchone() + if ack is None: + raise failure("LEASE_LOST") # Rolls back evidence, dirty, ACK and outbox together. + due = ( + now + timedelta(seconds=max(1, error.retry_after_seconds or 5)) + if retry and error + else None + ) + if reason == "target_changed": + due = now + elif reason not in {"retry", "verified"}: + due = None + if ( + reason == "inactive" + and context.configuration.activated_at is not None + and context.configuration.activated_at > now + ): + due = context.configuration.activated_at + if verified is not None: + due = verified.resource.expires_at + await self._park_on(connection, lease.scope, reason, due_at=due) + await self._schedule_account_on(connection, lease.scope.principal.account_id) + return ReportingMaterializerTurn( + "verified" if verified is not None else "failed", + reason, + stored.reporting_materialization_id, + ) + + async def _materializer_dirty_on( + self, connection: Any, outcome: ReportingMaterializationRecord, context: MaterializerContext + ) -> None: + from adcp.reporting.ledger.notification_events import delivery_dirty + + scope, reason, evidence = delivery_dirty(outcome, context.obligation) + await self._dirty_status(connection, scope, reason, after=evidence) + + from adcp.reporting.canonical_json import canonical_json_sha256_v1 + from adcp.reporting.ledger.pg import _json + from adcp.reporting.ledger.status_snapshot import settle_snapshot_on + + core = await settle_snapshot_on( + self, connection, account_id=scope.account_id, as_of=outcome.completed_at + ) + row = await ( + await connection.execute( + "INSERT INTO reporting_materializer_status_heads" + " (account_id,consumer_id,max_sequence)" + " VALUES (%s,%s,1) ON CONFLICT (account_id,consumer_id) DO UPDATE" + " SET max_sequence=reporting_materializer_status_heads.max_sequence+1" + " RETURNING max_sequence", + (scope.account_id, outcome.scope.consumer_id), + ) + ).fetchone() + assert row is not None + account = await ( + await connection.execute( + "UPDATE reporting_materializer_accounts SET captured_sequence=captured_sequence+1" + " WHERE account_id=%s RETURNING captured_sequence", + (scope.account_id,), + ) + ).fetchone() + assert account is not None + boundary = ReportingMaterializerBoundary( + outcome.scope.principal, + row[0], + account[0], + outcome.reporting_materialization_id, + outcome.completed_at, + private_snapshot(core, outcome.scope.principal), + await self._records(connection, outcome.scope.principal), + ).to_storage() + await connection.execute( + "INSERT INTO reporting_materializer_status_boundaries" + " (account_id,consumer_id,sequence,account_sequence,reporting_materialization_id," + " as_of,input,content_sha256)" + " VALUES (%s,%s,%s,%s,%s,%s,%s::jsonb,%s)", + ( + scope.account_id, + outcome.scope.consumer_id, + row[0], + account[0], + outcome.reporting_materialization_id, + outcome.completed_at, + _json(boundary), + canonical_json_sha256_v1(boundary), + ), + ) + + @materializer_errors + async def read_materializer_boundaries( + self, *, caller: ReportingDeliveryPrincipal, after: int = 0, limit: int = 100 + ) -> tuple[ReportingMaterializerBoundary, ...]: + if type(after) is not int or after < 0 or type(limit) is not int or not 1 <= limit <= 100: + raise ReportingMaterializerUsageError( + "materializer boundary reads require bounded positions" + ) + async with self._connection() as connection, connection.transaction(): + await self._lock_account(connection, caller.account_id) + rows = await ( + await connection.execute( + "SELECT input, content_sha256=reporting_payload_sha256(input)" + " FROM reporting_materializer_status_boundaries" + " WHERE account_id=%s AND consumer_id=%s AND sequence>%s" + " ORDER BY sequence LIMIT %s", + (caller.account_id, caller.consumer_id, after, limit), + ) + ).fetchall() + if any(not row[1] for row in rows): + raise failure("HISTORY_CORRUPT") + return tuple(decode_materializer_boundary(row[0]) for row in rows) + + @materializer_errors + async def import_pending_materialization( + self, + *, + scope: ReportingDeliveryScope, + reporting_materialization_id: str, + original_external_id: str, + keys: tuple[ReportingVerificationKey, ...], + ) -> None: + """Explicit operator recovery after verifying the original external identity. + + Never infer legacy effect history. An unknown/non-SDK external identity + requires operator cleanup and a public terminal outcome before recovery. + """ + await self.materializer_ready() + async with self._connection() as connection, connection.transaction(): + await self._lock_account(connection, scope.principal.account_id) + context = await self._materializer_context_on(connection, scope) + attempt = next( + ( + r + for r in context.records + if isinstance(r, ReportingMaterializationAttempt) + and r.reporting_materialization_id == reporting_materialization_id + and r.scope == scope + ), + None, + ) + if attempt is None or context.outcome(attempt) is not None: + raise failure("BINDING_MISMATCH") + if not context.resumable(attempt): + raise failure("HISTORY_CORRUPT") + key = key_for(context.binding, context.obligation, keys) + request = ReportingDestinationRequest.from_binding(context.binding, attempt, key) + if original_external_id != request.external_id: + raise failure("BINDING_MISMATCH") + await connection.execute( + "SELECT reporting_materializer_wake(%s)", (scope.principal.account_id,) + ) + await connection.execute( + "INSERT INTO reporting_materializer_candidates" + " (account_id,consumer_id,delivery_config_id,delivery_config_version," + " reporting_obligation_id)" + " VALUES (%s,%s,%s,%s,%s) ON CONFLICT DO NOTHING", + _scope_args(scope), + ) + await connection.execute( + "INSERT INTO reporting_materializer_work" + " (account_id,consumer_id,delivery_config_id,delivery_config_version," + " reporting_obligation_id,reporting_revision_id,reporting_materialization_id," + " generation,binding_sha256,verification_key_sha256,external_id,imported," + " notifications_enabled)" + " SELECT account_id,consumer_id,delivery_config_id,delivery_config_version," + " reporting_obligation_id,%s,%s,generation,%s,%s,%s,TRUE,%s" + f" FROM reporting_materializer_candidates WHERE {_SCOPE_WHERE}" # nosec B608 + " ON CONFLICT (account_id,consumer_id,reporting_materialization_id) DO NOTHING", + ( + attempt.reporting_revision_id, + reporting_materialization_id, + request.binding_fingerprint, + verification_key_id(key), + request.external_id, + self._notifications_enabled, + *_scope_args(scope), + ), + ) + await connection.execute( + "UPDATE reporting_materializer_work SET due_at=clock_timestamp()" + " WHERE account_id=%s AND consumer_id=%s AND reporting_materialization_id=%s" + " AND state='pending' AND lease_token IS NULL", + (scope.principal.account_id, scope.consumer_id, reporting_materialization_id), + ) diff --git a/src/adcp/reporting/materializer/required_schema.json b/src/adcp/reporting/materializer/required_schema.json new file mode 100644 index 000000000..483a548bd --- /dev/null +++ b/src/adcp/reporting/materializer/required_schema.json @@ -0,0 +1,750 @@ +{ + "column:reporting_materializer_accounts.account_id": { + "enabled": true, + "fingerprint": "fdc4b549138bf90a5af5ed00b87a86ff9a0cbea553e7f017b4d95510708f6f88" + }, + "column:reporting_materializer_accounts.captured_sequence": { + "enabled": true, + "fingerprint": "42202005517b72e082eb22c9eceb2ac0252815e5df700c83eb50c54cfeb46297" + }, + "column:reporting_materializer_accounts.due_at": { + "enabled": true, + "fingerprint": "6f1466ce5d0aaac8471e39834b9c4b1f85d6f7169d9238a245ac035ff518e0fc" + }, + "column:reporting_materializer_accounts.served_at": { + "enabled": true, + "fingerprint": "ff6b592f18aa2fe4a2d7393bd4976409093f2650300d7073f5f485bed633309c" + }, + "column:reporting_materializer_candidates.account_id": { + "enabled": true, + "fingerprint": "fdc4b549138bf90a5af5ed00b87a86ff9a0cbea553e7f017b4d95510708f6f88" + }, + "column:reporting_materializer_candidates.consumer_id": { + "enabled": true, + "fingerprint": "fdc4b549138bf90a5af5ed00b87a86ff9a0cbea553e7f017b4d95510708f6f88" + }, + "column:reporting_materializer_candidates.delivery_config_id": { + "enabled": true, + "fingerprint": "fdc4b549138bf90a5af5ed00b87a86ff9a0cbea553e7f017b4d95510708f6f88" + }, + "column:reporting_materializer_candidates.delivery_config_version": { + "enabled": true, + "fingerprint": "54a5f9962598caa8723435f8b55bd5e15823053af5dd63d531663997537dc6ba" + }, + "column:reporting_materializer_candidates.due_at": { + "enabled": true, + "fingerprint": "6f1466ce5d0aaac8471e39834b9c4b1f85d6f7169d9238a245ac035ff518e0fc" + }, + "column:reporting_materializer_candidates.generation": { + "enabled": true, + "fingerprint": "69ade56844c40f1606daf51b6a1de9e07d8b838db8f35a89e7ef1576035794be" + }, + "column:reporting_materializer_candidates.reason": { + "enabled": true, + "fingerprint": "806b02ec486fb1a02f57be76cf01aba1287baf2d669233d70849bcb0df7c558c" + }, + "column:reporting_materializer_candidates.reporting_obligation_id": { + "enabled": true, + "fingerprint": "fdc4b549138bf90a5af5ed00b87a86ff9a0cbea553e7f017b4d95510708f6f88" + }, + "column:reporting_materializer_candidates.served_at": { + "enabled": true, + "fingerprint": "ff6b592f18aa2fe4a2d7393bd4976409093f2650300d7073f5f485bed633309c" + }, + "column:reporting_materializer_discovery.account_id": { + "enabled": true, + "fingerprint": "fdc4b549138bf90a5af5ed00b87a86ff9a0cbea553e7f017b4d95510708f6f88" + }, + "column:reporting_materializer_discovery.after_obligation_id": { + "enabled": true, + "fingerprint": "128b66e02f14946100273f112c144af7605d626124d20fe16c1ba30c6b19ae3a" + }, + "column:reporting_materializer_discovery.complete": { + "enabled": true, + "fingerprint": "981e469ff869d932309f9e6aab9b07ff394c7439fe2431281320e25b41c4198d" + }, + "column:reporting_materializer_discovery.consumer_id": { + "enabled": true, + "fingerprint": "fdc4b549138bf90a5af5ed00b87a86ff9a0cbea553e7f017b4d95510708f6f88" + }, + "column:reporting_materializer_discovery.delivery_config_id": { + "enabled": true, + "fingerprint": "fdc4b549138bf90a5af5ed00b87a86ff9a0cbea553e7f017b4d95510708f6f88" + }, + "column:reporting_materializer_discovery.delivery_config_version": { + "enabled": true, + "fingerprint": "54a5f9962598caa8723435f8b55bd5e15823053af5dd63d531663997537dc6ba" + }, + "column:reporting_materializer_notification_events.account_id": { + "enabled": true, + "fingerprint": "fdc4b549138bf90a5af5ed00b87a86ff9a0cbea553e7f017b4d95510708f6f88" + }, + "column:reporting_materializer_notification_events.admission_epoch": { + "enabled": true, + "fingerprint": "42202005517b72e082eb22c9eceb2ac0252815e5df700c83eb50c54cfeb46297" + }, + "column:reporting_materializer_notification_events.cause_generation": { + "enabled": true, + "fingerprint": "54a5f9962598caa8723435f8b55bd5e15823053af5dd63d531663997537dc6ba" + }, + "column:reporting_materializer_notification_events.cause_id": { + "enabled": true, + "fingerprint": "fdc4b549138bf90a5af5ed00b87a86ff9a0cbea553e7f017b4d95510708f6f88" + }, + "column:reporting_materializer_notification_events.cause_kind": { + "enabled": true, + "fingerprint": "fdc4b549138bf90a5af5ed00b87a86ff9a0cbea553e7f017b4d95510708f6f88" + }, + "column:reporting_materializer_notification_events.consumer_namespace": { + "enabled": true, + "fingerprint": "fdc4b549138bf90a5af5ed00b87a86ff9a0cbea553e7f017b4d95510708f6f88" + }, + "column:reporting_materializer_notification_events.fired_at": { + "enabled": true, + "fingerprint": "1cac4e73af11a8ecafd646ef6a0ff6ecb087d46f150408dcfebc630fe1bf5e1e" + }, + "column:reporting_materializer_notification_events.notification_id": { + "enabled": true, + "fingerprint": "fdc4b549138bf90a5af5ed00b87a86ff9a0cbea553e7f017b4d95510708f6f88" + }, + "column:reporting_materializer_notification_events.notification_type": { + "enabled": true, + "fingerprint": "fdc4b549138bf90a5af5ed00b87a86ff9a0cbea553e7f017b4d95510708f6f88" + }, + "column:reporting_materializer_notification_events.reporting_materialization_id": { + "enabled": true, + "fingerprint": "a0d17631e6e95ba976e4615a034020bc089fa63601fb91205ebb35798a39c0de" + }, + "column:reporting_materializer_notification_events.snapshot": { + "enabled": true, + "fingerprint": "ac355fc16c02b70cb0a24afee8214cdce5f5cbfdc7fd1630786d5101932ecfa4" + }, + "column:reporting_materializer_notification_expansions.account_id": { + "enabled": true, + "fingerprint": "fdc4b549138bf90a5af5ed00b87a86ff9a0cbea553e7f017b4d95510708f6f88" + }, + "column:reporting_materializer_notification_expansions.claim_count": { + "enabled": true, + "fingerprint": "42202005517b72e082eb22c9eceb2ac0252815e5df700c83eb50c54cfeb46297" + }, + "column:reporting_materializer_notification_expansions.consumer_namespace": { + "enabled": true, + "fingerprint": "fdc4b549138bf90a5af5ed00b87a86ff9a0cbea553e7f017b4d95510708f6f88" + }, + "column:reporting_materializer_notification_expansions.due_at": { + "enabled": true, + "fingerprint": "1cac4e73af11a8ecafd646ef6a0ff6ecb087d46f150408dcfebc630fe1bf5e1e" + }, + "column:reporting_materializer_notification_expansions.emission_generation": { + "enabled": true, + "fingerprint": "54a5f9962598caa8723435f8b55bd5e15823053af5dd63d531663997537dc6ba" + }, + "column:reporting_materializer_notification_expansions.error_code": { + "enabled": true, + "fingerprint": "5b92595d0b54d473a3a3818455845f1fe0dc20b48cfe0faf63061a82d1a3cb02" + }, + "column:reporting_materializer_notification_expansions.lease_expires_at": { + "enabled": true, + "fingerprint": "6f1466ce5d0aaac8471e39834b9c4b1f85d6f7169d9238a245ac035ff518e0fc" + }, + "column:reporting_materializer_notification_expansions.lease_token": { + "enabled": true, + "fingerprint": "5b92595d0b54d473a3a3818455845f1fe0dc20b48cfe0faf63061a82d1a3cb02" + }, + "column:reporting_materializer_notification_expansions.notification_id": { + "enabled": true, + "fingerprint": "fdc4b549138bf90a5af5ed00b87a86ff9a0cbea553e7f017b4d95510708f6f88" + }, + "column:reporting_materializer_notification_expansions.state": { + "enabled": true, + "fingerprint": "a1ea96aa09ee40a7801ee346ca59f35c6ceccd207397a2b7bfd70d130d159091" + }, + "column:reporting_materializer_status_boundaries.account_id": { + "enabled": true, + "fingerprint": "fdc4b549138bf90a5af5ed00b87a86ff9a0cbea553e7f017b4d95510708f6f88" + }, + "column:reporting_materializer_status_boundaries.account_sequence": { + "enabled": true, + "fingerprint": "54a5f9962598caa8723435f8b55bd5e15823053af5dd63d531663997537dc6ba" + }, + "column:reporting_materializer_status_boundaries.as_of": { + "enabled": true, + "fingerprint": "1cac4e73af11a8ecafd646ef6a0ff6ecb087d46f150408dcfebc630fe1bf5e1e" + }, + "column:reporting_materializer_status_boundaries.consumer_id": { + "enabled": true, + "fingerprint": "fdc4b549138bf90a5af5ed00b87a86ff9a0cbea553e7f017b4d95510708f6f88" + }, + "column:reporting_materializer_status_boundaries.content_sha256": { + "enabled": true, + "fingerprint": "fdc4b549138bf90a5af5ed00b87a86ff9a0cbea553e7f017b4d95510708f6f88" + }, + "column:reporting_materializer_status_boundaries.input": { + "enabled": true, + "fingerprint": "ac355fc16c02b70cb0a24afee8214cdce5f5cbfdc7fd1630786d5101932ecfa4" + }, + "column:reporting_materializer_status_boundaries.outcome_namespace": { + "enabled": true, + "fingerprint": "37f064bd049ffa20c99bccfcb2ddd77b4e7b65fc6471d7683087ed0beec2098e" + }, + "column:reporting_materializer_status_boundaries.reporting_materialization_id": { + "enabled": true, + "fingerprint": "fdc4b549138bf90a5af5ed00b87a86ff9a0cbea553e7f017b4d95510708f6f88" + }, + "column:reporting_materializer_status_boundaries.sequence": { + "enabled": true, + "fingerprint": "54a5f9962598caa8723435f8b55bd5e15823053af5dd63d531663997537dc6ba" + }, + "column:reporting_materializer_status_heads.account_id": { + "enabled": true, + "fingerprint": "fdc4b549138bf90a5af5ed00b87a86ff9a0cbea553e7f017b4d95510708f6f88" + }, + "column:reporting_materializer_status_heads.consumer_id": { + "enabled": true, + "fingerprint": "fdc4b549138bf90a5af5ed00b87a86ff9a0cbea553e7f017b4d95510708f6f88" + }, + "column:reporting_materializer_status_heads.max_sequence": { + "enabled": true, + "fingerprint": "54a5f9962598caa8723435f8b55bd5e15823053af5dd63d531663997537dc6ba" + }, + "column:reporting_materializer_work.account_id": { + "enabled": true, + "fingerprint": "fdc4b549138bf90a5af5ed00b87a86ff9a0cbea553e7f017b4d95510708f6f88" + }, + "column:reporting_materializer_work.acknowledged_at": { + "enabled": true, + "fingerprint": "6f1466ce5d0aaac8471e39834b9c4b1f85d6f7169d9238a245ac035ff518e0fc" + }, + "column:reporting_materializer_work.admission_epoch": { + "enabled": true, + "fingerprint": "42202005517b72e082eb22c9eceb2ac0252815e5df700c83eb50c54cfeb46297" + }, + "column:reporting_materializer_work.attempt_namespace": { + "enabled": true, + "fingerprint": "48602ba37bcbee2d9c8104042cc7868262df4d288eaf1543e387b9a059a08117" + }, + "column:reporting_materializer_work.binding_sha256": { + "enabled": true, + "fingerprint": "fdc4b549138bf90a5af5ed00b87a86ff9a0cbea553e7f017b4d95510708f6f88" + }, + "column:reporting_materializer_work.completion_token": { + "enabled": true, + "fingerprint": "88223bccb5aae7ddfe4b3feacd193586f6699cbfaa3507ae6c21250775392e25" + }, + "column:reporting_materializer_work.consumer_id": { + "enabled": true, + "fingerprint": "fdc4b549138bf90a5af5ed00b87a86ff9a0cbea553e7f017b4d95510708f6f88" + }, + "column:reporting_materializer_work.created_at": { + "enabled": true, + "fingerprint": "336df3243b293695d8321965e92f26d3f132e11514ed9678fb5e64e0015fa580" + }, + "column:reporting_materializer_work.delivery_config_id": { + "enabled": true, + "fingerprint": "fdc4b549138bf90a5af5ed00b87a86ff9a0cbea553e7f017b4d95510708f6f88" + }, + "column:reporting_materializer_work.delivery_config_version": { + "enabled": true, + "fingerprint": "54a5f9962598caa8723435f8b55bd5e15823053af5dd63d531663997537dc6ba" + }, + "column:reporting_materializer_work.due_at": { + "enabled": true, + "fingerprint": "336df3243b293695d8321965e92f26d3f132e11514ed9678fb5e64e0015fa580" + }, + "column:reporting_materializer_work.external_id": { + "enabled": true, + "fingerprint": "fdc4b549138bf90a5af5ed00b87a86ff9a0cbea553e7f017b4d95510708f6f88" + }, + "column:reporting_materializer_work.generation": { + "enabled": true, + "fingerprint": "54a5f9962598caa8723435f8b55bd5e15823053af5dd63d531663997537dc6ba" + }, + "column:reporting_materializer_work.imported": { + "enabled": true, + "fingerprint": "981e469ff869d932309f9e6aab9b07ff394c7439fe2431281320e25b41c4198d" + }, + "column:reporting_materializer_work.lease_token": { + "enabled": true, + "fingerprint": "88223bccb5aae7ddfe4b3feacd193586f6699cbfaa3507ae6c21250775392e25" + }, + "column:reporting_materializer_work.lease_until": { + "enabled": true, + "fingerprint": "6f1466ce5d0aaac8471e39834b9c4b1f85d6f7169d9238a245ac035ff518e0fc" + }, + "column:reporting_materializer_work.notifications_enabled": { + "enabled": true, + "fingerprint": "1abe3a1c570fbe885784dab5d979307c373d50f567ff4481ce3996869bf58fed" + }, + "column:reporting_materializer_work.reason": { + "enabled": true, + "fingerprint": "806b02ec486fb1a02f57be76cf01aba1287baf2d669233d70849bcb0df7c558c" + }, + "column:reporting_materializer_work.reporting_materialization_id": { + "enabled": true, + "fingerprint": "fdc4b549138bf90a5af5ed00b87a86ff9a0cbea553e7f017b4d95510708f6f88" + }, + "column:reporting_materializer_work.reporting_obligation_id": { + "enabled": true, + "fingerprint": "fdc4b549138bf90a5af5ed00b87a86ff9a0cbea553e7f017b4d95510708f6f88" + }, + "column:reporting_materializer_work.reporting_revision_id": { + "enabled": true, + "fingerprint": "fdc4b549138bf90a5af5ed00b87a86ff9a0cbea553e7f017b4d95510708f6f88" + }, + "column:reporting_materializer_work.retry_allowed": { + "enabled": true, + "fingerprint": "981e469ff869d932309f9e6aab9b07ff394c7439fe2431281320e25b41c4198d" + }, + "column:reporting_materializer_work.state": { + "enabled": true, + "fingerprint": "675d9766cc6787c63f91ee16167a1de38069be7ad44f6d8362bd498885ae1355" + }, + "column:reporting_materializer_work.verification_key_sha256": { + "enabled": true, + "fingerprint": "fdc4b549138bf90a5af5ed00b87a86ff9a0cbea553e7f017b4d95510708f6f88" + }, + "constraint:reporting_materializer_accounts.reporting_materializer_accounts_captured_sequence_check": { + "enabled": true, + "fingerprint": "1d2836d91f73454d02a4299e43a4ed52d0df6f60d4a1c62358b213b1e2f184b5" + }, + "constraint:reporting_materializer_accounts.reporting_materializer_accounts_pkey": { + "enabled": true, + "fingerprint": "e65d70e61c89a93d66c4c4f4c59ef755d0ff526b0fcdd1d9ae6d830d2abea913" + }, + "constraint:reporting_materializer_candidates.reporting_materializer_candid_account_id_delivery_config_i_fkey": { + "enabled": true, + "fingerprint": "f2fd7665f778fae1bec63184c8468e0212256b25009a44659abd7284bc54b8a7" + }, + "constraint:reporting_materializer_candidates.reporting_materializer_candidates_account_id_fkey": { + "enabled": true, + "fingerprint": "56b5c2865d458269356ade49fde975a0ebaf0a1338a80846132d5ec172dcddf5" + }, + "constraint:reporting_materializer_candidates.reporting_materializer_candidates_delivery_config_version_check": { + "enabled": true, + "fingerprint": "aa64b81d15585376b0e6a0904344e39bc5010257097cee5c8118967efdac4d34" + }, + "constraint:reporting_materializer_candidates.reporting_materializer_candidates_generation_check": { + "enabled": true, + "fingerprint": "04ce269eb45cc51febf8cbf44b760185b1eb2fd4b1935d6b53ecae063496539e" + }, + "constraint:reporting_materializer_candidates.reporting_materializer_candidates_pkey": { + "enabled": true, + "fingerprint": "eeabaa92e67458b97f1288ad0ca744a7a6b2bdcf24bed3511258141cf0635498" + }, + "constraint:reporting_materializer_candidates.reporting_materializer_candidates_reason_check": { + "enabled": true, + "fingerprint": "d2dc21e781a468c735dd8b305b5405e779c85c27657683c19570cb1ba4b94aa7" + }, + "constraint:reporting_materializer_discovery.reporting_materializer_discov_account_id_delivery_config_i_fkey": { + "enabled": true, + "fingerprint": "4761471309c35947c8d3928dce4595b4d455e7de6166aeb78a5de220465cb64a" + }, + "constraint:reporting_materializer_discovery.reporting_materializer_discovery_delivery_config_version_check": { + "enabled": true, + "fingerprint": "aa64b81d15585376b0e6a0904344e39bc5010257097cee5c8118967efdac4d34" + }, + "constraint:reporting_materializer_discovery.reporting_materializer_discovery_pkey": { + "enabled": true, + "fingerprint": "cb0658b07aa5249a6934cca6248293967db4ab1a848d706c3d5310fa98f08785" + }, + "constraint:reporting_materializer_notification_events.reporting_materializer_notifi_account_id_consumer_namespac_fkey": { + "enabled": true, + "fingerprint": "9274f47b408013aa1d44c57310026f2178212e0e3ec823f2a61d21cc08efd2f1" + }, + "constraint:reporting_materializer_notification_events.reporting_materializer_notifi_account_id_consumer_namespace_key": { + "enabled": true, + "fingerprint": "7156cd6623f45f181906d2886b53dfba82bfdfc6cdcbdd0017c1f1cd7ceae407" + }, + "constraint:reporting_materializer_notification_events.reporting_materializer_notification_ev_consumer_namespace_check": { + "enabled": true, + "fingerprint": "69951e411156e1739417ca750e81d5d806b2fcccea7e1914674f730ca6433b58" + }, + "constraint:reporting_materializer_notification_events.reporting_materializer_notification_eve_notification_type_check": { + "enabled": true, + "fingerprint": "4dc2b2e5d5fe003a3b98e1832dd453c9115cc3d03d59d3cb23258e90605548e3" + }, + "constraint:reporting_materializer_notification_events.reporting_materializer_notification_even_cause_generation_check": { + "enabled": true, + "fingerprint": "b43ce8b2f74254027bcc0d382aa0bda21801bd6539f4ea3365679718c5cd80ba" + }, + "constraint:reporting_materializer_notification_events.reporting_materializer_notification_event_admission_epoch_check": { + "enabled": true, + "fingerprint": "abd3e698b241244e9e751e940838ef5bcecdd79d21aab371f4b595818b5b1f95" + }, + "constraint:reporting_materializer_notification_events.reporting_materializer_notification_events_cause_kind_check": { + "enabled": true, + "fingerprint": "4d213a62ba48d8d428c2811ddc5039b1349d3a41943509a85062b43235ab669d" + }, + "constraint:reporting_materializer_notification_events.reporting_materializer_notification_events_check": { + "enabled": true, + "fingerprint": "bc5fdbb84edccb22f49869598e550f0f72ee45323f0852be3438c8c5d5a387fc" + }, + "constraint:reporting_materializer_notification_events.reporting_materializer_notification_events_check1": { + "enabled": true, + "fingerprint": "3d95e12acbdfe40cfd7c36ea0c4cceddc8106395e3165b6c8034dd14783ba596" + }, + "constraint:reporting_materializer_notification_events.reporting_materializer_notification_events_check2": { + "enabled": true, + "fingerprint": "abcd49a900cec31d6a71175389de38cf44ffe6d16fa9083e45fe82967313db78" + }, + "constraint:reporting_materializer_notification_events.reporting_materializer_notification_events_check3": { + "enabled": true, + "fingerprint": "66bffc213553425f6a34852e7394536bdd550304c09cdb472a70223bcadb9f7b" + }, + "constraint:reporting_materializer_notification_events.reporting_materializer_notification_events_pkey": { + "enabled": true, + "fingerprint": "e8d63f44b25700915bcb3cb69e2e24b98cf21fd304f36a2e9c2992236309309d" + }, + "constraint:reporting_materializer_notification_expansions.reporting_materializer_notif_account_id_consumer_namespac_fkey1": { + "enabled": true, + "fingerprint": "6819542fa33be0e7a42bead23f6729b2dfb82d447b59543aaa96071054d56289" + }, + "constraint:reporting_materializer_notification_expansions.reporting_materializer_notification_e_emission_generation_check": { + "enabled": true, + "fingerprint": "5b7b2793bd710c7eef8c02003fd8adb1df91d7c10659364fe37d7f6351d4bf5e" + }, + "constraint:reporting_materializer_notification_expansions.reporting_materializer_notification_expansions_error_code_check": { + "enabled": true, + "fingerprint": "502085808cda7e7ec1e45e3d88257a660bdc65b59a71727b473303a362b9cf9d" + }, + "constraint:reporting_materializer_notification_expansions.reporting_materializer_notification_expansions_pkey": { + "enabled": true, + "fingerprint": "8ac920e8e1ed5a59417d5d1ffb9bea608bbeb64dbcd1a431240370fe555fe6f0" + }, + "constraint:reporting_materializer_notification_expansions.reporting_materializer_notification_expansions_state_check": { + "enabled": true, + "fingerprint": "12b7a569e49f4bf374d5dade9ec0a0d812e4dea251366467c95bcd58bee90b9d" + }, + "constraint:reporting_materializer_status_boundaries.reporting_materializer_status_account_id_consumer_id_outco_fkey": { + "enabled": true, + "fingerprint": "e3e10c3f3d574511ee5944b8bade5d927099809b0a45539ce8502caa9ebb05bf" + }, + "constraint:reporting_materializer_status_boundaries.reporting_materializer_status_account_id_consumer_id_repor_fkey": { + "enabled": true, + "fingerprint": "837c9aa80214bc7b0b25cd299aec9cad71de4bb616eefd0960ae5ca57efb1683" + }, + "constraint:reporting_materializer_status_boundaries.reporting_materializer_status_account_id_consumer_id_report_key": { + "enabled": true, + "fingerprint": "72945097549e77da2940c9cc5e54db645c12c0d5ef3bed0d4d7e7ba5cc93ffc1" + }, + "constraint:reporting_materializer_status_boundaries.reporting_materializer_status_b_account_id_account_sequence_key": { + "enabled": true, + "fingerprint": "460b11d8840751c3e894245e33c24e704407f4b3ca42bbd9e832cb48a0c3b24d" + }, + "constraint:reporting_materializer_status_boundaries.reporting_materializer_status_boundarie_outcome_namespace_check": { + "enabled": true, + "fingerprint": "60e879d7d85bbf368a4f30d08481dd59f5aa1d92eb6943b0e60a8e8e219c2ff2" + }, + "constraint:reporting_materializer_status_boundaries.reporting_materializer_status_boundaries_account_sequence_check": { + "enabled": true, + "fingerprint": "1a64697a81e3b39433d851f9916df1964cbbc260aa54dd6fee4076c7e805c70e" + }, + "constraint:reporting_materializer_status_boundaries.reporting_materializer_status_boundaries_check": { + "enabled": true, + "fingerprint": "6db9e761fce16f6deaabe04581b3813f3495382b5edf905c2c01a772151eed89" + }, + "constraint:reporting_materializer_status_boundaries.reporting_materializer_status_boundaries_check1": { + "enabled": true, + "fingerprint": "6c5865ad0c28d00ec74194418def9d764ddf85b230f1dfb6d14c6e505eda3737" + }, + "constraint:reporting_materializer_status_boundaries.reporting_materializer_status_boundaries_check2": { + "enabled": true, + "fingerprint": "307105bea1a76e214540a207ab8dddcffb26853845a89236454a6e9ecb0508d3" + }, + "constraint:reporting_materializer_status_boundaries.reporting_materializer_status_boundaries_check3": { + "enabled": true, + "fingerprint": "65d0dca50d7ca308123252fcc784fb427fd857d8bf3092bd85fc8fff5787d7ec" + }, + "constraint:reporting_materializer_status_boundaries.reporting_materializer_status_boundaries_check4": { + "enabled": true, + "fingerprint": "0d806a2b956a1e8947dcc43c634352ed52c1c3f158235145b2fe5a96db5fb379" + }, + "constraint:reporting_materializer_status_boundaries.reporting_materializer_status_boundaries_check5": { + "enabled": true, + "fingerprint": "b330c171bc0bb87ac12aa97aca4d4ae7167e0c07d7b61b3b688c63df9588d927" + }, + "constraint:reporting_materializer_status_boundaries.reporting_materializer_status_boundaries_check6": { + "enabled": true, + "fingerprint": "9b7f5755a4d4093ab3618e85ff78ea92e931d1895d601ee48f781dcbca07155d" + }, + "constraint:reporting_materializer_status_boundaries.reporting_materializer_status_boundaries_content_sha256_check": { + "enabled": true, + "fingerprint": "2afe58d90df96e397cb8e6a941e52fcf8e01b7835024dd3b4ba67141a3578505" + }, + "constraint:reporting_materializer_status_boundaries.reporting_materializer_status_boundaries_input_check": { + "enabled": true, + "fingerprint": "1403f246fdde17015118d212847ed5d8169df3adc0364598cbd5e82be066e1bf" + }, + "constraint:reporting_materializer_status_boundaries.reporting_materializer_status_boundaries_input_check1": { + "enabled": true, + "fingerprint": "fadd88bcfddae6b78d4fc68c90018f5c21ba800b5beb4ddfb2d5805da7703a2b" + }, + "constraint:reporting_materializer_status_boundaries.reporting_materializer_status_boundaries_pkey": { + "enabled": true, + "fingerprint": "00dd5f9ba4e0d0face7ab27fe5e52ffa9c4101943c95afb8ec1badb401c7010d" + }, + "constraint:reporting_materializer_status_boundaries.reporting_materializer_status_boundaries_sequence_check": { + "enabled": true, + "fingerprint": "554d491362648e795a6567528a6244977d5df276216297f35e26e4750f869dcf" + }, + "constraint:reporting_materializer_status_heads.reporting_materializer_status_heads_max_sequence_check": { + "enabled": true, + "fingerprint": "ee47acec8d450eed7f99b637596847d21bc449a12317a605fd6b6cb0fad6aa79" + }, + "constraint:reporting_materializer_status_heads.reporting_materializer_status_heads_pkey": { + "enabled": true, + "fingerprint": "4d853add149814edf9eadd3f752bf43f1f7164cba2bd079f0349d4d540a7f20c" + }, + "constraint:reporting_materializer_work.reporting_materializer_work_account_id_consumer_id_attempt_fkey": { + "enabled": true, + "fingerprint": "e3a1d24fef8da0fbe5587a7f7887a4cb9adf0133071070a09c950db1474304c8" + }, + "constraint:reporting_materializer_work.reporting_materializer_work_account_id_consumer_id_deliver_fkey": { + "enabled": true, + "fingerprint": "9390df97c6b965e9f16f700a4bd4e90392e60f89a15b6a9840d05682cdcc532f" + }, + "constraint:reporting_materializer_work.reporting_materializer_work_account_id_consumer_id_external_key": { + "enabled": true, + "fingerprint": "78641c1ca086a76ec1a81540664d98811ffb2adb51d5dc27db4d307f428bd8cd" + }, + "constraint:reporting_materializer_work.reporting_materializer_work_account_id_reporting_obligatio_fkey": { + "enabled": true, + "fingerprint": "a0dca2cc4a27c9d4380eadb1513d30fe6383849657fc5e0d0f93c4b4c4ed4c08" + }, + "constraint:reporting_materializer_work.reporting_materializer_work_admission_epoch_check": { + "enabled": true, + "fingerprint": "abd3e698b241244e9e751e940838ef5bcecdd79d21aab371f4b595818b5b1f95" + }, + "constraint:reporting_materializer_work.reporting_materializer_work_attempt_namespace_check": { + "enabled": true, + "fingerprint": "639b8bfe70df1945046d65f693e67c45a4015e0e318356d77a97f5e4b3a7b629" + }, + "constraint:reporting_materializer_work.reporting_materializer_work_binding_sha256_check": { + "enabled": true, + "fingerprint": "c9db9bf2eca41364cec2fc19983d43e23fedf6711e5c4deb5f1d80aed317fd2b" + }, + "constraint:reporting_materializer_work.reporting_materializer_work_check": { + "enabled": true, + "fingerprint": "a6826ba06594c4200bda46be2704e03cb9d234d36538780211c04c2b2392e61b" + }, + "constraint:reporting_materializer_work.reporting_materializer_work_check1": { + "enabled": true, + "fingerprint": "311f8f3d7b44fd93e0b4ed5945b86997d4c6fd3beea28eca693b1e506466676d" + }, + "constraint:reporting_materializer_work.reporting_materializer_work_check2": { + "enabled": true, + "fingerprint": "3b94ebc10b0f1c882069a518c6496ff87db17e6b3405aecdb2f9aca7b158c644" + }, + "constraint:reporting_materializer_work.reporting_materializer_work_check3": { + "enabled": true, + "fingerprint": "fc30826fbb5c547751d5b430e90d54631bc537ca3af856785d68bb4a350cf9cb" + }, + "constraint:reporting_materializer_work.reporting_materializer_work_check4": { + "enabled": true, + "fingerprint": "50525da46b6ed71b60656e35fbba42e4fcbb45719c4d2032c2eccf35b971fafc" + }, + "constraint:reporting_materializer_work.reporting_materializer_work_external_id_check": { + "enabled": true, + "fingerprint": "df053aff1b660dec5635dbcc1043c90a9b338c3d871b72d4ed26ec60d465d278" + }, + "constraint:reporting_materializer_work.reporting_materializer_work_generation_check": { + "enabled": true, + "fingerprint": "04ce269eb45cc51febf8cbf44b760185b1eb2fd4b1935d6b53ecae063496539e" + }, + "constraint:reporting_materializer_work.reporting_materializer_work_pkey": { + "enabled": true, + "fingerprint": "fb2ee6241427fec8a9ae91d2940abe9e20b25f47554bd2d0a7c4046eb7704c41" + }, + "constraint:reporting_materializer_work.reporting_materializer_work_reason_check": { + "enabled": true, + "fingerprint": "d2dc21e781a468c735dd8b305b5405e779c85c27657683c19570cb1ba4b94aa7" + }, + "constraint:reporting_materializer_work.reporting_materializer_work_state_check": { + "enabled": true, + "fingerprint": "2e00e4b40728462003ac3d7f891ca8354aaeee6e19d808f7809030ff4db7758a" + }, + "constraint:reporting_materializer_work.reporting_materializer_work_verification_key_sha256_check": { + "enabled": true, + "fingerprint": "5442191b821cf162c8dfe50da9181b02acbf254eeb477c27b98e69d1fd26134b" + }, + "function:reporting_materializer_binding_dirty()": { + "enabled": true, + "fingerprint": "eea2454f1ded0e1bc966fac4cb567e43520182595e9961cd6052c6b7f5cf94e3" + }, + "function:reporting_materializer_dirty(a text, c text, d text, v bigint, o text)": { + "enabled": true, + "fingerprint": "bbcf34da712e0fae850148f133223786b751f6b03bc71c4c6d4f3f55bb78586b" + }, + "function:reporting_materializer_expansion_guard()": { + "enabled": true, + "fingerprint": "cb98dadaf9c1932f1ff1671b4cd2a7509488c6f00eacbeacb6b7cf069ea72356" + }, + "function:reporting_materializer_retained_guard()": { + "enabled": true, + "fingerprint": "f3a494b6d28a4e5340a3f4707da937a765122016eb68d93572ca05c982bae49b" + }, + "function:reporting_materializer_source_dirty()": { + "enabled": true, + "fingerprint": "49bb0040dd74ba1af0f4f0ecf7b308cff4d8f6cad76be064dbe7ea46e71aee3e" + }, + "function:reporting_materializer_wake(a text)": { + "enabled": true, + "fingerprint": "c0487a973b7faba9ee0766ebd845b67f44486f5e31cdb2cacb85fd7805a636e7" + }, + "function:reporting_materializer_work_guard()": { + "enabled": true, + "fingerprint": "53beb6d2d06c61acbdbd6963c9f4869421d98af3842ee4b574c885061ddcd955" + }, + "index:reporting_materializer_accounts.reporting_materializer_account_due": { + "enabled": true, + "fingerprint": "aa5282c87c241a1fb4231aedf5c2729ed8b7e4d72c5f6fddd93001fba5edf4b3" + }, + "index:reporting_materializer_accounts.reporting_materializer_account_wakeup": { + "enabled": true, + "fingerprint": "fd3be523100f148ee3d372fe749107f426ea21c1c5e22062c77329c8b3e5404d" + }, + "index:reporting_materializer_accounts.reporting_materializer_accounts_pkey": { + "enabled": true, + "fingerprint": "8b2d5d80f629b4f7a049604a72b947d79d365ab67ba4caaebc48d0ea8163167b" + }, + "index:reporting_materializer_candidates.reporting_materializer_candidate_due": { + "enabled": true, + "fingerprint": "3be6530dbb55b15eaada3576100fac5fc4d2b9852e848b3163cef3e2bc66ba48" + }, + "index:reporting_materializer_candidates.reporting_materializer_candidate_publication": { + "enabled": true, + "fingerprint": "999e962294bc6d17632cf79252bfa9be4a1cff222238b55ed920cad9a0eb24c3" + }, + "index:reporting_materializer_candidates.reporting_materializer_candidates_pkey": { + "enabled": true, + "fingerprint": "851a791d7161fb93f6d3e0001fb59d46ecf58b96c3df1f9cfad8983b59b7699b" + }, + "index:reporting_materializer_discovery.reporting_materializer_discovery_pending": { + "enabled": true, + "fingerprint": "be5e169a272b75f8ff83ccffb7ef1af3b629456a405c4170f5f4f40611a14860" + }, + "index:reporting_materializer_discovery.reporting_materializer_discovery_pkey": { + "enabled": true, + "fingerprint": "b6b8787db0e681f9bee61e99514097d8d0c150fcb78102e439a92c0483521f62" + }, + "index:reporting_materializer_notification_events.reporting_materializer_notifi_account_id_consumer_namespace_key": { + "enabled": true, + "fingerprint": "8d588106a4322296872f083deffc88263341ba7b62217919ce1cb64ba5c7175b" + }, + "index:reporting_materializer_notification_events.reporting_materializer_notification_events_pkey": { + "enabled": true, + "fingerprint": "d9a07bc15c257dc9c5f0832be98dca66e6fd908f12c4f739ff294a8a95ceb736" + }, + "index:reporting_materializer_notification_expansions.reporting_materializer_notification_due": { + "enabled": true, + "fingerprint": "b5ce1d459ed7406b956d3c9144b95693791ce956229e62aa419ca9dc92d91b03" + }, + "index:reporting_materializer_notification_expansions.reporting_materializer_notification_expansions_pkey": { + "enabled": true, + "fingerprint": "d58d479e1ce11640e782f92435570967ae778f19e3ae38e5c1d7d1c819123920" + }, + "index:reporting_materializer_status_boundaries.reporting_materializer_status_account_id_consumer_id_report_key": { + "enabled": true, + "fingerprint": "9608f9409e9efd9d58e70c1d3e6bb814b3e61bdd50c51dc8680fd4376520ca1c" + }, + "index:reporting_materializer_status_boundaries.reporting_materializer_status_b_account_id_account_sequence_key": { + "enabled": true, + "fingerprint": "da68c1f5003f093b5cf98c7ea057638c0c4690884972298b529433c3c457633d" + }, + "index:reporting_materializer_status_boundaries.reporting_materializer_status_boundaries_pkey": { + "enabled": true, + "fingerprint": "1862208bb1b96147d96b83f2f772208eb62dc39470d6eca03431b76f5ec07c6b" + }, + "index:reporting_materializer_status_heads.reporting_materializer_status_heads_pkey": { + "enabled": true, + "fingerprint": "6f3fd3013e95dc10388625b673ae88b6aac381066f81981825039dfaf913f412" + }, + "index:reporting_materializer_work.reporting_materializer_one_pending": { + "enabled": true, + "fingerprint": "c8fca3e3da79cc0c5488f1b7003b93169c727337b0fbf5e4b472878d1002aa14" + }, + "index:reporting_materializer_work.reporting_materializer_work_account_id_consumer_id_external_key": { + "enabled": true, + "fingerprint": "ab5fd65a6f6e47fff0b4d35473a7e6d05973d83721ba412722ae2d5dea03f4c6" + }, + "index:reporting_materializer_work.reporting_materializer_work_due": { + "enabled": true, + "fingerprint": "86ca449c8a2ab3161dad3dc6cfcca8e7fe45e01d3ac73472be058917463f865b" + }, + "index:reporting_materializer_work.reporting_materializer_work_pkey": { + "enabled": true, + "fingerprint": "a36b975739037fdb8f2eefa757f3265be6c06afe19ec8340fd8fa32669cf6220" + }, + "index:reporting_obligations.reporting_materializer_obligation_discovery": { + "enabled": true, + "fingerprint": "e715e2c28cbb6c25f9bb78d7c53422884ef9fa83c3740b4d1859bee1800c6855" + }, + "index:reporting_reconciliation_records.reporting_materializer_binding_discovery": { + "enabled": true, + "fingerprint": "c09880d92551983d9aec30e6a7cb34058db7f473849d8cc6d8a194baee1707a0" + }, + "table:reporting_materializer_accounts": { + "enabled": true, + "fingerprint": "1f824779ff80f110344420b019786663d8c9beaad230da90e0439795e734ccda" + }, + "table:reporting_materializer_candidates": { + "enabled": true, + "fingerprint": "1f824779ff80f110344420b019786663d8c9beaad230da90e0439795e734ccda" + }, + "table:reporting_materializer_discovery": { + "enabled": true, + "fingerprint": "1f824779ff80f110344420b019786663d8c9beaad230da90e0439795e734ccda" + }, + "table:reporting_materializer_notification_events": { + "enabled": true, + "fingerprint": "1f824779ff80f110344420b019786663d8c9beaad230da90e0439795e734ccda" + }, + "table:reporting_materializer_notification_expansions": { + "enabled": true, + "fingerprint": "1f824779ff80f110344420b019786663d8c9beaad230da90e0439795e734ccda" + }, + "table:reporting_materializer_status_boundaries": { + "enabled": true, + "fingerprint": "1f824779ff80f110344420b019786663d8c9beaad230da90e0439795e734ccda" + }, + "table:reporting_materializer_status_heads": { + "enabled": true, + "fingerprint": "1f824779ff80f110344420b019786663d8c9beaad230da90e0439795e734ccda" + }, + "table:reporting_materializer_work": { + "enabled": true, + "fingerprint": "1f824779ff80f110344420b019786663d8c9beaad230da90e0439795e734ccda" + }, + "trigger:reporting_configurations.reporting_materializer_configuration": { + "enabled": true, + "fingerprint": "68c28dd89618041f28c1e1fb5e833d1af17b14224cbe54cde4f0b3a62dc13407" + }, + "trigger:reporting_materializer_notification_events.reporting_materializer_event_immutable": { + "enabled": true, + "fingerprint": "faf323248f4100b2d93324789dcf1435eddfc55aa970b04e08cb1ef8705e1062" + }, + "trigger:reporting_materializer_notification_expansions.reporting_materializer_pre_activation_guard": { + "enabled": true, + "fingerprint": "f3132d843dd232dca85cb174aaf3d398a959d26bcfbcfd622e6e93be932fc23e" + }, + "trigger:reporting_materializer_status_boundaries.reporting_materializer_boundary_immutable": { + "enabled": true, + "fingerprint": "54cabb176a7b94c44c16c1e420d948dd8cf8462807ae41ac9d1f8fd3cfa391da" + }, + "trigger:reporting_materializer_work.reporting_materializer_guard": { + "enabled": true, + "fingerprint": "c157e0d74d41e71eaf8b1c5c391ff40231a6c909ddcdc891f3e046c76152d219" + }, + "trigger:reporting_obligations.reporting_materializer_obligation": { + "enabled": true, + "fingerprint": "cbea44a53e0542a797454a9af681915c5933dd92e3e00520b64a8f2849937234" + }, + "trigger:reporting_reconciliation_records.reporting_materializer_binding": { + "enabled": true, + "fingerprint": "45497bd65d22b5709f9ed2e3f863e3585ffc1d54a211e1fa76306237824612da" + }, + "trigger:reporting_revisions.reporting_materializer_publication": { + "enabled": true, + "fingerprint": "8afe04a8f60e5f863981505f688b55295a42694e6835951c30750470b1162b5b" + } +} diff --git a/src/adcp/reporting/materializer/schema.py b/src/adcp/reporting/materializer/schema.py new file mode 100644 index 000000000..c6c42ff25 --- /dev/null +++ b/src/adcp/reporting/materializer/schema.py @@ -0,0 +1,43 @@ +"""Materializer readiness is independent of every prior mandatory manifest.""" + +from __future__ import annotations + +import json +from importlib.resources import files +from typing import Any + +from adcp.reporting.ledger.store import LedgerConflictError +from adcp.reporting.outbox._schema import REQUIRED_OBJECTS, schema_objects, validate_schema + + +async def validate_materializer_schema(connection: Any, *, notifications: bool = False) -> None: + required = json.loads( + files("adcp.reporting.materializer").joinpath("required_schema.json").read_text() + ) + if not required: + raise LedgerConflictError("MATERIALIZER_SCHEMA_UNREADY", "install the materializer schema") + installed = None + try: + installed = await schema_objects(connection) + except Exception: + installed = None + if installed is None: + raise LedgerConflictError( + "MATERIALIZER_SCHEMA_UNREADY", "materializer schema catalog is unavailable" + ) + if any(installed.get(key) != expected for key, expected in required.items()): + raise LedgerConflictError("MATERIALIZER_SCHEMA_UNREADY", "install the materializer schema") + # Inherit the reviewed foundation prerequisites without appending B2 objects + # to A's manifest or requiring optional notification delivery infrastructure. + for key, expected in REQUIRED_OBJECTS.items(): + if any( + word in key + for word in ("notification_", "webhook_", "status_dirty", "status_checkpoints") + ): + continue + if installed.get(key) != expected: + raise LedgerConflictError( + "MATERIALIZER_SCHEMA_UNREADY", "install the retained evidence schema" + ) + if notifications: + await validate_schema(connection) diff --git a/src/adcp/reporting/materializer/service.py b/src/adcp/reporting/materializer/service.py new file mode 100644 index 000000000..11a564f7a --- /dev/null +++ b/src/adcp/reporting/materializer/service.py @@ -0,0 +1,157 @@ +"""Service-owned heartbeat around destination I/O outside database transactions.""" + +from __future__ import annotations + +import asyncio +from dataclasses import dataclass, field +from datetime import datetime, timedelta, timezone + +from adcp.reporting.materializer._errors import materializer_errors +from adcp.reporting.materializer.contracts import ( + ReportingDestinationWriter, + ReportingIOContext, + ReportingWriterError, + ReportingWriterFailure, + _join_tasks, + failure, +) +from adcp.reporting.materializer.verification import ReportingDestinationIO +from adcp.reporting.materializer.work import ( + ReportingMaterializerLease, + ReportingMaterializerStore, + ReportingMaterializerTurn, + validate_lease_seconds, +) + + +class _LeaseHeartbeat: + def __init__( + self, + store: ReportingMaterializerStore, + lease: ReportingMaterializerLease, + seconds: int, + cancel: asyncio.Event, + ) -> None: + self.store, self.lease, self.seconds, self.cancel = store, lease, seconds, cancel + self.stopped = asyncio.Event() + self.lost = False + + async def checkpoint(self) -> None: + if self.lost: + raise failure("LEASE_LOST") + + async def run(self) -> None: + while not self.stopped.is_set(): + try: + await asyncio.wait_for(self.stopped.wait(), self.seconds / 3) + return + except asyncio.TimeoutError: + # The heartbeat interval elapsed; renew the owned lease below. + pass + held = False + try: + held = await self.store.renew_materialization( + self.lease, lease_seconds=self.seconds + ) + except asyncio.CancelledError: + raise + except Exception: + held = False # Driver/provider bodies never escape service diagnostics. + if not held: + self.lost = True + self.cancel.set() + return + + +@dataclass(frozen=True) +class ReportingMaterializerService: + """One autonomous turn; schedule repeatedly, including after process restart. + + The destination must implement its advertised conditional/idempotent write + semantics durably. A timeout or interrupted outcome commit retains the same + immutable attempt. The SDK opens fresh authorization sessions for write and + paginated readback; the service checks current ledger authority before each. + """ + + store: ReportingMaterializerStore = field(repr=False) + io: ReportingDestinationIO = field(repr=False) + writer: ReportingDestinationWriter = field(repr=False) + lease_seconds: int = 30 + io_timeout_seconds: int = 300 + + def __post_init__(self) -> None: + validate_lease_seconds(self.lease_seconds) + if type(self.io_timeout_seconds) is not int or not 1 <= self.io_timeout_seconds <= 3600: + raise ValueError("materializer I/O deadline requires 1..3600 seconds") + if type(self.io) is not ReportingDestinationIO: + raise failure("UNSUPPORTED_VERIFICATION") + + @materializer_errors + async def run_once(self) -> ReportingMaterializerTurn: + keys = tuple( + v.key + for v in self.io.registry.verifiers + if v.key.capability in self.writer.capabilities + ) + reserved = await self.store.claim_materialization( + keys=keys, lease_seconds=self.lease_seconds + ) + if isinstance(reserved, ReportingMaterializerTurn): + return reserved + cancel = asyncio.Event() + heartbeat = _LeaseHeartbeat(self.store, reserved, self.lease_seconds, cancel) + task = asyncio.create_task(heartbeat.run()) + context = ReportingIOContext( + datetime.now(timezone.utc) + timedelta(seconds=self.io_timeout_seconds), + cancel, + heartbeat, + ) + canceled = False + turn = ReportingMaterializerTurn( + "pending", "effect_unknown", reserved.attempt.reporting_materialization_id + ) + try: + try: + await self.store.authorize_materialization(reserved) + target = reserved.context + if target.delivery is None: + raise failure("BINDING_MISMATCH") + prepared = await self.io.registry.prepare( + key=reserved.request.verification_key, + binding=target.binding, + delivery=target.delivery, + obligation=target.obligation, + revisions=target.revisions, + attempt=reserved.attempt, + reader=self.store, + context=context, + ) + await self.store.authorize_materialization(reserved) + locator = await self.io.write(prepared, context=context) + await self.store.authorize_materialization(reserved) + verified = await self.io.verify(prepared, locator, context=context) + turn = await self.store.finish_materialization( + reserved, prepared=prepared, verified=verified + ) + except ReportingWriterError as exc: + record = exc.failure + # The writer owns whether a failure is known and terminal. All + # uncertain I/O and lost leases keep the external identity. + if record.code in {"DEADLINE_EXCEEDED", "LEASE_LOST"}: + record = ReportingWriterFailure(record.code, "same_identity", "unknown") + turn = await self.store.finish_materialization(reserved, error=record) + except asyncio.CancelledError: + canceled = not heartbeat.lost + except Exception: + # Includes commit/connection loss and failed atomic outbox enqueue. + # The lease expires; a restart reuses the original identity. + turn = ReportingMaterializerTurn( + "pending", "effect_unknown", reserved.attempt.reporting_materialization_id + ) + finally: + heartbeat.stopped.set() + task.cancel() + canceled = await _join_tasks(task) or canceled + if canceled: + raise asyncio.CancelledError + return turn diff --git a/src/adcp/reporting/materializer/verification.py b/src/adcp/reporting/materializer/verification.py index 308b7880d..3fd36dcc7 100644 --- a/src/adcp/reporting/materializer/verification.py +++ b/src/adcp/reporting/materializer/verification.py @@ -10,10 +10,13 @@ import asyncio import base64 import hashlib +import hmac +import json import re +import secrets from collections.abc import AsyncIterator, Sequence from contextlib import asynccontextmanager -from dataclasses import dataclass, field, replace +from dataclasses import asdict, dataclass, field, replace from datetime import datetime, timedelta, timezone from decimal import Decimal, localcontext from typing import Any, Protocol, cast @@ -63,6 +66,8 @@ ) from adcp.reporting.revision_selection import select_reporting_revision +_VERIFICATION_SEAL_KEY = secrets.token_bytes(32) + class ReportingRevisionRowReader(Protocol): async def read_revision_rows( @@ -75,8 +80,14 @@ async def read_revision_rows( ) -> ReportingRowPage: ... +class _VerifiedProvenance(_ClosedValue): + # Non-dataclass slots deliberately stay out of B1's three-field constructor, + # dataclasses.fields/asdict, Pydantic schemas and serialized observations. + __slots__ = ("_sdk_seal", "_materializer_fence") + + @dataclass(frozen=True, slots=True) -class ReportingVerifiedDestination(_ClosedValue): +class ReportingVerifiedDestination(_VerifiedProvenance): """Immutable SDK observations. B2 alone owns atomic publication/readiness.""" request: ReportingDestinationRequest @@ -85,6 +96,43 @@ class ReportingVerifiedDestination(_ClosedValue): def __post_init__(self) -> None: _freeze_fields(self) + object.__setattr__(self, "_sdk_seal", b"") + object.__setattr__(self, "_materializer_fence", None) + + +def _verification_seal(value: ReportingVerifiedDestination) -> bytes: + from adcp.reporting.canonical_json import canonical_json_utf8_v1 + + content = json.loads( + json.dumps( + [ + asdict(value.request), + asdict(value.resource), + asdict(value.verification), + getattr(value, "_materializer_fence", None), + ], + default=lambda at: at.isoformat(), + ) + ) + return hmac.digest(_VERIFICATION_SEAL_KEY, canonical_json_utf8_v1(content), "sha256") + + +def validate_verified_destination( + value: ReportingVerifiedDestination, request: ReportingDestinationRequest, *, token: str +) -> None: + """Require unmodified evidence minted by this process's SDK readback. + + A public value constructor remains source compatible with B1. Constructed, + copied or restored claims cannot authorize B2 finish: restart repeats the + actual readback. The seal is never a persisted destination credential. + """ + if ( + type(value) is not ReportingVerifiedDestination + or value.request != request + or getattr(value, "_materializer_fence", None) != token + or not hmac.compare_digest(getattr(value, "_sdk_seal", b""), _verification_seal(value)) + ): + raise failure("DESTINATION_CORRUPT") def validate_materialization_target( @@ -925,9 +973,17 @@ async def _verify_destination( native_observed_through=native.verification_path if native else None, verified_format=cap.format, ) - return ReportingVerifiedDestination( + result = ReportingVerifiedDestination( prepared.request, replace(resource, manifest_sha256=manifest_digest), verification ) + # Only the service-owned heartbeat binds readback to its reservation. B1 + # standalone verification remains available without conferring B2 authority. + from adcp.reporting.materializer.service import _LeaseHeartbeat + + if type(context.heartbeat) is _LeaseHeartbeat: + object.__setattr__(result, "_materializer_fence", context.heartbeat.lease.token) + object.__setattr__(result, "_sdk_seal", _verification_seal(result)) + return result def _native( diff --git a/src/adcp/reporting/materializer/work.py b/src/adcp/reporting/materializer/work.py new file mode 100644 index 000000000..e2f306f15 --- /dev/null +++ b/src/adcp/reporting/materializer/work.py @@ -0,0 +1,285 @@ +"""Optional durable work contracts, independent of the foundation store protocols.""" + +from __future__ import annotations + +import hashlib +from dataclasses import asdict, dataclass, field +from datetime import datetime +from typing import Literal, Protocol, get_args, runtime_checkable +from uuid import UUID + +from adcp.reporting.canonical_json import canonical_json_utf8_v1 +from adcp.reporting.evidence import aware_utc, reporting_identifier +from adcp.reporting.ledger._delivery_state import latest_check +from adcp.reporting.ledger.delivery_models import ( + MaterializationFailure, + ReportingDeliveryRecord, + ReportingDeliveryScope, + ReportingDestinationBinding, + ReportingMaterializationAttempt, + ReportingMaterializationRecord, + ReportingObligationDeliveryRecord, +) +from adcp.reporting.ledger.models import ( + ReportingConfiguration, + ReportingObligationRecord, + ReportingRevisionRecord, +) +from adcp.reporting.materializer._errors import ReportingMaterializerUsageError +from adcp.reporting.materializer.contracts import ( + ReportingDestinationRequest, + ReportingPreparedRevision, + ReportingVerificationKey, + ReportingWriterFailure, + failure, +) +from adcp.reporting.materializer.verification import ( + ReportingRevisionRowReader, + ReportingVerifiedDestination, + _same_definition, +) +from adcp.reporting.revision_selection import select_reporting_revision + +MaterializerReason = Literal[ + "ready", + "verified", + "retry", + "inactive", + "revision_not_ready", + "revision_unreadable", + "target_changed", + "history_corrupt", + "legacy_pending", + "legacy_terminal", + "component_unavailable", + "binding_changed", + "effect_unknown", + "operator_required", +] + + +def scope_id(scope: ReportingDeliveryScope) -> str: + return hashlib.sha256(canonical_json_utf8_v1(asdict(scope))).hexdigest() + + +def verification_key_id(key: ReportingVerificationKey) -> str: + return hashlib.sha256(canonical_json_utf8_v1(asdict(key))).hexdigest() + + +def key_for( + binding: ReportingDestinationBinding, + obligation: ReportingObligationRecord, + keys: tuple[ReportingVerificationKey, ...], + *, + required: str | None = None, +) -> ReportingVerificationKey: + matches = tuple( + key + for key in keys + if (required is None or verification_key_id(key) == required) + and _same_definition(key, obligation.definition) + and (key.report_definition_id, key.reporting_profile) + == (obligation.report_definition_id, obligation.reporting_profile) + and ( + key.capability.method, + key.capability.transport, + key.capability.format, + key.capability.verification_profile, + ) + == (binding.method, binding.transport, binding.format, binding.verification_profile) + ) + if len(matches) != 1: + raise failure("UNSUPPORTED_VERIFICATION") + return matches[0] + + +@dataclass(frozen=True) +class MaterializerContext: + configuration: ReportingConfiguration + obligation: ReportingObligationRecord + binding: ReportingDestinationBinding + delivery: ReportingObligationDeliveryRecord | None + revisions: tuple[ReportingRevisionRecord, ...] + records: tuple[ReportingDeliveryRecord, ...] + + @property + def scope(self) -> ReportingDeliveryScope: + return ReportingDeliveryScope( + self.obligation.generation_key, + self.binding.consumer_id, + self.obligation.reporting_obligation_id, + ) + + def selection(self, now: datetime) -> tuple[ReportingRevisionRecord | None, MaterializerReason]: + config = self.configuration + if ( + config.activated_at is None + or config.activated_at > now + or (config.deactivated_at is not None and config.deactivated_at <= now) + ): + return None, "inactive" + selected = select_reporting_revision( + self.revisions, + account_id=self.scope.principal.account_id, + reporting_obligation_id=self.scope.reporting_obligation_id, + required_finality=self.obligation.required_finality, + ) + if selected.kind == "corrupt": + return None, "history_corrupt" + if selected.kind != "selected": + return None, "revision_not_ready" + if not selected.revision.readable: + return selected.revision, "revision_unreadable" + return selected.revision, "ready" + + def attempts(self, revision_id: str) -> tuple[ReportingMaterializationAttempt, ...]: + return tuple( + sorted( + ( + r + for r in self.records + if isinstance(r, ReportingMaterializationAttempt) + and r.scope == self.scope + and r.reporting_revision_id == revision_id + ), + key=lambda r: r.attempt, + ) + ) + + def outcome( + self, attempt: ReportingMaterializationAttempt + ) -> ReportingMaterializationRecord | None: + return next( + ( + r + for r in self.records + if isinstance(r, ReportingMaterializationRecord) and r.key == attempt.key + ), + None, + ) + + def pending_attempts(self) -> tuple[ReportingMaterializationAttempt, ...]: + return tuple( + r + for r in self.records + if isinstance(r, ReportingMaterializationAttempt) + and r.scope == self.scope + and self.outcome(r) is None + ) + + def resumable(self, attempt: ReportingMaterializationAttempt) -> bool: + history = self.attempts(attempt.reporting_revision_id) + return ( + bool(history) + and history[-1] == attempt + and tuple(a.attempt for a in history) == tuple(range(1, len(history) + 1)) + and self.pending_attempts() == (attempt,) + ) + + def retained_success( + self, outcome: ReportingMaterializationRecord, now: datetime + ) -> tuple[MaterializerReason, datetime | None]: + check = latest_check(self.records, outcome.reporting_materialization_id, at=now) + if ( + outcome.resource is None + or outcome.resource.expires_at <= now + or (check is not None and check.state != "readable") + ): + return "operator_required", None + return "verified", outcome.resource.expires_at + + +@dataclass(frozen=True) +class ReportingMaterializerLease: + """A short fenced reservation. The token grants no destination authorization.""" + + scope: ReportingDeliveryScope + generation: int + token: str = field(repr=False) + expires_at: datetime + attempt: ReportingMaterializationAttempt + request: ReportingDestinationRequest + context: MaterializerContext + notifications_enabled: bool + + def __post_init__(self) -> None: + invalid = False + try: + invalid = ( + type(self.generation) is not int + or self.generation < 1 + or type(self.notifications_enabled) is not bool + or UUID(self.token).version != 4 + or self.attempt.scope != self.scope + or self.context.scope != self.scope + or self.request + != ReportingDestinationRequest.from_binding( + self.context.binding, self.attempt, self.request.verification_key + ) + ) + object.__setattr__(self, "expires_at", aware_utc(self.expires_at)) + except (ValueError, TypeError, AttributeError): + invalid = True + if invalid: + raise failure("BINDING_MISMATCH") + + +@dataclass(frozen=True) +class ReportingMaterializerTurn: + state: Literal["idle", "discovered", "parked", "pending", "verified", "failed"] + reason: MaterializerReason | None = None + reporting_materialization_id: str | None = None + + def __post_init__(self) -> None: + if self.state not in {"idle", "discovered", "parked", "pending", "verified", "failed"} or ( + self.reason is not None and self.reason not in get_args(MaterializerReason) + ): + raise ValueError("invalid materializer turn") + if self.reporting_materialization_id is not None: + reporting_identifier(self.reporting_materialization_id, maximum=255) + + +@runtime_checkable +class ReportingMaterializerStore(ReportingRevisionRowReader, Protocol): + """Opt-in service primitives. Old structural store implementations stay valid.""" + + async def claim_materialization( + self, *, keys: tuple[ReportingVerificationKey, ...], lease_seconds: int = 30 + ) -> ReportingMaterializerLease | ReportingMaterializerTurn: ... + + async def renew_materialization( + self, lease: ReportingMaterializerLease, *, lease_seconds: int + ) -> bool: ... + + async def authorize_materialization(self, lease: ReportingMaterializerLease) -> None: ... + + async def finish_materialization( + self, + lease: ReportingMaterializerLease, + *, + prepared: ReportingPreparedRevision | None = None, + verified: ReportingVerifiedDestination | None = None, + error: ReportingWriterFailure | None = None, + ) -> ReportingMaterializerTurn: ... + + +def validate_lease_seconds(value: int) -> None: + if type(value) is not int or not 3 <= value <= 300: + raise ReportingMaterializerUsageError("materializer leases require 3..300 seconds") + + +def public_failure(error: ReportingWriterFailure) -> MaterializationFailure: + if error.code in {"DESTINATION_CORRUPT", "SOURCE_INVALID"}: + return "CONTENT_CORRUPT" + if error.code == "WRITE_FAILED": + return "WRITE_FAILED" + if error.code in { + "RESOURCE_UNAVAILABLE", + "CURRENT_REVISION_CHANGED", + "REVISION_NOT_READY", + "AUTHORIZATION_DENIED", + "BINDING_MISMATCH", + "HISTORY_CORRUPT", + }: + return "RESOURCE_UNAVAILABLE" + return "VERIFICATION_FAILED" diff --git a/tests/conformance/reporting/_durable_materializer_support.py b/tests/conformance/reporting/_durable_materializer_support.py new file mode 100644 index 000000000..1f0c52b8d --- /dev/null +++ b/tests/conformance/reporting/_durable_materializer_support.py @@ -0,0 +1,349 @@ +"""Shared deterministic vectors; PostgreSQL work always uses real database time.""" + +from contextlib import asynccontextmanager +from copy import deepcopy +from dataclasses import dataclass, replace +from datetime import datetime, timedelta, timezone + +from adcp.reporting.ledger import ( + ReportingDeliveryScope, + ReportingDestinationBinding, + ReportingMaterializationRecord, + ReportingRevisionRecord, + revision_content_sha256, +) +from adcp.reporting.materializer import ( + ReferenceReportingDestinationWriter, + ReferenceReportingResolver, + ReportingDestinationIO, + ReportingRevisionVerifierRegistry, + reference_digest, + reference_verifier, +) +from adcp.reporting.materializer.memory import InMemoryReportingMaterializerStore +from adcp.reporting.materializer.service import ReportingMaterializerService +from adcp.reporting.materializer.work import ReportingMaterializerLease + +from ._generation_support import END, START, configuration, isolated_reporting_pool, obligation_for +from ._materializer_support import io_context, reference_rows +from ._reliable_support import ManualClock + + +@dataclass +class DurableHarness: + store: object + clock: object + pool: object = None + + async def image(self): + """Every table/collection changed by an SDK transaction, including heads.""" + if self.pool is None: + return deepcopy( + {k: v for k, v in vars(self.store).items() if k not in {"_clock", "_lock"}} + ) + from psycopg import sql + + result = {} + async with self.pool.connection() as c: + tables = await ( + await c.execute( + "SELECT tablename FROM pg_tables WHERE schemaname=current_schema()" + " AND starts_with(tablename,'reporting_') ORDER BY tablename" + ) + ).fetchall() + for (table,) in tables: + result[table] = await ( + await c.execute( + sql.SQL("SELECT to_jsonb(t) FROM {} t ORDER BY to_jsonb(t)::text").format( + sql.Identifier(table) + ) + ) + ).fetchall() + return result + + async def expire(self): + """Persist expiry, then let the production fence observe it; no timing sleeps.""" + if self.pool is None: + self.clock.advance(timedelta(seconds=1)) + for work in self.store._materializer_work.values(): + if not work.acked: + work.lease_until = self.clock() if work.token is not None else None + work.due_at = self.clock() + for candidate in self.store._materializer_candidates.values(): + if candidate.due_at is not None: + candidate.due_at = self.clock() + return + async with self.pool.connection() as c, c.transaction(): + await self.store._lock_account(c, "acct_a") + await c.execute( + "UPDATE reporting_materializer_work SET lease_until=CASE" + " WHEN lease_token IS NOT NULL THEN clock_timestamp() END," + " due_at=clock_timestamp() WHERE state='pending'" + ) + await c.execute( + "UPDATE reporting_materializer_candidates SET due_at=clock_timestamp()" + " WHERE due_at IS NOT NULL" + ) + await c.execute("UPDATE reporting_materializer_accounts SET due_at=clock_timestamp()") + + async def queue(self): + if self.pool is None: + state = self.store._materializer_outbox + if state is None: + return (), () + return tuple(state.events.values()), tuple(w.state for w in state.expansions.values()) + async with self.pool.connection() as c: + events = await ( + await c.execute("SELECT snapshot FROM reporting_materializer_notification_events") + ).fetchall() + expansions = await ( + await c.execute("SELECT state FROM reporting_materializer_notification_expansions") + ).fetchall() + return tuple(r[0] for r in events), tuple(r[0] for r in expansions) + + async def dirty(self): + """Ordered status projection work, identical shape on both backends.""" + if self.pool is None: + state = self.store._notification_state + records = () if state is None else tuple(state.dirty) + else: + from adcp.reporting.ledger.notification_models import decode_dirty + + async with self.pool.connection() as c: + rows = await ( + await c.execute( + "SELECT snapshot FROM reporting_status_dirty ORDER BY account_id,sequence" + ) + ).fetchall() + records = tuple(decode_dirty(row[0]) for row in rows) + return tuple((r.reason, r.after) for r in records) + + async def ordinary_events(self): + """The old shared notification queue, which B2 must never publish into.""" + if self.pool is None: + state = self.store._notification_state + return ( + () if state is None else tuple(e.notification_type for e in state.events.values()) + ) + async with self.pool.connection() as c: + rows = await ( + await c.execute("SELECT notification_type FROM reporting_notification_events") + ).fetchall() + return tuple(row[0] for row in rows) + + async def works(self): + if self.pool is None: + return tuple( + (w.request.external_id, "acked" if w.acked else "pending", w.generation) + for w in self.store._materializer_work.values() + ) + async with self.pool.connection() as c: + return tuple( + await ( + await c.execute( + "SELECT external_id,state,generation FROM reporting_materializer_work" + " ORDER BY created_at,reporting_materialization_id" + ) + ).fetchall() + ) + + +@asynccontextmanager +async def durable_harness(backend, *, notifications=False): + clock = ManualClock(datetime.now(timezone.utc)) + if backend == "memory": + yield DurableHarness( + InMemoryReportingMaterializerStore(clock=clock, notifications=notifications), clock + ) + else: + from adcp.reporting.materializer.pg import PgReportingMaterializerStore + + async with isolated_reporting_pool(autocommit=True) as pool: + store = PgReportingMaterializerStore(pool=pool, notifications=notifications) + await store.create_schema() + yield DurableHarness(store, clock, pool) + + +@dataclass +class DurableCase: + store: object + config: object + obligation: object + binding: object + revision: object + rows: object + verifier: object + registry: object + writer: object + resolver: object + io: object + + @property + def scope(self): + return ReportingDeliveryScope( + self.config.generation_key, + self.binding.consumer_id, + self.obligation.reporting_obligation_id, + ) + + @property + def keys(self): + return (self.verifier.key,) + + def service(self, **kwargs): + return ReportingMaterializerService(self.store, self.io, self.writer, **kwargs) + + async def claim(self, **kwargs): + for _ in range(4): + result = await self.store.claim_materialization(keys=self.keys, **kwargs) + if getattr(result, "state", None) != "discovered": + return result + raise AssertionError("discovery did not converge") + + async def verified(self, lease): + from adcp.reporting.materializer.service import _LeaseHeartbeat + + assert isinstance(lease, ReportingMaterializerLease) + target = lease.context + context = io_context() + # The same SDK-owned I/O component used by the service; the test holds + # its explicit lease while injecting individual finish boundaries. + context = replace(context, heartbeat=_LeaseHeartbeat(self.store, lease, 30, context.cancel)) + prepared = await self.registry.prepare( + key=self.verifier.key, + binding=target.binding, + delivery=target.delivery, + obligation=target.obligation, + revisions=target.revisions, + attempt=lease.attempt, + reader=self.store, + context=context, + ) + locator = await self.io.write(prepared, context=context) + verified = await self.io.verify(prepared, locator, context=context) + return prepared, verified + + async def outcomes(self): + snapshot = await self.store.read_reconciliation_snapshot(caller=self.scope.principal) + return tuple(r for r in snapshot.records if isinstance(r, ReportingMaterializationRecord)) + + async def publish( + self, revision_id="revision-official", *, finality="official", supersedes=None + ): + pairs = self.revision.control_totals + revision = replace( + self.revision, + reporting_revision_id=revision_id, + finality=finality, + revision_content_sha256=revision_content_sha256( + reporting_revision_id=revision_id, + row_count=len(self.rows), + control_totals=pairs, + reporting_rows=self.rows, + control_total_evidence=self.revision.managed_control_totals, + ), + finality_basis="source_final" if finality == "official" else None, + finality_policy_id="reference-final" if finality == "official" else None, + finalized_at=END if finality == "official" else None, + supersedes_reporting_revision_id=supersedes, + ) + return await self.store.commit_revision(revision, self.rows) + + +async def durable_case( + store, + *, + count=1, + account="acct_a", + consumer="https://buyer.example.test/agent", + finality="snapshot", + required="snapshot", + active=True, + binding=True, + legacy_definition=False, + reconciliation_mode="delivery_only", +): + verifier = reference_verifier() + if legacy_definition: + verifier = replace( + verifier, + key=replace( + verifier.key, + definition=replace( + verifier.key.definition, + monetary_metric_units=(), + monetary_control_total_units=(), + ), + ), + ) + registry = ReportingRevisionVerifierRegistry((verifier,)) + config = replace( + configuration(account), + deactivated_at=None if active else END, + definition=verifier.key.definition, + report_definition_id=verifier.key.report_definition_id, + required_finality=required, + ) + await store.put_configuration(config) + obligation = await store.commit_obligation(obligation_for(config)) + cap = verifier.key.capability + destination = ReportingDestinationBinding( + config.generation_key, + consumer, + "destination", + "trusted-reference-binding", + cap.method, + cap.transport, + cap.verification_profile, + reconciliation_mode, + "analytics", + 400, + START, + cap.format, + ("reference-v1",), + "available", + ) + if binding: + await store.put_destination_binding(destination) + rows = reference_rows(count) + _, totals = verifier.canonicalize(rows) + pairs = tuple((t.name, t.value) for t in totals) + revision = ReportingRevisionRecord( + f"revision-{account}", + account, + obligation.reporting_obligation_id, + finality, + revision_content_sha256( + reporting_revision_id=f"revision-{account}", + row_count=count, + control_totals=pairs, + reporting_rows=rows, + control_total_evidence=totals, + ), + count, + pairs, + END, + END, + END, + finality_basis="source_final" if finality == "official" else None, + finality_policy_id="reference-final" if finality == "official" else None, + finalized_at=END if finality == "official" else None, + canonical_content_digest=reference_digest(verifier, rows), + managed_control_totals=totals, + ) + await store.commit_revision(revision, rows) + writer = ReferenceReportingDestinationWriter((cap,)) + resolver = ReferenceReportingResolver(writer, registry, (destination,)) + return DurableCase( + store, + config, + obligation, + destination, + revision, + rows, + verifier, + registry, + writer, + resolver, + ReportingDestinationIO(registry, resolver), + ) diff --git a/tests/conformance/reporting/_materializer_frozen.py b/tests/conformance/reporting/_materializer_frozen.py new file mode 100644 index 000000000..ce7d0106c --- /dev/null +++ b/tests/conformance/reporting/_materializer_frozen.py @@ -0,0 +1,298 @@ +"""Copied out of the checkout and run with -I in an installed frozen wheel.""" + +import asyncio +import hashlib +import importlib +import json +import sys +import traceback +from dataclasses import replace +from pathlib import Path + + +async def main(settings): + from psycopg_pool import AsyncConnectionPool + + import adcp.reporting.ledger as ledger_module + from adcp.reporting.ledger import PgReportingLedgerStore + + origins = {} + for name, expected in settings["modules"].items(): + module = importlib.import_module(name) + assert hashlib.sha256(Path(module.__file__).read_bytes()).hexdigest() == expected + origins[name] = str(Path(module.__file__).resolve()) + workspace = Path(settings["workspace"]).resolve() + assert not any(Path(p).resolve().is_relative_to(workspace) for p in sys.path) + for name, module in tuple(sys.modules.items()): + if name == "adcp" or name.startswith("adcp."): + if getattr(module, "__file__", None): + assert not Path(module.__file__).resolve().is_relative_to(workspace) + assert "site-packages" in module.__file__ + has_records = hasattr(ledger_module, "PgReportingReconciliationStore") + store_type = ( + ledger_module.PgReportingReconciliationStore if has_records else PgReportingLedgerStore + ) + async with AsyncConnectionPool( + settings["conninfo"], kwargs=settings["kwargs"], min_size=2, max_size=4, open=False + ) as pool: + async with pool.connection() as connection: + database = await ( + await connection.execute( + "SELECT current_setting('server_version_num')::int," + " current_setting('server_encoding'),datcollate,datctype" + " FROM pg_database WHERE datname=current_database()" + ) + ).fetchone() + assert 160000 <= database[0] < 170000 and database[1] == "UTF8" + store = store_type(pool=pool) + if settings["action"] == "install": + await store.create_schema() + if settings["artifact"] in {"c", "b1"}: + from adcp.reporting.outbox import PgStatusNotificationStore + + await PgStatusNotificationStore( + store_type(pool=pool, notifications=True) + ).create_schema() + if settings["artifact"] in {"a", "b", "c", "b1"}: + from adcp.reporting.outbox._schema import validate_schema + + async with pool.connection() as connection: + await validate_schema(connection) + return { + "artifact": settings["artifact"], + "sha": settings["sha"], + "installed": True, + "origins": origins, + "module_hashes": settings["modules"], + "database": database, + "notifications_ready": ( + True if settings["artifact"] in {"a", "b", "c", "b1"} else None + ), + } + + notifications_ready = None + if settings["artifact"] in {"a", "b", "c", "b1"}: + from adcp.reporting.ledger.notification_models import ReportingNotificationError + from adcp.reporting.outbox._schema import validate_schema + + async with pool.connection() as connection: + try: + await validate_schema(connection) + notifications_ready = True + except ReportingNotificationError: + notifications_ready = False + # Before AND after B2: exactly the reviewed C/B1 envelope. A's + # aggregate closes; B/C/B1 per-object required manifests stay ready. + assert notifications_ready == (settings["artifact"] != "a") + + configs = await store.list_configurations(account_id="acct_a") + assert len(configs) == 1 + obligation = await store.get_obligation( + account_id="acct_a", reporting_obligation_id="rpo_acct_a" + ) + assert obligation is not None + revisions = await store.list_revisions( + account_id="acct_a", reporting_obligation_id=obligation.reporting_obligation_id + ) + assert len(revisions) == 1 and revisions[0].row_count == 3 + snapshot = await store.open_snapshot(account_id="acct_a", filters_fingerprint="frozen-b2") + page = await store.read_page( + snapshot=snapshot, + consumer_id=settings["consumer"], + delivery_config_ids=None, + media_buy_ids=None, + offset=0, + limit=100, + changes_after_sequence=None, + ) + assert len(page.obligations) == len(page.revisions) == 1 + assert not page.has_more + records = 0 + if has_records: + principal = ledger_module.ReportingDeliveryPrincipal("acct_a", settings["consumer"]) + retained = await store.read_reconciliation_snapshot(caller=principal) + assert len(retained.records) == (1 if settings["action"] == "baseline" else 4) + assert [r.status for r in retained.records if r.kind == "materialization"] == [ + "available" + ] * (settings["action"] != "baseline") + records = len(retained.records) + # Permitted ordinary old writes remain functional; legacy materializer + # writers are deliberately drained, not run beside the new worker. + status = None + projector_turns = 0 + if settings["artifact"] in {"c", "b1"} and settings["action"] != "baseline": + from adcp.reporting.outbox import PgStatusNotificationStore + + store = store_type(pool=pool, notifications=True) + status = PgStatusNotificationStore(store) + await status.baseline(account_id="acct_a") + await store.put_configuration( + replace(configs[0], status_retention_days=configs[0].status_retention_days + 1) + ) + for readable in (False, True): + await store.set_revision_readable( + account_id="acct_a", + reporting_revision_id=revisions[0].reporting_revision_id, + readable=readable, + ) + if status is not None: + while (await status.project_one(account_id="acct_a")).did_work: + projector_turns += 1 + assert projector_turns < 32 + assert ( + await store.get_revision( + account_id="acct_a", reporting_revision_id=revisions[0].reporting_revision_id + ) + ).readable + workers = {} + event_identities = {} + if settings["artifact"] in {"a", "b", "c", "b1"} and settings["action"] != "baseline": + from adcp.reporting.ledger.notification_models import decode_event + from adcp.reporting.outbox import ( + PgReportingOutbox, + ReportingEnvelopeCipher, + ReportingNotificationWorker, + ) + + class EmptySubscriptions: + async def list_active(self, **kwargs): + assert kwargs["notification_type"] != "reporting.delivery_ready" + return () + + async def get_active(self, **kwargs): + raise AssertionError("empty recipient membership has no HTTP delivery") + + outboxes = {"ordinary": PgReportingOutbox(pool=pool)} + if settings["artifact"] in {"c", "b1"}: + from adcp.reporting.outbox.status_pg import PgReportingStatusOutbox + + outboxes["status"] = PgReportingStatusOutbox(pool=pool) + for name, outbox in outboxes.items(): + events = await outbox.list_events(account_id="acct_a") + if name == "ordinary": + assert len(events) == 1 + event = events[0] + assert event.account_id == "acct_a" and event.consumer_namespace == "" + assert event.notification_type == "reporting.ledger_changed" + assert event.cause.kind == "revision_published" + assert event.cause.reporting_revision_id == "revision-acct_a" + assert event.cause.finality == "snapshot" and event.cause_generation == 1 + assert event.cause.supersedes_reporting_revision_id is None + else: + # Six known Core views (configuration + obligation for the + # seller and two callers), each complete -> action -> complete + # for this closed period with snapshot-required finality. + # Materialization captures add no C reconciliation semantics. + expected = { + ( + ("acct_a", consumer, "daily", 1, kind, obligation_id), + generation, + previous, + health, + ) + for consumer in ( + "", + settings["consumer"], + "https://buyer.example.test/isolated", + ) + for kind, obligation_id in ( + ("configuration", ""), + ("obligation", "rpo_acct_a"), + ) + for generation, previous, health in ( + (1, "complete", "action_required"), + (2, "action_required", "complete"), + ) + } + assert len(events) == len(expected) == 12 + assert all( + e.notification_type == "reporting.status_changed" + and e.cause.kind == "status_changed" + and e.cause_generation == e.cause.checkpoint_generation + and len(e.cause.issue_ids) == int(e.cause.health == "action_required") + for e in events + ) + assert { + ( + e.cause.scope.checkpoint_key, + e.cause_generation, + e.cause.previous_health, + e.cause.health, + ) + for e in events + } == expected + assert len({e.causal_key for e in events}) == len(events) + identities = { + (e.account_id, e.consumer_namespace, e.notification_id): e for e in events + } + assert len(identities) == len(events) + seen = set() + + class ObservedOutbox: + def __getattr__(self, attribute): + return getattr(outbox, attribute) + + async def claim_expansion(self, **kwargs): + lease = await outbox.claim_expansion(**kwargs) + if lease is not None: + identity = ( + lease.account_id, + lease.consumer_namespace, + lease.notification_id, + ) + assert identity in identities and identity not in seen + assert decode_event(lease.event) == identities[identity] + seen.add(identity) + return lease + + worker = ReportingNotificationWorker( + outbox=ObservedOutbox(), + subscriptions=EmptySubscriptions(), + cipher=ReportingEnvelopeCipher(b"e" * 32), + ) + for _ in events: + materializer_operation_3 = await worker.expand_one(account_id="acct_a") + assert materializer_operation_3 + assert seen == identities.keys() + materializer_operation_1 = await worker.deliver_one(account_id="acct_a") + assert not materializer_operation_1 + materializer_operation_2 = await worker.expand_one(account_id="acct_a") + assert not materializer_operation_2 + assert await outbox.list_events(account_id="acct_a") == events + workers[name] = len(seen) + event_identities[name] = [ + {"notification_id": e.notification_id, "causal_key": e.causal_key} + for e in events + ] + assert workers["ordinary"] == 1 # Positive control: an actual Core event was claimed. + if status is not None: + assert projector_turns > 0 and workers["status"] > 0 + return { + "artifact": settings["artifact"], + "sha": settings["sha"], + "installed": True, + "core_records": 2, + "managed_records": records, + "ordinary_writes": True, + "workers": workers, + "event_identities": event_identities, + "projector_turns": projector_turns, + "notifications_ready": notifications_ready, + "origins": origins, + "module_hashes": settings["modules"], + "database": database, + } + + +if __name__ == "__main__": + try: + result = asyncio.run(main(json.load(sys.stdin))) + except Exception as error: + result = { + "failure": type(error).__name__, + "frames": [ + [Path(frame.filename).name, frame.lineno] + for frame in traceback.extract_tb(error.__traceback__) + ], + } + print(json.dumps(result)) diff --git a/tests/conformance/reporting/_materializer_installed.py b/tests/conformance/reporting/_materializer_installed.py index dc5f5c4f2..c56349360 100644 --- a/tests/conformance/reporting/_materializer_installed.py +++ b/tests/conformance/reporting/_materializer_installed.py @@ -6,10 +6,13 @@ import importlib.util import json import sys +from dataclasses import fields from datetime import datetime, timedelta, timezone from importlib.resources import files from pathlib import Path +from pydantic import TypeAdapter + async def main(): config = json.load(sys.stdin) @@ -39,9 +42,13 @@ async def main(): revision_content_sha256, ) from adcp.reporting.materializer import ( + InMemoryReportingMaterializerStore, ReportingDestinationBinding, + ReportingDestinationIO, ReportingMaterializationAttempt, + ReportingMaterializerService, ReportingObligationDeliveryRecord, + ReportingVerifiedDestination, reference_digest, reference_verifier, ) @@ -63,6 +70,8 @@ async def main(): files("adcp.reporting.ledger").joinpath("reporting_status_selector_version.sql").is_file() ) assert files("adcp.reporting.outbox").joinpath("required_status_selector_schema.json").is_file() + assert files("adcp.reporting.ledger").joinpath("reporting_materializer.sql").is_file() + assert files("adcp.reporting.materializer").joinpath("required_schema.json").is_file() start = datetime(2026, 9, 1, tzinfo=timezone.utc) schedule = ReportingScheduleSpec("PT1H", "PT1H", period_anchor=start) period = derive_period(schedule, account_timezone="UTC", ordinal=0) @@ -170,9 +179,37 @@ async def main(): ) ) assert all(result.verification.row_count == count for result in results) + codec = TypeAdapter(ReportingVerifiedDestination) + assert {item.name for item in fields(results[0])} == { + "request", + "resource", + "verification", + } + assert set(json.loads(codec.dump_json(results[0]))) == { + "request", + "resource", + "verification", + } + assert codec.validate_json(codec.dump_json(results[0])) == results[0] assert results[0].request.external_id == results[1].request.external_id assert destination.writer.write_effects == 1 and not destination.writer.production_eligible assert destination.writer.open_count == destination.writer.close_count == 4 + durable = InMemoryReportingMaterializerStore(notifications=False) + await durable.put_configuration(configuration) + await durable.commit_obligation(obligation) + await durable.put_destination_binding(binding) + await durable.commit_revision(revision, rows) + destination = example.development_destination(binding) + service = ReportingMaterializerService( + durable, + ReportingDestinationIO(destination.registry, destination.resolver), + destination.writer, + ) + materializer_operation_1 = await service.run_once() + assert (materializer_operation_1).state == "verified" + boundaries = await durable.read_materializer_boundaries(caller=scope.principal) + assert len(boundaries) == boundaries[0].sequence == boundaries[0].account_sequence == 1 + assert durable._materializer_outbox is None workspace = Path(config["workspace"]).resolve() assert all(not Path(path).resolve().is_relative_to(workspace) for path in sys.path) assert all( @@ -182,7 +219,17 @@ async def main(): if getattr(module, "__file__", None) ) assert not any(name.startswith("psycopg") for name in sys.modules) - print(json.dumps({"python": "3.10", "rows": [0, 501], "installed": True, "assets": actual})) + print( + json.dumps( + { + "python": "3.10", + "rows": [0, 501], + "installed": True, + "assets": actual, + "durable": True, + } + ) + ) asyncio.run(main()) diff --git a/tests/conformance/reporting/_materializer_process.py b/tests/conformance/reporting/_materializer_process.py new file mode 100644 index 000000000..a6267a225 --- /dev/null +++ b/tests/conformance/reporting/_materializer_process.py @@ -0,0 +1,181 @@ +"""Real worker process and conditional file destination, solely for crash tests.""" + +import asyncio +import hashlib +import importlib +import json +import os +import sys +from pathlib import Path +from uuid import uuid4 + + +def emit(point, **values): + print(json.dumps({"point": point, **values}), flush=True) + + +async def read(): + return json.loads(await asyncio.to_thread(sys.stdin.readline)) + + +async def main(): + settings = await read() + from psycopg import AsyncConnection + from psycopg_pool import AsyncConnectionPool + from pydantic import TypeAdapter + + from adcp.reporting.ledger._delivery_state import decode_record + from adcp.reporting.materializer import ( + PgReportingMaterializerStore, + ReferenceReportingDestinationWriter, + ReferenceReportingResolver, + ReportingDestinationIO, + ReportingMaterializerService, + ReportingRevisionVerifierRegistry, + reference_verifier, + ) + from adcp.reporting.materializer.reference import _Artifact, _Session + + origins = {} + if settings.get("installed"): + installed = settings["installed"] + workspace = Path(installed["workspace"]).resolve() + for name, expected in installed["modules"].items(): + module = importlib.import_module(name) + path = Path(module.__file__).resolve() + assert hashlib.sha256(path.read_bytes()).hexdigest() == expected + origins[name] = str(path) + assert not any(Path(p).resolve().is_relative_to(workspace) for p in sys.path) + for name, module in tuple(sys.modules.items()): + if name == "adcp" or name.startswith("adcp."): + if getattr(module, "__file__", None): + assert not Path(module.__file__).resolve().is_relative_to(workspace) + assert "site-packages" in module.__file__ + assert list(sys.version_info[:2]) == installed["python"] + + if settings.get("action") == "install": + from adcp.reporting.ledger import LedgerConflictError + + async with AsyncConnectionPool( + settings["conninfo"], kwargs=settings["kwargs"], min_size=1, max_size=1, open=False + ) as pool: + store = PgReportingMaterializerStore(pool=pool, notifications=settings["notifications"]) + try: + await store.materializer_ready() + except LedgerConflictError: + # An empty schema must reject readiness before installation. + pass + else: + raise AssertionError("empty schema must be unready") + await store.create_schema() + await store.create_schema() + assert await store.materializer_ready() + emit("done", state="installed", origins=origins) + return + + async def hit(point): + if point == settings.get("pause"): + emit(point) + materializer_operation_1 = await read() + assert (materializer_operation_1)["continue"] + + class Connection(AsyncConnection): + async def execute(self, query, params=None, **kwargs): + result = await super().execute(query, params, **kwargs) + if isinstance(query, str) and query.startswith( + "INSERT INTO reporting_materializer_notification_events" + ): + await hit("after_event") + return result + + class Store(PgReportingMaterializerStore): + async def claim_materialization(self, **kwargs): + result = await super().claim_materialization(**kwargs) + if hasattr(result, "attempt"): + await hit("reserved") + return result + + async def _commit_record_on(self, connection, record, **kwargs): + result = await super()._commit_record_on(connection, record, **kwargs) + if record.kind == "materialization": + await hit("after_outcome") + return result + + async def _materializer_dirty_on(self, *args): + await super()._materializer_dirty_on(*args) + await hit("after_capture") + + async def finish_materialization(self, *args, **kwargs): + if kwargs.get("verified") is not None: + await hit("after_readback") + result = await super().finish_materialization(*args, **kwargs) + if result.state == "verified": + await hit("after_commit") + return result + + verifier = reference_verifier() + registry = ReportingRevisionVerifierRegistry((verifier,)) + writer = ReferenceReportingDestinationWriter((verifier.key.capability,)) + reference = ReferenceReportingResolver(writer, registry, (decode_record(settings["binding"]),)) + artifact_codec = TypeAdapter(_Artifact) + directory = Path(settings["destination"]) + + class Session(_Session): + async def write(self, content): + await hit("before_write") + path = directory / self.request.external_id + if path.exists(): + writer._artifacts[self.request.external_id] = artifact_codec.validate_json( + path.read_bytes() + ) + locator = await super().write(content) + if not path.exists(): + temporary = directory / ("staged-" + uuid4().hex) + temporary.write_bytes( + artifact_codec.dump_json(writer._artifacts[self.request.external_id]) + ) + with temporary.open("rb") as stream: + os.fsync(stream.fileno()) + try: + os.link(temporary, path) # Conditional atomic publication, no overwrite. + except FileExistsError: + writer._artifacts[self.request.external_id] = artifact_codec.validate_json( + path.read_bytes() + ) + locator = await super().write(content) + finally: + temporary.unlink() + descriptor = os.open(directory, os.O_RDONLY) + try: + os.fsync(descriptor) + finally: + os.close(descriptor) + await hit("after_write") + return locator + + class Resolver: + def resolve(self, request, *, phase, context): + return Session(reference, request, phase, context) + + async with AsyncConnectionPool( + settings["conninfo"], + kwargs=settings["kwargs"], + connection_class=Connection, + min_size=1, + max_size=1, + open=False, + ) as pool: + store = Store(pool=pool, notifications=settings["notifications"]) + service = ReportingMaterializerService( + store, ReportingDestinationIO(registry, Resolver()), writer, lease_seconds=90 + ) + result = await service.run_once() + emit("done", state=result.state, reason=result.reason, origins=origins) + + +if __name__ == "__main__": + try: + asyncio.run(main()) + except BaseException as error: + emit("failed", classification=type(error).__name__) + sys.exit(1) diff --git a/tests/conformance/reporting/test_reporting_materializer_contracts.py b/tests/conformance/reporting/test_reporting_materializer_contracts.py index e164315c5..42392aa33 100644 --- a/tests/conformance/reporting/test_reporting_materializer_contracts.py +++ b/tests/conformance/reporting/test_reporting_materializer_contracts.py @@ -227,7 +227,7 @@ async def checkpoint(self): assert calls == 1 and case.writer.open_count == 0 -def test_curated_all_exports_resolve_and_do_not_add_materializer_sql(): +def test_curated_all_exports_resolve_with_the_isolated_optional_materializer(): for name in ( "adcp.reporting.materializer", "adcp.reporting.revision_selection", @@ -239,9 +239,13 @@ def test_curated_all_exports_resolve_and_do_not_add_materializer_sql(): for public in module.__all__: assert getattr(module, public) is not None module = importlib.import_module("adcp.reporting.materializer") - assert not any( - name.endswith(("Coordinator", "Service", "Lease", "WorkQueue")) for name in module.__all__ - ) + # B1 deliberately stopped before orchestration. B2.1 adds only this optional + # service/lease surface; it does not promote the reference writer or tiers. + assert { + name + for name in module.__all__ + if name.endswith(("Coordinator", "Service", "Lease", "WorkQueue")) + } == {"ReportingMaterializerService", "ReportingMaterializerLease"} assert inspect.isclass(module.ReportingDestinationSession) assert get_type_hints(module.ReportingDestinationSession.write)["content"] is ( module.ReportingPreparedRevision diff --git a/tests/conformance/reporting/test_reporting_materializer_durable.py b/tests/conformance/reporting/test_reporting_materializer_durable.py new file mode 100644 index 000000000..153c252d8 --- /dev/null +++ b/tests/conformance/reporting/test_reporting_materializer_durable.py @@ -0,0 +1,433 @@ +"""Shared durable state machine: real PostgreSQL and deterministic memory.""" + +from dataclasses import fields, replace + +import pytest +from pydantic import TypeAdapter + +from adcp.reporting.ledger import ReportingMaterializationAttempt +from adcp.reporting.materializer import ( + ReportingVerifiedDestination, + ReportingWriterError, + ReportingWriterFailure, +) +from adcp.reporting.materializer.work import ReportingMaterializerLease + +from ._durable_materializer_support import durable_case, durable_harness +from ._generation_support import configuration, obligation_for, revision_for + +pytestmark = pytest.mark.parametrize("backend", ["memory", "postgres"]) + + +async def test_core_records_do_not_create_managed_work_capture_or_readiness(backend): + async with durable_harness(backend, notifications=True) as h: + config = configuration() + obligation = obligation_for(config) + revision, rows = revision_for(obligation) + await h.store.put_configuration(config) + await h.store.commit_obligation(obligation) + await h.store.commit_revision(revision, rows) + materializer_operation_1 = await h.store.claim_materialization(keys=()) + assert (materializer_operation_1).state == "idle" + assert not await h.works() and await h.queue() == ((), ()) + + +async def test_durable_verified_success_still_cannot_advertise_unactivated_readiness(backend): + from adcp.reporting.ledger.notification_models import ReportingNotificationError + from adcp.reporting.outbox import ( + InMemoryReportingOutbox, + PgReportingOutbox, + ReportingEnvelopeCipher, + ReportingNotificationWorker, + ) + + class NoSubscriptions: + async def list_active(self, **kwargs): + raise AssertionError("capability inspection must not enumerate subscriptions") + + async def get_active(self, **kwargs): + raise AssertionError("capability inspection must not authorize an HTTP delivery") + + async with durable_harness(backend, notifications=True) as h: + case = await durable_case(h.store) + materializer_operation_2 = await case.service().run_once() + assert (materializer_operation_2).state == "verified" + outbox = ( + PgReportingOutbox(pool=h.pool) + if h.pool is not None + else InMemoryReportingOutbox(h.store) + ) + worker = ReportingNotificationWorker( + outbox=outbox, + subscriptions=NoSubscriptions(), + cipher=ReportingEnvelopeCipher(b"e" * 32), + ) + # The new store is deliberately not in the production composition's + # approved identity set until B2.4 proves projection/mount readiness. + with pytest.raises(ReportingNotificationError, match="notification_chain_unready"): + await worker.advertised_notifications( + h.store, account_id="acct_a", ready_scope=case.scope + ) + assert len((await h.queue())[0]) == 1 and (await h.queue())[1] == ("quarantined",) + + +async def test_materializer_store_closes_core_advertisement_until_b24_admits_it(backend): + """Pin the full extent of the closed advertisement so B2.4 must act deliberately. + + `advertised_notifications` matches the ledger by exact `type(...)`, so both + materializer stores lose Core `reporting.ledger_changed` too, not only the + Managed/Reconciled claims. Fail closed is correct here, but it is broader + than a tier veto and an adopter switching store classes must be told. + """ + from adcp.reporting.ledger.notification_models import ReportingNotificationError + from adcp.reporting.outbox import ( + InMemoryReportingOutbox, + PgReportingOutbox, + ReportingEnvelopeCipher, + ReportingNotificationWorker, + ) + + class NoSubscriptions: + async def list_active(self, **kwargs): + return () + + async def get_active(self, **kwargs): + return None + + async def advertise(store, outbox): + return await ReportingNotificationWorker( + outbox=outbox, + subscriptions=NoSubscriptions(), + cipher=ReportingEnvelopeCipher(b"e" * 32), + ).advertised_notifications(store, account_id="acct_a", ready_scope=None) + + async with durable_harness(backend, notifications=True) as h: + if h.pool is None: + from adcp.reporting.ledger.delivery import InMemoryReportingReconciliationStore + + reviewed = InMemoryReportingReconciliationStore(notifications=True) + reviewed_outbox = InMemoryReportingOutbox(reviewed) + outbox = InMemoryReportingOutbox(h.store) + else: + from adcp.reporting.ledger.delivery_pg import PgReportingReconciliationStore + + reviewed = PgReportingReconciliationStore(pool=h.pool, notifications=True) + reviewed_outbox = PgReportingOutbox(pool=h.pool) + outbox = PgReportingOutbox(pool=h.pool) + # The reviewed store still advertises the Core notification. + materializer_operation_3 = await advertise(reviewed, reviewed_outbox) + assert (materializer_operation_3)["ledger_notification"] == ("reporting.ledger_changed") + with pytest.raises(ReportingNotificationError, match="notification_chain_unready"): + await advertise(h.store, outbox) + + +@pytest.mark.parametrize("notifications", [False, True]) +@pytest.mark.parametrize("count", [0, 501]) +async def test_verified_finish_captures_private_inputs_acks_and_quarantines_exact_event( + backend, notifications, count +): + async with durable_harness(backend, notifications=notifications) as h: + case = await durable_case(h.store, count=count) + lease = await case.claim() + assert lease.attempt.attempt == 1 + assert lease.context.delivery.created_at == lease.attempt.created_at + prepared, evidence = await case.verified(lease) + result = await h.store.finish_materialization(lease, prepared=prepared, verified=evidence) + assert result.state == "verified" + assert case.writer.write_effects == 1 + assert case.writer.open_count == case.writer.close_count == 2 + outcomes = await case.outcomes() + assert len(outcomes) == 1 and outcomes[0].verification.row_count == count + boundaries = await h.store.read_materializer_boundaries(caller=case.scope.principal) + assert len(boundaries) == 1 + assert boundaries[0].as_of == outcomes[0].completed_at + assert boundaries[0].core.consumer_ids == (case.binding.consumer_id,) + assert outcomes[0] in boundaries[0].reconciliation + assert ( + await h.store.read_materializer_boundaries( + caller=replace(case.scope.principal, consumer_id="another-buyer") + ) + == () + ) + events, expansions = await h.queue() + assert len(events) == int(notifications) + assert expansions == (("quarantined",) if notifications else ()) + assert (await h.works())[0][1] == "acked" + materializer_operation_4 = await h.store.finish_materialization( + lease, prepared=prepared, verified=evidence + ) + assert materializer_operation_4 == result + assert await h.queue() == (events, expansions) + assert await h.store.read_materializer_boundaries(caller=case.scope.principal) == boundaries + + +async def test_service_owns_reservation_verification_and_finish(backend): + async with durable_harness(backend) as h: + case = await durable_case(h.store) + materializer_operation_5 = await case.service().run_once() + assert (materializer_operation_5).state == "verified" + materializer_operation_6 = await case.service().run_once() + assert (materializer_operation_6).state in {"idle", "discovered"} + assert case.writer.write_effects == 1 + + +async def test_invalid_caller_arguments_stay_actionable_and_claim_no_external_effect(backend): + """A caller-argument rejection must not masquerade as an unknown external effect. + + The closed-error guard previously rewrote these deterministic ``ValueError``s + into ``RESOURCE_UNAVAILABLE``/``same_identity``/``unknown``, which tells an + adopter a destination write may have started and hides what to correct. + """ + async with durable_harness(backend) as h: + case = await durable_case(h.store) + before = await h.image() + calls = ( + ( + "materializer leases", + lambda: h.store.claim_materialization(keys=case.keys, lease_seconds=1), + ), + ( + "materializer leases", + lambda: h.store.claim_materialization(keys=case.keys, lease_seconds=301), + ), + ( + "materializer boundary reads", + lambda: h.store.read_materializer_boundaries(caller=case.scope.principal, after=-1), + ), + ( + "materializer boundary reads", + lambda: h.store.read_materializer_boundaries(caller=case.scope.principal, limit=0), + ), + ) + for expected, call in calls: + with pytest.raises(ValueError) as caught: + await call() + assert not isinstance(caught.value, ReportingWriterError) + assert str(caught.value).startswith(expected) + assert not hasattr(caught.value, "failure") + assert await h.image() == before + assert case.writer.write_effects == 0 + + +async def test_unknown_effect_resumes_same_attempt_external_identity_and_rejects_old_fence(backend): + async with durable_harness(backend) as h: + case = await durable_case(h.store) + first = await case.claim() + prepared, evidence = await case.verified(first) + result = await h.store.finish_materialization( + first, error=ReportingWriterFailure("WRITE_FAILED", "same_identity", "unknown") + ) + assert result.state == "pending" + assert not await case.outcomes() + await h.expire() + second = await case.claim() + assert second.attempt == first.attempt + assert second.request.external_id == first.request.external_id + assert second.token != first.token + materializer_operation_7 = await h.store.finish_materialization( + first, prepared=prepared, verified=evidence + ) + assert (materializer_operation_7).state == "pending" + prepared, evidence = await case.verified(second) + assert case.writer.write_effects == 1 + materializer_operation_8 = await h.store.finish_materialization( + second, prepared=prepared, verified=evidence + ) + assert (materializer_operation_8).state == "verified" + + +@pytest.mark.parametrize("change", ["official", "unreadable", "deactivated"]) +async def test_changed_authority_finishes_safe_failure_without_reusing_artifact(backend, change): + async with durable_harness(backend, notifications=True) as h: + case = await durable_case(h.store) + first = await case.claim() + prepared, evidence = await case.verified(first) + if change == "official": + await case.publish() + elif change == "unreadable": + await h.store.set_revision_readable( + account_id=case.config.account_id, + reporting_revision_id=case.revision.reporting_revision_id, + readable=False, + ) + else: + await h.store.put_configuration( + replace(case.config, deactivated_at=case.revision.created_at) + ) + result = await h.store.finish_materialization(first, prepared=prepared, verified=evidence) + assert result.state == "failed" + assert (await case.outcomes())[0].failure_code == "RESOURCE_UNAVAILABLE" + assert await h.queue() == ((), ()) + next_work = await case.claim() + if change == "official": + assert next_work.attempt.reporting_revision_id == "revision-official" + assert next_work.attempt.attempt == 1 + assert next_work.request.external_id != first.request.external_id + else: + assert not isinstance(next_work, ReportingMaterializerLease) + if change == "unreadable": + await h.store.set_revision_readable( + account_id=case.config.account_id, + reporting_revision_id=case.revision.reporting_revision_id, + readable=True, + ) + next_work = await case.claim() + assert ( + next_work.attempt.reporting_revision_id == first.attempt.reporting_revision_id + ) + assert next_work.attempt.attempt == 2 + + +async def test_known_service_failure_allows_next_attempt_but_pending_does_not(backend): + async with durable_harness(backend) as h: + case = await durable_case(h.store) + first = await case.claim() + materializer_operation_9 = await case.claim() + assert not isinstance(materializer_operation_9, ReportingMaterializerLease) + await h.store.finish_materialization( + first, error=ReportingWriterFailure("WRITE_FAILED", "new_attempt", "not_started") + ) + await h.expire() + second = await case.claim() + assert second.attempt.attempt == 2 + assert second.request.external_id != first.request.external_id + + +async def test_sdk_seal_rejects_fabricated_or_modified_readback(backend): + async with durable_harness(backend) as h: + case = await durable_case(h.store) + lease = await case.claim() + prepared, evidence = await case.verified(lease) + forged = ReportingVerifiedDestination( + evidence.request, evidence.resource, evidence.verification + ) + with pytest.raises(ReportingWriterError): + await h.store.finish_materialization(lease, prepared=prepared, verified=forged) + codec = TypeAdapter(ReportingVerifiedDestination) + assert {f.name for f in fields(evidence)} == {"request", "resource", "verification"} + assert set(codec.dump_python(evidence, mode="json")) == { + "request", + "resource", + "verification", + } + restored = codec.validate_json(codec.dump_json(evidence)) + assert restored == evidence # B1's serializable observation contract survives. + with pytest.raises(ReportingWriterError): + await h.store.finish_materialization(lease, prepared=prepared, verified=restored) + object.__setattr__(evidence, "verification", replace(evidence.verification, row_count=7)) + with pytest.raises(ReportingWriterError): + await h.store.finish_materialization(lease, prepared=prepared, verified=evidence) + assert not await case.outcomes() + assert (await h.works())[0][1] == "pending" + + +async def test_prior_lease_readback_cannot_finish_a_new_fence_in_the_same_process(backend): + async with durable_harness(backend, notifications=True) as h: + case = await durable_case(h.store) + old = await case.claim() + prepared, proof = await case.verified(old) + await h.expire() + resumed = await case.claim() + assert resumed.attempt == old.attempt and resumed.token != old.token + with pytest.raises(ReportingWriterError): + await h.store.finish_materialization(resumed, prepared=prepared, verified=proof) + assert not await case.outcomes() and await h.queue() == ((), ()) + prepared, proof = await case.verified(resumed) + materializer_operation_10 = await h.store.finish_materialization( + resumed, prepared=prepared, verified=proof + ) + assert (materializer_operation_10).state == "verified" + assert case.writer.write_effects == 1 + + +async def test_finish_rechecks_sdk_evidence_after_acquiring_transaction_locks(backend, monkeypatch): + async with durable_harness(backend, notifications=True) as h: + case = await durable_case(h.store) + lease = await case.claim() + prepared, proof = await case.verified(lease) + before = await h.image() + cls = type(h.store) + if h.pool is None: + original = cls._held + + def held(self, reserved): + result = original(self, reserved) + object.__setattr__(proof, "verification", replace(proof.verification, row_count=7)) + return result + + monkeypatch.setattr(cls, "_held", held) + else: + original = cls._held_on + + async def held(self, connection, reserved): + result = await original(self, connection, reserved) + object.__setattr__(proof, "verification", replace(proof.verification, row_count=7)) + return result + + monkeypatch.setattr(cls, "_held_on", held) + with pytest.raises(ReportingWriterError): + await h.store.finish_materialization(lease, prepared=prepared, verified=proof) + assert await h.image() == before + assert not await case.outcomes() and await h.queue() == ((), ()) + + +async def test_legacy_pending_requires_explicit_exact_external_history_import(backend): + async with durable_harness(backend) as h: + case = await durable_case(h.store) + # The public API remains independent; this predates any service reservation. + from datetime import timedelta + + from adcp.reporting.ledger import ReportingObligationDeliveryRecord + from adcp.reporting.materializer import ReportingDestinationRequest + + await h.store.bind_obligation_delivery( + ReportingObligationDeliveryRecord( + case.scope, + "USD", + case.revision.created_at + timedelta(days=400), + case.revision.created_at, + ) + ) + attempt = ReportingMaterializationAttempt( + case.scope, case.revision.reporting_revision_id, "legacy", 1, case.revision.created_at + ) + await h.store.commit_materialization_attempt(attempt) + result = await case.claim() + assert result.reason == "legacy_pending" + assert not await h.works() + with pytest.raises(ReportingWriterError): + await h.store.import_pending_materialization( + scope=case.scope, + reporting_materialization_id="legacy", + original_external_id="unknown", + keys=case.keys, + ) + identity = ReportingDestinationRequest.from_binding( + case.binding, attempt, case.verifier.key + ).external_id + await h.store.import_pending_materialization( + scope=case.scope, + reporting_materialization_id="legacy", + original_external_id=identity, + keys=case.keys, + ) + lease = await case.claim() + assert lease.attempt == attempt and lease.request.external_id == identity + + +@pytest.mark.parametrize( + "required,finality,expected", + [ + ("snapshot", "snapshot", "verified"), + ("official", "snapshot", "parked"), + ("official", "official", "verified"), + ], +) +async def test_finality_and_late_delivery_binding(backend, required, finality, expected): + async with durable_harness(backend) as h: + case = await durable_case(h.store, required=required, finality=finality, binding=False) + materializer_operation_11 = await case.service().run_once() + assert (materializer_operation_11).state == "idle" + await h.store.put_destination_binding(case.binding) + materializer_operation_12 = await case.service().run_once() + assert (materializer_operation_12).state == expected diff --git a/tests/conformance/reporting/test_reporting_materializer_history.py b/tests/conformance/reporting/test_reporting_materializer_history.py new file mode 100644 index 000000000..5c9ae7716 --- /dev/null +++ b/tests/conformance/reporting/test_reporting_materializer_history.py @@ -0,0 +1,334 @@ +"""History, late binding, private captured ordering and selector corruption.""" + +from dataclasses import replace +from datetime import timedelta + +import pytest + +from adcp.reporting.ledger import ( + ReportingMaterializationAttempt, + ReportingMaterializationRecord, + ReportingObligationDeliveryRecord, + ReportingRevisionReceiptRecord, +) +from adcp.reporting.materializer import ReportingDestinationRequest, ReportingWriterError + +from ._durable_materializer_support import durable_case, durable_harness + +pytestmark = pytest.mark.parametrize("backend", ["memory", "postgres"]) + + +async def test_exact_source_replay_does_not_dirty_an_inflight_generation(backend): + async with durable_harness(backend, notifications=True) as h: + case = await durable_case(h.store) + lease = await case.claim() + await h.store.put_configuration(case.config) + await h.store.commit_revision(case.revision, case.rows) + await h.store.set_revision_readable( + account_id="acct_a", + reporting_revision_id=case.revision.reporting_revision_id, + readable=True, + ) + prepared, evidence = await case.verified(lease) + materializer_operation_1 = await h.store.finish_materialization( + lease, prepared=prepared, verified=evidence + ) + assert (materializer_operation_1).state == "verified" + + +async def test_selected_official_never_uses_snapshot_artifact_and_preserves_both_histories(backend): + async with durable_harness(backend, notifications=True) as h: + case = await durable_case(h.store) + first = await case.claim() + prepared, evidence = await case.verified(first) + await h.store.finish_materialization(first, prepared=prepared, verified=evidence) + original = await case.outcomes() + official = await case.publish() + await h.store.set_revision_readable( + account_id="acct_a", + reporting_revision_id=official.reporting_revision_id, + readable=False, + ) + materializer_operation_2 = await case.claim() + assert (materializer_operation_2).reason == "revision_unreadable" + assert await case.outcomes() == original + await h.store.set_revision_readable( + account_id="acct_a", reporting_revision_id=official.reporting_revision_id, readable=True + ) + second = await case.claim() + assert first.attempt.attempt == second.attempt.attempt == 1 + assert first.attempt.reporting_revision_id != second.attempt.reporting_revision_id + assert first.request.external_id != second.request.external_id + prepared, evidence = await case.verified(second) + await h.store.finish_materialization(second, prepared=prepared, verified=evidence) + assert len(await case.outcomes()) == 2 + assert (await case.outcomes())[0] == original[0] + assert case.writer.write_effects == 2 + + +@pytest.mark.parametrize("corruption", ["fork", "cycle", "multiple_official"]) +async def test_corrupt_topology_parks_without_attempt_or_hot_loop(backend, corruption, monkeypatch): + async with durable_harness(backend, notifications=True) as h: + case = await durable_case(h.store) + + def corrupt(context): + first = case.revision + second = replace( + first, + reporting_revision_id="second", + supersedes_reporting_revision_id=first.reporting_revision_id, + ) + if corruption == "fork": + revisions = (first, second, replace(second, reporting_revision_id="third")) + elif corruption == "cycle": + revisions = (replace(first, supersedes_reporting_revision_id="second"), second) + else: + official = replace( + first, + finality="official", + finality_basis="source_final", + finality_policy_id="policy", + finalized_at=first.created_at, + ) + revisions = (official, replace(official, reporting_revision_id="second")) + return replace(context, revisions=revisions) + + cls = type(h.store) + if backend == "memory": + original = cls._context + + def read(self, scope): + return corrupt(original(self, scope)) + + method = "_context" + else: + original = cls._materializer_context_on + + async def read(self, connection, scope): + return corrupt(await original(self, connection, scope)) + + method = "_materializer_context_on" + with monkeypatch.context() as patch: + patch.setattr(cls, method, read) + materializer_operation_14 = await case.claim() + assert (materializer_operation_14).reason == "history_corrupt" + materializer_operation_15 = await case.claim() + assert (materializer_operation_15).state == "idle" + assert not await h.works() and await h.queue() == ((), ()) + + +async def test_inactive_late_binding_waits_for_activation_without_allocating_history(backend): + async with durable_harness(backend) as h: + case = await durable_case(h.store, active=False, binding=False) + await h.store.put_destination_binding(case.binding) + materializer_operation_3 = await case.claim() + assert (materializer_operation_3).reason == "inactive" + assert not await h.works() + await h.store.put_configuration(replace(case.config, deactivated_at=None)) + materializer_operation_4 = await case.claim() + assert (materializer_operation_4).attempt.attempt == 1 + + +async def test_private_consumer_boundaries_keep_a_durable_account_order(backend): + from adcp.reporting.ledger.notification_models import ReportingNotificationError + + async with durable_harness(backend, notifications=True) as h: + first = await durable_case(h.store) + second = await durable_case(h.store, consumer="https://buyer.example.test/second") + for case in (first, second): + lease = await case.claim() + # Account-fair discovery chooses the canonical consumer order. + target = first if lease.scope == first.scope else second + prepared, evidence = await target.verified(lease) + await h.store.finish_materialization(lease, prepared=prepared, verified=evidence) + boundaries = [ + (await h.store.read_materializer_boundaries(caller=case.scope.principal))[0] + for case in (first, second) + ] + assert {b.sequence for b in boundaries} == {1} + assert {b.account_sequence for b in boundaries} == {1, 2} + for boundary in boundaries: + assert boundary.core.consumer_ids == (boundary.caller.consumer_id,) + assert all( + getattr(r, "consumer_id", getattr(getattr(r, "scope", None), "consumer_id", None)) + == boundary.caller.consumer_id + for r in boundary.reconciliation + ) + with pytest.raises(ReportingNotificationError): + replace(boundary, core=replace(boundary.core, consumer_ids=("another-consumer",))) + with pytest.raises(ReportingNotificationError): + replace(boundary, reporting_materialization_id="missing-outcome") + with pytest.raises(ReportingNotificationError): + replace(boundary, reconciliation=()) + assert len((await h.queue())[0]) == 2 + + +async def test_rejected_consumer_receipt_does_not_dirty_materializer_or_allocate_retry(backend): + async with durable_harness(backend, notifications=True) as h: + case = await durable_case(h.store, reconciliation_mode="consumer_receipt") + materializer_operation_5 = await case.service().run_once() + assert (materializer_operation_5).state == "verified" + outcome = (await case.outcomes())[0] + before = await h.works() + await h.store.record_revision_receipt( + ReportingRevisionReceiptRecord( + case.scope, + "receipt-rejected-0001", + case.revision.reporting_revision_id, + outcome.reporting_materialization_id, + "rejected", + case.binding.verification_profile, + 0, + (), + outcome.completed_at, + rejection_codes=("ROW_COUNT_MISMATCH",), + ) + ) + materializer_operation_6 = await case.claim() + assert (materializer_operation_6).state in {"idle", "discovered"} + assert await h.works() == before + + +async def test_tampered_lease_cannot_cross_principal_or_generation(backend): + async with durable_harness(backend) as h: + case = await durable_case(h.store) + lease = await case.claim() + for changes in ( + {"scope": replace(lease.scope, consumer_id="another-consumer")}, + {"attempt": replace(lease.attempt, attempt=2)}, + {"token": "invalid"}, + ): + with pytest.raises(ReportingWriterError): + replace(lease, **changes) + assert not await case.outcomes() + + +async def test_publication_does_not_bypass_legacy_pending_on_an_older_revision(backend): + async with durable_harness(backend, notifications=True) as h: + case = await durable_case(h.store) + await h.store.bind_obligation_delivery( + ReportingObligationDeliveryRecord( + case.scope, + "USD", + case.revision.created_at + timedelta(days=400), + case.revision.created_at, + ) + ) + legacy = ReportingMaterializationAttempt( + case.scope, case.revision.reporting_revision_id, "legacy", 1, case.revision.created_at + ) + await h.store.commit_materialization_attempt(legacy) + official = await case.publish() + materializer_operation_7 = await case.claim() + assert (materializer_operation_7).reason == "legacy_pending" + assert not await h.works() + identity = ReportingDestinationRequest.from_binding( + case.binding, legacy, case.verifier.key + ).external_id + await h.store.import_pending_materialization( + scope=case.scope, + reporting_materialization_id="legacy", + original_external_id=identity, + keys=case.keys, + ) + # Exact recovery concludes the obsolete attempt safely before the + # current official gets its own first attempt; no old artifact is reused. + materializer_operation_8 = await case.service().run_once() + assert (materializer_operation_8).state == "failed" + assert case.writer.write_effects == 0 + current = await case.claim() + assert current.attempt.reporting_revision_id == official.reporting_revision_id + assert current.attempt.attempt == 1 + assert len(await h.works()) == 2 and await h.queue() == ((), ()) + + +async def test_external_terminal_write_parks_owned_pending_history_without_a_hot_loop(backend): + async with durable_harness(backend, notifications=True) as h: + case = await durable_case(h.store) + lease = await case.claim() + await h.store.commit_materialization( + ReportingMaterializationRecord( + case.scope, + lease.attempt.reporting_revision_id, + lease.attempt.reporting_materialization_id, + "failed", + lease.attempt.created_at, + failure_code="WRITE_FAILED", + ) + ) + await h.expire() + materializer_operation_9 = await case.claim() + assert (materializer_operation_9).reason == "history_corrupt" + materializer_operation_10 = await case.claim() + assert (materializer_operation_10).state == "idle" + assert len(await h.works()) == 1 + assert not await h.store.read_materializer_boundaries(caller=case.scope.principal) + assert await h.queue() == ((), ()) + + +async def test_public_terminal_outcome_keeps_projection_dirty_work_without_readiness(backend): + """Suppressing the fenced readiness event must not drop ordinary projection work. + + Adopters keep persisting materialization outcomes directly while the durable + service runs. Before this regression the materializer store silently stopped + marking the status scope dirty for those writes, so the projector never saw + the outcome and `get_reporting_status` stayed stale indefinitely. + """ + from adcp.reporting.ledger._delivery_state import change_id + from adcp.reporting.ledger.notification_models import ReportingStatusEvidence + + async with durable_harness(backend, notifications=True) as h: + case = await durable_case(h.store) + lease = await case.claim() + outcome = ReportingMaterializationRecord( + case.scope, + lease.attempt.reporting_revision_id, + lease.attempt.reporting_materialization_id, + "failed", + lease.attempt.created_at, + failure_code="WRITE_FAILED", + ) + before = await h.dirty() + stored, created = await h.store.commit_materialization(outcome) + assert created + added = (await h.dirty())[len(before) :] + assert [reason for reason, _ in added] == ["materialization"] + assert added[0][1] == ReportingStatusEvidence(stored.kind, change_id(stored)) + # The readiness intent still belongs only to a fenced verified finish. + assert await h.queue() == ((), ()) + assert "reporting.delivery_ready" not in await h.ordinary_events() + + +async def test_reserved_attempt_keeps_the_finish_transaction_as_the_only_dirty_work(backend): + """Reservation stays short: the terminal finish owns the projection work.""" + async with durable_harness(backend, notifications=True) as h: + case = await durable_case(h.store) + before = await h.dirty() + lease = await case.claim() + assert (await h.dirty())[len(before) :] == () + prepared, verified = await case.verified(lease) + materializer_operation_11 = await h.store.finish_materialization( + lease, prepared=prepared, verified=verified + ) + assert (materializer_operation_11).state == "verified" + assert [reason for reason, _ in (await h.dirty())[len(before) :]] == ["materialization"] + + +async def test_restored_exact_component_can_explicitly_resume_parked_identity(backend): + async with durable_harness(backend, notifications=True) as h: + case = await durable_case(h.store) + lease = await case.claim() + await h.expire() + materializer_operation_12 = await h.store.claim_materialization(keys=()) + assert (materializer_operation_12).reason == "component_unavailable" + materializer_operation_13 = await case.claim() + assert (materializer_operation_13).state == "idle" + await h.store.import_pending_materialization( + scope=case.scope, + reporting_materialization_id=lease.attempt.reporting_materialization_id, + original_external_id=lease.request.external_id, + keys=case.keys, + ) + resumed = await case.claim() + assert resumed.attempt == lease.attempt and resumed.request == lease.request + assert resumed.token != lease.token diff --git a/tests/conformance/reporting/test_reporting_materializer_installed_pg.py b/tests/conformance/reporting/test_reporting_materializer_installed_pg.py new file mode 100644 index 000000000..86e4d7e49 --- /dev/null +++ b/tests/conformance/reporting/test_reporting_materializer_installed_pg.py @@ -0,0 +1,115 @@ +"""Non-editable VCS/sdist SQL installation and killed/resumed PostgreSQL workers.""" + +import asyncio +import hashlib +import json +import os +import shutil +import sys +from pathlib import Path + +import pytest + +from adcp.reporting.materializer import PgReportingMaterializerStore + +from ._durable_materializer_support import DurableHarness, durable_case +from ._generation_support import isolated_reporting_pool, require_rolling_database +from .test_reporting_materializer_packaging import ROOT, b1_wheels, built_distribution, run_step +from .test_reporting_materializer_process import worker + +# Expose the exact existing fixtures without rebuilding a current module as an +# alleged historical artifact. These are the new head's two distribution paths. +__all__ = ["b1_wheels", "built_distribution"] + + +@pytest.fixture(scope="module", params=["vcs", "sdist"]) +def installed_materializer(request): + require_rolling_database() + b1_wheels = request.getfixturevalue("b1_wheels") + path, wheels, _ = b1_wheels + interpreter = os.environ.get("ADCP_PYTHON310") or sys.executable + environment = path / f"materializer-pg-{request.param}" + run_step( + [interpreter, "-m", "venv", str(environment)], + label=f"materializer-{request.param}-pg-environment", + cwd=path, + ) + python = environment / "bin/python" + installer = ( + [shutil.which("uv"), "pip", "install", "--python", str(python)] + if shutil.which("uv") + else [str(python), "-m", "pip", "install"] + ) + run_step( + [*installer, f"{wheels[request.param]}[pg]"], + label=f"materializer-{request.param}-pg-install", + cwd=path, + timeout=180, + ) + script = path / f"materializer_pg_{request.param}.py" + shutil.copy2(Path(__file__).with_name("_materializer_process.py"), script) + version = json.loads( + run_step( + [str(python), "-I", "-c", "import json,sys;print(json.dumps(sys.version_info[:2]))"], + label="materializer-installed-python-version", + cwd=path, + ) + ) + if os.environ.get("ADCP_PYTHON310"): + assert version == [3, 10] + modules = {} + for name in ("materializer/pg.py", "materializer/service.py", "materializer/verification.py"): + modules["adcp.reporting." + name.removesuffix(".py").replace("/", ".")] = hashlib.sha256( + (ROOT / "src/adcp/reporting" / name).read_bytes() + ).hexdigest() + return path, python, script, {"workspace": str(ROOT), "python": version, "modules": modules} + + +@pytest.mark.parametrize("notifications", [False, True]) +async def test_installed_sql_and_process_restart_preserve_original_effect( + installed_materializer, notifications, tmp_path +): + path, python, script, installed = installed_materializer + async with isolated_reporting_pool(autocommit=True) as pool: + result = json.loads( + await asyncio.to_thread( + run_step, + [str(python), "-I", str(script)], + label="materializer-installed-sql-readiness", + cwd=path, + value={ + "conninfo": pool.conninfo, + "kwargs": pool.kwargs, + "notifications": notifications, + "action": "install", + "installed": installed, + }, + timeout=60, + ) + ) + assert result["point"] == "done" and result["state"] == "installed" + h = DurableHarness( + PgReportingMaterializerStore(pool=pool, notifications=notifications), None, pool + ) + case = await durable_case(h.store, count=501) + options = dict( + python=python, script=script, installed=installed, notifications=notifications + ) + async with worker(h, case, tmp_path, pause="after_write", **options) as child: + await child.event("after_write") + before = await h.works() + await child.kill() + assert not await case.outcomes() + assert await h.queue() == ((), ()) + await h.expire() + async with worker(h, case, tmp_path, **options) as child: + done = await child.event("done") + assert done["state"] == "verified" and done["origins"] == result["origins"] + materializer_operation_1 = await asyncio.wait_for(child.process.wait(), 5) + assert materializer_operation_1 == 0 + after = await h.works() + assert len(after) == 1 and after[0][0] == before[0][0] and after[0][1] == "acked" + assert len(tuple(tmp_path.glob("rwm_*"))) == 1 + assert len(await h.store.read_materializer_boundaries(caller=case.scope.principal)) == 1 + assert len((await h.queue())[0]) == int(notifications) + print(json.dumps({"installed": installed, "origins": done["origins"]}), flush=True) diff --git a/tests/conformance/reporting/test_reporting_materializer_migration.py b/tests/conformance/reporting/test_reporting_materializer_migration.py new file mode 100644 index 000000000..10deb5a80 --- /dev/null +++ b/tests/conformance/reporting/test_reporting_materializer_migration.py @@ -0,0 +1,296 @@ +"""Isolated manifests, atomic migration, bounded discovery and database fences.""" + +import asyncio +import json +from dataclasses import replace +from importlib.resources import files + +import pytest + +from adcp.reporting.ledger import LedgerConflictError, PgReportingReconciliationStore, derive_period +from adcp.reporting.materializer import PgReportingMaterializerStore +from adcp.reporting.outbox._schema import REQUIRED_OBJECTS, schema_objects, validate_schema +from adcp.reporting.outbox.status_schema import REQUIRED_STATUS_OBJECTS + +from ._durable_materializer_support import durable_case, durable_harness +from ._generation_support import isolated_reporting_pool, obligation_for + +SQL = files("adcp.reporting.ledger").joinpath("reporting_materializer.sql").read_text() +MANIFEST = json.loads( + files("adcp.reporting.materializer").joinpath("required_schema.json").read_text() +) + + +@pytest.mark.parametrize("autocommit", [False, True]) +async def test_populated_repeated_and_concurrent_install_preserves_all_old_objects_and_rows( + autocommit, +): + async with isolated_reporting_pool(autocommit=autocommit) as pool: + old = PgReportingReconciliationStore(pool=pool) + await old.create_schema() + case = await durable_case(old) + async with pool.connection() as c: + original = await schema_objects(c) + physical = await ( + await c.execute( + "SELECT tableoid::regclass::text,ctid::text,xmin::text,to_jsonb(r)" + " FROM reporting_reconciliation_records r ORDER BY record_id" + ) + ).fetchall() + new = PgReportingMaterializerStore(pool=pool) + with pytest.raises(LedgerConflictError, match="materializer schema"): + await new.materializer_ready() + await asyncio.gather(*(new.create_schema() for _ in range(3))) + for _ in range(2): + await new.create_schema() + assert await new.materializer_ready() + async with pool.connection() as c: + actual = await schema_objects(c) + # The populated rc.6 case also installs its exact waiver-binding objects. + waiver_objects = { + key: value + for key, value in REQUIRED_STATUS_OBJECTS.items() + if "reporting_issue_waiver_bindings" in key + } + assert len(waiver_objects) == 10 + assert ( + {key: actual[key] for key in original} + == original + == { + **REQUIRED_OBJECTS, + **waiver_objects, + } + ) + assert { + key: value for key, value in actual.items() if "reporting_materializer_" in key + } == MANIFEST + assert ( + await ( + await c.execute( + "SELECT tableoid::regclass::text,ctid::text,xmin::text,to_jsonb(r)" + " FROM reporting_reconciliation_records r ORDER BY record_id" + ) + ).fetchall() + == physical + ) + await validate_schema(c, activity=True) + case.store = new + materializer_operation_1 = await case.service().run_once() + assert (materializer_operation_1).state == "verified" + # Discovery is persisted, and reinstall cannot invalidate leased generations. + before = await new.read_reconciliation_snapshot(caller=case.scope.principal) + await new.create_schema() + assert ( + await new.read_reconciliation_snapshot(caller=case.scope.principal) + ).records == before.records + + +async def test_interrupted_migration_is_invisible_and_restart_converges(): + async with isolated_reporting_pool(autocommit=True) as pool: + await PgReportingReconciliationStore(pool=pool).create_schema() + entered, release = asyncio.Event(), asyncio.Event() + + async def install(): + async with pool.connection() as c, c.transaction(): + await c.execute(SQL) + entered.set() + await release.wait() + + task = asyncio.create_task(install()) + await asyncio.wait_for(entered.wait(), 10) + async with pool.connection() as c: + assert ( + await ( + await c.execute("SELECT to_regclass('reporting_materializer_work')") + ).fetchone() + )[0] is None + task.cancel() + with pytest.raises(asyncio.CancelledError): + await task + store = PgReportingMaterializerStore(pool=pool) + with pytest.raises(LedgerConflictError): + await store.materializer_ready() + await store.create_schema() + assert await store.materializer_ready() + + +@pytest.mark.parametrize( + "damage", + [ + "DROP INDEX reporting_materializer_work_due", + "ALTER TABLE reporting_materializer_work DISABLE TRIGGER reporting_materializer_guard", + "ALTER TABLE reporting_materializer_work ALTER COLUMN generation DROP NOT NULL", + ( + "ALTER TABLE reporting_materializer_notification_expansions" + " ALTER COLUMN state SET DEFAULT 'pending'" + ), + "DROP TABLE reporting_materializer_status_boundaries CASCADE", + ( + "ALTER TABLE reporting_reconciliation_records" + " DISABLE TRIGGER reporting_reconciliation_guard" + ), + ], +) +@pytest.mark.parametrize("notifications", [False, True]) +async def test_partial_mismatched_and_disabled_guard_schemas_fail_closed(damage, notifications): + async with durable_harness("postgres", notifications=notifications) as h: + case = await durable_case(h.store) + async with h.pool.connection() as c: + await c.execute(damage) + with pytest.raises(LedgerConflictError): + await case.claim() + assert not await h.works() + + +async def test_backfill_is_bounded_indexed_restartable_and_fair_between_accounts(): + async with isolated_reporting_pool(autocommit=True) as pool: + old = PgReportingReconciliationStore(pool=pool) + await old.create_schema() + cases = [await durable_case(old, account=name) for name in ("acct_a", "acct_b")] + first = cases[0] + for ordinal in range(1, 70): + period = derive_period(first.config.schedule, account_timezone="UTC", ordinal=ordinal) + await old.commit_obligation( + replace( + obligation_for(first.config), + reporting_obligation_id=f"pending-{ordinal:03d}", + period=period, + scope_resolved_at=period.end, + automated_recovery_deadline_at=period.expected_at + + first.config.automated_recovery_window, + ) + ) + store = PgReportingMaterializerStore(pool=pool) + await store.create_schema() + claimed = [] + for _ in range(4): + result = await store.claim_materialization(keys=first.keys) + if hasattr(result, "attempt"): + claimed.append(result.scope.principal.account_id) + assert "acct_b" in claimed # A's large backlog cannot starve B. + for _ in range(80): + store = PgReportingMaterializerStore(pool=pool) # Restart each turn. + result = await store.claim_materialization(keys=first.keys) + if getattr(result, "state", None) == "idle": + break + async with pool.connection() as c: + assert ( + await ( + await c.execute("SELECT count(*) FROM reporting_materializer_candidates") + ).fetchone() + )[0] == 71 + assert ( + await ( + await c.execute( + "SELECT bool_and(complete) FROM reporting_materializer_discovery" + ) + ).fetchone() + )[0] + await c.execute("SET enable_seqscan=off") + (now,) = await (await c.execute("SELECT clock_timestamp()")).fetchone() + plans = [] + for query, params in ( + ( + "SELECT account_id,consumer_id,delivery_config_id,delivery_config_version" + " FROM reporting_reconciliation_records" + " WHERE record_kind='destination_binding'", + (), + ), + ( + "SELECT account_id FROM reporting_materializer_accounts WHERE due_at<=%s" + " ORDER BY served_at,account_id LIMIT 16", + (now,), + ), + ( + "SELECT reporting_obligation_id FROM reporting_obligations WHERE account_id=%s" + " AND delivery_config_id=%s AND delivery_config_version=%s" + " AND reporting_obligation_id>%s ORDER BY reporting_obligation_id LIMIT 32", + ("acct_a", "daily", 1, ""), + ), + ( + "SELECT reporting_materialization_id FROM reporting_materializer_work" + " WHERE account_id=%s AND state='pending' AND due_at<=%s" + " ORDER BY due_at LIMIT 1", + ("acct_a", now), + ), + ): + plan = await (await c.execute("EXPLAIN (FORMAT JSON) " + query, params)).fetchone() + plans.append(json.dumps(plan)) + assert all("Seq Scan" not in plan and "Index" in plan for plan in plans) + + +async def test_enabled_work_cannot_be_resumed_by_notifications_disabled_store(): + async with durable_harness("postgres", notifications=True) as h: + case = await durable_case(h.store) + lease = await case.claim() + await h.expire() + disabled = PgReportingMaterializerStore(pool=h.pool, notifications=False) + result = await disabled.claim_materialization(keys=case.keys) + assert result.reason == "component_unavailable" + assert (await h.works())[0][0] == lease.request.external_id + assert (await h.works())[0][1] == "pending" + + +async def test_bounded_sampling_walks_past_a_full_page_of_busy_account_locks(): + async with durable_harness("postgres") as h: + busy = [f"busy-{i:02d}" for i in range(16)] + for account in busy: + await durable_case(h.store, account=account) + available = await durable_case(h.store, account="zz-available") + async with h.pool.connection() as connection, connection.transaction(): + for account in busy: + await h.store._lock_account(connection, account) + materializer_operation_3 = await available.claim() + assert (materializer_operation_3).state == "idle" + selected = await available.claim() + assert selected.scope.principal.account_id == "zz-available" + assert len(await h.works()) == 1 + + +async def test_pre_activation_event_never_promotes_and_old_outbox_cannot_claim_it(): + from adcp.reporting.ledger.notification_models import decode_event + from adcp.reporting.outbox import PgReportingOutbox + + async with durable_harness("postgres", notifications=True) as h: + case = await durable_case(h.store) + materializer_operation_2 = await case.service().run_once() + assert (materializer_operation_2).state == "verified" + before = await h.queue() + # Remove only the ordinary Core event's pending expansion via its own + # worker protocol. Materializer records remain in their isolated queue. + outbox = PgReportingOutbox(pool=h.pool) + from datetime import datetime, timezone + + while lease := await outbox.claim_expansion( + account_id="acct_a", now=datetime.now(timezone.utc), lease_seconds=30 + ): + assert decode_event(lease.event).notification_type != "reporting.delivery_ready" + await outbox.finish_expansion(lease, now=datetime.now(timezone.utc), state="suppressed") + from psycopg import errors + + for mutation in ( + "UPDATE reporting_materializer_notification_events SET admission_epoch=1", + "UPDATE reporting_materializer_notification_expansions SET state='pending'", + "UPDATE reporting_materializer_work SET admission_epoch=1", + ): + with pytest.raises(errors.CheckViolation): + async with h.pool.connection() as c, c.transaction(): + await c.execute(mutation) + await case.publish() + await h.store.put_configuration( + replace(case.config, deactivated_at=case.revision.created_at) + ) + assert await h.queue() == before + + +async def test_schema_loss_after_reservation_fails_authorization_before_external_write(): + async with durable_harness("postgres", notifications=True) as h: + case = await durable_case(h.store) + lease = await case.claim() + async with h.pool.connection() as c: + await c.execute("DROP INDEX reporting_materializer_work_due") + with pytest.raises(LedgerConflictError): + await h.store.authorize_materialization(lease) + assert case.writer.write_effects == 0 + assert not await case.outcomes() + assert len(await h.works()) == 1 and (await h.works())[0][1] == "pending" diff --git a/tests/conformance/reporting/test_reporting_materializer_packaging.py b/tests/conformance/reporting/test_reporting_materializer_packaging.py index 007a279bf..36187037b 100644 --- a/tests/conformance/reporting/test_reporting_materializer_packaging.py +++ b/tests/conformance/reporting/test_reporting_materializer_packaging.py @@ -45,15 +45,14 @@ def b1_wheels(built_distribution): for relative in ( "ledger/reporting_status_selector_version.sql", "outbox/required_status_selector_schema.json", + "ledger/reporting_materializer.sql", + "materializer/required_schema.json", ): assert ( vcs.read(f"adcp/reporting/{relative}") == wheel.read(f"adcp/reporting/{relative}") == (ROOT / "src/adcp/reporting" / relative).read_bytes() ) - assert not any( - "reporting_materializer_" in name and name.endswith(".sql") for name in vcs.namelist() - ) return ( path, {"vcs": vcs_wheel, "sdist": sdist_wheel}, @@ -95,6 +94,10 @@ def test_python310_installed_wheel_exports_verifier_reference_and_strict_adopter shutil.copy2(Path(__file__).with_name("_materializer_installed.py"), smoke) shutil.copy2(ROOT / "examples/reporting_destination_writer.py", example) shutil.copy2(ROOT / "tests/type_checks/reporting_destination_writer.py", adopter) + durable_example = path / f"durable_example_{kind}.py" + durable_adopter = path / f"durable_adopter_{kind}.py" + shutil.copy2(ROOT / "examples/reporting_durable_materializer.py", durable_example) + shutil.copy2(ROOT / "tests/type_checks/reporting_durable_materializer.py", durable_adopter) result = json.loads( run_step( [str(python), "-I", str(smoke)], @@ -104,7 +107,13 @@ def test_python310_installed_wheel_exports_verifier_reference_and_strict_adopter timeout=120, ) ) - assert result == {"python": "3.10", "rows": [0, 501], "installed": True, "assets": hashes} + assert result == { + "python": "3.10", + "rows": [0, 501], + "installed": True, + "assets": hashes, + "durable": True, + } config = path / "mypy.ini" config.write_text( "[mypy]\npython_version = 3.10\nstrict = True\n" @@ -122,6 +131,8 @@ def test_python310_installed_wheel_exports_verifier_reference_and_strict_adopter "--no-incremental", str(adopter), str(example), + str(durable_adopter), + str(durable_example), ], label=f"b1-{kind}-installed-adopter-types", cwd=path, diff --git a/tests/conformance/reporting/test_reporting_materializer_process.py b/tests/conformance/reporting/test_reporting_materializer_process.py new file mode 100644 index 000000000..ceb451f94 --- /dev/null +++ b/tests/conformance/reporting/test_reporting_materializer_process.py @@ -0,0 +1,158 @@ +"""SIGKILL across external-effect and outcome commit boundaries; durable restart.""" + +import asyncio +import json +import sys +from contextlib import asynccontextmanager +from pathlib import Path + +import pytest + +from adcp.reporting.ledger._delivery_state import payload + +from ._durable_materializer_support import durable_case, durable_harness + + +class Child: + def __init__(self, process): + self.process = process + + async def event(self, point): + line = await asyncio.wait_for(self.process.stdout.readline(), 30) + assert line, "materializer worker exited before its boundary" + result = json.loads(line) + assert result["point"] == point, result + return result + + async def send(self, value): + self.process.stdin.write(json.dumps(value).encode() + b"\n") + await self.process.stdin.drain() + + async def kill(self): + if self.process.returncode is None: + self.process.kill() + await asyncio.wait_for(self.process.wait(), 5) + + +async def visible(h): + """MVCC observation while the child deliberately holds the account lock.""" + async with h.pool.connection() as connection: + return await ( + await connection.execute( + "SELECT (SELECT count(*) FROM reporting_reconciliation_records" + " WHERE record_kind='materialization')," + " (SELECT count(*) FROM reporting_materializer_status_boundaries)" + ) + ).fetchone() + + +@asynccontextmanager +async def worker( + h, case, directory, *, pause=None, notifications=True, installed=None, python=None, script=None +): + process = await asyncio.create_subprocess_exec( + str(python or sys.executable), + *(["-I"] if installed else []), + str(script or Path(__file__).with_name("_materializer_process.py")), + stdin=asyncio.subprocess.PIPE, + stdout=asyncio.subprocess.PIPE, + stderr=asyncio.subprocess.DEVNULL, + ) + child = Child(process) + try: + await child.send( + { + "conninfo": h.pool.conninfo, + "kwargs": h.pool.kwargs, + "binding": payload(case.binding), + "destination": str(directory), + "pause": pause, + "notifications": notifications, + "installed": installed, + } + ) + yield child + finally: + await child.kill() + + +@pytest.mark.parametrize("notifications", [False, True]) +@pytest.mark.parametrize( + "boundary", + [ + "reserved", + "before_write", + "after_write", + "after_readback", + "after_outcome", + "after_capture", + "after_commit", + ], +) +async def test_real_process_crash_resume_preserves_external_identity_and_atomic_finish( + boundary, notifications, tmp_path +): + async with durable_harness("postgres", notifications=notifications) as h: + case = await durable_case(h.store, count=501) + async with worker(h, case, tmp_path, pause=boundary, notifications=notifications) as child: + await child.event(boundary) + work_before = await h.works() + assert len(work_before) == 1 + committed = boundary == "after_commit" + assert await visible(h) == (int(committed), int(committed)) + assert len((await h.queue())[0]) == int(committed and notifications) + await child.kill() + await h.expire() + async with worker(h, case, tmp_path, notifications=notifications) as child: + result = await child.event("done") + assert result["state"] in ( + {"idle", "parked", "discovered"} if committed else {"verified"} + ) + materializer_operation_1 = await asyncio.wait_for(child.process.wait(), 5) + assert materializer_operation_1 == 0 + after = await h.works() + assert len(after) == 1 and after[0][0] == work_before[0][0] and after[0][1] == "acked" + assert len(tuple(tmp_path.glob("rwm_*"))) == 1 + assert len(await case.outcomes()) == 1 + assert len(await h.store.read_materializer_boundaries(caller=case.scope.principal)) == 1 + events, expansions = await h.queue() + assert len(events) == int(notifications) + assert expansions == (("quarantined",) if notifications else ()) + + +async def test_crash_after_actual_enabled_logical_enqueue_rolls_back_before_restart(tmp_path): + async with durable_harness("postgres", notifications=True) as h: + case = await durable_case(h.store) + async with worker(h, case, tmp_path, pause="after_event") as child: + await child.event("after_event") + assert await visible(h) == (0, 0) + assert await h.queue() == ((), ()) + await child.kill() + assert not await h.store.read_materializer_boundaries(caller=case.scope.principal) + await h.expire() + async with worker(h, case, tmp_path) as child: + materializer_operation_2 = await child.event("done") + assert (materializer_operation_2)["state"] == "verified" + materializer_operation_3 = await asyncio.wait_for(child.process.wait(), 5) + assert materializer_operation_3 == 0 + assert len(tuple(tmp_path.glob("rwm_*"))) == 1 + assert len((await h.queue())[0]) == 1 + + +async def test_two_real_workers_reserve_once_without_global_candidate_locks(tmp_path): + async with durable_harness("postgres", notifications=True) as h: + case = await durable_case(h.store) + async with worker(h, case, tmp_path, pause="reserved") as first: + await first.event("reserved") + async with worker(h, case, tmp_path) as second: + materializer_operation_6 = await second.event("done") + assert (materializer_operation_6)["state"] in {"idle", "discovered"} + materializer_operation_7 = await asyncio.wait_for(second.process.wait(), 5) + assert materializer_operation_7 == 0 + assert not tuple(tmp_path.glob("rwm_*")) + await first.send({"continue": True}) + materializer_operation_4 = await first.event("done") + assert (materializer_operation_4)["state"] == "verified" + materializer_operation_5 = await asyncio.wait_for(first.process.wait(), 5) + assert materializer_operation_5 == 0 + assert len(await h.works()) == 1 diff --git a/tests/conformance/reporting/test_reporting_materializer_rolling.py b/tests/conformance/reporting/test_reporting_materializer_rolling.py new file mode 100644 index 000000000..46eaf7956 --- /dev/null +++ b/tests/conformance/reporting/test_reporting_materializer_rolling.py @@ -0,0 +1,260 @@ +"""Actual built/installed frozen artifacts, not renamed or path-selected modules.""" + +import asyncio +import hashlib +import json +import shutil +import sys +import tarfile +from pathlib import Path + +import pytest + +from adcp.reporting.ledger import LedgerConflictError, PgReportingReconciliationStore +from adcp.reporting.materializer import PgReportingMaterializerStore +from adcp.reporting.outbox._schema import schema_objects + +from ._durable_materializer_support import DurableHarness, durable_case +from ._generation_support import isolated_reporting_pool, require_rolling_database +from .test_reporting_notification_packaging import ROOT, run_step + +ARTIFACTS = { + "beta15": "3e76aa54623529a3dda01cd690b8a5c287c75641", + "records": "3c405a21f978ed9d3208611bb4a7a8434a056933", + "integration": "037de4ac822ecefb2f95d32c15c297fb4c45d683", + "a": "17ee407ae3978c8a2bb54437287afbf9dafb8130", + "b": "0f34c666ac1961e9832fce43ef0ef6937b3c1dde", + "c": "967b6e286301d7e5d089aea6fdbb90bea8ee5a16", + "b1": "5487f2bdef23c5102118b305be9e868228f6ce61", +} + + +def build_frozen(artifact, tmp_path_factory, request): + require_rolling_database() + sha = ARTIFACTS[artifact] + root = tmp_path_factory.mktemp(f"materializer-{artifact}") + request.addfinalizer(lambda: shutil.rmtree(root)) + archive, source, dist, environment = ( + root / n for n in ("source.tar.gz", "source", "dist", "installed") + ) + source.mkdir() + run_step( + ["git", "archive", "--format=tar.gz", "-1", f"--output={archive}", sha], + label=f"{artifact}-exact-git-archive", + cwd=ROOT, + ) + with tarfile.open(archive) as tar: + tar.extractall(source, filter="data") + archive.unlink() + run_step( + [sys.executable, "-m", "build", "--wheel", "--outdir", str(dist), str(source)], + label=f"{artifact}-build-frozen-wheel", + cwd=root, + timeout=180, + ) + wheel = next(dist.glob("*.whl")) + run_step( + [sys.executable, "-m", "venv", str(environment)], + label=f"{artifact}-isolated-environment", + cwd=root, + ) + python = environment / "bin/python" + installer = ( + [shutil.which("uv"), "pip", "install", "--python", str(python)] + if shutil.which("uv") + else [str(python), "-m", "pip", "install"] + ) + run_step( + [*installer, f"{wheel}[pg]"], + label=f"{artifact}-install-frozen-wheel", + cwd=root, + timeout=180, + ) + script = root / "frozen.py" + shutil.copy2(Path(__file__).with_name("_materializer_frozen.py"), script) + modules = {} + for relative in ( + "ledger/pg.py", + "ledger/delivery_pg.py", + "outbox/worker.py", + "outbox/status_pg.py", + "materializer/contracts.py", + "materializer/verification.py", + ): + path = source / "src/adcp/reporting" / relative + if path.exists(): + modules["adcp.reporting." + relative.removesuffix(".py").replace("/", ".")] = ( + hashlib.sha256(path.read_bytes()).hexdigest() + ) + settings = { + "artifact": artifact, + "sha": sha, + "modules": modules, + "workspace": str(ROOT), + "wheel_sha256": hashlib.sha256(wheel.read_bytes()).hexdigest(), + } + # The exact source hashes and wheel survive in the evidence log; the build + # tree contains 2.2 GiB of cached schemas and is not an execution dependency. + print(json.dumps({"frozen_build": settings}), flush=True) + shutil.rmtree(source) + return ( + root, + python, + script, + settings, + ) + + +@pytest.fixture(scope="module") +def installed_parent(tmp_path_factory, request): + return build_frozen("b1", tmp_path_factory, request) + + +@pytest.fixture(scope="module", params=tuple(ARTIFACTS)) +def installed_frozen(request, tmp_path_factory, installed_parent): + if request.param == "b1": + return installed_parent + return build_frozen(request.param, tmp_path_factory, request) + + +async def frozen_call(artifact, pool, action, **kwargs): + root, python, script, settings = artifact + settings = { + **settings, + "conninfo": pool.conninfo, + "kwargs": pool.kwargs, + "action": action, + **kwargs, + } + raw = await asyncio.to_thread( + run_step, + [str(python), "-I", str(script)], + label=f"{settings['artifact']}-{action}", + cwd=root, + value=settings, + timeout=60, + ) + result = json.loads(raw) + assert "failure" not in result, result + return result + + +async def test_installed_old_reader_writer_and_workers_on_populated_materializer_schema( + installed_frozen, + installed_parent, +): + async with isolated_reporting_pool(autocommit=True) as pool: + evidence = await frozen_call(installed_frozen, pool, "install") + assert evidence["installed"] + # The approved B1 wheel installs the comparison baseline, including + # reviewed C's additive capture and B1's unactivated selector fence. + parent = await frozen_call(installed_parent, pool, "install") + assert parent["sha"] == ARTIFACTS["b1"] + baseline_store = PgReportingReconciliationStore(pool=pool, notifications=True) + # beta.15 can read its original definition shape. Unit declarations are + # an A prerequisite, not a B2 schema or decoder change. New Managed facts + # stay in A's separate feed and never enter beta.15's closed Core feed. + before_url_principals = evidence["artifact"] in { + "beta15", + "records", + "integration", + "a", + "b", + } + case = await durable_case( + baseline_store, + count=3, + consumer=( + "frozen-buyer" if before_url_principals else "https://buyer.example.test/agent" + ), + legacy_definition=evidence["artifact"] == "beta15", + ) + # URL principals became readable in C. Earlier binaries still read their + # opaque caller's records while a separate URL consumer has populated + # materializations/captures/events in the same account and schema. + sibling = await durable_case( + baseline_store, + count=3, + consumer="https://buyer.example.test/isolated", + legacy_definition=evidence["artifact"] == "beta15", + ) + baseline = await frozen_call( + installed_frozen, pool, "baseline", consumer=case.binding.consumer_id + ) + assert baseline["ordinary_writes"] and baseline["core_records"] == 2 + async with pool.connection() as connection: + old_objects = await schema_objects(connection) + store = PgReportingMaterializerStore(pool=pool, notifications=True) + with pytest.raises(LedgerConflictError): + await store.materializer_ready() + await store.create_schema() + async with pool.connection() as connection: + objects = await schema_objects(connection) + assert {key: objects[key] for key in old_objects} == old_objects + assert all("reporting_materializer_" in key for key in objects.keys() - old_objects.keys()) + case.store = sibling.store = store + materializer_operation_1 = await case.service().run_once() + assert (materializer_operation_1).state == "verified" + materializer_operation_2 = await sibling.service().run_once() + assert (materializer_operation_2).state == "verified" + h = DurableHarness(store, None, pool) + before = await h.queue() + async with pool.connection() as connection: + immutable_before = [ + await (await connection.execute(f"SELECT * FROM {table} ORDER BY 1,2,3")).fetchall() + for table in ( + "reporting_materializer_work", + "reporting_materializer_status_boundaries", + "reporting_materializer_status_heads", + "reporting_materializer_notification_events", + "reporting_materializer_notification_expansions", + ) + ] + captured_heads = await ( + await connection.execute( + "SELECT account_id,captured_sequence FROM reporting_materializer_accounts" + " ORDER BY account_id" + ) + ).fetchall() + result = await frozen_call( + installed_frozen, pool, "exercise", consumer=case.binding.consumer_id + ) + assert result["core_records"] == 2 and result["ordinary_writes"] + assert result["managed_records"] == (0 if result["artifact"] == "beta15" else 4) + assert result["notifications_ready"] == baseline["notifications_ready"] + assert await h.queue() == before + async with pool.connection() as connection: + immutable_after = [ + await (await connection.execute(f"SELECT * FROM {table} ORDER BY 1,2,3")).fetchall() + for table in ( + "reporting_materializer_work", + "reporting_materializer_status_boundaries", + "reporting_materializer_status_heads", + "reporting_materializer_notification_events", + "reporting_materializer_notification_expansions", + ) + ] + assert ( + await ( + await connection.execute( + "SELECT account_id,captured_sequence FROM reporting_materializer_accounts" + " ORDER BY account_id" + ) + ).fetchall() + ) == captured_heads + assert immutable_after == immutable_before + assert before[1] == ("quarantined", "quarantined") + assert await store.materializer_ready() + print( + json.dumps( + { + **result, + "native": evidence, + "baseline": baseline, + "baseline_installer": parent, + "additive_objects": len(objects.keys() - old_objects.keys()), + "wheel_sha256": installed_frozen[3]["wheel_sha256"], + } + ), + flush=True, + ) diff --git a/tests/conformance/reporting/test_reporting_materializer_service.py b/tests/conformance/reporting/test_reporting_materializer_service.py new file mode 100644 index 000000000..97292ab4e --- /dev/null +++ b/tests/conformance/reporting/test_reporting_materializer_service.py @@ -0,0 +1,321 @@ +"""Long I/O, reauthorization, cleanup and uncertain-effect service convergence.""" + +import asyncio +from dataclasses import replace +from datetime import timedelta + +import pytest + +from adcp.reporting.ledger import ( + ReportingMaterializationAttempt, + ReportingMaterializationCheck, + ReportingObligationDeliveryRecord, +) +from adcp.reporting.materializer import ( + ReportingDestinationIO, + ReportingMaterializerService, + ReportingWriterError, + ReportingWriterFailure, +) + +from ._durable_materializer_support import durable_case, durable_harness +from .test_reporting_materializer_lifecycle import SECRET, Phases, safe_exception + + +@pytest.mark.parametrize("backend", ["memory", "postgres"]) +@pytest.mark.parametrize("stage", ["write-before", "write-after", "object", "readback-resolve"]) +async def test_cancellation_and_restart_never_allocate_a_new_external_identity( + backend, stage, tmp_path, caplog +): + async with durable_harness(backend, notifications=True) as h: + case = await durable_case(h.store, count=501) + owner = Phases(case, tmp_path, stage, "cancel") + service = ReportingMaterializerService( + h.store, ReportingDestinationIO(case.registry, owner), case.writer, lease_seconds=3 + ) + before = set(asyncio.all_tasks()) + task = asyncio.create_task(service.run_once()) + await asyncio.wait_for(owner.entered.wait(), 10) + initial = await h.works() + task.cancel(SECRET) + with pytest.raises(asyncio.CancelledError) as error: + await asyncio.wait_for(task, 10) + safe_exception(error.value) + assert all(s._closed and s._credential is None for s in owner.sessions) + assert not await case.outcomes() and await h.queue() == ((), ()) + assert not list(tmp_path.iterdir()) + assert not (set(asyncio.all_tasks()) - before) + await h.expire() + materializer_operation_1 = await case.service().run_once() + assert (materializer_operation_1).state == "verified" + works = await h.works() + assert len(works) == 1 and works[0][0] == initial[0][0] + assert case.writer.write_effects == 1 + assert case.writer.open_count == case.writer.close_count + assert SECRET not in caplog.text + + +@pytest.mark.parametrize("stage", ["write-after", "object"]) +async def test_service_heartbeat_keeps_long_io_alive_with_size_one_postgres_pool( + stage, tmp_path, monkeypatch +): + from adcp.reporting.materializer import PgReportingMaterializerStore + + async with durable_harness("postgres", notifications=True) as h: + from psycopg_pool import AsyncConnectionPool + + case = await durable_case(h.store, count=501) + async with AsyncConnectionPool( + h.pool.conninfo, kwargs=h.pool.kwargs, min_size=1, max_size=1, open=False + ) as single: + store = PgReportingMaterializerStore(pool=single, notifications=True) + case.store = store + owner = Phases(case, tmp_path, stage, "wait") + renewed, turns = asyncio.Event(), [] + original = store.renew_materialization + + async def renew(*args, **kwargs): + held = await original(*args, **kwargs) + turns.append(held) + if len(turns) == 4: + renewed.set() + return held + + monkeypatch.setattr(store, "renew_materialization", renew) + service = ReportingMaterializerService( + store, ReportingDestinationIO(case.registry, owner), case.writer, lease_seconds=3 + ) + task = asyncio.create_task(service.run_once()) + await asyncio.wait_for(owner.entered.wait(), 10) + await asyncio.wait_for(renewed.wait(), 10) # Four DB-time heartbeats > one whole lease. + assert all(turns) and len(turns) >= 4 + materializer_operation_7 = await store.claim_materialization( + keys=case.keys, lease_seconds=3 + ) + assert not hasattr(materializer_operation_7, "attempt") + owner.release.set() + materializer_operation_8 = await asyncio.wait_for(task, 10) + assert (materializer_operation_8).state == "verified" + assert case.writer.write_effects == 1 + assert case.writer.open_count == case.writer.close_count == 2 + + +@pytest.mark.parametrize("backend", ["memory", "postgres"]) +@pytest.mark.parametrize("change", ["rotate", "revoke", "official", "readability"]) +async def test_write_to_readback_window_reauthorizes_exact_principal_and_current_generation( + backend, change, tmp_path +): + async with durable_harness(backend, notifications=True) as h: + case = await durable_case(h.store) + owner = Phases(case, tmp_path, "write-after", "wait") + service = ReportingMaterializerService( + h.store, ReportingDestinationIO(case.registry, owner), case.writer + ) + task = asyncio.create_task(service.run_once()) + await asyncio.wait_for(owner.entered.wait(), 10) + if change == "rotate": + case.resolver.rotate() + elif change == "revoke": + case.resolver.revoke(case.scope.principal) + elif change == "official": + await case.publish() + else: + await h.store.set_revision_readable( + account_id="acct_a", + reporting_revision_id=case.revision.reporting_revision_id, + readable=False, + ) + owner.release.set() + turn = await asyncio.wait_for(task, 10) + assert turn.state == ("verified" if change == "rotate" else "failed") + assert len((await h.queue())[0]) == int(change == "rotate") + assert case.writer.open_count == case.writer.close_count + assert all(s._credential is None for s in owner.sessions) + + +@pytest.mark.parametrize("backend", ["memory", "postgres"]) +@pytest.mark.parametrize("state", ["corrupt", "unavailable"]) +async def test_later_health_loss_never_resets_or_retries_successful_history(backend, state): + async with durable_harness(backend) as h: + case = await durable_case(h.store) + materializer_operation_2 = await case.service().run_once() + assert (materializer_operation_2).state == "verified" + outcome = (await case.outcomes())[0] + await h.store.record_materialization_check( + ReportingMaterializationCheck( + case.scope, + outcome.reporting_materialization_id, + "health", + state, + outcome.completed_at, + ) + ) + result = await case.claim() + assert result.reason == "operator_required" + materializer_operation_3 = await case.claim() + assert (materializer_operation_3).state == "idle" + assert len(await h.works()) == 1 + assert await case.outcomes() == (outcome,) + + +async def test_retention_loss_between_readback_and_finish_is_immutable_safe_failure(): + async with durable_harness("memory", notifications=True) as h: + case = await durable_case(h.store) + lease = await case.claim(lease_seconds=300) + prepared, evidence = await case.verified(lease) + h.clock.advance(timedelta(seconds=60)) + result = await h.store.finish_materialization(lease, prepared=prepared, verified=evidence) + assert result.state == "failed" and await h.queue() == ((), ()) + assert (await case.outcomes())[0].failure_code == "RESOURCE_UNAVAILABLE" + + +@pytest.mark.parametrize("backend", ["memory", "postgres"]) +@pytest.mark.parametrize("terminal", ["verified", "failed"]) +async def test_public_next_attempt_is_still_permitted_after_any_terminal_outcome(backend, terminal): + async with durable_harness(backend) as h: + case = await durable_case(h.store) + lease = await case.claim() + if terminal == "verified": + prepared, evidence = await case.verified(lease) + await h.store.finish_materialization(lease, prepared=prepared, verified=evidence) + else: + await h.store.finish_materialization( + lease, error=ReportingWriterFailure("WRITE_FAILED", "never", "not_started") + ) + outcome = (await case.outcomes())[0] + attempt = replace( + lease.attempt, + attempt=2, + reporting_materialization_id="operator-next", + created_at=outcome.completed_at, + ) + await h.store.commit_materialization_attempt(attempt) + snapshot = await h.store.read_reconciliation_snapshot(caller=case.scope.principal) + assert [ + r.attempt for r in snapshot.records if isinstance(r, ReportingMaterializationAttempt) + ] == [1, 2] + await h.expire() + materializer_operation_4 = await case.claim() + assert not hasattr(materializer_operation_4, "attempt") + assert len(await h.works()) == 1 + + +@pytest.mark.parametrize("backend", ["memory", "postgres"]) +@pytest.mark.parametrize("source", ["writer_metadata", "store_reservation"]) +async def test_service_entry_point_closes_errors_before_reservation(backend, source, monkeypatch): + async with durable_harness(backend, notifications=True) as h: + case = await durable_case(h.store) + monkeypatch.setattr(type(h.store), "__repr__", lambda self: SECRET) + assert SECRET not in repr(case.service()) + if source == "writer_metadata": + + def broken_metadata(self): + raise RuntimeError(SECRET) + + monkeypatch.setattr(type(case.writer), "capabilities", property(broken_metadata)) + else: + + async def broken_reservation(self, **kwargs): + raise RuntimeError(SECRET) + + monkeypatch.setattr(type(h.store), "claim_materialization", broken_reservation) + with pytest.raises(ReportingWriterError) as error: + await case.service().run_once() + safe_exception(error.value) + assert not await h.works() and await h.queue() == ((), ()) + assert case.writer.write_effects == 0 + + +@pytest.mark.parametrize("backend", ["memory", "postgres"]) +async def test_driver_failure_is_a_closed_error_without_provider_context(backend, monkeypatch): + async with durable_harness(backend) as h: + case = await durable_case(h.store) + if backend == "memory": + + def fail(*args, **kwargs): + raise OSError(SECRET) + + monkeypatch.setattr(type(h.store), "_context", fail) + else: + + async def fail(*args, **kwargs): + raise OSError(SECRET) + + monkeypatch.setattr(type(h.store), "_materializer_context_on", fail) + with pytest.raises(ReportingWriterError) as error: + await case.claim() + safe_exception(error.value) + + +@pytest.mark.parametrize("backend", ["memory", "postgres"]) +@pytest.mark.parametrize("cause", ["timeout", "cleanup"]) +async def test_timeout_or_cleanup_failure_after_effect_resumes_original_identity( + backend, cause, tmp_path, caplog +): + async with durable_harness(backend, notifications=True) as h: + case = await durable_case(h.store) + # This fixture's short I/O deadline must not be its artifact lifetime. + # Recovery reuses a conditional object that still meets the full + # required retention window, independently of the timed-out request. + await h.store.bind_obligation_delivery( + ReportingObligationDeliveryRecord( + case.scope, "USD", h.clock() + timedelta(days=401), h.clock() + ) + ) + owner = Phases( + case, + tmp_path, + "write-after" if cause == "timeout" else "close", + "wait" if cause == "timeout" else "error", + ) + service = ReportingMaterializerService( + h.store, + ReportingDestinationIO(case.registry, owner), + case.writer, + io_timeout_seconds=1 if cause == "timeout" else 30, + ) + materializer_operation_5 = await asyncio.wait_for(service.run_once(), 10) + assert (materializer_operation_5).state == "pending" + before = await h.works() + assert not await case.outcomes() and await h.queue() == ((), ()) + assert case.writer.write_effects == 1 + assert all(s._closed and s._credential is None for s in owner.sessions) + assert not tuple(tmp_path.iterdir()) and SECRET not in caplog.text + await h.expire() + materializer_operation_6 = await case.service().run_once() + assert (materializer_operation_6).state == "verified" + assert (await h.works())[0][0] == before[0][0] + assert case.writer.write_effects == 1 + + +@pytest.mark.parametrize("backend", ["memory", "postgres"]) +async def test_heartbeat_cancels_io_after_expiry_and_competing_worker_steals_fence( + backend, tmp_path +): + async with durable_harness(backend, notifications=True) as h: + case = await durable_case(h.store) + owner = Phases(case, tmp_path, "write-after", "wait") + service = ReportingMaterializerService( + h.store, ReportingDestinationIO(case.registry, owner), case.writer, lease_seconds=3 + ) + task = asyncio.create_task(service.run_once()) + try: + await asyncio.wait_for(owner.entered.wait(), 10) + before = await h.works() + await h.expire() + winner = await case.claim() + materializer_operation_9 = await asyncio.wait_for(task, 10) + assert (materializer_operation_9).state == "pending" + assert all(s._closed and s._credential is None for s in owner.sessions) + assert not tuple(tmp_path.iterdir()) + assert not await case.outcomes() and await h.queue() == ((), ()) + prepared, evidence = await case.verified(winner) + materializer_operation_10 = await h.store.finish_materialization( + winner, prepared=prepared, verified=evidence + ) + assert (materializer_operation_10).state == "verified" + assert (await h.works())[0][0] == before[0][0] + assert case.writer.write_effects == 1 + finally: + task.cancel() + await asyncio.gather(task, return_exceptions=True) diff --git a/tests/conformance/reporting/test_reporting_materializer_transactions.py b/tests/conformance/reporting/test_reporting_materializer_transactions.py new file mode 100644 index 000000000..9788fba5a --- /dev/null +++ b/tests/conformance/reporting/test_reporting_materializer_transactions.py @@ -0,0 +1,315 @@ +"""Faults at every reserve/finish write boundary with notifications off and on.""" + +from contextlib import contextmanager +from dataclasses import replace +from datetime import timedelta + +import pytest + +from adcp.reporting.ledger import ReportingMaterializationRecord +from adcp.reporting.materializer import ReportingWriterError +from adcp.reporting.materializer.memory import InMemoryReportingMaterializerStore + +from ._durable_materializer_support import durable_case, durable_harness + + +@pytest.mark.parametrize("notifications", [False, True]) +async def test_failed_memory_snapshot_does_not_leak_transaction_ownership(notifications): + class Uncopyable: + def __deepcopy__(self, memo): + raise ValueError("injected snapshot fault") + + async with durable_harness("memory", notifications=notifications) as h: + case = await durable_case(h.store) + changed = replace(case.config, status_retention_days=case.config.status_retention_days + 1) + h.store._injected_snapshot_fault = Uncopyable() + with pytest.raises(ValueError, match="injected snapshot fault"): + await h.store.put_configuration(changed) + del h.store._injected_snapshot_fault + before = await h.image() + with pytest.raises(RuntimeError, match="rollback subsequent transaction"): + async with h.store.transaction(): + await h.store.put_configuration(changed) + raise RuntimeError("rollback subsequent transaction") + assert await h.image() == before + + +@contextmanager +def postgres_failure(monkeypatch, prefix): + from psycopg import AsyncConnection + + original = AsyncConnection.execute + hit = [] + + async def execute(self, query, *args, **kwargs): + result = await original(self, query, *args, **kwargs) + if isinstance(query, str) and query.startswith(prefix): + hit.append(self.pgconn.backend_pid) + raise OSError("injected transaction boundary") + return result + + with monkeypatch.context() as patch: + patch.setattr(AsyncConnection, "execute", execute) + yield hit + + +@pytest.mark.parametrize("backend", ["memory", "postgres"]) +@pytest.mark.parametrize("notifications", [False, True]) +@pytest.mark.parametrize("position", ["delivery", "attempt", "work", "lease"]) +async def test_reservation_rolls_back_attempt_work_delivery_and_sequence_heads( + backend, notifications, position, monkeypatch +): + async with durable_harness(backend, notifications=notifications) as h: + case = await durable_case(h.store) + before = await h.image() + if backend == "postgres": + prefixes = { + "delivery": "INSERT INTO reporting_reconciliation_records", + "attempt": "INSERT INTO reporting_reconciliation_changes", + "work": "INSERT INTO reporting_materializer_work", + "lease": "UPDATE reporting_materializer_work SET lease_token=", + } + # Count explicitly so attempt failure happens after the delivery's + # own complete insertion, not at its earlier feed append. + if position == "attempt": + from adcp.reporting.materializer.pg import PgReportingMaterializerStore + + original = PgReportingMaterializerStore._commit_record_on + + async def fail(self, connection, record, **kwargs): + result = await original(self, connection, record, **kwargs) + if record.kind == "materialization_attempt": + raise OSError("injected transaction boundary") + return result + + with monkeypatch.context() as patch: + patch.setattr(PgReportingMaterializerStore, "_commit_record_on", fail) + with pytest.raises(ReportingWriterError): + await case.claim() + else: + with postgres_failure(monkeypatch, prefixes[position]) as hit: + with pytest.raises(ReportingWriterError): + await case.claim() + assert len(hit) == 1 + else: + cls = InMemoryReportingMaterializerStore + if position in {"delivery", "attempt"}: + original = cls._commit_record_unlocked + + def fail(self, record, **kwargs): + result = original(self, record, **kwargs) + if ( + record.kind + == { + "delivery": "obligation_delivery", + "attempt": "materialization_attempt", + }[position] + ): + raise OSError("injected transaction boundary") + return result + + method = "_commit_record_unlocked" + else: + original = cls._lease + + def fail(self, *args): + if position == "lease": + original(self, *args) + raise OSError("injected transaction boundary") + + method = "_lease" + with monkeypatch.context() as patch: + patch.setattr(cls, method, fail) + with pytest.raises(ReportingWriterError): + await case.claim() + assert await h.image() == before + lease = await case.claim() + assert lease.attempt.attempt == 1 + assert len(await h.works()) == 1 + + +@pytest.mark.parametrize("backend", ["memory", "postgres"]) +@pytest.mark.parametrize("notifications", [False, True]) +@pytest.mark.parametrize("position", ["outcome", "status_head", "account_head", "boundary", "ack"]) +async def test_finish_rolls_back_terminal_capture_ack_and_all_initialized_heads( + backend, notifications, position, monkeypatch +): + async with durable_harness(backend, notifications=notifications) as h: + case = await durable_case(h.store) + lease = await case.claim() + prepared, verified = await case.verified(lease) + before = await h.image() + + async def finish(): + return await h.store.finish_materialization(lease, prepared=prepared, verified=verified) + + if backend == "postgres": + prefixes = { + "outcome": "INSERT INTO reporting_reconciliation_records", + "status_head": "INSERT INTO reporting_materializer_status_heads", + "account_head": "UPDATE reporting_materializer_accounts SET captured_sequence=", + "boundary": "INSERT INTO reporting_materializer_status_boundaries", + "ack": "UPDATE reporting_materializer_work SET state='acked'", + } + with postgres_failure(monkeypatch, prefixes[position]) as hit: + with pytest.raises(ReportingWriterError): + await finish() + assert len(hit) == 1 + else: + import adcp.reporting.materializer.memory as memory + + cls = InMemoryReportingMaterializerStore + if position == "outcome": + original = cls._commit_record_unlocked + + def fail(self, record, **kwargs): + result = original(self, record, **kwargs) + if isinstance(record, ReportingMaterializationRecord): + raise OSError("injected transaction boundary") + return result + + target, method = cls, "_commit_record_unlocked" + elif position in {"status_head", "account_head"}: + + def fail(*args, **kwargs): + raise OSError("injected transaction boundary") + + target, method = memory, "ReportingMaterializerBoundary" + else: + method = "_materializer_dirty" if position == "boundary" else "_park" + target, original = cls, getattr(cls, method) + + def fail(self, *args, **kwargs): + original(self, *args, **kwargs) + raise OSError("injected transaction boundary") + + with monkeypatch.context() as patch: + patch.setattr(target, method, fail) + with pytest.raises(ReportingWriterError): + await finish() + assert await h.image() == before + materializer_operation_1 = await finish() + assert (materializer_operation_1).state == "verified" + assert len(await case.outcomes()) == 1 + assert len(await h.store.read_materializer_boundaries(caller=case.scope.principal)) == 1 + + +@pytest.mark.parametrize("backend", ["memory", "postgres"]) +@pytest.mark.parametrize("position", ["event", "expansion_work"]) +async def test_enabled_logical_enqueue_failure_never_downgrades_to_polling( + backend, position, monkeypatch +): + async with durable_harness(backend, notifications=True) as h: + case = await durable_case(h.store) + lease = await case.claim() + prepared, verified = await case.verified(lease) + before = await h.image() + + async def finish(): + return await h.store.finish_materialization(lease, prepared=prepared, verified=verified) + + if backend == "postgres": + table = "events" if position == "event" else "expansions" + with postgres_failure( + monkeypatch, f"INSERT INTO reporting_materializer_notification_{table}" + ) as hit: + with pytest.raises(ReportingWriterError): + await finish() + assert len(hit) == 1 + else: + import adcp.reporting.outbox.memory as memory + from adcp.reporting.materializer.capture import MaterializerNotificationState + + if position == "event": + target, method = memory, "_Work" + + def fail(*args, **kwargs): + raise OSError("injected transaction boundary") + + else: + target, method = MaterializerNotificationState, "enqueue" + original = target.enqueue + + def fail(self, event): + original(self, event) + raise OSError("injected transaction boundary") + + with monkeypatch.context() as patch: + patch.setattr(target, method, fail) + with pytest.raises(ReportingWriterError): + await finish() + assert await h.image() == before + assert await h.queue() == ((), ()) + materializer_operation_2 = await finish() + assert (materializer_operation_2).state == "verified" + events, work = await h.queue() + assert len(events) == 1 and work == ("quarantined",) + + +@pytest.mark.parametrize("backend", ["memory", "postgres"]) +async def test_missing_enabled_logical_event_cannot_ack_a_verified_finish(backend, monkeypatch): + async with durable_harness(backend, notifications=True) as h: + case = await durable_case(h.store) + lease = await case.claim() + prepared, evidence = await case.verified(lease) + before = await h.image() + if backend == "memory": + from adcp.reporting.materializer.capture import MaterializerNotificationState + + def empty(self, event): + pass + + monkeypatch.setattr(MaterializerNotificationState, "enqueue", empty) + else: + import adcp.reporting.materializer.pg as pg + + async def empty(connection, event): + pass + + monkeypatch.setattr(pg, "enqueue_materializer_event_on", empty) + with pytest.raises(ReportingWriterError): + await h.store.finish_materialization(lease, prepared=prepared, verified=evidence) + assert await h.image() == before + assert await h.queue() == ((), ()) + + +@pytest.mark.parametrize("backend", ["memory", "postgres"]) +@pytest.mark.parametrize("notifications", [False, True]) +async def test_expiry_inside_finish_rolls_back_outcome_capture_event_and_ack( + backend, notifications, monkeypatch +): + async with durable_harness(backend, notifications=notifications) as h: + case = await durable_case(h.store) + lease = await case.claim(lease_seconds=3) + prepared, evidence = await case.verified(lease) + before = await h.image() + cls = type(h.store) + if backend == "memory": + original = cls._materializer_dirty + + def expire(self, *args): + original(self, *args) + h.clock.advance(timedelta(seconds=4)) + + method = "_materializer_dirty" + else: + original = cls._materializer_dirty_on + + async def expire(self, connection, *args): + await original(self, connection, *args) + await connection.execute( + "UPDATE reporting_materializer_work SET lease_until=clock_timestamp()" + " WHERE state='pending'" + ) + + method = "_materializer_dirty_on" + with monkeypatch.context() as patch: + patch.setattr(cls, method, expire) + with pytest.raises(ReportingWriterError): + await h.store.finish_materialization(lease, prepared=prepared, verified=evidence) + assert await h.image() == before and await h.queue() == ((), ()) + await h.expire() + materializer_operation_3 = await case.service().run_once() + assert (materializer_operation_3).state == "verified" + assert case.writer.write_effects == 1 + assert len(await h.works()) == 1 diff --git a/tests/conformance/reporting/test_reporting_notification_packaging.py b/tests/conformance/reporting/test_reporting_notification_packaging.py index a068aec69..6679a909e 100644 --- a/tests/conformance/reporting/test_reporting_notification_packaging.py +++ b/tests/conformance/reporting/test_reporting_notification_packaging.py @@ -30,6 +30,45 @@ ROOT = Path(__file__).resolve().parents[3] +def redacted_stage_stderr(stderr): + """Keep known error signatures, never arbitrary build/provider prose. + + Even an unfamiliar credential format cannot escape an allowlist of fixed + labels. Line positions and a digest distinguish otherwise unclassified + traces without logging URLs, environment values or exception messages. + """ + signatures = { + "No space left on device": "disk_full", + "[Errno 28]": "disk_full", + "PermissionError": "permission_denied", + "FileNotFoundError": "file_missing", + "ModuleNotFoundError": "module_missing", + "BackendUnavailable": "backend_unavailable", + "subprocess-exited-with-error": "subprocess_failed", + "No matching distribution found": "distribution_unavailable", + "Temporary failure in name resolution": "dns_failure", + "ConnectionError": "connection_failure", + "ReadTimeout": "read_timeout", + "SyntaxError": "syntax_error", + "AssertionError": "assertion_failed", + } + lines = stderr.splitlines() + trace = [] + for position, line in enumerate(lines): + found = sorted({tag for marker, tag in signatures.items() if marker in line}) + if found: + trace.append([position + 1, found]) + return json.dumps( + { + "lines": len(lines), + "sha256": hashlib.sha256(stderr.encode()).hexdigest(), + "trace": trace[-8:], + "classification": "recognized" if trace else "unclassified", + }, + separators=(",", ":"), + ) + + def run_step(command, *, label, cwd, value=None, timeout=120): started = time.monotonic() process = subprocess.Popen( @@ -44,7 +83,7 @@ def run_step(command, *, label, cwd, value=None, timeout=120): ) print(f"notification_distribution stage={label} pid={process.pid} started", flush=True) try: - stdout, _ = process.communicate( + stdout, stderr = process.communicate( json.dumps(value) if value is not None else None, timeout=timeout, ) @@ -79,10 +118,21 @@ def run_step(command, *, label, cwd, value=None, timeout=120): f" pid={process.pid} exit={process.returncode} cleanup={cleanup}" f" elapsed_ms={int((time.monotonic() - started) * 1000)}" f" stdout_chars={len(stdout)} stderr_chars={len(stderr)}" + f" stderr={redacted_stage_stderr(stderr)}" ) from None # Do not turn package-manager/provider output into test diagnostics. - assert process.returncode == 0, f"notification distribution {label}: exit {process.returncode}" - print(f"notification_distribution stage={label} passed", flush=True) + if process.returncode != 0: + raise AssertionError( + f"notification distribution {label}: exit {process.returncode} pid={process.pid}" + f" elapsed_ms={int((time.monotonic() - started) * 1000)}" + f" stdout_chars={len(stdout)} stderr_chars={len(stderr)}" + f" stderr={redacted_stage_stderr(stderr)}" + ) + print( + f"notification_distribution stage={label} passed" + f" elapsed_ms={int((time.monotonic() - started) * 1000)}", + flush=True, + ) return stdout @@ -96,9 +146,39 @@ def test_distribution_subprocess_deadline_is_bounded_and_sanitized(tmp_path): ) +def test_distribution_failure_diagnostics_classify_without_echoing_prose(tmp_path): + body = ( + "https://index-user:index-password@example.test/simple?signature=private\n" + "Authorization: Bearer opaque-credential\n" + "TOKEN_WITH_UNFAMILIAR_FORMAT=private-configuration\n" + "OSError: [Errno 28] No space left on device\n" + ) + with pytest.raises(AssertionError) as captured: + run_step( + [ + sys.executable, + "-c", + "import sys;sys.stderr.write(sys.stdin.read());raise SystemExit(3)", + ], + label="diagnostic_probe", + cwd=tmp_path, + value=body, + ) + message = str(captured.value) + assert "disk_full" in message and "exit 3" in message + assert len(message) < 1024 + assert all(word not in message for word in ("private", "index-user", "opaque-credential")) + unknown = redacted_stage_stderr("unrecognized provider body and secret configuration") + assert "unclassified" in unknown and "provider" not in unknown and "secret" not in unknown + + @pytest.fixture(scope="module") -def built_distribution(tmp_path_factory): +def built_distribution(tmp_path_factory, request): path = tmp_path_factory.mktemp("reporting-outbox-distribution") + # A module can allocate several installed environments and copied schemas. + # Remove only this fixture's tree after all its dependent fixtures/processes + # finish. Keep the optional fingerprint-checked immutable cache separately. + request.addfinalizer(lambda: shutil.rmtree(path)) cache = os.environ.get("ADCP_REPORTING_DISTRIBUTION") sources = [ ROOT / name @@ -196,8 +276,12 @@ def installed_distribution(built_distribution, request): resolve_reporting_consumer, ) from adcp.reporting.ledger import InMemoryReportingLedgerStore, ReportingProducer -from adcp.reporting.ledger import ReportingStatusSnapshot, StatusProjectionInput, project_status_scope -from adcp.reporting.outbox import ReportingStatusNotificationLifecycle, ReportingStatusService, StatusChanged +from adcp.reporting.ledger import ( + ReportingStatusSnapshot, StatusProjectionInput, project_status_scope, +) +from adcp.reporting.outbox import ( + ReportingStatusNotificationLifecycle, ReportingStatusService, StatusChanged, +) from adcp.validation.schema_loader import get_named_validator import inspect, sys @@ -216,7 +300,9 @@ def installed_distribution(built_distribution, request): assert "adcp[pg]" in str(error), str(error) else: raise AssertionError("PgReportingOutbox must raise the [pg] install hint") -from adcp.reporting.outbox import PgStatusNotificationStore, PgReportingStatusOutbox, PgReportingActivityUnionStore +from adcp.reporting.outbox import ( + PgStatusNotificationStore, PgReportingStatusOutbox, PgReportingActivityUnionStore, +) for constructor in (lambda: PgStatusNotificationStore(None), lambda: PgReportingStatusOutbox(pool=None), lambda: PgReportingActivityUnionStore(None, None)): @@ -383,7 +469,8 @@ async def get_active(self, *, account_id, subscriber_id, notification_type): clock=clock, ) projector = ReportingActivityProjector(outbox) - assert await ReportingActivitySupport(worker, store, projector).durable() + materializer_operation_1 = await ReportingActivitySupport(worker, store, projector).durable() + assert materializer_operation_1 events = await outbox.list_events(account_id="acct_a") expanded_1 = await worker.expand_one(account_id="acct_a") assert expanded_1 @@ -446,7 +533,9 @@ async def get_active(self, *, account_id, subscriber_id, notification_type): status_subscription = replace(subscription, event_types=("reporting.status_changed",)) class StatusConfigurations: async def list_active(self, *, account_id, notification_type): - return (status_subscription,) if account_id == status_subscription.account_id else () + return ( + (status_subscription,) if account_id == status_subscription.account_id else () + ) async def get_active(self, *, account_id, subscriber_id, notification_type): return status_subscription if account_id == status_subscription.account_id else None status_worker = ReportingNotificationWorker(outbox=status.outbox, @@ -466,9 +555,11 @@ async def get_active(self, *, account_id, subscriber_id, notification_type): status_operation_4 = await status.project_one(account_id="acct_a") assert not (status_operation_4).did_work assert await status.outbox.list_events(account_id="acct_a") == status_events - status_lease = await status.outbox.claim_delivery(account_id="acct_a", now=clock(), lease_seconds=60) + status_lease = await status.outbox.claim_delivery( + account_id="acct_a", now=clock(), lease_seconds=60) retry = cipher.open(status_lease.delivery).prepared - assert retry.body == status_body.body and retry.idempotency_key == status_body.idempotency_key + assert retry.body == status_body.body + assert retry.idempotency_key == status_body.idempotency_key status_operation_5 = await status.outbox.finish_delivery(status_lease, now=clock(), state="complete") assert status_operation_5 status_operation_6 = await status.outbox.reemit(account_id="acct_a", consumer_namespace="", diff --git a/tests/conformance/reporting/test_reporting_status_process_matrix.py b/tests/conformance/reporting/test_reporting_status_process_matrix.py index e3c039441..e3c8e2d4d 100644 --- a/tests/conformance/reporting/test_reporting_status_process_matrix.py +++ b/tests/conformance/reporting/test_reporting_status_process_matrix.py @@ -146,7 +146,8 @@ async def database_seed(pool, case="expected", *, baseline=True): await c.execute( "UPDATE reporting_status_accounts SET baseline_complete=TRUE," " baseline_highwater=%s, dirty_sequence=%s, baseline_at=%s," - " replay_lifecycles=%s::jsonb, selector_target_version=2, selector_transition='complete' WHERE account_id='acct_a'", + " replay_lifecycles=%s::jsonb, selector_target_version=2," + " selector_transition='complete' WHERE account_id='acct_a'", (through, through, snapshot.as_of, _replay_storage(snapshot)), ) return ledger, status, at, escalation diff --git a/tests/type_checks/reporting_durable_materializer.py b/tests/type_checks/reporting_durable_materializer.py new file mode 100644 index 000000000..7ff9b7f4c --- /dev/null +++ b/tests/type_checks/reporting_durable_materializer.py @@ -0,0 +1,32 @@ +"""The optional durable protocol and public lazy imports remain strict-adopter APIs.""" + +from typing_extensions import assert_type + +from adcp.reporting.materializer import ( + InMemoryReportingMaterializerStore, + PgReportingMaterializerStore, + ReportingDestinationIO, + ReportingDestinationWriter, + ReportingMaterializerLease, + ReportingMaterializerService, + ReportingMaterializerStore, + ReportingMaterializerTurn, + ReportingVerificationKey, +) + + +async def adopter( + postgres: PgReportingMaterializerStore, + memory: InMemoryReportingMaterializerStore, + io: ReportingDestinationIO, + writer: ReportingDestinationWriter, + key: ReportingVerificationKey, +) -> None: + store: ReportingMaterializerStore = postgres + store = memory + service = ReportingMaterializerService(store, io, writer) + assert_type(await service.run_once(), ReportingMaterializerTurn) + lease = await store.claim_materialization(keys=(key,)) + if isinstance(lease, ReportingMaterializerLease): + assert_type(await store.renew_materialization(lease, lease_seconds=30), bool) + await store.authorize_materialization(lease)