diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 7b3581c4f..d2edf5f5b 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -12,6 +12,7 @@ on: - conductor/1167b2-durable-managed-reporting - conductor/reporting-receipt-ingress-b22 - conductor/reporting-frozen-account-feed-b23 + - conductor/reporting-schema-proof-receipt-diagnostics-hardening # Default @adcp/sdk runner alias for storyboard jobs. Tracks the current # stable @adcp/sdk release via the ``latest`` npm dist-tag. @@ -109,7 +110,7 @@ jobs: - name: Run adopter type-check suite if: matrix.python-version == '3.12' - run: mypy --strict tests/type_checks/ examples/reporting_webhook_activity.py examples/reporting_status_notifications.py examples/reporting_destination_writer.py examples/reporting_durable_materializer.py examples/reporting_receipt_ingress.py + run: mypy --strict tests/type_checks/ examples/reporting_webhook_activity.py examples/reporting_status_notifications.py examples/reporting_destination_writer.py examples/reporting_durable_materializer.py examples/reporting_receipt_ingress.py examples/reporting_production.py - name: Enforce adopter type-check fixture contract if: matrix.python-version == '3.12' @@ -200,6 +201,10 @@ jobs: --ignore=tests/conformance/reporting/test_reporting_materializer_installed_pg.py \ --ignore-glob='tests/conformance/reporting/test_reporting_receipt_*.py' \ --ignore-glob='tests/conformance/reporting/test_reporting_feed_*.py' \ + --ignore-glob='tests/conformance/reporting/test_reporting_production*.py' \ + --ignore-glob='tests/conformance/reporting/test_reporting_projection*.py' \ + --ignore=tests/conformance/reporting/test_reporting_tier_projection.py \ + --ignore=tests/conformance/reporting/test_reporting_schedule_schema.py \ -v -ra ;; process) @@ -216,7 +221,7 @@ jobs: runs-on: ubuntu-latest timeout-minutes: 5 permissions: {} - needs: [pg-conformance, pg-reporting-status, pg-reporting-materializer, pg-reporting-receipts, pg-reporting-receipt-compatibility, pg-reporting-feed, pg-reporting-feed-compatibility, pg-reporting-feed-installed] + needs: [pg-conformance, pg-reporting-status, pg-reporting-materializer, pg-reporting-receipts, pg-reporting-receipt-compatibility, pg-reporting-feed, pg-reporting-feed-compatibility, pg-reporting-feed-installed, pg-reporting-production, pg-reporting-production-compatibility, pg-reporting-production-installed] if: ${{ always() }} steps: - name: Require every Postgres conformance lane @@ -229,8 +234,11 @@ jobs: FEED_RESULT: ${{ needs.pg-reporting-feed.result }} FEED_COMPATIBILITY_RESULT: ${{ needs.pg-reporting-feed-compatibility.result }} FEED_INSTALLED_RESULT: ${{ needs.pg-reporting-feed-installed.result }} + PRODUCTION_RESULT: ${{ needs.pg-reporting-production.result }} + PRODUCTION_COMPATIBILITY_RESULT: ${{ needs.pg-reporting-production-compatibility.result }} + PRODUCTION_INSTALLED_RESULT: ${{ needs.pg-reporting-production-installed.result }} run: | - if [ "$PG_RESULT" != "success" ] || [ "$STATUS_RESULT" != "success" ] || [ "$MATERIALIZER_RESULT" != "success" ] || [ "$RECEIPTS_RESULT" != "success" ] || [ "$RECEIPT_COMPATIBILITY_RESULT" != "success" ] || [ "$FEED_RESULT" != "success" ] || [ "$FEED_COMPATIBILITY_RESULT" != "success" ] || [ "$FEED_INSTALLED_RESULT" != "success" ]; then + if [ "$PG_RESULT" != "success" ] || [ "$STATUS_RESULT" != "success" ] || [ "$MATERIALIZER_RESULT" != "success" ] || [ "$RECEIPTS_RESULT" != "success" ] || [ "$RECEIPT_COMPATIBILITY_RESULT" != "success" ] || [ "$FEED_RESULT" != "success" ] || [ "$FEED_COMPATIBILITY_RESULT" != "success" ] || [ "$FEED_INSTALLED_RESULT" != "success" ] || [ "$PRODUCTION_RESULT" != "success" ] || [ "$PRODUCTION_COMPATIBILITY_RESULT" != "success" ] || [ "$PRODUCTION_INSTALLED_RESULT" != "success" ]; then echo "Postgres conformance matrix result: $PG_RESULT" echo "Reporting status conformance result: $STATUS_RESULT" echo "Durable materializer conformance result: $MATERIALIZER_RESULT" @@ -239,6 +247,9 @@ jobs: echo "Frozen feed conformance result: $FEED_RESULT" echo "Feed rolling compatibility result: $FEED_COMPATIBILITY_RESULT" echo "Installed frozen feed result: $FEED_INSTALLED_RESULT" + echo "pg-reporting-production result: $PRODUCTION_RESULT" + echo "pg-reporting-production-compatibility result: $PRODUCTION_COMPATIBILITY_RESULT" + echo "pg-reporting-production-installed result: $PRODUCTION_INSTALLED_RESULT" exit 1 fi echo "All Postgres conformance lanes passed" @@ -661,6 +672,177 @@ jobs: ${{ runner.temp }}/hardening-installed-evidence if-no-files-found: error + pg-reporting-production: + name: Production reporting status, ownership and notification contracts + runs-on: ubuntu-latest + permissions: + contents: read + timeout-minutes: 35 + services: + postgres: + image: postgres:16 + env: + POSTGRES_HOST_AUTH_METHOD: trust + POSTGRES_DB: adcp_production_test + ports: ["5432:5432"] + options: >- + --health-cmd pg_isready --health-interval 5s + --health-timeout 5s --health-retries 10 + steps: + - uses: actions/checkout@v6 + - uses: actions/setup-python@v6 + with: + python-version: "3.12" + cache: pip + cache-dependency-path: pyproject.toml + - name: Install test dependencies + run: pip install -e ".[dev,pg]" + - name: Run complete production and projection conformance + shell: bash + timeout-minutes: 30 + env: + ADCP_PG_TEST_URL: postgresql://postgres@localhost:5432/adcp_production_test + run: | + # Explicit pytest paths bypass --ignore; preselect and reject empty globs. + shopt -s nullglob + source_tests=() + for test in tests/conformance/reporting/test_reporting_production*.py; do + case "$test" in + *test_reporting_production_packaging.py|*test_reporting_production_rolling.py) continue ;; + esac + source_tests+=("$test") + done + projection_tests=(tests/conformance/reporting/test_reporting_projection*.py) + if (( ${#source_tests[@]} == 0 || ${#projection_tests[@]} == 0 )); then + echo "Production or projection test selection is empty" + exit 1 + fi + python scripts/reporting_test_harness.py pytest \ + "${source_tests[@]}" \ + "${projection_tests[@]}" \ + tests/conformance/reporting/test_reporting_tier_projection.py \ + tests/conformance/reporting/test_reporting_schedule_schema.py \ + tests/test_reporting_capability_models.py \ + tests/test_reporting_revision_ownership.py \ + tests/test_reporting_production_public.py \ + tests/test_schema_datetime_formats.py \ + -v -s -ra | tee pg-reporting-production-evidence.log + - name: Preserve production contract evidence + if: always() + uses: actions/upload-artifact@v7 + with: + name: pg-reporting-production-evidence-${{ github.run_attempt }} + path: pg-reporting-production-evidence.log + if-no-files-found: error + + pg-reporting-production-installed: + name: Installed production reporting (${{ matrix.cell }}, Python 3.10) + runs-on: ubuntu-latest + permissions: + contents: read + timeout-minutes: 40 + strategy: + fail-fast: false + matrix: + cell: [base-vcs, base-sdist, pg-vcs, pg-sdist] + services: + postgres: + image: postgres:16 + env: + POSTGRES_HOST_AUTH_METHOD: trust + POSTGRES_DB: adcp_production_installed_test + ports: ["5432:5432"] + options: >- + --health-cmd pg_isready --health-interval 5s + --health-timeout 5s --health-retries 10 + steps: + - uses: actions/checkout@v6 + - uses: actions/setup-python@v6 + id: production-python310 + with: + python-version: "3.10" + - uses: actions/setup-python@v6 + with: + python-version: "3.12" + cache: pip + cache-dependency-path: pyproject.toml + - name: Install test dependencies + run: pip install -e ".[dev,pg]" + - name: Run installed source-equivalent production contract + shell: bash + timeout-minutes: 35 + env: + ADCP_PG_TEST_URL: postgresql://postgres@localhost:5432/adcp_production_installed_test + ADCP_PYTHON310: ${{ steps.production-python310.outputs.python-path }} + ADCP_PRODUCTION_EVIDENCE: ${{ runner.temp }}/production-installed-evidence + PRODUCTION_CELL: ${{ matrix.cell }} + run: | + python scripts/reporting_test_harness.py pytest \ + tests/conformance/reporting/test_reporting_production_packaging.py \ + -k "$PRODUCTION_CELL" -v -s -ra | tee pg-reporting-production-installed.log + - name: Preserve installed origins, original inner logs and exact asset hashes + if: always() + uses: actions/upload-artifact@v7 + with: + name: pg-reporting-production-installed-${{ matrix.cell }}-${{ github.run_attempt }} + path: | + pg-reporting-production-installed.log + ${{ runner.temp }}/production-installed-evidence + if-no-files-found: error + + pg-reporting-production-compatibility: + name: B2.3 and hardening to installed B2.4 activation and restart + runs-on: ubuntu-latest + permissions: + contents: read + timeout-minutes: 50 + services: + postgres: + image: postgres:16 + env: + POSTGRES_HOST_AUTH_METHOD: trust + POSTGRES_DB: adcp_production_rolling_test + ports: ["5432:5432"] + options: >- + --health-cmd pg_isready --health-interval 5s + --health-timeout 5s --health-retries 10 + steps: + - uses: actions/checkout@v6 + - name: Fetch integrated feed and hardening comparison artifacts + timeout-minutes: 1 + run: git fetch --no-tags origin 2d777ace7b4bf8be519ce0abd4fd0a25ed4f1da7 e16eb8cf3074cabd45aab42840950f05ad6d2b43 + - uses: actions/setup-python@v6 + id: production-python310 + with: + python-version: "3.10" + - uses: actions/setup-python@v6 + with: + python-version: "3.12" + cache: pip + cache-dependency-path: pyproject.toml + - name: Install test dependencies + run: pip install -e ".[dev,pg]" + - name: Run actual historical page one, activation, SIGKILL and cold continuations + shell: bash + timeout-minutes: 42 + env: + ADCP_PG_TEST_URL: postgresql://postgres@localhost:5432/adcp_production_rolling_test + ADCP_PYTHON310: ${{ steps.production-python310.outputs.python-path }} + ADCP_PRODUCTION_EVIDENCE: ${{ runner.temp }}/production-rolling-evidence + run: | + python scripts/reporting_test_harness.py pytest \ + tests/conformance/reporting/test_reporting_production_rolling.py \ + -v -s -ra | tee pg-reporting-production-rolling.log + - name: Preserve exact installed historical continuity and fence evidence + if: always() + uses: actions/upload-artifact@v7 + with: + name: pg-reporting-production-rolling-${{ github.run_attempt }} + path: | + pg-reporting-production-rolling.log + ${{ runner.temp }}/production-rolling-evidence + if-no-files-found: error + conventional-commits: name: Validate conventional commit format runs-on: ubuntu-latest diff --git a/.github/workflows/pr-title-check.yml b/.github/workflows/pr-title-check.yml index de0669486..3975fd4a7 100644 --- a/.github/workflows/pr-title-check.yml +++ b/.github/workflows/pr-title-check.yml @@ -3,7 +3,7 @@ name: PR Title Check on: pull_request: types: [opened, edited, synchronize, reopened] - branches: [main, conductor/1167b2-durable-managed-reporting, conductor/reporting-receipt-ingress-b22, conductor/reporting-frozen-account-feed-b23] + branches: [main, conductor/1167b2-durable-managed-reporting, conductor/reporting-receipt-ingress-b22, conductor/reporting-frozen-account-feed-b23, conductor/reporting-schema-proof-receipt-diagnostics-hardening] permissions: contents: read diff --git a/docs/reporting-production.md b/docs/reporting-production.md new file mode 100644 index 000000000..e4b7a1ab2 --- /dev/null +++ b/docs/reporting-production.md @@ -0,0 +1,346 @@ +# Production reporting and versioned status + +`adcp.reporting.production` composes the existing durable producer, materializer, +receipt ingress and frozen feed with versioned private status. Use +[`examples/reporting_production.py`](../examples/reporting_production.py) for the +typed composition and authenticated MCP/A2A lifecycle. Existing Core polling +and eligible Core notification deployments keep their existing composition. + +## Provider and source contracts + +Each `ReportingProductionOffering` binds a complete public offering to one +actual `ReportingProducer`, source offering and installed verifier. A +`ReportingProductionDestination` supplies complete `ReportingProductionMethod` +values and resolves each authorized destination into a +`ReportingProductionDestinationBinding`. Pattern or transport labels alone +are insufficient: provider, destination modes, access mode, orchestration, +reader compatibility and the selected destination must match. Supply opaque +references, never credentials or bearer URLs. The same provider resolves +fresh, separately authorized write and readback sessions. + +The source implements `ReportingProductionSource.configuration_binding()`. +Construct its result with `ReportingProductionSourceBinding.for_configuration()` +using the authenticated account/catalog mapping and effective source capability +digest. Every media-buy/product pair is explicit. Report-definition IDs do not +identify products. Account, configuration ID and generation are all part of the +binding. Multiple offerings can share a definition; they still need distinct +applicable source contracts. Supported-offering discovery can precede the first +account binding. + +Admission durably freezes the source mapping, generation semantics and selected +provider method. Source acquisition and materialization compare the current +authoritative contracts to those frozen values. Withdrawal or incompatible +mutation fails closed; it cannot reinterpret an old generation. Resolve and +admit a new generation for a different contract. Historical pages and accepted +receipt evidence retain their original values. + +`ReportingProductionConfigurationTask` wraps the adopter's authenticated account +task. Its callback receives an `admit` function; call it with a +`ReportingConfigurationAdmission` containing the complete requested public +configuration and trusted resolved records before returning ready or inactive +state, including replay. The SDK checks actual schedule, scope, provider method, +finality, source generation and returned coverage. It completes the account's +versioned activation before returning ready, so newly admitted accounts need no +separate account-enumeration worker. Account provisioning and its durable +idempotency remain the account implementation's responsibility; do not claim +unsupported account features in its capability model. This producer admits +explicit full media-buy scopes and fixed-duration schedules that it can prove. +It does not turn an unsupported dynamic or partial scope into full coverage. + +Managed delivery-only polling needs the source, durable writer, verified +readback, complete status/exact reads and configuration route. It does not need +the receipt route or an HTTP notification worker. Reconciled additionally needs +official finality, canonical verification, and mounted revision/adjustment +receipt ingress. Neither the development writer nor the in-memory conformance +store advertises production durability. Capability checks bind the actual +mounted handler and running components; adopters do not maintain readiness +booleans. Protected capability fields cannot be supplied through `extra=`. + +## Migration, drain and activation + +1. Stop and drain autonomous legacy materializer writers. Resolve or explicitly + import uncertain legacy effects using the + [materializer recovery procedure](reporting-durable-materializer.md). Keep the + original external idempotency identity for pending SDK attempts. +2. Stop and drain old status projectors and clock sweepers. Ordinary compatible + historical readers/writers are a different compatibility claim from running + those incompatible autonomous workers. +3. Call `await store.create_schema()` on `PgReportingProductionStore` before + starting workers. Bootstrap remains transactional, serialized and repeatable. + Keep the private configuration lease-fairness objects: they deliberately + remain outside the old mandatory manifests. +4. Construct the verifier registry, trusted source/provider adapters, producer, + materializer, `PgReportingStatusProjection` and `ReportingProductionSupport`. + Mount **that support's handler** on the authenticated transports before + `await support.start()`. The typed example uses the combined server's startup + and shutdown hooks. +5. Activate existing accounts with `await support.activate(account_id=...)`, or + let validated configuration admission activate the affected account. Activation + fences old projection writers and incrementally consumes preserved captured + boundaries and baselines. An interruption resumes those original inputs; + it does not reconstruct historical status from today's records. + +The isolated projection and production manifests do not append objects to the +older ledger, materializer, receipt-ingestion or frozen-feed manifests. Old or +partial installations cannot prove production readiness. Catalog proofs are +positive caches scoped to the concrete support/pool and invalidation epoch; +live component and route checks still run. For later DDL, stop/drain support, +migrate, construct fresh support, validate, and restart. Arbitrary serving-time +DDL or search-path mutation is not automatically detected. Bound database +statement execution at the adopter/database layer. +The support also binds the pool's concrete identity and checks its open state +on each readiness request. Closing or replacing that pool withdraws the claim +even while the immutable catalog proof remains cached; rebuild the support for +a new pool. +The projection queue and all configured notification queues must retain the +same pool as the ledger. Warm discovery performs no pool checkout and stays +available when that valid pool is temporarily saturated. + +Activation never promotes epoch-zero work or readiness. Pending attempts keep +their original work identity, epoch and permanently quarantined events through +resume, replay and restart. Only genuinely new qualified work enters the +production epoch. The verified terminal result, immutable status capture, work +ACK and enabled logical `reporting.delivery_ready` enqueue share one transaction. +Enqueue failure rolls all of them back. An uncertain external effect resumes +the same destination identity; it must not allocate a fresh attempt. + +With notifications disabled, no logical enqueue occurs and complete polling +remains available. With notifications enabled, the logical queue is mandatory; +a broken enabled path fails closed. Optional `production_notification_workers` +compose the real Core, versioned status and production queues for crash-safe +recipient expansion and delivery. They never drain quarantined readiness. +Supply `ReportingProductionSigning(resolver, algorithms, brand_json_url=...)` with the actual +resolver's RFC 9421 algorithms. Its public signing declaration and every +resolved key must agree. Production notification registration and dispatch +reject legacy Bearer/HMAC fallback; rotation keeps the advertised algorithm +contract. Private key material stays in the resolver and sender. The operator +must publish its brand document and corresponding signing keys at the declared +identity; the SDK does not assert that a supplied URL proves external ownership. +The declaration is RFC 9421 `adcp/webhook-signing/v1`, the exact supplied +algorithm set, no legacy fallback, and an 86,400-second retry horizon. Each +queue reserves the first HTTP attempt and its key/body binding before peer I/O, +using database time in PostgreSQL. Window insertion, HTTP attempt and ordinal +reservation share one transaction; a failed reservation leaves none of them. +That committed reservation anchors an immutable +deadline: attempts are permitted strictly before `started_at + 86400 seconds`, +and refused at or after that instant. A reservation with unknown HTTP effect is +still the original anchor. Retries, key rotation, crashes and restart cannot extend it; expired +deliveries are suppressed with the existing closed `lease_expired` code. +Backoff is capped at the original deadline, including after configuration changes. +An HTTP attempt reserved before that deadline may finish afterward; the limit +prevents another attempt, and does not rewrite an already observed result. +A clock earlier than the retained first-attempt timestamp fails closed. +Delivery records and protected payload bindings are retained. Drain and rebuild +the support to change algorithms or operator identity. Receivers must retain +old public verification keys and authenticated deduplication state for the +advertised interval. The tests use public verification with pinned fixture +keys; external brand/JWKS discovery and live interoperability remain separate +cross-language gates. + +In the immutable `3.2.0-rc.3` capability schema, the descriptions at +`/properties/webhook_signing/properties/delivery_retry_horizon_seconds` and +`/properties/identity/description` require these declarations for applicable 3.2 +agents. The JSON fields deliberately remain optional for older documents. +Unmodified schema validation therefore **accepts their omission**. The mounted +semantic assertions in `test_reporting_production_readiness.py` establish the +missing-declaration defect; `test_reporting_production_signing_schema.py` +separately preserves the original schema acceptance and normative text. The +reservation, clock, restart, signature and expiry tests establish actual +behavior. This differs from the #1179 cached-schema rejection described below. + +The three workers also retain actual attempt activity through the inherited +SDK reservation/outcome transaction. `ReportingActivityProjector(worker.outbox)` +provides the corresponding authenticated account-activity read; applications +may compose that existing optional account surface. Expiry creates no fictitious +HTTP attempt or outcome. Reporting polling and immutable receipt evidence remain +the recovery path if the receiver did not acknowledge before the deadline. +Recipient fanout is separate from the finish transaction's immutable logical +enqueue. Stopping delivery may accumulate eligible pending events; it cannot +manufacture, promote or re-identify historical readiness. + +## Captured status, schedule and ownership + +Status keeps the canonical consumer private even when consumer feedback is off. +Revision selection examines complete history first. A unique official wins over +an unlinked retained snapshot even before the official has an artifact. Managed +needs that selected readable verified artifact. Reconciled also needs accepted +selected-official evidence and an accepted current receipt leaf for every +applicable official adjustment. Later valid artifacts or health degradation do +not erase accepted evidence or successful-materialization counts. Consumer +rejection is never a materializer retry signal. + +A generation owes only complete periods whose start is at or after activation +and strictly before deactivation. A period already begun at deactivation remains +owed in full, including its SLA. The producer and captured feed use the same rule. +For the current `3.2-rc.6` contract, an open summary's `next_expected_at` is its +nearest future obligation due time. A **complete summary** instead reports the +nearest future period **start**, strictly after the captured `ledger_as_of`, +across the committed generations in scope. It is absent when no such period +exists. This forecast creates, counts and leases no future obligation, and does +not change scope closure, coverage or health. Complete periods responses do not +inherit the summary-only forecast. Civil-time and DST boundaries are retained. + +The immutable `3.2.0-rc.3` cache and exact-version Python correction for #1179 +remain available for historical validation. Direct historical projections and +stored rc.3 walks retain their original representation; they do not acquire the +new rc.6 forecast. The current SDK does not advertise rc.3 as a live client or +server pin. New mounts, advertised schemas and rendering use the supported +packaged version. Cursor/checkpoint version mismatches fail closed only after +caller and signed-position verification. Changing a production mount's captured +protocol pin invalidates readiness even after a successful schema proof. + +Integrated parents created representation-one snapshots without a protocol +filter marker. Their original pages and checkpoints remain usable under rc.6, +before and after activation, without re-projecting or modifying their captured +bytes. Caller, signature and every original semantic filter must still match. +This exception is restricted to unversioned representation one without revision +ownership; newer version-bound representations retain strict pin matching. A +legacy checkpoint can start a fresh rc.6 walk, which records the new marker. + +These are Python source and conformance boundaries, not TypeScript, release or +cross-language acceptance. The tests in `test_reporting_projection_rc6.py` cover +current mounted transports, clients and schema agreement, frozen historical +bytes and continuations, civil/DST boundaries and absence of future work. The +exact original rc.3 rejection and version-scoped correction remain separately +recorded in `test_reporting_schedule_schema.py`. + +Producer turns keep the 64-period maximum. Durable closing positions and bounded +pending queues rotate fairly without rescanning all history. A readable snapshot +completes an acquisition, not its declared settling policy: the queue retains +that obligation through restatement cadence and the settling window. It retires +a snapshot-only policy at its terminal boundary, or a closing policy only after +an official revision is actually committed. A not-ready official source leaves +work pending. Persisted observation checkpoints survive fresh workers and keep +no-op refreshes from replaying or changing execution identity. Lease acquisition +takes the account lock before configuration rows and preserves account isolation. +Lease acquisition, release and recovery are bookkeeping, not new reporting +observations or materializer targets. The production PostgreSQL path retains +the inherited trigger and cancels only its lease-only candidate increment in +the same account-locked transaction. It never rewrites a pending attempt's +generation, epoch or external identity. Real configuration, revision and +readability changes retain their original fences. +PostgreSQL checks at most 32 admitted configuration candidates per lease turn. +A rejected source binding advances a separate durable probe rank so it cannot +permanently occupy that window. Rejection does not acquire a configuration lease +or change its frozen binding, pending acquisition, or external identity. +A held account lock instead advances a read-only sampling hint, because no +rank can be changed without that account lock. The hint belongs to one store +instance and one selected set of producer keys. Each pass examines at most 32 +candidates; an empty tail may wrap once to the beginning. Only a committed +turn updates the hint. A successful lease clears it and uses the durable +turn-primary order again, so an unlocked earlier account is revisited. A new +store begins at that same durable order; it may revisit one bounded window +before continuing. Concurrent workers can repeat a sample, but the account +lock, lease predicate and durable ranks still determine actual acquisition. +Hints never create work, acquire leases or alter frozen generation identities. + +Persisted PostgreSQL timestamps can omit trailing fractional zeros. Reporting +decoders accept the resulting aware precision and offset forms on Python 3.10 +without changing the captured bytes or represented microsecond instant. Naive, +invalid and excess-precision values remain refused. + +The shared public JSON Schema `date-time` checker has a different role: it +validates RFC 3339 wire strings without parsing them into Python timestamps. +It accepts arbitrary positive fractional widths as specified by +[RFC 3339 section 5.6](https://www.rfc-editor.org/rfc/rfc3339#section-5.6), including +the five-digit PostgreSQL form, on Python 3.10–3.13; validation preserves the exact input. +Its existing calendar, aware-offset, ASCII and seconds `00..59` requirements +remain enforced. In particular, the persisted decoder's six-digit bound and +historical seconds-bearing offsets are not imported into public validation. +This correction affects named schemas, task request/response validation and +the MCP/A2A validation paths that use them. It does not change a cached schema, +the separate #1179 conditional exception, or public-model work in #1190. +`test_schema_datetime_formats.py` checks the actual named/task schemas, +fractional precision, invalid inputs and `oneOf` selection; the configuration +mount tests check the unchanged raw timestamp through all three mounts. + +Opt-in revision ownership uses page-local +`ext.adcp.reporting_revision_ownership` version 1 bindings. Every returned revision +has exactly one owner and empty opted-in pages explicitly carry empty bindings. +The full bounded buyer walk checks ownership, dependencies and counts after +all pages arrive, with defaults of 2,048 pages and 200,000 records; +all-pages-absent remains conservative legacy mode. An exact +revision's binding alone cannot prove an otherwise unknown obligation. + +An in-flight B2.3 snapshot retains its original representation, ownership mode, +membership, order, counts and checkpoint after migration/activation/restart. +New snapshots may opt into the new representation. Authorization is checked on +every request: revocation can deny a continuation but cannot rebuild its history. + +For #1180 the four public task/notification fields are nullable with default +`None` in canonical and bundled generated models. Missing values stay absent in +standalone and nested serialization. Readiness requires Managed, receipts require +Reconciled, and ledger/status notifications are independently opt-in. The former +global reporting serializer mask is removed; generation owns the model contract. + +## Recovery and operational boundaries + +Close support with `await support.aclose()` before replacing components or +migrating. Preserve pending work, immutable journals, snapshots and exact receipt +batch responses. Restart with the same admitted contracts, complete migration and +let the owned bounded workers converge. Inspect typed closed failure codes; +provider bodies and credential contexts are not persistence or diagnostic data. +An unexpected owned worker failure latches the composition unready, wakes the +shared stop signal and emits one `ERROR` record on `adcp.reporting.production` +with code `REPORTING_PRODUCTION_WORKER_STOPPED` and a closed boundary label +(`producer`, `materializer`, `projection`, `sweeper` or `notifications`). Route +that logger to the operator's alert sink. Records contain no exception text, +traceback, provider body, request identity or ambient logging context. Expected +`ReportingNotificationError` outcomes and cancellation stop the owned loops +without this unexpected-failure signal. A late in-flight error after an already +requested stop does not create a second alert or turn cancellation into an +unexpected-failure signal. Existing notification guards still +check readiness before sampling and before dispatch; already-reserved work may +finish under its existing transaction and deadline rules. Drain with `aclose()`, +repair the failed component and construct fresh support to recover; the failed +instance never silently restarts or regains its capability claim. A failing log +sink cannot prevent the stop latch or change the safe public error. +The [receipt ingress](reporting-receipt-ingress.md), +[frozen feed](reporting-frozen-feed.md) and original materializer recovery +contracts continue to apply. + +Full buyer adjustment/submission automation and the `client.reporting` facade +remain later buyer work. They do not substitute for seller financial validation. +This slice remains open and unmerged pending independent exact-head review and +the separately gated downstream interoperability program. + +## Seller acceptance ownership + +The PR evidence index binds executed commands, counts, artifacts and tested +head/tree to every row below. An upstream implementation identity identifies +an input; it does not replace execution against the final B2.4 child. +The mounted MCP and A2A 0.3/1.0 tests use in-process ASGI. Separate real-process +and SIGKILL tests establish restart behavior; these do not establish live TCP +or the later cross-language interoperability gate. + +| Requirement | Implementation owner | Current integration coverage | +| --- | --- | --- | +| Durable materializer | B2.1; B2.4 admission | Reserve/verify/finish/ACK/enabled enqueue, faults, leases, uncertain-effect identities and permanent epoch-zero quarantine. | +| Canonical authenticated consumer | B2.2; B2.4 reads | MCP/A2A trusted resolver, authorization on replay, account collisions and feedback-off/on private reads. | +| Immutable receipt transaction | B2.2 | Mixed receipt, feed, captured status and ordinal commit/rollback in both stores and notification modes. | +| Exact batch replay and mount | B2.2 | Shape preflight, semantic outcomes, original order/timestamps and concurrent/crashed final-response replay. | +| Middleware/schema boundary | B2.2; B2.4 | Actual pinned/unpinned mounts, diagnostics, #1179 narrow schema correction and #1180 nested/public models. | +| Receipt financial graph | B2.2 writes; B2.4 projection | Selected official, accepted artifact stability, rejected-leaf replacement, accepted terminality and official adjustments. | +| Frozen authorized combined feed | B2.3; B2.4 activation | Actual historical page one, installed child activation/SIGKILL, exact remaining bytes and captured schedules. | +| Feed checkpoint/closure | B2.3; B2.4 ownership | Scope-bound compact tokens, full dependency closure/counts, deterministic walk and unchanged final checkpoint. | +| Versioned status capture | B2.4 | Original captured boundaries/baselines, old-writer fence, ordered reconciliation-only and reversible health transitions. | +| Tier-correct status | B2.4; #1179 | Strict financial evidence, immutable counts/retention and committed future schedules, including complete health and DST. | +| Private scope and buyer ownership | B2.3 inputs; B2.4 wire/walk | Exact page-local ownership, malformed/mixed/conflicting walks, cross-page dependencies and conservative legacy compatibility. | +| Production tier capabilities | B2.4; #1180 | Full provider/source contracts, live component/mount checks, empty-seller discovery, polling and signing/retry truthfulness. | +| Rolling compatibility | Every slice | Eleven distinct historical inputs, isolated manifests, populated/repeated/interrupted migration, installed floor runtimes and actual restarts. | + +## Integrated comparison and release-note limits + +The production rolling controls compare the integrated B2.3 artifact +`2d777ace7b4bf8be519ce0abd4fd0a25ed4f1da7` and integrated hardening artifact +`e16eb8cf3074cabd45aab42840950f05ad6d2b43`. Their frozen bytes, origins and +continuations must be established by fresh installed CI. Earlier snapshots are +not qualified by these comparisons. + +The release notes must retain all eight exclusions: pre-`17ee407a` A, +pre-`0f34c666` B, pre-`967b6e28` C, pre-`5487f2bd` B1, +pre-`3fd62121` B2.1, pre-`09fd87f7` B2.2, pre-`2d777ace` B2.3, +and pre-`e16eb8cf` hardening. In particular, old A whole-trigger startup after C +is not supported, and historical false notification-readiness results do not +become healthy through later source integration. These notes do not qualify +simultaneous old autonomous writers or release/activation acceptance. diff --git a/examples/reporting_production.py b/examples/reporting_production.py new file mode 100644 index 000000000..c5ea501a8 --- /dev/null +++ b/examples/reporting_production.py @@ -0,0 +1,121 @@ +"""One production seller composition, with actual provider and source contracts. + +The adopter owns trusted source/product mappings, provider grants, the account +task and token verification. The SDK owns admission, bounded discovery, the +materializer, captured status, receipts, exact reads and optional notification +delivery. Migrate and drain older workers before starting this composition. +""" + +from __future__ import annotations + +from adcp.decisioning.registry import BuyerAgentRegistry +from adcp.reporting.materializer import ( + ReportingDestinationIO, + ReportingMaterializerService, + ReportingRevisionVerifierRegistry, +) +from adcp.reporting.outbox import ( + ReportingEnvelopeCipher, + ReportingNotificationWorker, + ReportingSubscriptionResolver, +) +from adcp.reporting.production import ( + PgReportingProductionStore, + ReportingProductionConfigurationTask, + ReportingProductionDestination, + ReportingProductionOffering, + ReportingProductionSigning, + ReportingProductionSupport, + production_notification_workers, +) +from adcp.reporting.projection import PgReportingStatusProjection +from adcp.reporting.receipts import ReceiptAccountResolver +from adcp.server import serve +from adcp.server.auth import BearerTokenAuth, auth_context_factory + + +async def compose_reporting( + store: PgReportingProductionStore, + *, + destination: ReportingProductionDestination, + registry: ReportingRevisionVerifierRegistry, + offerings: tuple[ReportingProductionOffering, ...], + configuration_task: ReportingProductionConfigurationTask, + resolve_account: ReceiptAccountResolver, + buyer_agents: BuyerAgentRegistry | None = None, + consumer_status_enabled: bool = False, + subscriptions: ReportingSubscriptionResolver | None = None, + signing: ReportingProductionSigning | None = None, + cipher: ReportingEnvelopeCipher | None = None, +) -> ReportingProductionSupport: + """Prepare after draining old autonomous materializers/projectors/sweepers. + + Each offering names its actual producer, effective source offering and + installed verifier. Its source implements configuration_binding() with a + ReportingProductionSourceBinding.for_configuration(...) built from the + trusted account/catalog mapping. Product IDs are explicit; neither SDK nor + adopter may infer them from a report-definition ID. + + The destination implements configuration_binding() with the complete + ReportingProductionDestinationBinding resolved from its provider grant. + Credentials are acquired only inside separate write/readback sessions. + A changed method or source generation requires a new admitted identity. + + The account task calls its supplied admit(ReportingConfigurationAdmission) + for each ready/inactive reporting result, including replay. This completes + the account's activation before ready can be returned. Discovery needs no + first account, and workers do not require an adopter account inventory. + """ + await store.create_schema() + projection = PgReportingStatusProjection( + store, consumer_status_enabled=consumer_status_enabled, revision_ownership=True + ) + workers: tuple[ReportingNotificationWorker, ...] = () + if subscriptions is not None: + if cipher is None or signing is None: + raise ValueError( + "notification delivery requires an envelope cipher and signing contract" + ) + workers = production_notification_workers( + store, projection, subscriptions=subscriptions, signing=signing, cipher=cipher + ) + elif signing is not None or cipher is not None: + raise ValueError("notification delivery requires the subscription resolver") + return ReportingProductionSupport( + ReportingMaterializerService( + store, ReportingDestinationIO(registry, destination), destination + ), + projection, + offerings=offerings, + configuration_task=configuration_task, + resolve_account=resolve_account, + buyer_agents=buyer_agents, + notification_workers=workers, + ) + + +def serve_reporting( + support: ReportingProductionSupport, + *, + auth: BearerTokenAuth, + public_url: str, + allowed_hosts: tuple[str, ...], +) -> None: + """Mount both authenticated transports, then start and drain the SDK lifecycle. + + Configure store notifications explicitly. Polling needs no HTTP worker; + enabled notifications always retain atomic logical enqueue, even while + recipient delivery is stopped. Historical accounts may additionally be + activated by the operator with await support.activate(account_id=...). + """ + serve( + support.handler, + name="reporting-production", + transport="both", + auth=auth, + context_factory=auth_context_factory, + public_url=public_url, + allowed_hosts=allowed_hosts, + on_startup=[support.start], + on_shutdown=[support.aclose], + ) diff --git a/pyproject.toml b/pyproject.toml index 52793d9f9..0a37f50b8 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -199,6 +199,8 @@ adcp = [ "reporting/materializer/*.json", "reporting/receipts/*.json", "reporting/feed/*.json", + "reporting/projection/*.json", + "reporting/production/*.json", # PREVIEW: vendored sync_reporting_status schemas. They are the runtime # validator for the wire conditionals codegen cannot express, so the wheel # must carry them. Removed with the rest of _preview/ at rc.2. diff --git a/scripts/post_generate_fixes.py b/scripts/post_generate_fixes.py index 411c274f1..73d258113 100644 --- a/scripts/post_generate_fixes.py +++ b/scripts/post_generate_fixes.py @@ -2558,6 +2558,124 @@ def fix_unchanged_literal_defaults() -> None: print(" No unchanged field defaults needed fixing") +def fix_reporting_capability_defaults() -> None: + """Keep optional reporting promises absent in both generated model graphs. + + JSON Schema ``const`` restricts a supplied value; it does not advertise a + capability when the property is absent. Correct the annotations/defaults, + not just serialization (#1180). The scoped serializer also preserves + omission when an ordinary Pydantic parent does not set exclude_none. + """ + optional = { + "reliable_reporting_version", + "managed_delivery", + "reconciled_billing", + "configuration_task", + "status_task", + "consumer_status_task", + "revision_content_task", + "receipt_task", + "readiness_notification", + "status_notification", + "ledger_notification", + "supports_webhook_activity", + } + targets = ( + OUTPUT_DIR / "core/reporting_delivery_capabilities.py", + OUTPUT_DIR / "bundled/protocol/get_adcp_capabilities_response.py", + ) + for path in targets: + source = path.read_text() + lines = source.splitlines(keepends=True) + offsets = [0] + for line in lines: + offsets.append(offsets[-1] + len(line)) + changes: list[tuple[int, int, str]] = [] + classes = [ + node + for node in ast.parse(source).body + if isinstance(node, ast.ClassDef) + and re.fullmatch(r"ReportingDelivery(?:Capabilities)?\d*", node.name) + ] + if not classes: + raise ValueError(f"reporting capability model missing from {path.name}") + for node in classes: + for field in node.body: + if not ( + isinstance(field, ast.AnnAssign) + and isinstance(field.target, ast.Name) + and field.target.id in optional + and field.value is not None + ): + continue + annotation = field.annotation + if ( + isinstance(annotation, ast.Subscript) + and isinstance(annotation.value, ast.Name) + and annotation.value.id == "Annotated" + and isinstance(annotation.slice, ast.Tuple) + ): + annotation = annotation.slice.elts[0] + text = ast.get_source_segment(source, annotation) + assert text is not None + if not any( + isinstance(part, ast.Constant) and part.value is None + for part in ast.walk(annotation) + ): + changes.append( + ( + offsets[annotation.lineno - 1] + annotation.col_offset, + offsets[annotation.end_lineno - 1] + annotation.end_col_offset, + text + " | None", + ) + ) + default = field.value + changes.append( + ( + offsets[default.lineno - 1] + default.col_offset, + offsets[default.end_lineno - 1] + default.end_col_offset, + "None", + ) + ) + if not any( + isinstance(method, ast.FunctionDef) and method.name == "_validate_reporting_tiers" + for method in node.body + ): + methods = f""" + @model_validator(mode='after') + def _validate_reporting_tiers(self) -> {node.name}: + if self.reconciled_billing is True and self.managed_delivery is not True: + raise ValueError('reconciled_billing requires managed_delivery') + if self.readiness_notification is not None and self.managed_delivery is not True: + raise ValueError('readiness_notification requires managed_delivery') + if self.receipt_task is not None and self.reconciled_billing is not True: + raise ValueError('receipt_task requires reconciled_billing') + return self + + @model_serializer(mode='wrap') + def _omit_absent_reporting_promises( + self, handler: SerializerFunctionWrapHandler + ) -> dict[str, Any]: + return {{key: value for key, value in handler(self).items() if value is not None}} +""" + changes.append((offsets[node.end_lineno], offsets[node.end_lineno], methods)) + for start, end, text in sorted(changes, reverse=True): + source = source[:start] + text + source[end:] + imports = ( + "from typing import Any\n" + "from pydantic import SerializerFunctionWrapHandler, model_serializer, model_validator\n" + ) + if "from pydantic import SerializerFunctionWrapHandler," not in source: + source = source.replace( + "from __future__ import annotations\n", + "from __future__ import annotations\n\n" + imports, + 1, + ) + ast.parse(source) + path.write_text(source) + print(f" {path.relative_to(OUTPUT_DIR)}: optional reporting promises and tier validation") + + def fix_protocol_envelope_status_default() -> None: """Default response envelope status to completed for ergonomic construction. @@ -5915,6 +6033,7 @@ def main(argv: list[str] | None = None): widen_extension_point_lists_to_sequence, fix_canceled_literal_defaults, fix_unchanged_literal_defaults, + fix_reporting_capability_defaults, fix_protocol_envelope_status_default, fix_trusted_match_runtime_validators, fix_beta3_secure_url_constraints, diff --git a/src/adcp/reporting/_reconcile.py b/src/adcp/reporting/_reconcile.py index 3868b53bb..7cfc90c48 100644 --- a/src/adcp/reporting/_reconcile.py +++ b/src/adcp/reporting/_reconcile.py @@ -40,10 +40,13 @@ post_consumer_statuses, resolve_checkpointed_leaves, ) +from adcp.reporting.ownership import ReportingOwnershipError, page_revision_ownership from adcp.reporting.revision_selection import RevisionHistoryEntry, select_reporting_revision from adcp.types import ( GetReportingStatusRequest, GetReportingStatusResponse, + ReportingAdjustment, + ReportingAdjustmentReceipt, ReportingCanonicalContentDigest, ReportingControlTotal, ReportingDeliveryCapabilities, @@ -172,6 +175,11 @@ class ReportingLedger: #: from "new claim, must supersede", and re-filing the same claim under a #: new id churns the chain for no reason. consumer_statuses: list[Any] = field(default_factory=list) + # None is the all-pages-absent legacy mode. An empty mapping is an explicit + # new-mode empty snapshot; do not collapse those two meanings. + revision_ownership: dict[str, str] | None = None + adjustments: list[ReportingAdjustment] = field(default_factory=list) + adjustment_receipts: list[ReportingAdjustmentReceipt] = field(default_factory=list) @dataclass(frozen=True) @@ -296,9 +304,19 @@ async def load_reporting_ledger( request: GetReportingStatusRequest, *, max_snapshot_restarts: int = 2, + max_pages: int = 2048, + max_records: int = 200_000, ) -> ReportingLedger: """Exhaust a stable periods cursor and verify its declared record count.""" - + if ( + type(max_snapshot_restarts) is not int + or max_snapshot_restarts < 0 + or type(max_pages) is not int + or max_pages < 1 + or type(max_records) is not int + or max_records < 1 + ): + raise ValueError("reporting walk bounds must be positive (restarts may be zero)") base = request.model_dump(mode="json", exclude_none=True) base["view"] = "periods" base.pop("pagination", None) @@ -309,6 +327,11 @@ async def load_reporting_ledger( materializations: dict[str, ReportingMaterialization] = {} receipts: dict[str, ReportingReceipt] = {} consumer_statuses: dict[str, Any] = {} + adjustments: dict[str, ReportingAdjustment] = {} + adjustment_receipts: dict[str, ReportingAdjustmentReceipt] = {} + ownership: dict[str, str] = {} + ownership_mode: bool | None = None + frozen_metadata: str | None = None cursor: str | None = None seen_cursors: set[str] = set() snapshot_id: str | None = None @@ -317,7 +340,7 @@ async def load_reporting_ledger( scope: BaseModel | None = None total_count: int | None = None - while True: + for _page_number in range(max_pages): payload = dict(base) if cursor: payload["pagination"] = {"cursor": cursor} @@ -330,6 +353,38 @@ async def load_reporting_ledger( "STATUS_READ_FAILED", "get_reporting_status did not return a completed periods view", ) + raw_page = response.model_dump(mode="json", exclude_none=True) + if "ext" in response.model_fields_set and response.ext is None: + raw_page["ext"] = None + try: + local = page_revision_ownership(raw_page) + except ReportingOwnershipError: + raise ReportingReconciliationError( + "INVALID_REVISION_OWNERSHIP", "invalid page-local revision ownership" + ) from None + mode = local is not None + if ownership_mode is not None and mode != ownership_mode: + raise ReportingReconciliationError( + "INVALID_REVISION_OWNERSHIP", "mixed ownership modes within one snapshot" + ) + ownership_mode = mode + for revision_id, owner_id in (local or {}).items(): + if revision_id in ownership and ownership[revision_id] != owner_id: + raise ReportingReconciliationError( + "INVALID_REVISION_OWNERSHIP", "ownership changed within one snapshot" + ) + ownership[revision_id] = owner_id + metadata = _json( + { + k: raw_page.get(k) + for k in ("changes_checkpoint", "next_expected_at", "health", "issues") + } + ) + if mode and frozen_metadata is not None and metadata != frozen_metadata: + raise ReportingReconciliationError( + "SNAPSHOT_CHANGED", "frozen projection changed" + ) + frozen_metadata = metadata pagination = response.pagination if ( not response.ledger_snapshot_id @@ -359,6 +414,10 @@ async def load_reporting_ledger( account_id = response.account_id scope = response.scope total_count = pagination.total_count + if total_count is not None and total_count > max_records: + raise ReportingReconciliationError( + "LEDGER_LIMIT_EXCEEDED", "ledger record limit exceeded" + ) for obligation in response.periods or []: _add_immutable( obligations, @@ -377,6 +436,17 @@ async def load_reporting_ledger( ) for receipt in response.receipts or []: _add_immutable(receipts, receipt.reporting_receipt_id, receipt, "receipt") + for adjustment in response.adjustments or []: + _add_immutable( + adjustments, adjustment.reporting_adjustment_id, adjustment, "adjustment" + ) + for adjustment_receipt in response.adjustment_receipts or []: + _add_immutable( + adjustment_receipts, + adjustment_receipt.reporting_receipt_id, + adjustment_receipt, + "adjustment receipt", + ) for status in getattr(response, "consumer_statuses", None) or []: _add_immutable( consumer_statuses, @@ -385,14 +455,36 @@ async def load_reporting_ledger( "consumer status", ) + if ( + sum( + len(records) + for records in ( + obligations, + revisions, + materializations, + receipts, + consumer_statuses, + adjustments, + adjustment_receipts, + ) + ) + > max_records + ): + raise ReportingReconciliationError( + "LEDGER_LIMIT_EXCEEDED", "ledger record limit exceeded" + ) if not pagination.has_more: break cursor = pagination.cursor - if not cursor or cursor in seen_cursors: + if not cursor or len(cursor) > 2048 or cursor in seen_cursors: raise ReportingReconciliationError( "CURSOR_LOOP", "ledger pagination did not advance" ) seen_cursors.add(cursor) + else: + raise ReportingReconciliationError( + "LEDGER_LIMIT_EXCEEDED", "ledger page limit exceeded" + ) count = ( len(obligations) @@ -400,6 +492,8 @@ async def load_reporting_ledger( + len(materializations) + len(receipts) + len(consumer_statuses) + + len(adjustments) + + len(adjustment_receipts) ) if total_count is not None and total_count != count: raise ReportingReconciliationError( @@ -410,7 +504,7 @@ async def load_reporting_ledger( raise ReportingReconciliationError( "EMPTY_LEDGER_RESPONSE", "get_reporting_status returned no ledger page" ) - return ReportingLedger( + ledger = ReportingLedger( snapshot_id, ledger_as_of, account_id, @@ -420,13 +514,92 @@ async def load_reporting_ledger( list(materializations.values()), list(receipts.values()), list(consumer_statuses.values()), + ownership if ownership_mode else None, + list(adjustments.values()), + list(adjustment_receipts.values()), ) + if ownership_mode: + _validate_owned_ledger(ledger) + return ledger except ReportingReconciliationError as error: if error.code != "SNAPSHOT_CHANGED" or restart == max_snapshot_restarts: raise raise ReportingReconciliationError("SNAPSHOT_CHANGED", "ledger never stabilized") +def _validate_owned_ledger(ledger: ReportingLedger) -> None: + """Validate explicit ownership only after all bounded pages are present.""" + owners = {o.reporting_obligation_id: o for o in ledger.obligations} + revisions = {r.reporting_revision_id: r for r in ledger.revisions} + bindings = ledger.revision_ownership + + def invalid() -> None: + raise ReportingReconciliationError( + "INVALID_REVISION_OWNERSHIP", "incomplete or inconsistent ownership dependencies" + ) + + if bindings is None or set(bindings) != set(revisions): + invalid() + assert bindings is not None + if any(o.account_id != ledger.account_id for o in owners.values()): + invalid() + for revision_id, owner_id in bindings.items(): + if owner_id not in owners or not _revision_matches_obligation( + revisions[revision_id], owners[owner_id] + ): + invalid() + predecessor = revisions[revision_id].supersedes_reporting_revision_id + if predecessor is not None and bindings.get(predecessor) != owner_id: + invalid() + for owner in owners.values(): + if ( + sum(o == owner.reporting_obligation_id for o in bindings.values()) + != owner.revision_count + ): + invalid() + materials = {m.reporting_materialization_id: m for m in ledger.materializations} + owned_evidence: tuple[ReportingMaterialization | ReportingReceipt, ...] = ( + *ledger.materializations, + *ledger.receipts, + ) + for item in owned_evidence: + if bindings.get(item.reporting_revision_id) != item.reporting_obligation_id: + invalid() + for receipt in ledger.receipts: + material = materials.get(receipt.reporting_materialization_id) + if material is None or ( + material.reporting_revision_id != receipt.reporting_revision_id + or material.reporting_obligation_id != receipt.reporting_obligation_id + ): + invalid() + adjustments = {a.reporting_adjustment_id: a for a in ledger.adjustments} + for adjustment in adjustments.values(): + revision = revisions.get(adjustment.adjusts_reporting_revision_id) + if revision is None or _enum(revision.finality) != "official": + invalid() + for adjustment_receipt in ledger.adjustment_receipts: + target_adjustment = adjustments.get(adjustment_receipt.reporting_adjustment_id) + if ( + target_adjustment is None + or target_adjustment.adjusts_reporting_revision_id + != adjustment_receipt.adjusts_reporting_revision_id + ): + invalid() + + +def _owned_revisions( + obligation: ReportingObligation, ledger: ReportingLedger +) -> list[ReportingRevision]: + if ledger.revision_ownership is not None: + return [ + r + for r in ledger.revisions + if ledger.revision_ownership.get(r.reporting_revision_id) + == obligation.reporting_obligation_id + ] + return [r for r in ledger.revisions if _revision_matches_obligation(r, obligation)] + + def _select_current( obligation: ReportingObligation, ledger: ReportingLedger ) -> tuple[ReportingRevision | None, ReportingMaterialization | None, list[str]]: @@ -459,6 +632,18 @@ def _select_current( ) and item.reporting_revision_id not in owned_elsewhere ] + if ledger.revision_ownership is not None: + candidates = _owned_revisions(obligation, ledger) + elif any( + not any( + m.reporting_revision_id == item.reporting_revision_id for m in ledger.materializations + ) + and sum(_revision_matches_obligation(item, o) for o in ledger.obligations) > 1 + for item in candidates + ): + # Counts and equal semantic scopes are not an ownership declaration. + # An unmaterialized revision may still belong to either obligation. + reasons.append("AMBIGUOUS_REVISION_OWNERSHIP") receipts = [ item for item in ledger.receipts @@ -761,11 +946,20 @@ def evaluate_reporting_ledger( expected_periods: list[ExpectedReportingPeriod] | None = None, now: datetime | None = None, ) -> ReportingReconciliationResult: + if ledger.revision_ownership is not None: + _validate_owned_ledger(ledger) now = now or datetime.now(timezone.utc) outcomes: list[ObligationReconciliation] = [] unique_revisions: dict[str, ReportingRevision] = {} for obligation in ledger.obligations: revision, materialization, reasons = _select_current(obligation, ledger) + if obligation.adjustment_count or any( + a.adjusts_reporting_revision_id == getattr(revision, "reporting_revision_id", None) + for a in ledger.adjustments + ): + # Loading ownership/dependencies is additive. The separately owned + # buyer adjustment evidence/submission workflow remains required. + reasons.append("ADJUSTMENT_RECONCILIATION_REQUIRED") if _enum(obligation.health) != "complete": reasons.append(f"OBLIGATION_{_enum(obligation.health).upper()}") if ( @@ -909,11 +1103,7 @@ async def reconcile_reporting_core( definition=pinned_definition, revisions={item.reporting_revision_id: item for item in ledger.revisions}, obligation_revisions={ - obligation.reporting_obligation_id: [ - revision - for revision in ledger.revisions - if _revision_matches_obligation(revision, obligation) - ] + obligation.reporting_obligation_id: _owned_revisions(obligation, ledger) for obligation in ledger.obligations }, current_statuses=ledger.consumer_statuses, diff --git a/src/adcp/reporting/_timestamp.py b/src/adcp/reporting/_timestamp.py new file mode 100644 index 000000000..074577e8f --- /dev/null +++ b/src/adcp/reporting/_timestamp.py @@ -0,0 +1,43 @@ +"""Lossless aware timestamps for reporting boundaries on every supported Python.""" + +from __future__ import annotations + +import re +from datetime import datetime, timedelta, timezone + +from adcp.reporting.evidence import aware_utc + +_TIMESTAMP = re.compile( + r"(?P[0-9]{4}-[0-9]{2}-[0-9]{2}T[0-9]{2}:[0-9]{2}:[0-9]{2}" + r"(?:\.[0-9]{1,6})?)" + r"(?:Z|(?P[+-])(?P[01][0-9]|2[0-3]):(?P[0-5][0-9])" + r"(?::(?P[0-5][0-9])(?:\.(?P[0-9]{1,6}))?)?)" +) + + +def aware_timestamp(value: str) -> datetime: + """Decode the exact microsecond instant without changing captured wire bytes. + + PostgreSQL JSON omits trailing fractional zeros. Python 3.10's ISO parser + only accepts three or six fractional digits. Padding supplies equivalent + zeros; excess precision, naive values and invalid dates remain refused. + Historical PostgreSQL time zones can also carry an offset in seconds. + """ + matched = _TIMESTAMP.fullmatch(value) if type(value) is str else None + if matched is None: + raise ValueError("reporting timestamp requires an aware microsecond instant") + normalized = re.sub(r"\.([0-9]+)", lambda m: "." + m[1].ljust(6, "0"), matched["local"]) + try: + # datetime.fromisoformat also drops subsecond offsets when their + # whole-second part is zero. Construct the exact offset independently. + offset = timedelta( + hours=int(matched["hours"] or 0), + minutes=int(matched["minutes"] or 0), + seconds=int(matched["seconds"] or 0), + microseconds=int((matched["microseconds"] or "").ljust(6, "0")), + ) + if matched["sign"] == "-": + offset = -offset + return aware_utc(datetime.fromisoformat(normalized).replace(tzinfo=timezone(offset))) + except (ValueError, OverflowError): + raise ValueError("reporting timestamp requires an aware microsecond instant") from None diff --git a/src/adcp/reporting/feed/errors.py b/src/adcp/reporting/feed/errors.py index e00d1d694..052a9b0ff 100644 --- a/src/adcp/reporting/feed/errors.py +++ b/src/adcp/reporting/feed/errors.py @@ -7,6 +7,7 @@ FeedErrorCode = Literal[ "INVALID_REQUEST", "INVALID_CHECKPOINT", + "REPORTING_FEED_VERSION_MISMATCH", "UNAUTHORIZED", "REPORTING_FEED_SCHEMA_UNREADY", "REPORTING_FEED_HISTORY_CORRUPT", @@ -17,6 +18,10 @@ _MESSAGES: dict[FeedErrorCode, str] = { "INVALID_REQUEST": "supply a periods request with valid reporting filters and pagination", "INVALID_CHECKPOINT": "restart the reporting walk; this position is unavailable for this scope", + "REPORTING_FEED_VERSION_MISMATCH": ( + "continue with the original walk's reporting representation;" + " start a new walk without a cursor or checkpoint when changing protocol version" + ), "UNAUTHORIZED": "the reporting account or authenticated consumer is unavailable", "REPORTING_FEED_SCHEMA_UNREADY": "install and verify the isolated reporting feed schema", "REPORTING_FEED_HISTORY_CORRUPT": "retained reporting feed evidence requires operator repair", diff --git a/src/adcp/reporting/feed/memory.py b/src/adcp/reporting/feed/memory.py index 7ddfd641b..f0bad7eef 100644 --- a/src/adcp/reporting/feed/memory.py +++ b/src/adcp/reporting/feed/memory.py @@ -33,6 +33,9 @@ class InMemoryReportingFeedStore(InMemoryReportingReceiptStore): _reporting_feed_snapshots: dict[str, tuple[bytes, str, bytes]] + def _feed_projection_options(self, caller: ReportingDeliveryPrincipal) -> dict[str, Any]: + return {} + def _feed_snapshot( self, snapshot_id: str, caller: ReportingDeliveryPrincipal ) -> StoredFeedSnapshot | None: @@ -90,6 +93,7 @@ def _capture_feed( receipt_boundaries=tuple( b for b in getattr(self, "_receipt_boundaries", ()) if b.caller == caller ), + **self._feed_projection_options(caller), ) @storage_errors diff --git a/src/adcp/reporting/feed/pg.py b/src/adcp/reporting/feed/pg.py index 778a0b9e4..867246426 100644 --- a/src/adcp/reporting/feed/pg.py +++ b/src/adcp/reporting/feed/pg.py @@ -9,7 +9,7 @@ from collections.abc import Awaitable, Callable from dataclasses import dataclass from importlib.resources import files -from typing import Any +from typing import Any, Literal from adcp.reporting.canonical_json import canonical_json_utf8_v1 from adcp.reporting.feed._errors import storage_errors @@ -41,6 +41,9 @@ class _CapturedFeed: changes: tuple[ReportingReconciliationChange, ...] materializer: tuple[dict[str, Any], ...] receipts: tuple[dict[str, Any], ...] + representation_version: Literal[1, 2] = 1 + revision_ownership: bool = False + activated_consumer_status_enabled: bool | None = None @dataclass(frozen=True, repr=False) @@ -71,6 +74,9 @@ def _prepare_feed( decode_materializer_boundary(r) for r in captured.materializer ), receipt_boundaries=tuple(decode_receipt_boundary(r) for r in captured.receipts), + representation_version=captured.representation_version, + revision_ownership=captured.revision_ownership, + activated_consumer_status_enabled=captured.activated_consumer_status_enabled, ) stored = StoredFeedSnapshot(snapshot, secrets.token_bytes(32)) document = canonical_json_utf8_v1(snapshot.to_storage()) diff --git a/src/adcp/reporting/feed/projection.py b/src/adcp/reporting/feed/projection.py index ed56e9cb3..bb9468409 100644 --- a/src/adcp/reporting/feed/projection.py +++ b/src/adcp/reporting/feed/projection.py @@ -3,7 +3,7 @@ from __future__ import annotations from dataclasses import replace -from typing import Any, NoReturn, cast +from typing import Any, Literal, NoReturn, cast from uuid import uuid4 from pydantic import TypeAdapter @@ -39,6 +39,7 @@ ReportingRevisionReceiptRecord, ) from adcp.reporting.ledger.notification_models import ReportingStatusScope +from adcp.reporting.ledger.schedule import next_reporting_expectation from adcp.reporting.ledger.status import ( _adjustment_to_wire, _consumer_status_to_wire, @@ -83,6 +84,9 @@ def capture_feed( consumer_status_enabled: bool, materializer_boundaries: tuple[ReportingMaterializerBoundary, ...] = (), receipt_boundaries: tuple[ReportingReceiptBoundary, ...] = (), + representation_version: Literal[1, 2] = 1, + revision_ownership: bool = False, + activated_consumer_status_enabled: bool | None = None, ) -> ReportingFeedSnapshot: """Project detached histories captured under one account-lock boundary. @@ -93,7 +97,15 @@ def capture_feed( order. The two sequence spaces are never compared or collapsed with max(). Closure records retain their original sort keys even below changes_after. """ - if core.account_id != caller.account_id: + if ( + core.account_id != caller.account_id + or representation_version not in {1, 2} + or (revision_ownership and representation_version != 2) + or ( + activated_consumer_status_enabled is not None + and activated_consumer_status_enabled != consumer_status_enabled + ) + ): _corrupt() # Filter foreign statements before deriving maxima, projection or membership. # IDs must resolve unambiguously; scope metadata never supplies ownership. @@ -231,13 +243,20 @@ def capture_feed( projection = StatusProjectionInput( core, ReportingStatusScope( - caller.account_id, consumer_id=caller.consumer_id if consumer_status_enabled else None + caller.account_id, + consumer_id=( + caller.consumer_id + if (representation_version == 2 or consumer_status_enabled) + else None + ), ), delivery_config_ids=tuple(scoped["delivery_config_ids"] or ()), media_buy_ids=tuple(scoped["media_buy_ids"] or ()), feed_purposes=tuple(scoped["feed_purposes"] or ()), period_start=_parse(scoped["period_start"]), period_end=_parse(scoped["period_end"]), + reconciliation=records if representation_version == 2 else None, + consumer_status_enabled=consumer_status_enabled, ) scope_result = project_status_scope(projection) selected_owners = {p.obligation.reporting_obligation_id for p in scope_result.obligations} @@ -292,12 +311,22 @@ def capture_feed( issues=result.issues, statuses=projected_obligation.statuses, ) + if projected_obligation.reconciliation is not None: + wires[identity].update(projected_obligation.reconciliation.wire) elif kind == "revision": owner_id = record.reporting_obligation_id if owner_id not in owners: _corrupt() revision_id = record_id wires[identity] = _revision_to_wire(record, owners[owner_id]) + if ( + representation_version == 2 + and owners[owner_id].generation_key in bindings + and record.canonical_content_digest is not None + ): + wires[identity][ + "canonical_content_digest" + ] = record.canonical_content_digest.to_wire() if record.supersedes_reporting_revision_id is not None: predecessor = revisions.get(record.supersedes_reporting_revision_id) if predecessor is None or predecessor.reporting_obligation_id != owner_id: @@ -310,6 +339,10 @@ def capture_feed( _corrupt() owner_id = target.reporting_obligation_id wires[identity] = _adjustment_to_wire(record) + if representation_version == 2 and owners[owner_id].generation_key in bindings: + from adcp.reporting.ledger.delivery import adjustment_to_wire + + wires[identity] = adjustment_to_wire(record) elif kind == "consumer_status": owner_id, revision_id = record.reporting_obligation_id, record.reporting_revision_id if owner_id is None and revision_id is not None: @@ -465,8 +498,8 @@ def capture_feed( _corrupt() inputs = { "version": 1, - "projection_version": 1, - "ownership_mode": "absent", + "projection_version": representation_version, + "ownership_mode": "bindings" if revision_ownership else "absent", "consumer_status_enabled": consumer_status_enabled, "core": _CORE.dump_python(frozen_core, mode="json"), "reconciliation": [payload(r) for r in records], @@ -501,6 +534,41 @@ def capture_feed( "health": scope_result.health, "issues": [issue.to_wire() for issue in scope_result.issues], } + if representation_version == 2 and not ( + scope_result.health == "complete" and "adcp_version" in filters + ): + configurations = tuple( + c + for c in scope_result.configurations + if (not finalities or c.required_finality in finalities) + and ( + not healths + or project_status_scope( + replace( + projection, + scope=ReportingStatusScope( + c.account_id, c.generation_key, consumer_id=caller.consumer_id + ), + ) + ).health + in healths + ) + ) + obligations = tuple( + p.obligation + for p in scope_result.obligations + if (not healths or p.projection.health in healths) + and (not finalities or p.obligation.required_finality in finalities) + ) + next_expected = next_reporting_expectation( + configurations, + obligations, + as_of=core.as_of, + period_start=projection.period_start, + period_end=projection.period_end, + ) + if next_expected is not None: + common["next_expected_at"] = next_expected.isoformat().replace("+00:00", "Z") return ReportingFeedSnapshot( caller, "rpfs_" + uuid4().hex, @@ -514,4 +582,6 @@ def capture_feed( for i in sorted(selected, key=keys.__getitem__) ), canonical_json_utf8_v1(inputs), + representation_version, + "bindings" if revision_ownership else "absent", ) diff --git a/src/adcp/reporting/feed/request.py b/src/adcp/reporting/feed/request.py index 3aaaa197e..7a81c3529 100644 --- a/src/adcp/reporting/feed/request.py +++ b/src/adcp/reporting/feed/request.py @@ -12,6 +12,11 @@ from jsonschema import Draft7Validator, FormatChecker +from adcp._version import ( + is_adcp_version_at_least, + normalize_to_release_precision, + resolve_adcp_version, +) from adcp.reporting.feed.errors import ReportingFeedError from adcp.reporting.receipts.wire import _IDENTITY_FIELDS from adcp.validation.schema_loader import get_portable_schema @@ -151,6 +156,14 @@ def parse(cls, request: dict[str, Any]) -> FeedRequest: "reporting_revision_id", } } + version = normalize_to_release_precision( + request.get("adcp_version") or resolve_adcp_version(None) + ) + if is_adcp_version_at_least(version, "3.2-rc.6"): + # New snapshots bind their rendering contract. Integrated + # parents' unversioned v1 walks retain their captured bytes; + # StoredFeedSnapshot checks that narrow compatibility case. + filters["adcp_version"] = version for name in ( "delivery_config_ids", "media_buy_ids", diff --git a/src/adcp/reporting/feed/snapshot.py b/src/adcp/reporting/feed/snapshot.py index 90147eb58..91f4a3e3a 100644 --- a/src/adcp/reporting/feed/snapshot.py +++ b/src/adcp/reporting/feed/snapshot.py @@ -74,7 +74,7 @@ class ReportingFeedSnapshot: records: tuple[ReportingFeedRecord, ...] = field(repr=False) inputs_json: bytes = field(repr=False) representation_version: int = 1 - ownership_mode: Literal["absent"] = "absent" + ownership_mode: Literal["absent", "bindings"] = "absent" @property def total_count(self) -> int: @@ -121,8 +121,8 @@ def decode_snapshot(value: Any) -> ReportingFeedSnapshot: or type(value["version"]) is not int or value["version"] != 1 or type(value["representation_version"]) is not int - or value["representation_version"] != 1 - or value["ownership_mode"] != "absent" + or (value["representation_version"], value["ownership_mode"]) + not in {(1, "absent"), (2, "absent"), (2, "bindings")} or type(value["filters"]) is not str or type(json.loads(value["filters"])) is not dict ): @@ -165,9 +165,39 @@ def decode_snapshot(value: Any) -> ReportingFeedSnapshot: canonical_json_utf8_v1(value["common"]), records, canonical_json_utf8_v1(value["inputs"]), + value["representation_version"], + value["ownership_mode"], ) if result.to_storage() != value: raise ValueError + if result.representation_version == 2: + inputs = value["inputs"] + if ( + inputs["projection_version"] != 2 + or inputs["ownership_mode"] != result.ownership_mode + ): + raise ValueError + core = inputs["core"] + owners = {o["reporting_obligation_id"] for o in core["obligations"]} + revisions = core["revisions"] + expected = {r["reporting_revision_id"]: r["reporting_obligation_id"] for r in revisions} + if len(expected) != len(revisions) or not set(expected.values()).issubset(owners): + raise ValueError + bindings = inputs["revision_ownership"] + if ( + type(bindings) is not list + or len(bindings) != len(expected) + or any( + type(b) is not dict + or set(b) != {"reporting_revision_id", "reporting_obligation_id"} + for b in bindings + ) + or {b["reporting_revision_id"]: b["reporting_obligation_id"] for b in bindings} + != expected + ): + raise ValueError + if any(r.record_id not in expected for r in records if r.kind == "revision"): + raise ValueError except (ValueError, TypeError, KeyError, IndexError, RecursionError): result = None if result is None: @@ -208,7 +238,6 @@ def check( position = decoded[3] if ( snapshot.caller != caller - or snapshot.filters_json != request.filters_json or decoded[1] != kind or decoded[2] != snapshot.snapshot_id or type(position) is not int @@ -218,6 +247,30 @@ def check( or not hmac.compare_digest(token, self.token(kind, position)) ): raise ReportingFeedError("INVALID_CHECKPOINT") + if snapshot.filters_json != request.filters_json: + captured_filters = json.loads(snapshot.filters_json) + proposed_filters = request.filters + if ( + snapshot.representation_version == 1 + and snapshot.ownership_mode == "absent" + and "adcp_version" not in captured_filters + and proposed_filters.get("adcp_version") == "3.2-rc.6" + ): + # Integrated parents persisted unversioned v1 filters. Their + # immutable representation has no complete-summary forecast; + # the rc.6 marker must not invalidate those original walks. + # Caller, position and signature were verified above. Every + # previously bound semantic filter must still match exactly. + proposed_filters.pop("adcp_version") + if captured_filters != proposed_filters: + raise ReportingFeedError("INVALID_CHECKPOINT") + return position + # Only an authenticated, correctly signed position can disclose + # this actionable version boundary. Other callers/tokens retain + # the indistinguishable INVALID_CHECKPOINT error above. + if captured_filters.get("adcp_version") != proposed_filters.get("adcp_version"): + raise ReportingFeedError("REPORTING_FEED_VERSION_MISMATCH") + raise ReportingFeedError("INVALID_CHECKPOINT") return position def page(self, offset: int, limit: int) -> dict[str, Any]: @@ -239,6 +292,19 @@ def page(self, offset: int, limit: int) -> dict[str, Any]: **({"cursor": self.token("cursor", end)} if more else {}), }, ) + if snapshot.ownership_mode == "bindings": + from adcp.reporting.ownership import ReportingOwnershipError, with_revision_ownership + + try: + ownership: dict[str, str] = {} + for item in snapshot.inputs["revision_ownership"]: + revision, owner = item["reporting_revision_id"], item["reporting_obligation_id"] + if revision in ownership: + raise ReportingOwnershipError() + ownership[revision] = owner + result = with_revision_ownership(result, ownership) + except (KeyError, TypeError, ReportingOwnershipError): + raise ReportingFeedError("REPORTING_FEED_HISTORY_CORRUPT") from None return result diff --git a/src/adcp/reporting/ledger/models.py b/src/adcp/reporting/ledger/models.py index af00c5449..3e9588138 100644 --- a/src/adcp/reporting/ledger/models.py +++ b/src/adcp/reporting/ledger/models.py @@ -26,6 +26,7 @@ from dataclasses import dataclass, field from datetime import datetime, timedelta, timezone from typing import Any, Literal +from zoneinfo import ZoneInfo from adcp.reporting.currency import validate_currency, validate_currency_units from adcp.reporting.evidence import ( @@ -233,6 +234,33 @@ def __post_init__(self) -> None: raise ValueError("expected_at cannot precede the period end") +def _schedule_clock( + schedule: ReportingScheduleSpec, account_timezone: str +) -> tuple[ZoneInfo, timedelta, datetime]: + zone = ZoneInfo(schedule.timezone_name(account_timezone)) + duration = iso_duration_to_timedelta(schedule.period_duration) + if duration <= timedelta(0): + raise ValueError("period_duration must be positive") + anchor = ( + _utc(schedule.period_anchor) + if schedule.period_anchor is not None + else datetime(1970, 1, 1, tzinfo=timezone.utc) + ) + return zone, duration, anchor.astimezone(zone).replace(tzinfo=None) + + +def _period_instants( + schedule: ReportingScheduleSpec, account_timezone: str, ordinal: int +) -> tuple[datetime, datetime]: + zone, duration, anchor = _schedule_clock(schedule, account_timezone) + # Civil-time boundaries use the first occurrence of an ambiguous local + # time. A spring-forward gap can collapse a slot to zero elapsed time; + # the shared schedule iterator skips that slot, never inventing a report. + start = (anchor + duration * ordinal).replace(tzinfo=zone).astimezone(timezone.utc) + end = (anchor + duration * (ordinal + 1)).replace(tzinfo=zone).astimezone(timezone.utc) + return start, end + + def derive_period( schedule: ReportingScheduleSpec, *, @@ -253,27 +281,11 @@ def derive_period( begins at the next boundary -- a partial first period would be reported as complete and understate delivery. """ - from zoneinfo import ZoneInfo - zone_name = schedule.timezone_name(account_timezone) - zone = ZoneInfo(zone_name) - duration = iso_duration_to_timedelta(schedule.period_duration) - if duration <= timedelta(0): - raise ValueError("period_duration must be positive") sla = iso_duration_to_timedelta(schedule.delivery_sla) - - anchor = ( - _utc(schedule.period_anchor) - if schedule.period_anchor is not None - else datetime(1970, 1, 1, tzinfo=timezone.utc) - ) # Walk whole periods from the anchor in local wall-clock terms so a DST # transition shifts the instant without changing which period it is. - local_anchor = anchor.astimezone(zone).replace(tzinfo=None) - start_local = local_anchor + duration * ordinal - end_local = local_anchor + duration * (ordinal + 1) - start = start_local.replace(tzinfo=zone).astimezone(timezone.utc) - end = end_local.replace(tzinfo=zone).astimezone(timezone.utc) + start, end = _period_instants(schedule, account_timezone, ordinal) if activated_at is not None and _utc(activated_at) > start: raise ValueError( @@ -299,23 +311,18 @@ def first_ordinal_after( A configuration activated at 00:20 with hourly aligned periods owes ``[01:00, 02:00)`` first, not a 40-minute stub. """ - ordinal = 0 - # Seek coarsely then step back, so a long-lived configuration does not walk - # every period since the epoch one at a time. - step = 1 << 20 - while step: - candidate = derive_period( - schedule, account_timezone=account_timezone, ordinal=ordinal + step - ) - if _utc(candidate.start) <= _utc(activated_at): - ordinal += step - else: - step //= 2 - while True: - candidate = derive_period(schedule, account_timezone=account_timezone, ordinal=ordinal) - if _utc(candidate.start) >= _utc(activated_at): - return ordinal + zone, duration, anchor = _schedule_clock(schedule, account_timezone) + at = _utc(activated_at) + local = at.astimezone(zone).replace(tzinfo=None) + ordinal = (local - anchor) // duration + # Seek directly in civil time, then resolve timezone folds/gaps against + # instants. This also supports an explicit anchor after activation without + # scanning from the Unix epoch or constructing overflowing probe dates. + while _period_instants(schedule, account_timezone, ordinal)[0] < at: ordinal += 1 + while _period_instants(schedule, account_timezone, ordinal - 1)[0] >= at: + ordinal -= 1 + return ordinal @dataclass(frozen=True) diff --git a/src/adcp/reporting/ledger/producer.py b/src/adcp/reporting/ledger/producer.py index 2650df1e0..62134481e 100644 --- a/src/adcp/reporting/ledger/producer.py +++ b/src/adcp/reporting/ledger/producer.py @@ -35,7 +35,7 @@ from collections.abc import Awaitable, Callable, Mapping, Sequence from dataclasses import dataclass, field, replace from datetime import datetime, timedelta, timezone -from typing import Any, TypeAlias +from typing import TYPE_CHECKING, Any, TypeAlias from adcp.reporting.canonical_json import canonical_json_utf8_v1 from adcp.reporting.currency import ( @@ -50,7 +50,6 @@ ReportingObligationRecord, ReportingPeriodBoundary, ReportingRevisionRecord, - derive_period, iso_duration_to_timedelta, ) from adcp.reporting.ledger.store import ( @@ -80,6 +79,10 @@ parse_verified_source_batch_manifest_v1, ) +if TYPE_CHECKING: + from adcp.reporting.ledger.producer_progress import ReportingProducerProgress + from adcp.reporting.materializer.verification import ReportingRevisionVerifier + __all__ = [ "CurrencyResolver", "FixedCurrencyResolver", @@ -236,6 +239,7 @@ def __init__( max_periods_per_turn: int = 64, clock: Callable[[], datetime] | None = None, currency_resolver: CurrencyResolver | None = None, + revision_verifier: ReportingRevisionVerifier | None = None, ) -> None: self._source = source self._offerings = offerings @@ -251,6 +255,7 @@ def __init__( if currency_resolver is not None else FixedCurrencyResolver(offerings.currency) ) + self._revision_verifier = revision_verifier @property def store(self) -> ReportingLedgerStore: @@ -413,8 +418,12 @@ async def _close_elapsed_periods( *, now: datetime, ) -> list[ReportingObligationRecord]: + from adcp.reporting.ledger.producer_progress import ReportingProducerProgress + + progress = self._store if isinstance(self._store, ReportingProducerProgress) else None + after = None if progress is None else await progress.producer_closed_through(configuration) committed: list[ReportingObligationRecord] = [] - for boundary in self._elapsed_periods(configuration, now=now): + for boundary in self._elapsed_periods(configuration, now=now, after=after): existing = await self._store.find_obligation( account_id=configuration.account_id, delivery_config_id=configuration.delivery_config_id, @@ -423,6 +432,11 @@ async def _close_elapsed_periods( period_end=boundary.end, ) if existing is not None: + if progress is not None: + await progress.commit_producer_period( + configuration, existing, previous_end=after + ) + after = boundary.end continue obligation = ReportingObligationRecord( reporting_obligation_id=self._obligation_id(configuration, boundary), @@ -449,48 +463,51 @@ async def _close_elapsed_periods( resolved = self._currency_resolver(configuration, obligation) currency = await resolved if inspect.isawaitable(resolved) else resolved obligation = replace(obligation, currency=validate_currency(currency)) - stored = await self._store.commit_obligation(obligation) + stored = ( + await self._store.commit_obligation(obligation) + if progress is None + else await progress.commit_producer_period( + configuration, obligation, previous_end=after + ) + ) + after = boundary.end committed.append(stored) turn.obligations_committed.append(stored.reporting_obligation_id) return committed def _elapsed_periods( - self, configuration: ReportingConfiguration, *, now: datetime + self, + configuration: ReportingConfiguration, + *, + now: datetime, + after: datetime | None = None, ) -> list[ReportingPeriodBoundary]: """Every eligible period that has closed but is not yet obligated. - A period is eligible once its *end* is at or before now. A snapshot - taken exactly at the boundary does not expose it -- the obligation - appears in the first snapshot strictly after it, which is the rule both - sides derive independently. + A period is eligible once its *end* is at or before now. Activation + owes the first full period; deactivation after a period has started + retains that whole period and its original SLA. Polling forecasts use + the same committed-generation iterator. """ - from adcp.reporting.ledger.models import first_ordinal_after + from itertools import islice + + from adcp.reporting.ledger.schedule import committed_periods - activated_at = configuration.activated_at - if activated_at is None: - return [] - ordinal = first_ordinal_after( - configuration.schedule, - account_timezone=configuration.account_timezone, - activated_at=activated_at, - ) boundaries: list[ReportingPeriodBoundary] = [] - for _ in range(self._max_periods_per_turn): - boundary = derive_period( - configuration.schedule, - account_timezone=configuration.account_timezone, - ordinal=ordinal, - ) + near = ( + None + if after is None + else after + iso_duration_to_timedelta(configuration.schedule.delivery_sla) + ) + periods = ( + period + for period in committed_periods(configuration, near=near) + if after is None or period.end > after + ) + for boundary in islice(periods, self._max_periods_per_turn): if _utc(boundary.end) > _utc(now): break - if configuration.deactivated_at is not None and _utc(boundary.start) >= _utc( - configuration.deactivated_at - ): - # Deactivation still owes a period that already started, but - # not one that had not begun when the configuration stopped. - break boundaries.append(boundary) - ordinal += 1 return boundaries @staticmethod @@ -516,6 +533,11 @@ def _obligation_id( async def _acquire_pending( self, configuration: ReportingConfiguration, turn: WorkerTurn, *, now: datetime ) -> None: + from adcp.reporting.ledger.producer_progress import ReportingProducerProgress + + if isinstance(self._store, ReportingProducerProgress): + await self._acquire_progress(self._store, configuration, turn, now=now) + return for boundary in self._elapsed_periods(configuration, now=now): obligation = await self._store.find_obligation( account_id=configuration.account_id, @@ -552,6 +574,48 @@ async def _acquire_pending( turn.slices_failed.append(obligation.reporting_obligation_id) self._note_escalation(obligation, turn, now=now) + async def _acquire_progress( + self, + progress: ReportingProducerProgress, + configuration: ReportingConfiguration, + turn: WorkerTurn, + *, + now: datetime, + ) -> None: + identifiers = await progress.next_producer_obligations( + configuration, now=now, limit=self._max_periods_per_turn + ) + for identifier in identifiers: + obligation = await self._store.get_obligation( + account_id=configuration.account_id, reporting_obligation_id=identifier + ) + if obligation is None or obligation.generation_key != configuration.generation_key: + raise LedgerConflictError("HISTORY_UNAVAILABLE", "producer history is unavailable") + policy = self._settling_policy(configuration, obligation) + finished = policy is None + try: + if policy is None: + await self.acquire_obligation(configuration, obligation, turn=turn, now=now) + else: + finished = await self._acquire_with_settling_policy( + configuration, obligation, policy=policy, turn=turn, now=now + ) + except (ReportingCurrencyError, LedgerConflictError) as error: + if isinstance(error, LedgerConflictError) and error.code not in { + "HISTORY_UNAVAILABLE", + "EMPTY_DENOMINATOR", + }: + raise + turn.slices_failed.append(identifier) + self._note_escalation(obligation, turn, now=now) + # Retain the existing corrupt-history parking check. A transient + # currency failure during settling must not retire a readable leaf. + finished = policy is None or isinstance(error, LedgerConflictError) + if finished: + await progress.finish_producer_acquisition( + configuration, reporting_obligation_id=identifier + ) + def _settling_policy( self, configuration: ReportingConfiguration, @@ -597,14 +661,19 @@ async def _acquire_with_settling_policy( policy: _SettlingPolicy, turn: WorkerTurn, now: datetime, - ) -> None: + ) -> bool: + """Return whether policy-controlled acquisition may leave the pending queue. + + A readable snapshot completes one acquisition, not the settling policy. + The progress store retains its rotating work item until the policy ends. + """ checkpoint_store = self._restatement_store() revisions = await self._store.list_revisions( account_id=obligation.account_id, reporting_obligation_id=obligation.reporting_obligation_id, ) if any(item.finality == "official" for item in revisions): - return + return True if not revisions: await self.acquire_obligation( configuration, @@ -614,7 +683,7 @@ async def _acquire_with_settling_policy( target_finality="snapshot", track_settling=True, ) - return + return False checkpoint = await checkpoint_store.get_restatement_checkpoint( account_id=obligation.account_id, @@ -642,10 +711,10 @@ async def _acquire_with_settling_policy( target_finality="snapshot", track_settling=True, ) - return + return False if policy.official_close_lag is None or self._offerings.official_offering_id is None: - return + return True closes_at = max( settles_at, _utc(obligation.period.end) + policy.official_close_lag, @@ -660,6 +729,14 @@ async def _acquire_with_settling_policy( target_finality="official", track_settling=True, ) + # An attempted close can still return not-ready. Retire only after + # the authoritative revision was actually committed. + revisions = await self._store.list_revisions( + account_id=obligation.account_id, + reporting_obligation_id=obligation.reporting_obligation_id, + ) + return any(item.finality == "official" for item in revisions) + return False async def acquire_obligation( self, @@ -724,6 +801,14 @@ async def acquire_obligation( f"this producer declares no source offering for {finality} reporting", ) + from adcp.reporting.ledger.producer_progress import ReportingProducerProgress + + constituents = ( + await self._store.producer_constituents(configuration, obligation) + if isinstance(self._store, ReportingProducerProgress) + else None + ) + checkpoint_store = self._restatement_store() if track_settling else None checkpoint = ( await checkpoint_store.get_restatement_checkpoint( @@ -741,6 +826,7 @@ async def acquire_obligation( finality=finality, now=now, observation=observation, + constituents=constituents, ) cancel = asyncio.Event() try: @@ -967,6 +1053,10 @@ async def commit_revision_from_manifest( source_publication_id=manifest.publication_id, source_manifest_sha256=manifest.content_fingerprint.split(":", 1)[-1], ) + if self._revision_verifier is not None: + from adcp.reporting.materializer.publication import verified_publication + + revision = verified_publication(self._revision_verifier, obligation, revision, rows) committed = await self._store.commit_revision(revision, rows) turn.revisions_committed.append(committed.reporting_revision_id) return committed @@ -1069,6 +1159,7 @@ def _build_slice( finality: str | None = None, now: datetime, observation: int = 0, + constituents: tuple[ReportingConstituent, ...] | None = None, ) -> ReportingSourceSliceRequestV1: """Freeze one slice request from the obligation. @@ -1087,15 +1178,19 @@ def _build_slice( behavior depends on wall time. """ offering = self._source.capabilities.offering(offering_id) - constituents: list[ReportingConstituent] = [ - MediaBuyConstituentV1( - constituent_id=media_buy_id, - product_id=obligation.report_definition_id, - media_buy_id=media_buy_id, - ) - for media_buy_id in obligation.media_buy_ids - ] - if not constituents: + resolved_constituents: list[ReportingConstituent] = ( + list(constituents) + if constituents is not None + else [ + MediaBuyConstituentV1( + constituent_id=media_buy_id, + product_id=obligation.report_definition_id, + media_buy_id=media_buy_id, + ) + for media_buy_id in obligation.media_buy_ids + ] + ) + if not resolved_constituents: raise LedgerConflictError( "EMPTY_DENOMINATOR", "an obligation with no media buys has no source work; it is a platform-owned " @@ -1151,8 +1246,8 @@ def _build_slice( trigger="scheduled_poll", coverage=ReportingSourceCoverageRequestV1( expected="full", - constituents=constituents, - denominator_fingerprint=coverage_denominator_fingerprint_v1(constituents), + constituents=resolved_constituents, + denominator_fingerprint=coverage_denominator_fingerprint_v1(resolved_constituents), ), requested_metrics=list(self._offerings.requested_metrics), requested_dimensions=list(self._offerings.requested_dimensions), diff --git a/src/adcp/reporting/ledger/producer_progress.py b/src/adcp/reporting/ledger/producer_progress.py new file mode 100644 index 000000000..dfdd4715d --- /dev/null +++ b/src/adcp/reporting/ledger/producer_progress.py @@ -0,0 +1,93 @@ +"""Optional bounded producer progress; the original ledger protocol stays intact.""" + +from __future__ import annotations + +from collections.abc import Sequence +from datetime import datetime +from typing import Literal, Protocol, runtime_checkable + +from adcp.reporting.ledger.models import ( + ReportingConfiguration, + ReportingObligationRecord, + ReportingRevisionRecord, + iso_duration_to_timedelta, +) +from adcp.reporting.ledger.schedule import committed_periods +from adcp.reporting.ledger.store import LedgerConflictError +from adcp.reporting.revision_selection import select_reporting_revision +from adcp.reporting.source import ReportingConstituent + + +@runtime_checkable +class ReportingProducerProgress(Protocol): + """Durable, generation-scoped closing position and indexed unfinished work. + + Only the production participant installs this optional path. An acquired + configuration still uses the original lease/fairness and source execution + identities. Closing position, obligation and its work item co-commit. + Acquisition completion reselects immutable history under the account lock. + """ + + async def producer_closed_through( + self, configuration: ReportingConfiguration + ) -> datetime | None: ... + + async def producer_constituents( + self, configuration: ReportingConfiguration, obligation: ReportingObligationRecord + ) -> tuple[ReportingConstituent, ...]: ... + + async def commit_producer_period( + self, + configuration: ReportingConfiguration, + obligation: ReportingObligationRecord, + *, + previous_end: datetime | None, + ) -> ReportingObligationRecord: ... + + async def next_producer_obligations( + self, configuration: ReportingConfiguration, *, now: datetime, limit: int + ) -> tuple[str, ...]: ... + + async def finish_producer_acquisition( + self, configuration: ReportingConfiguration, *, reporting_obligation_id: str + ) -> None: ... + + +def check_next_period( + configuration: ReportingConfiguration, + obligation: ReportingObligationRecord, + previous_end: datetime | None, +) -> None: + near = ( + None + if previous_end is None + else previous_end + iso_duration_to_timedelta(configuration.schedule.delivery_sla) + ) + expected = next( + ( + p + for p in committed_periods(configuration, near=near) + if previous_end is None or p.end > previous_end + ), + None, + ) + if obligation.generation_key != configuration.generation_key or expected != obligation.period: + raise LedgerConflictError("HISTORY_UNAVAILABLE", "producer closing position is invalid") + + +def acquisition_state( + obligation: ReportingObligationRecord, revisions: Sequence[ReportingRevisionRecord] +) -> Literal["pending", "settled", "parked"]: + """The existing acquire-obligation predicates, without a new re-read policy.""" + selection = select_reporting_revision( + revisions, + account_id=obligation.account_id, + reporting_obligation_id=obligation.reporting_obligation_id, + required_finality=obligation.required_finality, + ) + if selection.kind == "corrupt": + return "parked" + current = selection.revision if selection.kind == "selected" else None + if current is not None and (current.finality == "official" or current.readable): + return "settled" + return "pending" diff --git a/src/adcp/reporting/ledger/reconciliation_projection.py b/src/adcp/reporting/ledger/reconciliation_projection.py new file mode 100644 index 000000000..c671a4913 --- /dev/null +++ b/src/adcp/reporting/ledger/reconciliation_projection.py @@ -0,0 +1,349 @@ +"""Pure tier evidence over one captured private record history. + +Selection precedes artifact/receipt inspection. Acceptance is validated against +its immutable admission prefix; later artifacts and checks cannot revoke it. +Current readability is a separate condition for delivery health. +""" + +from __future__ import annotations + +import hashlib +import json +from dataclasses import asdict, dataclass +from datetime import datetime +from typing import Any, Literal + +from adcp.reporting.canonical_json import canonical_json_utf8_v1 +from adcp.reporting.ledger._delivery_state import ( + _verify_materialization, + _verify_receipt, + adjustment_sha256, + current_receipt, + fingerprint, + iso, + principal, + record_identity, +) +from adcp.reporting.ledger.delivery import ReportingMaterializationView +from adcp.reporting.ledger.delivery_models import ( + ReportingAdjustmentReceiptRecord, + ReportingDeliveryPrincipal, + ReportingDeliveryRecord, + ReportingDestinationBinding, + ReportingMaterializationAttempt, + ReportingMaterializationCheck, + ReportingMaterializationRecord, + ReportingObligationDeliveryRecord, + ReportingRevisionReceiptRecord, +) +from adcp.reporting.ledger.health import issue_id_for +from adcp.reporting.ledger.models import ( + ReportingAdjustmentRecord, + ReportingIssue, + ReportingObligationRecord, + ReportingRevisionRecord, +) +from adcp.reporting.ledger.store import LedgerConflictError +from adcp.reporting.revision_selection import select_reporting_revision + + +@dataclass(frozen=True) +class ReconciliationProjection: + wire_json: bytes + evidence_json: bytes + issues: tuple[ReportingIssue, ...] = () + satisfied: bool = True + deadlines: tuple[datetime, ...] = () + + @property + def wire(self) -> dict[str, Any]: + return dict(json.loads(self.wire_json)) + + +def project_reconciliation( + obligation: ReportingObligationRecord, + revisions: tuple[ReportingRevisionRecord, ...], + adjustments: tuple[ReportingAdjustmentRecord, ...], + records: tuple[ReportingDeliveryRecord, ...], + *, + consumer_id: str | None, + as_of: datetime, +) -> ReconciliationProjection: + """All inputs belong to the captured boundary; no live store is consulted.""" + selection = select_reporting_revision( + revisions, + account_id=obligation.account_id, + reporting_obligation_id=obligation.reporting_obligation_id, + required_finality=obligation.required_finality, + ) + revision = selection.revision if selection.kind == "selected" else None + owned_ids = {r.reporting_revision_id for r in revisions} + owned_adjustments = tuple( + a for a in adjustments if a.adjusts_reporting_revision_id in owned_ids + ) + caller = ReportingDeliveryPrincipal(obligation.account_id, consumer_id) if consumer_id else None + history = tuple(r for r in records if caller is not None and principal(r) == caller) + scoped = tuple( + r + for r in history + if ( + r.generation_key == obligation.generation_key + if isinstance(r, ReportingDestinationBinding) + else r.scope.reporting_obligation_id == obligation.reporting_obligation_id + ) + ) + if len({record_identity(r) for r in scoped}) != len(scoped): + raise LedgerConflictError("HISTORY_UNAVAILABLE", "reconciliation history is inconsistent") + evidence: dict[str, Any] = { + "reporting_obligation_id": obligation.reporting_obligation_id, + "consumer_id": consumer_id, + "selected": revision.reporting_revision_id if revision else None, + "records": [fingerprint(r) for r in scoped], + "adjustments": [ + [ + a.reporting_adjustment_id, + hashlib.sha256( + json.dumps(asdict(a), default=iso, sort_keys=True).encode() + ).hexdigest(), + ] + for a in owned_adjustments + ], + } + wire: dict[str, Any] = {"adjustment_count": len(owned_adjustments)} + bindings = [r for r in scoped if isinstance(r, ReportingDestinationBinding)] + if len(bindings) > 1: + raise LedgerConflictError("HISTORY_UNAVAILABLE", "reconciliation binding is inconsistent") + if not bindings: + if scoped: + raise LedgerConflictError( + "HISTORY_UNAVAILABLE", "reconciliation binding is unavailable" + ) + return ReconciliationProjection( + canonical_json_utf8_v1(wire), canonical_json_utf8_v1(evidence) + ) + binding = bindings[0] + deliveries = [r for r in scoped if isinstance(r, ReportingObligationDeliveryRecord)] + if len(deliveries) > 1: + raise LedgerConflictError("HISTORY_UNAVAILABLE", "reconciliation delivery is inconsistent") + delivery = deliveries[0] if deliveries else None + attempts = { + r.reporting_materialization_id: r + for r in scoped + if isinstance(r, ReportingMaterializationAttempt) + } + outcomes = tuple(r for r in scoped if isinstance(r, ReportingMaterializationRecord)) + successes = tuple(r for r in outcomes if r.status in {"available", "delivered"}) + receipts = tuple(r for r in scoped if isinstance(r, ReportingRevisionReceiptRecord)) + adjustment_receipts = tuple( + r for r in scoped if isinstance(r, ReportingAdjustmentReceiptRecord) + ) + checks = tuple(r for r in scoped if isinstance(r, ReportingMaterializationCheck)) + by_revision = {r.reporting_revision_id: r for r in revisions} + by_adjustment = {a.reporting_adjustment_id: a for a in owned_adjustments} + readable: dict[str, bool] = {} + deadlines: set[datetime] = set() + for outcome in outcomes: + attempt, target = attempts.get(outcome.reporting_materialization_id), by_revision.get( + outcome.reporting_revision_id + ) + if ( + attempt is None + or target is None + or delivery is None + or attempt.scope != outcome.scope + or attempt.reporting_revision_id != outcome.reporting_revision_id + ): + raise LedgerConflictError( + "HISTORY_UNAVAILABLE", "materialization dependencies are unavailable" + ) + if outcome.status != "failed": + _verify_materialization(outcome, binding, delivery, target, obligation) + view = ReportingMaterializationView( + attempt, + binding, + outcome, + tuple( + c + for c in checks + if c.reporting_materialization_id == outcome.reporting_materialization_id + ), + ) + readable[outcome.reporting_materialization_id] = view.readable_at(as_of) and target.readable + if outcome.resource is not None: + deadlines.add(outcome.resource.expires_at) + leaves = {} + for index, record in enumerate(history): + if record not in (*receipts, *adjustment_receipts): + continue + if isinstance(record, ReportingRevisionReceiptRecord): + target = by_revision.get(record.reporting_revision_id) + if target is None: + raise LedgerConflictError("HISTORY_UNAVAILABLE", "receipt revision is unavailable") + _verify_receipt(record, history[: index + 1], target) + leaves[("revision", record.reporting_revision_id)] = current_receipt(history, record) + elif isinstance(record, ReportingAdjustmentReceiptRecord): + adjustment = by_adjustment.get(record.reporting_adjustment_id) + if ( + adjustment is None + or adjustment.adjusts_reporting_revision_id != record.adjusts_reporting_revision_id + or ( + record.status == "accepted" + and record.observed_adjustment_sha256 != adjustment_sha256(adjustment) + ) + ): + raise LedgerConflictError( + "HISTORY_UNAVAILABLE", "adjustment receipt is inconsistent" + ) + leaves[("adjustment", record.reporting_adjustment_id)] = current_receipt( + history, record + ) + selected_successes = tuple( + r + for r in successes + if revision is not None and r.reporting_revision_id == revision.reporting_revision_id + ) + current = max( + selected_successes, + key=lambda r: attempts[r.reporting_materialization_id].attempt, + default=None, + ) + artifact_readable = bool(current and readable[current.reporting_materialization_id]) + evidence["readable"] = readable + # Counts describe immutable successful outcomes, not today's health checks. + wire.update( + destination_ref=binding.destination_ref, + reconciliation_mode=binding.reconciliation_mode, + materialization_count=len(outcomes), + successful_materialization_count=len(successes), + ) + retained = [o.resource.expires_at for o in successes if o.resource is not None] + if retained: + wire["resource_retained_until"] = iso(min(retained)) + requires_receipt = binding.reconciliation_mode == "consumer_receipt" + selected_receipt = ( + leaves.get(("revision", revision.reporting_revision_id)) if revision else None + ) + accepted = bool(selected_receipt and selected_receipt.status == "accepted") + applicable = tuple( + a + for a in owned_adjustments + if revision is not None + and a.adjusts_reporting_revision_id == revision.reporting_revision_id + ) + pending_adjustments = tuple( + a + for a in applicable + if ( + (leaf := leaves.get(("adjustment", a.reporting_adjustment_id))) is None + or leaf.status != "accepted" + ) + ) + if requires_receipt: + wire.update( + receipt_count=len(receipts), + accepted_receipt_count=sum(r.status == "accepted" for r in receipts), + adjustment_receipt_count=len(adjustment_receipts), + accepted_adjustment_receipt_count=sum( + r.status == "accepted" for r in adjustment_receipts + ), + pending_adjustment_count=len(pending_adjustments), + ) + rejected = bool(selected_receipt and selected_receipt.status == "rejected") or any( + (leaf := leaves.get(("adjustment", a.reporting_adjustment_id))) is not None + and leaf.status == "rejected" + for a in applicable + ) + wire["reconciliation_status"] = ( + "rejected" + if rejected + else "accepted" if accepted and not pending_adjustments else "pending" + ) + else: + wire["reconciliation_status"] = "not_required" + issues: list[ReportingIssue] = [] + + def issue(code: str, *, buyer: bool = False, immediate: bool = False) -> None: + if not immediate and as_of < obligation.period.expected_at: + return + severity: Literal["delayed", "action_required"] = ( + "action_required" + if immediate or as_of >= obligation.automated_recovery_deadline_at + else "delayed" + ) + # An immutable evidence change differentiates a recurring artifact/receipt + # failure while the ordered checkpoint retains its monotonic generation. + occurrence = hashlib.sha256(canonical_json_utf8_v1(evidence)).hexdigest() + issues.append( + ReportingIssue( + issue_id_for( + "tier-status-v2", + obligation.account_id, + consumer_id, + obligation.reporting_obligation_id, + code, + occurrence, + ), + code, + severity, + "buyer" if buyer else "seller", + "contact_buyer" if buyer else "contact_seller", + reporting_obligation_id=obligation.reporting_obligation_id, + delivery_config_id=obligation.delivery_config_id, + delivery_config_version=obligation.delivery_config_version, + feed_purpose=obligation.feed_purpose, + media_buy_ids=obligation.media_buy_ids, + expected_at=obligation.period.expected_at, + ) + ) + + if revision is not None: + if not artifact_readable: + expired = ( + current is not None + and current.resource is not None + and current.resource.expires_at <= as_of + ) + issue( + "RESOURCE_EXPIRED" if expired else "DELIVERY_FAILED", immediate=current is not None + ) + if requires_receipt: + if revision.finality != "official" or revision.canonical_content_digest is None: + issue("HISTORY_UNAVAILABLE", immediate=True) + if not accepted: + issue( + "RECEIPT_REJECTED" if selected_receipt else "RECEIPT_REQUIRED", + buyer=True, + immediate=True, + ) + for adjustment in pending_adjustments: + leaf = leaves.get(("adjustment", adjustment.reporting_adjustment_id)) + issue( + "ADJUSTMENT_RECEIPT_REJECTED" if leaf else "ADJUSTMENT_RECEIPT_REQUIRED", + buyer=True, + immediate=True, + ) + satisfied = bool( + revision is not None + and artifact_readable + and ( + not requires_receipt + or ( + revision.finality == "official" + and revision.canonical_content_digest is not None + and accepted + and not pending_adjustments + ) + ) + ) + # This bound is independent of transient corruption/readability; never + # claim the provider's retention beyond the frozen generation commitment. + if "resource_retained_until" in wire: + assert delivery is not None + wire["resource_retained_until"] = iso(min(delivery.resource_retained_until, *retained)) + return ReconciliationProjection( + canonical_json_utf8_v1(wire), + canonical_json_utf8_v1(evidence), + tuple(issues), + satisfied, + tuple(sorted(deadlines)), + ) diff --git a/src/adcp/reporting/ledger/reporting_production.sql b/src/adcp/reporting/ledger/reporting_production.sql new file mode 100644 index 000000000..209363432 --- /dev/null +++ b/src/adcp/reporting/ledger/reporting_production.sql @@ -0,0 +1,669 @@ +-- B2.4 production admission. Earlier work/queues and their mandatory manifests +-- remain byte-for-byte unchanged. The SDK uses the same materializer state +-- machine with a closed connection adapter for these new participants. +DO $production$ +BEGIN + PERFORM pg_advisory_xact_lock(hashtext('adcp.reporting.schema'), hashtext(current_schema())); + CREATE TABLE IF NOT EXISTS reporting_production_delivery_windows ( + account_id TEXT COLLATE "C" NOT NULL, + idempotency_key TEXT COLLATE "C" NOT NULL, + queue TEXT NOT NULL CHECK (queue IN ('core','status','ready')), + body_sha256 TEXT NOT NULL CHECK (body_sha256 ~ '^[0-9a-f]{64}$'), + started_at TIMESTAMPTZ NOT NULL, + expires_at TIMESTAMPTZ NOT NULL, + PRIMARY KEY (account_id,idempotency_key), + CHECK (expires_at=started_at+interval '86400 seconds') + ); + IF NOT EXISTS (SELECT 1 FROM pg_trigger + WHERE tgrelid='reporting_production_delivery_windows'::regclass + AND tgname='reporting_production_delivery_window_immutable') THEN + CREATE TRIGGER reporting_production_delivery_window_immutable BEFORE UPDATE OR DELETE + ON reporting_production_delivery_windows FOR EACH ROW + EXECUTE FUNCTION reporting_receipt_ingestion_immutable(); + END IF; + CREATE TABLE IF NOT EXISTS reporting_production_accounts ( + account_id TEXT COLLATE "C" PRIMARY KEY REFERENCES reporting_projection_accounts, + admission_epoch BIGINT NOT NULL DEFAULT 2 CHECK (admission_epoch=2), + activated_at TIMESTAMPTZ NOT NULL DEFAULT clock_timestamp(), + policy JSONB NOT NULL CHECK (jsonb_typeof(policy)='object'), + CHECK (jsonb_typeof(policy->'verification_keys') IS NOT DISTINCT FROM 'array'), + CHECK (jsonb_array_length(policy->'verification_keys') > 0), + CHECK (jsonb_typeof(policy->'notifications_enabled') IS NOT DISTINCT FROM 'boolean') + ); + IF NOT EXISTS (SELECT 1 FROM pg_trigger + WHERE tgrelid='reporting_production_accounts'::regclass + AND tgname='reporting_production_activation_immutable') THEN + CREATE TRIGGER reporting_production_activation_immutable BEFORE UPDATE OR DELETE + ON reporting_production_accounts FOR EACH ROW + EXECUTE FUNCTION reporting_receipt_ingestion_immutable(); + END IF; + CREATE TABLE IF NOT EXISTS reporting_production_work ( + account_id TEXT COLLATE "C" NOT NULL REFERENCES reporting_production_accounts, + consumer_id TEXT COLLATE "C" NOT NULL, + delivery_config_id TEXT COLLATE "C" NOT NULL, + delivery_config_version BIGINT NOT NULL, + reporting_obligation_id TEXT COLLATE "C" NOT NULL, + reporting_revision_id TEXT COLLATE "C" NOT NULL, + reporting_materialization_id TEXT COLLATE "C" NOT NULL, + attempt_namespace TEXT COLLATE "C" NOT NULL DEFAULT 'materialization_attempt' + CHECK (attempt_namespace = 'materialization_attempt'), + generation BIGINT NOT NULL CHECK (generation > 0), + binding_sha256 TEXT COLLATE "C" NOT NULL CHECK (binding_sha256 ~ '^[0-9a-f]{64}$'), + verification_key_sha256 TEXT COLLATE "C" NOT NULL + CHECK (verification_key_sha256 ~ '^[0-9a-f]{64}$'), + external_id TEXT COLLATE "C" NOT NULL CHECK (external_id ~ '^rwm_[0-9a-f]{64}$'), + state TEXT COLLATE "C" NOT NULL DEFAULT 'pending' CHECK (state IN ('pending','acked')), + retry_allowed BOOLEAN NOT NULL DEFAULT FALSE, + reason TEXT COLLATE "C" NOT NULL DEFAULT 'ready' CHECK (reason IN ( + 'ready','verified','retry','inactive','revision_not_ready','revision_unreadable', + 'target_changed','history_corrupt','legacy_pending','legacy_terminal', + 'component_unavailable','binding_changed','effect_unknown','operator_required')), + created_at TIMESTAMPTZ NOT NULL DEFAULT clock_timestamp(), + acknowledged_at TIMESTAMPTZ, + completion_token UUID, + lease_token UUID, + lease_until TIMESTAMPTZ, + due_at TIMESTAMPTZ NOT NULL DEFAULT clock_timestamp(), + imported BOOLEAN NOT NULL DEFAULT FALSE, + notifications_enabled BOOLEAN NOT NULL, + -- Only new reservations enter this epoch; no historical work is copied. + admission_epoch BIGINT NOT NULL DEFAULT 2 CHECK (admission_epoch = 2), + PRIMARY KEY (account_id, consumer_id, reporting_materialization_id), + UNIQUE (account_id, consumer_id, external_id), + FOREIGN KEY (account_id, consumer_id, delivery_config_id, delivery_config_version, + reporting_obligation_id) REFERENCES reporting_materializer_candidates, + FOREIGN KEY (account_id, consumer_id, attempt_namespace, reporting_materialization_id) + REFERENCES reporting_reconciliation_records(account_id, consumer_id, namespace, record_id), + FOREIGN KEY (account_id, reporting_obligation_id, reporting_revision_id) + REFERENCES reporting_revisions(account_id, reporting_obligation_id, reporting_revision_id), + CHECK ((lease_token IS NULL) = (lease_until IS NULL)), + CHECK ((state = 'acked') = (acknowledged_at IS NOT NULL)), + CHECK ((state = 'acked') = (completion_token IS NOT NULL)), + CHECK (state <> 'acked' OR lease_token IS NULL), + CHECK (NOT retry_allowed OR state = 'acked') + ); + CREATE UNIQUE INDEX IF NOT EXISTS reporting_production_one_pending + ON reporting_production_work (account_id, consumer_id, delivery_config_id, + delivery_config_version, reporting_obligation_id) WHERE state = 'pending'; + CREATE INDEX IF NOT EXISTS reporting_production_work_due + ON reporting_production_work (account_id, due_at, reporting_materialization_id) + WHERE state = 'pending'; + + CREATE TABLE IF NOT EXISTS reporting_production_status_heads ( + account_id TEXT COLLATE "C" NOT NULL, + consumer_id TEXT COLLATE "C" NOT NULL, + max_sequence BIGINT NOT NULL CHECK (max_sequence > 0), + PRIMARY KEY (account_id, consumer_id) + ); + CREATE TABLE IF NOT EXISTS reporting_production_status_boundaries ( + account_id TEXT COLLATE "C" NOT NULL, + consumer_id TEXT COLLATE "C" NOT NULL, + sequence BIGINT NOT NULL CHECK (sequence > 0), + account_sequence BIGINT NOT NULL CHECK (account_sequence > 0), + reporting_materialization_id TEXT COLLATE "C" NOT NULL, + outcome_namespace TEXT COLLATE "C" NOT NULL DEFAULT 'materialization' + CHECK (outcome_namespace = 'materialization'), + as_of TIMESTAMPTZ NOT NULL, + input JSONB NOT NULL, + content_sha256 TEXT COLLATE "C" NOT NULL CHECK (content_sha256 ~ '^[0-9a-f]{64}$'), + PRIMARY KEY (account_id, consumer_id, sequence), + UNIQUE (account_id, consumer_id, reporting_materialization_id), + UNIQUE (account_id, account_sequence), + FOREIGN KEY (account_id, consumer_id, reporting_materialization_id) + REFERENCES reporting_production_work, + FOREIGN KEY (account_id, consumer_id, outcome_namespace, reporting_materialization_id) + REFERENCES reporting_reconciliation_records(account_id, consumer_id, namespace, record_id), + CHECK ((input->>'version')::integer IS NOT DISTINCT FROM 1), + CHECK ((input->>'account_id') IS NOT DISTINCT FROM account_id), + CHECK ((input->>'consumer_id') IS NOT DISTINCT FROM consumer_id), + CHECK ((input->>'reporting_materialization_id') IS NOT DISTINCT FROM reporting_materialization_id), + CHECK ((input->>'sequence')::bigint IS NOT DISTINCT FROM sequence), + CHECK ((input->>'account_sequence')::bigint IS NOT DISTINCT FROM account_sequence), + CHECK ((input->>'as_of')::timestamptz IS NOT DISTINCT FROM as_of), + CHECK (content_sha256 = reporting_payload_sha256(input)), + CHECK (input - ARRAY['version','account_id','consumer_id','reporting_materialization_id', + 'sequence','account_sequence','as_of','core','reconciliation'] = '{}'::jsonb) + ); + CREATE TABLE IF NOT EXISTS reporting_production_notification_events ( + account_id TEXT COLLATE "C" NOT NULL, + notification_id TEXT COLLATE "C" NOT NULL, + notification_type TEXT COLLATE "C" NOT NULL CHECK (notification_type='reporting.delivery_ready'), + cause_kind TEXT COLLATE "C" NOT NULL CHECK (cause_kind='materialization_ready'), + cause_id TEXT COLLATE "C" NOT NULL, + cause_generation BIGINT NOT NULL CHECK (cause_generation=1), + consumer_namespace TEXT COLLATE "C" NOT NULL CHECK (length(consumer_namespace)>0), + admission_epoch BIGINT NOT NULL DEFAULT 2 CHECK (admission_epoch = 2), + fired_at TIMESTAMPTZ NOT NULL, + snapshot JSONB NOT NULL, + reporting_materialization_id TEXT COLLATE "C" GENERATED ALWAYS AS + (snapshot #>> '{cause,reporting_materialization_id}') STORED, + PRIMARY KEY (account_id, consumer_namespace, notification_id), + UNIQUE (account_id, consumer_namespace, notification_type, cause_kind, cause_id, cause_generation), + UNIQUE (account_id, consumer_namespace, notification_id, cause_kind, cause_id, cause_generation), + FOREIGN KEY (account_id, consumer_namespace, reporting_materialization_id) + REFERENCES reporting_production_status_boundaries + (account_id, consumer_id, reporting_materialization_id), + CHECK ((snapshot->>'account_id') IS NOT DISTINCT FROM account_id), + CHECK ((snapshot->>'notification_id') IS NOT DISTINCT FROM notification_id), + CHECK (cause_id::jsonb IS NOT DISTINCT FROM + jsonb_build_array(consumer_namespace, reporting_materialization_id)), + CHECK ((snapshot #>> '{cause,consumer_id}') IS NOT DISTINCT FROM consumer_namespace) + ); + CREATE TABLE IF NOT EXISTS reporting_production_notification_expansions ( + account_id TEXT COLLATE "C" NOT NULL, + consumer_namespace TEXT COLLATE "C" NOT NULL, + notification_id TEXT COLLATE "C" NOT NULL, + emission_generation BIGINT NOT NULL CHECK (emission_generation>0), + state TEXT NOT NULL DEFAULT 'pending' CHECK + (state IN ('pending','leased','complete','suppressed','quarantined')), + due_at TIMESTAMPTZ NOT NULL, + lease_token TEXT, + lease_expires_at TIMESTAMPTZ, + claim_count BIGINT NOT NULL DEFAULT 0, + error_code TEXT CHECK (error_code IN ( + 'network','retryable_http','permanent_http','signing_unavailable', + 'permanent_scope','subscription_unavailable','subscription_changed', + 'invalid_configuration','invalid_payload','integrity_failure','lease_expired')), + PRIMARY KEY (account_id, consumer_namespace, notification_id, emission_generation), + FOREIGN KEY (account_id, consumer_namespace, notification_id) + REFERENCES reporting_production_notification_events + ); + CREATE INDEX IF NOT EXISTS reporting_production_notification_due + ON reporting_production_notification_expansions (account_id, due_at) + WHERE state IN ('pending','leased'); + + CREATE TABLE IF NOT EXISTS reporting_production_notification_deliveries ( + account_id TEXT COLLATE "C" NOT NULL, + delivery_id TEXT COLLATE "C" NOT NULL, + subscriber_id TEXT COLLATE "C" NOT NULL, + principal_id TEXT COLLATE "C" NOT NULL, + notification_id TEXT COLLATE "C" NOT NULL, + notification_type TEXT COLLATE "C" NOT NULL CHECK (notification_type = 'reporting.delivery_ready'), + emission_generation BIGINT NOT NULL CHECK (emission_generation > 0), + idempotency_key TEXT COLLATE "C" NOT NULL, + destination_sha256 TEXT NOT NULL, + subscription_fingerprint TEXT NOT NULL, + signing_scope_id TEXT COLLATE "C", + cause_kind TEXT COLLATE "C" NOT NULL CHECK (cause_kind = 'materialization_ready'), + cause_id TEXT COLLATE "C" NOT NULL, + cause_generation BIGINT NOT NULL CHECK (cause_generation > 0), + consumer_namespace TEXT COLLATE "C" NOT NULL, + auth_mode TEXT NOT NULL, + body_sha256 TEXT NOT NULL, + envelope_version INTEGER NOT NULL, + key_version TEXT COLLATE "C" NOT NULL, + envelope BYTEA NOT NULL, + state TEXT NOT NULL DEFAULT 'pending' CHECK + (state IN ('pending', 'leased', 'complete', 'suppressed', 'quarantined')), + due_at TIMESTAMPTZ NOT NULL, + lease_token TEXT, + lease_expires_at TIMESTAMPTZ, + claim_count BIGINT NOT NULL DEFAULT 0, + error_code TEXT CHECK (error_code IN ( + 'network', 'retryable_http', 'permanent_http', 'signing_unavailable', + 'permanent_scope', 'subscription_unavailable', 'subscription_changed', + 'invalid_configuration', 'invalid_payload', 'integrity_failure', 'lease_expired')), + PRIMARY KEY (account_id, consumer_namespace, delivery_id), + UNIQUE (account_id, consumer_namespace, notification_id, emission_generation, subscriber_id), + UNIQUE (account_id, consumer_namespace, idempotency_key), + CHECK (length(principal_id) > 0 AND (consumer_namespace = '' OR consumer_namespace = principal_id)), + FOREIGN KEY (account_id, consumer_namespace, notification_id, cause_kind, cause_id, cause_generation) + REFERENCES reporting_production_notification_events + (account_id, consumer_namespace, notification_id, cause_kind, cause_id, cause_generation), + FOREIGN KEY (account_id, consumer_namespace, notification_id, emission_generation) + REFERENCES reporting_production_notification_expansions + (account_id, consumer_namespace, notification_id, emission_generation) + ); + CREATE INDEX IF NOT EXISTS reporting_production_notification_deliveries_due + ON reporting_production_notification_deliveries (account_id, due_at) + WHERE state IN ('pending', 'leased'); + + + CREATE TABLE IF NOT EXISTS reporting_production_webhook_attempt_heads ( + account_id TEXT COLLATE "C" NOT NULL, + consumer_namespace TEXT COLLATE "C" NOT NULL, + principal_id TEXT COLLATE "C" NOT NULL CHECK (length(principal_id) > 0), + subscriber_id TEXT COLLATE "C" NOT NULL, + idempotency_key TEXT COLLATE "C" NOT NULL, + last_attempt BIGINT NOT NULL CHECK (last_attempt > 0), + PRIMARY KEY (account_id, consumer_namespace, principal_id, subscriber_id, idempotency_key) + ); + CREATE TABLE IF NOT EXISTS reporting_production_webhook_attempts ( + account_id TEXT COLLATE "C" NOT NULL, + principal_id TEXT COLLATE "C" NOT NULL, + subscriber_id TEXT COLLATE "C" NOT NULL, + notification_id TEXT COLLATE "C" NOT NULL, + idempotency_key TEXT COLLATE "C" NOT NULL, + attempt BIGINT NOT NULL CHECK (attempt > 0), + delivery_id TEXT COLLATE "C" NOT NULL, + consumer_namespace TEXT COLLATE "C" NOT NULL, + lease_token TEXT NOT NULL, + reservation_token TEXT NOT NULL, + binding JSONB NOT NULL, + fired_at TIMESTAMPTZ NOT NULL, + url TEXT NOT NULL CONSTRAINT reporting_production_webhook_url_safe + CHECK (length(url) <= 8192 AND url !~ '[?#@]' AND url ~ '^https?://'), + payload_size_bytes BIGINT NOT NULL CHECK (payload_size_bytes >= 0), + status TEXT NOT NULL DEFAULT 'pending' CHECK + (status IN ('pending', 'success', 'failed', 'timeout', 'connection_error')), + completed_at TIMESTAMPTZ, + http_status_code INTEGER, + response_time_ms BIGINT CHECK (response_time_ms >= 0), + PRIMARY KEY (account_id, consumer_namespace, principal_id, subscriber_id, idempotency_key, attempt), + UNIQUE (account_id, consumer_namespace, principal_id, delivery_id, lease_token), + FOREIGN KEY (account_id, consumer_namespace, delivery_id) + REFERENCES reporting_production_notification_deliveries, + FOREIGN KEY (account_id, consumer_namespace, principal_id, subscriber_id, idempotency_key) + REFERENCES reporting_production_webhook_attempt_heads + (account_id, consumer_namespace, principal_id, subscriber_id, idempotency_key), + CONSTRAINT reporting_production_webhook_identity CHECK ( + (binding->>'account_id') IS NOT DISTINCT FROM account_id AND + (binding->>'principal_id') IS NOT DISTINCT FROM principal_id AND + (binding->>'subscriber_id') IS NOT DISTINCT FROM subscriber_id AND + (binding->>'notification_id') IS NOT DISTINCT FROM notification_id AND + (binding->>'idempotency_key') IS NOT DISTINCT FROM idempotency_key AND + (binding->>'delivery_id') IS NOT DISTINCT FROM delivery_id AND + (binding->>'consumer_namespace') IS NOT DISTINCT FROM consumer_namespace), + CONSTRAINT reporting_production_webhook_completion CHECK ((status = 'pending') = (completed_at IS NULL)), + CONSTRAINT reporting_production_webhook_timestamps CHECK (completed_at >= fired_at), + CONSTRAINT reporting_production_webhook_outcome CHECK ( + (status IN ('pending', 'timeout', 'connection_error') + AND http_status_code IS NULL AND response_time_ms IS NULL) + OR (status IN ('success', 'failed') AND http_status_code BETWEEN 100 AND 599 + AND http_status_code IS NOT NULL AND response_time_ms IS NOT NULL + AND ((status = 'success') = (http_status_code BETWEEN 200 AND 299)))) + ); + CREATE INDEX IF NOT EXISTS reporting_production_webhook_activity_newest ON reporting_production_webhook_attempts + (account_id, principal_id, fired_at DESC, notification_id DESC, + idempotency_key DESC, subscriber_id DESC, attempt DESC, delivery_id DESC); + CREATE INDEX IF NOT EXISTS reporting_production_webhook_activity_retention ON reporting_production_webhook_attempts + (account_id, principal_id, completed_at) WHERE completed_at IS NOT NULL; + + -- Retention never resets a logical delivery's sequence, even after all + -- terminal attempts are purged. Writers always lock parent, then head. + CREATE OR REPLACE FUNCTION reporting_production_webhook_head_guard() + RETURNS TRIGGER LANGUAGE plpgsql AS $head_guard$ + BEGIN + IF TG_OP = 'DELETE' OR + (TG_OP = 'INSERT' AND NEW.last_attempt <> 1) OR + (TG_OP = 'UPDATE' AND (NEW.last_attempt <> OLD.last_attempt + 1 OR + (to_jsonb(NEW) - 'last_attempt') <> (to_jsonb(OLD) - 'last_attempt'))) THEN + RAISE EXCEPTION 'reporting activity counter must advance and be retained' + USING ERRCODE = '23514'; + END IF; + RETURN NEW; + END; + $head_guard$; + IF NOT EXISTS (SELECT 1 FROM pg_trigger WHERE tgrelid = 'reporting_production_webhook_attempt_heads'::regclass + AND tgname = 'reporting_production_webhook_head_guard' AND NOT tgisinternal) THEN + CREATE TRIGGER reporting_production_webhook_head_guard BEFORE INSERT OR UPDATE OR DELETE + ON reporting_production_webhook_attempt_heads FOR EACH ROW EXECUTE FUNCTION reporting_production_webhook_head_guard(); + END IF; + + CREATE OR REPLACE FUNCTION reporting_production_webhook_attempt_guard() + RETURNS TRIGGER LANGUAGE plpgsql AS $guard$ + DECLARE + delivery reporting_production_notification_deliveries; + BEGIN + IF TG_OP = 'DELETE' THEN + IF OLD.completed_at IS NULL THEN + RAISE EXCEPTION 'pending reporting activity must be retained' USING ERRCODE = '23514'; + END IF; + RETURN OLD; + END IF; + IF TG_OP = 'UPDATE' AND ( + OLD.status <> 'pending' OR NEW.status = 'pending' OR + (to_jsonb(NEW) - ARRAY['status','completed_at','http_status_code','response_time_ms']) <> + (to_jsonb(OLD) - ARRAY['status','completed_at','http_status_code','response_time_ms'])) THEN + RAISE EXCEPTION 'immutable reporting activity reservation' USING ERRCODE = '23514'; + END IF; + -- The opaque reservation token fences terminalization in the UPDATE + -- predicate. A known late response may finish its own reservation after + -- lease expiry/reclaim; it must not require or modify the parent lease. + IF TG_OP = 'UPDATE' THEN + RETURN NEW; + END IF; + IF TG_OP = 'INSERT' AND NEW.status <> 'pending' THEN + RAISE EXCEPTION 'reporting activity requires reservation' USING ERRCODE = '23514'; + END IF; + SELECT * INTO delivery FROM reporting_production_notification_deliveries + WHERE account_id = NEW.account_id AND principal_id = NEW.principal_id + AND consumer_namespace = NEW.consumer_namespace AND delivery_id = NEW.delivery_id + AND subscriber_id = NEW.subscriber_id AND notification_id = NEW.notification_id + AND idempotency_key = NEW.idempotency_key + AND state = 'leased' AND lease_token = NEW.lease_token + AND lease_expires_at > coalesce(NEW.completed_at, NEW.fired_at) FOR UPDATE; + IF NOT FOUND OR + NEW.binding->>'account_id' IS DISTINCT FROM NEW.account_id OR + NEW.binding->>'principal_id' IS DISTINCT FROM NEW.principal_id OR + NEW.binding->>'subscriber_id' IS DISTINCT FROM NEW.subscriber_id OR + NEW.binding->>'notification_id' IS DISTINCT FROM NEW.notification_id OR + NEW.binding->>'idempotency_key' IS DISTINCT FROM NEW.idempotency_key OR + NEW.binding->>'delivery_id' IS DISTINCT FROM NEW.delivery_id OR + NEW.binding->>'consumer_namespace' IS DISTINCT FROM NEW.consumer_namespace OR + NEW.binding->>'notification_type' IS DISTINCT FROM delivery.notification_type OR + NEW.binding->>'body_sha256' IS DISTINCT FROM delivery.body_sha256 THEN + RAISE EXCEPTION 'reporting activity lease or identity mismatch' USING ERRCODE = '23514'; + END IF; + RETURN NEW; + END; + $guard$; + IF NOT EXISTS (SELECT 1 FROM pg_trigger WHERE tgrelid = 'reporting_production_webhook_attempts'::regclass + AND tgname = 'reporting_production_webhook_attempt_guard' AND NOT tgisinternal) THEN + CREATE TRIGGER reporting_production_webhook_attempt_guard BEFORE INSERT OR UPDATE OR DELETE + ON reporting_production_webhook_attempts FOR EACH ROW EXECUTE FUNCTION reporting_production_webhook_attempt_guard(); + END IF; + + EXECUTE $function$ + CREATE OR REPLACE FUNCTION reporting_production_retained_guard() RETURNS trigger + LANGUAGE plpgsql AS $body$ + BEGIN + RAISE EXCEPTION 'reporting_materializer_evidence_immutable' USING ERRCODE='23514'; + END + $body$ $function$; + IF NOT EXISTS (SELECT 1 FROM pg_trigger WHERE tgrelid='reporting_production_status_boundaries'::regclass + AND tgname='reporting_production_boundary_immutable') THEN + CREATE TRIGGER reporting_production_boundary_immutable BEFORE UPDATE OR DELETE + ON reporting_production_status_boundaries FOR EACH ROW + EXECUTE FUNCTION reporting_production_retained_guard(); + END IF; + IF NOT EXISTS (SELECT 1 FROM pg_trigger WHERE tgrelid='reporting_production_notification_events'::regclass + AND tgname='reporting_production_event_immutable') THEN + CREATE TRIGGER reporting_production_event_immutable BEFORE UPDATE OR DELETE + ON reporting_production_notification_events FOR EACH ROW + EXECUTE FUNCTION reporting_production_retained_guard(); + END IF; + + EXECUTE $function$ + CREATE OR REPLACE FUNCTION reporting_production_work_guard() RETURNS trigger + LANGUAGE plpgsql AS $body$ + DECLARE a JSONB; result RECORD; activation RECORD; + BEGIN + IF TG_OP = 'DELETE' THEN + RAISE EXCEPTION 'reporting_materializer_history_immutable' USING ERRCODE='23514'; + END IF; + IF TG_OP = 'UPDATE' AND ( + (to_jsonb(NEW) - ARRAY['lease_token','lease_until','due_at','state','retry_allowed', + 'reason','acknowledged_at','completion_token']) IS DISTINCT FROM + (to_jsonb(OLD) - ARRAY['lease_token','lease_until','due_at','state','retry_allowed', + 'reason','acknowledged_at','completion_token']) OR OLD.state = 'acked' + ) THEN + RAISE EXCEPTION 'reporting_materializer_identity_immutable' USING ERRCODE='23514'; + END IF; + SELECT payload INTO a FROM reporting_reconciliation_records + WHERE account_id = NEW.account_id AND consumer_id = NEW.consumer_id + AND namespace = 'materialization_attempt' AND record_id = NEW.reporting_materialization_id; + IF a IS NULL OR a->>'reporting_revision_id' IS DISTINCT FROM NEW.reporting_revision_id + OR a #>> '{scope,reporting_obligation_id}' IS DISTINCT FROM NEW.reporting_obligation_id + OR a #>> '{scope,generation_key,delivery_config_id}' IS DISTINCT FROM NEW.delivery_config_id + OR (a #>> '{scope,generation_key,delivery_config_version}')::bigint + IS DISTINCT FROM NEW.delivery_config_version THEN + RAISE EXCEPTION 'reporting_materializer_attempt_mismatch' USING ERRCODE='23514'; + END IF; + SELECT activated_at, policy INTO activation FROM reporting_production_accounts + WHERE account_id=NEW.account_id; + IF NOT FOUND OR NEW.admission_epoch <> 2 OR NEW.imported + OR (a->>'created_at')::timestamptz < activation.activated_at + OR NOT (activation.policy->'verification_keys' ? NEW.verification_key_sha256) + OR (activation.policy->>'notifications_enabled')::boolean + IS DISTINCT FROM NEW.notifications_enabled THEN + RAISE EXCEPTION 'reporting_production_admission_required' USING ERRCODE='23514'; + END IF; + IF TG_OP='INSERT' AND EXISTS (SELECT 1 FROM reporting_materializer_work previous_work + WHERE previous_work.account_id=NEW.account_id AND previous_work.consumer_id=NEW.consumer_id + AND (previous_work.reporting_materialization_id=NEW.reporting_materialization_id OR + (previous_work.state='pending' AND previous_work.delivery_config_id=NEW.delivery_config_id + AND previous_work.delivery_config_version=NEW.delivery_config_version + AND previous_work.reporting_obligation_id=NEW.reporting_obligation_id))) THEN + RAISE EXCEPTION 'reporting_production_historical_work_conflict' USING ERRCODE='23514'; + END IF; + IF NEW.state = 'acked' THEN + IF TG_OP <> 'UPDATE' OR OLD.state <> 'pending' OR OLD.lease_token IS NULL + OR OLD.lease_until <= clock_timestamp() + OR NEW.completion_token IS DISTINCT FROM OLD.lease_token THEN + RAISE EXCEPTION 'reporting_materializer_fence_required' USING ERRCODE='23514'; + END IF; + SELECT payload INTO result FROM reporting_reconciliation_records + WHERE account_id = NEW.account_id AND consumer_id = NEW.consumer_id + AND namespace = 'materialization' AND record_id = NEW.reporting_materialization_id; + IF NOT FOUND OR (NEW.retry_allowed AND result.payload->>'status' <> 'failed') THEN + RAISE EXCEPTION 'reporting_materializer_outcome_required' USING ERRCODE='23514'; + END IF; + IF NOT EXISTS (SELECT 1 FROM reporting_production_status_boundaries b + WHERE b.account_id=NEW.account_id AND b.consumer_id=NEW.consumer_id + AND b.reporting_materialization_id=NEW.reporting_materialization_id + AND b.as_of=NEW.acknowledged_at + AND b.as_of=(result.payload->>'completed_at')::timestamptz) THEN + RAISE EXCEPTION 'reporting_materializer_capture_required' USING ERRCODE='23514'; + END IF; + IF NEW.notifications_enabled AND result.payload->>'status' <> 'failed' + AND NOT EXISTS (SELECT 1 FROM reporting_production_notification_events e + WHERE e.account_id=NEW.account_id AND e.consumer_namespace=NEW.consumer_id + AND e.reporting_materialization_id=NEW.reporting_materialization_id + AND e.admission_epoch=NEW.admission_epoch) THEN + RAISE EXCEPTION 'reporting_materializer_event_required' USING ERRCODE='23514'; + END IF; + END IF; + RETURN NEW; + END + $body$ $function$; + + IF NOT EXISTS (SELECT 1 FROM pg_trigger WHERE tgrelid='reporting_production_work'::regclass + AND tgname='reporting_production_guard') THEN + CREATE TRIGGER reporting_production_guard BEFORE INSERT OR UPDATE OR DELETE + ON reporting_production_work FOR EACH ROW + EXECUTE FUNCTION reporting_production_work_guard(); + END IF; + CREATE OR REPLACE FUNCTION reporting_production_old_reservation_guard() RETURNS trigger + LANGUAGE plpgsql AS $function$ + DECLARE activation TIMESTAMPTZ; attempt JSONB; + BEGIN + PERFORM pg_advisory_xact_lock(hashtext('adcp.reporting:' || NEW.account_id)); + SELECT activated_at INTO activation FROM reporting_production_accounts + WHERE account_id=NEW.account_id; + IF FOUND THEN + SELECT payload INTO attempt FROM reporting_reconciliation_records + WHERE account_id=NEW.account_id AND consumer_id=NEW.consumer_id + AND namespace='materialization_attempt' + AND record_id=NEW.reporting_materialization_id; + IF NOT NEW.imported OR attempt IS NULL + OR (attempt->>'created_at')::timestamptz >= activation THEN + RAISE EXCEPTION 'reporting_production_old_worker_fenced' USING ERRCODE='23514'; + END IF; + END IF; + RETURN NEW; + END + $function$; + IF NOT EXISTS (SELECT 1 FROM pg_trigger WHERE tgrelid='reporting_materializer_work'::regclass + AND tgname='reporting_production_old_reservation_guard') THEN + CREATE TRIGGER reporting_production_old_reservation_guard BEFORE INSERT + ON reporting_materializer_work FOR EACH ROW + EXECUTE FUNCTION reporting_production_old_reservation_guard(); + END IF; + CREATE TABLE IF NOT EXISTS reporting_production_generations ( + account_id TEXT COLLATE "C" NOT NULL, + delivery_config_id TEXT COLLATE "C" NOT NULL, + delivery_config_version INTEGER NOT NULL, + producer_key TEXT COLLATE "C" NOT NULL CHECK (producer_key ~ '^[0-9a-f]{64}$'), + source_binding JSONB NOT NULL CHECK (jsonb_typeof(source_binding)='object'), + PRIMARY KEY (account_id,delivery_config_id,delivery_config_version), + FOREIGN KEY (account_id,delivery_config_id,delivery_config_version) + REFERENCES reporting_configurations + ); + CREATE INDEX IF NOT EXISTS reporting_production_source_generations + ON reporting_production_generations(producer_key,account_id,delivery_config_id,delivery_config_version); + IF NOT EXISTS (SELECT 1 FROM pg_trigger WHERE tgrelid='reporting_production_generations'::regclass + AND tgname='reporting_production_generation_immutable') THEN + CREATE TRIGGER reporting_production_generation_immutable BEFORE UPDATE OR DELETE + ON reporting_production_generations FOR EACH ROW + EXECUTE FUNCTION reporting_receipt_ingestion_immutable(); + END IF; + CREATE TABLE IF NOT EXISTS reporting_production_destination_bindings ( + account_id TEXT COLLATE "C" NOT NULL, + consumer_id TEXT COLLATE "C" NOT NULL, + delivery_config_id TEXT COLLATE "C" NOT NULL, + delivery_config_version INTEGER NOT NULL, + method JSONB NOT NULL CHECK (jsonb_typeof(method)='object'), + PRIMARY KEY (account_id,consumer_id,delivery_config_id,delivery_config_version), + FOREIGN KEY (account_id,delivery_config_id,delivery_config_version) + REFERENCES reporting_production_generations + ); + IF NOT EXISTS (SELECT 1 FROM pg_trigger + WHERE tgrelid='reporting_production_destination_bindings'::regclass + AND tgname='reporting_production_destination_immutable') THEN + CREATE TRIGGER reporting_production_destination_immutable BEFORE UPDATE OR DELETE + ON reporting_production_destination_bindings FOR EACH ROW + EXECUTE FUNCTION reporting_receipt_ingestion_immutable(); + END IF; +END +$production$; + +-- Global due sampling for the owned optional workers; account claims retain +-- the original SDK outbox transactions and recipient ordering. +DO $production_delivery_discovery$ +BEGIN + PERFORM pg_advisory_xact_lock(hashtext('adcp.reporting.schema'), hashtext(current_schema())); + CREATE INDEX IF NOT EXISTS reporting_production_core_expansions_due + ON reporting_notification_expansions (due_at,account_id) WHERE state IN ('pending','leased'); + CREATE INDEX IF NOT EXISTS reporting_production_core_deliveries_due + ON reporting_notification_deliveries (due_at,account_id) WHERE state IN ('pending','leased'); + CREATE INDEX IF NOT EXISTS reporting_production_status_expansions_due + ON reporting_projection_notification_expansions (due_at,account_id) WHERE state IN ('pending','leased'); + CREATE INDEX IF NOT EXISTS reporting_production_status_deliveries_due + ON reporting_projection_notification_deliveries (due_at,account_id) WHERE state IN ('pending','leased'); + CREATE INDEX IF NOT EXISTS reporting_production_ready_expansions_due + ON reporting_production_notification_expansions (due_at,account_id) WHERE state IN ('pending','leased'); + CREATE INDEX IF NOT EXISTS reporting_production_ready_deliveries_due + ON reporting_production_notification_deliveries (due_at,account_id) WHERE state IN ('pending','leased'); +END +$production_delivery_discovery$; + +-- Generation-scoped producer closing and unfinished acquisition work. The +-- cursor never replaces original obligations, revision history or source keys. +DO $production_source_progress$ +BEGIN + PERFORM pg_advisory_xact_lock(hashtext('adcp.reporting.schema'), hashtext(current_schema())); + -- Rejected admitted generations advance through the bounded discovery + -- window without acquiring an ordinary lease or changing frozen bindings. + CREATE TABLE IF NOT EXISTS reporting_production_source_probe_turns ( + account_id TEXT COLLATE "C" NOT NULL, + delivery_config_id TEXT COLLATE "C" NOT NULL, + delivery_config_version INTEGER NOT NULL, + probe_turn BIGINT NOT NULL CHECK (probe_turn>0), + PRIMARY KEY(account_id,delivery_config_id,delivery_config_version), + FOREIGN KEY(account_id,delivery_config_id,delivery_config_version) + REFERENCES reporting_production_generations + ); + CREATE TABLE IF NOT EXISTS reporting_production_source_progress ( + account_id TEXT COLLATE "C" NOT NULL, + delivery_config_id TEXT COLLATE "C" NOT NULL, + delivery_config_version INTEGER NOT NULL, + closed_through TIMESTAMPTZ, + acquisition_turn BIGINT NOT NULL DEFAULT 0 CHECK (acquisition_turn>=0), + PRIMARY KEY(account_id,delivery_config_id,delivery_config_version), + FOREIGN KEY(account_id,delivery_config_id,delivery_config_version) + REFERENCES reporting_production_generations + ); + CREATE TABLE IF NOT EXISTS reporting_production_source_work ( + account_id TEXT COLLATE "C" NOT NULL, + delivery_config_id TEXT COLLATE "C" NOT NULL, + delivery_config_version INTEGER NOT NULL, + reporting_obligation_id TEXT COLLATE "C" NOT NULL REFERENCES reporting_obligations, + period_end TIMESTAMPTZ NOT NULL, + acquisition_turn BIGINT NOT NULL DEFAULT 0 CHECK (acquisition_turn>=0), + state TEXT COLLATE "C" NOT NULL DEFAULT 'pending' + CHECK (state IN ('pending','settled','parked')), + PRIMARY KEY(account_id,reporting_obligation_id), + FOREIGN KEY(account_id,delivery_config_id,delivery_config_version) + REFERENCES reporting_production_generations + ); + CREATE INDEX IF NOT EXISTS reporting_production_source_pending + ON reporting_production_source_work + (account_id,delivery_config_id,delivery_config_version,acquisition_turn,reporting_obligation_id) + INCLUDE(period_end) WHERE state='pending'; + + CREATE OR REPLACE FUNCTION reporting_production_source_progress_guard() RETURNS trigger + LANGUAGE plpgsql AS $function$ + BEGIN + IF TG_OP='DELETE' OR (TG_OP='UPDATE' AND ( + (to_jsonb(NEW)-ARRAY['closed_through','acquisition_turn']) IS DISTINCT FROM + (to_jsonb(OLD)-ARRAY['closed_through','acquisition_turn']) + OR NEW.acquisition_turn < OLD.acquisition_turn + OR (OLD.closed_through IS NOT NULL AND + (NEW.closed_through IS NULL OR NEW.closed_through < OLD.closed_through)))) THEN + RAISE EXCEPTION 'reporting_production_source_progress_immutable' USING ERRCODE='23514'; + END IF; + IF NEW.closed_through IS NOT NULL AND NOT EXISTS ( + SELECT 1 FROM reporting_obligations o WHERE o.account_id=NEW.account_id + AND o.delivery_config_id=NEW.delivery_config_id + AND o.delivery_config_version=NEW.delivery_config_version + AND o.period_end=NEW.closed_through + ) THEN + RAISE EXCEPTION 'reporting_production_source_obligation_required' USING ERRCODE='23514'; + END IF; + RETURN NEW; + END + $function$; + CREATE OR REPLACE FUNCTION reporting_production_source_work_guard() RETURNS trigger + LANGUAGE plpgsql AS $function$ + BEGIN + IF TG_OP='DELETE' OR (TG_OP='UPDATE' AND ( + (to_jsonb(NEW)-ARRAY['state','acquisition_turn']) IS DISTINCT FROM + (to_jsonb(OLD)-ARRAY['state','acquisition_turn']) + OR NEW.acquisition_turn < OLD.acquisition_turn)) THEN + RAISE EXCEPTION 'reporting_production_source_identity_immutable' USING ERRCODE='23514'; + END IF; + IF NOT EXISTS (SELECT 1 FROM reporting_obligations o + WHERE o.reporting_obligation_id=NEW.reporting_obligation_id + AND o.account_id=NEW.account_id AND o.delivery_config_id=NEW.delivery_config_id + AND o.delivery_config_version=NEW.delivery_config_version AND o.period_end=NEW.period_end) THEN + RAISE EXCEPTION 'reporting_production_source_obligation_required' USING ERRCODE='23514'; + END IF; + RETURN NEW; + END + $function$; + CREATE OR REPLACE FUNCTION reporting_production_source_dirty() RETURNS trigger + LANGUAGE plpgsql AS $function$ + BEGIN + -- SDK domain writers already hold the account lock before their row + -- mutation. New triggers retain that order and do no external I/O. + INSERT INTO reporting_production_source_work + (account_id,delivery_config_id,delivery_config_version,reporting_obligation_id,period_end) + SELECT o.account_id,o.delivery_config_id,o.delivery_config_version, + o.reporting_obligation_id,o.period_end FROM reporting_obligations o + JOIN reporting_production_generations g + USING(account_id,delivery_config_id,delivery_config_version) + WHERE o.reporting_obligation_id=NEW.reporting_obligation_id + AND o.account_id=NEW.account_id + ON CONFLICT(account_id,reporting_obligation_id) DO UPDATE SET state='pending'; + RETURN NEW; + END + $function$; + IF NOT EXISTS (SELECT 1 FROM pg_trigger + WHERE tgrelid='reporting_production_source_progress'::regclass + AND tgname='reporting_production_source_progress_guard') THEN + CREATE TRIGGER reporting_production_source_progress_guard BEFORE INSERT OR UPDATE OR DELETE + ON reporting_production_source_progress FOR EACH ROW + EXECUTE FUNCTION reporting_production_source_progress_guard(); + END IF; + IF NOT EXISTS (SELECT 1 FROM pg_trigger + WHERE tgrelid='reporting_production_source_work'::regclass + AND tgname='reporting_production_source_work_guard') THEN + CREATE TRIGGER reporting_production_source_work_guard BEFORE INSERT OR UPDATE OR DELETE + ON reporting_production_source_work FOR EACH ROW + EXECUTE FUNCTION reporting_production_source_work_guard(); + END IF; + IF NOT EXISTS (SELECT 1 FROM pg_trigger + WHERE tgrelid='reporting_obligations'::regclass + AND tgname='reporting_production_source_obligation') THEN + CREATE TRIGGER reporting_production_source_obligation AFTER INSERT ON reporting_obligations + FOR EACH ROW EXECUTE FUNCTION reporting_production_source_dirty(); + END IF; + IF NOT EXISTS (SELECT 1 FROM pg_trigger + WHERE tgrelid='reporting_revisions'::regclass + AND tgname='reporting_production_source_revision') THEN + CREATE TRIGGER reporting_production_source_revision AFTER INSERT OR UPDATE OF readable + ON reporting_revisions FOR EACH ROW EXECUTE FUNCTION reporting_production_source_dirty(); + END IF; +END +$production_source_progress$; diff --git a/src/adcp/reporting/ledger/reporting_projection.sql b/src/adcp/reporting/ledger/reporting_projection.sql new file mode 100644 index 000000000..04b4d0f86 --- /dev/null +++ b/src/adcp/reporting/ledger/reporting_projection.sql @@ -0,0 +1,242 @@ +-- B2.4 v2 capture and activation. All earlier manifests/SQL remain immutable. +-- create_schema() installs this inside the same serialized schema transaction. +DO $migration$ +DECLARE source_name TEXT; +BEGIN + PERFORM pg_advisory_xact_lock(hashtext('adcp.reporting.schema'), hashtext(current_schema())); + PERFORM account_id FROM reporting_status_scope_checkpoints LIMIT 0; + + CREATE TABLE IF NOT EXISTS reporting_projection_accounts ( + account_id TEXT COLLATE "C" PRIMARY KEY, + version INTEGER NOT NULL DEFAULT 2 CHECK (version = 2), + activated_at TIMESTAMPTZ NOT NULL, + notifications_enabled BOOLEAN NOT NULL, + consumer_status_enabled BOOLEAN NOT NULL, + ownership_enabled BOOLEAN NOT NULL, + policy JSONB NOT NULL, + legacy_through BIGINT NOT NULL CHECK (legacy_through >= 0), + legacy_capture_through BIGINT NOT NULL CHECK (legacy_capture_through >= 0), + legacy_capture_cursor BIGINT NOT NULL DEFAULT 0 + CHECK (legacy_capture_cursor BETWEEN 0 AND legacy_capture_through), + legacy_generation_floor BIGINT NOT NULL CHECK (legacy_generation_floor >= 0), + max_sequence BIGINT NOT NULL DEFAULT 0 CHECK (max_sequence >= 0), + cursor BIGINT NOT NULL DEFAULT 0 CHECK (cursor BETWEEN 0 AND max_sequence), + checkpoint_floor BIGINT NOT NULL DEFAULT 0 CHECK (checkpoint_floor >= 0), + current_input JSONB, + current_as_of TIMESTAMPTZ, + CHECK ((current_input IS NULL) = (current_as_of IS NULL)) + ); + CREATE INDEX IF NOT EXISTS reporting_projection_pending + ON reporting_projection_accounts (account_id) WHERE cursor < max_sequence; + CREATE INDEX IF NOT EXISTS reporting_projection_activation_pending + ON reporting_projection_accounts (account_id) WHERE current_input IS NULL; + CREATE TABLE IF NOT EXISTS reporting_projection_writes ( + account_id TEXT COLLATE "C" NOT NULL REFERENCES reporting_projection_accounts, + transaction_id TEXT COLLATE "C" NOT NULL, + PRIMARY KEY (account_id, transaction_id) + ); + CREATE TABLE IF NOT EXISTS reporting_projection_inputs ( + account_id TEXT COLLATE "C" NOT NULL REFERENCES reporting_projection_accounts, + sequence BIGINT NOT NULL CHECK (sequence > 0), + transaction_id TEXT COLLATE "C" NOT NULL, + as_of TIMESTAMPTZ NOT NULL, + input JSONB NOT NULL, + content_sha256 TEXT COLLATE "C" NOT NULL, + PRIMARY KEY (account_id, sequence), + UNIQUE (account_id, transaction_id), + CHECK (content_sha256 = reporting_receipt_ingestion_sha256(input)), + CHECK ((input->>'version')::integer IS NOT DISTINCT FROM 2), + CHECK ((input->>'account_id') IS NOT DISTINCT FROM account_id), + CHECK ((input->>'as_of')::timestamptz IS NOT DISTINCT FROM as_of) + ); + IF NOT EXISTS (SELECT 1 FROM pg_trigger WHERE tgrelid='reporting_projection_inputs'::regclass + AND tgname='reporting_projection_input_immutable') THEN + CREATE TRIGGER reporting_projection_input_immutable BEFORE UPDATE OR DELETE + ON reporting_projection_inputs FOR EACH ROW + EXECUTE FUNCTION reporting_receipt_ingestion_immutable(); + END IF; + CREATE TABLE IF NOT EXISTS reporting_projection_legacy_baselines ( + account_id TEXT COLLATE "C" NOT NULL REFERENCES reporting_projection_accounts, + scope_key TEXT COLLATE "C" NOT NULL, + checkpoint JSONB NOT NULL, + content_sha256 TEXT COLLATE "C" NOT NULL, + PRIMARY KEY(account_id,scope_key), + CHECK (content_sha256=reporting_receipt_ingestion_sha256(checkpoint)) + ); + CREATE TABLE IF NOT EXISTS reporting_projection_legacy_inputs ( + account_id TEXT COLLATE "C" NOT NULL REFERENCES reporting_projection_accounts, + account_sequence BIGINT NOT NULL CHECK (account_sequence>0), + consumer_id TEXT COLLATE "C" NOT NULL, + kind TEXT COLLATE "C" NOT NULL CHECK (kind IN ('materializer','receipt')), + input JSONB NOT NULL, + content_sha256 TEXT COLLATE "C" NOT NULL, + PRIMARY KEY(account_id,account_sequence), + CHECK (content_sha256=reporting_receipt_ingestion_sha256(input)), + CHECK (input->>'account_id' IS NOT DISTINCT FROM account_id), + CHECK (input->>'consumer_id' IS NOT DISTINCT FROM consumer_id), + CHECK ((input->>'account_sequence')::bigint IS NOT DISTINCT FROM account_sequence) + ); + CREATE TABLE IF NOT EXISTS reporting_projection_legacy_steps ( + account_id TEXT COLLATE "C" NOT NULL, + account_sequence BIGINT NOT NULL, + scope_key TEXT COLLATE "C" NOT NULL, + input JSONB NOT NULL, + content_sha256 TEXT COLLATE "C" NOT NULL, + PRIMARY KEY(account_id,account_sequence,scope_key), + FOREIGN KEY(account_id,account_sequence) + REFERENCES reporting_projection_legacy_inputs, + CHECK (content_sha256=reporting_receipt_ingestion_sha256(input)), + CHECK ((input->>'admission_epoch')::integer IS NOT DISTINCT FROM 0) + ); + CREATE TABLE IF NOT EXISTS reporting_projection_legacy_checkpoints ( + account_id TEXT COLLATE "C" NOT NULL REFERENCES reporting_projection_accounts, + consumer_id TEXT COLLATE "C" NOT NULL, + scope_key TEXT COLLATE "C" NOT NULL, + checkpoint JSONB NOT NULL, + content_sha256 TEXT COLLATE "C" NOT NULL, + CHECK (content_sha256=reporting_receipt_ingestion_sha256(checkpoint)), + PRIMARY KEY(account_id,scope_key) + ); + CREATE INDEX IF NOT EXISTS reporting_projection_legacy_consumer + ON reporting_projection_legacy_checkpoints(account_id,consumer_id,scope_key); + FOREACH source_name IN ARRAY ARRAY['reporting_projection_legacy_baselines', + 'reporting_projection_legacy_inputs','reporting_projection_legacy_steps'] LOOP + IF NOT EXISTS (SELECT 1 FROM pg_trigger WHERE tgrelid=source_name::regclass + AND tgname='reporting_projection_legacy_immutable') THEN + EXECUTE format('CREATE TRIGGER reporting_projection_legacy_immutable' + ' BEFORE UPDATE OR DELETE ON %I FOR EACH ROW' + ' EXECUTE FUNCTION reporting_receipt_ingestion_immutable()',source_name); + END IF; + END LOOP; +END +$migration$; + +DO $migration$ +DECLARE source_name TEXT; +BEGIN + PERFORM pg_advisory_xact_lock(hashtext('adcp.reporting.schema'), hashtext(current_schema())); + ALTER TABLE reporting_status_scope_checkpoints + ADD COLUMN IF NOT EXISTS projection_writer_floor INTEGER NOT NULL DEFAULT 1; + CREATE INDEX IF NOT EXISTS reporting_projection_due_clock + ON reporting_status_scope_checkpoints (next_due_at,account_id) + WHERE projection_writer_floor=2 AND next_due_at IS NOT NULL; + IF NOT EXISTS (SELECT 1 FROM pg_constraint + WHERE conrelid='reporting_status_scope_checkpoints'::regclass + AND conname='reporting_projection_writer_floor') THEN + ALTER TABLE reporting_status_scope_checkpoints + ADD CONSTRAINT reporting_projection_writer_floor CHECK (projection_writer_floor IN (1,2)); + END IF; + + CREATE OR REPLACE FUNCTION reporting_projection_checkpoint_guard() RETURNS trigger + LANGUAGE plpgsql AS $function$ + DECLARE floor INTEGER; + BEGIN + floor := CASE WHEN TG_OP='INSERT' THEN NEW.projection_writer_floor + ELSE OLD.projection_writer_floor END; + IF floor=2 OR EXISTS (SELECT 1 FROM reporting_projection_accounts + WHERE account_id=NEW.account_id) THEN + IF current_setting('adcp.reporting.projection_version',true) IS DISTINCT FROM '2' THEN + RAISE EXCEPTION 'status_projection_writer_fenced' USING ERRCODE='23514'; + END IF; + NEW.projection_writer_floor := 2; + END IF; + RETURN NEW; + END + $function$; + IF NOT EXISTS (SELECT 1 FROM pg_trigger + WHERE tgrelid='reporting_status_scope_checkpoints'::regclass + AND tgname='reporting_projection_checkpoint_guard') THEN + CREATE TRIGGER reporting_projection_checkpoint_guard BEFORE INSERT OR UPDATE + ON reporting_status_scope_checkpoints FOR EACH ROW + EXECUTE FUNCTION reporting_projection_checkpoint_guard(); + END IF; + + CREATE OR REPLACE FUNCTION reporting_projection_document(owner TEXT, at_time TIMESTAMPTZ) + RETURNS JSONB LANGUAGE plpgsql STABLE AS $function$ + DECLARE core JSONB; records JSONB; counts JSONB := '{}'; name TEXT; + BEGIN + core := reporting_status_projection_input(owner, at_time); + SELECT coalesce(jsonb_agg(jsonb_build_object( + 'consumer_id', c.consumer_id, 'sequence', c.seq, 'record', r.payload) + ORDER BY c.consumer_id COLLATE "C", c.seq), '[]') INTO records + FROM reporting_reconciliation_changes c JOIN reporting_reconciliation_records r + USING(account_id,consumer_id,namespace,record_id) + WHERE c.account_id=owner; + FOREACH name IN ARRAY ARRAY['configurations','obligations','revisions','statuses', + 'lifecycles','issue_scopes','adjustments','changes'] LOOP + counts := counts || jsonb_build_object(name, jsonb_array_length(core->name)); + END LOOP; + RETURN jsonb_build_object('version',2,'account_id',owner,'as_of',at_time, + 'core_format','sql-v1','core',core,'reconciliation',records, + 'counts',counts || jsonb_build_object('reconciliation',jsonb_array_length(records))); + END + $function$; + + CREATE OR REPLACE FUNCTION reporting_projection_capture(owner TEXT) RETURNS BIGINT + LANGUAGE plpgsql AS $function$ + DECLARE sequence BIGINT; document JSONB; at_time TIMESTAMPTZ; existing BIGINT; + BEGIN + PERFORM pg_advisory_xact_lock(hashtext('adcp.reporting:' || owner)); + SELECT i.sequence INTO existing FROM reporting_projection_inputs i + WHERE account_id=owner AND transaction_id=pg_current_xact_id()::text; + IF FOUND THEN RETURN existing; END IF; + at_time := nullif(current_setting('adcp.reporting.projection_clock',true),'')::timestamptz; + at_time := coalesce(at_time,clock_timestamp()); + UPDATE reporting_projection_accounts SET max_sequence=max_sequence+1 + WHERE account_id=owner RETURNING max_sequence INTO sequence; + IF NOT FOUND THEN RAISE EXCEPTION 'status_projection_activation_required' USING ERRCODE='23514'; END IF; + document := reporting_projection_document(owner,at_time); + INSERT INTO reporting_projection_inputs + (account_id,sequence,transaction_id,as_of,input,content_sha256) + VALUES(owner,sequence,pg_current_xact_id()::text,at_time,document, + reporting_receipt_ingestion_sha256(document)); + RETURN sequence; + END + $function$; + CREATE OR REPLACE FUNCTION reporting_projection_mark() RETURNS trigger + LANGUAGE plpgsql AS $function$ + BEGIN + IF current_setting('adcp.reporting.projection_internal',true)='on' OR NOT EXISTS + (SELECT 1 FROM reporting_projection_accounts WHERE account_id=NEW.account_id) THEN + RETURN NEW; + END IF; + IF TG_OP='UPDATE' AND to_jsonb(NEW)=to_jsonb(OLD) THEN RETURN NEW; END IF; + IF TG_TABLE_NAME='reporting_configurations' AND TG_OP='UPDATE' AND + (to_jsonb(NEW)-ARRAY['lease_worker_id','lease_expires_at']) = + (to_jsonb(OLD)-ARRAY['lease_worker_id','lease_expires_at']) THEN + RETURN NEW; + END IF; + PERFORM pg_advisory_xact_lock(hashtext('adcp.reporting:' || NEW.account_id)); + IF EXISTS (SELECT 1 FROM reporting_projection_inputs + WHERE account_id=NEW.account_id AND transaction_id=pg_current_xact_id()::text) THEN + RAISE EXCEPTION 'status_projection_boundary_already_captured' USING ERRCODE='23514'; + END IF; + INSERT INTO reporting_projection_writes VALUES(NEW.account_id,pg_current_xact_id()::text) + ON CONFLICT DO NOTHING; + RETURN NEW; + END + $function$; + CREATE OR REPLACE FUNCTION reporting_projection_commit() RETURNS trigger + LANGUAGE plpgsql AS $function$ + BEGIN + PERFORM reporting_projection_capture(NEW.account_id); + RETURN NEW; + END + $function$; + IF NOT EXISTS (SELECT 1 FROM pg_trigger WHERE tgrelid='reporting_projection_writes'::regclass + AND tgname='reporting_projection_commit') THEN + CREATE CONSTRAINT TRIGGER reporting_projection_commit AFTER INSERT + ON reporting_projection_writes DEFERRABLE INITIALLY DEFERRED + FOR EACH ROW EXECUTE FUNCTION reporting_projection_commit(); + END IF; + FOREACH source_name IN ARRAY ARRAY['reporting_configurations','reporting_obligations', + 'reporting_revisions','reporting_adjustments','reporting_consumer_statuses', + 'reporting_issue_lifecycle','reporting_issue_status_scopes','reporting_reconciliation_changes'] LOOP + IF NOT EXISTS (SELECT 1 FROM pg_trigger WHERE tgrelid=source_name::regclass + AND tgname='reporting_projection_mark') THEN + EXECUTE format('CREATE TRIGGER reporting_projection_mark AFTER INSERT OR UPDATE ON %I' + ' FOR EACH ROW EXECUTE FUNCTION reporting_projection_mark()',source_name); + END IF; + END LOOP; +END +$migration$; diff --git a/src/adcp/reporting/ledger/reporting_projection_feed.sql b/src/adcp/reporting/ledger/reporting_projection_feed.sql new file mode 100644 index 000000000..9f5311909 --- /dev/null +++ b/src/adcp/reporting/ledger/reporting_projection_feed.sql @@ -0,0 +1,43 @@ +-- B2.4 snapshots use a new table; legacy B2.3 rows keep their original representation. +DO $migration$ +BEGIN + PERFORM pg_advisory_xact_lock(hashtext('adcp.reporting.projection.feed.schema')); + CREATE TABLE IF NOT EXISTS reporting_projection_feed_snapshots ( + account_id TEXT COLLATE "C" NOT NULL, + consumer_id TEXT COLLATE "C" NOT NULL, + snapshot_id TEXT COLLATE "C" NOT NULL CHECK (snapshot_id ~ '^rpfs_[0-9a-f]{32}$'), + as_of TIMESTAMPTZ NOT NULL, + representation_version INTEGER NOT NULL CHECK (representation_version = 2), + ownership_mode TEXT COLLATE "C" NOT NULL CHECK (ownership_mode IN ('absent','bindings')), + document TEXT NOT NULL, + content_sha256 TEXT COLLATE "C" NOT NULL, + signing_key BYTEA NOT NULL CHECK (octet_length(signing_key) = 32), + PRIMARY KEY (account_id, consumer_id, snapshot_id), + UNIQUE (snapshot_id), + CHECK (content_sha256 = encode(sha256(convert_to(document,'UTF8')), 'hex')), + CHECK (content_sha256 = reporting_receipt_ingestion_sha256(document::jsonb)), + CHECK ((document::jsonb->>'version')::integer IS NOT DISTINCT FROM 1), + CHECK ((document::jsonb->>'account_id') IS NOT DISTINCT FROM account_id), + CHECK ((document::jsonb->>'consumer_id') IS NOT DISTINCT FROM consumer_id), + CHECK ((document::jsonb->>'snapshot_id') IS NOT DISTINCT FROM snapshot_id), + CHECK ((document::jsonb->>'as_of')::timestamptz IS NOT DISTINCT FROM as_of), + CHECK ((document::jsonb->>'representation_version')::integer IS NOT DISTINCT FROM representation_version), + CHECK ((document::jsonb->>'ownership_mode') IS NOT DISTINCT FROM ownership_mode), + CHECK (jsonb_typeof(document::jsonb->'records') IS NOT DISTINCT FROM 'array'), + CHECK (jsonb_array_length(document::jsonb->'records') IS NOT DISTINCT FROM (document::jsonb->>'total_count')::integer), + CHECK (jsonb_typeof(document::jsonb->'inputs') IS NOT DISTINCT FROM 'object'), + CHECK (document::jsonb - ARRAY['version','representation_version','ownership_mode', + 'account_id','consumer_id','snapshot_id','as_of','after','through','filters', + 'common','total_count','records','inputs'] = '{}'::jsonb) + ); + CREATE OR REPLACE FUNCTION reporting_projection_feed_immutable() + RETURNS TRIGGER LANGUAGE plpgsql AS $function$ + BEGIN + RAISE EXCEPTION 'reporting feed snapshot is immutable' USING ERRCODE = '23514'; + END + $function$; + DROP TRIGGER IF EXISTS reporting_projection_feed_immutable ON reporting_projection_feed_snapshots; + CREATE TRIGGER reporting_projection_feed_immutable BEFORE UPDATE OR DELETE ON reporting_projection_feed_snapshots + FOR EACH ROW EXECUTE FUNCTION reporting_projection_feed_immutable(); +END +$migration$; diff --git a/src/adcp/reporting/ledger/reporting_projection_notifications.sql b/src/adcp/reporting/ledger/reporting_projection_notifications.sql new file mode 100644 index 000000000..feb9ad8dc --- /dev/null +++ b/src/adcp/reporting/ledger/reporting_projection_notifications.sql @@ -0,0 +1,296 @@ +-- B2.4 status events have isolated queues. A/B/C workers cannot claim them. +-- The same reviewed checkpoint, fanout and delivery state machines apply. +DO $projection_notifications$ +BEGIN + PERFORM pg_advisory_xact_lock(hashtext('adcp.reporting.schema'), hashtext(current_schema())); + CREATE TABLE IF NOT EXISTS reporting_projection_notification_events ( + account_id TEXT COLLATE "C" NOT NULL, + notification_id TEXT COLLATE "C" NOT NULL, + notification_type TEXT COLLATE "C" NOT NULL CHECK + (notification_type = 'reporting.status_changed'), + cause_kind TEXT COLLATE "C" NOT NULL CHECK + (cause_kind = 'status_changed'), + cause_id TEXT COLLATE "C" NOT NULL CHECK (length(cause_id) > 0), + cause_generation BIGINT NOT NULL CHECK (cause_generation > 0), + consumer_namespace TEXT COLLATE "C" NOT NULL, + delivery_config_id TEXT COLLATE "C" NOT NULL, + version BIGINT NOT NULL CHECK (version > 0), + scope_kind TEXT COLLATE "C" NOT NULL CHECK (scope_kind IN ('configuration','obligation')), + obligation_namespace TEXT COLLATE "C" NOT NULL, + fingerprint TEXT NOT NULL CHECK (fingerprint ~ '^[0-9a-f]{64}$'), + admission_epoch BIGINT NOT NULL DEFAULT 2 CHECK (admission_epoch=2), + fired_at TIMESTAMPTZ NOT NULL, + snapshot JSONB NOT NULL, + PRIMARY KEY (account_id, consumer_namespace, notification_id), + UNIQUE (account_id, consumer_namespace, delivery_config_id, version, + scope_kind, obligation_namespace, cause_generation), + FOREIGN KEY (account_id, consumer_namespace, delivery_config_id, version, + scope_kind, obligation_namespace) + REFERENCES reporting_status_scope_checkpoints, + CHECK ((scope_kind = 'configuration') = (obligation_namespace = '')), + UNIQUE (account_id, consumer_namespace, notification_type, + cause_kind, cause_id, cause_generation), + UNIQUE (account_id, consumer_namespace, notification_id, cause_kind, cause_id, cause_generation), + CHECK ((snapshot #>> '{cause,kind}') IS NOT DISTINCT FROM cause_kind), + CHECK ((snapshot #>> '{cause,fingerprint}') IS NOT DISTINCT FROM fingerprint), + CHECK ((snapshot #>> '{cause,checkpoint_generation}')::bigint IS NOT DISTINCT FROM cause_generation), + CHECK ((snapshot #>> '{cause,scope,account_id}') IS NOT DISTINCT FROM account_id), + CHECK ((snapshot #>> '{cause,scope,generation_key,account_id}') IS NOT DISTINCT FROM account_id), + CHECK (coalesce(snapshot #>> '{cause,scope,consumer_id}', '') = consumer_namespace), + CHECK ((snapshot #>> '{cause,scope,generation_key,delivery_config_id}') IS NOT DISTINCT FROM delivery_config_id), + CHECK ((snapshot #>> '{cause,scope,generation_key,delivery_config_version}')::bigint IS NOT DISTINCT FROM version), + CHECK (coalesce(snapshot #>> '{cause,scope,reporting_obligation_id}', '') = obligation_namespace), + CHECK ((snapshot->>'account_id') IS NOT DISTINCT FROM account_id AND + (snapshot->>'notification_id') IS NOT DISTINCT FROM notification_id), + CHECK ((snapshot->>'cause_generation')::bigint IS NOT DISTINCT FROM cause_generation), + CHECK ((snapshot->>'fired_at')::timestamptz IS NOT DISTINCT FROM fired_at) + ); + CREATE TABLE IF NOT EXISTS reporting_projection_notification_expansions ( + account_id TEXT COLLATE "C" NOT NULL, + consumer_namespace TEXT COLLATE "C" NOT NULL DEFAULT '', + notification_id TEXT COLLATE "C" NOT NULL, + emission_generation BIGINT NOT NULL CHECK (emission_generation > 0), + state TEXT NOT NULL DEFAULT 'pending' CHECK + (state IN ('pending', 'leased', 'complete', 'suppressed', 'quarantined')), + due_at TIMESTAMPTZ NOT NULL, + lease_token TEXT, + lease_expires_at TIMESTAMPTZ, + claim_count BIGINT NOT NULL DEFAULT 0, + error_code TEXT CHECK (error_code IN ( + 'network', 'retryable_http', 'permanent_http', 'signing_unavailable', + 'permanent_scope', 'subscription_unavailable', 'subscription_changed', + 'invalid_configuration', 'invalid_payload', 'integrity_failure', 'lease_expired')), + PRIMARY KEY (account_id, consumer_namespace, notification_id, emission_generation), + FOREIGN KEY (account_id, consumer_namespace, notification_id) + REFERENCES reporting_projection_notification_events (account_id, consumer_namespace, notification_id) + ); + CREATE INDEX IF NOT EXISTS reporting_projection_notification_expansions_due + ON reporting_projection_notification_expansions (account_id, due_at) + WHERE state IN ('pending', 'leased'); + + CREATE TABLE IF NOT EXISTS reporting_projection_notification_deliveries ( + account_id TEXT COLLATE "C" NOT NULL, + delivery_id TEXT COLLATE "C" NOT NULL, + subscriber_id TEXT COLLATE "C" NOT NULL, + principal_id TEXT COLLATE "C" NOT NULL, + notification_id TEXT COLLATE "C" NOT NULL, + notification_type TEXT COLLATE "C" NOT NULL CHECK (notification_type = 'reporting.status_changed'), + emission_generation BIGINT NOT NULL CHECK (emission_generation > 0), + idempotency_key TEXT COLLATE "C" NOT NULL, + destination_sha256 TEXT NOT NULL, + subscription_fingerprint TEXT NOT NULL, + signing_scope_id TEXT COLLATE "C", + cause_kind TEXT COLLATE "C" NOT NULL CHECK (cause_kind = 'status_changed'), + cause_id TEXT COLLATE "C" NOT NULL, + cause_generation BIGINT NOT NULL CHECK (cause_generation > 0), + consumer_namespace TEXT COLLATE "C" NOT NULL, + auth_mode TEXT NOT NULL, + body_sha256 TEXT NOT NULL, + envelope_version INTEGER NOT NULL, + key_version TEXT COLLATE "C" NOT NULL, + envelope BYTEA NOT NULL, + state TEXT NOT NULL DEFAULT 'pending' CHECK + (state IN ('pending', 'leased', 'complete', 'suppressed', 'quarantined')), + due_at TIMESTAMPTZ NOT NULL, + lease_token TEXT, + lease_expires_at TIMESTAMPTZ, + claim_count BIGINT NOT NULL DEFAULT 0, + error_code TEXT CHECK (error_code IN ( + 'network', 'retryable_http', 'permanent_http', 'signing_unavailable', + 'permanent_scope', 'subscription_unavailable', 'subscription_changed', + 'invalid_configuration', 'invalid_payload', 'integrity_failure', 'lease_expired')), + PRIMARY KEY (account_id, consumer_namespace, delivery_id), + UNIQUE (account_id, consumer_namespace, notification_id, emission_generation, subscriber_id), + UNIQUE (account_id, consumer_namespace, idempotency_key), + CHECK (length(principal_id) > 0 AND (consumer_namespace = '' OR consumer_namespace = principal_id)), + FOREIGN KEY (account_id, consumer_namespace, notification_id, cause_kind, cause_id, cause_generation) + REFERENCES reporting_projection_notification_events + (account_id, consumer_namespace, notification_id, cause_kind, cause_id, cause_generation), + FOREIGN KEY (account_id, consumer_namespace, notification_id, emission_generation) + REFERENCES reporting_projection_notification_expansions + (account_id, consumer_namespace, notification_id, emission_generation) + ); + CREATE INDEX IF NOT EXISTS reporting_projection_notification_deliveries_due + ON reporting_projection_notification_deliveries (account_id, due_at) + WHERE state IN ('pending', 'leased'); + + + CREATE TABLE IF NOT EXISTS reporting_projection_webhook_attempt_heads ( + account_id TEXT COLLATE "C" NOT NULL, + consumer_namespace TEXT COLLATE "C" NOT NULL, + principal_id TEXT COLLATE "C" NOT NULL CHECK (length(principal_id) > 0), + subscriber_id TEXT COLLATE "C" NOT NULL, + idempotency_key TEXT COLLATE "C" NOT NULL, + last_attempt BIGINT NOT NULL CHECK (last_attempt > 0), + PRIMARY KEY (account_id, consumer_namespace, principal_id, subscriber_id, idempotency_key) + ); + CREATE TABLE IF NOT EXISTS reporting_projection_webhook_attempts ( + account_id TEXT COLLATE "C" NOT NULL, + principal_id TEXT COLLATE "C" NOT NULL, + subscriber_id TEXT COLLATE "C" NOT NULL, + notification_id TEXT COLLATE "C" NOT NULL, + idempotency_key TEXT COLLATE "C" NOT NULL, + attempt BIGINT NOT NULL CHECK (attempt > 0), + delivery_id TEXT COLLATE "C" NOT NULL, + consumer_namespace TEXT COLLATE "C" NOT NULL, + lease_token TEXT NOT NULL, + reservation_token TEXT NOT NULL, + binding JSONB NOT NULL, + fired_at TIMESTAMPTZ NOT NULL, + url TEXT NOT NULL CONSTRAINT reporting_projection_webhook_url_safe + CHECK (length(url) <= 8192 AND url !~ '[?#@]' AND url ~ '^https?://'), + payload_size_bytes BIGINT NOT NULL CHECK (payload_size_bytes >= 0), + status TEXT NOT NULL DEFAULT 'pending' CHECK + (status IN ('pending', 'success', 'failed', 'timeout', 'connection_error')), + completed_at TIMESTAMPTZ, + http_status_code INTEGER, + response_time_ms BIGINT CHECK (response_time_ms >= 0), + PRIMARY KEY (account_id, consumer_namespace, principal_id, subscriber_id, idempotency_key, attempt), + UNIQUE (account_id, consumer_namespace, principal_id, delivery_id, lease_token), + FOREIGN KEY (account_id, consumer_namespace, delivery_id) + REFERENCES reporting_projection_notification_deliveries, + FOREIGN KEY (account_id, consumer_namespace, principal_id, subscriber_id, idempotency_key) + REFERENCES reporting_projection_webhook_attempt_heads + (account_id, consumer_namespace, principal_id, subscriber_id, idempotency_key), + CONSTRAINT reporting_projection_webhook_identity CHECK ( + (binding->>'account_id') IS NOT DISTINCT FROM account_id AND + (binding->>'principal_id') IS NOT DISTINCT FROM principal_id AND + (binding->>'subscriber_id') IS NOT DISTINCT FROM subscriber_id AND + (binding->>'notification_id') IS NOT DISTINCT FROM notification_id AND + (binding->>'idempotency_key') IS NOT DISTINCT FROM idempotency_key AND + (binding->>'delivery_id') IS NOT DISTINCT FROM delivery_id AND + (binding->>'consumer_namespace') IS NOT DISTINCT FROM consumer_namespace), + CONSTRAINT reporting_projection_webhook_completion CHECK ((status = 'pending') = (completed_at IS NULL)), + CONSTRAINT reporting_projection_webhook_timestamps CHECK (completed_at >= fired_at), + CONSTRAINT reporting_projection_webhook_outcome CHECK ( + (status IN ('pending', 'timeout', 'connection_error') + AND http_status_code IS NULL AND response_time_ms IS NULL) + OR (status IN ('success', 'failed') AND http_status_code BETWEEN 100 AND 599 + AND http_status_code IS NOT NULL AND response_time_ms IS NOT NULL + AND ((status = 'success') = (http_status_code BETWEEN 200 AND 299)))) + ); + CREATE INDEX IF NOT EXISTS reporting_projection_webhook_activity_newest ON reporting_projection_webhook_attempts + (account_id, principal_id, fired_at DESC, notification_id DESC, + idempotency_key DESC, subscriber_id DESC, attempt DESC, delivery_id DESC); + CREATE INDEX IF NOT EXISTS reporting_projection_webhook_activity_retention ON reporting_projection_webhook_attempts + (account_id, principal_id, completed_at) WHERE completed_at IS NOT NULL; + + -- Retention never resets a logical delivery's sequence, even after all + -- terminal attempts are purged. Writers always lock parent, then head. + CREATE OR REPLACE FUNCTION reporting_projection_webhook_head_guard() + RETURNS TRIGGER LANGUAGE plpgsql AS $head_guard$ + BEGIN + IF TG_OP = 'DELETE' OR + (TG_OP = 'INSERT' AND NEW.last_attempt <> 1) OR + (TG_OP = 'UPDATE' AND (NEW.last_attempt <> OLD.last_attempt + 1 OR + (to_jsonb(NEW) - 'last_attempt') <> (to_jsonb(OLD) - 'last_attempt'))) THEN + RAISE EXCEPTION 'reporting activity counter must advance and be retained' + USING ERRCODE = '23514'; + END IF; + RETURN NEW; + END; + $head_guard$; + IF NOT EXISTS (SELECT 1 FROM pg_trigger WHERE tgrelid = 'reporting_projection_webhook_attempt_heads'::regclass + AND tgname = 'reporting_projection_webhook_head_guard' AND NOT tgisinternal) THEN + CREATE TRIGGER reporting_projection_webhook_head_guard BEFORE INSERT OR UPDATE OR DELETE + ON reporting_projection_webhook_attempt_heads FOR EACH ROW EXECUTE FUNCTION reporting_projection_webhook_head_guard(); + END IF; + + CREATE OR REPLACE FUNCTION reporting_projection_webhook_attempt_guard() + RETURNS TRIGGER LANGUAGE plpgsql AS $guard$ + DECLARE + delivery reporting_projection_notification_deliveries; + BEGIN + IF TG_OP = 'DELETE' THEN + IF OLD.completed_at IS NULL THEN + RAISE EXCEPTION 'pending reporting activity must be retained' USING ERRCODE = '23514'; + END IF; + RETURN OLD; + END IF; + IF TG_OP = 'UPDATE' AND ( + OLD.status <> 'pending' OR NEW.status = 'pending' OR + (to_jsonb(NEW) - ARRAY['status','completed_at','http_status_code','response_time_ms']) <> + (to_jsonb(OLD) - ARRAY['status','completed_at','http_status_code','response_time_ms'])) THEN + RAISE EXCEPTION 'immutable reporting activity reservation' USING ERRCODE = '23514'; + END IF; + -- The opaque reservation token fences terminalization in the UPDATE + -- predicate. A known late response may finish its own reservation after + -- lease expiry/reclaim; it must not require or modify the parent lease. + IF TG_OP = 'UPDATE' THEN + RETURN NEW; + END IF; + IF TG_OP = 'INSERT' AND NEW.status <> 'pending' THEN + RAISE EXCEPTION 'reporting activity requires reservation' USING ERRCODE = '23514'; + END IF; + SELECT * INTO delivery FROM reporting_projection_notification_deliveries + WHERE account_id = NEW.account_id AND principal_id = NEW.principal_id + AND consumer_namespace = NEW.consumer_namespace AND delivery_id = NEW.delivery_id + AND subscriber_id = NEW.subscriber_id AND notification_id = NEW.notification_id + AND idempotency_key = NEW.idempotency_key + AND state = 'leased' AND lease_token = NEW.lease_token + AND lease_expires_at > coalesce(NEW.completed_at, NEW.fired_at) FOR UPDATE; + IF NOT FOUND OR + NEW.binding->>'account_id' IS DISTINCT FROM NEW.account_id OR + NEW.binding->>'principal_id' IS DISTINCT FROM NEW.principal_id OR + NEW.binding->>'subscriber_id' IS DISTINCT FROM NEW.subscriber_id OR + NEW.binding->>'notification_id' IS DISTINCT FROM NEW.notification_id OR + NEW.binding->>'idempotency_key' IS DISTINCT FROM NEW.idempotency_key OR + NEW.binding->>'delivery_id' IS DISTINCT FROM NEW.delivery_id OR + NEW.binding->>'consumer_namespace' IS DISTINCT FROM NEW.consumer_namespace OR + NEW.binding->>'notification_type' IS DISTINCT FROM delivery.notification_type OR + NEW.binding->>'body_sha256' IS DISTINCT FROM delivery.body_sha256 THEN + RAISE EXCEPTION 'reporting activity lease or identity mismatch' USING ERRCODE = '23514'; + END IF; + RETURN NEW; + END; + $guard$; + IF NOT EXISTS (SELECT 1 FROM pg_trigger WHERE tgrelid = 'reporting_projection_webhook_attempts'::regclass + AND tgname = 'reporting_projection_webhook_attempt_guard' AND NOT tgisinternal) THEN + CREATE TRIGGER reporting_projection_webhook_attempt_guard BEFORE INSERT OR UPDATE OR DELETE + ON reporting_projection_webhook_attempts FOR EACH ROW EXECUTE FUNCTION reporting_projection_webhook_attempt_guard(); + END IF; + + CREATE OR REPLACE FUNCTION reporting_projection_event_guard() + RETURNS TRIGGER LANGUAGE plpgsql AS $event_guard$ + DECLARE checkpoint reporting_status_scope_checkpoints; ids jsonb; + BEGIN + IF NOT EXISTS (SELECT 1 FROM reporting_projection_accounts + WHERE account_id=NEW.account_id AND current_input IS NOT NULL + AND notifications_enabled) THEN + RAISE EXCEPTION 'status_projection_activation_required' USING ERRCODE='23514'; + END IF; + SELECT * INTO checkpoint FROM reporting_status_scope_checkpoints + WHERE account_id=NEW.account_id AND consumer_namespace=NEW.consumer_namespace + AND delivery_config_id=NEW.delivery_config_id AND version=NEW.version + AND scope_kind=NEW.scope_kind AND obligation_namespace=NEW.obligation_namespace; + IF NOT FOUND OR NOT checkpoint.initialized OR NOT checkpoint.publishable + OR checkpoint.generation<>NEW.cause_generation OR checkpoint.fingerprint<>NEW.fingerprint + OR checkpoint.snapshot->>'health' IS DISTINCT FROM NEW.snapshot #>> '{cause,health}' THEN + RAISE EXCEPTION 'invalid status event checkpoint' USING ERRCODE='23514'; + END IF; + SELECT coalesce(jsonb_agg(issue_id ORDER BY issue_id COLLATE "C"), '[]') INTO ids + FROM (SELECT DISTINCT i->>'issue_id' COLLATE "C" AS issue_id + FROM jsonb_array_elements(checkpoint.snapshot->'issues') i + ORDER BY issue_id LIMIT 16) selected; + IF ids IS DISTINCT FROM NEW.snapshot #> '{cause,issue_ids}' THEN + RAISE EXCEPTION 'invalid status event issues' USING ERRCODE='23514'; + END IF; + RETURN NEW; + END + $event_guard$; + IF NOT EXISTS (SELECT 1 FROM pg_trigger + WHERE tgrelid='reporting_projection_notification_events'::regclass + AND tgname='reporting_projection_event_guard') THEN + CREATE TRIGGER reporting_projection_event_guard BEFORE INSERT + ON reporting_projection_notification_events FOR EACH ROW + EXECUTE FUNCTION reporting_projection_event_guard(); + END IF; + IF NOT EXISTS (SELECT 1 FROM pg_trigger + WHERE tgrelid='reporting_projection_notification_events'::regclass + AND tgname='reporting_projection_event_immutable') THEN + CREATE TRIGGER reporting_projection_event_immutable BEFORE UPDATE OR DELETE + ON reporting_projection_notification_events FOR EACH ROW + EXECUTE FUNCTION reporting_notification_immutable(); + END IF; +END +$projection_notifications$; diff --git a/src/adcp/reporting/ledger/schedule.py b/src/adcp/reporting/ledger/schedule.py new file mode 100644 index 000000000..9467a88b1 --- /dev/null +++ b/src/adcp/reporting/ledger/schedule.py @@ -0,0 +1,121 @@ +"""One captured-generation clock for producer obligations and status forecasts. + +A generation owes full periods starting at/after activation and strictly before +deactivation. Deactivation after a period starts keeps that entire period and +its original SLA. Forecasting never creates an obligation or changes health. +""" + +from __future__ import annotations + +from collections.abc import Iterator, Sequence +from datetime import datetime + +from adcp.reporting.ledger.models import ( + ReportingConfiguration, + ReportingObligationRecord, + ReportingPeriodBoundary, + _period_instants, + _schedule_clock, + derive_period, + first_ordinal_after, + iso_duration_to_timedelta, +) + + +def committed_periods( + configuration: ReportingConfiguration, + *, + near: datetime | None = None, + near_start: datetime | None = None, +) -> Iterator[ReportingPeriodBoundary]: + """Yield full committed periods, optionally seeking near an expected-at time. + + ``near`` is an optimization, not a filter: the caller still compares exact + instants. Two predecessor civil slots retain the period containing the + requested instant, including its DST fold. No wall clock is consulted. + ``near_start`` seeks around a period start independently of its SLA. + """ + activated = configuration.activated_at + if activated is None: + return + schedule, timezone = configuration.schedule, configuration.account_timezone + zone, duration, anchor = _schedule_clock(schedule, timezone) + ordinal = first_ordinal_after(schedule, account_timezone=timezone, activated_at=activated) + seek = near_start + if seek is None and near is not None: + seek = near - iso_duration_to_timedelta(schedule.delivery_sla) + if seek is not None: + candidate = first_ordinal_after( + schedule, + account_timezone=timezone, + activated_at=seek, + ) + ordinal = max(ordinal, candidate - 2) + while True: + start, end = _period_instants(schedule, timezone, ordinal) + if configuration.deactivated_at is not None and start >= configuration.deactivated_at: + return + local_start = anchor + duration * ordinal + if end > start and start.astimezone(zone).replace(tzinfo=None) == local_start: + yield derive_period(schedule, account_timezone=timezone, ordinal=ordinal) + ordinal += 1 + + +def next_reporting_expectation( + configurations: Sequence[ReportingConfiguration], + obligations: Sequence[ReportingObligationRecord], + *, + as_of: datetime, + period_start: datetime | None = None, + period_end: datetime | None = None, +) -> datetime | None: + """Nearest future due instant in the selected frozen schedule/period scope. + + Existing obligations remain commitments even if their registry generation + has aged out. They supplement, rather than replace, captured schedules. + This value is independent of current health and of record pagination. + """ + future = [ + item.period.expected_at + for item in obligations + if item.period.expected_at > as_of + and (period_start is None or item.period.end > period_start) + and (period_end is None or item.period.start < period_end) + ] + for configuration in configurations: + # A requested historical horizon does not create an unbounded walk. + # Seek close to the later of the SLA cutoff and its lower period edge. + near = as_of + if period_start is not None: + near = max( + near, period_start + iso_duration_to_timedelta(configuration.schedule.delivery_sla) + ) + for period in committed_periods(configuration, near=near): + if period_end is not None and period.start >= period_end: + break + if period.expected_at <= as_of or ( + period_start is not None and period.end <= period_start + ): + continue + future.append(period.expected_at) + break + return min(future) if future else None + + +def next_reporting_period_start( + configurations: Sequence[ReportingConfiguration], *, as_of: datetime +) -> datetime | None: + """rc.6 complete-summary forecast outside the closed evaluated horizon. + + Only captured committed generations supply this forecast. A completed + obligation's due time is not a period start, and a future forecast does + not create, count or lease an obligation. The producer still uses the + identical full-period activation/deactivation and civil-time boundaries. + """ + future = [] + for configuration in configurations: + for period in committed_periods(configuration, near_start=as_of): + if period.start > as_of: + future.append(period.start) + break + return min(future) if future else None diff --git a/src/adcp/reporting/ledger/status.py b/src/adcp/reporting/ledger/status.py index b01628698..62e98850a 100644 --- a/src/adcp/reporting/ledger/status.py +++ b/src/adcp/reporting/ledger/status.py @@ -35,8 +35,14 @@ from datetime import datetime, timezone from typing import Any, Literal +from adcp._version import ( + is_adcp_version_at_least, + normalize_to_release_precision, + resolve_adcp_version, +) from adcp.reporting.canonical_json import canonical_json_utf8_v1 from adcp.reporting.ledger.consumer_status import condition_after_waiver +from adcp.reporting.ledger.delivery_models import ReportingDeliveryRecord from adcp.reporting.ledger.models import ( ConsumerStatusRecord, ReportingAdjustmentRecord, @@ -48,6 +54,7 @@ ReportingRevisionRecord, ) from adcp.reporting.ledger.notification_models import ReportingStatusScope +from adcp.reporting.ledger.schedule import next_reporting_expectation, next_reporting_period_start from adcp.reporting.ledger.status_projection import ( ReportingStatusSnapshot, StatusProjectionInput, @@ -219,6 +226,8 @@ def render_snapshot( *, caller: ReportingStatusCaller, snapshot: ReportingStatusSnapshot, + reconciliation: tuple[ReportingDeliveryRecord, ...] | None = None, + revision_ownership: bool = False, ) -> dict[str, Any]: """Render captured database evidence without any store calls or clock reads.""" view = request.get("view", "summary") @@ -227,21 +236,51 @@ def render_snapshot( if snapshot.account_id != caller.account_id: raise LedgerConflictError("LOOKUP_UNAVAILABLE", "status is unavailable to this caller") filters = _filters(request) + version = normalize_to_release_precision( + request.get("adcp_version") or resolve_adcp_version(None) + ) + complete_start_forecast = is_adcp_version_at_least(version, "3.2-rc.6") + if complete_start_forecast: + # Bind the new wire contract without relabelling explicit rc.3 snapshots. + filters["adcp_version"] = version scope = ReportingStatusScope( caller.account_id, - consumer_id=caller.consumer_id if self._consumer_status_enabled else None, + consumer_id=( + caller.consumer_id + if self._consumer_status_enabled or reconciliation is not None + else None + ), ) snapshot_id = ( "rpls_" + _fingerprint( [ caller.account_id, - caller.consumer_id if self._consumer_status_enabled else None, + ( + caller.consumer_id + if self._consumer_status_enabled or reconciliation is not None + else None + ), filters, snapshot.max_sequence, ] )[:32] ) + if reconciliation is not None: + from adcp.reporting.ledger._delivery_state import fingerprint, principal + + reconciliation = tuple( + r + for r in reconciliation + if principal(r).account_id == caller.account_id + and principal(r).consumer_id == caller.consumer_id + ) + snapshot_id = ( + "rpls_" + + _fingerprint( + [snapshot_id, [fingerprint(r) for r in reconciliation], _iso(snapshot.as_of)] + )[:32] + ) offset = 0 lower = _checkpoint_sequence(request.get("changes_after")) or 0 cursor = (request.get("pagination") or {}).get("cursor") @@ -277,6 +316,8 @@ def render_snapshot( tuple(filters["feed_purposes"] or ()), _parse(filters["period_start"]), _parse(filters["period_end"]), + reconciliation=reconciliation, + consumer_status_enabled=self._consumer_status_enabled, ) result = project_status_scope(value) if result.intents: @@ -311,7 +352,7 @@ def render_snapshot( ), None, ) - return { + response = { **common, "revision": _revision_to_wire(revision, owner), "adjustments": [ @@ -323,6 +364,19 @@ def render_snapshot( "receipts": [], "pagination": {"total_count": 1, "has_more": False}, } + if reconciliation is not None: + from adcp.reporting.projection.wire import exact_revision_evidence + + response.update( + exact_revision_evidence(snapshot, revision, owner, reconciliation, caller) + ) + if revision_ownership: + from adcp.reporting.ownership import with_revision_ownership + + response = with_revision_ownership( + response, {revision.reporting_revision_id: revision.reporting_obligation_id} + ) + return response common["scope"] = _scope_to_wire( result.configurations, ledger_as_of=snapshot.as_of, @@ -342,17 +396,49 @@ def render_snapshot( if self._consumer_status_enabled: counts["consumer_status_pending"] = result.pending_count watermark = _scope_data_through(p.projection for p in result.obligations) - next_expected = _next_expected(obligations, ledger_as_of=snapshot.as_of) + configurations = tuple( + c + for c in result.configurations + if (not request.get("finality") or c.required_finality in request["finality"]) + and ( + not request.get("health") + or project_status_scope( + replace( + value, + scope=ReportingStatusScope( + c.account_id, c.generation_key, consumer_id=scope.consumer_id + ), + ) + ).health + in request["health"] + ) + ) + selected_obligations = tuple( + p.obligation + for p in result.obligations + if ( + not request.get("finality") + or p.obligation.required_finality in request["finality"] + ) + and (not request.get("health") or p.projection.health in request["health"]) + ) + next_expected = ( + next_reporting_period_start(configurations, as_of=snapshot.as_of) + if complete_start_forecast and result.health == "complete" + else next_reporting_expectation( + configurations, + selected_obligations, + as_of=snapshot.as_of, + period_start=value.period_start, + period_end=value.period_end, + ) + ) return { **common, "health": result.health, "coverage": _coverage_roll_up(obligations, as_of=snapshot.as_of), "data_through": _iso(watermark) if watermark else None, - **( - {"next_expected_at": next_expected} - if next_expected is not None and result.health != "complete" - else {} - ), + **({"next_expected_at": _iso(next_expected)} if next_expected is not None else {}), "obligation_counts": counts, "issues": [i.to_wire() for i in result.issues], } @@ -529,17 +615,6 @@ def _scope_data_through(projections: Any) -> datetime | None: return min(watermarks) if watermarks else None -def _next_expected( - obligations: Sequence[ReportingObligationRecord], *, ledger_as_of: datetime -) -> str | None: - upcoming = [ - item.period.expected_at - for item in obligations - if _utc(item.period.expected_at) > _utc(ledger_as_of) - ] - return _iso(min(upcoming)) if upcoming else None - - def _scope_to_wire( configurations: Sequence[ReportingConfiguration], *, diff --git a/src/adcp/reporting/ledger/status_projection.py b/src/adcp/reporting/ledger/status_projection.py index 67d7ee157..10c746d12 100644 --- a/src/adcp/reporting/ledger/status_projection.py +++ b/src/adcp/reporting/ledger/status_projection.py @@ -11,7 +11,7 @@ from collections.abc import Sequence from dataclasses import asdict, dataclass, replace from datetime import datetime, timedelta -from typing import Any, Literal, cast +from typing import TYPE_CHECKING, Any, Literal, cast from adcp.reporting.canonical_json import canonical_json_utf8_v1 from adcp.reporting.ledger.consumer_status import ( @@ -24,6 +24,7 @@ stale_received_grace_deadline, waiver_covers_mismatch, ) +from adcp.reporting.ledger.delivery_models import ReportingDeliveryRecord from adcp.reporting.ledger.health import ( ObligationProjection, aggregate_reporting_health, @@ -51,6 +52,9 @@ ) from adcp.reporting.revision_selection import select_reporting_revision +if TYPE_CHECKING: + from adcp.reporting.ledger.reconciliation_projection import ReconciliationProjection + @dataclass(frozen=True) class ReportingStatusSnapshot: @@ -89,6 +93,10 @@ class StatusProjectionInput: feed_purposes: tuple[str, ...] = () period_start: datetime | None = None period_end: datetime | None = None + # None selects the immutable legacy C representation. Versioned callers + # pass the complete captured account history, even when it is empty. + reconciliation: tuple[ReportingDeliveryRecord, ...] | None = None + consumer_status_enabled: bool = True @dataclass(frozen=True) @@ -97,6 +105,7 @@ class StatusObligationProjection: projection: ObligationProjection revisions: tuple[ReportingRevisionRecord, ...] statuses: tuple[ConsumerStatusRecord, ...] + reconciliation: ReconciliationProjection | None = None @dataclass(frozen=True) @@ -468,7 +477,7 @@ def status_retained_from( return max( ( ( - max(as_of - timedelta(days=c.status_retention_days), c.activated_at) + max(as_of - timedelta(days=c.status_retention_days), min(c.activated_at, as_of)) if c.activated_at is not None else as_of - timedelta(days=c.status_retention_days) ) @@ -543,7 +552,9 @@ def _project(value: StatusProjectionInput) -> tuple[StatusProjectionResult, set[ intents = tuple( i for i in lifecycle_intents(snapshot) - if _selected(i.scope, scope) and i.scope.generation_key in generations + if _selected(i.scope, scope) + and i.scope.generation_key in generations + and (value.consumer_status_enabled or i.scope.consumer_id is None) ) live = {i.issue_key: i for i in snapshot.lifecycles if i.live} issue_scopes = dict(snapshot.issue_scopes) @@ -617,7 +628,9 @@ def _project(value: StatusProjectionInput) -> tuple[StatusProjectionResult, set[ statuses = tuple( s for s in snapshot.statuses - if s.consumer_id == scope.consumer_id and status_matches_obligation(s, obligation) + if value.consumer_status_enabled + and s.consumer_id == scope.consumer_id + and status_matches_obligation(s, obligation) ) projection = project_obligation_health( obligation, @@ -648,7 +661,8 @@ def _project(value: StatusProjectionInput) -> tuple[StatusProjectionResult, set[ projection = replace(projection, health="action_required") current = current_consumer_statement(statuses) if ( - scope.consumer_id is not None + value.consumer_status_enabled + and scope.consumer_id is not None and current is None and (snapshot.as_of >= obligation.automated_recovery_deadline_at) ): @@ -699,8 +713,55 @@ def _project(value: StatusProjectionInput) -> tuple[StatusProjectionResult, set[ candidates.add( lifecycle.opened_at + value.escalation.consumer_mismatch_escalation ) + reconciliation = None + if value.reconciliation is not None: + from adcp.reporting.ledger.reconciliation_projection import project_reconciliation + from adcp.reporting.ledger.store import LedgerConflictError + + try: + reconciliation = project_reconciliation( + obligation, + revisions, + snapshot.adjustments, + value.reconciliation, + consumer_id=scope.consumer_id, + as_of=snapshot.as_of, + ) + except LedgerConflictError: + local.append( + ReportingIssue( + issue_id_for( + "reconciliation-history-v2", + snapshot.account_id, + scope.consumer_id, + obligation.reporting_obligation_id, + ), + "HISTORY_UNAVAILABLE", + "action_required", + "seller", + "contact_seller", + reporting_obligation_id=obligation.reporting_obligation_id, + delivery_config_id=obligation.delivery_config_id, + delivery_config_version=obligation.delivery_config_version, + feed_purpose=obligation.feed_purpose, + ) + ) + projection = replace(projection, health="action_required", satisfied=False) + else: + local.extend(reconciliation.issues) + candidates.update(reconciliation.deadlines) + if not reconciliation.satisfied: + projection = replace(projection, satisfied=False) + if projection.health in {"healthy", "complete"}: + projection = replace(projection, health="waiting") + if any(i.severity == "action_required" for i in local): + projection = replace(projection, health="action_required") + elif any(i.severity == "delayed" for i in local): + projection = replace(projection, health="delayed") projection = replace(projection, issues=_sorted_issues(local)) - projected.append(StatusObligationProjection(obligation, projection, revisions, statuses)) + projected.append( + StatusObligationProjection(obligation, projection, revisions, statuses, reconciliation) + ) issues.extend(local) candidates.update( ( @@ -722,7 +783,11 @@ def _project(value: StatusProjectionInput) -> tuple[StatusProjectionResult, set[ break mismatch_keys.add(condition_key) for status in snapshot.statuses: - if status.superseded or status.consumer_id != scope.consumer_id: + if ( + not value.consumer_status_enabled + or status.superseded + or status.consumer_id != scope.consumer_id + ): continue if status.generation_key not in generations or scope.reporting_obligation_id is not None: continue @@ -808,6 +873,19 @@ def _project(value: StatusProjectionInput) -> tuple[StatusProjectionResult, set[ "period_end": value.period_end.isoformat() if value.period_end else None, }, } + if value.reconciliation is not None: + canonical["version"] = 2 + canonical["reconciliation"] = [ + ( + p.reconciliation.evidence_json.decode("utf-8") + if p.reconciliation is not None + else { + "reporting_obligation_id": p.obligation.reporting_obligation_id, + "unavailable": True, + } + ) + for p in projected + ] fingerprint = hashlib.sha256(canonical_json_utf8_v1(canonical)).hexdigest() return ( StatusProjectionResult( diff --git a/src/adcp/reporting/ledger/status_server.py b/src/adcp/reporting/ledger/status_server.py index 23a95f378..2f073cd32 100644 --- a/src/adcp/reporting/ledger/status_server.py +++ b/src/adcp/reporting/ledger/status_server.py @@ -5,6 +5,8 @@ from collections.abc import Awaitable, Callable from typing import Any +from adcp._version import is_adcp_version_at_least, resolve_adcp_version +from adcp.exceptions import ConfigurationError from adcp.reporting.ledger.status import ReportingStatusCaller, ReportingStatusHandler from adcp.server.base import ADCPHandler, ToolContext from adcp.types import GetReportingStatusRequest @@ -24,19 +26,38 @@ class ReportingStatusNotificationHandler(ADCPHandler[ToolContext]): """ def __init__( - self, status: ReportingStatusHandler, *, resolve_caller: ReportingStatusCallerResolver + self, + status: ReportingStatusHandler, + *, + resolve_caller: ReportingStatusCallerResolver, + adcp_version: str | None = None, ) -> None: super().__init__() self.reporting_status_handler = status self._resolve_status_caller = resolve_caller + self._adcp_version = resolve_adcp_version(adcp_version) + if not is_adcp_version_at_least(self._adcp_version, "3.2-rc.3"): + raise ConfigurationError( + "reporting mounts require a supported AdCP 3.2 reporting contract; " + "use 3.2-rc.3 for retained walks or omit the pin for the packaged default" + ) + + def get_adcp_version(self) -> str: + """Public per-mount protocol pin, shared by schema and rendering.""" + return self._adcp_version async def get_reporting_status( self, params: GetReportingStatusRequest | dict[str, Any], context: ToolContext | None = None ) -> dict[str, Any]: request = ( - params + dict(params) if isinstance(params, dict) else params.model_dump(mode="json", exclude_unset=True) ) + request["adcp_version"] = ( + context.resolved_adcp_version + if context is not None and context.resolved_adcp_version is not None + else request.get("adcp_version") or self.get_adcp_version() + ) caller = await self._resolve_status_caller(request, context) return await self.reporting_status_handler.handle(request, caller=caller) diff --git a/src/adcp/reporting/ledger/status_snapshot.py b/src/adcp/reporting/ledger/status_snapshot.py index fd6bcbd05..18c31257d 100644 --- a/src/adcp/reporting/ledger/status_snapshot.py +++ b/src/adcp/reporting/ledger/status_snapshot.py @@ -7,6 +7,7 @@ from datetime import datetime from typing import TYPE_CHECKING, Any, Protocol, runtime_checkable +from adcp.reporting._timestamp import aware_timestamp from adcp.reporting.ledger.models import ConsumerStatusRecord from adcp.reporting.ledger.notification_models import ReportingNotificationError from adcp.reporting.ledger.status_projection import ( @@ -258,11 +259,7 @@ def decode(key: str, columns: str, builder: Callable[[Any], Any]) -> tuple[Any, result = [] for row in raw.get(key, []): values = [ - ( - datetime.fromisoformat(row[n]) - if n in dates and row.get(n) is not None - else row.get(n) - ) + (aware_timestamp(row[n]) if n in dates and row.get(n) is not None else row.get(n)) for n in names ] result.append(builder(values)) @@ -304,7 +301,7 @@ def decode(key: str, columns: str, builder: Callable[[Any], Any]) -> tuple[Any, consumers.update(i.consumer_id for i in lifecycles if i.consumer_id is not None) return ReportingStatusSnapshot( account_id=raw["account_id"], - as_of=datetime.fromisoformat(raw["as_of"]), + as_of=aware_timestamp(raw["as_of"]), configurations=configurations, obligations=decode("obligations", _OBLIGATION_COLUMNS, _obligation_from_row), revisions=decode("revisions", _REVISION_COLUMNS, _revision_from_row), diff --git a/src/adcp/reporting/ledger/store.py b/src/adcp/reporting/ledger/store.py index 549553ca2..f7773697a 100644 --- a/src/adcp/reporting/ledger/store.py +++ b/src/adcp/reporting/ledger/store.py @@ -1647,6 +1647,9 @@ def _obligation_for( # -- leasing --------------------------------------------------------- + def _configuration_lease_eligible(self, configuration: ReportingConfiguration) -> bool: + return True + async def lease_period_close( self, *, worker_id: str, now: datetime, lease_seconds: float ) -> LeasedConfiguration | None: @@ -1674,6 +1677,8 @@ async def lease_period_close( ] ] = [] for key in self._configurations: + if not self._configuration_lease_eligible(self._configurations[key]): + continue turn = self._lease_turns.get(key, 0) held = self._leases.get(key) tail = (key.account_id, key.delivery_config_id, key.delivery_config_version) diff --git a/src/adcp/reporting/materializer/memory.py b/src/adcp/reporting/materializer/memory.py index dd922be0a..7dd91f36e 100644 --- a/src/adcp/reporting/materializer/memory.py +++ b/src/adcp/reporting/materializer/memory.py @@ -24,6 +24,7 @@ ) from adcp.reporting.ledger.models import ReportingConfiguration from adcp.reporting.ledger.notification_events import delivery_dirty, materialization_event +from adcp.reporting.ledger.notification_models import ReportingDomainEvent from adcp.reporting.ledger.store import LedgerConflictError from adcp.reporting.materializer._errors import ( ReportingMaterializerUsageError, @@ -89,6 +90,8 @@ class _Work: class InMemoryReportingMaterializerStore(InMemoryReportingReconciliationStore): + _materializer_writer_epoch = 0 + def __init__(self, **kwargs: Any) -> None: super().__init__(**kwargs) self._materializer_candidates: dict[ReportingDeliveryScope, _Candidate] = {} @@ -243,6 +246,9 @@ def _park( candidate.reason, candidate.due_at = reason, due return ReportingMaterializerTurn("parked", reason) + def _materializer_candidate_enabled(self, account_id: str) -> bool: + return True + @materializer_errors async def claim_materialization( self, @@ -265,7 +271,10 @@ async def claim_materialization( candidates = [ c for c in self._materializer_candidates.values() - if c.due_at is not None and c.due_at <= now and c.scope not in pending + if c.due_at is not None + and c.due_at <= now + and c.scope not in pending + and self._materializer_candidate_enabled(c.scope.principal.account_id) ] accounts = {w.scope.principal.account_id for w in works} | { c.scope.principal.account_id for c in candidates @@ -313,6 +322,10 @@ async def claim_materialization( return self._park( candidate, "legacy_terminal" if owned is None else "operator_required" ) + try: + admission_epoch = self._new_admission_epoch(context, key) + except ReportingWriterError: + return self._park(candidate, "component_unavailable") if context.delivery is None: if context.obligation.currency is None: return self._park(candidate, "operator_required") @@ -340,6 +353,7 @@ async def claim_materialization( request, now, self._notification_state is not None, + admission_epoch=admission_epoch, ) self._materializer_work[self._work_key(attempt)] = work self._park(candidate, "ready") @@ -351,6 +365,8 @@ def _lease( keys: tuple[ReportingVerificationKey, ...], seconds: int, ) -> ReportingMaterializerLease | ReportingMaterializerTurn: + if work.admission_epoch > self._materializer_writer_epoch: + raise failure("UNSUPPORTED_VERIFICATION") if work.notifications_enabled != (self._notification_state is not None): return self._park_work(work, "component_unavailable") try: @@ -382,6 +398,7 @@ def _lease( work.request, context, work.notifications_enabled, + work.admission_epoch, ) def _park_work(self, work: _Work, reason: MaterializerReason) -> ReportingMaterializerTurn: @@ -404,6 +421,8 @@ def _held(self, lease: ReportingMaterializerLease) -> _Work | None: or work.generation != lease.generation or work.notifications_enabled != lease.notifications_enabled or work.notifications_enabled != (self._notification_state is not None) + or work.admission_epoch != lease.admission_epoch + or work.admission_epoch > self._materializer_writer_epoch ): raise failure("BINDING_MISMATCH") return work @@ -546,15 +565,7 @@ async def finish_materialization( ) if event is None: raise failure("BINDING_MISMATCH") - assert self._materializer_outbox is not None - self._materializer_outbox.enqueue(event) - if ( - self._materializer_outbox.events.get( - (event.account_id, event.consumer_namespace, event.notification_id) - ) - != event - ): - raise failure("BINDING_MISMATCH") + self._enqueue_materializer(event, lease) if self._held(lease) is None: raise failure("LEASE_LOST") work.completion_token = work.token @@ -591,6 +602,26 @@ async def finish_materialization( stored.reporting_materialization_id, ) + def _new_admission_epoch( + self, context: MaterializerContext, key: ReportingVerificationKey + ) -> int: + return 0 + + def _enqueue_materializer( + self, event: ReportingDomainEvent, lease: ReportingMaterializerLease + ) -> None: + if lease.admission_epoch != 0: + raise failure("UNSUPPORTED_VERIFICATION") + assert self._materializer_outbox is not None + self._materializer_outbox.enqueue(event) + if ( + self._materializer_outbox.events.get( + (event.account_id, event.consumer_namespace, event.notification_id) + ) + != event + ): + raise failure("BINDING_MISMATCH") + def _materializer_dirty( self, outcome: ReportingMaterializationRecord, context: MaterializerContext ) -> None: @@ -601,11 +632,12 @@ def _materializer_dirty( core = settle_memory_snapshot(self, scope.account_id) core = replace(core, as_of=outcome.completed_at) - sequence = self._materializer_status_heads.get(outcome.scope.principal, 0) + 1 - self._materializer_status_heads[outcome.scope.principal] = sequence + heads, boundaries = self._materializer_capture_collections(outcome) + sequence = heads.get(outcome.scope.principal, 0) + 1 + heads[outcome.scope.principal] = sequence account_sequence = self._materializer_account_heads.get(scope.account_id, 0) + 1 self._materializer_account_heads[scope.account_id] = account_sequence - self._materializer_boundaries.append( + boundaries.append( ReportingMaterializerBoundary( outcome.scope.principal, sequence, @@ -617,6 +649,11 @@ def _materializer_dirty( ) ) + def _materializer_capture_collections( + self, outcome: ReportingMaterializationRecord + ) -> tuple[dict[ReportingDeliveryPrincipal, int], list[ReportingMaterializerBoundary]]: + return self._materializer_status_heads, self._materializer_boundaries + @materializer_errors async def read_materializer_boundaries( self, *, caller: ReportingDeliveryPrincipal, after: int = 0, limit: int = 100 diff --git a/src/adcp/reporting/materializer/pg.py b/src/adcp/reporting/materializer/pg.py index 990ce2405..fe01e7805 100644 --- a/src/adcp/reporting/materializer/pg.py +++ b/src/adcp/reporting/materializer/pg.py @@ -526,6 +526,7 @@ async def _lease_on( request, context, work["notifications_enabled"], + work["admission_epoch"], ) async def _park_work_on( @@ -570,6 +571,7 @@ async def _held_on( or work["binding_sha256"] != lease.request.binding_fingerprint or work["notifications_enabled"] != lease.notifications_enabled or work["notifications_enabled"] != self._notifications_enabled + or work["admission_epoch"] != lease.admission_epoch ): raise failure("BINDING_MISMATCH") return work diff --git a/src/adcp/reporting/materializer/publication.py b/src/adcp/reporting/materializer/publication.py new file mode 100644 index 000000000..70e3b0070 --- /dev/null +++ b/src/adcp/reporting/materializer/publication.py @@ -0,0 +1,63 @@ +"""SDK-owned canonical evidence before an immutable source revision is committed.""" + +from __future__ import annotations + +import hashlib +from collections.abc import Sequence +from dataclasses import replace +from decimal import Decimal +from typing import Any + +from adcp.reporting.evidence import ReportingCanonicalDigest +from adcp.reporting.ledger.models import ReportingObligationRecord, ReportingRevisionRecord +from adcp.reporting.materializer.contracts import failure +from adcp.reporting.materializer.verification import ReportingRevisionVerifier, _same_definition + + +def verified_publication( + verifier: ReportingRevisionVerifier, + obligation: ReportingObligationRecord, + revision: ReportingRevisionRecord, + rows: Sequence[dict[str, Any]], +) -> ReportingRevisionRecord: + """Validate the source rows and totals; never reinterpret an existing revision. + + Core publications retain their original representation when no verifier is + installed. Production publications use the exact same installed contract as + destination verification, before the first immutable commit. + """ + from adcp.reporting.ledger.producer import revision_content_sha256 + + key = verifier.key + if ( + not _same_definition(key, obligation.definition) + or (key.report_definition_id, key.reporting_profile) + != (obligation.report_definition_id, obligation.reporting_profile) + or revision.row_count != len(rows) + ): + raise failure("SOURCE_INVALID") + encoded, totals = verifier.canonicalize(rows) + expected = {t.name: Decimal(t.value) for t in totals} + actual = {name: Decimal(value) for name, value in revision.control_totals} + if len(actual) != len(revision.control_totals) or expected != actual: + raise failure("SOURCE_INVALID") + contract = key.canonicalization + pairs = tuple((t.name, t.value) for t in totals) + return replace( + revision, + control_totals=pairs, + managed_control_totals=totals, + canonical_content_digest=ReportingCanonicalDigest( + hashlib.sha256(b"[" + b",".join(encoded) + b"]").hexdigest(), + contract.canonicalization_id, + contract.canonicalization_uri, + contract.canonicalization_sha256, + ), + revision_content_sha256=revision_content_sha256( + reporting_revision_id=revision.reporting_revision_id, + row_count=revision.row_count, + control_totals=pairs, + reporting_rows=rows, + control_total_evidence=totals, + ), + ) diff --git a/src/adcp/reporting/materializer/work.py b/src/adcp/reporting/materializer/work.py index e2f306f15..9bed997e6 100644 --- a/src/adcp/reporting/materializer/work.py +++ b/src/adcp/reporting/materializer/work.py @@ -201,6 +201,7 @@ class ReportingMaterializerLease: request: ReportingDestinationRequest context: MaterializerContext notifications_enabled: bool + admission_epoch: int = 0 def __post_init__(self) -> None: invalid = False @@ -209,6 +210,8 @@ def __post_init__(self) -> None: type(self.generation) is not int or self.generation < 1 or type(self.notifications_enabled) is not bool + or type(self.admission_epoch) is not int + or self.admission_epoch not in {0, 2} or UUID(self.token).version != 4 or self.attempt.scope != self.scope or self.context.scope != self.scope diff --git a/src/adcp/reporting/outbox/_activity_pg.py b/src/adcp/reporting/outbox/_activity_pg.py index 80103ebb6..8549b2858 100644 --- a/src/adcp/reporting/outbox/_activity_pg.py +++ b/src/adcp/reporting/outbox/_activity_pg.py @@ -104,6 +104,12 @@ async def _activity_fence(self, conn: Any, lease: DeliveryLease) -> datetime | N async def reserve_attempt( self, lease: DeliveryLease, *, request: ActivityRequest, now: datetime + ) -> WebhookAttempt | None: + async with self._activity_transaction() as conn: + return await self._reserve_attempt_on(conn, lease, request=request) + + async def _reserve_attempt_on( + self, conn: Any, lease: DeliveryLease, *, request: ActivityRequest ) -> WebhookAttempt | None: from adcp.reporting.outbox.pg import database_now @@ -111,47 +117,46 @@ async def reserve_attempt( consumer = canonical_consumer(b.principal_id) request = ActivityRequest(request.url, request.payload_size_bytes) key = (b.account_id, consumer, b.subscriber_id, b.idempotency_key) - async with self._activity_transaction() as conn: - if await self._activity_fence(conn, lease) is None: - return None - duplicate = await ( - await conn.execute( - "SELECT 1 FROM reporting_webhook_attempts WHERE account_id = %s" - " AND principal_id = %s AND consumer_namespace = %s" - " AND delivery_id = %s AND lease_token = %s", - (b.account_id, consumer, b.consumer_namespace, b.delivery_id, lease.token), - ) - ).fetchone() - if duplicate is not None: - return None - number = await self._next_attempt_on(conn, b) - at = await database_now(conn, self._clock) - # The row stays locked from the fence through this commit. A later - # reclaim can never mutate this reservation, including after purge. - if at >= lease.expires_at: - # Roll back the increment as well; no reservation means no HTTP. - raise ReportingNotificationError("activity_lease_expired") - reservation = token_hex(32) + if await self._activity_fence(conn, lease) is None: + return None + duplicate = await ( await conn.execute( - "INSERT INTO reporting_webhook_attempts (account_id, principal_id, subscriber_id," - " idempotency_key, notification_id, attempt, delivery_id, consumer_namespace," - " lease_token, reservation_token, binding, fired_at, url, payload_size_bytes)" - " VALUES (%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s::jsonb,%s,%s,%s)", - ( - *key, - b.notification_id, - number, - b.delivery_id, - b.consumer_namespace, - lease.token, - reservation, - json.dumps(asdict(b)), - at, - request.url, - request.payload_size_bytes, - ), + "SELECT 1 FROM reporting_webhook_attempts WHERE account_id = %s" + " AND principal_id = %s AND consumer_namespace = %s" + " AND delivery_id = %s AND lease_token = %s", + (b.account_id, consumer, b.consumer_namespace, b.delivery_id, lease.token), ) - return WebhookAttempt(b, number, lease.token, reservation, at, request) + ).fetchone() + if duplicate is not None: + return None + number = await self._next_attempt_on(conn, b) + at = await database_now(conn, self._clock) + # The row stays locked from the fence through this commit. A later + # reclaim can never mutate this reservation, including after purge. + if at >= lease.expires_at: + # Roll back the increment as well; no reservation means no HTTP. + raise ReportingNotificationError("activity_lease_expired") + reservation = token_hex(32) + await conn.execute( + "INSERT INTO reporting_webhook_attempts (account_id, principal_id, subscriber_id," + " idempotency_key, notification_id, attempt, delivery_id, consumer_namespace," + " lease_token, reservation_token, binding, fired_at, url, payload_size_bytes)" + " VALUES (%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s::jsonb,%s,%s,%s)", + ( + *key, + b.notification_id, + number, + b.delivery_id, + b.consumer_namespace, + lease.token, + reservation, + json.dumps(asdict(b)), + at, + request.url, + request.payload_size_bytes, + ), + ) + return WebhookAttempt(b, number, lease.token, reservation, at, request) async def _next_attempt_on(self, conn: Any, binding: DeliveryBinding) -> int: b = binding diff --git a/src/adcp/reporting/outbox/memory.py b/src/adcp/reporting/outbox/memory.py index 4976696b0..ea5794453 100644 --- a/src/adcp/reporting/outbox/memory.py +++ b/src/adcp/reporting/outbox/memory.py @@ -335,6 +335,12 @@ async def finish_delivery( async def reserve_attempt( self, lease: DeliveryLease, *, request: ActivityRequest, now: datetime + ) -> WebhookAttempt | None: + async with self._store._lock: + return self._reserve_attempt_locked(lease, request=request, now=now) + + def _reserve_attempt_locked( + self, lease: DeliveryLease, *, request: ActivityRequest, now: datetime ) -> WebhookAttempt | None: binding = lease.delivery.binding consumer = canonical_consumer(binding.principal_id) @@ -346,29 +352,28 @@ async def reserve_attempt( binding.subscriber_id, binding.idempotency_key, ) - async with self._store._lock: - item = self._state.deliveries.get( - (binding.account_id, binding.consumer_namespace, binding.delivery_id) - ) - if ( - item is None - or item[0] != lease.delivery - or item[1].expires_at != lease.expires_at - or not item[1].held(lease.token, now) - ): - return None - if any( - row.lease_token == lease.token and row.binding == binding - for row in self._state.activity.values() - ): - return None - number = self._state.activity_heads.get(key, 0) + 1 - attempt = WebhookAttempt( - binding, number, lease.token, token_hex(32), aware_utc(now), request - ) - self._state.activity_heads[key] = number - self._state.activity[(*key, number)] = attempt - return attempt + item = self._state.deliveries.get( + (binding.account_id, binding.consumer_namespace, binding.delivery_id) + ) + if ( + item is None + or item[0] != lease.delivery + or item[1].expires_at != lease.expires_at + or not item[1].held(lease.token, now) + ): + return None + if any( + row.lease_token == lease.token and row.binding == binding + for row in self._state.activity.values() + ): + return None + number = self._state.activity_heads.get(key, 0) + 1 + attempt = WebhookAttempt( + binding, number, lease.token, token_hex(32), aware_utc(now), request + ) + self._state.activity_heads[key] = number + self._state.activity[(*key, number)] = attempt + return attempt async def complete_attempt( self, attempt: WebhookAttempt, *, outcome: ActivityOutcome, now: datetime diff --git a/src/adcp/reporting/outbox/status_memory.py b/src/adcp/reporting/outbox/status_memory.py index e3a9b89d0..2732a4eef 100644 --- a/src/adcp/reporting/outbox/status_memory.py +++ b/src/adcp/reporting/outbox/status_memory.py @@ -7,8 +7,12 @@ from secrets import token_hex from typing import Any, Literal +from adcp.reporting.ledger.delivery_models import ReportingDeliveryRecord from adcp.reporting.ledger.models import ReportingDeliveryEscalation -from adcp.reporting.ledger.notification_models import ReportingNotificationError +from adcp.reporting.ledger.notification_models import ( + ReportingDomainEvent, + ReportingNotificationError, +) from adcp.reporting.ledger.status_projection import ( ReportingStatusSnapshot, StatusProjectionInput, @@ -42,6 +46,11 @@ class _StatusMemoryState: class InMemoryReportingStatusOutbox(InMemoryReportingOutbox): + def __init__(self, store: InMemoryReportingLedgerStore) -> None: + if store._status_notification_state is None: + raise ValueError("construct a status projection participant first") + self._store = store + @property def _state(self) -> NotificationState: state: _StatusMemoryState = self._store._status_notification_state @@ -92,6 +101,7 @@ async def create_schema(self) -> None: pass def _cursor(self, account_id: str) -> int: + self._projection_fence(account_id) account = self._state.accounts.get(account_id) if account is None: raise ReportingNotificationError("status_baseline_required") @@ -99,8 +109,16 @@ def _cursor(self, account_id: str) -> int: raise ReportingNotificationError("status_policy_conflict") return account[0] + def _projection_fence(self, account_id: str) -> None: + if ( + account_id in getattr(self.ledger, "_projection_accounts", {}) + and getattr(self, "_projection_version", 1) != 2 + ): + raise ReportingNotificationError("status_projection_writer_fenced") + async def baseline(self, *, account_id: str) -> bool: async with self.ledger._mutation(): + self._projection_fence(account_id) if account_id in self._state.accounts: if not self._needs_rebuild(account_id): self._cursor(account_id) @@ -144,6 +162,7 @@ def _needs_rebuild(self, account_id: str) -> bool: ) def _rebuild(self, account_id: str) -> StatusTurn: + self._projection_fence(account_id) if not self._needs_rebuild(account_id): return StatusTurn(False) if self._state.selector_accounts.get(account_id) != "transitioning": @@ -200,8 +219,16 @@ async def rebuild_one(self) -> StatusTurn: return self._rebuild(account_id) if account_id is not None else StatusTurn(False) def _apply( - self, snapshot: ReportingStatusSnapshot, *, through: int, baseline: bool = False + self, + snapshot: ReportingStatusSnapshot, + *, + through: int, + baseline: bool = False, + reconciliation: tuple[ReportingDeliveryRecord, ...] | None = None, + consumer_status_enabled: bool = True, + enqueue: bool = True, ) -> int: + self._projection_fence(snapshot.account_id) snapshot = settled_replay(snapshot) events = 0 scopes = {s.checkpoint_key: s for s in projection_scopes(snapshot)} @@ -211,7 +238,15 @@ def _apply( if c.scope.account_id == snapshot.account_id ) for _, scope in sorted(scopes.items()): - result = project_status_scope(StatusProjectionInput(snapshot, scope, self.escalation)) + result = project_status_scope( + StatusProjectionInput( + snapshot, + scope, + self.escalation, + reconciliation=reconciliation, + consumer_status_enabled=consumer_status_enabled, + ) + ) checkpoint, event = advance_checkpoint( self._state.checkpoints.get(scope.checkpoint_key), result, @@ -220,11 +255,14 @@ def _apply( baseline=baseline, ) self._state.checkpoints[scope.checkpoint_key] = checkpoint - if event is not None: - self._state.outbox.enqueue(event) + if event is not None and enqueue: + self._enqueue_status(event) events += 1 return events + def _enqueue_status(self, event: ReportingDomainEvent) -> None: + self._state.outbox.enqueue(event) + def _project(self, account_id: str) -> StatusTurn: cursor = self._cursor(account_id) assert self.ledger._notification_state is not None diff --git a/src/adcp/reporting/outbox/status_pg.py b/src/adcp/reporting/outbox/status_pg.py index 4ac861d90..eb59a5cf8 100644 --- a/src/adcp/reporting/outbox/status_pg.py +++ b/src/adcp/reporting/outbox/status_pg.py @@ -17,6 +17,7 @@ from pydantic import TypeAdapter +from adcp.reporting.ledger.delivery_models import ReportingDeliveryRecord from adcp.reporting.ledger.models import ReportingDeliveryEscalation, ReportingIssueLifecycle from adcp.reporting.ledger.notification_models import ( ReportingDomainEvent, @@ -291,6 +292,9 @@ async def _apply_on( *, through: int, baseline: bool = False, + reconciliation: tuple[ReportingDeliveryRecord, ...] | None = None, + consumer_status_enabled: bool = True, + enqueue: bool = True, ) -> int: await self._lock_scopes_on(connection, snapshot) snapshot = settled_replay(snapshot) @@ -311,7 +315,15 @@ async def _apply_on( scope.checkpoint_key, ) ).fetchone() - result = project_status_scope(StatusProjectionInput(snapshot, scope, self.escalation)) + result = project_status_scope( + StatusProjectionInput( + snapshot, + scope, + self.escalation, + reconciliation=reconciliation, + consumer_status_enabled=consumer_status_enabled, + ) + ) checkpoint, event = advance_checkpoint( _checkpoint(row), result, @@ -320,11 +332,14 @@ async def _apply_on( baseline=baseline, ) await self._write_on(connection, checkpoint) - if event is not None: - await _enqueue_on(connection, event) + if event is not None and enqueue: + await self._enqueue_status_on(connection, event) count += 1 return count + async def _enqueue_status_on(self, connection: Any, event: ReportingDomainEvent) -> None: + await _enqueue_on(connection, event) + async def baseline(self, *, account_id: str) -> bool: from adcp.reporting.outbox.status_schema import validate_status_schema diff --git a/src/adcp/reporting/outbox/worker.py b/src/adcp/reporting/outbox/worker.py index 0370a5cf3..27d95ae16 100644 --- a/src/adcp/reporting/outbox/worker.py +++ b/src/adcp/reporting/outbox/worker.py @@ -7,7 +7,7 @@ from collections.abc import Callable from dataclasses import dataclass from datetime import datetime, timedelta, timezone -from typing import TYPE_CHECKING +from typing import TYPE_CHECKING, Protocol import httpx @@ -60,6 +60,25 @@ class _Outcome: class _HttpObservation: reservation: WebhookAttempt | None = None started_ns: int = 0 + retry_window_expired: bool = False + retry_deadline: datetime | None = None + + +class ReportingDeliveryWindow(Protocol): + """Optional additive SDK admission for a durable per-key retry horizon.""" + + async def inspect( + self, lease: DeliveryLease, *, now: datetime + ) -> tuple[datetime | None, bool]: ... + + async def reserve_attempt( + self, + activity: ReportingActivityStore, + lease: DeliveryLease, + *, + request: ActivityRequest, + now: datetime, + ) -> tuple[WebhookAttempt | None, datetime | None, bool]: ... class ReportingNotificationWorker: @@ -85,6 +104,7 @@ def __init__( lease_seconds: float = 60, retry_seconds: float = 5, activity: ReportingActivityStore | None = None, + delivery_window: ReportingDeliveryWindow | None = None, ) -> None: if lease_seconds < 1 or retry_seconds <= 0: raise ValueError("positive retry and at least one second of lease are required") @@ -99,6 +119,7 @@ def __init__( if activity is not None and id(activity) != id(outbox): raise ReportingNotificationError("activity_requires_reporting_outbox") self.activity = activity + self.delivery_window = delivery_window async def advertised_notifications( self, @@ -203,16 +224,23 @@ async def deliver_one(self, *, account_id: str) -> bool: ) if lease is None: return False + observation = _HttpObservation() + if self.delivery_window is not None: + observation.retry_deadline, observation.retry_window_expired = ( + await self.delivery_window.inspect(lease, now=self._clock()) + ) try: opened = self.cipher.open(lease.delivery) except (ReportingNotificationError, ValueError, TypeError): outcome = _Outcome("quarantined", "integrity_failure") else: - observation = _HttpObservation() try: - outcome = await asyncio.wait_for( - self._attempt(lease, opened, observation), timeout=self.lease_seconds * 0.8 - ) + if observation.retry_window_expired: + outcome = _Outcome("suppressed", "lease_expired") + else: + outcome = await asyncio.wait_for( + self._attempt(lease, opened, observation), timeout=self.lease_seconds * 0.8 + ) except (TimeoutError, asyncio.TimeoutError): # Worker cancellation is not an observed HTTP timeout. A # reservation remains pending until a known result is ACKed. @@ -220,6 +248,9 @@ async def deliver_one(self, *, account_id: str) -> bool: return True outcome = _Outcome("pending", "network") now = self._clock() + retry_at = now + timedelta(seconds=self.retry_seconds) + if observation.retry_deadline is not None: + retry_at = min(retry_at, observation.retry_deadline) # A DB failure after HTTP acceptance is intentionally not converted to # success. Expiry/restart retries these exact protected body bytes/key. await self.outbox.finish_delivery( @@ -227,9 +258,7 @@ async def deliver_one(self, *, account_id: str) -> bool: now=now, state=outcome.state, error_code=outcome.error, - retry_at=( - now + timedelta(seconds=self.retry_seconds) if outcome.state == "pending" else None - ), + retry_at=retry_at if outcome.state == "pending" else None, ) return True @@ -291,14 +320,25 @@ async def current_fence() -> bool: callback_used = True if not await self.outbox.delivery_lease_current(lease, now=self._clock()): return False + if self.delivery_window is not None and self.activity is None: + raise ReportingNotificationError("activity_requires_reporting_outbox") if self.activity is not None: # Signing, URL/DNS preparation, and the final fence # precede this transaction. No awaitable preparation # remains between reservation and starting peer I/O. try: - observation.reservation = await self.activity.reserve_attempt( - lease, request=request, now=self._clock() - ) + if self.delivery_window is not None: + ( + observation.reservation, + observation.retry_deadline, + observation.retry_window_expired, + ) = await self.delivery_window.reserve_attempt( + self.activity, lease, request=request, now=self._clock() + ) + else: + observation.reservation = await self.activity.reserve_attempt( + lease, request=request, now=self._clock() + ) except ReportingNotificationError as error: if error.code == "activity_lease_expired": raise PreparedWebhookAttemptExpiredError( @@ -315,7 +355,9 @@ async def current_fence() -> bool: sender, opened.prepared, before_attempt=current_fence ) except PreparedWebhookAttemptExpiredError: - return _Outcome("pending", "lease_expired") + return _Outcome( + "suppressed" if observation.retry_window_expired else "pending", "lease_expired" + ) except ReportingNotificationError: # Failed/unknown reservation commit means no HTTP and no ACK. raise diff --git a/src/adcp/reporting/ownership.py b/src/adcp/reporting/ownership.py new file mode 100644 index 000000000..9b2b61d62 --- /dev/null +++ b/src/adcp/reporting/ownership.py @@ -0,0 +1,129 @@ +"""Additive, page-local revision ownership without changing protocol schemas. + +The extension is evidence, never authorization. A revision response can check +its own binding; only a complete periods walk can establish the named owner. +""" + +from __future__ import annotations + +from collections.abc import Mapping, Sequence +from copy import deepcopy +from typing import Any + +from adcp.reporting.evidence import reporting_identifier + +_NAME = "reporting_revision_ownership" + + +class ReportingOwnershipError(ValueError): + def __init__(self) -> None: + super().__init__("invalid reporting revision ownership") + + +def page_revision_ownership(page: Mapping[str, Any]) -> dict[str, str] | None: + """Read exactly one binding per returned revision, or the absent legacy mode. + + Duplicate bindings on one page are invalid, including identical duplicates. + The full-walk accumulator may deduplicate identical records on later pages. + Empty opted-in pages must contain an explicit empty bindings array. + """ + if "ext" not in page: + return None + ext = page["ext"] + if type(ext) is not dict: + raise ReportingOwnershipError() + if "adcp" not in ext: + return None + adcp = ext["adcp"] + if type(adcp) is not dict: + raise ReportingOwnershipError() + if _NAME not in adcp: + return None + value = adcp[_NAME] + if ( + type(value) is not dict + or set(value) != {"version", "bindings"} + # A2A's protobuf Struct represents every JSON number as a double. + # JSON Schema's integer 1 includes 1.0, but never booleans or strings. + or type(value["version"]) not in {int, float} + or value["version"] != 1 + or type(value["bindings"]) is not list + ): + raise ReportingOwnershipError() + bindings: dict[str, str] = {} + for binding in value["bindings"]: + if type(binding) is not dict or set(binding) != { + "reporting_revision_id", + "reporting_obligation_id", + }: + raise ReportingOwnershipError() + revision, owner = binding["reporting_revision_id"], binding["reporting_obligation_id"] + try: + reporting_identifier(revision, maximum=255) + reporting_identifier(owner, maximum=255) + except (ValueError, TypeError): + raise ReportingOwnershipError() from None + if revision in bindings: + raise ReportingOwnershipError() + bindings[revision] = owner + revisions = _page_revisions(page) + if type(revisions) is not list or any(type(r) is not dict for r in revisions): + raise ReportingOwnershipError() + ids = [r.get("reporting_revision_id") for r in revisions] + if ( + any(type(r) is not str for r in ids) + or len(set(ids)) != len(ids) + or set(ids) != set(bindings) + ): + raise ReportingOwnershipError() + return bindings + + +def _page_revisions(page: Mapping[str, Any]) -> Any: + names = {"revision", "reporting_revision", "revisions"}.intersection(page) + if len(names) > 1: + raise ReportingOwnershipError() + for name in ("revision", "reporting_revision"): + if name not in page: + continue + revision = page[name] + if type(revision) is not dict: + raise ReportingOwnershipError() + if "reporting_revision_binding" in page: + binding = page["reporting_revision_binding"] + if type(binding) is not dict or binding.get("reporting_revision_id") != revision.get( + "reporting_revision_id" + ): + raise ReportingOwnershipError() + return [revision] + return page.get("revisions", []) + + +def with_revision_ownership(page: Mapping[str, Any], bindings: Mapping[str, str]) -> dict[str, Any]: + """Merge the reserved namespace, rejecting an existing conflicting claim. + + The input is not mutated. Callers supply the already frozen ownership map; + only bindings for this page's revisions enter the response. + """ + result = deepcopy(dict(page)) + previous = page_revision_ownership(result) + revisions: Sequence[dict[str, Any]] = _page_revisions(result) + try: + local = { + r["reporting_revision_id"]: bindings[r["reporting_revision_id"]] for r in revisions + } + except (KeyError, TypeError): + raise ReportingOwnershipError() from None + if previous is not None and previous != local: + raise ReportingOwnershipError() + ext = result.setdefault("ext", {}) + adcp = ext.setdefault("adcp", {}) + adcp[_NAME] = { + "version": 1, + "bindings": [ + {"reporting_revision_id": revision, "reporting_obligation_id": owner} + for revision, owner in local.items() + ], + } + page_revision_ownership(result) + return result diff --git a/src/adcp/reporting/production/__init__.py b/src/adcp/reporting/production/__init__.py new file mode 100644 index 000000000..c68445505 --- /dev/null +++ b/src/adcp/reporting/production/__init__.py @@ -0,0 +1,67 @@ +"""Production reporting composition with explicit, frozen provider contracts. + +Mount one support's handler, start its owned lifecycle, then activate accounts +after migration and drain. The memory implementation is for conformance and +does not claim durability. PostgreSQL uses the shared optional driver guard. +""" + +from typing import TYPE_CHECKING, Any + +from adcp.reporting.production.configuration import ( + ConfigurationAdmission, + ConfigurationTask, + ReportingConfigurationAdmission, + ReportingProductionConfigurationTask, +) +from adcp.reporting.production.contracts import ( + ReportingProductionDestinationBinding, + ReportingProductionMethod, + ReportingProductionSource, + ReportingProductionSourceBinding, +) +from adcp.reporting.production.handler import ReportingProductionHandler +from adcp.reporting.production.memory import ( + InMemoryReportingProductionOutbox, + InMemoryReportingProductionStore, +) +from adcp.reporting.production.notifications import ( + ReportingProductionSigning, + production_notification_workers, +) +from adcp.reporting.production.offerings import ReportingProductionOffering +from adcp.reporting.production.service import ( + ReportingProductionDestination, + ReportingProductionSupport, +) + +if TYPE_CHECKING: + from adcp.reporting.production.pg import PgReportingProductionOutbox, PgReportingProductionStore + +__all__ = [ + "ConfigurationAdmission", + "ConfigurationTask", + "InMemoryReportingProductionOutbox", + "InMemoryReportingProductionStore", + "PgReportingProductionOutbox", + "PgReportingProductionStore", + "ReportingConfigurationAdmission", + "ReportingProductionConfigurationTask", + "ReportingProductionDestination", + "ReportingProductionDestinationBinding", + "ReportingProductionHandler", + "ReportingProductionMethod", + "ReportingProductionOffering", + "ReportingProductionSigning", + "ReportingProductionSource", + "ReportingProductionSourceBinding", + "ReportingProductionSupport", + "production_notification_workers", +] + + +def __getattr__(name: str) -> Any: + if name in {"PgReportingProductionOutbox", "PgReportingProductionStore"}: + from adcp.reporting.production import pg + + return getattr(pg, name) + raise AttributeError(name) diff --git a/src/adcp/reporting/production/_diagnostics.py b/src/adcp/reporting/production/_diagnostics.py new file mode 100644 index 000000000..f2c852cf1 --- /dev/null +++ b/src/adcp/reporting/production/_diagnostics.py @@ -0,0 +1,43 @@ +"""Closed operator signals for an unexpectedly stopped production worker.""" + +from __future__ import annotations + +import logging +from typing import Literal + +_WorkerBoundary = Literal[ + "producer", "materializer", "projection", "sweeper", "notifications", "worker" +] +_BOUNDARIES = frozenset( + {"producer", "materializer", "projection", "sweeper", "notifications", "worker"} +) +_LOGGER = logging.getLogger("adcp.reporting.production") + + +def _boundary_label(value: object) -> str: + return value if type(value) is str and value in _BOUNDARIES else "worker" + + +def _worker_stopped(*, boundary: _WorkerBoundary) -> None: + """No exception, provider/request object or ambient context enters the record.""" + if not _LOGGER.isEnabledFor(logging.ERROR): + return + record = logging.LogRecord( + _LOGGER.name, logging.ERROR, "", 0, "Reporting production worker stopped", (), None + ) + # Bypass ambient LogRecordFactory additions. Names may themselves carry + # adopter data, so this diagnostic does not retain them or a traceback. + record.threadName = None + record.processName = None + if hasattr(record, "taskName"): + record.taskName = None + record.__dict__.update( + code="REPORTING_PRODUCTION_WORKER_STOPPED", + boundary=_boundary_label(boundary), + ) + try: + _LOGGER.handle(record) + except Exception: + # The failed/stop latch has already committed. A broken operator sink + # cannot prevent sibling shutdown or replace the closed public error. + return diff --git a/src/adcp/reporting/production/configuration.py b/src/adcp/reporting/production/configuration.py new file mode 100644 index 000000000..fd81ee1af --- /dev/null +++ b/src/adcp/reporting/production/configuration.py @@ -0,0 +1,335 @@ +"""A typed admission boundary for the application's existing account task.""" + +from __future__ import annotations + +import json +from collections.abc import Awaitable, Callable, Mapping +from dataclasses import dataclass, field +from typing import TYPE_CHECKING, Any + +from adcp.decisioning.capabilities import Account as AccountCapabilities +from adcp.reporting._timestamp import aware_timestamp +from adcp.reporting.canonical_json import canonical_json_utf8_v1 +from adcp.reporting.ledger.delivery_models import ( + ReportingDeliveryPrincipal, + ReportingDestinationBinding, +) +from adcp.reporting.ledger.models import ReportingConfiguration +from adcp.reporting.ledger.store import LedgerConflictError +from adcp.reporting.materializer.contracts import failure +from adcp.server.base import ToolContext + +if TYPE_CHECKING: + from adcp.reporting.production.service import ReportingProductionSupport + + +@dataclass(frozen=True) +class ReportingConfigurationAdmission: + """Resolved by trusted account/provider code, never decoded from buyer JSON. + + The bound references are opaque; credentials stay in destination sessions. + The SDK validates the complete frozen tuple before admitting configuration. + """ + + offering_id: str + configuration: ReportingConfiguration + binding: ReportingDestinationBinding + configuration_wire: Mapping[str, Any] = field(kw_only=True, repr=False, compare=False) + _wire: bytes = field(init=False, repr=False) + + def __post_init__(self) -> None: + from adcp.validation.schema_loader import get_named_validator + + wire = canonical_json_utf8_v1(dict(self.configuration_wire)) + validator = get_named_validator("core/reporting-delivery-config.json") + if validator is None or next(validator.iter_errors(json.loads(wire)), None) is not None: + raise ValueError("configuration must satisfy the complete public contract") + object.__setattr__(self, "_wire", wire) + + def wire(self) -> dict[str, Any]: + return dict(json.loads(self._wire)) + + def check(self, support: ReportingProductionSupport) -> None: + from adcp.reporting.ledger.store import reject_reserved_authoritative_party + + config, binding, raw = self.configuration, self.binding, self.wire() + reject_reserved_authoritative_party(config) + if raw.get("authoritative_party", "seller") != "seller": + raise LedgerConflictError("UNSUPPORTED_FEATURE", "consumer authority is reserved") + offering = support._configuration_offering(config, binding, offering_id=self.offering_id) + schedule = offering.configuration_schedule(config) + supplied_schedule = dict(raw["schedule"]) + if "period_anchor" in supplied_schedule: + supplied_schedule["period_anchor"] = aware_timestamp( + supplied_schedule["period_anchor"] + ).isoformat() + schedule["period_anchor"] = aware_timestamp(schedule["period_anchor"]).isoformat() + scope = raw["scope"] + # This producer freezes an explicit full media-buy denominator. An + # application's dynamic all-buy or partial-coverage implementation must + # not be represented as this installed exact-generation contract. + if ( + raw["delivery_config_id"] != config.delivery_config_id + or raw["delivery_config_version"] != config.delivery_config_version + or raw["offering_id"] != self.offering_id + or raw["feed_purpose"] != config.feed_purpose + or raw["report_definition_id"] != config.report_definition_id + or raw["reporting_profile"] != config.reporting_profile + or raw["required_finality"] != config.required_finality + or raw["reconciliation_mode"] != binding.reconciliation_mode + or set(scope) != {"media_buy_ids"} + or set(scope["media_buy_ids"]) != set(config.media_buy_ids) + or raw["coverage_requirement"] != "full" + or supplied_schedule != schedule + or raw.get("method") != support._destination_binding(binding, offering).wire() + or raw["active"] != (config.activated_at is not None and config.deactivated_at is None) + or ( + "revocation_effective_at" in raw + and aware_timestamp(raw["revocation_effective_at"]) != config.deactivated_at + ) + ): + raise failure("BINDING_MISMATCH") + + +ConfigurationAdmission = Callable[[ReportingConfigurationAdmission], Awaitable[None]] +ConfigurationTask = Callable[ + [dict[str, Any], ToolContext | None, ConfigurationAdmission], Awaitable[dict[str, Any]] +] + + +@dataclass(frozen=True) +class ReportingProductionConfigurationTask: + """Compose the account implementation with enforced SDK reporting admission. + + ``handle`` remains the application's authenticated, caller-owned desired + state account task. It resolves provider grants and opaque bindings, calls + the supplied ``admit`` for every accepted reporting generation, and returns + the normal account response. The wrapper verifies returned ready states + against the actual stored records and that call's validated admissions. + A task cannot return a successful unsupported promise without a matching + admitted configuration, including on exact replay. + """ + + handle: ConfigurationTask = field(repr=False) + account: AccountCapabilities = field(repr=False, compare=False) + _account_wire: bytes = field(init=False, repr=False) + + def __post_init__(self) -> None: + from adcp.validation.schema_loader import get_named_validator + + raw = self.account.model_dump(mode="json", exclude_none=True, exclude_unset=True) + validator = get_named_validator("protocol/get-adcp-capabilities-response.json") + if ( + validator is None + or next( + validator.evolve(schema=validator.schema["properties"]["account"]).iter_errors(raw), + None, + ) + is not None + or raw.get("account_financials") is True + or any( + raw.get(name, {}).get("supported") is True + for name in ("notifications", "change_feed", "identity_updates") + ) + ): + raise ValueError( + "configuration task requires its actual account capabilities and mounted operations" + ) + object.__setattr__(self, "_account_wire", canonical_json_utf8_v1(raw)) + + def account_capabilities(self) -> dict[str, Any]: + return dict(json.loads(self._account_wire)) + + async def execute( + self, + support: ReportingProductionSupport, + request: dict[str, Any], + context: ToolContext | None, + ) -> dict[str, Any]: + from adcp.validation.schema_loader import get_named_validator + + validator = get_named_validator("account/sync-accounts-request.json") + if validator is None or next(validator.iter_errors(request), None) is not None: + raise LedgerConflictError("INVALID_REQUEST", "account request is invalid") + for entry in request["accounts"]: + configurations = entry.get("reporting_delivery_configs", ()) + keys = [(c["delivery_config_id"], c["delivery_config_version"]) for c in configurations] + if len(set(keys)) != len(keys): + raise LedgerConflictError( + "INVALID_REQUEST", "configuration generations must be unique" + ) + admitted: dict[tuple[str, str, int], ReportingConfigurationAdmission] = {} + + async def admit(value: ReportingConfigurationAdmission) -> None: + if request.get("dry_run") is True: + raise LedgerConflictError( + "INVALID_REQUEST", "dry runs cannot admit reporting state" + ) + if type(value) is not ReportingConfigurationAdmission or context is None: + raise LedgerConflictError("UNAUTHORIZED", "configuration authority is unavailable") + who = await support.handler._authorize( + {"account": {"account_id": value.configuration.account_id}}, context + ) + if who != value.binding.principal: + raise LedgerConflictError("UNAUTHORIZED", "configuration authority is unavailable") + matched = False + for entry in request["accounts"]: + if "reporting_delivery_configs" not in entry or "account" not in entry: + continue + try: + requested_caller = await support.handler._authorize( + {"account": entry["account"]}, context + ) + # An unresolved account cannot authorize this admission. + except Exception: # nosec B112 + continue + if requested_caller != who: + continue + desired = entry["reporting_delivery_configs"] + matched = value.wire() in desired or ( + value.configuration.deactivated_at is not None + and not any( + (c["delivery_config_id"], c["delivery_config_version"]) + == ( + value.configuration.delivery_config_id, + value.configuration.delivery_config_version, + ) + for c in desired + ) + ) + if matched: + break + if not matched: + raise LedgerConflictError( + "INVALID_REQUEST", "configuration differs from requested account state" + ) + support.validate_configuration(value) + await support._check_notifications(who.account_id) + key = ( + who.account_id, + value.configuration.delivery_config_id, + value.configuration.delivery_config_version, + ) + if key in admitted and admitted[key] != value: + raise LedgerConflictError( + "CONFIGURATION_GENERATION_IMMUTABLE", "configuration identity conflicts" + ) + await support.store.admit_production_configuration( + value.configuration, value.binding, offering_id=value.offering_id + ) + # The caller already entered the migrated, drained production + # lifecycle. Complete this account's versioned baseline before + # echoing ready; adopters need no account-enumeration worker or + # hand-maintained readiness flag. A failed activation remains + # unready and can resume under the same durable identities. + await support.activate(account_id=who.account_id) + admitted[key] = value + + response = await self.handle(dict(request), context, admit) + if not isinstance(response, Mapping): + raise LedgerConflictError( + "INVALID_REQUEST", "configuration task returned an invalid response" + ) + if response.get("errors") or request.get("dry_run") is True: + return dict(response) + for account in response.get("accounts", ()): + if account.get("action") == "failed": + continue + for state in account.get("reporting_delivery_configs", ()): + if state.get("state") not in {"ready", "inactive"}: + continue + config = state.get("configuration", {}) + key = ( + account.get("account_id"), + config.get("delivery_config_id"), + config.get("delivery_config_version"), + ) + value = admitted.get(key) + if value is None: + raise LedgerConflictError( + "REPORTING_CONFIGURATION_UNADMITTED", "configuration requires SDK admission" + ) + validator = get_named_validator("core/reporting-delivery-config-state.json") + if ( + validator is None + or next(validator.iter_errors(state), None) is not None + or config != value.wire() + ): + raise LedgerConflictError( + "REPORTING_CONFIGURATION_UNADMITTED", + "configuration state differs from admission", + ) + expected_state = ( + "inactive" if value.configuration.deactivated_at is not None else "ready" + ) + if state["state"] != expected_state: + raise LedgerConflictError( + "REPORTING_CONFIGURATION_UNADMITTED", "configuration lifecycle differs" + ) + for name in ("activated_at", "deactivated_at"): + actual_time = getattr(value.configuration, name) + supplied = state.get(name) + if supplied is not None and aware_timestamp(supplied) != actual_time: + raise LedgerConflictError( + "REPORTING_CONFIGURATION_UNADMITTED", "configuration lifecycle differs" + ) + coverage = state.get("current_coverage") + if state["state"] == "ready" and ( + not isinstance(coverage, dict) + or coverage["status"] != "full" + or set(coverage["media_buy_ids"]) != set(value.configuration.media_buy_ids) + or set(coverage["fully_covered_media_buy_ids"]) + != set(value.configuration.media_buy_ids) + or any( + coverage[k] + for k in ( + "partially_covered_media_buy_ids", + "unsupported_media_buy_ids", + "unknown_media_buy_ids", + "unsupported_package_ids", + "unknown_package_ids", + "limitations", + ) + ) + or set(coverage["package_ids"]) != set(coverage["covered_package_ids"]) + ): + raise LedgerConflictError( + "REPORTING_CONFIGURATION_UNADMITTED", "configuration coverage differs" + ) + if state["state"] == "inactive": + from adcp.reporting.ledger.models import derive_period, first_ordinal_after + + configuration = value.configuration + assert configuration.deactivated_at is not None + ordinal = first_ordinal_after( + configuration.schedule, + account_timezone=configuration.account_timezone, + activated_at=configuration.deactivated_at, + ) + cutoff = derive_period( + configuration.schedule, + account_timezone=configuration.account_timezone, + ordinal=ordinal, + ).start + if aware_timestamp(state["publication_stopped_at"]) != cutoff: + raise LedgerConflictError( + "REPORTING_CONFIGURATION_UNADMITTED", "configuration cutoff differs" + ) + who = ReportingDeliveryPrincipal( + value.configuration.account_id, value.binding.consumer_id + ) + actual = await support.store.get_destination_binding( + caller=who, generation_key=value.configuration.generation_key + ) + configs = await support.store.list_configurations(account_id=who.account_id) + support.validate_configuration(value) + if actual != value.binding or value.configuration not in configs: + raise LedgerConflictError( + "REPORTING_CONFIGURATION_UNADMITTED", + "configuration admission is unavailable", + ) + if state.get("destination_ref") != actual.destination_ref: + raise LedgerConflictError( + "REPORTING_CONFIGURATION_UNADMITTED", "configuration destination differs" + ) + return dict(response) diff --git a/src/adcp/reporting/production/contracts.py b/src/adcp/reporting/production/contracts.py new file mode 100644 index 000000000..3e2e1f6d9 --- /dev/null +++ b/src/adcp/reporting/production/contracts.py @@ -0,0 +1,221 @@ +"""Provider declarations and trusted, immutable source generation bindings.""" + +from __future__ import annotations + +import hashlib +import json +from collections.abc import Mapping +from dataclasses import dataclass, field +from typing import Any, Protocol, runtime_checkable + +from adcp.reporting.canonical_json import canonical_json_utf8_v1 +from adcp.reporting.ledger.delivery_models import ReportingDestinationBinding +from adcp.reporting.ledger.models import ReportingConfiguration, ReportingConfigurationGenerationKey +from adcp.reporting.ledger.store import _config_payload +from adcp.reporting.materializer.contracts import ReportingWriterCapability, failure +from adcp.reporting.source import ( + MediaBuyConstituentV1, + ReportingConstituent, + ReportingSourceCapabilitiesV1, + ReportingSourceExecutor, +) + + +@dataclass(frozen=True) +class ReportingProductionMethod: + """The actual provider's complete public method for one writer capability. + + This includes the provider, destination modes, access/producer identity and + reader requirements. A method advertised by an offering must equal this + declaration. The copied bytes cannot change through an adopter's mapping. + Runtime writer, resolver, verifier and authorization checks remain required. + """ + + capability: ReportingWriterCapability + method: Mapping[str, Any] = field(repr=False, compare=False) + _wire: bytes = field(init=False, repr=False) + + def __post_init__(self) -> None: + from adcp.validation.schema_loader import get_named_validator + + raw = json.loads(canonical_json_utf8_v1(dict(self.method))) + validator = get_named_validator("core/reporting-delivery-offering.json") + if ( + validator is None + or next( + validator.evolve(schema=validator.schema["properties"]["method"]).iter_errors(raw), + None, + ) + is not None + or raw.get("orchestration") != "producer_managed" + or (raw.get("pattern"), raw.get("transport"), raw.get("format")) + != (self.capability.method, self.capability.transport, self.capability.format) + ): + raise ValueError("production method must match the provider's exact writer capability") + object.__setattr__(self, "_wire", canonical_json_utf8_v1(raw)) + + def wire(self) -> dict[str, Any]: + return dict(json.loads(self._wire)) + + +@dataclass(frozen=True) +class ReportingProductionDestinationBinding: + """The provider's authorized, immutable destination contract for a caller. + + ``configuration`` is the complete secret-free public delivery method, + including the selected destination. The provider resolves it from its + trusted binding, not from a buyer's claim. Credentials remain in the + independently authorized write/readback sessions. The SDK freezes these + bytes at admission and compares them again before materialization. + """ + + binding: ReportingDestinationBinding = field(repr=False) + method: ReportingProductionMethod + configuration: Mapping[str, Any] = field(repr=False, compare=False) + _wire: bytes = field(init=False, repr=False) + + def __post_init__(self) -> None: + from adcp.reporting.evidence import consumer_reference, resource_location + from adcp.validation.schema_loader import get_named_validator + + if ( + type(self.binding) is not ReportingDestinationBinding + or type(self.method) is not ReportingProductionMethod + ): + raise ValueError("destination requires the exact provider binding and method") + raw = json.loads(canonical_json_utf8_v1(dict(self.configuration))) + validator = get_named_validator("core/reporting-delivery-method.json") + offered = self.method.wire() + if ( + validator is None + or next(validator.iter_errors(raw), None) is not None + or any(raw.get(k) != offered.get(k) for k in ("pattern", "transport", "orchestration")) + or (raw["pattern"] == "file_transfer" and raw["format"] != offered.get("format")) + or raw["destination"]["mode"] not in offered["destination_modes"] + ): + raise ValueError("destination must match the provider's complete method") + destination = raw["destination"] + if destination["mode"] == "existing": + if destination["destination_ref"] != self.binding.destination_ref: + raise ValueError("destination must match the immutable binding") + elif destination.get("provider") != offered.get("provider") or destination.get( + "access_mode" + ) != offered.get("access_mode"): + raise ValueError("destination must match the provider's complete method") + if "location" in destination: + resource_location(destination["location"]) + if "recipient" in destination: + consumer_reference(destination["recipient"]["identity"]) + object.__setattr__(self, "_wire", canonical_json_utf8_v1(raw)) + + def wire(self) -> dict[str, Any]: + return dict(json.loads(self._wire)) + + +@dataclass(frozen=True) +class ReportingProductionSourceBinding: + """Trusted account-to-source mapping, fixed for a configuration generation. + + ``media_buy_products`` comes from the authenticated source/account mapping, + never from buyer JSON or a report-definition identifier. The SDK persists + it with admission, checks the effective capability digest on every source + turn, and refuses a changed mapping. Reauthorization can withdraw a binding; + it cannot silently change historical scope. No credentials belong here. + """ + + generation_key: ReportingConfigurationGenerationKey + capabilities_sha256: str + media_buy_products: tuple[tuple[str, str], ...] + configuration_sha256: str = field(kw_only=True) + + def __post_init__(self) -> None: + from adcp.reporting.evidence import reporting_identifier, sha256_value + + if type(self.generation_key) is not ReportingConfigurationGenerationKey: + raise ValueError("source binding requires an exact configuration generation") + sha256_value(self.capabilities_sha256) + sha256_value(self.configuration_sha256) + pairs = tuple(tuple(pair) for pair in self.media_buy_products) + if any(len(pair) != 2 for pair in pairs): + raise ValueError("source binding requires media-buy/product pairs") + for media_buy_id, product_id in pairs: + reporting_identifier(media_buy_id, maximum=255) + reporting_identifier(product_id, maximum=255) + if len({pair[0] for pair in pairs}) != len(pairs): + raise ValueError("source binding media buys must be unique") + object.__setattr__(self, "media_buy_products", tuple(sorted(pairs))) + + @classmethod + def for_configuration( + cls, + configuration: ReportingConfiguration, + *, + capabilities_sha256: str, + media_buy_products: tuple[tuple[str, str], ...], + ) -> ReportingProductionSourceBinding: + """Freeze the exact generation semantics and explicitly resolved products. + + Lifecycle changes retain the same semantic generation, matching the + ledger's immutable configuration contract. Captured projection inputs + independently retain each historical activation/deactivation boundary. + """ + return cls( + configuration.generation_key, + capabilities_sha256, + media_buy_products, + configuration_sha256=hashlib.sha256( + canonical_json_utf8_v1(_config_payload(configuration)) + ).hexdigest(), + ) + + def document(self) -> dict[str, Any]: + key = self.generation_key + return { + "account_id": key.account_id, + "delivery_config_id": key.delivery_config_id, + "delivery_config_version": key.delivery_config_version, + "capabilities_sha256": self.capabilities_sha256, + "configuration_sha256": self.configuration_sha256, + "media_buy_products": [list(pair) for pair in self.media_buy_products], + } + + def check( + self, + configuration: ReportingConfiguration, + capabilities: ReportingSourceCapabilitiesV1, + offering_id: str, + ) -> None: + offering = capabilities.offering(offering_id) + if ( + self.generation_key != configuration.generation_key + or self.configuration_sha256 + != hashlib.sha256(canonical_json_utf8_v1(_config_payload(configuration))).hexdigest() + or capabilities.scope != "effective_account" + or self.capabilities_sha256 != capabilities.capabilities_sha256 + or {pair[0] for pair in self.media_buy_products} != set(configuration.media_buy_ids) + or (self.media_buy_products and "media_buy" not in offering.constituent_kinds) + or any(pair[1] not in offering.product_ids for pair in self.media_buy_products) + ): + raise failure("BINDING_MISMATCH") + + def constituents(self) -> tuple[ReportingConstituent, ...]: + return tuple( + MediaBuyConstituentV1( + constituent_id=media_buy_id, media_buy_id=media_buy_id, product_id=product_id + ) + for media_buy_id, product_id in self.media_buy_products + ) + + +@runtime_checkable +class ReportingProductionSource(ReportingSourceExecutor, Protocol): + """A source with an authenticated generation mapping available before I/O. + + Discovery may precede any account binding. Admission and every source turn + require the applicable binding, obtained from the source's trusted account + configuration. Returning ``None`` withdraws authorization for new work. + """ + + def configuration_binding( + self, configuration: ReportingConfiguration + ) -> ReportingProductionSourceBinding | None: ... diff --git a/src/adcp/reporting/production/delivery_window.py b/src/adcp/reporting/production/delivery_window.py new file mode 100644 index 000000000..244e0cd1e --- /dev/null +++ b/src/adcp/reporting/production/delivery_window.py @@ -0,0 +1,160 @@ +"""Immutable first-attempt deadlines for the three production delivery queues.""" + +from __future__ import annotations + +from dataclasses import dataclass +from datetime import datetime, timedelta +from typing import TYPE_CHECKING + +from adcp.reporting.evidence import aware_utc +from adcp.reporting.ledger.notification_models import ReportingNotificationError +from adcp.reporting.outbox.activity import ActivityRequest, ReportingActivityStore, WebhookAttempt +from adcp.reporting.outbox.memory import InMemoryReportingOutbox +from adcp.reporting.outbox.models import DeliveryLease +from adcp.reporting.production.memory import InMemoryReportingProductionStore + +if TYPE_CHECKING: + from adcp.reporting.production.pg import PgReportingProductionStore + +RETRY_HORIZON_SECONDS = 86400 + + +@dataclass(frozen=True) +class ProductionDeliveryWindow: + store: InMemoryReportingProductionStore | PgReportingProductionStore + queue: str + + @staticmethod + def _inspect( + saved: tuple[str, str, datetime, datetime] | None, + expected: tuple[str, str], + moment: datetime, + ) -> tuple[datetime | None, bool]: + if saved is None: + return None, False + if saved[:2] != expected or moment < saved[2]: + raise ReportingNotificationError("notification_retry_unready") + return saved[3], moment >= saved[3] + + async def inspect(self, lease: DeliveryLease, *, now: datetime) -> tuple[datetime | None, bool]: + binding = lease.delivery.binding + key = (binding.account_id, binding.idempotency_key) + expected = (self.queue, binding.body_sha256) + if isinstance(self.store, InMemoryReportingProductionStore): + async with self.store._lock: + return self._inspect( + self.store._production_delivery_windows.get(key), expected, aware_utc(now) + ) + from adcp.reporting.outbox.pg import database_now + + try: + async with self.store._connection() as connection, connection.transaction(): + saved = await ( + await connection.execute( + "SELECT queue,body_sha256,started_at,expires_at" + " FROM reporting_production_delivery_windows" + " WHERE account_id=%s AND idempotency_key=%s", + key, + ) + ).fetchone() + return self._inspect( + saved, expected, await database_now(connection, self.store._clock) + ) + except ReportingNotificationError: + raise + except Exception: + raise ReportingNotificationError("notification_retry_unready") from None + + async def reserve_attempt( + self, + activity: ReportingActivityStore, + lease: DeliveryLease, + *, + request: ActivityRequest, + now: datetime, + ) -> tuple[WebhookAttempt | None, datetime | None, bool]: + """Commit the immutable deadline with the original SDK HTTP reservation. + + False admission never produces an activity ordinal. A transaction + failing after either insertion rolls back the window, ordinal head and + attempt together. Unknown commit outcomes leave the original identity. + """ + binding = lease.delivery.binding + key = (binding.account_id, binding.idempotency_key) + expected = (self.queue, binding.body_sha256) + if self.queue not in {"core", "status", "ready"}: + raise ReportingNotificationError("notification_retry_unready") + if isinstance(self.store, InMemoryReportingProductionStore): + if ( + not isinstance(activity, InMemoryReportingOutbox) + or activity._store is not self.store + ): + raise ReportingNotificationError("notification_retry_unready") + async with self.store._mutation(): + at = aware_utc(now) + saved = self.store._production_delivery_windows.get(key) + deadline, expired = self._inspect(saved, expected, at) + if expired: + return None, deadline, True + attempt = activity._reserve_attempt_locked(lease, request=request, now=at) + if attempt is None: + return None, deadline, False + retained = self.store._production_delivery_windows.setdefault( + key, + ( + *expected, + attempt.fired_at, + attempt.fired_at + timedelta(seconds=RETRY_HORIZON_SECONDS), + ), + ) + return attempt, retained[3], False + + from adcp.reporting.outbox.pg import PgReportingOutbox, database_now + + if not isinstance(activity, PgReportingOutbox) or activity._pool is not self.store._pool: + raise ReportingNotificationError("notification_retry_unready") + try: + # The activity participant supplies its exact queue adapter and + # connection. Take the inherited account lock before its row lock. + async with activity._activity_transaction() as connection: + await self.store._lock_account(connection, binding.account_id) + moment = await database_now(connection, self.store._clock) + saved = await ( + await connection.execute( + "SELECT queue,body_sha256,started_at,expires_at" + " FROM reporting_production_delivery_windows" + " WHERE account_id=%s AND idempotency_key=%s", + key, + ) + ).fetchone() + deadline, expired = self._inspect(saved, expected, moment) + if expired: + return None, deadline, True + attempt = await activity._reserve_attempt_on(connection, lease, request=request) + if attempt is None: + return None, deadline, False + deadline, expired = self._inspect(saved, expected, attempt.fired_at) + if expired: + # Time can cross the deadline while reserving an ordinal. + # Roll back that provisional head and attempt, too. + raise ReportingNotificationError("notification_retry_expired") + await connection.execute( + "INSERT INTO reporting_production_delivery_windows" + " (account_id,idempotency_key,queue,body_sha256,started_at,expires_at)" + " VALUES(%s,%s,%s,%s,%s,%s) ON CONFLICT DO NOTHING", + ( + *key, + *expected, + attempt.fired_at, + attempt.fired_at + timedelta(seconds=RETRY_HORIZON_SECONDS), + ), + ) + return ( + attempt, + deadline or attempt.fired_at + timedelta(seconds=RETRY_HORIZON_SECONDS), + False, + ) + except ReportingNotificationError as error: + if error.code == "notification_retry_expired": + return None, deadline, True + raise diff --git a/src/adcp/reporting/production/handler.py b/src/adcp/reporting/production/handler.py new file mode 100644 index 000000000..6a77df295 --- /dev/null +++ b/src/adcp/reporting/production/handler.py @@ -0,0 +1,322 @@ +"""Authenticated production routes with private, revision-bound exact reads.""" + +from __future__ import annotations + +import hashlib +from typing import TYPE_CHECKING, Any + +from adcp.decisioning.context import RequestContext +from adcp.exceptions import ADCPTaskError +from adcp.reporting.canonical_json import canonical_json_utf8_v1 +from adcp.reporting.ledger.consumer_status import ConsumerStatusIngest +from adcp.reporting.ledger.delivery_models import ReportingDeliveryPrincipal +from adcp.reporting.ledger.notification_models import ReportingNotificationError +from adcp.reporting.ledger.store import LedgerConflictError, decode_cursor, encode_cursor +from adcp.reporting.receipts.errors import ReportingReceiptError +from adcp.reporting.receipts.handler import ( + ReceiptAccountResolver, + ReportingReceiptHandler, + _consumer, +) +from adcp.server.base import NotImplementedResponse, ToolContext +from adcp.server.responses import capabilities_response +from adcp.types import ( + Error, + GetAdcpCapabilitiesRequest, + GetMediaBuyDeliveryRequest, + GetReportingStatusRequest, + SyncAccountsRequest, + SyncReportingReceiptsRequest, + SyncReportingStatusRequest, +) + +if TYPE_CHECKING: + from adcp.decisioning.registry import BuyerAgentRegistry + from adcp.reporting.production.service import ReportingProductionSupport + + +def _request(value: Any) -> dict[str, Any]: + return ( + dict(value) + if isinstance(value, dict) + else dict(value.model_dump(mode="json", exclude_unset=True)) + ) + + +def _task_error(task: str, code: str, message: str) -> ADCPTaskError: + return ADCPTaskError(operation=task, errors=[Error(code=code, message=message)]) + + +class ReportingProductionHandler(ReportingReceiptHandler): + """Mount the same instance on MCP/A2A; the support owns its lifecycle.""" + + advertised_tools = { + "get_adcp_capabilities", + "get_reporting_status", + "get_media_buy_delivery", + "sync_accounts", + "sync_reporting_receipts", + "sync_reporting_status", + } + + def __init__( + self, + production: ReportingProductionSupport, + *, + resolve_account: ReceiptAccountResolver, + buyer_agents: BuyerAgentRegistry | None = None, + adcp_version: str | None = None, + ) -> None: + self.production = production + super().__init__( + production.store, + resolve_account=resolve_account, + buyer_agents=buyer_agents, + consumer_status_enabled=production.projection.consumer_status_enabled, + adcp_version=adcp_version, + ) + + def advertised_tools_for_instance(self) -> set[str]: + names = { + "get_adcp_capabilities", + "get_reporting_status", + "get_media_buy_delivery", + "sync_accounts", + } + if any(o.reconciled for o in self.production.offerings): + names.add("sync_reporting_receipts") + if self._feed_consumer_status_enabled: + names.add("sync_reporting_status") + return names + + async def get_reporting_status( + self, + params: GetReportingStatusRequest | dict[str, Any], + context: ToolContext | None = None, + ) -> dict[str, Any] | NotImplementedResponse: + return await super().get_reporting_status(params, context) + + async def sync_reporting_receipts( + self, + params: SyncReportingReceiptsRequest | dict[str, Any], + context: ToolContext | None = None, + ) -> dict[str, Any]: + if not any(o.reconciled for o in self.production.offerings): + raise _task_error( + "sync_reporting_receipts", "NOT_SUPPORTED", "receipt task is unavailable" + ) + return await super().sync_reporting_receipts(params, context) + + async def _authorize( + self, request: dict[str, Any], context: ToolContext | None + ) -> ReportingDeliveryPrincipal: + try: + if context is None or not isinstance(request.get("account"), dict): + raise ReportingReceiptError("UNAUTHORIZED") + consumer = await _consumer(context, self._receipt_registry) + account = await self._receipt_account_resolver( + dict(request["account"]), context, consumer + ) + if isinstance(context, RequestContext) and context.account.id != account: + raise ReportingReceiptError("UNAUTHORIZED") + return ReportingDeliveryPrincipal(account, consumer) + except Exception: + raise ReportingReceiptError("UNAUTHORIZED") from None + + async def get_adcp_capabilities( + self, + params: GetAdcpCapabilitiesRequest | dict[str, Any], + context: ToolContext | None = None, + ) -> dict[str, Any]: + response = capabilities_response( + ["media_buy"], + sandbox=False, + idempotency={"supported": False}, + adcp_version=self.production._protocol_version, + supported_versions=[self.production._protocol_version], + ) + response["account"] = self.production.configuration_task.account_capabilities() + reporting = await self.production.reporting_delivery() + if reporting: + response["media_buy"] = {"reporting_delivery": reporting} + response["experimental_features"] = ["media_buy.reporting_delivery"] + if any( + reporting.get(k) + for k in ("ledger_notification", "status_notification", "readiness_notification") + ): + from adcp.reporting.production.notifications import ( + signing_capabilities, + signing_identity, + ) + + response["webhook_signing"] = signing_capabilities(self.production) + response["identity"] = signing_identity(self.production) + return response + + async def sync_accounts( + self, + params: SyncAccountsRequest | dict[str, Any], + context: ToolContext | None = None, + ) -> dict[str, Any]: + try: + self.production._assert_components() + return await self.production.configuration_task.execute( + self.production, _request(params), context + ) + except LedgerConflictError as error: + code, message = error.code, str(error) + except ReportingReceiptError as error: + code, message = error.code, str(error) + except Exception: + code, message = ( + "REPORTING_CONFIGURATION_UNAVAILABLE", + "reporting configuration is unavailable", + ) + raise _task_error("sync_accounts", code, message) + + async def sync_reporting_status( + self, + params: SyncReportingStatusRequest | dict[str, Any], + context: ToolContext | None = None, + ) -> dict[str, Any] | NotImplementedResponse: + if not self._feed_consumer_status_enabled: + return self._not_supported("sync_reporting_status") + try: + request = _request(params) + caller = await self._authorize(request, context) + result = await ConsumerStatusIngest(self.production.store, enabled=True).handle( + request, + account_id=caller.account_id, + consumer_id=caller.consumer_id, + ) + if await self._authorize(request, context) != caller: + raise ReportingReceiptError("UNAUTHORIZED") + return result + except (ReportingReceiptError, LedgerConflictError) as error: + code, message = error.code, str(error) + except Exception: + code, message = "REPORTING_STATUS_UNAVAILABLE", "reporting status is unavailable" + raise _task_error("sync_reporting_status", code, message) + + async def get_media_buy_delivery( + self, + params: GetMediaBuyDeliveryRequest | dict[str, Any], + context: ToolContext | None = None, + ) -> dict[str, Any] | NotImplementedResponse: + request = _request(params) + if "reporting_revision_id" not in request: + return self._not_supported("get_media_buy_delivery") + try: + from adcp.validation.schema_loader import get_named_validator + + validator = get_named_validator("media-buy/get-media-buy-delivery-request.json") + if validator is None or next(validator.iter_errors(request), None) is not None: + raise LedgerConflictError("INVALID_REQUEST", "exact revision request is invalid") + caller = await self._authorize(request, context) + revision_id = request["reporting_revision_id"] + status_request = { + "account": request["account"], + "view": "revision", + "reporting_revision_id": revision_id, + } + # This is the actual private mounted status path, including exact + # ownership/visibility and complete captured reconciliation checks. + projected = await self.get_reporting_status(status_request, context) + if not isinstance(projected, dict) or "revision" not in projected: + raise LedgerConflictError("LOOKUP_UNAVAILABLE", "no such revision is available") + store = self.production.store + revision = await store.get_revision( + account_id=caller.account_id, reporting_revision_id=revision_id + ) + if revision is None or not revision.readable: + raise LedgerConflictError("LOOKUP_UNAVAILABLE", "no such revision is available") + pagination = request.get("pagination") or {} + limit = pagination.get("max_results", 50) + if type(limit) is not int or not 1 <= limit <= 100: + raise LedgerConflictError("INVALID_REQUEST", "exact revision page size is invalid") + binding_hash = hashlib.sha256( + canonical_json_utf8_v1( + { + "account": caller.account_id, + "consumer": caller.consumer_id, + "revision": revision_id, + "digest": revision.revision_content_sha256, + "count": revision.row_count, + "limit": limit, + } + ) + ).hexdigest() + token = pagination.get("cursor") + offset = 0 + if token is not None: + if not isinstance(token, str) or not token.startswith("rpr2.") or len(token) > 2048: + raise LedgerConflictError( + "INVALID_CURSOR", "exact revision cursor is unavailable" + ) + cursor = decode_cursor(token[5:]) + if ( + set(cursor) != {"v", "h", "p"} + or cursor["v"] != 2 + or cursor["h"] != binding_hash + or type(cursor["p"]) is not int + or not 0 <= cursor["p"] <= revision.row_count + ): + raise LedgerConflictError( + "INVALID_CURSOR", "exact revision cursor is unavailable" + ) + offset = cursor["p"] + page = await store.read_revision_rows( + account_id=caller.account_id, + reporting_revision_id=revision_id, + limit=limit, + cursor=( + encode_cursor({"revision": revision_id, "offset": offset}) if offset else None + ), + ) + if page.total_count != revision.row_count or page.reporting_revision_id != revision_id: + raise ReportingNotificationError("reporting_revision_content_unavailable") + if await self._authorize(request, context) != caller: + raise ReportingReceiptError("UNAUTHORIZED") + after = await store.get_revision( + account_id=caller.account_id, reporting_revision_id=revision_id + ) + if after != revision: + raise LedgerConflictError("LOOKUP_UNAVAILABLE", "no such revision is available") + wire = projected["revision"] + totals = ( + [r.to_wire() for r in revision.managed_control_totals] + if revision.managed_control_totals is not None + else [{"name": n, "value": v} for n, v in revision.control_totals] + ) + position: dict[str, Any] = {"total_count": page.total_count, "has_more": page.has_more} + if page.has_more: + position["cursor"] = "rpr2." + encode_cursor( + {"v": 2, "h": binding_hash, "p": offset + len(page.rows)} + ) + result = { + "status": "completed", + "reporting_period": { + "start": wire["period"]["start"], + "end": wire["period"]["end"], + }, + "media_buy_deliveries": [], + "reporting_revision": wire, + "reporting_revision_binding": { + "reporting_revision_id": revision_id, + "row_count": revision.row_count, + "control_totals": totals, + "content_sha256": revision.revision_content_sha256, + }, + "reporting_rows": list(page.rows), + "pagination": position, + } + if "ext" in projected: + result["ext"] = projected["ext"] + return result + except ADCPTaskError: + raise + except (ReportingReceiptError, LedgerConflictError) as error: + code, message = error.code, str(error) + except Exception: + code, message = "REPORTING_CONTENT_UNAVAILABLE", "reporting content is unavailable" + raise _task_error("get_media_buy_delivery", code, message) diff --git a/src/adcp/reporting/production/memory.py b/src/adcp/reporting/production/memory.py new file mode 100644 index 000000000..b804a23bc --- /dev/null +++ b/src/adcp/reporting/production/memory.py @@ -0,0 +1,380 @@ +"""Reference production transactions for shared state-machine conformance.""" + +from __future__ import annotations + +import json +import weakref +from dataclasses import dataclass +from datetime import datetime +from typing import TYPE_CHECKING, Any + +from adcp.reporting.canonical_json import canonical_json_utf8_v1 +from adcp.reporting.ledger.delivery_models import ( + ReportingDeliveryPrincipal, + ReportingDestinationBinding, + ReportingMaterializationRecord, +) +from adcp.reporting.ledger.models import ( + ReportingConfiguration, + ReportingConfigurationGenerationKey, + ReportingObligationRecord, +) +from adcp.reporting.ledger.notification_models import ( + ReportingDomainEvent, + ReportingNotificationError, +) +from adcp.reporting.ledger.producer_progress import acquisition_state, check_next_period +from adcp.reporting.ledger.store import LedgerConflictError +from adcp.reporting.materializer.capture import ReportingMaterializerBoundary +from adcp.reporting.materializer.contracts import ReportingVerificationKey, failure +from adcp.reporting.materializer.work import MaterializerContext, ReportingMaterializerLease +from adcp.reporting.outbox.memory import InMemoryReportingOutbox, NotificationState +from adcp.reporting.production.contracts import ReportingProductionSourceBinding +from adcp.reporting.projection.memory import InMemoryReportingProjectionStore +from adcp.reporting.source import ReportingConstituent + +if TYPE_CHECKING: + from adcp.reporting.materializer.memory import _Work + from adcp.reporting.production.service import ReportingProductionSupport + + +@dataclass(frozen=True) +class _Admission: + activated_at: datetime + policy: dict[str, Any] + + +@dataclass(frozen=True) +class _SourceWork: + obligation: ReportingObligationRecord + turn: int = 0 + state: str = "pending" + + +class InMemoryReportingProductionOutbox(InMemoryReportingOutbox): + """The ordinary SDK expansion/delivery state machine, in a separate collection.""" + + _store: InMemoryReportingProductionStore + + def __init__(self, store: InMemoryReportingProductionStore) -> None: + if store._production_outbox is None: + raise ReportingNotificationError("notifications_disabled") + self._store = store + + @property + def _state(self) -> NotificationState: + assert self._store._production_outbox is not None + return self._store._production_outbox + + +class InMemoryReportingProductionStore(InMemoryReportingProjectionStore): + """Matches production atomicity and epochs, without claiming durable storage.""" + + _materializer_writer_epoch = 2 + _production_support: weakref.ReferenceType[ReportingProductionSupport] | None = None + + def __init__(self, **kwargs: Any) -> None: + super().__init__(**kwargs) + self._production_accounts: dict[str, _Admission] = {} + self._production_delivery_windows: dict[ + tuple[str, str], tuple[str, str, datetime, datetime] + ] = {} + self._production_generations: dict[ReportingConfigurationGenerationKey, str] = {} + self._production_source_bindings: dict[ + ReportingConfigurationGenerationKey, ReportingProductionSourceBinding + ] = {} + self._production_destination_bindings: dict[ + tuple[ReportingConfigurationGenerationKey, str], bytes + ] = {} + self._production_outbox = ( + NotificationState() if self._notification_state is not None else None + ) + self._production_status_heads: dict[ReportingDeliveryPrincipal, int] = {} + self._production_boundaries: list[ReportingMaterializerBoundary] = [] + self._production_closed: dict[ReportingConfigurationGenerationKey, datetime] = {} + self._production_source_turns: dict[ReportingConfigurationGenerationKey, int] = {} + self._production_source_work: dict[str, _SourceWork] = {} + + def _owner(self) -> ReportingProductionSupport: + from adcp.reporting.production.service import production_owner + + return production_owner(self) + + async def admit_production_configuration( + self, + configuration: ReportingConfiguration, + binding: ReportingDestinationBinding, + *, + offering_id: str, + ) -> None: + offering = self._owner()._configuration_offering( + configuration, binding, offering_id=offering_id + ) + async with self._mutation(): + await self.put_configuration(configuration) + await self.put_destination_binding(binding) + self._enroll(configuration, offering._producer_key, binding) + + def _enroll( + self, + configuration: ReportingConfiguration, + producer_key: str, + destination: ReportingDestinationBinding, + ) -> None: + binding = self._owner()._source_binding(configuration, producer_key) + previous = self._production_generations.setdefault( + configuration.generation_key, producer_key + ) + previous_binding = self._production_source_bindings.setdefault( + configuration.generation_key, binding + ) + if previous != producer_key or previous_binding != binding: + raise ReportingNotificationError("reporting_production_source_conflict") + owner = self._owner() + offering = owner._configuration_offering( + configuration, destination, producer_key=producer_key + ) + document = canonical_json_utf8_v1(owner._destination_binding(destination, offering).wire()) + original = self._production_destination_bindings.setdefault( + (configuration.generation_key, destination.consumer_id), document + ) + if original != document: + raise ReportingNotificationError("reporting_production_destination_conflict") + + def _destination_document(self, binding: ReportingDestinationBinding) -> dict[str, Any] | None: + raw = self._production_destination_bindings.get( + (binding.generation_key, binding.consumer_id) + ) + return dict(json.loads(raw)) if raw is not None else None + + def _configuration_lease_eligible(self, configuration: ReportingConfiguration) -> bool: + if configuration.account_id not in self._production_accounts: + return False + producer_key = self._production_generations.get(configuration.generation_key) + binding = self._production_source_bindings.get(configuration.generation_key) + if producer_key not in self._owner()._producer_keys() or binding is None: + return False + try: + self._owner()._check_source_binding(configuration, producer_key, binding.document()) + except Exception: + return False + return True + + def _check_source_generation(self, configuration: ReportingConfiguration) -> None: + if ( + not self._configuration_lease_eligible(configuration) + or self._configurations.get(configuration.generation_key) != configuration + ): + raise LedgerConflictError("HISTORY_UNAVAILABLE", "producer generation is unavailable") + + def _wake_obligation(self, account_id: str, obligation_id: str) -> None: + super()._wake_obligation(account_id, obligation_id) + obligation = self._obligations.get(obligation_id) + if ( + obligation is not None + and obligation.account_id == account_id + and obligation.generation_key in self._production_generations + ): + previous = self._production_source_work.get(obligation_id) + self._production_source_work[obligation_id] = _SourceWork( + obligation, previous.turn if previous else 0 + ) + + async def producer_closed_through( + self, configuration: ReportingConfiguration + ) -> datetime | None: + async with self._lock: + self._check_source_generation(configuration) + return self._production_closed.get(configuration.generation_key) + + async def producer_constituents( + self, configuration: ReportingConfiguration, obligation: ReportingObligationRecord + ) -> tuple[ReportingConstituent, ...]: + async with self._lock: + self._check_source_generation(configuration) + if obligation.generation_key != configuration.generation_key or set( + obligation.media_buy_ids + ) != set(configuration.media_buy_ids): + raise LedgerConflictError("HISTORY_UNAVAILABLE", "source denominator differs") + return self._production_source_bindings[configuration.generation_key].constituents() + + async def commit_producer_period( + self, + configuration: ReportingConfiguration, + obligation: ReportingObligationRecord, + *, + previous_end: datetime | None, + ) -> ReportingObligationRecord: + check_next_period(configuration, obligation, previous_end) + async with self._mutation(): + self._check_source_generation(configuration) + current = self._production_closed.get(configuration.generation_key) + if current != previous_end and (current is None or current < obligation.period.end): + raise LedgerConflictError("HISTORY_UNAVAILABLE", "producer progress changed") + stored = await self.commit_obligation(obligation) + self._production_source_work.setdefault( + stored.reporting_obligation_id, _SourceWork(stored) + ) + self._production_closed[configuration.generation_key] = max( + obligation.period.end, current or obligation.period.end + ) + return stored + + async def next_producer_obligations( + self, configuration: ReportingConfiguration, *, now: datetime, limit: int + ) -> tuple[str, ...]: + if type(limit) is not int or not 1 <= limit <= 64: + raise ValueError("production acquisition limit must be in 1..64") + async with self._lock: + self._check_source_generation(configuration) + candidates = sorted( + (work.turn, work.obligation.reporting_obligation_id) + for work in self._production_source_work.values() + if work.obligation.generation_key == configuration.generation_key + and work.obligation.period.end <= now + and work.state == "pending" + )[:limit] + turn = self._production_source_turns.get(configuration.generation_key, 0) + for offset, (_, identifier) in enumerate(candidates, 1): + work = self._production_source_work[identifier] + self._production_source_work[identifier] = _SourceWork( + work.obligation, turn + offset + ) + self._production_source_turns[configuration.generation_key] = turn + len(candidates) + return tuple(identifier for _, identifier in candidates) + + async def finish_producer_acquisition( + self, configuration: ReportingConfiguration, *, reporting_obligation_id: str + ) -> None: + async with self._lock: + self._check_source_generation(configuration) + work = self._production_source_work[reporting_obligation_id] + if work.obligation.generation_key != configuration.generation_key: + raise LedgerConflictError("HISTORY_UNAVAILABLE", "producer generation differs") + revisions = await self.list_revisions( + account_id=configuration.account_id, + reporting_obligation_id=reporting_obligation_id, + ) + self._production_source_work[reporting_obligation_id] = _SourceWork( + work.obligation, work.turn, acquisition_state(work.obligation, revisions) + ) + + async def _activate_production(self, *, account_id: str) -> bool: + owner = self._owner() + async with self._mutation(): + projection = self._projection_accounts.get(account_id) + if ( + projection is None + or not projection.ready + or projection.policy != owner.projection.policy + ): + raise ReportingNotificationError("status_projection_activation_required") + policy = owner._admission_policy() + current = self._production_accounts.get(account_id) + if current is not None: + if current.policy != policy: + raise ReportingNotificationError("reporting_production_policy_conflict") + return False + self._production_accounts[account_id] = _Admission(self._clock(), policy) + for _, _, record in self._retained_delivery_records(): + if ( + isinstance(record, ReportingDestinationBinding) + and record.principal.account_id == account_id + ): + configuration = self._configurations[record.generation_key] + try: + offering = owner._configuration_offering(configuration, record) + # Unsupported or unavailable bindings remain unadmitted. + except Exception: # nosec B112 + continue + self._enroll(configuration, offering._producer_key, record) + return True + + def _check_admission(self, lease: ReportingMaterializerLease) -> None: + if lease.admission_epoch != 2: + return + admission = self._production_accounts.get(lease.scope.principal.account_id) + if admission is None or lease.attempt.created_at < admission.activated_at: + raise failure("BINDING_MISMATCH") + self._owner()._check_context( + self._context(lease.scope), + lease.request.verification_key, + admission.policy, + self._production_generations.get(lease.scope.generation_key), + ( + self._production_source_bindings[lease.scope.generation_key].document() + if lease.scope.generation_key in self._production_source_bindings + else None + ), + self._destination_document(self._context(lease.scope).binding), + ) + + def _new_admission_epoch( + self, context: MaterializerContext, key: ReportingVerificationKey + ) -> int: + admission = self._production_accounts.get(context.scope.principal.account_id) + if admission is None: + raise ReportingNotificationError("reporting_production_activation_required") + self._owner()._check_context( + context, + key, + admission.policy, + self._production_generations.get(context.configuration.generation_key), + ( + self._production_source_bindings[context.configuration.generation_key].document() + if context.configuration.generation_key in self._production_source_bindings + else None + ), + self._destination_document(context.binding), + ) + return 2 + + def _materializer_candidate_enabled(self, account_id: str) -> bool: + return account_id in self._production_accounts + + def _held(self, lease: ReportingMaterializerLease) -> _Work | None: + work = super()._held(lease) + if work is not None: + self._check_admission(lease) + return work + + def _materializer_capture_collections( + self, outcome: ReportingMaterializationRecord + ) -> tuple[dict[ReportingDeliveryPrincipal, int], list[ReportingMaterializerBoundary]]: + work = self._materializer_work.get( + ( + outcome.scope.principal.account_id, + outcome.scope.consumer_id, + outcome.reporting_materialization_id, + ) + ) + if work is not None and work.admission_epoch == 2: + return self._production_status_heads, self._production_boundaries + return super()._materializer_capture_collections(outcome) + + def _enqueue_materializer( + self, event: ReportingDomainEvent, lease: ReportingMaterializerLease + ) -> None: + if lease.admission_epoch == 0: + return super()._enqueue_materializer(event, lease) + self._check_admission(lease) + if self._production_outbox is None: + raise failure("BINDING_MISMATCH") + self._production_outbox.enqueue(event) + if ( + self._production_outbox.events.get( + (event.account_id, event.consumer_namespace, event.notification_id) + ) + != event + ): + raise failure("BINDING_MISMATCH") + + async def read_production_boundaries( + self, *, caller: ReportingDeliveryPrincipal, after: int = 0, limit: int = 100 + ) -> tuple[ReportingMaterializerBoundary, ...]: + if type(after) is not int or after < 0 or type(limit) is not int or not 1 <= limit <= 100: + raise ValueError("production boundary reads require bounded positions") + async with self._lock: + return tuple( + b for b in self._production_boundaries if b.caller == caller and b.sequence > after + )[:limit] diff --git a/src/adcp/reporting/production/notifications.py b/src/adcp/reporting/production/notifications.py new file mode 100644 index 000000000..1b2224cb3 --- /dev/null +++ b/src/adcp/reporting/production/notifications.py @@ -0,0 +1,364 @@ +"""Optional owned notification delivery, independent of complete polling.""" + +from __future__ import annotations + +from dataclasses import dataclass, field +from typing import TYPE_CHECKING, Any + +import httpx + +from adcp.reporting.ledger.notification_models import ReportingNotificationError +from adcp.reporting.outbox.memory import InMemoryReportingOutbox +from adcp.reporting.outbox.routing import ( + ReportingEnvelopeCipher, + ReportingNotificationSubscription, + ReportingSigningMaterial, + ReportingSigningResolver, + ReportingSubscriptionResolver, +) +from adcp.reporting.outbox.worker import ReportingNotificationWorker +from adcp.reporting.production.delivery_window import ( + RETRY_HORIZON_SECONDS, + ProductionDeliveryWindow, +) +from adcp.reporting.production.memory import ( + InMemoryReportingProductionOutbox, + InMemoryReportingProductionStore, +) +from adcp.reporting.projection.memory import InMemoryReportingStatusProjection +from adcp.signing.crypto import ALLOWED_ALGS + +if TYPE_CHECKING: + from adcp.reporting.production.pg import PgReportingProductionStore + from adcp.reporting.production.service import ReportingProductionSupport + from adcp.reporting.projection.pg import PgReportingStatusProjection + + +@dataclass(frozen=True) +class ReportingProductionSigning: + """A declared RFC 9421 algorithm contract checked on every resolved key. + + Key material remains in the trusted resolver. The public declaration and + actual sender share this immutable contract; rotation cannot switch to an + unadvertised algorithm or a legacy authentication path. + """ + + resolver: ReportingSigningResolver = field(repr=False) + algorithms: tuple[str, ...] + brand_json_url: str = field(kw_only=True) + + def __post_init__(self) -> None: + object.__setattr__(self, "algorithms", tuple(self.algorithms)) + if ( + not self.algorithms + or len(set(self.algorithms)) != len(self.algorithms) + or not set(self.algorithms) <= ALLOWED_ALGS + or not callable(getattr(self.resolver, "resolve", None)) + ): + raise ReportingNotificationError("notification_signing_unready") + try: + if type(self.brand_json_url) is not str: + raise ValueError + identity = httpx.URL(self.brand_json_url) + valid = ( + identity.scheme == "https" + and bool(identity.host) + and not identity.userinfo + and not identity.query + and not identity.fragment + and identity.port in (None, 443) + ) + except (TypeError, ValueError, httpx.InvalidURL): + valid = False + if not valid: + raise ReportingNotificationError("notification_signing_unready") + + async def resolve( + self, *, account_id: str, principal_id: str, signing_scope_id: str + ) -> ReportingSigningMaterial: + material = await self.resolver.resolve( + account_id=account_id, principal_id=principal_id, signing_scope_id=signing_scope_id + ) + if type( + material + ) is not ReportingSigningMaterial or material.advertised_algorithms != frozenset( + self.algorithms + ): + raise ReportingNotificationError("notification_signing_unready") + ReportingSigningMaterial.__post_init__(material) + return material + + def wire(self) -> dict[str, Any]: + return { + "supported": True, + "profile": "adcp/webhook-signing/v1", + "algorithms": list(self.algorithms), + "legacy_hmac_fallback": False, + "delivery_retry_horizon_seconds": RETRY_HORIZON_SECONDS, + } + + +@dataclass(frozen=True) +class _SignedSubscriptions: + resolver: ReportingSubscriptionResolver = field(repr=False) + + @staticmethod + def _check(value: ReportingNotificationSubscription) -> ReportingNotificationSubscription: + if type(value) is not ReportingNotificationSubscription or value.authentication is not None: + raise ReportingNotificationError("notification_signing_unready") + return value + + async def list_active( + self, *, account_id: str, notification_type: str + ) -> tuple[ReportingNotificationSubscription, ...]: + values = await self.resolver.list_active( + account_id=account_id, notification_type=notification_type + ) + return tuple(self._check(v) for v in values) + + async def get_active( + self, *, account_id: str, subscriber_id: str, notification_type: str + ) -> ReportingNotificationSubscription | None: + value = await self.resolver.get_active( + account_id=account_id, subscriber_id=subscriber_id, notification_type=notification_type + ) + return self._check(value) if value is not None else None + + +def production_notification_workers( + store: InMemoryReportingProductionStore | PgReportingProductionStore, + projection: InMemoryReportingStatusProjection | PgReportingStatusProjection, + *, + subscriptions: ReportingSubscriptionResolver, + cipher: ReportingEnvelopeCipher, + signing: ReportingProductionSigning, +) -> tuple[ReportingNotificationWorker, ...]: + """Build all three real SDK queue workers for ``ReportingProductionSupport``. + + The support schedules these workers itself. Omitting them keeps polling + and enabled atomic logical enqueue available, without advertising push. + Retained epoch-zero readiness queues are never among these participants. + """ + from adcp.reporting.outbox.pg import PgReportingOutbox + from adcp.reporting.production.pg import PgReportingProductionOutbox, PgReportingProductionStore + + if ( + projection.ledger is not store + or not projection.policy["notifications_enabled"] + or type(signing) is not ReportingProductionSigning + ): + raise ReportingNotificationError("notification_chain_unready") + ReportingProductionSigning.__post_init__(signing) + signed_subscriptions = _SignedSubscriptions(subscriptions) + outboxes: tuple[Any, ...] + if type(store) is InMemoryReportingProductionStore: + outboxes = ( + InMemoryReportingOutbox(store), + projection.outbox, + InMemoryReportingProductionOutbox(store), + ) + elif type(store) is PgReportingProductionStore: + outboxes = ( + PgReportingOutbox(pool=store._pool, clock=store._clock), + projection.outbox, + PgReportingProductionOutbox(pool=store._pool, clock=store._clock), + ) + else: + raise ReportingNotificationError("notification_chain_unready") + return tuple( + ReportingNotificationWorker( + outbox=outbox, + subscriptions=signed_subscriptions, + cipher=cipher, + signing=signing, + activity=outbox, + clock=store._clock, + delivery_window=ProductionDeliveryWindow(store, queue), + ) + for outbox, queue in zip(outboxes, ("core", "status", "ready")) + ) + + +def worker_identity(worker: ReportingNotificationWorker) -> tuple[int, ...]: + return tuple( + id(component) + for component in ( + worker, + worker.outbox, + worker.subscriptions, + worker.cipher, + worker.signing, + worker.activity, + worker.delivery_window, + getattr(worker.signing, "resolver", None), + getattr(worker.signing, "algorithms", None), + getattr(worker.signing, "brand_json_url", None), + getattr(worker.subscriptions, "resolver", None), + ) + ) + + +def check_workers(support: ReportingProductionSupport) -> None: + workers = support.notification_workers + if not workers: + return + if ( + type(workers) is not tuple + or len(workers) != 3 + or not support.notifications_enabled + or any(type(w) is not ReportingNotificationWorker for w in workers) + or any(type(w.cipher) is not ReportingEnvelopeCipher for w in workers) + or any(type(w.signing) is not ReportingProductionSigning for w in workers) + or any(type(w.subscriptions) is not _SignedSubscriptions for w in workers) + or any(type(w.delivery_window) is not ProductionDeliveryWindow for w in workers) + or any(id(w.activity) != id(w.outbox) for w in workers) + or any( + w.subscriptions is not workers[0].subscriptions + or w.signing is not workers[0].signing + or w.cipher is not workers[0].cipher + for w in workers + ) + or workers[1].outbox is not support.projection.outbox + or tuple(worker_identity(w) for w in workers) != support._notification_identity + ): + raise ReportingNotificationError("notification_chain_unready") + expected: tuple[type[Any], ...] + if isinstance(support.store, InMemoryReportingProductionStore): + from adcp.reporting.projection.memory import InMemoryReportingProjectionOutbox + + expected = ( + InMemoryReportingOutbox, + InMemoryReportingProjectionOutbox, + InMemoryReportingProductionOutbox, + ) + linked = all(getattr(w.outbox, "_store", None) is support.store for w in workers) + else: + from adcp.reporting.outbox.pg import PgReportingOutbox + from adcp.reporting.production.pg import PgReportingProductionOutbox + from adcp.reporting.projection.notifications import PgReportingProjectionOutbox + + expected = (PgReportingOutbox, PgReportingProjectionOutbox, PgReportingProductionOutbox) + linked = all(getattr(w.outbox, "_pool", None) is support.store._pool for w in workers) + if not linked or tuple(type(w.outbox) for w in workers) != expected: + raise ReportingNotificationError("notification_chain_unready") + for worker, queue in zip(workers, ("core", "status", "ready")): + window = worker.delivery_window + if ( + not isinstance(window, ProductionDeliveryWindow) + or window.store is not support.store + or window.queue != queue + ): + raise ReportingNotificationError("notification_chain_unready") + signing = workers[0].signing + assert isinstance(signing, ReportingProductionSigning) + ReportingProductionSigning.__post_init__(signing) + subscriptions = workers[0].subscriptions + assert isinstance(subscriptions, _SignedSubscriptions) + if not all( + callable(getattr(subscriptions.resolver, method, None)) + for method in ("list_active", "get_active") + ): + raise ReportingNotificationError("notification_chain_unready") + + +def signing_capabilities(support: ReportingProductionSupport) -> dict[str, Any]: + check_workers(support) + signing = support.notification_workers[0].signing + assert isinstance(signing, ReportingProductionSigning) + return signing.wire() + + +def signing_identity(support: ReportingProductionSupport) -> dict[str, str]: + check_workers(support) + signing = support.notification_workers[0].signing + assert isinstance(signing, ReportingProductionSigning) + return {"brand_json_url": signing.brand_json_url} + + +async def check_account_notifications(support: ReportingProductionSupport, account_id: str) -> None: + """Resolve trusted registrations/signing before admitting this account. + + This check does not replace dispatch's authorization/revocation checks. + A supported empty seller needs no invented account to advertise discovery. + """ + import asyncio + + check_workers(support) + if not support.notification_workers: + return + worker = support.notification_workers[0] + for event_type in ( + "reporting.ledger_changed", + "reporting.status_changed", + "reporting.delivery_ready", + ): + subscriptions = await asyncio.wait_for( + worker.subscriptions.list_active(account_id=account_id, notification_type=event_type), + timeout=worker.lease_seconds * 0.8, + ) + if not isinstance(subscriptions, (tuple, list)): + raise ReportingNotificationError("notification_chain_unready") + identifiers = set() + for subscription in subscriptions: + if ( + type(subscription) is not ReportingNotificationSubscription + or subscription.account_id != account_id + or subscription.subscriber_id in identifiers + or event_type not in subscription.event_types + or not ( + subscription.active and subscription.authorized and subscription.proof_valid + ) + ): + raise ReportingNotificationError("notification_chain_unready") + ReportingNotificationSubscription.__post_init__(subscription) + identifiers.add(subscription.subscriber_id) + sender = await asyncio.wait_for( + worker._sender(subscription), timeout=worker.lease_seconds * 0.8 + ) + await sender.aclose() + check_workers(support) + + +async def next_account(worker: ReportingNotificationWorker, *, delivery: bool) -> str | None: + """Sample one indexed due queue row; never enumerate adopter accounts.""" + from adcp.reporting.outbox.pg import PgReportingOutbox, database_now + + outbox = worker.outbox + if isinstance(outbox, InMemoryReportingOutbox): + now = worker._clock() + async with outbox._store._lock: + pending = ( + ((key[0], work) for key, (_, work) in outbox._state.deliveries.items()) + if delivery + else ((key[0], work) for key, work in outbox._state.expansions.items()) + ) + values = [(work.due_at, account) for account, work in pending if work.available(now)] + return min(values)[1] if values else None + if not isinstance(outbox, PgReportingOutbox): + raise ReportingNotificationError("notification_chain_unready") + # Both identifiers are closed SDK literals. The concrete queue adapter + # selects the matching isolated table on this same actual connection. + table = "reporting_notification_deliveries" if delivery else "reporting_notification_expansions" + async with outbox._connection() as connection: + now = await database_now(connection, outbox._clock) + row = await ( + await connection.execute( + f"SELECT account_id FROM {table}" # nosec B608 + " WHERE due_at<=%s AND (state='pending' OR" + " (state='leased' AND lease_expires_at<=%s))" + " ORDER BY due_at,account_id LIMIT 1", + (now, now), + ) + ).fetchone() + return str(row[0]) if row is not None else None + + +async def notification_turn(support: ReportingProductionSupport) -> None: + for worker in support.notification_workers: + for delivery in (False, True): + support._assert_components() + account_id = await next_account(worker, delivery=delivery) + if account_id is not None: + support._assert_components() + operation = worker.deliver_one if delivery else worker.expand_one + await operation(account_id=account_id) diff --git a/src/adcp/reporting/production/offerings.py b/src/adcp/reporting/production/offerings.py new file mode 100644 index 000000000..932ffc8f6 --- /dev/null +++ b/src/adcp/reporting/production/offerings.py @@ -0,0 +1,352 @@ +"""Atomic offerings bind public promises to the actual producer and verifier.""" + +from __future__ import annotations + +import hashlib +import json +from dataclasses import dataclass, field +from datetime import datetime +from typing import Any + +from adcp.reporting._timestamp import aware_timestamp +from adcp.reporting.canonical_json import canonical_json_utf8_v1 +from adcp.reporting.ledger.delivery_models import ReportingDestinationBinding +from adcp.reporting.ledger.models import ( + ReportingConfiguration, + _schedule_clock, + iso_duration_to_timedelta, +) +from adcp.reporting.ledger.producer import ReportingProducer +from adcp.reporting.materializer.contracts import ReportingVerificationKey, failure +from adcp.reporting.materializer.verification import _same_definition +from adcp.reporting.production.contracts import ( + ReportingProductionSource, + ReportingProductionSourceBinding, +) +from adcp.reporting.source import ( + AuthoritativeOfferingV1, + ProvisionalSnapshotOfferingV1, + ReportingSourceCapabilitiesV1, + ReportingSourceStagedObjectReader, + iso_duration_milliseconds_v1, +) +from adcp.types import ReportingDeliveryOffering + + +@dataclass(frozen=True) +class ReportingProductionOffering: + """One public offering and the exact running components that implement it. + + The wire model is copied into immutable bytes so mutating an adopter's + Pydantic model after construction cannot change the advertised contract. + Capability availability is still checked against live components. + """ + + offering: ReportingDeliveryOffering = field(repr=False, compare=False) + producer: ReportingProducer = field(repr=False, compare=False) + verification_key: ReportingVerificationKey + source_offering_id: str + _wire: bytes = field(init=False, repr=False) + _producer_key: str = field(init=False, repr=False) + _source_identity: int = field(init=False, repr=False) + _reader_identity: int = field(init=False, repr=False) + + def __post_init__(self) -> None: + checked = ReportingDeliveryOffering.model_validate( + self.offering.model_dump(mode="json", exclude_none=True) + ) + raw = checked.model_dump(mode="json", exclude_none=True) + from adcp.validation.schema_loader import get_named_validator + + validator = get_named_validator("core/reporting-delivery-offering.json") + if validator is None or next(validator.iter_errors(raw), None) is not None: + raise ValueError("offering must satisfy the complete public contract") + profile, definition, key = ( + raw["reporting_profile"], + self.verification_key.definition, + self.verification_key, + ) + method = raw.get("method") + if ( + method is None + or method.get("orchestration") != "producer_managed" + or (method["pattern"], method["transport"], method.get("format")) + != (key.capability.method, key.capability.transport, key.capability.format) + or (raw["report_definition_id"], profile["id"]) + != (key.report_definition_id, key.reporting_profile) + or (raw["report_definition_uri"], raw["report_definition_sha256"].lower()) + != (definition.report_definition_uri, definition.report_definition_sha256) + or ( + profile["version"], + profile["schema_uri"], + profile["schema_sha256"].lower(), + profile["schema_dialect"], + profile["schema_ref_policy"], + ) + != ( + definition.schema_version, + definition.schema_uri, + definition.schema_sha256, + definition.schema_dialect, + definition.schema_ref_policy, + ) + or not raw["supported_finality"] + or len(set(raw["supported_finality"])) != len(raw["supported_finality"]) + ): + raise ValueError("offering must match its installed producer and exact verifier") + if raw["reconciliation_mode"] == "consumer_receipt": + if ( + raw["supported_finality"] != ["official"] + or key.capability.verification_profile != "canonical_digest" + or ( + profile.get("canonicalization_id"), + profile.get("canonicalization_uri"), + str(profile.get("canonicalization_sha256", "")).lower(), + ) + != ( + key.canonicalization.canonicalization_id, + key.canonicalization.canonicalization_uri, + key.canonicalization.canonicalization_sha256, + ) + ): + raise ValueError("reconciled offerings require official canonical receipt evidence") + elif raw["feed_purpose"] == "billing": + raise ValueError("billing offerings require consumer receipts") + object.__setattr__(self, "_wire", canonical_json_utf8_v1(raw)) + object.__setattr__(self, "_source_identity", id(self.producer._source)) + object.__setattr__(self, "_reader_identity", id(self.producer._object_reader)) + object.__setattr__( + self, + "_producer_key", + hashlib.sha256( + canonical_json_utf8_v1( + { + "offering": raw, + "source_offering_id": self.source_offering_id, + "publication_namespace": self.producer._offerings.publication_namespace, + "source_scope": dict(self.producer._offerings.source_scope), + } + ) + ).hexdigest(), + ) + self.check_source() + + @property + def offering_id(self) -> str: + return str(self.wire()["offering_id"]) + + @property + def reconciled(self) -> bool: + return bool(self.wire()["reconciliation_mode"] == "consumer_receipt") + + def wire(self) -> dict[str, Any]: + return dict(json.loads(self._wire)) + + def check_source(self, *, effective: bool = False) -> ReportingSourceCapabilitiesV1: + source = self.producer._source + if ( + id(source) != self._source_identity + or not isinstance(source, ReportingProductionSource) + or id(self.producer._object_reader) != self._reader_identity + or not isinstance(self.producer._object_reader, ReportingSourceStagedObjectReader) + ): + raise failure("BINDING_MISMATCH") + capabilities = ReportingSourceCapabilitiesV1.model_validate( + source.capabilities.model_dump(mode="json", exclude_none=True) + ) + if effective and capabilities.scope != "effective_account": + raise failure("BINDING_MISMATCH") + raw = self.wire() + source_offering = capabilities.offering(self.source_offering_id) + contract, key = source_offering.contract, self.verification_key + source_values = contract.model_dump(mode="json") + expected = { + "report_definition_id": key.report_definition_id, + "reporting_profile": key.reporting_profile, + } + expected.update( + { + name: getattr(key.definition, name) + for name in ( + "report_definition_uri", + "report_definition_sha256", + "schema_version", + "schema_uri", + "schema_sha256", + "schema_dialect", + "schema_ref_policy", + ) + } + ) + finality = raw["supported_finality"] + configured = self.producer._offerings + if ( + any(source_values.get(name) != value for name, value in expected.items()) + or source_offering.publication_namespace != configured.publication_namespace + or dict(configured.source_scope) != capabilities.source_scope + or not source_offering.provider_execution.supports_cancellation + or ( + "official" in finality + and ( + not isinstance(source_offering, AuthoritativeOfferingV1) + or configured.official_offering_id != self.source_offering_id + or ( + self.reconciled + and source_offering.correction_policy != "immutable_correction" + ) + ) + ) + or ( + "snapshot" in finality + and ( + not isinstance(source_offering, ProvisionalSnapshotOfferingV1) + or configured.snapshot_offering_id != self.source_offering_id + ) + ) + ): + raise failure("UNSUPPORTED_VERIFICATION") + schedule = raw["schedule"] + duration = iso_duration_milliseconds_v1(schedule["period_duration"]) + if not ( + iso_duration_milliseconds_v1(source_offering.windowing.minimum_window) + <= duration + <= iso_duration_milliseconds_v1(source_offering.windowing.maximum_window) + ) or iso_duration_milliseconds_v1(schedule["delivery_sla"]) < iso_duration_milliseconds_v1( + source_offering.worst_case_availability_lag + ): + raise failure("UNSUPPORTED_VERIFICATION") + if isinstance( + source_offering, ProvisionalSnapshotOfferingV1 + ) and duration < iso_duration_milliseconds_v1(source_offering.fastest_safe_cadence): + raise failure("UNSUPPORTED_VERIFICATION") + if ( + schedule["alignment"] == "source_timezone" + and schedule.get("period_timezone") != source_offering.source_timezone + ): + raise failure("UNSUPPORTED_VERIFICATION") + return capabilities + + def source_binding( + self, configuration: ReportingConfiguration + ) -> ReportingProductionSourceBinding: + try: + capabilities = self.check_source(effective=True) + source = self.producer._source + assert isinstance(source, ReportingProductionSource) + binding = source.configuration_binding(configuration) + if type(binding) is not ReportingProductionSourceBinding: + raise failure("BINDING_MISMATCH") + binding.check(configuration, capabilities, self.source_offering_id) + return binding + except Exception: + raise failure("BINDING_MISMATCH") from None + + def configuration_schedule(self, configuration: ReportingConfiguration) -> dict[str, Any]: + """Project a proven legacy clock into the public schedule vocabulary. + + Existing captured clocks and their closed storage decoder stay intact. + New production admission requires the normative public phase; an old + explicit anchor is compatible only when it produces the same periods. + Calendar-month/year clocks unsupported by that decoder are refused. + """ + offered = self.wire()["schedule"] + schedule = configuration.schedule + alignment = offered["alignment"] + expected_alignment = ( + alignment if alignment in {"utc", "account_timezone"} else "custom_timezone" + ) + zone, duration, anchor = _schedule_clock(schedule, configuration.account_timezone) + if ( + schedule.alignment != expected_alignment + or schedule.period_duration != offered["period_duration"] + or schedule.delivery_sla != offered["delivery_sla"] + or (alignment != "billing_cycle" and (anchor - datetime(1970, 1, 1)) % duration) + or (alignment == "billing_cycle" and schedule.period_anchor is None) + ): + raise failure("BINDING_MISMATCH") + result: dict[str, Any] = { + "period_duration": schedule.period_duration, + "delivery_sla": schedule.delivery_sla, + "alignment": alignment, + } + if alignment in {"source_timezone", "billing_cycle"}: + result["period_timezone"] = zone.key + if alignment == "billing_cycle": + assert schedule.period_anchor is not None + result["period_anchor"] = schedule.period_anchor.isoformat() + if ( + offered.get("period_timezone_policy") == "fixed" + or offered.get("period_anchor_policy") == "fixed" + ) and result.get("period_timezone") != offered.get("period_timezone"): + raise failure("BINDING_MISMATCH") + if offered.get( + "period_anchor_policy" + ) == "fixed" and schedule.period_anchor != aware_timestamp(offered["period_anchor"]): + raise failure("BINDING_MISMATCH") + if ( + alignment == "source_timezone" + and zone.key + != self.check_source(effective=True).offering(self.source_offering_id).source_timezone + ): + raise failure("BINDING_MISMATCH") + return result + + def check_configuration( + self, + configuration: ReportingConfiguration, + binding: ReportingDestinationBinding, + *, + retention_days: int, + ) -> None: + capabilities = self.check_source(effective=True) + self.source_binding(configuration) + self.configuration_schedule(configuration) + raw, key = self.wire(), self.verification_key + schedule, offered = configuration.schedule, raw["schedule"] + if ( + binding.generation_key != configuration.generation_key + or ( + configuration.report_definition_id, + configuration.reporting_profile, + configuration.feed_purpose, + configuration.required_finality, + ) + != ( + key.report_definition_id, + key.reporting_profile, + raw["feed_purpose"], + raw["supported_finality"][0], + ) + or not _same_definition(key, configuration.definition) + or configuration.authoritative_party != "seller" + or binding.reconciliation_mode != raw["reconciliation_mode"] + or binding.feed_purpose != raw["feed_purpose"] + or (binding.method, binding.transport, binding.format, binding.verification_profile) + != ( + key.capability.method, + key.capability.transport, + key.capability.format, + key.capability.verification_profile, + ) + or binding.resource_retention_days != retention_days + or binding.reader_compatibility != tuple(raw["method"].get("reader_compatibility", ())) + or schedule.period_duration != offered["period_duration"] + or iso_duration_to_timedelta(schedule.delivery_sla) + != iso_duration_to_timedelta(offered["delivery_sla"]) + or iso_duration_milliseconds_v1(schedule.delivery_sla) + < iso_duration_milliseconds_v1( + capabilities.offering(self.source_offering_id).worst_case_availability_lag + ) + or ( + offered.get("period_anchor_policy") == "fixed" + and ( + schedule.period_anchor is None + or schedule.period_anchor != aware_timestamp(str(offered.get("period_anchor"))) + ) + ) + or ( + offered.get("period_timezone_policy") == "fixed" + and schedule.period_timezone != offered.get("period_timezone") + ) + ): + raise failure("BINDING_MISMATCH") diff --git a/src/adcp/reporting/production/pg.py b/src/adcp/reporting/production/pg.py new file mode 100644 index 000000000..9d25821e0 --- /dev/null +++ b/src/adcp/reporting/production/pg.py @@ -0,0 +1,755 @@ +"""Production admission selects new participants in the single B2.1 transaction.""" + +from __future__ import annotations + +import asyncio +import json +import weakref +from collections.abc import AsyncIterator +from contextlib import asynccontextmanager +from contextvars import ContextVar +from datetime import datetime, timedelta +from importlib.resources import files +from typing import TYPE_CHECKING, Any + +from adcp.reporting.ledger._delivery_state import decode_record +from adcp.reporting.ledger.delivery_models import ( + ReportingDeliveryPrincipal, + ReportingDeliveryScope, + ReportingDestinationBinding, +) +from adcp.reporting.ledger.models import ReportingConfiguration, ReportingObligationRecord +from adcp.reporting.ledger.notification_models import ReportingNotificationError +from adcp.reporting.ledger.pg import _configuration_from_row +from adcp.reporting.ledger.producer_progress import acquisition_state, check_next_period +from adcp.reporting.ledger.store import LeasedConfiguration, LedgerConflictError, _utc +from adcp.reporting.materializer.capture import ReportingMaterializerBoundary +from adcp.reporting.materializer.contracts import ( + ReportingPreparedRevision, + ReportingVerificationKey, + ReportingWriterFailure, +) +from adcp.reporting.materializer.verification import ReportingVerifiedDestination +from adcp.reporting.materializer.work import ( + MaterializerContext, + ReportingMaterializerLease, + ReportingMaterializerTurn, + key_for, +) +from adcp.reporting.outbox.status_pg import PgReportingStatusOutbox +from adcp.reporting.projection.pg import PgReportingProjectionStore +from adcp.reporting.source import ReportingConstituent + +if TYPE_CHECKING: + from adcp.reporting.production.service import ReportingProductionSupport + +_EPOCH: ContextVar[tuple[int, int] | None] = ContextVar( + "reporting_production_operation", default=None +) +_ADMISSION_CONNECTION: ContextVar[tuple[int, object, Any] | None] = ContextVar( + "reporting_production_configuration_connection", default=None +) + +_SOURCE_CONFIGURATION = ( + "SELECT c.delivery_config_id,c.delivery_config_version,c.account_id," + " c.report_definition_id,c.reporting_profile,c.feed_purpose," + " c.required_finality,c.account_timezone,c.schedule,c.media_buy_ids," + " c.activated_at,c.deactivated_at," + " c.automated_recovery_seconds,c.status_retention_days,c.definition," + " c.authoritative_party,g.producer_key,g.source_binding FROM reporting_configurations c" + " JOIN reporting_production_generations g" + " USING(account_id,delivery_config_id,delivery_config_version)" + " JOIN reporting_production_accounts a ON a.account_id=g.account_id" + " WHERE c.account_id=%s AND c.delivery_config_id=%s" + " AND c.delivery_config_version=%s AND g.producer_key=ANY(%s)" +) + +_OWNED = ( + "(SELECT account_id,consumer_id,delivery_config_id,delivery_config_version," + "reporting_obligation_id,reporting_materialization_id,state,retry_allowed" + " FROM reporting_materializer_work UNION ALL" + " SELECT account_id,consumer_id,delivery_config_id,delivery_config_version," + "reporting_obligation_id,reporting_materialization_id,state,retry_allowed" + " FROM reporting_production_work)" +) + +_ProducerSample = tuple[int, datetime | None, str, str, int] + + +class _ProductionConnection: + """Only fixed SDK identifiers are selected; account/writer ordering stays intact.""" + + def __init__(self, connection: Any) -> None: + self.connection = connection + + def transaction(self) -> Any: + return self.connection.transaction() + + async def execute(self, query: str, params: Any = None) -> Any: + # A known terminal B2.1 failure remains eligible for an N+1 retry, but + # pending/unknown effects in either epoch prohibit a new reservation. + if query.startswith("SELECT retry_allowed FROM reporting_materializer_work"): + query = query.replace("reporting_materializer_work", _OWNED + " owned", 1) + elif "SELECT 1 FROM reporting_materializer_work w WHERE" in query: + query = query.replace( + "FROM reporting_materializer_work w WHERE", "FROM " + _OWNED + " w WHERE" + ) + else: + for old, new in ( + ("reporting_materializer_notification_", "reporting_production_notification_"), + ("reporting_materializer_status_", "reporting_production_status_"), + ("reporting_materializer_work", "reporting_production_work"), + ): + query = query.replace(old, new) + return await self.connection.execute(query, params) + + +class _ProductionQueueConnection: + def __init__(self, connection: Any) -> None: + self.connection = connection + + def transaction(self) -> Any: + return self.connection.transaction() + + async def execute(self, query: str, params: Any = None) -> Any: + for old, new in ( + ("reporting_notification_", "reporting_production_notification_"), + ("reporting_webhook_", "reporting_production_webhook_"), + ): + query = query.replace(old, new) + return await self.connection.execute(query, params) + + +class PgReportingProductionOutbox(PgReportingStatusOutbox): + """Generic crash-safe delivery/fanout and private activity on the admitted queue.""" + + @asynccontextmanager + async def _connection(self) -> AsyncIterator[Any]: + async with self._pool.connection() as connection: + yield _ProductionQueueConnection(connection) + + async def create_schema(self) -> None: + await PgReportingProductionStore(pool=self._pool, notifications=True).create_schema() + + +class PgReportingProductionStore(PgReportingProjectionStore): + """The production store retains all ordinary legacy reader/writer APIs. + + New work requires the live SDK composition and its mounted applicable + routes. Pending epoch-zero work uses its original tables, identity and + permanently quarantined enqueue. Installation alone admits nothing. + """ + + _production_support: weakref.ReferenceType[ReportingProductionSupport] | None = None + _production_lease_samples: dict[tuple[str, ...], _ProducerSample] | None = None + + def _owner(self) -> ReportingProductionSupport: + from adcp.reporting.production.service import production_owner + + return production_owner(self) + + @asynccontextmanager + async def _connection(self) -> AsyncIterator[Any]: + admission = _ADMISSION_CONNECTION.get() + if admission is not None and admission[:2] == (id(self), asyncio.current_task()): + yield admission[2] + return + async with super()._connection() as connection: + if _EPOCH.get() == (id(self), 2): + yield _ProductionConnection(connection) + else: + yield connection + + async def admit_production_configuration( + self, + configuration: ReportingConfiguration, + binding: ReportingDestinationBinding, + *, + offering_id: str, + ) -> None: + offering = self._owner()._configuration_offering( + configuration, binding, offering_id=offering_id + ) + async with self._connection() as connection, connection.transaction(): + await self._lock_account(connection, configuration.account_id) + token = _ADMISSION_CONNECTION.set((id(self), asyncio.current_task(), connection)) + try: + await self.put_configuration(configuration) + await self.put_destination_binding(binding) + await self._enroll_on(connection, configuration, offering._producer_key, binding) + finally: + _ADMISSION_CONNECTION.reset(token) + + async def _enroll_on( + self, + connection: Any, + configuration: ReportingConfiguration, + producer_key: str, + destination: ReportingDestinationBinding, + ) -> None: + key = configuration.generation_key + identity = (key.account_id, key.delivery_config_id, key.delivery_config_version) + binding = self._owner()._source_binding(configuration, producer_key).document() + await connection.execute( + "INSERT INTO reporting_production_generations VALUES(%s,%s,%s,%s,%s::jsonb)" + " ON CONFLICT DO NOTHING", + (*identity, producer_key, json.dumps(binding)), + ) + row = await ( + await connection.execute( + "SELECT producer_key,source_binding FROM reporting_production_generations" + " WHERE account_id=%s AND delivery_config_id=%s AND delivery_config_version=%s", + identity, + ) + ).fetchone() + if row is None or row != (producer_key, binding): + raise ReportingNotificationError("reporting_production_source_conflict") + owner = self._owner() + offering = owner._configuration_offering( + configuration, destination, producer_key=producer_key + ) + document = owner._destination_binding(destination, offering).wire() + destination_identity = ( + key.account_id, + destination.consumer_id, + key.delivery_config_id, + key.delivery_config_version, + ) + await connection.execute( + "INSERT INTO reporting_production_destination_bindings VALUES(%s,%s,%s,%s,%s::jsonb)" + " ON CONFLICT DO NOTHING", + (*destination_identity, json.dumps(document)), + ) + original = await ( + await connection.execute( + "SELECT method FROM reporting_production_destination_bindings" + " WHERE account_id=%s AND consumer_id=%s AND delivery_config_id=%s" + " AND delivery_config_version=%s", + destination_identity, + ) + ).fetchone() + if original is None or original[0] != document: + raise ReportingNotificationError("reporting_production_destination_conflict") + + async def lease_period_close( + self, *, worker_id: str, now: datetime, lease_seconds: float + ) -> LeasedConfiguration | None: + keys = self._owner()._producer_keys() + if not keys: + return None + moment = _utc(now) + expires = moment + timedelta(seconds=lease_seconds) + if expires <= moment: + raise ValueError("producer lease duration must be positive") + if self._production_lease_samples is None: + self._production_lease_samples = {} + after = self._production_lease_samples.get(keys) + following: _ProducerSample | None = None + result = None + async with self._connection() as connection, connection.transaction(): + # Discover a bounded set without locking configuration rows. The + # inherited configuration trigger takes the account lock, so that + # lock must precede the row lock here, just as it does in activation. + # A busy account cannot advance a durable rank while another + # transaction holds its lock. Continue a read-only sample instead + # of repeatedly trying the same prefix. At most one nonempty + # 32-row window is examined; an empty tail may wrap once. + for _ in range(2): + continuation = ( + " AND (greatest(coalesce(t.lease_turn,0),coalesce(p.probe_turn,0))," + " coalesce(c.lease_expires_at,'-infinity'::timestamptz)," + " c.account_id,c.delivery_config_id,c.delivery_config_version)" + " > (%s,coalesce(%s::timestamptz,'-infinity'::timestamptz),%s,%s,%s)" + if after is not None + else "" + ) + query = ( + "SELECT c.account_id,c.delivery_config_id,c.delivery_config_version," # nosec B608 + " greatest(coalesce(t.lease_turn,0),coalesce(p.probe_turn,0))," + " c.lease_expires_at" + " FROM reporting_production_generations g" + " JOIN reporting_production_accounts a ON a.account_id=g.account_id" + " JOIN reporting_configurations c" + " ON (c.account_id,c.delivery_config_id,c.delivery_config_version)=" + " (g.account_id,g.delivery_config_id,g.delivery_config_version)" + " LEFT JOIN adcp_reporting_configuration_lease_turns t" + " ON (t.account_id,t.delivery_config_id,t.delivery_config_version)=" + " (g.account_id,g.delivery_config_id,g.delivery_config_version)" + " LEFT JOIN reporting_production_source_probe_turns p" + " ON (p.account_id,p.delivery_config_id,p.delivery_config_version)=" + " (g.account_id,g.delivery_config_id,g.delivery_config_version)" + " WHERE g.producer_key=ANY(%s)" + " AND (c.lease_expires_at IS NULL OR c.lease_expires_at<=%s)" + + continuation + + " ORDER BY greatest(coalesce(t.lease_turn,0),coalesce(p.probe_turn,0))," + " c.lease_expires_at NULLS FIRST," + " c.account_id,c.delivery_config_id,c.delivery_config_version LIMIT 32" + ) + # Only the fixed SDK continuation above changes this SQL; + # every cursor value and source identity remains a parameter. + rows = await ( + await connection.execute(query, (list(keys), moment, *(after or ()))) + ).fetchall() + if rows or after is None: + break + after = None + for candidate in rows: + row = candidate[:3] + following = (candidate[3], candidate[4], row[0], row[1], row[2]) + locked = await ( + await connection.execute( + "SELECT pg_try_advisory_xact_lock(hashtext('adcp.reporting:' || %s))", + (row[0],), + ) + ).fetchone() + if not locked[0]: + continue + current = await ( + await connection.execute(_SOURCE_CONFIGURATION, (*row, list(keys))) + ).fetchone() + if current is None: + continue + configuration = _configuration_from_row(current[:16]) + try: + self._owner()._check_source_binding(configuration, current[16], current[17]) + except Exception: + # A permanently revoked generation must not occupy the + # first bounded window forever. This is a probe, not a + # lease: preserve ordinary lease ranks and all source and + # external identities. Its durable rank shares the same + # ordering clock as successful configuration acquisitions. + await connection.execute( + "INSERT INTO reporting_production_source_probe_turns" + " (account_id,delivery_config_id,delivery_config_version,probe_turn)" + " VALUES(%s,%s,%s,nextval('adcp_reporting_configuration_lease_turn_seq'))" + " ON CONFLICT(account_id,delivery_config_id,delivery_config_version)" + " DO UPDATE SET probe_turn=" + "nextval('adcp_reporting_configuration_lease_turn_seq')", + tuple(row), + ) + continue + acquired = await ( + await connection.execute( + "UPDATE reporting_configurations SET lease_worker_id=%s,lease_expires_at=%s" + " WHERE account_id=%s AND delivery_config_id=%s" + " AND delivery_config_version=%s" + " AND (lease_expires_at IS NULL OR lease_expires_at<=%s)" + " RETURNING account_id", + (worker_id, expires, *row, moment), + ) + ).fetchone() + if acquired is None: + continue + await self._retain_materializer_generation_on_lease_change(connection, tuple(row)) + await connection.execute( + "INSERT INTO adcp_reporting_configuration_lease_turns" + " (account_id,delivery_config_id,delivery_config_version,lease_turn)" + " VALUES(%s,%s,%s,nextval('adcp_reporting_configuration_lease_turn_seq'))" + " ON CONFLICT(account_id,delivery_config_id,delivery_config_version)" + " DO UPDATE SET lease_turn=" + "nextval('adcp_reporting_configuration_lease_turn_seq')", + tuple(row), + ) + result = LeasedConfiguration(row[0], row[1], row[2], expires) + break + # Hints are per store and selected producer keys, not durable work or + # leases. Publish a hint only after commit; a failed mutation retries + # the same window. Successful acquisition returns to the durable + # turn-primary order. A fresh store starts there too. Concurrent hints + # may cause a bounded revisit, but cannot authorize or fence any work. + if result is None and following is not None: + self._production_lease_samples[keys] = following + else: + self._production_lease_samples.pop(keys, None) + return result + + async def release_period_close(self, lease: LeasedConfiguration, *, worker_id: str) -> None: + async with self._connection() as connection, connection.transaction(): + await self._lock_account(connection, lease.account_id) + identity = (lease.account_id, lease.delivery_config_id, lease.delivery_config_version) + released = await ( + await connection.execute( + "UPDATE reporting_configurations SET lease_worker_id=NULL,lease_expires_at=NULL" + " WHERE account_id=%s AND delivery_config_id=%s AND delivery_config_version=%s" + " AND lease_worker_id=%s AND lease_expires_at=%s RETURNING account_id", + (*identity, worker_id, lease.lease_expires_at), + ) + ).fetchone() + if released is not None: + await self._retain_materializer_generation_on_lease_change(connection, identity) + + async def _retain_materializer_generation_on_lease_change( + self, connection: Any, identity: tuple[str, str, int] + ) -> None: + # The immutable inherited trigger treats *every* configuration UPDATE + # as source invalidation, including lease-only bookkeeping. These two + # SDK statements change only lease fields, under the account lock. + # Cancel just their one trigger increment in the same transaction; + # the wakeup remains harmless. No intervening source write can be + # hidden, no committed generation goes backwards, and pending work's + # original generation/epoch/external identity is never rewritten. + # A real configuration, revision or readability change still executes + # the original trigger without this correction and fences old work. + await connection.execute( + "UPDATE reporting_materializer_candidates SET generation=generation-1" + " WHERE account_id=%s AND delivery_config_id=%s AND delivery_config_version=%s", + identity, + ) + + @asynccontextmanager + async def _source_connection( + self, configuration: ReportingConfiguration + ) -> AsyncIterator[tuple[Any, tuple[str, str, int]]]: + keys = self._owner()._producer_keys() + key = configuration.generation_key + identity = (key.account_id, key.delivery_config_id, key.delivery_config_version) + async with self._connection() as connection, connection.transaction(): + await self._lock_account(connection, key.account_id) + row = await ( + await connection.execute( + _SOURCE_CONFIGURATION, + (*identity, list(keys)), + ) + ).fetchone() + if row is None or _configuration_from_row(row[:16]) != configuration: + raise LedgerConflictError("HISTORY_UNAVAILABLE", "producer generation unavailable") + self._owner()._check_source_binding(configuration, row[16], row[17]) + await connection.execute( + "INSERT INTO reporting_production_source_progress" + " (account_id,delivery_config_id,delivery_config_version) VALUES(%s,%s,%s)" + " ON CONFLICT DO NOTHING", + identity, + ) + token = _ADMISSION_CONNECTION.set((id(self), asyncio.current_task(), connection)) + try: + yield connection, identity + finally: + _ADMISSION_CONNECTION.reset(token) + + async def producer_constituents( + self, configuration: ReportingConfiguration, obligation: ReportingObligationRecord + ) -> tuple[ReportingConstituent, ...]: + async with self._source_connection(configuration) as (connection, identity): + if obligation.generation_key != configuration.generation_key or set( + obligation.media_buy_ids + ) != set(configuration.media_buy_ids): + raise LedgerConflictError("HISTORY_UNAVAILABLE", "source denominator differs") + row = await ( + await connection.execute( + "SELECT producer_key,source_binding FROM reporting_production_generations" + " WHERE account_id=%s AND delivery_config_id=%s AND delivery_config_version=%s", + identity, + ) + ).fetchone() + binding = self._owner()._check_source_binding(configuration, row[0], row[1]) + return binding.constituents() + + async def producer_closed_through( + self, configuration: ReportingConfiguration + ) -> datetime | None: + async with self._source_connection(configuration) as (connection, identity): + row = await ( + await connection.execute( + "SELECT closed_through FROM reporting_production_source_progress" + " WHERE account_id=%s AND delivery_config_id=%s AND delivery_config_version=%s", + identity, + ) + ).fetchone() + return row[0] if row is not None else None + + async def commit_producer_period( + self, + configuration: ReportingConfiguration, + obligation: ReportingObligationRecord, + *, + previous_end: datetime | None, + ) -> ReportingObligationRecord: + check_next_period(configuration, obligation, previous_end) + async with self._source_connection(configuration) as (connection, identity): + row = await ( + await connection.execute( + "SELECT closed_through FROM reporting_production_source_progress" + " WHERE account_id=%s AND delivery_config_id=%s AND delivery_config_version=%s", + identity, + ) + ).fetchone() + current = row[0] + if current != previous_end and (current is None or current < obligation.period.end): + raise LedgerConflictError("HISTORY_UNAVAILABLE", "producer progress changed") + stored = await self.commit_obligation(obligation) + await connection.execute( + "INSERT INTO reporting_production_source_work" + " (account_id,delivery_config_id,delivery_config_version,reporting_obligation_id," + " period_end) VALUES(%s,%s,%s,%s,%s) ON CONFLICT DO NOTHING", + (*identity, stored.reporting_obligation_id, stored.period.end), + ) + await connection.execute( + "UPDATE reporting_production_source_progress" + " SET closed_through=greatest(closed_through,%s)" + " WHERE account_id=%s AND delivery_config_id=%s AND delivery_config_version=%s", + (stored.period.end, *identity), + ) + return stored + + async def next_producer_obligations( + self, configuration: ReportingConfiguration, *, now: datetime, limit: int + ) -> tuple[str, ...]: + if type(limit) is not int or not 1 <= limit <= 64: + raise ValueError("production acquisition limit must be in 1..64") + async with self._source_connection(configuration) as (connection, identity): + rows = await ( + await connection.execute( + "SELECT reporting_obligation_id FROM reporting_production_source_work" + " WHERE account_id=%s AND delivery_config_id=%s AND delivery_config_version=%s" + " AND state='pending' AND period_end<=%s" + " ORDER BY acquisition_turn,reporting_obligation_id LIMIT %s FOR UPDATE", + (*identity, now, limit), + ) + ).fetchall() + if not rows: + return () + head = await ( + await connection.execute( + "UPDATE reporting_production_source_progress" + " SET acquisition_turn=acquisition_turn+%s" + " WHERE account_id=%s AND delivery_config_id=%s AND delivery_config_version=%s" + " RETURNING acquisition_turn", + (len(rows), *identity), + ) + ).fetchone() + for offset, row in enumerate(rows, 1): + await connection.execute( + "UPDATE reporting_production_source_work SET acquisition_turn=%s" + " WHERE account_id=%s AND reporting_obligation_id=%s", + (head[0] - len(rows) + offset, identity[0], row[0]), + ) + return tuple(row[0] for row in rows) + + async def finish_producer_acquisition( + self, configuration: ReportingConfiguration, *, reporting_obligation_id: str + ) -> None: + async with self._source_connection(configuration) as (connection, identity): + obligation = await self.get_obligation( + account_id=identity[0], reporting_obligation_id=reporting_obligation_id + ) + if obligation is None or obligation.generation_key != configuration.generation_key: + raise LedgerConflictError("HISTORY_UNAVAILABLE", "producer generation differs") + revisions = await self.list_revisions( + account_id=identity[0], reporting_obligation_id=reporting_obligation_id + ) + await connection.execute( + "UPDATE reporting_production_source_work SET state=%s" + " WHERE account_id=%s AND delivery_config_id=%s AND delivery_config_version=%s" + " AND reporting_obligation_id=%s", + (acquisition_state(obligation, revisions), *identity, reporting_obligation_id), + ) + + @asynccontextmanager + async def _lease_epoch(self, lease: ReportingMaterializerLease) -> AsyncIterator[None]: + token = _EPOCH.set((id(self), lease.admission_epoch)) + try: + yield + finally: + _EPOCH.reset(token) + + async def create_schema(self) -> None: + async with self._connection() as connection, connection.transaction(): + await self._create_schema_on(connection) + root = files("adcp.reporting.ledger") + for name in ( + "reporting_materializer.sql", + "reporting_receipt_ingestion.sql", + "reporting_feed.sql", + "reporting_status_notifications.sql", + "reporting_status_selector_version.sql", + "reporting_projection.sql", + "reporting_projection_notifications.sql", + "reporting_projection_feed.sql", + "reporting_production.sql", + ): + await connection.execute(root.joinpath(name).read_text()) + + async def materializer_ready(self) -> bool: + owner = self._owner() + if not await owner._schema_ready(): + raise ReportingNotificationError("reporting_production_schema_unready") + owner._assert_components() + return True + + async def _activate_production(self, *, account_id: str) -> bool: + owner = self._owner() + await self.materializer_ready() + async with self._connection() as connection, connection.transaction(): + await self._lock_account(connection, account_id) + owner._assert_components() + projection = await ( + await connection.execute( + "SELECT policy FROM reporting_projection_accounts" + " WHERE account_id=%s AND current_input IS NOT NULL", + (account_id,), + ) + ).fetchone() + if projection is None or projection[0] != owner.projection.policy: + raise ReportingNotificationError("status_projection_activation_required") + policy = owner._admission_policy() + current = await ( + await connection.execute( + "SELECT policy FROM reporting_production_accounts WHERE account_id=%s", + (account_id,), + ) + ).fetchone() + if current is not None: + if current[0] != policy: + raise ReportingNotificationError("reporting_production_policy_conflict") + return False + await connection.execute( + "INSERT INTO reporting_production_accounts(account_id,policy) VALUES(%s,%s::jsonb)", + (account_id, json.dumps(policy)), + ) + configurations = { + c.generation_key: c + for c in await self._list_configurations_on(connection, account_id=account_id) + } + bindings = await ( + await connection.execute( + "SELECT payload FROM reporting_reconciliation_records" + " WHERE account_id=%s AND namespace='destination_binding'", + (account_id,), + ) + ).fetchall() + for (document,) in bindings: + binding = decode_record(document) + if not isinstance(binding, ReportingDestinationBinding): + raise ReportingNotificationError("reporting_production_history_corrupt") + configuration = configurations[binding.generation_key] + try: + offering = owner._configuration_offering(configuration, binding) + # Unsupported or unavailable bindings remain unadmitted. + except Exception: # nosec B112 + continue + await self._enroll_on(connection, configuration, offering._producer_key, binding) + return True + + async def _materializer_context_on( + self, connection: Any, scope: ReportingDeliveryScope + ) -> MaterializerContext: + context = await super()._materializer_context_on(connection, scope) + if isinstance(connection, _ProductionConnection): + owner = self._owner() + row = await ( + await connection.execute( + "SELECT a.policy,g.producer_key,g.source_binding,d.method" + " FROM reporting_production_accounts a" + " LEFT JOIN reporting_production_generations g" + " ON g.account_id=a.account_id AND g.delivery_config_id=%s" + " AND g.delivery_config_version=%s" + " LEFT JOIN reporting_production_destination_bindings d" + " ON (d.account_id,d.delivery_config_id,d.delivery_config_version)=" + " (g.account_id,g.delivery_config_id,g.delivery_config_version)" + " AND d.consumer_id=%s WHERE a.account_id=%s", + ( + scope.generation_key.delivery_config_id, + scope.generation_key.delivery_config_version, + scope.consumer_id, + scope.principal.account_id, + ), + ) + ).fetchone() + if row is None: + raise ReportingNotificationError("reporting_production_activation_required") + owner._check_context( + context, + key_for(context.binding, context.obligation, owner.keys), + row[0], + row[1], + row[2], + row[3], + ) + return context + + async def _claim_account_on( + self, + connection: Any, + account_id: str, + keys: tuple[ReportingVerificationKey, ...], + lease_seconds: int, + ) -> ReportingMaterializerLease | ReportingMaterializerTurn: + activated = await ( + await connection.execute( + "SELECT 1 FROM reporting_production_accounts WHERE account_id=%s", (account_id,) + ) + ).fetchone() + if activated is None: + return ReportingMaterializerTurn("idle") + old = await ( + await connection.execute( + "SELECT to_jsonb(w) FROM reporting_materializer_work w WHERE account_id=%s" + " AND state='pending' AND due_at<=clock_timestamp()" + " AND (lease_until IS NULL OR lease_until<=clock_timestamp())" + " ORDER BY due_at,reporting_materialization_id LIMIT 1 FOR UPDATE", + (account_id,), + ) + ).fetchone() + if old is not None: + return await super()._lease_on(connection, old[0], keys, lease_seconds) + return await super()._claim_account_on( + _ProductionConnection(connection), account_id, keys, lease_seconds + ) + + async def _schedule_account_on(self, connection: Any, account_id: str) -> None: + if isinstance(connection, _ProductionConnection): + connection = connection.connection + # _OWNED contains only fixed SDK tables/columns; account values are bound. + await connection.execute( + "UPDATE reporting_materializer_accounts SET due_at=(SELECT min(due) FROM (" # nosec B608 + " SELECT due_at AS due FROM reporting_materializer_work" + " WHERE account_id=%s AND state='pending'" + " UNION ALL SELECT due_at FROM reporting_production_work" + " WHERE account_id=%s AND state='pending'" + " UNION ALL SELECT c.due_at FROM reporting_materializer_candidates c" + " WHERE c.account_id=%s AND c.due_at IS NOT NULL AND NOT EXISTS (SELECT 1 FROM " + + _OWNED + + " w WHERE w.account_id=c.account_id AND w.consumer_id=c.consumer_id" # nosec B608 + " AND w.delivery_config_id=c.delivery_config_id" + " AND w.delivery_config_version=c.delivery_config_version" + " AND w.reporting_obligation_id=c.reporting_obligation_id AND w.state='pending')" + " UNION ALL SELECT clock_timestamp() FROM reporting_materializer_discovery" + " WHERE account_id=%s AND NOT complete) ready) WHERE account_id=%s", + (account_id,) * 5, + ) + + async def renew_materialization( + self, lease: ReportingMaterializerLease, *, lease_seconds: int = 30 + ) -> bool: + async with self._lease_epoch(lease): + return await super().renew_materialization(lease, lease_seconds=lease_seconds) + + async def authorize_materialization(self, lease: ReportingMaterializerLease) -> None: + async with self._lease_epoch(lease): + await super().authorize_materialization(lease) + + async def finish_materialization( + self, + lease: ReportingMaterializerLease, + *, + prepared: ReportingPreparedRevision | None = None, + verified: ReportingVerifiedDestination | None = None, + error: ReportingWriterFailure | None = None, + ) -> ReportingMaterializerTurn: + if lease.admission_epoch == 2: + self._owner()._assert_components() + async with self._lease_epoch(lease): + return await super().finish_materialization( + lease, prepared=prepared, verified=verified, error=error + ) + + async def read_production_boundaries( + self, *, caller: ReportingDeliveryPrincipal, after: int = 0, limit: int = 100 + ) -> tuple[ReportingMaterializerBoundary, ...]: + token = _EPOCH.set((id(self), 2)) + try: + return await super().read_materializer_boundaries( + caller=caller, after=after, limit=limit + ) + finally: + _EPOCH.reset(token) diff --git a/src/adcp/reporting/production/required_schema.json b/src/adcp/reporting/production/required_schema.json new file mode 100644 index 000000000..587a22303 --- /dev/null +++ b/src/adcp/reporting/production/required_schema.json @@ -0,0 +1,1326 @@ +{ + "column:reporting_production_accounts.account_id": { + "enabled": true, + "fingerprint": "fdc4b549138bf90a5af5ed00b87a86ff9a0cbea553e7f017b4d95510708f6f88" + }, + "column:reporting_production_accounts.activated_at": { + "enabled": true, + "fingerprint": "336df3243b293695d8321965e92f26d3f132e11514ed9678fb5e64e0015fa580" + }, + "column:reporting_production_accounts.admission_epoch": { + "enabled": true, + "fingerprint": "b636d6e865960a7e2527ed11b86a9a63fff025d6106672308d86fc9630c328ab" + }, + "column:reporting_production_accounts.policy": { + "enabled": true, + "fingerprint": "ac355fc16c02b70cb0a24afee8214cdce5f5cbfdc7fd1630786d5101932ecfa4" + }, + "column:reporting_production_delivery_windows.account_id": { + "enabled": true, + "fingerprint": "fdc4b549138bf90a5af5ed00b87a86ff9a0cbea553e7f017b4d95510708f6f88" + }, + "column:reporting_production_delivery_windows.body_sha256": { + "enabled": true, + "fingerprint": "10ccaa0dc3b93d48a1f32c7ef2352a11676632e1e871d77ef9ef4393eea15d27" + }, + "column:reporting_production_delivery_windows.expires_at": { + "enabled": true, + "fingerprint": "1cac4e73af11a8ecafd646ef6a0ff6ecb087d46f150408dcfebc630fe1bf5e1e" + }, + "column:reporting_production_delivery_windows.idempotency_key": { + "enabled": true, + "fingerprint": "fdc4b549138bf90a5af5ed00b87a86ff9a0cbea553e7f017b4d95510708f6f88" + }, + "column:reporting_production_delivery_windows.queue": { + "enabled": true, + "fingerprint": "10ccaa0dc3b93d48a1f32c7ef2352a11676632e1e871d77ef9ef4393eea15d27" + }, + "column:reporting_production_delivery_windows.started_at": { + "enabled": true, + "fingerprint": "1cac4e73af11a8ecafd646ef6a0ff6ecb087d46f150408dcfebc630fe1bf5e1e" + }, + "column:reporting_production_destination_bindings.account_id": { + "enabled": true, + "fingerprint": "fdc4b549138bf90a5af5ed00b87a86ff9a0cbea553e7f017b4d95510708f6f88" + }, + "column:reporting_production_destination_bindings.consumer_id": { + "enabled": true, + "fingerprint": "fdc4b549138bf90a5af5ed00b87a86ff9a0cbea553e7f017b4d95510708f6f88" + }, + "column:reporting_production_destination_bindings.delivery_config_id": { + "enabled": true, + "fingerprint": "fdc4b549138bf90a5af5ed00b87a86ff9a0cbea553e7f017b4d95510708f6f88" + }, + "column:reporting_production_destination_bindings.delivery_config_version": { + "enabled": true, + "fingerprint": "64be57437fdc0a07a97985c2aa058031f8082db7251bdb4d5afa1a9b088de97a" + }, + "column:reporting_production_destination_bindings.method": { + "enabled": true, + "fingerprint": "ac355fc16c02b70cb0a24afee8214cdce5f5cbfdc7fd1630786d5101932ecfa4" + }, + "column:reporting_production_generations.account_id": { + "enabled": true, + "fingerprint": "fdc4b549138bf90a5af5ed00b87a86ff9a0cbea553e7f017b4d95510708f6f88" + }, + "column:reporting_production_generations.delivery_config_id": { + "enabled": true, + "fingerprint": "fdc4b549138bf90a5af5ed00b87a86ff9a0cbea553e7f017b4d95510708f6f88" + }, + "column:reporting_production_generations.delivery_config_version": { + "enabled": true, + "fingerprint": "64be57437fdc0a07a97985c2aa058031f8082db7251bdb4d5afa1a9b088de97a" + }, + "column:reporting_production_generations.producer_key": { + "enabled": true, + "fingerprint": "fdc4b549138bf90a5af5ed00b87a86ff9a0cbea553e7f017b4d95510708f6f88" + }, + "column:reporting_production_generations.source_binding": { + "enabled": true, + "fingerprint": "ac355fc16c02b70cb0a24afee8214cdce5f5cbfdc7fd1630786d5101932ecfa4" + }, + "column:reporting_production_notification_deliveries.account_id": { + "enabled": true, + "fingerprint": "fdc4b549138bf90a5af5ed00b87a86ff9a0cbea553e7f017b4d95510708f6f88" + }, + "column:reporting_production_notification_deliveries.auth_mode": { + "enabled": true, + "fingerprint": "10ccaa0dc3b93d48a1f32c7ef2352a11676632e1e871d77ef9ef4393eea15d27" + }, + "column:reporting_production_notification_deliveries.body_sha256": { + "enabled": true, + "fingerprint": "10ccaa0dc3b93d48a1f32c7ef2352a11676632e1e871d77ef9ef4393eea15d27" + }, + "column:reporting_production_notification_deliveries.cause_generation": { + "enabled": true, + "fingerprint": "54a5f9962598caa8723435f8b55bd5e15823053af5dd63d531663997537dc6ba" + }, + "column:reporting_production_notification_deliveries.cause_id": { + "enabled": true, + "fingerprint": "fdc4b549138bf90a5af5ed00b87a86ff9a0cbea553e7f017b4d95510708f6f88" + }, + "column:reporting_production_notification_deliveries.cause_kind": { + "enabled": true, + "fingerprint": "fdc4b549138bf90a5af5ed00b87a86ff9a0cbea553e7f017b4d95510708f6f88" + }, + "column:reporting_production_notification_deliveries.claim_count": { + "enabled": true, + "fingerprint": "42202005517b72e082eb22c9eceb2ac0252815e5df700c83eb50c54cfeb46297" + }, + "column:reporting_production_notification_deliveries.consumer_namespace": { + "enabled": true, + "fingerprint": "fdc4b549138bf90a5af5ed00b87a86ff9a0cbea553e7f017b4d95510708f6f88" + }, + "column:reporting_production_notification_deliveries.delivery_id": { + "enabled": true, + "fingerprint": "fdc4b549138bf90a5af5ed00b87a86ff9a0cbea553e7f017b4d95510708f6f88" + }, + "column:reporting_production_notification_deliveries.destination_sha256": { + "enabled": true, + "fingerprint": "10ccaa0dc3b93d48a1f32c7ef2352a11676632e1e871d77ef9ef4393eea15d27" + }, + "column:reporting_production_notification_deliveries.due_at": { + "enabled": true, + "fingerprint": "1cac4e73af11a8ecafd646ef6a0ff6ecb087d46f150408dcfebc630fe1bf5e1e" + }, + "column:reporting_production_notification_deliveries.emission_generation": { + "enabled": true, + "fingerprint": "54a5f9962598caa8723435f8b55bd5e15823053af5dd63d531663997537dc6ba" + }, + "column:reporting_production_notification_deliveries.envelope": { + "enabled": true, + "fingerprint": "554c34e416bd4546469b42d5773bc7c59b2104366ffa5259a2838c64cd826e57" + }, + "column:reporting_production_notification_deliveries.envelope_version": { + "enabled": true, + "fingerprint": "64be57437fdc0a07a97985c2aa058031f8082db7251bdb4d5afa1a9b088de97a" + }, + "column:reporting_production_notification_deliveries.error_code": { + "enabled": true, + "fingerprint": "5b92595d0b54d473a3a3818455845f1fe0dc20b48cfe0faf63061a82d1a3cb02" + }, + "column:reporting_production_notification_deliveries.idempotency_key": { + "enabled": true, + "fingerprint": "fdc4b549138bf90a5af5ed00b87a86ff9a0cbea553e7f017b4d95510708f6f88" + }, + "column:reporting_production_notification_deliveries.key_version": { + "enabled": true, + "fingerprint": "fdc4b549138bf90a5af5ed00b87a86ff9a0cbea553e7f017b4d95510708f6f88" + }, + "column:reporting_production_notification_deliveries.lease_expires_at": { + "enabled": true, + "fingerprint": "6f1466ce5d0aaac8471e39834b9c4b1f85d6f7169d9238a245ac035ff518e0fc" + }, + "column:reporting_production_notification_deliveries.lease_token": { + "enabled": true, + "fingerprint": "5b92595d0b54d473a3a3818455845f1fe0dc20b48cfe0faf63061a82d1a3cb02" + }, + "column:reporting_production_notification_deliveries.notification_id": { + "enabled": true, + "fingerprint": "fdc4b549138bf90a5af5ed00b87a86ff9a0cbea553e7f017b4d95510708f6f88" + }, + "column:reporting_production_notification_deliveries.notification_type": { + "enabled": true, + "fingerprint": "fdc4b549138bf90a5af5ed00b87a86ff9a0cbea553e7f017b4d95510708f6f88" + }, + "column:reporting_production_notification_deliveries.principal_id": { + "enabled": true, + "fingerprint": "fdc4b549138bf90a5af5ed00b87a86ff9a0cbea553e7f017b4d95510708f6f88" + }, + "column:reporting_production_notification_deliveries.signing_scope_id": { + "enabled": true, + "fingerprint": "85b88fb407e112399f25b5dc8830dfdcb2a8271ecf00e70b662f6917b4a002cd" + }, + "column:reporting_production_notification_deliveries.state": { + "enabled": true, + "fingerprint": "1a1ab7c892bfef3ca42f00cf764453bc5e0578b3a82ecf81da90b56816df493f" + }, + "column:reporting_production_notification_deliveries.subscriber_id": { + "enabled": true, + "fingerprint": "fdc4b549138bf90a5af5ed00b87a86ff9a0cbea553e7f017b4d95510708f6f88" + }, + "column:reporting_production_notification_deliveries.subscription_fingerprint": { + "enabled": true, + "fingerprint": "10ccaa0dc3b93d48a1f32c7ef2352a11676632e1e871d77ef9ef4393eea15d27" + }, + "column:reporting_production_notification_events.account_id": { + "enabled": true, + "fingerprint": "fdc4b549138bf90a5af5ed00b87a86ff9a0cbea553e7f017b4d95510708f6f88" + }, + "column:reporting_production_notification_events.admission_epoch": { + "enabled": true, + "fingerprint": "b636d6e865960a7e2527ed11b86a9a63fff025d6106672308d86fc9630c328ab" + }, + "column:reporting_production_notification_events.cause_generation": { + "enabled": true, + "fingerprint": "54a5f9962598caa8723435f8b55bd5e15823053af5dd63d531663997537dc6ba" + }, + "column:reporting_production_notification_events.cause_id": { + "enabled": true, + "fingerprint": "fdc4b549138bf90a5af5ed00b87a86ff9a0cbea553e7f017b4d95510708f6f88" + }, + "column:reporting_production_notification_events.cause_kind": { + "enabled": true, + "fingerprint": "fdc4b549138bf90a5af5ed00b87a86ff9a0cbea553e7f017b4d95510708f6f88" + }, + "column:reporting_production_notification_events.consumer_namespace": { + "enabled": true, + "fingerprint": "fdc4b549138bf90a5af5ed00b87a86ff9a0cbea553e7f017b4d95510708f6f88" + }, + "column:reporting_production_notification_events.fired_at": { + "enabled": true, + "fingerprint": "1cac4e73af11a8ecafd646ef6a0ff6ecb087d46f150408dcfebc630fe1bf5e1e" + }, + "column:reporting_production_notification_events.notification_id": { + "enabled": true, + "fingerprint": "fdc4b549138bf90a5af5ed00b87a86ff9a0cbea553e7f017b4d95510708f6f88" + }, + "column:reporting_production_notification_events.notification_type": { + "enabled": true, + "fingerprint": "fdc4b549138bf90a5af5ed00b87a86ff9a0cbea553e7f017b4d95510708f6f88" + }, + "column:reporting_production_notification_events.reporting_materialization_id": { + "enabled": true, + "fingerprint": "a0d17631e6e95ba976e4615a034020bc089fa63601fb91205ebb35798a39c0de" + }, + "column:reporting_production_notification_events.snapshot": { + "enabled": true, + "fingerprint": "ac355fc16c02b70cb0a24afee8214cdce5f5cbfdc7fd1630786d5101932ecfa4" + }, + "column:reporting_production_notification_expansions.account_id": { + "enabled": true, + "fingerprint": "fdc4b549138bf90a5af5ed00b87a86ff9a0cbea553e7f017b4d95510708f6f88" + }, + "column:reporting_production_notification_expansions.claim_count": { + "enabled": true, + "fingerprint": "42202005517b72e082eb22c9eceb2ac0252815e5df700c83eb50c54cfeb46297" + }, + "column:reporting_production_notification_expansions.consumer_namespace": { + "enabled": true, + "fingerprint": "fdc4b549138bf90a5af5ed00b87a86ff9a0cbea553e7f017b4d95510708f6f88" + }, + "column:reporting_production_notification_expansions.due_at": { + "enabled": true, + "fingerprint": "1cac4e73af11a8ecafd646ef6a0ff6ecb087d46f150408dcfebc630fe1bf5e1e" + }, + "column:reporting_production_notification_expansions.emission_generation": { + "enabled": true, + "fingerprint": "54a5f9962598caa8723435f8b55bd5e15823053af5dd63d531663997537dc6ba" + }, + "column:reporting_production_notification_expansions.error_code": { + "enabled": true, + "fingerprint": "5b92595d0b54d473a3a3818455845f1fe0dc20b48cfe0faf63061a82d1a3cb02" + }, + "column:reporting_production_notification_expansions.lease_expires_at": { + "enabled": true, + "fingerprint": "6f1466ce5d0aaac8471e39834b9c4b1f85d6f7169d9238a245ac035ff518e0fc" + }, + "column:reporting_production_notification_expansions.lease_token": { + "enabled": true, + "fingerprint": "5b92595d0b54d473a3a3818455845f1fe0dc20b48cfe0faf63061a82d1a3cb02" + }, + "column:reporting_production_notification_expansions.notification_id": { + "enabled": true, + "fingerprint": "fdc4b549138bf90a5af5ed00b87a86ff9a0cbea553e7f017b4d95510708f6f88" + }, + "column:reporting_production_notification_expansions.state": { + "enabled": true, + "fingerprint": "1a1ab7c892bfef3ca42f00cf764453bc5e0578b3a82ecf81da90b56816df493f" + }, + "column:reporting_production_source_probe_turns.account_id": { + "enabled": true, + "fingerprint": "fdc4b549138bf90a5af5ed00b87a86ff9a0cbea553e7f017b4d95510708f6f88" + }, + "column:reporting_production_source_probe_turns.delivery_config_id": { + "enabled": true, + "fingerprint": "fdc4b549138bf90a5af5ed00b87a86ff9a0cbea553e7f017b4d95510708f6f88" + }, + "column:reporting_production_source_probe_turns.delivery_config_version": { + "enabled": true, + "fingerprint": "64be57437fdc0a07a97985c2aa058031f8082db7251bdb4d5afa1a9b088de97a" + }, + "column:reporting_production_source_probe_turns.probe_turn": { + "enabled": true, + "fingerprint": "54a5f9962598caa8723435f8b55bd5e15823053af5dd63d531663997537dc6ba" + }, + "column:reporting_production_source_progress.account_id": { + "enabled": true, + "fingerprint": "fdc4b549138bf90a5af5ed00b87a86ff9a0cbea553e7f017b4d95510708f6f88" + }, + "column:reporting_production_source_progress.acquisition_turn": { + "enabled": true, + "fingerprint": "42202005517b72e082eb22c9eceb2ac0252815e5df700c83eb50c54cfeb46297" + }, + "column:reporting_production_source_progress.closed_through": { + "enabled": true, + "fingerprint": "6f1466ce5d0aaac8471e39834b9c4b1f85d6f7169d9238a245ac035ff518e0fc" + }, + "column:reporting_production_source_progress.delivery_config_id": { + "enabled": true, + "fingerprint": "fdc4b549138bf90a5af5ed00b87a86ff9a0cbea553e7f017b4d95510708f6f88" + }, + "column:reporting_production_source_progress.delivery_config_version": { + "enabled": true, + "fingerprint": "64be57437fdc0a07a97985c2aa058031f8082db7251bdb4d5afa1a9b088de97a" + }, + "column:reporting_production_source_work.account_id": { + "enabled": true, + "fingerprint": "fdc4b549138bf90a5af5ed00b87a86ff9a0cbea553e7f017b4d95510708f6f88" + }, + "column:reporting_production_source_work.acquisition_turn": { + "enabled": true, + "fingerprint": "42202005517b72e082eb22c9eceb2ac0252815e5df700c83eb50c54cfeb46297" + }, + "column:reporting_production_source_work.delivery_config_id": { + "enabled": true, + "fingerprint": "fdc4b549138bf90a5af5ed00b87a86ff9a0cbea553e7f017b4d95510708f6f88" + }, + "column:reporting_production_source_work.delivery_config_version": { + "enabled": true, + "fingerprint": "64be57437fdc0a07a97985c2aa058031f8082db7251bdb4d5afa1a9b088de97a" + }, + "column:reporting_production_source_work.period_end": { + "enabled": true, + "fingerprint": "1cac4e73af11a8ecafd646ef6a0ff6ecb087d46f150408dcfebc630fe1bf5e1e" + }, + "column:reporting_production_source_work.reporting_obligation_id": { + "enabled": true, + "fingerprint": "fdc4b549138bf90a5af5ed00b87a86ff9a0cbea553e7f017b4d95510708f6f88" + }, + "column:reporting_production_source_work.state": { + "enabled": true, + "fingerprint": "675d9766cc6787c63f91ee16167a1de38069be7ad44f6d8362bd498885ae1355" + }, + "column:reporting_production_status_boundaries.account_id": { + "enabled": true, + "fingerprint": "fdc4b549138bf90a5af5ed00b87a86ff9a0cbea553e7f017b4d95510708f6f88" + }, + "column:reporting_production_status_boundaries.account_sequence": { + "enabled": true, + "fingerprint": "54a5f9962598caa8723435f8b55bd5e15823053af5dd63d531663997537dc6ba" + }, + "column:reporting_production_status_boundaries.as_of": { + "enabled": true, + "fingerprint": "1cac4e73af11a8ecafd646ef6a0ff6ecb087d46f150408dcfebc630fe1bf5e1e" + }, + "column:reporting_production_status_boundaries.consumer_id": { + "enabled": true, + "fingerprint": "fdc4b549138bf90a5af5ed00b87a86ff9a0cbea553e7f017b4d95510708f6f88" + }, + "column:reporting_production_status_boundaries.content_sha256": { + "enabled": true, + "fingerprint": "fdc4b549138bf90a5af5ed00b87a86ff9a0cbea553e7f017b4d95510708f6f88" + }, + "column:reporting_production_status_boundaries.input": { + "enabled": true, + "fingerprint": "ac355fc16c02b70cb0a24afee8214cdce5f5cbfdc7fd1630786d5101932ecfa4" + }, + "column:reporting_production_status_boundaries.outcome_namespace": { + "enabled": true, + "fingerprint": "37f064bd049ffa20c99bccfcb2ddd77b4e7b65fc6471d7683087ed0beec2098e" + }, + "column:reporting_production_status_boundaries.reporting_materialization_id": { + "enabled": true, + "fingerprint": "fdc4b549138bf90a5af5ed00b87a86ff9a0cbea553e7f017b4d95510708f6f88" + }, + "column:reporting_production_status_boundaries.sequence": { + "enabled": true, + "fingerprint": "54a5f9962598caa8723435f8b55bd5e15823053af5dd63d531663997537dc6ba" + }, + "column:reporting_production_status_heads.account_id": { + "enabled": true, + "fingerprint": "fdc4b549138bf90a5af5ed00b87a86ff9a0cbea553e7f017b4d95510708f6f88" + }, + "column:reporting_production_status_heads.consumer_id": { + "enabled": true, + "fingerprint": "fdc4b549138bf90a5af5ed00b87a86ff9a0cbea553e7f017b4d95510708f6f88" + }, + "column:reporting_production_status_heads.max_sequence": { + "enabled": true, + "fingerprint": "54a5f9962598caa8723435f8b55bd5e15823053af5dd63d531663997537dc6ba" + }, + "column:reporting_production_webhook_attempt_heads.account_id": { + "enabled": true, + "fingerprint": "fdc4b549138bf90a5af5ed00b87a86ff9a0cbea553e7f017b4d95510708f6f88" + }, + "column:reporting_production_webhook_attempt_heads.consumer_namespace": { + "enabled": true, + "fingerprint": "fdc4b549138bf90a5af5ed00b87a86ff9a0cbea553e7f017b4d95510708f6f88" + }, + "column:reporting_production_webhook_attempt_heads.idempotency_key": { + "enabled": true, + "fingerprint": "fdc4b549138bf90a5af5ed00b87a86ff9a0cbea553e7f017b4d95510708f6f88" + }, + "column:reporting_production_webhook_attempt_heads.last_attempt": { + "enabled": true, + "fingerprint": "54a5f9962598caa8723435f8b55bd5e15823053af5dd63d531663997537dc6ba" + }, + "column:reporting_production_webhook_attempt_heads.principal_id": { + "enabled": true, + "fingerprint": "fdc4b549138bf90a5af5ed00b87a86ff9a0cbea553e7f017b4d95510708f6f88" + }, + "column:reporting_production_webhook_attempt_heads.subscriber_id": { + "enabled": true, + "fingerprint": "fdc4b549138bf90a5af5ed00b87a86ff9a0cbea553e7f017b4d95510708f6f88" + }, + "column:reporting_production_webhook_attempts.account_id": { + "enabled": true, + "fingerprint": "fdc4b549138bf90a5af5ed00b87a86ff9a0cbea553e7f017b4d95510708f6f88" + }, + "column:reporting_production_webhook_attempts.attempt": { + "enabled": true, + "fingerprint": "54a5f9962598caa8723435f8b55bd5e15823053af5dd63d531663997537dc6ba" + }, + "column:reporting_production_webhook_attempts.binding": { + "enabled": true, + "fingerprint": "ac355fc16c02b70cb0a24afee8214cdce5f5cbfdc7fd1630786d5101932ecfa4" + }, + "column:reporting_production_webhook_attempts.completed_at": { + "enabled": true, + "fingerprint": "6f1466ce5d0aaac8471e39834b9c4b1f85d6f7169d9238a245ac035ff518e0fc" + }, + "column:reporting_production_webhook_attempts.consumer_namespace": { + "enabled": true, + "fingerprint": "fdc4b549138bf90a5af5ed00b87a86ff9a0cbea553e7f017b4d95510708f6f88" + }, + "column:reporting_production_webhook_attempts.delivery_id": { + "enabled": true, + "fingerprint": "fdc4b549138bf90a5af5ed00b87a86ff9a0cbea553e7f017b4d95510708f6f88" + }, + "column:reporting_production_webhook_attempts.fired_at": { + "enabled": true, + "fingerprint": "1cac4e73af11a8ecafd646ef6a0ff6ecb087d46f150408dcfebc630fe1bf5e1e" + }, + "column:reporting_production_webhook_attempts.http_status_code": { + "enabled": true, + "fingerprint": "4340876cb26818ac55a8d51cfbc7047e90fc4d7e44f570ee454b4552beb351a1" + }, + "column:reporting_production_webhook_attempts.idempotency_key": { + "enabled": true, + "fingerprint": "fdc4b549138bf90a5af5ed00b87a86ff9a0cbea553e7f017b4d95510708f6f88" + }, + "column:reporting_production_webhook_attempts.lease_token": { + "enabled": true, + "fingerprint": "10ccaa0dc3b93d48a1f32c7ef2352a11676632e1e871d77ef9ef4393eea15d27" + }, + "column:reporting_production_webhook_attempts.notification_id": { + "enabled": true, + "fingerprint": "fdc4b549138bf90a5af5ed00b87a86ff9a0cbea553e7f017b4d95510708f6f88" + }, + "column:reporting_production_webhook_attempts.payload_size_bytes": { + "enabled": true, + "fingerprint": "54a5f9962598caa8723435f8b55bd5e15823053af5dd63d531663997537dc6ba" + }, + "column:reporting_production_webhook_attempts.principal_id": { + "enabled": true, + "fingerprint": "fdc4b549138bf90a5af5ed00b87a86ff9a0cbea553e7f017b4d95510708f6f88" + }, + "column:reporting_production_webhook_attempts.reservation_token": { + "enabled": true, + "fingerprint": "10ccaa0dc3b93d48a1f32c7ef2352a11676632e1e871d77ef9ef4393eea15d27" + }, + "column:reporting_production_webhook_attempts.response_time_ms": { + "enabled": true, + "fingerprint": "992336704a95e12ec6e959825c59fa2e51cddc5f1568af6bebaf12f03ac5655f" + }, + "column:reporting_production_webhook_attempts.status": { + "enabled": true, + "fingerprint": "1a1ab7c892bfef3ca42f00cf764453bc5e0578b3a82ecf81da90b56816df493f" + }, + "column:reporting_production_webhook_attempts.subscriber_id": { + "enabled": true, + "fingerprint": "fdc4b549138bf90a5af5ed00b87a86ff9a0cbea553e7f017b4d95510708f6f88" + }, + "column:reporting_production_webhook_attempts.url": { + "enabled": true, + "fingerprint": "10ccaa0dc3b93d48a1f32c7ef2352a11676632e1e871d77ef9ef4393eea15d27" + }, + "column:reporting_production_work.account_id": { + "enabled": true, + "fingerprint": "fdc4b549138bf90a5af5ed00b87a86ff9a0cbea553e7f017b4d95510708f6f88" + }, + "column:reporting_production_work.acknowledged_at": { + "enabled": true, + "fingerprint": "6f1466ce5d0aaac8471e39834b9c4b1f85d6f7169d9238a245ac035ff518e0fc" + }, + "column:reporting_production_work.admission_epoch": { + "enabled": true, + "fingerprint": "b636d6e865960a7e2527ed11b86a9a63fff025d6106672308d86fc9630c328ab" + }, + "column:reporting_production_work.attempt_namespace": { + "enabled": true, + "fingerprint": "48602ba37bcbee2d9c8104042cc7868262df4d288eaf1543e387b9a059a08117" + }, + "column:reporting_production_work.binding_sha256": { + "enabled": true, + "fingerprint": "fdc4b549138bf90a5af5ed00b87a86ff9a0cbea553e7f017b4d95510708f6f88" + }, + "column:reporting_production_work.completion_token": { + "enabled": true, + "fingerprint": "88223bccb5aae7ddfe4b3feacd193586f6699cbfaa3507ae6c21250775392e25" + }, + "column:reporting_production_work.consumer_id": { + "enabled": true, + "fingerprint": "fdc4b549138bf90a5af5ed00b87a86ff9a0cbea553e7f017b4d95510708f6f88" + }, + "column:reporting_production_work.created_at": { + "enabled": true, + "fingerprint": "336df3243b293695d8321965e92f26d3f132e11514ed9678fb5e64e0015fa580" + }, + "column:reporting_production_work.delivery_config_id": { + "enabled": true, + "fingerprint": "fdc4b549138bf90a5af5ed00b87a86ff9a0cbea553e7f017b4d95510708f6f88" + }, + "column:reporting_production_work.delivery_config_version": { + "enabled": true, + "fingerprint": "54a5f9962598caa8723435f8b55bd5e15823053af5dd63d531663997537dc6ba" + }, + "column:reporting_production_work.due_at": { + "enabled": true, + "fingerprint": "336df3243b293695d8321965e92f26d3f132e11514ed9678fb5e64e0015fa580" + }, + "column:reporting_production_work.external_id": { + "enabled": true, + "fingerprint": "fdc4b549138bf90a5af5ed00b87a86ff9a0cbea553e7f017b4d95510708f6f88" + }, + "column:reporting_production_work.generation": { + "enabled": true, + "fingerprint": "54a5f9962598caa8723435f8b55bd5e15823053af5dd63d531663997537dc6ba" + }, + "column:reporting_production_work.imported": { + "enabled": true, + "fingerprint": "981e469ff869d932309f9e6aab9b07ff394c7439fe2431281320e25b41c4198d" + }, + "column:reporting_production_work.lease_token": { + "enabled": true, + "fingerprint": "88223bccb5aae7ddfe4b3feacd193586f6699cbfaa3507ae6c21250775392e25" + }, + "column:reporting_production_work.lease_until": { + "enabled": true, + "fingerprint": "6f1466ce5d0aaac8471e39834b9c4b1f85d6f7169d9238a245ac035ff518e0fc" + }, + "column:reporting_production_work.notifications_enabled": { + "enabled": true, + "fingerprint": "1abe3a1c570fbe885784dab5d979307c373d50f567ff4481ce3996869bf58fed" + }, + "column:reporting_production_work.reason": { + "enabled": true, + "fingerprint": "806b02ec486fb1a02f57be76cf01aba1287baf2d669233d70849bcb0df7c558c" + }, + "column:reporting_production_work.reporting_materialization_id": { + "enabled": true, + "fingerprint": "fdc4b549138bf90a5af5ed00b87a86ff9a0cbea553e7f017b4d95510708f6f88" + }, + "column:reporting_production_work.reporting_obligation_id": { + "enabled": true, + "fingerprint": "fdc4b549138bf90a5af5ed00b87a86ff9a0cbea553e7f017b4d95510708f6f88" + }, + "column:reporting_production_work.reporting_revision_id": { + "enabled": true, + "fingerprint": "fdc4b549138bf90a5af5ed00b87a86ff9a0cbea553e7f017b4d95510708f6f88" + }, + "column:reporting_production_work.retry_allowed": { + "enabled": true, + "fingerprint": "981e469ff869d932309f9e6aab9b07ff394c7439fe2431281320e25b41c4198d" + }, + "column:reporting_production_work.state": { + "enabled": true, + "fingerprint": "675d9766cc6787c63f91ee16167a1de38069be7ad44f6d8362bd498885ae1355" + }, + "column:reporting_production_work.verification_key_sha256": { + "enabled": true, + "fingerprint": "fdc4b549138bf90a5af5ed00b87a86ff9a0cbea553e7f017b4d95510708f6f88" + }, + "constraint:reporting_production_accounts.reporting_production_accounts_account_id_fkey": { + "enabled": true, + "fingerprint": "53a472fec66c9e990740ba025807acc57c5795bd0a84ea1ad29f8447fcfd0c85" + }, + "constraint:reporting_production_accounts.reporting_production_accounts_admission_epoch_check": { + "enabled": true, + "fingerprint": "6cfd9db9dd38e8c29346710c69de6ff07379d1bef2e9e5b1aba297a3ffa833f2" + }, + "constraint:reporting_production_accounts.reporting_production_accounts_pkey": { + "enabled": true, + "fingerprint": "e65d70e61c89a93d66c4c4f4c59ef755d0ff526b0fcdd1d9ae6d830d2abea913" + }, + "constraint:reporting_production_accounts.reporting_production_accounts_policy_check": { + "enabled": true, + "fingerprint": "80f89f47761fc22c72aabd7083963e72e4113bcfe00f0923f45be306e540b837" + }, + "constraint:reporting_production_accounts.reporting_production_accounts_policy_check1": { + "enabled": true, + "fingerprint": "e83adfb83c455b7b7141555b4fc6c67370d787abf3c77c87001d3aa622239510" + }, + "constraint:reporting_production_accounts.reporting_production_accounts_policy_check2": { + "enabled": true, + "fingerprint": "5c2314fc1a16a0761359a7885fc31c52f3a91e1a80ad7ea305a6cb3807503502" + }, + "constraint:reporting_production_accounts.reporting_production_accounts_policy_check3": { + "enabled": true, + "fingerprint": "39bd47a78f6779fe1881995a88ce9ee48123e77e936ef12b30ed231a44019c11" + }, + "constraint:reporting_production_delivery_windows.reporting_production_delivery_windows_body_sha256_check": { + "enabled": true, + "fingerprint": "08bcbac24dab1de32cebf5ea27d21c0762ef54f2fd47ce6045a5815ee395b1ab" + }, + "constraint:reporting_production_delivery_windows.reporting_production_delivery_windows_check": { + "enabled": true, + "fingerprint": "f675507fe20dae95352895867c06a71166f3889144d28389539d43a345a358f5" + }, + "constraint:reporting_production_delivery_windows.reporting_production_delivery_windows_pkey": { + "enabled": true, + "fingerprint": "c49724177245a89c200c46c61b54b60e7988cddb577d3ba58c67f771cf2ba204" + }, + "constraint:reporting_production_delivery_windows.reporting_production_delivery_windows_queue_check": { + "enabled": true, + "fingerprint": "d4ad040b32660abca511e5442252d8b9e1be8be6e7db8a009c464b51d0a56345" + }, + "constraint:reporting_production_destination_bindings.reporting_production_destinat_account_id_delivery_config_i_fkey": { + "enabled": true, + "fingerprint": "75c62699e078bab9886b32e70b19f33a126b20459e13e71b7a1aab4b4a02e4fb" + }, + "constraint:reporting_production_destination_bindings.reporting_production_destination_bindings_method_check": { + "enabled": true, + "fingerprint": "a1d15c03d0e951caea89b55170544a51d76bac60017539348d3f55a73caca80b" + }, + "constraint:reporting_production_destination_bindings.reporting_production_destination_bindings_pkey": { + "enabled": true, + "fingerprint": "cb0658b07aa5249a6934cca6248293967db4ab1a848d706c3d5310fa98f08785" + }, + "constraint:reporting_production_generations.reporting_production_generati_account_id_delivery_config_i_fkey": { + "enabled": true, + "fingerprint": "4761471309c35947c8d3928dce4595b4d455e7de6166aeb78a5de220465cb64a" + }, + "constraint:reporting_production_generations.reporting_production_generations_pkey": { + "enabled": true, + "fingerprint": "d9ba22090ae520014ae689fbeb3206c705853c9227e68c533acd51317287e944" + }, + "constraint:reporting_production_generations.reporting_production_generations_producer_key_check": { + "enabled": true, + "fingerprint": "50fbb9f237cebdab6477e2b7e6a469e50607d1fb5d4ad21505f9fe5f0e8f8fc8" + }, + "constraint:reporting_production_generations.reporting_production_generations_source_binding_check": { + "enabled": true, + "fingerprint": "4fdc2c43438889e1348918f91fb9793d2241bc40abfef17de4b9cc822f6c39ca" + }, + "constraint:reporting_production_notification_deliveries.reporting_production_notific_account_id_consumer_namespac_fkey2": { + "enabled": true, + "fingerprint": "290b23049e9cd59676d28813c39b0b7ee4319a728220816e72ebaee448450298" + }, + "constraint:reporting_production_notification_deliveries.reporting_production_notific_account_id_consumer_namespac_fkey3": { + "enabled": true, + "fingerprint": "8673e8804fc529b047821250805cf1f6350f041bf84c658933c53e64c2a5e8aa" + }, + "constraint:reporting_production_notification_deliveries.reporting_production_notifica_account_id_consumer_namespac_key2": { + "enabled": true, + "fingerprint": "91982910079d78e1b6162e69e1e40d0f98ab055b811726cd8a9357c4aa9cfd22" + }, + "constraint:reporting_production_notification_deliveries.reporting_production_notifica_account_id_consumer_namespac_key3": { + "enabled": true, + "fingerprint": "bece3b7de1c22f3a600c89c175f942f83ecff078142418072b1bc1b42a670c18" + }, + "constraint:reporting_production_notification_deliveries.reporting_production_notification_del_emission_generation_check": { + "enabled": true, + "fingerprint": "5b7b2793bd710c7eef8c02003fd8adb1df91d7c10659364fe37d7f6351d4bf5e" + }, + "constraint:reporting_production_notification_deliveries.reporting_production_notification_deliv_notification_type_check": { + "enabled": true, + "fingerprint": "4dc2b2e5d5fe003a3b98e1832dd453c9115cc3d03d59d3cb23258e90605548e3" + }, + "constraint:reporting_production_notification_deliveries.reporting_production_notification_delive_cause_generation_check": { + "enabled": true, + "fingerprint": "d93961696665129cf254c93dae4377741c5cef5fe9d373c6a5cb36226e5a69e2" + }, + "constraint:reporting_production_notification_deliveries.reporting_production_notification_deliveries_cause_kind_check": { + "enabled": true, + "fingerprint": "4d213a62ba48d8d428c2811ddc5039b1349d3a41943509a85062b43235ab669d" + }, + "constraint:reporting_production_notification_deliveries.reporting_production_notification_deliveries_check": { + "enabled": true, + "fingerprint": "a61e7698dd266b3a4abee2c2716b00919f047e85bcd888a90c1a5bc5350c4d6e" + }, + "constraint:reporting_production_notification_deliveries.reporting_production_notification_deliveries_error_code_check": { + "enabled": true, + "fingerprint": "502085808cda7e7ec1e45e3d88257a660bdc65b59a71727b473303a362b9cf9d" + }, + "constraint:reporting_production_notification_deliveries.reporting_production_notification_deliveries_pkey": { + "enabled": true, + "fingerprint": "126da0131155e9b09e48d24466c5f61b7b6dc1540a68c465c52b59769637820d" + }, + "constraint:reporting_production_notification_deliveries.reporting_production_notification_deliveries_state_check": { + "enabled": true, + "fingerprint": "12b7a569e49f4bf374d5dade9ec0a0d812e4dea251366467c95bcd58bee90b9d" + }, + "constraint:reporting_production_notification_events.reporting_production_notifica_account_id_consumer_namespac_fkey": { + "enabled": true, + "fingerprint": "39075eefddb9b204eef01670600946cd8e67c770bc795b66256ab4974afdef81" + }, + "constraint:reporting_production_notification_events.reporting_production_notifica_account_id_consumer_namespac_key1": { + "enabled": true, + "fingerprint": "1d1f2cee412feec063fd1340b0d7773adfc55cad1f740e21bb45f75acf185b8c" + }, + "constraint:reporting_production_notification_events.reporting_production_notifica_account_id_consumer_namespace_key": { + "enabled": true, + "fingerprint": "7156cd6623f45f181906d2886b53dfba82bfdfc6cdcbdd0017c1f1cd7ceae407" + }, + "constraint:reporting_production_notification_events.reporting_production_notification_even_consumer_namespace_check": { + "enabled": true, + "fingerprint": "69951e411156e1739417ca750e81d5d806b2fcccea7e1914674f730ca6433b58" + }, + "constraint:reporting_production_notification_events.reporting_production_notification_event_notification_type_check": { + "enabled": true, + "fingerprint": "4dc2b2e5d5fe003a3b98e1832dd453c9115cc3d03d59d3cb23258e90605548e3" + }, + "constraint:reporting_production_notification_events.reporting_production_notification_events_admission_epoch_check": { + "enabled": true, + "fingerprint": "6cfd9db9dd38e8c29346710c69de6ff07379d1bef2e9e5b1aba297a3ffa833f2" + }, + "constraint:reporting_production_notification_events.reporting_production_notification_events_cause_generation_check": { + "enabled": true, + "fingerprint": "b43ce8b2f74254027bcc0d382aa0bda21801bd6539f4ea3365679718c5cd80ba" + }, + "constraint:reporting_production_notification_events.reporting_production_notification_events_cause_kind_check": { + "enabled": true, + "fingerprint": "4d213a62ba48d8d428c2811ddc5039b1349d3a41943509a85062b43235ab669d" + }, + "constraint:reporting_production_notification_events.reporting_production_notification_events_check": { + "enabled": true, + "fingerprint": "bc5fdbb84edccb22f49869598e550f0f72ee45323f0852be3438c8c5d5a387fc" + }, + "constraint:reporting_production_notification_events.reporting_production_notification_events_check1": { + "enabled": true, + "fingerprint": "3d95e12acbdfe40cfd7c36ea0c4cceddc8106395e3165b6c8034dd14783ba596" + }, + "constraint:reporting_production_notification_events.reporting_production_notification_events_check2": { + "enabled": true, + "fingerprint": "abcd49a900cec31d6a71175389de38cf44ffe6d16fa9083e45fe82967313db78" + }, + "constraint:reporting_production_notification_events.reporting_production_notification_events_check3": { + "enabled": true, + "fingerprint": "66bffc213553425f6a34852e7394536bdd550304c09cdb472a70223bcadb9f7b" + }, + "constraint:reporting_production_notification_events.reporting_production_notification_events_pkey": { + "enabled": true, + "fingerprint": "e8d63f44b25700915bcb3cb69e2e24b98cf21fd304f36a2e9c2992236309309d" + }, + "constraint:reporting_production_notification_expansions.reporting_production_notific_account_id_consumer_namespac_fkey1": { + "enabled": true, + "fingerprint": "543a59e86b26fff09da6144c4685bfd8e2489795644ec4e72668afd29fdcd324" + }, + "constraint:reporting_production_notification_expansions.reporting_production_notification_exp_emission_generation_check": { + "enabled": true, + "fingerprint": "5b7b2793bd710c7eef8c02003fd8adb1df91d7c10659364fe37d7f6351d4bf5e" + }, + "constraint:reporting_production_notification_expansions.reporting_production_notification_expansions_error_code_check": { + "enabled": true, + "fingerprint": "502085808cda7e7ec1e45e3d88257a660bdc65b59a71727b473303a362b9cf9d" + }, + "constraint:reporting_production_notification_expansions.reporting_production_notification_expansions_pkey": { + "enabled": true, + "fingerprint": "8ac920e8e1ed5a59417d5d1ffb9bea608bbeb64dbcd1a431240370fe555fe6f0" + }, + "constraint:reporting_production_notification_expansions.reporting_production_notification_expansions_state_check": { + "enabled": true, + "fingerprint": "12b7a569e49f4bf374d5dade9ec0a0d812e4dea251366467c95bcd58bee90b9d" + }, + "constraint:reporting_production_source_probe_turns.reporting_production_source_p_account_id_delivery_config_i_fkey": { + "enabled": true, + "fingerprint": "75c62699e078bab9886b32e70b19f33a126b20459e13e71b7a1aab4b4a02e4fb" + }, + "constraint:reporting_production_source_probe_turns.reporting_production_source_probe_turns_pkey": { + "enabled": true, + "fingerprint": "d9ba22090ae520014ae689fbeb3206c705853c9227e68c533acd51317287e944" + }, + "constraint:reporting_production_source_probe_turns.reporting_production_source_probe_turns_probe_turn_check": { + "enabled": true, + "fingerprint": "58f1f55f151556fb2aef6678d8418605bef80f3141755502759c72efceaa7a62" + }, + "constraint:reporting_production_source_progress.reporting_production_source__account_id_delivery_config_i_fkey1": { + "enabled": true, + "fingerprint": "75c62699e078bab9886b32e70b19f33a126b20459e13e71b7a1aab4b4a02e4fb" + }, + "constraint:reporting_production_source_progress.reporting_production_source_progress_acquisition_turn_check": { + "enabled": true, + "fingerprint": "43f2b66f59d2dade9afd34a2fb5182710e40cd49ab12d2fed8b569ce4cdd5b9e" + }, + "constraint:reporting_production_source_progress.reporting_production_source_progress_pkey": { + "enabled": true, + "fingerprint": "d9ba22090ae520014ae689fbeb3206c705853c9227e68c533acd51317287e944" + }, + "constraint:reporting_production_source_work.reporting_production_source_w_account_id_delivery_config_i_fkey": { + "enabled": true, + "fingerprint": "75c62699e078bab9886b32e70b19f33a126b20459e13e71b7a1aab4b4a02e4fb" + }, + "constraint:reporting_production_source_work.reporting_production_source_work_acquisition_turn_check": { + "enabled": true, + "fingerprint": "43f2b66f59d2dade9afd34a2fb5182710e40cd49ab12d2fed8b569ce4cdd5b9e" + }, + "constraint:reporting_production_source_work.reporting_production_source_work_pkey": { + "enabled": true, + "fingerprint": "d355e8dc9dcf7840e1eb3d2f21f27c8423668327fe68c988c3df2f0918a2c454" + }, + "constraint:reporting_production_source_work.reporting_production_source_work_reporting_obligation_id_fkey": { + "enabled": true, + "fingerprint": "52080b2461e9d04a0fc27011d8aa726299ff01b18975d49f00c9644137b78b06" + }, + "constraint:reporting_production_source_work.reporting_production_source_work_state_check": { + "enabled": true, + "fingerprint": "81b58b0df487102bccee7def2dc2f710efaa046347c5f0fd4c746d596459a5a3" + }, + "constraint:reporting_production_status_boundaries.reporting_production_status_b_account_id_consumer_id_outco_fkey": { + "enabled": true, + "fingerprint": "e3e10c3f3d574511ee5944b8bade5d927099809b0a45539ce8502caa9ebb05bf" + }, + "constraint:reporting_production_status_boundaries.reporting_production_status_b_account_id_consumer_id_repor_fkey": { + "enabled": true, + "fingerprint": "250741cd0a316028a3f4963b83b558e7daaeb0fc65a3aca9c610e7b7bf1c020d" + }, + "constraint:reporting_production_status_boundaries.reporting_production_status_b_account_id_consumer_id_report_key": { + "enabled": true, + "fingerprint": "72945097549e77da2940c9cc5e54db645c12c0d5ef3bed0d4d7e7ba5cc93ffc1" + }, + "constraint:reporting_production_status_boundaries.reporting_production_status_bou_account_id_account_sequence_key": { + "enabled": true, + "fingerprint": "460b11d8840751c3e894245e33c24e704407f4b3ca42bbd9e832cb48a0c3b24d" + }, + "constraint:reporting_production_status_boundaries.reporting_production_status_boundaries_account_sequence_check": { + "enabled": true, + "fingerprint": "1a64697a81e3b39433d851f9916df1964cbbc260aa54dd6fee4076c7e805c70e" + }, + "constraint:reporting_production_status_boundaries.reporting_production_status_boundaries_check": { + "enabled": true, + "fingerprint": "6db9e761fce16f6deaabe04581b3813f3495382b5edf905c2c01a772151eed89" + }, + "constraint:reporting_production_status_boundaries.reporting_production_status_boundaries_check1": { + "enabled": true, + "fingerprint": "6c5865ad0c28d00ec74194418def9d764ddf85b230f1dfb6d14c6e505eda3737" + }, + "constraint:reporting_production_status_boundaries.reporting_production_status_boundaries_check2": { + "enabled": true, + "fingerprint": "307105bea1a76e214540a207ab8dddcffb26853845a89236454a6e9ecb0508d3" + }, + "constraint:reporting_production_status_boundaries.reporting_production_status_boundaries_check3": { + "enabled": true, + "fingerprint": "65d0dca50d7ca308123252fcc784fb427fd857d8bf3092bd85fc8fff5787d7ec" + }, + "constraint:reporting_production_status_boundaries.reporting_production_status_boundaries_check4": { + "enabled": true, + "fingerprint": "0d806a2b956a1e8947dcc43c634352ed52c1c3f158235145b2fe5a96db5fb379" + }, + "constraint:reporting_production_status_boundaries.reporting_production_status_boundaries_check5": { + "enabled": true, + "fingerprint": "b330c171bc0bb87ac12aa97aca4d4ae7167e0c07d7b61b3b688c63df9588d927" + }, + "constraint:reporting_production_status_boundaries.reporting_production_status_boundaries_check6": { + "enabled": true, + "fingerprint": "9b7f5755a4d4093ab3618e85ff78ea92e931d1895d601ee48f781dcbca07155d" + }, + "constraint:reporting_production_status_boundaries.reporting_production_status_boundaries_content_sha256_check": { + "enabled": true, + "fingerprint": "2afe58d90df96e397cb8e6a941e52fcf8e01b7835024dd3b4ba67141a3578505" + }, + "constraint:reporting_production_status_boundaries.reporting_production_status_boundaries_input_check": { + "enabled": true, + "fingerprint": "1403f246fdde17015118d212847ed5d8169df3adc0364598cbd5e82be066e1bf" + }, + "constraint:reporting_production_status_boundaries.reporting_production_status_boundaries_input_check1": { + "enabled": true, + "fingerprint": "fadd88bcfddae6b78d4fc68c90018f5c21ba800b5beb4ddfb2d5805da7703a2b" + }, + "constraint:reporting_production_status_boundaries.reporting_production_status_boundaries_outcome_namespace_check": { + "enabled": true, + "fingerprint": "60e879d7d85bbf368a4f30d08481dd59f5aa1d92eb6943b0e60a8e8e219c2ff2" + }, + "constraint:reporting_production_status_boundaries.reporting_production_status_boundaries_pkey": { + "enabled": true, + "fingerprint": "00dd5f9ba4e0d0face7ab27fe5e52ffa9c4101943c95afb8ec1badb401c7010d" + }, + "constraint:reporting_production_status_boundaries.reporting_production_status_boundaries_sequence_check": { + "enabled": true, + "fingerprint": "554d491362648e795a6567528a6244977d5df276216297f35e26e4750f869dcf" + }, + "constraint:reporting_production_status_heads.reporting_production_status_heads_max_sequence_check": { + "enabled": true, + "fingerprint": "ee47acec8d450eed7f99b637596847d21bc449a12317a605fd6b6cb0fad6aa79" + }, + "constraint:reporting_production_status_heads.reporting_production_status_heads_pkey": { + "enabled": true, + "fingerprint": "4d853add149814edf9eadd3f752bf43f1f7164cba2bd079f0349d4d540a7f20c" + }, + "constraint:reporting_production_webhook_attempt_heads.reporting_production_webhook_attempt_heads_last_attempt_check": { + "enabled": true, + "fingerprint": "5bf9c354d680faec2bf3155f8c9b27323d7893564b30688a039ba19fc8b8d5d3" + }, + "constraint:reporting_production_webhook_attempt_heads.reporting_production_webhook_attempt_heads_pkey": { + "enabled": true, + "fingerprint": "490860c80b1129b15ab44f458a1a967382fc8b073f694ac959bf938282a74af5" + }, + "constraint:reporting_production_webhook_attempt_heads.reporting_production_webhook_attempt_heads_principal_id_check": { + "enabled": true, + "fingerprint": "bfc70b4b01cc74b9c93c630928d7cde9b0134c3d6614e2cbb326604c865c2cac" + }, + "constraint:reporting_production_webhook_attempts.reporting_production_webhook__account_id_consumer_namespac_fkey": { + "enabled": true, + "fingerprint": "37d7e7994745bab9c9bf7313f10cd46b9c94d08e615964b518e0aa3431f6d4f1" + }, + "constraint:reporting_production_webhook_attempts.reporting_production_webhook__account_id_consumer_namespace_key": { + "enabled": true, + "fingerprint": "b8546a68def520db0e6139c898e3e2a76318e05548a3de1b6e568fbf3a75ee63" + }, + "constraint:reporting_production_webhook_attempts.reporting_production_webhook_account_id_consumer_namespac_fkey1": { + "enabled": true, + "fingerprint": "b8ce3a7a8a92fef844563b5e9850186bbf6b150861584017a7c764241b062db1" + }, + "constraint:reporting_production_webhook_attempts.reporting_production_webhook_attempts_attempt_check": { + "enabled": true, + "fingerprint": "e05925f8a9fe41263902b6b2a72bca02958627e079787e99987820a52cb47d71" + }, + "constraint:reporting_production_webhook_attempts.reporting_production_webhook_attempts_payload_size_bytes_check": { + "enabled": true, + "fingerprint": "95548ff5519cec8b0ff110ad66f7b77d77e6015ef8c54a52e09cd4d7d80a7118" + }, + "constraint:reporting_production_webhook_attempts.reporting_production_webhook_attempts_pkey": { + "enabled": true, + "fingerprint": "5580d4f19f5ba257093ef71127d72dbba9669ac2db9b2f116f246d7f760522b4" + }, + "constraint:reporting_production_webhook_attempts.reporting_production_webhook_attempts_response_time_ms_check": { + "enabled": true, + "fingerprint": "6698890e9bba25670f73e3a5a7a7a7f9b664ab2571637eba582f22863c92db4d" + }, + "constraint:reporting_production_webhook_attempts.reporting_production_webhook_attempts_status_check": { + "enabled": true, + "fingerprint": "1c821d6bf41e5137a6263eae308d0a8648b7f0c95154724e5430e8c630aec30e" + }, + "constraint:reporting_production_webhook_attempts.reporting_production_webhook_completion": { + "enabled": true, + "fingerprint": "adc02762a63dcbf72330ca318884e6236fb2418cb7e1111f861ee74f98b53316" + }, + "constraint:reporting_production_webhook_attempts.reporting_production_webhook_identity": { + "enabled": true, + "fingerprint": "1e3f9725baa47d118c5b2efeeafbd5a2429400c1f667188fd14c4f2795114bf8" + }, + "constraint:reporting_production_webhook_attempts.reporting_production_webhook_outcome": { + "enabled": true, + "fingerprint": "cc0cc61f029bb3b28629e12a42c6f6ddbfd943156e1b90e5eeb2346aed678e00" + }, + "constraint:reporting_production_webhook_attempts.reporting_production_webhook_timestamps": { + "enabled": true, + "fingerprint": "0175d921479ed64020d88874f3b6ac822a9979f3ec8c3780d89bb9e0e3e3556c" + }, + "constraint:reporting_production_webhook_attempts.reporting_production_webhook_url_safe": { + "enabled": true, + "fingerprint": "3dd08bbc446317552434fb91bebd96e2a534395454b436910450f909ce2ab88e" + }, + "constraint:reporting_production_work.reporting_production_work_account_id_consumer_id_attempt_n_fkey": { + "enabled": true, + "fingerprint": "e3a1d24fef8da0fbe5587a7f7887a4cb9adf0133071070a09c950db1474304c8" + }, + "constraint:reporting_production_work.reporting_production_work_account_id_consumer_id_delivery__fkey": { + "enabled": true, + "fingerprint": "9390df97c6b965e9f16f700a4bd4e90392e60f89a15b6a9840d05682cdcc532f" + }, + "constraint:reporting_production_work.reporting_production_work_account_id_consumer_id_external_i_key": { + "enabled": true, + "fingerprint": "78641c1ca086a76ec1a81540664d98811ffb2adb51d5dc27db4d307f428bd8cd" + }, + "constraint:reporting_production_work.reporting_production_work_account_id_fkey": { + "enabled": true, + "fingerprint": "e6d7ea3bd4ffb8c9ad4e4c2307804a624d79764a63069a9a9762f49a2a092bee" + }, + "constraint:reporting_production_work.reporting_production_work_account_id_reporting_obligation__fkey": { + "enabled": true, + "fingerprint": "a0dca2cc4a27c9d4380eadb1513d30fe6383849657fc5e0d0f93c4b4c4ed4c08" + }, + "constraint:reporting_production_work.reporting_production_work_admission_epoch_check": { + "enabled": true, + "fingerprint": "6cfd9db9dd38e8c29346710c69de6ff07379d1bef2e9e5b1aba297a3ffa833f2" + }, + "constraint:reporting_production_work.reporting_production_work_attempt_namespace_check": { + "enabled": true, + "fingerprint": "639b8bfe70df1945046d65f693e67c45a4015e0e318356d77a97f5e4b3a7b629" + }, + "constraint:reporting_production_work.reporting_production_work_binding_sha256_check": { + "enabled": true, + "fingerprint": "c9db9bf2eca41364cec2fc19983d43e23fedf6711e5c4deb5f1d80aed317fd2b" + }, + "constraint:reporting_production_work.reporting_production_work_check": { + "enabled": true, + "fingerprint": "a6826ba06594c4200bda46be2704e03cb9d234d36538780211c04c2b2392e61b" + }, + "constraint:reporting_production_work.reporting_production_work_check1": { + "enabled": true, + "fingerprint": "311f8f3d7b44fd93e0b4ed5945b86997d4c6fd3beea28eca693b1e506466676d" + }, + "constraint:reporting_production_work.reporting_production_work_check2": { + "enabled": true, + "fingerprint": "3b94ebc10b0f1c882069a518c6496ff87db17e6b3405aecdb2f9aca7b158c644" + }, + "constraint:reporting_production_work.reporting_production_work_check3": { + "enabled": true, + "fingerprint": "fc30826fbb5c547751d5b430e90d54631bc537ca3af856785d68bb4a350cf9cb" + }, + "constraint:reporting_production_work.reporting_production_work_check4": { + "enabled": true, + "fingerprint": "50525da46b6ed71b60656e35fbba42e4fcbb45719c4d2032c2eccf35b971fafc" + }, + "constraint:reporting_production_work.reporting_production_work_external_id_check": { + "enabled": true, + "fingerprint": "df053aff1b660dec5635dbcc1043c90a9b338c3d871b72d4ed26ec60d465d278" + }, + "constraint:reporting_production_work.reporting_production_work_generation_check": { + "enabled": true, + "fingerprint": "04ce269eb45cc51febf8cbf44b760185b1eb2fd4b1935d6b53ecae063496539e" + }, + "constraint:reporting_production_work.reporting_production_work_pkey": { + "enabled": true, + "fingerprint": "fb2ee6241427fec8a9ae91d2940abe9e20b25f47554bd2d0a7c4046eb7704c41" + }, + "constraint:reporting_production_work.reporting_production_work_reason_check": { + "enabled": true, + "fingerprint": "d2dc21e781a468c735dd8b305b5405e779c85c27657683c19570cb1ba4b94aa7" + }, + "constraint:reporting_production_work.reporting_production_work_state_check": { + "enabled": true, + "fingerprint": "2e00e4b40728462003ac3d7f891ca8354aaeee6e19d808f7809030ff4db7758a" + }, + "constraint:reporting_production_work.reporting_production_work_verification_key_sha256_check": { + "enabled": true, + "fingerprint": "5442191b821cf162c8dfe50da9181b02acbf254eeb477c27b98e69d1fd26134b" + }, + "function:reporting_production_old_reservation_guard()": { + "enabled": true, + "fingerprint": "a46a6daef7fc2a07be432cf5415abf3df043a0a3f6097b7644f1f3eb956a87b7" + }, + "function:reporting_production_retained_guard()": { + "enabled": true, + "fingerprint": "f3a494b6d28a4e5340a3f4707da937a765122016eb68d93572ca05c982bae49b" + }, + "function:reporting_production_source_dirty()": { + "enabled": true, + "fingerprint": "248a91ace6ab30967841ad2cfed33b7554e1b9524ef9ebbcc833d42eccaa956f" + }, + "function:reporting_production_source_progress_guard()": { + "enabled": true, + "fingerprint": "c1a4a8e578deb4bab1f4a277c23b7605e77dd6b327f88c2c3b038d54fe1ee315" + }, + "function:reporting_production_source_work_guard()": { + "enabled": true, + "fingerprint": "34fb03589c1ce3021d94f650b881df24b18789d78b187a9a2e4966b73f4f7bec" + }, + "function:reporting_production_webhook_attempt_guard()": { + "enabled": true, + "fingerprint": "6b996c7e32c7afe0804181f1966812f834aca9653235e7d3ec4ed18590447a63" + }, + "function:reporting_production_webhook_head_guard()": { + "enabled": true, + "fingerprint": "4e448e0df3d56a5b04bc01ca41a4c5afbb8e601a4699d2e6dd0e3922ed2f0bf8" + }, + "function:reporting_production_work_guard()": { + "enabled": true, + "fingerprint": "89e538b8d4aa47ed74d8d789d0d49a7f66b9360bac7455b21708a227eced1d33" + }, + "index:reporting_notification_deliveries.reporting_production_core_deliveries_due": { + "enabled": true, + "fingerprint": "faeefdc68a8158d06454340a165077ae8ae8c3c281afe6644ad60df092615bcc" + }, + "index:reporting_notification_expansions.reporting_production_core_expansions_due": { + "enabled": true, + "fingerprint": "2b408cc42bd605749529aba6df2ce2833ea6c973787b09e32feab68494cdb7e5" + }, + "index:reporting_production_accounts.reporting_production_accounts_pkey": { + "enabled": true, + "fingerprint": "151886856aad940481552b97892b6ec86cb2418479fbffa8fcde29bd3c26968f" + }, + "index:reporting_production_delivery_windows.reporting_production_delivery_windows_pkey": { + "enabled": true, + "fingerprint": "ba988fbea01cd07a302653122095f8ebfb30e0e5498d2c4877c08dafa9e407ca" + }, + "index:reporting_production_destination_bindings.reporting_production_destination_bindings_pkey": { + "enabled": true, + "fingerprint": "24b1631ba99eb166db21d1d6f4804cfcce250ba559d6934fb952bc8d7c512ae1" + }, + "index:reporting_production_generations.reporting_production_generations_pkey": { + "enabled": true, + "fingerprint": "52db7741f2374442bbbcc253d7376bf7af21b4f65e8e01dbd395486b024c1657" + }, + "index:reporting_production_generations.reporting_production_source_generations": { + "enabled": true, + "fingerprint": "df6bb3c5045c17bd05895151a79d292a4f57b06d82d6a9001b124b9490658370" + }, + "index:reporting_production_notification_deliveries.reporting_production_notifica_account_id_consumer_namespac_key2": { + "enabled": true, + "fingerprint": "4762eb7ec4d24b1b248ea600d2421d675827fd683569f48515e04794a939578f" + }, + "index:reporting_production_notification_deliveries.reporting_production_notifica_account_id_consumer_namespac_key3": { + "enabled": true, + "fingerprint": "86594fa7c0fb8e3d56b87ad0d41d555874649ef6e7bb0b9f299b70e9d9fe0f85" + }, + "index:reporting_production_notification_deliveries.reporting_production_notification_deliveries_due": { + "enabled": true, + "fingerprint": "b169ff747e8e54a0f0977b5bbef0f9f2b206edd2aeba44b47c91e963d52bc422" + }, + "index:reporting_production_notification_deliveries.reporting_production_notification_deliveries_pkey": { + "enabled": true, + "fingerprint": "61d1e3319019b1c0b5471a8c099e5bdfa0ff46df98e51a89ab4f2f4c2791c208" + }, + "index:reporting_production_notification_deliveries.reporting_production_ready_deliveries_due": { + "enabled": true, + "fingerprint": "0f8de7ab8666254f2c4b74246708c2dd59666ed9ec59fd2d6523116724988ed6" + }, + "index:reporting_production_notification_events.reporting_production_notifica_account_id_consumer_namespac_key1": { + "enabled": true, + "fingerprint": "627f488a483bbc20f6adcb9c6f8217efdf665915ff07e9f046cf8d95495230d7" + }, + "index:reporting_production_notification_events.reporting_production_notifica_account_id_consumer_namespace_key": { + "enabled": true, + "fingerprint": "ad2bfee21400b1b027e96a27d57a6b72261071e0772a548fd8aa356bf7cfcaca" + }, + "index:reporting_production_notification_events.reporting_production_notification_events_pkey": { + "enabled": true, + "fingerprint": "011f2487b90e534412c03aa5c5b506cb819f6f0268471ae76377cab257a9ec12" + }, + "index:reporting_production_notification_expansions.reporting_production_notification_due": { + "enabled": true, + "fingerprint": "bc56cbd2460e73c3f1d1f17157d1443b4640e1d4e836b453f632b6c54bb863ea" + }, + "index:reporting_production_notification_expansions.reporting_production_notification_expansions_pkey": { + "enabled": true, + "fingerprint": "3f7ade1e36b9b44bdc4c05dddbf220bf121af7317b28febcf1dc5bb5236fa98b" + }, + "index:reporting_production_notification_expansions.reporting_production_ready_expansions_due": { + "enabled": true, + "fingerprint": "85b0655e19a3764f6eb71d8d0963056c844e09f5daae7f64ed821501e1c712b4" + }, + "index:reporting_production_source_probe_turns.reporting_production_source_probe_turns_pkey": { + "enabled": true, + "fingerprint": "9ef04a971b13db799196deb2251b35586a66ce783cc8c0b28c66165544ea9dac" + }, + "index:reporting_production_source_progress.reporting_production_source_progress_pkey": { + "enabled": true, + "fingerprint": "af80661aff369fc8a0a6d226ce94059e7ae95d8823fa5e34d1f3c47e4c3cdfd8" + }, + "index:reporting_production_source_work.reporting_production_source_pending": { + "enabled": true, + "fingerprint": "7e903d0a69de84d0763092ef6974055b4a7ce74884169456f5db72108998c59c" + }, + "index:reporting_production_source_work.reporting_production_source_work_pkey": { + "enabled": true, + "fingerprint": "b2bbf3fefe4e804ed8ea85ad6a0d9dcb5cb9845794b7e9d120e41781f6d693cf" + }, + "index:reporting_production_status_boundaries.reporting_production_status_b_account_id_consumer_id_report_key": { + "enabled": true, + "fingerprint": "31d4a0205b67c703c2329137212a899eebe9701a81f9bc4e6996481a156497da" + }, + "index:reporting_production_status_boundaries.reporting_production_status_bou_account_id_account_sequence_key": { + "enabled": true, + "fingerprint": "d9e0b980715b67bdcafc91339ba799c18815c4e19835796d40343976a5d0e5c1" + }, + "index:reporting_production_status_boundaries.reporting_production_status_boundaries_pkey": { + "enabled": true, + "fingerprint": "f60fc5fec06ce7bced70c766321327a6cbf96449546b0af9f7187971b67f0e7c" + }, + "index:reporting_production_status_heads.reporting_production_status_heads_pkey": { + "enabled": true, + "fingerprint": "df3abee748f8edaa4ddac8806a21c37a3a7b623fcb489f0fb3f581344fee1830" + }, + "index:reporting_production_webhook_attempt_heads.reporting_production_webhook_attempt_heads_pkey": { + "enabled": true, + "fingerprint": "da0aa49ff5efb1becacef3f40bde178be69bff369146e22d36f0bdbd9de2853d" + }, + "index:reporting_production_webhook_attempts.reporting_production_webhook__account_id_consumer_namespace_key": { + "enabled": true, + "fingerprint": "27fde93ad0125cfe9be4389b0a0fa9c399d172ee5775ed8cdb0f84b0d88d9396" + }, + "index:reporting_production_webhook_attempts.reporting_production_webhook_activity_newest": { + "enabled": true, + "fingerprint": "240921000941d702fa20c6457c0381d975c2b748935951653fab1bfc74172ff4" + }, + "index:reporting_production_webhook_attempts.reporting_production_webhook_activity_retention": { + "enabled": true, + "fingerprint": "a0b89634c6fdf96ecba7d7f7c6afdf64f61faa96d1c57fc32c6dba89e6561792" + }, + "index:reporting_production_webhook_attempts.reporting_production_webhook_attempts_pkey": { + "enabled": true, + "fingerprint": "4579c6dc33d4f2b51a51c12dc2002ce98476d5ff4c075d481a98fd27da9bb998" + }, + "index:reporting_production_work.reporting_production_one_pending": { + "enabled": true, + "fingerprint": "2f959d945d4dfaf5ec73b3407f4d7694470582cb7991a3fa9aa3f4b6877a366f" + }, + "index:reporting_production_work.reporting_production_work_account_id_consumer_id_external_i_key": { + "enabled": true, + "fingerprint": "c5a14351456c141c94f42726b9375e570b74cca97cdc85f4d8426b1139b9fe22" + }, + "index:reporting_production_work.reporting_production_work_due": { + "enabled": true, + "fingerprint": "ef3ca445274e81f60764e4a7130dc16f01c4297ad37a2fce495e249096b6314c" + }, + "index:reporting_production_work.reporting_production_work_pkey": { + "enabled": true, + "fingerprint": "deec29bab9f4b09d6f060b62e705f64b0e9a48e654dffe2597fb3b9b1ba7037f" + }, + "index:reporting_projection_notification_deliveries.reporting_production_status_deliveries_due": { + "enabled": true, + "fingerprint": "b5f0b4ffc23790dbe584e52c95b9fdb353d8926738ba95cf3829e9d89fba4572" + }, + "index:reporting_projection_notification_expansions.reporting_production_status_expansions_due": { + "enabled": true, + "fingerprint": "6f6175f294b31bbf3c2425db8bdfefc68fba8f5aee7d254f5c3eed7cf5fa610f" + }, + "table:reporting_production_accounts": { + "enabled": true, + "fingerprint": "1f824779ff80f110344420b019786663d8c9beaad230da90e0439795e734ccda" + }, + "table:reporting_production_delivery_windows": { + "enabled": true, + "fingerprint": "1f824779ff80f110344420b019786663d8c9beaad230da90e0439795e734ccda" + }, + "table:reporting_production_destination_bindings": { + "enabled": true, + "fingerprint": "1f824779ff80f110344420b019786663d8c9beaad230da90e0439795e734ccda" + }, + "table:reporting_production_generations": { + "enabled": true, + "fingerprint": "1f824779ff80f110344420b019786663d8c9beaad230da90e0439795e734ccda" + }, + "table:reporting_production_notification_deliveries": { + "enabled": true, + "fingerprint": "1f824779ff80f110344420b019786663d8c9beaad230da90e0439795e734ccda" + }, + "table:reporting_production_notification_events": { + "enabled": true, + "fingerprint": "1f824779ff80f110344420b019786663d8c9beaad230da90e0439795e734ccda" + }, + "table:reporting_production_notification_expansions": { + "enabled": true, + "fingerprint": "1f824779ff80f110344420b019786663d8c9beaad230da90e0439795e734ccda" + }, + "table:reporting_production_source_probe_turns": { + "enabled": true, + "fingerprint": "1f824779ff80f110344420b019786663d8c9beaad230da90e0439795e734ccda" + }, + "table:reporting_production_source_progress": { + "enabled": true, + "fingerprint": "1f824779ff80f110344420b019786663d8c9beaad230da90e0439795e734ccda" + }, + "table:reporting_production_source_work": { + "enabled": true, + "fingerprint": "1f824779ff80f110344420b019786663d8c9beaad230da90e0439795e734ccda" + }, + "table:reporting_production_status_boundaries": { + "enabled": true, + "fingerprint": "1f824779ff80f110344420b019786663d8c9beaad230da90e0439795e734ccda" + }, + "table:reporting_production_status_heads": { + "enabled": true, + "fingerprint": "1f824779ff80f110344420b019786663d8c9beaad230da90e0439795e734ccda" + }, + "table:reporting_production_webhook_attempt_heads": { + "enabled": true, + "fingerprint": "1f824779ff80f110344420b019786663d8c9beaad230da90e0439795e734ccda" + }, + "table:reporting_production_webhook_attempts": { + "enabled": true, + "fingerprint": "1f824779ff80f110344420b019786663d8c9beaad230da90e0439795e734ccda" + }, + "table:reporting_production_work": { + "enabled": true, + "fingerprint": "1f824779ff80f110344420b019786663d8c9beaad230da90e0439795e734ccda" + }, + "trigger:reporting_materializer_work.reporting_production_old_reservation_guard": { + "enabled": true, + "fingerprint": "e5f11febe88890b585f56ea796477656255225842e3abdcc42f158e3a3a10d7c" + }, + "trigger:reporting_obligations.reporting_production_source_obligation": { + "enabled": true, + "fingerprint": "9c3f01405aa80debf4b965494b6f2dff91f74b82f7b2242cb9df1d676821a656" + }, + "trigger:reporting_production_accounts.reporting_production_activation_immutable": { + "enabled": true, + "fingerprint": "f9426c100b6504a95c9bc1920ef446ff125d44b29bcd0ccf624872babd526d64" + }, + "trigger:reporting_production_delivery_windows.reporting_production_delivery_window_immutable": { + "enabled": true, + "fingerprint": "00babf492b6604846ede741a81f3a9cc270c0287df22a487c2de983205aa30b7" + }, + "trigger:reporting_production_destination_bindings.reporting_production_destination_immutable": { + "enabled": true, + "fingerprint": "0bd06a51562e44e8344367738c6a37b45c3b4fd6b9afca8678cacb55a434ed99" + }, + "trigger:reporting_production_generations.reporting_production_generation_immutable": { + "enabled": true, + "fingerprint": "d25848af82ae00800061fc9aaa9f2d64c0088527d228e6be033fff363451840b" + }, + "trigger:reporting_production_notification_events.reporting_production_event_immutable": { + "enabled": true, + "fingerprint": "8c3f2542634b9faca85f1266fbc6310eeb7f561e13572d5857c44a36044d37e1" + }, + "trigger:reporting_production_source_progress.reporting_production_source_progress_guard": { + "enabled": true, + "fingerprint": "5ea39da121c62e94e8d7c9c60844018872b94feab2671d461943f063efcbd90c" + }, + "trigger:reporting_production_source_work.reporting_production_source_work_guard": { + "enabled": true, + "fingerprint": "e03a1bcb6f6226a9bcce2e966227a0f238c0a333dcc53b582c727b1f712aa761" + }, + "trigger:reporting_production_status_boundaries.reporting_production_boundary_immutable": { + "enabled": true, + "fingerprint": "b744d5024657cf26ee4dd93e0e7ed4d2b1180c70479ffeefbf2ac2b8f7dcf15e" + }, + "trigger:reporting_production_webhook_attempt_heads.reporting_production_webhook_head_guard": { + "enabled": true, + "fingerprint": "42a7f06e3a7162412afe91173067f5bf85f99d7dac0ccc4d0035ddec97355599" + }, + "trigger:reporting_production_webhook_attempts.reporting_production_webhook_attempt_guard": { + "enabled": true, + "fingerprint": "0ef9aa027bdcd90909ed611dba2a25b353cbadf1d7b880309b25abba09102f53" + }, + "trigger:reporting_production_work.reporting_production_guard": { + "enabled": true, + "fingerprint": "71ae8906dce02251a41e4924e9b2160772d9526bbde775f1fd881808a9422832" + }, + "trigger:reporting_revisions.reporting_production_source_revision": { + "enabled": true, + "fingerprint": "d0e846596c4a33cf4ba19de7688bdcf21a2de3771af81d8e0098cbe2468d3a2d" + } +} diff --git a/src/adcp/reporting/production/schema.py b/src/adcp/reporting/production/schema.py new file mode 100644 index 000000000..b3cf06712 --- /dev/null +++ b/src/adcp/reporting/production/schema.py @@ -0,0 +1,24 @@ +"""Separate production participants; no inherited mandatory manifest is widened.""" + +from __future__ import annotations + +import json +from importlib.resources import files +from typing import Any + +from adcp.reporting.ledger.notification_models import ReportingNotificationError +from adcp.reporting.outbox._schema import schema_objects +from adcp.reporting.projection.schema import validate_projection_schema + + +async def validate_production_schema(connection: Any, *, notifications: bool = False) -> None: + try: + required = json.loads( + files("adcp.reporting.production").joinpath("required_schema.json").read_text() + ) + actual = await schema_objects(connection) + if not required or any(actual.get(k) != v for k, v in required.items()): + raise ValueError + await validate_projection_schema(connection, notifications=notifications) + except Exception: + raise ReportingNotificationError("reporting_production_schema_unready") from None diff --git a/src/adcp/reporting/production/service.py b/src/adcp/reporting/production/service.py new file mode 100644 index 000000000..1f486f9cf --- /dev/null +++ b/src/adcp/reporting/production/service.py @@ -0,0 +1,711 @@ +"""One live SDK composition for admitted reporting work and truthful discovery.""" + +from __future__ import annotations + +import asyncio +import hashlib +import json +import weakref +from collections.abc import Mapping +from contextvars import ContextVar +from dataclasses import dataclass +from datetime import timedelta +from typing import TYPE_CHECKING, Any, Protocol, runtime_checkable + +from adcp.reporting.ledger.delivery_models import ReportingDestinationBinding +from adcp.reporting.ledger.models import ReportingConfiguration +from adcp.reporting.ledger.notification_models import ReportingNotificationError +from adcp.reporting.materializer.contracts import ( + ReportingDestinationResolver, + ReportingDestinationWriter, + ReportingVerificationKey, + failure, +) +from adcp.reporting.materializer.reference import ( + ReferenceReportingDestinationWriter, + ReferenceReportingResolver, +) +from adcp.reporting.materializer.service import ReportingMaterializerService +from adcp.reporting.materializer.verification import ReportingDestinationIO +from adcp.reporting.materializer.work import MaterializerContext, verification_key_id +from adcp.reporting.production._diagnostics import _worker_stopped, _WorkerBoundary +from adcp.reporting.production.configuration import ( + ReportingConfigurationAdmission, + ReportingProductionConfigurationTask, +) +from adcp.reporting.production.contracts import ( + ReportingProductionDestinationBinding, + ReportingProductionMethod, + ReportingProductionSourceBinding, +) +from adcp.reporting.production.memory import InMemoryReportingProductionStore +from adcp.reporting.production.offerings import ReportingProductionOffering +from adcp.reporting.projection.memory import InMemoryReportingStatusProjection +from adcp.reporting.receipts.handler import ReceiptAccountResolver + +if TYPE_CHECKING: + from adcp.decisioning.registry import BuyerAgentRegistry + from adcp.reporting.ledger.producer import ReportingProducer + from adcp.reporting.outbox.worker import ReportingNotificationWorker + from adcp.reporting.production.pg import PgReportingProductionStore + from adcp.reporting.projection.pg import PgReportingStatusProjection + from adcp.server.base import ADCPHandler + + +@runtime_checkable +class ReportingProductionDestination( + ReportingDestinationWriter, ReportingDestinationResolver, Protocol +): + """An actual provider's bounded retention and revocation commitments. + + These promises accompany the writer's exact capabilities. Every write and + independent readback still opens a separately authorized SDK session. A + service cannot make the development writer eligible by decorating it. + """ + + @property + def resource_retention_days(self) -> int: ... + + @property + def authorization_revocation_seconds(self) -> int: ... + + @property + def delivery_methods(self) -> tuple[ReportingProductionMethod, ...]: ... + + def configuration_binding( + self, binding: ReportingDestinationBinding + ) -> ReportingProductionDestinationBinding | None: ... + + +def _method_ready( + writer: ReportingDestinationWriter, offering: ReportingProductionOffering +) -> bool: + return ( + isinstance(writer, ReportingProductionDestination) + and offering.verification_key.capability in writer.capabilities + and any( + type(method) is ReportingProductionMethod + and method.capability == offering.verification_key.capability + and method.wire() == offering.wire()["method"] + for method in writer.delivery_methods + ) + ) + + +def production_owner( + store: InMemoryReportingProductionStore | PgReportingProductionStore, +) -> ReportingProductionSupport: + reference = store._production_support + owner = reference() if reference is not None else None + if owner is None or owner.store is not store: + raise ReportingNotificationError("reporting_production_component_unready") + owner._assert_components() + return owner + + +@dataclass(frozen=True) +class _MountedProduction: + mount: weakref.ReferenceType[Any] + dispatcher: weakref.ReferenceType[Any] + transport: str + entries: dict[str, tuple[int, int]] + + def current(self) -> dict[str, tuple[int, int]]: + dispatcher = self.dispatcher() + if self.mount() is None or dispatcher is None: + return {} + try: + if self.transport == "mcp": + return { + name: (id(tool), id(tool.fn)) + for name, tool in dispatcher._tool_manager._tools.items() + } + if self.transport == "a2a": + return {name: (id(fn), id(fn)) for name, fn in dispatcher._tool_callers.items()} + except (AttributeError, TypeError): + return {} + return {} + + +def register_production_mount( + handler: ADCPHandler[Any], mount: object, *, transport: str, dispatcher: object +) -> None: + """Called only after the SDK has installed the transport's real dispatch map.""" + from adcp.reporting.production.handler import ReportingProductionHandler + + if type(handler) is ReportingProductionHandler: + proof = _MountedProduction(weakref.ref(mount), weakref.ref(dispatcher), transport, {}) + proof.entries.update(proof.current()) + handler.production._mounts.append(proof) + + +class ReportingProductionSupport: + """Compose, mount, start, then activate accounts after draining old workers. + + The owned worker performs indexed producer/materializer/projector turns. + Stopping it withdraws production claims and prevents fresh admission. An + optional HTTP notification worker is independent of polling readiness; + enabled logical enqueue is always part of the materializer transaction. + + Migrate with ``store.create_schema()`` before construction/start. For DDL, + drain and close this support, migrate, and construct fresh support. Schema + proofs do not detect arbitrary serving-time DDL or search-path changes. + """ + + def __init__( + self, + materializer: ReportingMaterializerService, + projection: InMemoryReportingStatusProjection | PgReportingStatusProjection, + *, + offerings: tuple[ReportingProductionOffering, ...], + configuration_task: ReportingProductionConfigurationTask, + resolve_account: ReceiptAccountResolver, + buyer_agents: BuyerAgentRegistry | None = None, + automated_recovery_window: timedelta = timedelta(hours=6), + status_retention_days: int = 400, + notification_workers: tuple[ReportingNotificationWorker, ...] = (), + poll_seconds: float = 0.25, + adcp_version: str | None = None, + ) -> None: + from adcp.reporting.outbox.worker import ReportingNotificationWorker + from adcp.reporting.production.handler import ReportingProductionHandler + from adcp.reporting.production.pg import PgReportingProductionStore + from adcp.reporting.projection.pg import PgReportingStatusProjection + + if ( + type(materializer) is not ReportingMaterializerService + or type(materializer.store) + not in {InMemoryReportingProductionStore, PgReportingProductionStore} + or type(projection) + not in {InMemoryReportingStatusProjection, PgReportingStatusProjection} + or projection.ledger is not materializer.store + or type(configuration_task) is not ReportingProductionConfigurationTask + or type(offerings) is not tuple + or not offerings + or any(type(o) is not ReportingProductionOffering for o in offerings) + or len({o.offering_id for o in offerings}) != len(offerings) + or type(status_retention_days) is not int + or not 1 <= status_retention_days <= 36500 + or automated_recovery_window <= timedelta(0) + or not 0.01 <= poll_seconds <= 60 + or type(notification_workers) is not tuple + or any(type(w) is not ReportingNotificationWorker for w in notification_workers) + ): + raise ValueError( + "production support requires exact SDK components and bounded promises" + ) + assert isinstance( + materializer.store, (InMemoryReportingProductionStore, PgReportingProductionStore) + ) + self.store = materializer.store + previous = self.store._production_support + if previous is not None and previous() is not None: + raise ReportingNotificationError("reporting_production_owner_conflict") + self.materializer, self.projection = materializer, projection + self.offerings, self.configuration_task = offerings, configuration_task + self.automated_recovery_window, self.status_retention_days = ( + automated_recovery_window, + status_retention_days, + ) + self.notification_workers, self.poll_seconds = notification_workers, poll_seconds + from adcp.reporting.production.notifications import worker_identity + + self._notification_identity = tuple(worker_identity(w) for w in notification_workers) + self.handler = ReportingProductionHandler( + self, + resolve_account=resolve_account, + buyer_agents=buyer_agents, + adcp_version=adcp_version, + ) + self._mounts: list[_MountedProduction] = [] + self._task: asyncio.Task[None] | None = None + self._notification_task: asyncio.Task[None] | None = None + self._stop = asyncio.Event() + self._started = asyncio.Event() + self._notification_started = asyncio.Event() + self._closed = False + self._failed = False + self._producer_turn: ContextVar[ReportingProducer | None] = ContextVar( + "reporting_production_source_turn", default=None + ) + self._schema_task: asyncio.Task[bool] | None = None + self._schema_epoch = 0 + self._schema_positive: tuple[int, int] | None = None + self._protocol_version = self.handler.get_adcp_version() + self._components = self._component_ids() + self._methods = self._handler_methods() + self.store._production_support = weakref.ref(self) + self._assert_components(running=False, mounted=False) + + @property + def keys(self) -> tuple[ReportingVerificationKey, ...]: + return tuple(dict.fromkeys(o.verification_key for o in self.offerings)) + + @property + def notifications_enabled(self) -> bool: + return bool(self.projection.policy["notifications_enabled"]) + + def _component_ids(self) -> tuple[int, ...]: + return tuple( + id(v) + for v in ( + self.store, + getattr(self.store, "_pool", None), + self.materializer, + self.materializer.io, + self.materializer.io.registry, + self.materializer.io.resolver, + self.materializer.writer, + self.projection, + self.projection.outbox, + self.configuration_task, + *self.offerings, + *self.notification_workers, + ) + ) + + def _handler_methods(self) -> tuple[object, ...]: + return tuple( + getattr(getattr(self.handler, name), "__func__", None) + for name in ( + "get_reporting_status", + "get_media_buy_delivery", + "sync_reporting_receipts", + "sync_reporting_status", + "sync_accounts", + "get_adcp_capabilities", + "get_adcp_version", + ) + ) + + def _mounted(self, *, receipts: bool = False) -> bool: + required = { + "get_adcp_capabilities", + "get_reporting_status", + "get_media_buy_delivery", + "sync_accounts", + } + if receipts: + required.add("sync_reporting_receipts") + if self.projection.consumer_status_enabled: + required.add("sync_reporting_status") + live = [proof for proof in self._mounts if proof.mount() is not None] + return bool(live) and all( + required <= proof.entries.keys() + and all(proof.current().get(name) == proof.entries[name] for name in required) + for proof in live + ) + + def _assert_components(self, *, running: bool = True, mounted: bool = True) -> None: + from adcp.reporting.production.notifications import check_workers + + check_workers(self) + writer = self.materializer.writer + if ( + self._closed + or self._failed + or (running and (self._task is None or self._task.done())) + or ( + running + and self.notification_workers + and (self._notification_task is None or self._notification_task.done()) + ) + or self._components != self._component_ids() + or self._methods != self._handler_methods() + or type(self.handler._adcp_version) is not str + or self.handler._adcp_version != self._protocol_version + or ( + (running or mounted) + and not isinstance(self.store, InMemoryReportingProductionStore) + and self.store._pool.closed is not False + ) + or self.materializer.store is not self.store + or self.projection.ledger is not self.store + or not self._projection_outbox_linked() + or self.handler.receipt_store is not self.store + or self.handler.reporting_feed_store is not self.store + or self.handler._feed_consumer_status_enabled != self.projection.consumer_status_enabled + or self.store._projection_read_policy != self.projection.policy + or type(self.materializer.io) is not ReportingDestinationIO + or isinstance(writer, ReferenceReportingDestinationWriter) + or isinstance(self.materializer.io.resolver, ReferenceReportingResolver) + or not isinstance(writer, ReportingProductionDestination) + or writer is not self.materializer.io.resolver + or writer.production_eligible is not True + or type(writer.resource_retention_days) is not int + or not 1 <= writer.resource_retention_days <= 36500 + or type(writer.authorization_revocation_seconds) is not int + or not 0 <= writer.authorization_revocation_seconds <= 86400 + or (mounted and not self._mounted()) + ): + raise ReportingNotificationError("reporting_production_component_unready") + + def _projection_outbox_linked(self) -> bool: + # The projection queue participates even when optional HTTP delivery + # workers are absent. Its connection ownership cannot ride a cached + # catalog proof belonging to a different pool or in-memory ledger. + if isinstance(self.store, InMemoryReportingProductionStore): + from adcp.reporting.projection.memory import InMemoryReportingProjectionOutbox + + return ( + type(self.projection.outbox) is InMemoryReportingProjectionOutbox + and self.projection.outbox._store is self.store + ) + from adcp.reporting.projection.notifications import PgReportingProjectionOutbox + + return ( + type(self.projection.outbox) is PgReportingProjectionOutbox + and self.projection.outbox._pool is self.store._pool + ) + + def _offering_ready(self, offering: ReportingProductionOffering) -> bool: + try: + if ( + offering.producer.store is not self.store + or type(offering.producer._max_periods_per_turn) is not int + or not 1 <= offering.producer._max_periods_per_turn <= 64 + or offering.producer.escalation != self.projection.escalation + or not _method_ready(self.materializer.writer, offering) + or (offering.reconciled and not self._mounted(receipts=True)) + ): + return False + verifier = self.materializer.io.registry.require(offering.verification_key) + profile = offering.wire()["reporting_profile"] + contract = json.loads(verifier.canonicalization_bytes) + definition = json.loads(verifier.definition_bytes) + if ( + offering.producer._revision_verifier is not verifier + or profile["primary_keys"] != contract["primary_keys"] + or profile["grain"] != definition.get("grain") + ): + return False + offering.check_source() + return True + except Exception: + return False + + def _admission_policy(self) -> dict[str, Any]: + self._assert_components() + # This fixes the installed contract, not its current authorization. + # Each reservation separately proves its own offering; an unrelated + # provider outage must not rewrite this epoch or veto healthy offerings. + ready = self.offerings + return { + "version": 2, + "verification_keys": sorted({verification_key_id(o.verification_key) for o in ready}), + "offerings": sorted(hashlib.sha256(o._wire).hexdigest() for o in ready), + "producers": sorted(o._producer_key for o in ready), + "notifications_enabled": self.notifications_enabled, + "projection": self.projection.policy, + } + + def _configuration_offering( + self, + configuration: ReportingConfiguration, + binding: ReportingDestinationBinding, + *, + offering_id: str | None = None, + producer_key: str | None = None, + ) -> ReportingProductionOffering: + self._assert_components() + writer = self.materializer.writer + assert isinstance(writer, ReportingProductionDestination) + matches = [] + for offering in self.offerings: + if (offering_id is not None and offering.offering_id != offering_id) or ( + producer_key is not None and offering._producer_key != producer_key + ): + continue + if not self._offering_ready(offering): + continue + try: + offering.check_configuration( + configuration, binding, retention_days=writer.resource_retention_days + ) + self._destination_binding(binding, offering) + # A failed proof never qualifies this offering. + except Exception: # nosec B112 + continue + matches.append(offering) + if ( + len(matches) != 1 + or configuration.automated_recovery_window != self.automated_recovery_window + or configuration.status_retention_days < self.status_retention_days + ): + raise failure("BINDING_MISMATCH") + return matches[0] + + def _destination_binding( + self, binding: ReportingDestinationBinding, offering: ReportingProductionOffering + ) -> ReportingProductionDestinationBinding: + try: + writer = self.materializer.writer + assert isinstance(writer, ReportingProductionDestination) + resolved = writer.configuration_binding(binding) + if ( + type(resolved) is not ReportingProductionDestinationBinding + or resolved.binding != binding + or resolved.method.capability != offering.verification_key.capability + or resolved.method.wire() != offering.wire()["method"] + ): + raise failure("BINDING_MISMATCH") + return resolved + except Exception: + raise failure("BINDING_MISMATCH") from None + + def _producer_keys(self) -> tuple[str, ...]: + self._assert_components() + producer = self._producer_turn.get() + return tuple( + o._producer_key + for o in self.offerings + if o.producer is producer and self._offering_ready(o) + ) + + def validate_configuration(self, value: ReportingConfigurationAdmission) -> None: + value.check(self) + + def _source_binding( + self, configuration: ReportingConfiguration, producer_key: str + ) -> ReportingProductionSourceBinding: + self._assert_components() + matches = [ + offering + for offering in self.offerings + if offering._producer_key == producer_key and self._offering_ready(offering) + ] + if len(matches) != 1: + raise failure("BINDING_MISMATCH") + return matches[0].source_binding(configuration) + + def _check_source_binding( + self, + configuration: ReportingConfiguration, + producer_key: str, + document: dict[str, Any] | None, + ) -> ReportingProductionSourceBinding: + binding = self._source_binding(configuration, producer_key) + if binding.document() != document: + raise failure("BINDING_MISMATCH") + return binding + + def _check_context( + self, + context: MaterializerContext, + key: ReportingVerificationKey, + policy: dict[str, Any], + producer_key: str | None, + source_binding: dict[str, Any] | None, + destination_binding: dict[str, Any] | None, + ) -> None: + if producer_key is None: + raise failure("BINDING_MISMATCH") + offering = self._configuration_offering( + context.configuration, context.binding, producer_key=producer_key + ) + self._check_source_binding(context.configuration, producer_key, source_binding) + if self._destination_binding(context.binding, offering).wire() != destination_binding: + raise failure("BINDING_MISMATCH") + if ( + policy != self._admission_policy() + or offering.verification_key != key + or verification_key_id(key) not in policy["verification_keys"] + or context.obligation.generation_key != context.configuration.generation_key + ): + raise failure("BINDING_MISMATCH") + + def invalidate_schema_validation(self) -> None: + """Drain first; an older scan cannot publish a new epoch's proof.""" + self._schema_epoch += 1 + self._schema_positive = None + self._schema_task = None + + async def _schema_ready(self) -> bool: + from adcp.reporting.production.pg import PgReportingProductionStore + from adcp.reporting.production.schema import validate_production_schema + + if not isinstance(self.store, PgReportingProductionStore): + return False + store = self.store + pool, epoch = store._pool, self._schema_epoch + identity = (id(pool), epoch) + if self._schema_positive == identity: + return True + task = self._schema_task + if task is None: + + async def scan() -> bool: + try: + async with pool.connection() as connection: + await validate_production_schema( + connection, notifications=self.notifications_enabled + ) + if epoch == self._schema_epoch and pool is store._pool: + self._schema_positive = identity + return True + return False + finally: + if self._schema_task is asyncio.current_task(): + self._schema_task = None + + task = asyncio.create_task(scan()) + self._schema_task = task + task.add_done_callback(lambda done: None if done.cancelled() else done.exception()) + if task.get_loop() is not asyncio.get_running_loop(): + return False + return await asyncio.shield(task) + + async def start(self) -> None: + if self._task is not None: + self._assert_components() + return + self._assert_components(running=False) + if ( + not isinstance(self.store, InMemoryReportingProductionStore) + and not await self._schema_ready() + ): + raise ReportingNotificationError("reporting_production_schema_unready") + self._assert_components(running=False) + self._task = asyncio.create_task(self._run(), name="adcp-reporting-production") + if self.notification_workers: + self._notification_task = asyncio.create_task( + self._run_notifications(), name="adcp-reporting-production-notifications" + ) + await self._started.wait() + if self.notification_workers: + await self._notification_started.wait() + self._assert_components() + + async def _run(self) -> None: + boundary: _WorkerBoundary = "producer" + try: + while not self._stop.is_set(): + # Each producer leases a generation through the reviewed fair + # indexed acquisition path; no account enumeration is required. + for producer in dict.fromkeys(o.producer for o in self.offerings): + boundary = "producer" + token = self._producer_turn.set(producer) + try: + await producer.run_worker() + finally: + self._producer_turn.reset(token) + boundary = "materializer" + await self.materializer.run_once() + boundary = "projection" + await self.projection.rebuild_one() + boundary = "sweeper" + await self.projection.sweep_one() + self._started.set() + try: + await asyncio.wait_for(self._stop.wait(), self.poll_seconds) + except asyncio.TimeoutError: + pass + except asyncio.CancelledError: + self._stop.set() + raise + except Exception as error: + already_stopping = self._stop.is_set() + self._failed = True + self._stop.set() + if not already_stopping and not isinstance(error, ReportingNotificationError): + _worker_stopped(boundary=boundary) + finally: + self._started.set() + + async def _run_notifications(self) -> None: + from adcp.reporting.production.notifications import notification_turn + + try: + while not self._stop.is_set(): + await notification_turn(self) + self._notification_started.set() + try: + await asyncio.wait_for(self._stop.wait(), self.poll_seconds) + except asyncio.TimeoutError: + pass + except asyncio.CancelledError: + self._stop.set() + raise + except Exception as error: + already_stopping = self._stop.is_set() + self._failed = True + self._stop.set() + if not already_stopping and not isinstance(error, ReportingNotificationError): + _worker_stopped(boundary="notifications") + finally: + self._notification_started.set() + + async def _check_notifications(self, account_id: str) -> None: + from adcp.reporting.production.notifications import check_account_notifications + + self._assert_components() + try: + await check_account_notifications(self, account_id) + except Exception: + raise ReportingNotificationError("notification_chain_unready") from None + self._assert_components() + + async def activate(self, *, account_id: str) -> bool: + await self._check_notifications(account_id) + await self.projection.activate(account_id=account_id) + return await self.store._activate_production(account_id=account_id) + + async def aclose(self) -> None: + self._stop.set() + task = self._task + if task is not None: + # Drain short database turns instead of interrupting psycopg's + # transaction entry/exit. Long I/O already has SDK-owned deadlines. + await asyncio.gather(task, return_exceptions=True) + if self._notification_task is not None: + await asyncio.gather(self._notification_task, return_exceptions=True) + self._notification_task = None + self._task = None + self._closed = True + proof = self._schema_task + if proof is not None: + await asyncio.gather(proof, return_exceptions=True) + if self.store._production_support is not None and self.store._production_support() is self: + self.store._production_support = None + + async def reporting_delivery(self, *, extra: Mapping[str, Any] | None = None) -> dict[str, Any]: + # Validate protected extra even when the concrete surface is unready. + producer = self.offerings[0].producer + payload = producer.advertised_reporting_delivery( + consumer_status_task=self.projection.consumer_status_enabled, + offerings=[], + automated_recovery_window=self.automated_recovery_window, + status_retention_days=self.status_retention_days, + extra=extra, + ) + try: + if self._stop.is_set(): + return {} + self._assert_components() + if not await self._schema_ready(): + return {} + # Catalog proof is cached; live component/mount ownership is not. + # It must still hold after an awaited first scan or invalidation. + self._assert_components() + offerings = [o for o in self.offerings if self._offering_ready(o)] + if not offerings: + return {} + writer = self.materializer.writer + assert isinstance(writer, ReportingProductionDestination) + payload.update( + offerings=[o.wire() for o in offerings], + managed_delivery=True, + resource_retention_days=writer.resource_retention_days, + authorization_revocation_seconds=writer.authorization_revocation_seconds, + ) + if any(o.reconciled for o in offerings): + payload.update(reconciled_billing=True, receipt_task="sync_reporting_receipts") + if self.notification_workers: + payload.update( + ledger_notification="reporting.ledger_changed", + readiness_notification="reporting.delivery_ready", + status_notification="reporting.status_changed", + ) + return payload + except Exception: + return {} diff --git a/src/adcp/reporting/projection/__init__.py b/src/adcp/reporting/projection/__init__.py new file mode 100644 index 000000000..22823c9b0 --- /dev/null +++ b/src/adcp/reporting/projection/__init__.py @@ -0,0 +1,36 @@ +"""Versioned private reporting projection, independent of optional delivery workers.""" + +from typing import TYPE_CHECKING, Any + +from adcp.reporting.projection.capture import ReportingProjectionInput +from adcp.reporting.projection.memory import ( + InMemoryReportingProjectionOutbox, + InMemoryReportingProjectionStore, + InMemoryReportingStatusProjection, +) + +if TYPE_CHECKING: + from adcp.reporting.projection.notifications import PgReportingProjectionOutbox + from adcp.reporting.projection.pg import PgReportingProjectionStore, PgReportingStatusProjection + +__all__ = [ + "InMemoryReportingProjectionOutbox", + "InMemoryReportingProjectionStore", + "InMemoryReportingStatusProjection", + "PgReportingProjectionOutbox", + "PgReportingProjectionStore", + "PgReportingStatusProjection", + "ReportingProjectionInput", +] + + +def __getattr__(name: str) -> Any: + if name in {"PgReportingProjectionStore", "PgReportingStatusProjection"}: + from adcp.reporting.projection import pg + + return getattr(pg, name) + if name == "PgReportingProjectionOutbox": + from adcp.reporting.projection.notifications import PgReportingProjectionOutbox + + return PgReportingProjectionOutbox + raise AttributeError(name) diff --git a/src/adcp/reporting/projection/capture.py b/src/adcp/reporting/projection/capture.py new file mode 100644 index 000000000..3d6978317 --- /dev/null +++ b/src/adcp/reporting/projection/capture.py @@ -0,0 +1,191 @@ +"""Closed immutable v2 inputs preserve legacy C/B2.1/B2.2 document shapes.""" + +from __future__ import annotations + +import json +from dataclasses import dataclass, field, fields, is_dataclass, replace +from datetime import datetime, timedelta, timezone +from typing import Any +from zoneinfo import ZoneInfo + +from pydantic import TypeAdapter, ValidationError +from pydantic_core import TzInfo + +from adcp.reporting._timestamp import aware_timestamp +from adcp.reporting.canonical_json import canonical_json_utf8_v1 +from adcp.reporting.evidence import aware_utc +from adcp.reporting.ledger._delivery_state import decode_record, payload, principal, record_identity +from adcp.reporting.ledger.delivery_models import ReportingDeliveryRecord +from adcp.reporting.ledger.notification_models import ReportingNotificationError +from adcp.reporting.ledger.status_projection import ReportingStatusSnapshot +from adcp.reporting.ledger.status_snapshot import snapshot_from_storage + +_CORE = TypeAdapter(ReportingStatusSnapshot) +_COLLECTIONS = ( + "configurations", + "obligations", + "revisions", + "statuses", + "lifecycles", + "issue_scopes", + "adjustments", + "changes", + "reconciliation", +) + + +def _require_frozen(value: Any) -> None: + if value is None or type(value) in {str, bytes, int, bool, float, timedelta}: + return + if type(value) is datetime and type(value.tzinfo) in {timezone, ZoneInfo, TzInfo}: + # Datetime itself is immutable; a user-defined tzinfo need not be. + # These closed decoder timezone types cannot mutate shared history. + return + if type(value) is tuple: + for item in value: + _require_frozen(item) + return + if is_dataclass(value) and getattr(type(value), "__dataclass_params__").frozen: + for item in fields(value): + _require_frozen(getattr(value, item.name)) + return + raise ReportingNotificationError("status_projection_history_corrupt") + + +@dataclass(frozen=True) +class ReportingProjectionInput: + core: ReportingStatusSnapshot = field(repr=False) + reconciliation: tuple[ReportingDeliveryRecord, ...] = field(repr=False) + document: bytes = field(repr=False) + + def __post_init__(self) -> None: + # Structural validation also protects direct internal construction: + # frozen outer dataclasses alone do not make nested lists/dicts safe. + _require_frozen(self.core) + _require_frozen(self.reconciliation) + if type(self.document) is not bytes: + raise ReportingNotificationError("status_projection_history_corrupt") + + def __deepcopy__(self, memo: dict[int, Any]) -> ReportingProjectionInput: + # Decoding closes every collection into tuples of frozen records. A + # rollback copies the mutable account cursor, queues and input list; + # the immutable historical values can safely remain shared. Copying + # every earlier full-account snapshot on each new source mutation made + # bounded production catch-up grow cubically in retained history. + memo[id(self)] = self + return self + + def at(self, at: datetime) -> ReportingProjectionInput: + """Evaluate a deadline on the same captured history, never today's records.""" + return replace(self, core=replace(self.core, as_of=aware_utc(at))) + + +def capture_memory_input( + core: ReportingStatusSnapshot, + changes: tuple[tuple[int, Any, ReportingDeliveryRecord], ...], +) -> ReportingProjectionInput: + raw = _CORE.dump_python(core, mode="json") + entries = [ + {"consumer_id": who.consumer_id, "sequence": seq, "record": payload(record)} + for seq, who, record in changes + if who.account_id == core.account_id + ] + document = { + "version": 2, + "account_id": core.account_id, + "as_of": core.as_of.isoformat(), + "core_format": "typed-v1", + "core": raw, + "reconciliation": entries, + "counts": {k: len(entries) if k == "reconciliation" else len(raw[k]) for k in _COLLECTIONS}, + } + return decode_projection_input(document) + + +def decode_projection_input(value: Any) -> ReportingProjectionInput: + try: + if ( + type(value) is not dict + or set(value) + != {"version", "account_id", "as_of", "core_format", "core", "reconciliation", "counts"} + or type(value["version"]) is not int + or value["version"] != 2 + or value["core_format"] not in {"typed-v1", "sql-v1"} + or type(value["core"]) is not dict + or type(value["reconciliation"]) is not list + or type(value["counts"]) is not dict + or set(value["counts"]) != set(_COLLECTIONS) + ): + raise ValueError + # Closure counts are checked before interpreting any financial evidence. + for name in _COLLECTIONS: + rows = value["reconciliation"] if name == "reconciliation" else value["core"][name] + count = value["counts"][name] + if type(rows) is not list or type(count) is not int or count != len(rows): + raise ValueError + core = ( + _CORE.validate_python(value["core"]) + if value["core_format"] == "typed-v1" + else snapshot_from_storage(value["core"]) + ) + if core.account_id != value["account_id"] or core.as_of != aware_timestamp(value["as_of"]): + raise ValueError + positions: dict[str, int] = {} + identities: set[tuple[str, tuple[str, str]]] = set() + records = [] + for row in value["reconciliation"]: + if type(row) is not dict or set(row) != {"consumer_id", "sequence", "record"}: + raise ValueError + record = decode_record(row["record"]) + who = principal(record) + if who.account_id != core.account_id or who.consumer_id != row["consumer_id"]: + raise ValueError + sequence = row["sequence"] + if type(sequence) is not int or sequence != positions.get(who.consumer_id, 0) + 1: + raise ValueError + positions[who.consumer_id] = sequence + identity = who.consumer_id, record_identity(record) + if identity in identities: + raise ValueError + identities.add(identity) + records.append(record) + if not set(positions).issubset(core.consumer_ids): + raise ValueError + for name in ( + "configurations", + "obligations", + "revisions", + "statuses", + "lifecycles", + "adjustments", + ): + if any(row.account_id != core.account_id for row in getattr(core, name)): + raise ValueError + return ReportingProjectionInput(core, tuple(records), canonical_json_utf8_v1(value)) + except (ValueError, TypeError, KeyError, AttributeError, ValidationError): + raise ReportingNotificationError("status_projection_history_corrupt") from None + + +def source_identity(value: ReportingProjectionInput) -> bytes: + """Memory transaction change detection excludes only the observation clock.""" + raw = json.loads(value.document) + raw.pop("as_of") + raw["core"].pop("as_of") + return canonical_json_utf8_v1(raw) + + +def with_projection_core( + value: ReportingProjectionInput, core: ReportingStatusSnapshot +) -> ReportingProjectionInput: + """Runtime replay state is separate from each immutable source boundary.""" + raw = json.loads(value.document) + raw.update( + core_format="typed-v1", + core=_CORE.dump_python(core, mode="json"), + as_of=core.as_of.isoformat(), + ) + raw["counts"] = { + k: len(raw["reconciliation"]) if k == "reconciliation" else len(raw["core"][k]) + for k in _COLLECTIONS + } + return decode_projection_input(raw) diff --git a/src/adcp/reporting/projection/history.py b/src/adcp/reporting/projection/history.py new file mode 100644 index 000000000..095f19109 --- /dev/null +++ b/src/adcp/reporting/projection/history.py @@ -0,0 +1,119 @@ +"""Replay retained private boundaries without creating active notifications. + +The old materializer and receipt captures are complete for their own caller. +They are never joined to today's configuration, artifacts or other consumers. +Their derived checkpoints use the same pure projector and generation primitive +as live projection. The original readiness queues and external identities are +not read, copied or modified here. +""" + +from __future__ import annotations + +import hashlib +from collections.abc import Mapping +from dataclasses import dataclass +from typing import Any, Literal + +from pydantic import TypeAdapter + +from adcp.reporting.canonical_json import canonical_json_utf8_v1 +from adcp.reporting.ledger.models import ReportingDeliveryEscalation +from adcp.reporting.ledger.notification_models import ( + ReportingNotificationError, + event_storage, +) +from adcp.reporting.ledger.status_projection import ( + StatusProjectionInput, + project_status_scope, + projection_scopes, +) +from adcp.reporting.materializer.capture import ( + ReportingMaterializerBoundary, + decode_materializer_boundary, +) +from adcp.reporting.outbox.status import StatusCheckpoint, advance_checkpoint, settled_replay +from adcp.reporting.receipts.capture import ReportingReceiptBoundary, decode_receipt_boundary + +HistoricalBoundary = ReportingMaterializerBoundary | ReportingReceiptBoundary +HistoricalKind = Literal["materializer", "receipt"] +_CHECKPOINT = TypeAdapter(StatusCheckpoint) + + +def decode_boundary(kind: HistoricalKind, value: dict[str, Any]) -> HistoricalBoundary: + if kind == "materializer": + return decode_materializer_boundary(value) + if kind == "receipt": + return decode_receipt_boundary(value) + raise ReportingNotificationError("status_projection_history_corrupt") + + +def checkpoint_key(checkpoint: StatusCheckpoint) -> str: + return hashlib.sha256(canonical_json_utf8_v1(checkpoint.scope.checkpoint_key)).hexdigest() + + +def checkpoint_document(checkpoint: StatusCheckpoint) -> dict[str, Any]: + result: dict[str, Any] = _CHECKPOINT.dump_python(checkpoint, mode="json") + return result + + +def decode_checkpoint(document: dict[str, Any]) -> StatusCheckpoint: + value = _CHECKPOINT.validate_python(document) + if checkpoint_document(value) != document: + raise ReportingNotificationError("status_projection_history_corrupt") + return value + + +@dataclass(frozen=True) +class HistoricalStep: + checkpoint: StatusCheckpoint + event: dict[str, Any] | None + + def document(self) -> dict[str, Any]: + return { + "admission_epoch": 0, + "checkpoint": checkpoint_document(self.checkpoint), + "event": self.event, + } + + +def project_boundary( + boundary: HistoricalBoundary, + previous: Mapping[str, StatusCheckpoint], + *, + baselines: Mapping[str, StatusCheckpoint], + source_sequence: int, + escalation: ReportingDeliveryEscalation | None, + consumer_status_enabled: bool, +) -> tuple[HistoricalStep, ...]: + core = settled_replay(boundary.core) + scopes = { + s.checkpoint_key: s + for s in projection_scopes(core) + if s.consumer_id == boundary.caller.consumer_id + } + scopes.update( + (c.scope.checkpoint_key, c.scope) + for c in (*baselines.values(), *previous.values()) + if c.scope.account_id == boundary.caller.account_id + and c.scope.consumer_id == boundary.caller.consumer_id + ) + results = [] + for scope in sorted(scopes.values(), key=lambda s: s.checkpoint_key): + result = project_status_scope( + StatusProjectionInput( + core, + scope, + escalation, + reconciliation=boundary.reconciliation, + consumer_status_enabled=consumer_status_enabled, + ) + ) + key = hashlib.sha256(canonical_json_utf8_v1(scope.checkpoint_key)).hexdigest() + checkpoint, event = advance_checkpoint( + previous.get(key, baselines.get(key)), + result, + fired_at=boundary.as_of, + source_sequence=source_sequence, + ) + results.append(HistoricalStep(checkpoint, event_storage(event) if event else None)) + return tuple(results) diff --git a/src/adcp/reporting/projection/memory.py b/src/adcp/reporting/projection/memory.py new file mode 100644 index 000000000..258b04734 --- /dev/null +++ b/src/adcp/reporting/projection/memory.py @@ -0,0 +1,411 @@ +"""Memory conformance participant with the same unconditional rollback boundary.""" + +from __future__ import annotations + +import asyncio +from collections.abc import AsyncIterator +from contextlib import asynccontextmanager +from contextvars import ContextVar +from copy import deepcopy +from dataclasses import dataclass, field, replace +from typing import Any + +from adcp.reporting.feed.memory import InMemoryReportingFeedStore +from adcp.reporting.ledger.delivery_models import ReportingDeliveryPrincipal +from adcp.reporting.ledger.models import ReportingDeliveryEscalation +from adcp.reporting.ledger.notification_models import ( + ReportingDomainEvent, + ReportingNotificationError, +) +from adcp.reporting.ledger.status_projection import with_replay_lifecycles +from adcp.reporting.ledger.status_snapshot import memory_snapshot +from adcp.reporting.ledger.store import _MEMORY_TRANSACTION +from adcp.reporting.outbox.memory import NotificationState +from adcp.reporting.outbox.status import ( + StatusCheckpoint, + StatusDueLease, + StatusTurn, + escalation_identity, + settled_replay, +) +from adcp.reporting.outbox.status_memory import ( + InMemoryReportingStatusOutbox, + InMemoryStatusNotificationStore, + _StatusMemoryState, +) +from adcp.reporting.projection.capture import ( + ReportingProjectionInput, + capture_memory_input, + source_identity, + with_projection_core, +) +from adcp.reporting.projection.history import ( + HistoricalBoundary, + checkpoint_key, + project_boundary, +) + +_REPLAY: ContextVar[object | None] = ContextVar("reporting_projection_replay", default=None) + + +@dataclass +class _ProjectionAccount: + policy: dict[str, Any] + checkpoint_floor: int + current: ReportingProjectionInput + source: bytes + cursor: int = 0 + inputs: list[ReportingProjectionInput] = field(default_factory=list) + ready: bool = False + legacy: tuple[HistoricalBoundary, ...] = () + legacy_cursor: int = 0 + baselines: dict[str, StatusCheckpoint] = field(default_factory=dict) + historical_checkpoints: dict[str, StatusCheckpoint] = field(default_factory=dict) + historical_steps: list[tuple[int, dict[str, Any]]] = field(default_factory=list) + + +class InMemoryReportingProjectionOutbox(InMemoryReportingStatusOutbox): + _store: InMemoryReportingProjectionStore + + @property + def _state(self) -> NotificationState: + state = self._store._projection_outbox + if state is None: + raise ReportingNotificationError("notifications_disabled") + return state + + +class InMemoryReportingProjectionStore(InMemoryReportingFeedStore): + """Reference semantics, never a production durability claim.""" + + _projection_read_policy: dict[str, Any] + _projection_read_escalation: ReportingDeliveryEscalation | None + + def __init__(self, **kwargs: Any) -> None: + super().__init__(**kwargs) + self._projection_accounts: dict[str, _ProjectionAccount] = {} + self._projection_outbox = ( + NotificationState() if self._notification_state is not None else None + ) + + def _capture_projection(self, account_id: str) -> ReportingProjectionInput: + return capture_memory_input( + memory_snapshot(self, account_id), tuple(getattr(self, "_delivery_records", ())) + ) + + @asynccontextmanager + async def _mutation(self) -> AsyncIterator[None]: + nested = _MEMORY_TRANSACTION.get() == (id(self), asyncio.current_task()) + async with super()._mutation(): + yield + if not nested and _REPLAY.get() != id(self): + for account_id, state in self._projection_accounts.items(): + captured = self._capture_projection(account_id) + identity = source_identity(captured) + if identity != state.source: + state.inputs.append(captured) + state.source = identity + + def _feed_projection_options(self, caller: ReportingDeliveryPrincipal) -> dict[str, Any]: + state = self._projection_accounts.get(caller.account_id) + if state is None or not state.ready: + return {} + return { + "representation_version": 2, + "revision_ownership": state.policy["ownership_enabled"], + "activated_consumer_status_enabled": state.policy["consumer_status_enabled"], + } + + async def read_projection_input( + self, *, caller: ReportingDeliveryPrincipal + ) -> ReportingProjectionInput: + async with self._lock: + state = self._projection_accounts.get(caller.account_id) + if state is None or not state.ready: + raise ReportingNotificationError("status_projection_activation_required") + return state.inputs[-1].at(self._clock()) + + async def read_tier_status( + self, + request: dict[str, Any], + *, + caller: ReportingDeliveryPrincipal, + consumer_status_enabled: bool = False, + ) -> dict[str, Any]: + from adcp.reporting.ledger.status import ReportingStatusCaller, ReportingStatusHandler + from adcp.reporting.projection.wire import render_tier_status + + async with self._lock: + state = self._projection_accounts.get(caller.account_id) + value = state.inputs[-1].at(self._clock()) if state and state.ready else None + policy = state.policy if state and state.ready else None + if value is None or policy is None: + return await ReportingStatusHandler( + self, consumer_status_enabled=consumer_status_enabled + ).handle(request, caller=ReportingStatusCaller(caller.account_id, caller.consumer_id)) + return render_tier_status(self, request, value, caller, policy, consumer_status_enabled) + + +class InMemoryReportingStatusProjection(InMemoryStatusNotificationStore): + _projection_version = 2 + ledger: InMemoryReportingProjectionStore + + def __init__( + self, + ledger: InMemoryReportingProjectionStore, + *, + consumer_status_enabled: bool = False, + revision_ownership: bool = False, + escalation: ReportingDeliveryEscalation | None = None, + ) -> None: + if not isinstance(ledger, InMemoryReportingProjectionStore): + raise ReportingNotificationError("status_projection_component_unready") + if type(consumer_status_enabled) is not bool or type(revision_ownership) is not bool: + raise ValueError("projection feature selections must be booleans") + self.ledger, self.escalation = ledger, escalation + self.consumer_status_enabled, self.revision_ownership = ( + consumer_status_enabled, + revision_ownership, + ) + if ledger._status_notification_state is None: + ledger._status_notification_state = _StatusMemoryState() + self.outbox = InMemoryReportingProjectionOutbox(ledger) + ledger._projection_read_policy = self.policy + ledger._projection_read_escalation = escalation + + def _enqueue_status(self, event: ReportingDomainEvent) -> None: + self.outbox._state.enqueue(event) + + async def checkpoints(self, *, account_id: str) -> tuple[StatusCheckpoint, ...]: + # A checkpoint contains a mutable JSON projection. Never lend that + # dictionary to a reader or retain it in a caller's published result. + return deepcopy(await super().checkpoints(account_id=account_id)) + + @property + def policy(self) -> dict[str, Any]: + return { + "version": 2, + "escalation": escalation_identity(self.escalation), + "consumer_status_enabled": self.consumer_status_enabled, + "ownership_enabled": self.revision_ownership, + "notifications_enabled": self.ledger._notification_state is not None, + } + + @asynccontextmanager + async def _transaction(self) -> AsyncIterator[None]: + token = _REPLAY.set(id(self.ledger)) + try: + async with self.ledger._mutation(): + yield + finally: + _REPLAY.reset(token) + + def _account(self, account_id: str) -> _ProjectionAccount: + state = self.ledger._projection_accounts.get(account_id) + if state is None: + raise ReportingNotificationError("status_projection_activation_required") + if state.policy != self.policy: + raise ReportingNotificationError("status_projection_policy_conflict") + return state + + def _cursor(self, account_id: str) -> int: + self._account(account_id) + return super()._cursor(account_id) + + async def baseline_ready(self, *, account_id: str) -> bool: + async with self.ledger._lock: + if account_id not in self.ledger._projection_accounts: + return False + return self._account(account_id).ready + + def _apply_value( + self, value: ReportingProjectionInput, *, through: int, silent: bool = False + ) -> int: + return self._apply( + settled_replay(value.core), + through=through, + reconciliation=value.reconciliation, + consumer_status_enabled=self.consumer_status_enabled, + enqueue=self.ledger._notification_state is not None and not silent, + ) + + async def activate(self, *, account_id: str) -> bool: + started = await self._begin_activation(account_id=account_id) + while True: + async with self._transaction(): + state = self._account(account_id) + if state.ready: + return started + self._activation_step(account_id) + + async def _begin_activation(self, *, account_id: str) -> bool: + async with self._transaction(): + if account_id in self.ledger._projection_accounts: + self._account(account_id) + return False + old = self._state.accounts.get(account_id) + notifications = self.ledger._notification_state + through = ( + max( + (d.sequence for d in notifications.dirty if d.scope.account_id == account_id), + default=0, + ) + if notifications + else 0 + ) + if old is not None: + if old[1] != escalation_identity(self.escalation): + raise ReportingNotificationError("status_policy_conflict") + if old[0] != through or self._needs_rebuild(account_id): + raise ReportingNotificationError("status_projection_legacy_drain_required") + checkpoints = [ + c for c in self._state.checkpoints.values() if c.scope.account_id == account_id + ] + now = self.ledger._clock() + if any( + (c.lease_expires_at and c.lease_expires_at > now) + or (c.next_due_at and c.next_due_at <= now) + for c in checkpoints + ): + raise ReportingNotificationError("status_projection_legacy_drain_required") + value = self.ledger._capture_projection(account_id) + floor = max((c.source_sequence for c in checkpoints), default=0) + legacy = tuple( + sorted( + ( + b + for b in ( + *self.ledger._materializer_boundaries, + *getattr(self.ledger, "_receipt_boundaries", ()), + ) + if b.caller.account_id == account_id + ), + key=lambda b: b.account_sequence, + ) + ) + expected = self.ledger._materializer_account_heads.get(account_id, 0) + if len(legacy) != expected or any( + b.account_sequence != n for n, b in enumerate(legacy, 1) + ): + raise ReportingNotificationError("status_projection_history_corrupt") + self.ledger._projection_accounts[account_id] = _ProjectionAccount( + self.policy, + floor + expected, + value, + source_identity(value), + inputs=[value], + legacy=legacy, + baselines={checkpoint_key(c): c for c in checkpoints}, + ) + self._state.accounts[account_id] = (through, escalation_identity(self.escalation)) + self._state.selector_accounts[account_id] = "complete" + return True + + def _activation_step(self, account_id: str) -> StatusTurn: + state = self._account(account_id) + if state.legacy_cursor < len(state.legacy): + boundary = state.legacy[state.legacy_cursor] + for step in project_boundary( + boundary, + state.historical_checkpoints, + baselines=state.baselines, + source_sequence=state.checkpoint_floor + - len(state.legacy) + + boundary.account_sequence, + escalation=self.escalation, + consumer_status_enabled=self.consumer_status_enabled, + ): + state.historical_checkpoints[checkpoint_key(step.checkpoint)] = step.checkpoint + state.historical_steps.append((boundary.account_sequence, step.document())) + self._state.checkpoints[step.checkpoint.scope.checkpoint_key] = step.checkpoint + state.legacy_cursor += 1 + return StatusTurn(True) + self._apply_value(state.inputs[0], through=state.checkpoint_floor + 1, silent=True) + state.cursor, state.ready = 1, True + return StatusTurn(True) + + async def baseline(self, *, account_id: str) -> bool: + return await self.activate(account_id=account_id) + + def _project_version(self, account_id: str) -> StatusTurn: + state = self._account(account_id) + if not state.ready: + return self._activation_step(account_id) + following = state.inputs[state.cursor] if state.cursor < len(state.inputs) else None + due = min( + ( + c.next_due_at + for c in self._state.checkpoints.values() + if c.scope.account_id == account_id and c.next_due_at is not None + ), + default=None, + ) + cursor = state.cursor + if ( + due is not None + and due <= self.ledger._clock() + and (following is None or due < following.core.as_of) + ): + value = state.current.at(due) + elif following is not None: + value, cursor = following, cursor + 1 + else: + return StatusTurn(False) + if value.core.as_of < state.current.core.as_of: + raise ReportingNotificationError("status_projection_clock_regressed") + value = with_projection_core( + value, settled_replay(with_replay_lifecycles(value.core, state.current.core)) + ) + count = self._apply_value(value, through=state.checkpoint_floor + cursor) + state.current, state.cursor = value, cursor + return StatusTurn(True, count) + + async def project_one(self, *, account_id: str) -> StatusTurn: + async with self._transaction(): + return self._project_version(account_id) + + async def rebuild_one(self) -> StatusTurn: + async with self._transaction(): + for account_id, state in sorted(self.ledger._projection_accounts.items()): + if state.cursor < len(state.inputs) and state.policy == self.policy: + return self._project_version(account_id) + return StatusTurn(False) + + async def complete_due(self, lease: StatusDueLease) -> StatusTurn: + async with self._transaction(): + self._account(lease.scope.account_id) + checkpoint = self._state.checkpoints.get(lease.scope.checkpoint_key) + if ( + checkpoint is None + or checkpoint.lease_token != lease.token + or checkpoint.lease_expires_at != lease.expires_at + or lease.expires_at <= self.ledger._clock() + ): + return StatusTurn(False) + result = self._project_version(lease.scope.account_id) + current = self._state.checkpoints[lease.scope.checkpoint_key] + if current.lease_token != lease.token or lease.expires_at <= self.ledger._clock(): + raise ReportingNotificationError("status_lease_lost") + self._state.checkpoints[lease.scope.checkpoint_key] = replace( + current, + lease_token=None, + lease_expires_at=None, + ) + return result + + async def sweep_one(self) -> StatusTurn: + async with self.ledger._lock: + at = self.ledger._clock() + due = sorted( + (c.next_due_at, c.scope.account_id) + for c in self._state.checkpoints.values() + if c.next_due_at is not None + and c.next_due_at <= at + and c.scope.account_id in self.ledger._projection_accounts + and self.ledger._projection_accounts[c.scope.account_id].policy == self.policy + and (c.lease_expires_at is None or c.lease_expires_at <= at) + ) + if not due: + return StatusTurn(False) + lease = await self.claim_due(account_id=due[0][1]) + return await self.complete_due(lease) if lease is not None else StatusTurn(False) diff --git a/src/adcp/reporting/projection/notifications.py b/src/adcp/reporting/projection/notifications.py new file mode 100644 index 000000000..8349cd2a8 --- /dev/null +++ b/src/adcp/reporting/projection/notifications.py @@ -0,0 +1,41 @@ +"""Isolated v2 status queues using the original SDK fanout/delivery transactions.""" + +from __future__ import annotations + +from collections.abc import AsyncIterator +from contextlib import asynccontextmanager +from typing import Any + +from adcp.reporting.outbox.status_pg import PgReportingStatusOutbox + + +class _ProjectionQueueConnection: + """Closed identifiers only; shares the caller's actual connection/transaction.""" + + def __init__(self, connection: Any) -> None: + self.connection = connection + + def transaction(self) -> Any: + return self.connection.transaction() + + async def execute(self, query: str, params: Any = None) -> Any: + for old, new in ( + ("reporting_status_notification_", "reporting_projection_notification_"), + ("reporting_status_webhook_", "reporting_projection_webhook_"), + ("reporting_notification_", "reporting_projection_notification_"), + ("reporting_webhook_", "reporting_projection_webhook_"), + ): + query = query.replace(old, new) + return await self.connection.execute(query, params) + + +class PgReportingProjectionOutbox(PgReportingStatusOutbox): + @asynccontextmanager + async def _connection(self) -> AsyncIterator[Any]: + async with self._pool.connection() as connection: + yield _ProjectionQueueConnection(connection) + + async def create_schema(self) -> None: + from adcp.reporting.projection.pg import PgReportingProjectionStore + + await PgReportingProjectionStore(pool=self._pool, notifications=True).create_schema() diff --git a/src/adcp/reporting/projection/pg.py b/src/adcp/reporting/projection/pg.py new file mode 100644 index 000000000..b24e723eb --- /dev/null +++ b/src/adcp/reporting/projection/pg.py @@ -0,0 +1,632 @@ +"""Connection-bound v2 capture over the reviewed feed and status participants.""" + +from __future__ import annotations + +import asyncio +import json +from collections.abc import AsyncIterator +from contextlib import asynccontextmanager +from dataclasses import replace +from importlib.resources import files +from typing import Any + +from adcp.reporting.feed.pg import PgReportingFeedStore, _CapturedFeed, _PreparedFeed +from adcp.reporting.feed.snapshot import StoredFeedSnapshot +from adcp.reporting.ledger.delivery_models import ReportingDeliveryPrincipal +from adcp.reporting.ledger.models import ReportingDeliveryEscalation +from adcp.reporting.ledger.notification_models import ( + ReportingDomainEvent, + ReportingNotificationError, +) +from adcp.reporting.ledger.status_projection import with_replay_lifecycles +from adcp.reporting.ledger.status_snapshot import persist_replay_lifecycles_on +from adcp.reporting.outbox.pg import database_now +from adcp.reporting.outbox.status import ( + StatusDueLease, + StatusTurn, + escalation_identity, + settled_replay, +) +from adcp.reporting.outbox.status_pg import _CHECKPOINT as _LEGACY_CHECKPOINT +from adcp.reporting.outbox.status_pg import PgStatusNotificationStore, _enqueue_on +from adcp.reporting.outbox.status_pg import _checkpoint as _legacy_checkpoint +from adcp.reporting.projection.capture import ( + ReportingProjectionInput, + decode_projection_input, + with_projection_core, +) +from adcp.reporting.projection.history import ( + checkpoint_document, + checkpoint_key, + decode_boundary, + decode_checkpoint, + project_boundary, +) +from adcp.reporting.projection.notifications import ( + PgReportingProjectionOutbox, + _ProjectionQueueConnection, +) +from adcp.reporting.projection.schema import validate_projection_schema + + +class _ProjectionFeedConnection: + """Closed SDK table selection; no adopter-supplied SQL identifiers.""" + + def __init__(self, connection: Any) -> None: + self.connection = connection + + async def execute(self, query: str, params: Any = None) -> Any: + return await self.connection.execute( + query.replace("reporting_feed_snapshots", "reporting_projection_feed_snapshots"), params + ) + + +class PgReportingProjectionStore(PgReportingFeedStore): + """Optional v2 store. Installation alone neither activates tiers nor sends readiness.""" + + _projection_read_policy: dict[str, Any] + _projection_read_escalation: ReportingDeliveryEscalation | None + + async def create_schema(self) -> None: + async with self._connection() as connection, connection.transaction(): + await self._create_schema_on(connection) + root = files("adcp.reporting.ledger") + for name in ( + "reporting_materializer.sql", + "reporting_receipt_ingestion.sql", + "reporting_feed.sql", + "reporting_status_notifications.sql", + "reporting_status_selector_version.sql", + "reporting_projection.sql", + "reporting_projection_notifications.sql", + "reporting_projection_feed.sql", + ): + await connection.execute(root.joinpath(name).read_text()) + + async def _feed_snapshot_on( + self, connection: Any, snapshot_id: str, caller: ReportingDeliveryPrincipal + ) -> StoredFeedSnapshot | None: + legacy = await super()._feed_snapshot_on(connection, snapshot_id, caller) + if legacy is not None: + return legacy + return await super()._feed_snapshot_on( + _ProjectionFeedConnection(connection), snapshot_id, caller + ) + + async def _save_feed_snapshot_on(self, connection: Any, stored: _PreparedFeed) -> None: + if stored.snapshot.representation_version == 2: + connection = _ProjectionFeedConnection(connection) + await super()._save_feed_snapshot_on(connection, stored) + + async def _capture_feed_on( + self, connection: Any, caller: ReportingDeliveryPrincipal + ) -> _CapturedFeed: + captured = await super()._capture_feed_on(connection, caller) + row = await ( + await connection.execute( + "SELECT ownership_enabled,consumer_status_enabled" + " FROM reporting_projection_accounts WHERE account_id=%s" + " AND current_input IS NOT NULL", + (caller.account_id,), + ) + ).fetchone() + if row is None: + return captured + await validate_projection_schema(connection, notifications=self._notifications_enabled) + return replace( + captured, + representation_version=2, + revision_ownership=row[0], + activated_consumer_status_enabled=row[1], + ) + + async def read_projection_input( + self, *, caller: ReportingDeliveryPrincipal + ) -> ReportingProjectionInput: + async with self._connection() as connection, connection.transaction(): + await self._lock_account(connection, caller.account_id) + await validate_projection_schema(connection, notifications=self._notifications_enabled) + row = await ( + await connection.execute( + "SELECT input,content_sha256=reporting_receipt_ingestion_sha256(input)" + " FROM reporting_projection_inputs WHERE account_id=%s" + " AND EXISTS (SELECT 1 FROM reporting_projection_accounts a" + " WHERE a.account_id=reporting_projection_inputs.account_id" + " AND a.current_input IS NOT NULL)" + " ORDER BY sequence DESC LIMIT 1", + (caller.account_id,), + ) + ).fetchone() + if row is None: + raise ReportingNotificationError("status_projection_activation_required") + if not row[1]: + raise ReportingNotificationError("status_projection_history_corrupt") + value = decode_projection_input(row[0]) + return value.at(await database_now(connection, self._clock)) + + async def read_tier_status( + self, + request: dict[str, Any], + *, + caller: ReportingDeliveryPrincipal, + consumer_status_enabled: bool = False, + ) -> dict[str, Any]: + from adcp.reporting.ledger.status import ReportingStatusCaller, ReportingStatusHandler + from adcp.reporting.projection.wire import render_tier_status + + async with self._connection() as connection: + row = await ( + await connection.execute( + "SELECT policy FROM reporting_projection_accounts WHERE account_id=%s" + " AND current_input IS NOT NULL", + (caller.account_id,), + ) + ).fetchone() + if row is None: + return await ReportingStatusHandler( + self, consumer_status_enabled=consumer_status_enabled + ).handle(request, caller=ReportingStatusCaller(caller.account_id, caller.consumer_id)) + value = await self.read_projection_input(caller=caller) + return await asyncio.to_thread( + render_tier_status, self, request, value, caller, row[0], consumer_status_enabled + ) + + +class PgReportingStatusProjection(PgStatusNotificationStore): + """v2 input/cursor lifecycle reusing C's pure projector, checkpoint and event transaction.""" + + _projection_version = 2 + ledger: PgReportingProjectionStore + + def __init__( + self, + ledger: PgReportingProjectionStore, + *, + consumer_status_enabled: bool = False, + revision_ownership: bool = False, + escalation: ReportingDeliveryEscalation | None = None, + ) -> None: + if not isinstance(ledger, PgReportingProjectionStore): + raise ReportingNotificationError("status_projection_component_unready") + if type(consumer_status_enabled) is not bool or type(revision_ownership) is not bool: + raise ValueError("projection feature selections must be booleans") + self.ledger, self.escalation = ledger, escalation + self.consumer_status_enabled, self.revision_ownership = ( + consumer_status_enabled, + revision_ownership, + ) + self.outbox = PgReportingProjectionOutbox(pool=ledger._pool, clock=ledger._clock) + ledger._projection_read_policy = self.policy + ledger._projection_read_escalation = escalation + + async def _enqueue_status_on(self, connection: Any, event: ReportingDomainEvent) -> None: + await _enqueue_on(_ProjectionQueueConnection(connection), event) + + @property + def policy(self) -> dict[str, Any]: + return { + "version": 2, + "escalation": escalation_identity(self.escalation), + "consumer_status_enabled": self.consumer_status_enabled, + "ownership_enabled": self.revision_ownership, + "notifications_enabled": self.ledger._notifications_enabled, + } + + async def create_schema(self) -> None: + await self.ledger.create_schema() + + @asynccontextmanager + async def _transaction(self, account_id: str) -> AsyncIterator[Any]: + async with self.ledger._connection() as connection, connection.transaction(): + await connection.execute( + "SELECT set_config('adcp.reporting.projection_version','2',true)" + ) + await connection.execute( + "SELECT set_config('adcp.reporting.selector_semantics_version','2',true)" + ) + await connection.execute( + "SELECT set_config('adcp.reporting.projection_internal','on',true)" + ) + await self.ledger._lock_account(connection, account_id) + if self.ledger._clock is not None: + await connection.execute( + "SELECT set_config('adcp.reporting.projection_clock',%s,true)", + (self.ledger._clock().isoformat(),), + ) + yield connection + + async def _state_on(self, connection: Any, account_id: str) -> tuple[Any, ...]: + row = await ( + await connection.execute( + "SELECT policy,cursor,max_sequence,checkpoint_floor,current_input,current_as_of" + " FROM reporting_projection_accounts WHERE account_id=%s FOR UPDATE", + (account_id,), + ) + ).fetchone() + if row is None: + raise ReportingNotificationError("status_projection_activation_required") + if row[0] != self.policy: + raise ReportingNotificationError("status_projection_policy_conflict") + return tuple(row) + + async def _account_on(self, connection: Any, account_id: str) -> int: + await self._state_on(connection, account_id) + return await super()._account_on(connection, account_id) + + async def baseline_ready(self, *, account_id: str) -> bool: + async with self._transaction(account_id) as connection: + await validate_projection_schema( + connection, notifications=self.ledger._notifications_enabled + ) + row = await ( + await connection.execute( + "SELECT policy,current_input IS NOT NULL FROM reporting_projection_accounts" + " WHERE account_id=%s", + (account_id,), + ) + ).fetchone() + if row is None: + return False + if row[0] != self.policy: + raise ReportingNotificationError("status_projection_policy_conflict") + return bool(row[1]) + + async def activate(self, *, account_id: str) -> bool: + """Fence, replay captured private history, then activate the frozen baseline. + + Each retained input advances in its own transaction. Cancellation or a + process crash resumes at that exact cursor. The derived historical + events stay in an immutable epoch-zero journal, never a delivery queue. + Old C projectors/sweepers must first be stopped and drained. + """ + started = await self._begin_activation(account_id=account_id) + while True: + async with self._transaction(account_id) as connection: + state = await self._state_on(connection, account_id) + if state[4] is not None: + return started + await self._activation_step_on(connection, account_id) + + async def _begin_activation(self, *, account_id: str) -> bool: + async with self._transaction(account_id) as connection: + await validate_projection_schema( + connection, notifications=self.ledger._notifications_enabled + ) + existing = await ( + await connection.execute( + "SELECT policy FROM reporting_projection_accounts WHERE account_id=%s", + (account_id,), + ) + ).fetchone() + if existing is not None: + await self._state_on(connection, account_id) + return False + now = await database_now(connection, self.ledger._clock) + old = await ( + await connection.execute( + "SELECT dirty_sequence,baseline_complete,policy FROM reporting_status_accounts" + " WHERE account_id=%s FOR UPDATE", + (account_id,), + ) + ).fetchone() + dirty = await ( + await connection.execute( + "SELECT coalesce(max_sequence,0) FROM reporting_status_dirty_heads" + " WHERE account_id=%s", + (account_id,), + ) + ).fetchone() + through = int(dirty[0]) if dirty else 0 + if old is not None and old[1]: + if old[2] != escalation_identity(self.escalation): + raise ReportingNotificationError("status_policy_conflict") + if int(old[0]) != through or await self._needs_rebuild_on(connection, account_id): + raise ReportingNotificationError("status_projection_legacy_drain_required") + rows = await ( + await connection.execute( + "SELECT source_sequence,lease_expires_at,next_due_at FROM" + " reporting_status_scope_checkpoints" + " WHERE account_id=%s ORDER BY" + " consumer_namespace,delivery_config_id,version,scope_kind," + " obligation_namespace FOR UPDATE", + (account_id,), + ) + ).fetchall() + if any( + (r[1] is not None and r[1] > now) or (r[2] is not None and r[2] <= now) + for r in rows + ): + raise ReportingNotificationError("status_projection_legacy_drain_required") + floor = max((int(r[0]) for r in rows), default=0) + # The inherited SDK column list is fixed; the account is bound. + old_checkpoints = [ + _legacy_checkpoint(r) + for r in await ( + await connection.execute( + f"SELECT {_LEGACY_CHECKPOINT} FROM reporting_status_scope_checkpoints" # nosec B608 + " WHERE account_id=%s", + (account_id,), + ) + ).fetchall() + ] + generation_floor = max((c.generation for c in old_checkpoints if c), default=0) + legacy_head = await ( + await connection.execute( + "SELECT captured_sequence FROM reporting_materializer_accounts" + " WHERE account_id=%s", + (account_id,), + ) + ).fetchone() + legacy_through = int(legacy_head[0]) if legacy_head else 0 + await connection.execute( + "INSERT INTO" + " reporting_status_accounts(account_id,policy,baseline_complete,dirty_sequence," + "baseline_highwater,baseline_at,selector_target_version,selector_transition)" + " VALUES(%s,%s::jsonb,TRUE,%s,%s,%s,2,'complete')" + " ON CONFLICT(account_id) DO NOTHING", + ( + account_id, + json.dumps(escalation_identity(self.escalation)), + through, + through, + now, + ), + ) + await connection.execute( + "INSERT INTO" + " reporting_projection_accounts(account_id,activated_at,notifications_enabled," + "consumer_status_enabled,ownership_enabled,policy,legacy_through,checkpoint_floor," + "legacy_capture_through,legacy_generation_floor)" + " VALUES(%s,%s,%s,%s,%s,%s::jsonb,%s,%s,%s,%s)", + ( + account_id, + now, + self.ledger._notifications_enabled, + self.consumer_status_enabled, + self.revision_ownership, + json.dumps(self.policy), + through, + floor + legacy_through, + legacy_through, + generation_floor, + ), + ) + for checkpoint in old_checkpoints: + if checkpoint is None: + continue + document = json.dumps(checkpoint_document(checkpoint)) + await connection.execute( + "INSERT INTO reporting_projection_legacy_baselines VALUES" + " (%s,%s,%s::jsonb,reporting_receipt_ingestion_sha256(%s::jsonb))", + (account_id, checkpoint_key(checkpoint), document, document), + ) + await connection.execute( + "UPDATE reporting_status_scope_checkpoints SET projection_writer_floor=2," + "lease_token=NULL,lease_expires_at=NULL WHERE account_id=%s", + (account_id,), + ) + await connection.execute("SELECT reporting_projection_capture(%s)", (account_id,)) + return True + + async def _activation_step_on(self, connection: Any, account_id: str) -> StatusTurn: + metadata = await ( + await connection.execute( + "SELECT legacy_capture_through,legacy_capture_cursor," + "checkpoint_floor FROM reporting_projection_accounts WHERE account_id=%s", + (account_id,), + ) + ).fetchone() + through, cursor, floor = map(int, metadata) + if cursor < through: + row = await ( + await connection.execute( + "SELECT kind,input,content_sha256=reporting_receipt_ingestion_sha256(input)," + " count(*) OVER (PARTITION BY account_sequence)" + " FROM (SELECT 'materializer' AS kind,account_sequence,input,content_sha256" + " FROM reporting_materializer_status_boundaries WHERE account_id=%s" + " AND account_sequence>%s AND account_sequence<=%s UNION ALL" + " SELECT 'receipt',account_sequence,input,content_sha256" + " FROM reporting_receipt_ingestion_boundaries WHERE account_id=%s" + " AND account_sequence>%s AND account_sequence<=%s) b" + " ORDER BY account_sequence LIMIT 1", + (account_id, cursor, through, account_id, cursor, through), + ) + ).fetchone() + if row is None or not row[2] or row[3] != 1: + raise ReportingNotificationError("status_projection_history_corrupt") + boundary = decode_boundary(row[0], row[1]) + if boundary.account_sequence != cursor + 1 or boundary.caller.account_id != account_id: + raise ReportingNotificationError("status_projection_history_corrupt") + previous_rows = await ( + await connection.execute( + "SELECT scope_key,checkpoint FROM reporting_projection_legacy_checkpoints" + " WHERE account_id=%s AND consumer_id=%s ORDER BY scope_key", + (account_id, boundary.caller.consumer_id), + ) + ).fetchall() + previous = {r[0]: decode_checkpoint(r[1]) for r in previous_rows} + baseline_rows = await ( + await connection.execute( + "SELECT scope_key,checkpoint FROM reporting_projection_legacy_baselines" + " WHERE account_id=%s ORDER BY scope_key", + (account_id,), + ) + ).fetchall() + baselines = {r[0]: decode_checkpoint(r[1]) for r in baseline_rows} + steps = project_boundary( + boundary, + previous, + baselines=baselines, + source_sequence=floor - through + boundary.account_sequence, + escalation=self.escalation, + consumer_status_enabled=self.consumer_status_enabled, + ) + raw = json.dumps(row[1]) + await connection.execute( + "INSERT INTO reporting_projection_legacy_inputs VALUES" + " (%s,%s,%s,%s,%s::jsonb,reporting_receipt_ingestion_sha256(%s::jsonb))", + ( + account_id, + boundary.account_sequence, + boundary.caller.consumer_id, + row[0], + raw, + raw, + ), + ) + await self._lock_scopes_on(connection, boundary.core) + for step in steps: + key = checkpoint_key(step.checkpoint) + document = json.dumps(step.document()) + await connection.execute( + "INSERT INTO reporting_projection_legacy_steps VALUES" + " (%s,%s,%s,%s::jsonb,reporting_receipt_ingestion_sha256(%s::jsonb))", + (account_id, boundary.account_sequence, key, document, document), + ) + checkpoint = json.dumps(checkpoint_document(step.checkpoint)) + await connection.execute( + "INSERT INTO reporting_projection_legacy_checkpoints VALUES" + " (%s,%s,%s,%s::jsonb,reporting_receipt_ingestion_sha256(%s::jsonb))" + " ON CONFLICT(account_id,scope_key) DO UPDATE SET" + " checkpoint=EXCLUDED.checkpoint,content_sha256=EXCLUDED.content_sha256", + (account_id, boundary.caller.consumer_id, key, checkpoint, checkpoint), + ) + # Advance the original guarded checkpoint once per retained + # boundary. Only the immutable epoch-zero journal receives the + # event; activation never releases it into an active queue. + await self._write_on(connection, step.checkpoint) + await connection.execute( + "UPDATE reporting_projection_accounts SET legacy_capture_cursor=%s" + " WHERE account_id=%s", + (boundary.account_sequence, account_id), + ) + return StatusTurn(True) + row = await ( + await connection.execute( + "SELECT input FROM reporting_projection_inputs WHERE account_id=%s AND sequence=1", + (account_id,), + ) + ).fetchone() + value = decode_projection_input(row[0]) + await self._lock_scopes_on(connection, value.core) + await self._apply_value_on(connection, value, through=floor + 1, silent=True) + await connection.execute( + "UPDATE reporting_projection_accounts SET" + " cursor=1,current_input=%s::jsonb,current_as_of=%s WHERE account_id=%s", + (value.document.decode(), value.core.as_of, account_id), + ) + return StatusTurn(True) + + async def baseline(self, *, account_id: str) -> bool: + return await self.activate(account_id=account_id) + + async def _apply_value_on( + self, + connection: Any, + value: ReportingProjectionInput, + *, + through: int, + silent: bool = False, + ) -> int: + return await self._apply_on( + connection, + settled_replay(value.core), + through=through, + reconciliation=value.reconciliation, + consumer_status_enabled=self.consumer_status_enabled, + enqueue=self.ledger._notifications_enabled and not silent, + ) + + async def _project_version_on(self, connection: Any, account_id: str) -> StatusTurn: + state = await self._state_on(connection, account_id) + if state[4] is None: + return await self._activation_step_on(connection, account_id) + cursor, floor = int(state[1]), int(state[3]) + row = await ( + await connection.execute( + "SELECT sequence,input,content_sha256=reporting_receipt_ingestion_sha256(input)" + " FROM reporting_projection_inputs WHERE account_id=%s AND sequence>%s" + " ORDER BY sequence LIMIT 1", + (account_id, cursor), + ) + ).fetchone() + next_value = None + if row is not None: + if row[0] != cursor + 1 or not row[2]: + raise ReportingNotificationError("status_projection_history_corrupt") + next_value = decode_projection_input(row[1]) + now = await database_now(connection, self.ledger._clock) + deadline = await ( + await connection.execute( + "SELECT min(next_due_at) FROM reporting_status_scope_checkpoints" + " WHERE account_id=%s", + (account_id,), + ) + ).fetchone() + due = deadline[0] if deadline else None + if due is not None and due <= now and (next_value is None or due < next_value.core.as_of): + value = decode_projection_input(state[4]).at(due) + if due < state[5]: + raise ReportingNotificationError("status_projection_clock_regressed") + elif next_value is not None: + value, cursor = next_value, int(row[0]) + if value.core.as_of < state[5]: + raise ReportingNotificationError("status_projection_clock_regressed") + else: + return StatusTurn(False) + prior = decode_projection_input(state[4]) + value = with_projection_core( + value, settled_replay(with_replay_lifecycles(value.core, prior.core)) + ) + await persist_replay_lifecycles_on(self.ledger, connection, value.core) + count = await self._apply_value_on(connection, value, through=floor + cursor) + await connection.execute( + "UPDATE reporting_projection_accounts SET" + " cursor=%s,current_input=%s::jsonb,current_as_of=%s" + " WHERE account_id=%s", + (cursor, value.document.decode(), value.core.as_of, account_id), + ) + return StatusTurn(True, count) + + async def project_one(self, *, account_id: str) -> StatusTurn: + async with self._transaction(account_id) as connection: + return await self._project_version_on(connection, account_id) + + async def rebuild_one(self) -> StatusTurn: + async with self.ledger._connection() as connection: + row = await ( + await connection.execute( + "SELECT account_id FROM reporting_projection_accounts WHERE cursor StatusTurn: + async with self._transaction(lease.scope.account_id) as connection: + await self._state_on(connection, lease.scope.account_id) + if not await self._held_on(connection, lease): + return StatusTurn(False) + result = await self._project_version_on(connection, lease.scope.account_id) + if not await self._ack_on(connection, lease): + raise ReportingNotificationError("status_lease_lost") + return result + + async def sweep_one(self) -> StatusTurn: + async with self.ledger._connection() as connection: + at = await database_now(connection, self.ledger._clock) + row = await ( + await connection.execute( + "SELECT c.account_id FROM reporting_status_scope_checkpoints c" + " JOIN reporting_projection_accounts a ON a.account_id=c.account_id" + " WHERE c.next_due_at<=%s AND a.policy=%s::jsonb" + " AND (c.lease_expires_at IS NULL OR c.lease_expires_at<=%s)" + " ORDER BY c.next_due_at,c.account_id LIMIT 1", + (at, json.dumps(self.policy), at), + ) + ).fetchone() + if row is None: + return StatusTurn(False) + lease = await self.claim_due(account_id=row[0]) + return await self.complete_due(lease) if lease is not None else StatusTurn(False) diff --git a/src/adcp/reporting/projection/required_schema.json b/src/adcp/reporting/projection/required_schema.json new file mode 100644 index 000000000..19d957b28 --- /dev/null +++ b/src/adcp/reporting/projection/required_schema.json @@ -0,0 +1,1270 @@ +{ + "column:reporting_projection_accounts.account_id": { + "enabled": true, + "fingerprint": "fdc4b549138bf90a5af5ed00b87a86ff9a0cbea553e7f017b4d95510708f6f88" + }, + "column:reporting_projection_accounts.activated_at": { + "enabled": true, + "fingerprint": "1cac4e73af11a8ecafd646ef6a0ff6ecb087d46f150408dcfebc630fe1bf5e1e" + }, + "column:reporting_projection_accounts.checkpoint_floor": { + "enabled": true, + "fingerprint": "42202005517b72e082eb22c9eceb2ac0252815e5df700c83eb50c54cfeb46297" + }, + "column:reporting_projection_accounts.consumer_status_enabled": { + "enabled": true, + "fingerprint": "1abe3a1c570fbe885784dab5d979307c373d50f567ff4481ce3996869bf58fed" + }, + "column:reporting_projection_accounts.current_as_of": { + "enabled": true, + "fingerprint": "6f1466ce5d0aaac8471e39834b9c4b1f85d6f7169d9238a245ac035ff518e0fc" + }, + "column:reporting_projection_accounts.current_input": { + "enabled": true, + "fingerprint": "1f539b84d6adb1a8577add320aabb81fdd5c614ad01b46c7b77eb55b995a6556" + }, + "column:reporting_projection_accounts.cursor": { + "enabled": true, + "fingerprint": "42202005517b72e082eb22c9eceb2ac0252815e5df700c83eb50c54cfeb46297" + }, + "column:reporting_projection_accounts.legacy_capture_cursor": { + "enabled": true, + "fingerprint": "42202005517b72e082eb22c9eceb2ac0252815e5df700c83eb50c54cfeb46297" + }, + "column:reporting_projection_accounts.legacy_capture_through": { + "enabled": true, + "fingerprint": "54a5f9962598caa8723435f8b55bd5e15823053af5dd63d531663997537dc6ba" + }, + "column:reporting_projection_accounts.legacy_generation_floor": { + "enabled": true, + "fingerprint": "54a5f9962598caa8723435f8b55bd5e15823053af5dd63d531663997537dc6ba" + }, + "column:reporting_projection_accounts.legacy_through": { + "enabled": true, + "fingerprint": "54a5f9962598caa8723435f8b55bd5e15823053af5dd63d531663997537dc6ba" + }, + "column:reporting_projection_accounts.max_sequence": { + "enabled": true, + "fingerprint": "42202005517b72e082eb22c9eceb2ac0252815e5df700c83eb50c54cfeb46297" + }, + "column:reporting_projection_accounts.notifications_enabled": { + "enabled": true, + "fingerprint": "1abe3a1c570fbe885784dab5d979307c373d50f567ff4481ce3996869bf58fed" + }, + "column:reporting_projection_accounts.ownership_enabled": { + "enabled": true, + "fingerprint": "1abe3a1c570fbe885784dab5d979307c373d50f567ff4481ce3996869bf58fed" + }, + "column:reporting_projection_accounts.policy": { + "enabled": true, + "fingerprint": "ac355fc16c02b70cb0a24afee8214cdce5f5cbfdc7fd1630786d5101932ecfa4" + }, + "column:reporting_projection_accounts.version": { + "enabled": true, + "fingerprint": "cec0cb8bc536b9a98fca05facc40ec8bb053622ba9030f5cb1888fe47da07e39" + }, + "column:reporting_projection_feed_snapshots.account_id": { + "enabled": true, + "fingerprint": "fdc4b549138bf90a5af5ed00b87a86ff9a0cbea553e7f017b4d95510708f6f88" + }, + "column:reporting_projection_feed_snapshots.as_of": { + "enabled": true, + "fingerprint": "1cac4e73af11a8ecafd646ef6a0ff6ecb087d46f150408dcfebc630fe1bf5e1e" + }, + "column:reporting_projection_feed_snapshots.consumer_id": { + "enabled": true, + "fingerprint": "fdc4b549138bf90a5af5ed00b87a86ff9a0cbea553e7f017b4d95510708f6f88" + }, + "column:reporting_projection_feed_snapshots.content_sha256": { + "enabled": true, + "fingerprint": "fdc4b549138bf90a5af5ed00b87a86ff9a0cbea553e7f017b4d95510708f6f88" + }, + "column:reporting_projection_feed_snapshots.document": { + "enabled": true, + "fingerprint": "10ccaa0dc3b93d48a1f32c7ef2352a11676632e1e871d77ef9ef4393eea15d27" + }, + "column:reporting_projection_feed_snapshots.ownership_mode": { + "enabled": true, + "fingerprint": "fdc4b549138bf90a5af5ed00b87a86ff9a0cbea553e7f017b4d95510708f6f88" + }, + "column:reporting_projection_feed_snapshots.representation_version": { + "enabled": true, + "fingerprint": "64be57437fdc0a07a97985c2aa058031f8082db7251bdb4d5afa1a9b088de97a" + }, + "column:reporting_projection_feed_snapshots.signing_key": { + "enabled": true, + "fingerprint": "554c34e416bd4546469b42d5773bc7c59b2104366ffa5259a2838c64cd826e57" + }, + "column:reporting_projection_feed_snapshots.snapshot_id": { + "enabled": true, + "fingerprint": "fdc4b549138bf90a5af5ed00b87a86ff9a0cbea553e7f017b4d95510708f6f88" + }, + "column:reporting_projection_inputs.account_id": { + "enabled": true, + "fingerprint": "fdc4b549138bf90a5af5ed00b87a86ff9a0cbea553e7f017b4d95510708f6f88" + }, + "column:reporting_projection_inputs.as_of": { + "enabled": true, + "fingerprint": "1cac4e73af11a8ecafd646ef6a0ff6ecb087d46f150408dcfebc630fe1bf5e1e" + }, + "column:reporting_projection_inputs.content_sha256": { + "enabled": true, + "fingerprint": "fdc4b549138bf90a5af5ed00b87a86ff9a0cbea553e7f017b4d95510708f6f88" + }, + "column:reporting_projection_inputs.input": { + "enabled": true, + "fingerprint": "ac355fc16c02b70cb0a24afee8214cdce5f5cbfdc7fd1630786d5101932ecfa4" + }, + "column:reporting_projection_inputs.sequence": { + "enabled": true, + "fingerprint": "54a5f9962598caa8723435f8b55bd5e15823053af5dd63d531663997537dc6ba" + }, + "column:reporting_projection_inputs.transaction_id": { + "enabled": true, + "fingerprint": "fdc4b549138bf90a5af5ed00b87a86ff9a0cbea553e7f017b4d95510708f6f88" + }, + "column:reporting_projection_legacy_baselines.account_id": { + "enabled": true, + "fingerprint": "fdc4b549138bf90a5af5ed00b87a86ff9a0cbea553e7f017b4d95510708f6f88" + }, + "column:reporting_projection_legacy_baselines.checkpoint": { + "enabled": true, + "fingerprint": "ac355fc16c02b70cb0a24afee8214cdce5f5cbfdc7fd1630786d5101932ecfa4" + }, + "column:reporting_projection_legacy_baselines.content_sha256": { + "enabled": true, + "fingerprint": "fdc4b549138bf90a5af5ed00b87a86ff9a0cbea553e7f017b4d95510708f6f88" + }, + "column:reporting_projection_legacy_baselines.scope_key": { + "enabled": true, + "fingerprint": "fdc4b549138bf90a5af5ed00b87a86ff9a0cbea553e7f017b4d95510708f6f88" + }, + "column:reporting_projection_legacy_checkpoints.account_id": { + "enabled": true, + "fingerprint": "fdc4b549138bf90a5af5ed00b87a86ff9a0cbea553e7f017b4d95510708f6f88" + }, + "column:reporting_projection_legacy_checkpoints.checkpoint": { + "enabled": true, + "fingerprint": "ac355fc16c02b70cb0a24afee8214cdce5f5cbfdc7fd1630786d5101932ecfa4" + }, + "column:reporting_projection_legacy_checkpoints.consumer_id": { + "enabled": true, + "fingerprint": "fdc4b549138bf90a5af5ed00b87a86ff9a0cbea553e7f017b4d95510708f6f88" + }, + "column:reporting_projection_legacy_checkpoints.content_sha256": { + "enabled": true, + "fingerprint": "fdc4b549138bf90a5af5ed00b87a86ff9a0cbea553e7f017b4d95510708f6f88" + }, + "column:reporting_projection_legacy_checkpoints.scope_key": { + "enabled": true, + "fingerprint": "fdc4b549138bf90a5af5ed00b87a86ff9a0cbea553e7f017b4d95510708f6f88" + }, + "column:reporting_projection_legacy_inputs.account_id": { + "enabled": true, + "fingerprint": "fdc4b549138bf90a5af5ed00b87a86ff9a0cbea553e7f017b4d95510708f6f88" + }, + "column:reporting_projection_legacy_inputs.account_sequence": { + "enabled": true, + "fingerprint": "54a5f9962598caa8723435f8b55bd5e15823053af5dd63d531663997537dc6ba" + }, + "column:reporting_projection_legacy_inputs.consumer_id": { + "enabled": true, + "fingerprint": "fdc4b549138bf90a5af5ed00b87a86ff9a0cbea553e7f017b4d95510708f6f88" + }, + "column:reporting_projection_legacy_inputs.content_sha256": { + "enabled": true, + "fingerprint": "fdc4b549138bf90a5af5ed00b87a86ff9a0cbea553e7f017b4d95510708f6f88" + }, + "column:reporting_projection_legacy_inputs.input": { + "enabled": true, + "fingerprint": "ac355fc16c02b70cb0a24afee8214cdce5f5cbfdc7fd1630786d5101932ecfa4" + }, + "column:reporting_projection_legacy_inputs.kind": { + "enabled": true, + "fingerprint": "fdc4b549138bf90a5af5ed00b87a86ff9a0cbea553e7f017b4d95510708f6f88" + }, + "column:reporting_projection_legacy_steps.account_id": { + "enabled": true, + "fingerprint": "fdc4b549138bf90a5af5ed00b87a86ff9a0cbea553e7f017b4d95510708f6f88" + }, + "column:reporting_projection_legacy_steps.account_sequence": { + "enabled": true, + "fingerprint": "54a5f9962598caa8723435f8b55bd5e15823053af5dd63d531663997537dc6ba" + }, + "column:reporting_projection_legacy_steps.content_sha256": { + "enabled": true, + "fingerprint": "fdc4b549138bf90a5af5ed00b87a86ff9a0cbea553e7f017b4d95510708f6f88" + }, + "column:reporting_projection_legacy_steps.input": { + "enabled": true, + "fingerprint": "ac355fc16c02b70cb0a24afee8214cdce5f5cbfdc7fd1630786d5101932ecfa4" + }, + "column:reporting_projection_legacy_steps.scope_key": { + "enabled": true, + "fingerprint": "fdc4b549138bf90a5af5ed00b87a86ff9a0cbea553e7f017b4d95510708f6f88" + }, + "column:reporting_projection_notification_deliveries.account_id": { + "enabled": true, + "fingerprint": "fdc4b549138bf90a5af5ed00b87a86ff9a0cbea553e7f017b4d95510708f6f88" + }, + "column:reporting_projection_notification_deliveries.auth_mode": { + "enabled": true, + "fingerprint": "10ccaa0dc3b93d48a1f32c7ef2352a11676632e1e871d77ef9ef4393eea15d27" + }, + "column:reporting_projection_notification_deliveries.body_sha256": { + "enabled": true, + "fingerprint": "10ccaa0dc3b93d48a1f32c7ef2352a11676632e1e871d77ef9ef4393eea15d27" + }, + "column:reporting_projection_notification_deliveries.cause_generation": { + "enabled": true, + "fingerprint": "54a5f9962598caa8723435f8b55bd5e15823053af5dd63d531663997537dc6ba" + }, + "column:reporting_projection_notification_deliveries.cause_id": { + "enabled": true, + "fingerprint": "fdc4b549138bf90a5af5ed00b87a86ff9a0cbea553e7f017b4d95510708f6f88" + }, + "column:reporting_projection_notification_deliveries.cause_kind": { + "enabled": true, + "fingerprint": "fdc4b549138bf90a5af5ed00b87a86ff9a0cbea553e7f017b4d95510708f6f88" + }, + "column:reporting_projection_notification_deliveries.claim_count": { + "enabled": true, + "fingerprint": "42202005517b72e082eb22c9eceb2ac0252815e5df700c83eb50c54cfeb46297" + }, + "column:reporting_projection_notification_deliveries.consumer_namespace": { + "enabled": true, + "fingerprint": "fdc4b549138bf90a5af5ed00b87a86ff9a0cbea553e7f017b4d95510708f6f88" + }, + "column:reporting_projection_notification_deliveries.delivery_id": { + "enabled": true, + "fingerprint": "fdc4b549138bf90a5af5ed00b87a86ff9a0cbea553e7f017b4d95510708f6f88" + }, + "column:reporting_projection_notification_deliveries.destination_sha256": { + "enabled": true, + "fingerprint": "10ccaa0dc3b93d48a1f32c7ef2352a11676632e1e871d77ef9ef4393eea15d27" + }, + "column:reporting_projection_notification_deliveries.due_at": { + "enabled": true, + "fingerprint": "1cac4e73af11a8ecafd646ef6a0ff6ecb087d46f150408dcfebc630fe1bf5e1e" + }, + "column:reporting_projection_notification_deliveries.emission_generation": { + "enabled": true, + "fingerprint": "54a5f9962598caa8723435f8b55bd5e15823053af5dd63d531663997537dc6ba" + }, + "column:reporting_projection_notification_deliveries.envelope": { + "enabled": true, + "fingerprint": "554c34e416bd4546469b42d5773bc7c59b2104366ffa5259a2838c64cd826e57" + }, + "column:reporting_projection_notification_deliveries.envelope_version": { + "enabled": true, + "fingerprint": "64be57437fdc0a07a97985c2aa058031f8082db7251bdb4d5afa1a9b088de97a" + }, + "column:reporting_projection_notification_deliveries.error_code": { + "enabled": true, + "fingerprint": "5b92595d0b54d473a3a3818455845f1fe0dc20b48cfe0faf63061a82d1a3cb02" + }, + "column:reporting_projection_notification_deliveries.idempotency_key": { + "enabled": true, + "fingerprint": "fdc4b549138bf90a5af5ed00b87a86ff9a0cbea553e7f017b4d95510708f6f88" + }, + "column:reporting_projection_notification_deliveries.key_version": { + "enabled": true, + "fingerprint": "fdc4b549138bf90a5af5ed00b87a86ff9a0cbea553e7f017b4d95510708f6f88" + }, + "column:reporting_projection_notification_deliveries.lease_expires_at": { + "enabled": true, + "fingerprint": "6f1466ce5d0aaac8471e39834b9c4b1f85d6f7169d9238a245ac035ff518e0fc" + }, + "column:reporting_projection_notification_deliveries.lease_token": { + "enabled": true, + "fingerprint": "5b92595d0b54d473a3a3818455845f1fe0dc20b48cfe0faf63061a82d1a3cb02" + }, + "column:reporting_projection_notification_deliveries.notification_id": { + "enabled": true, + "fingerprint": "fdc4b549138bf90a5af5ed00b87a86ff9a0cbea553e7f017b4d95510708f6f88" + }, + "column:reporting_projection_notification_deliveries.notification_type": { + "enabled": true, + "fingerprint": "fdc4b549138bf90a5af5ed00b87a86ff9a0cbea553e7f017b4d95510708f6f88" + }, + "column:reporting_projection_notification_deliveries.principal_id": { + "enabled": true, + "fingerprint": "fdc4b549138bf90a5af5ed00b87a86ff9a0cbea553e7f017b4d95510708f6f88" + }, + "column:reporting_projection_notification_deliveries.signing_scope_id": { + "enabled": true, + "fingerprint": "85b88fb407e112399f25b5dc8830dfdcb2a8271ecf00e70b662f6917b4a002cd" + }, + "column:reporting_projection_notification_deliveries.state": { + "enabled": true, + "fingerprint": "1a1ab7c892bfef3ca42f00cf764453bc5e0578b3a82ecf81da90b56816df493f" + }, + "column:reporting_projection_notification_deliveries.subscriber_id": { + "enabled": true, + "fingerprint": "fdc4b549138bf90a5af5ed00b87a86ff9a0cbea553e7f017b4d95510708f6f88" + }, + "column:reporting_projection_notification_deliveries.subscription_fingerprint": { + "enabled": true, + "fingerprint": "10ccaa0dc3b93d48a1f32c7ef2352a11676632e1e871d77ef9ef4393eea15d27" + }, + "column:reporting_projection_notification_events.account_id": { + "enabled": true, + "fingerprint": "fdc4b549138bf90a5af5ed00b87a86ff9a0cbea553e7f017b4d95510708f6f88" + }, + "column:reporting_projection_notification_events.admission_epoch": { + "enabled": true, + "fingerprint": "b636d6e865960a7e2527ed11b86a9a63fff025d6106672308d86fc9630c328ab" + }, + "column:reporting_projection_notification_events.cause_generation": { + "enabled": true, + "fingerprint": "54a5f9962598caa8723435f8b55bd5e15823053af5dd63d531663997537dc6ba" + }, + "column:reporting_projection_notification_events.cause_id": { + "enabled": true, + "fingerprint": "fdc4b549138bf90a5af5ed00b87a86ff9a0cbea553e7f017b4d95510708f6f88" + }, + "column:reporting_projection_notification_events.cause_kind": { + "enabled": true, + "fingerprint": "fdc4b549138bf90a5af5ed00b87a86ff9a0cbea553e7f017b4d95510708f6f88" + }, + "column:reporting_projection_notification_events.consumer_namespace": { + "enabled": true, + "fingerprint": "fdc4b549138bf90a5af5ed00b87a86ff9a0cbea553e7f017b4d95510708f6f88" + }, + "column:reporting_projection_notification_events.delivery_config_id": { + "enabled": true, + "fingerprint": "fdc4b549138bf90a5af5ed00b87a86ff9a0cbea553e7f017b4d95510708f6f88" + }, + "column:reporting_projection_notification_events.fingerprint": { + "enabled": true, + "fingerprint": "10ccaa0dc3b93d48a1f32c7ef2352a11676632e1e871d77ef9ef4393eea15d27" + }, + "column:reporting_projection_notification_events.fired_at": { + "enabled": true, + "fingerprint": "1cac4e73af11a8ecafd646ef6a0ff6ecb087d46f150408dcfebc630fe1bf5e1e" + }, + "column:reporting_projection_notification_events.notification_id": { + "enabled": true, + "fingerprint": "fdc4b549138bf90a5af5ed00b87a86ff9a0cbea553e7f017b4d95510708f6f88" + }, + "column:reporting_projection_notification_events.notification_type": { + "enabled": true, + "fingerprint": "fdc4b549138bf90a5af5ed00b87a86ff9a0cbea553e7f017b4d95510708f6f88" + }, + "column:reporting_projection_notification_events.obligation_namespace": { + "enabled": true, + "fingerprint": "fdc4b549138bf90a5af5ed00b87a86ff9a0cbea553e7f017b4d95510708f6f88" + }, + "column:reporting_projection_notification_events.scope_kind": { + "enabled": true, + "fingerprint": "fdc4b549138bf90a5af5ed00b87a86ff9a0cbea553e7f017b4d95510708f6f88" + }, + "column:reporting_projection_notification_events.snapshot": { + "enabled": true, + "fingerprint": "ac355fc16c02b70cb0a24afee8214cdce5f5cbfdc7fd1630786d5101932ecfa4" + }, + "column:reporting_projection_notification_events.version": { + "enabled": true, + "fingerprint": "54a5f9962598caa8723435f8b55bd5e15823053af5dd63d531663997537dc6ba" + }, + "column:reporting_projection_notification_expansions.account_id": { + "enabled": true, + "fingerprint": "fdc4b549138bf90a5af5ed00b87a86ff9a0cbea553e7f017b4d95510708f6f88" + }, + "column:reporting_projection_notification_expansions.claim_count": { + "enabled": true, + "fingerprint": "42202005517b72e082eb22c9eceb2ac0252815e5df700c83eb50c54cfeb46297" + }, + "column:reporting_projection_notification_expansions.consumer_namespace": { + "enabled": true, + "fingerprint": "128b66e02f14946100273f112c144af7605d626124d20fe16c1ba30c6b19ae3a" + }, + "column:reporting_projection_notification_expansions.due_at": { + "enabled": true, + "fingerprint": "1cac4e73af11a8ecafd646ef6a0ff6ecb087d46f150408dcfebc630fe1bf5e1e" + }, + "column:reporting_projection_notification_expansions.emission_generation": { + "enabled": true, + "fingerprint": "54a5f9962598caa8723435f8b55bd5e15823053af5dd63d531663997537dc6ba" + }, + "column:reporting_projection_notification_expansions.error_code": { + "enabled": true, + "fingerprint": "5b92595d0b54d473a3a3818455845f1fe0dc20b48cfe0faf63061a82d1a3cb02" + }, + "column:reporting_projection_notification_expansions.lease_expires_at": { + "enabled": true, + "fingerprint": "6f1466ce5d0aaac8471e39834b9c4b1f85d6f7169d9238a245ac035ff518e0fc" + }, + "column:reporting_projection_notification_expansions.lease_token": { + "enabled": true, + "fingerprint": "5b92595d0b54d473a3a3818455845f1fe0dc20b48cfe0faf63061a82d1a3cb02" + }, + "column:reporting_projection_notification_expansions.notification_id": { + "enabled": true, + "fingerprint": "fdc4b549138bf90a5af5ed00b87a86ff9a0cbea553e7f017b4d95510708f6f88" + }, + "column:reporting_projection_notification_expansions.state": { + "enabled": true, + "fingerprint": "1a1ab7c892bfef3ca42f00cf764453bc5e0578b3a82ecf81da90b56816df493f" + }, + "column:reporting_projection_webhook_attempt_heads.account_id": { + "enabled": true, + "fingerprint": "fdc4b549138bf90a5af5ed00b87a86ff9a0cbea553e7f017b4d95510708f6f88" + }, + "column:reporting_projection_webhook_attempt_heads.consumer_namespace": { + "enabled": true, + "fingerprint": "fdc4b549138bf90a5af5ed00b87a86ff9a0cbea553e7f017b4d95510708f6f88" + }, + "column:reporting_projection_webhook_attempt_heads.idempotency_key": { + "enabled": true, + "fingerprint": "fdc4b549138bf90a5af5ed00b87a86ff9a0cbea553e7f017b4d95510708f6f88" + }, + "column:reporting_projection_webhook_attempt_heads.last_attempt": { + "enabled": true, + "fingerprint": "54a5f9962598caa8723435f8b55bd5e15823053af5dd63d531663997537dc6ba" + }, + "column:reporting_projection_webhook_attempt_heads.principal_id": { + "enabled": true, + "fingerprint": "fdc4b549138bf90a5af5ed00b87a86ff9a0cbea553e7f017b4d95510708f6f88" + }, + "column:reporting_projection_webhook_attempt_heads.subscriber_id": { + "enabled": true, + "fingerprint": "fdc4b549138bf90a5af5ed00b87a86ff9a0cbea553e7f017b4d95510708f6f88" + }, + "column:reporting_projection_webhook_attempts.account_id": { + "enabled": true, + "fingerprint": "fdc4b549138bf90a5af5ed00b87a86ff9a0cbea553e7f017b4d95510708f6f88" + }, + "column:reporting_projection_webhook_attempts.attempt": { + "enabled": true, + "fingerprint": "54a5f9962598caa8723435f8b55bd5e15823053af5dd63d531663997537dc6ba" + }, + "column:reporting_projection_webhook_attempts.binding": { + "enabled": true, + "fingerprint": "ac355fc16c02b70cb0a24afee8214cdce5f5cbfdc7fd1630786d5101932ecfa4" + }, + "column:reporting_projection_webhook_attempts.completed_at": { + "enabled": true, + "fingerprint": "6f1466ce5d0aaac8471e39834b9c4b1f85d6f7169d9238a245ac035ff518e0fc" + }, + "column:reporting_projection_webhook_attempts.consumer_namespace": { + "enabled": true, + "fingerprint": "fdc4b549138bf90a5af5ed00b87a86ff9a0cbea553e7f017b4d95510708f6f88" + }, + "column:reporting_projection_webhook_attempts.delivery_id": { + "enabled": true, + "fingerprint": "fdc4b549138bf90a5af5ed00b87a86ff9a0cbea553e7f017b4d95510708f6f88" + }, + "column:reporting_projection_webhook_attempts.fired_at": { + "enabled": true, + "fingerprint": "1cac4e73af11a8ecafd646ef6a0ff6ecb087d46f150408dcfebc630fe1bf5e1e" + }, + "column:reporting_projection_webhook_attempts.http_status_code": { + "enabled": true, + "fingerprint": "4340876cb26818ac55a8d51cfbc7047e90fc4d7e44f570ee454b4552beb351a1" + }, + "column:reporting_projection_webhook_attempts.idempotency_key": { + "enabled": true, + "fingerprint": "fdc4b549138bf90a5af5ed00b87a86ff9a0cbea553e7f017b4d95510708f6f88" + }, + "column:reporting_projection_webhook_attempts.lease_token": { + "enabled": true, + "fingerprint": "10ccaa0dc3b93d48a1f32c7ef2352a11676632e1e871d77ef9ef4393eea15d27" + }, + "column:reporting_projection_webhook_attempts.notification_id": { + "enabled": true, + "fingerprint": "fdc4b549138bf90a5af5ed00b87a86ff9a0cbea553e7f017b4d95510708f6f88" + }, + "column:reporting_projection_webhook_attempts.payload_size_bytes": { + "enabled": true, + "fingerprint": "54a5f9962598caa8723435f8b55bd5e15823053af5dd63d531663997537dc6ba" + }, + "column:reporting_projection_webhook_attempts.principal_id": { + "enabled": true, + "fingerprint": "fdc4b549138bf90a5af5ed00b87a86ff9a0cbea553e7f017b4d95510708f6f88" + }, + "column:reporting_projection_webhook_attempts.reservation_token": { + "enabled": true, + "fingerprint": "10ccaa0dc3b93d48a1f32c7ef2352a11676632e1e871d77ef9ef4393eea15d27" + }, + "column:reporting_projection_webhook_attempts.response_time_ms": { + "enabled": true, + "fingerprint": "992336704a95e12ec6e959825c59fa2e51cddc5f1568af6bebaf12f03ac5655f" + }, + "column:reporting_projection_webhook_attempts.status": { + "enabled": true, + "fingerprint": "1a1ab7c892bfef3ca42f00cf764453bc5e0578b3a82ecf81da90b56816df493f" + }, + "column:reporting_projection_webhook_attempts.subscriber_id": { + "enabled": true, + "fingerprint": "fdc4b549138bf90a5af5ed00b87a86ff9a0cbea553e7f017b4d95510708f6f88" + }, + "column:reporting_projection_webhook_attempts.url": { + "enabled": true, + "fingerprint": "10ccaa0dc3b93d48a1f32c7ef2352a11676632e1e871d77ef9ef4393eea15d27" + }, + "column:reporting_projection_writes.account_id": { + "enabled": true, + "fingerprint": "fdc4b549138bf90a5af5ed00b87a86ff9a0cbea553e7f017b4d95510708f6f88" + }, + "column:reporting_projection_writes.transaction_id": { + "enabled": true, + "fingerprint": "fdc4b549138bf90a5af5ed00b87a86ff9a0cbea553e7f017b4d95510708f6f88" + }, + "column:reporting_status_scope_checkpoints.projection_writer_floor": { + "enabled": true, + "fingerprint": "7d6ce545a69d9f3e873875690ed9ccbec7114a9e32ad4820babca3f1c97b38ea" + }, + "constraint:reporting_projection_accounts.reporting_projection_accounts_check": { + "enabled": true, + "fingerprint": "ea35b760e040eef6a722d0b1ecf8a6e9f8d9d57feb6ac045caa2037b6b048a9b" + }, + "constraint:reporting_projection_accounts.reporting_projection_accounts_check1": { + "enabled": true, + "fingerprint": "77fe4d1cb567d39e9e6dae13605e0a9a50ec80cc8905c9c826d06394dc4f02bd" + }, + "constraint:reporting_projection_accounts.reporting_projection_accounts_check2": { + "enabled": true, + "fingerprint": "03eea3ba4967a34e66d84e32942c137d2548855973811cd4c301744098613880" + }, + "constraint:reporting_projection_accounts.reporting_projection_accounts_checkpoint_floor_check": { + "enabled": true, + "fingerprint": "4a5954bcfbdb5854908b976ed27044b8c64b4ad502f436f3333ef82973a44f8f" + }, + "constraint:reporting_projection_accounts.reporting_projection_accounts_legacy_capture_through_check": { + "enabled": true, + "fingerprint": "c5e638b4dc3130b99175082d33af80a2960e7c864129283767dd049e595aa4e8" + }, + "constraint:reporting_projection_accounts.reporting_projection_accounts_legacy_generation_floor_check": { + "enabled": true, + "fingerprint": "eeadbe3edcffa3355d8dc736af0580bf1c97a9b7a404213c277af268f1e10e75" + }, + "constraint:reporting_projection_accounts.reporting_projection_accounts_legacy_through_check": { + "enabled": true, + "fingerprint": "af3dcbbc378afb98c3ed16dd682316f9dfb4d66ac9d12b9118edecdc554cd19b" + }, + "constraint:reporting_projection_accounts.reporting_projection_accounts_max_sequence_check": { + "enabled": true, + "fingerprint": "0fc3b5ce464cce8adb79b8a8e8379c84713d2c37902b8b969cbaa0ac5322b0ae" + }, + "constraint:reporting_projection_accounts.reporting_projection_accounts_pkey": { + "enabled": true, + "fingerprint": "e65d70e61c89a93d66c4c4f4c59ef755d0ff526b0fcdd1d9ae6d830d2abea913" + }, + "constraint:reporting_projection_accounts.reporting_projection_accounts_version_check": { + "enabled": true, + "fingerprint": "905e3e7e12f64037c03d855fe84bf4319694539049e337e0ee0fcffb46d392a4" + }, + "constraint:reporting_projection_feed_snapshots.reporting_projection_feed_snapshot_representation_version_check": { + "enabled": true, + "fingerprint": "b4bb98e6c439f502c4ae47a541ac87e553b808e2b41efeb354d1b6e97c029caa" + }, + "constraint:reporting_projection_feed_snapshots.reporting_projection_feed_snapshots_check": { + "enabled": true, + "fingerprint": "00c281a801bf6afa7206453af0c2ec9839dae0df4420b9b7f6a1990eeb7c358f" + }, + "constraint:reporting_projection_feed_snapshots.reporting_projection_feed_snapshots_check1": { + "enabled": true, + "fingerprint": "375f2992cfd0b07e7dfefe6ae5c2aaa8fd16007d5964e91e8372a28c09aa684f" + }, + "constraint:reporting_projection_feed_snapshots.reporting_projection_feed_snapshots_check2": { + "enabled": true, + "fingerprint": "84972326afd68c2ff32f599ad0f129c22b91072d0eb4e1380613a91ded47e6da" + }, + "constraint:reporting_projection_feed_snapshots.reporting_projection_feed_snapshots_check3": { + "enabled": true, + "fingerprint": "18aad02260bda2aba9f6e51349520ac53ba3901ac25e26877d4b06c9d92832e3" + }, + "constraint:reporting_projection_feed_snapshots.reporting_projection_feed_snapshots_check4": { + "enabled": true, + "fingerprint": "ba2bd76a123a1c07c02d86609a9275fccedb28c2a9049ac87e5441c6c4e8aa64" + }, + "constraint:reporting_projection_feed_snapshots.reporting_projection_feed_snapshots_check5": { + "enabled": true, + "fingerprint": "fbb1a305ea8b9cd37c90a3612dccb065ca57d04405711dcba64a98bff46e5761" + }, + "constraint:reporting_projection_feed_snapshots.reporting_projection_feed_snapshots_check6": { + "enabled": true, + "fingerprint": "d6b940c0765d3ae2e773a1cba51ae8b478820bbe9bc593e005115421d1e9a415" + }, + "constraint:reporting_projection_feed_snapshots.reporting_projection_feed_snapshots_check7": { + "enabled": true, + "fingerprint": "87a2ec39393813ff33449cd7a02cc4738d1e91d5b6406e212b78372def2c1778" + }, + "constraint:reporting_projection_feed_snapshots.reporting_projection_feed_snapshots_document_check": { + "enabled": true, + "fingerprint": "84bdfff00c1f410606570fb522ab542616825c122b05a4e5d7b35d9937dd70e4" + }, + "constraint:reporting_projection_feed_snapshots.reporting_projection_feed_snapshots_document_check1": { + "enabled": true, + "fingerprint": "763668adc07dbcc7c5275dab0bbd3b9f0b6833893acd9725f4e9913d09c3c1e9" + }, + "constraint:reporting_projection_feed_snapshots.reporting_projection_feed_snapshots_document_check2": { + "enabled": true, + "fingerprint": "b393d759417c024e203b8521b8ba1e34d76e85dda5a8dfcdb1882ed2a56568f2" + }, + "constraint:reporting_projection_feed_snapshots.reporting_projection_feed_snapshots_document_check3": { + "enabled": true, + "fingerprint": "2053806dff13a9802996d32867a3858703e7e9a03b11735d216582cbde48ab4b" + }, + "constraint:reporting_projection_feed_snapshots.reporting_projection_feed_snapshots_document_check4": { + "enabled": true, + "fingerprint": "bf8ed87ea3b56d6eb3d7c0efa3575695b2b9e56f32bc0b3d172cc2395bcb1c9d" + }, + "constraint:reporting_projection_feed_snapshots.reporting_projection_feed_snapshots_ownership_mode_check": { + "enabled": true, + "fingerprint": "b642ee29e8813655ec63bb50bd53a601ac4da60cd3d4d31028340f55729aa362" + }, + "constraint:reporting_projection_feed_snapshots.reporting_projection_feed_snapshots_pkey": { + "enabled": true, + "fingerprint": "69c464c154977fe93cbcf866d711d9dc62357603a4a2cc104b280b44984d3533" + }, + "constraint:reporting_projection_feed_snapshots.reporting_projection_feed_snapshots_signing_key_check": { + "enabled": true, + "fingerprint": "4beb9e7f9146e3095acdb7d1dd3867766108e3ab93313deb9b6b0953e992c024" + }, + "constraint:reporting_projection_feed_snapshots.reporting_projection_feed_snapshots_snapshot_id_check": { + "enabled": true, + "fingerprint": "da3012902db7fe22f476ac4cca0d315f27456cace1dbb9f2ba4b3c9eae2d0a9e" + }, + "constraint:reporting_projection_feed_snapshots.reporting_projection_feed_snapshots_snapshot_id_key": { + "enabled": true, + "fingerprint": "c97a22e931dedffcfbd8ab22824035126745be64aa5d616a90e33563643ca749" + }, + "constraint:reporting_projection_inputs.reporting_projection_inputs_account_id_fkey": { + "enabled": true, + "fingerprint": "53a472fec66c9e990740ba025807acc57c5795bd0a84ea1ad29f8447fcfd0c85" + }, + "constraint:reporting_projection_inputs.reporting_projection_inputs_account_id_transaction_id_key": { + "enabled": true, + "fingerprint": "32924d7285616a887a0008aa1f0b8c418caae69c7429278d01064b64cba91dae" + }, + "constraint:reporting_projection_inputs.reporting_projection_inputs_check": { + "enabled": true, + "fingerprint": "88c2bcdd52446d3f8a7c5e21b16f277d231d44a17842e8c625d02e31cc16dd22" + }, + "constraint:reporting_projection_inputs.reporting_projection_inputs_check1": { + "enabled": true, + "fingerprint": "6db9e761fce16f6deaabe04581b3813f3495382b5edf905c2c01a772151eed89" + }, + "constraint:reporting_projection_inputs.reporting_projection_inputs_check2": { + "enabled": true, + "fingerprint": "b330c171bc0bb87ac12aa97aca4d4ae7167e0c07d7b61b3b688c63df9588d927" + }, + "constraint:reporting_projection_inputs.reporting_projection_inputs_input_check": { + "enabled": true, + "fingerprint": "cac041ba097e7ddad3ed595027b946cfb0c4a8b75bf187f5d2d9608fdb7174ac" + }, + "constraint:reporting_projection_inputs.reporting_projection_inputs_pkey": { + "enabled": true, + "fingerprint": "146d890eadd6744b5c6a7bfdbe840580d36e46d49ece8c02eff7c076f4edaec3" + }, + "constraint:reporting_projection_inputs.reporting_projection_inputs_sequence_check": { + "enabled": true, + "fingerprint": "554d491362648e795a6567528a6244977d5df276216297f35e26e4750f869dcf" + }, + "constraint:reporting_projection_legacy_baselines.reporting_projection_legacy_baselines_account_id_fkey": { + "enabled": true, + "fingerprint": "53a472fec66c9e990740ba025807acc57c5795bd0a84ea1ad29f8447fcfd0c85" + }, + "constraint:reporting_projection_legacy_baselines.reporting_projection_legacy_baselines_check": { + "enabled": true, + "fingerprint": "c05b78095cbb771e19c3758c560aa94e1b86c593ea90c3242012c8164d995ad5" + }, + "constraint:reporting_projection_legacy_baselines.reporting_projection_legacy_baselines_pkey": { + "enabled": true, + "fingerprint": "b7ae4c1159881c6b7e8cdd2a56418a730fc904db9deb2d993d4cf8fb81ad9cd8" + }, + "constraint:reporting_projection_legacy_checkpoints.reporting_projection_legacy_checkpoints_account_id_fkey": { + "enabled": true, + "fingerprint": "53a472fec66c9e990740ba025807acc57c5795bd0a84ea1ad29f8447fcfd0c85" + }, + "constraint:reporting_projection_legacy_checkpoints.reporting_projection_legacy_checkpoints_check": { + "enabled": true, + "fingerprint": "c05b78095cbb771e19c3758c560aa94e1b86c593ea90c3242012c8164d995ad5" + }, + "constraint:reporting_projection_legacy_checkpoints.reporting_projection_legacy_checkpoints_pkey": { + "enabled": true, + "fingerprint": "b7ae4c1159881c6b7e8cdd2a56418a730fc904db9deb2d993d4cf8fb81ad9cd8" + }, + "constraint:reporting_projection_legacy_inputs.reporting_projection_legacy_inputs_account_id_fkey": { + "enabled": true, + "fingerprint": "53a472fec66c9e990740ba025807acc57c5795bd0a84ea1ad29f8447fcfd0c85" + }, + "constraint:reporting_projection_legacy_inputs.reporting_projection_legacy_inputs_account_sequence_check": { + "enabled": true, + "fingerprint": "1a64697a81e3b39433d851f9916df1964cbbc260aa54dd6fee4076c7e805c70e" + }, + "constraint:reporting_projection_legacy_inputs.reporting_projection_legacy_inputs_check": { + "enabled": true, + "fingerprint": "88c2bcdd52446d3f8a7c5e21b16f277d231d44a17842e8c625d02e31cc16dd22" + }, + "constraint:reporting_projection_legacy_inputs.reporting_projection_legacy_inputs_check1": { + "enabled": true, + "fingerprint": "6db9e761fce16f6deaabe04581b3813f3495382b5edf905c2c01a772151eed89" + }, + "constraint:reporting_projection_legacy_inputs.reporting_projection_legacy_inputs_check2": { + "enabled": true, + "fingerprint": "6c5865ad0c28d00ec74194418def9d764ddf85b230f1dfb6d14c6e505eda3737" + }, + "constraint:reporting_projection_legacy_inputs.reporting_projection_legacy_inputs_check3": { + "enabled": true, + "fingerprint": "0d806a2b956a1e8947dcc43c634352ed52c1c3f158235145b2fe5a96db5fb379" + }, + "constraint:reporting_projection_legacy_inputs.reporting_projection_legacy_inputs_kind_check": { + "enabled": true, + "fingerprint": "b97ee75a4c9b39ff41cb90a8dc2faa6e4c985bbd6e2c96e4c77d6fb50f10e6cf" + }, + "constraint:reporting_projection_legacy_inputs.reporting_projection_legacy_inputs_pkey": { + "enabled": true, + "fingerprint": "5584c208839db0f8f7bc64ae38ae5a6f97c23493e9c4c19648e1e46b7fdb3f61" + }, + "constraint:reporting_projection_legacy_steps.reporting_projection_legacy_st_account_id_account_sequence_fkey": { + "enabled": true, + "fingerprint": "c9d5184cf4f53772f44f1fa97dbd3225d856113e8243d49e05177194a6d3573e" + }, + "constraint:reporting_projection_legacy_steps.reporting_projection_legacy_steps_check": { + "enabled": true, + "fingerprint": "88c2bcdd52446d3f8a7c5e21b16f277d231d44a17842e8c625d02e31cc16dd22" + }, + "constraint:reporting_projection_legacy_steps.reporting_projection_legacy_steps_input_check": { + "enabled": true, + "fingerprint": "f0f3bd2238cf6933df16e0f0cb3a2eb9049bd3e20150af23067d04760ae6189f" + }, + "constraint:reporting_projection_legacy_steps.reporting_projection_legacy_steps_pkey": { + "enabled": true, + "fingerprint": "54a1db9ba984eb27a50dc481d6e21cd2d187af90355713309f7b2f6c81927f40" + }, + "constraint:reporting_projection_notification_deliveries.reporting_projection_notific_account_id_consumer_namespac_fkey2": { + "enabled": true, + "fingerprint": "1e6be1e6f690856d3bc97d14b0e5837a60cc50d6f28432d40ec3f106398ed35d" + }, + "constraint:reporting_projection_notification_deliveries.reporting_projection_notific_account_id_consumer_namespac_fkey3": { + "enabled": true, + "fingerprint": "c33f0f1c9aaadd0bce3ad32c95b885e7d75af1fe37517c27e47f381d57f696b3" + }, + "constraint:reporting_projection_notification_deliveries.reporting_projection_notifica_account_id_consumer_namespac_key3": { + "enabled": true, + "fingerprint": "91982910079d78e1b6162e69e1e40d0f98ab055b811726cd8a9357c4aa9cfd22" + }, + "constraint:reporting_projection_notification_deliveries.reporting_projection_notifica_account_id_consumer_namespac_key4": { + "enabled": true, + "fingerprint": "bece3b7de1c22f3a600c89c175f942f83ecff078142418072b1bc1b42a670c18" + }, + "constraint:reporting_projection_notification_deliveries.reporting_projection_notification_del_emission_generation_check": { + "enabled": true, + "fingerprint": "5b7b2793bd710c7eef8c02003fd8adb1df91d7c10659364fe37d7f6351d4bf5e" + }, + "constraint:reporting_projection_notification_deliveries.reporting_projection_notification_deliv_notification_type_check": { + "enabled": true, + "fingerprint": "ee42647bcbefde76d58a6b7231c68bc1c4ef047e2523f779b4c28bcd9792eb47" + }, + "constraint:reporting_projection_notification_deliveries.reporting_projection_notification_delive_cause_generation_check": { + "enabled": true, + "fingerprint": "d93961696665129cf254c93dae4377741c5cef5fe9d373c6a5cb36226e5a69e2" + }, + "constraint:reporting_projection_notification_deliveries.reporting_projection_notification_deliveries_cause_kind_check": { + "enabled": true, + "fingerprint": "10f7027028f807d2ac122ae119f6de51d5c3b54217dcf78d0371b7d92ac4e942" + }, + "constraint:reporting_projection_notification_deliveries.reporting_projection_notification_deliveries_check": { + "enabled": true, + "fingerprint": "a61e7698dd266b3a4abee2c2716b00919f047e85bcd888a90c1a5bc5350c4d6e" + }, + "constraint:reporting_projection_notification_deliveries.reporting_projection_notification_deliveries_error_code_check": { + "enabled": true, + "fingerprint": "502085808cda7e7ec1e45e3d88257a660bdc65b59a71727b473303a362b9cf9d" + }, + "constraint:reporting_projection_notification_deliveries.reporting_projection_notification_deliveries_pkey": { + "enabled": true, + "fingerprint": "126da0131155e9b09e48d24466c5f61b7b6dc1540a68c465c52b59769637820d" + }, + "constraint:reporting_projection_notification_deliveries.reporting_projection_notification_deliveries_state_check": { + "enabled": true, + "fingerprint": "12b7a569e49f4bf374d5dade9ec0a0d812e4dea251366467c95bcd58bee90b9d" + }, + "constraint:reporting_projection_notification_events.reporting_projection_notifica_account_id_consumer_namespac_fkey": { + "enabled": true, + "fingerprint": "ee26a73f446c794ba69fc056f5fb1b8e094b25ae48743fe5e23045749e84f1c2" + }, + "constraint:reporting_projection_notification_events.reporting_projection_notifica_account_id_consumer_namespac_key1": { + "enabled": true, + "fingerprint": "7156cd6623f45f181906d2886b53dfba82bfdfc6cdcbdd0017c1f1cd7ceae407" + }, + "constraint:reporting_projection_notification_events.reporting_projection_notifica_account_id_consumer_namespac_key2": { + "enabled": true, + "fingerprint": "1d1f2cee412feec063fd1340b0d7773adfc55cad1f740e21bb45f75acf185b8c" + }, + "constraint:reporting_projection_notification_events.reporting_projection_notifica_account_id_consumer_namespace_key": { + "enabled": true, + "fingerprint": "a21acc6c4e405fcbf7711fe5a82c55206c994c6c969b498f9afe35af859ed206" + }, + "constraint:reporting_projection_notification_events.reporting_projection_notification_event_notification_type_check": { + "enabled": true, + "fingerprint": "ee42647bcbefde76d58a6b7231c68bc1c4ef047e2523f779b4c28bcd9792eb47" + }, + "constraint:reporting_projection_notification_events.reporting_projection_notification_events_admission_epoch_check": { + "enabled": true, + "fingerprint": "6cfd9db9dd38e8c29346710c69de6ff07379d1bef2e9e5b1aba297a3ffa833f2" + }, + "constraint:reporting_projection_notification_events.reporting_projection_notification_events_cause_generation_check": { + "enabled": true, + "fingerprint": "d93961696665129cf254c93dae4377741c5cef5fe9d373c6a5cb36226e5a69e2" + }, + "constraint:reporting_projection_notification_events.reporting_projection_notification_events_cause_id_check": { + "enabled": true, + "fingerprint": "e86db06c50f414ddc137693ad14cc7faaf8c3805591871c6d001cc5438d31a71" + }, + "constraint:reporting_projection_notification_events.reporting_projection_notification_events_cause_kind_check": { + "enabled": true, + "fingerprint": "10f7027028f807d2ac122ae119f6de51d5c3b54217dcf78d0371b7d92ac4e942" + }, + "constraint:reporting_projection_notification_events.reporting_projection_notification_events_check": { + "enabled": true, + "fingerprint": "efada5fd861f242458a8c57b2bf1c97280cd3c266d285050679afb2bbff363cb" + }, + "constraint:reporting_projection_notification_events.reporting_projection_notification_events_check1": { + "enabled": true, + "fingerprint": "36694052bb1b576de628eee965d5821ab01923e563295220ca21252603c2aa7a" + }, + "constraint:reporting_projection_notification_events.reporting_projection_notification_events_check10": { + "enabled": true, + "fingerprint": "627584412b64b780790cab6c0ca3a8f6e11a2b954eda505f58b709264a301956" + }, + "constraint:reporting_projection_notification_events.reporting_projection_notification_events_check11": { + "enabled": true, + "fingerprint": "1c4273a87391e21a4156104e7c9a556a70ddafaaca8810533b75335b14fa93bf" + }, + "constraint:reporting_projection_notification_events.reporting_projection_notification_events_check12": { + "enabled": true, + "fingerprint": "5cf1645ef1880c0fd61275e271b8ebfe33e12b4aa79646d82a1aef609c333b64" + }, + "constraint:reporting_projection_notification_events.reporting_projection_notification_events_check2": { + "enabled": true, + "fingerprint": "dc119dbfe2444526e08b41cedd271ea31deab1082f55816cc6d4257aa088f9d8" + }, + "constraint:reporting_projection_notification_events.reporting_projection_notification_events_check3": { + "enabled": true, + "fingerprint": "ffe54c1b8d166fbbfe66ebbf70dbd5ee8c8a61b09fdc57bbf2d5c9ccea8ad806" + }, + "constraint:reporting_projection_notification_events.reporting_projection_notification_events_check4": { + "enabled": true, + "fingerprint": "a2d30638b070f0f9f83bfaf782cc1a36cc89b2de31471a589f9568c174b8f7a1" + }, + "constraint:reporting_projection_notification_events.reporting_projection_notification_events_check5": { + "enabled": true, + "fingerprint": "7e8ae7f4e2f6d5b320600b088f0f7c9f31520fb499cfc8504a46b5fda4aa1180" + }, + "constraint:reporting_projection_notification_events.reporting_projection_notification_events_check6": { + "enabled": true, + "fingerprint": "49a10f7157a0b18ad541d7bf95ba43d28849ab45892d592d87bd720816325780" + }, + "constraint:reporting_projection_notification_events.reporting_projection_notification_events_check7": { + "enabled": true, + "fingerprint": "b7666e96be91fe368c981a8e7e0c3017823e34f97354428dd3d29c5a08a9c6a8" + }, + "constraint:reporting_projection_notification_events.reporting_projection_notification_events_check8": { + "enabled": true, + "fingerprint": "25fa4df322afde24dad3652593dffe4e696ceb9ab60b364f13d848839fea91f5" + }, + "constraint:reporting_projection_notification_events.reporting_projection_notification_events_check9": { + "enabled": true, + "fingerprint": "dafe7fb15d13195b1d2a6a1b4f62287d70b42543bc9ac156e32b1bce4f9c3c15" + }, + "constraint:reporting_projection_notification_events.reporting_projection_notification_events_fingerprint_check": { + "enabled": true, + "fingerprint": "e6f7fe9bd1925eb05eeab481da0229b0180558dd2448eda9e51e98faf31d6497" + }, + "constraint:reporting_projection_notification_events.reporting_projection_notification_events_pkey": { + "enabled": true, + "fingerprint": "e8d63f44b25700915bcb3cb69e2e24b98cf21fd304f36a2e9c2992236309309d" + }, + "constraint:reporting_projection_notification_events.reporting_projection_notification_events_scope_kind_check": { + "enabled": true, + "fingerprint": "990e8c59c76274efb82c87d70135717b5b47f7edbf86b52cdde0377a3f85194f" + }, + "constraint:reporting_projection_notification_events.reporting_projection_notification_events_version_check": { + "enabled": true, + "fingerprint": "e7b1e199b6f69994a833033a7770c2b6fcbf1b3ba77251b5df6bea466b0ee4b9" + }, + "constraint:reporting_projection_notification_expansions.reporting_projection_notific_account_id_consumer_namespac_fkey1": { + "enabled": true, + "fingerprint": "3325be861e9f98edd263b2b2e7952b31fde4d9463458f9044c426388b14c175a" + }, + "constraint:reporting_projection_notification_expansions.reporting_projection_notification_exp_emission_generation_check": { + "enabled": true, + "fingerprint": "5b7b2793bd710c7eef8c02003fd8adb1df91d7c10659364fe37d7f6351d4bf5e" + }, + "constraint:reporting_projection_notification_expansions.reporting_projection_notification_expansions_error_code_check": { + "enabled": true, + "fingerprint": "502085808cda7e7ec1e45e3d88257a660bdc65b59a71727b473303a362b9cf9d" + }, + "constraint:reporting_projection_notification_expansions.reporting_projection_notification_expansions_pkey": { + "enabled": true, + "fingerprint": "8ac920e8e1ed5a59417d5d1ffb9bea608bbeb64dbcd1a431240370fe555fe6f0" + }, + "constraint:reporting_projection_notification_expansions.reporting_projection_notification_expansions_state_check": { + "enabled": true, + "fingerprint": "12b7a569e49f4bf374d5dade9ec0a0d812e4dea251366467c95bcd58bee90b9d" + }, + "constraint:reporting_projection_webhook_attempt_heads.reporting_projection_webhook_attempt_heads_last_attempt_check": { + "enabled": true, + "fingerprint": "5bf9c354d680faec2bf3155f8c9b27323d7893564b30688a039ba19fc8b8d5d3" + }, + "constraint:reporting_projection_webhook_attempt_heads.reporting_projection_webhook_attempt_heads_pkey": { + "enabled": true, + "fingerprint": "490860c80b1129b15ab44f458a1a967382fc8b073f694ac959bf938282a74af5" + }, + "constraint:reporting_projection_webhook_attempt_heads.reporting_projection_webhook_attempt_heads_principal_id_check": { + "enabled": true, + "fingerprint": "bfc70b4b01cc74b9c93c630928d7cde9b0134c3d6614e2cbb326604c865c2cac" + }, + "constraint:reporting_projection_webhook_attempts.reporting_projection_webhook__account_id_consumer_namespac_fkey": { + "enabled": true, + "fingerprint": "160e4a76316d7669011f46848f4053b65a149f749fcec376a0318c21593a9931" + }, + "constraint:reporting_projection_webhook_attempts.reporting_projection_webhook__account_id_consumer_namespace_key": { + "enabled": true, + "fingerprint": "b8546a68def520db0e6139c898e3e2a76318e05548a3de1b6e568fbf3a75ee63" + }, + "constraint:reporting_projection_webhook_attempts.reporting_projection_webhook_account_id_consumer_namespac_fkey1": { + "enabled": true, + "fingerprint": "ed1785d5d7786390edfd842f1a7184f234f35e136ff3b23fe5993a8451f34200" + }, + "constraint:reporting_projection_webhook_attempts.reporting_projection_webhook_attempts_attempt_check": { + "enabled": true, + "fingerprint": "e05925f8a9fe41263902b6b2a72bca02958627e079787e99987820a52cb47d71" + }, + "constraint:reporting_projection_webhook_attempts.reporting_projection_webhook_attempts_payload_size_bytes_check": { + "enabled": true, + "fingerprint": "95548ff5519cec8b0ff110ad66f7b77d77e6015ef8c54a52e09cd4d7d80a7118" + }, + "constraint:reporting_projection_webhook_attempts.reporting_projection_webhook_attempts_pkey": { + "enabled": true, + "fingerprint": "5580d4f19f5ba257093ef71127d72dbba9669ac2db9b2f116f246d7f760522b4" + }, + "constraint:reporting_projection_webhook_attempts.reporting_projection_webhook_attempts_response_time_ms_check": { + "enabled": true, + "fingerprint": "6698890e9bba25670f73e3a5a7a7a7f9b664ab2571637eba582f22863c92db4d" + }, + "constraint:reporting_projection_webhook_attempts.reporting_projection_webhook_attempts_status_check": { + "enabled": true, + "fingerprint": "1c821d6bf41e5137a6263eae308d0a8648b7f0c95154724e5430e8c630aec30e" + }, + "constraint:reporting_projection_webhook_attempts.reporting_projection_webhook_completion": { + "enabled": true, + "fingerprint": "adc02762a63dcbf72330ca318884e6236fb2418cb7e1111f861ee74f98b53316" + }, + "constraint:reporting_projection_webhook_attempts.reporting_projection_webhook_identity": { + "enabled": true, + "fingerprint": "1e3f9725baa47d118c5b2efeeafbd5a2429400c1f667188fd14c4f2795114bf8" + }, + "constraint:reporting_projection_webhook_attempts.reporting_projection_webhook_outcome": { + "enabled": true, + "fingerprint": "cc0cc61f029bb3b28629e12a42c6f6ddbfd943156e1b90e5eeb2346aed678e00" + }, + "constraint:reporting_projection_webhook_attempts.reporting_projection_webhook_timestamps": { + "enabled": true, + "fingerprint": "0175d921479ed64020d88874f3b6ac822a9979f3ec8c3780d89bb9e0e3e3556c" + }, + "constraint:reporting_projection_webhook_attempts.reporting_projection_webhook_url_safe": { + "enabled": true, + "fingerprint": "3dd08bbc446317552434fb91bebd96e2a534395454b436910450f909ce2ab88e" + }, + "constraint:reporting_projection_writes.reporting_projection_commit": { + "enabled": true, + "fingerprint": "20ed755d29980bfc007ba63a8f0b50277c734a0703525b0c3ebeb1ebdc466162" + }, + "constraint:reporting_projection_writes.reporting_projection_writes_account_id_fkey": { + "enabled": true, + "fingerprint": "53a472fec66c9e990740ba025807acc57c5795bd0a84ea1ad29f8447fcfd0c85" + }, + "constraint:reporting_projection_writes.reporting_projection_writes_pkey": { + "enabled": true, + "fingerprint": "4d672603e38a33518c9c2166994348a7ec73c90f7b0c1f2afc73155bb6afdbc6" + }, + "constraint:reporting_status_scope_checkpoints.reporting_projection_writer_floor": { + "enabled": true, + "fingerprint": "2938b1784f2a320559cccf185d99ce7a8bb663f166eee8cbe883585e54b7814c" + }, + "function:reporting_projection_capture(owner text)": { + "enabled": true, + "fingerprint": "aa5ec57f464d517011526cdd6671e247e49ad622744a81a8d992f2db6b0b8007" + }, + "function:reporting_projection_checkpoint_guard()": { + "enabled": true, + "fingerprint": "326574129aa5b54b23eb18705b4898b600458a8b746b9a04a04a44e2575bec7b" + }, + "function:reporting_projection_commit()": { + "enabled": true, + "fingerprint": "da0a2d33359f39e067c9402d8b5474df4cb2eef1c1a49f48aa5e266c75aa7666" + }, + "function:reporting_projection_document(owner text, at_time timestamp with time zone)": { + "enabled": true, + "fingerprint": "33911a2307e5b4564bdbf6766e70ab46a1f81c5c8e5b37b88905a69f65a1c466" + }, + "function:reporting_projection_event_guard()": { + "enabled": true, + "fingerprint": "64f503d73db42c174921cbb9cbaf822383e39d35607c2ce0d9ff315c2c5f3074" + }, + "function:reporting_projection_feed_immutable()": { + "enabled": true, + "fingerprint": "b17f620de435a86c8815c3efbace4b602166bad7dc95f923434d402cddcd63f5" + }, + "function:reporting_projection_mark()": { + "enabled": true, + "fingerprint": "fa1c7a33b889525a065b07bff321794530ba8fbff996e901f6666bc75684536d" + }, + "function:reporting_projection_webhook_attempt_guard()": { + "enabled": true, + "fingerprint": "622d83f4fd58d38d06f2eef2223f011fe16eb9758438ae3155dcb0a2e5ddfe0b" + }, + "function:reporting_projection_webhook_head_guard()": { + "enabled": true, + "fingerprint": "4e448e0df3d56a5b04bc01ca41a4c5afbb8e601a4699d2e6dd0e3922ed2f0bf8" + }, + "index:reporting_projection_accounts.reporting_projection_accounts_pkey": { + "enabled": true, + "fingerprint": "4b805ae5bc3f3d509fb5f6d1551af8a6fa2065a9d6151a7f218f08c2be7a5292" + }, + "index:reporting_projection_accounts.reporting_projection_activation_pending": { + "enabled": true, + "fingerprint": "0dd7ddc05637321a246cf9236a202c9c4f96836993fdc2de75ff5fc7d6a625a8" + }, + "index:reporting_projection_accounts.reporting_projection_pending": { + "enabled": true, + "fingerprint": "97566fb1a05d93b85a923bc721b7e551bb6256184835c2caf1c089c870ed4585" + }, + "index:reporting_projection_feed_snapshots.reporting_projection_feed_snapshots_pkey": { + "enabled": true, + "fingerprint": "fbbf876488b7acbc5964c465cc63e579ec21c6036708a4baa5f8a4f36d582cd6" + }, + "index:reporting_projection_feed_snapshots.reporting_projection_feed_snapshots_snapshot_id_key": { + "enabled": true, + "fingerprint": "74d4af408b949a2021008e0e4cb1fa46fec81746804548c9333da4f4ca275f36" + }, + "index:reporting_projection_inputs.reporting_projection_inputs_account_id_transaction_id_key": { + "enabled": true, + "fingerprint": "e79866e916616016aad04b889700e57adb174019d81190ac1d9db4e8c3c475dc" + }, + "index:reporting_projection_inputs.reporting_projection_inputs_pkey": { + "enabled": true, + "fingerprint": "b74110aa7db352e4c0d8dd971620b983d43d9fca140e3da3e4379f12c3dbcf93" + }, + "index:reporting_projection_legacy_baselines.reporting_projection_legacy_baselines_pkey": { + "enabled": true, + "fingerprint": "33808c42f95cdd713a7067c81a772edd709fce85c5aa1a1f83329cf58bb7b6a2" + }, + "index:reporting_projection_legacy_checkpoints.reporting_projection_legacy_checkpoints_pkey": { + "enabled": true, + "fingerprint": "17288d53dcfa13cbec3c058b98485cf8063da6a752ff20fbe1d6493ef50bb2d2" + }, + "index:reporting_projection_legacy_checkpoints.reporting_projection_legacy_consumer": { + "enabled": true, + "fingerprint": "43909bb4e3297a0e79795b308e6157724c50ac63559e9d5bf50d565de1db9e62" + }, + "index:reporting_projection_legacy_inputs.reporting_projection_legacy_inputs_pkey": { + "enabled": true, + "fingerprint": "c3717266852eee124f070047263fc5d0c76b6680416b7c72f74834d1b25beaa9" + }, + "index:reporting_projection_legacy_steps.reporting_projection_legacy_steps_pkey": { + "enabled": true, + "fingerprint": "572e483043d844b80f0bcb05162d210c233eafe1776235f197475b6fc23631ea" + }, + "index:reporting_projection_notification_deliveries.reporting_projection_notifica_account_id_consumer_namespac_key3": { + "enabled": true, + "fingerprint": "db9f6646990410974ecfe24a7dd3b1d4eefc35678756d91789f37a8e15640344" + }, + "index:reporting_projection_notification_deliveries.reporting_projection_notifica_account_id_consumer_namespac_key4": { + "enabled": true, + "fingerprint": "41c096310d5ca152e99b979a7ba591b3276ee4c1dbb858a243201f2c93720cc9" + }, + "index:reporting_projection_notification_deliveries.reporting_projection_notification_deliveries_due": { + "enabled": true, + "fingerprint": "dd243bfbf67a1d56f4773583a92efd477ebc262e772706d4c759bbbe0ccb5236" + }, + "index:reporting_projection_notification_deliveries.reporting_projection_notification_deliveries_pkey": { + "enabled": true, + "fingerprint": "39973dc1c3de7bdd4bb1ae7ab98477ebbda282ff9e93398b8499e4bc7169e8fc" + }, + "index:reporting_projection_notification_events.reporting_projection_notifica_account_id_consumer_namespac_key1": { + "enabled": true, + "fingerprint": "ffcbea4686e819d18554bf3c7357122e90aaf14a4227ab321e1bcf15248b84d8" + }, + "index:reporting_projection_notification_events.reporting_projection_notifica_account_id_consumer_namespac_key2": { + "enabled": true, + "fingerprint": "cc35d20e8e088c65dfc86e25fd6f4f67e8644307d25b32bab6c545e144bd0e87" + }, + "index:reporting_projection_notification_events.reporting_projection_notifica_account_id_consumer_namespace_key": { + "enabled": true, + "fingerprint": "ca07092e6e38681fee63f6717cf10f1446bf3b8bc117a73cde19b5dcc987be5c" + }, + "index:reporting_projection_notification_events.reporting_projection_notification_events_pkey": { + "enabled": true, + "fingerprint": "0e42c2e06c2b9faccd04e0ac781ccc0a7ff684ba827e84523a70abd427ade222" + }, + "index:reporting_projection_notification_expansions.reporting_projection_notification_expansions_due": { + "enabled": true, + "fingerprint": "a654a298089afe3acfff950faacaf4bcddc198db1680756ce3a6cc5cceba2905" + }, + "index:reporting_projection_notification_expansions.reporting_projection_notification_expansions_pkey": { + "enabled": true, + "fingerprint": "37720f22ee293ab422ad013f058a90c1d02fcc12ff547c424a724eb3b32c2501" + }, + "index:reporting_projection_webhook_attempt_heads.reporting_projection_webhook_attempt_heads_pkey": { + "enabled": true, + "fingerprint": "34ead5983456c814e390fcd1951ece7d38545783c6b9edc52a568ea623e05475" + }, + "index:reporting_projection_webhook_attempts.reporting_projection_webhook__account_id_consumer_namespace_key": { + "enabled": true, + "fingerprint": "c7e1c8f8be8ff6fe8798a05bbaa74b478de73e887433c948989d1e750bc85593" + }, + "index:reporting_projection_webhook_attempts.reporting_projection_webhook_activity_newest": { + "enabled": true, + "fingerprint": "199287b543822d45c89d735206c1636e8d0978bea77a8240b7793640fade97d3" + }, + "index:reporting_projection_webhook_attempts.reporting_projection_webhook_activity_retention": { + "enabled": true, + "fingerprint": "f6f99131566c8279a4b3d4006a2122954acedeb287bd9f59eff8dfafb67c9963" + }, + "index:reporting_projection_webhook_attempts.reporting_projection_webhook_attempts_pkey": { + "enabled": true, + "fingerprint": "b2c98959ee7fa3d182d6fea57f2ddaa5beb726da8f97a7081e1b9b59da78cffe" + }, + "index:reporting_projection_writes.reporting_projection_writes_pkey": { + "enabled": true, + "fingerprint": "23e19a2a5adb02bec7292061246f159726c855318b1686e01b88357cdf1a11a9" + }, + "index:reporting_status_scope_checkpoints.reporting_projection_due_clock": { + "enabled": true, + "fingerprint": "f7343b98d419d3e2d87d25f88469d6c1e92a9838a6f359452e58712c4f406a5e" + }, + "table:reporting_projection_accounts": { + "enabled": true, + "fingerprint": "1f824779ff80f110344420b019786663d8c9beaad230da90e0439795e734ccda" + }, + "table:reporting_projection_feed_snapshots": { + "enabled": true, + "fingerprint": "1f824779ff80f110344420b019786663d8c9beaad230da90e0439795e734ccda" + }, + "table:reporting_projection_inputs": { + "enabled": true, + "fingerprint": "1f824779ff80f110344420b019786663d8c9beaad230da90e0439795e734ccda" + }, + "table:reporting_projection_legacy_baselines": { + "enabled": true, + "fingerprint": "1f824779ff80f110344420b019786663d8c9beaad230da90e0439795e734ccda" + }, + "table:reporting_projection_legacy_checkpoints": { + "enabled": true, + "fingerprint": "1f824779ff80f110344420b019786663d8c9beaad230da90e0439795e734ccda" + }, + "table:reporting_projection_legacy_inputs": { + "enabled": true, + "fingerprint": "1f824779ff80f110344420b019786663d8c9beaad230da90e0439795e734ccda" + }, + "table:reporting_projection_legacy_steps": { + "enabled": true, + "fingerprint": "1f824779ff80f110344420b019786663d8c9beaad230da90e0439795e734ccda" + }, + "table:reporting_projection_notification_deliveries": { + "enabled": true, + "fingerprint": "1f824779ff80f110344420b019786663d8c9beaad230da90e0439795e734ccda" + }, + "table:reporting_projection_notification_events": { + "enabled": true, + "fingerprint": "1f824779ff80f110344420b019786663d8c9beaad230da90e0439795e734ccda" + }, + "table:reporting_projection_notification_expansions": { + "enabled": true, + "fingerprint": "1f824779ff80f110344420b019786663d8c9beaad230da90e0439795e734ccda" + }, + "table:reporting_projection_webhook_attempt_heads": { + "enabled": true, + "fingerprint": "1f824779ff80f110344420b019786663d8c9beaad230da90e0439795e734ccda" + }, + "table:reporting_projection_webhook_attempts": { + "enabled": true, + "fingerprint": "1f824779ff80f110344420b019786663d8c9beaad230da90e0439795e734ccda" + }, + "table:reporting_projection_writes": { + "enabled": true, + "fingerprint": "1f824779ff80f110344420b019786663d8c9beaad230da90e0439795e734ccda" + }, + "trigger:reporting_adjustments.reporting_projection_mark": { + "enabled": true, + "fingerprint": "192ccdbcab9b7193a85990dcef614ef0be68fd79582525856b3c00e793d361b4" + }, + "trigger:reporting_configurations.reporting_projection_mark": { + "enabled": true, + "fingerprint": "09cb0a6ab2c13da199badf822de029b0290387a098bb0365844e72d4f74a08bb" + }, + "trigger:reporting_consumer_statuses.reporting_projection_mark": { + "enabled": true, + "fingerprint": "91d3dff97f063184909c23620c5bcd6a0cc1c07bae8c888b58a80c3a2342c5bd" + }, + "trigger:reporting_issue_lifecycle.reporting_projection_mark": { + "enabled": true, + "fingerprint": "2793d5d63ec5e17d63695f5264ca8a78f30f4353d967b2acf12d3dda1f420634" + }, + "trigger:reporting_issue_status_scopes.reporting_projection_mark": { + "enabled": true, + "fingerprint": "b6ec60e605cd956fda57ffec137fb4530ba2c41d071351d626aa65113ae4afba" + }, + "trigger:reporting_obligations.reporting_projection_mark": { + "enabled": true, + "fingerprint": "83f851a94ccf29c50b0d9ca2d8e5d5ee4edf9d3801ae806077dc47f7113a8230" + }, + "trigger:reporting_projection_feed_snapshots.reporting_projection_feed_immutable": { + "enabled": true, + "fingerprint": "4a3f5552f5c940234144e1300ff526fc54f7376c076b457a47e48d38848d06b4" + }, + "trigger:reporting_projection_inputs.reporting_projection_input_immutable": { + "enabled": true, + "fingerprint": "714e3c04ac1ffdf34f8435915e3829fcef55bd3df8ebe4b719f5d14414b6fcb1" + }, + "trigger:reporting_projection_legacy_baselines.reporting_projection_legacy_immutable": { + "enabled": true, + "fingerprint": "68c9d1f4db22039ae8ebfa2b315b0325c512d138906833c9658341279aaf1d2d" + }, + "trigger:reporting_projection_legacy_inputs.reporting_projection_legacy_immutable": { + "enabled": true, + "fingerprint": "8005ae52bd5e328ff3c7774768ab875b3eea36993d9fbdedb6db2d3766aafd8f" + }, + "trigger:reporting_projection_legacy_steps.reporting_projection_legacy_immutable": { + "enabled": true, + "fingerprint": "0979972b1537ecd106497c92511ad512a255e185646b4416c25355aad8aeaab5" + }, + "trigger:reporting_projection_notification_events.reporting_projection_event_guard": { + "enabled": true, + "fingerprint": "6ad8a2eb5e1c4b6ddf040412d468f2805b3695b055b8258dbc4bf07fed36fc4b" + }, + "trigger:reporting_projection_notification_events.reporting_projection_event_immutable": { + "enabled": true, + "fingerprint": "69fe101856c6f93e27f3366027a54e6c005953d8b1fd09fc089c8642b723a925" + }, + "trigger:reporting_projection_webhook_attempt_heads.reporting_projection_webhook_head_guard": { + "enabled": true, + "fingerprint": "f3ad96da6db3a669fc802e18373b8a8e158f0b0ca2aea04ca1b9683107dd714c" + }, + "trigger:reporting_projection_webhook_attempts.reporting_projection_webhook_attempt_guard": { + "enabled": true, + "fingerprint": "d4e53e26100053f79451da61382f1a587e382b15f37cc2b31411288278c8dc3a" + }, + "trigger:reporting_projection_writes.reporting_projection_commit": { + "enabled": true, + "fingerprint": "23308ca5359667c11fa7aef1a70801d6f250b3a03cc986e8c8482fe4c5344d0c" + }, + "trigger:reporting_reconciliation_changes.reporting_projection_mark": { + "enabled": true, + "fingerprint": "c94a3a33a4596e81710c3c803834638949e83d563c784ca4cc28efbf8cf8f918" + }, + "trigger:reporting_revisions.reporting_projection_mark": { + "enabled": true, + "fingerprint": "10aba6894d1730ce630923537d17a97bfb2eb984889d43cdf28c396820a0aa01" + }, + "trigger:reporting_status_scope_checkpoints.reporting_projection_checkpoint_guard": { + "enabled": true, + "fingerprint": "0403ed1d78051471207cda0c361b95ead9995a0a200574e65ad3bc98214f7a40" + } +} diff --git a/src/adcp/reporting/projection/schema.py b/src/adcp/reporting/projection/schema.py new file mode 100644 index 000000000..9b46a0592 --- /dev/null +++ b/src/adcp/reporting/projection/schema.py @@ -0,0 +1,26 @@ +"""B2.4's additive manifest never widens an earlier feature's required objects.""" + +from __future__ import annotations + +import json +from importlib.resources import files +from typing import Any + +from adcp.reporting.feed.schema import validate_feed_schema +from adcp.reporting.ledger.notification_models import ReportingNotificationError +from adcp.reporting.outbox._schema import schema_objects +from adcp.reporting.outbox.status_schema import validate_status_schema + + +async def validate_projection_schema(connection: Any, *, notifications: bool = False) -> None: + try: + required = json.loads( + files("adcp.reporting.projection").joinpath("required_schema.json").read_text() + ) + actual = await schema_objects(connection) + if not required or any(actual.get(k) != v for k, v in required.items()): + raise ValueError + await validate_feed_schema(connection, notifications=notifications) + await validate_status_schema(connection) + except Exception: + raise ReportingNotificationError("status_projection_schema_unready") from None diff --git a/src/adcp/reporting/projection/wire.py b/src/adcp/reporting/projection/wire.py new file mode 100644 index 000000000..64346120a --- /dev/null +++ b/src/adcp/reporting/projection/wire.py @@ -0,0 +1,161 @@ +"""Tier-correct exact views over one captured, caller-private financial history.""" + +from __future__ import annotations + +from typing import Any, Protocol, runtime_checkable + +from adcp.reporting.ledger.delivery import ( + ReportingMaterializationView, + adjustment_to_wire, + materialization_to_wire, + receipt_to_wire, + revision_to_wire, +) +from adcp.reporting.ledger.delivery_models import ( + ReportingAdjustmentReceiptRecord, + ReportingDeliveryPrincipal, + ReportingDeliveryRecord, + ReportingDestinationBinding, + ReportingMaterializationAttempt, + ReportingMaterializationCheck, + ReportingMaterializationRecord, + ReportingRevisionReceiptRecord, +) +from adcp.reporting.ledger.models import ReportingObligationRecord, ReportingRevisionRecord +from adcp.reporting.ledger.reconciliation_projection import project_reconciliation +from adcp.reporting.ledger.status import ReportingStatusCaller, ReportingStatusHandler +from adcp.reporting.ledger.status_projection import ReportingStatusSnapshot +from adcp.reporting.ledger.store import LedgerConflictError +from adcp.reporting.materializer.capture import private_snapshot +from adcp.reporting.outbox.status import settled_replay +from adcp.reporting.projection.capture import ReportingProjectionInput + + +@runtime_checkable +class ReportingTierStatusStore(Protocol): + async def read_tier_status( + self, + request: dict[str, Any], + *, + caller: ReportingDeliveryPrincipal, + consumer_status_enabled: bool = False, + ) -> dict[str, Any]: ... + + +def render_tier_status( + store: Any, + request: dict[str, Any], + value: ReportingProjectionInput, + caller: ReportingDeliveryPrincipal, + policy: dict[str, Any], + consumer_status_enabled: bool, +) -> dict[str, Any]: + if ( + getattr(store, "_projection_read_policy", None) != policy + or policy["consumer_status_enabled"] != consumer_status_enabled + ): + raise LedgerConflictError( + "STATUS_PROJECTION_UNAVAILABLE", "reporting projection is unavailable" + ) + return ReportingStatusHandler( + store, + consumer_status_enabled=consumer_status_enabled, + escalation=getattr(store, "_projection_read_escalation", None), + ).render_snapshot( + request, + caller=ReportingStatusCaller(caller.account_id, caller.consumer_id), + snapshot=settled_replay(private_snapshot(value.core, caller)), + reconciliation=value.reconciliation, + revision_ownership=policy["ownership_enabled"], + ) + + +def exact_revision_evidence( + core: ReportingStatusSnapshot, + revision: ReportingRevisionRecord, + owner: ReportingObligationRecord | None, + records: tuple[ReportingDeliveryRecord, ...], + caller: ReportingStatusCaller, +) -> dict[str, Any]: + if owner is None or owner.account_id != caller.account_id: + raise LedgerConflictError( + "LOOKUP_UNAVAILABLE", "no such revision is available to this caller" + ) + history = tuple( + r for r in core.revisions if r.reporting_obligation_id == owner.reporting_obligation_id + ) + project_reconciliation( + owner, history, core.adjustments, records, consumer_id=caller.consumer_id, as_of=core.as_of + ) + binding = next( + ( + r + for r in records + if isinstance(r, ReportingDestinationBinding) + and r.generation_key == owner.generation_key + ), + None, + ) + if binding is None: + return {} + attempts = { + r.reporting_materialization_id: r + for r in records + if isinstance(r, ReportingMaterializationAttempt) + and r.reporting_revision_id == revision.reporting_revision_id + } + artifacts = [ + r + for r in records + if isinstance(r, ReportingMaterializationRecord) + and r.reporting_revision_id == revision.reporting_revision_id + ] + adjustments = tuple( + a + for a in core.adjustments + if a.adjusts_reporting_revision_id == revision.reporting_revision_id + ) + adjustment_ids = {a.reporting_adjustment_id for a in adjustments} + receipts = [ + r + for r in records + if isinstance(r, ReportingRevisionReceiptRecord) + and r.reporting_revision_id == revision.reporting_revision_id + ] + adjustment_receipts = [ + r + for r in records + if isinstance(r, ReportingAdjustmentReceiptRecord) + and r.reporting_adjustment_id in adjustment_ids + ] + return { + "revision": revision_to_wire(revision, obligation=owner), + "adjustments": [adjustment_to_wire(a) for a in adjustments], + "materializations": [ + materialization_to_wire( + ReportingMaterializationView( + attempts[r.reporting_materialization_id], + binding, + r, + tuple( + c + for c in records + if isinstance(c, ReportingMaterializationCheck) + and c.reporting_materialization_id == r.reporting_materialization_id + ), + ), + obligation=owner, + ) + for r in artifacts + ], + "receipts": [receipt_to_wire(r) for r in receipts], + "adjustment_receipts": [receipt_to_wire(r) for r in adjustment_receipts], + "pagination": { + "total_count": 1 + + len(adjustments) + + len(artifacts) + + len(receipts) + + len(adjustment_receipts), + "has_more": False, + }, + } diff --git a/src/adcp/reporting/receipts/handler.py b/src/adcp/reporting/receipts/handler.py index 050017734..62a5db83d 100644 --- a/src/adcp/reporting/receipts/handler.py +++ b/src/adcp/reporting/receipts/handler.py @@ -5,9 +5,10 @@ from collections.abc import Awaitable, Callable from typing import Any +from adcp._version import is_adcp_version_at_least, resolve_adcp_version from adcp.decisioning.context import AuthInfo, RequestContext from adcp.decisioning.registry import BuyerAgent, BuyerAgentRegistry, HttpSigCredential -from adcp.exceptions import ADCPTaskError +from adcp.exceptions import ADCPTaskError, ConfigurationError from adcp.reporting.ledger.delivery_models import ReportingDeliveryPrincipal from adcp.reporting.ledger.notification_models import ReportingNotificationError from adcp.reporting.outbox.identity import canonical_consumer, resolve_reporting_consumer @@ -109,6 +110,7 @@ def __init__( resolve_account: ReceiptAccountResolver, buyer_agents: BuyerAgentRegistry | None = None, consumer_status_enabled: bool = False, + adcp_version: str | None = None, ) -> None: super().__init__() if not isinstance(store, ReportingReceiptBatchStore): @@ -122,6 +124,16 @@ def __init__( store if isinstance(store, ReportingFeedStore) else None ) self._feed_consumer_status_enabled = consumer_status_enabled + self._adcp_version = resolve_adcp_version(adcp_version) + if not is_adcp_version_at_least(self._adcp_version, "3.2-rc.3"): + raise ConfigurationError( + "reporting mounts require a supported AdCP 3.2 reporting contract; " + "use 3.2-rc.3 for retained walks or omit the pin for the packaged default" + ) + + def get_adcp_version(self) -> str: + """Select rendering and advertised MCP/A2A schemas for this mount.""" + return self._adcp_version def advertised_tools_for_instance(self) -> set[str]: return {TASK, "get_reporting_status"} if self.reporting_feed_store is not None else {TASK} @@ -140,10 +152,14 @@ async def get_reporting_status( if self.reporting_feed_store is None: return self._not_supported("get_reporting_status") request = ( - params + dict(params) if isinstance(params, dict) else params.model_dump(mode="json", exclude_unset=True) ) + if context is not None and context.resolved_adcp_version is not None: + request["adcp_version"] = context.resolved_adcp_version + else: + request.setdefault("adcp_version", self.get_adcp_version()) try: if request.get("view") == "periods": FeedRequest.parse(request) @@ -163,14 +179,14 @@ async def authorize() -> ReportingDeliveryPrincipal: raise ReportingFeedError("UNAUTHORIZED") from None caller = await authorize() - if request.get("view") == "periods": - async def reauthorize() -> None: - # A still-authorized alias must not change which account or - # canonical consumer owns the already captured boundary. - if await authorize() != caller: - raise ReportingFeedError("UNAUTHORIZED") + async def reauthorize() -> None: + # A still-authorized alias must not change which account or + # canonical consumer owns the already captured boundary. + if await authorize() != caller: + raise ReportingFeedError("UNAUTHORIZED") + if request.get("view") == "periods": response = await self.reporting_feed_store.read_reporting_feed( request, caller=caller, @@ -190,13 +206,24 @@ async def reauthorize() -> None: raise ReportingFeedError("INVALID_CHECKPOINT") if isinstance(self.receipt_store, ReportingLedgerStore): try: - return await ReportingStatusHandler( - self.receipt_store, - consumer_status_enabled=self._feed_consumer_status_enabled, - ).handle( - request, - caller=ReportingStatusCaller(caller.account_id, caller.consumer_id), - ) + from adcp.reporting.projection.wire import ReportingTierStatusStore + + if isinstance(self.receipt_store, ReportingTierStatusStore): + response = await self.receipt_store.read_tier_status( + request, + caller=caller, + consumer_status_enabled=self._feed_consumer_status_enabled, + ) + else: + response = await ReportingStatusHandler( + self.receipt_store, + consumer_status_enabled=self._feed_consumer_status_enabled, + ).handle( + request, + caller=ReportingStatusCaller(caller.account_id, caller.consumer_id), + ) + await reauthorize() + return response except LedgerConflictError as error: # Only the legacy Core projector exposes its established # domain errors. ACL/provider/feed failures stay redacted. diff --git a/src/adcp/server/a2a_server.py b/src/adcp/server/a2a_server.py index 4404e43e8..91d944c0c 100644 --- a/src/adcp/server/a2a_server.py +++ b/src/adcp/server/a2a_server.py @@ -1540,6 +1540,11 @@ def agent_card_url(request: Request) -> str: registered=list(executor.supported_skills), ) + if hasattr(handler, "production"): + from adcp.reporting.production.service import register_production_mount + + register_production_mount(handler, app, transport="a2a", dispatcher=executor) + return app diff --git a/src/adcp/server/mcp_tools.py b/src/adcp/server/mcp_tools.py index 5859b5d22..aba71b5b4 100644 --- a/src/adcp/server/mcp_tools.py +++ b/src/adcp/server/mcp_tools.py @@ -2447,7 +2447,15 @@ def get_tools_for_handler( input_schema = get_mcp_schema(name, "request", version=resolved_version) if input_schema is None: continue - definition = copy.deepcopy(tool) + # The current-model schemas can be large and are replaced below by + # the exact versioned wire schemas. Copy only retained metadata. + definition = copy.deepcopy( + { + key: value + for key, value in tool.items() + if key not in {"inputSchema", "outputSchema"} + } + ) if name == "sync_reporting_receipts": from adcp.reporting.receipts.wire import receipt_schema diff --git a/src/adcp/server/serve.py b/src/adcp/server/serve.py index 03752eb67..257a811bc 100644 --- a/src/adcp/server/serve.py +++ b/src/adcp/server/serve.py @@ -2572,6 +2572,10 @@ def create_mcp_server( ) _install_adcp_mcp_transport_methods(mcp) mcp._session_manager = _create_adcp_mcp_session_manager(mcp) + if hasattr(handler, "production"): + from adcp.reporting.production.service import register_production_mount + + register_production_mount(handler, mcp, transport="mcp", dispatcher=mcp) return mcp diff --git a/src/adcp/types/__init__.py b/src/adcp/types/__init__.py index 43e1c13d9..afe9cccbc 100644 --- a/src/adcp/types/__init__.py +++ b/src/adcp/types/__init__.py @@ -138,6 +138,8 @@ "PaymentTerms", "ReportUsageRequest", "ReportUsageResponse", + "ReportingAdjustment", + "ReportingAdjustmentReceipt", "ReportingBucket", "ReportingCanonicalContentDigest", "ReportingCanonicalizationContract", @@ -1820,6 +1822,8 @@ def __dir__() -> list[str]: RegistryAcceptancePolicyProfileReference, Renders, RepeatableAssetGroup, + ReportingAdjustment, + ReportingAdjustmentReceipt, ReportingAuthoritativeParty, ReportingBucket, ReportingCanonicalContentDigest, diff --git a/src/adcp/types/_eager.py b/src/adcp/types/_eager.py index ea23bd2fe..d01888864 100644 --- a/src/adcp/types/_eager.py +++ b/src/adcp/types/_eager.py @@ -345,6 +345,8 @@ RefineProposalsResponse, RegistryAcceptancePolicyProfileReference, Renders, + ReportingAdjustment, + ReportingAdjustmentReceipt, ReportingBucket, ReportingCanonicalContentDigest, ReportingCanonicalizationContract, @@ -1756,6 +1758,8 @@ def __init__(self, *args: object, **kwargs: object) -> None: "ReportPlanAdjustmentResponse", "ReportUsageRequest", "ReportUsageResponse", + "ReportingAdjustment", + "ReportingAdjustmentReceipt", "ReportingBucket", "ReportingCanonicalContentDigest", "ReportingCanonicalizationContract", diff --git a/src/adcp/types/base.py b/src/adcp/types/base.py index cfbfa0063..cb8be3fea 100644 --- a/src/adcp/types/base.py +++ b/src/adcp/types/base.py @@ -275,12 +275,11 @@ class AdCPBaseModel(BaseModel): model_config = ConfigDict(extra=_EXTRA_POLICY, defer_build=True) @model_serializer(mode="wrap") - def _explicit_reporting_wire_defaults(self, handler: SerializerFunctionWrapHandler) -> Any: - """Do not synthesize conditional reporting promises from generated defaults. + def _notification_config_wire_defaults(self, handler: SerializerFunctionWrapHandler) -> Any: + """Retain the unrelated product default only for product subscriptions. - The generated classes remain untouched. This wrapper also runs for - nested account notification configs and model_dump_json, without - mutating the adopter's model or its fields-set information. + Reporting capability defaults and tier validation live in their own + generated models; this wrapper does not mask reporting attributes. """ value = handler(self) if not isinstance(value, dict): @@ -295,31 +294,6 @@ def _explicit_reporting_wire_defaults(self, handler: SerializerFunctionWrapHandl str(getattr(e, "value", e)).startswith("product.") for e in events ): value.pop("product_payload_view", None) - elif { - "supported", - "reliable_reporting_version", - "managed_delivery", - "reconciled_billing", - "offerings", - "automated_recovery_window_seconds", - "status_retention_days", - } <= fields.keys(): - for field in ( - "reliable_reporting_version", - "managed_delivery", - "reconciled_billing", - "configuration_task", - "status_task", - "consumer_status_task", - "revision_content_task", - "receipt_task", - "readiness_notification", - "status_notification", - "ledger_notification", - "supports_webhook_activity", - ): - if field not in self.model_fields_set: - value.pop(field, None) return value def model_dump(self, **kwargs: Any) -> dict[str, Any]: diff --git a/src/adcp/types/generated_poc/bundled/protocol/get_adcp_capabilities_response.py b/src/adcp/types/generated_poc/bundled/protocol/get_adcp_capabilities_response.py index aff0aa170..09dd01722 100644 --- a/src/adcp/types/generated_poc/bundled/protocol/get_adcp_capabilities_response.py +++ b/src/adcp/types/generated_poc/bundled/protocol/get_adcp_capabilities_response.py @@ -4,6 +4,9 @@ from __future__ import annotations +from typing import Any +from pydantic import SerializerFunctionWrapHandler, model_serializer, model_validator + from datetime import date from adcp.types._str_enum import StrEnum from typing import Annotated, Any, Dict, Literal @@ -4640,61 +4643,61 @@ class ReportingDelivery(AdCPBaseModel): ) supported: Literal[True] reliable_reporting_version: Annotated[ - Literal['1.0'], + Literal['1.0'] | None, Field( description='Explicit adoption declaration for the proper-name AdCP 3.2 Reliable Reporting contract. Presence, together with supported: true and the media_buy.reporting_delivery experimental feature gate, is the affirmative machine-readable answer. Absence denotes the earlier experimental managed-reporting shape.' ), - ] = '1.0' + ] = None managed_delivery: Annotated[ bool | None, Field( description='Tier flag: this seller supports managed file, dataset-share, or warehouse delivery. Offerings whose method names a delivery pattern require this tier. When false or absent, every offering is API-delivered and Core-only.' ), - ] = False + ] = None reconciled_billing: Annotated[ bool | None, Field( description='Tier flag: this seller supports canonical-digest verification and authenticated consumer receipts for both report materializations and post-official adjustments through receipt_task. Offerings with reconciliation_mode consumer_receipt and billing-grade canonicalization require this tier.' ), - ] = False - configuration_task: Literal['sync_accounts'] = 'sync_accounts' - status_task: Literal['get_reporting_status'] = 'get_reporting_status' + ] = None + configuration_task: Literal['sync_accounts'] | None = None + status_task: Literal['get_reporting_status'] | None = None consumer_status_task: Annotated[ - Literal['sync_reporting_status'], + Literal['sync_reporting_status'] | None, Field( description='Opt-in consumer-status loop during the published migration window, becoming required Core in the next eligible minor after that window. Buyers call this seller-hosted task to record whether each expected reporting period was received, missing, or unreadable. Buyers expose no reverse endpoint, and the status is not a billing receipt.' ), - ] = 'sync_reporting_status' + ] = None revision_content_task: Annotated[ - Literal['get_media_buy_delivery'], + Literal['get_media_buy_delivery'] | None, Field( description='Reliable Reporting exact-content read: callers select reporting_revision_id and receive immutable revision metadata plus authoritative canonical reporting_rows.' ), - ] = 'get_media_buy_delivery' + ] = None receipt_task: Annotated[ - Literal['sync_reporting_receipts'], + Literal['sync_reporting_receipts'] | None, Field( description='Required when reconciled_billing is true: the task consumers call to submit and read back authenticated revision and adjustment receipts.' ), - ] = 'sync_reporting_receipts' + ] = None readiness_notification: Annotated[ - Literal['reporting.delivery_ready'], + Literal['reporting.delivery_ready'] | None, Field( description='Optional managed-delivery-only positive-readiness doorbell. It names a materialization at a destination, so Core sellers MUST omit it.' ), - ] = 'reporting.delivery_ready' + ] = None status_notification: Annotated[ - Literal['reporting.status_changed'], + Literal['reporting.status_changed'] | None, Field( description='Optional tier-independent invalidation doorbell for health transitions in either direction, including clock-driven waiting-to-delayed and delayed-to-action_required. Valid for Core: it names no destination. Polling status_task remains the authoritative recovery path whether or not this is offered.' ), - ] = 'reporting.status_changed' + ] = None ledger_notification: Annotated[ - Literal['reporting.ledger_changed'], + Literal['reporting.ledger_changed'] | None, Field( description='Optional tier-independent invalidation for every newly committed revision or post-official adjustment, even when health does not change. Receivers repair through get_reporting_status changes_after; polling remains authoritative.' ), - ] = 'reporting.ledger_changed' + ] = None offerings: Annotated[ list[Offering], Field( @@ -4742,7 +4745,7 @@ class ReportingDelivery(AdCPBaseModel): ge=1, ), ] = None - supports_webhook_activity: bool | None = False + supports_webhook_activity: bool | None = None authorization_revocation_seconds: Annotated[ int | None, Field( @@ -4751,6 +4754,22 @@ class ReportingDelivery(AdCPBaseModel): ), ] = None + @model_validator(mode='after') + def _validate_reporting_tiers(self) -> ReportingDelivery: + if self.reconciled_billing is True and self.managed_delivery is not True: + raise ValueError('reconciled_billing requires managed_delivery') + if self.readiness_notification is not None and self.managed_delivery is not True: + raise ValueError('readiness_notification requires managed_delivery') + if self.receipt_task is not None and self.reconciled_billing is not True: + raise ValueError('receipt_task requires reconciled_billing') + return self + + @model_serializer(mode='wrap') + def _omit_absent_reporting_promises( + self, handler: SerializerFunctionWrapHandler + ) -> dict[str, Any]: + return {key: value for key, value in handler(self).items() if value is not None} + class TranslationTarget(AdCPBaseModel): model_config = ConfigDict( diff --git a/src/adcp/types/generated_poc/core/reporting_delivery_capabilities.py b/src/adcp/types/generated_poc/core/reporting_delivery_capabilities.py index e3dd37398..0cf914ecb 100644 --- a/src/adcp/types/generated_poc/core/reporting_delivery_capabilities.py +++ b/src/adcp/types/generated_poc/core/reporting_delivery_capabilities.py @@ -1,9 +1,12 @@ # generated by datamodel-codegen: # filename: core/reporting_delivery_capabilities.json -# timestamp: 2026-09-14T14:16:02+00:00 +# timestamp: 2026-09-18T13:06:39+00:00 from __future__ import annotations +from typing import Any +from pydantic import SerializerFunctionWrapHandler, model_serializer, model_validator + from typing import Annotated, Literal from adcp.types.base import AdCPBaseModel @@ -38,61 +41,61 @@ class ReportingDeliveryCapabilities(AdCPBaseModel): ) supported: Literal[True] reliable_reporting_version: Annotated[ - Literal['1.0'], + Literal['1.0'] | None, Field( description='Explicit adoption declaration for the proper-name AdCP 3.2 Reliable Reporting contract. Presence, together with supported: true and the media_buy.reporting_delivery experimental feature gate, is the affirmative machine-readable answer. Absence denotes the earlier experimental managed-reporting shape.' ), - ] = '1.0' + ] = None managed_delivery: Annotated[ bool | None, Field( description='Tier flag: this seller supports managed file, dataset-share, or warehouse delivery. Offerings whose method names a delivery pattern require this tier. When false or absent, every offering is API-delivered and Core-only.' ), - ] = False + ] = None reconciled_billing: Annotated[ bool | None, Field( description='Tier flag: this seller supports canonical-digest verification and authenticated consumer receipts for both report materializations and post-official adjustments through receipt_task. Offerings with reconciliation_mode consumer_receipt and billing-grade canonicalization require this tier.' ), - ] = False - configuration_task: Literal['sync_accounts'] = 'sync_accounts' - status_task: Literal['get_reporting_status'] = 'get_reporting_status' + ] = None + configuration_task: Literal['sync_accounts'] | None = None + status_task: Literal['get_reporting_status'] | None = None consumer_status_task: Annotated[ - Literal['sync_reporting_status'], + Literal['sync_reporting_status'] | None, Field( description='Opt-in consumer-status loop during the published migration window, becoming required Core in the next eligible minor after that window. Buyers call this seller-hosted task to record whether each expected reporting period was received, missing, or unreadable. Buyers expose no reverse endpoint, and the status is not a billing receipt.' ), - ] = 'sync_reporting_status' + ] = None revision_content_task: Annotated[ - Literal['get_media_buy_delivery'], + Literal['get_media_buy_delivery'] | None, Field( description='Reliable Reporting exact-content read: callers select reporting_revision_id and receive immutable revision metadata plus authoritative canonical reporting_rows.' ), - ] = 'get_media_buy_delivery' + ] = None receipt_task: Annotated[ - Literal['sync_reporting_receipts'], + Literal['sync_reporting_receipts'] | None, Field( description='Required when reconciled_billing is true: the task consumers call to submit and read back authenticated revision and adjustment receipts.' ), - ] = 'sync_reporting_receipts' + ] = None readiness_notification: Annotated[ - Literal['reporting.delivery_ready'], + Literal['reporting.delivery_ready'] | None, Field( description='Optional managed-delivery-only positive-readiness doorbell. It names a materialization at a destination, so Core sellers MUST omit it.' ), - ] = 'reporting.delivery_ready' + ] = None status_notification: Annotated[ - Literal['reporting.status_changed'], + Literal['reporting.status_changed'] | None, Field( description='Optional tier-independent invalidation doorbell for health transitions in either direction, including clock-driven waiting-to-delayed and delayed-to-action_required. Valid for Core: it names no destination. Polling status_task remains the authoritative recovery path whether or not this is offered.' ), - ] = 'reporting.status_changed' + ] = None ledger_notification: Annotated[ - Literal['reporting.ledger_changed'], + Literal['reporting.ledger_changed'] | None, Field( description='Optional tier-independent invalidation for every newly committed revision or post-official adjustment, even when health does not change. Receivers repair through get_reporting_status changes_after; polling remains authoritative.' ), - ] = 'reporting.ledger_changed' + ] = None offerings: Annotated[ list[reporting_delivery_offering.ReportingDeliveryOffering], Field( @@ -140,7 +143,7 @@ class ReportingDeliveryCapabilities(AdCPBaseModel): ge=1, ), ] = None - supports_webhook_activity: bool | None = False + supports_webhook_activity: bool | None = None authorization_revocation_seconds: Annotated[ int | None, Field( @@ -148,3 +151,19 @@ class ReportingDeliveryCapabilities(AdCPBaseModel): ge=0, ), ] = None + + @model_validator(mode='after') + def _validate_reporting_tiers(self) -> ReportingDeliveryCapabilities: + if self.reconciled_billing is True and self.managed_delivery is not True: + raise ValueError('reconciled_billing requires managed_delivery') + if self.readiness_notification is not None and self.managed_delivery is not True: + raise ValueError('readiness_notification requires managed_delivery') + if self.receipt_task is not None and self.reconciled_billing is not True: + raise ValueError('receipt_task requires reconciled_billing') + return self + + @model_serializer(mode='wrap') + def _omit_absent_reporting_promises( + self, handler: SerializerFunctionWrapHandler + ) -> dict[str, Any]: + return {key: value for key, value in handler(self).items() if value is not None} diff --git a/src/adcp/validation/schema_loader.py b/src/adcp/validation/schema_loader.py index 630dedf0f..14267f0ac 100644 --- a/src/adcp/validation/schema_loader.py +++ b/src/adcp/validation/schema_loader.py @@ -31,10 +31,10 @@ import threading import warnings from copy import deepcopy -from datetime import datetime +from datetime import date from importlib.resources import as_file, files from pathlib import Path -from typing import Any, Literal +from typing import Any, Literal, cast from urllib.parse import unquote, urlparse from adcp.validation.version import resolve_bundle_key @@ -56,7 +56,8 @@ r"^\d{4}-\d{2}-\d{2}[Tt]" r"(?:[01]\d|2[0-3]):[0-5]\d:[0-5]\d" r"(?:\.\d+)?" - r"(?:[Zz]|[+-](?:[01]\d|2[0-3]):[0-5]\d)$" + r"(?:[Zz]|[+-](?:[01]\d|2[0-3]):[0-5]\d)$", + re.ASCII, ) @@ -73,9 +74,12 @@ def _is_rfc3339_date_time(instance: Any) -> bool: return True if _RFC3339_DATE_TIME.fullmatch(instance) is None: return False - normalized = instance[:-1] + "+00:00" if instance.endswith(("Z", "z")) else instance try: - datetime.fromisoformat(normalized) + # The grammar already checks time and offset ranges. Validate only + # the calendar here: Python 3.10's datetime parser accepts only three + # or six fractional digits, whereas RFC 3339 permits any positive + # number. Validation must neither coerce nor truncate the wire value. + date.fromisoformat(instance[:10]) except ValueError: return False return True @@ -143,6 +147,10 @@ def __init__(self, root: _SchemaRoot, bundle_key: str) -> None: self.compiled: dict[tuple[str, Direction], Any] = {} self.named_compiled: dict[str, Any] = {} self.portable: dict[tuple[str, Direction], dict[str, Any]] = {} + # Serialized JSON keeps the immutable cached value private and makes + # every returned tree independent, including any repeated branches. + self.mcp_schemas: dict[tuple[str, Direction], str] = {} + self.mcp_schema_lock = threading.Lock() self.registry: dict[str, dict[str, Any]] = {} self._registry_loaded = False @@ -358,7 +366,7 @@ def _make_ref_resolver(state: _LoaderState, base_file: Path, schema: dict[str, A ) from exc _load_schema_registry(state) - base_uri = base_file.resolve().parent.as_uri() + "/" + base_uri = base_file.resolve().as_uri() def missing_local_reference(uri: str) -> Any: raise ValueError(f"schema reference is not in bundle {state.bundle_key}: {uri}") @@ -471,6 +479,45 @@ def visit(value: Any, *, root: bool = False) -> None: return normalized +def _effective_task_schema( + schema: dict[str, Any], tool_name: str, direction: Direction, *, bundle_key: str +) -> dict[str, Any]: + """Apply the SDK's captured-schedule contract without rewriting signed bundles. + + Reporting health describes existing evidence; it does not cancel a frozen + generation's future commitment (#1179). The 3.2.0-rc.3 status schema couples + the two at /allOf/2/then/not. Correct only that exact known rule and version. + Its if, scope closure and coverage requirements remain unchanged. A changed + or different-version rule is left intact for explicit compatibility review. + """ + if (tool_name, direction, bundle_key) != ("get_reporting_status", "sync", "3.2.0-rc.3"): + return schema + known_rule = { + "if": { + "properties": {"health": {"const": "complete"}}, + "required": ["health"], + }, + "then": { + "properties": { + "scope": { + "properties": { + "scope_closed": {"const": True}, + "coverage_complete": {"const": True}, + }, + "required": ["scope_closed", "coverage_complete"], + } + }, + "not": {"required": ["next_expected_at"]}, + }, + } + conditions = schema.get("allOf") + if not isinstance(conditions, list) or len(conditions) < 3 or conditions[2] != known_rule: + return schema + result = deepcopy(schema) + del result["allOf"][2]["then"]["not"] + return result + + def get_validator( tool_name: str, direction: Direction, @@ -507,6 +554,7 @@ def get_validator( return None if file.is_relative_to(state.root.bundled): schema = _normalize_bundled_schema_for_validation(schema) + schema = _effective_task_schema(schema, tool_name, direction, bundle_key=state.bundle_key) try: from jsonschema import Draft7Validator, FormatChecker @@ -596,7 +644,7 @@ def get_named_validator( from jsonschema import RefResolver resolver = RefResolver( - base_uri=file.resolve().parent.as_uri() + "/", + base_uri=file.resolve().as_uri(), referrer=schema, store=_reachable_schema_store(state, file, schema), ) @@ -645,7 +693,9 @@ def get_schema( if not isinstance(schema, dict): logger.warning("Schema %s is not a JSON object", file) return None - return deepcopy(schema) + return deepcopy( + _effective_task_schema(schema, tool_name, direction, bundle_key=state.bundle_key) + ) def get_named_schema_document( @@ -737,7 +787,11 @@ def get_portable_schema( schema = json.loads(file.read_text()) if not isinstance(schema, dict): raise ValueError("schema root is not an object") - portable = _self_contained_schema(state, file, schema) + portable = _self_contained_schema( + state, + file, + _effective_task_schema(schema, tool_name, direction, bundle_key=state.bundle_key), + ) except (OSError, json.JSONDecodeError, KeyError, ValueError) as exc: logger.warning("Failed to make schema %s portable for %s: %s", file, key, exc) return None @@ -872,35 +926,59 @@ def get_mcp_schema( Newer bundles provide self-contained production-profile schemas that remove duplicated descriptions and definitions. Releases without those artifacts fall back to their canonical versioned schema. + + Successful materializations belong to the immutable versioned loader + state. Each caller receives an independent, alias-free JSON tree; missing + or invalid schemas are not added to the materialization cache. """ state = _ensure_state(version) if state is None: return None key = (tool_name, direction) - file = state.mcp_index.get(key) or state.source_index.get(key) or state.file_index.get(key) - if file is None: - return None - try: - schema = json.loads(file.read_text()) - except (OSError, json.JSONDecodeError) as exc: - logger.warning( - "Failed to load MCP schema %s for %s::%s: %s", - file, - tool_name, - direction, - exc, - ) - return None - if not isinstance(schema, dict): - logger.warning("MCP schema %s is not a JSON object", file) - return None - try: - portable = _self_contained_schema(state, file, schema) - except (OSError, json.JSONDecodeError, KeyError, ValueError) as exc: - logger.warning("Failed to make MCP schema %s portable: %s", file, exc) - return None - compact = _strip_schema_annotations(portable) - return compact if isinstance(compact, dict) else None + cached = state.mcp_schemas.get(key) + if cached is None: + with state.mcp_schema_lock: + # Only one concurrent first caller traverses the reference graph. + cached = state.mcp_schemas.get(key) + if cached is None: + file = ( + state.mcp_index.get(key) + or state.source_index.get(key) + or state.file_index.get(key) + ) + if file is None: + return None + try: + schema = json.loads(file.read_text()) + except (OSError, json.JSONDecodeError) as exc: + logger.warning( + "Failed to load MCP schema %s for %s::%s: %s", + file, + tool_name, + direction, + exc, + ) + return None + if not isinstance(schema, dict): + logger.warning("MCP schema %s is not a JSON object", file) + return None + try: + portable = _self_contained_schema( + state, + file, + _effective_task_schema( + schema, tool_name, direction, bundle_key=state.bundle_key + ), + ) + except (OSError, json.JSONDecodeError, KeyError, ValueError) as exc: + logger.warning("Failed to make MCP schema %s portable: %s", file, exc) + return None + compact = _strip_schema_annotations(portable) + if not isinstance(compact, dict): + return None + cached = json.dumps(compact, separators=(",", ":")) + state.mcp_schemas[key] = cached + return cast(dict[str, Any], json.loads(cached)) def list_validator_keys(*, version: str | None = None) -> list[str]: diff --git a/tests/conformance/reporting/_feed_support.py b/tests/conformance/reporting/_feed_support.py index 3ef9ecfea..c976d0481 100644 --- a/tests/conformance/reporting/_feed_support.py +++ b/tests/conformance/reporting/_feed_support.py @@ -160,10 +160,9 @@ def __init__(self, h, *, feedback=False, **kwargs): resolve_account=self.resolve_account, buyer_agents=self.registry, consumer_status_enabled=feedback, + adcp_version=self.version, ) self.handler.get_reporting_status = self.idempotency.wrap(self.handler.get_reporting_status) - if kwargs.get("version") is not None: - self.handler.adcp_version = kwargs["version"] @asynccontextmanager async def sdk_clients(self, a2a_version, *, token="token-one"): @@ -215,7 +214,7 @@ def mcp_http(**kwargs): clients[protocol] = await stack.enter_async_context( ADCPClient( config, - adcp_version="3.2-rc.6", + adcp_version=self.version, force_a2a_version=a2a_version if protocol == "a2a" else None, httpx_client_factory=mcp_http if protocol == "mcp" else None, ) diff --git a/tests/conformance/reporting/_production_delivery_process.py b/tests/conformance/reporting/_production_delivery_process.py new file mode 100644 index 000000000..655238e12 --- /dev/null +++ b/tests/conformance/reporting/_production_delivery_process.py @@ -0,0 +1,134 @@ +"""Real SIGKILL after receiver acceptance; cold retry uses the durable deadline.""" + +import asyncio +import hashlib +import json +import sys +from datetime import datetime +from pathlib import Path +from types import SimpleNamespace + + +async def main(settings): + import pytest + from psycopg_pool import AsyncConnectionPool + + import adcp.reporting.production.delivery_window as window_module + from adcp.reporting.outbox.routing import ReportingEnvelopeCipher + from adcp.reporting.production.notifications import ( + ReportingProductionSigning, + production_notification_workers, + ) + from adcp.reporting.production.pg import PgReportingProductionStore + from adcp.reporting.projection.pg import PgReportingStatusProjection + + from ._reliable_support import ( + Barrier, + DeterministicReceiverStore, + FailurePlan, + ManualClock, + ScriptedNotificationReceiver, + ScriptedSigning, + ScriptedSubscriptions, + _BytesStore, + notification_subscription, + ) + from .test_reporting_production_notifications import EVENTS, retained_windows + + origin = Path(window_module.__file__).resolve() + assert hashlib.sha256(origin.read_bytes()).hexdigest() == settings["module_sha256"] + if settings["installed"]: + assert origin.is_relative_to(Path(sys.prefix)) and "site-packages" in str(origin) + clock = ManualClock(datetime.fromisoformat(settings["at"])) + async with AsyncConnectionPool( + settings["conninfo"], kwargs=settings["kwargs"], min_size=2, max_size=4, open=False + ) as pool: + store = PgReportingProductionStore(pool=pool, clock=clock, notifications=True) + await store.create_schema() + projection = PgReportingStatusProjection(store, revision_ownership=True) + failures = FailurePlan() + subscriptions = ScriptedSubscriptions(failures) + subscriptions.put( + notification_subscription( + subscriber="buyer", + principal=settings["consumer"], + events=EVENTS, + url="https://receiver.example.test/reporting", + ) + ) + resolver = ScriptedSigning(failures) + resolver.generation = 1 if settings["pause"] else 2 + signing = ReportingProductionSigning( + resolver, ("ed25519",), brand_json_url="https://seller.example.test/brand.json" + ) + worker = production_notification_workers( + store, + projection, + subscriptions=subscriptions, + cipher=ReportingEnvelopeCipher(b"b" * 32), + signing=signing, + )[settings["queue"]] + blobs = _BytesStore(pool) + await blobs.create_schema() + received = DeterministicReceiverStore(blobs, failures) + receiver = ScriptedNotificationReceiver( + SimpleNamespace(clock=clock, failures=failures, receiver=received) + ) + with pytest.MonkeyPatch.context() as patch: + receiver.install(patch) + if settings["pause"]: + accepted = Barrier() + failures.at("http.accepted", accepted) + task = asyncio.create_task(worker.deliver_one(account_id="acct_a")) + await accepted.wait() + windows = await retained_windows(SimpleNamespace(pool=pool)) + assert len(windows) == 1 + row = windows[0] + print( + json.dumps( + { + "point": "accepted_before_ack", + "started_at": row[4].isoformat(), + "expires_at": row[5].isoformat(), + "key_sha256": hashlib.sha256(row[1].encode()).hexdigest(), + "body_sha256": row[3], + "origin": str(origin), + } + ), + flush=True, + ) + await asyncio.to_thread(sys.stdin.readline) + accepted.release() + await task + raise AssertionError("paused child must be killed") + before = await retained_windows(SimpleNamespace(pool=pool)) + production_operation_1 = await worker.deliver_one(account_id="acct_a") + assert production_operation_1 + assert await retained_windows(SimpleNamespace(pool=pool)) == before + row = before[0] + states = await worker.outbox.list_deliveries(account_id="acct_a") + target = next(s for s in states if s.delivery.binding.idempotency_key == row[1]) + body = await received.read("acct_a", row[1]) + assert body is not None and hashlib.sha256(body).hexdigest() == row[3] + attempts = await worker.outbox.list_activity( + account_id="acct_a", consumer_id=settings["consumer"] + ) + result = { + "point": "done", + "http_calls": len(receiver.received), + "state": target.state, + "error_code": target.error_code, + "activity_count": len(attempts), + "key_sha256": hashlib.sha256(row[1].encode()).hexdigest(), + "body_sha256": row[3], + "started_at": row[4].isoformat(), + "expires_at": row[5].isoformat(), + "origin": str(origin), + } + if receiver.received: + assert "key-2" in receiver.received[0].headers["signature-input"] + return result + + +if __name__ == "__main__": + print(json.dumps(asyncio.run(main(json.loads(sys.stdin.readline())))), flush=True) diff --git a/tests/conformance/reporting/_production_installed.py b/tests/conformance/reporting/_production_installed.py new file mode 100644 index 000000000..c01c3f7b2 --- /dev/null +++ b/tests/conformance/reporting/_production_installed.py @@ -0,0 +1,192 @@ +"""Floor-runtime conformance with complete installed module/schema provenance.""" + +import contextlib +import hashlib +import importlib +import importlib.metadata +import importlib.util +import json +import os +import subprocess +import sys +import time +from importlib.resources import files +from pathlib import Path + +import pytest + + +def main(settings): + root = Path(settings["fixtures"]) + workspace = Path(settings["workspace"]) + evidence = Path(settings["evidence"]) + evidence.mkdir(parents=True, exist_ok=True, mode=0o700) + phases = [] + + def enter_phase(name): + # Closed phase names identify a failed preflight without publishing + # arbitrary child stderr, provider detail or runtime values. + phases.append(name) + (evidence / (settings["label"] + "-phases.json")).write_text( + json.dumps({"entered_phases": phases}) + "\n" + ) + + enter_phase("runtime") + assert sys.version_info[:2] == tuple(settings["python"]) + assert not any(Path(p).resolve().is_relative_to(workspace) for p in sys.path) + assert not (root / "adcp").exists() and not (root / "src").exists() + sys.path.insert(0, str(root)) + from tests.conformance.reporting._hardening_installed import Results + + enter_phase("installed_modules") + origins = {} + for name, expected in settings["modules"].items(): + path = Path(importlib.import_module(name).__file__).resolve() + assert path.is_relative_to(Path(sys.prefix)) and not path.is_relative_to(workspace) + assert hashlib.sha256(path.read_bytes()).hexdigest() == expected + origins[name] = str(path) + enter_phase("installed_assets") + for name, expected in settings["assets"].items(): + assert ( + hashlib.sha256(files("adcp.reporting").joinpath(name).read_bytes()).hexdigest() + == expected + ) + from adcp.validation import schema_loader + + enter_phase("installed_current_schemas") + assert set(settings["schemas"]) == {schema_loader._sdk_pinned_bundle_key()} + for version, schemas in settings["schemas"].items(): + resolved = schema_loader._resolve_schema_root(version) + assert resolved is not None + schema_root = resolved.root + assert schema_root.is_relative_to(Path(sys.prefix)) + for name, expected in schemas.items(): + assert hashlib.sha256((schema_root / name).read_bytes()).hexdigest() == expected + enter_phase("historical_reference_schemas") + reference = settings["historical_reference_schema"] + reference_root = Path(reference["root"]) + assert not reference_root.is_relative_to(Path(sys.prefix)) + assert not reference_root.is_relative_to(workspace) + assert reference["version"] not in settings["schemas"] + assert schema_loader._resolve_schema_root(reference["version"]).root == reference_root + + def reference_manifest(): + return { + str(p.relative_to(reference_root)): hashlib.sha256(p.read_bytes()).hexdigest() + for p in sorted(reference_root.rglob("*.json")) + } + + assert reference_manifest() == reference["files"] + enter_phase("optional_driver_boundary") + if settings["driver_absent"]: + assert importlib.util.find_spec("psycopg") is None + assert importlib.util.find_spec("psycopg_pool") is None + os.environ.pop("ADCP_PG_TEST_URL", None) + else: + assert os.environ.get("ADCP_PG_TEST_URL") + reference_inputs = evidence / (settings["label"] + "-historical-schema-inputs.json") + reference_bytes = (json.dumps(reference, indent=2, sort_keys=True) + "\n").encode() + reference_inputs.write_bytes(reference_bytes) + log = evidence / (settings["label"] + ".log") + recorder = Results() + command = [ + *(str(root / path) for path in settings["tests"]), + "-v", + "-s", + "-ra", + "-o", + "asyncio_mode=auto", + "-p", + "no:cacheprovider", + "--basetemp", + str(root / "temp"), + "--deselect=tests/test_reporting_capability_models.py::test_post_generation_repair_is_idempotent_for_both_actual_model_layouts", + ] + enter_phase("conformance") + started = time.monotonic() + with ( + log.open("x") as stream, + contextlib.redirect_stdout(stream), + contextlib.redirect_stderr(stream), + ): + code = int(pytest.main(command, plugins=[recorder])) + result = { + "command": command, + "pytest_exit": code, + "passed": recorder.passed, + "failed": recorder.failed, + "errors": recorder.errors, + "skipped": recorder.skipped, + "deselected": recorder.deselected, + "seconds": round(time.monotonic() - started, 3), + "log": str(log), + "bytes": log.stat().st_size, + "sha256": hashlib.sha256(log.read_bytes()).hexdigest(), + "valid": code == 0 and recorder.failed == recorder.errors == 0 and recorder.passed > 0, + } + if settings["driver_absent"]: + assert recorder.skipped > 0 + else: + result["valid"] &= recorder.skipped == 0 + result["valid"] &= recorder.deselected == 1 + config = root / "mypy.ini" + config.write_text( + "[mypy]\npython_version=3.10\nstrict=True\nplugins=adcp.types.mypy_plugin\nfollow_imports=silent\n" + ) + typing_command = [ + sys.executable, + "-I", + "-m", + "mypy", + "--config-file", + str(config), + "--strict", + "--no-incremental", + str(root / "adopter.py"), + ] + enter_phase("strict_adopter") + typed = subprocess.run(typing_command, cwd=root, capture_output=True, timeout=120) + typing_log = evidence / (settings["label"] + "-adopter.log") + typing_log.write_bytes(typed.stdout + typed.stderr) + result["valid"] &= typed.returncode == 0 + enter_phase("final_origins_and_reference_preservation") + for name, module in tuple(sys.modules.items()): + if (name == "adcp" or name.startswith("adcp.")) and getattr(module, "__file__", None): + assert Path(module.__file__).resolve().is_relative_to(Path(sys.prefix)) + assert reference_manifest() == reference["files"] + record = { + "python": sys.version, + "source_basis": settings["source_basis"], + "direct_url": json.loads( + importlib.metadata.distribution("adcp").read_text("direct_url.json") + ), + "origins": origins, + "distribution_version": importlib.metadata.version("adcp"), + "wheel_sha256": settings["wheel_sha256"], + "assets": settings["assets"], + "schemas": settings["schemas"], + "historical_reference_schema": { + "version": reference["version"], + "root": str(reference_root), + "origin": "copied immutable test reference; not a shipped SDK bundle", + "files": len(reference["files"]), + "inputs": str(reference_inputs), + "inputs_sha256": hashlib.sha256(reference_bytes).hexdigest(), + }, + "driver_absent": settings["driver_absent"], + "result": result, + "adopter": { + "command": typing_command, + "exit": typed.returncode, + "log": str(typing_log), + "bytes": typing_log.stat().st_size, + "sha256": hashlib.sha256(typing_log.read_bytes()).hexdigest(), + }, + } + (evidence / (settings["label"] + ".json")).write_text(json.dumps(record, indent=2) + "\n") + enter_phase("record_complete") + print(json.dumps(record), flush=True) + + +if __name__ == "__main__": + main(json.load(sys.stdin)) diff --git a/tests/conformance/reporting/_production_installed_process.py b/tests/conformance/reporting/_production_installed_process.py new file mode 100644 index 000000000..47ebbea16 --- /dev/null +++ b/tests/conformance/reporting/_production_installed_process.py @@ -0,0 +1,286 @@ +"""Installed activation, SIGKILL, and immutable legacy/new representation walks.""" + +import asyncio +import hashlib +import importlib +import json +import sys +from dataclasses import asdict +from importlib.resources import files +from pathlib import Path +from types import SimpleNamespace + + +async def main(settings): + root = Path(settings["fixtures"]) + assert not (root / "src").exists() and not (root / "adcp").exists() + sys.path.insert(0, str(root)) + assert sys.version_info[:2] == (3, 10) + origins = {} + for name, digest in settings["modules"].items(): + path = Path(importlib.import_module(name).__file__).resolve() + assert path.is_relative_to(Path(sys.prefix)) + assert hashlib.sha256(path.read_bytes()).hexdigest() == digest + origins[name] = str(path) + for name, digest in settings["assets"].items(): + assert ( + hashlib.sha256(files("adcp.reporting").joinpath(name).read_bytes()).hexdigest() + == digest + ) + + from psycopg_pool import AsyncConnectionPool + from pydantic import TypeAdapter + + from adcp.reporting.ledger import ReportingMaterializationAttempt + from adcp.reporting.ledger.delivery_models import ReportingDeliveryPrincipal + from adcp.reporting.ownership import page_revision_ownership + from tests.conformance.reporting._production_support import production_harness + from tests.conformance.reporting._production_transport import MountedProduction + from tests.conformance.reporting._projection_support import drain + from tests.conformance.reporting.test_reporting_production_lock_order import source_turn + + async with AsyncConnectionPool( + settings["conninfo"], kwargs=settings["kwargs"], min_size=2, max_size=4, open=False + ) as pool: + async with production_harness( + "postgres", + Path(settings["destination"]), + notifications=settings["notifications"], + count=0, + reconciled=True, + identity_prefix="b24-", + existing_pool=pool, + ) as h: + mount = MountedProduction(h) + subject = SimpleNamespace( + obligation=SimpleNamespace(account_id=settings["caller"]["account_id"]), + binding=SimpleNamespace(consumer_id=settings["caller"]["consumer_id"]), + ) + mount.authorize(subject) + historical = settings["historical_pending"] + original_attempt = TypeAdapter(ReportingMaterializationAttempt).validate_python( + historical["attempt"] + ) + original_id = original_attempt.reporting_materialization_id + + async def pending_identity(): + async with pool.connection() as c: + row = await ( + await c.execute( + "SELECT external_id,generation,admission_epoch,state" + " FROM reporting_materializer_work" + " WHERE account_id=%s AND consumer_id=%s" + " AND reporting_materialization_id=%s", + ( + original_attempt.scope.principal.account_id, + original_attempt.scope.consumer_id, + original_id, + ), + ) + ).fetchone() + assert row[:3] == ( + historical["external_id"], + historical["generation"], + historical["epoch"], + ) + records = await h.store.read_reconciliation_snapshot( + caller=original_attempt.scope.principal + ) + assert original_attempt in records.records + return row[3] + + if settings["pause"]: + assert await pending_identity() == "pending" + production_operation_1 = await h.production.activate(account_id="acct_a") + assert production_operation_1 + # Run the real producer's lease acquisition/release after + # activation while the actual parent's attempt is pending. + # The already-killed parent's lease deliberately spans setup. + # Persist its expiry as a bounded crash-recovery fault, without + # changing any original attempt, generation or external identity. + production_operation_2 = await source_turn(h.production) + assert (production_operation_2).leased is not None + async with pool.connection() as c, c.transaction(): + account = original_attempt.scope.principal.account_id + await h.store._lock_account(c, account) + await c.execute( + "UPDATE reporting_materializer_work SET lease_until=clock_timestamp()," + " due_at=clock_timestamp() WHERE account_id=%s AND consumer_id=%s" + " AND reporting_materialization_id=%s AND state='pending'", + (account, original_attempt.scope.consumer_id, original_id), + ) + await c.execute( + "UPDATE reporting_materializer_accounts SET due_at=clock_timestamp()" + " WHERE account_id=%s", + (account,), + ) + for _ in range(40): + turn = await h.production.materializer.run_once() + if turn.state == "verified": + assert turn.reporting_materialization_id == original_id + break + assert turn.state in {"idle", "discovered", "parked", "pending"} + await asyncio.sleep(0.05) + else: + raise AssertionError("historical pending work did not resume to verified") + assert h.item.writer.writes == 1 + assert await pending_identity() == "acked" + async with pool.connection() as c: + rows = await ( + await c.execute( + "SELECT (SELECT count(*)" + " FROM reporting_materializer_notification_events" + " WHERE reporting_materialization_id=%s AND admission_epoch=0)," + " (SELECT count(*)" + " FROM reporting_materializer_notification_expansions x" + " JOIN reporting_materializer_notification_events e" + " USING(account_id,consumer_namespace,notification_id)" + " WHERE e.reporting_materialization_id=%s AND x.state='quarantined')," + " (SELECT count(*) FROM reporting_production_notification_events" + " WHERE reporting_materialization_id=%s)", + (original_id,) * 3, + ) + ).fetchone() + assert rows == (int(settings["notifications"]), int(settings["notifications"]), 0) + await drain(h.projection, "acct_a") + else: + production_operation_3 = await h.production.activate(account_id="acct_a") + assert not production_operation_3 + assert await pending_identity() == "acked" + # A new eligible period was committed after both first pages. + # Finish it through bounded real turns if startup first handled + # another candidate. Neither snapshot may gain that membership. + if "new_revision_after_snapshot" in settings: + for _ in range(32): + outcomes = [ + r + for r in await h.item.outcomes() + if r.scope.generation_key == h.item.scope.generation_key + ] + if any( + r.reporting_revision_id == settings["new_revision_after_snapshot"] + and r.status == "delivered" + for r in outcomes + ): + break + await h.production.materializer.run_once() + else: + raise AssertionError("new eligible period did not complete after restart") + assert {r.reporting_revision_id for r in outcomes} == { + h.item.revision.reporting_revision_id, + settings["new_revision_after_snapshot"], + } + assert h.item.writer.writes == int("new_revision_after_snapshot" in settings) + caller = ReportingDeliveryPrincipal(**settings["caller"]) + query = { + "account": {"account_id": caller.account_id}, + "view": "periods", + "pagination": {"max_results": 1}, + } + + async def walk(client, request, transport): + pages = [] + request = json.loads(json.dumps(request)) + for _ in range(1000): + _, page = await mount.call( + client, "get_reporting_status", request, transport=transport + ) + assert "pagination" in page + pages.append(page) + if not page["pagination"]["has_more"]: + break + request["pagination"]["cursor"] = page["pagination"]["cursor"] + else: + raise AssertionError("installed cursor walk exceeded its bound") + assert len({p["changes_checkpoint"] for p in pages}) == 1 + snapshot = await h.store.read_reporting_feed_snapshot( + pages[0]["ledger_snapshot_id"], caller=caller + ) + return { + "pages": pages, + "binding": snapshot.binding, + "version": snapshot.representation_version, + "ownership_mode": snapshot.ownership_mode, + } + + async with mount.client() as client: + if settings["pause"]: + _, new_first = await mount.call(client, "get_reporting_status", query) + assert page_revision_ownership(new_first) is not None + expected_new = await walk( + client, + { + **query, + "pagination": { + "max_results": 1, + "cursor": new_first["pagination"]["cursor"], + }, + }, + "mcp", + ) + print( + json.dumps( + { + "point": "activated", + "origins": origins, + "first": new_first, + "new_remaining": expected_new, + "verification_key": asdict(h.item.verifier.key), + "pending_continuation": { + "state": "verified", + "epoch": 0, + "external_id": historical["external_id"], + "generation": historical["generation"], + "materialization_id": original_id, + "original_attempt_unchanged": True, + "original_quarantine_preserved": True, + "expiry_control": "persisted expiry after parent SIGKILL", + }, + } + ), + flush=True, + ) + await asyncio.to_thread(sys.stdin.readline) + raise AssertionError("activated process must be killed") + old_walks = [] + new_walks = [] + for transport in ("mcp", "a2a-0.3", "a2a-1.0"): + _, caps = await mount.call( + client, "get_adcp_capabilities", {}, transport=transport + ) + assert caps["media_buy"]["reporting_delivery"]["reconciled_billing"] is True + old_walks.append(await walk(client, settings["continuation"], transport)) + new_walks.append(await walk(client, settings["new_continuation"], transport)) + _, replay = await mount.call( + client, + "sync_reporting_receipts", + settings["receipt_request"], + transport=transport, + ) + assert replay == settings["receipt_response"] + assert old_walks[0] == old_walks[1] == old_walks[2] + assert new_walks[0] == new_walks[1] == new_walks[2] + assert all(page_revision_ownership(p) is None for p in old_walks[0]["pages"]) + assert all(page_revision_ownership(p) is not None for p in new_walks[0]["pages"]) + mount.grants.clear() + h.authorized_bindings.clear() + _, refused = await mount.call( + client, "get_reporting_status", settings["continuation"] + ) + assert "UNAUTHORIZED" in json.dumps(refused) + for name, module in tuple(sys.modules.items()): + if (name == "adcp" or name.startswith("adcp.")) and getattr( + module, "__file__", None + ): + assert Path(module.__file__).resolve().is_relative_to(Path(sys.prefix)) + return { + "point": "done", + "legacy": old_walks[0], + "new": new_walks[0], + "origins": origins, + "fresh_external_writes": h.item.writer.writes, + } + + +if __name__ == "__main__": + print(json.dumps(asyncio.run(main(json.loads(sys.stdin.readline())))), flush=True) diff --git a/tests/conformance/reporting/_production_legacy_process.py b/tests/conformance/reporting/_production_legacy_process.py new file mode 100644 index 000000000..2328e2ebb --- /dev/null +++ b/tests/conformance/reporting/_production_legacy_process.py @@ -0,0 +1,137 @@ +"""Executed by the real historical binary: incompatible C writers fail closed.""" + +import asyncio +import json +import sys +from dataclasses import asdict +from datetime import datetime, timezone +from pathlib import Path + + +async def main(settings): + import hashlib + + from psycopg_pool import AsyncConnectionPool + from pydantic import TypeAdapter + + import adcp.reporting.materializer.pg as materializer + import adcp.reporting.outbox.status_pg as implementation + from adcp.reporting.feed import PgReportingFeedStore + from adcp.reporting.materializer import ReportingVerificationKey + from adcp.reporting.materializer.work import ReportingMaterializerLease + from adcp.reporting.outbox.status_pg import PgStatusNotificationStore + + path = Path(implementation.__file__).resolve() + assert path.is_relative_to(Path(sys.prefix)) + assert hashlib.sha256(path.read_bytes()).hexdigest() == settings["module_sha256"] + materializer_path = Path(materializer.__file__).resolve() + assert materializer_path.is_relative_to(Path(sys.prefix)) + assert ( + hashlib.sha256(materializer_path.read_bytes()).hexdigest() + == settings["materializer_module_sha256"] + ) + async with AsyncConnectionPool( + settings["conninfo"], kwargs=settings["kwargs"], min_size=1, max_size=1, open=False + ) as pool: + store = PgReportingFeedStore( + pool=pool, + clock=lambda: datetime(2099, 1, 1, tzinfo=timezone.utc), + # This is an intentionally incompatible historical notification + # writer, even when the new deployment uses polling only. + notifications=settings["notifications"] if settings.get("pending") else True, + ) + key = TypeAdapter(ReportingVerificationKey).validate_python(settings["verification_key"]) + await store.materializer_ready() + if settings.get("pending"): + # The actual parent binary reserves on its original schema before + # any B2.4 migration. Its durable identity, not a reconstruction, + # must survive the process being killed and the child's activation. + for _ in range(16): + lease = await store.claim_materialization(keys=(key,), lease_seconds=300) + if isinstance(lease, ReportingMaterializerLease): + assert lease.attempt.reporting_revision_id == settings["revision_id"] + # The historical public lease has no epoch attribute. + # Read its real durable work row instead of assuming a + # child's newer dataclass shape or inventing a default. + async with pool.connection() as c: + identity = await ( + await c.execute( + "SELECT admission_epoch,generation,external_id" + " FROM reporting_materializer_work WHERE account_id=%s" + " AND consumer_id=%s AND reporting_materialization_id=%s", + ( + lease.scope.principal.account_id, + lease.scope.consumer_id, + lease.attempt.reporting_materialization_id, + ), + ) + ).fetchone() + assert identity == (0, lease.generation, lease.request.external_id) + return { + "point": "pending", + "attempt": TypeAdapter(type(lease.attempt)).dump_python( + lease.attempt, mode="json" + ), + "external_id": lease.request.external_id, + "generation": lease.generation, + "epoch": identity[0], + "lease_expires_at": lease.expires_at.isoformat(), + "scope": TypeAdapter(type(lease.scope)).dump_python( + lease.scope, mode="json" + ), + "verification_key": asdict(key), + "materializer_origin": str(materializer_path), + "origin": str(path), + } + raise AssertionError("actual parent did not reserve eligible pending work") + old = PgStatusNotificationStore(store) + checkpoints = await old.checkpoints(account_id="acct_a") + assert checkpoints + try: + async with old._transaction("acct_a") as connection: + await old._write_on(connection, checkpoints[0]) + except Exception as error: + assert getattr(error, "sqlstate", None) == "23514" + assert "status_projection_writer_fenced" in str(error) + else: + raise AssertionError("historical projector changed a v2 boundary") + try: + lease = await old.claim_due(account_id="acct_a") + except Exception as error: + # A legacy policy refusal or the actual row trigger is closed. + assert ( + getattr(error, "sqlstate", None) == "23514" + or getattr(error, "code", None) == "status_policy_conflict" + ) + sweeper = "fenced" + else: + assert lease is None + sweeper = "no_mutation" + try: + async with pool.connection() as connection, connection.transaction(): + await store._lock_account(connection, "acct_a") + # Execute the historical reserve algorithm, including its real + # discovery and selection. Keep the bounded turns in one + # transaction so the trigger's refusal proves full rollback. + for _ in range(16): + turn = await store._claim_account_on(connection, "acct_a", (key,), 30) + assert not hasattr(turn, "token"), "historical worker acquired new work" + raise AssertionError("historical reservation did not reach the production fence") + except Exception as error: + assert getattr(error, "sqlstate", None) == "23514" + assert "reporting_production_old_worker_fenced" in str(error) + return { + "historical_projection": "trigger_fenced", + "historical_sweeper": sweeper, + "historical_materializer": "reservation_trigger_fenced", + "materializer_origin": str(materializer_path), + "origin": str(path), + } + + +if __name__ == "__main__": + settings = json.loads(sys.stdin.readline()) + print(json.dumps(asyncio.run(main(settings))), flush=True) + if settings.get("pending"): + sys.stdin.readline() + raise AssertionError("historical pending worker must be killed") diff --git a/tests/conformance/reporting/_production_packaging.py b/tests/conformance/reporting/_production_packaging.py new file mode 100644 index 000000000..7cfd900f9 --- /dev/null +++ b/tests/conformance/reporting/_production_packaging.py @@ -0,0 +1,304 @@ +"""Build and inspect actual production distributions; fixtures never alias SDK source.""" + +import hashlib +import json +import os +import shutil +import subprocess +import tarfile +import zipfile +from pathlib import Path + +from .test_reporting_notification_packaging import ROOT, run_step + +ASSETS = ( + "ledger/reporting_materializer.sql", + "materializer/required_schema.json", + "ledger/reporting_receipt_ingestion.sql", + "receipts/required_schema.json", + "ledger/reporting_feed.sql", + "feed/required_schema.json", + "ledger/reporting_status_notifications.sql", + "outbox/required_status_schema.json", + "ledger/reporting_status_selector_version.sql", + "outbox/required_status_selector_schema.json", + "ledger/reporting_projection.sql", + "ledger/reporting_projection_notifications.sql", + "ledger/reporting_projection_feed.sql", + "projection/required_schema.json", + "ledger/reporting_production.sql", + "production/required_schema.json", +) +SCHEMAS = ( + "core/reporting-delivery-config-state.json", + "media-buy/get-reporting-status-response.json", + "bundled/media-buy/get-reporting-status-response.json", + "mcp/2026-07-28/profiles/production/media-buy/get-reporting-status-response.json", + "protocol/get-adcp-capabilities-response.json", + "bundled/protocol/get-adcp-capabilities-response.json", +) + + +def production_modules(): + paths = [ + p + for part in ("reporting/production", "reporting/projection") + for p in (ROOT / "src/adcp" / part).glob("*.py") + ] + paths += [ + ROOT / "src/adcp" / name + for name in ( + "reporting/ledger/status.py", + "reporting/ledger/status_snapshot.py", + "reporting/ledger/status_projection.py", + "reporting/ledger/reconciliation_projection.py", + "reporting/ledger/schedule.py", + "reporting/ledger/producer.py", + "reporting/ledger/producer_progress.py", + "reporting/materializer/memory.py", + "reporting/materializer/pg.py", + "reporting/materializer/publication.py", + "reporting/materializer/service.py", + "reporting/materializer/verification.py", + "reporting/feed/request.py", + "reporting/feed/errors.py", + "reporting/ledger/status_server.py", + "reporting/feed/snapshot.py", + "reporting/feed/projection.py", + "reporting/feed/memory.py", + "reporting/feed/pg.py", + "reporting/ownership.py", + "reporting/_timestamp.py", + "reporting/_reconcile.py", + "reporting/receipts/handler.py", + "reporting/outbox/memory.py", + "reporting/outbox/_activity_pg.py", + "reporting/outbox/worker.py", + "validation/schema_loader.py", + "types/base.py", + "types/generated_poc/core/reporting_delivery_capabilities.py", + "types/generated_poc/bundled/protocol/get_adcp_capabilities_response.py", + ) + ] + return { + "adcp." + + str(p.relative_to(ROOT / "src/adcp")) + .removesuffix(".py") + .replace("/", ".") + .removesuffix(".__init__"): hashlib.sha256(p.read_bytes()) + .hexdigest() + for p in paths + } + + +def inspect_distribution(wheel, source): + modules = production_modules() + assets = {name: (ROOT / "src/adcp/reporting" / name).read_bytes() for name in ASSETS} + with zipfile.ZipFile(wheel) as archive, tarfile.open(source) as sdist: + prefix = sdist.getnames()[0].split("/")[0] + for name, digest in modules.items(): + member = name.replace(".", "/") + ".py" + if member not in archive.namelist(): + member = name.replace(".", "/") + "/__init__.py" + raw = archive.read(member) + assert hashlib.sha256(raw).hexdigest() == digest + assert sdist.extractfile(f"{prefix}/src/{member}").read() == raw + for name, raw in assets.items(): + assert archive.read("adcp/reporting/" + name) == raw + assert sdist.extractfile(f"{prefix}/src/adcp/reporting/{name}").read() == raw + return modules, {name: hashlib.sha256(raw).hexdigest() for name, raw in assets.items()} + + +def copied_fixtures(root, label): + destination = root / ("production-" + label) + destination.mkdir(mode=0o700) + shutil.copytree( + ROOT / "tests", destination / "tests", ignore=shutil.ignore_patterns("__pycache__") + ) + shutil.copy2(ROOT / "examples/reporting_production.py", destination / "adopter.py") + assert not (destination / "adcp").exists() and not (destination / "src").exists() + return destination + + +def source_basis(wheel, source, *, evidence, label): + """Record actual build inputs; a dirty checkout's HEAD is lineage only.""" + + def git(*args): + return subprocess.check_output(["git", *args], cwd=ROOT) + + head, tree = (git("rev-parse", value).decode().strip() for value in ("HEAD", "HEAD^{tree}")) + dirty = bool(git("status", "--porcelain=v1", "-uall")) + members = [] + with tarfile.open(source) as archive: + prefix = archive.getnames()[0].split("/")[0] + "/" + for member in archive.getmembers(): + if not member.isfile(): + continue + relative = Path(member.name.removeprefix(prefix)) + if any(p.endswith(".egg-info") for p in relative.parts): + continue + path = ROOT / relative + if not path.is_file(): + continue # Generated sdist metadata is identified by its archive digest. + raw = archive.extractfile(member).read() + assert path.read_bytes() == raw, relative + members.append( + { + "path": str(relative), + "bytes": len(raw), + "sha256": hashlib.sha256(raw).hexdigest(), + } + ) + raw_manifest = (json.dumps(sorted(members, key=lambda r: r["path"]), indent=2) + "\n").encode() + evidence.mkdir(parents=True, exist_ok=True, mode=0o700) + manifest = evidence / (label + "-build-inputs.json") + with manifest.open("xb") as stream: + stream.write(raw_manifest) + basis = { + "kind": "development-export" if dirty else "git-commit", + "head": head, + "tree": tree, + "clean": not dirty, + "head_role": "parent lineage only" if dirty else "actual built commit", + "build_input_manifest": str(manifest), + "build_input_count": len(members), + "build_input_manifest_sha256": hashlib.sha256(raw_manifest).hexdigest(), + "sdist_sha256": hashlib.sha256(source.read_bytes()).hexdigest(), + "wheel_sha256": hashlib.sha256(wheel.read_bytes()).hexdigest(), + } + if dirty: + patch = git("diff", "--binary", "HEAD") + patch_file = evidence / (label + "-development.patch") + with patch_file.open("xb") as stream: + stream.write(patch) + basis["patch_sha256"] = hashlib.sha256(patch).hexdigest() + basis["patch"] = str(patch_file) + new_files = { + p.decode(): hashlib.sha256((ROOT / p.decode()).read_bytes()).hexdigest() + for p in git("ls-files", "--others", "--exclude-standard", "-z").split(b"\0") + if p and (ROOT / p.decode()).is_file() + } + new_manifest = evidence / (label + "-development-new-files.json") + raw_new = (json.dumps(new_files, indent=2, sort_keys=True) + "\n").encode() + with new_manifest.open("xb") as stream: + stream.write(raw_new) + basis["new_files_manifest"] = str(new_manifest) + basis["new_files_manifest_sha256"] = hashlib.sha256(raw_new).hexdigest() + if os.environ.get("ADCP_PRODUCTION_EVIDENCE"): + retained = evidence / (label + "-artifacts") + retained.mkdir(mode=0o700) + for path in (wheel, source): + shutil.copy2(path, retained / path.name) + basis["retained_artifacts"] = str(retained) + return basis + + +def historical_schema_fixture(root): + """Copy immutable rc.3 reference inputs, never claim them as wheel contents. + + The installed suite retains historical rejection and correction controls. + rc.3 is no longer a shipped bundle. The existing source-layout fallback + can read these explicit test inputs without changing installed SDK code, + its current bundle, or the public protocol-version allowlist. + """ + version = "3.2.0-rc.3" + source = ROOT / "schemas/cache" / version + destination = root / "schemas/cache" / version + assert not root.resolve().is_relative_to(ROOT.resolve()) + expected = { + str(p.relative_to(source)): hashlib.sha256(p.read_bytes()).hexdigest() + for p in sorted(source.rglob("*.json")) + } + assert expected + if not destination.exists(): + shutil.copytree(source, destination) + actual = { + str(p.relative_to(destination)): hashlib.sha256(p.read_bytes()).hexdigest() + for p in sorted(destination.rglob("*.json")) + } + assert actual == expected + return {"version": version, "root": str(destination), "files": expected} + + +def installed_production(root, python, wheel, source, *, label, driver_absent): + fixture_root = copied_fixtures(root, label) + script = fixture_root / "run_installed.py" + shutil.copy2(Path(__file__).with_name("_production_installed.py"), script) + installer = ( + [shutil.which("uv"), "pip", "install", "--python", str(python)] + if shutil.which("uv") + else [str(python), "-m", "pip", "install"] + ) + run_step( + [ + *installer, + "pytest==9.1.1", + "pytest-asyncio==1.4.0", + "respx==0.23.1", + "asgi-lifespan==2.1.0", + "mypy==1.20.2", + ], + label=label + "-production-conformance-dependencies", + cwd=root, + timeout=180, + ) + modules, assets = inspect_distribution(wheel, source) + tests = sorted( + { + *ROOT.glob("tests/conformance/reporting/test_reporting_production*.py"), + *ROOT.glob("tests/conformance/reporting/test_reporting_projection*.py"), + } + ) + tests = [ + p + for p in tests + if p.name + not in {"test_reporting_production_packaging.py", "test_reporting_production_rolling.py"} + ] + tests += [ + ROOT / name + for name in ( + "tests/conformance/reporting/test_reporting_tier_projection.py", + "tests/conformance/reporting/test_reporting_schedule_schema.py", + "tests/test_reporting_revision_ownership.py", + "tests/test_reporting_capability_models.py", + "tests/test_reporting_production_public.py", + "tests/test_schema_datetime_formats.py", + ) + ] + evidence = Path(os.environ.get("ADCP_PRODUCTION_EVIDENCE", str(root / "production-evidence"))) + settings = { + "workspace": str(ROOT), + "fixtures": str(fixture_root), + "label": label, + "modules": modules, + "assets": assets, + "schemas": { + version: { + name: hashlib.sha256( + (ROOT / "schemas/cache" / version / name).read_bytes() + ).hexdigest() + for name in SCHEMAS + } + for version in ("3.2.0-rc.6",) + }, + "historical_reference_schema": historical_schema_fixture(root), + "tests": [str(p.relative_to(ROOT)) for p in tests], + "driver_absent": driver_absent, + "python": [3, 10], + "source_basis": source_basis(wheel, source, evidence=evidence, label=label), + "wheel_sha256": hashlib.sha256(wheel.read_bytes()).hexdigest(), + "evidence": str(evidence), + } + result = json.loads( + run_step( + [str(python), "-I", str(script)], + label=label + "-installed-production", + cwd=fixture_root, + value=settings, + timeout=1800, + ) + ) + print(json.dumps({"installed_production": label, **result}), flush=True) + assert result["result"]["valid"], result["result"] + return result diff --git a/tests/conformance/reporting/_production_progress_process.py b/tests/conformance/reporting/_production_progress_process.py new file mode 100644 index 000000000..df5c93eca --- /dev/null +++ b/tests/conformance/reporting/_production_progress_process.py @@ -0,0 +1,57 @@ +"""Cold producer restart: committed bounded window, retained lease, then SIGKILL.""" + +import asyncio +import json +import sys +from datetime import timedelta +from pathlib import Path + + +async def main(): + from psycopg_pool import AsyncConnectionPool + + from ._production_support import production_harness + from .test_reporting_production_lock_order import source_turn + from .test_reporting_production_progress import turn_document + + settings = json.loads(await asyncio.to_thread(sys.stdin.readline)) + async with AsyncConnectionPool( + settings["conninfo"], kwargs=settings["kwargs"], min_size=2, max_size=4, open=False + ) as pool: + async with production_harness( + "postgres", Path(settings["path"]), count=0, periods=131, existing_pool=pool + ) as h: + support = h.production + producer = support.offerings[0].producer + source = producer._source + h.source_clock.advance(timedelta(hours=131, seconds=0 if settings["pause"] else 61)) + if settings["pause"]: + original = producer._acquire_pending + + async def pause(configuration, turn, *, now): + await original(configuration, turn, now=now) + print( + json.dumps( + {"point": "committed_before_release", **turn_document(turn, source)} + ), + flush=True, + ) + # The parent observes the real committed cursor and live + # configuration lease, then kills this process at this edge. + await asyncio.to_thread(sys.stdin.readline) + raise AssertionError("paused child must be killed") + + producer._acquire_pending = pause + turn = await source_turn(support) + result = turn_document(turn, source) + repeated = await source_turn(support) + assert not repeated.obligations_committed and not repeated.revisions_committed + assert result["executions"] == [ + r.identity.source_execution_key for r in source.requests + ] + await support.aclose() + print(json.dumps({"point": "done", **result}), flush=True) + + +if __name__ == "__main__": + asyncio.run(main()) diff --git a/tests/conformance/reporting/_production_support.py b/tests/conformance/reporting/_production_support.py new file mode 100644 index 000000000..c6e78668d --- /dev/null +++ b/tests/conformance/reporting/_production_support.py @@ -0,0 +1,643 @@ +"""Real independent SQLite destination, mounted SDK composition, shared store vectors. + +The destination is a test provider, not the development writer with a changed +eligibility flag. Its immutable rows and grants survive a new provider process. +""" + +import hashlib +import json +import sqlite3 +from contextlib import asynccontextmanager +from dataclasses import asdict, replace +from datetime import datetime, timedelta, timezone + +from adcp.decisioning.capabilities import Account as AccountCapabilities +from adcp.reporting.fixtures import redacted_capabilities +from adcp.reporting.inline_source import ( + FileSystemStagingStore, + InlineFetchResult, + InlineReportingSource, + SealedSlice, +) +from adcp.reporting.ledger import ReportingRevisionRecord, revision_content_sha256 +from adcp.reporting.ledger.delivery_models import ( + ReportingDestinationBinding, + ReportingResourceRecord, +) +from adcp.reporting.ledger.models import ReportingDeliveryEscalation +from adcp.reporting.ledger.producer import ProducerOfferings, ReportingProducer +from adcp.reporting.materializer import ( + ReportingDestinationIO, + ReportingRevisionVerifierRegistry, + reference_digest, + reference_verifier, +) +from adcp.reporting.materializer.contracts import ( + ReportingDestinationLocator, + ReportingDestinationPage, + ReportingDestinationSession, + ReportingWriterCapability, + binding_fingerprint, + failure, +) +from adcp.reporting.materializer.service import ReportingMaterializerService +from adcp.reporting.production.configuration import ReportingProductionConfigurationTask +from adcp.reporting.production.contracts import ( + ReportingProductionDestinationBinding, + ReportingProductionMethod, + ReportingProductionSourceBinding, +) +from adcp.reporting.production.memory import InMemoryReportingProductionStore +from adcp.reporting.production.offerings import ReportingProductionOffering +from adcp.reporting.production.service import ReportingProductionSupport +from adcp.reporting.projection.memory import InMemoryReportingStatusProjection +from adcp.reporting.receipts.errors import ReportingReceiptError +from adcp.reporting.source import ( + ReportingSourceCapabilitiesV1, + SourceBatchManifestReferenceV1, + reporting_source_capabilities_sha256_v1, +) +from adcp.server.serve import create_mcp_server +from adcp.types import ReportingDeliveryOffering + +from ._durable_materializer_support import DurableCase, DurableHarness +from ._generation_support import END, START, configuration, isolated_reporting_pool, obligation_for +from ._materializer_support import reference_rows +from ._reliable_support import ManualClock + + +class SQLiteDestination: + production_eligible = True + resource_retention_days = 400 + authorization_revocation_seconds = 0 + + def __init__(self, path, key): + self.path, self.key = path, key + self.capabilities = (key.capability,) + self.delivery_methods = ( + ReportingProductionMethod( + key.capability, + { + "pattern": key.capability.method, + "transport": key.capability.transport, + "format": key.capability.format, + "provider": {"domain": "fixture.example.test"}, + "orchestration": "producer_managed", + "destination_modes": ["provision"], + "reader_compatibility": ["fixture-sql-v1"], + }, + ), + ) + self.opens = self.closes = self.writes = 0 + with sqlite3.connect(path) as c: + c.execute("CREATE TABLE IF NOT EXISTS grants (binding TEXT PRIMARY KEY)") + c.execute( + "CREATE TABLE IF NOT EXISTS methods" + " (binding TEXT PRIMARY KEY, method TEXT NOT NULL)" + ) + c.execute( + "CREATE TABLE IF NOT EXISTS artifacts (id TEXT PRIMARY KEY, content TEXT NOT NULL)" + ) + + def grant(self, binding): + with sqlite3.connect(self.path) as c: + c.execute("INSERT OR IGNORE INTO grants VALUES (?)", (binding_fingerprint(binding),)) + c.execute( + "INSERT OR IGNORE INTO methods VALUES (?,?)", + ( + binding_fingerprint(binding), + json.dumps( + { + "pattern": binding.method, + "transport": binding.transport, + "orchestration": "producer_managed", + "destination": { + "mode": "provision", + "provider": {"domain": "fixture.example.test"}, + "location": "reporting/" + binding.destination_ref, + }, + } + ), + ), + ) + + def configuration_binding(self, binding): + with sqlite3.connect(self.path) as c: + row = c.execute( + "SELECT method FROM methods JOIN grants USING(binding) WHERE binding=?", + (binding_fingerprint(binding),), + ).fetchone() + if row is None: + return None + return ReportingProductionDestinationBinding( + binding, self.delivery_methods[0], json.loads(row[0]) + ) + + def revoke(self, binding): + with sqlite3.connect(self.path) as c: + c.execute("DELETE FROM grants WHERE binding=?", (binding_fingerprint(binding),)) + + def resolve(self, request, *, phase, context): + return _SQLiteSession(self, request, phase, context) + + +class _SQLiteSession(ReportingDestinationSession): + def __init__(self, provider, request, phase, context): + super().__init__(request, phase, context) + self.provider, self.connection = provider, None + + async def _open(self): + p = self.provider + self.connection = sqlite3.connect(p.path) + p.opens += 1 + if ( + self.request.verification_key != p.key + or self.connection.execute( + "SELECT 1 FROM grants WHERE binding=?", (self.request.binding_fingerprint,) + ).fetchone() + is None + ): + raise failure("AUTHORIZATION_DENIED") + + async def _close(self): + if self.connection is not None: + self.connection.close() + self.connection = None + self.provider.closes += 1 + + async def write(self, content): + assert self.phase == "write" and self.connection is not None + r = self.request + path = "fixture/" + hashlib.sha256(r.external_id.encode()).hexdigest() + row = self.connection.execute( + "SELECT content FROM artifacts WHERE id=?", (r.external_id,) + ).fetchone() + if row: + data = json.loads(row[0]) + if data["rows"] != [v.decode() for v in content.rows]: + raise failure("WRITE_FAILED") + else: + resource = ReportingResourceRecord( + "fixture_" + hashlib.sha256(r.external_id.encode()).hexdigest(), + "warehouse_relation", + path + "/table", + "immutable_location", + datetime.now(timezone.utc) + + timedelta(days=self.provider.resource_retention_days + 1), + reader_compatibility=content.binding.reader_compatibility, + ) + data = { + "rows": [v.decode() for v in content.rows], + "resource": asdict(resource), + "binding": r.binding_fingerprint, + "revision": r.reporting_revision_id, + } + data["resource"]["expires_at"] = resource.expires_at.isoformat() + self.connection.execute( + "INSERT INTO artifacts VALUES (?,?)", (r.external_id, json.dumps(data)) + ) + self.connection.commit() + self.provider.writes += 1 + resource = dict(data["resource"]) + resource["expires_at"] = datetime.fromisoformat(resource["expires_at"]) + resource["object_refs"] = tuple(resource["object_refs"]) + resource["reader_compatibility"] = tuple(resource["reader_compatibility"]) + return ReportingDestinationLocator( + r.external_id, r.binding_fingerprint, ReportingResourceRecord(**resource) + ) + + async def read_rows(self, locator, *, cursor, limit): + assert self.phase == "readback" and self.connection is not None + raw = self.connection.execute( + "SELECT content FROM artifacts WHERE id=?", (locator.external_id,) + ).fetchone() + if raw is None or locator.external_id != self.request.external_id: + raise failure("RESOURCE_UNAVAILABLE") + data = json.loads(raw[0]) + if data["binding"] != self.request.binding_fingerprint: + raise failure("AUTHORIZATION_DENIED") + offset = 0 if cursor is None else int(cursor) + rows = tuple(v.encode() for v in data["rows"][offset : offset + limit]) + following = offset + len(rows) + more = following < len(data["rows"]) + return ReportingDestinationPage( + data["revision"], + rows, + len(data["rows"]), + more, + str(following) if more else None, + self.request.verification_key.capability.format, + "destination", + ) + + +class SQLiteSeals: + def __init__(self, path): + self.path = path + with sqlite3.connect(path) as c: + c.execute( + "CREATE TABLE IF NOT EXISTS seals (account TEXT, execution TEXT," + " reference TEXT NOT NULL, manifest BLOB NOT NULL, PRIMARY KEY(account,execution))" + ) + + async def get(self, *, account_id, source_execution_key): + with sqlite3.connect(self.path) as c: + row = c.execute( + "SELECT reference,manifest FROM seals WHERE account=? AND execution=?", + (account_id, source_execution_key), + ).fetchone() + return ( + None + if row is None + else SealedSlice(SourceBatchManifestReferenceV1.model_validate_json(row[0]), row[1]) + ) + + async def put(self, *, account_id, source_execution_key, sealed): + with sqlite3.connect(self.path) as c: + c.execute( + "INSERT OR IGNORE INTO seals VALUES (?,?,?,?)", + ( + account_id, + source_execution_key, + sealed.reference.model_dump_json(), + sealed.manifest_bytes, + ), + ) + return await self.get(account_id=account_id, source_execution_key=source_execution_key) + + +class Source: + def __init__( + self, + key, + path, + rows=None, + *, + clock=None, + product_ids=("catalog-7391", "catalog-5820"), + official=False, + ): + raw = redacted_capabilities().model_dump(mode="json") + raw["offerings"] = [raw["offerings"][int(official)]] + self.source_id = raw["offerings"][0]["offering_id"] + if official: + raw["offerings"][0].update( + grain="fixed_window", + source_timezone="UTC", + source_local_ready_time="00:00", + days_after_period_end=0, + expected_availability_lag="PT0S", + windowing={ + "kind": "fixed_closed_window", + "minimum_window": "PT1H", + "maximum_window": "P1D", + "overlapping_windows_supported": False, + }, + ) + raw["offerings"][0]["product_ids"] = list(product_ids) + raw["offerings"][0]["contract"] = { + "report_definition_id": key.report_definition_id, + "reporting_profile": key.reporting_profile, + **{ + k: getattr(key.definition, k) + for k in ( + "report_definition_uri", + "report_definition_sha256", + "schema_version", + "schema_uri", + "schema_sha256", + "schema_dialect", + "schema_ref_policy", + ) + }, + } + raw["offerings"][0]["worst_case_availability_lag"] = "PT1H" + raw["capabilities_sha256"] = reporting_source_capabilities_sha256_v1(raw) + self.capabilities = ReportingSourceCapabilitiesV1.model_validate(raw) + self.bindings = {} + self.rows, self.requests = rows, [] + self.inline = InlineReportingSource( + capabilities=self.capabilities, + fetch=self.fetch, + staging=FileSystemStagingStore(path.with_suffix(".staging")), + seals=SQLiteSeals(path.with_suffix(".seals")), + constituent_of=lambda row, req: req.coverage.constituents[0].constituent_id, + clock=clock or (lambda: END), + ) + self.reader = self.inline.staging + + def bind_generation(self, configuration, *, product_id="catalog-7391"): + binding = ReportingProductionSourceBinding.for_configuration( + configuration, + capabilities_sha256=self.capabilities.capabilities_sha256, + media_buy_products=tuple( + (media_buy_id, product_id) for media_buy_id in configuration.media_buy_ids + ), + ) + self.bindings[configuration.generation_key] = binding + return binding + + def configuration_binding(self, configuration): + return self.bindings.get(configuration.generation_key) + + async def fetch(self, request): + assert self.rows is not None, "this seeded publication needs no new acquisition" + self.requests.append(request) + return InlineFetchResult(self.rows, data_through=request.period.end, currency="USD") + + async def execute(self, request, *, cancel, heartbeat=None): + return await self.inline.execute(request, cancel=cancel, heartbeat=heartbeat) + + +async def account_task(request, context, admit): + return {"accounts": []} + + +@asynccontextmanager +async def production_harness( + backend, + path, + *, + notifications=False, + count=503, + second_source=False, + source_publication=False, + notification_delivery=False, + periods=None, + existing_store=None, + existing_pool=None, + source_bindings=(), + account_handler=None, + reconciled=False, + feedback=False, + identity_prefix="", + poll_seconds=60, + source_factory=Source, + adcp_version=None, +): + from contextlib import AsyncExitStack + + async with AsyncExitStack() as stack: + clock = ManualClock(datetime.now(timezone.utc)) + pool = existing_pool + if existing_store is not None: + store = existing_store + clock = store._clock + elif backend == "memory": + store = InMemoryReportingProductionStore(clock=clock, notifications=notifications) + else: + from adcp.reporting.production.pg import PgReportingProductionStore + + if pool is None: + pool = await stack.enter_async_context(isolated_reporting_pool(autocommit=True)) + store = PgReportingProductionStore(pool=pool, notifications=notifications) + await store.create_schema() + h = DurableHarness(store, clock, pool) + cap = ReportingWriterCapability( + "warehouse_materialization", + "fixture-sql", + "jsonl", + "canonical_digest", + "destination", + "immutable_location", + "sha256", + "conditional_create", + ) + verifier = reference_verifier(cap) + key = verifier.key + config = replace( + configuration(), + delivery_config_id=identity_prefix + configuration().delivery_config_id, + deactivated_at=None if periods is None else START + timedelta(hours=periods), + definition=key.definition, + report_definition_id=key.report_definition_id, + feed_purpose="billing" if reconciled else "analytics", + required_finality="official" if reconciled else "snapshot", + ) + await store.put_configuration(config) + original_obligation = obligation_for(config) + obligation = await store.commit_obligation( + replace( + original_obligation, + reporting_obligation_id=identity_prefix + + original_obligation.reporting_obligation_id, + ) + ) + binding = ReportingDestinationBinding( + config.generation_key, + "https://buyer.example.test/agent", + "destination", + "trusted-provider-binding", + cap.method, + cap.transport, + cap.verification_profile, + "consumer_receipt" if reconciled else "delivery_only", + config.feed_purpose, + 400, + START, + cap.format, + ("fixture-sql-v1",), + "delivered", + ) + await store.put_destination_binding(binding) + rows = reference_rows(count) + _, totals = verifier.canonicalize(rows) + pairs = tuple((t.name, t.value) for t in totals) + revision = ReportingRevisionRecord( + identity_prefix + "production-revision", + config.account_id, + obligation.reporting_obligation_id, + config.required_finality, + revision_content_sha256( + reporting_revision_id=identity_prefix + "production-revision", + row_count=count, + control_totals=pairs, + reporting_rows=rows, + control_total_evidence=totals, + ), + count, + pairs, + END, + END, + END, + finality_basis="source_final" if reconciled else None, + finality_policy_id="fixture-official-v1" if reconciled else None, + finalized_at=END if reconciled else None, + canonical_content_digest=reference_digest(verifier, rows), + managed_control_totals=totals, + ) + if not source_publication: + await store.commit_revision(revision, rows) + writer = SQLiteDestination(path, key) + writer.grant(binding) + registry = ReportingRevisionVerifierRegistry((verifier,)) + io = ReportingDestinationIO(registry, writer) + item = DurableCase( + store, + config, + obligation, + binding, + revision, + rows, + verifier, + registry, + writer, + writer, + io, + ) + source_clock = ManualClock(END) + source = source_factory( + key, + path.with_name("source"), + rows if source_publication or periods is not None else None, + clock=source_clock, + official=reconciled, + ) + if not second_source: + source.bind_generation(config) + escalation = ReportingDeliveryEscalation() + producer = ReportingProducer( + source=source, + offerings=ProducerOfferings( + snapshot_offering_id=None if reconciled else source.source_id, + official_offering_id=( + source.source_id if reconciled else getattr(source, "official_source_id", None) + ), + publication_namespace=source.capabilities.offerings[0].publication_namespace, + source_scope=source.capabilities.source_scope, + ), + store=store, + escalation=escalation, + clock=source_clock, + revision_verifier=verifier, + object_reader=source.reader, + ) + if pool is None: + projection = InMemoryReportingStatusProjection( + store, + revision_ownership=True, + escalation=escalation, + consumer_status_enabled=feedback, + ) + else: + from adcp.reporting.projection.pg import PgReportingStatusProjection + + projection = PgReportingStatusProjection( + store, + revision_ownership=True, + escalation=escalation, + consumer_status_enabled=feedback, + ) + profile = { + "id": key.reporting_profile, + "version": key.definition.schema_version, + "schema_uri": key.definition.schema_uri, + "schema_sha256": key.definition.schema_sha256, + "schema_dialect": key.definition.schema_dialect, + "schema_ref_policy": key.definition.schema_ref_policy, + "grain": "row", + "primary_keys": ["row_id"], + "canonicalization_id": key.canonicalization.canonicalization_id, + "canonicalization_uri": key.canonicalization.canonicalization_uri, + "canonicalization_sha256": key.canonicalization.canonicalization_sha256, + } + offering = ReportingDeliveryOffering.model_validate( + { + "offering_id": "reconciled-fixture" if reconciled else "managed-fixture", + "feed_purpose": config.feed_purpose, + "report_definition_id": key.report_definition_id, + "report_definition_uri": key.definition.report_definition_uri, + "report_definition_sha256": key.definition.report_definition_sha256, + "reporting_profile": profile, + "schedule": { + "period_duration": "PT1H", + "alignment": "utc", + "delivery_sla": "PT1H", + }, + "supported_finality": [config.required_finality], + "reconciliation_mode": binding.reconciliation_mode, + "method": writer.delivery_methods[0].wire(), + } + ) + admitted = ReportingProductionOffering(offering, producer, key, source.source_id) + offerings = (admitted,) + if second_source: + other = Source(key, path.with_name("other-source")) + other_producer = ReportingProducer( + source=other, + offerings=producer._offerings, + store=store, + escalation=escalation, + clock=lambda: END, + revision_verifier=verifier, + object_reader=other.reader, + ) + offerings += ( + ReportingProductionOffering( + offering.model_copy(update={"offering_id": "managed-other"}), + other_producer, + key, + other.source_id, + ), + ) + + for configured, offering_id, product_id in source_bindings: + selected = next(o for o in offerings if o.offering_id == offering_id) + selected.producer._source.bind_generation(configured, product_id=product_id) + + h.authorized_bindings = {(config.account_id, binding.consumer_id)} + + async def authorize(account, context, consumer): + account_id = account.get("account_id") + if ( + account != {"account_id": account_id} + or (account_id, consumer) not in h.authorized_bindings + ): + raise ReportingReceiptError("UNAUTHORIZED") + return account_id + + workers = () + if notification_delivery: + from adcp.reporting.outbox.routing import ReportingEnvelopeCipher + from adcp.reporting.production.notifications import ( + ReportingProductionSigning, + production_notification_workers, + ) + + from ._reliable_support import FailurePlan, ScriptedSigning, ScriptedSubscriptions + + h.notification_failures = FailurePlan() + h.subscriptions = ScriptedSubscriptions(h.notification_failures) + h.signing = ScriptedSigning(h.notification_failures) + workers = production_notification_workers( + store, + projection, + subscriptions=h.subscriptions, + signing=ReportingProductionSigning( + h.signing, + ("ed25519",), + brand_json_url="https://seller.example.test/brand.json", + ), + cipher=ReportingEnvelopeCipher(b"b" * 32), + ) + support = ReportingProductionSupport( + ReportingMaterializerService(store, io, writer), + projection, + offerings=offerings, + configuration_task=ReportingProductionConfigurationTask( + account_handler or account_task, + AccountCapabilities(supported_billing=["operator"], require_operator_auth=True), + ), + resolve_account=authorize, + notification_workers=workers, + poll_seconds=poll_seconds, + adcp_version=adcp_version, + ) + h.production, h.projection, h.item = support, projection, item + h.source_clock = source_clock + h.mount = create_mcp_server(support.handler) + try: + await support.start() + yield h + finally: + await support.aclose() diff --git a/tests/conformance/reporting/_production_transport.py b/tests/conformance/reporting/_production_transport.py new file mode 100644 index 000000000..f9789ad6f --- /dev/null +++ b/tests/conformance/reporting/_production_transport.py @@ -0,0 +1,36 @@ +"""The actual production handler on authenticated MCP and both A2A mounts.""" + +import json + +from ._receipt_transport import MountedReceipts + + +class MountedProduction(MountedReceipts): + def __init__(self, h): + super().__init__(h) + self.handler = h.production.handler + self.version = self.handler.get_adcp_version() + + def authorize(self, s, *, token="token-one"): + super().authorize(s, token=token) + self.h.authorized_bindings.add((s.obligation.account_id, s.binding.consumer_id)) + + async def middleware(self, name, params, context, call_next): + return await call_next() + + async def call(self, client, task, request, *, transport="mcp", token="token-one"): + request = {"adcp_version": self.version, **request} + + def route(wire): + value = json.loads(wire) + if transport == "mcp": + value["params"]["name"] = task + else: + value["params"]["message"]["parts"][0]["data"]["skill"] = task + return json.dumps(value) + + if transport == "mcp": + return await self.mcp(client, request, token=token, mutate_wire=route) + return await self.a2a( + client, request, token=token, mutate_wire=route, v1=transport == "a2a-1.0" + ) diff --git a/tests/conformance/reporting/_projection_support.py b/tests/conformance/reporting/_projection_support.py new file mode 100644 index 000000000..5feeb53a5 --- /dev/null +++ b/tests/conformance/reporting/_projection_support.py @@ -0,0 +1,75 @@ +"""B2.4 shared execution: actual stores, no forged readiness certificate.""" + +from contextlib import asynccontextmanager + +import pytest + +from adcp.reporting.projection.memory import ( + InMemoryReportingProjectionStore, + InMemoryReportingStatusProjection, +) + +from ._durable_materializer_support import DurableHarness +from ._generation_support import isolated_reporting_pool +from ._reconciliation_support import Clock + + +@asynccontextmanager +async def projection_harness(backend, *, notifications=False, feedback=False, ownership=True): + clock = Clock() + if backend == "memory": + store = InMemoryReportingProjectionStore(clock=clock, notifications=notifications) + h = DurableHarness(store, clock) + h.projection = InMemoryReportingStatusProjection( + store, consumer_status_enabled=feedback, revision_ownership=ownership + ) + yield h + else: + from adcp.reporting.projection.pg import ( + PgReportingProjectionStore, + PgReportingStatusProjection, + ) + + async with isolated_reporting_pool(autocommit=True) as pool: + store = PgReportingProjectionStore(pool=pool, notifications=notifications) + await store.create_schema() + h = DurableHarness(store, clock, pool) + h.projection = PgReportingStatusProjection( + store, consumer_status_enabled=feedback, revision_ownership=ownership + ) + yield h + + +@pytest.fixture( + params=[("memory", False), ("memory", True), ("postgres", False), ("postgres", True)] +) +async def projections(request): + backend, notifications = request.param + async with projection_harness(backend, notifications=notifications) as h: + yield h + + +async def drain(projection, account_id): + turns = [] + for _ in range(100): + result = await projection.project_one(account_id=account_id) + if not result.did_work: + return turns + turns.append(result) + raise AssertionError("projection failed to converge") + + +async def inputs(h, account_id): + if h.pool is None: + return tuple(h.store._projection_accounts[account_id].inputs) + from adcp.reporting.projection.capture import decode_projection_input + + async with h.pool.connection() as c: + rows = await ( + await c.execute( + "SELECT input FROM reporting_projection_inputs" + " WHERE account_id=%s ORDER BY sequence", + (account_id,), + ) + ).fetchall() + return tuple(decode_projection_input(r[0]) for r in rows) diff --git a/tests/conformance/reporting/_receipt_transport.py b/tests/conformance/reporting/_receipt_transport.py index 1b25cb4f7..bf2e41376 100644 --- a/tests/conformance/reporting/_receipt_transport.py +++ b/tests/conformance/reporting/_receipt_transport.py @@ -54,6 +54,9 @@ def __init__(self, h, *, hydrated=False, registry_kind=None, version=None): self.registry = Registry() if registry_kind is not None else None self.sessions = {} self.counter = 0 + from adcp._version import normalize_to_release_precision, resolve_adcp_version + + self.version = normalize_to_release_precision(version or resolve_adcp_version(None)) self.idempotency = IdempotencyStore(backend=ForbiddenGenericCache()) self.handler = ReportingReceiptHandler( h.store, resolve_account=self.resolve_account, buyer_agents=self.registry @@ -63,8 +66,7 @@ def __init__(self, h, *, hydrated=False, registry_kind=None, version=None): self.handler.sync_reporting_receipts = self.idempotency.wrap( self.handler.sync_reporting_receipts ) - if version is not None: - self.handler.adcp_version = version + self.handler.get_adcp_version = lambda: self.version def authorize(self, s, *, token="token-one"): account, consumer = s.obligation.account_id, s.binding.consumer_id diff --git a/tests/conformance/reporting/test_reporting_feed_rolling.py b/tests/conformance/reporting/test_reporting_feed_rolling.py index 4c946b13d..b4bba4b57 100644 --- a/tests/conformance/reporting/test_reporting_feed_rolling.py +++ b/tests/conformance/reporting/test_reporting_feed_rolling.py @@ -19,6 +19,8 @@ from adcp.reporting.ledger.delivery import receipt_to_wire from adcp.reporting.materializer import PgReportingMaterializerStore from adcp.reporting.outbox._schema import schema_objects +from adcp.reporting.production.pg import PgReportingProductionStore +from adcp.reporting.production.schema import validate_production_schema from adcp.reporting.receipts import PgReportingReceiptStore from ._durable_materializer_support import DurableHarness, durable_case @@ -250,6 +252,17 @@ async def test_nine_actual_artifacts_preserve_ordinary_writes_and_frozen_b22_mou frozen = await feed.read_reporting_feed_snapshot( first["ledger_snapshot_id"], caller=case.scope.principal ) + # Final B2.4 integration: the actual old binaries below exercise + # their ordinary read/write contract on all new objects, before + # incompatible autonomous projectors are drained and activated. + production = PgReportingProductionStore(pool=pool, notifications=notifications) + await production.create_schema() + await production.create_schema() + async with pool.connection() as c: + production_objects = await schema_objects(c) + await validate_production_schema(c, notifications=notifications) + assert {k: production_objects[k] for k in new_objects} == new_objects + production_added = production_objects.keys() - new_objects.keys() after = await frozen_call( installed_feed_history, pool, "exercise", phase="after", **kwargs ) @@ -316,6 +329,8 @@ async def test_nine_actual_artifacts_preserve_ordinary_writes_and_frozen_b22_mou "historical_wheel_sha256": installed_feed_history[3]["wheel_sha256"], "b22_wheel_sha256": approved_feed_b22[3]["wheel_sha256"], "feed_objects": len(added), + "b24_additive_objects": len(production_added), + "b24_activation": False, "page_count": len(expected[0]), "quarantine_preserved": True, } diff --git a/tests/conformance/reporting/test_reporting_production.py b/tests/conformance/reporting/test_reporting_production.py new file mode 100644 index 000000000..b6310abcc --- /dev/null +++ b/tests/conformance/reporting/test_reporting_production.py @@ -0,0 +1,126 @@ +"""Admitted production work preserves exact transactions and epoch-zero history.""" + +import pytest + +from adcp.reporting.materializer import reference_digest +from adcp.reporting.materializer.work import ReportingMaterializerLease +from adcp.server.base import ToolContext +from adcp.validation.schema_loader import get_named_validator + +from ._production_support import production_harness +from ._projection_support import drain +from .test_reporting_production_lock_order import source_turn + + +@pytest.mark.parametrize("backend", ["memory", "postgres"]) +@pytest.mark.parametrize("count", [0, 503]) +async def test_actual_source_publication_builds_canonical_evidence(backend, count, tmp_path): + async with production_harness( + backend, tmp_path / "destination.sqlite", count=count, source_publication=True + ) as h: + support, item = h.production, h.item + source = support.offerings[0].producer._source + assert source.requests == [] + await support.activate(account_id=item.config.account_id) + turn = await source_turn(support) + assert not turn.slices_failed and len(turn.revisions_committed) == 1 + revisions = await h.store.list_revisions( + account_id=item.config.account_id, + reporting_obligation_id=item.obligation.reporting_obligation_id, + ) + assert len(revisions) == 1 + item.revision = revisions[0] + assert item.revision.canonical_content_digest == reference_digest(item.verifier, item.rows) + assert item.revision.managed_control_totals == item.verifier.canonicalize(item.rows)[1] + assert len(source.requests) == 1 + repeat = await source_turn(support) + assert not repeat.revisions_committed and len(source.requests) == 1 + result = await support.materializer.run_once() + assert result.state == "verified" + assert item.writer.writes == 1 + await drain(h.projection, item.config.account_id) + page = await support.handler.get_reporting_status( + {"account": {"account_id": item.config.account_id}, "view": "summary"}, + ToolContext(caller_identity=item.binding.consumer_id), + ) + assert page["health"] == "complete" + + +@pytest.mark.parametrize("backend", ["memory", "postgres"]) +@pytest.mark.parametrize("notifications", [False, True]) +async def test_actual_admission_verified_finish_and_private_polling( + backend, notifications, tmp_path +): + async with production_harness( + backend, tmp_path / "destination.sqlite", notifications=notifications + ) as h: + support, item = h.production, h.item + before = await support.reporting_delivery() + assert bool(before) == (backend == "postgres") + if before: + assert before["managed_delivery"] is True + assert "reconciled_billing" not in before + assert "receipt_task" not in before + assert "readiness_notification" not in before + validator = get_named_validator("core/reporting-delivery-capabilities.json") + assert validator is not None + assert not list(validator.iter_errors(before)) + assert "sync_reporting_receipts" not in support.handler.advertised_tools_for_instance() + production_operation_1 = await support.activate(account_id=item.config.account_id) + assert production_operation_1 + lease = await item.claim() + assert isinstance(lease, ReportingMaterializerLease) + assert lease.admission_epoch == 2 + prepared, verified = await item.verified(lease) + result = await h.store.finish_materialization(lease, prepared=prepared, verified=verified) + assert result.state == "verified" + assert item.writer.writes == 1 + assert item.writer.opens == item.writer.closes == 2 + assert not (await h.queue())[0] + assert await h.store.read_materializer_boundaries(caller=item.binding.principal) == () + assert len(await h.store.read_production_boundaries(caller=item.binding.principal)) == 1 + if h.pool is None: + queue = h.store._production_outbox + count = len(queue.events) if queue is not None else 0 + else: + async with h.pool.connection() as c: + count = ( + await ( + await c.execute( + "SELECT count(*) FROM reporting_production_notification_events" + ) + ).fetchone() + )[0] + assert count == int(notifications) + await drain(h.projection, item.config.account_id) + context = ToolContext(caller_identity=item.binding.consumer_id) + response = await support.handler.get_reporting_status( + {"account": {"account_id": item.config.account_id}, "view": "summary"}, context + ) + assert response["health"] == "complete" + rows = [] + request = { + "account": {"account_id": item.config.account_id}, + "reporting_revision_id": item.revision.reporting_revision_id, + "pagination": {"max_results": 100}, + } + for _ in range(10): + page = await support.handler.get_media_buy_delivery(request, context) + assert isinstance(page, dict) + validator = get_named_validator("media-buy/get-media-buy-delivery-response.json") + assert validator is not None + assert not list(validator.iter_errors(page)) + rows.extend(page["reporting_rows"]) + if not page["pagination"]["has_more"]: + break + assert len(page["pagination"]["cursor"]) <= 2048 + request["pagination"]["cursor"] = page["pagination"]["cursor"] + else: + pytest.fail("exact revision walk did not terminate") + assert rows == item.rows + repeat = await h.store.finish_materialization(lease, prepared=prepared, verified=verified) + assert repeat.state == "verified" + assert item.writer.writes == 1 + await support.aclose() + production_operation_2 = await support.reporting_delivery() + assert production_operation_2 == {} diff --git a/tests/conformance/reporting/test_reporting_production_bindings.py b/tests/conformance/reporting/test_reporting_production_bindings.py new file mode 100644 index 000000000..6073fc05e --- /dev/null +++ b/tests/conformance/reporting/test_reporting_production_bindings.py @@ -0,0 +1,231 @@ +"""Full provider methods and account-qualified frozen source mappings.""" + +import json +from dataclasses import replace + +import pytest + +from adcp.reporting.ledger.notification_models import ReportingNotificationError +from adcp.reporting.ledger.store import LedgerConflictError +from adcp.reporting.materializer.contracts import ReportingWriterError +from adcp.reporting.production.contracts import ReportingProductionMethod + +from ._generation_support import END, obligation_for +from ._production_support import production_harness +from ._production_transport import MountedProduction +from .test_reporting_production_lock_order import source_turn + + +async def source_documents(h): + if h.pool is None: + return sorted( + (key.account_id, key.delivery_config_id, key.delivery_config_version, value.document()) + for key, value in h.store._production_source_bindings.items() + ) + async with h.pool.connection() as connection: + return await ( + await connection.execute( + "SELECT account_id,delivery_config_id,delivery_config_version,source_binding" + " FROM reporting_production_generations" + " ORDER BY account_id,delivery_config_id,delivery_config_version" + ) + ).fetchall() + + +@pytest.mark.parametrize("backend", ["memory", "postgres"]) +@pytest.mark.parametrize("notifications", [False, True]) +async def test_full_method_controls_admission_acquisition_and_raw_discovery( + backend, notifications, tmp_path +): + async with production_harness( + backend, + tmp_path / "destination.sqlite", + count=0, + source_publication=True, + notifications=notifications, + ) as h: + support, item = h.production, h.item + offering = support.offerings[0] + producer, provider = offering.producer, item.writer + await support.activate(account_id=item.config.account_id) + mounted = MountedProduction(h) + mounted.authorize(item) + original = provider.delivery_methods + new = replace(item.config, delivery_config_version=2) + new_binding = replace(item.binding, generation_key=new.generation_key) + provider.grant(new_binding) + producer._source.bind_generation(new) + before = await source_documents(h) + async with mounted.client() as client: + for field, different in ( + ("provider", {"domain": "different-provider.example.test"}), + ("destination_modes", ["existing"]), + ("access_mode", "read_only"), + ("reader_compatibility", ["different-reader-v2"]), + ): + raw = original[0].wire() + raw[field] = different + provider.delivery_methods = ( + ReportingProductionMethod(item.verifier.key.capability, raw), + ) + assert provider.capabilities == (item.verifier.key.capability,) + try: + with pytest.raises(ReportingWriterError) as denied: + await h.store.admit_production_configuration( + new, new_binding, offering_id=offering.offering_id + ) + assert denied.value.failure.code == "BINDING_MISMATCH" + token = support._producer_turn.set(producer) + try: + with pytest.raises((LedgerConflictError, ReportingWriterError)): + await producer.acquire_obligation(item.config, item.obligation, now=END) + finally: + support._producer_turn.reset(token) + assert producer._source.requests == [] + production_operation_3 = await source_turn(support) + assert (production_operation_3).leased is None + for transport in ("mcp", "a2a-0.3", "a2a-1.0"): + _, caps = await mounted.call( + client, "get_adcp_capabilities", {}, transport=transport + ) + assert caps.get("status") == "completed", caps + assert "reporting_delivery" not in caps.get("media_buy", {}), caps + assert "different-provider" not in json.dumps(caps) + _, status = await mounted.call( + client, + "get_reporting_status", + {"account": {"account_id": item.config.account_id}, "view": "summary"}, + transport=transport, + ) + assert status.get("status") == "completed", status + assert status["health"] != "complete" + finally: + provider.delivery_methods = original + assert await source_documents(h) == before + _, restored = await mounted.call(client, "get_adcp_capabilities", {}) + assert restored.get("status") == "completed", restored + claims = restored.get("media_buy", {}).get("reporting_delivery", {}) + assert bool(claims.get("managed_delivery")) == (backend == "postgres") + assert "reconciled_billing" not in claims and "receipt_task" not in claims + result = await source_turn(support) + assert len(result.revisions_committed) == 1 and not result.slices_failed + + +@pytest.mark.parametrize("backend", ["memory", "postgres"]) +async def test_products_are_explicit_for_shared_definition_and_colliding_account_keys( + backend, tmp_path +): + path = tmp_path / "destination.sqlite" + async with production_harness( + backend, path, count=0, source_publication=True, second_source=True + ) as h: + support, item = h.production, h.item + first, second = support.offerings + outside = replace(item.config, account_id="acct_b") + peer = replace(item.config, delivery_config_id="other-source") + entries = ( + (item.config, item.obligation, item.binding, first, "catalog-7391"), + ( + outside, + obligation_for(outside), + replace(item.binding, generation_key=outside.generation_key), + first, + "catalog-5820", + ), + ( + peer, + replace(obligation_for(peer), reporting_obligation_id="peer-obligation"), + replace(item.binding, generation_key=peer.generation_key), + second, + "catalog-5820", + ), + ) + assert first.verification_key.definition == second.verification_key.definition + assert outside.media_buy_ids == item.config.media_buy_ids + assert outside.delivery_config_id == item.config.delivery_config_id + # Supported discovery does not need a first trusted account binding. + assert not first.producer._source.bindings and not second.producer._source.bindings + production_condition_1 = bool(await support.reporting_delivery()) == (backend == "postgres") + assert production_condition_1 + for config, obligation, binding, offering, product_id in entries: + source = offering.producer._source + source.rows = item.rows + source.bind_generation(config, product_id=product_id) + item.writer.grant(binding) + assert product_id != config.report_definition_id + await h.store.admit_production_configuration( + config, binding, offering_id=offering.offering_id + ) + await h.store.commit_obligation(obligation) + frozen = await source_documents(h) + assert len(frozen) == 3 + # Admission fixes bindings even before activation can claim work. + production_operation_2 = await source_turn(support) + assert (production_operation_2).leased is None + for account in ("acct_a", "acct_b"): + await support.activate(account_id=account) + for config, obligation, binding, offering, product_id in entries: + producer = offering.producer + token = support._producer_turn.set(producer) + try: + revision = await producer.acquire_obligation(config, obligation, now=END) + finally: + support._producer_turn.reset(token) + assert revision is not None + request = producer._source.requests[-1] + assert request.identity.account_id == config.account_id + assert request.identity.delivery_config_id == config.delivery_config_id + assert {c.product_id for c in request.coverage.constituents} == {product_id} + assert {c.media_buy_id for c in request.coverage.constituents} == set( + config.media_buy_ids + ) + assert await source_documents(h) == frozen + await support.aclose() + async with production_harness( + backend, + path, + count=0, + source_publication=True, + second_source=True, + existing_store=h.store if h.pool is None else None, + existing_pool=h.pool, + source_bindings=tuple( + (config, offering.offering_id, product) + for config, _, _, offering, product in entries + ), + ) as restarted: + fresh = restarted.production + assert await source_documents(restarted) == frozen + for config, obligation, binding, old_offering, product_id in entries: + offering = next( + o for o in fresh.offerings if o.offering_id == old_offering.offering_id + ) + offering.producer._source.bind_generation(config, product_id=product_id) + await restarted.store.admit_production_configuration( + config, binding, offering_id=offering.offering_id + ) + first_fresh = fresh.offerings[0] + source = first_fresh.producer._source + old = source.bindings[item.config.generation_key] + source.bind_generation(item.config, product_id="catalog-5820") + try: + with pytest.raises(ReportingNotificationError, match="source_conflict"): + await restarted.store.admit_production_configuration( + item.config, item.binding, offering_id=first_fresh.offering_id + ) + token = fresh._producer_turn.set(first_fresh.producer) + try: + with pytest.raises((LedgerConflictError, ReportingWriterError)): + await restarted.store.producer_constituents(item.config, item.obligation) + finally: + fresh._producer_turn.reset(token) + assert await source_documents(restarted) == frozen + finally: + source.bindings[item.config.generation_key] = old + token = fresh._producer_turn.set(first_fresh.producer) + try: + restored = await restarted.store.producer_constituents(item.config, item.obligation) + finally: + fresh._producer_turn.reset(token) + assert {c.product_id for c in restored} == {"catalog-7391"} + assert await source_documents(restarted) == frozen diff --git a/tests/conformance/reporting/test_reporting_production_configuration.py b/tests/conformance/reporting/test_reporting_production_configuration.py new file mode 100644 index 000000000..dcfafb029 --- /dev/null +++ b/tests/conformance/reporting/test_reporting_production_configuration.py @@ -0,0 +1,277 @@ +"""Frozen provider configuration, mounted admission and semantic replay checks.""" + +import copy +import json +import sqlite3 +from dataclasses import replace +from datetime import timedelta, timezone + +import pytest + +from adcp.reporting.ledger.notification_models import ReportingNotificationError +from adcp.reporting.materializer.contracts import ReportingWriterError, binding_fingerprint +from adcp.reporting.materializer.work import ReportingMaterializerLease +from adcp.reporting.production.configuration import ReportingConfigurationAdmission +from adcp.validation.schema_loader import get_named_validator + +from ._feed_support import feed_request, walk +from ._generation_support import END +from ._production_support import production_harness +from ._production_transport import MountedProduction +from ._receipt_transport import error_code +from .test_reporting_production_bindings import source_documents + + +def wire_configuration(h): + config, binding = h.item.config, h.item.binding + offering = h.production.offerings[0] + return { + "delivery_config_id": config.delivery_config_id, + "delivery_config_version": config.delivery_config_version, + "offering_id": offering.offering_id, + "active": config.deactivated_at is None, + "feed_purpose": config.feed_purpose, + "report_definition_id": config.report_definition_id, + "reporting_profile": config.reporting_profile, + "scope": {"media_buy_ids": list(config.media_buy_ids)}, + "coverage_requirement": "full", + "required_finality": config.required_finality, + "reconciliation_mode": binding.reconciliation_mode, + "schedule": offering.configuration_schedule(config), + "method": h.item.writer.configuration_binding(binding).wire(), + } + + +def state_for(h, configuration): + ids = list(h.item.config.media_buy_ids) + coverage = { + "status": "full", + "evaluated_at": END.isoformat(), + "media_buy_ids": ids, + "fully_covered_media_buy_ids": ids, + "partially_covered_media_buy_ids": [], + "unsupported_media_buy_ids": [], + "unknown_media_buy_ids": [], + "package_ids": [], + "covered_package_ids": [], + "unsupported_package_ids": [], + "unknown_package_ids": [], + "limitations": [], + } + return { + "configuration": configuration, + "state": "ready", + "destination_ref": h.item.binding.destination_ref, + "validated_at": END.isoformat(), + "activated_at": h.item.config.activated_at.isoformat(), + "current_coverage": coverage, + } + + +@pytest.mark.parametrize("backend", ["memory", "postgres"]) +@pytest.mark.parametrize("notifications", [False, True]) +@pytest.mark.parametrize("fractional_offset", [False, True]) +async def test_mounted_configuration_is_the_admitted_method_scope_and_schedule( + backend, notifications, fractional_offset, tmp_path +): + selected = {} + + async def handle(request, context, admit): + h = selected["h"] + wire = request["accounts"][0]["reporting_delivery_configs"][0] + await admit( + ReportingConfigurationAdmission( + h.production.offerings[0].offering_id, + h.item.config, + h.item.binding, + configuration_wire=wire, + ) + ) + state = state_for(h, copy.deepcopy(wire)) + if fractional_offset: + at = h.item.config.activated_at + assert at.microsecond == 0 + state["activated_at"] = ( + at.astimezone(timezone(timedelta(hours=5, minutes=45))).strftime( + "%Y-%m-%dT%H:%M:%S" + ) + + ".00000+05:45" + ) + if selected.get("response_mutation"): + state["configuration"]["method"]["destination"]["location"] = "other/location" + return { + "accounts": [ + { + "account_id": h.item.config.account_id, + "brand": {"domain": "advertiser.example.test"}, + "operator": "buyer.example.test", + "action": "unchanged", + "status": "active", + "billing": "operator", + "timezone": "UTC", + "reporting_delivery_configs": [state], + } + ] + } + + async with production_harness( + backend, + tmp_path / "destination.sqlite", + notifications=notifications, + count=0, + account_handler=handle, + ) as h: + selected["h"] = h + mounted = MountedProduction(h) + mounted.authorize(h.item) + desired = wire_configuration(h) + request = { + "idempotency_key": "configuration-exact-replay-0001", + "accounts": [ + { + "account": {"account_id": h.item.config.account_id}, + "reporting_delivery_configs": [desired], + } + ], + } + async with mounted.client() as client: + for transport in ("mcp", "a2a-0.3", "a2a-1.0"): + _, result = await mounted.call( + client, "sync_accounts", request, transport=transport + ) + assert result.get("status") == "completed", result + validator = get_named_validator("account/sync-accounts-response.json") + assert not list(validator.iter_errors(result)), result + assert ( + result["accounts"][0]["reporting_delivery_configs"][0]["configuration"] + == desired + ) + expected_time = h.item.config.activated_at.isoformat() + if fractional_offset: + expected_time = ( + h.item.config.activated_at.astimezone( + timezone(timedelta(hours=5, minutes=45)) + ).strftime("%Y-%m-%dT%H:%M:%S") + + ".00000+05:45" + ) + assert ( + result["accounts"][0]["reporting_delivery_configs"][0]["activated_at"] + == expected_time + ) + before = await h.image() + frozen = await source_documents(h) + for where, name, value in ( + ("method.destination", "location", "different/provider/location"), + ("method.destination", "provider", {"domain": "different.example.test"}), + ("schedule", "delivery_sla", "PT2H"), + ("scope", "media_buy_ids", ["unrelated-media-buy"]), + ("", "report_definition_id", "different-definition"), + ): + bad = copy.deepcopy(request) + target = bad["accounts"][0]["reporting_delivery_configs"][0] + for segment in where.split(".") if where else (): + target = target[segment] + target[name] = value + _, rejected = await mounted.call(client, "sync_accounts", bad) + assert error_code(rejected) == "REPORTING_CONFIGURATION_UNAVAILABLE", rejected + assert "different/provider" not in json.dumps(rejected) + assert await h.image() == before + assert await source_documents(h) == frozen + selected["response_mutation"] = True + _, rejected = await mounted.call(client, "sync_accounts", request) + assert error_code(rejected) == "REPORTING_CONFIGURATION_UNADMITTED", rejected + assert "other/location" not in json.dumps(rejected) + selected.pop("response_mutation") + # Replays reauthorize the same consumer, even after an earlier + # accepted configuration and with the same transport key. + h.authorized_bindings.clear() + _, rejected = await mounted.call(client, "sync_accounts", request, transport="a2a-1.0") + assert error_code(rejected) == "UNAUTHORIZED", rejected + + +async def destination_documents(h): + if h.pool is None: + return copy.deepcopy(h.store._production_destination_bindings) + async with h.pool.connection() as connection: + return await ( + await connection.execute( + "SELECT * FROM reporting_production_destination_bindings" + " ORDER BY account_id,consumer_id,delivery_config_id,delivery_config_version" + ) + ).fetchall() + + +@pytest.mark.parametrize("backend", ["memory", "postgres"]) +@pytest.mark.parametrize("notifications", [False, True]) +async def test_provider_binding_is_frozen_across_restart_and_later_mutation( + backend, notifications, tmp_path +): + path = tmp_path / "destination.sqlite" + async with production_harness(backend, path, notifications=notifications, count=0) as h: + support, item = h.production, h.item + await support.activate(account_id=item.config.account_id) + frozen = await destination_documents(h) + source = await source_documents(h) + request = feed_request(item) + first = await h.store.read_reporting_feed(request, caller=item.binding.principal) + walk_before = await walk(h.store, request, item.binding.principal, first=first) + with sqlite3.connect(path) as connection: + original = connection.execute( + "SELECT method FROM methods WHERE binding=?", (binding_fingerprint(item.binding),) + ).fetchone()[0] + changed = json.loads(original) + changed["destination"]["location"] = "new-location-same-label" + connection.execute( + "UPDATE methods SET method=? WHERE binding=?", + (json.dumps(changed), binding_fingerprint(item.binding)), + ) + assert item.writer.configuration_binding(item.binding).wire() == changed + before = await h.image() + with pytest.raises(ReportingNotificationError, match="destination_conflict"): + await h.store.admit_production_configuration( + item.config, item.binding, offering_id=support.offerings[0].offering_id + ) + assert await h.image() == before + denied = await item.claim() + assert not isinstance(denied, ReportingMaterializerLease), denied + assert item.writer.writes == 0 + assert await destination_documents(h) == frozen + assert await source_documents(h) == source + assert await walk(h.store, request, item.binding.principal, first=first) == walk_before + await support.aclose() + async with production_harness( + backend, + path, + notifications=notifications, + count=0, + existing_store=h.store if h.pool is None else None, + existing_pool=h.pool, + ) as restarted: + assert await destination_documents(restarted) == frozen + assert await source_documents(restarted) == source + assert ( + await walk(restarted.store, request, item.binding.principal, first=first) + == walk_before + ) + with sqlite3.connect(path) as connection: + connection.execute( + "UPDATE methods SET method=? WHERE binding=?", + (original, binding_fingerprint(item.binding)), + ) + await restarted.store.admit_production_configuration( + item.config, item.binding, offering_id=restarted.production.offerings[0].offering_id + ) + assert await destination_documents(restarted) == frozen + # The source mapping is also a semantic-generation contract, not + # just a lookup by business keys. Same key, changed definition fails. + old = restarted.production.offerings[0].producer._source.bindings[ + item.config.generation_key + ] + changed_config = replace(item.config, account_timezone="Etc/UTC") + with pytest.raises(ReportingWriterError): + old.check( + changed_config, + restarted.production.offerings[0].producer._source.capabilities, + restarted.production.offerings[0].source_offering_id, + ) + assert await source_documents(restarted) == source diff --git a/tests/conformance/reporting/test_reporting_production_lifecycle.py b/tests/conformance/reporting/test_reporting_production_lifecycle.py new file mode 100644 index 000000000..837332c43 --- /dev/null +++ b/tests/conformance/reporting/test_reporting_production_lifecycle.py @@ -0,0 +1,299 @@ +"""Actual source/I/O, mounted financial receipts, private frozen reads and health cycles.""" + +import copy +from dataclasses import replace +from datetime import timedelta + +import pytest + +from adcp.reporting.ledger import ( + ReportingAdjustmentRecord, + ReportingControlTotalRecord, + ReportingRevisionReceiptRecord, + revision_content_sha256, +) +from adcp.reporting.ledger.delivery import adjustment_to_wire, receipt_to_wire +from adcp.reporting.materializer.work import ReportingMaterializerLease +from adcp.reporting.ownership import page_revision_ownership +from adcp.validation.schema_loader import get_validator + +from ._generation_support import END +from ._production_support import production_harness +from ._production_transport import MountedProduction +from ._projection_support import drain +from ._receipt_transport import error_code +from .test_reporting_production_lock_order import source_turn +from .test_reporting_schedule_schema import assert_original_rejection + + +async def public_walk(mounted, client, request, *, first=None, transport="mcp"): + request = copy.deepcopy(request) + pages = [] + page = first + for _ in range(40): + if page is None: + _, page = await mounted.call( + client, "get_reporting_status", request, transport=transport + ) + assert page.get("status") == "completed", page + page_revision_ownership(page) + pages.append(page) + assert page["changes_checkpoint"] == pages[0]["changes_checkpoint"] + if not page["pagination"]["has_more"]: + return pages + request["pagination"]["cursor"] = page["pagination"]["cursor"] + page = None + pytest.fail("production cursor walk exceeded its bound") + + +async def generation(h): + account = h.item.config.account_id + await drain(h.projection, account) + values = [ + value.generation + for value in await h.projection.checkpoints(account_id=account) + if value.scope.consumer_id == h.item.binding.consumer_id + and value.scope.reporting_obligation_id == h.item.obligation.reporting_obligation_id + ] + assert values + return max(values) + + +async def observed_now(h): + # PostgreSQL owns materialization/receipt commit time. The app fixture's + # frozen clock predates I/O and is not a valid consumer observation time. + if h.pool is None: + return h.clock() + async with h.pool.connection() as connection: + return (await (await connection.execute("SELECT clock_timestamp()")).fetchone())[0] + + +@pytest.mark.parametrize("backend", ["memory", "postgres"]) +@pytest.mark.parametrize("notifications", [False, True]) +@pytest.mark.parametrize("feedback", [False, True]) +async def test_production_official_receipts_adjustment_and_frozen_wire_cycle( + backend, notifications, feedback, tmp_path +): + async with production_harness( + backend, + tmp_path / "provider.sqlite", + count=3, + source_publication=True, + reconciled=True, + notifications=notifications, + feedback=feedback, + ) as h: + support, item = h.production, h.item + snapshot_id = "retained-unlinked-snapshot" + old = replace( + item.revision, + reporting_revision_id=snapshot_id, + finality="snapshot", + finality_basis=None, + finality_policy_id=None, + finalized_at=None, + revision_content_sha256=revision_content_sha256( + reporting_revision_id=snapshot_id, + row_count=len(item.rows), + control_totals=item.revision.control_totals, + reporting_rows=item.rows, + control_total_evidence=item.revision.managed_control_totals, + ), + ) + await h.store.commit_revision(old, item.rows) + mounted = MountedProduction(h) + mounted.authorize(item) + await support.activate(account_id=item.config.account_id) + async with mounted.client() as client: + for transport in ("mcp", "a2a-0.3", "a2a-1.0"): + _, caps = await mounted.call( + client, "get_adcp_capabilities", {}, transport=transport + ) + assert caps.get("status") == "completed", caps + claims = caps.get("media_buy", {}).get("reporting_delivery", {}) + if backend == "postgres": + assert claims["managed_delivery"] and claims["reconciled_billing"] + assert claims["receipt_task"] == "sync_reporting_receipts" + # Complete polling requires no optional HTTP delivery worker. + assert "readiness_notification" not in claims + else: + assert not claims + selected = await item.claim() + assert not isinstance(selected, ReportingMaterializerLease) + assert item.writer.writes == 0 # never falls back to the retained snapshot + source_result = await source_turn(support) + assert len(source_result.revisions_committed) == 1 and not source_result.slices_failed + revisions = await h.store.list_revisions( + account_id=item.config.account_id, + reporting_obligation_id=item.obligation.reporting_obligation_id, + ) + official = next(r for r in revisions if r.finality == "official") + assert {r.reporting_revision_id for r in revisions} == { + snapshot_id, + official.reporting_revision_id, + } + item.revision = official + lease = await item.claim() + assert isinstance(lease, ReportingMaterializerLease) + assert lease.attempt.reporting_revision_id == official.reporting_revision_id + prepared, verified = await item.verified(lease) + production_operation_1 = await h.store.finish_materialization( + lease, prepared=prepared, verified=verified + ) + assert (production_operation_1).state == "verified" + before_receipts = await generation(h) + receipt = ReportingRevisionReceiptRecord( + item.scope, + "production-rejected", + official.reporting_revision_id, + lease.attempt.reporting_materialization_id, + "rejected", + item.binding.verification_profile, + official.row_count, + official.managed_control_totals, + await observed_now(h), + observed_canonical_content_digest=official.canonical_content_digest, + rejection_codes=("LOAD_FAILED",), + ) + request = { + "account": {"account_id": item.config.account_id}, + "idempotency_key": "production-rejected-0001", + "receipts": [receipt_to_wire(receipt)], + } + _, rejected = await mounted.call(client, "sync_reporting_receipts", request) + assert rejected["results"][0]["result"] == "recorded", rejected + after_rejection = await generation(h) + assert after_rejection > before_receipts + production_condition_2 = not isinstance(await item.claim(), ReportingMaterializerLease) + assert production_condition_2 + assert item.writer.writes == 1 # consumer rejection cannot schedule a retry + feed_request = { + "account": request["account"], + "view": "periods", + "pagination": {"max_results": 1}, + } + frozen = await public_walk(mounted, client, feed_request) + accepted = replace( + receipt, + reporting_receipt_id="production-accepted", + status="accepted", + supersedes_reporting_receipt_id=receipt.reporting_receipt_id, + rejection_codes=(), + observed_at=await observed_now(h), + ) + request = { + **request, + "idempotency_key": "production-accepted-0001", + "receipts": [receipt_to_wire(accepted)], + } + _, response = await mounted.call( + client, "sync_reporting_receipts", request, transport="a2a-1.0" + ) + assert response["results"][0]["result"] == "recorded", response + accepted_generation = await generation(h) + assert accepted_generation > after_rejection + + async def summary(): + _, value = await mounted.call( + client, + "get_reporting_status", + {"account": request["account"], "view": "summary"}, + ) + assert value.get("status") == "completed", value + return value + + assert (await summary())["health"] == "complete" + observed = await observed_now(h) + adjustment = ReportingAdjustmentRecord( + "production-adjustment", + item.config.account_id, + official.reporting_revision_id, + "source_correction", + END, + END + timedelta(days=30), + (("spend", "-0.50"),), + observed, + observed, + managed_control_total_deltas=( + ReportingControlTotalRecord("spend", "-0.50", "decimal", "USD"), + ), + ) + await h.store.commit_adjustment(adjustment) + pending_generation = await generation(h) + assert pending_generation > accepted_generation + assert (await summary())["health"] != "complete" + replacement = replace( + accepted, + reporting_receipt_id="forbidden-accepted-replacement", + supersedes_reporting_receipt_id=accepted.reporting_receipt_id, + ) + mixed = { + "account": request["account"], + "idempotency_key": "production-mixed-final-0001", + "receipts": [receipt_to_wire(replacement)], + "adjustment_receipts": [ + { + "reporting_receipt_id": "production-adjustment-accepted", + "reporting_adjustment_id": adjustment.reporting_adjustment_id, + "adjusts_reporting_revision_id": official.reporting_revision_id, + "status": "accepted", + "observed_adjustment_sha256": adjustment_to_wire(adjustment)[ + "canonical_adjustment_sha256" + ], + "observed_at": (await observed_now(h)).isoformat(), + } + ], + } + _, mixed_response = await mounted.call( + client, "sync_reporting_receipts", mixed, transport="a2a-0.3" + ) + assert mixed_response["results"][0]["reporting_receipt_id"] == ( + replacement.reporting_receipt_id + ) + assert mixed_response["results"][0]["errors"][0]["code"] == ( + "ACCEPTED_RECEIPT_TERMINAL" + ) + assert mixed_response["results"][1]["result"] == "recorded", mixed_response + complete_generation = await generation(h) + assert complete_generation > pending_generation + assert (await summary())["health"] == "complete" + for transport in ("mcp", "a2a-0.3", "a2a-1.0"): + _, complete = await mounted.call( + client, + "get_reporting_status", + {"account": request["account"], "view": "summary"}, + transport=transport, + ) + assert complete["health"] == "complete" + assert "next_expected_at" in complete + assert_original_rejection(complete) + get_validator("get_reporting_status", "sync").validate(complete) + _, replay = await mounted.call( + client, "sync_reporting_receipts", mixed, transport=transport + ) + assert replay == mixed_response + assert await generation(h) == complete_generation + assert ( + await public_walk( + mounted, client, feed_request, first=frozen[0], transport=transport + ) + == frozen + ) + _, exact = await mounted.call( + client, + "get_media_buy_delivery", + { + "account": request["account"], + "reporting_revision_id": official.reporting_revision_id, + }, + transport=transport, + ) + assert exact["reporting_rows"] == item.rows, exact + assert page_revision_ownership(exact) == { + official.reporting_revision_id: item.obligation.reporting_obligation_id + } + assert item.writer.writes == 1 + assert not (await h.queue())[0] # epoch-zero readiness stays quarantined + h.authorized_bindings.clear() + _, denied = await mounted.call(client, "sync_reporting_receipts", mixed) + assert error_code(denied) == "UNAUTHORIZED" diff --git a/tests/conformance/reporting/test_reporting_production_lock_order.py b/tests/conformance/reporting/test_reporting_production_lock_order.py new file mode 100644 index 000000000..a3b46b4e6 --- /dev/null +++ b/tests/conformance/reporting/test_reporting_production_lock_order.py @@ -0,0 +1,268 @@ +"""Coordinated real lock order, with the executable wrong-order control restored.""" + +import asyncio +import json +from dataclasses import replace +from datetime import timedelta +from types import MethodType + +import pytest + +from adcp.reporting.ledger.pg import PgReportingLedgerStore + +from ._generation_support import END +from ._production_support import production_harness + + +async def source_turn(support): + producer = support.offerings[0].producer + token = support._producer_turn.set(producer) + try: + return await producer.run_worker() + finally: + support._producer_turn.reset(token) + + +@pytest.mark.parametrize("wrong_order", [False, True]) +async def test_activation_and_producer_actual_trigger_lock_order( + wrong_order, tmp_path, monkeypatch +): + psycopg = pytest.importorskip("psycopg") + async with production_harness("postgres", tmp_path / "destination.sqlite") as h: + support, store, item = h.production, h.store, h.item + locked, release, executing = asyncio.Event(), asyncio.Event(), asyncio.Event() + pids = {} + original_step = support.projection._activation_step_on + original_execute = psycopg.AsyncConnection.execute + original_lease = store.lease_period_close.__func__ + tasks = [] + activation = None + + async def activation_step(connection, account_id): + # This phase already owns the real account lock and will acquire + # configuration FK locks while installing the live checkpoints. + # Earlier activation phases commit their own bounded transactions. + if asyncio.current_task() is activation and not locked.is_set(): + pids["activation"] = connection.info.backend_pid + locked.set() + await asyncio.wait_for(release.wait(), 5) + return await original_step(connection, account_id) + + async def observe(connection, query, *args, **kwargs): + if isinstance(query, str) and query.startswith( + "UPDATE reporting_configurations SET lease_worker_id" + ): + pids["producer"] = connection.info.backend_pid + executing.set() + return await original_execute(connection, query, *args, **kwargs) + + async def wait_for_actual_trigger_wait(): + await asyncio.wait_for(executing.wait(), 5) + async with h.pool.connection() as observer: + for _ in range(100): + row = await ( + await observer.execute( + "SELECT EXISTS(SELECT 1 FROM pg_locks WHERE pid=%s" + " AND locktype='advisory' AND NOT granted)," + " %s=ANY(pg_blocking_pids(%s))", + (pids["producer"], pids["activation"], pids["producer"]), + ) + ).fetchone() + if row == (True, True): + return + await asyncio.sleep(0.01) + pytest.fail("the producer did not block in the real account-lock trigger") + + # Both modes retain the inherited trigger and its original function. + async with h.pool.connection() as c: + trigger = await ( + await c.execute( + "SELECT t.tgenabled,pg_get_functiondef(t.tgfoid)" + " FROM pg_trigger t WHERE t.tgrelid='reporting_configurations'::regclass" + " AND t.tgname='reporting_materializer_configuration'" + ) + ).fetchone() + assert trigger[0] == "O" and "pg_advisory_xact_lock" in trigger[1] + with monkeypatch.context() as patch: + patch.setattr(support.projection, "_activation_step_on", activation_step) + patch.setattr(psycopg.AsyncConnection, "execute", observe) + if wrong_order: + # Restore the actual predecessor acquisition; do not emulate + # its result or replace the lock-taking database trigger. + patch.setattr( + store, + "lease_period_close", + MethodType(PgReportingLedgerStore.lease_period_close, store), + ) + activation = asyncio.create_task(support.activate(account_id=item.config.account_id)) + tasks.append(activation) + try: + await asyncio.wait_for(locked.wait(), 5) + producer = asyncio.create_task(source_turn(support)) + tasks.append(producer) + if wrong_order: + await wait_for_actual_trigger_wait() + else: + turn = await asyncio.wait_for(asyncio.shield(producer), 5) + assert turn.leased is None + assert not executing.is_set() + release.set() + results = await asyncio.wait_for( + asyncio.gather(activation, producer, return_exceptions=True), 8 + ) + finally: + release.set() + for task in tasks: + if not task.done(): + task.cancel() + await asyncio.gather(*tasks, return_exceptions=True) + assert store.lease_period_close.__func__ is original_lease + deadlocks = [r for r in results if isinstance(r, psycopg.errors.DeadlockDetected)] + assert len(deadlocks) == int(wrong_order) + assert all(not isinstance(r, BaseException) or r in deadlocks for r in results) + + async with h.pool.connection() as c: + rows = await ( + await c.execute( + "SELECT (SELECT count(*) FROM reporting_projection_accounts)," + " (SELECT count(*) FROM reporting_production_accounts)," + " (SELECT count(*) FROM reporting_production_work)," + " (SELECT count(*) FROM reporting_materializer_work)," + " (SELECT count(*) FROM reporting_production_notification_events)" + ) + ).fetchone() + lease = await ( + await c.execute( + "SELECT lease_worker_id,lease_expires_at FROM reporting_configurations" + " WHERE account_id=%s", + (item.config.account_id,), + ) + ).fetchone() + if isinstance(results[0], BaseException): + assert rows == (1, 0, 0, 0, 0) + assert not await support.projection.baseline_ready(account_id=item.config.account_id) + else: + assert rows == (1, 1, 0, 0, 0) + # A failed lease rolls back; a winning producer releases in its real + # finally block. Neither path can leak a lease or reserve epoch-zero I/O. + assert lease == (None, None) + assert ( + await store.get_revision( + account_id=item.config.account_id, + reporting_revision_id=item.revision.reporting_revision_id, + ) + == item.revision + ) + # The negative control restores the real methods before resuming the + # incomplete phase. Its original input and epoch-zero queues persist. + await support.activate(account_id=item.config.account_id) + assert await support.projection.baseline_ready(account_id=item.config.account_id) + if not wrong_order: + executing.clear() + with monkeypatch.context() as patch: + patch.setattr(psycopg.AsyncConnection, "execute", observe) + turn = await asyncio.wait_for(source_turn(support), 5) + assert turn.leased is not None and executing.is_set() + assert turn.revisions_committed == [] + print( + json.dumps( + { + "production_lock_order": "wrong_order_control" if wrong_order else "restored", + "actual_trigger": True, + "observed_trigger_wait": wrong_order, + "deadlocks": len(deadlocks), + "rollback_or_commit_verified": True, + "restored_acquisition": store.lease_period_close.__func__ is original_lease, + } + ) + ) + + +@pytest.mark.parametrize("backend", ["memory", "postgres"]) +async def test_selected_source_enrollment_and_expired_lease_fairness(backend, tmp_path): + async with production_harness( + backend, tmp_path / "destination.sqlite", second_source=True + ) as h: + support, store, item = h.production, h.store, h.item + selected, other = support.offerings + peer = replace(item.config, delivery_config_id="peer") + other_config = replace(item.config, delivery_config_id="other-source") + unadmitted = replace(item.config, delivery_config_id="unadmitted") + outside = replace(item.config, account_id="acct_b") + for configuration in (peer, other_config, unadmitted, outside): + await store.put_configuration(configuration) + before = { + account: await store.list_configurations(account_id=account) + for account in ("acct_a", "acct_b") + } + + async def acquire(offering, worker, now=END): + token = support._producer_turn.set(offering.producer) + try: + return await store.lease_period_close(worker_id=worker, now=now, lease_seconds=1) + finally: + support._producer_turn.reset(token) + + production_operation_1 = await acquire(selected, "preactivation") + assert production_operation_1 is None + production_operation_2 = await acquire(other, "preactivation") + assert production_operation_2 is None + await support.activate(account_id="acct_a") + # Two actual source instances intentionally have identical report + # contracts. Only explicit, verified offering admission chooses one. + production_operation_3 = await acquire(selected, "unbound") + assert production_operation_3 is None + for configuration, offering in ( + (item.config, selected), + (peer, selected), + (other_config, other), + ): + offering.producer._source.bind_generation(configuration) + binding = replace(item.binding, generation_key=configuration.generation_key) + item.writer.grant(binding) + await store.admit_production_configuration( + configuration, binding, offering_id=offering.offering_id + ) + first = await acquire(selected, "crashed") + second = await acquire(selected, "live") + assert first.generation_key == item.config.generation_key + assert second.generation_key == peer.generation_key + await store.release_period_close(second, worker_id="live") + expired = await acquire(selected, "replacement", END + timedelta(seconds=2)) + assert expired.generation_key == first.generation_key + await store.release_period_close(expired, worker_id="replacement") + isolated = await acquire(other, "other") + assert isolated.generation_key == other_config.generation_key + await store.release_period_close(isolated, worker_id="other") + assert { + account: await store.list_configurations(account_id=account) + for account in ("acct_a", "acct_b") + } == before + if h.pool is None: + turns = store._lease_turns + assert unadmitted.generation_key not in turns + assert outside.generation_key not in turns + assert not store._leases + else: + async with h.pool.connection() as c: + turns = await ( + await c.execute( + "SELECT account_id,delivery_config_id" + " FROM adcp_reporting_configuration_lease_turns" + " ORDER BY account_id,delivery_config_id" + ) + ).fetchall() + assert turns == [ + ("acct_a", "daily"), + ("acct_a", "other-source"), + ("acct_a", "peer"), + ] + assert ( + await ( + await c.execute( + "SELECT count(*) FROM reporting_configurations" + " WHERE lease_worker_id IS NOT NULL" + ) + ).fetchone() + )[0] == 0 + assert not (await h.queue())[0] diff --git a/tests/conformance/reporting/test_reporting_production_migration.py b/tests/conformance/reporting/test_reporting_production_migration.py new file mode 100644 index 000000000..363aefc75 --- /dev/null +++ b/tests/conformance/reporting/test_reporting_production_migration.py @@ -0,0 +1,265 @@ +"""Populated, concurrent and interrupted B2.4 bootstrap with old objects intact.""" + +import asyncio +import json +from copy import deepcopy +from datetime import timedelta +from importlib.resources import files + +import pytest + +from adcp.reporting.feed import PgReportingFeedStore +from adcp.reporting.ledger.notification_models import ReportingNotificationError +from adcp.reporting.materializer import ReportingMaterializerLease +from adcp.reporting.outbox._schema import schema_objects +from adcp.reporting.outbox.status_pg import PgStatusNotificationStore +from adcp.reporting.production.pg import PgReportingProductionStore +from adcp.reporting.production.schema import validate_production_schema +from adcp.reporting.projection.schema import validate_projection_schema + +from ._durable_materializer_support import DurableHarness, durable_case +from ._feed_support import feed_request, mixed_case, walk +from ._generation_support import isolated_reporting_pool +from ._reconciliation_support import Clock +from .test_reporting_feed_migration import fairness + + +def manifests(): + return { + package: json.loads( + files("adcp.reporting." + package).joinpath("required_schema.json").read_text() + ) + for package in ("materializer", "receipts", "feed", "projection", "production") + } + + +def original_rows(image, before): + # This one documented additive column fences newly activated checkpoints. + # Its default leaves old C checkpoints compatible until explicit activation. + result = deepcopy({key: image[key] for key in before}) + for (row,) in result.get("reporting_status_scope_checkpoints", []): + writer_floor = row.pop("projection_writer_floor", 1) + assert writer_floor == 1 + return result + + +@pytest.mark.parametrize("notifications", [False, True]) +@pytest.mark.parametrize("autocommit", [False, True]) +async def test_populated_repeat_concurrent_migration_keeps_history_fairness_and_frozen_pages( + notifications, autocommit +): + async with isolated_reporting_pool(autocommit=autocommit) as pool: + parent = PgReportingFeedStore(pool=pool, notifications=notifications) + await parent.create_schema() + # Install the optional historical C objects through their supported + # notification-enabled owner. The actual receipt/feed writer retains + # this cell's requested off/on mode. + old_projection = PgStatusNotificationStore( + PgReportingFeedStore(pool=pool, notifications=True) + ) + await old_projection.create_schema() + h = DurableHarness(parent, Clock(), pool) + case, request, response = await mixed_case(h) + pending = await durable_case(parent, account="pending-account") + lease = None + for _ in range(8): + candidate = await pending.claim() + if isinstance(candidate, ReportingMaterializerLease): + lease = candidate + break + assert lease is not None and lease.admission_epoch == 0 + if notifications: + await old_projection.baseline(account_id=case.obligation.account_id) + first = await parent.read_reporting_feed(feed_request(case), caller=case.binding.principal) + original = await parent.read_reporting_feed_snapshot( + first["ledger_snapshot_id"], caller=case.binding.principal + ) + expected = await walk(parent, feed_request(case), case.binding.principal, first=first) + before = await h.image() + old_turns = await fairness(pool) + async with pool.connection() as c: + old_objects = await schema_objects(c) + with pytest.raises(ReportingNotificationError): + await validate_production_schema(c, notifications=notifications) + child = PgReportingProductionStore(pool=pool, notifications=notifications) + await asyncio.wait_for(asyncio.gather(*(child.create_schema() for _ in range(3))), 30) + await child.create_schema() + async with pool.connection() as c: + current = await schema_objects(c) + await validate_projection_schema(c, notifications=notifications) + await validate_production_schema(c, notifications=notifications) + assert ( + await ( + await c.execute("SELECT count(*) FROM reporting_production_accounts") + ).fetchone() + )[0] == 0 + assert {key: current[key] for key in old_objects} == old_objects + required = manifests() + assert set(required["projection"]).isdisjoint(required["production"]) + assert set(current) - set(old_objects) == set(required["projection"]) | set( + required["production"] + ) + for objects in required.values(): + assert all(current.get(key) == value for key, value in objects.items()) + assert original_rows(await h.image(), before) == before + assert await fairness(pool) == old_turns + production_operation_1 = await parent.ingest_receipt_batch( + request, caller=case.binding.principal + ) + assert production_operation_1 == response + assert ( + await walk(child, feed_request(case), case.binding.principal, first=first) == expected + ) + assert ( + await child.read_reporting_feed_snapshot( + original.snapshot_id, caller=case.binding.principal + ) + == original + ) + assert original_rows(await h.image(), before) == before + print( + json.dumps( + { + "b24_migration": "concurrent-repeat", + "notifications": notifications, + "autocommit": autocommit, + "preserved_objects": len(old_objects), + "isolated_additions": { + key: len(required[key]) for key in ("projection", "production") + }, + "pending_epoch": lease.admission_epoch, + "old_pages": len(expected[0]), + } + ), + flush=True, + ) + + +@pytest.mark.parametrize("notifications", [False, True]) +async def test_interrupted_complete_migration_rolls_back_every_new_object( + notifications, monkeypatch +): + async with isolated_reporting_pool(autocommit=True) as pool: + parent = PgReportingFeedStore(pool=pool, notifications=notifications) + await parent.create_schema() + await PgStatusNotificationStore( + PgReportingFeedStore(pool=pool, notifications=True) + ).create_schema() + h = DurableHarness(parent, Clock(), pool) + case, request, response = await mixed_case(h) + before = await h.image() + async with pool.connection() as c: + original = await schema_objects(c) + child = PgReportingProductionStore(pool=pool, notifications=notifications) + entered = asyncio.Event() + from psycopg import AsyncConnection + + execute = AsyncConnection.execute + + async def interrupt(connection, query, *args, **kwargs): + result = await execute(connection, query, *args, **kwargs) + if isinstance(query, str) and query.startswith("-- B2.4 production admission."): + entered.set() + await asyncio.Event().wait() + return result + + with monkeypatch.context() as patch: + patch.setattr(AsyncConnection, "execute", interrupt) + task = asyncio.create_task(child.create_schema()) + try: + await asyncio.wait_for(entered.wait(), 20) + async with pool.connection() as c: + # Catalog deparsing can acquire a relation lock behind + # the intentionally paused ALTER TABLE. Observe raw MVCC + # catalog visibility now; compare every definition after + # cancellation releases those DDL locks. + assert ( + await ( + await c.execute( + "SELECT count(*) FROM pg_class c" + " JOIN pg_namespace n ON n.oid=c.relnamespace" + " WHERE n.nspname=current_schema()" + " AND c.relname='reporting_production_delivery_windows'" + ) + ).fetchone() + )[0] == 0 + finally: + task.cancel() + with pytest.raises(asyncio.CancelledError): + await task + assert await h.image() == before + async with pool.connection() as c: + assert await schema_objects(c) == original + await child.create_schema() + assert original_rows(await h.image(), before) == before + production_operation_2 = await parent.ingest_receipt_batch( + request, caller=case.binding.principal + ) + assert production_operation_2 == response + + +@pytest.mark.parametrize( + "damage", + [ + ( + "ALTER TABLE reporting_production_delivery_windows" + " DISABLE TRIGGER reporting_production_delivery_window_immutable" + ), + "ALTER TABLE reporting_production_delivery_windows ALTER COLUMN expires_at DROP NOT NULL", + "DROP TABLE reporting_production_delivery_windows", + "DROP INDEX reporting_production_source_pending", + ( + "ALTER TABLE reporting_projection_inputs" + " DISABLE TRIGGER reporting_projection_input_immutable" + ), + ( + "ALTER TABLE reporting_status_scope_checkpoints" + " DISABLE TRIGGER reporting_projection_checkpoint_guard" + ), + ], +) +@pytest.mark.parametrize("notifications", [False, True]) +async def test_partial_or_mismatched_production_objects_refuse_fresh_readiness( + damage, notifications +): + async with isolated_reporting_pool(autocommit=True) as pool: + store = PgReportingProductionStore(pool=pool, notifications=notifications) + await store.create_schema() + async with pool.connection() as c: + await c.execute(damage) + with pytest.raises(ReportingNotificationError) as caught: + await validate_production_schema(c, notifications=notifications) + assert caught.value.code == "reporting_production_schema_unready" + + +async def test_retry_window_is_immutable_and_repeated_bootstrap_never_restarts_deadline(): + async with isolated_reporting_pool(autocommit=True) as pool: + store = PgReportingProductionStore(pool=pool) + await store.create_schema() + started = Clock()() + row = ( + "account-window", + "window-key", + "core", + "a" * 64, + started, + started + timedelta(seconds=86400), + ) + async with pool.connection() as c: + await c.execute( + "INSERT INTO reporting_production_delivery_windows VALUES(%s,%s,%s,%s,%s,%s)", row + ) + for statement in ( + "UPDATE reporting_production_delivery_windows" + " SET expires_at=expires_at+interval '1 second'", + "DELETE FROM reporting_production_delivery_windows", + ): + async with pool.connection() as c: + with pytest.raises(Exception): + async with c.transaction(): + await c.execute(statement) + await store.create_schema() + async with pool.connection() as c: + assert await ( + await c.execute("SELECT * FROM reporting_production_delivery_windows") + ).fetchall() == [row] diff --git a/tests/conformance/reporting/test_reporting_production_notifications.py b/tests/conformance/reporting/test_reporting_production_notifications.py new file mode 100644 index 000000000..a8e98f910 --- /dev/null +++ b/tests/conformance/reporting/test_reporting_production_notifications.py @@ -0,0 +1,708 @@ +"""All actual production queues use durable fanout and signed at-least-once delivery.""" + +import asyncio +import json +from contextlib import asynccontextmanager +from dataclasses import replace +from datetime import timedelta +from types import SimpleNamespace + +import pytest + +from adcp.reporting.outbox import validate_notification_payload +from adcp.reporting.production.notifications import production_notification_workers +from adcp.signing.jwks import StaticJwksResolver +from adcp.signing.webhook_verifier import WebhookVerifyOptions, verify_webhook_signature + +from ._generation_support import configuration, obligation_for, revision_for +from ._production_support import production_harness +from ._production_transport import MountedProduction +from ._projection_support import drain +from ._reliable_support import ( + DeterministicReceiverStore, + ScriptedNotificationReceiver, + SimulatedCrash, + _BytesStore, + notification_subscription, + notification_verification_keys, +) + +EVENTS = ( + "reporting.ledger_changed", + "reporting.status_changed", + "reporting.delivery_ready", +) + + +@asynccontextmanager +async def queued_production(backend, tmp_path, monkeypatch): + async with production_harness( + backend, + tmp_path / "destination.sqlite", + count=0, + notifications=True, + notification_delivery=True, + ) as h: + item, support = h.item, h.production + for subscriber, principal in ( + ("buyer", item.binding.consumer_id), + ("second", item.binding.consumer_id), + ("outsider", "https://buyer.example.test/other"), + ): + h.subscriptions.put( + notification_subscription( + subscriber=subscriber, + principal=principal, + events=EVENTS, + url="https://receiver.example.test/reporting", + ) + ) + h.subscriptions.put(notification_subscription(account="acct_b", events=EVENTS)) + blobs = _BytesStore(h.pool) + await blobs.create_schema() + receiver_store = DeterministicReceiverStore(blobs, h.notification_failures) + receiver = ScriptedNotificationReceiver( + SimpleNamespace( + clock=h.clock, failures=h.notification_failures, receiver=receiver_store + ) + ) + receiver.install(monkeypatch) + # The harness's original Core event was already expanded at startup, + # before registrations existed. Publish a new ordinary Core revision + # through the real ledger transaction after registering recipients. + core = replace(configuration(), delivery_config_id="ordinary-core") + await h.store.put_configuration(core) + obligation = await h.store.commit_obligation( + replace(obligation_for(core), reporting_obligation_id="ordinary-core-obligation") + ) + revision, rows = revision_for(obligation, suffix="ordinary-core") + await h.store.commit_revision(revision, rows) + await support.activate(account_id=item.config.account_id) + production_operation_1 = await support.materializer.run_once() + assert (production_operation_1).state == "verified" + assert item.writer.writes == 1 + for readable in (False, True): + await h.store.set_revision_readable( + account_id=item.config.account_id, + reporting_revision_id=item.revision.reporting_revision_id, + readable=readable, + ) + await drain(h.projection, item.config.account_id) + if h.pool is not None: + mount = MountedProduction(h) + mount.authorize(item) + async with mount.client() as client: + for transport in ("mcp", "a2a-0.3", "a2a-1.0"): + _, response = await mount.call( + client, "get_adcp_capabilities", {}, transport=transport + ) + reporting = response["media_buy"]["reporting_delivery"] + assert reporting["managed_delivery"] is True + assert response["webhook_signing"] == { + "supported": True, + "profile": "adcp/webhook-signing/v1", + "algorithms": ["ed25519"], + "legacy_hmac_fallback": False, + "delivery_retry_horizon_seconds": 86400, + } + assert [ + reporting[k] + for k in ( + "ledger_notification", + "status_notification", + "readiness_notification", + ) + ] == list(EVENTS) + workers = support.notification_workers + await support.aclose() + assert all([await w.outbox.list_events(account_id="acct_a") for w in workers]) + h.receiver, h.receiver_store = receiver, receiver_store + h.workers = workers + yield h + + +def fresh_workers(h): + return production_notification_workers( + h.store, + h.projection, + subscriptions=h.subscriptions, + cipher=h.workers[0].cipher, + signing=h.workers[0].signing, + ) + + +async def expire_queue_lease(h, worker, *, expansion=False): + if h.pool is None: + h.clock.advance(timedelta(seconds=61)) + return + table = ( + "reporting_notification_expansions" if expansion else "reporting_notification_deliveries" + ) + # The fixed SDK queue adapter maps this identifier to its actual queue. + async with worker.outbox._connection() as c: + await c.execute( + f"UPDATE {table} SET lease_expires_at=clock_timestamp()-interval '1 second'" + " WHERE account_id=%s AND state='leased'", + ("acct_a",), + ) + + +async def retained_windows(h): + if h.pool is None: + return tuple( + sorted((*key, *value) for key, value in h.store._production_delivery_windows.items()) + ) + async with h.pool.connection() as connection: + return tuple( + await ( + await connection.execute( + "SELECT account_id,idempotency_key,queue,body_sha256,started_at,expires_at" + " FROM reporting_production_delivery_windows" + " ORDER BY account_id,idempotency_key" + ) + ).fetchall() + ) + + +def use_clock(h, moment): + h.clock.now = moment + h.store._clock = h.clock + # The status queue is retained by the projector; the other two are + # reconstructed by the factory. All database clock seams must agree. + for worker in h.workers: + worker.outbox._clock = h.clock + + +async def pin_current_clock(h): + moment = h.clock() + if h.pool is not None: + from adcp.reporting.outbox.pg import database_now + + async with h.pool.connection() as connection: + moment = await database_now(connection, None) + use_clock(h, moment) + + +@pytest.mark.parametrize("backend", ["memory", "postgres"]) +@pytest.mark.parametrize("fault", [False, True], ids=["same-anchor", "reservation-fault"]) +async def test_retry_window_and_http_attempt_share_the_reservation_boundary( + backend, fault, tmp_path, monkeypatch +): + from adcp.reporting.ledger.notification_models import ReportingNotificationError + + async with queued_production(backend, tmp_path, monkeypatch) as h: + worker = h.workers[2] + await pin_current_clock(h) + production_operation_2 = await worker.expand_one(account_id="acct_a") + assert production_operation_2 + calls = 0 + with monkeypatch.context() as patch: + if h.pool is None: + if fault: + import adcp.reporting.outbox.memory as memory + + original = memory.token_hex + + def fail(*args): + nonlocal calls + calls += 1 + if calls == 2: + raise ReportingNotificationError("injected_reservation_failure") + return original(*args) + + patch.setattr(memory, "token_hex", fail) + else: + original = worker.outbox._next_attempt_on + + async def step(*args): + result = await original(*args) + if fault: + raise ReportingNotificationError("injected_reservation_failure") + # Deterministic time passes while the transaction reserves + # its ordinal. The persisted anchor must use fired_at. + h.clock.advance(timedelta(microseconds=1)) + return result + + patch.setattr(worker.outbox, "_next_attempt_on", step) + if fault: + with pytest.raises(ReportingNotificationError, match="injected_reservation"): + await worker.deliver_one(account_id="acct_a") + else: + h.notification_failures.at("http.accepted", SimulatedCrash()) + with pytest.raises(SimulatedCrash): + await worker.deliver_one(account_id="acct_a") + windows = await retained_windows(h) + activity = await worker.outbox.list_activity( + account_id="acct_a", consumer_id=h.item.binding.consumer_id + ) + if fault: + if h.pool is None: + assert calls == 2 # after lease claim, at the HTTP reservation + assert not windows, "failed HTTP reservation retained a first-attempt window" + assert not activity + assert not h.receiver.received + else: + assert len(windows) == len(activity) == 1 + assert windows[0][4] == activity[0].fired_at + + +@pytest.mark.parametrize("backend", ["memory", "postgres"]) +@pytest.mark.parametrize("queue", [0, 1, 2], ids=["core", "status-v2", "ready-v2"]) +async def test_window_insert_failure_rolls_back_attempt_head_and_all_state( + backend, queue, tmp_path, monkeypatch +): + from adcp.reporting.ledger.notification_models import ReportingNotificationError + from adcp.reporting.outbox.activity import ActivityRequest + + async with queued_production(backend, tmp_path, monkeypatch) as h: + await pin_current_clock(h) + worker = h.workers[queue] + production_operation_3 = await worker.expand_one(account_id="acct_a") + assert production_operation_3 + lease = await worker.outbox.claim_delivery( + account_id="acct_a", now=h.clock(), lease_seconds=60 + ) + assert lease is not None + with monkeypatch.context() as patch: + if h.pool is None: + + class FaultWindows(dict): + def setdefault(self, *args): + super().setdefault(*args) + raise ReportingNotificationError("injected_window_failure") + + patch.setattr(h.store, "_production_delivery_windows", FaultWindows()) + else: + original = worker.outbox._connection + + class FaultConnection: + def __init__(self, connection): + self.connection = connection + + def transaction(self): + return self.connection.transaction() + + async def execute(self, query, params=None): + result = await self.connection.execute(query, params) + if query.startswith("INSERT INTO reporting_production_delivery_windows"): + raise ReportingNotificationError("injected_window_failure") + return result + + @asynccontextmanager + async def connection(): + async with original() as bound: + yield FaultConnection(bound) + + patch.setattr(worker.outbox, "_connection", connection) + before = await h.image() + with pytest.raises(ReportingNotificationError, match="injected_window_failure"): + await worker.delivery_window.reserve_attempt( + worker.outbox, + lease, + request=ActivityRequest("https://receiver.example.test/reporting", 1), + now=h.clock(), + ) + assert await h.image() == before + assert not await retained_windows(h) + assert not await worker.outbox.list_activity( + account_id="acct_a", consumer_id=lease.delivery.binding.principal_id + ) + attempt, deadline, expired = await worker.delivery_window.reserve_attempt( + worker.outbox, + lease, + request=ActivityRequest("https://receiver.example.test/reporting", 1), + now=h.clock(), + ) + assert not expired and attempt.attempt == 1 + assert deadline == attempt.fired_at + timedelta(seconds=86400) + + +@pytest.mark.parametrize("backend", ["memory", "postgres"]) +@pytest.mark.parametrize("queue", [0, 1, 2], ids=["core", "status-v2", "ready-v2"]) +async def test_timeout_backoff_duplicate_workers_and_configuration_change_keep_first_deadline( + backend, queue, tmp_path, monkeypatch, caplog +): + import httpx + + async with queued_production(backend, tmp_path, monkeypatch) as h: + await pin_current_clock(h) + for key in tuple(h.subscriptions.values): + if key[0] == "acct_a" and key[1] != "buyer": + del h.subscriptions.values[key] + worker = h.workers[queue] + production_operation_4 = await worker.expand_one(account_id="acct_a") + assert production_operation_4 + h.receiver.responses["buyer"].extend([429, httpx.ReadTimeout("controlled timeout")]) + production_operation_5 = await worker.deliver_one(account_id="acct_a") + assert production_operation_5 + original = await retained_windows(h) + assert len(original) == 1 + first = (await worker.outbox.list_deliveries(account_id="acct_a"))[0] + assert first.state == "pending" + duplicate = fresh_workers(h)[queue] + production_operation_6 = await asyncio.gather( + worker.deliver_one(account_id="acct_a"), duplicate.deliver_one(account_id="acct_a") + ) + assert production_operation_6 == [False, False] + h.clock.advance(timedelta(seconds=5)) + # A reconstructed worker with a longer retry interval cannot postpone + # expiration or turn it into another day of delivery eligibility. + restarted = fresh_workers(h)[queue] + restarted.retry_seconds = 2 * 86400 + production_operation_7 = await restarted.deliver_one(account_id="acct_a") + assert production_operation_7 + assert await retained_windows(h) == original + await h.store.put_configuration(replace(h.item.config, deactivated_at=h.clock())) + h.signing.generation = 2 + use_clock(h, original[0][5] - timedelta(microseconds=1)) + production_operation_8 = await fresh_workers(h)[queue].deliver_one(account_id="acct_a") + assert not production_operation_8 + use_clock(h, original[0][5]) + current, duplicate = fresh_workers(h)[queue], fresh_workers(h)[queue] + production_condition_9 = sorted( + await asyncio.gather( + current.deliver_one(account_id="acct_a"), + duplicate.deliver_one(account_id="acct_a"), + ) + ) == [False, True] + assert production_condition_9 + final = (await current.outbox.list_deliveries(account_id="acct_a"))[0] + assert final.delivery == first.delivery + assert (final.state, final.error_code) == ("suppressed", "lease_expired") + assert await retained_windows(h) == original + activity = await current.outbox.list_activity( + account_id="acct_a", consumer_id=h.item.binding.consumer_id + ) + assert [r.outcome.status for r in activity] == ["timeout", "failed"] + assert [r.attempt for r in activity] == [2, 1] + assert len(h.receiver.received) == 1 + production_operation_10 = await current.deliver_one(account_id="acct_b") + assert not production_operation_10 + assert not caplog.records + + +@pytest.mark.parametrize("backend", ["memory", "postgres"]) +@pytest.mark.parametrize("queue", [0, 1, 2], ids=["core", "status-v2", "ready-v2"]) +async def test_worker_timeout_after_reservation_preserves_original_window_and_pending_activity( + backend, queue, tmp_path, monkeypatch +): + from ._reliable_support import Barrier + + async with queued_production(backend, tmp_path, monkeypatch) as h: + await pin_current_clock(h) + for key in tuple(h.subscriptions.values): + if key[0] == "acct_a" and key[1] != "buyer": + del h.subscriptions.values[key] + worker = h.workers[queue] + worker.lease_seconds = 1 + production_operation_11 = await worker.expand_one(account_id="acct_a") + assert production_operation_11 + barrier = Barrier() + h.notification_failures.at("http.before", barrier) + task = asyncio.create_task(worker.deliver_one(account_id="acct_a")) + try: + await barrier.wait() + original = await retained_windows(h) + assert len(original) == 1 + use_clock(h, original[0][5]) + production_operation_19 = await asyncio.wait_for(task, 3) + assert production_operation_19 + finally: + barrier.release() + if not task.done(): + task.cancel() + await asyncio.gather(task, return_exceptions=True) + restarted = fresh_workers(h)[queue] + production_operation_12 = await restarted.deliver_one(account_id="acct_a") + assert production_operation_12 + assert await retained_windows(h) == original + activity = await restarted.outbox.list_activity( + account_id="acct_a", consumer_id=h.item.binding.consumer_id + ) + assert len(activity) == 1 and activity[0].outcome is None + assert not h.receiver.received + assert (await restarted.outbox.list_deliveries(account_id="acct_a"))[ + 0 + ].state == "suppressed" + + +async def test_pg_deadline_crossed_while_reserving_an_ordinal_rolls_it_back(tmp_path, monkeypatch): + async with queued_production("postgres", tmp_path, monkeypatch) as h: + worker = h.workers[2] + production_operation_13 = await worker.expand_one(account_id="acct_a") + assert production_operation_13 + h.notification_failures.at("http.accepted", SimulatedCrash()) + with pytest.raises(SimulatedCrash): + await worker.deliver_one(account_id="acct_a") + original = await retained_windows(h) + use_clock(h, original[0][5] - timedelta(microseconds=1)) + restarted = fresh_workers(h)[2] + before = await restarted.outbox.list_activity( + account_id="acct_a", consumer_id=h.item.binding.consumer_id + ) + original_next = restarted.outbox._next_attempt_on + + async def cross_deadline(*args): + ordinal = await original_next(*args) + h.clock.advance(timedelta(microseconds=1)) + return ordinal + + monkeypatch.setattr(restarted.outbox, "_next_attempt_on", cross_deadline) + production_operation_14 = await restarted.deliver_one(account_id="acct_a") + assert production_operation_14 + assert len(h.receiver.received) == 1 + assert await retained_windows(h) == original + assert ( + await restarted.outbox.list_activity( + account_id="acct_a", consumer_id=h.item.binding.consumer_id + ) + == before + ) + + +@pytest.mark.parametrize("backend", ["memory", "postgres"]) +@pytest.mark.parametrize("queue", [0, 1, 2], ids=["core", "status-v2", "ready-v2"]) +@pytest.mark.parametrize("offset", [-1, 0, 1], ids=["before", "exact", "after"]) +async def test_retry_horizon_is_immutable_across_crash_rotation_and_restart( + backend, queue, offset, tmp_path, monkeypatch +): + async with queued_production(backend, tmp_path, monkeypatch) as h: + worker = h.workers[queue] + production_operation_15 = await worker.expand_one(account_id="acct_a") + assert production_operation_15 + h.notification_failures.at("http.accepted", SimulatedCrash()) + with pytest.raises(SimulatedCrash): + await worker.deliver_one(account_id="acct_a") + received = h.receiver.received[0] + original = await retained_windows(h) + assert len(original) == 1 + row = original[0] + assert row[1] == received.idempotency_key + assert (row[5] - row[4]).total_seconds() == 86400 + # Only the deterministic database clock seam is advanced. The immutable + # persisted timestamps/bytes are not edited to fabricate expiration. + use_clock(h, row[5] + timedelta(microseconds=offset)) + h.signing.generation = 2 + restarted = fresh_workers(h)[queue] + production_operation_16 = await restarted.deliver_one(account_id="acct_a") + assert production_operation_16 + received_count = len(h.receiver.received) + assert received_count == (2 if offset < 0 else 1) + assert await retained_windows(h) == original + target = next( + value + for value in await restarted.outbox.list_deliveries(account_id="acct_a") + if value.delivery.binding.idempotency_key == received.idempotency_key + ) + assert (target.state, target.error_code) == ( + ("complete", None) if offset < 0 else ("suppressed", "lease_expired") + ) + assert await h.receiver_store.read("acct_a", received.idempotency_key) == received.body + # Expiry cannot erase the original ambiguous attempt. Existing public + # account-activity projection and authenticated polling still recover + # the retained history; no fabricated late HTTP outcome is inserted. + from adcp.decisioning.accounts import ResolveContext + from adcp.decisioning.context import AuthInfo + from adcp.reporting.outbox import ReportingActivityProjector + + principal = h.subscriptions.values[("acct_a", received.subscriber_id)].principal_id + activity = ReportingActivityProjector(restarted.outbox) + rows = await activity.for_account( + account_id="acct_a", + context=ResolveContext(auth_info=AuthInfo(kind="bearer", principal=principal)), + ) + assert len(rows) == (2 if offset < 0 else 1) + assert ( + await activity.for_account( + account_id="acct_b", + context=ResolveContext(auth_info=AuthInfo(kind="bearer", principal=principal)), + ) + == [] + ) + mount = MountedProduction(h) + mount.authorize(h.item) + async with mount.client() as client: + for transport in ("mcp", "a2a-0.3", "a2a-1.0"): + _, polling = await mount.call( + client, + "get_reporting_status", + {"account": {"account_id": "acct_a"}, "view": "periods"}, + transport=transport, + ) + assert polling["status"] == "completed" + + +@pytest.mark.parametrize("backend", ["memory", "postgres"]) +@pytest.mark.parametrize("queue", [0, 1, 2], ids=["core", "status-v2", "ready-v2"]) +async def test_actual_queues_retry_exact_signed_bytes_after_acceptance_before_ack( + backend, queue, tmp_path, monkeypatch +): + async with queued_production(backend, tmp_path, monkeypatch) as h: + original_quarantine = await h.queue() + for worker in h.workers: + for _ in range(100): + if not await worker.expand_one(account_id="acct_a"): + break + else: + pytest.fail("production fanout did not reach a bounded idle state") + worker = h.workers[queue] + assert await worker.outbox.list_deliveries(account_id="acct_a") + h.notification_failures.at("http.accepted", SimulatedCrash()) + with pytest.raises(SimulatedCrash): + await worker.deliver_one(account_id="acct_a") + first = h.receiver.received[-1] + assert json.loads(first.body)["notification_type"] == EVENTS[queue] + assert await h.receiver_store.read("acct_a", first.idempotency_key) == first.body + await expire_queue_lease(h, worker) + h.signing.generation = 2 + for restarted in fresh_workers(h): + for _ in range(100): + if not await restarted.deliver_one(account_id="acct_a"): + break + else: + pytest.fail("production delivery did not reach a bounded idle state") + assert { + r.state for r in await restarted.outbox.list_deliveries(account_id="acct_a") + } == {"complete"} + assert not await restarted.outbox.list_deliveries(account_id="acct_b") + repeated = [r for r in h.receiver.received if r.idempotency_key == first.idempotency_key] + assert len(repeated) == 2 and repeated[0].body == repeated[1].body + assert "key-1" in repeated[0].headers["signature-input"] + assert "key-2" in repeated[1].headers["signature-input"] + options = WebhookVerifyOptions( + jwks_resolver=StaticJwksResolver({"keys": notification_verification_keys()}), + clock=lambda: h.clock().timestamp(), + ) + assert {json.loads(r.body)["notification_type"] for r in h.receiver.received} == set(EVENTS) + for received in h.receiver.received: + value = json.loads(received.body) + validate_notification_payload(value) + verify_webhook_signature( + method="POST", + url="https://receiver.example.test" + received.target, + headers=received.headers, + body=received.body, + options=options, + ) + if value["notification_type"] == "reporting.delivery_ready": + assert received.subscriber_id in {"buyer", "second"} + assert received.account_id == "acct_a" + assert await h.queue() == original_quarantine + assert h.item.writer.writes == 1 + + +@pytest.mark.parametrize("backend", ["memory", "postgres"]) +@pytest.mark.parametrize("queue", [0, 1, 2], ids=["core", "status-v2", "ready-v2"]) +async def test_new_queue_fanout_failure_rolls_back_all_recipients( + backend, queue, tmp_path, monkeypatch +): + async with queued_production(backend, tmp_path, monkeypatch) as h: + worker = h.workers[queue] + lease = await worker.outbox.claim_expansion( + account_id="acct_a", now=h.clock(), lease_seconds=60 + ) + assert lease is not None + from adcp.reporting.ledger.notification_models import decode_event + + event = decode_event(lease.event) + subscriptions = await h.subscriptions.list_active( + account_id="acct_a", notification_type=event.notification_type + ) + deliveries = tuple( + worker.cipher.prepare(event, s, lease.emission_generation) + for s in subscriptions + if s.matches(event) + ) + assert len(deliveries) >= 2 + before = await h.image() + with monkeypatch.context() as patch: + if h.pool is None: + import adcp.reporting.outbox.memory as memory + + original = memory._finish + + def fail(*args, **kwargs): + original(*args, **kwargs) + raise RuntimeError("injected fanout finish failure") + + patch.setattr(memory, "_finish", fail) + else: + original = worker.outbox._insert_delivery + + async def fail(*args, **kwargs): + await original(*args, **kwargs) + raise RuntimeError("injected fanout insert failure") + + patch.setattr(worker.outbox, "_insert_delivery", fail) + with pytest.raises(RuntimeError, match="injected fanout"): + await worker.outbox.complete_expansion(lease, deliveries, now=h.clock()) + assert await h.image() == before + assert not await worker.outbox.list_deliveries(account_id="acct_a") + await expire_queue_lease(h, worker, expansion=True) + restarted = fresh_workers(h)[queue] + production_operation_17 = await restarted.expand_one(account_id="acct_a") + assert production_operation_17 + assert len(await restarted.outbox.list_deliveries(account_id="acct_a")) == len(deliveries) + + +@pytest.mark.parametrize("backend", ["memory", "postgres"]) +@pytest.mark.parametrize("queue", [0, 1, 2], ids=["core", "status-v2", "ready-v2"]) +@pytest.mark.parametrize("mutation", ["algorithm", "legacy", "transient", "permanent", "cancel"]) +async def test_current_signing_failure_never_sends_or_exposes_private_data( + backend, queue, mutation, tmp_path, monkeypatch, caplog +): + from cryptography.hazmat.primitives.asymmetric import ec + + from adcp.reporting.outbox.routing import ( + ReportingLegacyAuthentication, + ReportingSigningMaterial, + ) + from adcp.webhook_sender import ScopePermanentlyUnknown, ScopeTransientlyUnavailable + + async with queued_production(backend, tmp_path, monkeypatch) as h: + worker = h.workers[queue] + production_operation_18 = await worker.expand_one(account_id="acct_a") + assert production_operation_18 + quarantine = await h.queue() + + async def incompatible(**kwargs): + if mutation == "transient": + raise ScopeTransientlyUnavailable() + if mutation == "permanent": + raise ScopePermanentlyUnknown() + if mutation == "cancel": + raise asyncio.CancelledError() + return ReportingSigningMaterial( + ec.derive_private_key(1, ec.SECP256R1()), + "https://seller.example.test/keys#changed", + "ecdsa-p256-sha256", + frozenset({"ecdsa-p256-sha256"}), + ) + + with monkeypatch.context() as patch: + if mutation == "legacy": + for key, subscription in tuple(h.subscriptions.values.items()): + h.subscriptions.values[key] = replace( + subscription, + signing_scope_id=None, + authentication=ReportingLegacyAuthentication("Bearer", "fixture-only"), + ) + else: + patch.setattr(h.signing, "resolve", incompatible) + if mutation == "cancel": + with pytest.raises(asyncio.CancelledError): + await worker.deliver_one(account_id="acct_a") + else: + production_operation_20 = await worker.deliver_one(account_id="acct_a") + assert production_operation_20 + assert not h.receiver.received + assert not caplog.records + deliveries = await worker.outbox.list_deliveries(account_id="acct_a") + assert all(r.state != "complete" for r in deliveries) + if mutation == "cancel": + assert sum(r.state == "leased" for r in deliveries) == 1 + elif mutation == "permanent": + assert sum(r.state == "quarantined" for r in deliveries) == 1 + else: + assert {r.state for r in deliveries} == {"pending"} + assert await h.queue() == quarantine diff --git a/tests/conformance/reporting/test_reporting_production_packaging.py b/tests/conformance/reporting/test_reporting_production_packaging.py new file mode 100644 index 000000000..40119eb53 --- /dev/null +++ b/tests/conformance/reporting/test_reporting_production_packaging.py @@ -0,0 +1,56 @@ +"""Python 3.10 VCS/sdist installed B2.4, with and without optional PostgreSQL.""" + +import asyncio +import os +import shutil + +import pytest + +from ._generation_support import require_rolling_database +from ._production_packaging import installed_production +from .test_reporting_materializer_packaging import b1_wheels, built_distribution +from .test_reporting_notification_packaging import run_step + +__all__ = ["b1_wheels", "built_distribution"] + + +@pytest.mark.parametrize("kind", ["vcs", "sdist"]) +@pytest.mark.parametrize("drivers", [False, True], ids=["base", "pg"]) +async def test_floor_installed_production_contract(request, kind, drivers): + interpreter = os.environ.get("ADCP_PYTHON310") + if interpreter is None: + pytest.skip("ADCP_PYTHON310 supplies the installed floor runtime") + if drivers: + require_rolling_database() + root, wheels, _ = request.getfixturevalue("b1_wheels") + _, _, source = request.getfixturevalue("built_distribution") + label = kind + ("-pg" if drivers else "-base") + environment = root / ("production-python310-" + label) + await asyncio.to_thread( + run_step, + [interpreter, "-m", "venv", str(environment)], + label=label + "-environment", + cwd=root, + ) + python = environment / "bin/python" + installer = ( + [shutil.which("uv"), "pip", "install", "--python", str(python)] + if shutil.which("uv") + else [str(python), "-m", "pip", "install"] + ) + await asyncio.to_thread( + run_step, + [*installer, str(wheels[kind]) + ("[pg]" if drivers else "")], + label=label + "-install", + cwd=root, + timeout=180, + ) + await asyncio.to_thread( + installed_production, + root, + python, + wheels[kind], + source, + label=label, + driver_absent=not drivers, + ) diff --git a/tests/conformance/reporting/test_reporting_production_progress.py b/tests/conformance/reporting/test_reporting_production_progress.py new file mode 100644 index 000000000..741076493 --- /dev/null +++ b/tests/conformance/reporting/test_reporting_production_progress.py @@ -0,0 +1,609 @@ +"""Bounded committed-period progress, with a settled first acquisition window.""" + +import asyncio +import hashlib +import json +import sqlite3 +import sys +from contextlib import asynccontextmanager +from dataclasses import replace +from datetime import timedelta + +import pytest + +from ._generation_support import END, START +from ._production_support import production_harness +from .test_reporting_production_lock_order import source_turn + + +async def sample_configurations(h, *, count=32): + selected = h.production.offerings[0] + blocked = [h.item.config] + [ + replace(h.item.config, delivery_config_id=f"busy-{i:02}") for i in range(count - 1) + ] + outside = replace(h.item.config, account_id="acct_b") + for configuration in [*blocked, outside]: + selected.producer._source.bind_generation(configuration) + binding = replace(h.item.binding, generation_key=configuration.generation_key) + h.item.writer.grant(binding) + await h.store.admit_production_configuration( + configuration, binding, offering_id=selected.offering_id + ) + for account in ("acct_a", "acct_b"): + await h.production.activate(account_id=account) + return blocked, outside + + +async def lease_source(support, worker, *, index=0): + token = support._producer_turn.set(support.offerings[index].producer) + try: + return await support.store.lease_period_close(worker_id=worker, now=END, lease_seconds=30) + finally: + support._producer_turn.reset(token) + + +def observe_samples(monkeypatch): + psycopg = pytest.importorskip("psycopg") + original = psycopg.AsyncConnection.execute + samples = [] + + async def execute(connection, query, *args, **kwargs): + result = await original(connection, query, *args, **kwargs) + if isinstance(query, str) and query.startswith( + "SELECT c.account_id,c.delivery_config_id,c.delivery_config_version," + ): + assert query.endswith("LIMIT 32") + assert 0 <= result.rowcount <= 32 + samples.append(result.rowcount) + return result + + monkeypatch.setattr(psycopg.AsyncConnection, "execute", execute) + return samples + + +async def bounded_turn(call, samples): + start = len(samples) + result = await asyncio.wait_for(call(), 5) + fetched = samples[start:] + assert 1 <= len(fetched) <= 2 and sum(fetched) <= 32 + return result + + +def account_image(image, account): + return { + table: [row for row in rows if row[0].get("account_id") == account] + for table, rows in image.items() + } + + +@pytest.mark.parametrize("notifications", [False, True]) +async def test_busy_account_window_advances_wraps_and_revisits_after_unlock( + notifications, tmp_path, monkeypatch +): + async with production_harness( + "postgres", + tmp_path / "destination.sqlite", + count=0, + source_publication=True, + second_source=True, + notifications=notifications, + ) as h: + support, store = h.production, h.store + blocked, outside = await sample_configurations(h) + other = replace(h.item.config, delivery_config_id="other-source") + unadmitted = replace(h.item.config, delivery_config_id="000-unadmitted") + await store.put_configuration(unadmitted) + support.offerings[1].producer._source.bind_generation(other) + binding = replace(h.item.binding, generation_key=other.generation_key) + h.item.writer.grant(binding) + await store.admit_production_configuration( + other, binding, offering_id=support.offerings[1].offering_id + ) + source = support.offerings[0].producer._source + samples = observe_samples(monkeypatch) + before = await h.image() + async with h.pool.connection() as holder, holder.transaction(): + await store._lock_account(holder, "acct_a") + first = await bounded_turn(lambda: source_turn(support), samples) + assert first.leased is None and samples[-1] == 32 + # Another producer has its own hint; its blocked turn cannot erase + # the first producer's progress beyond the full 32-row window. + production_operation_1 = await bounded_turn( + lambda: lease_source(support, "other", index=1), samples + ) + assert production_operation_1 is None + assert await h.image() == before + second = await bounded_turn(lambda: source_turn(support), samples) + assert second.leased is not None, "locked prefix hid the eligible second account" + assert second.leased.generation_key == outside.generation_key + assert len(second.obligations_committed) == len(second.revisions_committed) == 1 + assert not second.slices_failed + assert account_image(await h.image(), "acct_a") == account_image(before, "acct_a") + assert len(source.requests) == 1 + assert source.requests[0].identity.account_id == "acct_b" + assert not support.offerings[1].producer._source.requests + # Successful acquisition resets discovery to the durable rank. + production_operation_2 = await bounded_turn(lambda: source_turn(support), samples) + assert (production_operation_2).leased is None + held = await bounded_turn(lambda: lease_source(support, "tail-held"), samples) + assert held is not None and held.generation_key == outside.generation_key + held_image = await h.image() + production_operation_3 = await bounded_turn(lambda: source_turn(support), samples) + assert (production_operation_3).leased is None + start = len(samples) + production_operation_4 = await bounded_turn(lambda: source_turn(support), samples) + assert (production_operation_4).leased is None + assert samples[start:] == [0, 32] # empty tail wraps once, never an unbounded scan + assert await h.image() == held_image + try: + returned = await bounded_turn(lambda: source_turn(support), samples) + assert returned.leased.generation_key in {c.generation_key for c in blocked} + assert len(returned.obligations_committed) == len(returned.revisions_committed) == 1 + assert not returned.slices_failed + finally: + await store.release_period_close(held, worker_id="tail-held") + assert len(source.requests) == 2 + assert {r.identity.account_id for r in source.requests} == {"acct_a", "acct_b"} + assert len({r.identity.source_execution_key for r in source.requests}) == 2 + async with h.pool.connection() as c: + turns = await ( + await c.execute( + "SELECT account_id,delivery_config_id" + " FROM adcp_reporting_configuration_lease_turns" + " ORDER BY account_id,delivery_config_id" + ) + ).fetchall() + assert turns == [ + ("acct_a", returned.leased.delivery_config_id), + ("acct_b", outside.delivery_config_id), + ] + assert await ( + await c.execute("SELECT count(*) FROM reporting_production_source_probe_turns") + ).fetchone() == (0,) + assert await ( + await c.execute( + "SELECT count(*) FROM reporting_configurations" + " WHERE lease_worker_id IS NOT NULL" + ) + ).fetchone() == (0,) + assert not (await h.queue())[0] + print( + json.dumps( + { + "busy_account_progress": { + "notifications": notifications, + "bound": 32, + "blocked_configurations": 32, + "publications": 2, + "separate_producer_hints": True, + "wrap_and_unlock": True, + "blocked_state_unchanged": True, + } + } + ), + flush=True, + ) + + +@pytest.mark.parametrize("notifications", [False, True]) +async def test_fresh_stores_and_concurrent_workers_preserve_bounded_sampling_and_acquisition( + notifications, tmp_path, monkeypatch +): + psycopg = pytest.importorskip("psycopg") + path = tmp_path / "destination.sqlite" + async with production_harness( + "postgres", + path, + count=0, + source_publication=True, + notifications=notifications, + ) as h: + # Two fresh services each perform a real startup turn. Even after + # those acquisitions, more than a full window belongs to the busy A. + blocked, outside = await sample_configurations(h, count=65) + async with h.pool.connection() as holder, holder.transaction(): + await h.store._lock_account(holder, "acct_a") + production_operation_5 = await source_turn(h.production) + assert (production_operation_5).leased is None + await h.production.aclose() + bindings = [ + (c, h.production.offerings[0].offering_id, "catalog-7391") for c in [*blocked, outside] + ] + async with ( + production_harness( + "postgres", + path, + count=0, + source_publication=True, + notifications=notifications, + existing_pool=h.pool, + source_bindings=bindings, + ) as fresh, + production_harness( + "postgres", + path, + count=0, + source_publication=True, + notifications=notifications, + existing_pool=h.pool, + source_bindings=bindings, + ) as peer, + ): + assert fresh.store is not h.store and peer.store is not fresh.store + sources = [v.production.offerings[0].producer._source for v in (fresh, peer)] + assert [len(s.requests) for s in sources] == [1, 1] + before = await h.image() + samples = observe_samples(monkeypatch) + original = psycopg.AsyncConnection.execute + holding, release = asyncio.Event(), asyncio.Event() + winner = None + + async def coordinate(connection, query, *args, **kwargs): + result = await original(connection, query, *args, **kwargs) + if ( + asyncio.current_task() is winner + and isinstance(query, str) + and query.startswith("UPDATE reporting_configurations SET lease_worker_id") + ): + # The real statement, including its inherited trigger, + # has executed under the account lock, but not committed. + holding.set() + await asyncio.wait_for(release.wait(), 5) + return result + + monkeypatch.setattr(psycopg.AsyncConnection, "execute", coordinate) + async with h.pool.connection() as holder, holder.transaction(): + await h.store._lock_account(holder, "acct_a") + for current in (fresh, peer): + turn = await bounded_turn(lambda: source_turn(current.production), samples) + assert turn.leased is None and samples[-1] == 32 + assert await h.image() == before + winner = asyncio.create_task(source_turn(fresh.production)) + try: + await asyncio.wait_for(holding.wait(), 5) + losing = await bounded_turn(lambda: source_turn(peer.production), samples) + assert losing.leased is None and not losing.revisions_committed + # Observe committed rows without requesting the account + # lock deliberately held by the winning transaction. + async with h.pool.connection() as observer: + assert await ( + await observer.execute( + "SELECT count(*) FROM reporting_revisions WHERE account_id=%s", + ("acct_b",), + ) + ).fetchone() == (0,) + release.set() + won = await asyncio.wait_for(winner, 5) + finally: + release.set() + if not winner.done(): + winner.cancel() + await asyncio.gather(winner, return_exceptions=True) + assert won.leased.generation_key == outside.generation_key + assert len(won.obligations_committed) == len(won.revisions_committed) == 1 + assert not won.slices_failed + assert account_image(await h.image(), "acct_a") == account_image(before, "acct_a") + # The losing worker resumes; it cannot duplicate B's committed + # source execution or fabricate an acquisition for the held A. + repeat = await source_turn(peer.production) + assert not repeat.slices_failed + snapshot = await h.store.read_status_snapshot(account_id="acct_b") + assert len(snapshot.obligations) == len(snapshot.revisions) == 1 + requests = [r for s in sources for r in s.requests if r.identity.account_id == "acct_b"] + assert len(requests) == 1 + print( + json.dumps( + { + "busy_account_fresh_concurrent": { + "notifications": notifications, + "bound": 32, + "blocked_configurations": 65, + "fresh_store_instances": 2, + "coordinated_actual_trigger": True, + "second_account_publications": 1, + } + } + ), + flush=True, + ) + + +@pytest.mark.parametrize("backend", ["memory", "postgres"]) +async def test_rejected_source_window_does_not_starve_later_admitted_generation( + backend, tmp_path, monkeypatch +): + path = tmp_path / "destination.sqlite" + async with production_harness(backend, path, count=0, source_publication=True) as h: + support, store, item = h.production, h.store, h.item + source = support.offerings[0].producer._source + blocked = [replace(item.config, delivery_config_id=f"blocked-{i:02}") for i in range(32)] + for configuration in blocked: + source.bind_generation(configuration) + binding = replace(item.binding, generation_key=configuration.generation_key) + item.writer.grant(binding) + await store.admit_production_configuration( + configuration, binding, offering_id=support.offerings[0].offering_id + ) + outside = replace(item.config, account_id="acct_b") + await store.put_configuration(outside) + await support.activate(account_id=item.config.account_id) + for configuration in blocked: + del source.bindings[configuration.generation_key] + before = await store.list_configurations(account_id=item.config.account_id) + if h.pool is not None: + import psycopg + + before_image = await h.image() + original_execute = psycopg.AsyncConnection.execute + + async def fail_after_probe(connection, query, *args, **kwargs): + result = await original_execute(connection, query, *args, **kwargs) + if isinstance(query, str) and query.startswith( + "INSERT INTO reporting_production_source_probe_turns" + ): + raise RuntimeError("injected probe commit failure") + return result + + with monkeypatch.context() as patch: + patch.setattr(psycopg.AsyncConnection, "execute", fail_after_probe) + with pytest.raises(RuntimeError, match="injected probe commit failure"): + await source_turn(support) + assert await h.image() == before_image + first = await source_turn(support) + assert not first.slices_failed + if h.pool is not None: + async with h.pool.connection() as c: + probes = await ( + await c.execute( + "SELECT account_id,delivery_config_id" + " FROM reporting_production_source_probe_turns" + " ORDER BY account_id,delivery_config_id" + ) + ).fetchall() + assert probes == [(item.config.account_id, v.delivery_config_id) for v in blocked] + assert await ( + await c.execute("SELECT count(*) FROM adcp_reporting_configuration_lease_turns") + ).fetchone() == (0,) + await support.aclose() + # A fresh store/service in PG uses persisted rejection progress. Memory + # keeps its state across a new service, without claiming process durability. + async with production_harness( + backend, + path, + count=0, + source_publication=True, + existing_store=store if backend == "memory" else None, + existing_pool=h.pool, + ) as fresh: + second = await source_turn(fresh.production) + assert not second.slices_failed + # start() itself executes a bounded producer turn. Count that + # actual acquisition as well as the explicitly requested turn. + fresh_source = fresh.production.offerings[0].producer._source + assert len(first.revisions_committed) + len(fresh_source.requests) == 1 + for completed in (first, second): + if completed.leased is not None: + assert completed.leased.generation_key == item.config.generation_key + repeat = await source_turn(fresh.production) + assert not repeat.obligations_committed and not repeat.revisions_committed + snapshot = await fresh.store.read_status_snapshot(account_id=item.config.account_id) + assert len(snapshot.obligations) == len(snapshot.revisions) == 1 + assert snapshot.obligations[0].generation_key == item.config.generation_key + assert not (await fresh.store.read_status_snapshot(account_id="acct_b")).obligations + assert ( + await fresh.store.list_configurations(account_id=item.config.account_id) == before + ) + if h.pool is not None: + after_image = await fresh.image() + for table in ( + "reporting_production_generations", + "reporting_production_destination_bindings", + "reporting_materializer_work", + "reporting_materializer_notification_events", + ): + assert after_image[table] == before_image[table] + print( + json.dumps( + { + "rejected_window": 32, + "backend": backend, + "publications": 1, + "fresh_store": backend == "postgres", + } + ), + flush=True, + ) + + +def turn_document(turn, source): + assert not turn.slices_failed + return { + "obligations": turn.obligations_committed, + "revisions": turn.revisions_committed, + "executions": [r.identity.source_execution_key for r in source.requests], + } + + +@asynccontextmanager +async def restarted_process(h, path, *, pause): + from .test_reporting_materializer_process import Child + + class ProgressChild(Child): + async def event(self, point): + line = await asyncio.wait_for(self.process.stdout.readline(), 120) + assert line, f"producer exited before {point}; retained diagnostic: {log}" + result = json.loads(line) + assert result["point"] == point, result + return result + + log = path.with_name( + "producer-restart-paused.log" if pause else "producer-restart-finished.log" + ) + with log.open("wb") as diagnostic: + process = await asyncio.create_subprocess_exec( + sys.executable, + "-m", + "tests.conformance.reporting._production_progress_process", + stdin=asyncio.subprocess.PIPE, + stdout=asyncio.subprocess.PIPE, + stderr=diagnostic, + ) + child = ProgressChild(process) + try: + await child.send( + { + "conninfo": h.pool.conninfo, + "kwargs": h.pool.kwargs, + "path": str(path), + "pause": pause, + } + ) + yield child + finally: + await child.kill() + print( + json.dumps( + { + "producer_restart_diagnostic": str(log), + "bytes": log.stat().st_size, + "sha256": hashlib.sha256(log.read_bytes()).hexdigest(), + } + ), + flush=True, + ) + + +@pytest.mark.parametrize("backend", ["memory", "postgres"]) +async def test_bounded_producer_advances_past_processed_first_window(backend, tmp_path): + async with production_harness( + backend, tmp_path / "destination.sqlite", count=0, periods=131 + ) as h: + support, item = h.production, h.item + producer = support.offerings[0].producer + source = producer._source + assert producer._max_periods_per_turn == 64 + h.source_clock.advance(timedelta(hours=131)) + # Same caller-selected ID in another account and a different retained + # generation must not acquire a position just because this source runs. + untouched = ( + replace(item.config, account_id="acct_b"), + replace(item.config, delivery_config_version=2), + ) + for config in untouched: + await h.store.put_configuration(config) + await support.activate(account_id=item.config.account_id) + print( + json.dumps({"progress_backend": backend, "phase": "activated", "bound": 64}), flush=True + ) + first = await source_turn(support) + assert not first.slices_failed + # The first period was already published before activation. The + # remaining first window is acquired once, using the real sealed source. + assert len(first.obligations_committed) == 63 + assert len(first.revisions_committed) == 63 + assert len(source.requests) == 63 + first_document = turn_document(first, source) + print( + json.dumps({"progress_backend": backend, "phase": "first_window", "count": 64}), + flush=True, + ) + await support.aclose() + path = tmp_path / "destination.sqlite" + if backend == "memory": + # This is a new service/source/projector over the reference store's + # retained state, not a claim of memory durability across processes. + async with production_harness( + backend, path, count=0, periods=131, existing_store=h.store + ) as fresh: + fresh.source_clock.advance(timedelta(hours=131, seconds=61)) + new_source = fresh.production.offerings[0].producer._source + second = turn_document(await source_turn(fresh.production), new_source) + previous_requests = len(new_source.requests) + third = turn_document(await source_turn(fresh.production), new_source) + third["executions"] = third["executions"][previous_requests:] + repeat = await source_turn(fresh.production) + assert not repeat.obligations_committed and not repeat.revisions_committed + assert len(new_source.requests) == 67 + else: + async with restarted_process(h, path, pause=True) as child: + second = await child.event("committed_before_release") + async with h.pool.connection() as c: + row = await ( + await c.execute( + "SELECT c.lease_worker_id,p.closed_through" + " FROM reporting_configurations c" + " JOIN reporting_production_source_progress p" + " USING(account_id,delivery_config_id,delivery_config_version)" + " WHERE c.account_id=%s AND c.delivery_config_id=%s" + " AND c.delivery_config_version=%s", + ( + item.config.account_id, + item.config.delivery_config_id, + item.config.delivery_config_version, + ), + ) + ).fetchone() + assert row[0] is not None and row[1] == START + timedelta(hours=128) + await child.kill() + assert child.process.returncode == -9 + async with restarted_process(h, path, pause=False) as child: + third = await child.event("done") + production_operation_6 = await asyncio.wait_for(child.process.wait(), 10) + assert production_operation_6 == 0 + assert len(second["obligations"]) == len(second["revisions"]) == 64 + assert len(third["obligations"]) == len(third["revisions"]) == 3 + executions = first_document["executions"] + second["executions"] + third["executions"] + assert len(executions) == len(set(executions)) == 130 + snapshot = await h.store.read_status_snapshot(account_id=item.config.account_id) + assert len(snapshot.obligations) == len(snapshot.revisions) == 131 + assert max(o.period.end for o in snapshot.obligations) == START + timedelta(hours=131) + assert {o.generation_key for o in snapshot.obligations} == {item.config.generation_key} + assert not (await h.store.read_status_snapshot(account_id="acct_b")).obligations + assert ( + await h.store.get_revision( + account_id=item.config.account_id, + reporting_revision_id=item.revision.reporting_revision_id, + ) + == item.revision + ) + with sqlite3.connect(tmp_path / "source.seals") as connection: + assert connection.execute("SELECT count(*) FROM seals").fetchone() == (130,) + if h.pool is None: + assert h.store._production_closed == { + item.config.generation_key: START + timedelta(hours=131) + } + assert {w.state for w in h.store._production_source_work.values()} == {"settled"} + assert not {c.generation_key for c in untouched} & set(h.store._production_closed) + else: + async with h.pool.connection() as c: + assert await ( + await c.execute( + "SELECT account_id,delivery_config_id,delivery_config_version," + " closed_through" + " FROM reporting_production_source_progress" + ) + ).fetchall() == [ + ( + item.config.account_id, + item.config.delivery_config_id, + 1, + START + timedelta(hours=131), + ) + ] + assert await ( + await c.execute( + "SELECT state,count(*) FROM reporting_production_source_work GROUP BY state" + ) + ).fetchall() == [("settled", 131)] + print( + json.dumps( + { + "progress_backend": backend, + "periods": 131, + "bound": 64, + "unique_acquisitions": 130, + "restart": "SIGKILL" if h.pool else "new-service", + } + ), + flush=True, + ) diff --git a/tests/conformance/reporting/test_reporting_production_readiness.py b/tests/conformance/reporting/test_reporting_production_readiness.py new file mode 100644 index 000000000..831327c36 --- /dev/null +++ b/tests/conformance/reporting/test_reporting_production_readiness.py @@ -0,0 +1,389 @@ +"""Actual route identity, positive schema proof and optional delivery lifecycle.""" + +import asyncio + +import pytest + +from adcp.reporting.ledger.notification_models import ReportingNotificationError +from adcp.server.a2a_server import create_a2a_server + +from ._production_support import production_harness +from ._production_transport import MountedProduction + + +@pytest.mark.parametrize("transport", ["mcp", "a2a"]) +@pytest.mark.parametrize("mutation", ["remove", "replace"]) +async def test_warm_proof_rechecks_the_actual_mount(transport, mutation, tmp_path): + async with production_harness("postgres", tmp_path / "destination.sqlite") as h: + support = h.production + app = create_a2a_server(support.handler) if transport == "a2a" else h.mount + proof = next(p for p in support._mounts if p.mount() is app) + dispatcher = proof.dispatcher() + mapping = ( + dispatcher._tool_manager._tools if transport == "mcp" else dispatcher._tool_callers + ) + original = mapping.pop("get_reporting_status") + if mutation == "replace": + mapping["get_reporting_status"] = mapping["get_adcp_capabilities"] + try: + production_operation_5 = await support.reporting_delivery() + assert production_operation_5 == {} + with pytest.raises(ReportingNotificationError, match="component_unready"): + await support.activate(account_id=h.item.config.account_id) + finally: + mapping["get_reporting_status"] = original + production_operation_1 = await support.reporting_delivery() + assert (production_operation_1)["managed_delivery"] is True + + +async def test_shared_scan_cancellation_and_post_scan_dynamic_check(tmp_path, monkeypatch): + import adcp.reporting.production.schema as schema + + async with production_harness("postgres", tmp_path / "destination.sqlite") as h: + support = h.production + original = schema.validate_production_schema + entered, release = asyncio.Event(), asyncio.Event() + scans = 0 + + async def paused(*args, **kwargs): + nonlocal scans + scans += 1 + entered.set() + await asyncio.wait_for(release.wait(), 5) + await original(*args, **kwargs) + + support.invalidate_schema_validation() + with monkeypatch.context() as patch: + patch.setattr(schema, "validate_production_schema", paused) + canceled = asyncio.create_task(support.reporting_delivery()) + await asyncio.wait_for(entered.wait(), 5) + callers = [asyncio.create_task(support.reporting_delivery()) for _ in range(4)] + canceled.cancel() + with pytest.raises(asyncio.CancelledError): + await canceled + removed = h.mount._tool_manager._tools.pop("get_reporting_status") + try: + release.set() + production_operation_8 = await asyncio.wait_for(asyncio.gather(*callers), 10) + assert production_operation_8 == [{}] * 4 + finally: + h.mount._tool_manager._tools["get_reporting_status"] = removed + assert scans == 1 + production_operation_6 = await support.reporting_delivery() + assert (production_operation_6)["managed_delivery"] is True + await h.store.materializer_ready() + assert scans == 1 + + +@pytest.mark.parametrize("notifications", [False, True]) +@pytest.mark.parametrize("mutation", ["closed", "replaced"]) +@pytest.mark.parametrize("reconciled", [False, True]) +async def test_warm_mounted_capability_rechecks_pool_lifecycle_and_identity( + notifications, mutation, reconciled, tmp_path, monkeypatch +): + from contextlib import AsyncExitStack + + import adcp.reporting.production.schema as schema + + pool_type = pytest.importorskip("psycopg_pool").AsyncConnectionPool + async with production_harness( + "postgres", + tmp_path / "destination.sqlite", + notifications=notifications, + notification_delivery=notifications, + reconciled=reconciled, + ) as h: + support = h.production + mounted = MountedProduction(h) + mounted.authorize(h.item) + async with mounted.client() as client, AsyncExitStack() as stack: + before = {} + for transport in ("mcp", "a2a-0.3", "a2a-1.0"): + _, before[transport] = await mounted.call( + client, "get_adcp_capabilities", {}, transport=transport + ) + assert before[transport]["media_buy"]["reporting_delivery"]["managed_delivery"] + if reconciled: + assert before[transport]["media_buy"]["reporting_delivery"][ + "reconciled_billing" + ] + replacement = None + if mutation == "replaced": + replacement = await stack.enter_async_context( + pool_type( + h.pool.conninfo, kwargs=h.pool.kwargs, min_size=1, max_size=2, open=False + ) + ) + await replacement.wait(timeout=5) + assert replacement.closed is False + else: + await h.pool.close() + assert h.pool.closed is True + assert not support._task.done() + scans = 0 + + async def no_scan(*args, **kwargs): + nonlocal scans + scans += 1 + raise AssertionError("component refusal must precede a catalog scan") + + with monkeypatch.context() as patch: + patch.setattr(schema, "validate_production_schema", no_scan) + if replacement is not None: + patch.setattr(h.store, "_pool", replacement) + for transport in before: + _, refused = await mounted.call( + client, "get_adcp_capabilities", {}, transport=transport + ) + assert "reporting_delivery" not in refused.get("media_buy", {}) + assert "webhook_signing" not in refused + expected_code = ( + "notification_chain_unready" + if replacement is not None and notifications + else "reporting_production_component_unready" + ) + with pytest.raises(ReportingNotificationError, match=expected_code): + await support.activate(account_id="acct_a") + assert scans == 0 + if mutation == "replaced": + for transport in before: + _, restored = await mounted.call( + client, "get_adcp_capabilities", {}, transport=transport + ) + assert restored == before[transport] + + +@pytest.mark.parametrize("queue", ["projection", "core", "ready"]) +@pytest.mark.parametrize("replacement_open", [False, True]) +async def test_warm_mounted_capability_rechecks_each_queue_pool( + queue, replacement_open, tmp_path, monkeypatch +): + from contextlib import AsyncExitStack + + import adcp.reporting.production.schema as schema + + pool_type = pytest.importorskip("psycopg_pool").AsyncConnectionPool + # The projection queue also participates without optional HTTP workers. + delivery = queue != "projection" + async with production_harness( + "postgres", + tmp_path / "destination.sqlite", + notifications=True, + notification_delivery=delivery, + reconciled=True, + ) as h: + mount = MountedProduction(h) + mount.authorize(h.item) + outbox = ( + h.projection.outbox + if queue == "projection" + else h.production.notification_workers[0 if queue == "core" else 2].outbox + ) + replacement = pool_type( + h.pool.conninfo, kwargs=h.pool.kwargs, min_size=1, max_size=1, open=False + ) + async with mount.client() as client, AsyncExitStack() as stack: + if replacement_open: + await stack.enter_async_context(replacement) + await replacement.wait(timeout=5) + assert replacement.closed is not replacement_open + baseline = {} + for transport in ("mcp", "a2a-0.3", "a2a-1.0"): + _, baseline[transport] = await mount.call( + client, "get_adcp_capabilities", {}, transport=transport + ) + assert baseline[transport]["media_buy"]["reporting_delivery"]["reconciled_billing"] + scans = 0 + + async def no_scan(*args, **kwargs): + nonlocal scans + scans += 1 + raise AssertionError("queue wiring checks must precede catalog proof") + + with monkeypatch.context() as patch: + patch.setattr(schema, "validate_production_schema", no_scan) + patch.setattr(outbox, "_pool", replacement) + for transport in baseline: + _, refused = await mount.call( + client, "get_adcp_capabilities", {}, transport=transport + ) + assert "reporting_delivery" not in refused.get("media_buy", {}) + assert "webhook_signing" not in refused + assert scans == 0 + for transport in baseline: + _, restored = await mount.call( + client, "get_adcp_capabilities", {}, transport=transport + ) + assert restored == baseline[transport] + + +@pytest.mark.parametrize("notifications", [False, True]) +async def test_warm_mounted_discovery_does_not_checkout_a_saturated_valid_pool( + notifications, tmp_path, monkeypatch +): + import adcp.reporting.production.schema as schema + + async with production_harness( + "postgres", + tmp_path / "destination.sqlite", + notifications=notifications, + notification_delivery=notifications, + reconciled=True, + ) as h: + mounted = MountedProduction(h) + mounted.authorize(h.item) + async with mounted.client() as client: + expected = {} + for transport in ("mcp", "a2a-0.3", "a2a-1.0"): + _, expected[transport] = await mounted.call( + client, "get_adcp_capabilities", {}, transport=transport + ) + assert expected[transport]["media_buy"]["reporting_delivery"]["reconciled_billing"] + connections = [] + try: + for _ in range(h.pool.max_size): + connections.append(await h.pool.getconn(timeout=5)) + assert h.pool.get_stats()["pool_available"] == 0 + assert h.pool.closed is False + scans = checkouts = 0 + + async def no_scan(*args, **kwargs): + nonlocal scans + scans += 1 + raise AssertionError("warm discovery must reuse the catalog proof") + + def no_connection(*args, **kwargs): + nonlocal checkouts + checkouts += 1 + raise AssertionError("warm discovery must not request a pool connection") + + with monkeypatch.context() as patch: + patch.setattr(schema, "validate_production_schema", no_scan) + patch.setattr(h.pool, "connection", no_connection) + for transport in expected: + _, result = await asyncio.wait_for( + mounted.call(client, "get_adcp_capabilities", {}, transport=transport), + 5, + ) + assert result == expected[transport] + assert scans == checkouts == 0 + finally: + for connection in connections: + await h.pool.putconn(connection) + + +@pytest.mark.parametrize("backend", ["memory", "postgres"]) +async def test_optional_delivery_is_owned_and_broken_enabled_chain_fails_closed(backend, tmp_path): + async with production_harness( + backend, tmp_path / "destination.sqlite", notifications=True, notification_delivery=True + ) as h: + support = h.production + assert support._notification_task is not None + assert not support._notification_task.done() + if backend == "postgres": + fields = await support.reporting_delivery() + assert fields["managed_delivery"] is True + assert fields["ledger_notification"] == "reporting.ledger_changed" + assert fields["readiness_notification"] == "reporting.delivery_ready" + assert fields["status_notification"] == "reporting.status_changed" + worker = support.notification_workers[-1] + outbox = worker.outbox + worker.outbox = support.notification_workers[0].outbox + try: + production_operation_7 = await support.reporting_delivery() + assert production_operation_7 == {} + with pytest.raises(ReportingNotificationError, match="notification_chain_unready"): + await support.activate(account_id=h.item.config.account_id) + finally: + worker.outbox = outbox + await support.activate(account_id=h.item.config.account_id) + assert h.subscriptions.lists + assert {event for _, event in h.subscriptions.lists} == { + "reporting.ledger_changed", + "reporting.status_changed", + "reporting.delivery_ready", + } + await support.aclose() + assert support._notification_task is None + production_operation_2 = await support.reporting_delivery() + assert production_operation_2 == {} + + +async def test_warm_catalog_proof_does_not_cache_signing_wiring(tmp_path, monkeypatch): + import adcp.reporting.production.schema as schema + + async with production_harness( + "postgres", tmp_path / "destination.sqlite", notifications=True, notification_delivery=True + ) as h: + support = h.production + production_operation_3 = await support.reporting_delivery() + assert (production_operation_3)["managed_delivery"] + mount = MountedProduction(h) + mount.authorize(h.item) + + async def no_scan(*args, **kwargs): + pytest.fail("warm immutable catalog proof was unnecessarily repeated") + + with monkeypatch.context() as patch: + patch.setattr(schema, "validate_production_schema", no_scan) + async with mount.client() as client: + for transport in ("mcp", "a2a-0.3", "a2a-1.0"): + _, valid = await mount.call( + client, "get_adcp_capabilities", {}, transport=transport + ) + assert valid["webhook_signing"]["algorithms"] == ["ed25519"] + # These are normative 3.2 obligations even though the + # additive schema fields remain optional for old agents. + assert valid["webhook_signing"]["delivery_retry_horizon_seconds"] == 86400 + assert valid["identity"]["brand_json_url"] == ( + "https://seller.example.test/brand.json" + ) + # Each mutation follows a successful schema proof. Every + # request must inspect the current concrete participant. + for target, name, replacement in ( + (support.notification_workers[0], "signing", h.signing), + (h.signing, "resolve", None), + (h.subscriptions, "get_active", None), + ): + with monkeypatch.context() as mutation: + mutation.setattr(target, name, replacement) + _, invalid = await mount.call( + client, "get_adcp_capabilities", {}, transport=transport + ) + assert "webhook_signing" not in invalid + assert "reporting_delivery" not in invalid.get("media_buy", {}) + with pytest.raises(ReportingNotificationError): + await support.activate(account_id="acct_a") + _, restored = await mount.call( + client, "get_adcp_capabilities", {}, transport=transport + ) + assert restored == valid + + +@pytest.mark.parametrize("backend", ["memory", "postgres"]) +async def test_current_key_contract_is_rechecked_before_account_activation( + backend, tmp_path, monkeypatch, caplog +): + from dataclasses import replace + + async with production_harness( + backend, tmp_path / "destination.sqlite", notifications=True, notification_delivery=True + ) as h: + from ._reliable_support import notification_subscription + + h.subscriptions.put(notification_subscription(principal=h.item.binding.consumer_id)) + original = h.signing.resolve + + async def incompatible(**kwargs): + material = await original(**kwargs) + return replace(material, advertised_algorithms={"ed25519", "ecdsa-p256-sha256"}) + + before = await h.image() + with monkeypatch.context() as patch: + patch.setattr(h.signing, "resolve", incompatible) + with pytest.raises(ReportingNotificationError, match="notification_chain_unready"): + await h.production.activate(account_id="acct_a") + assert await h.image() == before + assert not caplog.records + production_operation_4 = await h.production.activate(account_id="acct_a") + assert production_operation_4 diff --git a/tests/conformance/reporting/test_reporting_production_restart.py b/tests/conformance/reporting/test_reporting_production_restart.py new file mode 100644 index 000000000..b4c39f416 --- /dev/null +++ b/tests/conformance/reporting/test_reporting_production_restart.py @@ -0,0 +1,110 @@ +"""Real database/process restart of first-attempt windows in all three queues.""" + +import asyncio +import hashlib +import json +import sys +from contextlib import asynccontextmanager +from datetime import datetime, timedelta +from pathlib import Path + +import pytest + +import adcp.reporting.production.delivery_window as window_module + +from .test_reporting_materializer_process import Child +from .test_reporting_production_notifications import ( + pin_current_clock, + queued_production, + retained_windows, +) + + +@asynccontextmanager +async def delivery_child(h, path, *, queue, pause, at): + origin = Path(window_module.__file__).resolve() + log = path / ("accepted-crash.log" if pause else "cold-retry.log") + with log.open("wb") as diagnostic: + process = await asyncio.create_subprocess_exec( + sys.executable, + "-m", + "tests.conformance.reporting._production_delivery_process", + stdin=asyncio.subprocess.PIPE, + stdout=asyncio.subprocess.PIPE, + stderr=diagnostic, + ) + child = Child(process) + try: + await child.send( + { + "conninfo": h.pool.conninfo, + "kwargs": h.pool.kwargs, + "queue": queue, + "pause": pause, + "at": at.isoformat(), + "consumer": h.item.binding.consumer_id, + "installed": "site-packages" in str(origin), + "module_sha256": hashlib.sha256(origin.read_bytes()).hexdigest(), + } + ) + yield child + finally: + await child.kill() + print( + json.dumps( + { + "production_delivery_process_log": str(log), + "bytes": log.stat().st_size, + "sha256": hashlib.sha256(log.read_bytes()).hexdigest(), + } + ), + flush=True, + ) + + +@pytest.mark.parametrize("queue", [0, 1, 2], ids=["core", "status-v2", "ready-v2"]) +@pytest.mark.parametrize("offset", [-1, 0], ids=["before", "exact"]) +async def test_pg_sigkill_after_accepted_http_keeps_first_retry_window( + queue, offset, tmp_path, monkeypatch +): + async with queued_production("postgres", tmp_path, monkeypatch) as h: + await pin_current_clock(h) + # This process receives exactly this registration. Multi-recipient + # expansion/rollback is exercised separately; retaining registrations + # unknown to this child would suppress a different first delivery. + for key in tuple(h.subscriptions.values): + if key[0] == "acct_a" and key[1] != "buyer": + del h.subscriptions.values[key] + production_operation_1 = await h.workers[queue].expand_one(account_id="acct_a") + assert production_operation_1 + async with delivery_child(h, tmp_path, queue=queue, pause=True, at=h.clock()) as child: + first = await child.event("accepted_before_ack") + assert len(await retained_windows(h)) == 1 + await child.kill() + assert child.process.returncode == -9 + saved = await retained_windows(h) + at = datetime.fromisoformat(first["expires_at"]) + timedelta(microseconds=offset) + async with delivery_child(h, tmp_path, queue=queue, pause=False, at=at) as child: + restored = await child.event("done") + production_operation_2 = await asyncio.wait_for(child.process.wait(), 10) + assert production_operation_2 == 0 + assert restored["http_calls"] == int(offset < 0) + assert restored["activity_count"] == (2 if offset < 0 else 1) + assert (restored["state"], restored["error_code"]) == ( + ("complete", None) if offset < 0 else ("suppressed", "lease_expired") + ) + for key in ("key_sha256", "body_sha256", "started_at", "expires_at", "origin"): + assert restored[key] == first[key] + assert await retained_windows(h) == saved + print( + json.dumps( + { + "production_delivery_cold_restart": { + "queue": queue, + "offset_microseconds": offset, + **restored, + } + } + ), + flush=True, + ) diff --git a/tests/conformance/reporting/test_reporting_production_rolling.py b/tests/conformance/reporting/test_reporting_production_rolling.py new file mode 100644 index 000000000..dda49a44a --- /dev/null +++ b/tests/conformance/reporting/test_reporting_production_rolling.py @@ -0,0 +1,410 @@ +"""Actual approved B2.3 and hardening binaries across installed B2.4 activation.""" + +import asyncio +import hashlib +import json +import os +import shutil +import subprocess +import zipfile +from contextlib import asynccontextmanager +from dataclasses import asdict, replace +from datetime import timedelta +from pathlib import Path + +import pytest + +from adcp.reporting.ledger import revision_content_sha256 +from adcp.reporting.ledger.models import derive_period + +from ._feed_support import feed_harness, feed_request, mixed_case, walk +from ._production_packaging import copied_fixtures, inspect_distribution, source_basis +from ._production_support import production_harness +from .test_reporting_feed_hardening_installed import approved_b23 +from .test_reporting_feed_installed_pg import ( + b1_wheels, + built_distribution, + feed_wheels, + installed_feed, +) +from .test_reporting_feed_process import feed_process +from .test_reporting_materializer_process import Child +from .test_reporting_materializer_rolling import build_frozen +from .test_reporting_notification_packaging import ROOT, run_step + +__all__ = ["approved_b23", "b1_wheels", "built_distribution", "feed_wheels", "installed_feed"] +HARDENING = "e16eb8cf3074cabd45aab42840950f05ad6d2b43" + + +@pytest.fixture(scope="module", params=["b23", "hardening"]) +def production_parent(request, tmp_path_factory): + if request.param == "b23": + return request.getfixturevalue("approved_b23") + root, _, _, identity = build_frozen("hardening-b24", tmp_path_factory, request, sha=HARDENING) + interpreter = os.environ.get("ADCP_PYTHON310") + if interpreter is None: + pytest.skip("ADCP_PYTHON310 supplies the installed floor artifact") + environment = root / "python310" + run_step( + [interpreter, "-m", "venv", str(environment)], label="hardening-floor-environment", cwd=root + ) + python = environment / "bin/python" + wheel = next((root / "dist").glob("*.whl")) + installer = ( + [shutil.which("uv"), "pip", "install", "--python", str(python)] + if shutil.which("uv") + else [str(python), "-m", "pip", "install"] + ) + run_step( + [*installer, f"{wheel}[pg]", "asgi-lifespan==2.1.0"], + label="hardening-floor-install", + cwd=root, + timeout=180, + ) + from .test_reporting_feed_packaging import feed_modules + + with zipfile.ZipFile(wheel) as archive: + modules = {} + for name in {*feed_modules(), "adcp.reporting.outbox.status_pg"}: + member = name.replace(".", "/") + ".py" + if member not in archive.namelist(): + member = name.replace(".", "/") + "/__init__.py" + raw = archive.read(member) + assert raw == subprocess.check_output( + ["git", "show", f"{HARDENING}:src/{member}"], cwd=ROOT + ) + modules[name] = hashlib.sha256(raw).hexdigest() + script, helper = root / "feed_process.py", root / "receipt_transport.py" + shutil.copy2(Path(__file__).with_name("_feed_process.py"), script) + shutil.copy2(Path(__file__).with_name("_receipt_transport.py"), helper) + return ( + root, + python, + script, + helper, + { + **identity, + "modules": modules, + "python": [3, 10], + "tree": "c043d1e14c5071859f566e07cc9980058fa6ee07", + }, + wheel, + ) + + +@pytest.fixture(scope="module") +def production_install(installed_feed, feed_wheels, built_distribution, production_parent): + root, python, _, _, current = installed_feed + parent_label = production_parent[4]["sha"][:12] + label = parent_label + "-" + current["distribution"] + _, wheels, _ = feed_wheels + _, _, source = built_distribution + modules, assets = inspect_distribution(wheels[current["distribution"]], source) + fixture_root = copied_fixtures(root, label + "-restart") + script = fixture_root / "production_process.py" + shutil.copy2(Path(__file__).with_name("_production_installed_process.py"), script) + installer = ( + [shutil.which("uv"), "pip", "install", "--python", str(python)] + if shutil.which("uv") + else [str(python), "-m", "pip", "install"] + ) + run_step( + [*installer, "pytest==9.1.1", "pytest-asyncio==1.4.0", "respx==0.23.1"], + label="production-process-fixtures", + cwd=root, + timeout=180, + ) + return ( + python, + script, + { + **current, + "fixtures": str(fixture_root), + "modules": modules, + "assets": assets, + "source_basis": source_basis( + wheels[current["distribution"]], + source, + evidence=Path(os.environ.get("ADCP_PRODUCTION_EVIDENCE", root / "evidence")), + label=label + "-rolling", + ), + }, + ) + + +@asynccontextmanager +async def installed_child(python, script, settings, path): + log = path / settings.get( + "diagnostic_name", "activation.log" if settings["pause"] else "continuation.log" + ) + try: + with log.open("xb") as diagnostic: + process = await asyncio.create_subprocess_exec( + str(python), + "-I", + str(script), + stdin=asyncio.subprocess.PIPE, + stdout=asyncio.subprocess.PIPE, + stderr=diagnostic, + ) + + class ActivationChild(Child): + async def event(self, point): + line = await asyncio.wait_for(self.process.stdout.readline(), 120) + assert ( + line + ), f"installed process exited before {point}; diagnostic retained at {log}" + result = json.loads(line) + assert result["point"] == point + return result + + child = ActivationChild(process) + try: + await child.send(settings) + yield child + finally: + await child.kill() + finally: + raw = log.read_bytes() + retained = None + if os.environ.get("ADCP_PRODUCTION_EVIDENCE"): + evidence = Path(os.environ["ADCP_PRODUCTION_EVIDENCE"]) + evidence.mkdir(parents=True, exist_ok=True, mode=0o700) + retained = evidence / (settings["evidence_key"] + "-" + log.name) + with retained.open("xb") as stream: + stream.write(raw) + print( + json.dumps( + { + "installed_activation_process_log": str(log), + "retained_log": str(retained) if retained else None, + "bytes": len(raw), + "sha256": hashlib.sha256(raw).hexdigest(), + } + ), + flush=True, + ) + + +@pytest.mark.parametrize("notifications", [False, True]) +async def test_actual_parent_page_one_to_installed_activation_sigkill_and_complete_walk( + production_parent, production_install, notifications, tmp_path +): + old_root, old_python, old_script, old_helper, old, old_wheel = production_parent + python, script, current = production_install + evidence_key = f"{old['sha'][:12]}-{current['distribution']}-{int(notifications)}" + async with feed_harness("postgres", notifications=notifications) as h: + case, receipt_request, receipt_response = await mixed_case(h) + # Seed public ordinary records on the parent schema. The memory fixture + # supplies only deterministic input values and a provider grant; the + # installed parent, below, creates the actual attempt and durable work. + async with production_harness( + "memory", + tmp_path / "destination.sqlite", + notifications=notifications, + count=0, + reconciled=True, + identity_prefix="b24-", + ) as seed: + item = seed.item + await h.store.put_configuration(item.config) + await h.store.commit_obligation(item.obligation) + await h.store.put_destination_binding(item.binding) + await h.store.commit_revision(item.revision, item.rows) + key = asdict(item.verifier.key) + legacy_script = old_root / "production_legacy_process.py" + shutil.copy2(Path(__file__).with_name("_production_legacy_process.py"), legacy_script) + legacy_module = subprocess.check_output( + ["git", "show", f"{old['sha']}:src/adcp/reporting/outbox/status_pg.py"], cwd=ROOT + ) + materializer_module = subprocess.check_output( + ["git", "show", f"{old['sha']}:src/adcp/reporting/materializer/pg.py"], cwd=ROOT + ) + legacy_settings = { + "conninfo": h.pool.conninfo, + "kwargs": h.pool.kwargs, + "notifications": notifications, + "module_sha256": hashlib.sha256(legacy_module).hexdigest(), + "materializer_module_sha256": hashlib.sha256(materializer_module).hexdigest(), + "verification_key": key, + "evidence_key": evidence_key, + } + async with installed_child( + old_python, + legacy_script, + { + **legacy_settings, + "pending": True, + "pause": True, + "diagnostic_name": "historical-pending.log", + "revision_id": item.revision.reporting_revision_id, + }, + tmp_path, + ) as child: + pending = await child.event("pending") + await child.kill() + assert child.process.returncode == -9 + async with h.pool.connection() as connection: + assert ( + await ( + await connection.execute("SELECT to_regclass('reporting_production_accounts')") + ).fetchone() + )[0] is None + pending_before_migration = (await h.image())["reporting_materializer_work"] + options = { + "python": old_python, + "script": old_script, + "helper": old_helper, + "installed": old, + } + async with feed_process(h, case, feed_request(case), pause="committed", **options) as child: + first = (await child.event("committed"))["result"] + await child.kill() + assert child.process.returncode == -9 + snapshot = await h.store.read_reporting_feed_snapshot( + first["ledger_snapshot_id"], caller=case.binding.principal + ) + expected = await walk(h.store, feed_request(case), case.binding.principal, first=first) + # Change actual live evidence while the original binary's frozen pages + # stay open. No reconstruction of its original representation is used. + await h.store.set_revision_readable( + account_id="acct_a", + reporting_revision_id=case.revision.reporting_revision_id, + readable=False, + ) + settings = { + **current, + "conninfo": h.pool.conninfo, + "kwargs": h.pool.kwargs, + "destination": str(tmp_path / "destination.sqlite"), + "notifications": notifications, + "caller": {"account_id": "acct_a", "consumer_id": case.binding.consumer_id}, + "receipt_request": receipt_request, + "receipt_response": receipt_response, + "historical_pending": pending, + "evidence_key": evidence_key, + "pause": True, + } + async with installed_child(python, script, settings, tmp_path) as child: + activated = await child.event("activated") + await child.kill() + assert child.process.returncode == -9 + assert activated["pending_continuation"]["state"] == "verified" + assert activated["pending_continuation"]["external_id"] == pending["external_id"] + assert ( + await h.store.read_reporting_feed_snapshot( + snapshot.snapshot_id, caller=case.binding.principal + ) + == snapshot + ) + # Run the actual inherited binary's projector primitive and sweeper on + # this newly activated schema, with no child SDK imported into it. + # An actually eligible revision for the next period makes this an old + # reservation exclusion test, rather than an idle-worker observation. + selected = await h.store.get_revision( + account_id="acct_a", reporting_revision_id="b24-production-revision" + ) + assert selected is not None and selected.row_count == 0 + original_obligation = await h.store.get_obligation( + account_id="acct_a", reporting_obligation_id=selected.reporting_obligation_id + ) + period = derive_period( + item.config.schedule, account_timezone=item.config.account_timezone, ordinal=1 + ) + new_obligation = replace( + original_obligation, + reporting_obligation_id="b24-fence-next-period", + period=period, + scope_resolved_at=period.end, + automated_recovery_deadline_at=period.expected_at + + item.config.automated_recovery_window, + ) + await h.store.commit_obligation(new_obligation) + revision_id = "b24-fence-next-revision" + await h.store.commit_revision( + replace( + selected, + reporting_revision_id=revision_id, + reporting_obligation_id=new_obligation.reporting_obligation_id, + data_through=period.end, + observed_at=period.end, + finalized_at=period.end, + created_at=period.end + timedelta(seconds=1), + revision_content_sha256=revision_content_sha256( + reporting_revision_id=revision_id, + row_count=0, + control_totals=selected.control_totals, + reporting_rows=[], + control_total_evidence=selected.managed_control_totals, + ), + ), + [], + ) + before_old = await h.image() + fenced = json.loads( + await asyncio.to_thread( + run_step, + [str(old_python), "-I", str(legacy_script)], + label="actual-parent-projector-fence", + cwd=old_root, + value=legacy_settings, + timeout=90, + ) + ) + assert fenced["historical_projection"] == "trigger_fenced" + assert fenced["historical_materializer"] == "reservation_trigger_fenced" + assert await h.image() == before_old + settings.update( + pause=False, + new_revision_after_snapshot=revision_id, + continuation=feed_request( + case, pagination={"cursor": first["pagination"]["cursor"], "max_results": 1} + ), + new_continuation=feed_request( + case, + pagination={"cursor": activated["first"]["pagination"]["cursor"], "max_results": 1}, + ), + ) + async with installed_child(python, script, settings, tmp_path) as child: + result = await child.event("done") + production_operation_1 = await asyncio.wait_for(child.process.wait(), 10) + assert production_operation_1 == 0 + assert result["legacy"] == { + "pages": expected[0][1:], + "binding": snapshot.binding, + "version": snapshot.representation_version, + "ownership_mode": snapshot.ownership_mode, + } + assert result["new"] == activated["new_remaining"] + assert result["fresh_external_writes"] == 1 + assert result["new"]["version"] == 2 and result["new"]["ownership_mode"] == "bindings" + assert ( + await h.store.read_reporting_feed_snapshot( + snapshot.snapshot_id, caller=case.binding.principal + ) + == snapshot + ) + print( + json.dumps( + { + "b24_actual_parent_activation_restart": old["sha"], + "parent_tree": old["tree"], + "parent_wheel_sha256": hashlib.sha256(old_wheel.read_bytes()).hexdigest(), + "current": current, + "notifications": notifications, + "historical_pending": pending, + "pending_continuation": activated["pending_continuation"], + "pending_before_migration_sha256": hashlib.sha256( + json.dumps(pending_before_migration, sort_keys=True).encode() + ).hexdigest(), + "parent_fence": fenced, + "legacy_page_count": len(expected[0]), + "new_page_count": len(result["new"]["pages"]) + 1, + "legacy_snapshot_sha256": hashlib.sha256( + json.dumps(expected[0], sort_keys=True).encode() + ).hexdigest(), + "new_origins": result["origins"], + } + ), + flush=True, + ) diff --git a/tests/conformance/reporting/test_reporting_production_settling.py b/tests/conformance/reporting/test_reporting_production_settling.py new file mode 100644 index 000000000..0daa23858 --- /dev/null +++ b/tests/conformance/reporting/test_reporting_production_settling.py @@ -0,0 +1,196 @@ +"""Production progress must retain unfinished source restatement policy across turns.""" + +from datetime import timedelta +from functools import partial + +import pytest + +from adcp.reporting.inline_source import InlineReportingSource +from adcp.reporting.source import ( + ReportingSourceCapabilitiesV1, + reporting_source_capabilities_sha256_v1, +) + +from ._generation_support import END +from ._materializer_support import reference_rows +from ._production_support import Source, SQLiteSeals, production_harness +from .test_reporting_production_lock_order import source_turn + + +class SettlingSource(Source): + """Declare the complete policy before the production service binds its hash.""" + + def __init__(self, key, path, rows=None, *, close_officially=True, **kwargs): + super().__init__(key, path, rows, **kwargs) + raw = self.capabilities.model_dump(mode="json") + raw["offerings"][0].update( + restatement_window="PT3H", + restatement_cadence="PT1H", + official_close_lag="PT4H" if close_officially else None, + ) + self.official_source_id = None + if close_officially: + official = Source(key, path.with_name("official-contract"), rows, official=True) + self.official_source_id = official.source_id + raw["offerings"].append(official.capabilities.offerings[0].model_dump(mode="json")) + raw["capabilities_sha256"] = reporting_source_capabilities_sha256_v1(raw) + self.capabilities = ReportingSourceCapabilitiesV1.model_validate(raw) + self.official_ready = False + self.inline = InlineReportingSource( + capabilities=self.capabilities, + fetch=self.fetch, + staging=self.inline.staging, + seals=SQLiteSeals(path.with_suffix(".seals")), + constituent_of=lambda row, req: req.coverage.constituents[0].constituent_id, + clock=kwargs.get("clock") or (lambda: END), + ) + + async def fetch(self, request): + if request.publication_class == "AUTHORITATIVE" and not self.official_ready: + self.requests.append(request) + return None + return await super().fetch(request) + + +async def revisions(h): + return await h.store.list_revisions( + account_id=h.item.config.account_id, + reporting_obligation_id=h.item.obligation.reporting_obligation_id, + ) + + +async def pending(h): + producer = h.production.offerings[0].producer + token = h.production._producer_turn.set(producer) + try: + return await h.store.next_producer_obligations( + h.item.config, now=h.source_clock(), limit=64 + ) + finally: + h.production._producer_turn.reset(token) + + +@pytest.mark.parametrize("backend", ["memory", "postgres"]) +@pytest.mark.parametrize("close_officially", [False, True]) +async def test_progress_retains_policy_until_terminal_publication( + backend, close_officially, tmp_path +): + async with production_harness( + backend, + tmp_path / "destination.sqlite", + count=1, + source_publication=True, + periods=1, + source_factory=partial(SettlingSource, close_officially=close_officially), + ) as h: + await h.production.activate(account_id=h.item.config.account_id) + source = h.production.offerings[0].producer._source + first = await source_turn(h.production) + assert len(first.revisions_committed) == 1 + assert [r.finality for r in await revisions(h)] == ["snapshot"] + production_operation_1 = await pending(h) + assert production_operation_1 == (h.item.obligation.reporting_obligation_id,) + + h.source_clock.now = END + timedelta(minutes=59) + await source_turn(h.production) + assert len(source.requests) == 1 + h.source_clock.now = END + timedelta(hours=1) + unchanged = await source_turn(h.production) + assert not unchanged.revisions_committed + assert len(source.requests) == 2 + checkpoint = await h.store.get_restatement_checkpoint( + account_id=h.item.config.account_id, + reporting_obligation_id=h.item.obligation.reporting_obligation_id, + ) + assert checkpoint is not None and checkpoint.next_observation == 2 + await source_turn(h.production) + assert len(source.requests) == 2 + + source.rows = reference_rows(2) + h.source_clock.now = END + timedelta(hours=2) + changed = await source_turn(h.production) + assert len(changed.revisions_committed) == 1 + history = await revisions(h) + assert len(history) == 2 + assert history[1].supersedes_reporting_revision_id == history[0].reporting_revision_id + + h.source_clock.now = END + timedelta(hours=3) + await source_turn(h.production) + assert len(source.requests) == 3 + if not close_officially: + production_operation_5 = await pending(h) + assert production_operation_5 == () + return + production_operation_2 = await pending(h) + assert production_operation_2 == (h.item.obligation.reporting_obligation_id,) + h.source_clock.now = END + timedelta(hours=4) + not_ready = await source_turn(h.production) + assert not not_ready.revisions_committed + production_operation_3 = await pending(h) + assert production_operation_3 == (h.item.obligation.reporting_obligation_id,) + source.official_ready = True + completed = await source_turn(h.production) + assert len(completed.revisions_committed) == 1 + assert [r.finality for r in await revisions(h)] == ["snapshot", "snapshot", "official"] + production_operation_4 = await pending(h) + assert production_operation_4 == () + count = len(source.requests) + await source_turn(h.production) + assert len(source.requests) == count + + +@pytest.mark.parametrize("backend", ["memory", "postgres"]) +async def test_settling_checkpoint_and_pending_work_survive_fresh_service(backend, tmp_path): + path = tmp_path / "restart.sqlite" + async with production_harness( + backend, + path, + count=1, + source_publication=True, + periods=1, + source_factory=SettlingSource, + ) as h: + await h.production.activate(account_id=h.item.config.account_id) + initial = await source_turn(h.production) + assert len(initial.revisions_committed) == 1 + h.source_clock.now = END + timedelta(hours=1) + noop = await source_turn(h.production) + assert not noop.revisions_committed + original_source = h.production.offerings[0].producer._source + assert len(original_source.requests) == 2 + prior_keys = {r.identity.source_execution_key for r in original_source.requests} + await h.production.aclose() + existing = {"existing_store": h.store} if h.pool is None else {"existing_pool": h.pool} + async with production_harness( + backend, + path, + count=1, + source_publication=True, + periods=1, + source_factory=SettlingSource, + **existing, + ) as fresh: + source = fresh.production.offerings[0].producer._source + assert source is not original_source + if h.pool is not None: + assert fresh.store is not h.store + checkpoint = await fresh.store.get_restatement_checkpoint( + account_id=fresh.item.config.account_id, + reporting_obligation_id=fresh.item.obligation.reporting_obligation_id, + ) + assert checkpoint is not None and checkpoint.next_observation == 2 + assert not source.requests + source.rows = reference_rows(2) + fresh.source_clock.now = END + timedelta(hours=2) + changed = await source_turn(fresh.production) + assert len(changed.revisions_committed) == 1 + assert len(source.requests) == 1 + assert source.requests[0].identity.source_execution_key not in prior_keys + checkpoint = await fresh.store.get_restatement_checkpoint( + account_id=fresh.item.config.account_id, + reporting_obligation_id=fresh.item.obligation.reporting_obligation_id, + ) + assert checkpoint is not None and checkpoint.next_observation == 3 + assert len(await revisions(fresh)) == 2 + unfinished = await pending(fresh) + assert unfinished == (fresh.item.obligation.reporting_obligation_id,) diff --git a/tests/conformance/reporting/test_reporting_production_signing_schema.py b/tests/conformance/reporting/test_reporting_production_signing_schema.py new file mode 100644 index 000000000..fd1c578bd --- /dev/null +++ b/tests/conformance/reporting/test_reporting_production_signing_schema.py @@ -0,0 +1,118 @@ +"""3.2 normative declarations, distinct from intentionally optional JSON fields.""" + +import hashlib +import json +from copy import deepcopy + +import pytest + +from adcp.server.responses import capabilities_response +from adcp.validation import schema_loader + +from ._production_support import production_harness +from ._production_transport import MountedProduction + +PIN = "3.2.0-rc.3" +HORIZON = "/properties/webhook_signing/properties/delivery_retry_horizon_seconds" +IDENTITY = "/properties/identity/description" +CACHED = ( + ( + "protocol/get-adcp-capabilities-response.json", + 213991, + "b8bb9cbd19491f352a277b0a5e7a39d88ab1290481e0f48cc025e61be6e52be1", + ), + ( + "bundled/protocol/get-adcp-capabilities-response.json", + 802990, + "bb852633ddf0873d935ab296b284b8cdf84f1857126b6ee01d4af1338750e1c8", + ), +) + + +@pytest.mark.parametrize("relative,size,digest", CACHED) +def test_exact_cached_schema_accepts_omission_despite_normative_32_requirement( + relative, size, digest +): + root = schema_loader._resolve_schema_root(PIN) + assert root is not None + path = root.root / relative + original = path.read_bytes() + assert len(original) == size and hashlib.sha256(original).hexdigest() == digest + schema = json.loads(original) + props = schema["properties"] + horizon = props["webhook_signing"]["properties"]["delivery_retry_horizon_seconds"] + identity = props["identity"]["description"] + assert "A webhook-emitting AdCP 3.2 agent MUST populate" in horizon["description"] + assert "Retries do not extend the horizon" in horizon["description"] + assert "brand_json_url` MUST be present" in identity + validator = schema_loader.get_named_validator(relative, version=PIN) + assert validator is not None and validator.schema == schema + omitted = capabilities_response(["media_buy"], sandbox=False, idempotency={"supported": False}) + omitted["webhook_signing"] = { + "supported": True, + "profile": "adcp/webhook-signing/v1", + "algorithms": ["ed25519"], + "legacy_hmac_fallback": False, + } + # This is an executed ACCEPTANCE, never the #1179 cached rejection. + validator.validate(omitted) + complete = deepcopy(omitted) + complete["webhook_signing"]["delivery_retry_horizon_seconds"] = 86400 + complete["identity"] = {"brand_json_url": "https://seller.example.test/brand.json"} + validator.validate(complete) + for field, value in ( + ("delivery_retry_horizon_seconds", 86399), + ("delivery_retry_horizon_seconds", 604801), + ("delivery_retry_horizon_seconds", "86400"), + ("algorithms", ["hs256"]), + ("profile", "invented-signing-profile"), + ): + invalid = deepcopy(complete) + invalid["webhook_signing"][field] = value + assert not validator.is_valid(invalid) + invalid = deepcopy(complete) + invalid["identity"]["brand_json_url"] = "http://seller.example.test/brand.json" + assert not validator.is_valid(invalid) + assert path.read_bytes() == original + print( + json.dumps( + { + "cached_optional_signing_declarations": { + "version": PIN, + "file": str(path), + "uri": f"https://adcontextprotocol.org/schemas/{PIN}/{relative}", + "bytes": size, + "sha256": digest, + "dialect": schema["$schema"], + "normative_pointers": {HORIZON: horizon, IDENTITY: identity}, + "omitted_payload": omitted, + "unmodified_schema_result": "accepted", + "semantic_32_result": "missing required horizon and operator declaration", + } + }, + sort_keys=True, + ) + ) + + +async def test_actual_public_declarations_and_schema_optional_omissions_on_all_mounts(tmp_path): + async with production_harness( + "postgres", tmp_path / "destination.sqlite", notifications=True, notification_delivery=True + ) as h: + mount = MountedProduction(h) + mount.authorize(h.item) + async with mount.client() as client: + for transport in ("mcp", "a2a-0.3", "a2a-1.0"): + _, raw = await mount.call(client, "get_adcp_capabilities", {}, transport=transport) + assert raw["webhook_signing"]["delivery_retry_horizon_seconds"] == 86400 + assert raw["identity"]["brand_json_url"] == ( + "https://seller.example.test/brand.json" + ) + for relative, _, _ in CACHED: + validator = schema_loader.get_named_validator(relative, version=PIN) + assert validator is not None + validator.validate(raw) + omitted = deepcopy(raw) + del omitted["webhook_signing"]["delivery_retry_horizon_seconds"] + del omitted["identity"] + validator.validate(omitted) diff --git a/tests/conformance/reporting/test_reporting_production_transactions.py b/tests/conformance/reporting/test_reporting_production_transactions.py new file mode 100644 index 000000000..ab07902e8 --- /dev/null +++ b/tests/conformance/reporting/test_reporting_production_transactions.py @@ -0,0 +1,411 @@ +"""Admitted finish atomicity and permanent quarantine across production activation.""" + +from datetime import timedelta + +import pytest + +from adcp.reporting.ledger import ( + ReportingMaterializationAttempt, + ReportingMaterializationRecord, + ReportingObligationDeliveryRecord, +) +from adcp.reporting.materializer import ReportingDestinationRequest, ReportingWriterError +from adcp.reporting.materializer.memory import InMemoryReportingMaterializerStore +from adcp.reporting.materializer.work import ReportingMaterializerLease + +from ._feed_support import feed_request, walk +from ._production_support import production_harness +from ._projection_support import drain +from .test_reporting_materializer_transactions import postgres_failure +from .test_reporting_production_lock_order import source_turn + + +async def production_queue(h): + if h.pool is None: + state = h.store._production_outbox + return ( + ((), ()) + if state is None + else ( + tuple(state.events.values()), + tuple(w.state for w in state.expansions.values()), + ) + ) + async with h.pool.connection() as c: + events = await ( + await c.execute("SELECT snapshot FROM reporting_production_notification_events") + ).fetchall() + work = await ( + await c.execute("SELECT state FROM reporting_production_notification_expansions") + ).fetchall() + return tuple(r[0] for r in events), tuple(r[0] for r in work) + + +@pytest.mark.parametrize("backend", ["memory", "postgres"]) +@pytest.mark.parametrize("notifications", [False, True]) +@pytest.mark.parametrize("position", ["outcome", "status_head", "account_head", "boundary", "ack"]) +async def test_production_finish_fault_restores_every_row_head_capture_and_frozen_page( + backend, notifications, position, monkeypatch, tmp_path +): + async with production_harness( + backend, tmp_path / "destination.sqlite", notifications=notifications, count=1 + ) as h: + item = h.item + await h.production.activate(account_id=item.config.account_id) + lease = await item.claim() + assert isinstance(lease, ReportingMaterializerLease) and lease.admission_epoch == 2 + prepared, verified = await item.verified(lease) + first = await h.store.read_reporting_feed(feed_request(item), caller=item.binding.principal) + frozen = await walk(h.store, feed_request(item), item.binding.principal, first=first) + old_queue, ordinary = await h.queue(), await h.ordinary_events() + before = await h.image() + + async def finish(): + return await h.store.finish_materialization(lease, prepared=prepared, verified=verified) + + if h.pool is not None: + prefixes = { + "outcome": "INSERT INTO reporting_reconciliation_records", + "status_head": "INSERT INTO reporting_production_status_heads", + "account_head": "UPDATE reporting_materializer_accounts SET captured_sequence=", + "boundary": "INSERT INTO reporting_production_status_boundaries", + "ack": "UPDATE reporting_production_work SET state='acked'", + } + with postgres_failure(monkeypatch, prefixes[position]) as hit: + with pytest.raises(ReportingWriterError): + await finish() + assert len(hit) == 1 + else: + import adcp.reporting.materializer.memory as memory + + cls = InMemoryReportingMaterializerStore + if position == "outcome": + original = cls._commit_record_unlocked + + def fail(self, record, **kwargs): + result = original(self, record, **kwargs) + if isinstance(record, ReportingMaterializationRecord): + raise OSError("injected production finish") + return result + + target, method = cls, "_commit_record_unlocked" + elif position in {"status_head", "account_head"}: + + def fail(*args, **kwargs): + raise OSError("injected production finish") + + target, method = memory, "ReportingMaterializerBoundary" + else: + method = "_materializer_dirty" if position == "boundary" else "_park" + target, original = cls, getattr(cls, method) + + def fail(self, *args, **kwargs): + original(self, *args, **kwargs) + raise OSError("injected production finish") + + with monkeypatch.context() as patch: + patch.setattr(target, method, fail) + with pytest.raises(ReportingWriterError): + await finish() + assert await h.image() == before + assert ( + await walk(h.store, feed_request(item), item.binding.principal, first=first) == frozen + ) + production_operation_1 = await finish() + assert (production_operation_1).state == "verified" + assert len(await item.outcomes()) == 1 + assert len(await h.store.read_production_boundaries(caller=item.binding.principal)) == 1 + assert await h.store.read_materializer_boundaries(caller=item.binding.principal) == () + assert len((await production_queue(h))[0]) == int(notifications) + assert await h.queue() == old_queue and await h.ordinary_events() == ordinary + committed = await h.image() + production_operation_2 = await finish() + assert (production_operation_2).state == "verified" + assert await h.image() == committed + assert item.writer.writes == 1 + + +@pytest.mark.parametrize("backend", ["memory", "postgres"]) +@pytest.mark.parametrize("position", ["event", "expansion", "missing_event"]) +async def test_enabled_production_enqueue_is_required_inside_verified_finish( + backend, position, monkeypatch, tmp_path +): + async with production_harness( + backend, tmp_path / "destination.sqlite", notifications=True, count=1 + ) as h: + item = h.item + await h.production.activate(account_id=item.config.account_id) + lease = await item.claim() + prepared, verified = await item.verified(lease) + before = await h.image() + + async def finish(): + return await h.store.finish_materialization(lease, prepared=prepared, verified=verified) + + if h.pool is not None and position != "missing_event": + table = "events" if position == "event" else "expansions" + with postgres_failure( + monkeypatch, f"INSERT INTO reporting_production_notification_{table}" + ) as hit: + with pytest.raises(ReportingWriterError): + await finish() + assert len(hit) == 1 + else: + with monkeypatch.context() as patch: + if h.pool is not None: + import adcp.reporting.materializer.pg as pg + + async def empty(connection, event): + pass + + patch.setattr(pg, "enqueue_materializer_event_on", empty) + else: + import adcp.reporting.outbox.memory as memory + + if position == "event": + + def fail(*args, **kwargs): + raise OSError("injected production event") + + patch.setattr(memory, "_Work", fail) + else: + original = memory.NotificationState.enqueue + + def enqueue(self, event): + if position != "missing_event": + original(self, event) + raise OSError("injected production expansion") + + patch.setattr(memory.NotificationState, "enqueue", enqueue) + with pytest.raises(ReportingWriterError): + await finish() + assert await h.image() == before + assert await production_queue(h) == ((), ()) + production_operation_3 = await finish() + assert (production_operation_3).state == "verified" + events, work = await production_queue(h) + assert len(events) == 1 and work == ("pending",) + + +@pytest.mark.parametrize("backend", ["memory", "postgres"]) +@pytest.mark.parametrize("notifications", [False, True]) +async def test_explicit_old_pending_import_keeps_epoch_and_external_identity_after_activation( + backend, notifications, tmp_path, monkeypatch +): + path = tmp_path / "destination.sqlite" + async with production_harness(backend, path, notifications=notifications, count=1) as h: + item = h.item + await h.store.bind_obligation_delivery( + ReportingObligationDeliveryRecord( + item.scope, + "USD", + item.revision.created_at + timedelta(days=400), + item.revision.created_at, + ) + ) + attempt = ReportingMaterializationAttempt( + item.scope, + item.revision.reporting_revision_id, + "preactivation-import", + 1, + item.revision.created_at, + ) + await h.store.commit_materialization_attempt(attempt) + external = ReportingDestinationRequest.from_binding( + item.binding, attempt, item.verifier.key + ).external_id + await h.store.import_pending_materialization( + scope=item.scope, + reporting_materialization_id=attempt.reporting_materialization_id, + original_external_id=external, + keys=item.keys, + ) + await h.production.activate(account_id=item.config.account_id) + original = await h.works() + await h.production.aclose() + observed = [] + original_finish = type(h.store).finish_materialization + + async def observe_finish(self, lease, **kwargs): + result = await original_finish(self, lease, **kwargs) + observed.append((lease, kwargs, result)) + return result + + monkeypatch.setattr(type(h.store), "finish_materialization", observe_finish) + async with production_harness( + backend, + path, + notifications=notifications, + count=1, + existing_store=h.store if h.pool is None else None, + existing_pool=h.pool, + ) as fresh: + # start() waits for its first real worker turn. The cold support + # has already resumed and completed the original pending effect. + assert await fresh.works() == tuple((key, "acked", gen) for key, _, gen in original) + assert len(observed) == 1 + lease, arguments, result = observed[0] + await fresh.production.activate(account_id=item.config.account_id) + assert lease.admission_epoch == 0 + assert lease.attempt == attempt and lease.request.external_id == external + assert result.state == "verified" + assert fresh.item.writer.writes == 1 + assert await production_queue(fresh) == ((), ()) + events, work = await fresh.queue() + assert len(events) == int(notifications) + assert work == (("quarantined",) if notifications else ()) + assert await fresh.store.read_production_boundaries(caller=item.binding.principal) == () + assert ( + len(await fresh.store.read_materializer_boundaries(caller=item.binding.principal)) + == 1 + ) + frozen = await fresh.image() + production_operation_5 = await fresh.store.finish_materialization(lease, **arguments) + assert (production_operation_5).state == "verified" + assert await fresh.image() == frozen + await drain(fresh.projection, item.config.account_id) + assert await production_queue(fresh) == ((), ()) + assert await fresh.queue() == (events, work) + + +@pytest.mark.parametrize("backend", ["memory", "postgres"]) +@pytest.mark.parametrize("notifications", [False, True]) +@pytest.mark.parametrize("source_changes", [None, "readability", "configuration"]) +async def test_producer_lease_bookkeeping_preserves_io_but_real_source_change_fences_it( + backend, notifications, source_changes, tmp_path +): + async with production_harness( + backend, tmp_path / "destination.sqlite", notifications=notifications, count=1 + ) as h: + item = h.item + await h.production.activate(account_id=item.config.account_id) + lease = await item.claim() + assert isinstance(lease, ReportingMaterializerLease) and lease.admission_epoch == 2 + prepared, verified = await item.verified(lease) + first = await h.store.read_reporting_feed(feed_request(item), caller=item.binding.principal) + frozen = await walk(h.store, feed_request(item), item.binding.principal, first=first) + # The source takes and releases its real fair configuration lease + # while a verified external effect is still awaiting its fenced finish. + turn = await source_turn(h.production) + assert turn.leased is not None and not turn.revisions_committed + producer = h.production.offerings[0].producer + token = h.production._producer_turn.set(producer) + try: + + async def acquire(worker, seconds): + return await h.store.lease_period_close( + worker_id=worker, + now=h.source_clock() + timedelta(seconds=seconds), + lease_seconds=1, + ) + + crashed = await acquire("crashed-producer", 0) + assert crashed is not None + production_operation_6 = await acquire("duplicate-producer", 0) + assert production_operation_6 is None + recovered = await acquire("recovered-producer", 2) + assert recovered is not None + await h.store.release_period_close(crashed, worker_id="crashed-producer") + production_operation_7 = await acquire("duplicate-producer", 2) + assert production_operation_7 is None + await h.store.release_period_close(recovered, worker_id="recovered-producer") + for seconds in (3, 4): + repeated = await acquire("scheduled-producer", seconds) + assert repeated is not None + await h.store.release_period_close(repeated, worker_id="scheduled-producer") + finally: + h.production._producer_turn.reset(token) + assert ( + await walk(h.store, feed_request(item), item.binding.principal, first=first) == frozen + ) + if source_changes == "readability": + await h.store.set_revision_readable( + account_id=item.config.account_id, + reporting_revision_id=item.revision.reporting_revision_id, + readable=False, + ) + elif source_changes == "configuration": + from dataclasses import replace + + await h.store.put_configuration(replace(item.config, deactivated_at=h.clock())) + result = await h.store.finish_materialization(lease, prepared=prepared, verified=verified) + assert result.state == ("failed" if source_changes else "verified") + outcomes = await item.outcomes() + assert len(outcomes) == 1 + assert ( + outcomes[0].reporting_materialization_id == lease.attempt.reporting_materialization_id + ) + assert item.writer.writes == 1 + assert len((await production_queue(h))[0]) == int(notifications and not source_changes) + assert await h.queue() == ((), ()) + committed = await h.image() + production_operation_4 = await h.store.finish_materialization( + lease, prepared=prepared, verified=verified + ) + assert production_operation_4 == result + assert await h.image() == committed + + +@pytest.mark.parametrize("notifications", [False, True]) +@pytest.mark.parametrize("release", [False, True]) +async def test_pg_lease_bookkeeping_fault_rolls_back_configuration_candidate_and_fairness( + notifications, release, monkeypatch, tmp_path +): + async with production_harness( + "postgres", tmp_path / "destination.sqlite", notifications=notifications, count=1 + ) as h: + await h.production.activate(account_id=h.item.config.account_id) + pending = await h.item.claim() + assert isinstance(pending, ReportingMaterializerLease) + producer = h.production.offerings[0].producer + token = h.production._producer_turn.set(producer) + try: + + async def acquire(): + return await h.store.lease_period_close( + worker_id="bookkeeping-fault", now=h.source_clock(), lease_seconds=30 + ) + + lease = await acquire() if release else None + before = await h.image() + async with h.pool.connection() as c: + ranks = await ( + await c.execute( + "SELECT * FROM adcp_reporting_configuration_lease_turns" + " ORDER BY account_id,delivery_config_id,delivery_config_version" + ) + ).fetchall() + with postgres_failure( + monkeypatch, "UPDATE reporting_materializer_candidates SET generation=generation-1" + ) as hit: + with pytest.raises(OSError, match="injected transaction boundary"): + if release: + await h.store.release_period_close(lease, worker_id="bookkeeping-fault") + else: + await acquire() + assert len(hit) == 1 + assert await h.image() == before + async with h.pool.connection() as c: + assert ( + await ( + await c.execute( + "SELECT * FROM adcp_reporting_configuration_lease_turns" + " ORDER BY account_id,delivery_config_id,delivery_config_version" + ) + ).fetchall() + == ranks + ) + lease = lease if release else await acquire() + assert lease is not None + await h.store.release_period_close(lease, worker_id="bookkeeping-fault") + # A stale or duplicate release cannot remove another generation. + image = await h.image() + await h.store.release_period_close(lease, worker_id="bookkeeping-fault") + assert await h.image() == image + prepared, verified = await h.item.verified(pending) + production_operation_8 = await h.store.finish_materialization( + pending, prepared=prepared, verified=verified + ) + assert (production_operation_8).state == "verified" + finally: + h.production._producer_turn.reset(token) diff --git a/tests/conformance/reporting/test_reporting_production_worker_failure.py b/tests/conformance/reporting/test_reporting_production_worker_failure.py new file mode 100644 index 000000000..60ebef1d0 --- /dev/null +++ b/tests/conformance/reporting/test_reporting_production_worker_failure.py @@ -0,0 +1,299 @@ +"""Owned worker failures stop the composition and emit only closed diagnostics.""" + +import asyncio +import json +import logging + +import pytest + +from adcp.reporting.ledger.notification_models import ReportingNotificationError + +from ._production_support import production_harness +from ._production_transport import MountedProduction + + +def _fault_target(harness, boundary): + if boundary == "producer": + return harness.production.offerings[0].producer, "run_worker" + if boundary == "materializer": + return type(harness.production.materializer), "run_once" + if boundary == "projection": + return harness.projection, "rebuild_one" + if boundary == "sweeper": + return harness.projection, "sweep_one" + import adcp.reporting.production.notifications as notifications + + # The running loop binds notification_turn at startup; its live sampling + # call is the actual owned boundary to fault after a healthy first turn. + return notifications, "next_account" + + +_CASES = [ + (boundary, delivery) + for boundary in ("producer", "materializer", "projection", "sweeper", "notifications") + for delivery in (False, True) + if delivery or boundary != "notifications" +] + + +@pytest.mark.parametrize("backend", ["memory", "postgres"]) +@pytest.mark.parametrize("boundary,delivery", _CASES) +async def test_unexpected_worker_failure_has_closed_operator_diagnostic( + backend, boundary, delivery, tmp_path, monkeypatch, caplog +): + entered, release = asyncio.Event(), asyncio.Event() + private_marker = "private-worker-failure-canary" + + async def fail(*args, **kwargs): + entered.set() + await release.wait() + raise RuntimeError(private_marker) + + async with production_harness( + backend, + tmp_path / "destination.sqlite", + notifications=delivery, + notification_delivery=delivery, + count=0, + poll_seconds=0.02, + ) as h: + support = h.production + mount = MountedProduction(h) + mount.authorize(h.item) + async with mount.client() as client: + for transport in ("mcp", "a2a-0.3", "a2a-1.0"): + _, before = await mount.call( + client, "get_adcp_capabilities", {}, transport=transport + ) + if backend == "postgres": + assert before["media_buy"]["reporting_delivery"]["managed_delivery"] + else: + assert "reporting_delivery" not in before.get("media_buy", {}) + target, method = _fault_target(h, boundary) + with monkeypatch.context() as patch: + patch.setattr(target, method, fail) + original_factory = logging.getLogRecordFactory() + + def ambient_factory(*args, **kwargs): + record = original_factory(*args, **kwargs) + record.private_request_context = private_marker + return record + + try: + await asyncio.wait_for(entered.wait(), 5) + caplog.clear() + logging.setLogRecordFactory(ambient_factory) + task = ( + support._notification_task if boundary == "notifications" else support._task + ) + task.set_name(private_marker) + release.set() + await asyncio.wait_for(asyncio.shield(task), 5) + records = [r for r in caplog.records if r.name == "adcp.reporting.production"] + assert len(records) == 1, "unexpected owned failure needs one operator signal" + record = records[0] + assert record.levelno == logging.ERROR + assert record.code == "REPORTING_PRODUCTION_WORKER_STOPPED" + assert record.boundary == boundary + assert record.getMessage() == "Reporting production worker stopped" + assert not record.args and record.exc_info is None and record.stack_info is None + assert record.pathname == "" + assert record.threadName is None and record.processName is None + assert getattr(record, "taskName", None) is None + assert private_marker not in json.dumps(record.__dict__, default=str) + assert support._failed and support._stop.is_set() + tasks = [ + t for t in (support._task, support._notification_task) if t is not None + ] + await asyncio.wait_for(asyncio.gather(*tasks), 5) + assert all(t.done() for t in tasks) + for transport in ("mcp", "a2a-0.3", "a2a-1.0"): + _, after = await mount.call( + client, "get_adcp_capabilities", {}, transport=transport + ) + assert "reporting_delivery" not in after.get("media_buy", {}) + assert "webhook_signing" not in after + with pytest.raises(ReportingNotificationError, match="component_unready"): + await support.activate(account_id=h.item.config.account_id) + finally: + logging.setLogRecordFactory(original_factory) + release.set() + support._stop.set() + + +@pytest.mark.parametrize("backend", ["memory", "postgres"]) +@pytest.mark.parametrize("boundary", ["producer", "notifications"]) +@pytest.mark.parametrize("kind", ["domain", "cancellation"]) +async def test_expected_worker_stop_is_silent_and_drains_sibling( + backend, boundary, kind, tmp_path, monkeypatch, caplog +): + entered, release = asyncio.Event(), asyncio.Event() + + async def fail(*args, **kwargs): + entered.set() + await release.wait() + if kind == "cancellation": + raise asyncio.CancelledError + raise ReportingNotificationError("notification_chain_unready") + + async with production_harness( + backend, + tmp_path / "destination.sqlite", + notifications=True, + notification_delivery=True, + count=0, + poll_seconds=0.02, + ) as h: + support = h.production + target, method = _fault_target(h, boundary) + with monkeypatch.context() as patch: + patch.setattr(target, method, fail) + try: + await asyncio.wait_for(entered.wait(), 5) + caplog.clear() + release.set() + task = support._notification_task if boundary == "notifications" else support._task + if kind == "cancellation": + with pytest.raises(asyncio.CancelledError): + await asyncio.wait_for(asyncio.shield(task), 5) + else: + await asyncio.wait_for(asyncio.shield(task), 5) + assert support._stop.is_set(), "the owned sibling must be woken immediately" + tasks = [t for t in (support._task, support._notification_task) if t is not None] + await asyncio.wait_for(asyncio.gather(*tasks, return_exceptions=True), 5) + assert all(t.done() for t in tasks) + assert not [r for r in caplog.records if r.name == "adcp.reporting.production"] + production_operation_1 = await support.reporting_delivery() + assert production_operation_1 == {} + finally: + release.set() + support._stop.set() + + +@pytest.mark.parametrize("backend", ["memory", "postgres"]) +@pytest.mark.parametrize("delivery", [False, True]) +async def test_failing_operator_sink_cannot_prevent_owned_shutdown( + backend, delivery, tmp_path, monkeypatch +): + entered, release = asyncio.Event(), asyncio.Event() + observed = [] + + async def fail(): + entered.set() + await release.wait() + raise RuntimeError("private-provider-diagnostic-canary") + + def broken_sink(record): + observed.append(record) + raise RuntimeError("private-operator-diagnostic-canary") + + async with production_harness( + backend, + tmp_path / "destination.sqlite", + notifications=delivery, + notification_delivery=delivery, + count=0, + poll_seconds=0.02, + ) as h: + support = h.production + with monkeypatch.context() as patch: + patch.setattr(support.offerings[0].producer, "run_worker", fail) + patch.setattr(logging.getLogger("adcp.reporting.production"), "handle", broken_sink) + try: + await asyncio.wait_for(entered.wait(), 5) + release.set() + tasks = [t for t in (support._task, support._notification_task) if t is not None] + await asyncio.wait_for(asyncio.gather(*tasks), 5) + assert support._failed and support._stop.is_set() + assert len(observed) == 1 + assert "canary" not in json.dumps(observed[0].__dict__, default=str) + production_operation_2 = await support.reporting_delivery() + assert production_operation_2 == {} + with pytest.raises(ReportingNotificationError, match="component_unready"): + await support.activate(account_id=h.item.config.account_id) + finally: + release.set() + support._stop.set() + + +@pytest.mark.parametrize("invalid", ["private-boundary-canary", []], ids=["string", "unhashable"]) +def test_operator_boundary_is_runtime_allowlisted(invalid, caplog): + from typing import get_args + + from adcp.reporting.production._diagnostics import ( + _BOUNDARIES, + _worker_stopped, + _WorkerBoundary, + ) + + assert set(get_args(_WorkerBoundary)) == _BOUNDARIES + _worker_stopped(boundary=invalid) + records = [r for r in caplog.records if r.name == "adcp.reporting.production"] + assert len(records) == 1 and records[0].boundary == "worker" + assert "canary" not in json.dumps(records[0].__dict__, default=str) + + +@pytest.mark.parametrize("backend", ["memory", "postgres"]) +@pytest.mark.parametrize("stop", ["cancellation", "close"]) +async def test_stopped_composition_does_not_alert_on_late_inflight_failure( + backend, stop, tmp_path, monkeypatch, caplog +): + import adcp.reporting.production.notifications as notifications + + producer_entered, notification_entered = asyncio.Event(), asyncio.Event() + producer_release, notification_release = asyncio.Event(), asyncio.Event() + + async def producer_wait(): + producer_entered.set() + await producer_release.wait() + raise RuntimeError("private-late-worker-canary") + + async def notification_wait(*args, **kwargs): + notification_entered.set() + await notification_release.wait() + if stop == "cancellation": + raise asyncio.CancelledError + return None + + async with production_harness( + backend, + tmp_path / "destination.sqlite", + notifications=True, + notification_delivery=True, + count=0, + poll_seconds=0.02, + ) as h: + support = h.production + closing = None + with monkeypatch.context() as patch: + patch.setattr(support.offerings[0].producer, "run_worker", producer_wait) + patch.setattr(notifications, "next_account", notification_wait) + try: + await asyncio.wait_for( + asyncio.gather(producer_entered.wait(), notification_entered.wait()), 5 + ) + caplog.clear() + if stop == "cancellation": + notification_release.set() + with pytest.raises(asyncio.CancelledError): + await asyncio.wait_for(asyncio.shield(support._notification_task), 5) + else: + closing = asyncio.create_task(support.aclose()) + await asyncio.wait_for(support._stop.wait(), 5) + assert support._stop.is_set() + assert not support._task.done() + producer_release.set() + notification_release.set() + tasks = [t for t in (support._task, support._notification_task) if t is not None] + await asyncio.wait_for(asyncio.gather(*tasks, return_exceptions=True), 5) + if closing is not None: + await asyncio.wait_for(closing, 5) + assert not [r for r in caplog.records if r.name == "adcp.reporting.production"] + production_operation_3 = await support.reporting_delivery() + assert production_operation_3 == {} + finally: + producer_release.set() + notification_release.set() + support._stop.set() + if closing is not None: + await asyncio.gather(closing, return_exceptions=True) diff --git a/tests/conformance/reporting/test_reporting_projection_capture.py b/tests/conformance/reporting/test_reporting_projection_capture.py new file mode 100644 index 000000000..5ac5be4f3 --- /dev/null +++ b/tests/conformance/reporting/test_reporting_projection_capture.py @@ -0,0 +1,146 @@ +"""Ordered capture, checkpoint, activation, rollback and frozen feed integration.""" + +from dataclasses import replace +from datetime import timedelta + +import pytest + +from adcp.reporting.ledger import ReportingMaterializationCheck +from adcp.reporting.ledger.notification_models import ReportingNotificationError +from adcp.reporting.ownership import page_revision_ownership + +from ._feed_support import feed_request, walk +from ._generation_support import END +from ._projection_support import drain, inputs, projections +from ._receipt_support import adjustment_for, receipt_case, request_for + +__all__ = ["projections"] + + +async def test_captured_receipt_adjustment_and_readability_cycles_are_not_collapsed(projections): + h = projections + s = await receipt_case(h) + account = s.obligation.account_id + production_operation_1 = await h.projection.activate(account_id=account) + assert production_operation_1 + assert await h.projection.baseline_ready(account_id=account) + starting = next( + c.generation + for c in await h.projection.checkpoints(account_id=account) + if c.scope.reporting_obligation_id == s.obligation.reporting_obligation_id + and c.scope.consumer_id == s.binding.consumer_id + ) + baseline = await inputs(h, account) + assert len(baseline) == 1 + await h.store.ingest_receipt_batch(request_for(s), caller=s.binding.principal) + rejected = ReportingMaterializationCheck( + s.attempt.scope, + s.attempt.reporting_materialization_id, + "projection-corruption", + "corrupt", + END + timedelta(seconds=30), + ) + await h.store.record_materialization_check(rejected) + await h.store.record_materialization_check( + replace( + rejected, + check_id="projection-recovery", + state="readable", + checked_at=END + timedelta(seconds=31), + ) + ) + item = await adjustment_for(h, s) + await h.store.ingest_receipt_batch( + { + "account": {"account_id": account}, + "idempotency_key": "projection-adjustment-0001", + "adjustment_receipts": [item], + }, + caller=s.binding.principal, + ) + frozen = await inputs(h, account) + assert len(frozen) == 6 + # All five transitions are pending while today's rows already look recovered. + assert len(frozen[0].reconciliation) + 4 == len(frozen[-1].reconciliation) + transitions = await drain(h.projection, account) + assert len(transitions) == 5 + checkpoints = await h.projection.checkpoints(account_id=account) + obligation = next( + c + for c in checkpoints + if c.scope.reporting_obligation_id == s.obligation.reporting_obligation_id + and c.scope.consumer_id == s.binding.consumer_id + ) + assert obligation.generation == starting + 5 + assert obligation.snapshot["health"] == "complete" + assert ( + all(t.events > 0 for t in transitions) + if h.projection.policy["notifications_enabled"] + else all(t.events == 0 for t in transitions) + ) + assert await inputs(h, account) == frozen + before = await h.image() + await h.store.ingest_receipt_batch(request_for(s), caller=s.binding.principal) + assert await h.image() == before + production_operation_2 = await h.projection.activate(account_id=account) + assert not production_operation_2 + + +async def test_activation_preserves_legacy_snapshot_and_new_pages_have_exact_local_ownership( + projections, +): + h = projections + s = await receipt_case(h) + req = feed_request(s) + first = await h.store.read_reporting_feed(req, caller=s.binding.principal) + legacy = await walk(h.store, req, s.binding.principal, first=first) + production_operation_3 = await h.projection.activate(account_id=s.obligation.account_id) + assert production_operation_3 + resumed = await walk(h.store, req, s.binding.principal, first=first) + assert resumed == legacy + new = await walk(h.store, req, s.binding.principal) + for page in new[0]: + bindings = page_revision_ownership(page) + assert bindings is not None + assert set(bindings) == {r["reporting_revision_id"] for r in page.get("revisions", [])} + assert all(owner == s.obligation.reporting_obligation_id for owner in bindings.values()) + assert new[0][0]["changes_checkpoint"] != first["changes_checkpoint"] + + +async def test_capture_failure_rolls_back_every_source_collection_or_row(projections, monkeypatch): + h = projections + s = await receipt_case(h) + await h.projection.activate(account_id=s.obligation.account_id) + before = await h.image() + if h.pool is None: + + def fail(self, account_id): + raise ReportingNotificationError("status_projection_history_corrupt") + + monkeypatch.setattr(type(h.store), "_capture_projection", fail) + with pytest.raises(ReportingNotificationError): + await h.store.ingest_receipt_batch(request_for(s), caller=s.binding.principal) + monkeypatch.undo() + else: + from psycopg import sql + + async with h.pool.connection() as c: + await c.execute( + "CREATE FUNCTION test_projection_failure() RETURNS trigger LANGUAGE plpgsql" + " AS $$BEGIN RAISE EXCEPTION 'injected'; END$$" + ) + await c.execute( + "CREATE TRIGGER test_projection_failure" + " BEFORE INSERT ON reporting_projection_inputs" + " FOR EACH ROW EXECUTE FUNCTION test_projection_failure()" + ) + try: + with pytest.raises(Exception): + await h.store.ingest_receipt_batch(request_for(s), caller=s.binding.principal) + finally: + async with h.pool.connection() as c: + await c.execute( + sql.SQL("DROP TRIGGER test_projection_failure ON reporting_projection_inputs") + ) + await c.execute("DROP FUNCTION test_projection_failure()") + assert await h.image() == before diff --git a/tests/conformance/reporting/test_reporting_projection_history.py b/tests/conformance/reporting/test_reporting_projection_history.py new file mode 100644 index 000000000..6a69890ad --- /dev/null +++ b/tests/conformance/reporting/test_reporting_projection_history.py @@ -0,0 +1,266 @@ +"""Retained preactivation inputs, incremental cutover and permanent quarantine.""" + +from dataclasses import replace +from datetime import datetime, timedelta, timezone + +import pytest + +from adcp.reporting.ledger import ( + ReportingAdjustmentReceiptRecord, + ReportingAdjustmentRecord, + ReportingControlTotalRecord, + ReportingRevisionReceiptRecord, +) +from adcp.reporting.ledger.delivery import adjustment_to_wire +from adcp.reporting.ledger.notification_models import ReportingNotificationError +from adcp.reporting.projection.history import checkpoint_document, checkpoint_key + +from ._durable_materializer_support import durable_case +from ._projection_support import projection_harness, projections + +__all__ = ["projections"] + + +async def history_image(h, account): + if h.pool is None: + state = h.store._projection_accounts[account] + return [(sequence, document) for sequence, document in state.historical_steps] + async with h.pool.connection() as c: + return await ( + await c.execute( + "SELECT account_sequence,input FROM reporting_projection_legacy_steps" + " WHERE account_id=%s ORDER BY account_sequence,scope_key", + (account,), + ) + ).fetchall() + + +async def prepare_history(h): + h.clock.now = datetime.now(timezone.utc) + first = await durable_case( + h.store, required="official", finality="official", reconciliation_mode="consumer_receipt" + ) + second = await durable_case( + h.store, + required="official", + finality="official", + reconciliation_mode="consumer_receipt", + consumer="https://buyer.example.test/other", + ) + for _ in range(2): + lease = await first.claim() + case = first if lease.scope == first.scope else second + prepared, evidence = await case.verified(lease) + await h.store.finish_materialization(lease, prepared=prepared, verified=evidence) + outcome = (await first.outcomes())[0] + h.clock.now = datetime.now(timezone.utc) + receipt = ReportingRevisionReceiptRecord( + first.scope, + "historical-receipt", + first.revision.reporting_revision_id, + outcome.reporting_materialization_id, + "accepted", + first.binding.verification_profile, + first.revision.row_count, + first.revision.managed_control_totals, + h.clock(), + observed_canonical_content_digest=first.revision.canonical_content_digest, + ) + await h.store.record_revision_receipt(receipt) + adjustment = ReportingAdjustmentRecord( + "historical-adjustment", + first.config.account_id, + first.revision.reporting_revision_id, + "source_correction", + first.obligation.period.end, + first.obligation.period.end + timedelta(days=30), + (("spend", "-0.50"),), + h.clock(), + h.clock(), + managed_control_total_deltas=( + ReportingControlTotalRecord("spend", "-0.50", "decimal", "USD"), + ), + ) + await h.store.commit_adjustment(adjustment) + rejected = ReportingAdjustmentReceiptRecord( + first.scope, + "historical-adjustment-rejected", + adjustment.reporting_adjustment_id, + first.revision.reporting_revision_id, + "rejected", + adjustment_to_wire(adjustment)["canonical_adjustment_sha256"], + h.clock(), + rejection_codes=("CONTROL_TOTAL_MISMATCH",), + ) + await h.store.record_adjustment_receipt(rejected) + await h.store.record_adjustment_receipt( + replace( + rejected, + reporting_receipt_id="historical-adjustment-accepted", + status="accepted", + supersedes_reporting_receipt_id=rejected.reporting_receipt_id, + rejection_codes=(), + ) + ) + h.clock.now = datetime.now(timezone.utc) + return first, second + + +async def original_inputs(h, cases): + result = [] + for case in cases: + records = ( + *await h.store.read_materializer_boundaries(caller=case.scope.principal), + *await h.store.read_receipt_boundaries(caller=case.scope.principal), + ) + result.extend(b.to_storage() for b in records) + return tuple(result) + + +async def test_captured_history_replays_after_interruption_without_promoting_readiness( + projections, monkeypatch +): + await history_replay(projections, monkeypatch, legacy_baseline=False) + + +@pytest.mark.parametrize("backend", ["memory", "postgres"]) +async def test_captured_history_preserves_the_existing_c_baseline(backend, monkeypatch): + async with projection_harness(backend, notifications=True) as h: + await history_replay(h, monkeypatch, legacy_baseline=True) + + +async def history_replay(h, monkeypatch, *, legacy_baseline): + first, second = await prepare_history(h) + account = first.config.account_id + baselines = () + if legacy_baseline: + if h.pool is None: + from adcp.reporting.outbox.status_memory import InMemoryStatusNotificationStore + + old = InMemoryStatusNotificationStore(h.store) + else: + from adcp.reporting.outbox.status_pg import PgStatusNotificationStore + + old = PgStatusNotificationStore(h.store) + production_operation_5 = await old.baseline(account_id=account) + assert production_operation_5 + baselines = await old.checkpoints(account_id=account) + assert baselines + original_queue = await h.queue() + originals = await original_inputs(h, (first, second)) + assert len(originals) == 5 + production_operation_1 = await h.projection._begin_activation(account_id=account) + assert production_operation_1 + assert not await h.projection.baseline_ready(account_id=account) + if h.pool is None: + archived = h.store._projection_accounts[account].baselines + assert archived == {checkpoint_key(c): c for c in baselines} + else: + async with h.pool.connection() as c: + archived = dict( + await ( + await c.execute( + "SELECT scope_key,checkpoint FROM reporting_projection_legacy_baselines" + " WHERE account_id=%s", + (account,), + ) + ).fetchall() + ) + assert archived == {checkpoint_key(c): checkpoint_document(c) for c in baselines} + production_operation_2 = await h.projection.project_one(account_id=account) + assert (production_operation_2).did_work + before = await history_image(h, account) + assert before + # An interrupted invocation has a committed first input; the next complete + # transaction fails after preparing its replay, and must advance nothing. + if h.pool is None: + import adcp.reporting.projection.memory as module + + original = module.project_boundary + + def failure(*args, **kwargs): + original(*args, **kwargs) + raise RuntimeError("injected replay transaction failure") + + with monkeypatch.context() as patch: + patch.setattr(module, "project_boundary", failure) + with pytest.raises(RuntimeError): + await h.projection.project_one(account_id=account) + else: + async with h.pool.connection() as c: + await c.execute( + "CREATE FUNCTION fail_history() RETURNS trigger LANGUAGE plpgsql AS" + " $$BEGIN RAISE EXCEPTION 'injected replay failure'; END$$" + ) + await c.execute( + "CREATE TRIGGER fail_history BEFORE INSERT ON reporting_projection_legacy_steps" + " FOR EACH ROW EXECUTE FUNCTION fail_history()" + ) + try: + with pytest.raises(Exception): + await h.projection.project_one(account_id=account) + finally: + async with h.pool.connection() as c: + await c.execute("DROP TRIGGER fail_history ON reporting_projection_legacy_steps") + await c.execute("DROP FUNCTION fail_history()") + assert await history_image(h, account) == before + projection = type(h.projection)( + h.store, + consumer_status_enabled=False, + revision_ownership=True, + ) + production_operation_3 = await projection.activate(account_id=account) + assert not production_operation_3 + assert await projection.baseline_ready(account_id=account) + history = await history_image(h, account) + personal = [ + doc["checkpoint"] + for _, doc in history + if doc["checkpoint"]["scope"]["consumer_id"] == first.binding.consumer_id + and doc["checkpoint"]["scope"]["reporting_obligation_id"] + == first.obligation.reporting_obligation_id + ] + assert [c["snapshot"]["health"] for c in personal] == [ + "action_required", + "complete", + "action_required", + "complete", + ] + assert [c["generation"] for c in personal] == [1, 2, 3, 4] + assert all(doc["admission_epoch"] == 0 for _, doc in history) + assert {doc["checkpoint"]["scope"]["consumer_id"] for _, doc in history} == { + first.binding.consumer_id, + second.binding.consumer_id, + } + assert await h.queue() == original_queue + current = await projection.checkpoints(account_id=account) + assert ( + next( + c + for c in current + if c.scope.consumer_id == first.binding.consumer_id + and c.scope.reporting_obligation_id == first.obligation.reporting_obligation_id + ).generation + >= 4 + ) + assert await original_inputs(h, (first, second)) == originals + production_operation_4 = await projection.activate(account_id=account) + assert not production_operation_4 + + +async def test_missing_retained_capture_refuses_activation(projections): + h = projections + first, _ = await prepare_history(h) + account = first.config.account_id + if h.pool is None: + h.store._materializer_account_heads[account] += 1 + else: + async with h.pool.connection() as c: + await c.execute( + "UPDATE reporting_materializer_accounts SET captured_sequence=captured_sequence+1" + " WHERE account_id=%s", + (account,), + ) + with pytest.raises(ReportingNotificationError, match="status_projection_history_corrupt"): + await h.projection.activate(account_id=account) + assert not await h.projection.baseline_ready(account_id=account) diff --git a/tests/conformance/reporting/test_reporting_projection_legacy_feed.py b/tests/conformance/reporting/test_reporting_projection_legacy_feed.py new file mode 100644 index 000000000..d33424105 --- /dev/null +++ b/tests/conformance/reporting/test_reporting_projection_legacy_feed.py @@ -0,0 +1,82 @@ +"""Unversioned representation-one walks survive the protocol-filter addition. + +These are legacy-format store controls. The separate rolling lane supplies +actual installed-parent evidence; this fixture does not claim that provenance. +""" + +import json +from dataclasses import replace + +import pytest + +from adcp.reporting.canonical_json import canonical_json_utf8_v1 +from adcp.reporting.feed import ReportingFeedError + +from ._feed_support import feed_request, mixed_case, walk +from ._projection_support import projection_harness + + +@pytest.mark.parametrize("backend", ["memory", "postgres"]) +@pytest.mark.parametrize("notifications", [False, True]) +async def test_legacy_unversioned_walk_survives_activation_and_current_protocol( + backend, notifications +): + async with projection_harness(backend, notifications=notifications) as h: + scenario, _, _ = await mixed_case(h) + caller = scenario.binding.principal + # The historical direct pin produces the pre-marker filter format. + # Before activation its representation is the parent's original v1. + old_request = feed_request(scenario, adcp_version="3.2-rc.3") + first = await h.store.read_reporting_feed(old_request, caller=caller) + snapshot = await h.store.read_reporting_feed_snapshot( + first["ledger_snapshot_id"], caller=caller + ) + assert snapshot.representation_version == 1 + assert snapshot.ownership_mode == "absent" + assert "adcp_version" not in json.loads(snapshot.filters_json) + stored_bytes = canonical_json_utf8_v1(snapshot.to_storage()) + expected = await walk(h.store, old_request, caller, first=first) + current_request = feed_request(scenario) + + for activated in (False, True): + if activated: + await h.projection.activate(account_id=caller.account_id) + continued = await walk(h.store, current_request, caller, first=first) + assert continued == expected + retained = await h.store.read_reporting_feed_snapshot( + snapshot.snapshot_id, caller=caller + ) + assert canonical_json_utf8_v1(retained.to_storage()) == stored_bytes + + # Caller, filter and signature failures remain indistinguishable. + continuation = feed_request( + scenario, + pagination={"cursor": first["pagination"]["cursor"], "max_results": 1}, + ) + for changed, principal in ( + ({**continuation, "media_buy_ids": ["different"]}, caller), + (continuation, replace(caller, consumer_id="different")), + ( + { + **continuation, + "pagination": {"cursor": first["pagination"]["cursor"] + "x"}, + }, + caller, + ), + ): + with pytest.raises(ReportingFeedError) as error: + await h.store.read_reporting_feed(changed, caller=principal) + assert error.value.code == "INVALID_CHECKPOINT" + + # A legacy checkpoint can start a new current-version walk without + # modifying its original boundary or any persisted old page. + repaired = await h.store.read_reporting_feed( + feed_request(scenario, changes_after=expected[2]), caller=caller + ) + newer = await h.store.read_reporting_feed_snapshot( + repaired["ledger_snapshot_id"], caller=caller + ) + assert newer.snapshot_id != snapshot.snapshot_id + assert newer.after == snapshot.through + assert json.loads(newer.filters_json)["adcp_version"] == "3.2-rc.6" + assert newer.representation_version == 2 diff --git a/tests/conformance/reporting/test_reporting_projection_memory_rollback.py b/tests/conformance/reporting/test_reporting_projection_memory_rollback.py new file mode 100644 index 000000000..e4964a77b --- /dev/null +++ b/tests/conformance/reporting/test_reporting_projection_memory_rollback.py @@ -0,0 +1,160 @@ +"""Shared frozen inputs never share mutable transaction or public response state.""" + +from copy import deepcopy +from dataclasses import FrozenInstanceError, replace +from datetime import timedelta, tzinfo + +import pytest + +from adcp.reporting.canonical_json import canonical_json_utf8_v1 +from adcp.reporting.ledger.notification_models import ReportingNotificationError +from adcp.reporting.projection.capture import ReportingProjectionInput + +from ._feed_support import feed_request, walk +from ._projection_support import projection_harness +from ._receipt_support import receipt_case, request_for + + +class InjectedMutationError(Exception): + pass + + +async def test_shared_capture_rejects_a_mutable_timezone(): + class MutableZone(tzinfo): + offset = timedelta(0) + + def utcoffset(self, dt): + return self.offset + + def dst(self, dt): + return timedelta(0) + + async with projection_harness("memory", notifications=False) as h: + case = await receipt_case(h) + await h.projection.activate(account_id=case.obligation.account_id) + captured = h.store._projection_accounts[case.obligation.account_id].inputs[0] + zone = MutableZone() + core = replace(captured.core, as_of=captured.core.as_of.replace(tzinfo=zone)) + with pytest.raises(ReportingNotificationError, match="status_projection_history_corrupt"): + ReportingProjectionInput(core, captured.reconciliation, captured.document) + + +@pytest.mark.parametrize("notifications", [False, True]) +@pytest.mark.parametrize("point", ["receipt_ordinal", "projection_capture", "feed_save"]) +async def test_memory_rollback_preserves_frozen_history_and_nested_public_values( + notifications, point, monkeypatch +): + async with projection_harness("memory", notifications=notifications) as h: + case = await receipt_case(h) + caller, account = case.binding.principal, case.obligation.account_id + await h.projection.activate(account_id=account) + request = feed_request(case) + first = await h.store.read_reporting_feed(request, caller=caller) + pages, records, checkpoint = await walk(h.store, request, caller, first=first) + public_checkpoints = await h.projection.checkpoints(account_id=account) + captured = h.store._projection_accounts[account].inputs[0] + captured_bytes = captured.document + assert deepcopy(captured) is captured + before = await h.image() + published = deepcopy((first, pages, records, checkpoint, public_checkpoints)) + + # Returned JSON is mutable by design; changing it must not lend a way + # to mutate retained bytes, a checkpoint, or a subsequent page. + first["ext"]["adcp"]["reporting_revision_ownership"]["bindings"].append( + {"reporting_revision_id": "caller-only", "reporting_obligation_id": "caller-only"} + ) + public_checkpoints[0].snapshot["issues"].append({"code": "CALLER_ONLY"}) + frozen = await h.store.read_reporting_feed_snapshot( + first["ledger_snapshot_id"], caller=caller + ) + frozen_bytes = canonical_json_utf8_v1(frozen.to_storage()) + frozen.inputs["core"]["revisions"][0]["readable"] = False + assert await h.image() == before + first, pages, records, checkpoint, public_checkpoints = published + retained = deepcopy(published) + + injected = False + + def fail_with_nested_changes(): + nonlocal injected + injected = True + # A failure after mutating actual transaction-owned nested JSON + # must restore it, as well as ordinary immutable domain records. + state = h.store._status_notification_state + next(iter(state.checkpoints.values())).snapshot["issues"].append( + {"code": "TRANSACTION_ONLY", "details": {"partial": [1, 2]}} + ) + h.store._new_projection_collection = {"sequence_head": 1, "partial": ["new"]} + raise InjectedMutationError(point) + + name = { + "receipt_ordinal": "_append_receipt_result", + "projection_capture": "_capture_projection", + "feed_save": "_save_feed_snapshot", + }[point] + original = getattr(type(h.store), name) + + def fail_after(self, *args, **kwargs): + original(self, *args, **kwargs) + fail_with_nested_changes() + + with monkeypatch.context() as patch: + patch.setattr(type(h.store), name, fail_after) + with pytest.raises(Exception): + async with h.store.transaction(): + await h.store.set_revision_readable( + account_id=account, + reporting_revision_id=case.revision.reporting_revision_id, + readable=False, + ) + if point == "feed_save": + await h.store.read_reporting_feed(request, caller=caller) + else: + await h.store.ingest_receipt_batch(request_for(case), caller=caller) + assert injected + assert not hasattr(h.store, "_new_projection_collection") + assert await h.image() == before + assert (first, pages, records, checkpoint, public_checkpoints) == retained + assert await h.projection.checkpoints(account_id=account) == public_checkpoints + assert h.store._projection_accounts[account].inputs[0] is captured + assert captured.document == captured_bytes + resumed = await walk(h.store, request, caller, first=first) + assert resumed == (pages, records, checkpoint) + reread = await h.store.read_reporting_feed_snapshot( + first["ledger_snapshot_id"], caller=caller + ) + assert canonical_json_utf8_v1(reread.to_storage()) == frozen_bytes + with pytest.raises(FrozenInstanceError): + captured.core.revisions[0].readable = False + with pytest.raises(ReportingNotificationError): + ReportingProjectionInput( + replace(captured.core, configurations=list(captured.core.configurations)), + captured.reconciliation, + captured.document, + ) + # Restore the fault and prove the real transaction can still commit. + result = await h.store.ingest_receipt_batch(request_for(case), caller=caller) + assert result["results"][0]["result"] == "recorded" + assert captured.document == captured_bytes + + +@pytest.mark.parametrize("notifications", [False, True]) +async def test_failed_first_feed_capture_removes_new_collection(notifications, monkeypatch): + async with projection_harness("memory", notifications=notifications) as h: + case = await receipt_case(h) + await h.projection.activate(account_id=case.obligation.account_id) + assert not hasattr(h.store, "_reporting_feed_snapshots") + before = await h.image() + original = type(h.store)._save_feed_snapshot + + def fail(self, stored): + original(self, stored) + assert self._reporting_feed_snapshots + raise InjectedMutationError("first feed") + + with monkeypatch.context() as patch: + patch.setattr(type(h.store), "_save_feed_snapshot", fail) + with pytest.raises(Exception): + await h.store.read_reporting_feed(feed_request(case), caller=case.binding.principal) + assert await h.image() == before + assert not hasattr(h.store, "_reporting_feed_snapshots") diff --git a/tests/conformance/reporting/test_reporting_projection_notifications.py b/tests/conformance/reporting/test_reporting_projection_notifications.py new file mode 100644 index 000000000..83b2604f4 --- /dev/null +++ b/tests/conformance/reporting/test_reporting_projection_notifications.py @@ -0,0 +1,74 @@ +"""New status queue identity, old-worker exclusion and same-transaction rollback.""" + +import pytest + +from adcp.reporting.outbox.status_memory import InMemoryReportingStatusOutbox + +from ._projection_support import projection_harness +from ._receipt_support import receipt_case, request_for + + +@pytest.mark.parametrize("backend", ["memory", "postgres"]) +@pytest.mark.parametrize("fault", [False, True]) +async def test_status_queue_isolation_and_atomic_projection(backend, fault, monkeypatch): + async with projection_harness(backend, notifications=True) as h: + case = await receipt_case(h) + account = case.obligation.account_id + if h.pool is None: + old = InMemoryReportingStatusOutbox(h.store) + else: + from adcp.reporting.outbox.status_pg import PgReportingStatusOutbox + + old = PgReportingStatusOutbox(pool=h.pool) + assert await old.list_events(account_id=account) == () + original_readiness = await h.queue() + await h.projection.activate(account_id=account) + assert await h.projection.outbox.list_events(account_id=account) == () + await h.store.ingest_receipt_batch(request_for(case), caller=case.binding.principal) + before = await h.image() + if fault: + if h.pool is None: + original = h.projection._enqueue_status + + def fail(event): + original(event) + raise RuntimeError("injected status enqueue failure") + + method = "_enqueue_status" + else: + original = h.projection._enqueue_status_on + + async def fail(connection, event): + await original(connection, event) + raise RuntimeError("injected status enqueue failure") + + method = "_enqueue_status_on" + with monkeypatch.context() as patch: + patch.setattr(h.projection, method, fail) + with pytest.raises(RuntimeError, match="injected status enqueue failure"): + await h.projection.project_one(account_id=account) + assert await h.image() == before + turn = await h.projection.project_one(account_id=account) + assert turn.did_work and turn.events > 0 + events = await h.projection.outbox.list_events(account_id=account) + assert len(events) == turn.events + assert all(e.notification_type == "reporting.status_changed" for e in events) + assert await old.list_events(account_id=account) == () + production_operation_1 = await old.claim_expansion( + account_id=account, now=h.clock(), lease_seconds=30 + ) + assert production_operation_1 is None + assert await h.queue() == original_readiness + # Persist a real expansion lease, then verify a competing incarnation + # cannot claim it and a separately constructed new outbox resumes it. + current = h.projection.outbox + lease = await current.claim_expansion(account_id=account, now=h.clock(), lease_seconds=30) + assert lease is not None + restarted = type(current)(h.store) if h.pool is None else type(current)(pool=h.pool) + await restarted.complete_expansion(lease, (), now=h.clock()) + assert await current.list_events(account_id=account) == events + assert await h.queue() == original_readiness + await h.store.ingest_receipt_batch(request_for(case), caller=case.binding.principal) + production_operation_2 = await h.projection.project_one(account_id=account) + assert not (production_operation_2).did_work + assert await current.list_events(account_id=account) == events diff --git a/tests/conformance/reporting/test_reporting_projection_rc6.py b/tests/conformance/reporting/test_reporting_projection_rc6.py new file mode 100644 index 000000000..6f74f43a7 --- /dev/null +++ b/tests/conformance/reporting/test_reporting_projection_rc6.py @@ -0,0 +1,581 @@ +"""The rc.6 forecast contract at producer, frozen store and public boundaries.""" + +from __future__ import annotations + +from copy import deepcopy +from dataclasses import replace +from datetime import datetime, timedelta, timezone +from types import SimpleNamespace + +import pytest +from jsonschema.validators import validator_for + +from adcp.reporting.canonical_json import canonical_json_utf8_v1 +from adcp.reporting.feed.errors import ReportingFeedError +from adcp.reporting.ledger import ProducerOfferings, ReportingProducer, ReportingScheduleSpec +from adcp.reporting.ledger.delivery_models import ReportingDeliveryPrincipal +from adcp.reporting.ledger.status import ReportingStatusCaller, ReportingStatusHandler +from adcp.types import GetReportingStatusRequest +from adcp.validation.schema_loader import get_named_validator, get_validator + +from ._feed_support import MountedFeed, feed_harness, walk +from ._generation_support import START, UncalledSource, configuration, revision_for +from ._projection_support import projection_harness +from ._receipt_transport import error_code +from .test_reporting_schedule_schema import formats + +RC3 = "3.2-rc.3" +RC6 = "3.2-rc.6" +CONSUMER = "https://buyer.example.test/agent" + + +@pytest.fixture(autouse=True) +def _a2a_compat_send_and_aggregate(): + # Use the real public async-generator transport instead of the unit mock shim. + pass + + +@pytest.mark.parametrize("backend", ["memory", "postgres"]) +@pytest.mark.parametrize("version", [RC6]) +async def test_production_discovery_advertises_its_usable_reporting_pin(backend, version, tmp_path): + from ._production_support import production_harness + from ._production_transport import MountedProduction + + async with production_harness( + backend, tmp_path / "discovery.sqlite", adcp_version=version + ) as h: + mounted = MountedProduction(h) + mounted.authorize(h.item) + async with mounted.client() as client: + for transport in ("mcp", "a2a-0.3", "a2a-1.0"): + _, caps = await mounted.call( + client, "get_adcp_capabilities", {}, transport=transport + ) + assert caps["adcp"]["supported_versions"] == [version] + assert caps["adcp_version"] == version + + +@pytest.mark.parametrize("version", ["3.0", "3.1", RC3]) +async def test_production_mount_rejects_releases_without_reporting_schemas(version, tmp_path): + from adcp.exceptions import ConfigurationError + + from ._production_support import production_harness + + with pytest.raises(ConfigurationError): + async with production_harness( + "memory", tmp_path / "unsupported.sqlite", adcp_version=version + ): + pass + + +@pytest.mark.parametrize("mutation", ["value", "method"]) +async def test_warm_production_proof_cannot_hide_a_changed_protocol_pin( + mutation, tmp_path, monkeypatch +): + from ._production_support import production_harness + from ._production_transport import MountedProduction + + async with production_harness("postgres", tmp_path / "pin.sqlite", adcp_version=RC6) as h: + mounted = MountedProduction(h) + mounted.authorize(h.item) + async with mounted.client() as client: + _, before = await mounted.call(client, "get_adcp_capabilities", {}) + assert before["media_buy"]["reporting_delivery"]["managed_delivery"] + if mutation == "value": + monkeypatch.setattr(h.production.handler, "_adcp_version", RC3) + else: + monkeypatch.setattr(h.production.handler, "get_adcp_version", lambda: RC3) + for transport in ("mcp", "a2a-0.3", "a2a-1.0"): + _, after = await mounted.call( + client, "get_adcp_capabilities", {}, transport=transport + ) + assert after["adcp_version"] == RC6 + assert after["adcp"]["supported_versions"] == [RC6] + assert not after.get("media_buy", {}).get("reporting_delivery") + + +async def scheduled(h, *, now=0.5, complete=True): + h.clock.now = START + timedelta(hours=now) + h.store._clock = h.clock + config = replace(configuration(), deactivated_at=None) + await h.store.put_configuration(config) + producer = ReportingProducer( + source=UncalledSource(), offerings=ProducerOfferings(), store=h.store + ) + obligations = await producer.close_elapsed_periods(config, now=h.clock()) + if complete: + for obligation in obligations: + revision, rows = revision_for(obligation, suffix=obligation.reporting_obligation_id) + await h.store.commit_revision(revision, rows) + if hasattr(h, "projection"): + await h.projection.activate(account_id=config.account_id) + return config, obligations + + +def mount(h, config, version): + mounted = MountedFeed(h, version=version, hydrated=True, registry_kind="oauth") + mounted.authorize( + SimpleNamespace(obligation=config, binding=SimpleNamespace(consumer_id=CONSUMER)) + ) + return mounted + + +def request(version, view="summary"): + return {"adcp_version": version, "account": {"account_id": "acct_a"}, "view": view} + + +def capture_clock(h, monkeypatch): + if h.pool is not None: + from adcp.reporting.feed import pg + + async def now(connection): + # Same account-locked SQL read with an explicit conformance instant. + # No projection, persisted snapshot or producer is substituted. + return ( + await (await connection.execute("SELECT %s::timestamptz", (h.clock(),))).fetchone() + )[0] + + monkeypatch.setattr(pg, "_now", now) + + +@pytest.mark.parametrize("backend", ["memory", "postgres"]) +@pytest.mark.parametrize("notifications", [False, True]) +async def test_complete_forecast_uses_period_start_without_creating_future_work( + backend, notifications +): + async with feed_harness(backend, notifications=notifications) as h: + config, obligations = await scheduled(h) + assert obligations == [] + before = await h.image() + captured = await h.store.read_status_snapshot(account_id=config.account_id) + handler = ReportingStatusHandler(h.store) + caller = ReportingStatusCaller(config.account_id, CONSUMER) + old = handler.render_snapshot(request(RC3), caller=caller, snapshot=captured) + new = handler.render_snapshot(request(RC6), caller=caller, snapshot=captured) + default = handler.render_snapshot({}, caller=caller, snapshot=captured) + assert old["next_expected_at"] == "2026-09-01T02:00:00Z" + assert new["next_expected_at"] == default["next_expected_at"] == "2026-09-01T01:00:00Z" + for field in ("health", "scope", "coverage", "obligation_counts", "issues", "ledger_as_of"): + assert old[field] == new[field] == default[field] + assert new["health"] == "complete" + assert new["obligation_counts"]["total"] == 0 + assert new["coverage"]["media_buy_ids"] == [] + assert new["ledger_snapshot_id"] != old["ledger_snapshot_id"] + for relative in ( + "media-buy/get-reporting-status-response.json", + "bundled/media-buy/get-reporting-status-response.json", + ): + raw = get_named_validator(relative, version=RC6) + assert raw is not None + raw.validate(new) + assert await h.image() == before + await h.store.put_configuration(replace(config, deactivated_at=START)) + h.clock.now += timedelta(hours=8) + assert handler.render_snapshot(request(RC6), caller=caller, snapshot=captured) == new + assert handler.render_snapshot(request(RC3), caller=caller, snapshot=captured) == old + + +@pytest.mark.parametrize("backend", ["memory", "postgres"]) +@pytest.mark.parametrize("mode", ["core", "projection"]) +@pytest.mark.parametrize("version", [RC6]) +@pytest.mark.parametrize("complete", [False, True]) +async def test_mounted_summary_periods_schema_and_client_use_the_same_pin( + backend, mode, version, complete, monkeypatch +): + factory = feed_harness if mode == "core" else projection_harness + async with factory(backend) as h: + config, obligations = await scheduled(h, now=0.5 if complete else 1.5, complete=complete) + capture_clock(h, monkeypatch) + mounted = mount(h, config, version) + before_work = await h.works() + async with mounted.client() as client: + _, inventory = await mounted.mcp(client, inventory=True) + schema = next( + t["outputSchema"] for t in inventory["tools"] if t["name"] == "get_reporting_status" + ) + advertised = validator_for(schema)(schema, format_checker=formats()) + for view in ("summary", "periods"): + for protocol in ("mcp", "a2a-0.3", "a2a-1.0"): + call = mounted.mcp if protocol == "mcp" else mounted.a2a + kwargs = {} if protocol == "mcp" else {"v1": protocol == "a2a-1.0"} + _, raw = await call(client, request(version, view), **kwargs) + assert "health" in raw, raw + assert (raw["health"] == "complete") == complete + advertised.validate(raw) + get_validator("get_reporting_status", "sync", version=version).validate(raw) + if complete and view == "summary": + assert raw["next_expected_at"] == ( + "2026-09-01T01:00:00Z" if version == RC6 else "2026-09-01T02:00:00Z" + ) + assert raw["obligation_counts"]["total"] == 0 + elif complete and view == "periods": + assert raw["periods"] == [] + assert ("next_expected_at" in raw) == ( + version == RC3 and mode == "projection" + ) + elif view == "summary" or mode == "projection": + assert raw["next_expected_at"] == "2026-09-01T02:00:00Z" + if view == "periods": + assert len(raw["periods"]) == len(obligations) + # Actual SDK transports, generated response parsing and client validators. + for a2a_version in ("0.3", "1.0"): + async with mounted.sdk_clients(a2a_version) as (clients, observed): + for client in clients.values(): + result = await client.get_reporting_status( + GetReportingStatusRequest.model_validate(request(version)) + ) + assert result.success, result + data = result.data.model_dump(mode="json", exclude_none=True) + assert data["next_expected_at"] == ( + "2026-09-01T01:00:00Z" + if complete and version == RC6 + else "2026-09-01T02:00:00Z" + ) + assert observed and all(p[2]["adcp_version"] == version for p in observed) + assert await h.works() == before_work + + +@pytest.mark.parametrize("backend", ["memory", "postgres"]) +@pytest.mark.parametrize("version", [RC3, RC6]) +async def test_frozen_continuations_retain_bytes_and_reject_cross_version_positions( + backend, version, monkeypatch +): + async with projection_harness(backend) as h: + config, _ = await scheduled(h, now=1.5) + capture_clock(h, monkeypatch) + caller = ReportingDeliveryPrincipal("acct_a", CONSUMER) + req = {**request(version, "periods"), "pagination": {"max_results": 1}} + first = await h.store.read_reporting_feed(req, caller=caller) + assert first["pagination"]["has_more"] + snapshot = await h.store.read_reporting_feed_snapshot( + first["ledger_snapshot_id"], caller=caller + ) + document = canonical_json_utf8_v1(snapshot.to_storage()) + assert ("next_expected_at" in first) == (version == RC3) + other = RC6 if version == RC3 else RC3 + pagination = {"max_results": 1, "cursor": first["pagination"]["cursor"]} + for position in ( + {"pagination": pagination}, + {"changes_after": first["changes_checkpoint"]}, + ): + with pytest.raises(ReportingFeedError) as error: + await h.store.read_reporting_feed( + {**req, **position, "adcp_version": other}, caller=caller + ) + assert error.value.code == "REPORTING_FEED_VERSION_MISMATCH" + original_pages, original_rows, checkpoint = await walk(h.store, req, caller, first=first) + await h.store.put_configuration(replace(config, deactivated_at=START)) + h.clock.now += timedelta(hours=8) + if h.pool is None: + from adcp.reporting.projection.memory import InMemoryReportingProjectionStore + + store = InMemoryReportingProjectionStore(clock=h.clock) + for key, value in vars(h.store).items(): + if key not in {"_clock", "_lock"}: + vars(store)[key] = deepcopy(value) + else: + from adcp.reporting.projection.pg import PgReportingProjectionStore + + store = PgReportingProjectionStore(pool=h.pool, clock=h.clock) + h.store = store + repeated, rows, repeated_checkpoint = await walk(store, req, caller, first=first) + assert ( + repeated == original_pages + and rows == original_rows + and repeated_checkpoint == checkpoint + ) + snapshot = await store.read_reporting_feed_snapshot( + first["ledger_snapshot_id"], caller=caller + ) + assert canonical_json_utf8_v1(snapshot.to_storage()) == document + if version == RC3: + # Historical stored bytes remain replayable through the store API. + # The current SDK does not advertise rc.3 as a live server/client pin. + return + mounted = mount(h, config, version) + continuation = {**req, "pagination": pagination} + async with mounted.client() as client: + for call in (mounted.mcp, mounted.a2a): + _, raw = await call(client, continuation) + assert raw == original_pages[1] + _, crossed = await call(client, {**continuation, "adcp_version": other}) + assert error_code(crossed) == "VERSION_UNSUPPORTED", crossed + + +@pytest.mark.parametrize("backend", ["memory", "postgres"]) +@pytest.mark.parametrize("version", [RC6]) +async def test_status_notification_mount_pin_controls_rendering_without_optional_validation( + backend, version +): + from adcp.reporting.ledger.status_server import ReportingStatusNotificationHandler + from adcp.reporting.receipts.handler import _consumer + + async with feed_harness(backend) as h: + config, _ = await scheduled(h) + mounted = mount(h, config, version) + + async def resolve_caller(params, context): + consumer = await _consumer(context, mounted.registry) + account = await mounted.resolve_account(params["account"], context, consumer) + return ReportingStatusCaller(account, consumer) + + mounted.handler = ReportingStatusNotificationHandler( + ReportingStatusHandler(h.store), + resolve_caller=resolve_caller, + adcp_version=version, + ) + async with mounted.client(validation=None) as client: + _, inventory = await mounted.mcp(client, inventory=True) + schema = next( + t["outputSchema"] for t in inventory["tools"] if t["name"] == "get_reporting_status" + ) + for call in (mounted.mcp, mounted.a2a): + _, result = await call(client, request(version)) + assert result["next_expected_at"] == ( + "2026-09-01T01:00:00Z" if version == RC6 else "2026-09-01T02:00:00Z" + ) + validator_for(schema)(schema, format_checker=formats()).validate(result) + _, rejected = await call(client, request(RC3 if version == RC6 else RC6)) + assert error_code(rejected) == "VERSION_UNSUPPORTED", rejected + + +@pytest.mark.parametrize("backend", ["memory", "postgres"]) +async def test_rc6_nearest_captured_generation_and_historical_scope_filters(backend): + async with feed_harness(backend) as h: + h.clock.now = START + timedelta(minutes=30) + h.store._clock = h.clock + first = replace( + configuration(), + delivery_config_id="forecast-first", + deactivated_at=None, + schedule=ReportingScheduleSpec("PT2H", "PT1H", "utc", period_anchor=START), + ) + anchor = START + timedelta(minutes=45) + second = replace( + first, + delivery_config_id="forecast-second", + activated_at=anchor, + schedule=ReportingScheduleSpec("PT1H", "PT1H", "utc", period_anchor=anchor), + ) + foreign = replace( + second, + account_id="acct_b", + activated_at=START, + schedule=replace(second.schedule, period_anchor=START + timedelta(minutes=40)), + ) + producer = ReportingProducer( + source=UncalledSource(), offerings=ProducerOfferings(), store=h.store + ) + for config in (first, second, foreign): + await h.store.put_configuration(config) + production_operation_2 = await producer.close_elapsed_periods(config, now=h.clock()) + assert production_operation_2 == [] + captured = await h.store.read_status_snapshot(account_id="acct_a") + handler = ReportingStatusHandler(h.store) + caller = ReportingStatusCaller("acct_a", CONSUMER) + before = await h.image() + common = {"adcp_version": RC6, "view": "summary"} + expected = { + "next_expected_at": "2026-09-01T00:45:00Z", + "health": "complete", + } + original = handler.render_snapshot(common, caller=caller, snapshot=captured) + for filters, forecast in ( + ({}, expected["next_expected_at"]), + ({"delivery_config_ids": [first.delivery_config_id]}, "2026-09-01T02:00:00Z"), + ({"delivery_config_ids": [second.delivery_config_id]}, expected["next_expected_at"]), + ({"delivery_config_ids": ["absent"]}, None), + ({"feed_purposes": ["billing"]}, None), + ({"media_buy_ids": ["foreign-buy"]}, None), + ( + { + "delivery_config_ids": [first.delivery_config_id], + "period": {"start": START.isoformat(), "end": h.clock().isoformat()}, + }, + "2026-09-01T02:00:00Z", + ), + ): + result = handler.render_snapshot( + {**common, **filters}, caller=caller, snapshot=captured + ) + assert result["health"] == "complete" and result["obligation_counts"]["total"] == 0 + assert result.get("next_expected_at") == forecast + assert result["ledger_as_of"] == "2026-09-01T00:30:00Z" + get_named_validator( + "media-buy/get-reporting-status-response.json", version=RC6 + ).validate(result) + assert all(original[key] == value for key, value in expected.items()) + assert await h.image() == before + await h.store.put_configuration(replace(second, deactivated_at=anchor)) + h.clock.now += timedelta(days=1) + assert handler.render_snapshot(common, caller=caller, snapshot=captured) == original + + +@pytest.mark.parametrize("backend", ["memory", "postgres"]) +@pytest.mark.parametrize( + "start,following", + [ + ("2026-03-08T05:00:00+00:00", "2026-03-09T04:00:00Z"), + ("2026-11-01T04:00:00+00:00", "2026-11-02T05:00:00Z"), + ], +) +async def test_rc6_period_start_forecast_retains_civil_dst_and_offset_instants( + backend, start, following +): + start = datetime.fromisoformat(start) + async with feed_harness(backend) as h: + h.clock.now = (start + timedelta(minutes=30)).astimezone( + timezone(timedelta(hours=5, minutes=30)) + ) + h.store._clock = h.clock + config = replace( + configuration(), + activated_at=start, + deactivated_at=None, + account_timezone="America/New_York", + schedule=ReportingScheduleSpec("P1D", "PT1H", "account_timezone", period_anchor=start), + ) + await h.store.put_configuration(config) + producer = ReportingProducer( + source=UncalledSource(), offerings=ProducerOfferings(), store=h.store + ) + production_operation_1 = await producer.close_elapsed_periods(config, now=h.clock()) + assert production_operation_1 == [] + caller = ReportingStatusCaller(config.account_id, CONSUMER) + handler = ReportingStatusHandler(h.store) + snapshot = await h.store.read_status_snapshot(account_id=config.account_id) + result = handler.render_snapshot(request(RC6), caller=caller, snapshot=snapshot) + assert result["next_expected_at"] == following + assert result["obligation_counts"]["total"] == 0 and result["health"] == "complete" + boundary = datetime.fromisoformat(following.replace("Z", "+00:00")) + h.clock.now = boundary + obligations = await producer.close_elapsed_periods(config, now=h.clock()) + assert len(obligations) == 1 and obligations[0].period.end == boundary + assert obligations[0].period.expected_at == boundary + timedelta(hours=1) + revision, rows = revision_for(obligations[0]) + await h.store.commit_revision(revision, rows) + next_day = await handler.handle(request(RC6), caller=caller) + assert next_day["next_expected_at"] == (boundary + timedelta(days=1)).isoformat().replace( + "+00:00", "Z" + ) + assert next_day["obligation_counts"]["total"] == 1 + assert handler.render_snapshot(request(RC6), caller=caller, snapshot=snapshot) == result + + +@pytest.mark.parametrize("backend", ["memory", "postgres"]) +@pytest.mark.parametrize("version", [RC6]) +@pytest.mark.parametrize("reconciled", [False, True]) +async def test_admitted_producer_verified_artifact_and_receipt_keep_the_future_summary( + backend, version, reconciled, tmp_path +): + from adcp.reporting.ledger import ReportingRevisionReceiptRecord + from adcp.reporting.ledger.delivery import receipt_to_wire + + from ._production_support import production_harness + from ._production_transport import MountedProduction + from ._projection_support import drain + from .test_reporting_production_lifecycle import observed_now + from .test_reporting_production_lock_order import source_turn + + async with production_harness( + backend, + tmp_path / "rc6-provider.sqlite", + count=3, + source_publication=True, + reconciled=reconciled, + adcp_version=version, + ) as h: + support, item = h.production, h.item + assert support.handler.get_adcp_version() == version + await support.activate(account_id=item.config.account_id) + turn = await source_turn(support) + assert not turn.slices_failed and len(turn.revisions_committed) == 1 + revisions = await h.store.list_revisions( + account_id=item.config.account_id, + reporting_obligation_id=item.obligation.reporting_obligation_id, + ) + assert len(revisions) == 1 + item.revision = revisions[0] + materialized = await support.materializer.run_once() + assert materialized.state == "verified" and item.writer.writes == 1 + mounted = MountedProduction(h) + mounted.authorize(item) + async with mounted.client() as client: + if reconciled: + receipt = ReportingRevisionReceiptRecord( + item.scope, + "rc6-production-accepted", + item.revision.reporting_revision_id, + materialized.reporting_materialization_id, + "accepted", + item.binding.verification_profile, + item.revision.row_count, + item.revision.managed_control_totals, + await observed_now(h), + observed_canonical_content_digest=item.revision.canonical_content_digest, + ) + _, accepted = await mounted.call( + client, + "sync_reporting_receipts", + { + "account": {"account_id": item.config.account_id}, + "idempotency_key": "rc6-production-receipt", + "receipts": [receipt_to_wire(receipt)], + }, + transport="a2a-1.0", + ) + assert accepted["results"][0]["result"] == "recorded", accepted + await drain(h.projection, item.config.account_id) + await h.store.put_configuration( + replace(item.config, deactivated_at=item.obligation.period.end) + ) + anchor = ((await observed_now(h)) + timedelta(days=1)).replace( + minute=0, second=0, microsecond=0 + ) + future = replace( + item.config, + delivery_config_version=2, + activated_at=anchor, + deactivated_at=None, + schedule=replace(item.config.schedule, period_anchor=anchor), + ) + producer = support.offerings[0].producer + producer._source.bind_generation(future) + destination = replace(item.binding, generation_key=future.generation_key) + item.writer.grant(destination) + await h.store.admit_production_configuration( + future, destination, offering_id=support.offerings[0].offering_id + ) + idle = await source_turn(support) + assert not idle.slices_failed and not idle.revisions_committed + assert len(producer._source.requests) == 1 + before_work = await h.works() + expectation = anchor if version == RC6 else anchor + timedelta(hours=2) + for transport in ("mcp", "a2a-0.3", "a2a-1.0"): + for view in ("summary", "periods"): + _, page = await mounted.call( + client, "get_reporting_status", request(version, view), transport=transport + ) + assert page["health"] == "complete", page + get_validator("get_reporting_status", "sync", version=version).validate(page) + if version == RC6: + for relative in ( + "media-buy/get-reporting-status-response.json", + "bundled/media-buy/get-reporting-status-response.json", + ): + get_named_validator(relative, version=version).validate(page) + if view == "summary": + assert page["next_expected_at"] == expectation.isoformat().replace( + "+00:00", "Z" + ) + assert page["obligation_counts"]["total"] == 1 + elif version == RC6: + assert "next_expected_at" not in page + assert await h.works() == before_work + async with MountedFeed.sdk_clients(mounted, "1.0") as (clients, observed): + for client in clients.values(): + result = await client.get_reporting_status( + GetReportingStatusRequest.model_validate(request(version)) + ) + assert result.success and result.data.health == "complete", result + assert observed and all(p[2]["adcp_version"] == version for p in observed) diff --git a/tests/conformance/reporting/test_reporting_projection_schedule.py b/tests/conformance/reporting/test_reporting_projection_schedule.py new file mode 100644 index 000000000..f6b005452 --- /dev/null +++ b/tests/conformance/reporting/test_reporting_projection_schedule.py @@ -0,0 +1,182 @@ +"""Versioned complete forecasts preserve activation, due times and captured state.""" + +from dataclasses import replace +from datetime import datetime, timedelta, timezone +from itertools import islice + +import pytest + +from adcp.reporting.ledger import ProducerOfferings, ReportingProducer, ReportingScheduleSpec +from adcp.reporting.ledger.schedule import committed_periods, next_reporting_expectation +from adcp.reporting.ledger.status import ReportingStatusCaller, ReportingStatusHandler +from adcp.types import GetReportingStatusResponse +from adcp.validation.schema_loader import get_validator + +from ._generation_support import START, UncalledSource, configuration, revision_for +from ._reliable_support import reliable_factory + + +@pytest.fixture( + params=[("memory", False), ("memory", True), ("postgres", False), ("postgres", True)] +) +async def schedules(request): + backend, notifications = request.param + async with reliable_factory(backend, notifications=notifications) as harness: + yield harness + + +def hour(value): + return START + timedelta(hours=value) + + +@pytest.mark.parametrize( + "activation,deactivation,now,legacy_due,rc6_start,closed", + [ + (0, None, 0.5, 2, 1, 0), # mid-period, no obligation or coverage yet + (0, None, 1, 2, 2, 1), # closes at 01:00, due at 02:00 + (0, None, 1.5, 2, 2, 1), + (0, None, 2, 3, 3, 2), # strictly future expectation at the exact SLA + (0, None, 2.5, 3, 3, 2), + (3, None, 0.5, 5, 3, 0), # already committed future activation + (0.5, None, 0.5, 3, 1, 0), # first full period only + (1, None, 1, 3, 2, 0), + (0, 3, 0.5, 2, 1, 0), # future deactivation + (0, 1, 2, None, None, 1), # stop exactly at the next start + (0, 0.5, 0.5, 2, None, 0), # begun full period remains owed + (0, 0.5, 2, None, None, 1), + (1, 1, 0.5, None, None, 0), # no committed active interval + (None, None, 0.5, None, None, 0), + ], +) +@pytest.mark.parametrize("adcp_version", ["3.2-rc.3", "3.2-rc.6"]) +async def test_complete_forecast_retains_versioned_activation_and_due_boundaries( + schedules, activation, deactivation, now, legacy_due, rc6_start, closed, adcp_version +): + h = schedules + h.clock.now = hour(now) + config = replace( + configuration(), + activated_at=hour(activation) if activation is not None else None, + deactivated_at=hour(deactivation) if deactivation is not None else None, + ) + await h.store.put_configuration(config) + producer = ReportingProducer( + source=UncalledSource(), offerings=ProducerOfferings(), store=h.store + ) + obligations = await producer.close_elapsed_periods(config, now=h.clock()) + assert len(obligations) == closed + production_operation_1 = await producer.close_elapsed_periods(config, now=h.clock()) + assert production_operation_1 == [] + # Completing all existing evidence must not erase tomorrow's commitment. + for obligation in obligations: + revision, rows = revision_for(obligation, suffix=obligation.reporting_obligation_id) + await h.store.commit_revision(revision, rows) + handler = ReportingStatusHandler(h.store) + for consumer in ("buyer-one", "buyer-two"): + raw = await handler.handle( + {"adcp_version": adcp_version}, caller=ReportingStatusCaller("acct_a", consumer) + ) + GetReportingStatusResponse.model_validate(raw) + validator = get_validator("get_reporting_status", "sync", version=adcp_version) + assert validator is not None + validator.validate(raw) + assert raw["health"] == "complete" + assert raw["obligation_counts"]["total"] == closed + # Complete rc.6 forecasts name the next start, never the obligation due time. + expected = rc6_start if adcp_version == "3.2-rc.6" else legacy_due + assert raw.get("next_expected_at") == ( + hour(expected).isoformat().replace("+00:00", "Z") if expected is not None else None + ) + if not closed: + assert raw["coverage"]["media_buy_ids"] == [] + assert raw["issues"] == [] + + +@pytest.mark.parametrize("adcp_version,expected_minute", [("3.2-rc.3", 10), ("3.2-rc.6", 0)]) +async def test_nearest_generation_and_account_filters_use_captured_not_current_configuration( + schedules, adcp_version, expected_minute +): + h = schedules + h.clock.now = hour(0.5) + first = replace(configuration(), deactivated_at=None) + second = replace( + first, delivery_config_version=2, schedule=replace(first.schedule, delivery_sla="PT10M") + ) + foreign = replace( + first, account_id="acct_b", schedule=replace(first.schedule, delivery_sla="PT0S") + ) + await h.store.put_configuration(first) + await h.store.put_configuration(second) + await h.store.put_configuration(foreign) + handler = ReportingStatusHandler(h.store) + caller = ReportingStatusCaller("acct_a", "buyer") + captured = await h.store.read_status_snapshot(account_id="acct_a") + expected = hour(1) + timedelta(minutes=expected_minute) + request = {"adcp_version": adcp_version} + original = handler.render_snapshot(request, caller=caller, snapshot=captured) + assert original["next_expected_at"] == expected.isoformat().replace("+00:00", "Z") + await h.store.put_configuration(replace(second, deactivated_at=hour(0))) + h.clock.now = hour(5) + assert handler.render_snapshot(request, caller=caller, snapshot=captured) == original + for filters in ( + {"delivery_config_ids": ["absent"]}, + {"feed_purposes": ["billing"]}, + {"media_buy_ids": ["foreign-buy"]}, + {"period": {"start": hour(-2).isoformat(), "end": hour(0).isoformat()}}, + ): + assert "next_expected_at" not in handler.render_snapshot( + {**request, **filters}, caller=caller, snapshot=captured + ) + + +@pytest.mark.parametrize( + "date,hours", [("2026-03-08T05:00:00+00:00", 23), ("2026-11-01T04:00:00+00:00", 25)] +) +def test_civil_days_across_dst_preserve_the_captured_timezone_and_sla(date, hours): + start = datetime.fromisoformat(date) + config = replace( + configuration(), + activated_at=start, + deactivated_at=start + timedelta(days=3), + account_timezone="America/New_York", + schedule=ReportingScheduleSpec("P1D", "PT1H", "account_timezone", period_anchor=start), + ) + first = next(committed_periods(config)) + assert first.start == start + assert first.end - first.start == timedelta(hours=hours) + assert ( + next_reporting_expectation((config,), (), as_of=start + timedelta(hours=1)) + == first.expected_at + ) + assert next_reporting_expectation((config,), (), as_of=first.expected_at) > first.expected_at + + +@pytest.mark.parametrize("duration", ["PT1H", "PT10M"]) +def test_nonexistent_civil_slots_are_skipped_without_duplicate_or_reordered_periods(duration): + start = datetime(2026, 3, 8, 5, tzinfo=timezone.utc) + config = replace( + configuration(), + activated_at=start, + deactivated_at=start + timedelta(hours=5), + schedule=ReportingScheduleSpec( + duration, "PT0S", "custom_timezone", "America/New_York", start + ), + ) + periods = list(committed_periods(config)) + assert all(a.end == b.start for a, b in zip(periods, periods[1:])) + assert len({p.period_key for p in periods}) == len(periods) + for period in periods: + near = period.start + (period.end - period.start) / 2 + assert next_reporting_expectation((config,), (), as_of=near) == period.expected_at + + +def test_empty_scope_and_explicit_anchor_before_or_after_activation(): + assert next_reporting_expectation((), (), as_of=START) is None + config = replace( + configuration(), + deactivated_at=None, + schedule=replace(configuration().schedule, period_anchor=hour(10)), + ) + assert [p.start for p in islice(committed_periods(config), 2)] == [START, hour(1)] + offset = START.astimezone(timezone(timedelta(hours=5, minutes=30))) + assert next_reporting_expectation((config,), (), as_of=offset) == hour(2) diff --git a/tests/conformance/reporting/test_reporting_projection_timestamps.py b/tests/conformance/reporting/test_reporting_projection_timestamps.py new file mode 100644 index 000000000..7b1918d48 --- /dev/null +++ b/tests/conformance/reporting/test_reporting_projection_timestamps.py @@ -0,0 +1,114 @@ +"""PostgreSQL JSON timestamps retain their exact instant on the Python floor.""" + +from copy import deepcopy +from dataclasses import replace +from datetime import datetime, timedelta, timezone + +import pytest + +from adcp.reporting._timestamp import aware_timestamp +from adcp.reporting.canonical_json import canonical_json_utf8_v1 +from adcp.reporting.ledger.notification_models import ReportingNotificationError +from adcp.reporting.ledger.status_snapshot import snapshot_from_storage +from adcp.reporting.projection.capture import decode_projection_input +from adcp.validation.schema_loader import get_named_validator + +from ._generation_support import configuration +from ._projection_support import projection_harness + + +@pytest.mark.parametrize("microsecond", [1, 100000, 120000, 123000, 123400, 123450, 123456]) +@pytest.mark.parametrize("offset", ["+00:00:00", "-00:00:00", "+05:45:03", "-02:30:01"]) +def test_fractional_offsets_preserve_the_exact_instant(microsecond, offset): + hours, minutes, seconds = map(int, offset[1:].split(":")) + delta = timedelta(hours=hours, minutes=minutes, seconds=seconds, microseconds=microsecond) + if offset[0] == "-": + delta = -delta + value = "2026-09-01T12:00:00.12345" + offset + "." + f"{microsecond:06d}".rstrip("0") + expected = datetime(2026, 9, 1, 12, 0, 0, 123450, tzinfo=timezone.utc) - delta + assert aware_timestamp(value) == expected + + +@pytest.mark.parametrize("fraction", ["", ".1", ".12345", ".123456"]) +def test_utc_z_is_aware_and_lossless(fraction): + microsecond = int(fraction.lstrip(".").ljust(6, "0")) + assert aware_timestamp("2026-09-01T12:00:00" + fraction + "Z") == datetime( + 2026, 9, 1, 12, 0, 0, microsecond, tzinfo=timezone.utc + ) + + +@pytest.mark.parametrize("microsecond", [0, 100000, 120000, 123000, 123400, 123450, 123456]) +@pytest.mark.parametrize("zone", ["UTC", "Asia/Kathmandu", "America/St_Johns", "Europe/Paris"]) +async def test_actual_pg_json_precision_and_offset_survive_captured_decode(microsecond, zone): + # Paris before 1911 also exercises a real seconds-bearing historical offset. + at = datetime(1890 if zone == "Europe/Paris" else 2026, 9, 1, 12, tzinfo=timezone.utc) + at = at.replace(microsecond=microsecond) + async with projection_harness("postgres") as h: + config = configuration() + config = replace( + config, + schedule=replace(config.schedule, period_anchor=at - timedelta(days=1)), + activated_at=at - timedelta(days=1), + deactivated_at=at + timedelta(days=1), + ) + await h.store.put_configuration(config) + async with h.pool.connection() as c: + await c.execute("SELECT set_config('TimeZone',%s,false)", (zone,)) + row = await ( + await c.execute( + "SELECT reporting_projection_document(%s,%s), to_jsonb(%s::timestamptz)", + (config.account_id, at, at), + ) + ).fetchone() + document, timestamp = row + assert document["as_of"] == document["core"]["as_of"] == timestamp + fraction = timestamp.split("T", 1)[1].split("+", 1)[0].split("-", 1)[0] + fraction = fraction.partition(".")[2] + assert fraction == (f"{microsecond:06d}".rstrip("0") if microsecond else "") + raw = canonical_json_utf8_v1(document) + core = snapshot_from_storage(document["core"]) + decoded = decode_projection_input(document) + assert core == decoded.core + assert core.as_of == at and core.as_of.microsecond == microsecond + assert core.configurations == (config,) + assert decoded.document == raw == canonical_json_utf8_v1(document) + # Exercise the actual PostgreSQL representation through the public + # named-schema path as well. RFC 3339 excludes the seconds-bearing + # historical Paris offset that the private lossless decoder permits. + validator = get_named_validator( + "core/reporting-delivery-config-state.json", version="3.2.0-rc.3" + ) + assert validator is not None + field = validator.evolve(schema=validator.schema["properties"]["activated_at"]) + assert field.is_valid(timestamp) is (zone != "Europe/Paris") + assert canonical_json_utf8_v1(document) == raw + + +@pytest.mark.parametrize( + "timestamp", + [ + "2026-09-01T12:00:00.12345", + "2026-09-01T12:00:00", + "2026-09-01T12:00:00.1234567+00:00", + "2026-09-01T12:00:00.12x45+00:00", + "2026-09-01T25:00:00.12345+00:00", + "2026-02-30T12:00:00.12345+00:00", + "2026-09-01T12:00:00.12345+25:00", + "2026-09-01T12:00:00.12345+01:60", + "2026-09-01T12:00:00.12345+00:09:60", + "not-a-timestamp", + ], +) +async def test_sql_boundary_rejects_invalid_naive_or_precision_losing_timestamps(timestamp): + async with projection_harness("postgres") as h: + async with h.pool.connection() as c: + row = await ( + await c.execute( + "SELECT reporting_projection_document('acct_a',%s)", + (datetime(2026, 9, 1, tzinfo=timezone.utc),), + ) + ).fetchone() + document = deepcopy(row[0]) + document["as_of"] = document["core"]["as_of"] = timestamp + with pytest.raises(ReportingNotificationError, match="status_projection_history_corrupt"): + decode_projection_input(document) diff --git a/tests/conformance/reporting/test_reporting_projection_waiver_rc6.py b/tests/conformance/reporting/test_reporting_projection_waiver_rc6.py new file mode 100644 index 000000000..a0b888647 --- /dev/null +++ b/tests/conformance/reporting/test_reporting_projection_waiver_rc6.py @@ -0,0 +1,72 @@ +"""Production projection inputs preserve the exact-scoped rc.6 waiver contract.""" + +from dataclasses import replace +from datetime import timedelta + +import pytest + +from adcp.reporting.ledger.status_projection import mismatch_key + +from ._projection_support import drain, inputs, projection_harness +from ._receipt_support import receipt_case +from .test_reporting_notification_outbox import statement + + +@pytest.mark.parametrize("backend", ["memory", "postgres"]) +@pytest.mark.parametrize("notifications", [False, True]) +async def test_projection_captures_exact_waiver_and_rearms_later_statement(backend, notifications): + async with projection_harness(backend, notifications=notifications, feedback=True) as h: + scenario = await receipt_case( + h, finality="snapshot", billing=False, reconciliation_mode="delivery_only" + ) + original = replace( + statement(scenario.obligation, consumer=scenario.binding.consumer_id), + consumer_status="unreadable", + failure_code="access_denied", + ) + await h.store.record_consumer_status(original) + await h.projection.activate(account_id=scenario.obligation.account_id) + request = {"view": "summary", "adcp_version": "3.2-rc.6"} + before = await h.store.read_tier_status( + request, caller=scenario.binding.principal, consumer_status_enabled=True + ) + assert before["health"] == "action_required" + + waived = await h.store.set_issue_state( + issue_key=mismatch_key(original), + account_id=scenario.obligation.account_id, + state="waived", + at=h.clock(), + external_ref="private-bilateral-audit", + ) + await drain(h.projection, scenario.obligation.account_id) + recovered = await h.store.read_tier_status( + request, caller=scenario.binding.principal, consumer_status_enabled=True + ) + assert recovered["health"] == "complete" and recovered["issues"] == [] + captured = await inputs(h, scenario.obligation.account_id) + assert any( + row.issue_id == waived.issue_id and row.waived_conflict_sha256 + for row in captured[-1].core.lifecycles + ) + + h.clock.now += timedelta(seconds=1) + later = replace( + original, + reporting_status_id="next-mismatch", + supersedes_reporting_status_id=original.reporting_status_id, + recorded_at=h.clock(), + status_as_of=h.clock(), + ) + await h.store.record_consumer_status(later) + await drain(h.projection, scenario.obligation.account_id) + current = await h.store.read_tier_status( + request, caller=scenario.binding.principal, consumer_status_enabled=True + ) + assert current["health"] == "action_required" + assert current["issues"][0]["reporting_status_id"] == later.reporting_status_id + assert current["issues"][0]["issue_id"] != waived.issue_id + retained = await h.store.get_issue( + account_id=scenario.obligation.account_id, issue_key=waived.issue_key + ) + assert retained == waived diff --git a/tests/conformance/reporting/test_reporting_projection_wire.py b/tests/conformance/reporting/test_reporting_projection_wire.py new file mode 100644 index 000000000..37c7ade4d --- /dev/null +++ b/tests/conformance/reporting/test_reporting_projection_wire.py @@ -0,0 +1,77 @@ +"""Authenticated tier reads use the same captured private inputs in every view.""" + +import json +from functools import partial + +import pytest + +from adcp.reporting.ownership import page_revision_ownership + +from ._feed_support import MountedFeed, feed_request, second_consumer +from ._projection_support import projection_harness +from ._receipt_support import receipt_case +from ._receipt_transport import error_code + + +@pytest.mark.parametrize("backend", ["memory", "postgres"]) +@pytest.mark.parametrize("notifications", [False, True]) +@pytest.mark.parametrize("feedback", [False, True]) +async def test_mounted_private_summary_revision_and_page_walk(backend, notifications, feedback): + async with projection_harness(backend, notifications=notifications, feedback=feedback) as h: + first = await receipt_case(h) + second = await second_consumer(h, first, "https://buyer.example/second") + await h.projection.activate(account_id=first.obligation.account_id) + mounted = MountedFeed(h, feedback=feedback, hydrated=True, registry_kind="oauth") + mounted.authorize(first) + mounted.authorize(second, token="token-two") + async with mounted.client() as client: + for call in (mounted.mcp, mounted.a2a, partial(mounted.a2a, v1=True)): + summaries = [] + for item, token, receipt_count in ( + (first, "token-one", 0), + (second, "token-two", 1), + ): + request = feed_request(item, view="summary") + del request["pagination"] + _, summary = await call(client, request, token=token) + assert summary.get("status") == "completed", summary + summaries.append(summary) + request.update( + view="revision", + reporting_revision_id=item.revision.reporting_revision_id, + ) + _, exact = await call(client, request, token=token) + assert exact.get("status") == "completed", exact + assert len(exact["receipts"]) == receipt_count + assert len(exact["materializations"]) == 1 + assert page_revision_ownership(exact) == { + item.revision.reporting_revision_id: item.obligation.reporting_obligation_id + } + assert exact["revision"]["revision_content_sha256"] == ( + item.revision.revision_content_sha256 + ) + records = [] + request = feed_request(item) + for _ in range(20): + _, page = await call(client, request, token=token) + assert page.get("status") == "completed", page + page_revision_ownership(page) + records.extend(page["receipts"]) + if not page["pagination"]["has_more"]: + break + request["pagination"]["cursor"] = page["pagination"]["cursor"] + else: + pytest.fail("bounded mounted walk did not finish") + assert len(records) == receipt_count + assert summaries[0]["ledger_snapshot_id"] != summaries[1]["ledger_snapshot_id"] + assert summaries[0]["health"] != summaries[1]["health"] + assert second.binding.consumer_id not in json.dumps(summaries[0]) + mounted.grants.remove((first.obligation.account_id, first.binding.consumer_id)) + for view in ("summary", "revision", "periods"): + request = feed_request(first, view=view) + if view != "periods": + del request["pagination"] + if view == "revision": + request["reporting_revision_id"] = first.revision.reporting_revision_id + _, denied = await mounted.mcp(client, request) + assert error_code(denied) == "UNAUTHORIZED" diff --git a/tests/conformance/reporting/test_reporting_schedule_schema.py b/tests/conformance/reporting/test_reporting_schedule_schema.py new file mode 100644 index 000000000..fa039cd53 --- /dev/null +++ b/tests/conformance/reporting/test_reporting_schedule_schema.py @@ -0,0 +1,252 @@ +"""#1179: execute the exact cached rejection and the version-scoped SDK correction. + +This is Python evidence, not approval of another SDK or a cross-language pin. +Every designated blocking compatible lane must independently succeed with this +scenario; an unsupported-schema outcome is only valid in an unsupported lane. +""" + +import hashlib +import json +from copy import deepcopy +from dataclasses import replace +from datetime import timedelta +from types import SimpleNamespace + +import pytest +from jsonschema import Draft7Validator, FormatChecker +from jsonschema.validators import validator_for + +from adcp.reporting.ledger import InMemoryReportingLedgerStore +from adcp.reporting.ledger.status import ReportingStatusCaller, ReportingStatusHandler +from adcp.types import GetReportingStatusResponse +from adcp.validation import schema_loader + +from ._feed_support import MountedFeed, feed_harness +from ._generation_support import START, configuration +from ._projection_support import projection_harness + +PIN = "3.2.0-rc.3" +RULE = "/allOf/2/then/not" +CACHED = ( + ( + "media-buy/get-reporting-status-response.json", + 30561, + "498774fa2a15ce1487183d3df4f982436936427d104879279524ed5d38d8e726", + ), + ( + "bundled/media-buy/get-reporting-status-response.json", + 314212, + "578ea8233c4022c528dee3b53bcb88884b36033c71d77a0eb00bee4bc67c72e3", + ), + ( + "mcp/2026-07-28/profiles/production/media-buy/get-reporting-status-response.json", + 171397, + "6b68df71ef16d3317a20e2b4f23f4c385f1f9bbb36bddc80515c719e56868b74", + ), +) + + +def formats(): + checker = FormatChecker() + checker.checks("date-time")(schema_loader._is_rfc3339_date_time) + return checker + + +async def deterministic_summary(): + store = InMemoryReportingLedgerStore(clock=lambda: START + timedelta(minutes=30)) + await store.put_configuration(replace(configuration(), deactivated_at=None)) + return await ReportingStatusHandler(store).handle( + {"adcp_version": "3.2-rc.3"}, + caller=ReportingStatusCaller("acct_a", "https://buyer.example.test/agent"), + ) + + +def assert_original_rejection(raw, relative=CACHED[0][0]): + validator = schema_loader.get_named_validator(relative, version=PIN) + assert validator is not None + if relative.startswith("mcp/"): + validator = validator_for(validator.schema)(validator.schema, format_checker=formats()) + without_expectation = {k: v for k, v in raw.items() if k != "next_expected_at"} + validator.validate(without_expectation) + errors = list(validator.iter_errors(raw)) + assert len(errors) == 1, errors + error = errors[0] + assert error.validator == "not" + assert list(error.absolute_schema_path) == ["allOf", 2, "then", "not"] + return { + "validator": error.validator, + "schema_pointer": RULE, + "instance_path": list(error.absolute_path), + "message": error.message, + } + + +def invalid_summaries(raw): + for field in ("scope_closed", "coverage_complete"): + for value in (False, None): + invalid = deepcopy(raw) + if value is None: + del invalid["scope"][field] + else: + invalid["scope"][field] = value + yield invalid + for value in (None, 5, "2026-09-01", "2026-09-01T02:00:00", "not-a-time"): + yield {**raw, "next_expected_at": value} + yield {**raw, "health": "pretend-complete"} + yield {**raw, "status": "pretend-completed"} + yield {**raw, "view": "periods"} # summary cannot evade the periods requirements + invalid = deepcopy(raw) + invalid["obligation_counts"]["total"] = "0" + yield invalid + + +@pytest.mark.parametrize("relative,expected_bytes,expected_sha256", CACHED) +async def test_exact_unmodified_cached_rule_rejects_the_required_payload( + relative, expected_bytes, expected_sha256 +): + root = schema_loader._resolve_schema_root(PIN) + assert root is not None + file = root.root / relative + original = file.read_bytes() + assert len(original) == expected_bytes + assert hashlib.sha256(original).hexdigest() == expected_sha256 + schema = json.loads(original) + assert schema["$schema"] == ( + "https://json-schema.org/draft/2020-12/schema" + if relative.startswith("mcp/") + else "http://json-schema.org/draft-07/schema#" + ) + validator = schema_loader.get_named_validator(relative, version=PIN) + assert validator is not None and validator.schema == schema + raw = await deterministic_summary() + assert raw["health"] == "complete" + assert raw["scope"]["scope_closed"] is raw["scope"]["coverage_complete"] is True + assert raw["next_expected_at"] == "2026-09-01T02:00:00Z" + rejection = assert_original_rejection(raw, relative) + effective = schema_loader._effective_task_schema( + schema, "get_reporting_status", "sync", bundle_key=PIN + ) + reconstructed = deepcopy(effective) + reconstructed["allOf"][2]["then"]["not"] = {"required": ["next_expected_at"]} + assert reconstructed == schema # only this pointer differs; all guards survive + assert file.read_bytes() == original + print( + json.dumps( + { + "cached_rule_reproduction": { + "schema_file": str(file), + "schema_uri": f"https://adcontextprotocol.org/schemas/{PIN}/{relative}", + "version": PIN, + "bytes": len(original), + "sha256": expected_sha256, + "dialect": schema["$schema"], + "payload": raw, + "exact_rejection": rejection, + "effective_correction": {"removed_pointer": RULE, "other_changes": []}, + } + }, + sort_keys=True, + ) + ) + + +async def test_effective_validation_advertisement_and_remaining_constraints(): + raw = await deterministic_summary() + GetReportingStatusResponse.model_validate(raw) + validator = schema_loader.get_validator("get_reporting_status", "sync", version=PIN) + assert validator is not None + validators = [validator] + for load in ( + schema_loader.get_schema, + schema_loader.get_portable_schema, + schema_loader.get_mcp_schema, + ): + schema = load("get_reporting_status", "sync", version=PIN) + assert schema is not None + validators.append(validator_for(schema)(deepcopy(schema), format_checker=formats())) + saved = deepcopy(schema) + schema.clear() + assert load("get_reporting_status", "sync", version=PIN) == saved + for validator in validators: + validator.validate(raw) + for invalid in invalid_summaries(raw): + assert not validator.is_valid(invalid), invalid + + +@pytest.mark.parametrize("mutation", ["other-pin", "changed-if", "changed-then", "moved-rule"]) +def test_correction_is_limited_to_the_known_rule_and_version(mutation): + schema = schema_loader.get_named_schema_document(CACHED[0][0], version=PIN) + assert schema is not None + version = PIN + if mutation == "other-pin": + version = "3.2.0-beta.4" + elif mutation == "changed-if": + schema["allOf"][2]["if"]["properties"]["health"]["const"] = "healthy" + elif mutation == "changed-then": + schema["allOf"][2]["then"]["required"] = ["unreviewed-condition"] + else: + schema["allOf"].insert(0, {}) + original = deepcopy(schema) + assert ( + schema_loader._effective_task_schema( + schema, "get_reporting_status", "sync", bundle_key=version + ) + == original + ) + assert schema == original + + +@pytest.mark.parametrize("backend", ["memory", "postgres"]) +@pytest.mark.parametrize("mode", ["core", "projection"]) +@pytest.mark.parametrize("notifications", [False, True]) +@pytest.mark.parametrize("version", [None, "3.2.0-rc.6"]) +async def test_complete_future_expectation_on_actual_summary_mounts( + backend, mode, notifications, version +): + factory = feed_harness if mode == "core" else projection_harness + async with factory(backend, notifications=notifications) as h: + h.clock.now = START + timedelta(minutes=30) + h.store._clock = h.clock # supported deterministic store clock, including PG captures + config = replace(configuration(), deactivated_at=None) + await h.store.put_configuration(config) + if mode == "projection": + await h.projection.activate(account_id=config.account_id) + mounted = MountedFeed(h, version=version, hydrated=True, registry_kind="oauth") + identity = SimpleNamespace( + obligation=config, + binding=SimpleNamespace(consumer_id="https://buyer.example.test/agent"), + ) + mounted.authorize(identity) + request = { + "adcp_version": "3.2-rc.6", + "account": {"account_id": config.account_id}, + "view": "summary", + } + async with mounted.client() as client: + _, inventory = await mounted.mcp(client, inventory=True) + output = next( + t["outputSchema"] for t in inventory["tools"] if t["name"] == "get_reporting_status" + ) + advertised = Draft7Validator(output, format_checker=formats()) + for path in ("/.well-known/agent.json", "/.well-known/agent-card.json"): + card = await client.get(path) + assert card.status_code == 200 + assert "get_reporting_status" in {s["id"] for s in card.json()["skills"]} + results = [] + for transport in ("mcp", "a2a-0.3", "a2a-1.0"): + if transport == "mcp": + _, raw = await mounted.mcp(client, request) + else: + _, raw = await mounted.a2a(client, request, v1=transport == "a2a-1.0") + assert raw.get("health") == "complete", raw + assert raw["next_expected_at"] == "2026-09-01T01:00:00Z" + assert_original_rejection(raw) + schema_loader.get_validator("get_reporting_status", "sync", version=PIN).validate( + raw + ) + advertised.validate(raw) + GetReportingStatusResponse.model_validate(raw) + for invalid in invalid_summaries(raw): + assert not advertised.is_valid(invalid), invalid + results.append(raw) + assert results[0] == results[1] == results[2] diff --git a/tests/conformance/reporting/test_reporting_tier_projection.py b/tests/conformance/reporting/test_reporting_tier_projection.py new file mode 100644 index 000000000..19d58831c --- /dev/null +++ b/tests/conformance/reporting/test_reporting_tier_projection.py @@ -0,0 +1,152 @@ +"""One captured tier projection over the real memory/PostgreSQL financial graph.""" + +from dataclasses import replace +from datetime import timedelta + +import pytest + +from adcp.reporting.ledger import ReportingMaterializationCheck +from adcp.reporting.ledger.notification_models import ReportingStatusScope +from adcp.reporting.ledger.reconciliation_projection import project_reconciliation +from adcp.reporting.ledger.status_projection import StatusProjectionInput, project_status_scope +from adcp.reporting.outbox.status import advance_checkpoint + +from ._generation_support import END +from ._receipt_support import adjustment_for, receipt_case, receipts, request_for + +__all__ = ["receipts"] + + +async def captured(h, s, *, feedback=False): + core = await h.store.read_status_snapshot(account_id=s.obligation.account_id) + private = await h.store.read_reconciliation_snapshot(caller=s.binding.principal) + core = replace(core, as_of=h.clock.now) + scope = ReportingStatusScope( + core.account_id, + s.binding.generation_key, + consumer_id=s.binding.consumer_id, + reporting_obligation_id=s.obligation.reporting_obligation_id, + feed_purpose=s.obligation.feed_purpose, + ) + return StatusProjectionInput( + core, scope, reconciliation=private.records, consumer_status_enabled=feedback + ) + + +def projected(value): + result = project_status_scope(value) + assert len(result.obligations) == 1 + tier = result.obligations[0].reconciliation + assert tier is not None + return result, tier + + +async def test_receipts_adjustments_and_reversible_health_keep_each_ordered_generation(receipts): + h = receipts + s = await receipt_case(h) + first = await captured(h, s) + result, tier = projected(first) + assert tier.wire["reconciliation_status"] == "pending" + assert tier.wire["successful_materialization_count"] == 1 + previous, _ = advance_checkpoint(None, result, fired_at=h.clock.now, source_sequence=1) + await h.store.ingest_receipt_batch(request_for(s), caller=s.binding.principal) + accepted = await captured(h, s) + result, tier = projected(accepted) + assert tier.satisfied and tier.wire["reconciliation_status"] == "accepted" + previous, event = advance_checkpoint(previous, result, fired_at=h.clock.now, source_sequence=2) + assert event is not None and previous.generation == 2 + item = await adjustment_for(h, s) + pending_adjustment = await captured(h, s) + result, tier = projected(pending_adjustment) + assert not tier.satisfied and tier.wire["pending_adjustment_count"] == 1 + previous, event = advance_checkpoint(previous, result, fired_at=h.clock.now, source_sequence=3) + assert event is not None and previous.generation == 3 + await h.store.ingest_receipt_batch( + { + "account": {"account_id": s.obligation.account_id}, + "idempotency_key": "adjustment-batch-0001", + "adjustment_receipts": [item], + }, + caller=s.binding.principal, + ) + completed = await captured(h, s) + result, tier = projected(completed) + assert tier.satisfied and tier.wire["pending_adjustment_count"] == 0 + previous, event = advance_checkpoint(previous, result, fired_at=h.clock.now, source_sequence=4) + assert event is not None and previous.generation == 4 + # Replay every old captured input after later mutable state has changed. + assert projected(first)[1].wire["reconciliation_status"] == "pending" + assert projected(accepted)[1].wire["pending_adjustment_count"] == 0 + assert projected(pending_adjustment)[1].wire["pending_adjustment_count"] == 1 + + +@pytest.mark.parametrize("later", ["corrupt", "expired", "unreadable", "success", "failure"]) +async def test_accepted_artifact_evidence_survives_later_health_and_attempts(receipts, later): + h = receipts + s = await receipt_case(h) + await h.store.ingest_receipt_batch(request_for(s), caller=s.binding.principal) + if later == "corrupt": + await h.store.record_materialization_check( + ReportingMaterializationCheck( + s.attempt.scope, + s.attempt.reporting_materialization_id, + "later-corruption", + "corrupt", + END + timedelta(seconds=20), + ) + ) + elif later == "expired": + h.clock.now = s.outcome.resource.expires_at + elif later == "unreadable": + await h.store.set_revision_readable( + account_id=s.obligation.account_id, + reporting_revision_id=s.revision.reporting_revision_id, + readable=False, + ) + else: + attempt = replace(s.attempt, reporting_materialization_id="materialization-2", attempt=2) + await h.store.commit_materialization_attempt(attempt) + outcome = replace( + s.outcome, + reporting_materialization_id=attempt.reporting_materialization_id, + status=s.outcome.status if later == "success" else "failed", + resource=s.outcome.resource if later == "success" else None, + verification=s.outcome.verification if later == "success" else None, + failure_code=None if later == "success" else "WRITE_FAILED", + ) + await h.store.commit_materialization(outcome) + _, tier = projected(await captured(h, s)) + assert tier.wire["reconciliation_status"] == "accepted" + assert tier.wire["accepted_receipt_count"] == 1 + assert tier.wire["successful_materialization_count"] == (2 if later == "success" else 1) + assert tier.satisfied is (later in {"success", "failure"}) + assert tier.wire[ + "resource_retained_until" + ] == s.delivery.resource_retained_until.isoformat().replace("+00:00", "Z") + + +async def test_official_selection_precedes_available_snapshot_artifacts(receipts): + h = receipts + s = await receipt_case( + h, finality="snapshot", billing=False, reconciliation_mode="delivery_only" + ) + private = await h.store.read_reconciliation_snapshot(caller=s.binding.principal) + official = replace( + s.revision, + reporting_revision_id="unmaterialized-official", + finality="official", + finality_basis="source_final", + finality_policy_id="policy-1", + finalized_at=END, + ) + result = project_reconciliation( + replace(s.obligation, required_finality="official"), + (s.revision, official), + (), + private.records, + consumer_id=s.binding.consumer_id, + as_of=h.clock.now, + ) + assert not result.satisfied + assert result.wire["successful_materialization_count"] == 1 + assert result.wire["reconciliation_status"] == "not_required" diff --git a/tests/fixtures/public_api_snapshot.json b/tests/fixtures/public_api_snapshot.json index 7ac6e13f2..17bb99443 100644 --- a/tests/fixtures/public_api_snapshot.json +++ b/tests/fixtures/public_api_snapshot.json @@ -1363,6 +1363,8 @@ "ReportPlanOutcomeResponse", "ReportUsageRequest", "ReportUsageResponse", + "ReportingAdjustment", + "ReportingAdjustmentReceipt", "ReportingAuthoritativeParty", "ReportingBucket", "ReportingCanonicalContentDigest", diff --git a/tests/test_mcp_schema_materialization.py b/tests/test_mcp_schema_materialization.py new file mode 100644 index 000000000..338e5675d --- /dev/null +++ b/tests/test_mcp_schema_materialization.py @@ -0,0 +1,295 @@ +"""Versioned public schema construction is cached without sharing mutable state.""" + +from __future__ import annotations + +import hashlib +import json +import threading +from collections import Counter +from concurrent.futures import ThreadPoolExecutor +from copy import deepcopy +from typing import Any + +import httpx +import pytest +from asgi_lifespan import LifespanManager + +from adcp.server import ADCPHandler, create_mcp_server, mcp_tools +from adcp.server.a2a_server import create_a2a_server +from adcp.validation import schema_loader as loader +from adcp.validation.schema_validator import validate_request + +PINS = ("3.2.0-rc.3", "3.2.0-rc.6", "3.2.0-beta.6") + +# Canonical JSON digests captured from unchanged e9a1c8fc before adding +# the materialization cache. These are transformed public schemas, +# not a claim of byte identity with the signed upstream schema files. +EXPECTED_PUBLIC_SHA256 = { + "3.2.0-rc.3": "6656874ca37ea0732e65a5f0313c8ead7b56ed12460bdbae297c4c648fa26f14", + "3.2.0-rc.6": "d712168e85932dfabad8a76b49a24aa6411dc11748832d18dd0ae00ad7106b3e", + "3.2.0-beta.6": "e9a10b9f1ee5654a51219c91329089f591c953b972e57a913450a1b9acdb810c", +} + + +class SchemaHandler(ADCPHandler): + def __init__(self) -> None: + self.calls = 0 + + async def get_products(self, params: Any, context: Any = None) -> dict[str, Any]: + self.calls += 1 + return {"products": []} + + async def get_reporting_status(self, params: Any, context: Any = None) -> dict[str, Any]: + raise AssertionError("schema discovery must not execute a reporting task") + + +class PinnedSchemaHandler(SchemaHandler): + def __init__(self, version: str) -> None: + super().__init__() + self.version = version + + def get_adcp_version(self) -> str: + return self.version + + +@pytest.fixture(autouse=True) +def isolated_loader(): + loader._reset_for_tests() + yield + loader._reset_for_tests() + + +def canonical(value: Any) -> bytes: + return json.dumps(value, sort_keys=True, separators=(",", ":")).encode() + + +def mutable_ids(value: Any) -> set[int]: + """Reject aliases within a result as well as between independent results.""" + seen: set[int] = set() + pending = [value] + while pending: + node = pending.pop() + if isinstance(node, (dict, list)): + assert id(node) not in seen + seen.add(id(node)) + pending.extend(node.values() if isinstance(node, dict) else node) + return seen + + +def public_definitions(version: str) -> list[dict[str, Any]]: + return mcp_tools.get_tools_for_handler(PinnedSchemaHandler(version)) + + +def count_materializations(monkeypatch): + original = loader._self_contained_schema + calls: Counter[tuple[str, str]] = Counter() + lock = threading.Lock() + + def counted(state, path, schema): + with lock: + calls[state.bundle_key, str(path)] += 1 + return original(state, path, schema) + + monkeypatch.setattr(loader, "_self_contained_schema", counted) + return calls + + +@pytest.mark.parametrize("version", PINS) +def test_repeated_public_schema_construction_materializes_once(monkeypatch, version): + calls = count_materializations(monkeypatch) + first = loader.get_mcp_schema("get_products", "request", version=version) + assert first is not None + original = canonical(first) + second = loader.get_mcp_schema("get_products", "request", version=version) + assert canonical(second) == original + assert not mutable_ids(first).intersection(mutable_ids(second)) + first["properties"]["brief"] = {"type": "array"} + third = loader.get_mcp_schema("get_products", "request", version=version) + assert canonical(third) == original + assert not mutable_ids(second).intersection(mutable_ids(third)) + assert list(calls.values()) == [1] + + +def test_public_handler_pins_have_isolated_materializations(monkeypatch): + calls = count_materializations(monkeypatch) + saved = {} + for version in PINS: + definitions = public_definitions(version) + saved[version] = canonical(definitions) + assert hashlib.sha256(saved[version]).hexdigest() == EXPECTED_PUBLIC_SHA256[version] + for definition in definitions: + definition["inputSchema"].clear() + if "outputSchema" in definition: + definition["outputSchema"].clear() + for version in reversed(PINS): + definitions = public_definitions(version) + assert canonical(definitions) == saved[version] + assert {key[0] for key in calls} == set(PINS) + assert calls and set(calls.values()) == {1} + assert len(set(saved.values())) == len(PINS) + + +def test_loader_reset_discards_materializations(monkeypatch): + calls = count_materializations(monkeypatch) + first = loader.get_mcp_schema("get_products", "request", version=PINS[1]) + loader._reset_for_tests() + second = loader.get_mcp_schema("get_products", "request", version=PINS[1]) + assert first == second + assert not mutable_ids(first).intersection(mutable_ids(second)) + assert list(calls.values()) == [2] + + +def test_failed_materialization_is_not_cached(monkeypatch): + original = loader._self_contained_schema + attempts = 0 + + def transient_failure(*args, **kwargs): + nonlocal attempts + attempts += 1 + if attempts == 1: + raise ValueError("missing fixture reference") + return original(*args, **kwargs) + + monkeypatch.setattr(loader, "_self_contained_schema", transient_failure) + missing = loader.get_mcp_schema("get_products", "request", version=PINS[1]) + assert missing is None + restored = loader.get_mcp_schema("get_products", "request", version=PINS[1]) + assert restored is not None + cached = loader.get_mcp_schema("get_products", "request", version=PINS[1]) + assert cached == restored + assert attempts == 2 + + +def test_concurrent_first_callers_share_one_materialization(monkeypatch): + workers = 8 + ready = threading.Barrier(workers + 1) + entered = threading.Event() + release = threading.Event() + count_lock = threading.Lock() + original = loader._self_contained_schema + attempts = 0 + + def blocked(*args, **kwargs): + nonlocal attempts + with count_lock: + attempts += 1 + entered.set() + released = release.wait(timeout=20) + assert released + return original(*args, **kwargs) + + def load(): + ready.wait(timeout=20) + return loader.get_mcp_schema("get_products", "request", version=PINS[1]) + + monkeypatch.setattr(loader, "_self_contained_schema", blocked) + with ThreadPoolExecutor(max_workers=workers) as pool: + pending = [pool.submit(load) for _ in range(workers)] + try: + ready.wait(timeout=20) + started = entered.wait(timeout=20) + assert started + finally: + release.set() + results = [future.result(timeout=30) for future in pending] + assert results[0] is not None and all(value == results[0] for value in results) + seen: set[int] = set() + for result in results: + identities = mutable_ids(result) + assert not seen.intersection(identities) + seen.update(identities) + assert attempts == 1 + + +def test_pinned_schemas_do_not_copy_superseded_current_models(monkeypatch): + class SupersededSchema(dict): + def __deepcopy__(self, memo): + raise AssertionError("copied a current-model schema that the pin replaces") + + def unexpected_generation(*args, **kwargs): + raise AssertionError("pinned discovery generated current-model schemas") + + definition = next(t for t in mcp_tools.ADCP_TOOL_DEFINITIONS if t["name"] == "get_products") + monkeypatch.setitem(definition, "inputSchema", SupersededSchema()) + monkeypatch.setitem(definition, "outputSchema", SupersededSchema()) + monkeypatch.setattr(mcp_tools, "_ensure_pydantic_schemas_applied", unexpected_generation) + definitions = public_definitions(PINS[1]) + assert hashlib.sha256(canonical(definitions)).hexdigest() == (EXPECTED_PUBLIC_SHA256[PINS[1]]) + + +def test_current_model_fallback_retains_exact_definitions_and_mutation_isolation(): + first = mcp_tools.get_tools_for_handler(SchemaHandler()) + names = {definition["name"] for definition in first} + expected = [t for t in mcp_tools.ADCP_TOOL_DEFINITIONS if t["name"] in names] + assert canonical(first) == canonical(expected) + snapshot = canonical(first) + for definition in first: + mutable_ids(definition) + definition["inputSchema"].clear() + repeated = mcp_tools.get_tools_for_handler(SchemaHandler()) + assert canonical(repeated) == snapshot + + +def test_unsupported_public_pin_never_reuses_a_warm_supported_schema(): + public_definitions(PINS[1]) + with pytest.raises(ValueError, match="no bundled AdCP schemas"): + public_definitions("3.2.0-rc.999") + unsupported = loader.get_mcp_schema("get_products", "request", version="3.2.0-rc.999") + assert unsupported is None + + +@pytest.mark.parametrize("version", PINS) +@pytest.mark.asyncio +async def test_mutation_cannot_change_mounted_discovery_registration_or_validation(version): + initial = public_definitions(version) + expected = next(t for t in initial if t["name"] == "get_products") + expected_input = deepcopy(expected["inputSchema"]) + expected_output = deepcopy(expected["outputSchema"]) + expected["inputSchema"]["properties"]["brief"] = {"type": "array"} + expected["outputSchema"].clear() + handler = PinnedSchemaHandler(version) + mcp = create_mcp_server(handler, stateless_http=True, allowed_hosts=["test"]) + mcp.settings.json_response = True + app = mcp.streamable_http_app() + headers = {"accept": "application/json, text/event-stream"} + async with LifespanManager(app): + async with httpx.AsyncClient( + transport=httpx.ASGITransport(app=app), + base_url="http://test", + follow_redirects=True, + ) as client: + response = await client.post( + "/mcp/", + json={"jsonrpc": "2.0", "id": 1, "method": "tools/list", "params": {}}, + headers=headers, + ) + assert response.status_code == 200 + tools = response.json()["result"]["tools"] + advertised = next(t for t in tools if t["name"] == "get_products") + assert advertised["inputSchema"] == expected_input + assert advertised["outputSchema"] == expected_output + response = await client.post( + "/mcp/", + json={ + "jsonrpc": "2.0", + "id": 2, + "method": "tools/call", + "params": {"name": "get_products", "arguments": {"brief": []}}, + }, + headers=headers, + ) + assert response.status_code == 200 + result = response.json() + assert "error" in result or result.get("result", {}).get("isError") is True + assert handler.calls == 0 + assert not validate_request("get_products", {"brief": []}, version=version).valid + a2a = create_a2a_server(PinnedSchemaHandler(version), name="schema-materialization") + async with httpx.AsyncClient( + transport=httpx.ASGITransport(app=a2a), base_url="http://test" + ) as client: + for path in ("/.well-known/agent.json", "/.well-known/agent-card.json"): + response = await client.get(path) + assert response.status_code == 200 + assert "get_products" in {skill["id"] for skill in response.json()["skills"]} + repeated = public_definitions(version) + assert hashlib.sha256(canonical(repeated)).hexdigest() == (EXPECTED_PUBLIC_SHA256[version]) diff --git a/tests/test_reporting_capability_models.py b/tests/test_reporting_capability_models.py new file mode 100644 index 000000000..d8ae20449 --- /dev/null +++ b/tests/test_reporting_capability_models.py @@ -0,0 +1,174 @@ +"""Public reporting promises are nullable, explicit and tier-consistent (#1180).""" + +from __future__ import annotations + +import importlib +import json +import shutil +from pathlib import Path +from typing import get_args + +import pytest +from pydantic import BaseModel, ValidationError + +from adcp.types import GetAdcpCapabilitiesResponse, ReportingDeliveryCapabilities +from tests.test_reporting_ledger import _OFFERING + +PROMISES = { + "receipt_task": "sync_reporting_receipts", + "readiness_notification": "reporting.delivery_ready", + "status_notification": "reporting.status_changed", + "ledger_notification": "reporting.ledger_changed", +} + + +def capability(**fields): + return { + "supported": True, + "offerings": [_OFFERING], + "automated_recovery_window_seconds": 0, + "status_retention_days": 7, + **fields, + } + + +def response(raw): + return { + "status": "completed", + "adcp": { + "major_versions": [3], + "idempotency": {"supported": True, "replay_ttl_seconds": 3600}, + }, + "supported_protocols": ["media_buy"], + "media_buy": {"reporting_delivery": raw}, + } + + +@pytest.fixture(params=["public", "bundled"]) +def graph(request): + if request.param == "public": + return ReportingDeliveryCapabilities, GetAdcpCapabilitiesResponse + # Codegen's self-contained clone is deliberately tested as a separate graph. + module = importlib.import_module( + "adcp.types.generated_poc.bundled.protocol.get_adcp_capabilities_response" + ) + return module.ReportingDelivery, module.GetAdcpCapabilitiesResponse + + +@pytest.mark.parametrize( + "flags", + [ + {}, + {"managed_delivery": False}, + {"reconciled_billing": False}, + {"managed_delivery": False, "reconciled_billing": False}, + ], +) +def test_core_attributes_schema_and_json_round_trip_do_not_synthesize_promises(graph, flags): + model, envelope = graph + value = model.model_validate(capability(**flags)) + fields = set(value.model_fields_set) + schema = model.model_json_schema() + for name in PROMISES: + field = model.model_fields[name] + assert field.default is None + assert type(None) in get_args(field.annotation) + assert getattr(value, name) is None + assert schema["properties"][name]["default"] is None + assert {"type": "null"} in schema["properties"][name]["anyOf"] + for raw in ( + value.model_dump(), + value.model_dump(mode="json"), + json.loads(value.model_dump_json()), + ): + assert not PROMISES.keys() & raw.keys() + assert model.model_validate(raw).model_dump(mode="json") == value.model_dump(mode="json") + restored = model.model_validate_json(value.model_dump_json()) + assert restored.model_dump(mode="json") == value.model_dump(mode="json") + nested = envelope.model_validate(response(capability(**flags))) + for raw in (nested.model_dump(mode="json"), json.loads(nested.model_dump_json())): + assert not PROMISES.keys() & raw["media_buy"]["reporting_delivery"].keys() + restored = envelope.model_validate_json(json.dumps(raw)) + assert restored.media_buy.reporting_delivery.receipt_task is None + assert restored.media_buy.reporting_delivery.readiness_notification is None + assert fields == value.model_fields_set + + +@pytest.mark.parametrize( + "field,tier", + [("readiness_notification", "managed_delivery"), ("receipt_task", "reconciled_billing")], +) +@pytest.mark.parametrize("flag", [None, False]) +def test_inverse_tier_validation_runs_in_both_standalone_and_nested_graphs( + graph, field, tier, flag +): + model, envelope = graph + raw = capability(**{field: PROMISES[field], **({tier: flag} if flag is not None else {})}) + with pytest.raises(ValidationError, match=f"{field} requires {tier}"): + model.model_validate(raw) + with pytest.raises(ValidationError, match=f"{field} requires {tier}"): + envelope.model_validate(response(raw)) + + +@pytest.mark.parametrize("managed", [None, False]) +def test_reconciled_is_cumulative_even_without_a_receipt_task(graph, managed): + model, envelope = graph + raw = capability(reconciled_billing=True, managed_delivery=managed) + for cls, body in ((model, raw), (envelope, response(raw))): + with pytest.raises(ValidationError, match="reconciled_billing requires managed_delivery"): + cls.model_validate(body) + + +@pytest.mark.parametrize("field", PROMISES) +def test_each_explicit_supported_literal_survives_without_implying_other_promises(graph, field): + model, envelope = graph + flags = {"managed_delivery": True} if field == "readiness_notification" else {} + if field == "receipt_task": + flags = {"managed_delivery": True, "reconciled_billing": True} + raw = capability(**flags, **{field: PROMISES[field]}) + for cls, body in ((model, raw), (envelope, response(raw))): + value = cls.model_validate(body) + for output in (value.model_dump(mode="json"), json.loads(value.model_dump_json())): + block = output if cls is model else output["media_buy"]["reporting_delivery"] + assert {key: block[key] for key in PROMISES if key in block} == {field: PROMISES[field]} + with pytest.raises(ValidationError): + cls.model_validate( + {**raw, field: "unsupported"} + if cls is model + else response({**raw, field: "unsupported"}) + ) + + +def test_subclasses_inside_a_non_sdk_parent_omit_absent_promises(): + class Reporting(ReportingDeliveryCapabilities): + pass + + class OrdinaryParent(BaseModel): + reporting: Reporting + + value = OrdinaryParent.model_validate({"reporting": capability()}) + assert all(getattr(value.reporting, name) is None for name in PROMISES) + for raw in (value.model_dump(), json.loads(value.model_dump_json())): + assert not PROMISES.keys() & raw["reporting"].keys() + value = OrdinaryParent.model_validate( + {"reporting": capability(status_notification=PROMISES["status_notification"])} + ) + assert value.model_dump()["reporting"]["status_notification"] == PROMISES["status_notification"] + + +def test_post_generation_repair_is_idempotent_for_both_actual_model_layouts(tmp_path, monkeypatch): + from scripts import post_generate_fixes + + root = Path(__file__).parents[1] / "src/adcp/types/generated_poc" + targets = ( + "core/reporting_delivery_capabilities.py", + "bundled/protocol/get_adcp_capabilities_response.py", + ) + for relative in targets: + target = tmp_path / relative + target.parent.mkdir(parents=True, exist_ok=True) + shutil.copyfile(root / relative, target) + monkeypatch.setattr(post_generate_fixes, "OUTPUT_DIR", tmp_path) + before = [(tmp_path / name).read_bytes() for name in targets] + post_generate_fixes.fix_reporting_capability_defaults() + assert before == [(tmp_path / name).read_bytes() for name in targets] diff --git a/tests/test_reporting_production_public.py b/tests/test_reporting_production_public.py new file mode 100644 index 000000000..87322a1b3 --- /dev/null +++ b/tests/test_reporting_production_public.py @@ -0,0 +1,50 @@ +"""All public exports work without optional drivers; concrete PG classes are lazy.""" + +import subprocess +import sys + +import pytest + + +@pytest.mark.parametrize("module", ["adcp.reporting.production", "adcp.reporting.projection"]) +@pytest.mark.parametrize("drivers", [False, True]) +def test_public_exports_and_optional_database_drivers(module, drivers): + script = """ +import importlib, sys +drivers = sys.argv[2] == 'True' +if not drivers: + class NoDrivers: + def find_spec(self, fullname, path=None, target=None): + if fullname.split('.')[0] in {'psycopg', 'psycopg_pool'}: + raise ModuleNotFoundError(fullname) + sys.meta_path.insert(0, NoDrivers()) +module = importlib.import_module(sys.argv[1]) +assert len(module.__all__) == len(set(module.__all__)) +assert module.__name__ + '.pg' not in sys.modules +for name in module.__all__: + assert getattr(module, name) is not None, name +if not drivers: + assert 'psycopg' not in sys.modules + assert 'psycopg_pool' not in sys.modules + name = ('PgReportingProductionStore' if module.__name__.endswith('production') + else 'PgReportingProjectionStore') + try: + getattr(module, name)(pool=object()) + except ImportError as error: + assert 'pg' in str(error) + else: + raise AssertionError('PG store constructed without its driver') +try: + getattr(module, 'NoSuchReportingExport') +except AttributeError: + pass +else: + raise AssertionError('unknown public export was accepted') +""" + result = subprocess.run( + [sys.executable, "-I", "-c", script, module, str(drivers)], + capture_output=True, + text=True, + timeout=60, + ) + assert result.returncode == 0, result.stdout + result.stderr diff --git a/tests/test_reporting_revision_ownership.py b/tests/test_reporting_revision_ownership.py new file mode 100644 index 000000000..5ab6699f9 --- /dev/null +++ b/tests/test_reporting_revision_ownership.py @@ -0,0 +1,243 @@ +"""Ownership is checked across the whole public, bounded periods walk.""" + +from copy import deepcopy +from datetime import datetime, timezone + +import pytest + +from adcp.reporting import ( + ReportingReconciliationError, + evaluate_reporting_ledger, + load_reporting_ledger, +) +from adcp.reporting.ownership import ( + ReportingOwnershipError, + page_revision_ownership, + with_revision_ownership, +) +from adcp.types import GetReportingStatusRequest, GetReportingStatusResponse +from adcp.types.core import TaskResult, TaskStatus +from tests.test_reporting_reconciliation import REVISION, _obligation, _response + +ARRAYS = ( + "periods", + "revisions", + "materializations", + "receipts", + "adjustments", + "adjustment_receipts", +) +OWNER = "obligation-billing" +REVISION_ID = REVISION["reporting_revision_id"] + + +class Pages: + def __init__(self, pages): + self.pages, self.calls = pages, 0 + + async def get_reporting_status(self, request): + page = self.pages[self.calls % len(self.pages)] + self.calls += 1 + return TaskResult( + status=TaskStatus.COMPLETED, + data=GetReportingStatusResponse.model_validate(deepcopy(page)), + ) + + +def pages(raw=None): + raw = deepcopy(raw or _response()) + owners = {REVISION_ID: OWNER, "revision-other": "obligation-other"} + records = [(name, record) for name in ARRAYS for record in raw.get(name, [])] + result = [] + for index, (name, record) in enumerate(records): + page = {k: v for k, v in raw.items() if k not in ARRAYS} + page.update({a: [] for a in ARRAYS}) + page[name] = [record] + page["pagination"] = {"has_more": index + 1 < len(records), "total_count": len(records)} + if page["pagination"]["has_more"]: + page["pagination"]["cursor"] = f"cursor-{index}" + page["changes_checkpoint"] = "constant-checkpoint" + result.append(with_revision_ownership(page, owners)) + return result + + +async def load(values, **bounds): + return await load_reporting_ledger( + Pages(values), + GetReportingStatusRequest.model_validate( + {"account": {"account_id": "account-1"}, "view": "periods"} + ), + **bounds, + ) + + +def test_page_local_merge_preserves_keys_and_input_and_explicit_empty_mode(): + raw = { + "revisions": [{"reporting_revision_id": "r"}], + "ext": {"vendor": {"a": 1}, "adcp": {"other": "retained"}}, + } + original = deepcopy(raw) + result = with_revision_ownership(raw, {"r": "o", "not-on-page": "o"}) + assert raw == original + assert result["ext"]["vendor"] == {"a": 1} + assert result["ext"]["adcp"]["other"] == "retained" + assert page_revision_ownership(result) == {"r": "o"} + assert with_revision_ownership(result, {"r": "o"}) == result + assert page_revision_ownership(with_revision_ownership({"revisions": []}, {})) == {} + assert page_revision_ownership({"revisions": []}) is None + + +@pytest.mark.parametrize( + "reserved", + [ + None, + [], + "bad", + {}, + {"version": True, "bindings": []}, + {"version": 2, "bindings": []}, + {"version": 1, "bindings": {}}, + {"version": 1, "bindings": [], "unknown": 1}, + { + "version": 1, + "bindings": [{"reporting_revision_id": "r", "reporting_obligation_id": "o"}], + }, + ], +) +def test_malformed_reserved_values_and_extra_page_binding_fail(reserved): + with pytest.raises(ReportingOwnershipError): + page_revision_ownership( + {"revisions": [], "ext": {"adcp": {"reporting_revision_ownership": reserved}}} + ) + + +@pytest.mark.parametrize("ext", [None, [], "bad", {"adcp": None}, {"adcp": []}, {"adcp": "bad"}]) +def test_non_object_reserved_namespaces_are_not_legacy(ext): + with pytest.raises(ReportingOwnershipError): + with_revision_ownership({"revisions": [], "ext": ext}, {}) + + +def test_duplicate_missing_and_conflicting_binding_are_rejected(): + page = with_revision_ownership({"revisions": [{"reporting_revision_id": "r"}]}, {"r": "o"}) + duplicate = deepcopy(page) + bindings = duplicate["ext"]["adcp"]["reporting_revision_ownership"]["bindings"] + bindings.append(deepcopy(bindings[0])) + with pytest.raises(ReportingOwnershipError): + page_revision_ownership(duplicate) + with pytest.raises(ReportingOwnershipError): + with_revision_ownership(page, {"r": "other"}) + with pytest.raises(ReportingOwnershipError): + with_revision_ownership({"revisions": [{"reporting_revision_id": "r"}]}, {}) + + +@pytest.mark.parametrize("name", ["revision", "reporting_revision"]) +def test_exact_view_checks_supplied_binding_without_proving_an_unknown_owner(name): + raw = { + name: {"reporting_revision_id": "r"}, + "reporting_revision_binding": {"reporting_revision_id": "r"}, + } + page = with_revision_ownership(raw, {"r": "owner-needs-periods-walk"}) + assert page_revision_ownership(page) == {"r": "owner-needs-periods-walk"} + for other in {"revision", "reporting_revision", "revisions"} - {name}: + with pytest.raises(ReportingOwnershipError): + page_revision_ownership({**page, other: []}) + for binding in (None, {}, {"reporting_revision_id": "other"}): + with pytest.raises(ReportingOwnershipError): + page_revision_ownership({**page, "reporting_revision_binding": binding}) + + +def test_a2a_struct_numeric_version_preserves_integer_semantics(): + page = with_revision_ownership({"revisions": []}, {}) + reserved = page["ext"]["adcp"]["reporting_revision_ownership"] + reserved["version"] = 1.0 + assert page_revision_ownership(page) == {} + for bad in (True, "1", 1.5, float("nan"), float("inf")): + reserved["version"] = bad + with pytest.raises(ReportingOwnershipError): + page_revision_ownership(page) + + +async def test_page_size_one_dependencies_and_repeated_identical_metadata(): + values = pages() + # Revisions precede their owner and materialization on this wire walk. + values[0]["periods"], values[1]["periods"] = [], values[0]["periods"] + values[0]["revisions"], values[1]["revisions"] = values[1]["revisions"], [] + for i in (0, 1): + values[i].pop("ext") + values[i] = with_revision_ownership(values[i], {REVISION_ID: OWNER}) + repeated = deepcopy(values[0]) + repeated["pagination"]["cursor"] = "repeated-revision" + values.insert(1, repeated) + ledger = await load(values) + assert ledger.revision_ownership == {REVISION_ID: OWNER} + assert len(ledger.revisions) == len(ledger.obligations) == len(ledger.materializations) == 1 + + +@pytest.mark.parametrize( + "mutation", + [ + "mixed", + "unknown-owner", + "foreign-account", + "semantic", + "count", + "missing-revision", + "changed-owner", + ], +) +async def test_full_walk_rejects_inconsistent_ownership(mutation): + values = pages() + if mutation == "mixed": + values[-1].pop("ext") + elif mutation == "unknown-owner": + values[1]["ext"]["adcp"]["reporting_revision_ownership"]["bindings"][0][ + "reporting_obligation_id" + ] = "unknown" + elif mutation == "foreign-account": + values[0]["periods"][0]["account_id"] = "other-account" + elif mutation == "semantic": + values[1]["revisions"][0]["media_buy_ids"] = ["unknown-buy"] + elif mutation == "count": + values[0]["periods"][0]["revision_count"] = 2 + elif mutation == "missing-revision": + values[1]["revisions"] = [] + else: + extra = deepcopy(values[1]) + extra["pagination"]["cursor"] = "extra" + extra["ext"]["adcp"]["reporting_revision_ownership"]["bindings"][0][ + "reporting_obligation_id" + ] = "unknown" + values.insert(2, extra) + with pytest.raises(ReportingReconciliationError, match="ownership"): + await load(values) + + +async def test_explicit_ownership_separates_identical_scopes_and_legacy_stays_conservative(): + raw = _response() + raw["periods"].append(_obligation("obligation-other")) + raw["revisions"].append({**deepcopy(REVISION), "reporting_revision_id": "revision-other"}) + raw["periods"][1].update( + destination_ref=None, + materialization_count=None, + successful_materialization_count=None, + reconciliation_mode="delivery_only", + reconciliation_status="not_required", + health="complete", + ) + owned = await load(pages(raw)) + result = evaluate_reporting_ledger(owned, now=datetime(2026, 9, 3, tzinfo=timezone.utc)) + assert result.obligations[1].reporting_revision_id == "revision-other" + assert result.obligations[1].definitive + legacy_pages = pages(raw) + for page in legacy_pages: + page.pop("ext") + legacy = await load(legacy_pages) + assert legacy.revision_ownership is None + assert not evaluate_reporting_ledger(legacy).obligations[1].definitive + + +@pytest.mark.parametrize("bounds", [{"max_pages": 1}, {"max_records": 1}]) +async def test_walk_budgets_are_enforced(bounds): + with pytest.raises(ReportingReconciliationError) as error: + await load(pages(), **bounds) + assert error.value.code == "LEDGER_LIMIT_EXCEEDED" diff --git a/tests/test_schema_datetime_formats.py b/tests/test_schema_datetime_formats.py new file mode 100644 index 000000000..e2d7f333a --- /dev/null +++ b/tests/test_schema_datetime_formats.py @@ -0,0 +1,136 @@ +"""Public date-time format validation is independent of Python parser precision. + +These exercise the registered named and task validators on the actual cached +schemas. Persisted reporting timestamps have a separate, lossless decoder; +its microsecond precision and historical offset rules are not this contract. +""" + +from copy import deepcopy + +import pytest + +from adcp.validation.schema_loader import get_named_validator, get_validator +from adcp.validation.schema_validator import validate_request + +PIN = "3.2.0-rc.3" +FRACTIONS = ("", ".1", ".12", ".123", ".1234", ".12345", ".123456", ".123456789012") +OFFSETS = ("Z", "z", "+00:00", "-00:00", "+05:45", "-03:30", "+23:59") +VALID = tuple( + "2026-04-01T12:00:00" + fraction + offset for fraction in FRACTIONS for offset in OFFSETS +) + ( + "2024-02-29t12:00:00.00001z", + "2000-02-29T12:00:00Z", + "0001-01-01T00:00:00Z", + "9999-12-31T23:59:59.999999999Z", +) +INVALID = ( + "2026-02-29T12:00:00Z", + "1900-02-29T12:00:00Z", + "2026-02-30T12:00:00.12345Z", + "0000-01-01T00:00:00Z", + "2026-00-01T00:00:00Z", + "2026-13-01T00:00:00Z", + "2026-04-00T00:00:00Z", + "2026-04-31T00:00:00Z", + "2026-04-01T12:00:00.12345", + "2026-04-01T12:00:00", + "2026-04-01", + "2026-04-01T12:00:00.Z", + "2026-04-01T12:00:00,12345Z", + "2026-04-01T24:00:00Z", + "2026-04-01T12:60:00Z", + # Keep the existing checker's seconds 00..59 boundary; this correction + # does not commission leap-second support or new offset syntax. + "2016-12-31T23:59:60Z", + "2026-04-01T12:00:00+24:00", + "2026-04-01T12:00:00+05:60", + "2026-04-01T12:00:00+05:45:03", + "2026-04-01T12:00:00+0545", + "2026-04-01 12:00:00Z", + "20260401T120000Z", + "2026-W14-3T12:00:00Z", + "2026-04-01T12:00:00.\u0661Z", + "2026-04-01T1\u0662:00:00Z", + "2026-04-01T12:00:00+0\u0661:00", + "2026-04-01T12:00:00Z\n", + "2026-04-01T12:00:00Z trailing", + "not-a-timestamp", + None, + 5, + True, + {}, +) + + +def reporting_timestamp_field(): + validator = get_named_validator("core/reporting-delivery-config-state.json", version=PIN) + assert validator is not None + field = validator.schema["properties"]["activated_at"] + assert field["type"] == "string" and field["format"] == "date-time" + return validator.evolve(schema=field) + + +def request_with_timestamp(value): + return { + "proposal_id": "format-contract-proposal", + "total_budget": {"amount": 50000, "currency": "USD"}, + "start_time": value, + "end_time": "2030-06-30T23:59:59Z", + "idempotency_key": "format-contract-0001", + "brand": {"domain": "advertiser.example.test"}, + "account": {"account_id": "acct_format"}, + } + + +@pytest.mark.parametrize("value", VALID) +def test_named_reporting_format_accepts_fractional_precision_without_coercion(value): + original = value.encode() + reporting_timestamp_field().validate(value) + assert value.encode() == original + + +@pytest.mark.parametrize("value", INVALID) +def test_named_reporting_format_preserves_invalid_input_rejection(value): + assert not reporting_timestamp_field().is_valid(value) + + +@pytest.mark.parametrize("version", ["3.1", PIN]) +@pytest.mark.parametrize("value", VALID) +def test_public_request_format_preserves_exact_value_and_oneof(version, value): + payload = request_with_timestamp(value) + original = deepcopy(payload) + validator = get_validator("create_media_buy", "request", version=version) + assert validator is not None + validator.validate(payload) + outcome = validate_request("create_media_buy", payload, version=version) + assert outcome.valid, outcome.issues + assert payload == original + + +@pytest.mark.parametrize("version", ["3.1", PIN]) +@pytest.mark.parametrize("value", INVALID) +def test_public_request_format_preserves_invalid_input_rejection(version, value): + payload = request_with_timestamp(value) + original = deepcopy(payload) + outcome = validate_request("create_media_buy", payload, version=version) + assert not outcome.valid + assert any(issue.pointer == "/start_time" for issue in outcome.issues) + assert payload == original + + +@pytest.mark.parametrize("version", ["3.1", PIN]) +def test_public_request_asap_still_selects_only_the_const_branch(version): + payload = request_with_timestamp("asap") + assert validate_request("create_media_buy", payload, version=version).valid + payload["end_time"] = "asap" + assert not validate_request("create_media_buy", payload, version=version).valid + assert not reporting_timestamp_field().is_valid("asap") + + +@pytest.mark.parametrize("value", [None, 5, True, {}, []]) +def test_format_annotation_does_not_impose_a_string_type(value): + # JSON Schema format applies to strings. The actual cached field's type + # constraint, rather than a new format coercion, rejects nonstrings. + field = reporting_timestamp_field() + assert field.evolve(schema={"format": "date-time"}).is_valid(value) + assert not field.is_valid(value) diff --git a/tests/test_schema_loader_per_version.py b/tests/test_schema_loader_per_version.py index bb8e6bffc..70d679a79 100644 --- a/tests/test_schema_loader_per_version.py +++ b/tests/test_schema_loader_per_version.py @@ -161,6 +161,81 @@ def test_root_relative_legacy_refs_resolve_from_offline_registry( assert not invalid.valid +def test_schedule_correction_does_not_change_a_different_version( + synthetic_legacy_bundle: tuple[str, Path], +) -> None: + """A similar legacy rule requires its own explicit compatibility decision.""" + version, root = synthetic_legacy_bundle + pinned = _loader_mod.get_named_schema_document( + "media-buy/get-reporting-status-response.json", version="3.2.0-rc.3" + ) + assert pinned is not None + schema = {"allOf": [{}, {}, pinned["allOf"][2]]} + file = root / "bundled" / "get-reporting-status-response.json" + original = json.dumps(schema).encode() + file.write_bytes(original) + value = { + "health": "complete", + "scope": {"scope_closed": True, "coverage_complete": True}, + "next_expected_at": "2026-10-01T01:00:00Z", + } + validator = get_validator("get_reporting_status", "sync", version=version) + assert validator is not None and not validator.is_valid(value) + for load in ( + _loader_mod.get_schema, + _loader_mod.get_portable_schema, + _loader_mod.get_mcp_schema, + ): + assert load("get_reporting_status", "sync", version=version) == schema + assert file.read_bytes() == original + + +def test_modular_schema_without_id_resolves_its_own_fragments_offline( + synthetic_legacy_bundle: tuple[str, Path], monkeypatch: pytest.MonkeyPatch +) -> None: + """Canonical path refs do not require $id or a network round trip.""" + legacy_key, root = synthetic_legacy_bundle + child = { + "definitions": {"Choice": {"type": "string", "enum": ["producer_managed"]}}, + "type": "object", + "properties": {"orchestration": {"$ref": "#/definitions/Choice"}}, + "required": ["orchestration"], + } + (root / "core" / "method.json").write_text(json.dumps(child), encoding="utf-8") + request = { + "type": "object", + "definitions": {"Choice": {"type": "integer"}}, + "properties": { + "method": { + "$ref": f"https://adcontextprotocol.org/schemas/{legacy_key}/core/method.json" + }, + "count": {"$ref": "#/definitions/Choice"}, + }, + "required": ["method", "count"], + } + (root / "bundled" / "synthetic-tool-request.json").write_text( + json.dumps(request), encoding="utf-8" + ) + + def deny_remote(*args: object, **kwargs: object) -> None: + pytest.fail("validation attempted to retrieve a remote schema") + + import warnings + + with warnings.catch_warnings(): + warnings.simplefilter("ignore", DeprecationWarning) + from jsonschema import RefResolver + + monkeypatch.setattr(RefResolver, "resolve_remote", deny_remote) + valid = {"method": {"orchestration": "producer_managed"}, "count": 1} + assert validate_request("synthetic_tool", valid, version=legacy_key).valid + invalid = {"method": {"orchestration": "consumer_managed"}, "count": 1} + assert not validate_request("synthetic_tool", invalid, version=legacy_key).valid + assert not validate_request( + "synthetic_tool", {**valid, "count": "producer_managed"}, version=legacy_key + ).valid + + @pytest.mark.parametrize("scheme", ["https", "http"]) def test_canonical_reference_without_id_resolves_offline_or_fails_closed( synthetic_legacy_bundle: tuple[str, Path],