From 3e611508f0b773286784284e3b545a063c6e2675 Mon Sep 17 00:00:00 2001 From: Brian O'Kelley Date: Thu, 17 Sep 2026 10:17:43 +0000 Subject: [PATCH 01/10] feat(reporting): ingest receipts with authenticated durable replay Persist exact mixed-batch ordinals, immutable financial evidence and private captured status inputs through the existing account-locked materializer store. Bind authenticated MCP/A2A receipt ingress to lossless whole-request identity. Refs #1167 B2.2 of 4 within B2 of B1/B2. Frozen feed, versioned public status and production capability activation remain gated on B2.3 and B2.4. --- .github/workflows/ci.yml | 136 +++- .github/workflows/pr-title-check.yml | 2 +- docs/reporting-receipt-ingress.md | 194 ++++++ examples/reporting_receipt_ingress.py | 66 ++ pyproject.toml | 1 + scripts/reporting_test_harness.py | 64 ++ src/adcp/reporting/ledger/_delivery_state.py | 56 +- src/adcp/reporting/ledger/delivery.py | 4 +- src/adcp/reporting/ledger/delivery_pg.py | 15 +- .../ledger/reporting_receipt_ingestion.sql | 405 ++++++++++++ src/adcp/reporting/receipts/__init__.py | 32 + src/adcp/reporting/receipts/capture.py | 93 +++ src/adcp/reporting/receipts/errors.py | 31 + src/adcp/reporting/receipts/handler.py | 153 +++++ src/adcp/reporting/receipts/memory.py | 203 ++++++ src/adcp/reporting/receipts/pg.py | 386 +++++++++++ src/adcp/reporting/receipts/records.py | 105 +++ .../reporting/receipts/required_schema.json | 410 ++++++++++++ src/adcp/reporting/receipts/schema.py | 27 + src/adcp/reporting/receipts/store.py | 24 + src/adcp/reporting/receipts/transport.py | 195 ++++++ src/adcp/reporting/receipts/wire.py | 189 ++++++ src/adcp/server/a2a_server.py | 106 ++- src/adcp/server/idempotency/store.py | 14 + src/adcp/server/mcp_tools.py | 81 ++- src/adcp/server/serve.py | 71 +- .../reporting/_materializer_installed.py | 57 +- .../reporting/_materializer_process.py | 13 + .../conformance/reporting/_receipt_frozen.py | 282 ++++++++ .../reporting/_receipt_installed.py | 82 +++ .../conformance/reporting/_receipt_process.py | 80 +++ .../conformance/reporting/_receipt_support.py | 103 +++ .../reporting/_receipt_transport.py | 270 ++++++++ .../test_reporting_materializer_packaging.py | 9 + .../test_reporting_materializer_process.py | 1 + .../test_reporting_materializer_rolling.py | 6 +- .../test_reporting_receipt_batches.py | 388 +++++++++++ .../reporting/test_reporting_receipt_graph.py | 398 ++++++++++++ .../test_reporting_receipt_installed_pg.py | 139 ++++ .../test_reporting_receipt_materializer.py | 64 ++ .../test_reporting_receipt_migration.py | 133 ++++ .../test_reporting_receipt_process.py | 117 ++++ .../test_reporting_receipt_rolling.py | 235 +++++++ .../test_reporting_receipt_transactions.py | 339 ++++++++++ .../test_reporting_receipt_transports.py | 609 ++++++++++++++++++ .../type_checks/reporting_receipt_ingress.py | 40 ++ 46 files changed, 6381 insertions(+), 47 deletions(-) create mode 100644 docs/reporting-receipt-ingress.md create mode 100644 examples/reporting_receipt_ingress.py create mode 100644 scripts/reporting_test_harness.py create mode 100644 src/adcp/reporting/ledger/reporting_receipt_ingestion.sql create mode 100644 src/adcp/reporting/receipts/__init__.py create mode 100644 src/adcp/reporting/receipts/capture.py create mode 100644 src/adcp/reporting/receipts/errors.py create mode 100644 src/adcp/reporting/receipts/handler.py create mode 100644 src/adcp/reporting/receipts/memory.py create mode 100644 src/adcp/reporting/receipts/pg.py create mode 100644 src/adcp/reporting/receipts/records.py create mode 100644 src/adcp/reporting/receipts/required_schema.json create mode 100644 src/adcp/reporting/receipts/schema.py create mode 100644 src/adcp/reporting/receipts/store.py create mode 100644 src/adcp/reporting/receipts/transport.py create mode 100644 src/adcp/reporting/receipts/wire.py create mode 100644 tests/conformance/reporting/_receipt_frozen.py create mode 100644 tests/conformance/reporting/_receipt_installed.py create mode 100644 tests/conformance/reporting/_receipt_process.py create mode 100644 tests/conformance/reporting/_receipt_support.py create mode 100644 tests/conformance/reporting/_receipt_transport.py create mode 100644 tests/conformance/reporting/test_reporting_receipt_batches.py create mode 100644 tests/conformance/reporting/test_reporting_receipt_graph.py create mode 100644 tests/conformance/reporting/test_reporting_receipt_installed_pg.py create mode 100644 tests/conformance/reporting/test_reporting_receipt_materializer.py create mode 100644 tests/conformance/reporting/test_reporting_receipt_migration.py create mode 100644 tests/conformance/reporting/test_reporting_receipt_process.py create mode 100644 tests/conformance/reporting/test_reporting_receipt_rolling.py create mode 100644 tests/conformance/reporting/test_reporting_receipt_transactions.py create mode 100644 tests/conformance/reporting/test_reporting_receipt_transports.py create mode 100644 tests/type_checks/reporting_receipt_ingress.py diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index bf83c4956..ee326fa65 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -9,6 +9,7 @@ on: - conductor/reporting-webhook-activity-1168b - conductor/reporting-status-notifications-1168c - conductor/1167b1-materializer-contracts + - conductor/1167b2-durable-managed-reporting # Default @adcp/sdk runner alias for storyboard jobs. Tracks the current # stable @adcp/sdk release via the ``latest`` npm dist-tag. @@ -103,7 +104,7 @@ jobs: - name: Run adopter type-check suite if: matrix.python-version == '3.12' - run: mypy --strict tests/type_checks/ examples/reporting_webhook_activity.py examples/reporting_status_notifications.py examples/reporting_destination_writer.py examples/reporting_durable_materializer.py + run: mypy --strict tests/type_checks/ examples/reporting_webhook_activity.py examples/reporting_status_notifications.py examples/reporting_destination_writer.py examples/reporting_durable_materializer.py examples/reporting_receipt_ingress.py - name: Enforce adopter type-check fixture contract if: matrix.python-version == '3.12' @@ -116,12 +117,12 @@ jobs: - name: Run tests if: matrix.python-version != '3.12' timeout-minutes: 45 - run: pytest tests/ -v + run: python scripts/reporting_test_harness.py pytest tests/ -v -ra - name: Run tests with coverage if: matrix.python-version == '3.12' timeout-minutes: 45 - run: pytest tests/ -v --cov=src/adcp --cov-report=term-missing + run: python scripts/reporting_test_harness.py pytest tests/ -v -ra --cov=src/adcp --cov-report=term-missing pg-conformance: name: Postgres conformance tests (Postgres 16) @@ -183,7 +184,8 @@ jobs: env: ADCP_PG_TEST_URL: postgresql://postgres@localhost:5432/adcp_test run: | - pytest tests/conformance/signing/test_pg_replay_store.py \ + python scripts/reporting_test_harness.py pytest \ + tests/conformance/signing/test_pg_replay_store.py \ tests/conformance/signing/test_pg_replay_store_e2e.py \ tests/conformance/decisioning/test_pg_buyer_agent_registry.py \ tests/conformance/decisioning/test_pg_idempotency_backend.py \ @@ -191,11 +193,12 @@ jobs: tests/conformance/decisioning/test_pg_reference_workflow_queue.py \ tests/conformance/reporting/ \ --ignore-glob='tests/conformance/reporting/test_reporting_status_*.py' \ + --ignore-glob='tests/conformance/reporting/test_reporting_receipt_*.py' \ --ignore=tests/conformance/reporting/test_reporting_materializer_rolling.py \ --ignore=tests/conformance/reporting/test_reporting_materializer_process.py \ --ignore=tests/conformance/reporting/test_reporting_materializer_migration.py \ --ignore=tests/conformance/reporting/test_reporting_materializer_installed_pg.py \ - -v + -v -ra pg-reporting-status: name: Reporting status conformance (Postgres 16) @@ -249,7 +252,7 @@ jobs: - name: Run status semantics, transaction, clock/process and rolling gates env: ADCP_PG_TEST_URL: postgresql://postgres@localhost:5432/adcp_status_test - run: pytest tests/conformance/reporting/test_reporting_status_*.py -v + run: python scripts/reporting_test_harness.py pytest tests/conformance/reporting/test_reporting_status_*.py -v -ra pg-reporting-materializer: name: Durable materializer and frozen artifacts (Postgres 16) @@ -306,11 +309,12 @@ jobs: ADCP_PG_TEST_URL: postgresql://postgres@localhost:5432/adcp_materializer_test ADCP_PYTHON310: ${{ steps.materializer-python310.outputs.python-path }} run: | - pytest tests/conformance/reporting/test_reporting_materializer_rolling.py \ + python scripts/reporting_test_harness.py pytest \ + tests/conformance/reporting/test_reporting_materializer_rolling.py \ tests/conformance/reporting/test_reporting_materializer_migration.py \ tests/conformance/reporting/test_reporting_materializer_process.py \ tests/conformance/reporting/test_reporting_materializer_installed_pg.py \ - -v -s | tee materializer-evidence.log + -v -s -ra | tee materializer-evidence.log - name: Preserve installed module and worker evidence if: always() uses: actions/upload-artifact@v7 @@ -319,6 +323,122 @@ jobs: path: materializer-evidence.log if-no-files-found: error + pg-reporting-receipts: + name: Authenticated receipts and installed replay (Postgres 16) + runs-on: ubuntu-latest + # Separate databases avoid shared account advisory locks. Keep five minutes + # outside the bounded suite for checkout, installations and cleanup. + timeout-minutes: 35 + services: + postgres: + image: postgres:16 + env: + POSTGRES_HOST_AUTH_METHOD: trust + POSTGRES_DB: adcp_receipts_test + POSTGRES_INITDB_ARGS: "--encoding=UTF8 --lc-collate=C --lc-ctype=C" + ports: + - 5432:5432 + options: >- + --health-cmd pg_isready + --health-interval 5s + --health-timeout 5s + --health-retries 10 + steps: + - uses: actions/checkout@v6 + - uses: actions/setup-python@v6 + id: receipt-python310 + with: + python-version: "3.10" + - uses: actions/setup-python@v6 + with: + python-version: "3.12" + cache: pip + cache-dependency-path: pyproject.toml + - name: Install test dependencies + run: pip install -e ".[dev,pg]" + - name: Run receipt transactions, real mounts, crash and installed gates + shell: bash + timeout-minutes: 30 + env: + ADCP_PG_TEST_URL: postgresql://postgres@localhost:5432/adcp_receipts_test + ADCP_PYTHON310: ${{ steps.receipt-python310.outputs.python-path }} + run: | + python scripts/reporting_test_harness.py pytest \ + tests/conformance/reporting/test_reporting_receipt_batches.py \ + tests/conformance/reporting/test_reporting_receipt_graph.py \ + tests/conformance/reporting/test_reporting_receipt_transactions.py \ + tests/conformance/reporting/test_reporting_receipt_transports.py \ + tests/conformance/reporting/test_reporting_receipt_materializer.py \ + tests/conformance/reporting/test_reporting_receipt_migration.py \ + tests/conformance/reporting/test_reporting_receipt_process.py \ + tests/conformance/reporting/test_reporting_receipt_installed_pg.py \ + -v -s -ra | tee pg-reporting-receipts-evidence.log + - name: Preserve commands, counts, installed provenance and skip reasons + if: always() + uses: actions/upload-artifact@v7 + with: + name: pg-reporting-receipts-evidence-${{ github.run_attempt }} + path: pg-reporting-receipts-evidence.log + if-no-files-found: error + + pg-reporting-receipt-compatibility: + name: Receipt rolling compatibility (eight actual artifacts) + runs-on: ubuntu-latest + # Separate databases avoid shared account advisory locks. Keep five minutes + # outside the bounded suite for checkout, installations and cleanup. + timeout-minutes: 35 + services: + postgres: + image: postgres:16 + env: + POSTGRES_HOST_AUTH_METHOD: trust + POSTGRES_DB: adcp_receipt_rolling_test + POSTGRES_INITDB_ARGS: "--encoding=UTF8 --lc-collate=C --lc-ctype=C" + ports: + - 5432:5432 + options: >- + --health-cmd pg_isready + --health-interval 5s + --health-timeout 5s + --health-retries 10 + steps: + - uses: actions/checkout@v6 + - name: Fetch all eight exact approved historical artifacts + timeout-minutes: 2 + run: | + git fetch --no-tags --depth=1 origin \ + 3e76aa54623529a3dda01cd690b8a5c287c75641 \ + 3c405a21f978ed9d3208611bb4a7a8434a056933 \ + 037de4ac822ecefb2f95d32c15c297fb4c45d683 \ + 21bf443e7d850d1800ec8a6f2e4abec1c8f85541 \ + 198d50e61c74fb82aedbf2c77e06a0e200b91db6 \ + ea150fabd5ad90e3abf93f89729d2919f1c61798 \ + 1c91311ec28d25506d5db43f59d0c34936ecb8f7 \ + 8e18ca12b9a0c3750f80aa822058c02982ab3e52 + - uses: actions/setup-python@v6 + with: + python-version: "3.12" + cache: pip + cache-dependency-path: pyproject.toml + - name: Install test dependencies + run: pip install -e ".[dev,pg]" + - name: Run all historical binaries against approved parent and receipt schema + shell: bash + timeout-minutes: 30 + env: + ADCP_PG_TEST_URL: postgresql://postgres@localhost:5432/adcp_receipt_rolling_test + run: | + python scripts/reporting_test_harness.py pytest \ + tests/conformance/reporting/test_reporting_receipt_rolling.py \ + -v -s -ra | tee pg-reporting-receipt-compatibility-evidence.log + - name: Preserve commands, counts, installed provenance and skip reasons + if: always() + uses: actions/upload-artifact@v7 + with: + name: pg-reporting-receipt-compatibility-evidence-${{ github.run_attempt }} + path: pg-reporting-receipt-compatibility-evidence.log + if-no-files-found: error + conventional-commits: name: Validate conventional commit format runs-on: ubuntu-latest diff --git a/.github/workflows/pr-title-check.yml b/.github/workflows/pr-title-check.yml index 2a8f7a038..ce86531ea 100644 --- a/.github/workflows/pr-title-check.yml +++ b/.github/workflows/pr-title-check.yml @@ -3,7 +3,7 @@ name: PR Title Check on: pull_request: types: [opened, edited, synchronize, reopened] - branches: [main] + branches: [main, conductor/1167b2-durable-managed-reporting] permissions: contents: read diff --git a/docs/reporting-receipt-ingress.md b/docs/reporting-receipt-ingress.md new file mode 100644 index 000000000..6eaa1b25c --- /dev/null +++ b/docs/reporting-receipt-ingress.md @@ -0,0 +1,194 @@ +# Authenticated durable reporting receipts + +`adcp.reporting.receipts.ReportingReceiptHandler` mounts the experimental +`sync_reporting_receipts` task over the SDK's MCP and A2A transports. It accepts +revision and official-adjustment receipts, persists each item outcome, and +replays the original completed response after a process restart. Use +`PgReportingReceiptStore` in production. It extends the existing durable +materializer store; no second materializer or required legacy store method is +introduced. The memory store is a conformance/reference implementation. + +This is **B2.2 of 4 within B2 of B1/B2** (Refs #1167). B2.3 owns frozen authorized +combined feeds, checkpoints and dependency closure. B2.4 owns versioned public +status, private ownership projection, complete ownership-walk validation and +per-offering production capability proof. Installing this slice does not make +Managed Delivery or Reconciled Billing ready. The existing new-store notification +advertisement veto also covers Core `reporting.ledger_changed`; Core polling is +unaffected. A deployment needing current Core notification advertisement may +retain the reviewed Core store for that surface. B2.4 owns deliberate admission +of the new store once all actual components and mounts are proven ready. + +## Composition and identity + +[The typed example](../examples/reporting_receipt_ingress.py) composes the store, +live account authorization callback and optional buyer registry, then uses +`serve(transport="both", auth=..., context_factory=auth_context_factory)`. +The same store participates in +[the durable materializer composition](../examples/reporting_durable_materializer.py). +A token validator supplies trusted principals. A registry-backed API/OAuth or +signed-agent adapter supplies trusted `AuthInfo` and its credential reference; +the registry is consulted again on every call, including exact replay. + +`resolve_account(reference, context, canonical_consumer)` must resolve the exact +account reference and check the current application ACL. Return the canonical +storage account ID; unknown and denied accounts raise +`ReportingReceiptError("UNAUTHORIZED")`. An opaque account ID is sufficient; +a natural account key is resolved through the same callback. Authorization must +not be cached solely by the batch key. A hydrated `RequestContext.account.id` +must equal the resolved account. + +The SDK cross-checks all present trusted principals, `AuthInfo`, signed agent +identity and active `BuyerAgent.agent_url`, using the reviewed canonical consumer +resolver. A raw `ToolContext.caller_identity` may identify the consumer. +`RequestContext.caller_identity` is an **opaque AccountStore cache key** and is +never parsed as a consumer. `tenant_id` and body fields cannot supply identity. +Absent, anonymous, inactive or conflicting identities fail closed. Two consumers +in one account and one consumer in two accounts have independent batch keys and +receipt visibility. + +## Wire and replay contract + +Requests negotiate `adcp_version: "3.2-rc.3"`. This is the wire release spelling; +`3.2.0-rc.3` is the bundle's semantic-version spelling. Unnegotiated traffic keeps +the SDK's existing AdCP 3.0 behavior; unsupported versions remain unsupported. +Pinned, unpinned and fallback MCP inventories use an isolated schema overlay. +Cached upstream schemas, generated models and older required protocols remain +unchanged. + +Before any batch or receipt mutation the SDK validates the entire request: + +- Supply `receipts`, `adjustment_receipts`, or both. Every supplied array is + nonempty; the combined size is 1–100. IDs are unique across both arrays. +- Do not supply `received_at`. PostgreSQL assigns it inside the item transaction. +- Follow the exact receipt schema, including evidence and timestamp formats. + Reporting's canonical JSON profile uses safe integers and decimal strings; + nonintegral numbers, nonfinite numbers, duplicate JSON keys, unsafe integers and + ambiguous requests and strings outside PostgreSQL JSONB's Unicode domain + (U+0000 or surrogate code points) are rejected. Receipt-specific SQL hashing + preserves the canonical UTF-16 key ordering for all accepted Unicode keys in + `context` and `ext`; earlier schema functions and manifests stay unchanged. +- Account and canonical consumer come from the trusted resolver and context. + Request identity fields cannot override them. + +Malformed shape produces task-level `INVALID_REQUEST` with zero writes. +Semantic failures are durable item results. Unknown and unauthorized referenced +records use the same safe `REPORTING_RECORD_UNAVAILABLE` item shape. Results are +revision receipts in input order followed by adjustment receipts in input order. +A failure of either kind has top-level `reporting_receipt_id` and `errors`; +a success has `receipt` or `adjustment_receipt`, including server `received_at`. + +The immutable batch header is keyed by `(account, canonical consumer, +idempotency_key)`. It stores the canonical **whole request**, digest and expected +count; `context` and `ext` participate in identity. Every ordinal, including a +failure, is durable. The final response is persisted and validated against the +original ordered items. Changed content conflicts before further work. Replays +return original `recorded`/failure results and original server timestamps, without +adding `replayed` or applying a response enhancer. + +The SDK generic idempotency wrapper and middleware are bypassed for this task: +the generic cache lacks the resolved account and can bypass revoked +authorization. Application middleware must not independently cache receipt +responses or rewrite admitted receipt parameters. Receipt pre-validation hooks +may inspect an isolated copy, but changing its content is rejected. + +The production HTTP mounts retain bounded raw bytes in the current request's +ASGI scope because A2A protobuf `Struct` normalizes JSON integers to floats. +The A2A wrapper checks a uniquely identified receipt's Unicode encoding before +protobuf decoding. Receipt parameter extraction remains bound to the selected +route and normal transport authentication; account and consumer authorization +still run in the handler. Raw bytes are never sourced from client metadata, +shared context caches or request-ID lookups. Both the normal body limiter and +the receipt capture bound apply. The raw request must identify +one unambiguous standard receipt invocation; lossy direct/custom transport inputs +fail closed. All normal whole-shape, numeric, account and consumer checks still +run. This does not replace the A2A stack or alter other tasks' version handling. +Exact integral HTTP numeric spellings such as protobuf's `1.0` are decoded from +their original text without floating-point rounding and canonicalized as `1`. +An already rounded direct/custom parameter value cannot supply that evidence +and is rejected. Fractional text that would round to an integer is also rejected. + +## Financial evidence and transactions + +Revision receipts match the exact account, consumer, obligation, revision, +materialization, verification profile, digest and control totals. Acceptance +requires readable evidence for that artifact at observation. Receipt chains are +single, complete, acyclic chains with no cross-target edges. Only the current +rejected leaf may be replaced. An accepted leaf is terminal. Adjustment receipts +match the exact official revision, canonical adjustment digest, finality and +correction/creation/observation ordering. PostgreSQL predicates also enforce these +relationships for ordinary SQL inserts. + +An accepted receipt remains authoritative for its referenced immutable artifact +after another materialization succeeds or fails, or after that artifact expires +or becomes corrupt/unreadable. Current readability is assessed separately. +Acceptance does not promise current availability; later health does not rewrite +acceptance. Consumer rejection never allocates a materializer attempt or dirties +retry work. + +For each item the account advisory lock is acquired before the batch row lock. +One connection-bound transaction chooses the next durable ordinal and commits +its receipt, private caller feed entry, ordinary status-dirty behavior, required +immutable captured status input and exact ordinal outcome. A semantic failure +commits only that ordinal outcome. A storage/capture/result failure rolls the +whole current ordinal back; previously committed sibling outcomes remain. +Memory rollback is unconditional, including first-use collections and sequence +heads, with notifications both off and on. + +Captured receipt inputs are isolated epoch-zero records with their own caller +sequence and B2.1's existing account capture sequence. They retain exact private +consumer and obligation/revision dependencies even when consumer status is +disabled. They do not activate a public projection, release historical events or +produce readiness. B2.1's terminal outcome + captured status input + work ACK + +enabled logical enqueue remains the original transaction. Its preactivation +work/events stay permanently quarantined across replay and later activation. +Ordinary terminal materialization writes still mark status dirty without +readiness, and argument bounds remain actionable closed errors. + +## Migration, rollout and recovery + +1. Back up and migrate with `await store.create_schema()` using a deployment + connection. The additive `reporting_receipt_ingestion.sql` and independent + `receipts/required_schema.json` ship in both wheel distribution paths. Do not + modify A/B/C manifests or the **187-object** B2.1 materializer manifest. +2. Verify `await store.receipt_ingestion_ready()` on the actual installed store. + Empty, old, partial, disabled-trigger and mismatched schemas fail closed, + including completed replay. The migration is transactional, repeatable and + serialized; interrupted installation exposes no partial schema. +3. Drain legacy materialization writers before autonomous materializer activation. + Preserve [B2.1's legacy-pending recovery/import rules](reporting-durable-materializer.md). + Never invent an external idempotency history or reuse quarantined epoch-zero + evidence for later activation. Old ordinary writers are compatible during a + controlled rolling deployment; they are not autonomous workers to run in + parallel with the new service. +4. Mount the authenticated receipt handler and verify its actual MCP/A2A routes. + Deploy the same durable store to restarted instances. Keep account resolution + and registry revocation live. Do not advertise incomplete tier capabilities. +5. On timeout, cancellation, process death or uncertain commit, retry the identical + body and key. The durable prefix resumes with no duplicated receipt or changed + timestamp. Final-response failure retains earlier ordinals and rebuilds the + response from them. `IDEMPOTENCY_CONFLICT` requires recovering the original + submission; do not erase the batch to reuse its key. +6. For `RECEIPT_SCHEMA_UNREADY` repair the installed migration. For + `RECEIPT_HISTORY_CORRUPT` stop ingress and investigate the immutable evidence; + do not rewrite financial outcomes. `RECEIPT_STORAGE_UNAVAILABLE` is a safe + storage boundary error; retry the original body after service recovery. + Closed public errors omit provider bodies, credentials and SQL diagnostics. + +Keep batches, ordinals and captures at least as long as their referenced receipt +history and replay obligation. This slice does not prune them. Database owners +can bypass SQL protections, so migrations and repair remain operator-controlled; +application roles must not disable triggers or rewrite immutable rows. Rollback +means draining the new ingress and returning to compatible older ordinary +readers/writers while retaining the additive objects and records. Do not drop +financial history as a rollback strategy. B2.4 must separately drain/fence +incompatible status projectors before its activation. + +Historical reader/writer gates execute actual installed beta.15, records A, +foundation integration, outbox A, activity B, status C, B1 and the exact approved +B2.1 artifact on populated new records, comparing permitted old behavior before +and after this migration. Their reviewed PostgreSQL prerequisite is **UTF8/C +collation and ctype**. A's notification-readiness closure after C is compared on +both sides and does not excuse new regressions. Non-C probes are distinct from +this frozen-artifact gate. Optional notifications may stay explicitly disabled; +complete polling still depends on the later B2.3/B2.4 components. Full buyer +adjustment automation and `client.reporting` remain named downstream #1172 work. diff --git a/examples/reporting_receipt_ingress.py b/examples/reporting_receipt_ingress.py new file mode 100644 index 000000000..b0ac3e1c5 --- /dev/null +++ b/examples/reporting_receipt_ingress.py @@ -0,0 +1,66 @@ +"""Authenticated durable receipts on the existing materializer store and mounts. + +Install migrations during deployment, drain legacy materialization writers, and +supply the application's token validator and fresh account ACL. Receipt traffic +negotiates AdCP 3.2-rc.3. This composition does not activate production tier claims. +""" + +from __future__ import annotations + +from adcp.decisioning.registry import BuyerAgentRegistry +from adcp.reporting.receipts import ( + PgReportingReceiptStore, + ReceiptAccountResolver, + ReportingReceiptHandler, +) +from adcp.server import serve +from adcp.server.auth import BearerTokenAuth, auth_context_factory + + +async def compose_receipt_ingress( + store: PgReportingReceiptStore, + *, + resolve_account: ReceiptAccountResolver, + buyer_agents: BuyerAgentRegistry | None = None, +) -> ReportingReceiptHandler: + """Validate the installed schema and bind the live authorization boundary. + + The resolver receives the exact account reference and canonical consumer on + every request, including replay. Resolve natural account keys and check the + current ACL there; return a storage account ID or raise + ReportingReceiptError("UNAUTHORIZED"). It must never derive a consumer from + tenancy, request fields, or RequestContext.caller_identity's cache key. + + This same store can be passed to compose_materializer in + reporting_durable_materializer.py. No parallel materializer or readiness + switch is needed. Schema installation is an explicit deployment operation. + """ + await store.receipt_ingestion_ready() + return ReportingReceiptHandler( + store, resolve_account=resolve_account, buyer_agents=buyer_agents + ) + + +def serve_receipt_ingress( + handler: ReportingReceiptHandler, + *, + auth: BearerTokenAuth, + public_url: str, + allowed_hosts: tuple[str, ...], +) -> None: + """Use the production MCP and A2A mounts with one authenticated handler. + + Token validation supplies trusted Principal identities, never credentials in + request bodies. Registry-backed deployments populate AuthInfo credentials + through their verified adapter. Generic SDK idempotency middleware is + automatically bypassed for this task; the store owns durable replay. + """ + serve( + handler, + name="reporting-receipts", + transport="both", + auth=auth, + context_factory=auth_context_factory, + public_url=public_url, + allowed_hosts=allowed_hosts, + ) diff --git a/pyproject.toml b/pyproject.toml index 64997d47c..d0f4556b3 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -193,6 +193,7 @@ adcp = [ "reporting/outbox/*.json", "reporting/materializer/assets/*.json", "reporting/materializer/*.json", + "reporting/receipts/*.json", # PREVIEW: vendored sync_reporting_status schemas. They are the runtime # validator for the wire conditionals codegen cannot express, so the wheel # must carry them. Removed with the rest of _preview/ at rc.2. diff --git a/scripts/reporting_test_harness.py b/scripts/reporting_test_harness.py new file mode 100644 index 000000000..4302bdf55 --- /dev/null +++ b/scripts/reporting_test_harness.py @@ -0,0 +1,64 @@ +"""Run a reporting pytest gate in an audited private temporary directory. + +Usage: python scripts/reporting_test_harness.py pytest tests/... -v -s -ra +The child inherits the normal process umask. Failed diagnostics remain under the +printed root until the runner/workspace is removed; no live artifact is deleted. +""" + +from __future__ import annotations + +import json +import os +import stat +import subprocess +import sys +import tempfile +import time +from pathlib import Path + + +def main() -> int: + if len(sys.argv) < 2: + raise SystemExit("supply the pytest executable and gate arguments") + root = Path(tempfile.mkdtemp(prefix="adcp-reporting-gate-")).resolve() + root.chmod(0o700) + parent = root / "suite" + parent.mkdir(mode=0o700) + ancestors = [] + for path in (parent, root, *root.parents): + metadata = path.stat() + mode = stat.S_IMODE(metadata.st_mode) + if mode & 0o022 and not mode & stat.S_ISVTX: + raise SystemExit(f"unsafe reporting harness ancestor: {path}") + ancestors.append({"path": str(path), "mode": oct(mode), "uid": metadata.st_uid}) + mask = os.umask(0) + os.umask(mask) + command = [*sys.argv[1:], "--basetemp", str(parent / "run")] + print( + json.dumps( + { + "reporting_harness": str(root), + "umask": oct(mask), + "ancestors": ancestors, + "command": command, + } + ), + flush=True, + ) + started = time.monotonic() + result = subprocess.run(command, check=False) + print( + json.dumps( + { + "reporting_harness": str(root), + "exit_status": result.returncode, + "runtime_seconds": round(time.monotonic() - started, 3), + } + ), + flush=True, + ) + return result.returncode + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/src/adcp/reporting/ledger/_delivery_state.py b/src/adcp/reporting/ledger/_delivery_state.py index a385a5610..3c8fb6289 100644 --- a/src/adcp/reporting/ledger/_delivery_state.py +++ b/src/adcp/reporting/ledger/_delivery_state.py @@ -186,11 +186,44 @@ def current_receipt( for item in records if isinstance(item, _RECEIPTS) and receipt_chain(item) == receipt_chain(requested) ] - superseded = {item.supersedes_reporting_receipt_id for item in chain} + if not chain: + return None + by_id = {item.reporting_receipt_id: item for item in chain} + superseded = [ + item.supersedes_reporting_receipt_id + for item in chain + if item.supersedes_reporting_receipt_id is not None + ] leaves = [item for item in chain if item.reporting_receipt_id not in superseded] - if chain and len(leaves) != 1: + if len(by_id) != len(chain) or len(set(superseded)) != len(superseded) or len(leaves) != 1: + fail("REPORTING_HISTORY_CORRUPT") + # One apparent leaf alone does not rule out a disconnected cycle, a missing + # predecessor, an accepted predecessor, or a cross-target/generation edge. + visited: set[str] = set() + node: ReportingReceiptRecord | None = leaves[0] + while node is not None: + if ( + node.reporting_receipt_id in visited + or type(node) is not type(requested) + or node.scope != requested.scope + ): + fail("REPORTING_HISTORY_CORRUPT") + if isinstance(node, ReportingAdjustmentReceiptRecord) and isinstance( + requested, ReportingAdjustmentReceiptRecord + ): + if node.adjusts_reporting_revision_id != requested.adjusts_reporting_revision_id: + fail("REPORTING_HISTORY_CORRUPT") + visited.add(node.reporting_receipt_id) + predecessor_id = node.supersedes_reporting_receipt_id + if predecessor_id is None: + node = None + else: + node = by_id.get(predecessor_id) + if node is None or node.status != "rejected": + fail("REPORTING_HISTORY_CORRUPT") + if len(visited) != len(chain): fail("REPORTING_HISTORY_CORRUPT") - return leaves[0] if leaves else None + return leaves[0] def replay( @@ -201,6 +234,8 @@ def replay( ) if existing is None: return None + if isinstance(existing, _RECEIPTS): + current_receipt(records, existing) comparison = existing if ( isinstance(record, _RECEIPTS) @@ -551,15 +586,22 @@ def _verify_receipt( fail("REPORTING_TIME_INVALID") if record.status != "accepted": return - if record.observed_row_count != revision.row_count or sorted( - record.observed_control_totals, key=lambda item: item.name - ) != sorted(target.verification.control_totals, key=lambda item: item.name): + if ( + record.observed_row_count != revision.row_count + or record.observed_row_count != target.verification.row_count + or sorted(record.observed_control_totals, key=lambda item: item.name) + != sorted(target.verification.control_totals, key=lambda item: item.name) + ): fail("RECEIPT_TOTALS_MISMATCH") profile = record.verification_profile if ( ( record.observed_canonical_content_digest is not None - and record.observed_canonical_content_digest != revision.canonical_content_digest + and ( + record.observed_canonical_content_digest != revision.canonical_content_digest + or record.observed_canonical_content_digest + != target.verification.canonical_content_digest + ) ) or ( record.observed_manifest_sha256 is not None diff --git a/src/adcp/reporting/ledger/delivery.py b/src/adcp/reporting/ledger/delivery.py index d07bad3c4..0ebb981c9 100644 --- a/src/adcp/reporting/ledger/delivery.py +++ b/src/adcp/reporting/ledger/delivery.py @@ -17,6 +17,7 @@ adjustment_payload, adjustment_sha256, change_id, + current_receipt, decode_record, fail, iso, @@ -281,8 +282,7 @@ def current_receipts(self) -> tuple[ReportingReceiptRecord, ...]: for item in self.records if isinstance(item, (ReportingRevisionReceiptRecord, ReportingAdjustmentReceiptRecord)) ) - replaced = {item.supersedes_reporting_receipt_id for item in receipts} - return tuple(item for item in receipts if item.reporting_receipt_id not in replaced) + return tuple(item for item in receipts if current_receipt(self.records, item) == item) @property def terminal_acceptances(self) -> tuple[ReportingReceiptKey, ...]: diff --git a/src/adcp/reporting/ledger/delivery_pg.py b/src/adcp/reporting/ledger/delivery_pg.py index 274a98826..449fe8f14 100644 --- a/src/adcp/reporting/ledger/delivery_pg.py +++ b/src/adcp/reporting/ledger/delivery_pg.py @@ -137,12 +137,7 @@ async def _commit_record_on( if existing is not None: return cast(RecordT, existing), False context = await self._delivery_context(connection, candidate) - if self._clock is not None: - now = self._clock() - else: - time_row = await (await connection.execute("SELECT clock_timestamp()")).fetchone() - assert time_row is not None - now = time_row[0] + now = await self._delivery_time_on(connection, candidate) stored = validate_transition(candidate, records, context, now) await self._insert(connection, stored) await self._append_reconciliation_change(connection, stored) @@ -168,6 +163,14 @@ async def _commit_record_on( await self._dirty_status(connection, scope, reason, after=evidence) return cast(RecordT, stored), True + async def _delivery_time_on(self, connection: Any, record: ReportingDeliveryRecord) -> datetime: + """Connection-bound clock seam; old public clock overrides remain supported.""" + if self._clock is not None: + return self._clock() + time_row = await (await connection.execute("SELECT clock_timestamp()")).fetchone() + assert time_row is not None + return cast(datetime, time_row[0]) + async def _append_reconciliation_change( self, connection: Any, record: ReportingDeliveryRecord ) -> None: diff --git a/src/adcp/reporting/ledger/reporting_receipt_ingestion.sql b/src/adcp/reporting/ledger/reporting_receipt_ingestion.sql new file mode 100644 index 000000000..6ebcd1f02 --- /dev/null +++ b/src/adcp/reporting/ledger/reporting_receipt_ingestion.sql @@ -0,0 +1,405 @@ +-- B2.2 additive receipt ingress. No older manifest, decoder, or queue is replaced. +-- Execute in the caller's transaction; an interrupted migration leaves no partial feature. +DO $migration$ +BEGIN + PERFORM pg_advisory_xact_lock(hashtext('adcp.reporting.receipt_ingestion.schema')); + + -- Whole receipt requests may have arbitrary Unicode context/ext keys. + -- JCS sorts UTF-16 code units, which differs from the older closed-record + -- encoder's C ordering for supplementary characters. Keep this additive; + -- replacing reporting_payload_sha256 would invalidate approved manifests. + CREATE OR REPLACE FUNCTION reporting_receipt_ingestion_utf16(value TEXT) + RETURNS BYTEA LANGUAGE plpgsql IMMUTABLE STRICT AS $function$ + DECLARE + result BYTEA := ''::bytea; + point INTEGER; + i INTEGER; + BEGIN + FOR i IN 1..char_length(value) LOOP + point := ascii(substr(value,i,1)); + IF point > 65535 THEN + point := point - 65536; + result := result || decode(lpad(to_hex(55296 + (point >> 10)),4,'0') + || lpad(to_hex(56320 + (point & 1023)),4,'0'),'hex'); + ELSE + result := result || decode(lpad(to_hex(point),4,'0'),'hex'); + END IF; + END LOOP; + RETURN result; + END + $function$; + + CREATE OR REPLACE FUNCTION reporting_receipt_ingestion_canonical(document JSONB) + RETURNS TEXT LANGUAGE plpgsql IMMUTABLE STRICT AS $function$ + DECLARE + shape TEXT := jsonb_typeof(document); + parts TEXT; + BEGIN + IF shape = 'object' THEN + SELECT coalesce(string_agg(to_json(e.key)::text || ':' + || reporting_receipt_ingestion_canonical(e.value), ',' + ORDER BY reporting_receipt_ingestion_utf16(e.key)), '') INTO parts + FROM jsonb_each(document) e; + RETURN '{' || parts || '}'; + ELSIF shape = 'array' THEN + SELECT coalesce(string_agg(reporting_receipt_ingestion_canonical(e.value), ',' + ORDER BY e.ordinality), '') INTO parts + FROM jsonb_array_elements(document) WITH ORDINALITY e; + RETURN '[' || parts || ']'; + END IF; + -- The approved scalar rules already match this restricted JCS profile. + RETURN reporting_canonical_json(document); + END + $function$; + + CREATE OR REPLACE FUNCTION reporting_receipt_ingestion_sha256(document JSONB) + RETURNS TEXT LANGUAGE SQL IMMUTABLE STRICT AS $function$ + SELECT encode(sha256(convert_to(reporting_receipt_ingestion_canonical(document), 'UTF8')), 'hex') + $function$; + + CREATE TABLE IF NOT EXISTS reporting_receipt_ingestion_batches ( + account_id TEXT COLLATE "C" NOT NULL, + consumer_id TEXT COLLATE "C" NOT NULL, + idempotency_key TEXT COLLATE "C" NOT NULL CHECK (idempotency_key ~ '^[A-Za-z0-9_.:-]{16,255}$'), + canonical_request TEXT NOT NULL, + request_sha256 TEXT COLLATE "C" NOT NULL CHECK (request_sha256 ~ '^[0-9a-f]{64}$'), + expected_count INTEGER NOT NULL CHECK (expected_count BETWEEN 1 AND 100), + created_at TIMESTAMPTZ NOT NULL DEFAULT clock_timestamp(), + final_response JSONB, + final_sha256 TEXT COLLATE "C", + finalized_at TIMESTAMPTZ, + admission_epoch BIGINT NOT NULL DEFAULT 0 CHECK (admission_epoch = 0), + PRIMARY KEY (account_id, consumer_id, idempotency_key), + CHECK (request_sha256 = encode(sha256(convert_to(canonical_request, 'UTF8')), 'hex')), + CHECK (request_sha256 = reporting_receipt_ingestion_sha256(canonical_request::jsonb)), + CHECK (jsonb_typeof(canonical_request::jsonb) = 'object'), + CHECK ((canonical_request::jsonb->>'idempotency_key') IS NOT DISTINCT FROM idempotency_key), + CHECK ((final_response IS NULL) = (final_sha256 IS NULL)), + CHECK ((final_response IS NULL) = (finalized_at IS NULL)), + CHECK (final_response IS NULL OR final_sha256 = reporting_receipt_ingestion_sha256(final_response)) + ); + + CREATE TABLE IF NOT EXISTS reporting_receipt_ingestion_results ( + account_id TEXT COLLATE "C" NOT NULL, + consumer_id TEXT COLLATE "C" NOT NULL, + idempotency_key TEXT COLLATE "C" NOT NULL, + ordinal INTEGER NOT NULL CHECK (ordinal BETWEEN 0 AND 99), + receipt_kind TEXT NOT NULL CHECK (receipt_kind IN ('revision_receipt','adjustment_receipt')), + reporting_receipt_id TEXT COLLATE "C" NOT NULL CHECK (reporting_receipt_id ~ '^[A-Za-z0-9_.:-]{16,255}$'), + receipt_namespace TEXT COLLATE "C" NOT NULL DEFAULT 'receipt' CHECK (receipt_namespace = 'receipt'), + receipt_record_id TEXT COLLATE "C", + result JSONB NOT NULL, + content_sha256 TEXT COLLATE "C" NOT NULL, + recorded_at TIMESTAMPTZ NOT NULL DEFAULT clock_timestamp(), + PRIMARY KEY (account_id, consumer_id, idempotency_key, ordinal), + UNIQUE (account_id, consumer_id, idempotency_key, reporting_receipt_id), + FOREIGN KEY (account_id, consumer_id, idempotency_key) REFERENCES reporting_receipt_ingestion_batches, + FOREIGN KEY (account_id, consumer_id, receipt_namespace, receipt_record_id) + REFERENCES reporting_reconciliation_records(account_id, consumer_id, namespace, record_id), + CHECK (content_sha256 = reporting_receipt_ingestion_sha256(result)), + CHECK (result ? 'result' AND result->>'result' IS NOT NULL + AND result->>'result' IN ('recorded','unchanged','failed')), + CHECK ((result->>'result' = 'failed') = (receipt_record_id IS NULL)), + CHECK (receipt_record_id IS NULL OR receipt_record_id = reporting_receipt_id) + ); + + CREATE TABLE IF NOT EXISTS reporting_receipt_ingestion_heads ( + account_id TEXT COLLATE "C" NOT NULL, + consumer_id TEXT COLLATE "C" NOT NULL, + max_sequence BIGINT NOT NULL CHECK (max_sequence > 0), + PRIMARY KEY (account_id, consumer_id) + ); + CREATE INDEX IF NOT EXISTS reporting_receipt_ingestion_chain + ON reporting_reconciliation_records(account_id, consumer_id, receipt_chain_key) + WHERE namespace = 'receipt'; + CREATE TABLE IF NOT EXISTS reporting_receipt_ingestion_boundaries ( + account_id TEXT COLLATE "C" NOT NULL, + consumer_id TEXT COLLATE "C" NOT NULL, + sequence BIGINT NOT NULL CHECK (sequence > 0), + account_sequence BIGINT NOT NULL CHECK (account_sequence > 0), + reporting_receipt_id TEXT COLLATE "C" NOT NULL, + receipt_namespace TEXT COLLATE "C" NOT NULL DEFAULT 'receipt' CHECK (receipt_namespace = 'receipt'), + as_of TIMESTAMPTZ NOT NULL, + input JSONB NOT NULL, + content_sha256 TEXT COLLATE "C" NOT NULL, + PRIMARY KEY (account_id, consumer_id, sequence), + UNIQUE (account_id, consumer_id, reporting_receipt_id), + UNIQUE (account_id, account_sequence), + FOREIGN KEY (account_id, consumer_id, receipt_namespace, reporting_receipt_id) + REFERENCES reporting_reconciliation_records(account_id, consumer_id, namespace, record_id), + CHECK (content_sha256 = reporting_receipt_ingestion_sha256(input)), + CHECK ((input->>'version')::integer IS NOT DISTINCT FROM 1), + CHECK ((input->>'admission_epoch')::bigint IS NOT DISTINCT FROM 0), + CHECK ((input->>'account_id') IS NOT DISTINCT FROM account_id), + CHECK ((input->>'consumer_id') IS NOT DISTINCT FROM consumer_id), + CHECK ((input->>'reporting_receipt_id') IS NOT DISTINCT FROM reporting_receipt_id), + CHECK ((input->>'sequence')::bigint IS NOT DISTINCT FROM sequence), + CHECK ((input->>'account_sequence')::bigint IS NOT DISTINCT FROM account_sequence), + CHECK ((input->>'as_of')::timestamptz IS NOT DISTINCT FROM as_of), + CHECK (input - ARRAY['version','admission_epoch','account_id','consumer_id', + 'sequence','account_sequence','reporting_receipt_id','as_of','core','reconciliation'] = '{}'::jsonb) + ); + + CREATE OR REPLACE FUNCTION reporting_receipt_ingestion_immutable() + RETURNS TRIGGER LANGUAGE plpgsql AS $function$ + BEGIN + RAISE EXCEPTION 'receipt ingestion evidence is immutable' USING ERRCODE = '23514'; + END + $function$; + + CREATE OR REPLACE FUNCTION reporting_receipt_ingestion_batch_guard() + RETURNS TRIGGER LANGUAGE plpgsql AS $function$ + DECLARE + request JSONB; + revisions JSONB; + adjustments JSONB; + expected JSONB; + n INTEGER; + BEGIN + IF TG_OP = 'DELETE' THEN + RAISE EXCEPTION 'receipt batch is immutable' USING ERRCODE = '23514'; + END IF; + PERFORM pg_advisory_xact_lock(hashtext('adcp.reporting:' || NEW.account_id)); + IF TG_OP = 'UPDATE' THEN + IF (to_jsonb(NEW) - ARRAY['final_response','final_sha256','finalized_at']) + IS DISTINCT FROM (to_jsonb(OLD) - ARRAY['final_response','final_sha256','finalized_at']) + OR OLD.final_response IS NOT NULL OR NEW.final_response IS NULL THEN + RAISE EXCEPTION 'receipt batch is immutable' USING ERRCODE = '23514'; + END IF; + SELECT count(*), jsonb_agg(r.result ORDER BY r.ordinal) INTO n, expected + FROM reporting_receipt_ingestion_results r + WHERE (r.account_id,r.consumer_id,r.idempotency_key) + = (NEW.account_id,NEW.consumer_id,NEW.idempotency_key); + IF n <> NEW.expected_count OR NEW.final_response->'results' IS DISTINCT FROM expected + OR NEW.final_response->>'status' IS DISTINCT FROM 'completed' THEN + RAISE EXCEPTION 'receipt batch results are incomplete' USING ERRCODE = '23514'; + END IF; + RETURN NEW; + END IF; + request := NEW.canonical_request::jsonb; + revisions := coalesce(request->'receipts','[]'::jsonb); + adjustments := coalesce(request->'adjustment_receipts','[]'::jsonb); + IF jsonb_typeof(revisions) <> 'array' OR jsonb_typeof(adjustments) <> 'array' + OR (request ? 'receipts' AND jsonb_array_length(revisions) = 0) + OR (request ? 'adjustment_receipts' AND jsonb_array_length(adjustments) = 0) + OR jsonb_array_length(revisions) + jsonb_array_length(adjustments) <> NEW.expected_count + OR EXISTS (SELECT 1 FROM jsonb_array_elements(revisions || adjustments) i WHERE i ? 'received_at') + OR (SELECT count(DISTINCT i->>'reporting_receipt_id') + FROM jsonb_array_elements(revisions || adjustments) i) <> NEW.expected_count + OR NEW.final_response IS NOT NULL THEN + RAISE EXCEPTION 'receipt batch shape is invalid' USING ERRCODE = '23514'; + END IF; + RETURN NEW; + END + $function$; + + CREATE OR REPLACE FUNCTION reporting_receipt_ingestion_wire(document JSONB) + RETURNS JSONB LANGUAGE SQL IMMUTABLE AS $function$ + SELECT jsonb_strip_nulls(document - ARRAY['scope','kind','rejection_codes']) + || CASE WHEN document->>'kind' = 'revision_receipt' + THEN jsonb_build_object('reporting_obligation_id',document#>>'{scope,reporting_obligation_id}') + ELSE '{}'::jsonb END + || CASE WHEN jsonb_array_length(document->'rejection_codes') > 0 + THEN jsonb_build_object('rejection_codes',document->'rejection_codes') ELSE '{}'::jsonb END + || CASE WHEN jsonb_typeof(document->'observed_canonical_content_digest') = 'object' + THEN jsonb_build_object('observed_canonical_content_digest', + document->'observed_canonical_content_digest' || '{"algorithm":"sha256"}'::jsonb) + ELSE '{}'::jsonb END + $function$; + + CREATE OR REPLACE FUNCTION reporting_receipt_ingestion_result_guard() + RETURNS TRIGGER LANGUAGE plpgsql AS $function$ + DECLARE + batch reporting_receipt_ingestion_batches; + request JSONB; + item JSONB; + expected_kind TEXT; + wire_key TEXT; + evidence JSONB; + n INTEGER; + BEGIN + PERFORM pg_advisory_xact_lock(hashtext('adcp.reporting:' || NEW.account_id)); + SELECT * INTO batch FROM reporting_receipt_ingestion_batches b + WHERE (b.account_id,b.consumer_id,b.idempotency_key) + = (NEW.account_id,NEW.consumer_id,NEW.idempotency_key) FOR UPDATE; + SELECT count(*) INTO n FROM reporting_receipt_ingestion_results r + WHERE (r.account_id,r.consumer_id,r.idempotency_key) + = (NEW.account_id,NEW.consumer_id,NEW.idempotency_key); + IF batch.idempotency_key IS NULL OR batch.final_response IS NOT NULL + OR NEW.ordinal <> n OR NEW.ordinal >= batch.expected_count THEN + RAISE EXCEPTION 'receipt ordinal is unavailable' USING ERRCODE = '23514'; + END IF; + request := batch.canonical_request::jsonb; + n := jsonb_array_length(coalesce(request->'receipts','[]'::jsonb)); + IF NEW.ordinal < n THEN + expected_kind := 'revision_receipt'; wire_key := 'receipt'; + item := request->'receipts'->NEW.ordinal; + ELSE + expected_kind := 'adjustment_receipt'; wire_key := 'adjustment_receipt'; + item := request->'adjustment_receipts'->(NEW.ordinal - n); + END IF; + IF NEW.receipt_kind <> expected_kind + OR NEW.reporting_receipt_id IS DISTINCT FROM item->>'reporting_receipt_id' THEN + RAISE EXCEPTION 'receipt ordinal identity differs' USING ERRCODE = '23514'; + END IF; + IF NEW.result->>'result' = 'failed' THEN + IF NEW.result - ARRAY['result','reporting_receipt_id','errors'] <> '{}'::jsonb + OR NEW.result->>'reporting_receipt_id' IS DISTINCT FROM NEW.reporting_receipt_id + OR jsonb_typeof(NEW.result->'errors') IS DISTINCT FROM 'array' + OR jsonb_array_length(NEW.result->'errors') NOT BETWEEN 1 AND 16 THEN + RAISE EXCEPTION 'receipt failure shape differs' USING ERRCODE = '23514'; + END IF; + ELSE + SELECT reporting_receipt_ingestion_wire(r.payload) INTO evidence + FROM reporting_reconciliation_records r + WHERE (r.account_id,r.consumer_id,r.namespace,r.record_id,r.record_kind) + = (NEW.account_id,NEW.consumer_id,'receipt',NEW.reporting_receipt_id,NEW.receipt_kind); + IF evidence IS NULL OR NEW.result->wire_key IS DISTINCT FROM evidence + OR NEW.result - ARRAY['result',wire_key] <> '{}'::jsonb THEN + RAISE EXCEPTION 'receipt result evidence differs' USING ERRCODE = '23514'; + END IF; + END IF; + RETURN NEW; + END + $function$; + + -- Additional write predicates leave all approved reconciliation predicates + -- unchanged. They apply to raw SQL too, and only to newly written receipts. + CREATE OR REPLACE FUNCTION reporting_receipt_ingestion_graph() + RETURNS TRIGGER LANGUAGE plpgsql AS $function$ + DECLARE + revision reporting_revisions; + obligation reporting_obligations; + adjustment reporting_adjustments; + outcome JSONB; + observed TIMESTAMPTZ; + received TIMESTAMPTZ; + check_state TEXT; + chain_count INTEGER; + leaf_count INTEGER; + predecessor reporting_reconciliation_records; + visited TEXT[] := ARRAY[]::TEXT[]; + BEGIN + IF NEW.record_kind NOT IN ('revision_receipt','adjustment_receipt') THEN RETURN NEW; END IF; + PERFORM pg_advisory_xact_lock(hashtext('adcp.reporting:' || NEW.account_id)); + observed := (NEW.payload->>'observed_at')::timestamptz; + received := (NEW.payload->>'received_at')::timestamptz; + IF observed IS NULL OR received IS NULL OR observed > received OR received > clock_timestamp() + OR NOT EXISTS (SELECT 1 FROM reporting_reconciliation_records b + WHERE (b.account_id,b.consumer_id,b.delivery_config_id,b.delivery_config_version) + = (NEW.account_id,NEW.consumer_id,NEW.delivery_config_id,NEW.delivery_config_version) + AND b.record_kind='destination_binding' AND b.payload->>'reconciliation_mode'='consumer_receipt') THEN + RAISE EXCEPTION 'receipt target is unavailable' USING ERRCODE = '23514'; + END IF; + SELECT * INTO revision FROM reporting_revisions v + WHERE (v.account_id,v.reporting_obligation_id,v.reporting_revision_id) + = (NEW.account_id,NEW.reporting_obligation_id,NEW.reporting_revision_id); + SELECT * INTO obligation FROM reporting_obligations o + WHERE (o.account_id,o.reporting_obligation_id,o.delivery_config_id,o.delivery_config_version) + = (NEW.account_id,NEW.reporting_obligation_id,NEW.delivery_config_id,NEW.delivery_config_version); + IF revision.reporting_revision_id IS NULL OR obligation.reporting_obligation_id IS NULL THEN + RAISE EXCEPTION 'receipt target is unavailable' USING ERRCODE = '23514'; + END IF; + SELECT count(*) INTO chain_count FROM reporting_reconciliation_records r + WHERE (r.account_id,r.consumer_id,r.namespace,r.receipt_chain_key) + = (NEW.account_id,NEW.consumer_id,'receipt',NEW.receipt_chain_key); + IF chain_count > 0 THEN + SELECT count(*) INTO leaf_count FROM reporting_reconciliation_records r + WHERE (r.account_id,r.consumer_id,r.namespace,r.receipt_chain_key) + = (NEW.account_id,NEW.consumer_id,'receipt',NEW.receipt_chain_key) + AND NOT EXISTS (SELECT 1 FROM reporting_reconciliation_records s + WHERE (s.account_id,s.consumer_id,s.namespace,s.supersedes_receipt_id) + = (r.account_id,r.consumer_id,'receipt',r.record_id)); + SELECT r.* INTO predecessor FROM reporting_reconciliation_records r + WHERE (r.account_id,r.consumer_id,r.namespace,r.record_id) + = (NEW.account_id,NEW.consumer_id,'receipt',NEW.supersedes_receipt_id); + IF leaf_count <> 1 OR predecessor.record_id IS NULL THEN + RAISE EXCEPTION 'receipt replacement is unavailable' USING ERRCODE = '23514'; + END IF; + LOOP + IF predecessor.record_id IS NULL OR predecessor.record_id = ANY(visited) + OR predecessor.receipt_status <> 'rejected' + OR (predecessor.record_kind,predecessor.delivery_config_id, + predecessor.delivery_config_version,predecessor.reporting_obligation_id, + predecessor.reporting_revision_id,predecessor.receipt_chain_key) + IS DISTINCT FROM (NEW.record_kind,NEW.delivery_config_id, + NEW.delivery_config_version,NEW.reporting_obligation_id, + NEW.reporting_revision_id,NEW.receipt_chain_key) + OR predecessor.reporting_adjustment_id IS DISTINCT FROM NEW.reporting_adjustment_id THEN + RAISE EXCEPTION 'receipt history is inconsistent' USING ERRCODE = '23514'; + END IF; + visited := array_append(visited, predecessor.record_id); + EXIT WHEN predecessor.supersedes_receipt_id IS NULL; + SELECT r.* INTO predecessor FROM reporting_reconciliation_records r + WHERE (r.account_id,r.consumer_id,r.namespace,r.record_id) + = (NEW.account_id,NEW.consumer_id,'receipt',predecessor.supersedes_receipt_id); + END LOOP; + IF cardinality(visited) <> chain_count THEN + RAISE EXCEPTION 'receipt history is inconsistent' USING ERRCODE = '23514'; + END IF; + ELSIF NEW.supersedes_receipt_id IS NOT NULL THEN + RAISE EXCEPTION 'receipt replacement is unavailable' USING ERRCODE = '23514'; + END IF; + IF NEW.record_kind='adjustment_receipt' THEN + SELECT * INTO adjustment FROM reporting_adjustments a + WHERE (a.account_id,a.adjusts_reporting_revision_id,a.reporting_adjustment_id) + = (NEW.account_id,NEW.reporting_revision_id,NEW.reporting_adjustment_id); + IF adjustment.reporting_adjustment_id IS NULL OR revision.finality <> 'official' + OR revision.finalized_at IS NULL + OR NOT (obligation.period_end <= revision.finalized_at + AND revision.finalized_at <= revision.created_at + AND revision.finalized_at <= adjustment.correction_observed_at + AND adjustment.correction_observed_at <= adjustment.created_at + AND adjustment.accounting_period_start < adjustment.accounting_period_end + AND adjustment.created_at <= observed) THEN + RAISE EXCEPTION 'receipt adjustment order is invalid' USING ERRCODE = '23514'; + END IF; + ELSE + SELECT r.payload INTO outcome FROM reporting_reconciliation_records r + WHERE (r.account_id,r.consumer_id,r.delivery_config_id,r.delivery_config_version, + r.reporting_obligation_id,r.reporting_revision_id,r.reporting_materialization_id) + = (NEW.account_id,NEW.consumer_id,NEW.delivery_config_id,NEW.delivery_config_version, + NEW.reporting_obligation_id,NEW.reporting_revision_id,NEW.reporting_materialization_id) + AND r.record_kind='materialization' AND r.payload->>'status' IN ('available','delivered'); + IF outcome IS NULL OR observed < (outcome->>'completed_at')::timestamptz + OR NEW.payload->>'verification_profile' IS DISTINCT FROM outcome#>>'{verification,verification_profile}' THEN + RAISE EXCEPTION 'receipt artifact is unavailable' USING ERRCODE = '23514'; + END IF; + IF NEW.receipt_status='accepted' THEN + SELECT r.payload->>'state' INTO check_state FROM reporting_reconciliation_records r + WHERE r.account_id=NEW.account_id AND r.consumer_id=NEW.consumer_id + AND r.record_kind='materialization_check' + AND r.reporting_materialization_id=NEW.reporting_materialization_id + AND (r.payload->>'checked_at')::timestamptz <= observed + ORDER BY (r.payload->>'checked_at')::timestamptz DESC LIMIT 1; + IF (check_state IS NOT NULL AND check_state <> 'readable') + OR (NEW.payload->>'observed_row_count')::bigint + IS DISTINCT FROM (outcome#>>'{verification,row_count}')::bigint + OR (jsonb_typeof(NEW.payload->'observed_canonical_content_digest')='object' + AND reporting_wire_digest(NEW.payload->'observed_canonical_content_digest') + IS DISTINCT FROM reporting_wire_digest(outcome#>'{verification,canonical_content_digest}')) THEN + RAISE EXCEPTION 'receipt artifact evidence differs' USING ERRCODE = '23514'; + END IF; + END IF; + END IF; + RETURN NEW; + END + $function$; + + DROP TRIGGER IF EXISTS reporting_receipt_ingestion_batch ON reporting_receipt_ingestion_batches; + CREATE TRIGGER reporting_receipt_ingestion_batch BEFORE INSERT OR UPDATE OR DELETE + ON reporting_receipt_ingestion_batches FOR EACH ROW EXECUTE FUNCTION reporting_receipt_ingestion_batch_guard(); + DROP TRIGGER IF EXISTS reporting_receipt_ingestion_result ON reporting_receipt_ingestion_results; + CREATE TRIGGER reporting_receipt_ingestion_result BEFORE INSERT ON reporting_receipt_ingestion_results + FOR EACH ROW EXECUTE FUNCTION reporting_receipt_ingestion_result_guard(); + DROP TRIGGER IF EXISTS reporting_receipt_ingestion_result_immutable ON reporting_receipt_ingestion_results; + CREATE TRIGGER reporting_receipt_ingestion_result_immutable BEFORE UPDATE OR DELETE + ON reporting_receipt_ingestion_results FOR EACH ROW EXECUTE FUNCTION reporting_receipt_ingestion_immutable(); + DROP TRIGGER IF EXISTS reporting_receipt_ingestion_boundary_immutable ON reporting_receipt_ingestion_boundaries; + CREATE TRIGGER reporting_receipt_ingestion_boundary_immutable BEFORE UPDATE OR DELETE + ON reporting_receipt_ingestion_boundaries FOR EACH ROW EXECUTE FUNCTION reporting_receipt_ingestion_immutable(); + DROP TRIGGER IF EXISTS reporting_receipt_ingestion_graph ON reporting_reconciliation_records; + CREATE TRIGGER reporting_receipt_ingestion_graph BEFORE INSERT ON reporting_reconciliation_records + FOR EACH ROW EXECUTE FUNCTION reporting_receipt_ingestion_graph(); +END +$migration$; diff --git a/src/adcp/reporting/receipts/__init__.py b/src/adcp/reporting/receipts/__init__.py new file mode 100644 index 000000000..551286cc7 --- /dev/null +++ b/src/adcp/reporting/receipts/__init__.py @@ -0,0 +1,32 @@ +"""Durable seller receipt ingress. Optional PG driver is imported lazily.""" + +from typing import TYPE_CHECKING, Any + +from adcp.reporting.receipts.capture import ReportingReceiptBoundary +from adcp.reporting.receipts.errors import ReceiptErrorCode, ReportingReceiptError +from adcp.reporting.receipts.handler import ReceiptAccountResolver, ReportingReceiptHandler +from adcp.reporting.receipts.memory import InMemoryReportingReceiptStore +from adcp.reporting.receipts.store import ReportingReceiptBatchStore, ReportingReceiptCaptureStore + +if TYPE_CHECKING: + from adcp.reporting.receipts.pg import PgReportingReceiptStore + +__all__ = [ + "InMemoryReportingReceiptStore", + "PgReportingReceiptStore", + "ReceiptAccountResolver", + "ReceiptErrorCode", + "ReportingReceiptBatchStore", + "ReportingReceiptBoundary", + "ReportingReceiptCaptureStore", + "ReportingReceiptError", + "ReportingReceiptHandler", +] + + +def __getattr__(name: str) -> Any: + if name == "PgReportingReceiptStore": + from adcp.reporting.receipts.pg import PgReportingReceiptStore + + return PgReportingReceiptStore + raise AttributeError(name) diff --git a/src/adcp/reporting/receipts/capture.py b/src/adcp/reporting/receipts/capture.py new file mode 100644 index 000000000..ae5dc4260 --- /dev/null +++ b/src/adcp/reporting/receipts/capture.py @@ -0,0 +1,93 @@ +"""Immutable receipt dirty inputs; projection activation belongs to B2.4.""" + +from __future__ import annotations + +from dataclasses import dataclass, field +from datetime import datetime +from typing import Any + +from pydantic import TypeAdapter, ValidationError + +from adcp.reporting.evidence import aware_utc +from adcp.reporting.ledger._delivery_state import decode_record, payload, principal +from adcp.reporting.ledger.delivery_models import ( + ReportingAdjustmentReceiptRecord, + ReportingDeliveryPrincipal, + ReportingDeliveryRecord, + ReportingRevisionReceiptRecord, +) +from adcp.reporting.ledger.status_projection import ReportingStatusSnapshot +from adcp.reporting.receipts.errors import ReportingReceiptError + +_CORE = TypeAdapter(ReportingStatusSnapshot) + + +@dataclass(frozen=True) +class ReportingReceiptBoundary: + caller: ReportingDeliveryPrincipal + sequence: int + account_sequence: int + reporting_receipt_id: str + as_of: datetime + core: ReportingStatusSnapshot = field(repr=False) + reconciliation: tuple[ReportingDeliveryRecord, ...] = field(repr=False) + + def __post_init__(self) -> None: + receipts = tuple( + r + for r in self.reconciliation + if isinstance(r, (ReportingRevisionReceiptRecord, ReportingAdjustmentReceiptRecord)) + and r.reporting_receipt_id == self.reporting_receipt_id + ) + if ( + type(self.sequence) is not int + or self.sequence < 1 + or type(self.account_sequence) is not int + or self.account_sequence < self.sequence + or self.core.account_id != self.caller.account_id + or self.core.as_of != self.as_of + or self.core.consumer_ids != (self.caller.consumer_id,) + or any(s.consumer_id != self.caller.consumer_id for s in self.core.statuses) + or any( + i.consumer_id not in {None, self.caller.consumer_id} for i in self.core.lifecycles + ) + or any(principal(r) != self.caller for r in self.reconciliation) + or len(receipts) != 1 + or receipts[0].received_at != self.as_of + ): + raise ReportingReceiptError("RECEIPT_HISTORY_CORRUPT") + object.__setattr__(self, "as_of", aware_utc(self.as_of)) + + def to_storage(self) -> dict[str, Any]: + return { + "version": 1, + "admission_epoch": 0, + "account_id": self.caller.account_id, + "consumer_id": self.caller.consumer_id, + "sequence": self.sequence, + "account_sequence": self.account_sequence, + "reporting_receipt_id": self.reporting_receipt_id, + "as_of": self.as_of.isoformat(), + "core": _CORE.dump_python(self.core, mode="json"), + "reconciliation": [payload(r) for r in self.reconciliation], + } + + +def decode_receipt_boundary(value: dict[str, Any]) -> ReportingReceiptBoundary: + result = None + try: + if type(value) is dict and type(value.get("version")) is int and value["version"] == 1: + result = ReportingReceiptBoundary( + ReportingDeliveryPrincipal(value["account_id"], value["consumer_id"]), + value["sequence"], + value["account_sequence"], + value["reporting_receipt_id"], + datetime.fromisoformat(value["as_of"]), + _CORE.validate_python(value["core"]), + tuple(decode_record(r) for r in value["reconciliation"]), + ) + except (ValueError, TypeError, KeyError, ValidationError): + pass + if result is None or result.to_storage() != value: + raise ReportingReceiptError("RECEIPT_HISTORY_CORRUPT") + return result diff --git a/src/adcp/reporting/receipts/errors.py b/src/adcp/reporting/receipts/errors.py new file mode 100644 index 000000000..338dea1e6 --- /dev/null +++ b/src/adcp/reporting/receipts/errors.py @@ -0,0 +1,31 @@ +"""Closed, payload-free receipt ingress failures.""" + +from __future__ import annotations + +from typing import Literal + +ReceiptErrorCode = Literal[ + "INVALID_REQUEST", + "UNAUTHORIZED", + "IDEMPOTENCY_CONFLICT", + "RECEIPT_SCHEMA_UNREADY", + "RECEIPT_HISTORY_CORRUPT", + "RECEIPT_STORAGE_UNAVAILABLE", +] + +_MESSAGES: dict[ReceiptErrorCode, str] = { + "INVALID_REQUEST": "supply a valid receipt batch with 1..100 distinct receipt IDs", + "UNAUTHORIZED": "the reporting account or authenticated consumer is unavailable", + "IDEMPOTENCY_CONFLICT": "reuse the original receipt batch body or choose a new batch key", + "RECEIPT_SCHEMA_UNREADY": "install and verify the isolated receipt ingestion schema", + "RECEIPT_HISTORY_CORRUPT": "retained receipt batch evidence requires operator repair", + "RECEIPT_STORAGE_UNAVAILABLE": "receipt storage is unavailable; retry the same batch and key", +} + + +class ReportingReceiptError(RuntimeError): + """An actionable classification; never includes request, driver or auth details.""" + + def __init__(self, code: ReceiptErrorCode) -> None: + self.code = code + super().__init__(_MESSAGES[code]) diff --git a/src/adcp/reporting/receipts/handler.py b/src/adcp/reporting/receipts/handler.py new file mode 100644 index 000000000..70239e119 --- /dev/null +++ b/src/adcp/reporting/receipts/handler.py @@ -0,0 +1,153 @@ +"""One authenticated composition path for direct MCP/A2A and hydrated contexts.""" + +from __future__ import annotations + +from collections.abc import Awaitable, Callable +from typing import Any + +from adcp.decisioning.context import AuthInfo, RequestContext +from adcp.decisioning.registry import BuyerAgent, BuyerAgentRegistry, HttpSigCredential +from adcp.exceptions import ADCPTaskError +from adcp.reporting.ledger.delivery_models import ReportingDeliveryPrincipal +from adcp.reporting.ledger.notification_models import ReportingNotificationError +from adcp.reporting.outbox.identity import canonical_consumer, resolve_reporting_consumer +from adcp.reporting.receipts.errors import ReportingReceiptError +from adcp.reporting.receipts.store import ReportingReceiptBatchStore +from adcp.reporting.receipts.wire import TASK, validate_receipt_request +from adcp.server.base import ADCPHandler, ToolContext +from adcp.types import Error, SyncReportingReceiptsRequest + +ReceiptAccountResolver = Callable[[dict[str, Any], ToolContext, str], Awaitable[str]] +"""Resolve AND reauthorize the exact account reference for this consumer on every call. + +Return the canonical storage account ID, never a RequestContext cache key. A +natural-key account reference is resolved by this same application ACL boundary. +Raises ReportingReceiptError('UNAUTHORIZED') for unknown or denied accounts. +""" + + +async def _consumer(context: ToolContext, registry: BuyerAgentRegistry | None) -> str: + identities: set[str] = set() + auths: list[AuthInfo] = [] + agents: list[BuyerAgent] = [] + auth_value = getattr(context, "auth_info", None) + if auth_value is not None: + if not isinstance(auth_value, AuthInfo): + raise ReportingReceiptError("UNAUTHORIZED") + auths.append(auth_value) + agent_value = getattr(context, "buyer_agent", None) + if agent_value is not None: + if not isinstance(agent_value, BuyerAgent): + raise ReportingReceiptError("UNAUTHORIZED") + agents.append(agent_value) + auth_principal = getattr(context, "auth_principal", None) + if auth_principal is not None: + identities.add(canonical_consumer(auth_principal)) + # caller_identity is deliberately never inspected on RequestContext. + if not isinstance(context, RequestContext) and context.caller_identity is not None: + identities.add(canonical_consumer(context.caller_identity)) + for key in ("adcp.auth_info", "auth_info"): + auth = context.metadata.get(key) + if auth is not None: + if not isinstance(auth, AuthInfo): + raise ReportingReceiptError("UNAUTHORIZED") + auths.append(auth) + agent = context.metadata.get("adcp.buyer_agent") + if agent is not None: + if not isinstance(agent, BuyerAgent): + raise ReportingReceiptError("UNAUTHORIZED") + agents.append(agent) + if registry is not None: + if not auths: + raise ReportingReceiptError("UNAUTHORIZED") + for auth in auths: + credential = auth.credential + if credential is None: + raise ReportingReceiptError("UNAUTHORIZED") + resolved = ( + await registry.resolve_by_agent_url(credential.agent_url) + if isinstance(credential, HttpSigCredential) + else await registry.resolve_by_credential(credential) + ) + if resolved is None: + raise ReportingReceiptError("UNAUTHORIZED") + agents.append(resolved) + for agent in agents: + if agent.status != "active": + raise ReportingReceiptError("UNAUTHORIZED") + identities.add(resolve_reporting_consumer(auth_info=None, agent=agent)) + for auth in auths: + # A credential-only API/OAuth AuthInfo is allowed when a registry or + # trusted hydrated agent supplied the consumer. It is never an alias. + if ( + auth.principal is not None + or auth.agent_url is not None + or isinstance(auth.credential, HttpSigCredential) + ): + identities.add(resolve_reporting_consumer(auth_info=auth)) + if len(identities) != 1: + raise ReportingReceiptError("UNAUTHORIZED") + return identities.pop() + + +class ReportingReceiptHandler(ADCPHandler[ToolContext]): + """Mount only the durable receipt task; tier activation remains separately gated. + + ``resolve_account`` is an application ACL, called even for a completed + batch. ``buyer_agents`` optionally re-resolves API/OAuth/signed commercial + identity on every call. Neither transport tenancy nor body fields supply + the consumer. Authentication middleware must populate trusted context. + """ + + advertised_tools = {TASK} + + def __init__( + self, + store: ReportingReceiptBatchStore, + *, + resolve_account: ReceiptAccountResolver, + buyer_agents: BuyerAgentRegistry | None = None, + ) -> None: + super().__init__() + if not isinstance(store, ReportingReceiptBatchStore): + raise TypeError("receipt ingress requires an atomic ReportingReceiptBatchStore") + self.receipt_store = store + self._receipt_account_resolver = resolve_account + self._receipt_registry = buyer_agents + + async def sync_reporting_receipts( + self, + params: SyncReportingReceiptsRequest | dict[str, Any], + context: ToolContext | None = None, + ) -> dict[str, Any]: + request = ( + params + if isinstance(params, dict) + else params.model_dump(mode="json", exclude_unset=True) + ) + try: + validate_receipt_request(request) + if context is None: + raise ReportingReceiptError("UNAUTHORIZED") + try: + consumer = await _consumer(context, self._receipt_registry) + except ReportingNotificationError: + raise ReportingReceiptError("UNAUTHORIZED") from None + account = await self._receipt_account_resolver( + dict(request["account"]), context, consumer + ) + if isinstance(context, RequestContext) and context.account.id != account: + raise ReportingReceiptError("UNAUTHORIZED") + try: + caller = ReportingDeliveryPrincipal(account, consumer) + except (ValueError, TypeError): + raise ReportingReceiptError("UNAUTHORIZED") from None + return await self.receipt_store.ingest_receipt_batch(request, caller=caller) + except ReportingReceiptError as error: + code, message = error.code, str(error) + except Exception: + unavailable = ReportingReceiptError("RECEIPT_STORAGE_UNAVAILABLE") + code, message = unavailable.code, str(unavailable) + # Leave the exception scope before translating. Credential/ACL adapters + # may raise provider errors; neither logs nor exception chains retain them. + raise ADCPTaskError(operation=TASK, errors=[Error(code=code, message=message)]) diff --git a/src/adcp/reporting/receipts/memory.py b/src/adcp/reporting/receipts/memory.py new file mode 100644 index 000000000..577966a7d --- /dev/null +++ b/src/adcp/reporting/receipts/memory.py @@ -0,0 +1,203 @@ +"""Account-serialized receipt batches on the existing unconditional rollback model.""" + +from __future__ import annotations + +import json +from copy import deepcopy +from dataclasses import dataclass, replace +from datetime import datetime +from typing import Any + +from adcp.reporting.canonical_json import canonical_json_utf8_v1 +from adcp.reporting.ledger._delivery_state import RecordT +from adcp.reporting.ledger.delivery_models import ( + ReportingAdjustmentReceiptRecord, + ReportingDeliveryPrincipal, + ReportingReceiptRecord, + ReportingRevisionReceiptRecord, +) +from adcp.reporting.ledger.status_snapshot import settle_memory_snapshot +from adcp.reporting.ledger.store import LedgerConflictError +from adcp.reporting.materializer.capture import private_snapshot +from adcp.reporting.materializer.memory import InMemoryReportingMaterializerStore +from adcp.reporting.receipts.capture import ReportingReceiptBoundary +from adcp.reporting.receipts.errors import ReportingReceiptError +from adcp.reporting.receipts.records import ( + failed_result, + prepare_receipt, + receipt_record, + success_result, +) +from adcp.reporting.receipts.wire import ( + ReceiptBatch, + validate_receipt_response, + validate_receipt_results, +) +from adcp.server.helpers import inject_context + + +@dataclass +class _BatchState: + batch: ReceiptBatch + expected_count: int + created_at: datetime + results: tuple[dict[str, Any], ...] = () + response: bytes | None = None + + +class InMemoryReportingReceiptStore(InMemoryReportingMaterializerStore): + """Reference participant. No production tier or notification readiness claim.""" + + # Lazily initialized deliberately: first-use allocations must roll back too. + _receipt_batches: dict[tuple[str, str, str], _BatchState] + _receipt_boundaries: list[ReportingReceiptBoundary] + _receipt_status_heads: dict[ReportingDeliveryPrincipal, int] + + def _receipt_batch( + self, caller: ReportingDeliveryPrincipal, batch: ReceiptBatch + ) -> _BatchState: + if not hasattr(self, "_receipt_batches"): + self._receipt_batches = {} + key = caller.account_id, caller.consumer_id, batch.key + state = self._receipt_batches.get(key) + if state is None: + state = _BatchState(batch, len(batch.items), self._clock()) + self._receipt_batches[key] = state + elif state.batch.canonical_request != batch.canonical_request: + raise ReportingReceiptError("IDEMPOTENCY_CONFLICT") + if state.expected_count != len(batch.items) or len(state.results) > state.expected_count: + raise ReportingReceiptError("RECEIPT_HISTORY_CORRUPT") + validate_receipt_results(list(state.results), batch) + return state + + def _append_receipt_result(self, state: _BatchState, result: dict[str, Any]) -> None: + state.results = (*state.results, deepcopy(result)) + + def _save_receipt_response(self, state: _BatchState) -> None: + response = inject_context( + state.batch.request, {"status": "completed", "results": deepcopy(list(state.results))} + ) + validate_receipt_response(response, state.batch) + state.response = canonical_json_utf8_v1(response) + + def _assemble_receipt_response(self, state: _BatchState) -> dict[str, Any]: + if state.response is None: + raise ReportingReceiptError("RECEIPT_HISTORY_CORRUPT") + response: dict[str, Any] = json.loads(state.response) + validate_receipt_response(response, state.batch) + if response["results"] != list(state.results): + raise ReportingReceiptError("RECEIPT_HISTORY_CORRUPT") + return response + + async def ingest_receipt_batch( + self, request: dict[str, Any], *, caller: ReportingDeliveryPrincipal + ) -> dict[str, Any]: + batch = ReceiptBatch.parse(request) + while True: + # Each turn chooses the next *durable* ordinal under the account + # lock. Concurrent/resumed callers cooperate; no process-local batch + # lock, pending task, or TTL is a correctness dependency. + async with self._mutation(): + state = self._receipt_batch(caller, batch) + if state.response is not None: + return self._assemble_receipt_response(state) + if len(state.results) == state.expected_count: + self._save_receipt_response(state) + return self._assemble_receipt_response(state) + kind, item = batch.items[len(state.results)] + candidate = None + try: + revision_id = item.get( + "reporting_revision_id", item.get("adjusts_reporting_revision_id") + ) + revision = ( + self._revisions.get(revision_id) if isinstance(revision_id, str) else None + ) + obligation_id = item.get("reporting_obligation_id") + if ( + kind == "adjustment_receipt" + and revision is not None + and revision.account_id == caller.account_id + ): + obligation_id = revision.reporting_obligation_id + obligation = ( + self._obligations.get(obligation_id) + if isinstance(obligation_id, str) + else None + ) + candidate = receipt_record(kind, item, caller, obligation) + records = tuple(c.record for c in self._caller_changes(caller)) + stored, added = prepare_receipt( + candidate, + records, + self._delivery_context(candidate), + tuple( + r + for r in self._revisions.values() + if obligation is not None + and r.account_id == caller.account_id + and r.reporting_obligation_id == obligation.reporting_obligation_id + ), + self._clock(), + ) + result = success_result(stored, added) + except LedgerConflictError as error: + result = failed_result(item, error) + added = False + # Nothing in the semantic-error catch above mutates domain state. + # Any insertion/capture/result failure below escapes and rolls + # this entire ordinal back, with notifications both off and on. + if added: + assert candidate is not None + stored, inserted = self._commit_record_unlocked(candidate) + assert inserted + result = success_result(stored, True) + self._append_receipt_result(state, result) + + def _commit_record_unlocked( + self, record: RecordT, *, notify: bool = True, dirty: bool = True + ) -> tuple[RecordT, bool]: + stored, added = super()._commit_record_unlocked(record, notify=notify, dirty=dirty) + if added and isinstance( + stored, (ReportingRevisionReceiptRecord, ReportingAdjustmentReceiptRecord) + ): + self._capture_receipt(stored) + return stored, added + + def _capture_receipt(self, receipt: ReportingReceiptRecord) -> None: + assert receipt.received_at is not None + caller = receipt.scope.principal + if not hasattr(self, "_receipt_status_heads"): + self._receipt_status_heads = {} + if not hasattr(self, "_receipt_boundaries"): + self._receipt_boundaries = [] + sequence = self._receipt_status_heads.get(caller, 0) + 1 + self._receipt_status_heads[caller] = sequence + # Preserve cross-kind account order using B2.1's original capture clock. + # No materializer work, retry decision, or readiness event is created. + account_sequence = self._materializer_account_heads.get(caller.account_id, 0) + 1 + self._materializer_account_heads[caller.account_id] = account_sequence + core = replace(settle_memory_snapshot(self, caller.account_id), as_of=receipt.received_at) + self._receipt_boundaries.append( + ReportingReceiptBoundary( + caller, + sequence, + account_sequence, + receipt.reporting_receipt_id, + receipt.received_at, + private_snapshot(core, caller), + tuple(c.record for c in self._caller_changes(caller)), + ) + ) + + async def read_receipt_boundaries( + self, *, caller: ReportingDeliveryPrincipal, after: int = 0, limit: int = 100 + ) -> tuple[ReportingReceiptBoundary, ...]: + if type(after) is not int or after < 0 or type(limit) is not int or not 1 <= limit <= 100: + raise ReportingReceiptError("INVALID_REQUEST") + async with self._lock: + return tuple( + b + for b in getattr(self, "_receipt_boundaries", ()) + if b.caller == caller and b.sequence > after + )[:limit] diff --git a/src/adcp/reporting/receipts/pg.py b/src/adcp/reporting/receipts/pg.py new file mode 100644 index 000000000..673deec8a --- /dev/null +++ b/src/adcp/reporting/receipts/pg.py @@ -0,0 +1,386 @@ +"""Durable mixed-batch ordinals on the materializer's account-locked connection.""" + +from __future__ import annotations + +import json +from collections.abc import Callable, Coroutine +from datetime import datetime +from functools import wraps +from importlib.resources import files +from typing import Any, ParamSpec, TypeVar, cast + +from adcp.reporting.canonical_json import canonical_json_sha256_v1 +from adcp.reporting.ledger._delivery_state import RecordT +from adcp.reporting.ledger.delivery_models import ( + ReportingAdjustmentReceiptRecord, + ReportingDeliveryPrincipal, + ReportingDeliveryRecord, + ReportingReceiptRecord, + ReportingRevisionReceiptRecord, +) +from adcp.reporting.ledger.pg import ( + _OBLIGATION_COLUMNS, + _REVISION_COLUMNS, + _json, + _obligation_from_row, + _revision_from_row, +) +from adcp.reporting.ledger.status_snapshot import settle_snapshot_on +from adcp.reporting.ledger.store import LedgerConflictError +from adcp.reporting.materializer.capture import private_snapshot +from adcp.reporting.materializer.pg import PgReportingMaterializerStore, _now +from adcp.reporting.receipts.capture import ReportingReceiptBoundary, decode_receipt_boundary +from adcp.reporting.receipts.errors import ReportingReceiptError +from adcp.reporting.receipts.records import ( + failed_result, + prepare_receipt, + receipt_record, + success_result, +) +from adcp.reporting.receipts.schema import validate_receipt_schema +from adcp.reporting.receipts.wire import ( + ReceiptBatch, + ReceiptKind, + validate_receipt_response, + validate_receipt_results, +) +from adcp.server.helpers import inject_context + +_P = ParamSpec("_P") +_R = TypeVar("_R") + + +def _storage_errors( + fn: Callable[_P, Coroutine[Any, Any, _R]], +) -> Callable[_P, Coroutine[Any, Any, _R]]: + @wraps(fn) + async def wrapped(*args: _P.args, **kwargs: _P.kwargs) -> _R: + try: + return await fn(*args, **kwargs) + except ReportingReceiptError: + raise + except Exception: + unavailable = ReportingReceiptError("RECEIPT_STORAGE_UNAVAILABLE") + # Outside the driver exception scope: no SQL/provider detail in __context__. + raise unavailable + + return wrapped + + +class PgReportingReceiptStore(PgReportingMaterializerStore): + """One composition: old public stores plus optional ingestion and capture. + + No network I/O, session lock, or second pool participates in an ordinal. + All actual receipt timestamps come from PostgreSQL, including when an old + public conformance clock was supplied to the inherited constructor. + """ + + @_storage_errors + async def create_schema(self) -> None: + async with self._connection() as connection, connection.transaction(): + await self._create_schema_on(connection) + root = files("adcp.reporting.ledger") + await connection.execute(root.joinpath("reporting_materializer.sql").read_text()) + await connection.execute(root.joinpath("reporting_receipt_ingestion.sql").read_text()) + + @_storage_errors + async def receipt_ingestion_ready(self) -> bool: + async with self._connection() as connection: + await validate_receipt_schema(connection, notifications=self._notifications_enabled) + return True + + async def _delivery_time_on(self, connection: Any, record: ReportingDeliveryRecord) -> datetime: + if isinstance(record, (ReportingRevisionReceiptRecord, ReportingAdjustmentReceiptRecord)): + return await _now(connection) + return await super()._delivery_time_on(connection, record) + + async def _commit_record_on( + self, connection: Any, record: RecordT, *, notify: bool = True, dirty: bool = True + ) -> tuple[RecordT, bool]: + stored, added = await super()._commit_record_on( + connection, record, notify=notify, dirty=dirty + ) + if added and isinstance( + stored, (ReportingRevisionReceiptRecord, ReportingAdjustmentReceiptRecord) + ): + await self._capture_receipt_on(connection, stored) + return stored, added + + async def _receipt_batch_on( + self, connection: Any, caller: ReportingDeliveryPrincipal, batch: ReceiptBatch + ) -> tuple[Any, ...]: + key = caller.account_id, caller.consumer_id, batch.key + row = await ( + await connection.execute( + "SELECT canonical_request,request_sha256,expected_count,final_response,final_sha256" + " FROM reporting_receipt_ingestion_batches" + " WHERE account_id=%s AND consumer_id=%s AND idempotency_key=%s FOR UPDATE", + key, + ) + ).fetchone() + if row is None: + await connection.execute( + "INSERT INTO reporting_receipt_ingestion_batches" + " (account_id,consumer_id,idempotency_key,canonical_request," + " request_sha256,expected_count)" + " VALUES (%s,%s,%s,%s,%s,%s)", + (*key, batch.canonical_request.decode(), batch.digest, len(batch.items)), + ) + return batch.canonical_request.decode(), batch.digest, len(batch.items), None, None + if row[0] != batch.canonical_request.decode() or row[1] != batch.digest: + raise ReportingReceiptError("IDEMPOTENCY_CONFLICT") + if row[2] != len(batch.items): + raise ReportingReceiptError("RECEIPT_HISTORY_CORRUPT") + return tuple(row) + + async def _receipt_results_on( + self, connection: Any, caller: ReportingDeliveryPrincipal, batch: ReceiptBatch + ) -> list[dict[str, Any]]: + rows = await ( + await connection.execute( + "SELECT ordinal,receipt_kind,reporting_receipt_id,result,content_sha256" + " FROM reporting_receipt_ingestion_results" + " WHERE account_id=%s AND consumer_id=%s AND idempotency_key=%s ORDER BY ordinal", + (caller.account_id, caller.consumer_id, batch.key), + ) + ).fetchall() + items = batch.items + if len(rows) > len(items) or any( + r[0] != i + or r[1] != items[i][0] + or r[2] != items[i][1]["reporting_receipt_id"] + or canonical_json_sha256_v1(r[3]) != r[4] + for i, r in enumerate(rows) + ): + raise ReportingReceiptError("RECEIPT_HISTORY_CORRUPT") + results = [r[3] for r in rows] + validate_receipt_results(results, batch) + return results + + async def _prepare_receipt_on( + self, + connection: Any, + caller: ReportingDeliveryPrincipal, + kind: ReceiptKind, + item: dict[str, Any], + ) -> tuple[ReportingReceiptRecord, dict[str, Any], bool]: + revision_id = item.get("reporting_revision_id", item.get("adjusts_reporting_revision_id")) + revision = await ( + await connection.execute( + # SDK-owned column list; every request value is bound below. + f"SELECT {_REVISION_COLUMNS} FROM reporting_revisions" # nosec B608 + " WHERE account_id=%s AND reporting_revision_id=%s", + (caller.account_id, revision_id), + ) + ).fetchone() + obligation_id = item.get("reporting_obligation_id") + if kind == "adjustment_receipt" and revision is not None: + obligation_id = _revision_from_row(revision).reporting_obligation_id + row = await ( + await connection.execute( + # SDK-owned column list; every request value is bound below. + f"SELECT {_OBLIGATION_COLUMNS} FROM reporting_obligations" # nosec B608 + " WHERE account_id=%s AND reporting_obligation_id=%s", + (caller.account_id, obligation_id), + ) + ).fetchone() + obligation = _obligation_from_row(row) if row else None + candidate = receipt_record(kind, item, caller, obligation) + records = await self._records(connection, caller) + revisions = await ( + await connection.execute( + # SDK-owned column list; every request value is bound below. + f"SELECT {_REVISION_COLUMNS} FROM reporting_revisions" # nosec B608 + " WHERE account_id=%s AND reporting_obligation_id=%s", + (caller.account_id, obligation_id), + ) + ).fetchall() + stored, added = prepare_receipt( + candidate, + records, + await self._delivery_context(connection, candidate), + tuple(_revision_from_row(r) for r in revisions), + await _now(connection), + ) + return candidate, success_result(stored, added), added + + async def _insert_receipt_result_on( + self, + connection: Any, + caller: ReportingDeliveryPrincipal, + batch: ReceiptBatch, + ordinal: int, + result: dict[str, Any], + ) -> None: + kind, item = batch.items[ordinal] + await connection.execute( + "INSERT INTO reporting_receipt_ingestion_results" + " (account_id,consumer_id,idempotency_key,ordinal,receipt_kind,reporting_receipt_id," + " receipt_record_id,result,content_sha256) VALUES (%s,%s,%s,%s,%s,%s,%s,%s::jsonb,%s)", + ( + caller.account_id, + caller.consumer_id, + batch.key, + ordinal, + kind, + item["reporting_receipt_id"], + item["reporting_receipt_id"] if result["result"] != "failed" else None, + _json(result), + canonical_json_sha256_v1(result), + ), + ) + + async def _save_receipt_response_on( + self, + connection: Any, + caller: ReportingDeliveryPrincipal, + batch: ReceiptBatch, + results: list[dict[str, Any]], + ) -> dict[str, Any]: + response = inject_context(batch.request, {"status": "completed", "results": results}) + validate_receipt_response(response, batch) + await connection.execute( + "UPDATE reporting_receipt_ingestion_batches" + " SET final_response=%s::jsonb,final_sha256=%s,finalized_at=clock_timestamp()" + " WHERE account_id=%s AND consumer_id=%s AND idempotency_key=%s", + ( + _json(response), + canonical_json_sha256_v1(response), + caller.account_id, + caller.consumer_id, + batch.key, + ), + ) + return response + + def _assemble_receipt_response( + self, + response: dict[str, Any], + digest: str, + results: list[dict[str, Any]], + batch: ReceiptBatch, + ) -> dict[str, Any]: + validate_receipt_response(response, batch) + if canonical_json_sha256_v1(response) != digest or response["results"] != results: + raise ReportingReceiptError("RECEIPT_HISTORY_CORRUPT") + return cast(dict[str, Any], json.loads(_json(response))) + + @_storage_errors + async def ingest_receipt_batch( + self, request: dict[str, Any], *, caller: ReportingDeliveryPrincipal + ) -> dict[str, Any]: + batch = ReceiptBatch.parse(request) + checked = False + while True: + async with self._connection() as connection, connection.transaction(): + # Account advisory lock precedes the batch row; no lock pool or + # second connection can deadlock a size-one application pool. + await self._lock_account(connection, caller.account_id) + if not checked: + await validate_receipt_schema( + connection, notifications=self._notifications_enabled + ) + checked = True + row = await self._receipt_batch_on(connection, caller, batch) + results = await self._receipt_results_on(connection, caller, batch) + if row[3] is not None: + return self._assemble_receipt_response(row[3], row[4], results, batch) + if len(results) == len(batch.items): + response = await self._save_receipt_response_on( + connection, caller, batch, results + ) + return self._assemble_receipt_response( + response, canonical_json_sha256_v1(response), results, batch + ) + kind, item = batch.items[len(results)] + candidate = None + try: + candidate, result, added = await self._prepare_receipt_on( + connection, caller, kind, item + ) + except LedgerConflictError as error: + result, added = failed_result(item, error), False + # Only pure semantic preparation is caught. SQL/capture/result + # failures roll back this ordinal instead of recording a failure + # beside a partially inserted receipt. + if added: + assert candidate is not None + stored, inserted = await self._commit_record_on(connection, candidate) + assert inserted + result = success_result(stored, True) + await self._insert_receipt_result_on( + connection, caller, batch, len(results), result + ) + + async def _capture_receipt_on(self, connection: Any, receipt: ReportingReceiptRecord) -> None: + assert receipt.received_at is not None + caller = receipt.scope.principal + core = await settle_snapshot_on( + self, connection, account_id=caller.account_id, as_of=receipt.received_at + ) + row = await ( + await connection.execute( + "INSERT INTO reporting_receipt_ingestion_heads" + " (account_id,consumer_id,max_sequence)" + " VALUES (%s,%s,1) ON CONFLICT (account_id,consumer_id) DO UPDATE" + " SET max_sequence=reporting_receipt_ingestion_heads.max_sequence+1" + " RETURNING max_sequence", + (caller.account_id, caller.consumer_id), + ) + ).fetchone() + account = await ( + await connection.execute( + "UPDATE reporting_materializer_accounts SET captured_sequence=captured_sequence+1" + " WHERE account_id=%s RETURNING captured_sequence", + (caller.account_id,), + ) + ).fetchone() + if row is None or account is None: + raise ReportingReceiptError("RECEIPT_HISTORY_CORRUPT") + value = ReportingReceiptBoundary( + caller, + row[0], + account[0], + receipt.reporting_receipt_id, + receipt.received_at, + private_snapshot(core, caller), + await self._records(connection, caller), + ).to_storage() + await connection.execute( + "INSERT INTO reporting_receipt_ingestion_boundaries" + " (account_id,consumer_id,sequence,account_sequence,reporting_receipt_id," + " as_of,input,content_sha256)" + " VALUES (%s,%s,%s,%s,%s,%s,%s::jsonb,%s)", + ( + caller.account_id, + caller.consumer_id, + row[0], + account[0], + receipt.reporting_receipt_id, + receipt.received_at, + _json(value), + canonical_json_sha256_v1(value), + ), + ) + + @_storage_errors + async def read_receipt_boundaries( + self, *, caller: ReportingDeliveryPrincipal, after: int = 0, limit: int = 100 + ) -> tuple[ReportingReceiptBoundary, ...]: + if type(after) is not int or after < 0 or type(limit) is not int or not 1 <= limit <= 100: + raise ReportingReceiptError("INVALID_REQUEST") + async with self._connection() as connection, connection.transaction(): + await self._lock_account(connection, caller.account_id) + await validate_receipt_schema(connection, notifications=self._notifications_enabled) + rows = await ( + await connection.execute( + "SELECT input,content_sha256=reporting_receipt_ingestion_sha256(input)" + " FROM reporting_receipt_ingestion_boundaries" + " WHERE account_id=%s AND consumer_id=%s AND sequence>%s" + " ORDER BY sequence LIMIT %s", + (caller.account_id, caller.consumer_id, after, limit), + ) + ).fetchall() + if any(not r[1] for r in rows): + raise ReportingReceiptError("RECEIPT_HISTORY_CORRUPT") + return tuple(decode_receipt_boundary(r[0]) for r in rows) diff --git a/src/adcp/reporting/receipts/records.py b/src/adcp/reporting/receipts/records.py new file mode 100644 index 000000000..419cccd4e --- /dev/null +++ b/src/adcp/reporting/receipts/records.py @@ -0,0 +1,105 @@ +"""Pure preparation against exact, account-locked retained evidence.""" + +from __future__ import annotations + +from datetime import datetime +from typing import Any, cast + +from pydantic import TypeAdapter, ValidationError + +from adcp.reporting.evidence import ReportingCanonicalDigest +from adcp.reporting.ledger._delivery_state import DeliveryContext, fail, replay, validate_transition +from adcp.reporting.ledger.delivery import receipt_to_wire +from adcp.reporting.ledger.delivery_models import ( + ReportingAdjustmentReceiptRecord, + ReportingControlTotalRecord, + ReportingDeliveryPrincipal, + ReportingDeliveryRecord, + ReportingDeliveryScope, + ReportingReceiptRecord, + ReportingRevisionReceiptRecord, +) +from adcp.reporting.ledger.models import ( + ReportingObligationRecord, + ReportingRevisionRecord, +) +from adcp.reporting.ledger.store import LedgerConflictError +from adcp.reporting.receipts.wire import ReceiptKind +from adcp.reporting.revision_selection import select_reporting_revision + +_REVISION_RECEIPT = TypeAdapter(ReportingRevisionReceiptRecord) +_ADJUSTMENT_RECEIPT = TypeAdapter(ReportingAdjustmentReceiptRecord) + + +def receipt_record( + kind: ReceiptKind, + item: dict[str, Any], + caller: ReportingDeliveryPrincipal, + obligation: ReportingObligationRecord | None, +) -> ReportingReceiptRecord: + if obligation is None or obligation.account_id != caller.account_id: + fail("REPORTING_RECORD_UNAVAILABLE") + body = dict(item) + body.pop("reporting_obligation_id", None) + body["scope"] = ReportingDeliveryScope( + obligation.generation_key, caller.consumer_id, obligation.reporting_obligation_id + ) + value: ReportingReceiptRecord | None = None + try: + if kind == "revision_receipt": + body["observed_control_totals"] = tuple( + ReportingControlTotalRecord(**total) for total in body["observed_control_totals"] + ) + digest = body.get("observed_canonical_content_digest") + if digest is not None: + body["observed_canonical_content_digest"] = ReportingCanonicalDigest( + **{k: v for k, v in digest.items() if k != "algorithm"} + ) + adapter = _REVISION_RECEIPT if kind == "revision_receipt" else _ADJUSTMENT_RECEIPT + value = adapter.validate_python(body) + except (ValueError, TypeError, ValidationError): + pass + if value is None: + fail("INVALID_REPORTING_RECORD") + return value + + +def prepare_receipt( + record: ReportingReceiptRecord, + records: tuple[ReportingDeliveryRecord, ...], + context: DeliveryContext, + revisions: tuple[ReportingRevisionRecord, ...], + now: datetime, +) -> tuple[ReportingReceiptRecord, bool]: + existing = replay(record, records) + if existing is not None: + return cast(ReportingReceiptRecord, existing), False + if context.obligation is None: + fail("REPORTING_RECORD_UNAVAILABLE") + obligation = context.obligation + selection = select_reporting_revision( + revisions, + account_id=obligation.account_id, + reporting_obligation_id=obligation.reporting_obligation_id, + required_finality=obligation.required_finality, + ) + if selection.kind == "corrupt": + fail("REPORTING_HISTORY_CORRUPT") + # Historical snapshot receipts remain independently repairable after official + # publication. Evidence is bound to the supplied artifact, never the latest try. + return cast(ReportingReceiptRecord, validate_transition(record, records, context, now)), True + + +def success_result(record: ReportingReceiptRecord, added: bool) -> dict[str, Any]: + key = "receipt" if isinstance(record, ReportingRevisionReceiptRecord) else "adjustment_receipt" + return {"result": "recorded" if added else "unchanged", key: receipt_to_wire(record)} + + +def failed_result(item: dict[str, Any], error: LedgerConflictError) -> dict[str, Any]: + # The shared state machine uses closed codes. Never persist exception text or + # database details, and never distinguish an invisible target from a missing one. + return { + "result": "failed", + "reporting_receipt_id": item["reporting_receipt_id"], + "errors": [{"code": error.code, "message": "reporting receipt could not be recorded"}], + } diff --git a/src/adcp/reporting/receipts/required_schema.json b/src/adcp/reporting/receipts/required_schema.json new file mode 100644 index 000000000..3ec4c8a1f --- /dev/null +++ b/src/adcp/reporting/receipts/required_schema.json @@ -0,0 +1,410 @@ +{ + "column:reporting_receipt_ingestion_batches.account_id": { + "enabled": true, + "fingerprint": "fdc4b549138bf90a5af5ed00b87a86ff9a0cbea553e7f017b4d95510708f6f88" + }, + "column:reporting_receipt_ingestion_batches.admission_epoch": { + "enabled": true, + "fingerprint": "42202005517b72e082eb22c9eceb2ac0252815e5df700c83eb50c54cfeb46297" + }, + "column:reporting_receipt_ingestion_batches.canonical_request": { + "enabled": true, + "fingerprint": "10ccaa0dc3b93d48a1f32c7ef2352a11676632e1e871d77ef9ef4393eea15d27" + }, + "column:reporting_receipt_ingestion_batches.consumer_id": { + "enabled": true, + "fingerprint": "fdc4b549138bf90a5af5ed00b87a86ff9a0cbea553e7f017b4d95510708f6f88" + }, + "column:reporting_receipt_ingestion_batches.created_at": { + "enabled": true, + "fingerprint": "336df3243b293695d8321965e92f26d3f132e11514ed9678fb5e64e0015fa580" + }, + "column:reporting_receipt_ingestion_batches.expected_count": { + "enabled": true, + "fingerprint": "64be57437fdc0a07a97985c2aa058031f8082db7251bdb4d5afa1a9b088de97a" + }, + "column:reporting_receipt_ingestion_batches.final_response": { + "enabled": true, + "fingerprint": "1f539b84d6adb1a8577add320aabb81fdd5c614ad01b46c7b77eb55b995a6556" + }, + "column:reporting_receipt_ingestion_batches.final_sha256": { + "enabled": true, + "fingerprint": "85b88fb407e112399f25b5dc8830dfdcb2a8271ecf00e70b662f6917b4a002cd" + }, + "column:reporting_receipt_ingestion_batches.finalized_at": { + "enabled": true, + "fingerprint": "6f1466ce5d0aaac8471e39834b9c4b1f85d6f7169d9238a245ac035ff518e0fc" + }, + "column:reporting_receipt_ingestion_batches.idempotency_key": { + "enabled": true, + "fingerprint": "fdc4b549138bf90a5af5ed00b87a86ff9a0cbea553e7f017b4d95510708f6f88" + }, + "column:reporting_receipt_ingestion_batches.request_sha256": { + "enabled": true, + "fingerprint": "fdc4b549138bf90a5af5ed00b87a86ff9a0cbea553e7f017b4d95510708f6f88" + }, + "column:reporting_receipt_ingestion_boundaries.account_id": { + "enabled": true, + "fingerprint": "fdc4b549138bf90a5af5ed00b87a86ff9a0cbea553e7f017b4d95510708f6f88" + }, + "column:reporting_receipt_ingestion_boundaries.account_sequence": { + "enabled": true, + "fingerprint": "54a5f9962598caa8723435f8b55bd5e15823053af5dd63d531663997537dc6ba" + }, + "column:reporting_receipt_ingestion_boundaries.as_of": { + "enabled": true, + "fingerprint": "1cac4e73af11a8ecafd646ef6a0ff6ecb087d46f150408dcfebc630fe1bf5e1e" + }, + "column:reporting_receipt_ingestion_boundaries.consumer_id": { + "enabled": true, + "fingerprint": "fdc4b549138bf90a5af5ed00b87a86ff9a0cbea553e7f017b4d95510708f6f88" + }, + "column:reporting_receipt_ingestion_boundaries.content_sha256": { + "enabled": true, + "fingerprint": "fdc4b549138bf90a5af5ed00b87a86ff9a0cbea553e7f017b4d95510708f6f88" + }, + "column:reporting_receipt_ingestion_boundaries.input": { + "enabled": true, + "fingerprint": "ac355fc16c02b70cb0a24afee8214cdce5f5cbfdc7fd1630786d5101932ecfa4" + }, + "column:reporting_receipt_ingestion_boundaries.receipt_namespace": { + "enabled": true, + "fingerprint": "3c136011bb3ec377d8e534e35c5e0e338531ff0a5cae3e41945a2eec630c09e6" + }, + "column:reporting_receipt_ingestion_boundaries.reporting_receipt_id": { + "enabled": true, + "fingerprint": "fdc4b549138bf90a5af5ed00b87a86ff9a0cbea553e7f017b4d95510708f6f88" + }, + "column:reporting_receipt_ingestion_boundaries.sequence": { + "enabled": true, + "fingerprint": "54a5f9962598caa8723435f8b55bd5e15823053af5dd63d531663997537dc6ba" + }, + "column:reporting_receipt_ingestion_heads.account_id": { + "enabled": true, + "fingerprint": "fdc4b549138bf90a5af5ed00b87a86ff9a0cbea553e7f017b4d95510708f6f88" + }, + "column:reporting_receipt_ingestion_heads.consumer_id": { + "enabled": true, + "fingerprint": "fdc4b549138bf90a5af5ed00b87a86ff9a0cbea553e7f017b4d95510708f6f88" + }, + "column:reporting_receipt_ingestion_heads.max_sequence": { + "enabled": true, + "fingerprint": "54a5f9962598caa8723435f8b55bd5e15823053af5dd63d531663997537dc6ba" + }, + "column:reporting_receipt_ingestion_results.account_id": { + "enabled": true, + "fingerprint": "fdc4b549138bf90a5af5ed00b87a86ff9a0cbea553e7f017b4d95510708f6f88" + }, + "column:reporting_receipt_ingestion_results.consumer_id": { + "enabled": true, + "fingerprint": "fdc4b549138bf90a5af5ed00b87a86ff9a0cbea553e7f017b4d95510708f6f88" + }, + "column:reporting_receipt_ingestion_results.content_sha256": { + "enabled": true, + "fingerprint": "fdc4b549138bf90a5af5ed00b87a86ff9a0cbea553e7f017b4d95510708f6f88" + }, + "column:reporting_receipt_ingestion_results.idempotency_key": { + "enabled": true, + "fingerprint": "fdc4b549138bf90a5af5ed00b87a86ff9a0cbea553e7f017b4d95510708f6f88" + }, + "column:reporting_receipt_ingestion_results.ordinal": { + "enabled": true, + "fingerprint": "64be57437fdc0a07a97985c2aa058031f8082db7251bdb4d5afa1a9b088de97a" + }, + "column:reporting_receipt_ingestion_results.receipt_kind": { + "enabled": true, + "fingerprint": "10ccaa0dc3b93d48a1f32c7ef2352a11676632e1e871d77ef9ef4393eea15d27" + }, + "column:reporting_receipt_ingestion_results.receipt_namespace": { + "enabled": true, + "fingerprint": "3c136011bb3ec377d8e534e35c5e0e338531ff0a5cae3e41945a2eec630c09e6" + }, + "column:reporting_receipt_ingestion_results.receipt_record_id": { + "enabled": true, + "fingerprint": "85b88fb407e112399f25b5dc8830dfdcb2a8271ecf00e70b662f6917b4a002cd" + }, + "column:reporting_receipt_ingestion_results.recorded_at": { + "enabled": true, + "fingerprint": "336df3243b293695d8321965e92f26d3f132e11514ed9678fb5e64e0015fa580" + }, + "column:reporting_receipt_ingestion_results.reporting_receipt_id": { + "enabled": true, + "fingerprint": "fdc4b549138bf90a5af5ed00b87a86ff9a0cbea553e7f017b4d95510708f6f88" + }, + "column:reporting_receipt_ingestion_results.result": { + "enabled": true, + "fingerprint": "ac355fc16c02b70cb0a24afee8214cdce5f5cbfdc7fd1630786d5101932ecfa4" + }, + "constraint:reporting_receipt_ingestion_batches.reporting_receipt_ingestion_batches_admission_epoch_check": { + "enabled": true, + "fingerprint": "abd3e698b241244e9e751e940838ef5bcecdd79d21aab371f4b595818b5b1f95" + }, + "constraint:reporting_receipt_ingestion_batches.reporting_receipt_ingestion_batches_canonical_request_check": { + "enabled": true, + "fingerprint": "912af24f35b588c366cb2e7520d612baab36ec0e91e5b2489a8a8ef4fc3e2d68" + }, + "constraint:reporting_receipt_ingestion_batches.reporting_receipt_ingestion_batches_check": { + "enabled": true, + "fingerprint": "5fcb92d5ab815be1be7ba3958714ac48a157ee8e3780601e1807108704249a12" + }, + "constraint:reporting_receipt_ingestion_batches.reporting_receipt_ingestion_batches_check1": { + "enabled": true, + "fingerprint": "66f19a2cc809660aef7b568e84764eb65c35964a4523a6c245afc61e90678500" + }, + "constraint:reporting_receipt_ingestion_batches.reporting_receipt_ingestion_batches_check2": { + "enabled": true, + "fingerprint": "927d933d82230e62d52dd8e37bdc6a1879f7765f2060d9c9fcd3ad39494744c8" + }, + "constraint:reporting_receipt_ingestion_batches.reporting_receipt_ingestion_batches_check3": { + "enabled": true, + "fingerprint": "6540b709a9e52537489a92313773f473f9ef7e1e1d9d59cebeaf0490a49006f1" + }, + "constraint:reporting_receipt_ingestion_batches.reporting_receipt_ingestion_batches_check4": { + "enabled": true, + "fingerprint": "47d0a4b1d9cd02cfd02f0e8d88e2010a6f7c70c9d1a59ed6eedfd283e3e4ea09" + }, + "constraint:reporting_receipt_ingestion_batches.reporting_receipt_ingestion_batches_check5": { + "enabled": true, + "fingerprint": "1b2547b965ab254cabcecf3edab1bdf0c65848b46c52a9c0268629f5a5f1f44f" + }, + "constraint:reporting_receipt_ingestion_batches.reporting_receipt_ingestion_batches_expected_count_check": { + "enabled": true, + "fingerprint": "723d4bcaf0cee6c6aa069b6ed53e19524ba88a12da5a0c4de7809ee446b5bb6e" + }, + "constraint:reporting_receipt_ingestion_batches.reporting_receipt_ingestion_batches_idempotency_key_check": { + "enabled": true, + "fingerprint": "bb45ae8ff7c580eb4c52bca8b48aae27441c134868327a8627baf671e651dc6d" + }, + "constraint:reporting_receipt_ingestion_batches.reporting_receipt_ingestion_batches_pkey": { + "enabled": true, + "fingerprint": "bd14bccb4c52b10981823f13e78f2d623d0f82b2d8103fe009b0ead4e7c4d7ed" + }, + "constraint:reporting_receipt_ingestion_batches.reporting_receipt_ingestion_batches_request_sha256_check": { + "enabled": true, + "fingerprint": "e4296548e9e39b78a6b347016c0bccde754cd6a0afbbe2ed963a125a1bca1918" + }, + "constraint:reporting_receipt_ingestion_boundaries.reporting_receipt_ingestion_b_account_id_consumer_id_recei_fkey": { + "enabled": true, + "fingerprint": "e7f6d3026b2a48db5725fedb3296c4ea22741e47af12b0c59e30bb4e29f6e2ce" + }, + "constraint:reporting_receipt_ingestion_boundaries.reporting_receipt_ingestion_b_account_id_consumer_id_report_key": { + "enabled": true, + "fingerprint": "4da8231cf59acee96ee91b4f06240f128843063504be0c21755ed8748bfbc403" + }, + "constraint:reporting_receipt_ingestion_boundaries.reporting_receipt_ingestion_bou_account_id_account_sequence_key": { + "enabled": true, + "fingerprint": "460b11d8840751c3e894245e33c24e704407f4b3ca42bbd9e832cb48a0c3b24d" + }, + "constraint:reporting_receipt_ingestion_boundaries.reporting_receipt_ingestion_boundaries_account_sequence_check": { + "enabled": true, + "fingerprint": "1a64697a81e3b39433d851f9916df1964cbbc260aa54dd6fee4076c7e805c70e" + }, + "constraint:reporting_receipt_ingestion_boundaries.reporting_receipt_ingestion_boundaries_check": { + "enabled": true, + "fingerprint": "88c2bcdd52446d3f8a7c5e21b16f277d231d44a17842e8c625d02e31cc16dd22" + }, + "constraint:reporting_receipt_ingestion_boundaries.reporting_receipt_ingestion_boundaries_check1": { + "enabled": true, + "fingerprint": "6db9e761fce16f6deaabe04581b3813f3495382b5edf905c2c01a772151eed89" + }, + "constraint:reporting_receipt_ingestion_boundaries.reporting_receipt_ingestion_boundaries_check2": { + "enabled": true, + "fingerprint": "6c5865ad0c28d00ec74194418def9d764ddf85b230f1dfb6d14c6e505eda3737" + }, + "constraint:reporting_receipt_ingestion_boundaries.reporting_receipt_ingestion_boundaries_check3": { + "enabled": true, + "fingerprint": "53c1080ff5edc81171541e32d6cdaf69dfa2376bd4cc5bb31774ab2d4b773ff2" + }, + "constraint:reporting_receipt_ingestion_boundaries.reporting_receipt_ingestion_boundaries_check4": { + "enabled": true, + "fingerprint": "65d0dca50d7ca308123252fcc784fb427fd857d8bf3092bd85fc8fff5787d7ec" + }, + "constraint:reporting_receipt_ingestion_boundaries.reporting_receipt_ingestion_boundaries_check5": { + "enabled": true, + "fingerprint": "0d806a2b956a1e8947dcc43c634352ed52c1c3f158235145b2fe5a96db5fb379" + }, + "constraint:reporting_receipt_ingestion_boundaries.reporting_receipt_ingestion_boundaries_check6": { + "enabled": true, + "fingerprint": "b330c171bc0bb87ac12aa97aca4d4ae7167e0c07d7b61b3b688c63df9588d927" + }, + "constraint:reporting_receipt_ingestion_boundaries.reporting_receipt_ingestion_boundaries_input_check": { + "enabled": true, + "fingerprint": "1403f246fdde17015118d212847ed5d8169df3adc0364598cbd5e82be066e1bf" + }, + "constraint:reporting_receipt_ingestion_boundaries.reporting_receipt_ingestion_boundaries_input_check1": { + "enabled": true, + "fingerprint": "7e7cefdce39915938ec708771cbe2b61b774ad362d41d2d3ef2a7896c9baa177" + }, + "constraint:reporting_receipt_ingestion_boundaries.reporting_receipt_ingestion_boundaries_input_check2": { + "enabled": true, + "fingerprint": "908f04e9bfa67eef862e12260105785952531a776178a8b57ec271ff21b3aec5" + }, + "constraint:reporting_receipt_ingestion_boundaries.reporting_receipt_ingestion_boundaries_pkey": { + "enabled": true, + "fingerprint": "00dd5f9ba4e0d0face7ab27fe5e52ffa9c4101943c95afb8ec1badb401c7010d" + }, + "constraint:reporting_receipt_ingestion_boundaries.reporting_receipt_ingestion_boundaries_receipt_namespace_check": { + "enabled": true, + "fingerprint": "2c7db1fb7a2ce91e34550c4a9925c8dadadc94ef57d8e020e4ba8d7875c94b15" + }, + "constraint:reporting_receipt_ingestion_boundaries.reporting_receipt_ingestion_boundaries_sequence_check": { + "enabled": true, + "fingerprint": "554d491362648e795a6567528a6244977d5df276216297f35e26e4750f869dcf" + }, + "constraint:reporting_receipt_ingestion_heads.reporting_receipt_ingestion_heads_max_sequence_check": { + "enabled": true, + "fingerprint": "ee47acec8d450eed7f99b637596847d21bc449a12317a605fd6b6cb0fad6aa79" + }, + "constraint:reporting_receipt_ingestion_heads.reporting_receipt_ingestion_heads_pkey": { + "enabled": true, + "fingerprint": "4d853add149814edf9eadd3f752bf43f1f7164cba2bd079f0349d4d540a7f20c" + }, + "constraint:reporting_receipt_ingestion_results.reporting_receipt_ingestion_r_account_id_consumer_id_idemp_fkey": { + "enabled": true, + "fingerprint": "b95192b8aa7ab2d51fc9ba9dc1c88ee5c32d39da4725efb06cfb67b494588383" + }, + "constraint:reporting_receipt_ingestion_results.reporting_receipt_ingestion_r_account_id_consumer_id_idempo_key": { + "enabled": true, + "fingerprint": "3e31e8cc82fc1bc507bc26bd28159937ac3f8ab8fc55a9926fc476568e3e49b1" + }, + "constraint:reporting_receipt_ingestion_results.reporting_receipt_ingestion_r_account_id_consumer_id_recei_fkey": { + "enabled": true, + "fingerprint": "9af695720cddd0dcbc5cff566319d54a58086960a9ab861d8908b3d7a43db762" + }, + "constraint:reporting_receipt_ingestion_results.reporting_receipt_ingestion_results_check": { + "enabled": true, + "fingerprint": "e665b17e336517d76d15cf779c34c687a7ebb443ccb89e1fa2ae90cbb74120d7" + }, + "constraint:reporting_receipt_ingestion_results.reporting_receipt_ingestion_results_check1": { + "enabled": true, + "fingerprint": "3c812b76d750d500699370919d0e256db83181eab776d254f897a5f68fb5ff96" + }, + "constraint:reporting_receipt_ingestion_results.reporting_receipt_ingestion_results_check2": { + "enabled": true, + "fingerprint": "af7466439087470d4590ea91c3d7d7de23bf47af1424ed14c254c96727784149" + }, + "constraint:reporting_receipt_ingestion_results.reporting_receipt_ingestion_results_ordinal_check": { + "enabled": true, + "fingerprint": "681e853666470c336498661c1ed30b8e5ed60024aa72dead813393d62eca83ce" + }, + "constraint:reporting_receipt_ingestion_results.reporting_receipt_ingestion_results_pkey": { + "enabled": true, + "fingerprint": "330a35cc748968f52093f6d1707fbcf4ac136f7d76327874eedb54000f2bb793" + }, + "constraint:reporting_receipt_ingestion_results.reporting_receipt_ingestion_results_receipt_kind_check": { + "enabled": true, + "fingerprint": "807349018715c3cfc653a1889df10225848b212a749d4977dfb37f4799674474" + }, + "constraint:reporting_receipt_ingestion_results.reporting_receipt_ingestion_results_receipt_namespace_check": { + "enabled": true, + "fingerprint": "2c7db1fb7a2ce91e34550c4a9925c8dadadc94ef57d8e020e4ba8d7875c94b15" + }, + "constraint:reporting_receipt_ingestion_results.reporting_receipt_ingestion_results_reporting_receipt_id_check": { + "enabled": true, + "fingerprint": "db3e198b5d35078fd20db3918eca7084516ec6eb8d49efa1a6eec5a0a744c6f2" + }, + "constraint:reporting_receipt_ingestion_results.reporting_receipt_ingestion_results_result_check": { + "enabled": true, + "fingerprint": "c57c0272ed7fd01b0fbb2972e8d5144bbf4566e21a3fda58d7d521dcecdf499a" + }, + "function:reporting_receipt_ingestion_batch_guard()": { + "enabled": true, + "fingerprint": "ebaa3104d98c18b32d36b7af443877f80d2a6ea4d564be68053ad937ba80b4b3" + }, + "function:reporting_receipt_ingestion_canonical(document jsonb)": { + "enabled": true, + "fingerprint": "83894ca562527560f07f5f8e27e4bccf728e851f5f12652ebc82f8f04630ad3e" + }, + "function:reporting_receipt_ingestion_graph()": { + "enabled": true, + "fingerprint": "570d827fe99bcd57d5f2614c0ed146d5d9cbd51d0c816ee6262cbdff140ab5f7" + }, + "function:reporting_receipt_ingestion_immutable()": { + "enabled": true, + "fingerprint": "099d3850fab91c795d5516e3e99066e1d02598a71e5d71e8dee9c69d998abb57" + }, + "function:reporting_receipt_ingestion_result_guard()": { + "enabled": true, + "fingerprint": "628db026db8a5a70ccd3e9c66c3b5ef344c80da2a662cc97c488a103dac3dc4c" + }, + "function:reporting_receipt_ingestion_sha256(document jsonb)": { + "enabled": true, + "fingerprint": "6f95f80903a080155fd6ecc522fdc99a6f656914f3973b44b1cb46a3c393f228" + }, + "function:reporting_receipt_ingestion_utf16(value text)": { + "enabled": true, + "fingerprint": "e136f3cbdb9aed5e8d48defef32a573c6d4b26a3f52a952022d1246adc0dffca" + }, + "function:reporting_receipt_ingestion_wire(document jsonb)": { + "enabled": true, + "fingerprint": "1a01db876edf8d3f077375aa09edc56c1feb5ab658d7a5aa3cdb17c461675cb0" + }, + "index:reporting_receipt_ingestion_batches.reporting_receipt_ingestion_batches_pkey": { + "enabled": true, + "fingerprint": "65c0ec48dceaace248f672b59680b283319529a1fd41125a15a96733092c5387" + }, + "index:reporting_receipt_ingestion_boundaries.reporting_receipt_ingestion_b_account_id_consumer_id_report_key": { + "enabled": true, + "fingerprint": "54844be45542b67fafaef6dd517b51e5a7147877035e47f711af914b90422f26" + }, + "index:reporting_receipt_ingestion_boundaries.reporting_receipt_ingestion_bou_account_id_account_sequence_key": { + "enabled": true, + "fingerprint": "e6c0904fc74d63aef2188c36ee5788d297d8f6175a0d10d5fe658a8d8030cbe3" + }, + "index:reporting_receipt_ingestion_boundaries.reporting_receipt_ingestion_boundaries_pkey": { + "enabled": true, + "fingerprint": "fe4b6f5a34e2c778f77138231b76fd27e8ee56639e357293b9f4700553db0da7" + }, + "index:reporting_receipt_ingestion_heads.reporting_receipt_ingestion_heads_pkey": { + "enabled": true, + "fingerprint": "a6fbc55d5bd3cc94f0cfdb3c3ea72c1611cccb2aeccbd67c0576076149dd2804" + }, + "index:reporting_receipt_ingestion_results.reporting_receipt_ingestion_r_account_id_consumer_id_idempo_key": { + "enabled": true, + "fingerprint": "bb57e83db85651fd68160cc0b437f0a730a4587d9f7f832b7b10c1e6141b0281" + }, + "index:reporting_receipt_ingestion_results.reporting_receipt_ingestion_results_pkey": { + "enabled": true, + "fingerprint": "16d6c560b7bb8eb6e36ae8f59aaff25aa8f2efb906f2f6ce9795c4a753f8baff" + }, + "index:reporting_reconciliation_records.reporting_receipt_ingestion_chain": { + "enabled": true, + "fingerprint": "956676e509d0b5c196f9abaa090eb4a57e8b764838572951105b9096b68b36b1" + }, + "table:reporting_receipt_ingestion_batches": { + "enabled": true, + "fingerprint": "1f824779ff80f110344420b019786663d8c9beaad230da90e0439795e734ccda" + }, + "table:reporting_receipt_ingestion_boundaries": { + "enabled": true, + "fingerprint": "1f824779ff80f110344420b019786663d8c9beaad230da90e0439795e734ccda" + }, + "table:reporting_receipt_ingestion_heads": { + "enabled": true, + "fingerprint": "1f824779ff80f110344420b019786663d8c9beaad230da90e0439795e734ccda" + }, + "table:reporting_receipt_ingestion_results": { + "enabled": true, + "fingerprint": "1f824779ff80f110344420b019786663d8c9beaad230da90e0439795e734ccda" + }, + "trigger:reporting_receipt_ingestion_batches.reporting_receipt_ingestion_batch": { + "enabled": true, + "fingerprint": "d5c809b674a594c41801296da98b81ba108bd08414bfc3bb711b266a7d589ef0" + }, + "trigger:reporting_receipt_ingestion_boundaries.reporting_receipt_ingestion_boundary_immutable": { + "enabled": true, + "fingerprint": "1ccb241ae1c0a4a467c64cc246d1ae159f5c78621b7b5a549f862e4b808958d1" + }, + "trigger:reporting_receipt_ingestion_results.reporting_receipt_ingestion_result": { + "enabled": true, + "fingerprint": "053262f24d8e8762ff76615ce66db34be3403992d0aeec598c27ee709ee5835a" + }, + "trigger:reporting_receipt_ingestion_results.reporting_receipt_ingestion_result_immutable": { + "enabled": true, + "fingerprint": "3e71c52e6576becdc89161b72a6b986f83e0301400e9049ed88a64d3b1889a56" + }, + "trigger:reporting_reconciliation_records.reporting_receipt_ingestion_graph": { + "enabled": true, + "fingerprint": "05a5ce8a48fb20c3682b8baede61ad9a2391e06908a91a6148760eec5550f685" + } +} diff --git a/src/adcp/reporting/receipts/schema.py b/src/adcp/reporting/receipts/schema.py new file mode 100644 index 000000000..cb874c93e --- /dev/null +++ b/src/adcp/reporting/receipts/schema.py @@ -0,0 +1,27 @@ +"""Isolated ingestion readiness; never widens an A/B/C/B1/B2.1 manifest.""" + +from __future__ import annotations + +import json +from importlib.resources import files +from typing import Any + +from adcp.reporting.materializer.schema import validate_materializer_schema +from adcp.reporting.outbox._schema import schema_objects +from adcp.reporting.receipts.errors import ReportingReceiptError + + +async def validate_receipt_schema(connection: Any, *, notifications: bool = False) -> None: + ready = False + try: + required = json.loads( + files("adcp.reporting.receipts").joinpath("required_schema.json").read_text() + ) + installed = await schema_objects(connection) + if required and all(installed.get(key) == expected for key, expected in required.items()): + await validate_materializer_schema(connection, notifications=notifications) + ready = True + except Exception: + ready = False + if not ready: + raise ReportingReceiptError("RECEIPT_SCHEMA_UNREADY") diff --git a/src/adcp/reporting/receipts/store.py b/src/adcp/reporting/receipts/store.py new file mode 100644 index 000000000..a163893f4 --- /dev/null +++ b/src/adcp/reporting/receipts/store.py @@ -0,0 +1,24 @@ +"""Optional batch participant; no change to any required legacy store protocol.""" + +from __future__ import annotations + +from typing import Any, Protocol, runtime_checkable + +from adcp.reporting.ledger.delivery_models import ReportingDeliveryPrincipal +from adcp.reporting.receipts.capture import ReportingReceiptBoundary + + +@runtime_checkable +class ReportingReceiptBatchStore(Protocol): + """Trusted callers only. Each ordinal co-commits with all of its domain effects.""" + + async def ingest_receipt_batch( + self, request: dict[str, Any], *, caller: ReportingDeliveryPrincipal + ) -> dict[str, Any]: ... + + +@runtime_checkable +class ReportingReceiptCaptureStore(Protocol): + async def read_receipt_boundaries( + self, *, caller: ReportingDeliveryPrincipal, after: int = 0, limit: int = 100 + ) -> tuple[ReportingReceiptBoundary, ...]: ... diff --git a/src/adcp/reporting/receipts/transport.py b/src/adcp/reporting/receipts/transport.py new file mode 100644 index 000000000..6fe8b0109 --- /dev/null +++ b/src/adcp/reporting/receipts/transport.py @@ -0,0 +1,195 @@ +"""Request-local lossless receipt JSON, before transport/model normalization.""" + +from __future__ import annotations + +import json +from collections.abc import Awaitable, Callable +from decimal import Decimal, InvalidOperation +from typing import Any + +from adcp.reporting.canonical_json import MAX_SAFE_INTEGER +from adcp.reporting.receipts.wire import TASK + +# The standard serve() body limiter applies first. Direct mounts also bound +# this additional receipt-only capture; a larger body is rejected, never rounded. +MAX_RECEIPT_BODY_BYTES = 10 * 1024 * 1024 +RAW_RECEIPT_BODY_SCOPE_KEY = "adcp.receipt_ingress.raw_body" + + +def receipt_body_receive( + scope: dict[str, Any], + receive: Callable[[], Awaitable[Any]], + *, + limit: int = MAX_RECEIPT_BODY_BYTES, +) -> Callable[[], Awaitable[Any]]: + """Tee only the bytes delivered to this mounted HTTP request's decoder. + + No caller metadata, global cache, request ID lookup or auth shortcut is + involved. The existing upstream body cap still applies first. + """ + captured: bytearray | None = bytearray() + + async def capture() -> Any: + nonlocal captured + message = await receive() + if message.get("type") == "http.request": + chunk = message.get("body", b"") + if captured is not None: + if len(captured) + len(chunk) > limit: + captured = None + else: + captured.extend(chunk) + if not message.get("more_body", False): + scope[RAW_RECEIPT_BODY_SCOPE_KEY] = ( + bytes(captured) if captured is not None else None + ) + return message + + return capture + + +def _unique_object(pairs: list[tuple[str, Any]]) -> dict[str, Any]: + result: dict[str, Any] = {} + for key, value in pairs: + if key in result: + raise ValueError("ambiguous JSON object") + result[key] = value + return result + + +def _exact_integer(value: Decimal) -> int: + """Accept protobuf's 1.0 spelling only when its *raw text* is an exact integer. + + A binary float cannot establish this: 1.000000000000000000001 has already + rounded to 1.0 there. Decimal examines the authenticated HTTP bytes without + rounding and bounds the value before any potentially large integer allocation. + """ + try: + if ( + value.is_finite() + and value.copy_abs() <= MAX_SAFE_INTEGER + and value == value.to_integral_value() + ): + return int(value) + except (InvalidOperation, ValueError, OverflowError): + pass + raise ValueError("receipt JSON requires exact safe integers") + + +def _receipt_numbers(value: Any) -> Any: + if isinstance(value, Decimal): + return _exact_integer(value) + if type(value) is dict: + return {key: _receipt_numbers(item) for key, item in value.items()} + if type(value) is list: + return [_receipt_numbers(item) for item in value] + return value + + +def _raw_number(raw: str) -> Decimal: + try: + return Decimal(raw) + except InvalidOperation: + # Preserve route identification even for an unrepresentable exponent. + # This sentinel can only produce an invalid empty request below; it is + # never handed to a handler as an accepted numeric value. + return Decimal("NaN") + + +def _raw_json(body: str | bytes) -> Any: + return json.loads( + body, + object_pairs_hook=_unique_object, + parse_float=_raw_number, + parse_constant=_raw_number, + ) + + +def _a2a_receipt_invocation(body: bytes | None) -> dict[str, Any] | None: + try: + if body is None: + return None + envelope = _raw_json(body) + if ( + type(envelope) is not dict + or envelope.get("jsonrpc") != "2.0" + or envelope.get("method") + not in {"message/send", "message/stream", "SendMessage", "SendStreamingMessage"} + ): + return None + parts = envelope["params"]["message"]["parts"] + invocations = [] + for part in parts: + if type(part) is not dict: + return None + data = part.get("data") + if data is None and isinstance(part.get("text"), str): + try: + data = _raw_json(part["text"]) + except ValueError: + continue + if type(data) is dict and data.get("skill"): + invocations.append(data) + if len(invocations) == 1 and invocations[0]["skill"] == TASK: + return dict(invocations[0]) + except (ValueError, TypeError, KeyError, RecursionError): + pass + return None + + +def a2a_receipt_parameters(body: bytes | None) -> dict[str, Any] | None: + """Distinguish an exact receipt route from invalid receipt parameters. + + None means no uniquely identified standard invocation. An empty dictionary + means that invocation's parameters are invalid and must reach the ordinary + whole-shape rejection. Neither case can authorize a domain write. + """ + invocation = _a2a_receipt_invocation(body) + if invocation is None: + return None + try: + params = invocation.get("parameters") + if type(params) is dict: + return dict(_receipt_numbers(params)) + except (ValueError, TypeError, RecursionError): + pass + return {} + + +def a2a_receipt_has_invalid_unicode(body: bytes | None) -> bool: + """Scope the upstream protobuf codec error boundary to this exact raw route.""" + if body is None or _a2a_receipt_invocation(body) is None: + return False + try: + pending = [_raw_json(body)] + while pending: + value = pending.pop() + if isinstance(value, str) and any(0xD800 <= ord(c) <= 0xDFFF for c in value): + return True + if type(value) is dict: + pending.extend(value) + pending.extend(value.values()) + elif type(value) is list: + pending.extend(value) + except (ValueError, TypeError, RecursionError): + pass + return False + + +def mcp_receipt_parameters(body: bytes | None) -> dict[str, Any]: + """The already selected tools/call must match this exact receipt invocation.""" + try: + if body is None: + return {} + envelope = _raw_json(body) + if ( + type(envelope) is dict + and envelope.get("jsonrpc") == "2.0" + and envelope.get("method") == "tools/call" + and envelope["params"]["name"] == TASK + and type(envelope["params"].get("arguments")) is dict + ): + return dict(_receipt_numbers(envelope["params"]["arguments"])) + except (ValueError, TypeError, KeyError, RecursionError): + pass + return {} diff --git a/src/adcp/reporting/receipts/wire.py b/src/adcp/reporting/receipts/wire.py new file mode 100644 index 000000000..db0b9f065 --- /dev/null +++ b/src/adcp/reporting/receipts/wire.py @@ -0,0 +1,189 @@ +"""Lossless raw batch admission and isolated transport-schema overlays. + +Never edit cached upstream schemas or generated models. The portable canonical +schema retains conditionals omitted by model generation and compact MCP profiles. +""" + +from __future__ import annotations + +import hashlib +import json +from dataclasses import dataclass, field +from functools import lru_cache +from typing import Any, Literal, cast + +from jsonschema import Draft7Validator, FormatChecker + +from adcp.reporting.canonical_json import canonical_json_utf8_v1 +from adcp.reporting.receipts.errors import ReportingReceiptError +from adcp.validation.schema_loader import get_portable_schema + +TASK = "sync_reporting_receipts" +ReceiptKind = Literal["revision_receipt", "adjustment_receipt"] +_IDENTITY_FIELDS = frozenset( + { + "consumer", + "consumer_id", + "principal", + "principal_id", + "buyer", + "buyer_agent", + "buyer_agent_id", + "governance_agent", + "governance_principal", + "caller_identity", + "tenant_id", + "auth_info", + "auth_principal", + "agent_url", + } +) + + +def receipt_schema( + direction: Literal["request", "sync"], *, version: str | None = None +) -> dict[str, Any]: + """Independent mutable schema for pinned, unpinned and model-fallback mounts.""" + schema = get_portable_schema(TASK, direction, version=version) + if schema is None: + raise ReportingReceiptError("RECEIPT_SCHEMA_UNREADY") + if direction == "request": + schema["anyOf"] = [{"required": ["receipts"]}, {"required": ["adjustment_receipts"]}] + rules = schema.setdefault("allOf", []) + rules.append( + {"not": {"anyOf": [{"required": [name]} for name in sorted(_IDENTITY_FIELDS)]}} + ) + for name in ("receipts", "adjustment_receipts"): + array = schema["properties"][name] + array.update(minItems=1, maxItems=100) + array["items"] = {"allOf": [array["items"], {"not": {"required": ["received_at"]}}]} + # JSON Schema has no cross-array sum operator. These bounded implications + # express the exact combined limit without a proprietary validator keyword. + rules.extend( + { + "if": {"required": ["receipts"], "properties": {"receipts": {"minItems": n}}}, + "then": {"properties": {"adjustment_receipts": {"maxItems": 100 - n}}}, + } + for n in range(1, 101) + ) + return schema + + +@lru_cache(maxsize=2) +def _validator(direction: Literal["request", "sync"]) -> Any: + return Draft7Validator(receipt_schema(direction), format_checker=FormatChecker()) + + +def validate_receipt_request(request: object) -> None: + """Preflight the entire supplied shape before auth-dependent lookup or writes.""" + valid = False + try: + if type(request) is dict: + supplied = [ + request[name] for name in ("receipts", "adjustment_receipts") if name in request + ] + if supplied and all(type(items) is list and items for items in supplied): + items = [item for array in supplied for item in array] + ids = [item.get("reporting_receipt_id") for item in items if type(item) is dict] + valid = ( + 1 <= len(items) <= 100 + and len(ids) == len(items) + and all(isinstance(i, str) for i in ids) + and len(set(ids)) == len(ids) + and not _IDENTITY_FIELDS.intersection(request) + and _validator("request").is_valid(request) + ) + if valid: + # Also reject non-JSON values and non-finite/lossy JCS numbers. + canonical_json_utf8_v1(request) + # JSONB requires Unicode scalar values without U+0000. + # Reject unsupported keys/values before a header on both + # stores, including strings inside arbitrary context/ext. + pending: list[Any] = [request] + while pending: + value = pending.pop() + if isinstance(value, str) and any( + c == "\x00" or 0xD800 <= ord(c) <= 0xDFFF for c in value + ): + raise ValueError("unsupported JSON string") + if type(value) is dict: + pending.extend(value) + pending.extend(value.values()) + elif type(value) is list: + pending.extend(value) + except (ValueError, TypeError, OverflowError, RecursionError): + valid = False + if not valid: + raise ReportingReceiptError("INVALID_REQUEST") + + +def validate_receipt_results( + results: object, batch: ReceiptBatch, *, complete: bool = False +) -> None: + """Validate every durable prefix before resuming any domain writes.""" + valid = False + try: + if type(results) is list and len(results) <= len(batch.items): + if not results and not complete: + return + valid = _validator("sync").is_valid({"status": "completed", "results": results}) + ids = [ + ( + result["reporting_receipt_id"] + if result["result"] == "failed" + else result.get("receipt", result.get("adjustment_receipt"))[ + "reporting_receipt_id" + ] + ) + for result in results + ] + expected = [item["reporting_receipt_id"] for _, item in batch.items] + valid = valid and ids == (expected if complete else expected[: len(results)]) + for (kind, _), result in zip(batch.items, results): + if result["result"] != "failed": + key = "receipt" if kind == "revision_receipt" else "adjustment_receipt" + valid = valid and key in result + except (ValueError, TypeError, KeyError): + valid = False + if not valid: + raise ReportingReceiptError("RECEIPT_HISTORY_CORRUPT") + + +def validate_receipt_response(response: object, batch: ReceiptBatch) -> None: + """Validate exact ID coverage/order as well as the final wire schema.""" + if type(response) is not dict or not _validator("sync").is_valid(response): + raise ReportingReceiptError("RECEIPT_HISTORY_CORRUPT") + validate_receipt_results(response["results"], batch, complete=True) + + +@dataclass(frozen=True) +class ReceiptBatch: + """Private immutable whole-request identity, including context and extensions.""" + + canonical_request: bytes = field(repr=False) + + @classmethod + def parse(cls, request: dict[str, Any]) -> ReceiptBatch: + validate_receipt_request(request) + return cls(canonical_json_utf8_v1(request)) + + @property + def request(self) -> dict[str, Any]: + return cast(dict[str, Any], json.loads(self.canonical_request)) + + @property + def key(self) -> str: + return str(self.request["idempotency_key"]) + + @property + def digest(self) -> str: + return hashlib.sha256(self.canonical_request).hexdigest() + + @property + def items(self) -> tuple[tuple[ReceiptKind, dict[str, Any]], ...]: + request = self.request + arrays: tuple[tuple[ReceiptKind, str], ...] = ( + ("revision_receipt", "receipts"), + ("adjustment_receipt", "adjustment_receipts"), + ) + return tuple((kind, item) for kind, name in arrays for item in request.get(name, [])) diff --git a/src/adcp/server/a2a_server.py b/src/adcp/server/a2a_server.py index 729921e3a..6b7602810 100644 --- a/src/adcp/server/a2a_server.py +++ b/src/adcp/server/a2a_server.py @@ -159,17 +159,73 @@ def _get_decisioning_adcp_error_types() -> tuple[type[BaseException], ...]: class _A2ARequestContextMiddleware: """Make the originating HTTP request available during A2A dispatch.""" - def __init__(self, app: Any) -> None: + def __init__(self, app: Any, *, receipt_ingress: bool = False) -> None: self.app = app + self.receipt_ingress = receipt_ingress async def __call__(self, scope: Any, receive: Any, send: Any) -> None: if scope["type"] != "http": await self.app(scope, receive, send) return - token = _A2A_REQUEST_CONTEXT.set(Request(scope, receive=receive)) + downstream_receive = receive + if self.receipt_ingress: + from collections import deque + + from adcp.reporting.receipts.transport import ( + MAX_RECEIPT_BODY_BYTES, + RAW_RECEIPT_BODY_SCOPE_KEY, + a2a_receipt_has_invalid_unicode, + receipt_body_receive, + ) + + captured_receive = receipt_body_receive(scope, receive) + prefix = bytearray() + trailing = None + more_body = False + # Read ahead only to the bounded receipt capture limit, then replay + # the same bytes. Coalesce small chunks so one-byte/empty chunks + # cannot grow an unbounded queue of ASGI message dictionaries. + while True: + message = await captured_receive() + chunk = message.get("body", b"") + if ( + message.get("type") != "http.request" + or len(prefix) + len(chunk) > MAX_RECEIPT_BODY_BYTES + ): + trailing, more_body = message, True + break + prefix.extend(chunk) + more_body = message.get("more_body", False) + if not more_body: + break + buffered: deque[Any] = deque() + if prefix or trailing is None: + buffered.append( + {"type": "http.request", "body": bytes(prefix), "more_body": more_body} + ) + if trailing is not None: + buffered.append(trailing) + + async def replay_receive() -> Any: + return buffered.popleft() if buffered else await captured_receive() + + downstream_receive = replay_receive + if a2a_receipt_has_invalid_unicode(scope.get(RAW_RECEIPT_BODY_SCOPE_KEY)): + from starlette.responses import JSONResponse + + await JSONResponse( + { + "jsonrpc": "2.0", + "id": None, + "error": {"code": -32700, "message": "Invalid receipt JSON encoding"}, + }, + status_code=400, + )(scope, downstream_receive, send) + return + token = _A2A_REQUEST_CONTEXT.set(Request(scope, receive=downstream_receive)) try: - await self.app(scope, receive, send) + await self.app(scope, downstream_receive, send) finally: _A2A_REQUEST_CONTEXT.reset(token) @@ -558,13 +614,44 @@ def _parse_request(self, context: RequestContext) -> tuple[str | None, dict[str, JSON fallback). """ request = _A2A_REQUEST_CONTEXT.get() + parsed = None if request is not None: cached = request.scope.get(_A2A_PARSED_REQUEST_SCOPE_KEY) if cached is not None: - return cast(tuple[str | None, dict[str, Any]], cached) - if self._message_parser is not None: - return self._message_parser(context) - return self._default_parse_request(context) + parsed = cast(tuple[str | None, dict[str, Any]], cached) + if parsed is None: + if self._message_parser is not None: + parsed = self._message_parser(context) + else: + try: + parsed = self._default_parse_request(context) + except ValueError: + # The protobuf JSON printer refuses non-finite numbers. + # A standard raw receipt invocation still reaches mandatory + # strict preflight and a closed INVALID_REQUEST, without + # logging a protobuf serialization exception. + if request is not None: + from adcp.reporting.receipts.transport import ( + RAW_RECEIPT_BODY_SCOPE_KEY, + a2a_receipt_parameters, + ) + + raw = a2a_receipt_parameters(request.scope.get(RAW_RECEIPT_BODY_SCOPE_KEY)) + if raw is not None: + return "sync_reporting_receipts", raw + raise + if parsed[0] == "sync_reporting_receipts" and request is not None: + from adcp.reporting.receipts.transport import ( + RAW_RECEIPT_BODY_SCOPE_KEY, + a2a_receipt_parameters, + ) + + # The protobuf representation has already lost numeric lexemes. + # Only the standard raw invocation can authorize the batch body. + return parsed[0], ( + a2a_receipt_parameters(request.scope.get(RAW_RECEIPT_BODY_SCOPE_KEY)) or {} + ) + return parsed def _default_parse_request(self, context: RequestContext) -> tuple[str | None, dict[str, Any]]: """Built-in parser. Supports two formats: @@ -1419,7 +1506,10 @@ def agent_card_url(request: Request) -> str: # during executor dispatch. This is installed for direct # ``create_a2a_server`` adopters as well as the unified ``serve`` path, # independent of whether bearer-auth middleware is configured. - app.add_middleware(_A2ARequestContextMiddleware) + app.add_middleware( + _A2ARequestContextMiddleware, + receipt_ingress="sync_reporting_receipts" in executor.supported_skills, + ) # Startup log lives on the create_a2a_server path (symmetric with # MCP's _register_handler_tools). Moved out of diff --git a/src/adcp/server/idempotency/store.py b/src/adcp/server/idempotency/store.py index ebbee1f47..621b4e7d1 100644 --- a/src/adcp/server/idempotency/store.py +++ b/src/adcp/server/idempotency/store.py @@ -37,6 +37,7 @@ import weakref from collections.abc import AsyncIterator, Awaitable, Callable from contextlib import asynccontextmanager +from contextvars import ContextVar from functools import wraps from typing import Any @@ -52,6 +53,10 @@ logger = logging.getLogger(__name__) +# Set only by the SDK transport dispatcher, never from request metadata. This +# also covers middleware whose wrapped callable is named call_next/execute. +_RECEIPT_BATCH_DISPATCH: ContextVar[bool] = ContextVar("receipt_batch_dispatch", default=False) + # Registry of functions returned by IdempotencyStore.wrap. Read by # adcp.decisioning.validate_idempotency.is_wrapped() to reconcile the # adopter's declared IdempotencySupported capability against actual @@ -215,6 +220,15 @@ def wrap(self, handler: HandlerFn) -> HandlerFn: @wraps(handler) async def _wrapped(*args: Any, **kwargs: Any) -> Any: + if ( + _RECEIPT_BATCH_DISPATCH.get() + or getattr(handler, "__name__", None) == "sync_reporting_receipts" + ): + # Receipt ingress owns durable whole-batch replay after exact + # account/consumer authorization. This generic key omits the + # resolved account and cached hits bypass that authorization. + # It also adds replayed=True, changing the immutable response. + return await handler(*args, **kwargs) handler_self, hash_source, context = _resolve_call_args(args, kwargs) operation = getattr(handler, "__name__", "handler") diff --git a/src/adcp/server/mcp_tools.py b/src/adcp/server/mcp_tools.py index 716deb62a..ebb8cd421 100644 --- a/src/adcp/server/mcp_tools.py +++ b/src/adcp/server/mcp_tools.py @@ -540,9 +540,21 @@ def _widen_media_buy_output_schema_for_legacy_statuses( "properties": { "account": {"type": "object"}, "idempotency_key": {"type": "string"}, - "receipts": {"type": "array"}, + "receipts": { + "type": "array", + "minItems": 1, + "maxItems": 100, + "items": {"type": "object", "not": {"required": ["received_at"]}}, + }, + "adjustment_receipts": { + "type": "array", + "minItems": 1, + "maxItems": 100, + "items": {"type": "object", "not": {"required": ["received_at"]}}, + }, }, - "required": ["account", "idempotency_key", "receipts"], + "required": ["account", "idempotency_key"], + "anyOf": [{"required": ["receipts"]}, {"required": ["adjustment_receipts"]}], }, }, { @@ -2401,7 +2413,16 @@ def get_tools_for_handler( # The in-memory registry shares memoized schema subtrees to keep server # startup compact. Public definitions remain ordinary independently # mutable JSON values, matching the pre-memoization behavior. - return [_copy_json_without_aliases(tool) for tool in selected] + definitions = [_copy_json_without_aliases(tool) for tool in selected] + for definition in definitions: + if definition["name"] == "sync_reporting_receipts": + from adcp.reporting.receipts.wire import receipt_schema + + # Also runs when model generation fell back to hand-written + # stubs. Do not mutate cached upstream/generated schema inputs. + definition["inputSchema"] = receipt_schema("request") + definition["outputSchema"] = receipt_schema("sync") + return definitions if not list_validator_keys(version=resolved_version): raise ValueError( @@ -2419,6 +2440,13 @@ def get_tools_for_handler( if input_schema is None: continue definition = copy.deepcopy(tool) + if name == "sync_reporting_receipts": + from adcp.reporting.receipts.wire import receipt_schema + + definition["inputSchema"] = receipt_schema("request", version=resolved_version) + definition["outputSchema"] = receipt_schema("sync", version=resolved_version) + versioned.append(definition) + continue definition["inputSchema"] = input_schema output_schema = get_mcp_schema(name, "sync", version=resolved_version) if output_schema is not None: @@ -2745,11 +2773,48 @@ async def call_tool(params: dict[str, Any], context: ToolContext | None = None) raw_params = params # Preserve original wire params for context echo. + if method_name == "sync_reporting_receipts": + from adcp.reporting.receipts.errors import ReportingReceiptError + from adcp.reporting.receipts.wire import ReceiptBatch + + # Financial batches require the raw shape even with generic + # validation disabled. Coercion must not erase supplied empty + # arrays, read-only fields, timestamps, or request extensions. + try: + receipt_batch = ReceiptBatch.parse(raw_params) + raw_params = receipt_batch.request + except ReportingReceiptError as exc: + raise ADCPTaskError( + operation=method_name, errors=[Error(code=exc.code, message=str(exc))] + ) from None + if pre_validation_hooks: try: params = _apply_pre_validation_hooks( - pre_validation_hooks, method_name, dict(params) + pre_validation_hooks, + method_name, + ( + copy.deepcopy(params) + if method_name == "sync_reporting_receipts" + else dict(params) + ), ) + if method_name == "sync_reporting_receipts": + # A rewrite would change the identity the buyer authorized. + # Hooks may inspect this task; financial mutations fail closed. + try: + if ReceiptBatch.parse(params) != receipt_batch: + raise ReportingReceiptError("INVALID_REQUEST") + except ReportingReceiptError: + raise ADCPTaskError( + operation=method_name, + errors=[ + Error( + code="INVALID_REQUEST", + message="receipt requests cannot be rewritten", + ) + ], + ) from None except PreValidationHookError as exc: raise ADCPTaskError( operation=method_name, @@ -2959,7 +3024,7 @@ async def call_tool(params: dict[str, Any], context: ToolContext | None = None) ], ) from exc - if isinstance(params, dict): + if isinstance(params, dict) and method_name != "sync_reporting_receipts": params = _apply_unknown_field_policy( method_name, params, @@ -2987,7 +3052,9 @@ async def call_tool(params: dict[str, Any], context: ToolContext | None = None) ) call_params: Any = params - if params_model is not None and isinstance(params, dict): + if method_name == "sync_reporting_receipts": + call_params = raw_params + elif params_model is not None and isinstance(params, dict): try: call_params = params_model.model_validate(params) except ValidationError as exc: @@ -3142,7 +3209,7 @@ async def call_tool(params: dict[str, Any], context: ToolContext | None = None) # envelope is enhanced on the dedicated error paths # (``build_mcp_error_result`` / ``_send_adcp_error``), so skip # it here to avoid a double pass. - if "adcp_error" not in result: + if "adcp_error" not in result and method_name != "sync_reporting_receipts": _apply_response_enhancer(response_enhancer, method_name, result, ctx) if response_mode is not None and response_mode != "off" and isinstance(result, dict): diff --git a/src/adcp/server/serve.py b/src/adcp/server/serve.py index 618d260d9..c28b0a557 100644 --- a/src/adcp/server/serve.py +++ b/src/adcp/server/serve.py @@ -539,8 +539,36 @@ async def _dispatch_with_middleware( if not middleware: return await call_handler() + receipt_request = None + if skill_name == "sync_reporting_receipts": + from adcp.exceptions import ADCPTaskError + from adcp.reporting.receipts.errors import ReportingReceiptError + from adcp.reporting.receipts.wire import ReceiptBatch + from adcp.types import Error + + try: + receipt_request = ReceiptBatch.parse(params) + except ReportingReceiptError as error: + raise ADCPTaskError( + operation=skill_name, errors=[Error(code=error.code, message=str(error))] + ) from None + async def _step(index: int) -> Any: if index >= len(middleware): + if receipt_request is not None: + try: + if ReceiptBatch.parse(params) != receipt_request: + raise ReportingReceiptError("INVALID_REQUEST") + except ReportingReceiptError: + raise ADCPTaskError( + operation=skill_name, + errors=[ + Error( + code="INVALID_REQUEST", + message="receipt requests cannot be rewritten", + ) + ], + ) from None return await call_handler() mw = middleware[index] @@ -549,6 +577,14 @@ async def call_next() -> Any: return await mw(skill_name, params, context, call_next) + if skill_name == "sync_reporting_receipts": + from adcp.server.idempotency.store import _RECEIPT_BATCH_DISPATCH + + token = _RECEIPT_BATCH_DISPATCH.set(True) + try: + return await _step(0) + finally: + _RECEIPT_BATCH_DISPATCH.reset(token) return await _step(0) @@ -1635,7 +1671,8 @@ def _run_mcp_http( import anyio import uvicorn - host = getattr(mcp.settings, "host", "0.0.0.0") + # Intentional server listener fallback; deployment settings select the host. + host = getattr(mcp.settings, "host", "0.0.0.0") # nosec B104 port = int(mcp.settings.port) log_level = getattr(mcp.settings, "log_level", "INFO").lower() @@ -1722,7 +1759,8 @@ def _build_a2a_app( from adcp.server.a2a_server import create_a2a_server from adcp.server.discovery import resolve_base_url - resolved_base_url = resolve_base_url("0.0.0.0", port, base_url) + # The URL resolver translates the wildcard listener into a usable public URL. + resolved_base_url = resolve_base_url("0.0.0.0", port, base_url) # nosec B104 app = create_a2a_server( handler, @@ -1817,7 +1855,8 @@ def _serve_a2a( auth=auth, public_url=public_url, ) - sock = _bind_reusable_socket("0.0.0.0", resolved_port) + # Intentional public serving socket; authentication is composed by the app. + sock = _bind_reusable_socket("0.0.0.0", resolved_port) # nosec B104 try: # Same bind-boundary INFO as the MCP path so A2A adopters # also see one framework-controlled line confirming the @@ -2151,7 +2190,8 @@ def _serve_mcp_and_a2a( import uvicorn resolved_port = port or int(os.environ.get("PORT", "3001")) - resolved_host = host or os.environ.get("ADCP_HOST", "0.0.0.0") + # Intentional server default, configurable by argument or deployment env. + resolved_host = host or os.environ.get("ADCP_HOST", "0.0.0.0") # nosec B104 log_level = "info" app = _build_mcp_and_a2a_app( @@ -2443,7 +2483,10 @@ def create_mcp_server( from mcp.server.transport_security import TransportSecuritySettings resolved_port = port or int(os.environ.get("PORT", "3001")) - resolved_host = host if host is not None else (os.environ.get("ADCP_HOST") or "0.0.0.0") + # Intentional server default, configurable by argument or deployment env. + resolved_host = ( + host if host is not None else (os.environ.get("ADCP_HOST") or "0.0.0.0") # nosec B104 + ) mcp: Any = MCPServer(name, instructions=instructions) mcp.settings = _ADCPMCPSettingsProxy(mcp.settings) object.__setattr__(mcp.settings, "host", resolved_host) @@ -2609,6 +2652,10 @@ def streamable_http_app( class ADCPStreamableHTTPASGIApp: async def __call__(self, scope: Any, receive: Any, send: Any) -> None: + if getattr(mcp, "_adcp_receipt_ingress", False): + from adcp.reporting.receipts.transport import receipt_body_receive + + receive = receipt_body_receive(scope, receive, limit=max_request_body_size) token = _ADCP_MCP_REQUEST_CONTEXT.set(Request(scope, receive)) try: await streamable_http(scope, receive, send) @@ -2740,7 +2787,21 @@ def _register_tool( build_mcp_error_result, ) + if name == "sync_reporting_receipts": + object.__setattr__(mcp, "_adcp_receipt_ingress", True) + async def fn(**kwargs: Any) -> dict[str, Any]: + if name == "sync_reporting_receipts": + request_context = _get_starlette_request_for_dispatch() + if request_context is not None: + from adcp.reporting.receipts.transport import ( + RAW_RECEIPT_BODY_SCOPE_KEY, + mcp_receipt_parameters, + ) + + kwargs = mcp_receipt_parameters( + request_context.scope.get(RAW_RECEIPT_BODY_SCOPE_KEY) + ) # Caller identity: FastMCP does not expose an authenticated principal # at the SDK level (``Context.client_id`` is a session hint, not an # authenticated user). Sellers wire auth via HTTP middleware on diff --git a/tests/conformance/reporting/_materializer_installed.py b/tests/conformance/reporting/_materializer_installed.py index a7c7d62ed..e4fc6e391 100644 --- a/tests/conformance/reporting/_materializer_installed.py +++ b/tests/conformance/reporting/_materializer_installed.py @@ -6,7 +6,7 @@ import importlib.util import json import sys -from dataclasses import fields +from dataclasses import fields, replace from datetime import datetime, timedelta, timezone from importlib.resources import files from pathlib import Path @@ -26,6 +26,7 @@ async def main(): "adcp.reporting.revision_selection", "adcp.reporting.ledger", "adcp.reporting.outbox", + "adcp.reporting.receipts", ): module = importlib.import_module(name) assert len(module.__all__) == len(set(module.__all__)) @@ -72,6 +73,8 @@ async def main(): assert files("adcp.reporting.outbox").joinpath("required_status_selector_schema.json").is_file() assert files("adcp.reporting.ledger").joinpath("reporting_materializer.sql").is_file() assert files("adcp.reporting.materializer").joinpath("required_schema.json").is_file() + assert files("adcp.reporting.ledger").joinpath("reporting_receipt_ingestion.sql").is_file() + assert files("adcp.reporting.receipts").joinpath("required_schema.json").is_file() start = datetime(2026, 9, 1, tzinfo=timezone.utc) schedule = ReportingScheduleSpec("PT1H", "PT1H", period_anchor=start) period = derive_period(schedule, account_timezone="UTC", ordinal=0) @@ -209,6 +212,57 @@ async def main(): boundaries = await durable.read_materializer_boundaries(caller=scope.principal) assert len(boundaries) == boundaries[0].sequence == boundaries[0].account_sequence == 1 assert durable._materializer_outbox is None + from adcp.reporting.ledger import ReportingRevisionReceiptRecord, receipt_to_wire + from adcp.reporting.receipts import InMemoryReportingReceiptStore, ReportingReceiptHandler + from adcp.server import ToolContext + + receipt_store = InMemoryReportingReceiptStore(notifications=False) + await receipt_store.put_configuration(configuration) + await receipt_store.commit_obligation(obligation) + receipt_binding = replace(binding, reconciliation_mode="consumer_receipt") + await receipt_store.put_destination_binding(receipt_binding) + await receipt_store.commit_revision(revision, rows) + destination = example.development_destination(receipt_binding) + receipt_service = ReportingMaterializerService( + receipt_store, + ReportingDestinationIO(destination.registry, destination.resolver), + destination.writer, + ) + assert (await receipt_service.run_once()).state == "verified" + snapshot = await receipt_store.read_reconciliation_snapshot(caller=scope.principal) + outcome = next(r for r in snapshot.records if r.kind == "materialization") + verification = outcome.verification + receipt = ReportingRevisionReceiptRecord( + scope, + "installed-receipt-0001", + revision.reporting_revision_id, + outcome.reporting_materialization_id, + "accepted", + verification.verification_profile, + verification.row_count, + verification.control_totals, + datetime.now(timezone.utc), + observed_canonical_content_digest=verification.canonical_content_digest, + ) + request = { + "adcp_version": "3.2-rc.3", + "account": {"account_id": configuration.account_id}, + "idempotency_key": "installed-batch-0001", + "receipts": [receipt_to_wire(receipt)], + } + + async def authorize(reference, context, consumer): + assert reference == {"account_id": configuration.account_id} + assert consumer == scope.consumer_id + return configuration.account_id + + handler = ReportingReceiptHandler(receipt_store, resolve_account=authorize) + context = ToolContext(caller_identity=scope.consumer_id) + recorded = await handler.sync_reporting_receipts(request, context) + assert recorded["results"][0]["result"] == "recorded" + assert await handler.sync_reporting_receipts(request, context) == recorded + assert len(await receipt_store.read_receipt_boundaries(caller=scope.principal)) == 1 + assert receipt_store._materializer_outbox is None workspace = Path(config["workspace"]).resolve() assert all(not Path(path).resolve().is_relative_to(workspace) for path in sys.path) assert all( @@ -226,6 +280,7 @@ async def main(): "installed": True, "assets": actual, "durable": True, + "receipts": True, } ) ) diff --git a/tests/conformance/reporting/_materializer_process.py b/tests/conformance/reporting/_materializer_process.py index cc62fd3bf..e6fb93c01 100644 --- a/tests/conformance/reporting/_materializer_process.py +++ b/tests/conformance/reporting/_materializer_process.py @@ -6,6 +6,7 @@ import json import os import sys +from dataclasses import replace from pathlib import Path from uuid import uuid4 @@ -112,6 +113,18 @@ async def finish_materialization(self, *args, **kwargs): return result verifier = reference_verifier() + if settings.get("legacy_definition"): + verifier = replace( + verifier, + key=replace( + verifier.key, + definition=replace( + verifier.key.definition, + monetary_metric_units=(), + monetary_control_total_units=(), + ), + ), + ) registry = ReportingRevisionVerifierRegistry((verifier,)) writer = ReferenceReportingDestinationWriter((verifier.key.capability,)) reference = ReferenceReportingResolver(writer, registry, (decode_record(settings["binding"]),)) diff --git a/tests/conformance/reporting/_receipt_frozen.py b/tests/conformance/reporting/_receipt_frozen.py new file mode 100644 index 000000000..5c23aed9b --- /dev/null +++ b/tests/conformance/reporting/_receipt_frozen.py @@ -0,0 +1,282 @@ +"""Installed historical reader/writer probe, copied out and run with python -I.""" + +import asyncio +import hashlib +import importlib +import json +import sys +import traceback +from dataclasses import replace +from datetime import datetime, timezone +from importlib.resources import files +from pathlib import Path + + +async def main(settings): + from psycopg_pool import AsyncConnectionPool + + import adcp.reporting.ledger as ledger + + workspace = Path(settings["workspace"]).resolve() + origins = {} + for name, expected in settings["modules"].items(): + module = importlib.import_module(name) + path = Path(module.__file__).resolve() + assert hashlib.sha256(path.read_bytes()).hexdigest() == expected + assert not path.is_relative_to(workspace) and "site-packages" in str(path) + origins[name] = str(path) + assert not any(Path(p).resolve().is_relative_to(workspace) for p in sys.path) + for name, module in tuple(sys.modules.items()): + if (name == "adcp" or name.startswith("adcp.")) and getattr(module, "__file__", None): + assert "site-packages" in module.__file__ + assert not Path(module.__file__).resolve().is_relative_to(workspace) + async with AsyncConnectionPool( + settings["conninfo"], kwargs=settings["kwargs"], min_size=1, max_size=2, open=False + ) as pool: + if settings["artifact"] == "b21": + from adcp.reporting.materializer import PgReportingMaterializerStore + + store_type = PgReportingMaterializerStore + else: + store_type = getattr( + ledger, "PgReportingReconciliationStore", ledger.PgReportingLedgerStore + ) + options = ( + {"notifications": settings["notifications"]} + if settings["artifact"] not in {"beta15", "records", "integration"} + else {} + ) + store = store_type(pool=pool, **options) + if settings["action"] == "install": + assert settings["artifact"] == "b21" + await store.create_schema() + # C's projection is an independently installed optional component. + # Use this actual parent binary's reviewed Core composition when + # preparing the rolling baseline; the materializer migration does + # not implicitly install or activate C's projector. + from adcp.reporting.outbox import PgStatusNotificationStore + + await PgStatusNotificationStore( + ledger.PgReportingReconciliationStore(pool=pool, notifications=True) + ).create_schema() + assert await store.materializer_ready() + manifest = json.loads( + files("adcp.reporting.materializer").joinpath("required_schema.json").read_text() + ) + assert len(manifest) == 187 + return {"installed": True, "manifest_objects": len(manifest), "origins": origins} + account, consumer = settings["account"], settings["consumer"] + configs = await store.list_configurations(account_id=account) + assert len(configs) == 1 + obligations = await store.get_obligation( + account_id=account, reporting_obligation_id=settings["obligation"] + ) + assert obligations is not None + revisions = await store.list_revisions( + account_id=account, reporting_obligation_id=settings["obligation"] + ) + assert len(revisions) == 1 and revisions[0].finality == "official" + status, projector_turns = None, 0 + if settings["notifications"] and settings["artifact"] in {"c", "b1", "b21"}: + from adcp.reporting.outbox import PgStatusNotificationStore + + status = PgStatusNotificationStore( + ledger.PgReportingReconciliationStore(pool=pool, notifications=True) + ) + await status.baseline(account_id=account) + + async def project_ordinary_status(): + nonlocal projector_turns + if status is not None: + while (await status.project_one(account_id=account)).did_work: + projector_turns += 1 + assert projector_turns < 32 + + # All artifacts execute their own default-off ordinary Core read/write + # path on the exact parent before and after the additive migration. + await store.put_configuration( + replace(configs[0], status_retention_days=configs[0].status_retention_days + 1) + ) + for readable in (False, True): + await store.set_revision_readable( + account_id=account, + reporting_revision_id=revisions[0].reporting_revision_id, + readable=readable, + ) + await project_ordinary_status() + receipt_count, ordinary_materializer = 0, False + if hasattr(ledger, "ReportingDeliveryPrincipal"): + caller = ledger.ReportingDeliveryPrincipal(account, consumer) + snapshot = await store.read_reconciliation_snapshot(caller=caller) + available = [ + r for r in snapshot.records if r.kind == "materialization" and r.status != "failed" + ] + assert len(available) == 1 + receipts = [ + r for r in snapshot.records if r.kind in {"revision_receipt", "adjustment_receipt"} + ] + receipt_count = len(receipts) + assert receipt_count == settings["receipt_count"] + if receipt_count: + assert {r.kind for r in receipts} == {"revision_receipt", "adjustment_receipt"} + for receipt in receipts: + assert (await store.get_receipt(receipt.key)) == receipt + assert receipt.status == "accepted" and receipt.received_at is not None + # Public immutable terminal N+1 compatibility is deliberately + # separate from autonomous retry allocation. This actual old writer + # is run in isolation, never alongside an autonomous worker. + attempts = [r for r in snapshot.records if r.kind == "materialization_attempt"] + now = datetime.now(timezone.utc) + attempt = replace( + attempts[0], + reporting_materialization_id=f"frozen-{settings['phase']}", + attempt=len(attempts) + 1, + created_at=now, + ) + await store.commit_materialization_attempt(attempt) + outcome = replace( + available[0], + reporting_materialization_id=attempt.reporting_materialization_id, + status="failed", + resource=None, + verification=None, + failure_code="WRITE_FAILED", + completed_at=now, + ) + assert (await store.commit_materialization(outcome))[1] + assert not (await store.commit_materialization(outcome))[1] + read = await store.get_materialization(outcome.key) + assert read is not None and read.outcome == outcome + ordinary_materializer = True + await project_ordinary_status() + readiness = None + if settings["artifact"] in {"a", "b", "c", "b1", "b21"}: + from adcp.reporting.ledger.notification_models import ReportingNotificationError + from adcp.reporting.outbox._schema import validate_schema + + async with pool.connection() as connection: + try: + await validate_schema(connection) + readiness = True + except ReportingNotificationError: + readiness = False + assert readiness == (settings["artifact"] != "a") + materializer = None + if settings["artifact"] == "b21": + assert await store.materializer_ready() + boundaries = await store.read_materializer_boundaries(caller=caller) + assert len(boundaries) == 1 + assert boundaries[0].to_storage()["version"] == 1 + # The approved boundary decoder is closed: epoch is stored on its + # referenced work/event rows, not inside the captured input blob. + async with pool.connection() as connection: + epoch = await ( + await connection.execute( + "SELECT admission_epoch,state FROM reporting_materializer_work" + " WHERE account_id=%s AND consumer_id=%s" + " AND reporting_materialization_id=%s", + (account, consumer, boundaries[0].reporting_materialization_id), + ) + ).fetchone() + assert epoch == (0, "acked") + materializer = len( + json.loads( + files("adcp.reporting.materializer") + .joinpath("required_schema.json") + .read_text() + ) + ) + assert materializer == 187 + workers = {} + if settings["notifications"]: + from adcp.reporting.ledger.notification_models import decode_event + from adcp.reporting.outbox import ( + PgReportingOutbox, + ReportingEnvelopeCipher, + ReportingNotificationWorker, + ) + + class EmptySubscriptions: + async def list_active(self, **kwargs): + assert kwargs["notification_type"] != "reporting.delivery_ready" + return () + + async def get_active(self, **kwargs): + raise AssertionError("empty membership has no HTTP delivery") + + outboxes = {"ordinary": PgReportingOutbox(pool=pool)} + if status is not None: + outboxes["status"] = status.outbox + for name, outbox in outboxes.items(): + events = await outbox.list_events(account_id=account) + assert events and all( + event.notification_type != "reporting.delivery_ready" for event in events + ) + identities = { + (e.account_id, e.consumer_namespace, e.notification_id): e for e in events + } + seen = set() + + class ObservedOutbox: + def __getattr__(self, attribute): + return getattr(outbox, attribute) + + async def claim_expansion(self, **kwargs): + lease = await outbox.claim_expansion(**kwargs) + if lease is not None: + identity = ( + lease.account_id, + lease.consumer_namespace, + lease.notification_id, + ) + assert identity in identities and identity not in seen + assert decode_event(lease.event) == identities[identity] + seen.add(identity) + return lease + + worker = ReportingNotificationWorker( + outbox=ObservedOutbox(), + subscriptions=EmptySubscriptions(), + cipher=ReportingEnvelopeCipher(b"e" * 32), + ) + while await worker.expand_one(account_id=account): + assert len(seen) <= len(events) + assert not await worker.deliver_one(account_id=account) + assert await outbox.list_events(account_id=account) == events + workers[name] = len(seen) + if settings["phase"] == "before": + assert workers["ordinary"] > 0 # An actual old Core worker claimed real work. + if status is not None: + assert projector_turns > 0 and workers["status"] > 0 + return { + "ordinary_core": True, + "ordinary_materializer": ordinary_materializer, + "receipt_count": receipt_count, + "notification_readiness": readiness, + "materializer_manifest": materializer, + "projector_turns": projector_turns, + "workers": workers, + "origins": origins, + "sha": settings["sha"], + "artifact": settings["artifact"], + } + + +if __name__ == "__main__": + try: + result = asyncio.run(main(json.load(sys.stdin))) + except Exception as error: + result = { + "failure": type(error).__name__, + "safe_code": ( + error.code + if type(error).__module__ == "adcp.reporting.ledger.notification_models" + and type(error).__name__ == "ReportingNotificationError" + else None + ), + "frames": [ + [Path(frame.filename).name, frame.lineno] + for frame in traceback.extract_tb(error.__traceback__) + ], + } + print(json.dumps(result)) diff --git a/tests/conformance/reporting/_receipt_installed.py b/tests/conformance/reporting/_receipt_installed.py new file mode 100644 index 000000000..8d8c93c71 --- /dev/null +++ b/tests/conformance/reporting/_receipt_installed.py @@ -0,0 +1,82 @@ +"""Installed Python 3.10 SQL and authenticated HTTP replay, run outside the checkout.""" + +import asyncio +import hashlib +import importlib +import importlib.util +import json +import sys +import traceback +from pathlib import Path +from types import SimpleNamespace + + +async def main(settings): + from psycopg_pool import AsyncConnectionPool + + from adcp.reporting.receipts import PgReportingReceiptStore, ReportingReceiptError + + installed = settings["installed"] + workspace = Path(installed["workspace"]).resolve() + assert list(sys.version_info[:2]) == installed["python"] + origins = {} + for name, expected in installed["modules"].items(): + module = importlib.import_module(name) + path = Path(module.__file__).resolve() + assert hashlib.sha256(path.read_bytes()).hexdigest() == expected + origins[name] = str(path) + assert not any(Path(p).resolve().is_relative_to(workspace) for p in sys.path) + async with AsyncConnectionPool( + settings["conninfo"], kwargs=settings["kwargs"], min_size=1, max_size=1, open=False + ) as pool: + store = PgReportingReceiptStore(pool=pool, notifications=settings["notifications"]) + if settings["action"] == "install": + try: + await store.receipt_ingestion_ready() + except ReportingReceiptError as error: + assert error.code == "RECEIPT_SCHEMA_UNREADY" + else: + raise AssertionError("empty schema must fail closed") + await store.create_schema() + await store.create_schema() + assert await store.receipt_ingestion_ready() + result = {"installed": True} + else: + spec = importlib.util.spec_from_file_location("installed_transport", settings["helper"]) + transport = importlib.util.module_from_spec(spec) + spec.loader.exec_module(transport) + mount = transport.MountedReceipts(SimpleNamespace(store=store), hydrated=True) + caller = settings["caller"] + scenario = SimpleNamespace( + obligation=SimpleNamespace(account_id=caller["account_id"]), + binding=SimpleNamespace(consumer_id=caller["consumer_id"]), + ) + mount.authorize(scenario) + async with mount.client() as client: + _, result = await mount.mcp(client, settings["request"]) + assert [r["result"] for r in result["results"]] == ["recorded", "recorded"] + assert (await mount.a2a(client, settings["request"]))[1] == result + mount.grants.clear() + for call in (mount.mcp, mount.a2a): + _, denied = await call(client, settings["request"]) + assert transport.error_code(denied) == "UNAUTHORIZED" + assert len(mount.auth_calls) == 4 + for name, module in tuple(sys.modules.items()): + if (name == "adcp" or name.startswith("adcp.")) and getattr(module, "__file__", None): + assert "site-packages" in module.__file__ + assert not Path(module.__file__).resolve().is_relative_to(workspace) + return {"result": result, "origins": origins, "python": installed["python"]} + + +if __name__ == "__main__": + try: + result = asyncio.run(main(json.load(sys.stdin))) + except Exception as error: + result = { + "failure": type(error).__name__, + "frames": [ + [Path(frame.filename).name, frame.lineno] + for frame in traceback.extract_tb(error.__traceback__) + ], + } + print(json.dumps(result)) diff --git a/tests/conformance/reporting/_receipt_process.py b/tests/conformance/reporting/_receipt_process.py new file mode 100644 index 000000000..c76918c2a --- /dev/null +++ b/tests/conformance/reporting/_receipt_process.py @@ -0,0 +1,80 @@ +"""Real receipt worker. Pause while holding an ordinal transaction, then SIGKILL.""" + +import asyncio +import hashlib +import importlib +import json +import sys +from pathlib import Path + + +async def main(): + settings = json.loads(await asyncio.to_thread(sys.stdin.readline)) + from psycopg import AsyncConnection + from psycopg_pool import AsyncConnectionPool + + from adcp.reporting.ledger import ReportingDeliveryPrincipal + from adcp.reporting.receipts import PgReportingReceiptStore + + origins = {} + if settings.get("installed"): + installed = settings["installed"] + workspace = Path(installed["workspace"]).resolve() + assert list(sys.version_info[:2]) == installed["python"] + for name, expected in installed["modules"].items(): + module = importlib.import_module(name) + path = Path(module.__file__).resolve() + assert hashlib.sha256(path.read_bytes()).hexdigest() == expected + origins[name] = str(path) + assert not any(Path(p).resolve().is_relative_to(workspace) for p in sys.path) + for name, module in tuple(sys.modules.items()): + if (name == "adcp" or name.startswith("adcp.")) and getattr(module, "__file__", None): + assert "site-packages" in module.__file__ + assert not Path(module.__file__).resolve().is_relative_to(workspace) + + async def pause(point): + if settings.get("pause") == point: + print(json.dumps({"point": point}), flush=True) + assert json.loads(await asyncio.to_thread(sys.stdin.readline))["continue"] + + class Connection(AsyncConnection): + async def execute(self, query, params=None, **kwargs): + result = await super().execute(query, params, **kwargs) + if isinstance(query, str): + for prefix, point in ( + ("INSERT INTO reporting_receipt_ingestion_batches", "header"), + ("INSERT INTO reporting_reconciliation_records", "receipt"), + ("INSERT INTO reporting_reconciliation_changes", "feed"), + ("INSERT INTO reporting_receipt_ingestion_boundaries", "capture"), + ("INSERT INTO reporting_receipt_ingestion_results", "ordinal"), + ("UPDATE reporting_receipt_ingestion_batches", "final"), + ): + if query.startswith(prefix): + await pause(point) + return result + + class Store(PgReportingReceiptStore): + async def _receipt_results_on(self, connection, caller, batch): + result = await super()._receipt_results_on(connection, caller, batch) + if len(result) == 1: + await pause("between_ordinals") + return result + + async with AsyncConnectionPool( + settings["conninfo"], + kwargs=settings["kwargs"], + min_size=1, + max_size=1, + connection_class=Connection, + open=False, + ) as pool: + store = Store(pool=pool, notifications=settings["notifications"]) + result = await store.ingest_receipt_batch( + settings["request"], caller=ReportingDeliveryPrincipal(**settings["caller"]) + ) + await pause("committed") + print(json.dumps({"point": "done", "result": result, "origins": origins}), flush=True) + + +if __name__ == "__main__": + asyncio.run(main()) diff --git a/tests/conformance/reporting/_receipt_support.py b/tests/conformance/reporting/_receipt_support.py new file mode 100644 index 000000000..1181d7b73 --- /dev/null +++ b/tests/conformance/reporting/_receipt_support.py @@ -0,0 +1,103 @@ +"""Shared receipt-ingress vectors; production PG receipt time is database time.""" + +from contextlib import asynccontextmanager +from dataclasses import replace +from datetime import timedelta + +import pytest + +from adcp.reporting.ledger import ReportingAdjustmentRecord, ReportingControlTotalRecord +from adcp.reporting.ledger.delivery import adjustment_to_wire, receipt_to_wire +from adcp.reporting.receipts import InMemoryReportingReceiptStore + +from ._durable_materializer_support import DurableHarness +from ._generation_support import END, isolated_reporting_pool +from ._reconciliation_support import Clock, scenario + + +@asynccontextmanager +async def receipt_harness(backend, *, notifications=False, pool=None): + clock = Clock() + if backend == "memory": + yield DurableHarness( + InMemoryReportingReceiptStore(clock=clock, notifications=notifications), clock + ) + return + from adcp.reporting.receipts import PgReportingReceiptStore + + async with isolated_reporting_pool(autocommit=True) as pool: + store = PgReportingReceiptStore(pool=pool, clock=clock, notifications=notifications) + await store.create_schema() + yield DurableHarness(store, clock, pool) + + +@pytest.fixture( + params=[("memory", False), ("memory", True), ("postgres", False), ("postgres", True)] +) +async def receipts(request): + backend, notifications = request.param + async with receipt_harness(backend, notifications=notifications) as h: + yield h + + +async def receipt_case(h, **kwargs): + s = await scenario(h.store, **kwargs) + await h.store.commit_materialization(s.outcome) + return s + + +def request_for(s, *, key="receipt-batch-0001", **changes): + return { + "adcp_version": "3.2-rc.3", + "account": {"account_id": s.obligation.account_id}, + "idempotency_key": key, + "receipts": [receipt_to_wire(s.receipt)], + **changes, + } + + +async def adjustment_for(h, s, **changes): + adjustment = ReportingAdjustmentRecord( + "adjustment-1", + s.obligation.account_id, + s.revision.reporting_revision_id, + "source_correction", + END, + END + timedelta(days=30), + (("spend", "-1.50"),), + END + timedelta(seconds=5), + END + timedelta(seconds=6), + managed_control_total_deltas=( + ReportingControlTotalRecord("spend", "-1.50", "decimal", "EUR"), + ), + ) + adjustment = replace(adjustment, **changes) + await h.store.commit_adjustment(adjustment) + return { + "reporting_receipt_id": "adjustment-receipt-0001", + "reporting_adjustment_id": adjustment.reporting_adjustment_id, + "adjusts_reporting_revision_id": adjustment.adjusts_reporting_revision_id, + "status": "accepted", + "observed_adjustment_sha256": adjustment_to_wire(adjustment)["canonical_adjustment_sha256"], + "observed_at": (END + timedelta(seconds=7)).isoformat().replace("+00:00", "Z"), + } + + +async def batch_state(h): + if h.pool is None: + return tuple( + (len(s.results), s.response is not None) + for s in getattr(h.store, "_receipt_batches", {}).values() + ) + async with h.pool.connection() as c: + return tuple( + await ( + await c.execute( + "SELECT (SELECT count(*) FROM reporting_receipt_ingestion_results r" + " WHERE (r.account_id,r.consumer_id,r.idempotency_key)=" + "(b.account_id,b.consumer_id,b.idempotency_key))," + " final_response IS NOT NULL FROM reporting_receipt_ingestion_batches b" + " ORDER BY account_id,consumer_id,idempotency_key" + ) + ).fetchall() + ) diff --git a/tests/conformance/reporting/_receipt_transport.py b/tests/conformance/reporting/_receipt_transport.py new file mode 100644 index 000000000..1b25cb4f7 --- /dev/null +++ b/tests/conformance/reporting/_receipt_transport.py @@ -0,0 +1,270 @@ +"""Real unified production mounts, with deterministic trusted auth adapters.""" + +import json +from contextlib import asynccontextmanager + +import httpx +from asgi_lifespan import LifespanManager + +from adcp.decisioning import Account, AuthInfo, RequestContext +from adcp.decisioning.registry import ( + ApiKeyCredential, + BuyerAgent, + HttpSigCredential, + OAuthCredential, +) +from adcp.reporting.receipts import ReportingReceiptError, ReportingReceiptHandler +from adcp.server.auth import BearerTokenAuth, Principal, auth_context_factory +from adcp.server.idempotency import IdempotencyStore, MemoryBackend +from adcp.server.serve import _build_mcp_and_a2a_app + + +class Registry: + def __init__(self): + self.agents = {} + self.calls = [] + + async def resolve_by_agent_url(self, agent_url): + self.calls.append(("signed", agent_url)) + return self.agents.get(agent_url) + + async def resolve_by_credential(self, credential): + key = ( + credential.key_id if isinstance(credential, ApiKeyCredential) else credential.client_id + ) + self.calls.append((credential.kind, key)) + return self.agents.get(key) + + +class ForbiddenGenericCache(MemoryBackend): + async def get(self, *args, **kwargs): + raise AssertionError("receipt ingress reached the generic idempotency cache") + + +class MountedReceipts: + def __init__(self, h, *, hydrated=False, registry_kind=None, version=None): + self.h = h + self.hydrated = hydrated + self.registry_kind = registry_kind + self.tokens = {} + self.grants = set() + self.accounts = {} + self.auth_calls = [] + self.contexts = [] + self.registry = Registry() if registry_kind is not None else None + self.sessions = {} + self.counter = 0 + self.idempotency = IdempotencyStore(backend=ForbiddenGenericCache()) + self.handler = ReportingReceiptHandler( + h.store, resolve_account=self.resolve_account, buyer_agents=self.registry + ) + # Exercise both the direct method decorator and the common generic + # middleware whose wrapped function is named execute, not the task. + self.handler.sync_reporting_receipts = self.idempotency.wrap( + self.handler.sync_reporting_receipts + ) + if version is not None: + self.handler.adcp_version = version + + def authorize(self, s, *, token="token-one"): + account, consumer = s.obligation.account_id, s.binding.consumer_id + self.tokens[token] = Principal( + caller_identity=consumer, + tenant_id="one-shared-tenant", + metadata={"account_id": account, "credential_id": token}, + ) + self.grants.add((account, consumer)) + self.accounts[account] = account + if self.registry is not None: + agent = BuyerAgent(consumer, "Fixture buyer", "active") + self.registry.agents[token] = self.registry.agents[consumer] = agent + + async def resolve_account(self, reference, context, consumer): + self.auth_calls.append((dict(reference), consumer)) + account = self.accounts.get(reference.get("account_id")) + if (account, consumer) not in self.grants: + raise ReportingReceiptError("UNAUTHORIZED") + return account + + def context(self, meta): + raw = auth_context_factory(meta) + auth = raw.metadata.get("adcp.auth_info") + agent = None + if self.registry_kind is not None and raw.caller_identity is not None: + key = raw.metadata["credential_id"] + if self.registry_kind == "api_key": + credential = ApiKeyCredential("api_key", key) + elif self.registry_kind == "oauth": + credential = OAuthCredential("oauth", key, ("reporting:write",)) + else: + credential = HttpSigCredential("http_sig", key, raw.caller_identity, 1.0) + # This is the adapter output of trusted verification, never an + # inbound body field. API/OAuth consumer resolution is registry-owned. + auth = AuthInfo(kind=self.registry_kind, credential=credential) + raw.metadata["adcp.auth_info"] = auth + agent = BuyerAgent(raw.caller_identity, "Fixture buyer", "active") + if self.hydrated: + context = RequestContext( + account=Account(id=raw.metadata.get("account_id", "unresolved")), + caller_identity="opaque:AccountStore:cache:key:identical-for-both-consumers", + tenant_id=raw.tenant_id, + metadata=raw.metadata, + auth_info=auth, + buyer_agent=agent, + auth_principal=raw.caller_identity, + ) + else: + context = raw + self.contexts.append(context) + return context + + async def middleware(self, name, params, context, call_next): + async def execute(params, context): + return await call_next() + + return await self.idempotency.wrap(execute)(params, context) + + @asynccontextmanager + async def client(self, **options): + app = _build_mcp_and_a2a_app( + self.handler, + name="receipt-ingress", + port=3001, + host="127.0.0.1", + instructions=None, + test_controller=None, + context_factory=self.context, + middleware=[self.middleware], + allowed_hosts=["localhost"], + auth=BearerTokenAuth(validate_token=lambda token: self.tokens.get(token)), + **options, + ) + async with LifespanManager(app): + async with httpx.AsyncClient( + transport=httpx.ASGITransport(app=app), base_url="http://localhost" + ) as client: + yield client + + async def mcp( + self, client, request=None, *, token="token-one", inventory=False, mutate_wire=None + ): + headers = { + "accept": "application/json, text/event-stream", + "authorization": f"Bearer {token}", + } + if token not in self.sessions: + initial = await client.post( + "/mcp/", + headers=headers, + json={ + "jsonrpc": "2.0", + "id": 0, + "method": "initialize", + "params": { + "protocolVersion": "2025-06-18", + "capabilities": {}, + "clientInfo": {"name": "receipt-test", "version": "1"}, + }, + }, + ) + assert initial.status_code == 200, initial.text + self.sessions[token] = initial.headers.get("mcp-session-id") + if self.sessions[token] is not None: + headers["mcp-session-id"] = self.sessions[token] + self.counter += 1 + wire = json.dumps( + { + "jsonrpc": "2.0", + "id": self.counter, + "method": "tools/list" if inventory else "tools/call", + "params": ( + {} if inventory else {"name": "sync_reporting_receipts", "arguments": request} + ), + } + ) + response = await client.post( + "/mcp/", + headers={**headers, "content-type": "application/json"}, + content=mutate_wire(wire) if mutate_wire is not None else wire, + ) + if response.status_code != 200: + try: + payload = response.json() + except ValueError: + payload = {"unstructured_transport_error": True} + return response.status_code, payload + payload = next( + ( + json.loads(line[6:]) + for line in response.text.splitlines() + if line.startswith("data: ") + ), + None, + ) + if payload is None: + payload = response.json() + if inventory: + return 200, payload["result"] + result = payload.get("result", payload) + if "structuredContent" in result: + return 200, result["structuredContent"] + for part in result.get("content", []): + if part.get("type") == "text": + return 200, json.loads(part["text"]) + return 200, result + + async def a2a(self, client, request, *, token="token-one", mutate_wire=None, v1=False): + self.counter += 1 + envelope = { + "jsonrpc": "2.0", + "id": str(self.counter), + "method": "SendMessage" if v1 else "message/send", + "params": { + "message": { + "messageId": f"message-{self.counter}", + "role": "user", + "parts": [ + { + "kind": "data", + "data": {"skill": "sync_reporting_receipts", "parameters": request}, + } + ], + } + }, + } + if v1: + del envelope["params"]["message"]["parts"][0]["kind"] + envelope["params"]["message"]["role"] = "ROLE_USER" + wire = json.dumps(envelope) + response = await client.post( + "/", + headers={ + "authorization": f"Bearer {token}", + "content-type": "application/json", + "A2A-Version": "1.0" if v1 else "0.3", + }, + content=mutate_wire(wire) if mutate_wire is not None else wire, + ) + if response.status_code != 200: + try: + payload = response.json() + except ValueError: + payload = {"unstructured_transport_error": True} + return response.status_code, payload + payload = response.json() + result = payload.get("result", payload) + if "task" in result: + result = result["task"] + for artifact in result.get("artifacts", []): + for part in artifact.get("parts", []): + if "data" in part: + return 200, part["data"] + return 200, result + + +def error_code(payload): + if "adcp_error" in payload: + return payload["adcp_error"]["code"] + if "errors" in payload: + return payload["errors"][0]["code"] + raise AssertionError(payload) diff --git a/tests/conformance/reporting/test_reporting_materializer_packaging.py b/tests/conformance/reporting/test_reporting_materializer_packaging.py index 36187037b..d01ca1374 100644 --- a/tests/conformance/reporting/test_reporting_materializer_packaging.py +++ b/tests/conformance/reporting/test_reporting_materializer_packaging.py @@ -47,6 +47,8 @@ def b1_wheels(built_distribution): "outbox/required_status_selector_schema.json", "ledger/reporting_materializer.sql", "materializer/required_schema.json", + "ledger/reporting_receipt_ingestion.sql", + "receipts/required_schema.json", ): assert ( vcs.read(f"adcp/reporting/{relative}") @@ -98,6 +100,10 @@ def test_python310_installed_wheel_exports_verifier_reference_and_strict_adopter durable_adopter = path / f"durable_adopter_{kind}.py" shutil.copy2(ROOT / "examples/reporting_durable_materializer.py", durable_example) shutil.copy2(ROOT / "tests/type_checks/reporting_durable_materializer.py", durable_adopter) + receipt_example = path / f"receipt_example_{kind}.py" + receipt_adopter = path / f"receipt_adopter_{kind}.py" + shutil.copy2(ROOT / "examples/reporting_receipt_ingress.py", receipt_example) + shutil.copy2(ROOT / "tests/type_checks/reporting_receipt_ingress.py", receipt_adopter) result = json.loads( run_step( [str(python), "-I", str(smoke)], @@ -113,6 +119,7 @@ def test_python310_installed_wheel_exports_verifier_reference_and_strict_adopter "installed": True, "assets": hashes, "durable": True, + "receipts": True, } config = path / "mypy.ini" config.write_text( @@ -133,6 +140,8 @@ def test_python310_installed_wheel_exports_verifier_reference_and_strict_adopter str(example), str(durable_adopter), str(durable_example), + str(receipt_adopter), + str(receipt_example), ], label=f"b1-{kind}-installed-adopter-types", cwd=path, diff --git a/tests/conformance/reporting/test_reporting_materializer_process.py b/tests/conformance/reporting/test_reporting_materializer_process.py index 80ea527b8..865fb8788 100644 --- a/tests/conformance/reporting/test_reporting_materializer_process.py +++ b/tests/conformance/reporting/test_reporting_materializer_process.py @@ -69,6 +69,7 @@ async def worker( "pause": pause, "notifications": notifications, "installed": installed, + "legacy_definition": not case.config.definition.monetary_metric_units, } ) yield child diff --git a/tests/conformance/reporting/test_reporting_materializer_rolling.py b/tests/conformance/reporting/test_reporting_materializer_rolling.py index 6f09b41a3..80414b636 100644 --- a/tests/conformance/reporting/test_reporting_materializer_rolling.py +++ b/tests/conformance/reporting/test_reporting_materializer_rolling.py @@ -29,9 +29,9 @@ } -def build_frozen(artifact, tmp_path_factory, request): +def build_frozen(artifact, tmp_path_factory, request, *, sha=None): assert_c_collated_rolling_database() - sha = ARTIFACTS[artifact] + sha = sha or ARTIFACTS[artifact] root = tmp_path_factory.mktemp(f"materializer-{artifact}") request.addfinalizer(lambda: shutil.rmtree(root)) archive, source, dist, environment = ( @@ -80,6 +80,8 @@ def build_frozen(artifact, tmp_path_factory, request): "outbox/status_pg.py", "materializer/contracts.py", "materializer/verification.py", + "materializer/pg.py", + "materializer/capture.py", ): path = source / "src/adcp/reporting" / relative if path.exists(): diff --git a/tests/conformance/reporting/test_reporting_receipt_batches.py b/tests/conformance/reporting/test_reporting_receipt_batches.py new file mode 100644 index 000000000..b1d04a18c --- /dev/null +++ b/tests/conformance/reporting/test_reporting_receipt_batches.py @@ -0,0 +1,388 @@ +"""The same mixed-batch state machine runs in memory and on PostgreSQL 16.""" + +import asyncio +from copy import deepcopy +from dataclasses import replace +from datetime import timedelta + +import pytest + +from adcp.reporting.ledger import ReportingDeliveryPrincipal, ReportingMaterializationCheck +from adcp.reporting.ledger.delivery import receipt_to_wire +from adcp.reporting.receipts import ReportingReceiptError +from adcp.reporting.receipts.wire import ReceiptBatch, validate_receipt_response + +from ._receipt_support import adjustment_for, batch_state, receipt_case, receipts, request_for + +__all__ = ["receipts"] + + +async def test_mixed_batch_order_durable_failure_and_exact_response_replay(receipts): + h = receipts + s = await receipt_case(h, consumer_id="https://buyer.example.test/agent") + adjustment = await adjustment_for(h, s) + bad_revision = { + **receipt_to_wire(s.receipt), + "reporting_receipt_id": "revision-missing-0002", + "reporting_revision_id": "missing", + } + bad_adjustment = { + **adjustment, + "reporting_receipt_id": "adjustment-missing-0002", + "reporting_adjustment_id": "missing", + } + request = request_for( + s, + receipts=[bad_revision, receipt_to_wire(s.receipt)], + adjustment_receipts=[adjustment, bad_adjustment], + context={"trace": "original"}, + ) + result = await h.store.ingest_receipt_batch(request, caller=s.attempt.scope.principal) + validate_receipt_response(result, ReceiptBatch.parse(request)) + assert [r["result"] for r in result["results"]] == ["failed", "recorded", "recorded", "failed"] + assert result["results"][0]["reporting_receipt_id"] == bad_revision["reporting_receipt_id"] + assert result["results"][3]["reporting_receipt_id"] == bad_adjustment["reporting_receipt_id"] + assert result["results"][0]["errors"] == result["results"][3]["errors"] + assert result["context"] == {"trace": "original"} + captures = await h.store.read_receipt_boundaries(caller=s.attempt.scope.principal) + assert len(captures) == 2 and captures[0].account_sequence < captures[1].account_sequence + assert all(b.core.consumer_ids == (s.binding.consumer_id,) for b in captures) + # A later authorization call can replay after readability/clock changes; + # no item is recomputed and original recorded outcomes/time remain intact. + await h.store.set_revision_readable( + account_id=s.obligation.account_id, + reporting_revision_id=s.revision.reporting_revision_id, + readable=False, + ) + h.clock.now += timedelta(days=450) + before = await h.image() + replay = await h.store.ingest_receipt_batch(deepcopy(request), caller=s.attempt.scope.principal) + assert replay == result and "replayed" not in replay + assert await h.image() == before + replay["results"].clear() + assert await h.store.ingest_receipt_batch(request, caller=s.attempt.scope.principal) == result + + +@pytest.mark.parametrize( + "case", + [ + "absent", + "empty", + "empty_sibling", + "null", + "not_list", + "duplicate", + "cross_duplicate", + "too_many", + "received_at", + "adjustment_received_at", + "spoof", + "bad_late_shape", + ], +) +async def test_whole_shape_admission_has_zero_writes(receipts, case): + h = receipts + s = await receipt_case(h) + adjustment = await adjustment_for(h, s) + request = request_for(s) + if case == "absent": + request.pop("receipts") + if case == "empty": + request["receipts"] = [] + if case == "empty_sibling": + request["adjustment_receipts"] = [] + if case == "null": + request["adjustment_receipts"] = None + if case == "not_list": + request["receipts"] = {} + if case == "duplicate": + request["receipts"] *= 2 + if case == "cross_duplicate": + request["adjustment_receipts"] = [ + {**adjustment, "reporting_receipt_id": s.receipt.reporting_receipt_id} + ] + if case == "too_many": + request["receipts"] = [ + {**request["receipts"][0], "reporting_receipt_id": f"receipt-many-{i:06d}"} + for i in range(50) + ] + request["adjustment_receipts"] = [ + {**adjustment, "reporting_receipt_id": f"adjustment-many-{i:06d}"} for i in range(51) + ] + if case == "received_at": + request["receipts"][0]["received_at"] = s.receipt.observed_at.isoformat() + if case == "adjustment_received_at": + request["adjustment_receipts"] = [{**adjustment, "received_at": None}] + if case == "spoof": + request["consumer_id"] = "other" + if case == "bad_late_shape": + request["adjustment_receipts"] = [{"reporting_receipt_id": "invalid-late-0001"}] + before = await h.image() + with pytest.raises(ReportingReceiptError) as error: + await h.store.ingest_receipt_batch(request, caller=s.attempt.scope.principal) + assert error.value.code == "INVALID_REQUEST" + assert await h.image() == before + + +@pytest.mark.parametrize( + "change", ["order", "context", "extension", "target", "omitted_array", "time_spelling"] +) +async def test_whole_request_conflict_before_any_write(receipts, change): + h = receipts + s = await receipt_case(h) + request = request_for( + s, + receipts=[ + receipt_to_wire(s.receipt), + { + **receipt_to_wire(s.receipt), + "reporting_receipt_id": "receipt-second-0002", + "reporting_revision_id": "missing", + }, + ], + ) + await h.store.ingest_receipt_batch(request, caller=s.attempt.scope.principal) + changed = deepcopy(request) + if change == "order": + changed["receipts"].reverse() + if change == "context": + changed["context"] = {"trace": "changed"} + if change == "extension": + changed["ext"] = {"test": {"version": 1}} + if change == "target": + changed["receipts"][1]["reporting_revision_id"] = "another" + if change == "omitted_array": + changed["receipts"].pop() + if change == "time_spelling": + changed["receipts"][0]["observed_at"] = changed["receipts"][0]["observed_at"].replace( + "Z", "+00:00" + ) + before = await h.image() + with pytest.raises(ReportingReceiptError) as error: + await h.store.ingest_receipt_batch(changed, caller=s.attempt.scope.principal) + assert error.value.code == "IDEMPOTENCY_CONFLICT" + assert await h.image() == before + + +async def test_concurrent_same_batch_and_changed_body_converge(receipts): + h = receipts + s = await receipt_case(h) + adjustment = await adjustment_for(h, s) + request = request_for(s, adjustment_receipts=[adjustment]) + responses = await asyncio.gather( + *( + h.store.ingest_receipt_batch(deepcopy(request), caller=s.attempt.scope.principal) + for _ in range(8) + ) + ) + assert all(response == responses[0] for response in responses) + assert [r["result"] for r in responses[0]["results"]] == ["recorded", "recorded"] + assert await batch_state(h) == ((2, True),) + assert len(await h.store.read_receipt_boundaries(caller=s.attempt.scope.principal)) == 2 + other = {**request, "context": {"changed": True}} + responses = await asyncio.gather( + h.store.ingest_receipt_batch(request, caller=s.attempt.scope.principal), + h.store.ingest_receipt_batch(other, caller=s.attempt.scope.principal), + return_exceptions=True, + ) + assert responses[0]["results"][0]["result"] == "recorded" + assert ( + isinstance(responses[1], ReportingReceiptError) + and responses[1].code == "IDEMPOTENCY_CONFLICT" + ) + + +async def test_same_ids_keys_scoped_to_exact_accounts_and_consumers(receipts): + h = receipts + cases = [ + await receipt_case(h, account_id=a, consumer_id=c) + for a, c in ( + ("acct_a", "buyer"), + ("acct_a", "https://buyer.example.test/agent"), + ("acct_b", "buyer"), + ("acct:a", "b:c"), + ("acct:a:b", "c"), + ) + ] + for s in cases: + response = await h.store.ingest_receipt_batch( + request_for(s), caller=s.attempt.scope.principal + ) + assert response["results"][0]["result"] == "recorded" + assert len(await batch_state(h)) == len(cases) + s = cases[0] + unauthorized = await h.store.ingest_receipt_batch( + request_for(s, key="unavailable-key-0001"), + caller=ReportingDeliveryPrincipal("acct_a", "other"), + ) + unknown = await h.store.ingest_receipt_batch( + request_for( + s, + key="unavailable-key-0002", + receipts=[{**receipt_to_wire(s.receipt), "reporting_revision_id": "missing"}], + ), + caller=ReportingDeliveryPrincipal("acct_a", "other"), + ) + assert unauthorized == unknown + + +async def test_rejected_leaf_replacement_terminal_acceptance_and_no_retry_signal(receipts): + h = receipts + s = await receipt_case(h) + request = request_for( + s, + receipts=[ + {**receipt_to_wire(s.receipt), "status": "rejected", "rejection_codes": ["LOAD_FAILED"]} + ], + ) + work_before = await h.works() + first = await h.store.ingest_receipt_batch(request, caller=s.attempt.scope.principal) + assert first["results"][0]["receipt"]["status"] == "rejected" + assert await h.works() == work_before + corrected = { + **receipt_to_wire(s.receipt), + "reporting_receipt_id": "receipt-replacement-0002", + "supersedes_reporting_receipt_id": s.receipt.reporting_receipt_id, + } + accepted = await h.store.ingest_receipt_batch( + request_for(s, key="replacement-batch-0002", receipts=[corrected]), + caller=s.attempt.scope.principal, + ) + assert accepted["results"][0]["receipt"]["status"] == "accepted" + terminal = await h.store.ingest_receipt_batch( + request_for( + s, + key="terminal-batch-0003", + receipts=[ + { + **corrected, + "reporting_receipt_id": "receipt-after-accept-0003", + "supersedes_reporting_receipt_id": corrected["reporting_receipt_id"], + } + ], + ), + caller=s.attempt.scope.principal, + ) + assert terminal["results"][0]["errors"][0]["code"] == "ACCEPTED_RECEIPT_TERMINAL" + replay = await h.store.ingest_receipt_batch( + request_for(s, key="unchanged-batch-0004", receipts=[corrected]), + caller=s.attempt.scope.principal, + ) + assert replay["results"][0] == {**accepted["results"][0], "result": "unchanged"} + + +@pytest.mark.parametrize("later", ["success", "failure", "expiry", "corruption"]) +async def test_accepted_artifact_authority_survives_later_outcomes_separately_from_readability( + receipts, later +): + h = receipts + s = await receipt_case(h) + result = await h.store.ingest_receipt_batch(request_for(s), caller=s.attempt.scope.principal) + original = result["results"][0]["receipt"] + if later in {"success", "failure"}: + attempt = replace(s.attempt, reporting_materialization_id="materialization-2", attempt=2) + await h.store.commit_materialization_attempt(attempt) + outcome = replace( + s.outcome, reporting_materialization_id=attempt.reporting_materialization_id + ) + if later == "failure": + outcome = replace( + outcome, + status="failed", + resource=None, + verification=None, + failure_code="CONTENT_CORRUPT", + ) + await h.store.commit_materialization(outcome) + at = h.clock.now + if later == "expiry": + at = s.outcome.resource.expires_at + if later == "corruption": + await h.store.record_materialization_check( + ReportingMaterializationCheck( + s.attempt.scope, + s.attempt.reporting_materialization_id, + "check-corrupt", + "corrupt", + h.clock.now, + ) + ) + snapshot = await h.store.read_reconciliation_snapshot(caller=s.attempt.scope.principal) + assert len(snapshot.current_receipts) == 1 + assert receipt_to_wire(snapshot.current_receipts[0]) == original + view = snapshot.materialization(s.attempt.key) + assert view.outcome == s.outcome + assert view.readable_at(at) is (later not in {"expiry", "corruption"}) + assert ( + await h.store.ingest_receipt_batch(request_for(s), caller=s.attempt.scope.principal) + == result + ) + + +async def test_hundred_combined_results_and_adjustment_only_admission(receipts): + h = receipts + s = await receipt_case(h) + adjustment = await adjustment_for(h, s) + request = request_for( + s, + receipts=[ + {**receipt_to_wire(s.receipt), "reporting_receipt_id": f"revision-{i:010d}"} + for i in range(50) + ], + adjustment_receipts=[ + {**adjustment, "reporting_receipt_id": f"adjustment-{i:010d}"} for i in range(50) + ], + ) + result = await h.store.ingest_receipt_batch(request, caller=s.attempt.scope.principal) + assert len(result["results"]) == 100 + assert sum(r["result"] == "recorded" for r in result["results"]) == 2 + only = { + "account": request["account"], + "idempotency_key": "only-adjustment-0001", + "adjustment_receipts": [request["adjustment_receipts"][0]], + } + assert (await h.store.ingest_receipt_batch(only, caller=s.attempt.scope.principal))["results"][ + 0 + ]["result"] == "unchanged" + + +@pytest.mark.parametrize( + "context", + [ + {"field": "nul\x00value"}, + {"nul\x00key": "value"}, + {"field": "\ud800"}, + {"\udfff": "value"}, + {"field": "\ud83d\ude00"}, + ], +) +async def test_unrepresentable_jsonb_strings_fail_whole_shape_before_any_header(receipts, context): + h = receipts + s = await receipt_case(h) + before = await h.image() + with pytest.raises(ReportingReceiptError) as error: + await h.store.ingest_receipt_batch( + request_for(s, context=context), caller=s.binding.principal + ) + assert error.value.code == "INVALID_REQUEST" + assert await h.image() == before + + +async def test_whole_request_utf16_key_order_survives_storage_and_exact_replay(receipts): + h = receipts + s = await receipt_case(h) + context = { + "nested": {"\ue000": "BMP", "\U00010000": "supplementary"}, + "array": [{"\U0001f600": "face", "e\u0301": "combining", "x": "a\\b\n"}], + } + request = request_for(s, context=context) + response = await h.store.ingest_receipt_batch(request, caller=s.binding.principal) + assert response["results"][0]["result"] == "recorded" + assert response["context"] == context + assert await h.store.ingest_receipt_batch(request, caller=s.binding.principal) == response + before = await h.image() + changed = {**request, "context": {**context, "nested": {"\ue000": "changed"}}} + with pytest.raises(ReportingReceiptError) as error: + await h.store.ingest_receipt_batch(changed, caller=s.binding.principal) + assert error.value.code == "IDEMPOTENCY_CONFLICT" + assert await h.image() == before diff --git a/tests/conformance/reporting/test_reporting_receipt_graph.py b/tests/conformance/reporting/test_reporting_receipt_graph.py new file mode 100644 index 000000000..d632a9619 --- /dev/null +++ b/tests/conformance/reporting/test_reporting_receipt_graph.py @@ -0,0 +1,398 @@ +"""Financial predicates are shared by ingress and literal PostgreSQL writes.""" + +from dataclasses import replace +from datetime import timedelta + +import pytest + +from adcp.reporting.ledger import ReportingMaterializationCheck +from adcp.reporting.ledger._delivery_state import current_receipt +from adcp.reporting.ledger.delivery import receipt_to_wire +from adcp.reporting.ledger.store import LedgerConflictError +from adcp.reporting.receipts.records import receipt_record + +from ._generation_support import END +from ._receipt_support import adjustment_for, receipt_case, receipt_harness, receipts, request_for +from .test_reporting_reconciliation_sql import raw_insert + +__all__ = ["receipts"] + + +@pytest.mark.parametrize( + "fault,expected", + [ + ("rows", "RECEIPT_TOTALS_MISMATCH"), + ("totals", "RECEIPT_TOTALS_MISMATCH"), + ("currency", "RECEIPT_TOTALS_MISMATCH"), + ("digest", "RECEIPT_EVIDENCE_MISMATCH"), + ("manifest", "RECEIPT_EVIDENCE_MISMATCH"), + ("profile", "RECEIPT_PROFILE_MISMATCH"), + ("materialization", "REPORTING_RECORD_UNAVAILABLE"), + ("revision", "REPORTING_RECORD_UNAVAILABLE"), + ("obligation", "REPORTING_RECORD_UNAVAILABLE"), + ("future", "REPORTING_TIME_INVALID"), + ("before_outcome", "REPORTING_TIME_INVALID"), + ("duplicate_totals", "INVALID_REPORTING_RECORD"), + ], +) +async def test_exact_artifact_predicates_produce_durable_item_failures(receipts, fault, expected): + h = receipts + s = await receipt_case(h) + request = request_for(s) + item = request["receipts"][0] + if fault == "rows": + item["observed_row_count"] += 1 + elif fault == "totals": + item["observed_control_totals"][0]["value"] = "1000" + elif fault == "currency": + next(t for t in item["observed_control_totals"] if t["name"] == "spend")["unit"] = "GBP" + elif fault == "digest": + item["observed_canonical_content_digest"]["value"] = "f" * 64 + elif fault == "manifest": + item["observed_manifest_sha256"] = "f" * 64 + elif fault == "profile": + item["verification_profile"] = "manifest_checksums" + item["observed_manifest_sha256"] = "a" * 64 + elif fault == "materialization": + item["reporting_materialization_id"] = "unknown" + elif fault == "revision": + item["reporting_revision_id"] = "unknown" + elif fault == "obligation": + item["reporting_obligation_id"] = "unknown" + elif fault == "future": + item["observed_at"] = "2099-01-01T00:00:00Z" + elif fault == "before_outcome": + item["observed_at"] = END.isoformat() + else: + item["observed_control_totals"].append({**item["observed_control_totals"][0], "value": "2"}) + result = await h.store.ingest_receipt_batch(request, caller=s.binding.principal) + assert result["results"][0]["errors"][0]["code"] == expected + assert not await h.store.get_receipt(s.receipt.key) + assert not await h.store.read_receipt_boundaries(caller=s.binding.principal) + assert await h.store.ingest_receipt_batch(request, caller=s.binding.principal) == result + + +@pytest.mark.parametrize( + "fault", + [ + "rows", + "digest", + "manifest", + "profile", + "materialization", + "revision", + "obligation", + "readability", + ], +) +async def test_literal_sql_enforces_exact_artifact_and_readability_without_python_validator(fault): + async with receipt_harness("postgres") as h: + from psycopg import IntegrityError + + s = await receipt_case(h) + item = receipt_to_wire(s.receipt) + if fault == "rows": + item["observed_row_count"] += 1 + elif fault == "digest": + item["observed_canonical_content_digest"]["value"] = "f" * 64 + elif fault == "manifest": + item["observed_manifest_sha256"] = "f" * 64 + elif fault == "profile": + item["verification_profile"] = "manifest_checksums" + item["observed_manifest_sha256"] = "a" * 64 + elif fault == "materialization": + item["reporting_materialization_id"] = "unknown" + elif fault == "revision": + item["reporting_revision_id"] = "unknown" + elif fault == "obligation": + item["reporting_obligation_id"] = "unknown" + else: + await h.store.record_materialization_check( + ReportingMaterializationCheck( + s.attempt.scope, + s.attempt.reporting_materialization_id, + "corrupt-before-acceptance", + "corrupt", + s.receipt.observed_at - timedelta(microseconds=1), + ) + ) + candidate = receipt_record("revision_receipt", item, s.binding.principal, s.obligation) + if fault == "obligation": + candidate = replace( + candidate, scope=replace(candidate.scope, reporting_obligation_id="unknown") + ) + before = await h.image() + with pytest.raises(IntegrityError): + await raw_insert(h.pool, candidate) + assert await h.image() == before + + +@pytest.mark.parametrize( + "fault,expected", + [ + ("digest", "ADJUSTMENT_DIGEST_MISMATCH"), + ("revision", "REPORTING_RECORD_UNAVAILABLE"), + ("unknown", "REPORTING_RECORD_UNAVAILABLE"), + ("observed_before_adjustment", "ADJUSTMENT_ORDER_INVALID"), + ("correction_before_finality", "ADJUSTMENT_ORDER_INVALID"), + ], +) +async def test_official_adjustment_digest_ownership_and_order(receipts, fault, expected): + h = receipts + s = await receipt_case(h) + changes = ( + {"correction_observed_at": END - timedelta(seconds=1)} + if fault == "correction_before_finality" + else {} + ) + item = await adjustment_for(h, s, **changes) + if fault == "digest": + item["observed_adjustment_sha256"] = "f" * 64 + elif fault == "revision": + item["adjusts_reporting_revision_id"] = "unknown" + elif fault == "unknown": + item["reporting_adjustment_id"] = "unknown" + elif fault == "observed_before_adjustment": + item["observed_at"] = END.isoformat() + request = request_for(s, adjustment_receipts=[item]) + result = await h.store.ingest_receipt_batch(request, caller=s.binding.principal) + assert result["results"][0]["result"] == "recorded" + assert result["results"][1]["errors"][0]["code"] == expected + assert result["results"][1]["reporting_receipt_id"] == item["reporting_receipt_id"] + assert await h.store.ingest_receipt_batch(request, caller=s.binding.principal) == result + + +@pytest.mark.parametrize( + "fault", ["digest", "observed_before_adjustment", "correction_before_finality"] +) +async def test_literal_sql_adjustment_finality_digest_and_order(fault): + async with receipt_harness("postgres") as h: + from psycopg import IntegrityError + + s = await receipt_case(h) + changes = ( + {"correction_observed_at": END - timedelta(seconds=1)} + if fault == "correction_before_finality" + else {} + ) + item = await adjustment_for(h, s, **changes) + if fault == "digest": + item["observed_adjustment_sha256"] = "f" * 64 + elif fault == "observed_before_adjustment": + item["observed_at"] = END.isoformat() + candidate = receipt_record("adjustment_receipt", item, s.binding.principal, s.obligation) + before = await h.image() + with pytest.raises(IntegrityError): + await raw_insert(h.pool, candidate) + assert await h.image() == before + + +async def test_adjustment_rejected_current_leaf_replacement_and_accepted_terminality(receipts): + h = receipts + s = await receipt_case(h) + item = await adjustment_for(h, s) + rejected = {**item, "status": "rejected", "rejection_codes": ["LOAD_FAILED"]} + first = request_for(s, adjustment_receipts=[rejected]) + assert (await h.store.ingest_receipt_batch(first, caller=s.binding.principal))["results"][1][ + "result" + ] == "recorded" + accepted = { + **item, + "reporting_receipt_id": "accepted-adjustment-0002", + "supersedes_reporting_receipt_id": item["reporting_receipt_id"], + } + second = request_for(s, key="receipt-batch-0002", adjustment_receipts=[accepted]) + assert (await h.store.ingest_receipt_batch(second, caller=s.binding.principal))["results"][1][ + "result" + ] == "recorded" + third = request_for( + s, + key="receipt-batch-0003", + adjustment_receipts=[ + { + **accepted, + "reporting_receipt_id": "terminal-adjustment-0003", + "supersedes_reporting_receipt_id": accepted["reporting_receipt_id"], + } + ], + ) + result = await h.store.ingest_receipt_batch(third, caller=s.binding.principal) + assert result["results"][1]["errors"][0]["code"] == "ACCEPTED_RECEIPT_TERMINAL" + leaves = ( + await h.store.read_reconciliation_snapshot(caller=s.binding.principal) + ).current_receipts + assert {r.reporting_receipt_id for r in leaves} == { + s.receipt.reporting_receipt_id, + accepted["reporting_receipt_id"], + } + + +def damaged_chain(s, fault): + root = replace(s.receipt, status="rejected", rejection_codes=("LOAD_FAILED",)) + leaf = replace( + root, + reporting_receipt_id="leaf-receipt-0002", + supersedes_reporting_receipt_id=root.reporting_receipt_id, + ) + records = [root, leaf] + if fault == "fork": + records.append(replace(leaf, reporting_receipt_id="fork-receipt-0003")) + elif fault == "cycle": + records[0] = replace(root, supersedes_reporting_receipt_id=leaf.reporting_receipt_id) + elif fault == "gap": + records.pop(0) + elif fault == "cross_target": + records[0] = replace(root, reporting_revision_id="other-revision") + elif fault == "accepted_predecessor": + records[0] = replace(root, status="accepted", rejection_codes=()) + else: + cycle = replace( + root, + reporting_receipt_id="cycle-receipt-0003", + supersedes_reporting_receipt_id="cycle-receipt-0004", + ) + records += [ + cycle, + replace( + cycle, + reporting_receipt_id="cycle-receipt-0004", + supersedes_reporting_receipt_id=cycle.reporting_receipt_id, + ), + ] + return records, leaf + + +@pytest.mark.parametrize( + "fault", ["fork", "cycle", "gap", "cross_target", "accepted_predecessor", "disconnected_cycle"] +) +async def test_complete_receipt_chain_rejects_damage_with_one_apparent_leaf(fault): + async with receipt_harness("memory") as h: + s = await receipt_case(h) + records, leaf = damaged_chain(s, fault) + with pytest.raises(LedgerConflictError) as error: + current_receipt(tuple(records), leaf) + assert error.value.code == "REPORTING_HISTORY_CORRUPT" + + +@pytest.mark.parametrize( + "fault", ["fork", "cycle", "gap", "cross_target", "accepted_predecessor", "disconnected_cycle"] +) +async def test_literal_sql_blocks_forks_and_detects_damaged_existing_chains(fault): + from adcp.reporting.ledger._delivery_state import ( + change_id, + fingerprint, + payload, + storage_identity, + ) + from adcp.reporting.ledger.delivery_pg import _IDENTITY_COLUMNS + + from ._generation_support import NOW + + async with receipt_harness("postgres") as h: + from psycopg import IntegrityError, sql + from psycopg.types.json import Jsonb + + s = await receipt_case(h) + records, leaf = damaged_chain(s, fault) + # Deliberate owner-level corruption bypasses triggers only while seeding + # the invalid prior graph. The subsequent literal INSERT runs every + # production predicate, without a Python validator or head API. + from contextlib import nullcontext + + seed_before = await h.image() + # A fork cannot even be seeded with replica triggers: the reviewed + # unique successor index is an independent hard financial predicate. + with pytest.raises(IntegrityError) if fault == "fork" else nullcontext(): + async with h.pool.connection() as c, c.transaction(): + await c.execute("SET LOCAL session_replication_role = replica") + for record in records: + record = replace(record, received_at=NOW) + row = dict(zip(_IDENTITY_COLUMNS, storage_identity(record))) + row.update( + payload=Jsonb(payload(record)), + content_sha256=fingerprint(record), + change_id=change_id(record), + ) + await c.execute( + sql.SQL( + "INSERT INTO reporting_reconciliation_records ({}) VALUES ({})" + ).format( + sql.SQL(", ").join(map(sql.Identifier, row)), + sql.SQL(", ").join(sql.Placeholder() for _ in row), + ), + tuple(row.values()), + ) + if fault == "fork": + assert await h.image() == seed_before + return + new = replace( + leaf, + reporting_receipt_id="new-leaf-receipt-0099", + supersedes_reporting_receipt_id=leaf.reporting_receipt_id, + ) + before = await h.image() + with pytest.raises(IntegrityError): + await raw_insert(h.pool, new) + assert await h.image() == before + + +@pytest.mark.parametrize( + "method,profile,field,wrong", + [ + ("file_transfer", "canonical_digest", "observed_canonical_content_digest", None), + ("file_transfer", "manifest_checksums", "observed_manifest_sha256", "f" * 64), + ("file_transfer", "native_commit", "observed_native_version_ref", "wrong-version"), + ("dataset_share", "native_commit", "observed_native_version_ref", "wrong-version"), + ( + "warehouse_materialization", + "native_commit", + "observed_native_version_ref", + "wrong-version", + ), + ], +) +async def test_all_receipt_evidence_profiles_match_exact_immutable_artifact( + receipts, method, profile, field, wrong +): + from copy import deepcopy + + h = receipts + s = await receipt_case(h, method=method, profile=profile, billing=profile == "canonical_digest") + request = request_for(s) + bad = deepcopy(request) + if wrong is None: + bad["receipts"][0][field][ + "canonicalization_uri" + ] = "https://wrong.example.test/profile.json" + else: + bad["receipts"][0][field] = wrong + result = await h.store.ingest_receipt_batch(bad, caller=s.binding.principal) + assert result["results"][0]["errors"][0]["code"] == "RECEIPT_EVIDENCE_MISMATCH" + request["idempotency_key"] = "valid-artifact-receipt" + accepted = await h.store.ingest_receipt_batch(request, caller=s.binding.principal) + assert accepted["results"][0]["result"] == "recorded" + assert accepted["results"][0]["receipt"][field] == request["receipts"][0][field] + assert await h.store.ingest_receipt_batch(request, caller=s.binding.principal) == accepted + + +async def test_snapshot_adjustment_is_never_admitted_as_official_receipt_evidence(receipts): + h = receipts + s = await receipt_case(h, finality="snapshot", billing=False) + before = await h.image() + with pytest.raises(LedgerConflictError) as error: + await adjustment_for(h, s) + assert error.value.code == "ADJUSTMENT_REQUIRES_OFFICIAL" + assert await h.image() == before + item = { + "reporting_receipt_id": "snapshot-adjustment-0001", + "reporting_adjustment_id": "adjustment-1", + "adjusts_reporting_revision_id": s.revision.reporting_revision_id, + "status": "accepted", + "observed_at": s.receipt.observed_at.isoformat(), + "observed_adjustment_sha256": "0" * 64, + } + result = await h.store.ingest_receipt_batch( + request_for(s, adjustment_receipts=[item]), caller=s.binding.principal + ) + assert result["results"][0]["result"] == "recorded" + assert result["results"][1]["errors"][0]["code"] == "REPORTING_RECORD_UNAVAILABLE" diff --git a/tests/conformance/reporting/test_reporting_receipt_installed_pg.py b/tests/conformance/reporting/test_reporting_receipt_installed_pg.py new file mode 100644 index 000000000..bc8507a04 --- /dev/null +++ b/tests/conformance/reporting/test_reporting_receipt_installed_pg.py @@ -0,0 +1,139 @@ +"""Installed VCS/sdist Python 3.10 receipts through real transports after SIGKILL.""" + +import asyncio +import hashlib +import json +import shutil +from pathlib import Path + +import pytest + +from adcp.reporting.ledger.delivery import receipt_to_wire +from adcp.reporting.receipts import PgReportingReceiptStore + +from ._durable_materializer_support import DurableHarness +from ._generation_support import isolated_reporting_pool +from ._receipt_support import adjustment_for, receipt_case, request_for +from ._reconciliation_support import Clock +from .test_reporting_materializer_installed_pg import installed_materializer +from .test_reporting_materializer_packaging import ROOT, b1_wheels, built_distribution, run_step +from .test_reporting_receipt_process import worker + +__all__ = ["installed_materializer", "b1_wheels", "built_distribution"] + + +@pytest.fixture(scope="module") +def installed_receipts(installed_materializer): + root, python, _, original = installed_materializer + installer = ( + [shutil.which("uv"), "pip", "install", "--python", str(python)] + if shutil.which("uv") + else [str(python), "-m", "pip", "install"] + ) + run_step( + [*installer, "asgi-lifespan==2.1.0"], + label="receipt-installed-transport-test-lifespan", + cwd=root, + timeout=60, + ) + modules = dict(original["modules"]) + for name in ( + "reporting/receipts/pg.py", + "reporting/receipts/handler.py", + "reporting/receipts/wire.py", + "reporting/receipts/transport.py", + "server/mcp_tools.py", + "server/a2a_server.py", + "server/serve.py", + ): + modules["adcp." + name.removesuffix(".py").replace("/", ".")] = hashlib.sha256( + (ROOT / "src/adcp" / name).read_bytes() + ).hexdigest() + scripts = {} + for name in ("_receipt_process.py", "_receipt_installed.py", "_receipt_transport.py"): + scripts[name] = root / name + shutil.copy2(Path(__file__).with_name(name), scripts[name]) + return root, python, scripts, {**original, "modules": modules} + + +@pytest.mark.parametrize("notifications", [False, True]) +async def test_installed_receipt_sql_crash_and_mounted_replay(installed_receipts, notifications): + root, python, scripts, installed = installed_receipts + async with isolated_reporting_pool(autocommit=True) as pool: + settings = { + "installed": installed, + "conninfo": pool.conninfo, + "kwargs": pool.kwargs, + "notifications": notifications, + "action": "install", + } + + async def invoke(settings): + result = json.loads( + await asyncio.to_thread( + run_step, + [str(python), "-I", str(scripts["_receipt_installed.py"])], + label=f"receipt-installed-{settings['action']}", + cwd=root, + value=settings, + timeout=90, + ) + ) + assert "failure" not in result, result + return result + + ready = await invoke(settings) + assert ready["result"] == {"installed": True} + h = DurableHarness( + PgReportingReceiptStore(pool=pool, notifications=notifications), Clock(), pool + ) + s = await receipt_case(h) + adjustment = await adjustment_for(h, s) + request = request_for(s, adjustment_receipts=[adjustment]) + async with worker( + h, + s, + request, + pause="between_ordinals", + installed=installed, + python=python, + script=scripts["_receipt_process.py"], + ) as child: + await child.event("between_ordinals") + # The child owns the next ordinal's account lock. Kill it + # before using the public locked reader; ordinal zero is + # already durable and must survive that process death. + await child.kill() + original = await h.store.get_receipt(s.receipt.key) + assert original is not None + done = await invoke( + { + **settings, + "action": "replay", + "request": request, + "helper": str(scripts["_receipt_transport.py"]), + "caller": { + "account_id": s.obligation.account_id, + "consumer_id": s.binding.consumer_id, + }, + } + ) + assert done["result"]["results"][0]["receipt"] == receipt_to_wire(original) + assert done["origins"] == ready["origins"] + assert ( + await h.store.ingest_receipt_batch(request, caller=s.binding.principal) + == done["result"] + ) + assert len(await h.store.read_receipt_boundaries(caller=s.binding.principal)) == 2 + assert await h.queue() == ((), ()) + print( + json.dumps( + { + "installed_receipts": installed, + "notifications": notifications, + "origins": done["origins"], + "received_at": done["result"]["results"][0]["receipt"]["received_at"], + } + ), + flush=True, + ) diff --git a/tests/conformance/reporting/test_reporting_receipt_materializer.py b/tests/conformance/reporting/test_reporting_receipt_materializer.py new file mode 100644 index 000000000..8bbf63345 --- /dev/null +++ b/tests/conformance/reporting/test_reporting_receipt_materializer.py @@ -0,0 +1,64 @@ +"""Receipt composition preserves the one reviewed materializer transaction.""" + +from datetime import datetime, timezone + +import pytest + +from adcp.reporting.materializer import ReportingWriterError +from adcp.reporting.materializer.capture import MaterializerNotificationState + +from ._durable_materializer_support import durable_case +from ._receipt_support import receipt_harness +from ._reliable_support import ManualClock +from .test_reporting_receipt_transactions import fail_after + + +@pytest.mark.parametrize("backend", ["memory", "postgres"]) +async def test_receipt_store_atomic_finish_capture_ack_and_enabled_enqueue(backend, monkeypatch): + async with receipt_harness(backend, notifications=True) as h: + h.clock = ManualClock(datetime.now(timezone.utc)) + h.store._clock = h.clock if h.pool is None else None + case = await durable_case(h.store, reconciliation_mode="consumer_receipt") + lease = await case.claim() + prepared, verified = await case.verified(lease) + before = await h.image() + if h.pool is None: + fail_after(monkeypatch, MaterializerNotificationState, "enqueue") + else: + from psycopg import AsyncConnection + + fail_after( + monkeypatch, + AsyncConnection, + "execute", + asynchronous=True, + predicate=lambda args, _: isinstance(args[1], str) + and args[1].startswith("INSERT INTO reporting_materializer_notification_events"), + ) + with pytest.raises(ReportingWriterError): + await h.store.finish_materialization(lease, prepared=prepared, verified=verified) + monkeypatch.undo() + assert await h.image() == before + assert ( + await h.store.finish_materialization(lease, prepared=prepared, verified=verified) + ).state == "verified" + captured = await h.store.read_materializer_boundaries(caller=case.scope.principal) + assert len(captured) == 1 and captured[0].account_sequence == 1 + assert (await h.queue())[1] == ("quarantined",) + assert (await h.works())[0][1] == "acked" + assert not await h.store.read_receipt_boundaries(caller=case.scope.principal) + assert "reporting.delivery_ready" not in await h.ordinary_events() + + +@pytest.mark.parametrize("backend", ["memory", "postgres"]) +async def test_receipt_composition_preserves_complete_parent_notification_advertisement_veto( + backend, monkeypatch +): + # Reuse the reviewed same-store vector, now instantiated with the additive + # receipt store. This covers Core ledger_changed too, not just Managed. + from . import test_reporting_materializer_durable as parent_vectors + + monkeypatch.setattr(parent_vectors, "durable_harness", receipt_harness) + await parent_vectors.test_materializer_store_closes_core_advertisement_until_b24_admits_it( + backend + ) diff --git a/tests/conformance/reporting/test_reporting_receipt_migration.py b/tests/conformance/reporting/test_reporting_receipt_migration.py new file mode 100644 index 000000000..a4c19a527 --- /dev/null +++ b/tests/conformance/reporting/test_reporting_receipt_migration.py @@ -0,0 +1,133 @@ +"""Additive receipt manifests and migration preserve the approved materializer.""" + +import asyncio +import json +from importlib.resources import files + +import pytest + +from adcp.reporting.materializer import PgReportingMaterializerStore +from adcp.reporting.outbox._schema import schema_objects +from adcp.reporting.receipts import PgReportingReceiptStore, ReportingReceiptError + +from ._durable_materializer_support import DurableHarness, durable_case +from ._generation_support import isolated_reporting_pool +from ._receipt_support import receipt_case, receipt_harness, request_for +from ._reconciliation_support import Clock + +SQL = files("adcp.reporting.ledger").joinpath("reporting_receipt_ingestion.sql").read_text() +MANIFEST = json.loads(files("adcp.reporting.receipts").joinpath("required_schema.json").read_text()) +PARENT_MANIFEST = json.loads( + files("adcp.reporting.materializer").joinpath("required_schema.json").read_text() +) + + +@pytest.mark.parametrize("notifications", [False, True]) +@pytest.mark.parametrize("autocommit", [False, True]) +async def test_populated_repeated_concurrent_migration_preserves_parent_objects_captures_and_queues( + notifications, autocommit +): + async with isolated_reporting_pool(autocommit=autocommit) as pool: + old = PgReportingMaterializerStore(pool=pool, notifications=notifications) + await old.create_schema() + case = await durable_case(old) + assert (await case.service().run_once()).state == "verified" + h = DurableHarness(old, Clock(), pool) + before = await h.image() + captures = await old.read_materializer_boundaries(caller=case.scope.principal) + queue = await h.queue() + async with pool.connection() as c: + original = await schema_objects(c) + new = PgReportingReceiptStore(pool=pool, notifications=notifications) + with pytest.raises(ReportingReceiptError) as error: + await new.receipt_ingestion_ready() + assert error.value.code == "RECEIPT_SCHEMA_UNREADY" + await asyncio.gather(*(new.create_schema() for _ in range(3))) + await new.create_schema() + assert await new.receipt_ingestion_ready() + async with pool.connection() as c: + actual = await schema_objects(c) + assert {key: actual[key] for key in original} == original + assert len(PARENT_MANIFEST) == 187 + assert {key: actual[key] for key in PARENT_MANIFEST} == PARENT_MANIFEST + assert { + key: value for key, value in actual.items() if "reporting_receipt_ingestion_" in key + } == MANIFEST + after = await h.image() + assert {key: after[key] for key in before} == before + assert await new.read_materializer_boundaries(caller=case.scope.principal) == captures + assert await h.queue() == queue + assert queue[1] == (("quarantined",) if notifications else ()) + + +async def test_interrupted_receipt_migration_is_invisible_and_restart_converges(): + async with isolated_reporting_pool(autocommit=True) as pool: + await PgReportingMaterializerStore(pool=pool).create_schema() + entered, release = asyncio.Event(), asyncio.Event() + + async def migrate(): + async with pool.connection() as c, c.transaction(): + await c.execute(SQL) + entered.set() + await release.wait() + + task = asyncio.create_task(migrate()) + await asyncio.wait_for(entered.wait(), 10) + async with pool.connection() as c: + assert ( + await ( + await c.execute("SELECT to_regclass('reporting_receipt_ingestion_batches')") + ).fetchone() + )[0] is None + task.cancel() + with pytest.raises(asyncio.CancelledError): + await task + new = PgReportingReceiptStore(pool=pool) + with pytest.raises(ReportingReceiptError): + await new.receipt_ingestion_ready() + await new.create_schema() + assert await new.receipt_ingestion_ready() + + +@pytest.mark.parametrize( + "damage", + [ + "DROP INDEX reporting_receipt_ingestion_chain", + "ALTER TABLE reporting_receipt_ingestion_results " + "DISABLE TRIGGER reporting_receipt_ingestion_result", + "ALTER TABLE reporting_receipt_ingestion_batches ALTER COLUMN expected_count DROP NOT NULL", + "DROP TABLE reporting_receipt_ingestion_boundaries CASCADE", + "ALTER TABLE reporting_materializer_work DISABLE TRIGGER reporting_materializer_guard", + "ALTER TABLE reporting_reconciliation_records " + "DISABLE TRIGGER reporting_reconciliation_guard", + ], +) +@pytest.mark.parametrize("notifications", [False, True]) +async def test_old_partial_mismatched_schemas_fail_closed_even_for_completed_replay( + damage, notifications +): + async with receipt_harness("postgres", notifications=notifications) as h: + s = await receipt_case(h) + request = request_for(s) + await h.store.ingest_receipt_batch(request, caller=s.binding.principal) + async with h.pool.connection() as c: + await c.execute(damage) + before = await h.image() + with pytest.raises(ReportingReceiptError) as error: + await h.store.ingest_receipt_batch(request, caller=s.binding.principal) + assert error.value.code == "RECEIPT_SCHEMA_UNREADY" + assert await h.image() == before + + +async def test_receipt_store_does_not_claim_readiness_on_old_parent_schema(): + async with isolated_reporting_pool(autocommit=True) as pool: + old = PgReportingMaterializerStore(pool=pool, clock=Clock()) + await old.create_schema() + h = DurableHarness(old, Clock(), pool) + s = await receipt_case(h) + before = await h.image() + new = PgReportingReceiptStore(pool=pool) + with pytest.raises(ReportingReceiptError) as error: + await new.ingest_receipt_batch(request_for(s), caller=s.binding.principal) + assert error.value.code == "RECEIPT_SCHEMA_UNREADY" + assert await h.image() == before diff --git a/tests/conformance/reporting/test_reporting_receipt_process.py b/tests/conformance/reporting/test_reporting_receipt_process.py new file mode 100644 index 000000000..de35e553d --- /dev/null +++ b/tests/conformance/reporting/test_reporting_receipt_process.py @@ -0,0 +1,117 @@ +"""Actual process death and independent size-one pools resume durable ordinals.""" + +import asyncio +import sys +from contextlib import asynccontextmanager +from pathlib import Path + +import pytest + +from ._receipt_support import ( + adjustment_for, + batch_state, + receipt_case, + receipt_harness, + request_for, +) +from .test_reporting_materializer_process import Child + + +@asynccontextmanager +async def worker(h, s, request, *, pause=None, installed=None, python=None, script=None): + process = await asyncio.create_subprocess_exec( + str(python or sys.executable), + *(["-I"] if installed else []), + str(script or Path(__file__).with_name("_receipt_process.py")), + stdin=asyncio.subprocess.PIPE, + stdout=asyncio.subprocess.PIPE, + stderr=asyncio.subprocess.PIPE, + ) + child = Child(process) + try: + await child.send( + { + "conninfo": h.pool.conninfo, + "kwargs": h.pool.kwargs, + "caller": { + "account_id": s.obligation.account_id, + "consumer_id": s.binding.consumer_id, + }, + "request": request, + "pause": pause, + "notifications": h.store._notifications_enabled, + "installed": installed, + } + ) + yield child + finally: + await child.kill() + diagnostic = await process.stderr.read() + if process.returncode not in {0, -9}: + pytest.fail(diagnostic.decode()) + + +@pytest.mark.parametrize("notifications", [False, True]) +@pytest.mark.parametrize( + "point", + ["header", "receipt", "feed", "capture", "ordinal", "between_ordinals", "final", "committed"], +) +async def test_process_crash_resumes_original_ordinals_and_final_response(point, notifications): + async with receipt_harness("postgres", notifications=notifications) as h: + s = await receipt_case(h) + adjustment = await adjustment_for(h, s) + request = request_for(s, adjustment_receipts=[adjustment]) + async with worker(h, s, request, pause=point) as child: + await child.event(point) + expected = ( + 2 if point in {"final", "committed"} else (1 if point == "between_ordinals" else 0) + ) + if expected: + assert await batch_state(h) == ((expected, point == "committed"),) + else: + assert await batch_state(h) == () + # These are MVCC reads from another connection while the process + # owns the account lock. No partial receipt/feed/capture is visible. + async with h.pool.connection() as c: + row = await ( + await c.execute( + "SELECT (SELECT count(*) FROM reporting_reconciliation_records " + "WHERE namespace='receipt')," + " (SELECT count(*) FROM reporting_reconciliation_changes " + "WHERE namespace='receipt')," + " (SELECT count(*) FROM reporting_receipt_ingestion_boundaries)" + ) + ).fetchone() + assert row == (expected, expected, expected) + await child.kill() + original = await h.store.get_receipt(s.receipt.key) + async with worker(h, s, request) as resumed: + response = (await resumed.event("done"))["result"] + assert await asyncio.wait_for(resumed.process.wait(), 5) == 0 + assert [r["result"] for r in response["results"]] == ["recorded", "recorded"] + if original is not None: + from adcp.reporting.ledger.delivery import receipt_to_wire + + assert response["results"][0]["receipt"] == receipt_to_wire(original) + assert await h.store.ingest_receipt_batch(request, caller=s.binding.principal) == response + assert await batch_state(h) == ((2, True),) + assert len(await h.store.read_receipt_boundaries(caller=s.binding.principal)) == 2 + assert await h.queue() == ((), ()) + + +async def test_two_processes_serialize_with_size_one_pools_and_keep_original_recorded_outcomes(): + async with receipt_harness("postgres") as h: + s = await receipt_case(h) + request = request_for(s) + async with worker(h, s, request, pause="receipt") as first: + await first.event("receipt") + async with worker(h, s, request) as second: + pending = asyncio.create_task(second.event("done")) + await asyncio.sleep(0.1) + assert not pending.done() + await first.send({"continue": True}) + a = (await first.event("done"))["result"] + b = (await asyncio.wait_for(pending, 30))["result"] + assert a == b and a["results"][0]["result"] == "recorded" + assert await asyncio.wait_for(second.process.wait(), 5) == 0 + assert await asyncio.wait_for(first.process.wait(), 5) == 0 diff --git a/tests/conformance/reporting/test_reporting_receipt_rolling.py b/tests/conformance/reporting/test_reporting_receipt_rolling.py new file mode 100644 index 000000000..be41d13f8 --- /dev/null +++ b/tests/conformance/reporting/test_reporting_receipt_rolling.py @@ -0,0 +1,235 @@ +"""Eight actual historical installations on the approved B2.1 and B2.2 schemas.""" + +import asyncio +import json +import shutil +import sys +from datetime import datetime, timedelta, timezone +from pathlib import Path + +import pytest + +from adcp.reporting.ledger import ( + ReportingAdjustmentRecord, + ReportingControlTotalRecord, + ReportingRevisionReceiptRecord, +) +from adcp.reporting.ledger.delivery import adjustment_to_wire, receipt_to_wire +from adcp.reporting.materializer import PgReportingMaterializerStore +from adcp.reporting.outbox._schema import schema_objects +from adcp.reporting.receipts import PgReportingReceiptStore + +from ._durable_materializer_support import DurableHarness, durable_case +from ._generation_support import END, isolated_reporting_pool +from .test_reporting_materializer_process import worker +from .test_reporting_materializer_rolling import ARTIFACTS, build_frozen, frozen_call + +B21 = "8e18ca12b9a0c3750f80aa822058c02982ab3e52" + + +def receipt_probe(artifact): + root, python, _, settings = artifact + script = root / "receipt_frozen.py" + shutil.copy2(Path(__file__).with_name("_receipt_frozen.py"), script) + return root, python, script, settings + + +@pytest.fixture(scope="module") +def installed_b21(tmp_path_factory, request): + return receipt_probe(build_frozen("b21", tmp_path_factory, request, sha=B21)) + + +@pytest.fixture(scope="module", params=(*ARTIFACTS, "b21")) +def installed_receipt_history(request, tmp_path_factory, installed_b21): + if request.param == "b21": + return installed_b21 + return receipt_probe(build_frozen(request.param, tmp_path_factory, request)) + + +async def immutable_parent_rows(pool): + async with pool.connection() as connection: + return [ + await (await connection.execute(f"SELECT * FROM {table} ORDER BY 1,2,3")).fetchall() + for table in ( + "reporting_materializer_work", + "reporting_materializer_status_boundaries", + "reporting_materializer_status_heads", + "reporting_materializer_notification_events", + "reporting_materializer_notification_expansions", + ) + ] + + +async def test_actual_old_readers_and_writers_before_and_after_receipt_migration( + installed_receipt_history, installed_b21, tmp_path +): + artifact = installed_receipt_history[3]["artifact"] + # All eight default-off ordinary writers are required. A's inherited + # notification-readiness closure after C is compared separately on both + # sides. The four compatible notification writers also exercise enabled + # writes; no applicable case is hidden behind a parameter skip. + for notifications in [False, True] if artifact in {"b", "c", "b1", "b21"} else [False]: + async with isolated_reporting_pool(autocommit=True) as pool: + installer = await frozen_call( + installed_b21, pool, "install", notifications=notifications + ) + assert installer["manifest_objects"] == 187 + parent = PgReportingMaterializerStore(pool=pool, notifications=notifications) + case = await durable_case( + parent, + count=3, + finality="official", + required="official", + reconciliation_mode="consumer_receipt", + consumer=( + "frozen-buyer" + if artifact in {"beta15", "records", "integration", "a", "b"} + else "https://buyer.example.test/historical" + ), + legacy_definition=artifact == "beta15", + ) + # This is the actual approved B2.1 installation executing reserve / + # write / verification / finish / captured boundary / quarantine. + root, python, _, settings = installed_b21 + process_script = root / "parent_process.py" + shutil.copy2(Path(__file__).with_name("_materializer_process.py"), process_script) + destination = tmp_path / f"destination-{notifications}" + destination.mkdir(mode=0o700) + installed = {**settings, "python": list(sys.version_info[:2])} + h = DurableHarness(parent, None, pool) + async with worker( + h, + case, + destination, + python=python, + script=process_script, + installed=installed, + notifications=notifications, + ) as child: + produced = await child.event("done") + assert produced["state"] == "verified", produced + assert await asyncio.wait_for(child.process.wait(), 5) == 0 + outcome = (await case.outcomes())[0] + assert outcome.verification is not None + kwargs = { + "account": case.scope.principal.account_id, + "consumer": case.scope.principal.consumer_id, + "obligation": case.obligation.reporting_obligation_id, + "notifications": notifications, + } + before = await frozen_call( + installed_receipt_history, + pool, + "exercise", + phase="before", + receipt_count=0, + **kwargs, + ) + saved = await immutable_parent_rows(pool) + queue = await h.queue() + assert queue[1] == (("quarantined",) if notifications else ()) + async with pool.connection() as c: + old_objects = await schema_objects(c) + store = PgReportingReceiptStore(pool=pool, notifications=notifications) + await store.create_schema() + assert await store.receipt_ingestion_ready() + async with pool.connection() as c: + new_objects = await schema_objects(c) + assert {key: new_objects[key] for key in old_objects} == old_objects + assert all( + "reporting_receipt_ingestion_" in key + for key in new_objects.keys() - old_objects.keys() + ) + assert await immutable_parent_rows(pool) == saved + adjustment = ReportingAdjustmentRecord( + "frozen-adjustment", + case.config.account_id, + case.revision.reporting_revision_id, + "source_correction", + END, + END + timedelta(days=30), + (("spend", "-1.50"),), + END + timedelta(seconds=5), + END + timedelta(seconds=6), + managed_control_total_deltas=( + ReportingControlTotalRecord("spend", "-1.50", "decimal", "USD"), + ), + ) + await store.commit_adjustment(adjustment) + verification = outcome.verification + observed = datetime.now(timezone.utc).isoformat().replace("+00:00", "Z") + receipt = ReportingRevisionReceiptRecord( + case.scope, + "historical-revision-0001", + case.revision.reporting_revision_id, + outcome.reporting_materialization_id, + "accepted", + verification.verification_profile, + verification.row_count, + verification.control_totals, + datetime.fromisoformat(observed.replace("Z", "+00:00")), + observed_canonical_content_digest=verification.canonical_content_digest, + ) + request = { + "adcp_version": "3.2-rc.3", + "account": {"account_id": case.config.account_id}, + "idempotency_key": "historical-mixed-receipts", + "receipts": [receipt_to_wire(receipt)], + "adjustment_receipts": [ + { + "reporting_receipt_id": "historical-adjustment-0001", + "reporting_adjustment_id": adjustment.reporting_adjustment_id, + "adjusts_reporting_revision_id": case.revision.reporting_revision_id, + "observed_at": observed, + "status": "accepted", + "observed_adjustment_sha256": adjustment_to_wire(adjustment)[ + "canonical_adjustment_sha256" + ], + } + ], + } + response = await store.ingest_receipt_batch(request, caller=case.scope.principal) + assert [r["result"] for r in response["results"]] == ["recorded", "recorded"], response + captured = await store.read_receipt_boundaries(caller=case.scope.principal) + after = await frozen_call( + installed_receipt_history, + pool, + "exercise", + phase="after", + receipt_count=2, + **kwargs, + ) + assert before["ordinary_core"] and after["ordinary_core"] + assert before["notification_readiness"] == after["notification_readiness"] + assert ( + before["ordinary_materializer"] + == after["ordinary_materializer"] + == (artifact != "beta15") + ) + assert ( + await store.ingest_receipt_batch(request, caller=case.scope.principal) == response + ) + assert await store.read_receipt_boundaries(caller=case.scope.principal) == captured + assert len(captured) == 2 and captured[0].account_sequence > 1 + assert await h.queue() == queue + assert await immutable_parent_rows(pool) == saved + assert await parent.materializer_ready() + print( + json.dumps( + { + "receipt_rolling": artifact, + "notifications": notifications, + "parent": B21, + "parent_process_origins": produced["origins"], + "before": before, + "after": after, + "wheel_sha256": installed_receipt_history[3]["wheel_sha256"], + "additive_objects": len(new_objects.keys() - old_objects.keys()), + "parent_manifest": 187, + "receipt_count": 2, + "capture_count": len(captured), + "quarantine_preserved": True, + } + ), + flush=True, + ) diff --git a/tests/conformance/reporting/test_reporting_receipt_transactions.py b/tests/conformance/reporting/test_reporting_receipt_transactions.py new file mode 100644 index 000000000..e02dc9666 --- /dev/null +++ b/tests/conformance/reporting/test_reporting_receipt_transactions.py @@ -0,0 +1,339 @@ +"""Receipt, feed, captured dirty input, and ordinal share one rollback boundary.""" + +from copy import deepcopy + +import pytest + +from adcp.reporting.ledger.delivery import receipt_to_wire +from adcp.reporting.receipts import ReportingReceiptError + +from ._receipt_support import adjustment_for, batch_state, receipt_case, receipts, request_for + +__all__ = ["receipts"] + + +def fail_after(monkeypatch, owner, name, *, predicate=None, asynchronous=False): + original = getattr(owner, name) + if asynchronous: + + async def injected(*args, **kwargs): + result = await original(*args, **kwargs) + if predicate is None or predicate(args, kwargs): + raise RuntimeError("receipt fault") + return result + + else: + + def injected(*args, **kwargs): + result = original(*args, **kwargs) + if predicate is None or predicate(args, kwargs): + raise RuntimeError("receipt fault") + return result + + monkeypatch.setattr(owner, name, injected) + + +@pytest.mark.parametrize( + "point", + [ + "header", + "receipt", + "receipt_head", + "feed_head", + "feed", + "dirty", + "capture_head", + "account_capture_head", + "capture", + "ordinal", + ], +) +async def test_every_insertion_rolls_back_first_ordinal_in_both_notification_modes( + receipts, point, monkeypatch +): + h = receipts + s = await receipt_case(h) + request = request_for(s) + before = await h.image() + store_class = type(h.store) + if h.pool is None: + if point in {"receipt", "receipt_head", "feed_head", "feed"}: + # Memory publishes the evidence, caller feed and sequence in one + # list assignment; there are no independently mutable receipt heads. + method = "_append_reconciliation_change" + elif point in {"capture_head", "account_capture_head"}: + # Construction occurs after both newly allocated heads changed but + # before the boundary is appended. This catches incomplete rollback + # snapshots that forget first-use collections or sequence counters. + import adcp.reporting.receipts.memory as memory + + fail_after(monkeypatch, memory, "ReportingReceiptBoundary") + method = None + else: + method = { + "header": "_receipt_batch", + "dirty": "_capture_receipt", + "capture": "_capture_receipt", + "ordinal": "_append_receipt_result", + }[point] + if method is not None: + fail_after(monkeypatch, store_class, method) + else: + if point == "receipt_head": + async with h.pool.connection() as c: + await c.execute( + "CREATE FUNCTION receipt_test_fault() RETURNS trigger LANGUAGE plpgsql " + "AS $$ BEGIN RAISE EXCEPTION 'receipt fault'; END $$" + ) + await c.execute( + "CREATE TRIGGER receipt_test_fault AFTER INSERT OR UPDATE ON " + "reporting_receipt_heads FOR EACH ROW EXECUTE FUNCTION receipt_test_fault()" + ) + elif point in {"feed_head", "capture_head", "account_capture_head"}: + from psycopg import AsyncConnection + + needle = { + "feed_head": "INSERT INTO reporting_reconciliation_heads", + "capture_head": "INSERT INTO reporting_receipt_ingestion_heads", + "account_capture_head": ( + "UPDATE reporting_materializer_accounts SET captured_sequence" + ), + }[point] + fail_after( + monkeypatch, + AsyncConnection, + "execute", + predicate=lambda args, _: isinstance(args[1], str) and needle in args[1], + asynchronous=True, + ) + else: + method = { + "header": "_receipt_batch_on", + "receipt": "_insert", + "feed": "_append_reconciliation_change", + "dirty": "_capture_receipt_on", + "capture": "_capture_receipt_on", + "ordinal": "_insert_receipt_result_on", + }[point] + fail_after(monkeypatch, store_class, method, asynchronous=True) + with pytest.raises((RuntimeError, ReportingReceiptError)): + await h.store.ingest_receipt_batch(request, caller=s.attempt.scope.principal) + monkeypatch.undo() + assert await h.image() == before + if h.pool is not None and point == "receipt_head": + async with h.pool.connection() as c: + await c.execute("DROP TRIGGER receipt_test_fault ON reporting_receipt_heads") + result = await h.store.ingest_receipt_batch(request, caller=s.attempt.scope.principal) + assert result["results"][0]["result"] == "recorded" + + +@pytest.mark.parametrize("point", ["persistence", "assembly"]) +async def test_final_response_fault_preserves_prior_ordinals_and_original_recorded_results( + receipts, point, monkeypatch +): + h = receipts + s = await receipt_case(h) + adjustment = await adjustment_for(h, s) + request = request_for(s, adjustment_receipts=[adjustment]) + method = ( + "_assemble_receipt_response" + if point == "assembly" + else ("_save_receipt_response" if h.pool is None else "_save_receipt_response_on") + ) + fail_after( + monkeypatch, + type(h.store), + method, + asynchronous=h.pool is not None and point == "persistence", + ) + with pytest.raises((RuntimeError, ReportingReceiptError)): + await h.store.ingest_receipt_batch(request, caller=s.attempt.scope.principal) + monkeypatch.undo() + assert await batch_state(h) == ((2, False),) + original = await h.store.get_receipt(s.receipt.key) + before = await h.store.read_receipt_boundaries(caller=s.attempt.scope.principal) + assert len(before) == 2 + result = await h.store.ingest_receipt_batch(request, caller=s.attempt.scope.principal) + assert [r["result"] for r in result["results"]] == ["recorded", "recorded"] + assert result["results"][0]["receipt"] == receipt_to_wire(original) + assert await h.store.read_receipt_boundaries(caller=s.attempt.scope.principal) == before + assert await batch_state(h) == ((2, True),) + + +async def test_second_ordinal_failure_preserves_successful_sibling_and_durable_failure( + receipts, monkeypatch +): + h = receipts + s = await receipt_case(h) + adjustment = await adjustment_for(h, s) + request = request_for( + s, + receipts=[ + { + **receipt_to_wire(s.receipt), + "reporting_receipt_id": "missing-first-0001", + "reporting_revision_id": "missing", + }, + receipt_to_wire(s.receipt), + ], + adjustment_receipts=[adjustment], + ) + if h.pool is None: + # Fail after appending the second ordinal: its newly recorded receipt, + # private capture and result must roll back; ordinal-zero failure stays. + fail_after( + monkeypatch, + type(h.store), + "_append_receipt_result", + predicate=lambda args, _: len(args[1].results) == 2, + ) + else: + fail_after( + monkeypatch, + type(h.store), + "_insert_receipt_result_on", + predicate=lambda args, _: args[4] == 1, + asynchronous=True, + ) + with pytest.raises((RuntimeError, ReportingReceiptError)): + await h.store.ingest_receipt_batch(request, caller=s.attempt.scope.principal) + monkeypatch.undo() + assert await batch_state(h) == ((1, False),) + assert await h.store.get_receipt(s.receipt.key) is None + assert await h.store.read_receipt_boundaries(caller=s.attempt.scope.principal) == () + changed = deepcopy(request) + changed["receipts"][0]["reporting_revision_id"] = s.revision.reporting_revision_id + before = await h.image() + with pytest.raises(ReportingReceiptError) as error: + await h.store.ingest_receipt_batch(changed, caller=s.attempt.scope.principal) + assert error.value.code == "IDEMPOTENCY_CONFLICT" + assert await h.image() == before + result = await h.store.ingest_receipt_batch(request, caller=s.attempt.scope.principal) + assert [r["result"] for r in result["results"]] == ["failed", "recorded", "recorded"] + + +async def test_captured_inputs_remain_private_and_frozen_after_current_state_changes(receipts): + h = receipts + s = await receipt_case(h, consumer_id="https://buyer.example.test/one") + sibling = await receipt_case(h, consumer_id="https://buyer.example.test/two") + await h.store.ingest_receipt_batch(request_for(s), caller=s.attempt.scope.principal) + frozen = await h.store.read_receipt_boundaries(caller=s.attempt.scope.principal) + assert len(frozen) == 1 + assert not await h.store.read_receipt_boundaries(caller=sibling.attempt.scope.principal) + await h.store.ingest_receipt_batch(request_for(sibling), caller=sibling.attempt.scope.principal) + await h.store.set_revision_readable( + account_id=s.obligation.account_id, + reporting_revision_id=s.revision.reporting_revision_id, + readable=False, + ) + assert await h.store.read_receipt_boundaries(caller=s.attempt.scope.principal) == frozen + assert all( + ( + r.scope.consumer_id == s.binding.consumer_id + if hasattr(r, "scope") + else r.consumer_id == s.binding.consumer_id + ) + for r in frozen[0].reconciliation + ) + assert frozen[0].core.consumer_ids == (s.binding.consumer_id,) + assert all(r.readable for r in frozen[0].core.revisions) + other = await h.store.read_receipt_boundaries(caller=sibling.attempt.scope.principal) + assert frozen[0].account_sequence < other[0].account_sequence + + +@pytest.mark.parametrize("backend", ["memory", "postgres"]) +async def test_ordinary_status_dirty_insert_failure_rolls_back_receipt_and_ordinal( + backend, monkeypatch +): + from ._receipt_support import receipt_harness + + async with receipt_harness(backend, notifications=True) as h: + s = await receipt_case(h) + before = await h.image() + if h.pool is None: + fail_after(monkeypatch, type(h.store), "_dirty_status") + else: + from psycopg import AsyncConnection + + fail_after( + monkeypatch, + AsyncConnection, + "execute", + asynchronous=True, + predicate=lambda args, _: isinstance(args[1], str) + and args[1].startswith("INSERT INTO reporting_status_dirty"), + ) + with pytest.raises((RuntimeError, ReportingReceiptError)): + await h.store.ingest_receipt_batch(request_for(s), caller=s.binding.principal) + monkeypatch.undo() + assert await h.image() == before + assert (await h.store.ingest_receipt_batch(request_for(s), caller=s.binding.principal))[ + "results" + ][0]["result"] == "recorded" + + +@pytest.mark.parametrize("damage", ["id", "kind", "outcome", "received_at", "ordinal"]) +async def test_corrupt_durable_prefix_blocks_resume_before_sibling_mutation( + receipts, monkeypatch, damage +): + from adcp.reporting.canonical_json import canonical_json_sha256_v1 + from adcp.reporting.receipts.wire import ReceiptBatch + + h = receipts + s = await receipt_case(h) + request = request_for(s, adjustment_receipts=[await adjustment_for(h, s)]) + # Retain the first committed ordinal, then corrupt only that immutable + # prefix through a deliberately privileged fixture. Restore guards before + # asking the actual store to resume. + method = "_append_receipt_result" if h.pool is None else "_insert_receipt_result_on" + fail_after( + monkeypatch, + type(h.store), + method, + asynchronous=h.pool is not None, + predicate=( + (lambda args, _: len(args[1].results) == 2) + if h.pool is None + else (lambda args, _: args[4] == 1) + ), + ) + with pytest.raises((RuntimeError, ReportingReceiptError)): + await h.store.ingest_receipt_batch(request, caller=s.binding.principal) + monkeypatch.undo() + if h.pool is None: + state = next(iter(h.store._receipt_batches.values())) + value = deepcopy(state.results[0]) + else: + async with h.pool.connection() as c: + value = ( + await ( + await c.execute("SELECT result FROM reporting_receipt_ingestion_results") + ).fetchone() + )[0] + if damage == "id": + value["receipt"]["reporting_receipt_id"] = "unrelated-receipt-0001" + elif damage == "kind": + value["adjustment_receipt"] = value.pop("receipt") + elif damage == "outcome": + value["receipt"]["status"] = "unrecognized" + elif damage == "received_at": + del value["receipt"]["received_at"] + else: + value["reporting_receipt_id"] = "extra-ordinal" + if h.pool is None: + state.results = (value,) + else: + from psycopg.types.json import Jsonb + + async with h.pool.connection() as c, c.transaction(): + await c.execute("SET LOCAL session_replication_role = replica") + await c.execute( + "UPDATE reporting_receipt_ingestion_results SET result=%s,content_sha256=%s", + (Jsonb(value), canonical_json_sha256_v1(value)), + ) + before = await h.image() + with pytest.raises(ReportingReceiptError) as error: + await h.store.ingest_receipt_batch(request, caller=s.binding.principal) + assert error.value.code == "RECEIPT_HISTORY_CORRUPT" + assert await h.image() == before + assert len(ReceiptBatch.parse(request).items) == 2 diff --git a/tests/conformance/reporting/test_reporting_receipt_transports.py b/tests/conformance/reporting/test_reporting_receipt_transports.py new file mode 100644 index 000000000..5dd3b413b --- /dev/null +++ b/tests/conformance/reporting/test_reporting_receipt_transports.py @@ -0,0 +1,609 @@ +"""Mounted MCP/A2A auth, canonical identity, raw wire shape and durable replay.""" + +from copy import deepcopy +from dataclasses import replace + +import pytest +from google.protobuf.json_format import MessageToDict, ParseDict +from google.protobuf.struct_pb2 import Value +from jsonschema import Draft7Validator + +from adcp.decisioning import Account, AuthInfo, RequestContext +from adcp.decisioning.registry import BuyerAgent, HttpSigCredential +from adcp.exceptions import ADCPTaskError +from adcp.reporting.receipts.wire import receipt_schema +from adcp.server import ToolContext + +from ._receipt_support import adjustment_for, receipt_case, receipt_harness, receipts, request_for +from ._receipt_transport import MountedReceipts, error_code + +__all__ = ["receipts"] + + +@pytest.mark.parametrize("hydrated", [False, True]) +async def test_actual_mounts_reauthorize_exact_replay_and_bypass_both_generic_caches( + receipts, hydrated +): + h = receipts + one = await receipt_case(h, consumer_id="https://buyer.example.test/one") + two = await receipt_case(h, consumer_id="https://buyer.example.test/two") + mount = MountedReceipts(h, hydrated=hydrated) + mount.authorize(one) + mount.authorize(two, token="token-two") + adjustment = await adjustment_for(h, one) + request = request_for( + one, + adjustment_receipts=[adjustment], + context={"buyer": "original context", "\ue000": "BMP", "\U00010000": "supplementary"}, + ) + async with mount.client() as client: + code, first = await mount.mcp(client, request) + assert code == 200 and first.get("status") == "completed", first + assert [r["result"] for r in first["results"]] == ["recorded", "recorded"] + _, replay = await mount.a2a(client, deepcopy(request)) + assert replay == first + assert "replayed" not in replay + _, sibling = await mount.mcp(client, request, token="token-two") + assert [r["result"] for r in sibling["results"]] == ["recorded", "recorded"] + assert await h.store.get_receipt(one.receipt.key) + assert await h.store.get_receipt(two.receipt.key) + assert len(mount.auth_calls) == 3 + if hydrated: + assert len({c.caller_identity for c in mount.contexts}) == 1 + mount.grants.remove((one.obligation.account_id, one.binding.consumer_id)) + before = await h.image() + for call in (mount.mcp, mount.a2a): + _, denied = await call(client, request) + assert error_code(denied) == "UNAUTHORIZED" + assert await h.image() == before + assert len(mount.auth_calls) == 5 + del mount.tokens["token-one"] + for call in (mount.mcp, mount.a2a): + code, _ = await call(client, request) + assert code == 401 + + +async def test_same_consumer_key_across_two_accounts_is_not_a_generic_cache_collision(receipts): + h = receipts + first = await receipt_case(h, account_id="account:a", consumer_id="b:c") + second = await receipt_case(h, account_id="account:a:b", consumer_id="b:c") + mount = MountedReceipts(h) + mount.authorize(first) + mount.grants.add((second.obligation.account_id, second.binding.consumer_id)) + mount.accounts[second.obligation.account_id] = second.obligation.account_id + async with mount.client() as client: + _, a = await mount.mcp(client, request_for(first)) + _, b = await mount.a2a(client, request_for(second)) + assert a["results"][0]["result"] == b["results"][0]["result"] == "recorded" + _, replay = await mount.mcp(client, request_for(second)) + assert replay == b + changed = request_for(second, context={"changed": True}) + _, conflict = await mount.a2a(client, changed) + assert error_code(conflict) == "IDEMPOTENCY_CONFLICT" + + +@pytest.mark.parametrize("first_route", ["mcp", "a2a"]) +@pytest.mark.parametrize("v1", [False, True]) +async def test_protobuf_integer_spellings_preserve_whole_request_identity_and_timestamps( + receipts, first_route, v1 +): + h = receipts + s = await receipt_case(h) + mount = MountedReceipts(h, hydrated=True, version="3.2-rc.3") + mount.authorize(s) + request = request_for( + s, + context={"integer": 42, "nested": [100], "bounds": [-9007199254740991, 9007199254740991]}, + ) + # Use the actual protobuf conversion that an A2A client performs. Python's + # JSON printer then emits 1.0, so reading the original HTTP bytes must accept + # that exact integral spelling without rounding arbitrary fractional input. + proto = Value() + ParseDict(request, proto) + protobuf_request = MessageToDict(proto) + assert type(protobuf_request["receipts"][0]["observed_row_count"]) is float + assert type(protobuf_request["context"]["integer"]) is float + assert Draft7Validator(receipt_schema("request", version="3.2-rc.3")).is_valid(protobuf_request) + requests = {"mcp": request, "a2a": protobuf_request} + other = "a2a" if first_route == "mcp" else "mcp" + async with mount.client() as client: + _, first = await getattr(mount, first_route)( + client, requests[first_route], **({"v1": v1} if first_route == "a2a" else {}) + ) + assert first["results"][0]["result"] == "recorded", first + _, replay = await getattr(mount, other)( + client, requests[other], **({"v1": v1} if other == "a2a" else {}) + ) + assert replay == first + assert replay["context"] == request["context"] + _, exponent = await mount.mcp( + client, + request, + mutate_wire=lambda w: w.replace( + '"observed_row_count": 1', '"observed_row_count": 1.0e+0' + ), + ) + assert exponent == first + assert len(mount.auth_calls) == 3 + + +@pytest.mark.parametrize("chunk_size", [1, 257]) +async def test_request_bound_capture_preserves_chunked_bytes_and_read_ahead_overflow( + receipts, monkeypatch, chunk_size +): + h = receipts + s = await receipt_case(h) + mount = MountedReceipts(h, hydrated=True) + mount.authorize(s) + # Shorten only the read-ahead window; the raw capture still has its real + # 10 MiB bound. The rest of this valid request must reach the normal decoder. + monkeypatch.setattr("adcp.reporting.receipts.transport.MAX_RECEIPT_BODY_BYTES", 256) + request = request_for(s, context={"original": "same request bytes"}) + + def chunked(wire): + async def chunks(): + raw = wire.encode() + for i in range(0, len(raw), chunk_size): + yield b"" + yield raw[i : i + chunk_size] + + return chunks() + + async with mount.client() as client: + _, first = await mount.a2a(client, request, mutate_wire=chunked) + assert first["results"][0]["result"] == "recorded", first + assert (await mount.mcp(client, request))[1] == first + assert len(mount.auth_calls) == 2 + + +@pytest.mark.parametrize("registry_kind", ["api_key", "oauth", "http_sig"]) +async def test_api_oauth_signed_registry_identity_is_refreshed_on_mounted_replay( + receipts, registry_kind +): + h = receipts + s = await receipt_case(h, consumer_id="https://buyer.example.test/agent") + mount = MountedReceipts(h, hydrated=True, registry_kind=registry_kind) + mount.authorize(s) + async with mount.client() as client: + _, first = await mount.mcp(client, request_for(s)) + assert first["results"][0]["result"] == "recorded", first + _, replay = await mount.a2a(client, request_for(s)) + assert replay == first + assert len(mount.registry.calls) >= 2 + for key, agent in mount.registry.agents.items(): + mount.registry.agents[key] = replace(agent, status="blocked") + before = await h.image() + _, denied = await mount.a2a(client, request_for(s)) + assert error_code(denied) == "UNAUTHORIZED" + assert await h.image() == before + + +@pytest.mark.parametrize( + "bad", + [ + "absent", + "anonymous", + "tenant_only", + "cache_only", + "auth_conflict", + "signed_conflict", + "agent_conflict", + "metadata_conflict", + "account_conflict", + ], +) +async def test_all_trusted_identities_must_agree_and_cache_and_tenant_never_identify_consumer(bad): + async with receipt_harness("memory") as h: + s = await receipt_case(h, consumer_id="https://buyer.example.test/agent") + mount = MountedReceipts(h, hydrated=True) + mount.authorize(s) + consumer = s.binding.consumer_id + context = RequestContext( + caller_identity="cache-only", + account=Account(id=s.obligation.account_id), + auth_info=AuthInfo(kind="bearer", principal=consumer, credential=None), + auth_principal=consumer, + buyer_agent=BuyerAgent(consumer, "Buyer", "active"), + ) + if bad == "absent": + context = ToolContext() + elif bad == "anonymous": + context = ToolContext(caller_identity="anonymous") + elif bad == "tenant_only": + context = ToolContext(tenant_id=consumer) + elif bad == "cache_only": + context = replace( + context, + auth_info=None, + auth_principal=None, + buyer_agent=None, + caller_identity=consumer, + ) + elif bad == "auth_conflict": + context = replace( + context, auth_info=AuthInfo(kind="bearer", principal="other", credential=None) + ) + elif bad == "signed_conflict": + context = replace( + context, + auth_info=AuthInfo( + kind="http_sig", + principal=consumer, + credential=HttpSigCredential( + "http_sig", "key", "https://other.example.test/agent", 1.0 + ), + ), + ) + elif bad == "agent_conflict": + context = replace( + context, + buyer_agent=BuyerAgent("https://other.example.test/agent", "Other", "active"), + ) + elif bad == "metadata_conflict": + context.metadata["adcp.auth_info"] = AuthInfo( + kind="bearer", principal="other", credential=None + ) + else: + context = replace(context, account=replace(context.account, id="other-account")) + before = await h.image() + with pytest.raises(ADCPTaskError) as error: + await mount.handler.sync_reporting_receipts(request_for(s), context) + assert error.value.errors[0].code == "UNAUTHORIZED" + assert await h.image() == before + + +@pytest.mark.parametrize( + "bad", ["empty", "empty_adjustments", "received_at", "duplicate", "combined101", "spoof"] +) +async def test_actual_mount_shape_failure_is_task_invalid_request_with_no_writes(bad): + async with receipt_harness("memory") as h: + s = await receipt_case(h) + mount = MountedReceipts(h) + mount.authorize(s) + request = request_for(s) + if bad == "empty": + request["receipts"] = [] + elif bad == "empty_adjustments": + request["adjustment_receipts"] = [] + elif bad == "received_at": + request["receipts"][0]["received_at"] = "2026-09-01T02:00:00Z" + elif bad == "duplicate": + request["receipts"] *= 2 + elif bad == "spoof": + request["consumer_id"] = "someone-else" + else: + request["receipts"] = [ + {**request["receipts"][0], "reporting_receipt_id": f"receipt-{i:016d}"} + for i in range(101) + ] + before = await h.image() + async with mount.client(validation=None) as client: + for call in (mount.mcp, mount.a2a): + _, response = await call(client, request) + assert error_code(response) == "INVALID_REQUEST" + assert await h.image() == before + assert mount.auth_calls == [] + + +@pytest.mark.parametrize("version", [None, "3.2-rc.3"]) +@pytest.mark.parametrize("fallback", [False, True]) +async def test_pinned_unpinned_and_fallback_mcp_schemas_keep_combined_and_received_rules( + version, fallback, monkeypatch +): + async with receipt_harness("memory") as h: + s = await receipt_case(h) + adjustment = await adjustment_for(h, s) + mount = MountedReceipts(h, version=version) + mount.authorize(s) + if fallback: + monkeypatch.setattr( + "adcp.server.mcp_tools._ensure_pydantic_schemas_applied", lambda _: None + ) + async with mount.client() as client: + _, inventory = await mount.mcp(client, inventory=True) + definition = next( + t for t in inventory["tools"] if t["name"] == "sync_reporting_receipts" + ) + request = request_for(s, adjustment_receipts=[adjustment]) + validator = Draft7Validator(definition["inputSchema"]) + assert validator.is_valid(request) + assert validator.is_valid({k: v for k, v in request.items() if k != "receipts"}) + for change in ( + {"receipts": []}, + {"adjustment_receipts": []}, + {"receipts": [{**request["receipts"][0], "received_at": "2026-09-01T02:00:00Z"}]}, + {"receipts": request["receipts"] * 51, "adjustment_receipts": [adjustment] * 50}, + ): + assert not validator.is_valid({**request, **change}) + absent = { + k: v for k, v in request.items() if k not in {"receipts", "adjustment_receipts"} + } + assert not validator.is_valid(absent) + _, result = await mount.mcp(client, request) + assert Draft7Validator(definition["outputSchema"]).is_valid(result), result + del result["results"][0]["receipt"]["received_at"] + assert not Draft7Validator(definition["outputSchema"]).is_valid(result) + # Per-mount overlays never mutate another caller's cached schema. + definition["inputSchema"]["anyOf"] = [] + assert receipt_schema("request")["anyOf"] + + +async def test_receipt_hook_rewrite_rejected_and_response_enhancer_cannot_mutate_replay(): + async with receipt_harness("memory") as h: + s = await receipt_case(h) + mount = MountedReceipts(h) + mount.authorize(s) + request = request_for(s) + original = deepcopy(request) + + def rewrite(task, params): + params["receipts"][0]["consumer_commit_ref"] = "changed" + return params + + async with mount.client( + pre_validation_hooks={"sync_reporting_receipts": [rewrite]} + ) as client: + _, error = await mount.mcp(client, request) + assert error_code(error) == "INVALID_REQUEST" + assert request == original + assert not await h.store.get_receipt(s.receipt.key) + mount.sessions.clear() + + def enhancer(*args): + raise AssertionError("a receipt response must not be enhanced") + + async with mount.client(response_enhancer=enhancer) as client: + _, first = await mount.mcp(client, request) + _, replay = await mount.a2a(client, request) + assert first == replay and first["results"][0]["result"] == "recorded" + + +@pytest.mark.parametrize("v1", [False, True]) +async def test_a2a_to_mcp_replay_after_mount_restart_keeps_every_original_ordinal_and_timestamp( + receipts, v1 +): + h = receipts + s = await receipt_case(h, consumer_id="https://buyer.example.test/restarted") + adjustment = await adjustment_for(h, s) + request = request_for(s, adjustment_receipts=[adjustment]) + request["receipts"].insert( + 0, + { + **request["receipts"][0], + "reporting_receipt_id": "failed-ordinal-0000", + "reporting_revision_id": "unknown", + }, + ) + mount = MountedReceipts(h, hydrated=True) + mount.authorize(s) + async with mount.client() as client: + _, first = await mount.a2a(client, request, v1=v1) + assert [r["result"] for r in first["results"]] == ["failed", "recorded", "recorded"], first + if h.pool is not None: + from adcp.reporting.receipts import PgReportingReceiptStore + + h.store = PgReportingReceiptStore(pool=h.pool, notifications=h.store._notifications_enabled) + restarted = MountedReceipts(h, hydrated=True) + restarted.authorize(s) + async with restarted.client() as client: + _, replay = await restarted.mcp(client, request) + assert replay == first + _, again = await restarted.a2a(client, request, v1=v1) + assert again == first + assert len(restarted.auth_calls) == 2 + assert "replayed" not in again + + +@pytest.mark.parametrize( + "fault", + [ + "fractional", + "rounded_fraction", + "deep_rounded_fraction", + "unsafe_integer", + "unsafe_exponent", + "unrepresentable_exponent", + "negative_integer", + "nonfinite", + "boolean_count", + "numeric_string_field", + "duplicate_key", + "duplicate_nested", + "empty_array", + "metadata_shortcut", + "unpaired_surrogate", + ], +) +@pytest.mark.parametrize("route", ["mcp", "a2a"]) +async def test_real_raw_strict_json_and_numeric_admission_fail_before_batch_mutation( + receipts, fault, route +): + h = receipts + s = await receipt_case(h) + mount = MountedReceipts(h, version="3.2-rc.3") + mount.authorize(s) + request = request_for(s) + mutations = { + "fractional": lambda w: w.replace('"observed_row_count": 1', '"observed_row_count": 1.5'), + "rounded_fraction": lambda w: w.replace( + '"observed_row_count": 1', '"observed_row_count": 1.0000000000000001' + ), + "deep_rounded_fraction": lambda w: w.replace( + '"observed_row_count": 1', '"observed_row_count": 1.000000000000000000001' + ), + "unsafe_integer": lambda w: w.replace( + '"observed_row_count": 1', '"observed_row_count": 9007199254740993' + ), + "unsafe_exponent": lambda w: w.replace( + '"observed_row_count": 1', '"observed_row_count": 9.007199254740993e15' + ), + "unrepresentable_exponent": lambda w: w.replace( + '"observed_row_count": 1', '"observed_row_count": 1e99999999999999999999999' + ), + "negative_integer": lambda w: w.replace( + '"observed_row_count": 1', '"observed_row_count": -1.0' + ), + "nonfinite": lambda w: w.replace('"observed_row_count": 1', '"observed_row_count": NaN'), + "boolean_count": lambda w: w.replace( + '"observed_row_count": 1', '"observed_row_count": true' + ), + "duplicate_key": lambda w: w.replace( + '"account": {', '"account": {"account_id":"other"}, "account": {' + ), + "duplicate_nested": lambda w: w.replace( + '"observed_row_count": 1', '"observed_row_count": 999, "observed_row_count": 1' + ), + } + if fault == "empty_array": + request["receipts"] = [] + if fault == "unpaired_surrogate": + request["context"] = {"unicode": "\ud800"} + if fault == "numeric_string_field": + request["receipts"][0]["observed_control_totals"][0]["value"] = 1.0 + if fault == "metadata_shortcut": + request["receipts"] = [] + request["context"] = { + "adcp.receipt_ingress.raw_body": request_for(s), + "adcp.a2a_parsed_request": request_for(s), + } + before = await h.image() + async with mount.client(validation=None) as client: + _, result = await getattr(mount, route)(client, request, mutate_wire=mutations.get(fault)) + if fault == "unpaired_surrogate": + assert result["error"]["code"] == -32700, result + else: + assert error_code(result) == "INVALID_REQUEST", result + assert await h.image() == before + assert mount.auth_calls == [] + + +@pytest.mark.parametrize("route", ["mcp", "a2a"]) +async def test_malformed_numeric_token_is_rejected_by_the_actual_json_decoder(receipts, route): + h = receipts + s = await receipt_case(h) + mount = MountedReceipts(h, version="3.2-rc.3") + mount.authorize(s) + request = request_for(s) + before = await h.image() + async with mount.client() as client: + status, response = await getattr(mount, route)( + client, + request, + mutate_wire=lambda w: w.replace('"observed_row_count": 1', '"observed_row_count": 1e'), + ) + # This invalid JSON cannot reach task admission. Preserve each real + # transport's parse-error envelope instead of inventing a task result. + assert status in {200, 400, 422}, response + if status == 200: + assert response["error"]["code"] in {-32700, -32600, -32602}, response + assert await h.image() == before + assert mount.auth_calls == [] + + +@pytest.mark.parametrize("route", ["mcp", "a2a"]) +async def test_raw_capture_cannot_supply_auth_or_cross_request_body_or_override_actual_account( + route, +): + async with receipt_harness("memory") as h: + s = await receipt_case(h) + mount = MountedReceipts(h) + mount.authorize(s) + request = request_for(s) + async with mount.client() as client: + call = getattr(mount, route) + _, first = await call(client, request) + assert first["results"][0]["result"] == "recorded" + before = await h.image() + _, invalid = await call(client, {**request, "receipts": []}) + assert error_code(invalid) == "INVALID_REQUEST" + changed = { + **request, + "account": {"account_id": "other"}, + "context": {"adcp.receipt_ingress.raw_body": request}, + } + _, denied = await call(client, changed) + assert error_code(denied) == "UNAUTHORIZED" + if route == "mcp": + # Avoid a fresh initialize in this helper: the authenticated + # tool request itself must reject a forged token. + mount.sessions["forged"] = mount.sessions["token-one"] + assert (await call(client, request, token="forged"))[0] == 401 + assert await h.image() == before + + +async def test_unsupported_version_rejected_on_both_routes_before_ingress(): + async with receipt_harness("memory") as h: + s = await receipt_case(h) + mount = MountedReceipts(h) + mount.authorize(s) + before = await h.image() + async with mount.client() as client: + for call in (mount.mcp, mount.a2a): + _, denied = await call(client, request_for(s, adcp_version="99.0")) + assert error_code(denied) == "VERSION_UNSUPPORTED" + assert await h.image() == before + + +@pytest.mark.parametrize("route", ["mcp", "a2a"]) +async def test_raw_capture_size_bound_and_fallback_are_closed(monkeypatch, route): + from adcp.reporting.receipts.transport import receipt_body_receive + + async with receipt_harness("memory") as h: + s = await receipt_case(h) + mount = MountedReceipts(h) + mount.authorize(s) + + def bounded(scope, receive, **kwargs): + return receipt_body_receive(scope, receive, limit=128) + + monkeypatch.setattr("adcp.reporting.receipts.transport.receipt_body_receive", bounded) + before = await h.image() + async with mount.client() as client: + _, result = await getattr(mount, route)(client, request_for(s)) + assert error_code(result) == "INVALID_REQUEST" + assert await h.image() == before + + +@pytest.mark.parametrize("route", ["mcp", "a2a"]) +async def test_generic_middleware_cannot_replace_the_authenticated_whole_request(route): + async with receipt_harness("memory") as h: + s = await receipt_case(h) + mount = MountedReceipts(h, hydrated=True) + mount.authorize(s) + request = request_for(s, context={"attempt": 1}) + + async def rewrite(name, params, context, call_next): + params["context"]["attempt"] = 2 + return await call_next() + + mount.middleware = rewrite + before = await h.image() + async with mount.client() as client: + _, result = await getattr(mount, route)(client, request) + assert error_code(result) == "INVALID_REQUEST" + assert await h.image() == before + assert mount.auth_calls == [] + + +async def test_raw_mounted_requests_do_not_alias_account_consumer_pairs_or_cached_context(receipts): + h = receipts + first = await receipt_case(h, account_id="acct:a", consumer_id="b:c") + second = await receipt_case(h, account_id="acct:a:b", consumer_id="c") + mount = MountedReceipts(h, hydrated=True) + mount.authorize(first) + mount.authorize(second, token="token-two") + async with mount.client() as client: + _, a = await mount.mcp(client, request_for(first)) + _, b = await mount.a2a(client, request_for(second), token="token-two") + assert a["results"][0]["result"] == b["results"][0]["result"] == "recorded" + assert a != b + assert (await mount.a2a(client, request_for(first)))[1] == a + assert (await mount.mcp(client, request_for(second), token="token-two"))[1] == b + before = await h.image() + assert error_code((await mount.a2a(client, request_for(second)))[1]) == "UNAUTHORIZED" + assert ( + error_code((await mount.mcp(client, request_for(first), token="token-two"))[1]) + == "UNAUTHORIZED" + ) + assert await h.image() == before + assert len({c.caller_identity for c in mount.contexts}) == 1 diff --git a/tests/type_checks/reporting_receipt_ingress.py b/tests/type_checks/reporting_receipt_ingress.py new file mode 100644 index 000000000..c4f65756c --- /dev/null +++ b/tests/type_checks/reporting_receipt_ingress.py @@ -0,0 +1,40 @@ +"""Additive receipt composition keeps existing stores and structural protocols.""" + +from typing import Any + +from typing_extensions import assert_type + +from adcp.reporting.ledger import ReportingDeliveryPrincipal +from adcp.reporting.materializer import ReportingMaterializerStore +from adcp.reporting.receipts import ( + InMemoryReportingReceiptStore, + PgReportingReceiptStore, + ReceiptAccountResolver, + ReportingReceiptBatchStore, + ReportingReceiptBoundary, + ReportingReceiptCaptureStore, + ReportingReceiptHandler, +) + + +async def adopter( + postgres: PgReportingReceiptStore, + memory: InMemoryReportingReceiptStore, + caller: ReportingDeliveryPrincipal, + request: dict[str, Any], + resolve_account: ReceiptAccountResolver, +) -> ReportingMaterializerStore: + store: ReportingReceiptBatchStore = postgres + store = memory + capture: ReportingReceiptCaptureStore = postgres + capture = memory + materializer: ReportingMaterializerStore = postgres + materializer = memory + assert_type(await store.ingest_receipt_batch(request, caller=caller), dict[str, Any]) + assert_type( + await capture.read_receipt_boundaries(caller=caller), tuple[ReportingReceiptBoundary, ...] + ) + handler = ReportingReceiptHandler(store, resolve_account=resolve_account) + assert_type(handler.receipt_store, ReportingReceiptBatchStore) + assert_type(await postgres.receipt_ingestion_ready(), bool) + return materializer From 8f9612aaf75493edb4cce57a7bb95f4765419b56 Mon Sep 17 00:00:00 2001 From: Brian O'Kelley Date: Thu, 17 Sep 2026 12:37:48 +0000 Subject: [PATCH 02/10] test(reporting): close receipt foreign-target and adjustment-chain coverage Two owned C1 evidence gaps had no executed coverage at the candidate head. Literal-SQL relationship negatives substituted absent IDs rather than existing rows owned by another valid target, and the damaged-chain matrix built revision receipts only, leaving the adjustment-specific adjusts_reporting_revision_id predicate unexercised. No production defect was reproduced and no production code is changed. - Add foreign_targets(): a second complete, wholly valid obligation, official revision, materialization and adjustment in the next schedule period, plus a second consumer holding its own artifact for the caller's revision. - Assert existing-but-foreign account/consumer/obligation/revision/ materialization/adjustment references fail indistinguishably from absent records across memory and PostgreSQL in both notification modes, with no receipt, caller-feed or capture write and identical durable replay. - Pin the exact rejecting literal-SQL predicate per reference so this slice's tuples are attributable rather than merely covered by the inherited reconciliation guards. - Extend damaged_chain() to adjustment receipts and add memory and literal-SQL fork/cycle/gap/cross-target/accepted-predecessor/ disconnected-cycle cases. The adjustment chain key binds only reporting_adjustment_id, so a divergent official target stays inside the requested chain and no fork, gap or leaf-count rule can substitute. - Add function-body schema damage to the fail-closed migration matrix. Every object stays present and enabled while a financial predicate body is silently replaced, which dropped objects and disabled triggers cannot represent. Refs #1167 Co-Authored-By: Claude Opus 5 (1M context) --- .../conformance/reporting/_receipt_support.py | 136 ++++++++++- .../reporting/test_reporting_receipt_graph.py | 227 +++++++++++++++++- .../test_reporting_receipt_migration.py | 8 + 3 files changed, 363 insertions(+), 8 deletions(-) diff --git a/tests/conformance/reporting/_receipt_support.py b/tests/conformance/reporting/_receipt_support.py index 1181d7b73..cac2dd1b1 100644 --- a/tests/conformance/reporting/_receipt_support.py +++ b/tests/conformance/reporting/_receipt_support.py @@ -1,17 +1,23 @@ """Shared receipt-ingress vectors; production PG receipt time is database time.""" from contextlib import asynccontextmanager -from dataclasses import replace +from dataclasses import dataclass, replace from datetime import timedelta import pytest -from adcp.reporting.ledger import ReportingAdjustmentRecord, ReportingControlTotalRecord +from adcp.reporting.ledger import ( + ReportingAdjustmentRecord, + ReportingControlTotalRecord, + ReportingDeliveryScope, +) from adcp.reporting.ledger.delivery import adjustment_to_wire, receipt_to_wire +from adcp.reporting.ledger.models import derive_period +from adcp.reporting.ledger.producer import revision_content_sha256 from adcp.reporting.receipts import InMemoryReportingReceiptStore from ._durable_materializer_support import DurableHarness -from ._generation_support import END, isolated_reporting_pool +from ._generation_support import END, configuration, isolated_reporting_pool from ._reconciliation_support import Clock, scenario @@ -101,3 +107,127 @@ async def batch_state(h): ) ).fetchall() ) + + +@dataclass(frozen=True) +class ForeignTargets: + """Existing, wholly VALID records that belong to a different exact target. + + Nothing here is corrupt or unauthorized at the application boundary: every + row is a legitimate artifact for its own obligation, revision, consumer or + adjustment. A receipt that points at one of them from another target must + still be refused, and refused indistinguishably from an absent record. + """ + + obligation: object + revision: object + materialization_id: str + adjustment: object + consumer_id: str + consumer_materialization_id: str + + +async def foreign_targets(h, s): + """Commit one more complete valid period and one more valid consumer.""" + config = replace( + configuration(s.obligation.account_id), + feed_purpose=s.binding.feed_purpose, + required_finality=s.obligation.required_finality, + ) + period = derive_period(config.schedule, account_timezone=config.account_timezone, ordinal=1) + obligation = replace( + s.obligation, + reporting_obligation_id="rpo_foreign", + period=period, + scope_resolved_at=period.end, + automated_recovery_deadline_at=period.expected_at + config.automated_recovery_window, + ) + await h.store.commit_obligation(obligation) + scope = ReportingDeliveryScope( + obligation.generation_key, s.binding.consumer_id, obligation.reporting_obligation_id + ) + await h.store.bind_obligation_delivery( + replace(s.delivery, scope=scope, resource_retained_until=period.end + timedelta(days=400)) + ) + rows = [ + { + "media_buy_id": obligation.media_buy_ids[0], + "impressions": 5, + "spend": "12.50", + "currency": "EUR", + } + ] + revision = replace( + s.revision, + reporting_revision_id="revision-foreign", + reporting_obligation_id=obligation.reporting_obligation_id, + observed_at=period.end, + data_through=period.end, + created_at=period.end + timedelta(seconds=1), + finalized_at=period.end if s.revision.finality == "official" else None, + revision_content_sha256=revision_content_sha256( + reporting_revision_id="revision-foreign", + row_count=s.revision.row_count, + control_totals=s.revision.control_totals, + reporting_rows=rows, + control_total_evidence=s.revision.managed_control_totals, + ), + ) + await h.store.commit_revision(revision, rows) + completed = period.end + timedelta(seconds=3) + await h.store.commit_materialization_attempt( + replace( + s.attempt, + scope=scope, + reporting_revision_id=revision.reporting_revision_id, + reporting_materialization_id="materialization-foreign", + created_at=period.end + timedelta(seconds=2), + ) + ) + await h.store.commit_materialization( + replace( + s.outcome, + scope=scope, + reporting_revision_id=revision.reporting_revision_id, + reporting_materialization_id="materialization-foreign", + completed_at=completed, + resource=replace(s.outcome.resource, expires_at=completed + timedelta(days=400)), + verification=replace(s.outcome.verification, verified_at=completed), + ) + ) + adjustment = ReportingAdjustmentRecord( + "adjustment-foreign", + obligation.account_id, + revision.reporting_revision_id, + "source_correction", + period.end, + period.end + timedelta(days=30), + (("spend", "-1.50"),), + period.end + timedelta(seconds=5), + period.end + timedelta(seconds=6), + managed_control_total_deltas=( + ReportingControlTotalRecord("spend", "-1.50", "decimal", "EUR"), + ), + ) + await h.store.commit_adjustment(adjustment) + # One more valid consumer holding its own artifact for the caller's revision. + consumer_id = f"{s.binding.consumer_id}-foreign" + other = ReportingDeliveryScope( + s.obligation.generation_key, consumer_id, s.obligation.reporting_obligation_id + ) + await h.store.put_destination_binding(replace(s.binding, consumer_id=consumer_id)) + await h.store.bind_obligation_delivery(replace(s.delivery, scope=other)) + await h.store.commit_materialization_attempt( + replace(s.attempt, scope=other, reporting_materialization_id="materialization-consumer") + ) + await h.store.commit_materialization( + replace(s.outcome, scope=other, reporting_materialization_id="materialization-consumer") + ) + return ForeignTargets( + obligation, + revision, + "materialization-foreign", + adjustment, + consumer_id, + "materialization-consumer", + ) diff --git a/tests/conformance/reporting/test_reporting_receipt_graph.py b/tests/conformance/reporting/test_reporting_receipt_graph.py index d632a9619..2585b048f 100644 --- a/tests/conformance/reporting/test_reporting_receipt_graph.py +++ b/tests/conformance/reporting/test_reporting_receipt_graph.py @@ -6,13 +6,24 @@ import pytest from adcp.reporting.ledger import ReportingMaterializationCheck -from adcp.reporting.ledger._delivery_state import current_receipt +from adcp.reporting.ledger._delivery_state import current_receipt, receipt_chain from adcp.reporting.ledger.delivery import receipt_to_wire +from adcp.reporting.ledger.delivery_models import ( + ReportingAdjustmentReceiptRecord, + ReportingReceiptKey, +) from adcp.reporting.ledger.store import LedgerConflictError from adcp.reporting.receipts.records import receipt_record from ._generation_support import END -from ._receipt_support import adjustment_for, receipt_case, receipt_harness, receipts, request_for +from ._receipt_support import ( + adjustment_for, + foreign_targets, + receipt_case, + receipt_harness, + receipts, + request_for, +) from .test_reporting_reconciliation_sql import raw_insert __all__ = ["receipts"] @@ -227,8 +238,10 @@ async def test_adjustment_rejected_current_leaf_replacement_and_accepted_termina } -def damaged_chain(s, fault): - root = replace(s.receipt, status="rejected", rejection_codes=("LOAD_FAILED",)) +def damaged_chain(s, fault, root=None): + root = replace( + root if root is not None else s.receipt, status="rejected", rejection_codes=("LOAD_FAILED",) + ) leaf = replace( root, reporting_receipt_id="leaf-receipt-0002", @@ -242,7 +255,15 @@ def damaged_chain(s, fault): elif fault == "gap": records.pop(0) elif fault == "cross_target": - records[0] = replace(root, reporting_revision_id="other-revision") + # An adjustment chain key binds only reporting_adjustment_id, so a + # divergent adjusts_reporting_revision_id stays inside the same chain + # and only the dedicated official-target predicate can reject it. + field = ( + "adjusts_reporting_revision_id" + if isinstance(root, ReportingAdjustmentReceiptRecord) + else "reporting_revision_id" + ) + records[0] = replace(root, **{field: "other-revision"}) elif fault == "accepted_predecessor": records[0] = replace(root, status="accepted", rejection_codes=()) else: @@ -396,3 +417,199 @@ async def test_snapshot_adjustment_is_never_admitted_as_official_receipt_evidenc ) assert result["results"][0]["result"] == "recorded" assert result["results"][1]["errors"][0]["code"] == "REPORTING_RECORD_UNAVAILABLE" + + +_FOREIGN_SQL_PREDICATE = { + "materialization_other_obligation": "receipt artifact is unavailable", + "materialization_other_consumer": "receipt artifact is unavailable", + "revision_other_obligation": "receipt target is unavailable", + "obligation_other_target": "receipt target is unavailable", + "adjustment_other_revision": "receipt adjustment order is invalid", + "adjustment_revision_other_target": "receipt target is unavailable", +} +_FOREIGN_TARGETS = list(_FOREIGN_SQL_PREDICATE) + + +def _foreign_case(s, f, fault): + """One existing, valid but foreign reference, plus its truly absent twin. + + ``adjustment_other_revision`` deliberately observes *after* the foreign + adjustment was created. Every ordering comparison then holds on its own, so + ``adjusts_reporting_revision_id`` in the adjustment tuple is the only thing + left refusing the write. A nearer observation would be refused by the + ordering rule instead and could not attribute the ownership predicate. + """ + if fault == "materialization_other_obligation": + return "revision", {"reporting_materialization_id": f.materialization_id}, {} + if fault == "materialization_other_consumer": + return "revision", {"reporting_materialization_id": f.consumer_materialization_id}, {} + if fault == "revision_other_obligation": + return "revision", {"reporting_revision_id": f.revision.reporting_revision_id}, {} + if fault == "obligation_other_target": + return "revision", {"reporting_obligation_id": f.obligation.reporting_obligation_id}, {} + if fault == "adjustment_other_revision": + late = (f.adjustment.created_at + timedelta(seconds=1)).isoformat().replace("+00:00", "Z") + return ( + "adjustment", + {"reporting_adjustment_id": f.adjustment.reporting_adjustment_id, "observed_at": late}, + {"observed_at": late}, + ) + return "adjustment", {"adjusts_reporting_revision_id": f.revision.reporting_revision_id}, {} + + +@pytest.mark.parametrize("fault", _FOREIGN_TARGETS) +async def test_existing_foreign_targets_fail_exactly_like_absent_records(receipts, fault): + """A valid row owned by another target is not a usable receipt reference. + + Absence and misownership must be indistinguishable, so a caller cannot probe + another account's, consumer's, obligation's or revision's artifact history. + """ + h = receipts + s = await receipt_case(h) + f = await foreign_targets(h, s) + kind, foreign, shared = _foreign_case(s, f, fault) + base = await adjustment_for(h, s) if kind == "adjustment" else receipt_to_wire(s.receipt) + array = "adjustment_receipts" if kind == "adjustment" else "receipts" + # An adjustment batch keeps its ordinary revision receipt at ordinal zero. + index = 1 if kind == "adjustment" else 0 + + def build(overrides, key, receipt_id): + item = {**base, **shared, **overrides, "reporting_receipt_id": receipt_id} + return request_for(s, key=key, **{array: [item]}) + + failed_id = base["reporting_receipt_id"] + request = build(foreign, "receipt-batch-0001", failed_id) + absent_of = {name: f"absent-{name}" for name in foreign if name != "observed_at"} + control = build(absent_of, "receipt-batch-0002", "receipt-control-0001") + result = await h.store.ingest_receipt_batch(request, caller=s.binding.principal) + failure = result["results"][index] + assert failure["result"] == "failed" + assert failure["reporting_receipt_id"] == failed_id + assert failure["errors"][0]["code"] == "REPORTING_RECORD_UNAVAILABLE" + # The refused item is durably absent from evidence, the caller feed and capture. + assert not await h.store.get_receipt(ReportingReceiptKey(s.binding.principal, failed_id)) + boundaries = await h.store.read_receipt_boundaries(caller=s.binding.principal) + assert failed_id not in {b.reporting_receipt_id for b in boundaries} + assert len(boundaries) == index + # Identical durable replay, and an identical closed error for an absent row. + assert await h.store.ingest_receipt_batch(request, caller=s.binding.principal) == result + absent = await h.store.ingest_receipt_batch(control, caller=s.binding.principal) + assert absent["results"][index]["result"] == "failed" + assert absent["results"][index]["errors"] == failure["errors"] + + +@pytest.mark.parametrize("fault", _FOREIGN_TARGETS) +async def test_literal_sql_rejects_existing_foreign_targets(fault): + """The exact tuple predicates run without any Python validator or head API.""" + async with receipt_harness("postgres") as h: + from psycopg import IntegrityError + + s = await receipt_case(h) + f = await foreign_targets(h, s) + kind, foreign, shared = _foreign_case(s, f, fault) + obligation = s.obligation + if kind == "adjustment": + item = {**await adjustment_for(h, s), **shared, **foreign} + else: + item = {**receipt_to_wire(s.receipt), **shared, **foreign} + # The scope carries the obligation, so a foreign one replaces it. + obligation = f.obligation if "reporting_obligation_id" in foreign else s.obligation + item.pop("reporting_obligation_id", None) + item["reporting_obligation_id"] = obligation.reporting_obligation_id + candidate = receipt_record(f"{kind}_receipt", item, s.binding.principal, obligation) + before = await h.image() + with pytest.raises(IntegrityError) as error: + await raw_insert(h.pool, candidate) + # Attribution, not sole prevention: the inherited reconciliation guards + # independently refuse these same tuples, so a bare IntegrityError + # cannot show that this slice's predicates run at all. Asserting the + # exact message proves they reject first. Predicate *drift* is refused + # separately by the fingerprinted readiness manifest -- see + # test_reporting_receipt_migration.py's function-body damage cases. + assert error.value.diag.message_primary == _FOREIGN_SQL_PREDICATE[fault] + assert await h.image() == before + + +@pytest.mark.parametrize( + "fault", ["fork", "cycle", "gap", "cross_target", "accepted_predecessor", "disconnected_cycle"] +) +async def test_complete_adjustment_receipt_chain_rejects_damage(fault): + """Adjustment chains need their own topology and official-target checks.""" + async with receipt_harness("memory") as h: + s = await receipt_case(h) + root = receipt_record( + "adjustment_receipt", await adjustment_for(h, s), s.binding.principal, s.obligation + ) + records, leaf = damaged_chain(s, fault, root) + if fault == "cross_target": + # The divergent predecessor is inside the requested chain, so no + # fork/gap/leaf-count rule can stand in for the exact predicate. + assert receipt_chain(records[0]) == receipt_chain(leaf) + with pytest.raises(LedgerConflictError) as error: + current_receipt(tuple(records), leaf) + assert error.value.code == "REPORTING_HISTORY_CORRUPT" + + +@pytest.mark.parametrize( + "fault", ["fork", "cycle", "gap", "cross_target", "accepted_predecessor", "disconnected_cycle"] +) +async def test_literal_sql_blocks_damaged_adjustment_chains(fault): + from contextlib import nullcontext + + from adcp.reporting.ledger._delivery_state import ( + change_id, + fingerprint, + payload, + storage_identity, + ) + from adcp.reporting.ledger.delivery_pg import _IDENTITY_COLUMNS + + from ._generation_support import NOW + + async with receipt_harness("postgres") as h: + from psycopg import IntegrityError, sql + from psycopg.types.json import Jsonb + + s = await receipt_case(h) + root = receipt_record( + "adjustment_receipt", await adjustment_for(h, s), s.binding.principal, s.obligation + ) + records, leaf = damaged_chain(s, fault, root) + seed_before = await h.image() + # Owner-level seeding only; the final literal INSERT runs every trigger. + with pytest.raises(IntegrityError) if fault == "fork" else nullcontext(): + async with h.pool.connection() as c, c.transaction(): + await c.execute("SET LOCAL session_replication_role = replica") + for record in records: + record = replace(record, received_at=NOW) + row = dict(zip(_IDENTITY_COLUMNS, storage_identity(record))) + row.update( + payload=Jsonb(payload(record)), + content_sha256=fingerprint(record), + change_id=change_id(record), + ) + await c.execute( + sql.SQL( + "INSERT INTO reporting_reconciliation_records ({}) VALUES ({})" + ).format( + sql.SQL(", ").join(map(sql.Identifier, row)), + sql.SQL(", ").join(sql.Placeholder() for _ in row), + ), + tuple(row.values()), + ) + if fault == "fork": + assert await h.image() == seed_before + return + new = replace( + leaf, + reporting_receipt_id="new-adjustment-receipt-0099", + supersedes_reporting_receipt_id=leaf.reporting_receipt_id, + ) + before = await h.image() + with pytest.raises(IntegrityError) as error: + await raw_insert(h.pool, new) + assert error.value.diag.message_primary in { + "receipt replacement is unavailable", + "receipt history is inconsistent", + } + assert await h.image() == before diff --git a/tests/conformance/reporting/test_reporting_receipt_migration.py b/tests/conformance/reporting/test_reporting_receipt_migration.py index a4c19a527..b1ec57a65 100644 --- a/tests/conformance/reporting/test_reporting_receipt_migration.py +++ b/tests/conformance/reporting/test_reporting_receipt_migration.py @@ -100,6 +100,14 @@ async def migrate(): "ALTER TABLE reporting_materializer_work DISABLE TRIGGER reporting_materializer_guard", "ALTER TABLE reporting_reconciliation_records " "DISABLE TRIGGER reporting_reconciliation_guard", + # Every object still present and enabled, but a financial predicate body + # silently replaced. Dropped objects and disabled triggers cannot stand + # in for this: a weakened tuple predicate is exactly how a deployment + # would lose graph enforcement without any visible schema difference. + "CREATE OR REPLACE FUNCTION reporting_receipt_ingestion_graph()" + " RETURNS TRIGGER LANGUAGE plpgsql AS $damage$ BEGIN RETURN NEW; END $damage$", + "CREATE OR REPLACE FUNCTION reporting_receipt_ingestion_sha256(document JSONB)" + " RETURNS TEXT LANGUAGE SQL IMMUTABLE STRICT AS $damage$ SELECT repeat('0', 64) $damage$", ], ) @pytest.mark.parametrize("notifications", [False, True]) From 81c90724e9e5ae0af3313623b08625aad401df8f Mon Sep 17 00:00:00 2001 From: Brian O'Kelley Date: Thu, 17 Sep 2026 13:08:58 +0000 Subject: [PATCH 03/10] test(reporting): exercise the account tuple column at the receipt write predicate The six foreign-target cases held account_id fixed, so no literal-SQL case substituted a wholly valid target owned by another account. Store-level and mounted authorization denials cover a different boundary and never reach an INSERT, so they are not evidence for the write predicate itself. No production defect was reproduced and no production code is changed. - Add foreign_account() and extra_materialization(): one more complete, valid account with its own configuration, obligation, official revision, verified materialization and adjustment, plus a distinctly named second materialization so a foreign-account artifact reference is expressible. - Add test_literal_sql_rejects_valid_other_account_targets over the exact revision, obligation, materialization and adjustment references, asserting the exact rejecting predicate and an unchanged database image. - Narrow two comments that could be read as claiming exclusive necessity. The adjustment ownership predicate is attributable, not exclusively necessary, and the fork variant is refused by the unique successor index during owner seeding rather than at a trigger-enabled INSERT. Refs #1167 Co-Authored-By: Claude Opus 5 (1M context) --- .../conformance/reporting/_receipt_support.py | 38 +++++++++++ .../reporting/test_reporting_receipt_graph.py | 65 ++++++++++++++++++- 2 files changed, 100 insertions(+), 3 deletions(-) diff --git a/tests/conformance/reporting/_receipt_support.py b/tests/conformance/reporting/_receipt_support.py index cac2dd1b1..52ade8487 100644 --- a/tests/conformance/reporting/_receipt_support.py +++ b/tests/conformance/reporting/_receipt_support.py @@ -231,3 +231,41 @@ async def foreign_targets(h, s): consumer_id, "materialization-consumer", ) + + +async def extra_materialization(h, s, materialization_id, attempt_number): + """Commit one more valid attempt/materialization for s's exact scope.""" + created = s.attempt.created_at + timedelta(seconds=attempt_number) + completed = s.outcome.completed_at + timedelta(seconds=attempt_number) + await h.store.commit_materialization_attempt( + replace( + s.attempt, + reporting_materialization_id=materialization_id, + attempt=attempt_number, + created_at=created, + ) + ) + await h.store.commit_materialization( + replace( + s.outcome, + reporting_materialization_id=materialization_id, + completed_at=completed, + resource=replace(s.outcome.resource, expires_at=completed + timedelta(days=400)), + verification=replace(s.outcome.verification, verified_at=completed), + ) + ) + return materialization_id + + +async def foreign_account(h, *, account_id="acct_b"): + """Commit one more COMPLETE, VALID account and return its exact identifiers. + + Obligations, revisions and adjustments are globally keyed, so the same + identifier cannot exist under two accounts. The account relationship can + therefore only be probed by naming another account's real identifiers from + this account's receipt, which is what the returned values are for. + """ + other = await receipt_case(h, account_id=account_id) + adjustment = await adjustment_for(h, other, reporting_adjustment_id=f"adjustment-{account_id}") + materialization = await extra_materialization(h, other, f"materialization-{account_id}", 2) + return other, adjustment, materialization diff --git a/tests/conformance/reporting/test_reporting_receipt_graph.py b/tests/conformance/reporting/test_reporting_receipt_graph.py index 2585b048f..d3e92485f 100644 --- a/tests/conformance/reporting/test_reporting_receipt_graph.py +++ b/tests/conformance/reporting/test_reporting_receipt_graph.py @@ -18,6 +18,7 @@ from ._generation_support import END from ._receipt_support import ( adjustment_for, + foreign_account, foreign_targets, receipt_case, receipt_harness, @@ -435,9 +436,11 @@ def _foreign_case(s, f, fault): ``adjustment_other_revision`` deliberately observes *after* the foreign adjustment was created. Every ordering comparison then holds on its own, so - ``adjusts_reporting_revision_id`` in the adjustment tuple is the only thing - left refusing the write. A nearer observation would be refused by the - ordering rule instead and could not attribute the ownership predicate. + within this trigger ``adjusts_reporting_revision_id`` is the remaining + reason the adjustment tuple finds no row. A nearer observation would be + refused by the ordering rule instead and could not attribute the ownership + predicate. Inherited reconciliation guards still refuse the write if this + predicate is weakened, so this is attribution, not exclusive necessity. """ if fault == "materialization_other_obligation": return "revision", {"reporting_materialization_id": f.materialization_id}, {} @@ -577,6 +580,8 @@ async def test_literal_sql_blocks_damaged_adjustment_chains(fault): records, leaf = damaged_chain(s, fault, root) seed_before = await h.image() # Owner-level seeding only; the final literal INSERT runs every trigger. + # "fork" is the exception: the unique successor index refuses it during + # seeding, so that variant never reaches a trigger-enabled INSERT. with pytest.raises(IntegrityError) if fault == "fork" else nullcontext(): async with h.pool.connection() as c, c.transaction(): await c.execute("SET LOCAL session_replication_role = replica") @@ -613,3 +618,57 @@ async def test_literal_sql_blocks_damaged_adjustment_chains(fault): "receipt history is inconsistent", } assert await h.image() == before + + +_FOREIGN_ACCOUNT_SQL = { + "revision_other_account": "receipt target is unavailable", + "obligation_other_account": "receipt target is unavailable", + "materialization_other_account": "receipt artifact is unavailable", + "adjustment_other_account": "receipt adjustment order is invalid", +} + + +@pytest.mark.parametrize("fault", list(_FOREIGN_ACCOUNT_SQL)) +async def test_literal_sql_rejects_valid_other_account_targets(fault): + """The account column of each financial tuple, at the write predicate itself. + + Store-level and mounted authorization denials are a different boundary and + cannot stand in for this: they never reach an INSERT. Obligations, revisions + and adjustments are globally keyed, so a foreign-account reference is the + only way to exercise the account column with a wholly valid target row. + """ + async with receipt_harness("postgres") as h: + from psycopg import IntegrityError + + s = await receipt_case(h) + other, adjustment, materialization = await foreign_account(h) + if fault == "adjustment_other_account": + candidate = receipt_record( + "adjustment_receipt", await adjustment_for(h, s), s.binding.principal, s.obligation + ) + candidate = replace( + candidate, reporting_adjustment_id=adjustment["reporting_adjustment_id"] + ) + else: + candidate = receipt_record( + "revision_receipt", receipt_to_wire(s.receipt), s.binding.principal, s.obligation + ) + if fault == "revision_other_account": + candidate = replace( + candidate, reporting_revision_id=other.revision.reporting_revision_id + ) + elif fault == "obligation_other_account": + candidate = replace( + candidate, + scope=replace( + candidate.scope, + reporting_obligation_id=other.obligation.reporting_obligation_id, + ), + ) + else: + candidate = replace(candidate, reporting_materialization_id=materialization) + before = await h.image() + with pytest.raises(IntegrityError) as error: + await raw_insert(h.pool, candidate) + assert error.value.diag.message_primary == _FOREIGN_ACCOUNT_SQL[fault] + assert await h.image() == before From b71927a2a0a9e5ce1dae0cf0a7f62a359c26b722 Mon Sep 17 00:00:00 2001 From: Brian O'Kelley Date: Thu, 17 Sep 2026 14:15:43 +0000 Subject: [PATCH 04/10] fix(reporting): give period-close leasing a durable total fairness order CI caught `test_a_worker_that_releases_each_turn_reaches_every_accounts_generation[postgres]` leasing only `acct_a` across all nine release/release turns while the in-memory store stayed fair. The PostgreSQL store ordered candidates by `lease_expires_at NULLS FIRST` alone, and `release_period_close` clears that column, so every released generation tied at NULL and whichever tuple the scan yielded first was re-leased forever. The index comment already promised "least recently worked" ordering the query could not deliver. The failure was latent and layout-dependent, which is why it passed on earlier heads. Both stores also preferred an unheld generation over an expired one ahead of fairness, so a peer released on every turn stayed permanently NULL and outranked a generation whose worker crashed, which could then never close another period. - Add an additive, defaulted `lease_turn` column, a monotonic sequence and a distinctly named index. The name is new because `CREATE INDEX IF NOT EXISTS` would not upgrade an existing same-name index on an installed schema. - Stamp the turn on acquisition, not release: release only clears the lease, so there is nowhere else to record it. - Make the turn the primary ordering term in both stores, with expiry and the full generation key breaking exact ties, so the order is total, identical across backends and independent of physical layout or acceptance order. - Cover it behaviourally on both backends: release-each-turn rotation, a crashed generation not starved by a releasing peer, acquisition-time turn advance, key-order independence from acceptance order, and migration onto an already installed older schema with a mixed old/new writer. Rows written by older writers default to the never-leased rank, so nothing is starved. No required manifest key changes: the 187-object materializer and 102-object receipt manifests validate unchanged, and the retained-evidence migration comparison still fails if an upgrade stamps a non-default turn. Refs #1167 Co-Authored-By: Claude Opus 5 (1M context) --- docs/reporting-receipt-ingress.md | 7 + src/adcp/reporting/ledger/pg.py | 17 +- .../reporting/ledger/reporting_ledger.sql | 33 ++++ src/adcp/reporting/ledger/store.py | 40 +++-- .../test_reporting_generation_identity.py | 168 ++++++++++++++++++ .../test_reporting_generation_migration.py | 9 + 6 files changed, 259 insertions(+), 15 deletions(-) diff --git a/docs/reporting-receipt-ingress.md b/docs/reporting-receipt-ingress.md index 6eaa1b25c..c319ab205 100644 --- a/docs/reporting-receipt-ingress.md +++ b/docs/reporting-receipt-ingress.md @@ -150,6 +150,13 @@ readiness, and argument bounds remain actionable closed errors. connection. The additive `reporting_receipt_ingestion.sql` and independent `receipts/required_schema.json` ship in both wheel distribution paths. Do not modify A/B/C manifests or the **187-object** B2.1 materializer manifest. + This migration also carries an additive Core fix: `reporting_configurations` + gains a defaulted `lease_turn` column, a monotonic sequence and a distinctly + named index that give period-close leasing a durable, total fairness order. + Without it a worker that releases each turn can re-lease one generation + forever and never close any other account's periods. Rows written by older + writers default to the never-leased rank, so nothing is starved and no + required manifest key changes. 2. Verify `await store.receipt_ingestion_ready()` on the actual installed store. Empty, old, partial, disabled-trigger and mismatched schemas fail closed, including completed replay. The migration is transactional, repeatable and diff --git a/src/adcp/reporting/ledger/pg.py b/src/adcp/reporting/ledger/pg.py index 6c4a64d92..b1acd0ff0 100644 --- a/src/adcp/reporting/ledger/pg.py +++ b/src/adcp/reporting/ledger/pg.py @@ -1702,13 +1702,26 @@ async def lease_period_close( async with self._connection() as connection: row = await ( await connection.execute( + # `lease_turn` is the durable fairness rank and must be + # the primary ordering term. The WHERE clause has already + # excluded every live lease, so among the survivors the + # expiry carries no fairness information: ordering by it + # first starves a crashed generation forever, because a + # peer that is leased and released each turn is always NULL + # and NULL sorts first. Without the turn at all, every + # released generation ties and whichever tuple the scan + # yields first is re-leased forever. Expiry and the + # generation key only break exact turn ties, which keeps + # the order total and independent of physical layout. "UPDATE reporting_configurations SET lease_worker_id = %s," - " lease_expires_at = %s" + " lease_expires_at = %s," + " lease_turn = nextval('reporting_configurations_lease_turn_seq')" " WHERE (account_id, delivery_config_id, delivery_config_version) = (" " SELECT account_id, delivery_config_id, delivery_config_version" " FROM reporting_configurations" " WHERE lease_expires_at IS NULL OR lease_expires_at <= %s" - " ORDER BY lease_expires_at NULLS FIRST" + " ORDER BY lease_turn, lease_expires_at NULLS FIRST," + " account_id, delivery_config_id, delivery_config_version" " FOR UPDATE SKIP LOCKED" " LIMIT 1)" " RETURNING account_id, delivery_config_id, delivery_config_version", diff --git a/src/adcp/reporting/ledger/reporting_ledger.sql b/src/adcp/reporting/ledger/reporting_ledger.sql index f4c76c4f5..712f3173a 100644 --- a/src/adcp/reporting/ledger/reporting_ledger.sql +++ b/src/adcp/reporting/ledger/reporting_ledger.sql @@ -58,6 +58,39 @@ CREATE INDEX IF NOT EXISTS reporting_configurations_account_idx CREATE INDEX IF NOT EXISTS reporting_configurations_lease_idx ON reporting_configurations (lease_expires_at NULLS FIRST); +-- Durable round-robin fairness for period-close leasing. +-- +-- `reporting_configurations_lease_idx` above cannot deliver the "least recently +-- worked" order its own comment promises. `release_period_close` clears +-- `lease_expires_at`, so every released generation ties at NULL and the winner +-- is whichever tuple the scan happens to yield first: a worker that releases at +-- the end of every turn re-leases the same generation forever and never closes +-- any other account's periods. Ordering by expiry *first* is a second +-- starvation path -- the acquisition filter has already dropped every live +-- lease, so a peer that is released each turn is permanently NULL and outranks +-- a generation whose worker crashed, which then stays expired forever. +-- +-- `lease_turn` is therefore the primary fairness rank, stamped on acquisition +-- (release only clears the lease, so there is nowhere else to record it) and +-- persisted so it survives a worker restart. Expiry and the generation key only +-- break exact ties, which keeps the order total. +-- +-- Additive and defaulted: old readers never select it, and old writers that +-- INSERT without it get 0, which is the correct "never leased" rank, so a newly +-- accepted generation is served before any that already took a turn. The +-- sequence only has to be monotonic, so rollback gaps are harmless, and no +-- required manifest key changes. The index carries a distinct name rather than +-- redefining the legacy one, because `CREATE INDEX IF NOT EXISTS` would not +-- upgrade an existing same-name index on an already installed schema. +CREATE SEQUENCE IF NOT EXISTS reporting_configurations_lease_turn_seq AS BIGINT; + +ALTER TABLE reporting_configurations + ADD COLUMN IF NOT EXISTS lease_turn BIGINT NOT NULL DEFAULT 0; + +CREATE INDEX IF NOT EXISTS reporting_configurations_lease_fairness_idx + ON reporting_configurations (lease_turn, lease_expires_at NULLS FIRST, + account_id, delivery_config_id, delivery_config_version); + CREATE TABLE IF NOT EXISTS reporting_obligations ( reporting_obligation_id TEXT COLLATE "C" NOT NULL PRIMARY KEY, account_id TEXT COLLATE "C" NOT NULL, diff --git a/src/adcp/reporting/ledger/store.py b/src/adcp/reporting/ledger/store.py index 3c12e2041..6f25b531f 100644 --- a/src/adcp/reporting/ledger/store.py +++ b/src/adcp/reporting/ledger/store.py @@ -1568,26 +1568,40 @@ async def lease_period_close( async with self._lock: moment = _utc(now) - # Rank leasable generations the way the SQL store's - # `ORDER BY lease_expires_at NULLS FIRST` does -- unheld before - # expired, oldest expiry first -- then break the tie by whichever - # generation went longest without a turn. Without that last term a - # worker that releases at the end of every turn re-leases the same - # generation forever, and every other account's periods are never - # closed: starvation that only appears once two accounts can hold - # the same delivery_config_id. - ranked: list[tuple[tuple[int, float, int], ReportingConfigurationGenerationKey]] = [] + # Rank leasable generations exactly the way the SQL store's + # `ORDER BY lease_turn, lease_expires_at NULLS FIRST, ...` does: + # whichever generation went longest without a turn goes first. + # + # The turn has to be the *primary* term. The caller's `now` has + # already excluded every live lease, so among the survivors the + # expiry carries no fairness information -- and preferring unheld + # over expired ahead of the turn starves a crashed generation + # forever: a peer that is leased and released every turn is always + # unheld, so it wins every comparison while the generation whose + # worker died stays expired and never closes another period. + # Expiry and the generation key only break exact turn ties, so the + # order stays total and never depends on physical layout. + ranked: list[ + tuple[ + tuple[int, int, float, str, str, int], + ReportingConfigurationGenerationKey, + ] + ] = [] for key in self._configurations: turn = self._lease_turns.get(key, 0) held = self._leases.get(key) + tail = (key.account_id, key.delivery_config_id, key.delivery_config_version) if held is None: - ranked.append(((0, 0.0, turn), key)) + ranked.append(((turn, 0, 0.0, *tail), key)) elif _utc(held[1]) <= moment: - ranked.append(((1, _utc(held[1]).timestamp(), turn), key)) + ranked.append(((turn, 1, _utc(held[1]).timestamp(), *tail), key)) if not ranked: return None - # `min` keeps the first of equal ranks, so generations that have - # never been leased are handed out in the order they were accepted. + # The generation key is part of the rank, so the order is total and + # both stores make the same choice. Ranking by `min` alone would + # fall back to whichever generation this process happened to accept + # first, which the SQL store cannot reproduce and no adopter can + # observe consistently across a restart or a second worker. key = min(ranked, key=lambda item: item[0])[1] configuration = self._configurations[key] expires = moment + timedelta(seconds=lease_seconds) diff --git a/tests/conformance/reporting/test_reporting_generation_identity.py b/tests/conformance/reporting/test_reporting_generation_identity.py index 644bcffbe..cfbb3a859 100644 --- a/tests/conformance/reporting/test_reporting_generation_identity.py +++ b/tests/conformance/reporting/test_reporting_generation_identity.py @@ -185,6 +185,13 @@ async def test_a_worker_that_releases_each_turn_reaches_every_accounts_generatio always hands back the first leasable generation would close periods for one account forever and never reach the others -- invisible until two accounts share a ``delivery_config_id``, which is exactly what this change allows. + + Release clears ``lease_expires_at``, so ``ORDER BY lease_expires_at NULLS + FIRST`` alone is not a total order and the winner is whatever plan order + happens to apply; CI caught nine consecutive ``acct_a`` leases this way + while the in-memory store, which breaks the tie on turn, stayed fair. Both + stores now rank on a persisted turn advanced at acquisition, so reverting + that ordering fails this test deterministically rather than occasionally. """ accounts = ("acct_a", "acct_b", "acct_c") await asyncio.gather(*(store.put_configuration(configuration(name)) for name in accounts)) @@ -199,6 +206,167 @@ async def test_a_worker_that_releases_each_turn_reaches_every_accounts_generatio assert set(worked[: len(accounts)]) == set(accounts) +async def test_lease_turn_advances_on_acquisition_so_a_crashed_worker_cannot_starve_peers( + store: ReportingLedgerStore, +) -> None: + """Acquisition, not release, is where a generation loses its place in line. + + ``release_period_close`` only clears the lease, so the turn has to be + recorded when the lease is taken -- otherwise a worker that crashes + mid-close keeps the minimum turn forever and every expiry sweep hands the + same generation back. + + Ordering by expiry with ``NULLS FIRST`` hides that on its own: a never + worked generation always outranks an expired one, so the turn is only the + deciding term once two eligible generations share an expiry. This builds + exactly that state. ``acct_b`` is leased first and ``acct_a`` second, both + crash unreleased with the same expiry, so the generation that went longest + without a turn is ``acct_b`` even though ``acct_a`` sorts lower by key. A + store that advances the turn only on release ranks them equal and reaches + for ``acct_a``. + """ + # Created in this order so the later-acquired generation sorts lower by key. + await store.put_configuration(configuration("acct_b")) + crashed_first = await store.lease_period_close(worker_id="crash-1", now=NOW, lease_seconds=30) + assert crashed_first is not None and crashed_first.account_id == "acct_b" + await store.put_configuration(configuration("acct_a")) + crashed_second = await store.lease_period_close(worker_id="crash-2", now=NOW, lease_seconds=30) + assert crashed_second is not None and crashed_second.account_id == "acct_a" + assert crashed_first.lease_expires_at == crashed_second.lease_expires_at + # Never worked, so it must outrank both expired generations. + await store.put_configuration(configuration("acct_c")) + later = NOW + timedelta(seconds=31) + swept = [] + for _ in range(3): + lease = await store.lease_period_close(worker_id="solo", now=later, lease_seconds=30) + assert lease is not None + swept.append(lease.account_id) + assert swept == ["acct_c", "acct_b", "acct_a"] + + +async def test_lease_order_is_total_and_does_not_depend_on_acceptance_order( + store: ReportingLedgerStore, +) -> None: + """Both backends must choose the same generation, not this process's history. + + Never-leased generations all share the minimum turn, so something has to + break the tie. Ranking on "whichever generation this store accepted first" + is not reproducible: the SQL store cannot express it, it differs from the + in-memory store, and it does not survive a restart or a second worker. The + generation key is part of the rank instead, so these configurations are + handed out in key order even though they are accepted in the reverse. + """ + # Accepted in reverse key order, deliberately. + for name in ("acct_c", "acct_b", "acct_a"): + await store.put_configuration(configuration(name)) + worked = [] + for _ in range(3): + lease = await store.lease_period_close(worker_id="solo", now=NOW, lease_seconds=60) + assert lease is not None + worked.append(lease.account_id) + await store.release_period_close(lease, worker_id="solo") + assert worked == ["acct_a", "acct_b", "acct_c"] + + +async def test_a_crashed_generation_is_not_starved_by_a_peer_that_keeps_releasing( + store: ReportingLedgerStore, +) -> None: + """A permanently unheld peer must not outrank a lower-turn expired generation. + + The acquisition filter already drops every live lease, so among the + survivors the expiry carries no fairness information. If unheld sorted + ahead of expired, a peer that is leased and released on every turn would be + NULL forever and win every comparison, while the generation whose worker + died would stay expired and never close another period -- starvation with + no expiry sweep able to clear it. + """ + await asyncio.gather( + *(store.put_configuration(configuration(name)) for name in ("acct_a", "acct_b")) + ) + crashed = await store.lease_period_close(worker_id="crashes", now=NOW, lease_seconds=30) + assert crashed is not None and crashed.account_id == "acct_a" + # acct_a is still live here, so this can only take acct_b; it releases. + released = await store.lease_period_close(worker_id="polite", now=NOW, lease_seconds=30) + assert released is not None and released.account_id == "acct_b" + await store.release_period_close(released, worker_id="polite") + # Past acct_a's expiry both are leasable: acct_b unheld, acct_a expired. + later = NOW + timedelta(seconds=31) + recovered = await store.lease_period_close(worker_id="solo", now=later, lease_seconds=30) + assert recovered is not None + assert recovered.account_id == "acct_a" + + +async def test_lease_fairness_migrates_onto_an_already_installed_older_schema() -> None: + """The additive upgrade reaches an existing install and stays idempotent. + + A same-name ``CREATE INDEX IF NOT EXISTS`` would not upgrade an existing + index, so the total order is carried by a distinctly named additive index + plus a defaulted column, both of which an older installation gains on the + next ``create_schema()``. + """ + async with isolated_reporting_pool() as pool: + store = PgReportingLedgerStore(pool=pool, clock=lambda: NOW) + await store.create_schema() + async with pool.connection() as connection: + # Reduce the install to the pre-fairness shape an older binary left. + await connection.execute( + "DROP INDEX IF EXISTS reporting_configurations_lease_fairness_idx" + ) + await connection.execute( + "ALTER TABLE reporting_configurations DROP COLUMN IF EXISTS lease_turn" + ) + await connection.execute( + "DROP SEQUENCE IF EXISTS reporting_configurations_lease_turn_seq" + ) + for name in ("acct_a", "acct_b"): + await store.put_configuration(configuration(name)) + # Repeated migration is safe and restores the durable fairness column. + await store.create_schema() + await store.create_schema() + async with pool.connection() as connection: + row = await ( + await connection.execute( + "SELECT count(*) FROM information_schema.columns" + " WHERE table_schema = current_schema()" + " AND table_name = 'reporting_configurations'" + " AND column_name = 'lease_turn'" + ) + ).fetchone() + assert row is not None and row[0] == 1 + turns = await ( + await connection.execute("SELECT DISTINCT lease_turn FROM reporting_configurations") + ).fetchall() + # Rows written before the upgrade rank as never leased. + assert [t[0] for t in turns] == [0] + worked = [] + for _ in range(2): + lease = await store.lease_period_close(worker_id="solo", now=NOW, lease_seconds=60) + assert lease is not None + worked.append(lease.account_id) + await store.release_period_close(lease, worker_id="solo") + assert worked == ["acct_a", "acct_b"] + # A generation accepted by an older writer that does not know the column + # still defaults to the never-leased rank, so it is served before the + # generations that already took a turn rather than starved behind them. + async with pool.connection() as connection: + await connection.execute( + "INSERT INTO reporting_configurations" + " (delivery_config_id, delivery_config_version, account_id," + " report_definition_id, reporting_profile, feed_purpose, required_finality," + " account_timezone, schedule, media_buy_ids, activated_at," + " automated_recovery_seconds, status_retention_days, content_sha256)" + " SELECT delivery_config_id, delivery_config_version, 'acct_legacy'," + " report_definition_id, reporting_profile, feed_purpose, required_finality," + " account_timezone, schedule, media_buy_ids, activated_at," + " automated_recovery_seconds, status_retention_days, 'f' || content_sha256" + " FROM reporting_configurations WHERE account_id = %s", + ("acct_a",), + ) + legacy = await store.lease_period_close(worker_id="solo", now=NOW, lease_seconds=60) + assert legacy is not None + assert legacy.account_id == "acct_legacy" + + async def test_concurrent_period_closes_converge_within_each_account( store: ReportingLedgerStore, ) -> None: diff --git a/tests/conformance/reporting/test_reporting_generation_migration.py b/tests/conformance/reporting/test_reporting_generation_migration.py index d810a0f44..81c0277a8 100644 --- a/tests/conformance/reporting/test_reporting_generation_migration.py +++ b/tests/conformance/reporting/test_reporting_generation_migration.py @@ -74,6 +74,15 @@ async def _retained_rows(pool: AsyncConnectionPool) -> dict[str, list[Any]]: ) ).fetchall() result[table] = [row[0] for row in rows] + if table == "reporting_configurations": + # The durable period-close fairness turn is additive and + # defaulted. Dropping it only while it still holds the + # never-leased default keeps every byte of the pre-existing + # evidence under comparison: an upgrade that gave a retained + # generation a non-zero turn would still fail here. + for record in result[table]: + if record.get("lease_turn") == 0: + record.pop("lease_turn", None) if table == "reporting_obligations": # #1171 adds an explicit unknown currency. This test still # compares every byte of the pre-existing #1169 evidence. From 74b338d81653f4f65bc1520e638da10225f199f3 Mon Sep 17 00:00:00 2001 From: Brian O'Kelley Date: Thu, 17 Sep 2026 15:40:03 +0000 Subject: [PATCH 05/10] fix(reporting): give period-close leasing a durable total fairness order CI caught `test_a_worker_that_releases_each_turn_reaches_every_accounts_generation[postgres]` leasing only `acct_a` across all nine release/release turns while the in-memory store stayed fair. The PostgreSQL store ordered candidates by `lease_expires_at NULLS FIRST` alone, and `release_period_close` clears that column, so every released generation tied at NULL and whichever tuple the scan yielded first was re-leased forever. The index comment already promised "least recently worked" ordering the query could not deliver. The failure was latent and layout-dependent, which is why earlier heads were green. Both stores also preferred an unheld generation over an expired one ahead of fairness, so a peer released on every turn stayed permanently NULL and outranked a generation whose worker crashed, which could then never close another period. The rank is held outside the enumerated reporting_* catalog on purpose. schema_objects() enumerates every reporting_* table plus its columns, constraints, indexes and triggers, and the A/B+C suites compare the installed set to their manifests exhaustively, so a new column on reporting_configurations or a new reporting_* table breaks exact object identity for every older binary, while adding it to a required manifest would make old installations fail validation. - Add a private adcp_reporting_configuration_lease_turns table and sequence, joined lazily and upserted in the same transaction as the acquisition, so the rank can never advance without the lease nor the lease without the rank. - Stamp the rank on acquisition, not release: release only clears the lease, so there is nowhere else to record it. - Make the rank the primary ordering term in both stores, with expiry and the full generation key breaking exact ties, so the order is total, identical across backends and independent of physical layout or acceptance order. - Cover it behaviourally on both backends: release-each-turn rotation, a crashed generation not starved by a releasing peer, acquisition-time rank advance, key-order independence from acceptance order, same-transaction commit and rollback under autocommit, inert orphan ranks, no enumerated catalog object, and migration onto an already installed older schema with a mixed old/new writer. Measured on a freshly migrated schema: the ledger to materializer upgrade preserves all 453 enumerated objects byte-identically and adds exactly 187, all reporting_materializer_*; the materializer to receipt upgrade preserves all 640 and adds exactly 102, all reporting_receipt_ingestion_*. The three private objects enumerate as zero. Because they are invisible to the manifests, readiness cannot assert them: create_schema() must run before a worker starts, and the first lease otherwise fails loudly rather than degrading silently. Refs #1167 Co-Authored-By: Claude Opus 5 (1M context) --- docs/reporting-receipt-ingress.md | 18 +- src/adcp/reporting/ledger/pg.py | 60 ++++--- .../reporting/ledger/reporting_ledger.sql | 42 +++-- .../test_reporting_generation_identity.py | 158 +++++++++++++++--- 4 files changed, 210 insertions(+), 68 deletions(-) diff --git a/docs/reporting-receipt-ingress.md b/docs/reporting-receipt-ingress.md index c319ab205..27c807c53 100644 --- a/docs/reporting-receipt-ingress.md +++ b/docs/reporting-receipt-ingress.md @@ -150,13 +150,17 @@ readiness, and argument bounds remain actionable closed errors. connection. The additive `reporting_receipt_ingestion.sql` and independent `receipts/required_schema.json` ship in both wheel distribution paths. Do not modify A/B/C manifests or the **187-object** B2.1 materializer manifest. - This migration also carries an additive Core fix: `reporting_configurations` - gains a defaulted `lease_turn` column, a monotonic sequence and a distinctly - named index that give period-close leasing a durable, total fairness order. - Without it a worker that releases each turn can re-lease one generation - forever and never close any other account's periods. Rows written by older - writers default to the never-leased rank, so nothing is starved and no - required manifest key changes. + This migration also carries an additive Core fix: period-close leasing gains + a durable, total fairness order. Without it a worker that releases each turn + can re-lease one generation forever and never close any other account's + periods. The rank is held in a private `adcp_reporting_configuration_lease_turns` + table, outside the enumerated `reporting_*` catalog, so every existing + manifest and `reporting_configurations`' own shape stay byte-identical for + older readers. A generation with no rank row yet ranks as never leased, so + writers that predate the table starve nothing. Because the table is + deliberately invisible to the schema manifests, readiness cannot assert it: + run `create_schema()` before starting a worker, or the first lease fails + loudly on the missing relation rather than degrading silently. 2. Verify `await store.receipt_ingestion_ready()` on the actual installed store. Empty, old, partial, disabled-trigger and mismatched schemas fail closed, including completed replay. The migration is transactional, repeatable and diff --git a/src/adcp/reporting/ledger/pg.py b/src/adcp/reporting/ledger/pg.py index b1acd0ff0..b75451e80 100644 --- a/src/adcp/reporting/ledger/pg.py +++ b/src/adcp/reporting/ledger/pg.py @@ -1699,35 +1699,55 @@ async def lease_period_close( self, *, worker_id: str, now: datetime, lease_seconds: float ) -> LeasedConfiguration | None: expires = _utc(now) + timedelta(seconds=lease_seconds) - async with self._connection() as connection: + async with self._connection() as connection, connection.transaction(): + # The fairness rank is the primary ordering term. The WHERE clause + # has already excluded every live lease, so among the survivors the + # expiry carries no fairness information: ordering by it first + # starves a crashed generation forever, because a peer that is + # leased and released each turn is always NULL and NULL sorts + # first. Without the rank at all, every released generation ties + # and whichever tuple the scan yields first is re-leased forever. + # Expiry and the generation key only break exact ties, which keeps + # the order total and independent of physical layout. + # + # The rank is joined from a private table rather than held on + # `reporting_configurations`, so the enumerated `reporting_*` + # catalog that older binaries validate stays byte-identical. A + # generation with no row yet has never been leased, which is what + # COALESCE to 0 means, so a newly accepted generation is served + # before any that already took a turn. row = await ( await connection.execute( - # `lease_turn` is the durable fairness rank and must be - # the primary ordering term. The WHERE clause has already - # excluded every live lease, so among the survivors the - # expiry carries no fairness information: ordering by it - # first starves a crashed generation forever, because a - # peer that is leased and released each turn is always NULL - # and NULL sorts first. Without the turn at all, every - # released generation ties and whichever tuple the scan - # yields first is re-leased forever. Expiry and the - # generation key only break exact turn ties, which keeps - # the order total and independent of physical layout. "UPDATE reporting_configurations SET lease_worker_id = %s," - " lease_expires_at = %s," - " lease_turn = nextval('reporting_configurations_lease_turn_seq')" + " lease_expires_at = %s" " WHERE (account_id, delivery_config_id, delivery_config_version) = (" - " SELECT account_id, delivery_config_id, delivery_config_version" - " FROM reporting_configurations" - " WHERE lease_expires_at IS NULL OR lease_expires_at <= %s" - " ORDER BY lease_turn, lease_expires_at NULLS FIRST," - " account_id, delivery_config_id, delivery_config_version" - " FOR UPDATE SKIP LOCKED" + " SELECT c.account_id, c.delivery_config_id, c.delivery_config_version" + " FROM reporting_configurations c" + " LEFT JOIN adcp_reporting_configuration_lease_turns t" + " ON (t.account_id, t.delivery_config_id, t.delivery_config_version)" + " = (c.account_id, c.delivery_config_id, c.delivery_config_version)" + " WHERE c.lease_expires_at IS NULL OR c.lease_expires_at <= %s" + " ORDER BY COALESCE(t.lease_turn, 0), c.lease_expires_at NULLS FIRST," + " c.account_id, c.delivery_config_id, c.delivery_config_version" + " FOR UPDATE OF c SKIP LOCKED" " LIMIT 1)" " RETURNING account_id, delivery_config_id, delivery_config_version", (worker_id, expires, _utc(now)), ) ).fetchone() + if row is not None: + # Same transaction as the acquisition, so the rank can never + # advance without the lease or the lease without the rank. + await connection.execute( + "INSERT INTO adcp_reporting_configuration_lease_turns" + " (account_id, delivery_config_id, delivery_config_version, lease_turn)" + " VALUES (%s, %s, %s," + " nextval('adcp_reporting_configuration_lease_turn_seq'))" + " ON CONFLICT (account_id, delivery_config_id, delivery_config_version)" + " DO UPDATE SET lease_turn =" + " nextval('adcp_reporting_configuration_lease_turn_seq')", + (row[0], row[1], row[2]), + ) if row is None: return None return LeasedConfiguration( diff --git a/src/adcp/reporting/ledger/reporting_ledger.sql b/src/adcp/reporting/ledger/reporting_ledger.sql index 712f3173a..385aa340f 100644 --- a/src/adcp/reporting/ledger/reporting_ledger.sql +++ b/src/adcp/reporting/ledger/reporting_ledger.sql @@ -70,26 +70,32 @@ CREATE INDEX IF NOT EXISTS reporting_configurations_lease_idx -- lease, so a peer that is released each turn is permanently NULL and outranks -- a generation whose worker crashed, which then stays expired forever. -- --- `lease_turn` is therefore the primary fairness rank, stamped on acquisition --- (release only clears the lease, so there is nowhere else to record it) and --- persisted so it survives a worker restart. Expiry and the generation key only --- break exact ties, which keeps the order total. +-- The fairness rank therefore lives in its own private SDK table, stamped on +-- acquisition (release only clears the lease, so there is nowhere else to +-- record it) and persisted so it survives a worker restart. -- --- Additive and defaulted: old readers never select it, and old writers that --- INSERT without it get 0, which is the correct "never leased" rank, so a newly --- accepted generation is served before any that already took a turn. The --- sequence only has to be monotonic, so rollback gaps are harmless, and no --- required manifest key changes. The index carries a distinct name rather than --- redefining the legacy one, because `CREATE INDEX IF NOT EXISTS` would not --- upgrade an existing same-name index on an already installed schema. -CREATE SEQUENCE IF NOT EXISTS reporting_configurations_lease_turn_seq AS BIGINT; +-- The `adcp_` prefix is deliberate and load-bearing. `schema_objects()` +-- enumerates every table in the current schema whose name starts with +-- `reporting_`, and the A/B+C conformance suites assert that the installed +-- object set is *exactly* their manifests. A new `reporting_*` table, or a new +-- column on `reporting_configurations`, would therefore break exact object +-- identity for every older binary: adding it to the required manifest would +-- make old installations fail validation, and leaving it out would break the +-- exhaustive comparison. Outside that prefix the table is invisible to the +-- catalog contract, so old manifests, `reporting_configurations`'s shape and +-- its retained evidence all stay byte-identical. +CREATE SEQUENCE IF NOT EXISTS adcp_reporting_configuration_lease_turn_seq AS BIGINT; -ALTER TABLE reporting_configurations - ADD COLUMN IF NOT EXISTS lease_turn BIGINT NOT NULL DEFAULT 0; - -CREATE INDEX IF NOT EXISTS reporting_configurations_lease_fairness_idx - ON reporting_configurations (lease_turn, lease_expires_at NULLS FIRST, - account_id, delivery_config_id, delivery_config_version); +-- No foreign key: the reference is resolved by the lazy LEFT JOIN instead, so +-- nothing is added to `reporting_configurations` and a generation that is +-- deleted simply stops being joined. +CREATE TABLE IF NOT EXISTS adcp_reporting_configuration_lease_turns ( + account_id TEXT COLLATE "C" NOT NULL, + delivery_config_id TEXT COLLATE "C" NOT NULL, + delivery_config_version INTEGER NOT NULL, + lease_turn BIGINT NOT NULL, + PRIMARY KEY (account_id, delivery_config_id, delivery_config_version) +); CREATE TABLE IF NOT EXISTS reporting_obligations ( reporting_obligation_id TEXT COLLATE "C" NOT NULL PRIMARY KEY, diff --git a/tests/conformance/reporting/test_reporting_generation_identity.py b/tests/conformance/reporting/test_reporting_generation_identity.py index cfbb3a859..d5132bdf8 100644 --- a/tests/conformance/reporting/test_reporting_generation_identity.py +++ b/tests/conformance/reporting/test_reporting_generation_identity.py @@ -26,6 +26,7 @@ consumer_mismatch_issue_key, ) from adcp.reporting.ledger.pg import PgReportingLedgerStore +from adcp.reporting.outbox._schema import schema_objects from adcp.reporting.source import ReportingSourceSliceRequestV1 from tests.conformance.reporting._generation_support import ( END, @@ -299,10 +300,10 @@ async def test_a_crashed_generation_is_not_starved_by_a_peer_that_keeps_releasin async def test_lease_fairness_migrates_onto_an_already_installed_older_schema() -> None: """The additive upgrade reaches an existing install and stays idempotent. - A same-name ``CREATE INDEX IF NOT EXISTS`` would not upgrade an existing - index, so the total order is carried by a distinctly named additive index - plus a defaulted column, both of which an older installation gains on the - next ``create_schema()``. + The rank lives in a private `adcp_`-prefixed table precisely so that + `schema_objects()` -- which enumerates every `reporting_*` table in the + schema -- keeps reporting the exact object set that older binaries + validate. So the upgrade has to be proven on an install that predates it. """ async with isolated_reporting_pool() as pool: store = PgReportingLedgerStore(pool=pool, clock=lambda: NOW) @@ -310,34 +311,25 @@ async def test_lease_fairness_migrates_onto_an_already_installed_older_schema() async with pool.connection() as connection: # Reduce the install to the pre-fairness shape an older binary left. await connection.execute( - "DROP INDEX IF EXISTS reporting_configurations_lease_fairness_idx" + "DROP TABLE IF EXISTS adcp_reporting_configuration_lease_turns" ) await connection.execute( - "ALTER TABLE reporting_configurations DROP COLUMN IF EXISTS lease_turn" - ) - await connection.execute( - "DROP SEQUENCE IF EXISTS reporting_configurations_lease_turn_seq" + "DROP SEQUENCE IF EXISTS adcp_reporting_configuration_lease_turn_seq" ) for name in ("acct_a", "acct_b"): await store.put_configuration(configuration(name)) - # Repeated migration is safe and restores the durable fairness column. + # Repeated migration is safe and restores the durable fairness rank. await store.create_schema() await store.create_schema() async with pool.connection() as connection: - row = await ( + ranks = await ( await connection.execute( - "SELECT count(*) FROM information_schema.columns" - " WHERE table_schema = current_schema()" - " AND table_name = 'reporting_configurations'" - " AND column_name = 'lease_turn'" + "SELECT count(*) FROM adcp_reporting_configuration_lease_turns" ) ).fetchone() - assert row is not None and row[0] == 1 - turns = await ( - await connection.execute("SELECT DISTINCT lease_turn FROM reporting_configurations") - ).fetchall() - # Rows written before the upgrade rank as never leased. - assert [t[0] for t in turns] == [0] + # Generations accepted before the upgrade have no rank row at all, which + # is the never-leased rank rather than a privileged one. + assert ranks is not None and ranks[0] == 0 worked = [] for _ in range(2): lease = await store.lease_period_close(worker_id="solo", now=NOW, lease_seconds=60) @@ -345,8 +337,8 @@ async def test_lease_fairness_migrates_onto_an_already_installed_older_schema() worked.append(lease.account_id) await store.release_period_close(lease, worker_id="solo") assert worked == ["acct_a", "acct_b"] - # A generation accepted by an older writer that does not know the column - # still defaults to the never-leased rank, so it is served before the + # A generation accepted by an older writer that knows nothing about the + # private table still ranks as never leased, so it is served before the # generations that already took a turn rather than starved behind them. async with pool.connection() as connection: await connection.execute( @@ -367,6 +359,126 @@ async def test_lease_fairness_migrates_onto_an_already_installed_older_schema() assert legacy.account_id == "acct_legacy" +async def test_lease_fairness_adds_no_enumerated_reporting_catalog_object() -> None: + """Exact `reporting_*` object identity is the A/B+C compatibility contract. + + `schema_objects()` enumerates every current-schema table whose name starts + with `reporting_`, plus that table's columns, constraints, indexes and + triggers, and the status suites compare the installed set to their + manifests exhaustively. The fairness rank must therefore add nothing to + that set -- not a column on `reporting_configurations`, and not a new + `reporting_*` table either. + """ + async with isolated_reporting_pool() as pool: + store = PgReportingLedgerStore(pool=pool, clock=lambda: NOW) + await store.create_schema() + async with pool.connection() as connection: + installed = await schema_objects(connection) + await connection.execute( + "DROP TABLE IF EXISTS adcp_reporting_configuration_lease_turns" + ) + await connection.execute( + "DROP SEQUENCE IF EXISTS adcp_reporting_configuration_lease_turn_seq" + ) + without = await schema_objects(connection) + assert installed == without + assert not [k for k in installed if "lease_turn" in k or "fairness" in k] + # And the private objects really are the ones carrying the rank. + await store.create_schema() + for name in ("acct_a", "acct_b"): + await store.put_configuration(configuration(name)) + lease = await store.lease_period_close(worker_id="solo", now=NOW, lease_seconds=60) + assert lease is not None + async with pool.connection() as connection: + assert await schema_objects(connection) == installed + rows = await ( + await connection.execute( + "SELECT account_id, lease_turn" " FROM adcp_reporting_configuration_lease_turns" + ) + ).fetchall() + assert [(r[0], r[1] > 0) for r in rows] == [("acct_a", True)] + + +async def test_lease_and_its_fairness_rank_commit_or_roll_back_together() -> None: + """The rank lives in another table, so it must share the lease transaction. + + If the two statements could commit separately, a crash between them would + either hand out a lease whose generation never lost its place in line, or + advance the rank for a lease nobody holds. The rank update is forced to fail + here; the acquisition must roll back with it. + + The pool is deliberately autocommit: that is the mode in which an implicit + per-block transaction does not exist, so it is the only mode that can + witness the explicit transaction actually doing the work. + """ + async with isolated_reporting_pool(autocommit=True) as pool: + store = PgReportingLedgerStore(pool=pool, clock=lambda: NOW) + await store.create_schema() + await store.put_configuration(configuration("acct_a")) + async with pool.connection() as connection: + await connection.execute( + "ALTER TABLE adcp_reporting_configuration_lease_turns" + " ADD CONSTRAINT reject_rank CHECK (lease_turn < 0)" + ) + with pytest.raises(Exception): # noqa: B017,PT011 - driver integrity error + await store.lease_period_close(worker_id="solo", now=NOW, lease_seconds=60) + async with pool.connection() as connection: + held = await ( + await connection.execute( + "SELECT count(*) FROM reporting_configurations" + " WHERE lease_worker_id IS NOT NULL OR lease_expires_at IS NOT NULL" + ) + ).fetchone() + ranked = await ( + await connection.execute( + "SELECT count(*) FROM adcp_reporting_configuration_lease_turns" + ) + ).fetchone() + # Neither half survived. + assert held is not None and held[0] == 0 + assert ranked is not None and ranked[0] == 0 + # With the rank writable again the generation is still leasable. + async with pool.connection() as connection: + await connection.execute( + "ALTER TABLE adcp_reporting_configuration_lease_turns DROP CONSTRAINT reject_rank" + ) + lease = await store.lease_period_close(worker_id="solo", now=NOW, lease_seconds=60) + assert lease is not None and lease.account_id == "acct_a" + + +async def test_a_stale_fairness_rank_row_cannot_affect_another_generation() -> None: + """An orphan rank row is inert, and a re-put generation keeps its own rank. + + The rank table carries no foreign key, so a generation removed by an + operator can leave a row behind. The lazy join only matches on the exact + generation key, so such a row is never consulted for anyone else, and + ``put_configuration`` is immutable by key, so re-accepting the same + generation is the same generation and legitimately keeps its place in line. + """ + async with isolated_reporting_pool() as pool: + store = PgReportingLedgerStore(pool=pool, clock=lambda: NOW) + await store.create_schema() + for name in ("acct_a", "acct_b"): + await store.put_configuration(configuration(name)) + async with pool.connection() as connection: + await connection.execute( + "INSERT INTO adcp_reporting_configuration_lease_turns" + " (account_id, delivery_config_id, delivery_config_version, lease_turn)" + " VALUES ('acct_vanished', 'daily', 1, 999999)" + ) + worked = [] + for _ in range(2): + lease = await store.lease_period_close(worker_id="solo", now=NOW, lease_seconds=60) + assert lease is not None + worked.append(lease.account_id) + await store.release_period_close(lease, worker_id="solo") + assert worked == ["acct_a", "acct_b"] + # Re-accepting acct_a's exact generation does not reset its rank. + await store.put_configuration(configuration("acct_a")) + again = await store.lease_period_close(worker_id="solo", now=NOW, lease_seconds=60) + assert again is not None and again.account_id == "acct_a" + + async def test_concurrent_period_closes_converge_within_each_account( store: ReportingLedgerStore, ) -> None: From c7c4f09d0fdf6e46e2b9e601d69b639fcf233b28 Mon Sep 17 00:00:00 2001 From: Brian O'Kelley Date: Thu, 24 Sep 2026 16:20:30 +0000 Subject: [PATCH 06/10] fix(reporting): qualify receipt gates against integrated baselines --- .github/workflows/ci.yml | 27 ++++++++++++------- docs/reporting-receipt-ingress.md | 14 +++++++--- src/adcp/reporting/receipts/capture.py | 1 + src/adcp/reporting/receipts/records.py | 1 + src/adcp/reporting/receipts/transport.py | 5 ++++ src/adcp/server/serve.py | 1 + .../reporting/_materializer_installed.py | 6 +++-- .../conformance/reporting/_receipt_frozen.py | 9 ++++--- .../reporting/_receipt_installed.py | 3 ++- .../conformance/reporting/_receipt_process.py | 3 ++- .../test_reporting_generation_identity.py | 20 +++++++++++--- .../test_reporting_generation_migration.py | 16 ++++++++--- .../test_reporting_receipt_batches.py | 19 +++++++------ .../reporting/test_reporting_receipt_graph.py | 22 ++++++++------- .../test_reporting_receipt_installed_pg.py | 6 ++--- .../test_reporting_receipt_materializer.py | 7 ++--- .../test_reporting_receipt_migration.py | 3 ++- .../test_reporting_receipt_process.py | 14 +++++++--- .../test_reporting_receipt_rolling.py | 10 ++++--- .../test_reporting_receipt_transactions.py | 7 ++--- .../test_reporting_receipt_transports.py | 21 ++++++++------- 21 files changed, 141 insertions(+), 74 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 05dfdc8b7..0a8fc4319 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -196,7 +196,8 @@ jobs: --ignore=tests/conformance/reporting/test_reporting_materializer_process.py \ --ignore=tests/conformance/reporting/test_reporting_materializer_migration.py \ --ignore=tests/conformance/reporting/test_reporting_materializer_installed_pg.py \ - -v + --ignore-glob='tests/conformance/reporting/test_reporting_receipt_*.py' \ + -v -ra ;; process) python scripts/reporting_test_harness.py pytest tests/conformance/reporting/test_reporting_notification_process_matrix.py -v -ra @@ -212,7 +213,7 @@ jobs: runs-on: ubuntu-latest timeout-minutes: 5 permissions: {} - needs: [pg-conformance, pg-reporting-status, pg-reporting-materializer] + needs: [pg-conformance, pg-reporting-status, pg-reporting-materializer, pg-reporting-receipts, pg-reporting-receipt-compatibility] if: ${{ always() }} steps: - name: Require every Postgres conformance lane @@ -220,11 +221,15 @@ jobs: PG_RESULT: ${{ needs.pg-conformance.result }} STATUS_RESULT: ${{ needs.pg-reporting-status.result }} MATERIALIZER_RESULT: ${{ needs.pg-reporting-materializer.result }} + RECEIPTS_RESULT: ${{ needs.pg-reporting-receipts.result }} + RECEIPT_COMPATIBILITY_RESULT: ${{ needs.pg-reporting-receipt-compatibility.result }} run: | - if [ "$PG_RESULT" != "success" ] || [ "$STATUS_RESULT" != "success" ] || [ "$MATERIALIZER_RESULT" != "success" ]; then + if [ "$PG_RESULT" != "success" ] || [ "$STATUS_RESULT" != "success" ] || [ "$MATERIALIZER_RESULT" != "success" ] || [ "$RECEIPTS_RESULT" != "success" ] || [ "$RECEIPT_COMPATIBILITY_RESULT" != "success" ]; then echo "Postgres conformance matrix result: $PG_RESULT" echo "Reporting status conformance result: $STATUS_RESULT" echo "Durable materializer conformance result: $MATERIALIZER_RESULT" + echo "Authenticated receipt conformance result: $RECEIPTS_RESULT" + echo "Receipt rolling compatibility result: $RECEIPT_COMPATIBILITY_RESULT" exit 1 fi echo "All Postgres conformance lanes passed" @@ -352,13 +357,14 @@ jobs: # Separate databases avoid shared account advisory locks. Keep five minutes # outside the bounded suite for checkout, installations and cleanup. timeout-minutes: 35 + permissions: + contents: read services: postgres: image: postgres:16 env: POSTGRES_HOST_AUTH_METHOD: trust POSTGRES_DB: adcp_receipts_test - POSTGRES_INITDB_ARGS: "--encoding=UTF8 --lc-collate=C --lc-ctype=C" ports: - 5432:5432 options: >- @@ -410,13 +416,14 @@ jobs: # Separate databases avoid shared account advisory locks. Keep five minutes # outside the bounded suite for checkout, installations and cleanup. timeout-minutes: 35 + permissions: + contents: read services: postgres: image: postgres:16 env: POSTGRES_HOST_AUTH_METHOD: trust POSTGRES_DB: adcp_receipt_rolling_test - POSTGRES_INITDB_ARGS: "--encoding=UTF8 --lc-collate=C --lc-ctype=C" ports: - 5432:5432 options: >- @@ -433,11 +440,11 @@ jobs: 3e76aa54623529a3dda01cd690b8a5c287c75641 \ 3c405a21f978ed9d3208611bb4a7a8434a056933 \ 037de4ac822ecefb2f95d32c15c297fb4c45d683 \ - 21bf443e7d850d1800ec8a6f2e4abec1c8f85541 \ - 198d50e61c74fb82aedbf2c77e06a0e200b91db6 \ - ea150fabd5ad90e3abf93f89729d2919f1c61798 \ - 1c91311ec28d25506d5db43f59d0c34936ecb8f7 \ - 8e18ca12b9a0c3750f80aa822058c02982ab3e52 + 17ee407ae3978c8a2bb54437287afbf9dafb8130 \ + 0f34c666ac1961e9832fce43ef0ef6937b3c1dde \ + 967b6e286301d7e5d089aea6fdbb90bea8ee5a16 \ + 5487f2bdef23c5102118b305be9e868228f6ce61 \ + 3fd62121c96a074e3ea458c30c5224d6a586f169 - uses: actions/setup-python@v6 with: python-version: "3.12" diff --git a/docs/reporting-receipt-ingress.md b/docs/reporting-receipt-ingress.md index 27c807c53..7c7d8ae08 100644 --- a/docs/reporting-receipt-ingress.md +++ b/docs/reporting-receipt-ingress.md @@ -197,9 +197,15 @@ incompatible status projectors before its activation. Historical reader/writer gates execute actual installed beta.15, records A, foundation integration, outbox A, activity B, status C, B1 and the exact approved B2.1 artifact on populated new records, comparing permitted old behavior before -and after this migration. Their reviewed PostgreSQL prerequisite is **UTF8/C -collation and ctype**. A's notification-readiness closure after C is compared on -both sides and does not excuse new regressions. Non-C probes are distinct from -this frozen-artifact gate. Optional notifications may stay explicitly disabled; +and after this migration. The integrated pins are `17ee407a` (A), `0f34c666` (B), +`967b6e28` (C), `5487f2bd` (B1) and `3fd62121` (B2.1). These controls do not qualify +the earlier pre-integration A/B/C/B1/B2.1 snapshots. In particular, the old A +fingerprints and B2.1 scheduler ordering depend on database collation. The +integrated artifacts remove those dependencies; a C-only cluster would hide +portability regressions, so the gates require URL and drivers without a locale +pin. The pre-`17ee407a` A, pre-`0f34c666` B, pre-`967b6e28` C, pre-`5487f2bd` B1 +and pre-`3fd62121` B2.1 rolling exclusions must accompany release notes. +A's notification-readiness closure after C is compared on both sides and does +not excuse new regressions. Optional notifications may stay explicitly disabled; complete polling still depends on the later B2.3/B2.4 components. Full buyer adjustment automation and `client.reporting` remain named downstream #1172 work. diff --git a/src/adcp/reporting/receipts/capture.py b/src/adcp/reporting/receipts/capture.py index ae5dc4260..966e99069 100644 --- a/src/adcp/reporting/receipts/capture.py +++ b/src/adcp/reporting/receipts/capture.py @@ -87,6 +87,7 @@ def decode_receipt_boundary(value: dict[str, Any]) -> ReportingReceiptBoundary: tuple(decode_record(r) for r in value["reconciliation"]), ) except (ValueError, TypeError, KeyError, ValidationError): + # Raise the closed history error below without exposing decoder exception context. pass if result is None or result.to_storage() != value: raise ReportingReceiptError("RECEIPT_HISTORY_CORRUPT") diff --git a/src/adcp/reporting/receipts/records.py b/src/adcp/reporting/receipts/records.py index 419cccd4e..6c22b6787 100644 --- a/src/adcp/reporting/receipts/records.py +++ b/src/adcp/reporting/receipts/records.py @@ -58,6 +58,7 @@ def receipt_record( adapter = _REVISION_RECEIPT if kind == "revision_receipt" else _ADJUSTMENT_RECEIPT value = adapter.validate_python(body) except (ValueError, TypeError, ValidationError): + # Reject the record below without attaching private validation details as context. pass if value is None: fail("INVALID_REPORTING_RECORD") diff --git a/src/adcp/reporting/receipts/transport.py b/src/adcp/reporting/receipts/transport.py index 6fe8b0109..a854526cc 100644 --- a/src/adcp/reporting/receipts/transport.py +++ b/src/adcp/reporting/receipts/transport.py @@ -72,6 +72,7 @@ def _exact_integer(value: Decimal) -> int: ): return int(value) except (InvalidOperation, ValueError, OverflowError): + # Conversion failed; report the bounded integer error outside this handler. pass raise ValueError("receipt JSON requires exact safe integers") @@ -133,6 +134,7 @@ def _a2a_receipt_invocation(body: bytes | None) -> dict[str, Any] | None: if len(invocations) == 1 and invocations[0]["skill"] == TASK: return dict(invocations[0]) except (ValueError, TypeError, KeyError, RecursionError): + # Malformed envelopes cannot establish a unique receipt invocation. pass return None @@ -152,6 +154,7 @@ def a2a_receipt_parameters(body: bytes | None) -> dict[str, Any] | None: if type(params) is dict: return dict(_receipt_numbers(params)) except (ValueError, TypeError, RecursionError): + # The receipt route is known; return invalid parameters for ordinary rejection. pass return {} @@ -172,6 +175,7 @@ def a2a_receipt_has_invalid_unicode(body: bytes | None) -> bool: elif type(value) is list: pending.extend(value) except (ValueError, TypeError, RecursionError): + # Failed parsing establishes no Unicode diagnosis; ordinary validation still applies. pass return False @@ -191,5 +195,6 @@ def mcp_receipt_parameters(body: bytes | None) -> dict[str, Any]: ): return dict(_receipt_numbers(envelope["params"]["arguments"])) except (ValueError, TypeError, KeyError, RecursionError): + # The selected MCP route must receive invalid parameters, never a partial body. pass return {} diff --git a/src/adcp/server/serve.py b/src/adcp/server/serve.py index c28b0a557..96e03ce4f 100644 --- a/src/adcp/server/serve.py +++ b/src/adcp/server/serve.py @@ -369,6 +369,7 @@ def _get_starlette_request_for_dispatch() -> Any: try: return _ADCP_MCP_REQUEST_CONTEXT.get() except LookupError: + # No SDK dispatch is bound here; try the legacy test-harness context below. pass # MCP SDK v1 exposed this contextvar. MCP SDK v2 removed it, but keep diff --git a/tests/conformance/reporting/_materializer_installed.py b/tests/conformance/reporting/_materializer_installed.py index 02c889446..07b16a7ab 100644 --- a/tests/conformance/reporting/_materializer_installed.py +++ b/tests/conformance/reporting/_materializer_installed.py @@ -229,7 +229,8 @@ async def main(): ReportingDestinationIO(destination.registry, destination.resolver), destination.writer, ) - assert (await receipt_service.run_once()).state == "verified" + receipt_operation_1 = await receipt_service.run_once() + assert (receipt_operation_1).state == "verified" snapshot = await receipt_store.read_reconciliation_snapshot(caller=scope.principal) outcome = next(r for r in snapshot.records if r.kind == "materialization") verification = outcome.verification @@ -261,7 +262,8 @@ async def authorize(reference, context, consumer): context = ToolContext(caller_identity=scope.consumer_id) recorded = await handler.sync_reporting_receipts(request, context) assert recorded["results"][0]["result"] == "recorded" - assert await handler.sync_reporting_receipts(request, context) == recorded + receipt_operation_2 = await handler.sync_reporting_receipts(request, context) + assert receipt_operation_2 == recorded assert len(await receipt_store.read_receipt_boundaries(caller=scope.principal)) == 1 assert receipt_store._materializer_outbox is None workspace = Path(config["workspace"]).resolve() diff --git a/tests/conformance/reporting/_receipt_frozen.py b/tests/conformance/reporting/_receipt_frozen.py index 5c23aed9b..fbd82911c 100644 --- a/tests/conformance/reporting/_receipt_frozen.py +++ b/tests/conformance/reporting/_receipt_frozen.py @@ -143,8 +143,10 @@ async def project_ordinary_status(): failure_code="WRITE_FAILED", completed_at=now, ) - assert (await store.commit_materialization(outcome))[1] - assert not (await store.commit_materialization(outcome))[1] + receipt_operation_1 = await store.commit_materialization(outcome) + assert (receipt_operation_1)[1] + receipt_operation_2 = await store.commit_materialization(outcome) + assert not (receipt_operation_2)[1] read = await store.get_materialization(outcome.key) assert read is not None and read.outcome == outcome ordinary_materializer = True @@ -241,7 +243,8 @@ async def claim_expansion(self, **kwargs): ) while await worker.expand_one(account_id=account): assert len(seen) <= len(events) - assert not await worker.deliver_one(account_id=account) + receipt_operation_3 = await worker.deliver_one(account_id=account) + assert not receipt_operation_3 assert await outbox.list_events(account_id=account) == events workers[name] = len(seen) if settings["phase"] == "before": diff --git a/tests/conformance/reporting/_receipt_installed.py b/tests/conformance/reporting/_receipt_installed.py index 8d8c93c71..ef9381a26 100644 --- a/tests/conformance/reporting/_receipt_installed.py +++ b/tests/conformance/reporting/_receipt_installed.py @@ -55,7 +55,8 @@ async def main(settings): async with mount.client() as client: _, result = await mount.mcp(client, settings["request"]) assert [r["result"] for r in result["results"]] == ["recorded", "recorded"] - assert (await mount.a2a(client, settings["request"]))[1] == result + receipt_operation_1 = await mount.a2a(client, settings["request"]) + assert (receipt_operation_1)[1] == result mount.grants.clear() for call in (mount.mcp, mount.a2a): _, denied = await call(client, settings["request"]) diff --git a/tests/conformance/reporting/_receipt_process.py b/tests/conformance/reporting/_receipt_process.py index c76918c2a..7a336fe11 100644 --- a/tests/conformance/reporting/_receipt_process.py +++ b/tests/conformance/reporting/_receipt_process.py @@ -35,7 +35,8 @@ async def main(): async def pause(point): if settings.get("pause") == point: print(json.dumps({"point": point}), flush=True) - assert json.loads(await asyncio.to_thread(sys.stdin.readline))["continue"] + receipt_operation_1 = await asyncio.to_thread(sys.stdin.readline) + assert json.loads(receipt_operation_1)["continue"] class Connection(AsyncConnection): async def execute(self, query, params=None, **kwargs): diff --git a/tests/conformance/reporting/test_reporting_generation_identity.py b/tests/conformance/reporting/test_reporting_generation_identity.py index d5132bdf8..d55480681 100644 --- a/tests/conformance/reporting/test_reporting_generation_identity.py +++ b/tests/conformance/reporting/test_reporting_generation_identity.py @@ -145,7 +145,10 @@ async def test_concurrent_leases_and_releases_keep_accounts_separate( ) replacement = await store.lease_period_close(worker_id="next", now=NOW, lease_seconds=60) assert replacement is not None and replacement.generation_key == first.generation_key - assert await store.lease_period_close(worker_id="extra", now=NOW, lease_seconds=60) is None + receipt_operation_1 = await store.lease_period_close( + worker_id="extra", now=NOW, lease_seconds=60 + ) + assert receipt_operation_1 is None await asyncio.gather( store.release_period_close(second, worker_id="shared"), store.release_period_close(replacement, worker_id="next"), @@ -170,11 +173,17 @@ async def test_an_expired_lease_cannot_release_a_replacement_with_the_same_worke replacement = await store.lease_period_close(worker_id="shared", now=later, lease_seconds=60) assert replacement is not None and replacement.generation_key == expired.generation_key await store.release_period_close(expired, worker_id="shared") - assert await store.lease_period_close(worker_id="extra", now=later, lease_seconds=60) is None + receipt_operation_2 = await store.lease_period_close( + worker_id="extra", now=later, lease_seconds=60 + ) + assert receipt_operation_2 is None await store.release_period_close(replacement, worker_id="shared") reclaimed = await store.lease_period_close(worker_id="new", now=later, lease_seconds=60) assert reclaimed is not None and reclaimed.generation_key == expired.generation_key - assert await store.lease_period_close(worker_id="extra", now=later, lease_seconds=60) is None + receipt_operation_3 = await store.lease_period_close( + worker_id="extra", now=later, lease_seconds=60 + ) + assert receipt_operation_3 is None async def test_a_worker_that_releases_each_turn_reaches_every_accounts_generation( @@ -592,7 +601,10 @@ async def fetch(request: ReportingSourceSliceRequestV1) -> None: assert finished.leased is not None and finished.leased.account_id == "acct_a" probe = await store.lease_period_close(worker_id="probe", now=NOW, lease_seconds=60) assert probe is not None and probe.generation_key == configs[0].generation_key - assert await store.lease_period_close(worker_id="extra", now=NOW, lease_seconds=60) is None + receipt_operation_4 = await store.lease_period_close( + worker_id="extra", now=NOW, lease_seconds=60 + ) + assert receipt_operation_4 is None await store.release_period_close(probe, worker_id="probe") finish["acct_b"].set() turns = await asyncio.wait_for(asyncio.gather(*tasks), 10) diff --git a/tests/conformance/reporting/test_reporting_generation_migration.py b/tests/conformance/reporting/test_reporting_generation_migration.py index a7b5bd771..0182a2419 100644 --- a/tests/conformance/reporting/test_reporting_generation_migration.py +++ b/tests/conformance/reporting/test_reporting_generation_migration.py @@ -187,17 +187,22 @@ async def test_beta15_upgrade_preserves_all_evidence_and_survives_concurrent_boo account_id=old.account_id, reporting_revision_id=revision.reporting_revision_id ) assert rows.rows == ({"media_buy_id": "mb_acct_a", "impressions": 5},) - assert await store.commit_revision(revision, rows.rows) == revision + receipt_operation_1 = await store.commit_revision(revision, rows.rows) + assert receipt_operation_1 == revision statuses = await store.list_consumer_statuses( account_id=old.account_id, consumer_id="shared-buyer" ) assert len(statuses) == 1 and statuses[0].mismatch_code == "metric_missing" - assert await store.record_consumer_status(statuses[0]) == (statuses[0], False) + receipt_operation_2 = await store.record_consumer_status(statuses[0]) + assert receipt_operation_2 == (statuses[0], False) assert await _retained_rows(pool) == before # Replays append no new feed entries. # Existing leases survive, and releasing an old handle must not release # a new tenant's same-name generation even if the worker id is reused. - assert await store.lease_period_close(worker_id="extra", now=NOW, lease_seconds=60) is None + receipt_operation_3 = await store.lease_period_close( + worker_id="extra", now=NOW, lease_seconds=60 + ) + assert receipt_operation_3 is None other = configuration("acct_b") await store.put_configuration(other) other_lease = await store.lease_period_close( @@ -210,7 +215,10 @@ async def test_beta15_upgrade_preserves_all_evidence_and_survives_concurrent_boo worker_id="new-worker", now=NOW, lease_seconds=60 ) assert reclaimed is not None and reclaimed.generation_key == old.generation_key - assert await store.lease_period_close(worker_id="extra", now=NOW, lease_seconds=60) is None + receipt_operation_4 = await store.lease_period_close( + worker_id="extra", now=NOW, lease_seconds=60 + ) + assert receipt_operation_4 is None # The retained feed and its sequence continue; no history is renumbered. checkpoint = await store.open_snapshot(account_id=old.account_id, filters_fingerprint="") diff --git a/tests/conformance/reporting/test_reporting_receipt_batches.py b/tests/conformance/reporting/test_reporting_receipt_batches.py index b1d04a18c..46e359e56 100644 --- a/tests/conformance/reporting/test_reporting_receipt_batches.py +++ b/tests/conformance/reporting/test_reporting_receipt_batches.py @@ -60,7 +60,10 @@ async def test_mixed_batch_order_durable_failure_and_exact_response_replay(recei assert replay == result and "replayed" not in replay assert await h.image() == before replay["results"].clear() - assert await h.store.ingest_receipt_batch(request, caller=s.attempt.scope.principal) == result + receipt_operation_1 = await h.store.ingest_receipt_batch( + request, caller=s.attempt.scope.principal + ) + assert receipt_operation_1 == result @pytest.mark.parametrize( @@ -313,10 +316,10 @@ async def test_accepted_artifact_authority_survives_later_outcomes_separately_fr view = snapshot.materialization(s.attempt.key) assert view.outcome == s.outcome assert view.readable_at(at) is (later not in {"expiry", "corruption"}) - assert ( - await h.store.ingest_receipt_batch(request_for(s), caller=s.attempt.scope.principal) - == result + receipt_operation_2 = await h.store.ingest_receipt_batch( + request_for(s), caller=s.attempt.scope.principal ) + assert receipt_operation_2 == result async def test_hundred_combined_results_and_adjustment_only_admission(receipts): @@ -341,9 +344,8 @@ async def test_hundred_combined_results_and_adjustment_only_admission(receipts): "idempotency_key": "only-adjustment-0001", "adjustment_receipts": [request["adjustment_receipts"][0]], } - assert (await h.store.ingest_receipt_batch(only, caller=s.attempt.scope.principal))["results"][ - 0 - ]["result"] == "unchanged" + receipt_operation_3 = await h.store.ingest_receipt_batch(only, caller=s.attempt.scope.principal) + assert (receipt_operation_3)["results"][0]["result"] == "unchanged" @pytest.mark.parametrize( @@ -379,7 +381,8 @@ async def test_whole_request_utf16_key_order_survives_storage_and_exact_replay(r response = await h.store.ingest_receipt_batch(request, caller=s.binding.principal) assert response["results"][0]["result"] == "recorded" assert response["context"] == context - assert await h.store.ingest_receipt_batch(request, caller=s.binding.principal) == response + receipt_operation_4 = await h.store.ingest_receipt_batch(request, caller=s.binding.principal) + assert receipt_operation_4 == response before = await h.image() changed = {**request, "context": {**context, "nested": {"\ue000": "changed"}}} with pytest.raises(ReportingReceiptError) as error: diff --git a/tests/conformance/reporting/test_reporting_receipt_graph.py b/tests/conformance/reporting/test_reporting_receipt_graph.py index d3e92485f..92637ac71 100644 --- a/tests/conformance/reporting/test_reporting_receipt_graph.py +++ b/tests/conformance/reporting/test_reporting_receipt_graph.py @@ -81,7 +81,8 @@ async def test_exact_artifact_predicates_produce_durable_item_failures(receipts, assert result["results"][0]["errors"][0]["code"] == expected assert not await h.store.get_receipt(s.receipt.key) assert not await h.store.read_receipt_boundaries(caller=s.binding.principal) - assert await h.store.ingest_receipt_batch(request, caller=s.binding.principal) == result + receipt_operation_1 = await h.store.ingest_receipt_batch(request, caller=s.binding.principal) + assert receipt_operation_1 == result @pytest.mark.parametrize( @@ -171,7 +172,8 @@ async def test_official_adjustment_digest_ownership_and_order(receipts, fault, e assert result["results"][0]["result"] == "recorded" assert result["results"][1]["errors"][0]["code"] == expected assert result["results"][1]["reporting_receipt_id"] == item["reporting_receipt_id"] - assert await h.store.ingest_receipt_batch(request, caller=s.binding.principal) == result + receipt_operation_2 = await h.store.ingest_receipt_batch(request, caller=s.binding.principal) + assert receipt_operation_2 == result @pytest.mark.parametrize( @@ -205,18 +207,16 @@ async def test_adjustment_rejected_current_leaf_replacement_and_accepted_termina item = await adjustment_for(h, s) rejected = {**item, "status": "rejected", "rejection_codes": ["LOAD_FAILED"]} first = request_for(s, adjustment_receipts=[rejected]) - assert (await h.store.ingest_receipt_batch(first, caller=s.binding.principal))["results"][1][ - "result" - ] == "recorded" + receipt_operation_3 = await h.store.ingest_receipt_batch(first, caller=s.binding.principal) + assert (receipt_operation_3)["results"][1]["result"] == "recorded" accepted = { **item, "reporting_receipt_id": "accepted-adjustment-0002", "supersedes_reporting_receipt_id": item["reporting_receipt_id"], } second = request_for(s, key="receipt-batch-0002", adjustment_receipts=[accepted]) - assert (await h.store.ingest_receipt_batch(second, caller=s.binding.principal))["results"][1][ - "result" - ] == "recorded" + receipt_operation_4 = await h.store.ingest_receipt_batch(second, caller=s.binding.principal) + assert (receipt_operation_4)["results"][1]["result"] == "recorded" third = request_for( s, key="receipt-batch-0003", @@ -394,7 +394,8 @@ async def test_all_receipt_evidence_profiles_match_exact_immutable_artifact( accepted = await h.store.ingest_receipt_batch(request, caller=s.binding.principal) assert accepted["results"][0]["result"] == "recorded" assert accepted["results"][0]["receipt"][field] == request["receipts"][0][field] - assert await h.store.ingest_receipt_batch(request, caller=s.binding.principal) == accepted + receipt_operation_5 = await h.store.ingest_receipt_batch(request, caller=s.binding.principal) + assert receipt_operation_5 == accepted async def test_snapshot_adjustment_is_never_admitted_as_official_receipt_evidence(receipts): @@ -495,7 +496,8 @@ def build(overrides, key, receipt_id): assert failed_id not in {b.reporting_receipt_id for b in boundaries} assert len(boundaries) == index # Identical durable replay, and an identical closed error for an absent row. - assert await h.store.ingest_receipt_batch(request, caller=s.binding.principal) == result + receipt_operation_6 = await h.store.ingest_receipt_batch(request, caller=s.binding.principal) + assert receipt_operation_6 == result absent = await h.store.ingest_receipt_batch(control, caller=s.binding.principal) assert absent["results"][index]["result"] == "failed" assert absent["results"][index]["errors"] == failure["errors"] diff --git a/tests/conformance/reporting/test_reporting_receipt_installed_pg.py b/tests/conformance/reporting/test_reporting_receipt_installed_pg.py index bc8507a04..3d2488963 100644 --- a/tests/conformance/reporting/test_reporting_receipt_installed_pg.py +++ b/tests/conformance/reporting/test_reporting_receipt_installed_pg.py @@ -120,10 +120,10 @@ async def invoke(settings): ) assert done["result"]["results"][0]["receipt"] == receipt_to_wire(original) assert done["origins"] == ready["origins"] - assert ( - await h.store.ingest_receipt_batch(request, caller=s.binding.principal) - == done["result"] + receipt_operation_1 = await h.store.ingest_receipt_batch( + request, caller=s.binding.principal ) + assert receipt_operation_1 == done["result"] assert len(await h.store.read_receipt_boundaries(caller=s.binding.principal)) == 2 assert await h.queue() == ((), ()) print( diff --git a/tests/conformance/reporting/test_reporting_receipt_materializer.py b/tests/conformance/reporting/test_reporting_receipt_materializer.py index 8bbf63345..42dd494fa 100644 --- a/tests/conformance/reporting/test_reporting_receipt_materializer.py +++ b/tests/conformance/reporting/test_reporting_receipt_materializer.py @@ -39,9 +39,10 @@ async def test_receipt_store_atomic_finish_capture_ack_and_enabled_enqueue(backe await h.store.finish_materialization(lease, prepared=prepared, verified=verified) monkeypatch.undo() assert await h.image() == before - assert ( - await h.store.finish_materialization(lease, prepared=prepared, verified=verified) - ).state == "verified" + receipt_operation_1 = await h.store.finish_materialization( + lease, prepared=prepared, verified=verified + ) + assert (receipt_operation_1).state == "verified" captured = await h.store.read_materializer_boundaries(caller=case.scope.principal) assert len(captured) == 1 and captured[0].account_sequence == 1 assert (await h.queue())[1] == ("quarantined",) diff --git a/tests/conformance/reporting/test_reporting_receipt_migration.py b/tests/conformance/reporting/test_reporting_receipt_migration.py index b1ec57a65..d0af8b47e 100644 --- a/tests/conformance/reporting/test_reporting_receipt_migration.py +++ b/tests/conformance/reporting/test_reporting_receipt_migration.py @@ -31,7 +31,8 @@ async def test_populated_repeated_concurrent_migration_preserves_parent_objects_ old = PgReportingMaterializerStore(pool=pool, notifications=notifications) await old.create_schema() case = await durable_case(old) - assert (await case.service().run_once()).state == "verified" + receipt_operation_1 = await case.service().run_once() + assert (receipt_operation_1).state == "verified" h = DurableHarness(old, Clock(), pool) before = await h.image() captures = await old.read_materializer_boundaries(caller=case.scope.principal) diff --git a/tests/conformance/reporting/test_reporting_receipt_process.py b/tests/conformance/reporting/test_reporting_receipt_process.py index de35e553d..1c68447a5 100644 --- a/tests/conformance/reporting/test_reporting_receipt_process.py +++ b/tests/conformance/reporting/test_reporting_receipt_process.py @@ -87,13 +87,17 @@ async def test_process_crash_resumes_original_ordinals_and_final_response(point, original = await h.store.get_receipt(s.receipt.key) async with worker(h, s, request) as resumed: response = (await resumed.event("done"))["result"] - assert await asyncio.wait_for(resumed.process.wait(), 5) == 0 + receipt_operation_2 = await asyncio.wait_for(resumed.process.wait(), 5) + assert receipt_operation_2 == 0 assert [r["result"] for r in response["results"]] == ["recorded", "recorded"] if original is not None: from adcp.reporting.ledger.delivery import receipt_to_wire assert response["results"][0]["receipt"] == receipt_to_wire(original) - assert await h.store.ingest_receipt_batch(request, caller=s.binding.principal) == response + receipt_operation_1 = await h.store.ingest_receipt_batch( + request, caller=s.binding.principal + ) + assert receipt_operation_1 == response assert await batch_state(h) == ((2, True),) assert len(await h.store.read_receipt_boundaries(caller=s.binding.principal)) == 2 assert await h.queue() == ((), ()) @@ -113,5 +117,7 @@ async def test_two_processes_serialize_with_size_one_pools_and_keep_original_rec a = (await first.event("done"))["result"] b = (await asyncio.wait_for(pending, 30))["result"] assert a == b and a["results"][0]["result"] == "recorded" - assert await asyncio.wait_for(second.process.wait(), 5) == 0 - assert await asyncio.wait_for(first.process.wait(), 5) == 0 + receipt_operation_4 = await asyncio.wait_for(second.process.wait(), 5) + assert receipt_operation_4 == 0 + receipt_operation_3 = await asyncio.wait_for(first.process.wait(), 5) + assert receipt_operation_3 == 0 diff --git a/tests/conformance/reporting/test_reporting_receipt_rolling.py b/tests/conformance/reporting/test_reporting_receipt_rolling.py index be41d13f8..1145eb792 100644 --- a/tests/conformance/reporting/test_reporting_receipt_rolling.py +++ b/tests/conformance/reporting/test_reporting_receipt_rolling.py @@ -24,7 +24,7 @@ from .test_reporting_materializer_process import worker from .test_reporting_materializer_rolling import ARTIFACTS, build_frozen, frozen_call -B21 = "8e18ca12b9a0c3750f80aa822058c02982ab3e52" +B21 = "3fd62121c96a074e3ea458c30c5224d6a586f169" def receipt_probe(artifact): @@ -108,7 +108,8 @@ async def test_actual_old_readers_and_writers_before_and_after_receipt_migration ) as child: produced = await child.event("done") assert produced["state"] == "verified", produced - assert await asyncio.wait_for(child.process.wait(), 5) == 0 + receipt_operation_2 = await asyncio.wait_for(child.process.wait(), 5) + assert receipt_operation_2 == 0 outcome = (await case.outcomes())[0] assert outcome.verification is not None kwargs = { @@ -206,9 +207,10 @@ async def test_actual_old_readers_and_writers_before_and_after_receipt_migration == after["ordinary_materializer"] == (artifact != "beta15") ) - assert ( - await store.ingest_receipt_batch(request, caller=case.scope.principal) == response + receipt_operation_1 = await store.ingest_receipt_batch( + request, caller=case.scope.principal ) + assert receipt_operation_1 == response assert await store.read_receipt_boundaries(caller=case.scope.principal) == captured assert len(captured) == 2 and captured[0].account_sequence > 1 assert await h.queue() == queue diff --git a/tests/conformance/reporting/test_reporting_receipt_transactions.py b/tests/conformance/reporting/test_reporting_receipt_transactions.py index e02dc9666..9a80deb61 100644 --- a/tests/conformance/reporting/test_reporting_receipt_transactions.py +++ b/tests/conformance/reporting/test_reporting_receipt_transactions.py @@ -267,9 +267,10 @@ async def test_ordinary_status_dirty_insert_failure_rolls_back_receipt_and_ordin await h.store.ingest_receipt_batch(request_for(s), caller=s.binding.principal) monkeypatch.undo() assert await h.image() == before - assert (await h.store.ingest_receipt_batch(request_for(s), caller=s.binding.principal))[ - "results" - ][0]["result"] == "recorded" + receipt_operation_1 = await h.store.ingest_receipt_batch( + request_for(s), caller=s.binding.principal + ) + assert (receipt_operation_1)["results"][0]["result"] == "recorded" @pytest.mark.parametrize("damage", ["id", "kind", "outcome", "received_at", "ordinal"]) diff --git a/tests/conformance/reporting/test_reporting_receipt_transports.py b/tests/conformance/reporting/test_reporting_receipt_transports.py index 5dd3b413b..3fc3a877a 100644 --- a/tests/conformance/reporting/test_reporting_receipt_transports.py +++ b/tests/conformance/reporting/test_reporting_receipt_transports.py @@ -152,7 +152,8 @@ async def chunks(): async with mount.client() as client: _, first = await mount.a2a(client, request, mutate_wire=chunked) assert first["results"][0]["result"] == "recorded", first - assert (await mount.mcp(client, request))[1] == first + receipt_operation_1 = await mount.mcp(client, request) + assert (receipt_operation_1)[1] == first assert len(mount.auth_calls) == 2 @@ -527,7 +528,8 @@ async def test_raw_capture_cannot_supply_auth_or_cross_request_body_or_override_ # Avoid a fresh initialize in this helper: the authenticated # tool request itself must reject a forged token. mount.sessions["forged"] = mount.sessions["token-one"] - assert (await call(client, request, token="forged"))[0] == 401 + receipt_operation_6 = await call(client, request, token="forged") + assert (receipt_operation_6)[0] == 401 assert await h.image() == before @@ -597,13 +599,14 @@ async def test_raw_mounted_requests_do_not_alias_account_consumer_pairs_or_cache _, b = await mount.a2a(client, request_for(second), token="token-two") assert a["results"][0]["result"] == b["results"][0]["result"] == "recorded" assert a != b - assert (await mount.a2a(client, request_for(first)))[1] == a - assert (await mount.mcp(client, request_for(second), token="token-two"))[1] == b + receipt_operation_2 = await mount.a2a(client, request_for(first)) + assert (receipt_operation_2)[1] == a + receipt_operation_3 = await mount.mcp(client, request_for(second), token="token-two") + assert (receipt_operation_3)[1] == b before = await h.image() - assert error_code((await mount.a2a(client, request_for(second)))[1]) == "UNAUTHORIZED" - assert ( - error_code((await mount.mcp(client, request_for(first), token="token-two"))[1]) - == "UNAUTHORIZED" - ) + receipt_operation_4 = await mount.a2a(client, request_for(second)) + assert error_code((receipt_operation_4)[1]) == "UNAUTHORIZED" + receipt_operation_5 = await mount.mcp(client, request_for(first), token="token-two") + assert error_code((receipt_operation_5)[1]) == "UNAUTHORIZED" assert await h.image() == before assert len({c.caller_identity for c in mount.contexts}) == 1 From c4494c63b080dd44c4c36307d2942bc25c899411 Mon Sep 17 00:00:00 2001 From: Brian O'Kelley Date: Thu, 24 Sep 2026 16:25:11 +0000 Subject: [PATCH 07/10] fix(reporting): align receipt fixtures with the rc.6 contract --- .github/workflows/ci.yml | 10 ++-------- docs/reporting-receipt-ingress.md | 4 ++-- examples/reporting_receipt_ingress.py | 2 +- tests/conformance/reporting/_receipt_support.py | 2 +- .../reporting/test_reporting_receipt_rolling.py | 2 +- .../reporting/test_reporting_receipt_transports.py | 10 +++++----- 6 files changed, 12 insertions(+), 18 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 0a8fc4319..7ce8fb721 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -393,14 +393,8 @@ jobs: ADCP_PYTHON310: ${{ steps.receipt-python310.outputs.python-path }} run: | python scripts/reporting_test_harness.py pytest \ - tests/conformance/reporting/test_reporting_receipt_batches.py \ - tests/conformance/reporting/test_reporting_receipt_graph.py \ - tests/conformance/reporting/test_reporting_receipt_transactions.py \ - tests/conformance/reporting/test_reporting_receipt_transports.py \ - tests/conformance/reporting/test_reporting_receipt_materializer.py \ - tests/conformance/reporting/test_reporting_receipt_migration.py \ - tests/conformance/reporting/test_reporting_receipt_process.py \ - tests/conformance/reporting/test_reporting_receipt_installed_pg.py \ + tests/conformance/reporting/test_reporting_receipt_*.py \ + --ignore=tests/conformance/reporting/test_reporting_receipt_rolling.py \ -v -s -ra | tee pg-reporting-receipts-evidence.log - name: Preserve commands, counts, installed provenance and skip reasons if: always() diff --git a/docs/reporting-receipt-ingress.md b/docs/reporting-receipt-ingress.md index 7c7d8ae08..578e48690 100644 --- a/docs/reporting-receipt-ingress.md +++ b/docs/reporting-receipt-ingress.md @@ -48,8 +48,8 @@ receipt visibility. ## Wire and replay contract -Requests negotiate `adcp_version: "3.2-rc.3"`. This is the wire release spelling; -`3.2.0-rc.3` is the bundle's semantic-version spelling. Unnegotiated traffic keeps +Requests negotiate `adcp_version: "3.2-rc.6"`. This is the wire release spelling; +`3.2.0-rc.6` is the bundle's semantic-version spelling. Unnegotiated traffic keeps the SDK's existing AdCP 3.0 behavior; unsupported versions remain unsupported. Pinned, unpinned and fallback MCP inventories use an isolated schema overlay. Cached upstream schemas, generated models and older required protocols remain diff --git a/examples/reporting_receipt_ingress.py b/examples/reporting_receipt_ingress.py index b0ac3e1c5..740640ca1 100644 --- a/examples/reporting_receipt_ingress.py +++ b/examples/reporting_receipt_ingress.py @@ -2,7 +2,7 @@ Install migrations during deployment, drain legacy materialization writers, and supply the application's token validator and fresh account ACL. Receipt traffic -negotiates AdCP 3.2-rc.3. This composition does not activate production tier claims. +negotiates AdCP 3.2-rc.6. This composition does not activate production tier claims. """ from __future__ import annotations diff --git a/tests/conformance/reporting/_receipt_support.py b/tests/conformance/reporting/_receipt_support.py index 52ade8487..25b82b946 100644 --- a/tests/conformance/reporting/_receipt_support.py +++ b/tests/conformance/reporting/_receipt_support.py @@ -54,7 +54,7 @@ async def receipt_case(h, **kwargs): def request_for(s, *, key="receipt-batch-0001", **changes): return { - "adcp_version": "3.2-rc.3", + "adcp_version": "3.2-rc.6", "account": {"account_id": s.obligation.account_id}, "idempotency_key": key, "receipts": [receipt_to_wire(s.receipt)], diff --git a/tests/conformance/reporting/test_reporting_receipt_rolling.py b/tests/conformance/reporting/test_reporting_receipt_rolling.py index 1145eb792..0d8576b2a 100644 --- a/tests/conformance/reporting/test_reporting_receipt_rolling.py +++ b/tests/conformance/reporting/test_reporting_receipt_rolling.py @@ -172,7 +172,7 @@ async def test_actual_old_readers_and_writers_before_and_after_receipt_migration observed_canonical_content_digest=verification.canonical_content_digest, ) request = { - "adcp_version": "3.2-rc.3", + "adcp_version": "3.2-rc.6", "account": {"account_id": case.config.account_id}, "idempotency_key": "historical-mixed-receipts", "receipts": [receipt_to_wire(receipt)], diff --git a/tests/conformance/reporting/test_reporting_receipt_transports.py b/tests/conformance/reporting/test_reporting_receipt_transports.py index 3fc3a877a..1686a2c45 100644 --- a/tests/conformance/reporting/test_reporting_receipt_transports.py +++ b/tests/conformance/reporting/test_reporting_receipt_transports.py @@ -89,7 +89,7 @@ async def test_protobuf_integer_spellings_preserve_whole_request_identity_and_ti ): h = receipts s = await receipt_case(h) - mount = MountedReceipts(h, hydrated=True, version="3.2-rc.3") + mount = MountedReceipts(h, hydrated=True, version="3.2-rc.6") mount.authorize(s) request = request_for( s, @@ -103,7 +103,7 @@ async def test_protobuf_integer_spellings_preserve_whole_request_identity_and_ti protobuf_request = MessageToDict(proto) assert type(protobuf_request["receipts"][0]["observed_row_count"]) is float assert type(protobuf_request["context"]["integer"]) is float - assert Draft7Validator(receipt_schema("request", version="3.2-rc.3")).is_valid(protobuf_request) + assert Draft7Validator(receipt_schema("request", version="3.2-rc.6")).is_valid(protobuf_request) requests = {"mcp": request, "a2a": protobuf_request} other = "a2a" if first_route == "mcp" else "mcp" async with mount.client() as client: @@ -286,7 +286,7 @@ async def test_actual_mount_shape_failure_is_task_invalid_request_with_no_writes assert mount.auth_calls == [] -@pytest.mark.parametrize("version", [None, "3.2-rc.3"]) +@pytest.mark.parametrize("version", [None, "3.2-rc.6"]) @pytest.mark.parametrize("fallback", [False, True]) async def test_pinned_unpinned_and_fallback_mcp_schemas_keep_combined_and_received_rules( version, fallback, monkeypatch @@ -421,7 +421,7 @@ async def test_real_raw_strict_json_and_numeric_admission_fail_before_batch_muta ): h = receipts s = await receipt_case(h) - mount = MountedReceipts(h, version="3.2-rc.3") + mount = MountedReceipts(h, version="3.2-rc.6") mount.authorize(s) request = request_for(s) mutations = { @@ -482,7 +482,7 @@ async def test_real_raw_strict_json_and_numeric_admission_fail_before_batch_muta async def test_malformed_numeric_token_is_rejected_by_the_actual_json_decoder(receipts, route): h = receipts s = await receipt_case(h) - mount = MountedReceipts(h, version="3.2-rc.3") + mount = MountedReceipts(h, version="3.2-rc.6") mount.authorize(s) request = request_for(s) before = await h.image() From 4d13d4abb5c5e079a9d46041f36d50628e351857 Mon Sep 17 00:00:00 2001 From: Brian O'Kelley Date: Thu, 24 Sep 2026 16:37:36 +0000 Subject: [PATCH 08/10] test(reporting): align installed receipt probe with rc.6 --- tests/conformance/reporting/_materializer_installed.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/conformance/reporting/_materializer_installed.py b/tests/conformance/reporting/_materializer_installed.py index 07b16a7ab..4fd2d0358 100644 --- a/tests/conformance/reporting/_materializer_installed.py +++ b/tests/conformance/reporting/_materializer_installed.py @@ -247,7 +247,7 @@ async def main(): observed_canonical_content_digest=verification.canonical_content_digest, ) request = { - "adcp_version": "3.2-rc.3", + "adcp_version": "3.2-rc.6", "account": {"account_id": configuration.account_id}, "idempotency_key": "installed-batch-0001", "receipts": [receipt_to_wire(receipt)], From 5b23778f0f813f6ebfaf78d2770c520b3d8169a0 Mon Sep 17 00:00:00 2001 From: Brian O'Kelley Date: Thu, 24 Sep 2026 16:57:06 +0000 Subject: [PATCH 09/10] test(reporting): group receipt migration damage statements --- .../test_reporting_receipt_migration.py | 24 ++++++++++++------- 1 file changed, 16 insertions(+), 8 deletions(-) diff --git a/tests/conformance/reporting/test_reporting_receipt_migration.py b/tests/conformance/reporting/test_reporting_receipt_migration.py index d0af8b47e..71955facc 100644 --- a/tests/conformance/reporting/test_reporting_receipt_migration.py +++ b/tests/conformance/reporting/test_reporting_receipt_migration.py @@ -94,21 +94,29 @@ async def migrate(): "damage", [ "DROP INDEX reporting_receipt_ingestion_chain", - "ALTER TABLE reporting_receipt_ingestion_results " - "DISABLE TRIGGER reporting_receipt_ingestion_result", + ( + "ALTER TABLE reporting_receipt_ingestion_results " + "DISABLE TRIGGER reporting_receipt_ingestion_result" + ), "ALTER TABLE reporting_receipt_ingestion_batches ALTER COLUMN expected_count DROP NOT NULL", "DROP TABLE reporting_receipt_ingestion_boundaries CASCADE", "ALTER TABLE reporting_materializer_work DISABLE TRIGGER reporting_materializer_guard", - "ALTER TABLE reporting_reconciliation_records " - "DISABLE TRIGGER reporting_reconciliation_guard", + ( + "ALTER TABLE reporting_reconciliation_records " + "DISABLE TRIGGER reporting_reconciliation_guard" + ), # Every object still present and enabled, but a financial predicate body # silently replaced. Dropped objects and disabled triggers cannot stand # in for this: a weakened tuple predicate is exactly how a deployment # would lose graph enforcement without any visible schema difference. - "CREATE OR REPLACE FUNCTION reporting_receipt_ingestion_graph()" - " RETURNS TRIGGER LANGUAGE plpgsql AS $damage$ BEGIN RETURN NEW; END $damage$", - "CREATE OR REPLACE FUNCTION reporting_receipt_ingestion_sha256(document JSONB)" - " RETURNS TEXT LANGUAGE SQL IMMUTABLE STRICT AS $damage$ SELECT repeat('0', 64) $damage$", + ( + "CREATE OR REPLACE FUNCTION reporting_receipt_ingestion_graph()" + " RETURNS TRIGGER LANGUAGE plpgsql AS $damage$ BEGIN RETURN NEW; END $damage$" + ), + ( + "CREATE OR REPLACE FUNCTION reporting_receipt_ingestion_sha256(document JSONB)" + " RETURNS TEXT LANGUAGE SQL IMMUTABLE STRICT AS $damage$ SELECT repeat('0', 64) $damage$" + ), ], ) @pytest.mark.parametrize("notifications", [False, True]) From 624c1001ed42a210462ddb6de76806f9c1a81fc8 Mon Sep 17 00:00:00 2001 From: Brian O'Kelley Date: Thu, 24 Sep 2026 17:10:23 +0000 Subject: [PATCH 10/10] fix(ci): separate receipt rolling inputs before pytest --- .github/workflows/ci.yml | 15 +++++++++++++-- 1 file changed, 13 insertions(+), 2 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 7ce8fb721..139b8a3e8 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -392,9 +392,20 @@ jobs: ADCP_PG_TEST_URL: postgresql://postgres@localhost:5432/adcp_receipts_test ADCP_PYTHON310: ${{ steps.receipt-python310.outputs.python-path }} run: | + # Explicit pytest paths bypass --ignore; exclude rolling before invocation. + shopt -s nullglob + receipt_tests=() + for test_file in tests/conformance/reporting/test_reporting_receipt_*.py; do + if [[ "$test_file" != tests/conformance/reporting/test_reporting_receipt_rolling.py ]]; then + receipt_tests+=("$test_file") + fi + done + if (( ${#receipt_tests[@]} == 0 )); then + echo "No receipt test files selected" >&2 + exit 1 + fi python scripts/reporting_test_harness.py pytest \ - tests/conformance/reporting/test_reporting_receipt_*.py \ - --ignore=tests/conformance/reporting/test_reporting_receipt_rolling.py \ + "${receipt_tests[@]}" \ -v -s -ra | tee pg-reporting-receipts-evidence.log - name: Preserve commands, counts, installed provenance and skip reasons if: always()