diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index c94a0335f..11a78e79e 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -4,7 +4,7 @@ on: push: branches: [main, python-adcp-sdk-setup] pull_request: - branches: [main, conductor/reporting-webhook-activity-1168b] + branches: [main, conductor/reporting-webhook-activity-1168b, conductor/reporting-status-notifications-1168c] # Default @adcp/sdk runner alias for storyboard jobs. Tracks the current # stable @adcp/sdk release via the ``latest`` npm dist-tag. @@ -61,7 +61,18 @@ jobs: test: name: Test Python ${{ matrix.python-version }} runs-on: ubuntu-latest - timeout-minutes: 30 + # The job ceiling is not the suite's budget: it also has to absorb + # checkout, Python setup, the editable [dev] install and, on 3.12 only, + # ruff/mypy/mypy --strict/the type-ignore contract before pytest starts, + # then post-job cleanup after it ends. On an ubuntu-latest runner that + # pre-test work is ~1m30s and the plain suite is ~16m, but 3.12 adds + # coverage tracing on ~9.4k tests: at 30 minutes that leg was cancelled + # mid-run at 99% with no failing test, which reports as a red matrix and + # hides real signal. Bound the suite itself below (so a hang fails one + # named step with its own message instead of silently taking the job), and + # leave the job enough room that step bound plus setup and cleanup still + # fit with margin for runner variance. + timeout-minutes: 60 strategy: matrix: python-version: ["3.10", "3.11", "3.12", "3.13"] @@ -91,18 +102,24 @@ jobs: - name: Run adopter type-check suite if: matrix.python-version == '3.12' - run: mypy --strict tests/type_checks/ examples/reporting_webhook_activity.py examples/reporting_status_notifications.py + run: mypy --strict tests/type_checks/ examples/reporting_webhook_activity.py examples/reporting_status_notifications.py examples/reporting_destination_writer.py - name: Enforce adopter type-check fixture contract if: matrix.python-version == '3.12' run: python scripts/check_type_ignore_contract.py + # Bounded well above the observed ~16m (plain) and ~30m (coverage) runs + # so ordinary variance never trips it, and well below the job ceiling so + # a genuinely stuck suite still fails *this* step with a timeout rather + # than being cancelled as a whole job. - name: Run tests if: matrix.python-version != '3.12' + timeout-minutes: 45 run: pytest tests/ -v - name: Run tests with coverage if: matrix.python-version == '3.12' + timeout-minutes: 45 run: pytest tests/ -v --cov=src/adcp --cov-report=term-missing pg-conformance: @@ -223,12 +240,13 @@ jobs: steps: - uses: actions/checkout@v6 - - name: Fetch exact reviewed A and B compatibility artifacts + - name: Fetch exact reviewed A, B and C compatibility artifacts timeout-minutes: 1 run: | git fetch --no-tags --depth=1 origin \ 17ee407ae3978c8a2bb54437287afbf9dafb8130 \ - 0f34c666ac1961e9832fce43ef0ef6937b3c1dde + 0f34c666ac1961e9832fce43ef0ef6937b3c1dde \ + 967b6e286301d7e5d089aea6fdbb90bea8ee5a16 - name: Set up Python 3.12 uses: actions/setup-python@v6 diff --git a/MANIFEST.in b/MANIFEST.in index 3ccfe0191..6cd472b94 100644 --- a/MANIFEST.in +++ b/MANIFEST.in @@ -3,6 +3,7 @@ include README.md include LICENSE include MIGRATION*.md recursive-include src/adcp py.typed +recursive-include src/adcp/reporting/materializer/assets *.json # Bundled AdCP JSON schemas. ``scripts/bundle_schemas.py`` mirrors # ``schemas/cache/`` into ``src/adcp/_schemas/`` before ``python -m # build`` so the validator ships with the wheel. Keep distributions on diff --git a/docs/reporting-destination-writer.md b/docs/reporting-destination-writer.md new file mode 100644 index 000000000..01a66bfc2 --- /dev/null +++ b/docs/reporting-destination-writer.md @@ -0,0 +1,177 @@ +# Verified destination I/O — #1167B1 + +**B1 of B1/B2**, composed with integrated #1168C at +`967b6e286301d7e5d089aea6fdbb90bea8ee5a16`. Refs #1167. + +B1 supplies immutable public contracts, whole-history revision selection, +SDK-owned source/destination verification, and a deterministic development +destination. Import from `adcp.reporting.materializer` and +`adcp.reporting.revision_selection`; PostgreSQL is optional. The strict +[adopter example](../examples/reporting_destination_writer.py) accepts existing +frozen ledger records and returns verification observations. + +There is no durable materializer, discovery queue, lease manager, retry +allocator, final outcome transaction, or materializer migration in B1. +`ReferenceReportingDestinationWriter.production_eligible` is always `False`, +including at the type level. It cannot be promoted by configuration or +subclassing. Configuring it, completing its readback, or freezing a destination +binding never advertises `managed_delivery`, `reconciled_billing`, or +`reporting.delivery_ready`. The real outbox readiness helper suppresses that +claim until B2 can prove a complete durable materializer. The producer's `extra` +argument rejects SDK-owned task, tier and notification keys. + +## Trusted contracts and lifecycle + +`ReportingDestinationRequest` includes the exact account and canonical consumer +(including HTTPS BuyerAgent identities), configuration generation, trusted +destination and binding references, binding fingerprint, obligation, revision, +materialization ID and attempt. Its verification key includes the complete +frozen definition/schema, canonicalization and capability tuples. Aliases must +be resolved by trusted middleware before this boundary. Credentials are never +protocol arguments, request fields, locators, verification records or metadata. + +`ReportingWriterCapability` closes method, transport, format, profile, readback +path, immutable-location/native-version mode, SHA-256 checksum and conditional +or idempotent write semantics. An unsupported tuple fails before resolver or +writer I/O. The immutable registry holds explicitly installed contract bytes; +it has no registration mutation, plugin callback or network fallback. + +A resolver synchronously constructs an **unopened** SDK +`ReportingDestinationSession`. Resource acquisition and fresh authorization +belong in `_open`; all credentials and partial acquisitions belong to that +redacted, non-persistable session. `_close` must release every acquired resource +and any adopter spool, including after an interrupted `_open`. Override the +protected hooks and I/O methods, preserving the SDK-owned context manager, +redaction and exactly-once close. Read methods must resolve locators inside the +session's trusted destination namespace and reject another tenant's paths. + +Write and readback open separate sessions and independently authorize the same +frozen request, so rotation/revocation between phases is observable. The SDK +checks the resolved request before opening and before I/O. `ReportingIOContext` +carries an absolute UTC deadline, cancellation event and optional service-owned +heartbeat checkpoint. It uses Python-3.10-compatible primitives, joins canceled +tasks, shields bounded cleanup and re-raises a clean `CancelledError`. The +optional heartbeat is a call boundary; B1 starts no lease-extension loop. + +Writers receive SDK-produced immutable canonical row bytes inside +`ReportingPreparedRevision`, never mutable row dictionaries or arbitrary +metadata. They return `ReportingDestinationLocator` claims. Counts, checksums, +manifest hashes and native commit IDs supplied by a writer establish where and +what to read; they cannot establish verification. + +`ReportingWriterFailure` contains only a closed code, retry instruction, +retry-after seconds and external-effect state (`not_started`, `applied`, +`unknown`). Unexpected provider failures become closed diagnostics without +provider exception chains. Do not construct messages from provider text or +log session internals, signed URLs, credentials or provider bodies. + +The external idempotency identity includes all tenant, consumer, binding, +revision and attempt coordinates. Resuming the same pending attempt preserves +it; equal attempt numbers on different revisions do not collide. Unknown +effects cannot request a new attempt. **Public foundation persistence still +allows N+1 after any immutable terminal outcome.** B2's autonomous retry rule +will be narrower: N+1 only after its own known terminal failure, never while N +is pending. Consumer receipt rejection is unchanged. Before activating B2, +drain legacy materialization writers and explicitly recover/import legacy +pending identities; their external-effect history cannot be inferred. + +## Installed canonicalizer and destination verification + +The small built-in implementation supports pinned `adcp_jcs_rows_v1` contracts, +Draft 2020-12 schemas with local fragment references, direct sum metrics and +complete integer/fixed-scale decimal control totals. Decimal values are strings; +JSON integers must be exactly typed and within the JavaScript safe range. +The pinned schema's `x-adcp-control-total` annotation specifies value type, +optional unit and decimal scale. All declared metrics must be represented. +The reference definition, row schema and canonicalization golden vectors are +bundled package assets with exact byte hashes in both distribution paths. + +This deliberately bounded JSON subset rejects floats, subclasses, tuples, +Decimal/datetime/bytes/set values, duplicate JSON members, invalid UTF-8 and lone +surrogates. It preserves Unicode without normalization and applies JCS UTF-16 +member ordering. Primary-key row ordering is by canonical scalar-key-array +bytes; duplicate primary keys fail. Golden vectors must test empty content, +nontrivial row ordering and member ordering. SHA-256 hexadecimal evidence is +validated and compared semantically, accepting uppercase and producing lowercase. + +Preparation reads **every** frozen source page, including zero rows and 501+ +rows, rederives the Core digest and the canonical digest, and recomputes typed +totals before destination authorization. SDK source cursors bind revision and +offset; custom row readers return the same `ReportingRowPage` identity/cursor +contract. Both bundled ledger stores now reject a cursor issued for another +revision and any `read_revision_rows` page size outside 1..500 with +`INVALID_CURSOR` / `INVALID_PAGE_SIZE`; a caller that paged in larger windows +must split its walk. Stable totals, cursor progress, cycles, `has_more` pairing +and final count are enforced. Source/destination walks bound bytes, recursive +items, nesting, rows, pages, objects and chunks. + +A retained Core `ReportingDefinitionBinding` is only loosely constrained, so an +obligation may carry a definition the strict verification key cannot express -- +a versioned query URI, an uppercase or short digest, a legacy dialect or schema +version. Preparation, write and readback answer that with the closed +`BINDING_MISMATCH` failure rather than a raw `ValueError`. + +Readback independently verifies every logical destination row in order. File +verification also reads the exact manifest bytes and its closed schema, +identities, period, creation time, complete typed totals, ordered object +inventory, and every streamed JSONL object's checksum, length, row count and +content. Native verification observes the pinned version, location and required +consumer/destination path before and after all pages; each page repeats that +version. A native commit ID alone cannot satisfy canonical-digest verification. + +| Method | B1 reference format | Supported profiles | Required readback | +| --- | --- | --- | --- | +| File transfer | JSONL, uncompressed | canonical digest, manifest checksums | logical pages + exact manifest + every object | +| Dataset share | logical typed rows | canonical digest, native commit | representative-consumer rows + pinned native observations | +| Warehouse materialization | logical typed rows | canonical digest, native commit | destination rows + pinned native observations | + +The verifier returns `ReportingVerifiedDestination` only after these reads. +Corruption returns a closed failure and no reusable verified result. The public +foundation's materialization transition validator checks retained claims; it is +not an independent destination reader. Its persisted `MaterializationFailure` +variants and record shapes are unchanged. + +## Current revision and B2 finish seam + +The neutral selector returns `selected`, `not_ready` or `corrupt`. It validates +ownership, duplicate IDs, every predecessor, finality edges, connected snapshot +history, forks, cycles and multiple officials before selecting. Only empty +history or absence of a required official is ordinary not-ready. A unique +official wins over an intact retained snapshot chain; otherwise snapshot +finality selects its one unsuperseded leaf. Readability and materialization +availability never select a revision or allow fallback. + +Core health, producer acquisition, status projection/validation/lifecycle, +consumer planning and reconciliation use that selector. Producer corruption +fails before source or adapter I/O. Status emits a stable `HISTORY_UNAVAILABLE` +issue. The exported `current_required_revision(...) -> record | None` remains +source compatible; internal selection consumes the typed result. + +`validate_materialization_target` is a pure seam for B2's locked finish path. +It checks the selected/readable exact revision and frozen binding against the +prepared input. It provides no transaction or fencing claim. B2 must reselect +on the same account-locked connection and co-commit immutable outcome, +reconciliation feed, work acknowledgment, C dirty and readiness notification. +A stale-after-I/O attempt must close as a compatible public safe failure with +its richer reason isolated in B2 state; B1 does not add a persisted +`CURRENT_REVISION_CHANGED` variant. Neither B1 nor B2 merges autonomously. + +## C selector-epoch cutover + +B1's only SQL addition is the separately manifested **C checkpoint** migration +`reporting_status_selector_version.sql`. See the +[C rollout instructions](reporting-status-notifications.md#selector-epoch-cutover-1167b1). +Stop and drain old C projectors and sweepers before enabling v2 turns. A/B/C +Core and notification writers remain compatible; legacy materialization writers +must separately be drained before B2 activation. + +The frozen-C process gates exercise populated checkpoints, old `claim_due`, +old schema recreation, competing v2 projectors/sweepers, pool-local marker +cleanup and retained physical rows. Shared memory/PG vectors cover interrupted +fence, checkpoint/event and final-mark commits, ordered boundaries, late clocks, +retained scopes, unchanged fingerprints and once-only restart convergence. + +The C control is the integrated rc.6 artifact above, including exact waiver +bindings and locale-independent catalog validation. These controls do not +qualify pre-`967b6e28` C binaries, including the earlier `ea150fab` snapshot. +The existing pre-`17ee407a` A and pre-`0f34c666` B rolling limits remain. diff --git a/docs/reporting-status-notifications.md b/docs/reporting-status-notifications.md index 1cb2cecc8..e20b99bca 100644 --- a/docs/reporting-status-notifications.md +++ b/docs/reporting-status-notifications.md @@ -105,12 +105,63 @@ PostgreSQL. PostgreSQL exports load lazily and construction without the extra raises an actionable `adcp[pg]` installation hint. `PgStatusNotificationStore.create_schema()` executes the six existing ledger/A/B -steps plus `reporting_status_notifications.sql` in one serialized transaction. +steps plus `reporting_status_notifications.sql` and +`reporting_status_selector_version.sql` in one serialized transaction. External migration tools must execute that same chain atomically. The original `required_schema.json`, A/B objects, functions and constraints are unchanged. `required_status_schema.json` independently validates C status and C activity objects. Missing C DDL suppresses status without changing B readiness. +## Selector epoch cutover (#1167B1) + +B1 changes whole-history revision selection and directional feed issue scopes. +Install the additive checkpoint migration, then **stop and drain old C +projectors and sweepers before scheduling v2 turns**. Core/source, A/B outbox and +C HTTP/activity writers remain usable. Installing schema alone does not fence +old C; account cutover is explicit, persisted and separate from its completion. + +Drive `ReportingStatusProjector.rebuild_once()` (or +`ReportingStatusService.rebuild_selector_once()`) until idle. This uses indexed +discovery of every populated stale account compatible with the store's original +escalation policy; no adopter account list or full periodic scan is needed. +The existing service's `drain()` includes these turns. Use the same escalation +policy as the original baseline; an explicitly targeted mismatched policy fails +closed. Ordinary `project_one(account_id=...)` also resumes that account's +interrupted cutover. + +Each first turn takes the existing account advisory lock, locks its checkpoints +and commits a checkpoint-local v2 writer floor plus an account transitioning +policy. A separate `selector_semantics_version` remains stale until projection +has committed. The new guard examines only the checkpoint and a +transaction-local v2 marker, avoiding an account-row lookup from old row-only +due claims. Old pending lease identities are retained, but old claims, +completions, projectors and readiness fail closed after the fence. Migration +does not wait for those leases to expire. Transaction-local markers are cleared +when connections return to the pool. Old named guards and their manifests are +unchanged, so old `create_schema()` cannot remove the independent v2 guard. + +Subsequent short transactions drain captured source boundaries in `through` +order, replay overdue semantic deadlines chronologically, then project current +state and mark the account complete atomically. Checkpoint/event failure rolls +back the entire turn. The scope union includes all retained checkpoints, +including those no longer returned by current scope discovery; absent source +history has a stable `HISTORY_UNAVAILABLE` result and no obsolete due deadline. + +No baseline, scope key, event, queue or activity history is deleted/reset. The +six-column scope identity, generation, baseline highwater, dirty cursor and +leases retain their meaning. Selector epoch is non-key metadata; canonical +fingerprints retain `version: 1`. Unchanged canonical health/issues only advance +the selector epoch and emit no event. Changed topology or issue membership +emits the corrected status with its existing previous health and next monotone +generation. Newly baselined accounts start at v2 without migration events. + +Readiness requires current schema, baseline, target epoch and zero stale or +incomplete checkpoint migrations. Account transitions are isolated. The memory +reference imports old shared-state images as v1 and performs the same restartable +transition. These objects and `required_status_selector_schema.json` belong only +to C checkpoint semantics; B1 adds no materializer persistence. See +[B1 destination I/O](reporting-destination-writer.md) for the B2 rollout dependency. + Default-off Core lifecycle writes also work before the A outbox migration. That schema has no issue-scope table: reads derive scope from retained status evidence until migration makes scope persistence available. Notification-enabled writers diff --git a/examples/reporting_destination_writer.py b/examples/reporting_destination_writer.py new file mode 100644 index 000000000..647209384 --- /dev/null +++ b/examples/reporting_destination_writer.py @@ -0,0 +1,77 @@ +"""B1 development destination: verify frozen content without advertising Managed. + +The trusted application supplies a frozen binding and its existing attempt. +Keep this object across retries to retain the reference writer's in-process +idempotency history. It is intentionally not durable or production eligible. +There is no scheduler, retry allocation, persistence, or readiness publication. +""" + +from __future__ import annotations + +import asyncio +from collections.abc import Sequence +from dataclasses import dataclass +from datetime import datetime + +from adcp.reporting.ledger import ReportingObligationRecord, ReportingRevisionRecord +from adcp.reporting.materializer import ( + ReferenceReportingDestinationWriter, + ReferenceReportingResolver, + ReportingDestinationBinding, + ReportingDestinationIO, + ReportingIOContext, + ReportingMaterializationAttempt, + ReportingObligationDeliveryRecord, + ReportingRevisionRowReader, + ReportingRevisionVerifier, + ReportingRevisionVerifierRegistry, + ReportingVerifiedDestination, + reference_verifier, +) + + +@dataclass(frozen=True) +class DevelopmentDestination: + verifier: ReportingRevisionVerifier + registry: ReportingRevisionVerifierRegistry + resolver: ReferenceReportingResolver + writer: ReferenceReportingDestinationWriter + + async def verify_revision( + self, + *, + reader: ReportingRevisionRowReader, + binding: ReportingDestinationBinding, + delivery: ReportingObligationDeliveryRecord, + obligation: ReportingObligationRecord, + revisions: Sequence[ReportingRevisionRecord], + attempt: ReportingMaterializationAttempt, + deadline_at: datetime, + cancel: asyncio.Event, + ) -> ReportingVerifiedDestination: + # Complete source verification happens before the resolver can open a + # destination session. The same frozen attempt keeps its external ID. + prepared = await self.registry.prepare( + key=self.verifier.key, + reader=reader, + binding=binding, + delivery=delivery, + obligation=obligation, + revisions=revisions, + attempt=attempt, + context=ReportingIOContext(deadline_at, cancel), + ) + io = ReportingDestinationIO(self.registry, self.resolver) + locator = await io.write(prepared, context=ReportingIOContext(deadline_at, cancel)) + # A fresh session reauthorizes readback, including any intervening + # revocation/credential rotation. Credentials never enter these records. + return await io.verify(prepared, locator, context=ReportingIOContext(deadline_at, cancel)) + + +def development_destination(binding: ReportingDestinationBinding) -> DevelopmentDestination: + """Install the exact bundled JSONL example definition/canonicalization.""" + verifier = reference_verifier() + registry = ReportingRevisionVerifierRegistry((verifier,)) + writer = ReferenceReportingDestinationWriter((verifier.key.capability,)) + resolver = ReferenceReportingResolver(writer, registry, (binding,)) + return DevelopmentDestination(verifier, registry, resolver, writer) diff --git a/pyproject.toml b/pyproject.toml index 683993f00..6a2204e1e 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -195,6 +195,7 @@ adcp = [ "decisioning/pg/*.sql", "reporting/ledger/*.sql", "reporting/outbox/*.json", + "reporting/materializer/assets/*.json", # PREVIEW: vendored sync_reporting_status schemas. They are the runtime # validator for the wire conditionals codegen cannot express, so the wheel # must carry them. Removed with the rest of _preview/ at rc.2. diff --git a/src/adcp/reporting/__init__.py b/src/adcp/reporting/__init__.py index 89d4060bc..0e642143e 100644 --- a/src/adcp/reporting/__init__.py +++ b/src/adcp/reporting/__init__.py @@ -75,6 +75,8 @@ from adcp.reporting import fixtures as fixtures from adcp.reporting import inline_source as inline_source from adcp.reporting import ledger as ledger + from adcp.reporting import materializer as materializer + from adcp.reporting import revision_selection as revision_selection from adcp.reporting import service as service from adcp.reporting import source as source from adcp.reporting import testing as testing @@ -108,6 +110,8 @@ "fixtures", "inline_source", "ledger", + "materializer", + "revision_selection", "service", "source", "testing", diff --git a/src/adcp/reporting/_consumer.py b/src/adcp/reporting/_consumer.py index d21f171fc..7705da28a 100644 --- a/src/adcp/reporting/_consumer.py +++ b/src/adcp/reporting/_consumer.py @@ -40,6 +40,7 @@ from datetime import datetime, timedelta, timezone from typing import Any, Literal, Protocol +from adcp.reporting.revision_selection import RevisionHistoryEntry, select_reporting_revision from adcp.types import ( GetReportingStatusRequest, ReportingObligation, @@ -481,6 +482,14 @@ def plan_consumer_statuses( publishing nothing it demonstrably published, and ``received`` asserts bytes the buyer never looked at. + ``obligation_revisions`` must carry each due obligation's *complete* + retained history -- the same partition its ``revision_count`` declares. + Finality is chosen by whole-history selection + (:func:`~adcp.reporting.revision_selection.select_reporting_revision`), not + by a count, so a partition that is absent, short, long or structurally + damaged raises :class:`ConsumerStatusPlanError` instead of selecting from + it. An obligation with ``revision_count == 0`` needs no entry. + ``current_statuses`` is this caller's own status history from the same ledger read. An intent whose content matches the current leaf is skipped entirely -- re-filing an unchanged claim under a fresh id churns the chain @@ -544,6 +553,9 @@ def add(intent: ConsumerStatusIntent | None) -> None: continue reading = readings.get(obligation_id) + history = obligation_revisions.get(obligation_id) + if history is not None: + _has_required_revision(obligation, history) # Reject damage even when a reading exists. status: ConsumerStatusValue mismatch: MismatchCode | None = None failure: ReportingFailureCode | None = None @@ -795,22 +807,43 @@ def _expected_at_of(period: Any) -> datetime | None: def _has_required_revision( obligation: ReportingObligation, revisions: Sequence[ReportingRevision] | None ) -> bool: - """Whether the seller has published a revision meeting the required finality. - - Prefers the supplied revisions, because ``required_finality`` matters: an - obligation needing ``official`` is not satisfied by snapshots. Falls back to - ``revision_count``, which the spec defines as the number of distinct - revision records for this obligation in the snapshot, so a caller that did - not pass revisions still gets the coarse answer rather than a wrong one. - """ + """Validate the caller's complete obligation partition before choosing finality.""" if revisions is not None: - required = str(getattr(obligation.required_finality, "value", obligation.required_finality)) - return any( - required == "snapshot" - or str(getattr(item.finality, "value", item.finality)) == "official" - for item in revisions + result = select_reporting_revision( + tuple( + RevisionHistoryEntry( + "wire", + obligation.reporting_obligation_id, + item.reporting_revision_id, + str(getattr(item.finality, "value", item.finality)), + item.supersedes_reporting_revision_id, + ) + for item in revisions + ), + account_id="wire", + reporting_obligation_id=obligation.reporting_obligation_id, + required_finality=str( + getattr(obligation.required_finality, "value", obligation.required_finality) + ), + ) + if result.kind == "corrupt" or len(revisions) != obligation.revision_count: + raise ConsumerStatusPlanError( + f"obligation {obligation.reporting_obligation_id!r} was given " + f"{len(revisions)} revisions against a declared revision_count of " + f"{obligation.revision_count}, or a damaged chain; obligation_revisions must " + "be that obligation's complete retained history. Re-read the seller's " + "snapshot rather than filing a status against a history neither party can " + "reconcile" + ) + return result.kind == "selected" + if obligation.revision_count: + raise ConsumerStatusPlanError( + f"obligation {obligation.reporting_obligation_id!r} advertises " + f"{obligation.revision_count} revisions but obligation_revisions carries no entry " + "for it; pass its complete retained history. required_finality cannot be decided " + "from a count, and guessing would file a status nobody validated" ) - return bool(obligation.revision_count) + return False def _source_timezone(obligation: ReportingObligation) -> str: diff --git a/src/adcp/reporting/_reconcile.py b/src/adcp/reporting/_reconcile.py index 65c539730..3868b53bb 100644 --- a/src/adcp/reporting/_reconcile.py +++ b/src/adcp/reporting/_reconcile.py @@ -40,6 +40,7 @@ post_consumer_statuses, resolve_checkpointed_leaves, ) +from adcp.reporting.revision_selection import RevisionHistoryEntry, select_reporting_revision from adcp.types import ( GetReportingStatusRequest, GetReportingStatusResponse, @@ -489,54 +490,50 @@ def _select_current( history_incomplete = True if history_incomplete: reasons.append("ASSOCIATED_HISTORY_INCOMPLETE") - superseded = { - item.supersedes_reporting_revision_id - for item in candidates - if item.supersedes_reporting_revision_id - } - candidate_ids = {item.reporting_revision_id for item in candidates} if any(not _revision_matches_obligation(item, obligation) for item in candidates) or any( item.reporting_revision_id in revision_ids and item.reporting_obligation_id != obligation.reporting_obligation_id for item in ledger.materializations ): reasons.append("REVISION_SCOPE_MISMATCH") - by_id = {item.reporting_revision_id: item for item in candidates} - leaves = [item for item in candidates if item.reporting_revision_id not in superseded] - # Walk every leaf back through its predecessors. A supersession cycle leaves - # its members unreachable, so a broken history cannot hide behind an official - # close the way a leaf-only count would let it. - reachable: set[str] = set() - for leaf in leaves: - node: ReportingRevision | None = leaf - while node is not None and node.reporting_revision_id not in reachable: - reachable.add(node.reporting_revision_id) - predecessor = node.supersedes_reporting_revision_id - node = by_id.get(predecessor) if predecessor else None - if ( - any( - item.supersedes_reporting_revision_id - and item.supersedes_reporting_revision_id not in candidate_ids + selection = select_reporting_revision( + tuple( + RevisionHistoryEntry( + ledger.account_id, + obligation.reporting_obligation_id, + item.reporting_revision_id, + _enum(item.finality), + item.supersedes_reporting_revision_id, + ) for item in candidates + ), + account_id=ledger.account_id, + reporting_obligation_id=obligation.reporting_obligation_id, + required_finality=_enum(obligation.required_finality), + ) + if selection.kind == "corrupt": + reasons.append( + "AMBIGUOUS_REVISION_CHAIN" + if selection.reason + in { + "multiple_officials", + "forked_snapshot_history", + "disconnected_snapshot_history", + "duplicate_revision_id", + } + else "INCOMPLETE_REVISION_CHAIN" ) - or reachable != candidate_ids - ): - reasons.append("INCOMPLETE_REVISION_CHAIN") - # Publication selection precedes destination selection. An official close - # coexists with retained snapshots; it does not supersede their histories. - # A newer unmaterialized publication must never reveal an older snapshot as - # the current deliverable merely because that snapshot has a ready resource. - official = [item for item in candidates if _enum(item.finality) == "official"] - # Snapshot topology is judged on its own. Selecting the official close must - # never excuse a forked snapshot history the buyer cannot reconcile. - snapshot_leaves = [item for item in leaves if _enum(item.finality) != "official"] - if official and len(snapshot_leaves) > 1: - reasons.append("AMBIGUOUS_REVISION_CHAIN") - current = official or snapshot_leaves - if len(current) != 1: - reasons.append("MISSING_CURRENT_REVISION" if not current else "AMBIGUOUS_REVISION_CHAIN") return None, None, reasons - revision = current[0] + if selection.kind == "not_ready": + reasons.append( + "FINALITY_NOT_MET" + if selection.reason == "official_required" + else "MISSING_CURRENT_REVISION" + ) + return None, None, reasons + revision = next( + r for r in candidates if r.reporting_revision_id == selection.revision.reporting_revision_id + ) if ( not _coverage_is_full(obligation.coverage, obligation.media_buy_ids) or obligation.coverage.evaluated_at != obligation.scope_resolved_at diff --git a/src/adcp/reporting/ledger/__init__.py b/src/adcp/reporting/ledger/__init__.py index 78338fa43..bb49f260a 100644 --- a/src/adcp/reporting/ledger/__init__.py +++ b/src/adcp/reporting/ledger/__init__.py @@ -186,6 +186,14 @@ issue_is_retirable, reject_reserved_authoritative_party, ) +from adcp.reporting.revision_selection import ( + REPORTING_SELECTOR_VERSION, + ReportingRevisionCorrupt, + ReportingRevisionNotReady, + ReportingRevisionSelected, + ReportingRevisionSelection, + select_reporting_revision, +) if TYPE_CHECKING: from adcp.reporting.ledger.delivery_pg import PgReportingReconciliationStore @@ -196,6 +204,12 @@ ) __all__ = [ + "REPORTING_SELECTOR_VERSION", + "ReportingRevisionCorrupt", + "ReportingRevisionNotReady", + "ReportingRevisionSelected", + "ReportingRevisionSelection", + "select_reporting_revision", "ReportingStatusCallerResolver", "ReportingStatusNotificationHandler", "ReportingStatusSnapshot", diff --git a/src/adcp/reporting/ledger/consumer_status.py b/src/adcp/reporting/ledger/consumer_status.py index b82b44fb8..356b6147d 100644 --- a/src/adcp/reporting/ledger/consumer_status.py +++ b/src/adcp/reporting/ledger/consumer_status.py @@ -68,7 +68,7 @@ from typing import Any, Literal, cast from adcp.reporting.canonical_json import canonical_json_utf8_v1 -from adcp.reporting.ledger.health import current_required_revision, issue_id_for +from adcp.reporting.ledger.health import issue_id_for from adcp.reporting.ledger.models import ( ConsumerStatusRecord, ConsumerStatusValue, @@ -82,6 +82,7 @@ ReportingRevisionRecord, ) from adcp.reporting.ledger.store import LedgerConflictError, ReportingLedgerStore +from adcp.reporting.revision_selection import select_reporting_revision __all__ = [ "CONSUMER_STATUS_ENABLED", @@ -244,6 +245,14 @@ async def _open_issue_on_first_observation(self, stored: ConsumerStatusRecord) - account_id=stored.account_id, reporting_obligation_id=stored.reporting_obligation_id, ) + if obligation is None: + obligation = await self.store.find_obligation( + account_id=stored.account_id, + delivery_config_id=stored.delivery_config_id, + delivery_config_version=stored.delivery_config_version, + period_start=stored.period_start, + period_end=stored.period_end, + ) revisions = ( await self.store.list_revisions( account_id=stored.account_id, @@ -252,7 +261,18 @@ async def _open_issue_on_first_observation(self, stored: ConsumerStatusRecord) - if obligation is not None else () ) - current = current_required_revision(obligation, revisions) if obligation else None + current = None + if obligation is not None: + selection = select_reporting_revision( + revisions, + account_id=obligation.account_id, + reporting_obligation_id=obligation.reporting_obligation_id, + required_finality=obligation.required_finality, + ) + if selection.kind == "corrupt": + return # Corruption cannot establish or resolve consumer disagreement. + if selection.kind == "selected": + current = selection.revision if not consumer_statement_conflicts( current=stored, current_revision=current, revisions=revisions ): @@ -363,6 +383,32 @@ async def _resolve_named_records(self, record: ConsumerStatusRecord) -> None: "this statement", ) + if obligation is None: + obligation = await self.store.find_obligation( + account_id=record.account_id, + delivery_config_id=record.delivery_config_id, + delivery_config_version=record.delivery_config_version, + period_start=record.period_start, + period_end=record.period_end, + ) + required = None + if obligation is not None: + revisions = await self.store.list_revisions( + account_id=record.account_id, + reporting_obligation_id=obligation.reporting_obligation_id, + ) + selection = select_reporting_revision( + revisions, + account_id=obligation.account_id, + reporting_obligation_id=obligation.reporting_obligation_id, + required_finality=obligation.required_finality, + ) + if selection.kind == "corrupt": + raise LedgerConflictError( + "HISTORY_UNAVAILABLE", "the revision history requires repair" + ) + if selection.kind == "selected": + required = selection.revision if record.reporting_revision_id is None: return @@ -388,11 +434,6 @@ async def _resolve_named_records(self, record: ConsumerStatusRecord) -> None: # recording it would degrade the caller's own view over bytes neither # party stands behind any more. The buyer's move there is to re-read and # either accept or dispute the current revision. - revisions = await self.store.list_revisions( - account_id=record.account_id, - reporting_obligation_id=obligation.reporting_obligation_id, - ) - required = current_required_revision(obligation, revisions) if required is None or required.reporting_revision_id != record.reporting_revision_id: raise LedgerConflictError( "REVISION_NOT_CURRENTLY_REQUIRED", diff --git a/src/adcp/reporting/ledger/delivery_models.py b/src/adcp/reporting/ledger/delivery_models.py index b75bbfcdb..a7d693982 100644 --- a/src/adcp/reporting/ledger/delivery_models.py +++ b/src/adcp/reporting/ledger/delivery_models.py @@ -71,7 +71,7 @@ def _freeze(value: Any, annotation: Any) -> Any: except ValueError: continue elif origin is Literal: - if value in args and type(value) is type(args[0]): + if type(value) is type(args[0]) and value in args: return value elif origin is tuple and isinstance(value, (tuple, list)): if len(args) == 2 and args[1] is Ellipsis: diff --git a/src/adcp/reporting/ledger/health.py b/src/adcp/reporting/ledger/health.py index ed109c967..429e4e434 100644 --- a/src/adcp/reporting/ledger/health.py +++ b/src/adcp/reporting/ledger/health.py @@ -39,6 +39,7 @@ ReportingProductionStatus, ReportingRevisionRecord, ) +from adcp.reporting.revision_selection import select_reporting_revision __all__ = [ "ObligationProjection", @@ -107,12 +108,44 @@ def project_obligation_health( ) -> ObligationProjection: """Classify one obligation's immutable evidence at the snapshot's clock.""" boundary = _utc(ledger_as_of) - qualifying = [ - revision - for revision in revisions - if obligation.required_finality == "snapshot" or revision.finality == "official" - ] - current = _current_revision(qualifying) + selection = select_reporting_revision( + revisions, + account_id=obligation.account_id, + reporting_obligation_id=obligation.reporting_obligation_id, + required_finality=obligation.required_finality, + ) + current = selection.revision if selection.kind == "selected" else None + + if selection.kind == "corrupt": + return ObligationProjection( + health="action_required", + production_status="published" if revisions else "pending", + issues=( + ReportingIssue( + issue_id=issue_id_for( + "core-revision-history-corrupt-v2", + obligation.account_id, + obligation.reporting_obligation_id, + ), + code="HISTORY_UNAVAILABLE", + severity="action_required", + responsible_party="seller", + recommended_action="contact_seller", + reporting_obligation_id=obligation.reporting_obligation_id, + delivery_config_id=obligation.delivery_config_id, + delivery_config_version=obligation.delivery_config_version, + feed_purpose=obligation.feed_purpose, + media_buy_ids=obligation.media_buy_ids, + period_start=obligation.period.start, + period_end=obligation.period.end, + message=( + "The retained revision history is inconsistent; seller repair is required." + ), + ), + ), + satisfied=False, + current_revision=None, + ) if obligation.currency is None: return ObligationProjection( @@ -201,48 +234,18 @@ def current_required_revision( obligation: ReportingObligationRecord, revisions: Sequence[ReportingRevisionRecord], ) -> ReportingRevisionRecord | None: - """The revision the seller currently requires for this obligation. + """Source-compatible wrapper. Corrupt and not-ready histories both return None. - Shared by the health projection and the ``sync_reporting_status`` ingest on - purpose. If the two computed "current" differently, a buyer could file a - ``content_mismatch`` the ingest accepts and the projection then treats as - naming a superseded revision -- a statement permanently stuck disputing - bytes nobody stands behind. - - Applies the obligation's ``required_finality`` first, then takes the - unsuperseded leaf: an official revision is terminal so it wins outright, - and among snapshots the current one is whichever no other supersedes. + New callers should consume ``select_reporting_revision``'s discriminated + result so corruption can be parked for repair instead of retried as absence. """ - qualifying = [ - revision - for revision in revisions - if obligation.required_finality == "snapshot" or revision.finality == "official" - ] - return _current_revision(qualifying) - - -def _current_revision( - revisions: Sequence[ReportingRevisionRecord], -) -> ReportingRevisionRecord | None: - """The unsuperseded leaf of a revision chain. - - An official revision is terminal, so it wins outright. Among snapshots, - the current one is whichever no other snapshot supersedes. - """ - if not revisions: - return None - official = [item for item in revisions if item.finality == "official"] - if official: - return max(official, key=lambda item: (_utc(item.created_at), item.reporting_revision_id)) - superseded = { - item.supersedes_reporting_revision_id - for item in revisions - if item.supersedes_reporting_revision_id - } - leaves = [item for item in revisions if item.reporting_revision_id not in superseded] - if not leaves: - return None - return max(leaves, key=lambda item: (_utc(item.created_at), item.reporting_revision_id)) + result = select_reporting_revision( + revisions, + account_id=obligation.account_id, + reporting_obligation_id=obligation.reporting_obligation_id, + required_finality=obligation.required_finality, + ) + return result.revision if result.kind == "selected" else None def _overdue_issue( diff --git a/src/adcp/reporting/ledger/pg.py b/src/adcp/reporting/ledger/pg.py index aec7a6d59..df2d275ab 100644 --- a/src/adcp/reporting/ledger/pg.py +++ b/src/adcp/reporting/ledger/pg.py @@ -125,7 +125,6 @@ RestatementCheckpoint, check_issue_state_transition, configuration_lifecycle, - decode_cursor, encode_cursor, issue_is_retirable, managed_revision_metadata, @@ -985,7 +984,9 @@ async def read_revision_rows( cursor: str | None = None, limit: int = 500, ) -> ReportingRowPage: - offset = int(decode_cursor(cursor).get("offset", 0)) if cursor else 0 + from adcp.reporting.ledger.store import revision_row_offset + + offset = revision_row_offset(cursor, reporting_revision_id, limit) async with self._connection() as connection: owned = await ( await connection.execute( @@ -1007,6 +1008,7 @@ async def read_revision_rows( total = int(owned[0]) has_more = offset + limit < total return ReportingRowPage( + reporting_revision_id=reporting_revision_id, rows=tuple(row[0] for row in rows), total_count=total, has_more=has_more, diff --git a/src/adcp/reporting/ledger/producer.py b/src/adcp/reporting/ledger/producer.py index ad6a2ef8b..2650df1e0 100644 --- a/src/adcp/reporting/ledger/producer.py +++ b/src/adcp/reporting/ledger/producer.py @@ -60,6 +60,7 @@ RestatementCheckpoint, RestatementCheckpointStore, ) +from adcp.reporting.revision_selection import select_reporting_revision from adcp.reporting.source import ( MediaBuyConstituentV1, ProvisionalSnapshotOfferingV1, @@ -317,6 +318,20 @@ def advertised_reporting_delivery( payload["consumer_status_task"] = "sync_reporting_status" payload.update(self._escalation.to_wire()) if extra: + reserved = { + *payload, + "consumer_status_task", + "managed_delivery", + "reconciled_billing", + "reporting.delivery_ready", + "ledger_notification", + "readiness_notification", + "status_notification", + "supports_webhook_activity", + "receipt_task", + } + if any(key in reserved or key.endswith(("_task", "_notification")) for key in extra): + raise ValueError("extra cannot override SDK-owned reporting capabilities") payload.update(extra) return payload @@ -679,11 +694,20 @@ async def acquire_obligation( account_id=obligation.account_id, reporting_obligation_id=obligation.reporting_obligation_id, ) - if any(item.finality == "official" for item in revisions): + selection = select_reporting_revision( + revisions, + account_id=obligation.account_id, + reporting_obligation_id=obligation.reporting_obligation_id, + required_finality=obligation.required_finality, + ) + if selection.kind == "corrupt": + raise LedgerConflictError("HISTORY_UNAVAILABLE", "the revision history requires repair") + current = selection.revision if selection.kind == "selected" else None + if current is not None and current.finality == "official": # An official close is terminal. A later source correction is an # adjustment, never another acquisition. return None - satisfied = any(item.readable for item in revisions) + satisfied = current is not None and current.readable if satisfied and not restate: return None # Everything below needs the frozen code: the slice request carries it, @@ -881,9 +905,30 @@ async def commit_revision_from_manifest( account_id=obligation.account_id, reporting_obligation_id=obligation.reporting_obligation_id, ) - supersedes = None - if finality == "snapshot": - supersedes = self._current_snapshot_leaf(existing) + selection = select_reporting_revision( + existing, + account_id=obligation.account_id, + reporting_obligation_id=obligation.reporting_obligation_id, + required_finality="snapshot", + ) + # A retained official close coexists with the snapshot chain and wins + # whole-history selection outright, so it is never the snapshot leaf. + # Reading it as one would root a restatement at ``None`` and split the + # obligation into two snapshot roots -- a permanently corrupt history + # over immutable rows, with no repair path. + leaf = select_reporting_revision( + tuple(item for item in existing if item.finality == "snapshot"), + account_id=obligation.account_id, + reporting_obligation_id=obligation.reporting_obligation_id, + required_finality="snapshot", + ) + if selection.kind == "corrupt" or leaf.kind == "corrupt": + raise LedgerConflictError("HISTORY_UNAVAILABLE", "the revision history requires repair") + supersedes = ( + leaf.revision.reporting_revision_id + if finality == "snapshot" and leaf.kind == "selected" + else None + ) control_totals = tuple((total.name, total.value) for total in manifest.control_totals) revision_id = f"rpr_{manifest.publication_id[4:44]}" diff --git a/src/adcp/reporting/ledger/reporting_status_selector_version.sql b/src/adcp/reporting/ledger/reporting_status_selector_version.sql new file mode 100644 index 000000000..9e31cce8f --- /dev/null +++ b/src/adcp/reporting/ledger/reporting_status_selector_version.sql @@ -0,0 +1,66 @@ +-- #1167B1: C checkpoint semantics only. No materializer persistence. +-- Drain old C projectors/sweepers before cutover. A/B/C source and outbox +-- writers do not update checkpoints and remain compatible with this guard. +DO $selector$ +BEGIN + PERFORM pg_advisory_xact_lock(hashtext('adcp.reporting.schema'), hashtext(current_schema())); + ALTER TABLE reporting_status_scope_checkpoints + ADD COLUMN IF NOT EXISTS selector_semantics_version INTEGER NOT NULL DEFAULT 1; + ALTER TABLE reporting_status_scope_checkpoints + ADD COLUMN IF NOT EXISTS selector_writer_floor INTEGER NOT NULL DEFAULT 1; + ALTER TABLE reporting_status_accounts + ADD COLUMN IF NOT EXISTS selector_target_version INTEGER NOT NULL DEFAULT 1; + ALTER TABLE reporting_status_accounts + ADD COLUMN IF NOT EXISTS selector_transition TEXT NOT NULL DEFAULT 'pending'; + IF NOT EXISTS (SELECT 1 FROM pg_constraint + WHERE conrelid = 'reporting_status_scope_checkpoints'::regclass + AND conname = 'reporting_status_selector_versions') THEN + ALTER TABLE reporting_status_scope_checkpoints + ADD CONSTRAINT reporting_status_selector_versions CHECK ( + selector_semantics_version IN (1,2) AND selector_writer_floor IN (1,2) + AND selector_semantics_version <= selector_writer_floor); + END IF; + IF NOT EXISTS (SELECT 1 FROM pg_constraint + WHERE conrelid = 'reporting_status_accounts'::regclass + AND conname = 'reporting_status_selector_account_transition') THEN + ALTER TABLE reporting_status_accounts + ADD CONSTRAINT reporting_status_selector_account_transition CHECK ( + (selector_target_version = 1 AND selector_transition = 'pending') OR + (selector_target_version = 2 AND selector_transition IN ('transitioning','complete'))); + END IF; + CREATE INDEX IF NOT EXISTS reporting_status_selector_rebuild + ON reporting_status_scope_checkpoints (account_id) + WHERE selector_semantics_version <> 2 OR selector_writer_floor <> 2; + CREATE INDEX IF NOT EXISTS reporting_status_selector_accounts + ON reporting_status_accounts (account_id) + WHERE baseline_complete AND (selector_target_version <> 2 OR selector_transition <> 'complete'); +END +$selector$; + +CREATE OR REPLACE FUNCTION reporting_status_selector_writer_guard_v2() RETURNS trigger +LANGUAGE plpgsql AS $guard$ +BEGIN + -- Check only this checkpoint. An account lookup here would invert the + -- account -> checkpoint lock order against old row-lock-only due claims. + IF (NEW.selector_writer_floor >= 2 OR + (TG_OP = 'UPDATE' AND OLD.selector_writer_floor >= 2)) AND + (current_setting('adcp.reporting.selector_semantics_version', true) IS DISTINCT FROM '2' + OR NEW.selector_writer_floor < 2 + OR (TG_OP = 'UPDATE' AND NEW.selector_semantics_version < OLD.selector_semantics_version)) THEN + RAISE EXCEPTION 'reporting_status_selector_writer_drain_required'; + END IF; + RETURN NEW; +END +$guard$; + +DO $selector_guard$ +BEGIN + IF NOT EXISTS (SELECT 1 FROM pg_trigger + WHERE tgrelid = 'reporting_status_scope_checkpoints'::regclass + AND tgname = 'reporting_status_selector_writer_v2') THEN + CREATE TRIGGER reporting_status_selector_writer_v2 + BEFORE INSERT OR UPDATE ON reporting_status_scope_checkpoints + FOR EACH ROW EXECUTE FUNCTION reporting_status_selector_writer_guard_v2(); + END IF; +END +$selector_guard$; diff --git a/src/adcp/reporting/ledger/status_projection.py b/src/adcp/reporting/ledger/status_projection.py index 351216545..67d7ee157 100644 --- a/src/adcp/reporting/ledger/status_projection.py +++ b/src/adcp/reporting/ledger/status_projection.py @@ -49,6 +49,7 @@ ReportingStatusScope, validate_scope_refinement, ) +from adcp.reporting.revision_selection import select_reporting_revision @dataclass(frozen=True) @@ -237,7 +238,20 @@ def lifecycle_intents(snapshot: ReportingStatusSnapshot) -> tuple[StatusLifecycl for r in snapshot.revisions if owner is not None and r.reporting_obligation_id == owner.reporting_obligation_id ) - required = current_required_revision(owner, revisions) if owner else None + required = None + if owner is not None: + selection = select_reporting_revision( + revisions, + account_id=owner.account_id, + reporting_obligation_id=owner.reporting_obligation_id, + required_finality=owner.required_finality, + ) + if selection.kind == "corrupt": + # Keep prior consumer occurrences intact until seller history + # can establish agreement. Health exposes HISTORY_UNAVAILABLE. + continue + if selection.kind == "selected": + required = selection.revision conflicts = consumer_statement_conflicts( current=status, current_revision=required, revisions=revisions ) @@ -403,6 +417,11 @@ def _selected(scope: ReportingStatusScope, target: ReportingStatusScope) -> bool return ( scope.account_id == target.account_id and (scope.consumer_id is None or scope.consumer_id == target.consumer_id) + and ( + scope.feed_purpose is None + or target.feed_purpose is None + or scope.feed_purpose == target.feed_purpose + ) and ( scope.generation_key is None or target.generation_key is None @@ -495,6 +514,7 @@ def _project(value: StatusProjectionInput) -> tuple[StatusProjectionResult, set[ c for c in snapshot.configurations if (scope.generation_key is None or c.generation_key == scope.generation_key) + and (scope.feed_purpose is None or c.feed_purpose == scope.feed_purpose) and configuration_selected( c, delivery_config_ids=value.delivery_config_ids, @@ -510,6 +530,7 @@ def _project(value: StatusProjectionInput) -> tuple[StatusProjectionResult, set[ # restored/custom stores. Never erase its health by filtering through # only the extant configuration rows; apply the same typed selection. if (scope.generation_key is None or o.generation_key == scope.generation_key) + and (scope.feed_purpose is None or o.feed_purpose == scope.feed_purpose) and (not value.delivery_config_ids or o.delivery_config_id in value.delivery_config_ids) and (not value.feed_purposes or o.feed_purpose in value.feed_purposes) and ( @@ -530,6 +551,33 @@ def _project(value: StatusProjectionInput) -> tuple[StatusProjectionResult, set[ projected: list[StatusObligationProjection] = [] candidates: set[datetime] = set() pending = 0 + if (scope.reporting_obligation_id is not None and not obligations) or ( + scope.generation_key is not None and not configurations and not obligations + ): + # A retained checkpoint can outlive the registry/obligation that made + # its scope discoverable. Keep the row and its event identity, publish + # the absence honestly, and leave no obsolete clock deadline behind. + issues.append( + ReportingIssue( + issue_id=issue_id_for( + "retained-scope-history-unavailable-v2", + canonical_json_utf8_v1(asdict(scope)).hex(), + ), + code="HISTORY_UNAVAILABLE", + severity="action_required", + responsible_party="seller", + recommended_action="contact_seller", + reporting_obligation_id=scope.reporting_obligation_id, + delivery_config_id=( + scope.generation_key.delivery_config_id if scope.generation_key else None + ), + delivery_config_version=( + scope.generation_key.delivery_config_version if scope.generation_key else None + ), + feed_purpose=scope.feed_purpose, + message="The retained reporting scope has no available source history.", + ) + ) retained_from = status_retained_from(configurations, snapshot.as_of) if value.period_start is not None and value.period_start < retained_from: for configuration in configurations: diff --git a/src/adcp/reporting/ledger/status_snapshot.py b/src/adcp/reporting/ledger/status_snapshot.py index d6e0a3ef2..fd6bcbd05 100644 --- a/src/adcp/reporting/ledger/status_snapshot.py +++ b/src/adcp/reporting/ledger/status_snapshot.py @@ -14,7 +14,9 @@ StatusLifecycleIntent, apply_intents_to_snapshot, lifecycle_intents, + status_matches_obligation, ) +from adcp.reporting.revision_selection import select_reporting_revision if TYPE_CHECKING: from adcp.reporting.ledger.pg import PgReportingLedgerStore @@ -41,7 +43,6 @@ def validate_status_evidence( consume only already captured evidence and never acquire a connection. """ from adcp.reporting.ledger.consumer_status import validate_consumer_status_timing - from adcp.reporting.ledger.health import current_required_revision from adcp.reporting.ledger.store import LedgerConflictError generation = next( @@ -58,11 +59,7 @@ def validate_status_evidence( "SELLER_SNAPSHOT_EVIDENCE_INCOMPLETE", "seller snapshot evidence is incomplete" ) obligation = next( - ( - o - for o in snapshot.obligations - if o.reporting_obligation_id == status.reporting_obligation_id - ), + (o for o in snapshot.obligations if status_matches_obligation(status, o)), None, ) if status.reporting_obligation_id is not None: @@ -80,6 +77,23 @@ def validate_status_evidence( ): raise LedgerConflictError("OBLIGATION_IDENTITY_MISMATCH", "status evidence differs") validate_consumer_status_timing(status, generation, as_of=snapshot.as_of) + required = None + if obligation is not None: + revisions = tuple( + r + for r in snapshot.revisions + if r.reporting_obligation_id == obligation.reporting_obligation_id + ) + selection = select_reporting_revision( + revisions, + account_id=obligation.account_id, + reporting_obligation_id=obligation.reporting_obligation_id, + required_finality=obligation.required_finality, + ) + if selection.kind == "corrupt": + raise LedgerConflictError("HISTORY_UNAVAILABLE", "the revision history requires repair") + if selection.kind == "selected": + required = selection.revision if status.reporting_revision_id is None: return revision = next( @@ -92,12 +106,6 @@ def validate_status_evidence( ): raise LedgerConflictError("LOOKUP_UNAVAILABLE", "status evidence is unavailable") if status.consumer_status == "content_mismatch" and obligation is not None: - revisions = tuple( - r - for r in snapshot.revisions - if r.reporting_obligation_id == obligation.reporting_obligation_id - ) - required = current_required_revision(obligation, revisions) if required is None or required.reporting_revision_id != status.reporting_revision_id: raise LedgerConflictError( "REVISION_NOT_CURRENTLY_REQUIRED", "status revision is no longer required" diff --git a/src/adcp/reporting/ledger/store.py b/src/adcp/reporting/ledger/store.py index abcb77cb7..54bb592c5 100644 --- a/src/adcp/reporting/ledger/store.py +++ b/src/adcp/reporting/ledger/store.py @@ -206,6 +206,7 @@ class ReportingRowPage: total_count: int has_more: bool cursor: str | None + reporting_revision_id: str | None = None def encode_cursor(payload: dict[str, Any]) -> str: @@ -222,17 +223,46 @@ def encode_cursor(payload: dict[str, Any]) -> str: def decode_cursor(cursor: str) -> dict[str, Any]: padded = cursor + "=" * (-len(cursor) % 4) + + def unique_pairs(pairs: list[tuple[str, Any]]) -> dict[str, Any]: + result: dict[str, Any] = {} + for key, value in pairs: + if key in result: + raise ValueError("duplicate cursor member") + result[key] = value + return result + try: - payload = json.loads(base64.urlsafe_b64decode(padded.encode("ascii"))) - except Exception as error: - raise LedgerConflictError( - "INVALID_CURSOR", "the pagination cursor is not readable" - ) from error + payload = json.loads( + base64.urlsafe_b64decode(padded.encode("ascii")), object_pairs_hook=unique_pairs + ) + except Exception: + payload = None if not isinstance(payload, dict): raise LedgerConflictError("INVALID_CURSOR", "the pagination cursor is not readable") return payload +def revision_row_offset(cursor: str | None, revision_id: str, limit: int) -> int: + """Reject another revision's cursor and invalid count types before a row read.""" + if type(limit) is not int or not 1 <= limit <= 500: + raise LedgerConflictError("INVALID_PAGE_SIZE", "row page size must be between 1 and 500") + if cursor is None: + return 0 + if type(cursor) is not str or len(cursor) > 2048: + raise LedgerConflictError("INVALID_CURSOR", "the cursor does not bind this revision") + payload = decode_cursor(cursor) + offset = payload.get("offset") + if ( + set(payload) != {"revision", "offset"} + or payload["revision"] != revision_id + or type(offset) is not int + or offset < 0 + ): + raise LedgerConflictError("INVALID_CURSOR", "the cursor does not bind this revision") + return offset + + @runtime_checkable class ReportingLedgerStore(Protocol): """Durable home for obligations, revisions, adjustments, and statuses.""" @@ -1085,10 +1115,11 @@ async def read_revision_rows( if revision is None: raise LedgerConflictError("REVISION_NOT_FOUND", "no such revision for this account") rows = self._rows.get(reporting_revision_id, ()) - offset = int(decode_cursor(cursor).get("offset", 0)) if cursor else 0 + offset = revision_row_offset(cursor, reporting_revision_id, limit) window = rows[offset : offset + limit] has_more = offset + limit < len(rows) return ReportingRowPage( + reporting_revision_id=reporting_revision_id, rows=tuple(deepcopy(row) for row in window), total_count=len(rows), has_more=has_more, diff --git a/src/adcp/reporting/materializer/__init__.py b/src/adcp/reporting/materializer/__init__.py new file mode 100644 index 000000000..165d6d3d8 --- /dev/null +++ b/src/adcp/reporting/materializer/__init__.py @@ -0,0 +1,93 @@ +"""B1 public destination contracts and verification; no durable Managed service. + +Use the immutable registry to prepare all frozen source rows, then invoke write +and verify explicitly with separate authorization sessions. The reference +writer is exclusively for tests/development. B2 owns durable work, fencing, +retry allocation, final target reselection, and readiness transactions. +""" + +from adcp.reporting.ledger.delivery_models import ( + ReportingDeliveryPrincipal, + ReportingDestinationBinding, + ReportingMaterializationAttempt, + ReportingObligationDeliveryRecord, +) +from adcp.reporting.materializer._json import ( + ReportingVerificationLimits, + parse_reporting_json, + strict_reporting_json, +) +from adcp.reporting.materializer.contracts import ( + ReportingCanonicalization, + ReportingDestinationLocator, + ReportingDestinationPage, + ReportingDestinationRequest, + ReportingDestinationResolver, + ReportingDestinationSession, + ReportingDestinationWriter, + ReportingExternalEffect, + ReportingHeartbeat, + ReportingIOContext, + ReportingIOPhase, + ReportingNativeObservation, + ReportingPreparedRevision, + ReportingVerificationKey, + ReportingWriterCapability, + ReportingWriterError, + ReportingWriterFailure, + ReportingWriterFailureCode, + ReportingWriterRetry, +) +from adcp.reporting.materializer.reference import ( + ReferenceReportingDestinationWriter, + ReferenceReportingResolver, + reference_digest, + reference_verifier, +) +from adcp.reporting.materializer.verification import ( + ReportingDestinationIO, + ReportingRevisionRowReader, + ReportingRevisionVerifier, + ReportingRevisionVerifierRegistry, + ReportingVerifiedDestination, + validate_materialization_target, +) + +__all__ = [ + "ReferenceReportingDestinationWriter", + "ReferenceReportingResolver", + "ReportingCanonicalization", + "ReportingDeliveryPrincipal", + "ReportingDestinationBinding", + "ReportingDestinationIO", + "ReportingDestinationLocator", + "ReportingDestinationPage", + "ReportingDestinationRequest", + "ReportingDestinationResolver", + "ReportingDestinationSession", + "ReportingDestinationWriter", + "ReportingExternalEffect", + "ReportingHeartbeat", + "ReportingIOContext", + "ReportingIOPhase", + "ReportingMaterializationAttempt", + "ReportingNativeObservation", + "ReportingObligationDeliveryRecord", + "ReportingPreparedRevision", + "ReportingRevisionRowReader", + "ReportingRevisionVerifier", + "ReportingRevisionVerifierRegistry", + "ReportingVerificationKey", + "ReportingVerificationLimits", + "ReportingVerifiedDestination", + "ReportingWriterCapability", + "ReportingWriterError", + "ReportingWriterFailure", + "ReportingWriterFailureCode", + "ReportingWriterRetry", + "parse_reporting_json", + "reference_digest", + "reference_verifier", + "strict_reporting_json", + "validate_materialization_target", +] diff --git a/src/adcp/reporting/materializer/_json.py b/src/adcp/reporting/materializer/_json.py new file mode 100644 index 000000000..01ab467ea --- /dev/null +++ b/src/adcp/reporting/materializer/_json.py @@ -0,0 +1,154 @@ +"""Bounded exact JSON values at the new verification boundary.""" + +from __future__ import annotations + +import json +from dataclasses import dataclass, fields +from typing import cast + +from adcp.reporting.canonical_json import MAX_SAFE_INTEGER, JsonValue, canonical_json_utf8_v1 +from adcp.reporting.materializer.contracts import failure + + +@dataclass(frozen=True, slots=True) +class ReportingVerificationLimits: + max_depth: int = 32 + max_value_bytes: int = 1024 * 1024 + max_total_bytes: int = 64 * 1024 * 1024 + max_items: int = 1_000_000 + max_rows: int = 100_000 + max_pages: int = 10_000 + max_objects: int = 10_000 + max_chunks: int = 1_000_000 + + def __post_init__(self) -> None: + if any( + type(getattr(self, f.name)) is not int or getattr(self, f.name) < 1 + for f in fields(self) + ): + raise ValueError("verification limits require positive integer bounds") + if self.max_depth > 128: + raise ValueError("verification depth cannot exceed 128") + + +DEFAULT_LIMITS = ReportingVerificationLimits() + + +def strict_reporting_json( + value: object, limits: ReportingVerificationLimits = DEFAULT_LIMITS +) -> bytes: + """Encode exact JSON types, without coercion, subclass hooks or Unicode normalization. + + Integers must be JS-safe; exact decimals are strings. Floats, tuples, + subclasses and lone surrogates are deliberately outside this SDK profile. + """ + stack = [(value, 0)] + count, encoded_bytes = 0, 0 + while stack: + item, depth = stack.pop() + count += 1 + if depth > limits.max_depth or count > limits.max_items: + raise failure("LIMIT_EXCEEDED") + kind = type(item) + if kind is str: + text = cast(str, item) + if len(text) > limits.max_value_bytes: + raise failure("LIMIT_EXCEEDED") + encoded_bytes += 2 # Opening/closing quotes, including escaped UTF-8. + for char in text: + code = ord(char) + if 0xD800 <= code <= 0xDFFF: + raise failure("SOURCE_INVALID") + if char in '\\"\b\t\n\f\r': + encoded_bytes += 2 + elif code < 0x20: + encoded_bytes += 6 + else: + encoded_bytes += ( + 1 if code < 0x80 else 2 if code < 0x800 else 3 if code < 0x10000 else 4 + ) + if encoded_bytes > limits.max_value_bytes: + raise failure("LIMIT_EXCEEDED") + elif kind is int: + if abs(cast(int, item)) > MAX_SAFE_INTEGER: + raise failure("SOURCE_INVALID") + encoded_bytes += len(str(item)) + elif item is None or kind is bool: + encoded_bytes += 5 if item is False else 4 + elif kind is list: + values = cast(list[object], item) + if len(values) + count + len(stack) > limits.max_items: + raise failure("LIMIT_EXCEEDED") + encoded_bytes += 2 + max(0, len(values) - 1) + stack.extend((v, depth + 1) for v in values) + elif kind is dict: + mapping = cast(dict[object, object], item) + if len(mapping) * 2 + count + len(stack) > limits.max_items: + raise failure("LIMIT_EXCEEDED") + encoded_bytes += 2 + max(0, 2 * len(mapping) - 1) + for key, child in mapping.items(): + if type(key) is not str: + raise failure("SOURCE_INVALID") + stack.extend(((key, depth + 1), (child, depth + 1))) + else: + raise failure("SOURCE_INVALID") + if encoded_bytes > limits.max_value_bytes: + raise failure("LIMIT_EXCEEDED") + result = canonical_json_utf8_v1(value) + if len(result) > limits.max_value_bytes: + raise failure("LIMIT_EXCEEDED") + return result + + +def parse_reporting_json( + payload: bytes, limits: ReportingVerificationLimits = DEFAULT_LIMITS +) -> JsonValue: + """Reject ambiguous JSON before it can be normalized into a Python mapping.""" + if type(payload) is not bytes: + raise failure("SOURCE_INVALID") + if len(payload) > limits.max_value_bytes: + raise failure("LIMIT_EXCEEDED") + # Bound parser recursion before json.loads allocates a nested tree. + depth, quoted, escaped = 0, False, False + for byte in payload: + if quoted: + if escaped: + escaped = False + elif byte == 92: + escaped = True + elif byte == 34: + quoted = False + elif byte == 34: + quoted = True + elif byte in (91, 123): + depth += 1 + if depth > limits.max_depth: + raise failure("LIMIT_EXCEEDED") + elif byte in (93, 125): + depth -= 1 + + def pairs(values: list[tuple[str, JsonValue]]) -> dict[str, JsonValue]: + result: dict[str, JsonValue] = {} + for key, value in values: + if key in result: + raise ValueError + result[key] = value + return result + + def reject(value: str) -> JsonValue: + raise ValueError + + invalid = False + try: + value = json.loads( + payload.decode("utf-8"), + object_pairs_hook=pairs, + parse_constant=reject, + parse_float=reject, + ) + except (ValueError, UnicodeError, RecursionError): + invalid = True + if invalid: + raise failure("SOURCE_INVALID") + strict_reporting_json(value, limits) + return cast(JsonValue, value) diff --git a/src/adcp/reporting/materializer/assets/reference-canonicalization.json b/src/adcp/reporting/materializer/assets/reference-canonicalization.json new file mode 100644 index 000000000..27d9d867c --- /dev/null +++ b/src/adcp/reporting/materializer/assets/reference-canonicalization.json @@ -0,0 +1,42 @@ +{ + "contract_version": "1.0", + "media_type": "application/vnd.adcp.reporting-canonicalization+json", + "algorithm": "adcp_jcs_rows_v1", + "schema_sha256": "aa4927d97b5e5889c4d79bd5461da81657cafea75a61d266ab452bfa0fc5d799", + "primary_keys": [ + "row_id" + ], + "golden_vectors": { + "empty_report": { + "name": "empty", + "purpose": "empty_report", + "input_rows": [], + "canonical_utf8_base64": "W10=", + "sha256": "4f53cda18c2baa0c0354bb5f9a3ecbe5ed12ab4d8e11ba873c2f11161202b945" + }, + "ordering_encoding": { + "name": "unicode-order", + "purpose": "ordering_encoding", + "input_rows": [ + { + "spend": "2.50", + "impressions": 2, + "currency": "USD", + "row_id": "z", + "details": { + "": "é", + "😀": "é" + } + }, + { + "spend": "1.00", + "impressions": 1, + "currency": "USD", + "row_id": "a" + } + ], + "canonical_utf8_base64": "W3siY3VycmVuY3kiOiJVU0QiLCJpbXByZXNzaW9ucyI6MSwicm93X2lkIjoiYSIsInNwZW5kIjoiMS4wMCJ9LHsiY3VycmVuY3kiOiJVU0QiLCJkZXRhaWxzIjp7IvCfmIAiOiLDqSIsIu6AgCI6ImXMgSJ9LCJpbXByZXNzaW9ucyI6Miwicm93X2lkIjoieiIsInNwZW5kIjoiMi41MCJ9XQ==", + "sha256": "3fb72666e0a391ef931445b5cebe16f8a6a801594acc885d4f2b1f18fb9a8eff" + } + } +} diff --git a/src/adcp/reporting/materializer/assets/reference-definition.json b/src/adcp/reporting/materializer/assets/reference-definition.json new file mode 100644 index 000000000..ce87305dd --- /dev/null +++ b/src/adcp/reporting/materializer/assets/reference-definition.json @@ -0,0 +1,48 @@ +{ + "contract_version": "1.1", + "media_type": "application/vnd.adcp.reporting-definition+json", + "report_definition_id": "reference-report-v1", + "reporting_profile": "paid_media_delivery", + "grain": "row", + "source": { + "provider": { + "domain": "example.test" + }, + "system": "non-production-reference", + "api_version": "1", + "query_semantics": { + "fixture_only": true + } + }, + "calendar": { + "timezone_basis": "utc" + }, + "metrics": [ + { + "name": "impressions", + "source_expression": "impressions", + "aggregation": "sum" + }, + { + "name": "spend", + "source_expression": "spend", + "aggregation": "sum", + "unit": "USD" + } + ], + "dimensions": [ + "row_id" + ], + "restatement_policy": { + "source_requery_duration": "P1D", + "emit_only_on_content_change": true, + "official_correction_mode": "adjustments_only" + }, + "finality_policies": [ + { + "finality_policy_id": "reference-final", + "basis": "source_final", + "source_signal": "fixture_closed" + } + ] +} diff --git a/src/adcp/reporting/materializer/assets/reference-row-schema.json b/src/adcp/reporting/materializer/assets/reference-row-schema.json new file mode 100644 index 000000000..86f40809a --- /dev/null +++ b/src/adcp/reporting/materializer/assets/reference-row-schema.json @@ -0,0 +1,38 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "type": "object", + "properties": { + "row_id": { + "type": "string", + "minLength": 1 + }, + "impressions": { + "type": "integer", + "minimum": 0, + "maximum": 9007199254740991, + "x-adcp-control-total": { + "value_type": "integer" + } + }, + "spend": { + "type": "string", + "pattern": "^(0|[1-9][0-9]{0,37})\\.[0-9]{2}$", + "x-adcp-control-total": { + "value_type": "decimal", + "scale": 2, + "unit": "USD" + } + }, + "currency": { + "const": "USD" + }, + "details": {} + }, + "required": [ + "row_id", + "impressions", + "spend", + "currency" + ], + "additionalProperties": false +} diff --git a/src/adcp/reporting/materializer/contracts.py b/src/adcp/reporting/materializer/contracts.py new file mode 100644 index 000000000..be70ca58e --- /dev/null +++ b/src/adcp/reporting/materializer/contracts.py @@ -0,0 +1,646 @@ +"""B1 destination I/O contracts. No scheduling, persistence or capability claims. + +Resolvers construct a session synchronously; authorization and resource acquisition +happen in its protected ``_open`` method, inside the SDK's owned async lifecycle. +Allocate credentials only there, retain them only on the redacted session, and +release partial acquisitions in ``_close``. Protocol callers supply opaque IDs. +""" + +from __future__ import annotations + +import asyncio +import hashlib +import math +import re +from abc import ABC, abstractmethod +from collections.abc import AsyncIterator, Awaitable, Callable +from dataclasses import asdict, dataclass, field, replace +from datetime import datetime, timezone +from types import TracebackType +from typing import Any, Literal, Protocol, TypeVar + +from adcp.reporting.canonical_json import canonical_json_utf8_v1 +from adcp.reporting.evidence import ( + ReportingCanonicalDigest, + aware_utc, + destination_reference, + file_object_reference, + native_version_reference, + reporting_identifier, + sha256_value, +) +from adcp.reporting.ledger.delivery_models import ( + DeliveryMethod, + ReportingDeliveryPrincipal, + ReportingDestinationBinding, + ReportingFormat, + ReportingMaterializationAttempt, + ReportingObligationDeliveryRecord, + ReportingResourceRecord, + VerificationPath, + VerificationProfile, + _ClosedValue, + _freeze_fields, +) +from adcp.reporting.ledger.models import ( + ReportingConfigurationGenerationKey, + ReportingDefinitionBinding, + ReportingObligationRecord, + ReportingRevisionRecord, +) + +ReportingIOPhase = Literal["write", "readback"] +ReportingExternalEffect = Literal["not_started", "applied", "unknown"] +ReportingWriterRetry = Literal["never", "same_identity", "new_attempt"] +ReportingWriterFailureCode = Literal[ + "UNSUPPORTED_VERIFICATION", + "HISTORY_CORRUPT", + "REVISION_NOT_READY", + "CURRENT_REVISION_CHANGED", + "AUTHORIZATION_DENIED", + "BINDING_MISMATCH", + "SOURCE_INVALID", + "DESTINATION_CORRUPT", + "RESOURCE_UNAVAILABLE", + "WRITE_FAILED", + "DEADLINE_EXCEEDED", + "LEASE_LOST", + "LIMIT_EXCEEDED", +] + + +@dataclass(frozen=True, slots=True) +class ReportingWriterFailure(_ClosedValue): + """Safe diagnostics only. Unknown effects must retain the original identity. + + ``new_attempt`` is usable by B2 only for its own known terminal failure. + Public ledger persistence still allows N+1 after any immutable outcome. + Consumer receipt rejection is outside this contract. + """ + + code: ReportingWriterFailureCode + retry: ReportingWriterRetry = "never" + effect: ReportingExternalEffect = "not_started" + retry_after_seconds: int | None = None + + def __post_init__(self) -> None: + _freeze_fields(self) + if self.effect == "unknown" and self.retry == "new_attempt": + raise ValueError("unknown external effects require the original identity") + if self.retry_after_seconds is not None and self.retry_after_seconds < 0: + raise ValueError("retry delay must be nonnegative") + + +class ReportingWriterError(Exception): + """A closed failure; never pass provider prose or attach a provider cause.""" + + def __init__(self, failure: ReportingWriterFailure) -> None: + if type(failure) is not ReportingWriterFailure: + raise TypeError("writer errors require a closed failure") + self.failure = failure + super().__init__(failure.code) + + def __repr__(self) -> str: + return f"ReportingWriterError({self.failure.code})" + + +def failure(code: ReportingWriterFailureCode) -> ReportingWriterError: + return ReportingWriterError(ReportingWriterFailure(code)) + + +@dataclass(frozen=True, slots=True) +class ReportingWriterCapability(_ClosedValue): + method: DeliveryMethod + transport: str + format: ReportingFormat | None + verification_profile: VerificationProfile + verification_path: VerificationPath + immutability: Literal["immutable_location", "native_version"] + checksum: Literal["sha256"] + write_semantics: Literal["conditional_create", "idempotent"] + + def __post_init__(self) -> None: + _freeze_fields(self) + reporting_identifier(self.transport, maximum=64) + if re.fullmatch(r"[a-z][a-z0-9_.-]{0,63}", self.transport) is None: + raise ValueError("transport requires a public protocol label") + if self.method == "file_transfer" and self.format is None: + raise ValueError("file transfer requires a format") + if self.verification_profile == "manifest_checksums" and self.method != "file_transfer": + raise ValueError("manifest verification requires file transfer") + if ( + (self.method == "dataset_share" and self.verification_path != "representative_consumer") + or ( + self.method == "warehouse_materialization" + and self.verification_path != "destination" + ) + or ( + self.verification_profile == "native_commit" + and self.immutability != "native_version" + ) + or (self.immutability == "native_version" and self.verification_path == "producer") + ): + raise ValueError("capability requires its exact immutable observation path") + + +@dataclass(frozen=True, slots=True) +class ReportingCanonicalization(_ClosedValue): + canonicalization_id: str + canonicalization_uri: str + canonicalization_sha256: str + + def __post_init__(self) -> None: + _freeze_fields(self) + ReportingCanonicalDigest( + "0" * 64, + self.canonicalization_id, + self.canonicalization_uri, + self.canonicalization_sha256, + ) + object.__setattr__(self, "canonicalization_sha256", self.canonicalization_sha256.lower()) + + +@dataclass(frozen=True, slots=True) +class ReportingVerificationKey(_ClosedValue): + """Complete frozen identity, including method/format/profile/path and schema.""" + + report_definition_id: str + reporting_profile: str + definition: ReportingDefinitionBinding + canonicalization: ReportingCanonicalization + capability: ReportingWriterCapability + + def __post_init__(self) -> None: + _freeze_fields(self) + reporting_identifier(self.report_definition_id) + reporting_identifier(self.reporting_profile, maximum=128) + for value in (self.definition.report_definition_sha256, self.definition.schema_sha256): + if type(value) is not str: + raise ValueError("definition digests require exact strings") + sha256_value(value) + for uri in (self.definition.report_definition_uri, self.definition.schema_uri): + # Reuse the strict public HTTPS contract screen; never dereference it. + ReportingCanonicalDigest("0" * 64, "definition", uri, "0" * 64) + for value in (self.definition.schema_version, self.definition.schema_ref_policy): + reporting_identifier(value, maximum=128) + ReportingCanonicalDigest("0" * 64, "dialect", self.definition.schema_dialect, "0" * 64) + for units in ( + self.definition.monetary_metric_units, + self.definition.monetary_control_total_units, + ): + for name, unit in units: + reporting_identifier(name, maximum=128) + reporting_identifier(unit, maximum=32) + object.__setattr__( + self, + "definition", + replace( + self.definition, + report_definition_sha256=self.definition.report_definition_sha256.lower(), + schema_sha256=self.definition.schema_sha256.lower(), + ), + ) + + +def binding_fingerprint(binding: ReportingDestinationBinding) -> str: + value = asdict(binding) + value["created_at"] = binding.created_at.isoformat() + return hashlib.sha256(canonical_json_utf8_v1(value)).hexdigest() + + +@dataclass(frozen=True, slots=True) +class ReportingDestinationRequest(_ClosedValue): + """Trusted resolver input. Aliases must already resolve to the canonical consumer.""" + + principal: ReportingDeliveryPrincipal + generation: ReportingConfigurationGenerationKey + destination_ref: str + trusted_binding_ref: str = field(repr=False) + binding_fingerprint: str + verification_key: ReportingVerificationKey + reporting_obligation_id: str + reporting_revision_id: str + reporting_materialization_id: str + attempt: int + + def __post_init__(self) -> None: + _freeze_fields(self) + if self.principal.account_id != self.generation.account_id or self.attempt < 1: + raise ValueError("destination request requires an exact account and attempt") + reporting_identifier(self.generation.delivery_config_id) + if ( + type(self.generation.delivery_config_version) is not int + or self.generation.delivery_config_version < 1 + ): + raise ValueError("destination request requires an exact generation") + destination_reference(self.destination_ref) + destination_reference(self.trusted_binding_ref) + sha256_value(self.binding_fingerprint) + object.__setattr__(self, "binding_fingerprint", self.binding_fingerprint.lower()) + for value in ( + self.reporting_obligation_id, + self.reporting_revision_id, + self.reporting_materialization_id, + ): + reporting_identifier(value) + + @classmethod + def from_binding( + cls, + binding: ReportingDestinationBinding, + attempt: ReportingMaterializationAttempt, + key: ReportingVerificationKey, + ) -> ReportingDestinationRequest: + cap = key.capability + if ( + binding.principal != attempt.scope.principal + or binding.generation_key != attempt.scope.generation_key + or (binding.method, binding.transport, binding.format, binding.verification_profile) + != (cap.method, cap.transport, cap.format, cap.verification_profile) + or (binding.success_status == "delivered" and cap.verification_path != "destination") + ): + raise failure("BINDING_MISMATCH") + return cls( + binding.principal, + binding.generation_key, + binding.destination_ref, + binding.trusted_binding_ref, + binding_fingerprint(binding), + key, + attempt.scope.reporting_obligation_id, + attempt.reporting_revision_id, + attempt.reporting_materialization_id, + attempt.attempt, + ) + + @property + def external_id(self) -> str: + """Stable across pending retries; isolated across tenants and revision attempts.""" + return "rwm_" + hashlib.sha256(canonical_json_utf8_v1(asdict(self))).hexdigest() + + +@dataclass(frozen=True, slots=True) +class ReportingDestinationLocator(_ClosedValue): + """Writer claims identify what to read. They are never verification proof.""" + + external_id: str + binding_fingerprint: str + resource: ReportingResourceRecord + + def __post_init__(self) -> None: + _freeze_fields(self) + reporting_identifier(self.external_id) + sha256_value(self.binding_fingerprint) + object.__setattr__(self, "binding_fingerprint", self.binding_fingerprint.lower()) + + +@dataclass(frozen=True, slots=True) +class ReportingPreparedRevision(_ClosedValue): + """Bounded immutable canonical row bytes, produced before resolver I/O. + + This is not a durable reservation. Keep the original attempt identity when + an external effect is unknown. Never automatically import legacy pending + attempts: drain old writers and recover/import their identities explicitly. + """ + + request: ReportingDestinationRequest + obligation: ReportingObligationRecord = field(repr=False) + revision: ReportingRevisionRecord = field(repr=False) + delivery: ReportingObligationDeliveryRecord = field(repr=False) + binding: ReportingDestinationBinding = field(repr=False) + rows: tuple[bytes, ...] = field(repr=False) + + def __post_init__(self) -> None: + _freeze_fields(self) + + +@dataclass(frozen=True, slots=True) +class ReportingDestinationPage(_ClosedValue): + reporting_revision_id: str + rows: tuple[bytes, ...] = field(repr=False) + total_count: int + has_more: bool + cursor: str | None + format: ReportingFormat | None + verification_path: VerificationPath + native_version_ref: str | None = None + + def __post_init__(self) -> None: + _freeze_fields(self) + reporting_identifier(self.reporting_revision_id) + if self.total_count < 0 or self.has_more != (self.cursor is not None): + raise ValueError("destination page requires a paired cursor and valid total") + if self.cursor is not None: + reporting_identifier(self.cursor, maximum=2048) + if self.native_version_ref is not None: + native_version_reference(self.native_version_ref) + + +@dataclass(frozen=True, slots=True) +class ReportingNativeObservation(_ClosedValue): + location: str + native_version_ref: str + verification_path: Literal["representative_consumer", "destination"] + + def __post_init__(self) -> None: + _freeze_fields(self) + from adcp.reporting.evidence import resource_location + + resource_location(self.location) + native_version_reference(self.native_version_ref) + + +def object_path(value: str) -> str: + file_object_reference(value) + if "%" in value or ":" in value or any(part in {"", ".", ".."} for part in value.split("/")): + raise ValueError("object references require decoded relative path components") + return value + + +class ReportingHeartbeat(Protocol): + """B2 owns any parallel lease heartbeat. B1 only calls this checkpoint.""" + + async def checkpoint(self) -> None: ... + + +T = TypeVar("T") + + +@dataclass(frozen=True, slots=True) +class ReportingIOContext: + deadline_at: datetime + cancel: asyncio.Event = field(repr=False) + heartbeat: ReportingHeartbeat | None = field(default=None, repr=False) + close_timeout_seconds: float = 5.0 + + def __post_init__(self) -> None: + object.__setattr__(self, "deadline_at", aware_utc(self.deadline_at)) + if not math.isfinite(self.close_timeout_seconds) or self.close_timeout_seconds <= 0: + raise ValueError("close timeout must be finite and positive") + + async def run( + self, call: Callable[[], Awaitable[T]], *, effect: ReportingExternalEffect = "not_started" + ) -> T: + if self.heartbeat is not None: + await self._run(self.heartbeat.checkpoint, effect=effect) + return await self._run(call, effect=effect) + + async def _run(self, call: Callable[[], Awaitable[T]], *, effect: ReportingExternalEffect) -> T: + if self.cancel.is_set(): + raise asyncio.CancelledError + remaining = (self.deadline_at - datetime.now(timezone.utc)).total_seconds() + if remaining <= 0: + raise ReportingWriterError( + ReportingWriterFailure("DEADLINE_EXCEEDED", "same_identity", effect) + ) + + async def invoke() -> T: + return await call() + + task = asyncio.create_task(invoke()) + canceled = asyncio.create_task(self.cancel.wait()) + problem: ReportingWriterFailure | None = None + was_canceled = False + try: + done, _ = await asyncio.wait( + (task, canceled), timeout=remaining, return_when=asyncio.FIRST_COMPLETED + ) + if canceled in done: + was_canceled = True + elif task not in done: + problem = ReportingWriterFailure("DEADLINE_EXCEEDED", "same_identity", effect) + else: + try: + result = task.result() + except ReportingWriterError as exc: + problem = exc.failure + if ( + effect == "unknown" + and problem.effect == "not_started" + and problem.code + in {"RESOURCE_UNAVAILABLE", "DEADLINE_EXCEEDED", "LEASE_LOST"} + ): + problem = replace(problem, effect="unknown", retry="same_identity") + except Exception: + problem = ReportingWriterFailure( + "RESOURCE_UNAVAILABLE", "same_identity", effect + ) + except asyncio.CancelledError: + was_canceled = True + finally: + task.cancel() + canceled.cancel() + if await _join_tasks(task, canceled): + was_canceled = True + if was_canceled: + raise asyncio.CancelledError + # Outside the except suite: no provider __context__, even when inspected. + if problem is not None: + raise ReportingWriterError(problem) + return result + + +async def _join_tasks(*tasks: asyncio.Task[Any]) -> bool: + """Finish cancellation cleanup even if the caller is canceled repeatedly.""" + joined = asyncio.gather(*tasks, return_exceptions=True) + canceled = False + while not joined.done(): + try: + await asyncio.shield(joined) + except asyncio.CancelledError: + canceled = True + joined.result() + return canceled + + +class ReportingDestinationSession(ABC): + """Single-use SDK-owned lifecycle around an adopter's private provider session. + + Override only _open/_close and I/O methods. _close must tolerate partial + _open, finish promptly, and remove any adopter-owned temporary spool. The + SDK invokes it exactly once, shields cancellation, and joins all its tasks. + No credentials may be placed on the public request, descriptor or results. + """ + + def __init_subclass__(cls, **kwargs: Any) -> None: + super().__init_subclass__(**kwargs) + owned = { + "__repr__", + "__str__", + "__reduce__", + "__reduce_ex__", + "__getstate__", + "__aenter__", + "__aexit__", + "aclose", + "request", + "phase", + "context", + } + if owned.intersection(cls.__dict__): + raise TypeError("destination session lifecycle and redaction belong to the SDK") + + def __init__( + self, + request: ReportingDestinationRequest, + phase: ReportingIOPhase, + context: ReportingIOContext, + ) -> None: + if ( + type(request) is not ReportingDestinationRequest + or phase not in ("write", "readback") + or type(context) is not ReportingIOContext + ): + raise failure("BINDING_MISMATCH") + self._request, self._phase, self._context = request, phase, context + self._entered = False + self._closed = False + + def __repr__(self) -> str: + return "" + + __str__ = __repr__ + + @property + def request(self) -> ReportingDestinationRequest: + return self._request + + @property + def phase(self) -> ReportingIOPhase: + return self._phase + + @property + def context(self) -> ReportingIOContext: + return self._context + + def __reduce__(self) -> tuple[Any, ...]: + raise TypeError("destination sessions cannot be persisted") + + @abstractmethod + async def _open(self) -> None: ... + + @abstractmethod + async def _close(self) -> None: ... + + async def __aenter__(self) -> ReportingDestinationSession: + if self._entered or self._closed: + raise failure("BINDING_MISMATCH") + self._entered = True + problem: ReportingWriterFailure | None = None + canceled = False + try: + await self.context.run(self._open) + except asyncio.CancelledError: + canceled = True + except ReportingWriterError as exc: + problem = exc.failure + if canceled or problem is not None: + try: + await self.aclose() + except asyncio.CancelledError: + canceled = True + except ReportingWriterError: + pass + if canceled or self.context.cancel.is_set(): + raise asyncio.CancelledError + assert problem is not None + raise ReportingWriterError(problem) + return self + + async def __aexit__( + self, + kind: type[BaseException] | None, + value: BaseException | None, + traceback: TracebackType | None, + ) -> None: + try: + await self.aclose() + except ReportingWriterError: + if kind is None: + raise + if self.context.cancel.is_set(): + raise asyncio.CancelledError + if kind is None and datetime.now(timezone.utc) >= self.context.deadline_at: + raise ReportingWriterError( + ReportingWriterFailure("DEADLINE_EXCEEDED", "same_identity", "unknown") + ) + + async def aclose(self) -> None: + if self._closed: + return + self._closed = True + await _close_owned(self._close, self.context.close_timeout_seconds) + + async def write(self, content: ReportingPreparedRevision) -> ReportingDestinationLocator: + raise failure("UNSUPPORTED_VERIFICATION") + + async def read_rows( + self, locator: ReportingDestinationLocator, *, cursor: str | None, limit: int + ) -> ReportingDestinationPage: + raise failure("UNSUPPORTED_VERIFICATION") + + async def read_manifest(self, locator: ReportingDestinationLocator) -> bytes: + raise failure("UNSUPPORTED_VERIFICATION") + + async def list_objects(self, locator: ReportingDestinationLocator) -> tuple[str, ...]: + raise failure("UNSUPPORTED_VERIFICATION") + + def read_object( + self, locator: ReportingDestinationLocator, *, object_ref: str + ) -> AsyncIterator[bytes]: + raise failure("UNSUPPORTED_VERIFICATION") + + async def observe_native_version( + self, locator: ReportingDestinationLocator + ) -> ReportingNativeObservation: + raise failure("UNSUPPORTED_VERIFICATION") + + +class ReportingDestinationResolver(Protocol): + """Return an unopened session. _open reauthorizes each phase independently. + + Resolution must match every request coordinate, including consumer URL, + immutable trusted binding, definition/canonicalization and capability. A + resolver must never allocate resources before constructing the session. + """ + + def resolve( + self, + request: ReportingDestinationRequest, + *, + phase: ReportingIOPhase, + context: ReportingIOContext, + ) -> ReportingDestinationSession: ... + + +class ReportingDestinationWriter(Protocol): + @property + def capabilities(self) -> tuple[ReportingWriterCapability, ...]: ... + + @property + def production_eligible(self) -> bool: ... + + +async def _close_owned(close: Callable[[], Awaitable[None]], timeout: float) -> None: + """Cancellation-safe joining shared by the session and object stream lifecycles.""" + + async def invoke() -> None: + await close() + + task = asyncio.create_task(asyncio.wait_for(invoke(), timeout)) + canceled = False + while not task.done(): + try: + await asyncio.shield(task) + except asyncio.CancelledError: + canceled = True + except Exception: + break + if canceled or task.cancelled(): + if not task.cancelled(): + task.exception() # Retrieve any failure while propagating a clean cancellation. + raise asyncio.CancelledError + if task.exception() is not None: + raise ReportingWriterError( + ReportingWriterFailure("RESOURCE_UNAVAILABLE", "same_identity", "unknown") + ) diff --git a/src/adcp/reporting/materializer/reference.py b/src/adcp/reporting/materializer/reference.py new file mode 100644 index 000000000..fcfc09ae8 --- /dev/null +++ b/src/adcp/reporting/materializer/reference.py @@ -0,0 +1,368 @@ +"""Deterministic memory destination for tests/development. NEVER production support.""" + +from __future__ import annotations + +import hashlib +from collections.abc import AsyncIterator +from dataclasses import dataclass, field +from datetime import timedelta +from importlib.resources import files +from typing import Any, Literal, cast, final + +from adcp.reporting.evidence import ReportingCanonicalDigest +from adcp.reporting.ledger.delivery_models import ( + ReportingDeliveryPrincipal, + ReportingDestinationBinding, + ReportingResourceRecord, +) +from adcp.reporting.ledger.models import ReportingDefinitionBinding +from adcp.reporting.materializer._json import strict_reporting_json +from adcp.reporting.materializer.contracts import ( + ReportingCanonicalization, + ReportingDestinationLocator, + ReportingDestinationPage, + ReportingDestinationRequest, + ReportingDestinationSession, + ReportingIOContext, + ReportingIOPhase, + ReportingNativeObservation, + ReportingPreparedRevision, + ReportingVerificationKey, + ReportingWriterCapability, + ReportingWriterError, + ReportingWriterFailure, + binding_fingerprint, + failure, +) +from adcp.reporting.materializer.verification import ( + ReportingRevisionVerifier, + ReportingRevisionVerifierRegistry, +) + + +def reference_verifier( + capability: ReportingWriterCapability | None = None, +) -> ReportingRevisionVerifier: + """One installed example definition/canonicalization; no network resolution.""" + root = files("adcp.reporting.materializer").joinpath("assets") + definition = root.joinpath("reference-definition.json").read_bytes() + schema = root.joinpath("reference-row-schema.json").read_bytes() + contract = root.joinpath("reference-canonicalization.json").read_bytes() + capability = capability or ReportingWriterCapability( + "file_transfer", + "reference-memory", + "jsonl", + "canonical_digest", + "producer", + "immutable_location", + "sha256", + "conditional_create", + ) + return ReportingRevisionVerifier( + ReportingVerificationKey( + "reference-report-v1", + "paid_media_delivery", + ReportingDefinitionBinding( + "https://contracts.example.test/reference-definition.json", + hashlib.sha256(definition).hexdigest(), + "1.0.0", + "https://contracts.example.test/reference-row-schema.json", + hashlib.sha256(schema).hexdigest(), + monetary_metric_units=(("spend", "USD"),), + monetary_control_total_units=(("spend", "USD"),), + ), + ReportingCanonicalization( + "reference-jcs-rows-v1", + "https://contracts.example.test/reference-canonicalization.json", + hashlib.sha256(contract).hexdigest(), + ), + capability, + ), + definition, + schema, + contract, + ) + + +def reference_digest( + verifier: ReportingRevisionVerifier, rows: list[object] +) -> ReportingCanonicalDigest: + encoded, _ = verifier.canonicalize(rows) + contract = verifier.key.canonicalization + return ReportingCanonicalDigest( + hashlib.sha256(b"[" + b",".join(encoded) + b"]").hexdigest(), + contract.canonicalization_id, + contract.canonicalization_uri, + contract.canonicalization_sha256, + ) + + +@dataclass(frozen=True) +class _Artifact: + rows: tuple[bytes, ...] + manifest: bytes | None + objects: tuple[tuple[str, bytes], ...] + locator: ReportingDestinationLocator + + +@final +class ReferenceReportingDestinationWriter: + """Conditional, deterministic external identities in shared process memory. + + Losing this object loses its artifacts. No durability or Managed capability + follows from this type or its descriptors. The production flag is a constant + property, with no constructor/config override. + """ + + def __init_subclass__(cls, **kwargs: Any) -> None: + raise TypeError("the non-production reference writer cannot be promoted by subclassing") + + def __init__(self, capabilities: tuple[ReportingWriterCapability, ...]) -> None: + if type(capabilities) is not tuple or any( + type(c) is not ReportingWriterCapability for c in capabilities + ): + raise ValueError("reference capabilities require an immutable typed tuple") + self._capabilities = capabilities + self._artifacts: dict[str, _Artifact] = {} + self.write_effects = 0 + self.open_count = 0 + self.close_count = 0 + + @property + def capabilities(self) -> tuple[ReportingWriterCapability, ...]: + return self._capabilities + + @property + def production_eligible(self) -> Literal[False]: + return False + + def __repr__(self) -> str: + return "" + + +@dataclass(frozen=True) +class ReferenceReportingResolver: + writer: ReferenceReportingDestinationWriter + registry: ReportingRevisionVerifierRegistry + bindings: tuple[ReportingDestinationBinding, ...] = field(repr=False) + _revoked: set[ReportingDeliveryPrincipal] = field( + default_factory=set, init=False, repr=False, compare=False + ) + _rotations: list[int] = field( + default_factory=lambda: [0], init=False, repr=False, compare=False + ) + + def __post_init__(self) -> None: + if type(self.bindings) is not tuple or any( + type(b) is not ReportingDestinationBinding for b in self.bindings + ): + raise ValueError("reference resolver requires frozen bindings") + + def revoke(self, principal: ReportingDeliveryPrincipal) -> None: + self._revoked.add(principal) + + def rotate(self) -> None: + self._rotations[0] += 1 + + def resolve( + self, + request: ReportingDestinationRequest, + *, + phase: ReportingIOPhase, + context: ReportingIOContext, + ) -> ReportingDestinationSession: + return _Session(self, request, phase, context) + + +class _Session(ReportingDestinationSession): + def __init__( + self, + resolver: ReferenceReportingResolver, + request: ReportingDestinationRequest, + phase: ReportingIOPhase, + context: ReportingIOContext, + ) -> None: + super().__init__(request, phase, context) + self._resolver = resolver + self._credential: object | None = None + + async def _open(self) -> None: + resolver = self._resolver + resolver.writer.open_count += 1 + resolver.registry.require(self.request.verification_key) + if self.request.verification_key.capability not in resolver.writer.capabilities: + raise failure("UNSUPPORTED_VERIFICATION") + if self.request.principal in resolver._revoked or not any( + b.principal == self.request.principal + and b.generation_key == self.request.generation + and b.destination_ref == self.request.destination_ref + and b.trusted_binding_ref == self.request.trusted_binding_ref + and binding_fingerprint(b) == self.request.binding_fingerprint + for b in resolver.bindings + ): + raise failure("AUTHORIZATION_DENIED") + # A test credential exists only in the owned session and is discarded on + # every close. Real resolvers retrieve fresh credentials at this point. + self._credential = (object(), resolver._rotations[0]) + + async def _close(self) -> None: + self._credential = None + self._resolver.writer.close_count += 1 + + def _check(self, phase: ReportingIOPhase) -> None: + if self._credential is None or self._closed or self.phase != phase: + raise failure("AUTHORIZATION_DENIED") + + async def write(self, content: ReportingPreparedRevision) -> ReportingDestinationLocator: + self._check("write") + request = self.request + if content.request != request: + raise failure("BINDING_MISMATCH") + writer = self._resolver.writer + existing = writer._artifacts.get(request.external_id) + if existing is not None: + if existing.rows != content.rows: + raise ReportingWriterError( + ReportingWriterFailure("WRITE_FAILED", "never", "applied") + ) + return existing.locator + cap = request.verification_key.capability + prefix = request.external_id + objects: tuple[tuple[str, bytes], ...] = () + manifest: bytes | None = None + if cap.method == "file_transfer": + objects = tuple( + ( + f"{prefix}/part-{index // 250:06d}.jsonl", + b"".join(r + b"\n" for r in content.rows[index : index + 250]), + ) + for index in range(0, max(1, len(content.rows)), 250) + ) + period = content.obligation.period + manifest = strict_reporting_json( + { + "manifest_version": "1.0", + "complete": True, + "reporting_revision_id": request.reporting_revision_id, + "reporting_obligation_id": request.reporting_obligation_id, + "reporting_materialization_id": request.reporting_materialization_id, + "period": { + "start": period.start.isoformat(), + "end": period.end.isoformat(), + "source_timezone": period.source_timezone, + }, + "format": cap.format, + "compression": "none", + "files": [ + { + "object_ref": ref, + "size_bytes": len(raw), + "row_count": raw.count(b"\n"), + "sha256": hashlib.sha256(raw).hexdigest(), + } + for ref, raw in objects + ], + "total_size_bytes": sum(len(raw) for _, raw in objects), + "row_count": len(content.rows), + "control_totals": [ + t.to_wire() for t in content.revision.managed_control_totals or () + ], + "created_at": content.revision.created_at.isoformat(), + } + ) + resource = ReportingResourceRecord( + resource_ref=prefix, + kind=cast( + Literal["manifest", "dataset", "warehouse_relation"], + { + "file_transfer": "manifest", + "dataset_share": "dataset", + "warehouse_materialization": "warehouse_relation", + }[cap.method], + ), + location=f"{prefix}/manifest.json" if manifest is not None else f"{prefix}/table", + immutability=cap.immutability, + expires_at=max( + content.delivery.resource_retained_until, + self.context.deadline_at + timedelta(days=content.binding.resource_retention_days), + ), + native_version_ref=( + f"reference-{prefix}" if cap.immutability == "native_version" else None + ), + manifest_sha256=hashlib.sha256(manifest).hexdigest() if manifest is not None else None, + object_refs=tuple(ref for ref, _ in objects), + reader_compatibility=content.binding.reader_compatibility, + ) + locator = ReportingDestinationLocator( + request.external_id, request.binding_fingerprint, resource + ) + writer._artifacts[request.external_id] = _Artifact(content.rows, manifest, objects, locator) + writer.write_effects += 1 + return locator + + def _artifact(self, locator: ReportingDestinationLocator) -> _Artifact: + self._check("readback") + if ( + locator.external_id != self.request.external_id + or locator.binding_fingerprint != self.request.binding_fingerprint + ): + raise failure("BINDING_MISMATCH") + artifact = self._resolver.writer._artifacts.get(locator.external_id) + if artifact is None: + raise failure("RESOURCE_UNAVAILABLE") + if artifact.locator.resource.location != locator.resource.location: + raise failure("BINDING_MISMATCH") + return artifact + + async def read_rows( + self, locator: ReportingDestinationLocator, *, cursor: str | None, limit: int + ) -> ReportingDestinationPage: + artifact = self._artifact(locator) + offset = 0 + prefix = f"{self.request.external_id}:" + if cursor is not None: + if not cursor.startswith(prefix) or not cursor[len(prefix) :].isdigit(): + raise failure("BINDING_MISMATCH") + offset = int(cursor[len(prefix) :]) + window = artifact.rows[offset : offset + limit] + following = offset + len(window) + more = following < len(artifact.rows) + cap = self.request.verification_key.capability + return ReportingDestinationPage( + self.request.reporting_revision_id, + window, + len(artifact.rows), + more, + f"{prefix}{following}" if more else None, + cap.format, + cap.verification_path, + artifact.locator.resource.native_version_ref, + ) + + async def read_manifest(self, locator: ReportingDestinationLocator) -> bytes: + result = self._artifact(locator).manifest + if result is None: + raise failure("RESOURCE_UNAVAILABLE") + return result + + async def list_objects(self, locator: ReportingDestinationLocator) -> tuple[str, ...]: + return tuple(ref for ref, _ in self._artifact(locator).objects) + + async def read_object( + self, locator: ReportingDestinationLocator, *, object_ref: str + ) -> AsyncIterator[bytes]: + for ref, raw in self._artifact(locator).objects: + if ref == object_ref: + for start in range(0, len(raw), 4096): + yield raw[start : start + 4096] + return + raise failure("RESOURCE_UNAVAILABLE") + + async def observe_native_version( + self, locator: ReportingDestinationLocator + ) -> ReportingNativeObservation: + resource = self._artifact(locator).locator.resource + path = self.request.verification_key.capability.verification_path + if resource.native_version_ref is None or path == "producer": + raise failure("UNSUPPORTED_VERIFICATION") + return ReportingNativeObservation(resource.location, resource.native_version_ref, path) diff --git a/src/adcp/reporting/materializer/verification.py b/src/adcp/reporting/materializer/verification.py new file mode 100644 index 000000000..308b7880d --- /dev/null +++ b/src/adcp/reporting/materializer/verification.py @@ -0,0 +1,1128 @@ +"""SDK-owned verification of frozen source content and actual destination reads. + +The registry contains immutable, explicitly installed bytes. It never fetches a +URI or trusts a writer's digest as proof. B1 returns evidence only; B2 must +reselect and fence under its final account transaction before publishing it. +""" + +from __future__ import annotations + +import asyncio +import base64 +import hashlib +import re +from collections.abc import AsyncIterator, Sequence +from contextlib import asynccontextmanager +from dataclasses import dataclass, field, replace +from datetime import datetime, timedelta, timezone +from decimal import Decimal, localcontext +from typing import Any, Protocol, cast + +from jsonschema import Draft202012Validator + +from adcp.reporting.evidence import ReportingCanonicalDigest, ReportingControlTotalRecord +from adcp.reporting.ledger.delivery_models import ( + ReportingDestinationBinding, + ReportingMaterializationAttempt, + ReportingObligationDeliveryRecord, + ReportingPhysicalChecksum, + ReportingResourceRecord, + ReportingVerificationRecord, + _ClosedValue, + _freeze_fields, +) +from adcp.reporting.ledger.models import ( + ReportingDefinitionBinding, + ReportingObligationRecord, + ReportingRevisionRecord, +) +from adcp.reporting.ledger.producer import revision_content_sha256 +from adcp.reporting.ledger.store import LedgerConflictError, ReportingRowPage, revision_row_offset +from adcp.reporting.materializer._json import ( + ReportingVerificationLimits, + parse_reporting_json, + strict_reporting_json, +) +from adcp.reporting.materializer.contracts import ( + ReportingDestinationLocator, + ReportingDestinationPage, + ReportingDestinationRequest, + ReportingDestinationResolver, + ReportingDestinationSession, + ReportingIOContext, + ReportingIOPhase, + ReportingNativeObservation, + ReportingPreparedRevision, + ReportingVerificationKey, + ReportingWriterError, + ReportingWriterFailure, + _close_owned, + binding_fingerprint, + failure, + object_path, +) +from adcp.reporting.revision_selection import select_reporting_revision + + +class ReportingRevisionRowReader(Protocol): + async def read_revision_rows( + self, + *, + account_id: str, + reporting_revision_id: str, + cursor: str | None = None, + limit: int = 500, + ) -> ReportingRowPage: ... + + +@dataclass(frozen=True, slots=True) +class ReportingVerifiedDestination(_ClosedValue): + """Immutable SDK observations. B2 alone owns atomic publication/readiness.""" + + request: ReportingDestinationRequest + resource: ReportingResourceRecord + verification: ReportingVerificationRecord + + def __post_init__(self) -> None: + _freeze_fields(self) + + +def validate_materialization_target( + prepared: ReportingPreparedRevision, + *, + binding: ReportingDestinationBinding, + revisions: Sequence[ReportingRevisionRecord], +) -> None: + """Pure B2 finish seam. Caller must supply its locked, complete current history.""" + selection = select_reporting_revision( + revisions, + account_id=prepared.request.principal.account_id, + reporting_obligation_id=prepared.request.reporting_obligation_id, + required_finality=prepared.obligation.required_finality, + ) + if selection.kind == "corrupt": + raise failure("HISTORY_CORRUPT") + if ( + selection.kind != "selected" + or not selection.revision.readable + or selection.revision != prepared.revision + or binding_fingerprint(binding) != prepared.request.binding_fingerprint + ): + raise failure("CURRENT_REVISION_CHANGED") + + +@dataclass(frozen=True, slots=True) +class ReportingRevisionVerifier(_ClosedValue): + """One exact installed contract using the SDK's safe-integer JCS subset. + + Sum totals are declared on schema properties with ``x-adcp-control-total`` + (value_type, optional unit, decimal scale). This explicit pinned subset has + no expression evaluator, custom callbacks, mutable registration or network. + Unsupported formats and definition semantics fail during construction. + """ + + key: ReportingVerificationKey + definition_bytes: bytes = field(repr=False) + schema_bytes: bytes = field(repr=False) + canonicalization_bytes: bytes = field(repr=False) + limits: ReportingVerificationLimits = field(default_factory=ReportingVerificationLimits) + + def __post_init__(self) -> None: + _freeze_fields(self) + error = False + try: + self._runtime() + except Exception: + error = True + if error: + raise failure("UNSUPPORTED_VERIFICATION") + + def _runtime(self) -> _Canonicalizer: + key, definition = self.key, self.key.definition + if key.capability.format not in {"jsonl", None}: + raise failure("UNSUPPORTED_VERIFICATION") + if ( + key.capability.method == "file_transfer" + and key.capability.immutability != "immutable_location" + ): + raise failure("UNSUPPORTED_VERIFICATION") + if ( + definition.schema_dialect != "https://json-schema.org/draft/2020-12/schema" + or definition.schema_ref_policy != "local_fragment_only" + ): + raise failure("UNSUPPORTED_VERIFICATION") + for raw, expected in ( + (self.definition_bytes, definition.report_definition_sha256), + (self.schema_bytes, definition.schema_sha256), + (self.canonicalization_bytes, key.canonicalization.canonicalization_sha256), + ): + if hashlib.sha256(raw).hexdigest() != expected.lower(): + raise failure("UNSUPPORTED_VERIFICATION") + report = _mapping(parse_reporting_json(self.definition_bytes, self.limits)) + schema = _mapping(parse_reporting_json(self.schema_bytes, self.limits)) + contract = _mapping(parse_reporting_json(self.canonicalization_bytes, self.limits)) + if ( + report.get("report_definition_id") != key.report_definition_id + or report.get("reporting_profile") != key.reporting_profile + or schema.get("$schema") != definition.schema_dialect + or set(contract) + != { + "contract_version", + "media_type", + "algorithm", + "schema_sha256", + "primary_keys", + "golden_vectors", + } + or contract["contract_version"] != "1.0" + or contract["media_type"] != "application/vnd.adcp.reporting-canonicalization+json" + or contract["algorithm"] != "adcp_jcs_rows_v1" + or contract["schema_sha256"].lower() != definition.schema_sha256.lower() + ): + raise failure("UNSUPPORTED_VERIFICATION") + _local_schema(schema) + Draft202012Validator.check_schema(schema) + keys = contract["primary_keys"] + if ( + type(keys) is not list + or not keys + or any(type(k) is not str for k in keys) + or len(set(keys)) != len(keys) + ): + raise failure("UNSUPPORTED_VERIFICATION") + totals: list[_Total] = [] + properties = _mapping(schema.get("properties")) + metrics = report.get("metrics") + if type(metrics) is not list or not metrics: + raise failure("UNSUPPORTED_VERIFICATION") + for metric in metrics: + metric = _mapping(metric) + name = metric["name"] + prop = _mapping(properties[name]) + total = _mapping(prop["x-adcp-control-total"]) + if ( + metric.get("source_expression") != name + or metric.get("aggregation") != "sum" + or set(total) - {"value_type", "unit", "scale"} + or total.get("unit") != metric.get("unit") + or (total["value_type"], prop.get("type")) + not in {("integer", "integer"), ("decimal", "string")} + ): + raise failure("UNSUPPORTED_VERIFICATION") + scale = total.get("scale", 0) + if ( + type(scale) is not int + or not 0 <= scale <= 18 + or (total["value_type"] == "integer" and scale != 0) + ): + raise failure("UNSUPPORTED_VERIFICATION") + totals.append(_Total(name, total["value_type"], total.get("unit"), scale)) + if len({t.name for t in totals}) != len(totals): + raise failure("UNSUPPORTED_VERIFICATION") + if {t.name for t in totals} != { + name + for name, prop in properties.items() + if type(prop) is dict and "x-adcp-control-total" in prop + }: + raise failure("UNSUPPORTED_VERIFICATION") + if not set(keys).union(t.name for t in totals) <= set(schema.get("required", [])): + raise failure("UNSUPPORTED_VERIFICATION") + units = {t.name: t.unit for t in totals} + if any( + units.get(name) != unit + for name, unit in ( + *definition.monetary_metric_units, + *definition.monetary_control_total_units, + ) + ): + raise failure("UNSUPPORTED_VERIFICATION") + runtime = _Canonicalizer( + tuple(keys), tuple(totals), Draft202012Validator(schema), self.limits + ) + vectors = _mapping(contract["golden_vectors"]) + if not {"empty_report", "ordering_encoding"} <= set(vectors) or set(vectors) - { + "empty_report", + "ordering_encoding", + "additional", + }: + raise failure("UNSUPPORTED_VERIFICATION") + seen: set[str] = set() + for vector in [ + vectors["empty_report"], + vectors["ordering_encoding"], + *vectors.get("additional", []), + ]: + vector = _mapping(vector) + if ( + set(vector) != {"name", "purpose", "input_rows", "canonical_utf8_base64", "sha256"} + or vector["name"] in seen + ): + raise failure("UNSUPPORTED_VERIFICATION") + seen.add(vector["name"]) + rows, _ = runtime.canonicalize(vector["input_rows"]) + golden_bytes = base64.b64decode(vector["canonical_utf8_base64"], validate=True) + if ( + b"[" + b",".join(rows) + b"]" != golden_bytes + or hashlib.sha256(golden_bytes).hexdigest() != vector["sha256"].lower() + ): + raise failure("UNSUPPORTED_VERIFICATION") + if ( + vectors["empty_report"]["input_rows"] != [] + or vectors["empty_report"]["purpose"] != "empty_report" + ): + raise failure("UNSUPPORTED_VERIFICATION") + ordering = vectors["ordering_encoding"] + order_rows = ordering["input_rows"] + if ( + ordering["purpose"] != "ordering_encoding" + or len(order_rows) < 2 + or [runtime.order_key(r) for r in order_rows] + == sorted(runtime.order_key(r) for r in order_rows) + or not _unordered_members(order_rows) + ): + raise failure("UNSUPPORTED_VERIFICATION") + return runtime + + def canonicalize( + self, rows: Sequence[object] + ) -> tuple[tuple[bytes, ...], tuple[ReportingControlTotalRecord, ...]]: + """Useful to trusted publishers preparing expected evidence before storage.""" + return self._runtime().canonicalize(rows) + + +@dataclass(frozen=True, slots=True) +class ReportingRevisionVerifierRegistry(_ClosedValue): + verifiers: tuple[ReportingRevisionVerifier, ...] + + def __post_init__(self) -> None: + _freeze_fields(self) + if len({v.key for v in self.verifiers}) != len(self.verifiers): + raise failure("UNSUPPORTED_VERIFICATION") + + def require(self, key: ReportingVerificationKey) -> ReportingRevisionVerifier: + for verifier in self.verifiers: + if verifier.key == key: + return verifier + raise failure("UNSUPPORTED_VERIFICATION") + + async def prepare( + self, + *, + key: ReportingVerificationKey, + binding: ReportingDestinationBinding, + delivery: ReportingObligationDeliveryRecord, + obligation: ReportingObligationRecord, + revisions: Sequence[ReportingRevisionRecord], + attempt: ReportingMaterializationAttempt, + reader: ReportingRevisionRowReader, + context: ReportingIOContext, + ) -> ReportingPreparedRevision: + verifier = self.require(key) # Unsupported tuples fail before any source/resolver I/O. + selection = select_reporting_revision( + revisions, + account_id=obligation.account_id, + reporting_obligation_id=obligation.reporting_obligation_id, + required_finality=obligation.required_finality, + ) + if selection.kind == "corrupt": + raise failure("HISTORY_CORRUPT") + if selection.kind != "selected": + raise failure("REVISION_NOT_READY") + revision = selection.revision + _revision_metadata(revision) + if not revision.readable: + raise failure("REVISION_NOT_READY") + request = ReportingDestinationRequest.from_binding(binding, attempt, key) + if ( + not _same_definition(key, obligation.definition) + or (obligation.report_definition_id, obligation.reporting_profile) + != (key.report_definition_id, key.reporting_profile) + or request.generation != obligation.generation_key + or request.reporting_obligation_id != obligation.reporting_obligation_id + or request.reporting_revision_id != revision.reporting_revision_id + or delivery.scope != attempt.scope + or delivery.currency != obligation.currency + ): + raise failure("BINDING_MISMATCH") + _expected_digest(revision, key) + rows: list[dict[str, Any]] = [] + cursor: str | None = None + seen: set[str] = set() + budget = _Budget(verifier.limits) + while True: + budget.page() + page = await context.run( + lambda: reader.read_revision_rows( + account_id=obligation.account_id, + reporting_revision_id=revision.reporting_revision_id, + cursor=cursor, + limit=500, + ) + ) + if ( + type(page) is not ReportingRowPage + or type(page.rows) is not tuple + or page.reporting_revision_id != revision.reporting_revision_id + ): + raise failure("SOURCE_INVALID") + _page( + page.total_count, + page.has_more, + page.cursor, + len(page.rows), + len(rows), + revision.row_count, + seen, + ) + if page.cursor is not None: + cursor_invalid = False + try: + cursor_invalid = revision_row_offset( + page.cursor, revision.reporting_revision_id, 500 + ) != len(rows) + len(page.rows) + except (ValueError, LedgerConflictError): + cursor_invalid = True + if cursor_invalid: + raise failure("SOURCE_INVALID") + for row in page.rows: + encoded = strict_reporting_json(row, verifier.limits) + budget.add(encoded) + rows.append(_mapping(parse_reporting_json(encoded, verifier.limits))) + if not page.has_more: + break + cursor = page.cursor + if ( + revision_content_sha256( + reporting_revision_id=revision.reporting_revision_id, + row_count=revision.row_count, + control_totals=revision.control_totals, + reporting_rows=rows, + control_total_evidence=revision.managed_control_totals, + ) + != revision.revision_content_sha256.lower() + ): + raise failure("SOURCE_INVALID") + canonical_rows, totals = verifier.canonicalize(rows) + _verify_content(revision, key, canonical_rows, totals) + await context.run(_checkpoint) + return ReportingPreparedRevision( + request, obligation, revision, delivery, binding, canonical_rows + ) + + +@dataclass(frozen=True, slots=True) +class ReportingDestinationIO: + """Explicit one-shot write/readback operations; no coordinator or lease loop.""" + + registry: ReportingRevisionVerifierRegistry + resolver: ReportingDestinationResolver = field(repr=False) + + @asynccontextmanager + async def _session( + self, + prepared: ReportingPreparedRevision, + phase: ReportingIOPhase, + context: ReportingIOContext, + ) -> AsyncIterator[ReportingDestinationSession]: + self.registry.require(prepared.request.verification_key) + problem = False + canceled = False + try: + session = self.resolver.resolve(prepared.request, phase=phase, context=context) + except asyncio.CancelledError: + canceled = True + except Exception: + problem = True + if canceled: + raise asyncio.CancelledError + if problem: + raise ReportingWriterError( + ReportingWriterFailure("RESOURCE_UNAVAILABLE", "same_identity") + ) + if not isinstance(session, ReportingDestinationSession): + raise failure("BINDING_MISMATCH") + try: + _session_binding(session, prepared.request, phase, context) + except ReportingWriterError: + problem = True + if problem: + await session.aclose() + raise failure("BINDING_MISMATCH") + failure_record: ReportingWriterFailure | None = None + canceled = False + try: + async with session: + _session_binding(session, prepared.request, phase, context) + yield session + except asyncio.CancelledError: + canceled = True + except ReportingWriterError as exc: + failure_record = exc.failure + except Exception: + failure_record = ReportingWriterFailure( + "RESOURCE_UNAVAILABLE", "same_identity", "unknown" + ) + if canceled: + raise asyncio.CancelledError + if failure_record is not None: + raise ReportingWriterError(failure_record) + + async def write( + self, prepared: ReportingPreparedRevision, *, context: ReportingIOContext + ) -> ReportingDestinationLocator: + self._validate_prepared(prepared) + problem: ReportingWriterFailure | None = None + canceled = False + try: + async with self._session(prepared, "write", context) as session: + _session_binding(session, prepared.request, "write", context) + locator = await context.run(lambda: session.write(prepared), effect="unknown") + invalid_locator = False + try: + _locator_binding(locator, prepared.request) + except ReportingWriterError: + invalid_locator = True + if invalid_locator: + raise ReportingWriterError( + ReportingWriterFailure("BINDING_MISMATCH", "same_identity", "unknown") + ) + return locator + except asyncio.CancelledError: + canceled = True + except ReportingWriterError as exc: + problem = exc.failure + if canceled: + raise asyncio.CancelledError + assert problem is not None + raise ReportingWriterError(problem) + + def _validate_prepared(self, prepared: ReportingPreparedRevision) -> ReportingRevisionVerifier: + verifier = self.registry.require(prepared.request.verification_key) + request, obligation, binding = prepared.request, prepared.obligation, prepared.binding + cap = request.verification_key.capability + if ( + binding_fingerprint(binding) != request.binding_fingerprint + or request.principal != binding.principal + or request.generation != binding.generation_key + or request.destination_ref != binding.destination_ref + or request.trusted_binding_ref != binding.trusted_binding_ref + or request.generation != obligation.generation_key + or request.reporting_obligation_id != obligation.reporting_obligation_id + or request.reporting_revision_id != prepared.revision.reporting_revision_id + or prepared.revision.account_id != obligation.account_id + or prepared.revision.reporting_obligation_id != obligation.reporting_obligation_id + or not prepared.revision.readable + or prepared.delivery.scope.principal != request.principal + or prepared.delivery.scope.generation_key != request.generation + or prepared.delivery.scope.reporting_obligation_id != request.reporting_obligation_id + or prepared.delivery.currency != obligation.currency + or not _same_definition(verifier.key, obligation.definition) + or (obligation.report_definition_id, obligation.reporting_profile) + != (verifier.key.report_definition_id, verifier.key.reporting_profile) + or (binding.method, binding.transport, binding.format, binding.verification_profile) + != (cap.method, cap.transport, cap.format, cap.verification_profile) + or (binding.success_status == "delivered" and cap.verification_path != "destination") + ): + raise failure("BINDING_MISMATCH") + rows, totals = verifier.canonicalize( + [parse_reporting_json(r, verifier.limits) for r in prepared.rows] + ) + if rows != prepared.rows: + raise failure("SOURCE_INVALID") + _verify_content(prepared.revision, verifier.key, rows, totals) + return verifier + + async def verify( + self, + prepared: ReportingPreparedRevision, + locator: ReportingDestinationLocator, + *, + context: ReportingIOContext, + ) -> ReportingVerifiedDestination: + verifier = self._validate_prepared(prepared) + _locator_binding(locator, prepared.request) + problem: ReportingWriterFailure | None = None + canceled = False + try: + async with self._session(prepared, "readback", context) as session: + _session_binding(session, prepared.request, "readback", context) + return await _verify_destination(verifier, prepared, locator, session, context) + except asyncio.CancelledError: + canceled = True + except ReportingWriterError as exc: + problem = exc.failure + if canceled: + raise asyncio.CancelledError + assert problem is not None + if problem.code == "SOURCE_INVALID": + problem = ReportingWriterFailure("DESTINATION_CORRUPT", "new_attempt", "applied") + raise ReportingWriterError(problem) + + +def _same_definition( + key: ReportingVerificationKey, definition: ReportingDefinitionBinding | None +) -> bool: + """Total, closed comparison against the frozen verification key. + + A retained Core definition is only loosely constrained, so one this strict + key contract cannot even express -- a versioned query URI, an uppercase or + short digest, a legacy dialect -- is a binding mismatch. Rebinding it must + never surface a raw ``ValueError`` through the closed failure boundary. + """ + rebound: ReportingVerificationKey | None = None + if definition is not None: + try: + rebound = replace(key, definition=definition) + except Exception: + rebound = None + return rebound == key + + +def _session_binding( + session: ReportingDestinationSession, + request: ReportingDestinationRequest, + phase: ReportingIOPhase, + context: ReportingIOContext, +) -> None: + if session.request != request or session.phase != phase or session.context is not context: + raise failure("BINDING_MISMATCH") + + +def _locator_binding( + locator: ReportingDestinationLocator, request: ReportingDestinationRequest +) -> None: + if ( + type(locator) is not ReportingDestinationLocator + or locator.external_id != request.external_id + or locator.binding_fingerprint != request.binding_fingerprint + ): + raise failure("BINDING_MISMATCH") + # Logical row readers may use these paths before reading the manifest. + # Reject traversal before authorization or any readback method can see it. + invalid = False + try: + for ref in locator.resource.object_refs: + object_path(ref) + except ValueError: + invalid = True + if invalid: + raise failure("BINDING_MISMATCH") + + +def _revision_metadata(revision: ReportingRevisionRecord) -> None: + # Foundation records are source compatible dataclasses, not the strict + # destination boundary. Never let Python's bool/int equality bless counts + # or readability flags from a custom/legacy reader. + if ( + type(revision) is not ReportingRevisionRecord + or type(revision.row_count) is not int + or revision.row_count < 0 + or type(revision.readable) is not bool + or type(revision.readable_at_commit) is not bool + or type(revision.revision_content_sha256) is not str + or re.fullmatch(r"[0-9a-fA-F]{64}", revision.revision_content_sha256) is None + ): + raise failure("SOURCE_INVALID") + + +def _expected_digest( + revision: ReportingRevisionRecord, key: ReportingVerificationKey +) -> ReportingCanonicalDigest: + _revision_metadata(revision) + digest, canonical = revision.canonical_content_digest, key.canonicalization + if digest is None or ( + digest.canonicalization_id, + digest.canonicalization_uri, + digest.canonicalization_sha256.lower(), + ) != ( + canonical.canonicalization_id, + canonical.canonicalization_uri, + canonical.canonicalization_sha256, + ): + raise failure("UNSUPPORTED_VERIFICATION") + return digest + + +def _verify_content( + revision: ReportingRevisionRecord, + key: ReportingVerificationKey, + rows: tuple[bytes, ...], + totals: tuple[ReportingControlTotalRecord, ...], +) -> None: + expected = _expected_digest(revision, key) + if ( + len(rows) != revision.row_count + or _digest_rows(rows) != expected.value.lower() + or totals != revision.managed_control_totals + ): + raise failure("SOURCE_INVALID") + + +def _digest_rows(rows: Sequence[bytes]) -> str: + digest = hashlib.sha256(b"[") + for index, row in enumerate(rows): + if index: + digest.update(b",") + digest.update(row) + digest.update(b"]") + return digest.hexdigest() + + +def _mapping(value: object) -> dict[str, Any]: + if type(value) is not dict: + raise failure("SOURCE_INVALID") + return cast(dict[str, Any], value) + + +def _local_schema(schema: dict[str, Any]) -> None: + stack: list[object] = [schema] + while stack: + value = stack.pop() + if isinstance(value, dict): + if any(k in value for k in ("$id", "$dynamicRef", "$recursiveRef", "$vocabulary")) or ( + "$ref" in value + and (type(value["$ref"]) is not str or not value["$ref"].startswith("#")) + ): + raise failure("UNSUPPORTED_VERIFICATION") + stack.extend(value.values()) + elif isinstance(value, list): + stack.extend(value) + + +@dataclass(frozen=True) +class _Total: + name: str + value_type: Any + unit: str | None + scale: int + + +@dataclass +class _Canonicalizer: + keys: tuple[str, ...] + totals: tuple[_Total, ...] + validator: Any + limits: ReportingVerificationLimits + + def order_key(self, row: object) -> bytes: + row = _mapping(row) + values = [row[k] for k in self.keys if k in row] + if len(values) != len(self.keys) or any( + type(v) not in {str, int, bool, type(None)} for v in values + ): + raise failure("SOURCE_INVALID") + return strict_reporting_json(values, self.limits) + + def canonicalize( + self, rows: Sequence[object] + ) -> tuple[tuple[bytes, ...], tuple[ReportingControlTotalRecord, ...]]: + budget = _Budget(self.limits) + ordered: dict[bytes, bytes] = {} + sums = [Decimal(0) for _ in self.totals] + for row in rows: + encoded = strict_reporting_json(row, self.limits) + budget.add(encoded) + value = _mapping(row) + valid = False + try: + valid = self.validator.is_valid(value) + except Exception: + valid = False # No schema resolver/validation diagnostics leave this boundary. + if not valid: + raise failure("SOURCE_INVALID") + key = self.order_key(value) + if key in ordered: + raise failure("SOURCE_INVALID") + ordered[key] = encoded + for index, total in enumerate(self.totals): + item = value.get(total.name) + if total.value_type == "integer": + if type(item) is not int: + raise failure("SOURCE_INVALID") + elif type(item) is not str or re_decimal(item, total.scale) is False: + raise failure("SOURCE_INVALID") + with localcontext() as ctx: + ctx.prec = 128 + sums[index] += Decimal(item) + totals = tuple( + ReportingControlTotalRecord( + rule.name, + format(value, f".{rule.scale}f"), + rule.value_type, + rule.unit, + ) + for rule, value in zip(self.totals, sums) + ) + return tuple(ordered[k] for k in sorted(ordered)), totals + + +def re_decimal(value: str, scale: int) -> bool: + return bool( + re.fullmatch(r"-?(?:0|[1-9][0-9]{0,37})" + (rf"\.[0-9]{{{scale}}}" if scale else ""), value) + ) + + +@dataclass +class _Budget: + limits: ReportingVerificationLimits + total_bytes: int = 0 + items: int = 0 + rows: int = 0 + pages: int = 0 + + def page(self) -> None: + self.pages += 1 + if self.pages > self.limits.max_pages: + raise failure("LIMIT_EXCEEDED") + + def add(self, encoded: bytes) -> None: + self.total_bytes += len(encoded) + self.rows += 1 + stack: list[object] = [parse_reporting_json(encoded, self.limits)] + while stack: + item = stack.pop() + self.items += 1 + if isinstance(item, dict): + self.items += len(item) + stack.extend(item.values()) + elif isinstance(item, list): + stack.extend(item) + if ( + self.total_bytes > self.limits.max_total_bytes + or self.items > self.limits.max_items + or self.rows > self.limits.max_rows + ): + raise failure("LIMIT_EXCEEDED") + + +def _page( + total: int, + has_more: bool, + cursor: str | None, + count: int, + before: int, + expected: int, + seen: set[str], +) -> None: + if ( + type(total) is not int + or total != expected + or type(has_more) is not bool + or has_more != (cursor is not None) + or count > 500 + or before + count > expected + or (has_more and (count == 0 or before + count >= expected)) + or (not has_more and before + count != expected) + ): + raise failure("SOURCE_INVALID") + if cursor is not None: + if type(cursor) is not str or not cursor or len(cursor) > 2048 or cursor in seen: + raise failure("SOURCE_INVALID") + seen.add(cursor) + + +async def _verify_destination( + verifier: ReportingRevisionVerifier, + prepared: ReportingPreparedRevision, + locator: ReportingDestinationLocator, + session: ReportingDestinationSession, + context: ReportingIOContext, +) -> ReportingVerifiedDestination: + cap, resource = verifier.key.capability, locator.resource + expected_kind = { + "file_transfer": "manifest", + "dataset_share": "dataset", + "warehouse_materialization": "warehouse_relation", + }[cap.method] + if ( + resource.kind != expected_kind + or resource.immutability != cap.immutability + or resource.expires_at + < max( + prepared.delivery.resource_retained_until, + datetime.now(timezone.utc) + timedelta(days=prepared.binding.resource_retention_days), + ) + or resource.reader_compatibility != prepared.binding.reader_compatibility + ): + raise failure("SOURCE_INVALID") + native: ReportingNativeObservation | None = None + if cap.immutability == "native_version": + native = await context.run( + lambda: session.observe_native_version(locator), effect="unknown" + ) + _native(native, resource, cap.verification_path) + cursor: str | None = None + seen: set[str] = set() + budget = _Budget(verifier.limits) + destination: list[object] = [] + while True: + budget.page() + page = await context.run( + lambda: session.read_rows(locator, cursor=cursor, limit=500), effect="unknown" + ) + if ( + type(page) is not ReportingDestinationPage + or page.reporting_revision_id != prepared.request.reporting_revision_id + or page.format != cap.format + or page.verification_path != cap.verification_path + or page.native_version_ref != (native.native_version_ref if native else None) + ): + raise failure("SOURCE_INVALID") + _page( + page.total_count, + page.has_more, + page.cursor, + len(page.rows), + len(destination), + len(prepared.rows), + seen, + ) + for raw in page.rows: + value = parse_reporting_json(raw, verifier.limits) + row_bytes = strict_reporting_json(value, verifier.limits) + budget.add(raw) + if row_bytes != prepared.rows[len(destination)]: + raise failure("SOURCE_INVALID") + destination.append(value) + if not page.has_more: + break + cursor = page.cursor + encoded, totals = verifier.canonicalize(destination) + _verify_content(prepared.revision, verifier.key, encoded, totals) + checksums: tuple[ReportingPhysicalChecksum, ...] = () + manifest_digest: str | None = None + if cap.method == "file_transfer": + checksums, manifest_digest = await _verify_files( + verifier, prepared, locator, session, context + ) + if native is not None: + observed = await context.run( + lambda: session.observe_native_version(locator), effect="unknown" + ) + _native(observed, resource, cap.verification_path) + if observed != native: + raise failure("SOURCE_INVALID") + at = datetime.now(timezone.utc) + canonical = verifier.key.canonicalization + verification = ReportingVerificationRecord( + verified_at=at, + verification_path=cap.verification_path, + verification_profile=cap.verification_profile, + row_count=len(encoded), + control_totals=totals, + canonical_content_digest=( + ReportingCanonicalDigest( + _digest_rows(encoded), + canonical.canonicalization_id, + canonical.canonicalization_uri, + canonical.canonicalization_sha256, + ) + if cap.verification_profile == "canonical_digest" + else None + ), + physical_checksums=checksums, + native_version_ref=native.native_version_ref if native else None, + native_observed_through=native.verification_path if native else None, + verified_format=cap.format, + ) + return ReportingVerifiedDestination( + prepared.request, replace(resource, manifest_sha256=manifest_digest), verification + ) + + +def _native( + observed: ReportingNativeObservation, resource: ReportingResourceRecord, path: str +) -> None: + if type(observed) is not ReportingNativeObservation or ( + observed.location, + observed.native_version_ref, + observed.verification_path, + ) != (resource.location, resource.native_version_ref, path): + raise failure("SOURCE_INVALID") + + +async def _verify_files( + verifier: ReportingRevisionVerifier, + prepared: ReportingPreparedRevision, + locator: ReportingDestinationLocator, + session: ReportingDestinationSession, + context: ReportingIOContext, +) -> tuple[tuple[ReportingPhysicalChecksum, ...], str]: + raw = await context.run(lambda: session.read_manifest(locator), effect="unknown") + if type(raw) is not bytes or len(raw) > verifier.limits.max_value_bytes: + raise failure("SOURCE_INVALID") + digest = hashlib.sha256(raw).hexdigest() + if ( + locator.resource.manifest_sha256 is None + or digest != locator.resource.manifest_sha256.lower() + ): + raise failure("SOURCE_INVALID") + manifest = _mapping(parse_reporting_json(raw, verifier.limits)) + required = { + "manifest_version", + "complete", + "reporting_revision_id", + "reporting_obligation_id", + "reporting_materialization_id", + "period", + "format", + "compression", + "files", + "total_size_bytes", + "row_count", + "control_totals", + "created_at", + } + period = prepared.obligation.period + if ( + set(manifest) != required + or manifest["manifest_version"] != "1.0" + or manifest["complete"] is not True + or manifest["reporting_revision_id"] != prepared.request.reporting_revision_id + or manifest["reporting_obligation_id"] != prepared.request.reporting_obligation_id + or manifest["reporting_materialization_id"] != prepared.request.reporting_materialization_id + or manifest["format"] != verifier.key.capability.format + or manifest["compression"] != "none" + or manifest["period"] + != { + "start": period.start.isoformat(), + "end": period.end.isoformat(), + "source_timezone": period.source_timezone, + } + or type(manifest["row_count"]) is not int + or manifest["row_count"] != len(prepared.rows) + or manifest["control_totals"] + != [t.to_wire() for t in prepared.revision.managed_control_totals or ()] + or type(manifest["total_size_bytes"]) is not int + or manifest["total_size_bytes"] < 0 + ): + raise failure("SOURCE_INVALID") + created = None + try: + if type(manifest["created_at"]) is str and re.fullmatch( + r"\d{4}-\d{2}-\d{2}[Tt]\d{2}:\d{2}:\d{2}(?:\.\d+)?(?:[Zz]|[+-]\d{2}:\d{2})", + manifest["created_at"], + ): + created = datetime.fromisoformat( + manifest["created_at"].replace("Z", "+00:00").replace("z", "+00:00") + ) + except ValueError: + pass + if ( + created is None + or created.tzinfo is None + or created < prepared.revision.created_at + or created > datetime.now(timezone.utc) + ): + raise failure("SOURCE_INVALID") + entries = manifest["files"] + if type(entries) is not list or not 1 <= len(entries) <= verifier.limits.max_objects: + raise failure("SOURCE_INVALID") + refs = tuple(_checked_object_path(_mapping(entry).get("object_ref")) for entry in entries) + inventory = await context.run(lambda: session.list_objects(locator), effect="unknown") + if ( + len(set(refs)) != len(refs) + or refs != locator.resource.object_refs + or type(inventory) is not tuple + or inventory != refs + ): + raise failure("SOURCE_INVALID") + size, ordinal, chunks = 0, 0, 0 + checksums: list[ReportingPhysicalChecksum] = [] + for entry, ref in zip(entries, refs): + if ( + set(entry) != {"object_ref", "size_bytes", "row_count", "sha256"} + or type(entry["size_bytes"]) is not int + or type(entry["row_count"]) is not int + or entry["size_bytes"] < 0 + or entry["row_count"] < 0 + ): + raise failure("SOURCE_INVALID") + hashed, actual, count = hashlib.sha256(), 0, 0 + pending = b"" + stream = session.read_object(locator, object_ref=ref) + try: + while True: + # StopAsyncIteration is a stream boundary, not a provider failure. + chunk = await context.run(lambda: _next_chunk(stream), effect="unknown") + if chunk is None: + break + if type(chunk) is not bytes or not chunk: + raise failure("SOURCE_INVALID") + chunks += 1 + size += len(chunk) + actual += len(chunk) + if ( + chunks > verifier.limits.max_chunks + or size > verifier.limits.max_total_bytes + or actual > entry["size_bytes"] + ): + raise failure("LIMIT_EXCEEDED") + hashed.update(chunk) + pending += chunk + while b"\n" in pending: + line, pending = pending.split(b"\n", 1) + if ( + ordinal >= len(prepared.rows) + or strict_reporting_json( + parse_reporting_json(line, verifier.limits), verifier.limits + ) + != prepared.rows[ordinal] + ): + raise failure("SOURCE_INVALID") + ordinal += 1 + count += 1 + if len(pending) > verifier.limits.max_value_bytes: + raise failure("LIMIT_EXCEEDED") + finally: + close = getattr(stream, "aclose", None) + if close is not None: + await _close_owned(close, context.close_timeout_seconds) + if ( + pending + or actual != entry["size_bytes"] + or count != entry["row_count"] + or type(entry["sha256"]) is not str + or hashed.hexdigest() != entry["sha256"].lower() + ): + raise failure("SOURCE_INVALID") + checksums.append(ReportingPhysicalChecksum(ref, "sha256", hashed.hexdigest())) + if size != manifest["total_size_bytes"] or ordinal != len(prepared.rows): + raise failure("SOURCE_INVALID") + return tuple(checksums), digest + + +async def _next_chunk(stream: Any) -> bytes | None: + try: + return cast(bytes, await stream.__anext__()) + except StopAsyncIteration: + return None + + +async def _checkpoint() -> None: + """One service-owned cancellation/deadline/heartbeat boundary; never a loop.""" + + +def _checked_object_path(value: object) -> str: + valid = False + if type(value) is str: + try: + object_path(value) + valid = True + except ValueError: + pass + if not valid: + raise failure("SOURCE_INVALID") + return cast(str, value) + + +def _unordered_members(value: object) -> bool: + """The mandatory golden vector must exercise JCS member ordering as well.""" + if type(value) is dict: + mapping = cast(dict[str, object], value) + if list(mapping) != sorted(mapping, key=lambda name: name.encode("utf-16-be")): + return True + return any(_unordered_members(item) for item in mapping.values()) + if type(value) is list: + return any(_unordered_members(item) for item in cast(list[object], value)) + return False diff --git a/src/adcp/reporting/outbox/__init__.py b/src/adcp/reporting/outbox/__init__.py index 33ddfe60f..f6a0bc090 100644 --- a/src/adcp/reporting/outbox/__init__.py +++ b/src/adcp/reporting/outbox/__init__.py @@ -48,6 +48,7 @@ StatusCheckpoint, StatusDueLease, StatusNotificationStore, + StatusSelectorRebuildStore, StatusTurn, ) from adcp.reporting.outbox.status_memory import ( @@ -78,6 +79,7 @@ "StatusCheckpoint", "StatusDueLease", "StatusNotificationStore", + "StatusSelectorRebuildStore", "StatusTurn", "ReportingStatusProjector", "ReportingStatusSweeper", diff --git a/src/adcp/reporting/outbox/_capabilities.py b/src/adcp/reporting/outbox/_capabilities.py index 9779d53af..540bf4542 100644 --- a/src/adcp/reporting/outbox/_capabilities.py +++ b/src/adcp/reporting/outbox/_capabilities.py @@ -121,5 +121,7 @@ async def advertised_notifications( or frozen.currency != obligation.currency ): raise ReportingNotificationError("notification_chain_unready") - result["readiness_notification"] = "reporting.delivery_ready" + # B1 contracts and a frozen binding are not a durable materializer. + # B2 must supply a concrete verified write/finish readiness proof before + # a positive delivery_ready capability can be added here. return result diff --git a/src/adcp/reporting/outbox/required_status_selector_schema.json b/src/adcp/reporting/outbox/required_status_selector_schema.json new file mode 100644 index 000000000..14075e031 --- /dev/null +++ b/src/adcp/reporting/outbox/required_status_selector_schema.json @@ -0,0 +1,42 @@ +{ + "column:reporting_status_accounts.selector_target_version": { + "enabled": true, + "fingerprint": "7d6ce545a69d9f3e873875690ed9ccbec7114a9e32ad4820babca3f1c97b38ea" + }, + "column:reporting_status_accounts.selector_transition": { + "enabled": true, + "fingerprint": "1a1ab7c892bfef3ca42f00cf764453bc5e0578b3a82ecf81da90b56816df493f" + }, + "column:reporting_status_scope_checkpoints.selector_semantics_version": { + "enabled": true, + "fingerprint": "7d6ce545a69d9f3e873875690ed9ccbec7114a9e32ad4820babca3f1c97b38ea" + }, + "column:reporting_status_scope_checkpoints.selector_writer_floor": { + "enabled": true, + "fingerprint": "7d6ce545a69d9f3e873875690ed9ccbec7114a9e32ad4820babca3f1c97b38ea" + }, + "constraint:reporting_status_accounts.reporting_status_selector_account_transition": { + "enabled": true, + "fingerprint": "47b1f4b23693b8d717bbd0efbe388416c6d32a9bf15487b80402d79c7df40f47" + }, + "constraint:reporting_status_scope_checkpoints.reporting_status_selector_versions": { + "enabled": true, + "fingerprint": "27e9e3994ed88d04f38bd6272cef49110c92143746ceb5845cd183d0ce2ed146" + }, + "function:reporting_status_selector_writer_guard_v2()": { + "enabled": true, + "fingerprint": "6f9f6df5355a5cc64f411ebc601b30c1358d9969061f5b92f4339804f887e837" + }, + "index:reporting_status_accounts.reporting_status_selector_accounts": { + "enabled": true, + "fingerprint": "fd952ac0f3fb0dab96f35395e97e3e2493769e3dcc05bbba71d37a298027f281" + }, + "index:reporting_status_scope_checkpoints.reporting_status_selector_rebuild": { + "enabled": true, + "fingerprint": "c8b8633eebbfb756d627e9e8b19d6949770cbe5374f0d63eb54e457751ba02a2" + }, + "trigger:reporting_status_scope_checkpoints.reporting_status_selector_writer_v2": { + "enabled": true, + "fingerprint": "ad9a5b09fa436bb9bdd7d0e4f88f88eba60ed730864b099636e499a747d78eac" + } +} diff --git a/src/adcp/reporting/outbox/status.py b/src/adcp/reporting/outbox/status.py index e8ea0a5b0..a218400a7 100644 --- a/src/adcp/reporting/outbox/status.py +++ b/src/adcp/reporting/outbox/status.py @@ -8,7 +8,7 @@ from dataclasses import dataclass, field from datetime import datetime -from typing import Any, Protocol +from typing import Any, Protocol, runtime_checkable from uuid import uuid4 from adcp.reporting.ledger.models import ReportingDeliveryEscalation @@ -25,6 +25,7 @@ apply_intents_to_snapshot, lifecycle_intents, ) +from adcp.reporting.revision_selection import REPORTING_SELECTOR_VERSION @dataclass(frozen=True) @@ -47,6 +48,8 @@ class StatusCheckpoint: publishable: bool lease_token: str | None = field(default=None, repr=False) lease_expires_at: datetime | None = None + selector_semantics_version: int = REPORTING_SELECTOR_VERSION + selector_writer_floor: int = REPORTING_SELECTOR_VERSION @dataclass(frozen=True) @@ -85,6 +88,13 @@ async def release_due(self, lease: StatusDueLease) -> bool: ... async def checkpoints(self, *, account_id: str) -> tuple[StatusCheckpoint, ...]: ... +@runtime_checkable +class StatusSelectorRebuildStore(Protocol): + """Indexed, account-discovering C cutover seam; no materializer work queue.""" + + async def rebuild_one(self) -> StatusTurn: ... + + @dataclass(frozen=True) class ReportingStatusProjector: store: StatusNotificationStore @@ -92,6 +102,12 @@ class ReportingStatusProjector: async def run_once(self, *, account_id: str) -> StatusTurn: return await self.store.project_one(account_id=account_id) + async def rebuild_once(self) -> StatusTurn: + """Reproject one populated old scope's account, without an account list.""" + if not isinstance(self.store, StatusSelectorRebuildStore): + raise ReportingNotificationError("status_selector_rebuild_unsupported") + return await self.store.rebuild_one() + @dataclass(frozen=True) class ReportingStatusSweeper: @@ -171,4 +187,7 @@ def settled_replay(snapshot: ReportingStatusSnapshot) -> ReportingStatusSnapshot def escalation_identity(escalation: ReportingDeliveryEscalation | None) -> dict[str, Any]: - return escalation.to_wire() if escalation else {} + return { + **(escalation.to_wire() if escalation else {}), + "selector_semantics_version": REPORTING_SELECTOR_VERSION, + } diff --git a/src/adcp/reporting/outbox/status_memory.py b/src/adcp/reporting/outbox/status_memory.py index b20a094d7..e3a9b89d0 100644 --- a/src/adcp/reporting/outbox/status_memory.py +++ b/src/adcp/reporting/outbox/status_memory.py @@ -5,7 +5,7 @@ from dataclasses import dataclass, field, replace from datetime import timedelta from secrets import token_hex -from typing import Any +from typing import Any, Literal from adcp.reporting.ledger.models import ReportingDeliveryEscalation from adcp.reporting.ledger.notification_models import ReportingNotificationError @@ -27,6 +27,7 @@ escalation_identity, settled_replay, ) +from adcp.reporting.revision_selection import REPORTING_SELECTOR_VERSION @dataclass @@ -37,6 +38,7 @@ class _StatusMemoryState: ) outbox: NotificationState = field(default_factory=NotificationState) replay: dict[str, ReportingStatusSnapshot] = field(default_factory=dict) + selector_accounts: dict[str, Literal["transitioning", "complete"]] = field(default_factory=dict) class InMemoryReportingStatusOutbox(InMemoryReportingOutbox): @@ -58,6 +60,27 @@ def __init__( self.ledger, self.escalation = ledger, escalation if ledger._status_notification_state is None: ledger._status_notification_state = _StatusMemoryState() + state = ledger._status_notification_state + if not hasattr(state, "selector_accounts"): + state.selector_accounts = {} + for key, checkpoint in tuple(state.checkpoints.items()): + # An old shared-state image has no epoch fields. Decode by keyword + # rather than letting new dataclass class defaults label it v2. + if "selector_semantics_version" not in vars(checkpoint): + state.checkpoints[key] = StatusCheckpoint( + scope=checkpoint.scope, + fingerprint=checkpoint.fingerprint, + generation=checkpoint.generation, + snapshot=checkpoint.snapshot, + next_due_at=checkpoint.next_due_at, + source_sequence=checkpoint.source_sequence, + baseline=checkpoint.baseline, + publishable=checkpoint.publishable, + lease_token=checkpoint.lease_token, + lease_expires_at=checkpoint.lease_expires_at, + selector_semantics_version=1, + selector_writer_floor=1, + ) self.outbox = InMemoryReportingStatusOutbox(ledger) @property @@ -79,7 +102,8 @@ def _cursor(self, account_id: str) -> int: async def baseline(self, *, account_id: str) -> bool: async with self.ledger._mutation(): if account_id in self._state.accounts: - self._cursor(account_id) + if not self._needs_rebuild(account_id): + self._cursor(account_id) return False snapshot = settle_memory_snapshot(self.ledger, account_id) assert self.ledger._notification_state is not None @@ -94,21 +118,99 @@ async def baseline(self, *, account_id: str) -> bool: self._apply(snapshot, through=through, baseline=True) self._state.replay[account_id] = snapshot self._state.accounts[account_id] = (through, escalation_identity(self.escalation)) + self._state.selector_accounts[account_id] = "complete" return True async def baseline_ready(self, *, account_id: str) -> bool: async with self.ledger._mutation(): if account_id not in self._state.accounts: return False + if self._needs_rebuild(account_id): + return False self._cursor(account_id) return True + def _needs_rebuild(self, account_id: str) -> bool: + return account_id in self._state.accounts and ( + self._state.selector_accounts.get(account_id) != "complete" + or any( + c.scope.account_id == account_id + and ( + c.selector_semantics_version != REPORTING_SELECTOR_VERSION + or c.selector_writer_floor != REPORTING_SELECTOR_VERSION + ) + for c in self._state.checkpoints.values() + ) + ) + + def _rebuild(self, account_id: str) -> StatusTurn: + if not self._needs_rebuild(account_id): + return StatusTurn(False) + if self._state.selector_accounts.get(account_id) != "transitioning": + through, policy = self._state.accounts[account_id] + expected = escalation_identity(self.escalation) + if policy not in ( + expected, + {k: v for k, v in expected.items() if k != "selector_semantics_version"}, + ): + raise ReportingNotificationError("status_policy_conflict") + self._state.accounts[account_id] = (through, expected) + self._state.selector_accounts[account_id] = "transitioning" + for key, checkpoint in tuple(self._state.checkpoints.items()): + if checkpoint.scope.account_id == account_id: + self._state.checkpoints[key] = replace( + checkpoint, selector_writer_floor=REPORTING_SELECTOR_VERSION + ) + return StatusTurn(True) + turn = self._project(account_id) + if turn.did_work: + return turn + snapshot = settle_memory_snapshot(self.ledger, account_id) + deadlines = [ + c.next_due_at + for c in self._state.checkpoints.values() + if c.scope.account_id == account_id + and c.next_due_at is not None + and c.next_due_at <= snapshot.as_of + ] + if deadlines: + return StatusTurn( + True, + self._apply( + replace(snapshot, as_of=min(deadlines)), through=self._cursor(account_id) + ), + ) + count = self._apply(snapshot, through=self._cursor(account_id)) + self._state.replay[account_id] = snapshot + self._state.selector_accounts[account_id] = "complete" + return StatusTurn(True, count) + + async def rebuild_one(self) -> StatusTurn: + async with self.ledger._mutation(): + expected = escalation_identity(self.escalation) + legacy = {k: v for k, v in expected.items() if k != "selector_semantics_version"} + account_id = next( + ( + a + for a in sorted(self._state.accounts) + if self._needs_rebuild(a) and self._state.accounts[a][1] in (expected, legacy) + ), + None, + ) + return self._rebuild(account_id) if account_id is not None else StatusTurn(False) + def _apply( self, snapshot: ReportingStatusSnapshot, *, through: int, baseline: bool = False ) -> int: snapshot = settled_replay(snapshot) events = 0 - for scope in projection_scopes(snapshot): + scopes = {s.checkpoint_key: s for s in projection_scopes(snapshot)} + scopes.update( + (key, c.scope) + for key, c in self._state.checkpoints.items() + if c.scope.account_id == snapshot.account_id + ) + for _, scope in sorted(scopes.items()): result = project_status_scope(StatusProjectionInput(snapshot, scope, self.escalation)) checkpoint, event = advance_checkpoint( self._state.checkpoints.get(scope.checkpoint_key), @@ -146,6 +248,9 @@ def _project(self, account_id: str) -> StatusTurn: async def project_one(self, *, account_id: str) -> StatusTurn: async with self.ledger._mutation(): + rebuilt = self._rebuild(account_id) + if rebuilt.did_work: + return rebuilt return self._project(account_id) async def claim_due( @@ -154,6 +259,8 @@ async def claim_due( if lease_seconds <= 0: raise ValueError("lease_seconds must be positive") async with self.ledger._lock: + if self._needs_rebuild(account_id): + return None self._cursor(account_id) at = self.ledger._clock() for key, checkpoint in sorted(self._state.checkpoints.items()): @@ -183,6 +290,8 @@ async def complete_due(self, lease: StatusDueLease) -> StatusTurn: async def _complete_due(self, lease: StatusDueLease) -> StatusTurn: async with self.ledger._mutation(): + if self._needs_rebuild(lease.scope.account_id): + return StatusTurn(False) checkpoint = self._state.checkpoints.get(lease.scope.checkpoint_key) at = self.ledger._clock() if ( diff --git a/src/adcp/reporting/outbox/status_pg.py b/src/adcp/reporting/outbox/status_pg.py index 941fe9954..4ac861d90 100644 --- a/src/adcp/reporting/outbox/status_pg.py +++ b/src/adcp/reporting/outbox/status_pg.py @@ -58,7 +58,7 @@ ) _CHECKPOINT = ( "scope, fingerprint, generation, snapshot, next_due_at, source_sequence, baseline, publishable," - " lease_token, lease_expires_at, initialized" + " lease_token, lease_expires_at, selector_semantics_version, selector_writer_floor, initialized" ) _LIFECYCLE = TypeAdapter(ReportingIssueLifecycle) @@ -212,11 +212,19 @@ async def create_schema(self) -> None: .joinpath("reporting_status_notifications.sql") .read_text() ) + await connection.execute( + files("adcp.reporting.ledger") + .joinpath("reporting_status_selector_version.sql") + .read_text() + ) await validate_status_schema(connection) @asynccontextmanager async def _transaction(self, account_id: str) -> AsyncIterator[Any]: async with self.ledger._pool.connection() as connection, connection.transaction(): + await connection.execute( + "SELECT set_config('adcp.reporting.selector_semantics_version', '2', true)" + ) await self.ledger._lock_account(connection, account_id) yield connection @@ -240,8 +248,8 @@ async def _lock_scopes_on(self, connection: Any, snapshot: ReportingStatusSnapsh for scope in projection_scopes(snapshot): await connection.execute( f"INSERT INTO reporting_status_scope_checkpoints ({_KEY}, scope, fingerprint," # nosec B608 - " snapshot, source_sequence, baseline, publishable)" - ' VALUES (%s,%s,%s,%s,%s,%s,%s::jsonb,%s,\'{"health":"waiting"}\',0,FALSE,FALSE)' + " snapshot, source_sequence, baseline, publishable, selector_writer_floor)" + ' VALUES (%s,%s,%s,%s,%s,%s,%s::jsonb,%s,\'{"health":"waiting"}\',0,FALSE,FALSE,2)' f" ON CONFLICT ({_KEY}) DO NOTHING", # nosec B608 (*scope.checkpoint_key, json.dumps(asdict(scope)), "0" * 64), ) @@ -258,7 +266,8 @@ async def _write_on(self, connection: Any, checkpoint: StatusCheckpoint) -> None await connection.execute( "UPDATE reporting_status_scope_checkpoints SET scope=%s::jsonb, fingerprint=%s," " generation=%s, snapshot=%s::jsonb, next_due_at=%s, source_sequence=%s," - " baseline=%s, publishable=%s, initialized=TRUE" + " baseline=%s, publishable=%s, initialized=TRUE, selector_semantics_version=%s," + " selector_writer_floor=%s" f" WHERE {_WHERE}", # nosec B608 ( json.dumps(asdict(checkpoint.scope)), @@ -269,6 +278,8 @@ async def _write_on(self, connection: Any, checkpoint: StatusCheckpoint) -> None checkpoint.source_sequence, checkpoint.baseline, checkpoint.publishable, + checkpoint.selector_semantics_version, + checkpoint.selector_writer_floor, *checkpoint.scope.checkpoint_key, ), ) @@ -284,7 +295,15 @@ async def _apply_on( await self._lock_scopes_on(connection, snapshot) snapshot = settled_replay(snapshot) count = 0 - for scope in projection_scopes(snapshot): + rows = await ( + await connection.execute( + "SELECT scope FROM reporting_status_scope_checkpoints WHERE account_id=%s" + " ORDER BY account_id, consumer_namespace, delivery_config_id, version," + " scope_kind, obligation_namespace", + (snapshot.account_id,), + ) + ).fetchall() + for scope in (decode_status_scope(row[0]) for row in rows): row = await ( await connection.execute( f"SELECT {_CHECKPOINT} FROM reporting_status_scope_checkpoints WHERE {_WHERE}" # nosec B608 @@ -318,7 +337,8 @@ async def baseline(self, *, account_id: str) -> bool: ) ).fetchone() if row is not None and row[0]: - await self._account_on(connection, account_id) + if not await self._needs_rebuild_on(connection, account_id): + await self._account_on(connection, account_id) return False await connection.execute( "INSERT INTO reporting_status_accounts (account_id, policy) VALUES (%s,%s::jsonb)" @@ -338,6 +358,7 @@ async def baseline(self, *, account_id: str) -> bool: "UPDATE reporting_status_accounts SET baseline_complete=TRUE," " baseline_highwater=%s," " dirty_sequence=%s, baseline_at=%s, replay_lifecycles=%s::jsonb" + ", selector_target_version=2, selector_transition='complete'" " WHERE account_id=%s", (through, through, snapshot.as_of, _replay_storage(snapshot), account_id), ) @@ -355,7 +376,7 @@ async def baseline_ready(self, *, account_id: str) -> bool: (account_id,), ) ).fetchone() - if row is None or not row[0]: + if row is None or not row[0] or await self._needs_rebuild_on(connection, account_id): return False if row[1] != escalation_identity(self.escalation): raise ReportingNotificationError("status_policy_conflict") @@ -363,6 +384,108 @@ async def baseline_ready(self, *, account_id: str) -> bool: # account's current business health or waived issue occurrences. return True + async def _needs_rebuild_on(self, connection: Any, account_id: str) -> bool: + row = await ( + await connection.execute( + "SELECT selector_target_version <> 2 OR selector_transition <> 'complete'" + " OR EXISTS(SELECT 1 FROM reporting_status_scope_checkpoints c" + " WHERE c.account_id=a.account_id AND" + " (c.selector_semantics_version <> 2 OR c.selector_writer_floor <> 2))" + " FROM reporting_status_accounts a WHERE account_id=%s AND baseline_complete", + (account_id,), + ) + ).fetchone() + return bool(row and row[0]) + + async def _rebuild_on(self, connection: Any, account_id: str) -> StatusTurn: + if not await self._needs_rebuild_on(connection, account_id): + return StatusTurn(False) + row = await ( + await connection.execute( + "SELECT policy, selector_transition FROM reporting_status_accounts" + " WHERE account_id=%s FOR UPDATE", + (account_id,), + ) + ).fetchone() + expected = escalation_identity(self.escalation) + legacy = {k: v for k, v in expected.items() if k != "selector_semantics_version"} + if row[0] not in (expected, legacy): + raise ReportingNotificationError("status_policy_conflict") + if row[1] != "transitioning": + # Phase one commits a checkpoint-local writer floor. The guard + # never reads/locks an account, including for old due claimers. + await ( + await connection.execute( + "SELECT 1 FROM reporting_status_scope_checkpoints WHERE account_id=%s" + " ORDER BY account_id, consumer_namespace, delivery_config_id, version," + " scope_kind, obligation_namespace FOR UPDATE", + (account_id,), + ) + ).fetchall() + await connection.execute( + "UPDATE reporting_status_scope_checkpoints SET selector_writer_floor=2" + " WHERE account_id=%s AND selector_writer_floor <> 2", + (account_id,), + ) + await connection.execute( + "UPDATE reporting_status_accounts SET selector_target_version=2," + " selector_transition='transitioning', policy=%s::jsonb WHERE account_id=%s", + (json.dumps(expected), account_id), + ) + return StatusTurn(True) + # Phase two advances exactly one immutable boundary or deadline per + # transaction. A crash leaves the durable cursor at the last commit. + turn = await self._project_on(connection, account_id) + if turn.did_work: + return turn + snapshot = await settle_snapshot_on(self.ledger, connection, account_id=account_id) + through = await self._account_on(connection, account_id) + due = await ( + await connection.execute( + "SELECT min(next_due_at) FROM reporting_status_scope_checkpoints" + " WHERE account_id=%s AND next_due_at <= %s", + (account_id, snapshot.as_of), + ) + ).fetchone() + if due[0] is not None: + count = await self._apply_on( + connection, replace(snapshot, as_of=due[0]), through=through + ) + return StatusTurn(True, count) + count = await self._apply_on(connection, snapshot, through=through) + await connection.execute( + "UPDATE reporting_status_accounts SET selector_transition='complete'," + " replay_lifecycles=%s::jsonb WHERE account_id=%s", + (_replay_storage(snapshot), account_id), + ) + return StatusTurn(True, count) + + async def rebuild_one(self) -> StatusTurn: + """Discover incomplete C cutovers by index, without enumerating accounts.""" + expected = escalation_identity(self.escalation) + policies = ( + json.dumps(expected), + json.dumps({k: v for k, v in expected.items() if k != "selector_semantics_version"}), + ) + async with self.ledger._pool.connection() as connection: + row = await ( + await connection.execute( + "SELECT account_id FROM reporting_status_accounts WHERE baseline_complete" + " AND (selector_target_version <> 2 OR selector_transition <> 'complete')" + " AND policy IN (%s::jsonb,%s::jsonb)" + " UNION SELECT c.account_id FROM reporting_status_scope_checkpoints c" + " JOIN reporting_status_accounts a ON a.account_id=c.account_id" + " WHERE (c.selector_semantics_version <> 2 OR c.selector_writer_floor <> 2)" + " AND a.baseline_complete AND a.policy IN (%s::jsonb,%s::jsonb)" + " ORDER BY account_id LIMIT 1", + (*policies, *policies), + ) + ).fetchone() + if row is None: + return StatusTurn(False) + async with self._transaction(row[0]) as connection: + return await self._rebuild_on(connection, row[0]) + async def _project_on(self, connection: Any, account_id: str) -> StatusTurn: through = await self._account_on(connection, account_id) row = await ( @@ -394,6 +517,9 @@ async def _project_on(self, connection: Any, account_id: str) -> StatusTurn: async def project_one(self, *, account_id: str) -> StatusTurn: async with self._transaction(account_id) as connection: + rebuilt = await self._rebuild_on(connection, account_id) + if rebuilt.did_work: + return rebuilt return await self._project_on(connection, account_id) async def claim_due( @@ -401,7 +527,10 @@ async def claim_due( ) -> StatusDueLease | None: if lease_seconds <= 0: raise ValueError("lease_seconds must be positive") - async with self.ledger._pool.connection() as connection, connection.transaction(): + async with self._transaction(account_id) as connection: + if await self._needs_rebuild_on(connection, account_id): + return None + await self._account_on(connection, account_id) at = await database_now(connection, self.ledger._clock) row = await ( await connection.execute( @@ -452,6 +581,8 @@ async def _ack_on(self, connection: Any, lease: StatusDueLease) -> bool: async def complete_due(self, lease: StatusDueLease) -> StatusTurn: try: async with self._transaction(lease.scope.account_id) as connection: + if await self._needs_rebuild_on(connection, lease.scope.account_id): + return StatusTurn(False) await self._account_on(connection, lease.scope.account_id) if not await self._held_on(connection, lease): return StatusTurn(False) diff --git a/src/adcp/reporting/outbox/status_schema.py b/src/adcp/reporting/outbox/status_schema.py index d1f4c636f..c83a17dc6 100644 --- a/src/adcp/reporting/outbox/status_schema.py +++ b/src/adcp/reporting/outbox/status_schema.py @@ -12,6 +12,9 @@ REQUIRED_STATUS_OBJECTS: dict[str, dict[str, Any]] = json.loads( files("adcp.reporting.outbox").joinpath("required_status_schema.json").read_text() ) +REQUIRED_STATUS_SELECTOR_OBJECTS: dict[str, dict[str, Any]] = json.loads( + files("adcp.reporting.outbox").joinpath("required_status_selector_schema.json").read_text() +) async def validate_status_schema( @@ -46,3 +49,9 @@ async def validate_status_schema( else: continue raise ReportingNotificationError(f"status_schema_unready:{reason}:{key}") + if status: + if not REQUIRED_STATUS_SELECTOR_OBJECTS: + raise ReportingNotificationError("status_selector_schema_unready:manifest_missing") + for key, expected in REQUIRED_STATUS_SELECTOR_OBJECTS.items(): + if installed.get(key) != expected: + raise ReportingNotificationError(f"status_selector_schema_unready:{key}") diff --git a/src/adcp/reporting/outbox/status_service.py b/src/adcp/reporting/outbox/status_service.py index f82fa09e5..03275fca2 100644 --- a/src/adcp/reporting/outbox/status_service.py +++ b/src/adcp/reporting/outbox/status_service.py @@ -5,7 +5,7 @@ from typing import Protocol from adcp.reporting.ledger.notification_models import ReportingNotificationError -from adcp.reporting.outbox.status import StatusTurn +from adcp.reporting.outbox.status import StatusSelectorRebuildStore, StatusTurn from adcp.reporting.outbox.status_support import ReportingStatusSupport @@ -68,6 +68,14 @@ async def project_dirty_once(self, *, account_id: str) -> StatusTurn: raise ReportingNotificationError("status_projector_unavailable") return await self.support.projector.run_once(account_id=account_id) + async def rebuild_selector_once(self) -> StatusTurn: + """Indexed C cutover, including retained accounts outside current registrations.""" + self._open() + store = self.support.store + if not isinstance(store, StatusSelectorRebuildStore): + return StatusTurn(False) # Existing custom lifecycle protocols stay compatible. + return await store.rebuild_one() + async def sweep_due_once(self, *, account_id: str) -> StatusTurn: self._open(account_id) if self.support.sweeper is None: @@ -87,7 +95,7 @@ async def drain(self, *, max_turns: int = 100) -> int: if max_turns < 1: raise ValueError("max_turns must be positive") for turn in range(max_turns): - worked = False + worked = (await self.rebuild_selector_once()).did_work for account_id in sorted(set(self.support.account_ids)): worked = (await self.project_dirty_once(account_id=account_id)).did_work or worked worked = (await self.sweep_due_once(account_id=account_id)).did_work or worked diff --git a/src/adcp/reporting/revision_selection.py b/src/adcp/reporting/revision_selection.py new file mode 100644 index 000000000..a09365081 --- /dev/null +++ b/src/adcp/reporting/revision_selection.py @@ -0,0 +1,191 @@ +"""Strict, pure publication selection shared by seller and buyer projections. + +Validate the *whole* obligation history before choosing finality. Destination +state, timestamps and readability never choose a publication. In particular, +an official close does not supersede (or excuse damage in) the snapshot chain. +""" + +from __future__ import annotations + +from collections.abc import Sequence +from dataclasses import dataclass, field +from typing import Generic, Literal, Protocol, TypeVar + +__all__ = [ + "REPORTING_SELECTOR_VERSION", + "ReportingRevisionCorrupt", + "ReportingRevisionNotReady", + "ReportingRevisionSelected", + "ReportingRevisionSelection", + "RevisionHistoryEntry", + "select_reporting_revision", +] + +REPORTING_SELECTOR_VERSION = 2 + + +class _Revision(Protocol): + @property + def account_id(self) -> str: ... + + @property + def reporting_obligation_id(self) -> str: ... + + @property + def reporting_revision_id(self) -> str: ... + + @property + def finality(self) -> str: ... + + @property + def supersedes_reporting_revision_id(self) -> str | None: ... + + +R = TypeVar("R", bound=_Revision) + + +@dataclass(frozen=True, slots=True) +class RevisionHistoryEntry: + """Adapter for wire revisions whose ownership is established by their caller.""" + + account_id: str + reporting_obligation_id: str + reporting_revision_id: str + finality: str + supersedes_reporting_revision_id: str | None = None + + +@dataclass(frozen=True, slots=True) +class ReportingRevisionSelected(Generic[R]): + revision: R + kind: Literal["selected"] = field(default="selected", init=False) + + +@dataclass(frozen=True, slots=True) +class ReportingRevisionNotReady: + reason: Literal["empty_history", "official_required"] + kind: Literal["not_ready"] = field(default="not_ready", init=False) + + +@dataclass(frozen=True, slots=True) +class ReportingRevisionCorrupt: + # Closed diagnostics: no row data, credentials or provider errors. + reason: Literal[ + "ownership_mismatch", + "duplicate_revision_id", + "invalid_revision_identity", + "invalid_finality", + "missing_predecessor", + "cross_finality_edge", + "official_predecessor", + "forked_snapshot_history", + "disconnected_snapshot_history", + "revision_cycle", + "multiple_officials", + ] + kind: Literal["corrupt"] = field(default="corrupt", init=False) + + +ReportingRevisionSelection = ( + ReportingRevisionSelected[R] | ReportingRevisionNotReady | ReportingRevisionCorrupt +) + + +def select_reporting_revision( + revisions: Sequence[R], + *, + account_id: str, + reporting_obligation_id: str, + required_finality: str, +) -> ReportingRevisionSelection[R]: + """Linear-time, order-independent selection, including disconnected cycles. + + Only empty history or a missing required official is ordinary not-ready. + The caller supplies every retained revision belonging to the obligation, + without filtering by finality, readability, or existing materializations. + A selected unreadable revision remains selected: repair it, never fall back. + """ + if ( + type(account_id) is not str + or type(reporting_obligation_id) is not str + or any( + type(r.account_id) is not str + or type(r.reporting_obligation_id) is not str + or r.account_id != account_id + or r.reporting_obligation_id != reporting_obligation_id + for r in revisions + ) + ): + return ReportingRevisionCorrupt("ownership_mismatch") + if any( + type(r.reporting_revision_id) is not str + or not r.reporting_revision_id + or ( + r.supersedes_reporting_revision_id is not None + and type(r.supersedes_reporting_revision_id) is not str + ) + for r in revisions + ): + return ReportingRevisionCorrupt("invalid_revision_identity") + by_id = {r.reporting_revision_id: r for r in revisions} + if len(by_id) != len(revisions): + return ReportingRevisionCorrupt("duplicate_revision_id") + if ( + type(required_finality) is not str + or required_finality not in {"snapshot", "official"} + or any( + type(r.finality) is not str or r.finality not in {"snapshot", "official"} + for r in revisions + ) + ): + return ReportingRevisionCorrupt("invalid_finality") + if any( + r.supersedes_reporting_revision_id is not None + and r.supersedes_reporting_revision_id not in by_id + for r in revisions + ): + return ReportingRevisionCorrupt("missing_predecessor") + if any( + r.supersedes_reporting_revision_id is not None + and by_id[r.supersedes_reporting_revision_id].finality != r.finality + for r in revisions + ): + return ReportingRevisionCorrupt("cross_finality_edge") + if any( + r.finality == "official" and r.supersedes_reporting_revision_id is not None + for r in revisions + ): + return ReportingRevisionCorrupt("official_predecessor") + officials = [r for r in revisions if r.finality == "official"] + if len(officials) > 1: + return ReportingRevisionCorrupt("multiple_officials") + snapshots = [r for r in revisions if r.finality == "snapshot"] + successors: dict[str, str] = {} + for revision in snapshots: + predecessor = revision.supersedes_reporting_revision_id + if predecessor is not None: + if predecessor in successors: + return ReportingRevisionCorrupt("forked_snapshot_history") + successors[predecessor] = revision.reporting_revision_id + # Walk each component once. A unique-looking leaf cannot hide a cycle. + visited: set[str] = set() + for revision in snapshots: + path: set[str] = set() + node: str | None = revision.reporting_revision_id + while node is not None and node not in visited: + if node in path: + return ReportingRevisionCorrupt("revision_cycle") + path.add(node) + node = by_id[node].supersedes_reporting_revision_id + visited.update(path) + roots = [r for r in snapshots if r.supersedes_reporting_revision_id is None] + if snapshots and len(roots) != 1: + return ReportingRevisionCorrupt("disconnected_snapshot_history") + if not revisions: + return ReportingRevisionNotReady("empty_history") + if officials: + return ReportingRevisionSelected(officials[0]) + if required_finality == "official": + return ReportingRevisionNotReady("official_required") + leaf = next(r for r in snapshots if r.reporting_revision_id not in successors) + return ReportingRevisionSelected(leaf) diff --git a/src/adcp/reporting/service.py b/src/adcp/reporting/service.py index 50675297e..6d97c4e5f 100644 --- a/src/adcp/reporting/service.py +++ b/src/adcp/reporting/service.py @@ -781,21 +781,21 @@ def capability_block(self) -> dict[str, Any]: offerings[offering_id] = offering configurations = [binding.configuration for binding in self._bindings.values()] first = next(iter(self._bindings.values())).producer - extra: dict[str, Any] = { - "managed_delivery": self._materialization_worker is not None, - "reconciled_billing": self._reconciled_billing, - } - if self._reconciled_billing: - extra["receipt_task"] = "sync_reporting_receipts" - return first.advertised_reporting_delivery( + payload = first.advertised_reporting_delivery( consumer_status_task=self._consumer_status_enabled, offerings=[offerings[key] for key in sorted(offerings)], automated_recovery_window=max( item.automated_recovery_window for item in configurations ), status_retention_days=min(item.status_retention_days for item in configurations), - extra=extra, ) + # The service owns these installed-component declarations; they are not + # caller-provided producer extensions. + payload["managed_delivery"] = self._materialization_worker is not None + payload["reconciled_billing"] = self._reconciled_billing + if self._reconciled_billing: + payload["receipt_task"] = "sync_reporting_receipts" + return payload def inject_capabilities(self, response: Any) -> dict[str, Any]: """Merge the truthful reporting block into a base capability response.""" diff --git a/tests/conformance/reporting/_frozen_status_c.py b/tests/conformance/reporting/_frozen_status_c.py new file mode 100644 index 000000000..20e709f6e --- /dev/null +++ b/tests/conformance/reporting/_frozen_status_c.py @@ -0,0 +1,112 @@ +"""JSON-line control for the exact reviewed C artifact, imported with python -I. + +This file deliberately imports no workspace SDK before choosing the frozen +source. It is a test driver, not a production migration or recovery service. +""" + +import asyncio +import json +import sys +from dataclasses import asdict +from datetime import datetime +from pathlib import Path + + +def emit(value): + print(json.dumps(value, default=lambda v: v.isoformat()), flush=True) + + +async def read(): + return json.loads(await asyncio.to_thread(sys.stdin.readline)) + + +async def main(): + config = await read() + root = Path(config["source"]).resolve() + sys.path.insert(0, str(root / "src")) + from psycopg import AsyncConnection, Error + from psycopg_pool import AsyncConnectionPool + + import adcp.reporting.outbox.status_pg as module + from adcp.reporting.ledger import PgReportingReconciliationStore + from adcp.reporting.ledger.notification_models import ReportingNotificationError + from adcp.reporting.outbox import PgStatusNotificationStore + + assert Path(module.__file__).is_relative_to(root) + gate = None + now = datetime.fromisoformat(config["now"]) + + class HeldConnection(AsyncConnection): + async def execute(self, query, params=None, **kwargs): + nonlocal gate + result = await super().execute(query, params, **kwargs) + if ( + gate + and isinstance(query, str) + and query.startswith("UPDATE reporting_status_scope_checkpoints SET") + ): + if gate == "project" or "SET lease_token=" in query: + gate = None + emit({"held": True}) + destination_operation_1 = await read() + assert (destination_operation_1)["action"] == "release_hold" + return result + + async with AsyncConnectionPool( + config["conninfo"], + kwargs=config["kwargs"], + min_size=1, + max_size=1, + connection_class=HeldConnection, + open=False, + ) as pool: + ledger = PgReportingReconciliationStore(pool=pool, notifications=True, clock=lambda: now) + status = PgStatusNotificationStore(ledger) + leases = [] + emit({"origin": str(module.__file__)}) + while True: + command = await read() + action = command["action"] + account = command.get("account", "acct_a") + if "now" in command: + now = datetime.fromisoformat(command["now"]) + gate = command.get("hold") + try: + if action == "stop": + emit({"stopped": True}) + return + if action == "schema": + await status.create_schema() + result = True + elif action == "baseline": + result = await status.baseline(account_id=account) + elif action == "ready": + result = await status.baseline_ready(account_id=account) + elif action == "project": + result = asdict(await status.project_one(account_id=account)) + elif action == "claim": + lease = await status.claim_due(account_id=account, lease_seconds=60) + leases.append(lease) + result = None if lease is None else asdict(lease) + elif action == "complete": + result = asdict(await status.complete_due(leases[command.get("lease", -1)])) + elif action == "release": + result = await status.release_due(leases[command.get("lease", -1)]) + elif action == "source": + await ledger.set_revision_readable( + account_id=account, + reporting_revision_id=command["revision"], + readable=command["readable"], + ) + result = True + else: + raise AssertionError("unknown test command") + emit({"result": result}) + except ReportingNotificationError as exc: + emit({"error": exc.code}) + except Error as exc: + emit({"error": "database_fence", "sqlstate": exc.sqlstate}) + + +if __name__ == "__main__": + asyncio.run(main()) diff --git a/tests/conformance/reporting/_materializer_installed.py b/tests/conformance/reporting/_materializer_installed.py new file mode 100644 index 000000000..dc5f5c4f2 --- /dev/null +++ b/tests/conformance/reporting/_materializer_installed.py @@ -0,0 +1,188 @@ +"""Copied outside the checkout and executed with Python 3.10 -I against a wheel.""" + +import asyncio +import hashlib +import importlib +import importlib.util +import json +import sys +from datetime import datetime, timedelta, timezone +from importlib.resources import files +from pathlib import Path + + +async def main(): + config = json.load(sys.stdin) + assert sys.version_info[:2] == (3, 10) + assert importlib.util.find_spec("psycopg") is None + assert importlib.util.find_spec("psycopg_pool") is None + + assert "adcp.reporting.materializer" not in sys.modules + for name in ( + "adcp.reporting.materializer", + "adcp.reporting.revision_selection", + "adcp.reporting.ledger", + "adcp.reporting.outbox", + ): + module = importlib.import_module(name) + assert len(module.__all__) == len(set(module.__all__)) + for symbol in module.__all__: + getattr(module, symbol) + from adcp.reporting.ledger import ( + InMemoryReportingReconciliationStore, + ReportingConfiguration, + ReportingDeliveryScope, + ReportingObligationRecord, + ReportingRevisionRecord, + ReportingScheduleSpec, + derive_period, + revision_content_sha256, + ) + from adcp.reporting.materializer import ( + ReportingDestinationBinding, + ReportingMaterializationAttempt, + ReportingObligationDeliveryRecord, + reference_digest, + reference_verifier, + ) + + example_path = Path(config["example"]) + spec = importlib.util.spec_from_file_location("installed_example", example_path) + example = importlib.util.module_from_spec(spec) + sys.modules[spec.name] = example + spec.loader.exec_module(example) + verifier = reference_verifier() + actual = { + name: hashlib.sha256( + files("adcp.reporting.materializer").joinpath("assets", name).read_bytes() + ).hexdigest() + for name in config["assets"] + } + assert actual == config["assets"] + assert ( + files("adcp.reporting.ledger").joinpath("reporting_status_selector_version.sql").is_file() + ) + assert files("adcp.reporting.outbox").joinpath("required_status_selector_schema.json").is_file() + start = datetime(2026, 9, 1, tzinfo=timezone.utc) + schedule = ReportingScheduleSpec("PT1H", "PT1H", period_anchor=start) + period = derive_period(schedule, account_timezone="UTC", ordinal=0) + for count in (0, 501): + ledger = InMemoryReportingReconciliationStore() + configuration = ReportingConfiguration( + delivery_config_id="daily", + delivery_config_version=1, + account_id="account", + report_definition_id=verifier.key.report_definition_id, + reporting_profile=verifier.key.reporting_profile, + feed_purpose="analytics", + required_finality="snapshot", + schedule=schedule, + activated_at=start, + definition=verifier.key.definition, + ) + await ledger.put_configuration(configuration) + obligation = ReportingObligationRecord( + reporting_obligation_id="obligation", + account_id="account", + delivery_config_id="daily", + delivery_config_version=1, + report_definition_id=verifier.key.report_definition_id, + reporting_profile=verifier.key.reporting_profile, + feed_purpose="analytics", + period=period, + scope_resolved_at=period.end, + media_buy_ids=("buy",), + required_finality="snapshot", + automated_recovery_deadline_at=period.expected_at + timedelta(hours=1), + schedule=schedule, + definition=verifier.key.definition, + created_at=period.end, + currency="USD", + ) + await ledger.commit_obligation(obligation) + binding = ReportingDestinationBinding( + configuration.generation_key, + "https://buyer.example.test/agents/reporting", + "reference-destination", + "trusted-binding", + "file_transfer", + "reference-memory", + "canonical_digest", + "delivery_only", + "analytics", + 400, + start, + "jsonl", + ("reference-v1",), + ) + await ledger.put_destination_binding(binding) + scope = ReportingDeliveryScope( + configuration.generation_key, binding.consumer_id, obligation.reporting_obligation_id + ) + delivery = ReportingObligationDeliveryRecord( + scope, "USD", period.end + timedelta(days=400), period.end + ) + await ledger.bind_obligation_delivery(delivery) + rows = [ + {"row_id": f"{i:06d}", "impressions": i, "spend": "1.25", "currency": "USD"} + for i in range(count) + ] + _, totals = verifier.canonicalize(rows) + pairs = tuple((t.name, t.value) for t in totals) + revision = ReportingRevisionRecord( + "revision", + "account", + obligation.reporting_obligation_id, + "snapshot", + revision_content_sha256( + reporting_revision_id="revision", + row_count=count, + control_totals=pairs, + reporting_rows=rows, + control_total_evidence=totals, + ), + count, + pairs, + period.end, + period.end, + period.end, + canonical_content_digest=reference_digest(verifier, rows), + managed_control_totals=totals, + ) + await ledger.commit_revision(revision, rows) + attempt = ReportingMaterializationAttempt( + scope, revision.reporting_revision_id, "materialization", 1, period.end + ) + await ledger.commit_materialization_attempt(attempt) + destination = example.development_destination(binding) + results = [] + for _ in range(2): + results.append( + await destination.verify_revision( + reader=ledger, + binding=binding, + delivery=delivery, + obligation=obligation, + revisions=(revision,), + attempt=attempt, + deadline_at=datetime.now(timezone.utc) + timedelta(seconds=60), + cancel=asyncio.Event(), + ) + ) + assert all(result.verification.row_count == count for result in results) + assert results[0].request.external_id == results[1].request.external_id + assert destination.writer.write_effects == 1 and not destination.writer.production_eligible + assert destination.writer.open_count == destination.writer.close_count == 4 + workspace = Path(config["workspace"]).resolve() + assert all(not Path(path).resolve().is_relative_to(workspace) for path in sys.path) + assert all( + not Path(module.__file__).resolve().is_relative_to(workspace) + for name, module in sys.modules.items() + if name == "adcp" or name.startswith("adcp.") + if getattr(module, "__file__", None) + ) + assert not any(name.startswith("psycopg") for name in sys.modules) + print(json.dumps({"python": "3.10", "rows": [0, 501], "installed": True, "assets": actual})) + + +asyncio.run(main()) diff --git a/tests/conformance/reporting/_materializer_support.py b/tests/conformance/reporting/_materializer_support.py new file mode 100644 index 000000000..dba8a9054 --- /dev/null +++ b/tests/conformance/reporting/_materializer_support.py @@ -0,0 +1,178 @@ +"""Frozen content shared by the B1 pure, memory and installed-artifact gates.""" + +import asyncio +from dataclasses import dataclass, replace +from datetime import datetime, timedelta, timezone + +from adcp.reporting.ledger import ( + InMemoryReportingReconciliationStore, + ReportingDeliveryScope, + ReportingRevisionRecord, + revision_content_sha256, +) +from adcp.reporting.materializer import ( + ReferenceReportingDestinationWriter, + ReferenceReportingResolver, + ReportingDestinationBinding, + ReportingDestinationIO, + ReportingIOContext, + ReportingMaterializationAttempt, + ReportingObligationDeliveryRecord, + ReportingRevisionVerifierRegistry, + reference_digest, + reference_verifier, +) + +from ._generation_support import END, START, configuration, obligation_for + + +def io_context(seconds=30): + return ReportingIOContext( + datetime.now(timezone.utc) + timedelta(seconds=seconds), asyncio.Event() + ) + + +def reference_rows(count): + return [ + { + "row_id": f"{i:06d}", + "impressions": i % 3, + "spend": "1.25", + "currency": "USD", + "details": {"active": True, "values": [1, None, "é", "e\u0301"]}, + } + for i in range(count) + ] + + +@dataclass +class Case: + store: object + verifier: object + registry: object + binding: object + delivery: object + obligation: object + revision: object + attempt: object + rows: object + prepared: object + writer: object + resolver: object + io: object + + async def prepare(self, **changes): + args = dict( + key=self.verifier.key, + binding=self.binding, + delivery=self.delivery, + obligation=self.obligation, + revisions=(self.revision,), + attempt=self.attempt, + reader=self.store, + context=io_context(), + ) + args.update(changes) + return await self.registry.prepare(**args) + + +async def materializer_case( + count=1, + *, + capability=None, + consumer="https://buyer.example.test/agents/reporting", + account="acct_a", + store=None, + finality="snapshot", +): + verifier = reference_verifier(capability) + registry = ReportingRevisionVerifierRegistry((verifier,)) + config = replace( + configuration(account), + definition=verifier.key.definition, + report_definition_id=verifier.key.report_definition_id, + required_finality=finality, + ) + store = store or InMemoryReportingReconciliationStore(notifications=True) + await store.put_configuration(config) + obligation = await store.commit_obligation(obligation_for(config)) + cap = verifier.key.capability + binding = ReportingDestinationBinding( + config.generation_key, + consumer, + "destination", + "trusted-reference-binding", + cap.method, + cap.transport, + cap.verification_profile, + "delivery_only", + "analytics", + 400, + START, + cap.format, + ("reference-v1",), + "delivered" if cap.method == "warehouse_materialization" else "available", + ) + await store.put_destination_binding(binding) + delivery = ReportingObligationDeliveryRecord( + ReportingDeliveryScope(config.generation_key, consumer, obligation.reporting_obligation_id), + "USD", + END + timedelta(days=400), + END, + ) + await store.bind_obligation_delivery(delivery) + rows = reference_rows(count) + _, totals = verifier.canonicalize(rows) + pairs = tuple((t.name, t.value) for t in totals) + revision = ReportingRevisionRecord( + "revision-first", + account, + obligation.reporting_obligation_id, + finality, + revision_content_sha256( + reporting_revision_id="revision-first", + row_count=count, + control_totals=pairs, + reporting_rows=rows, + control_total_evidence=totals, + ), + count, + pairs, + END, + END, + END, + finality_basis="source_final" if finality == "official" else None, + finality_policy_id="reference-final" if finality == "official" else None, + finalized_at=END if finality == "official" else None, + canonical_content_digest=reference_digest(verifier, rows), + managed_control_totals=totals, + ) + await store.commit_revision(revision, rows) + attempt = ReportingMaterializationAttempt( + delivery.scope, + revision.reporting_revision_id, + "materialization-first", + 1, + END + timedelta(seconds=1), + ) + await store.commit_materialization_attempt(attempt) + writer = ReferenceReportingDestinationWriter((cap,)) + resolver = ReferenceReportingResolver(writer, registry, (binding,)) + io = ReportingDestinationIO(registry, resolver) + case = Case( + store, + verifier, + registry, + binding, + delivery, + obligation, + revision, + attempt, + rows, + None, + writer, + resolver, + io, + ) + case.prepared = await case.prepare() + return case diff --git a/tests/conformance/reporting/_reliable_support.py b/tests/conformance/reporting/_reliable_support.py index 872da1e8e..248cb3e3e 100644 --- a/tests/conformance/reporting/_reliable_support.py +++ b/tests/conformance/reporting/_reliable_support.py @@ -719,10 +719,27 @@ async def publication_records( ) await harness.store.put_destination_binding(binding) await harness.store.bind_obligation_delivery(delivery) - rows = await harness.store.read_revision_rows( - account_id=obligation.account_id, reporting_revision_id=revision.reporting_revision_id - ) - payload = b"".join(canonical_json_utf8_v1(row) + b"\n" for row in rows.rows) + all_rows = [] + cursor = None + seen = set() + while True: + page = await harness.store.read_revision_rows( + account_id=obligation.account_id, + reporting_revision_id=revision.reporting_revision_id, + cursor=cursor, + limit=500, + ) + assert page.reporting_revision_id == revision.reporting_revision_id + assert page.total_count == revision.row_count + assert page.has_more == (page.cursor is not None) + all_rows.extend(page.rows) + if not page.has_more: + break + assert page.rows and page.cursor not in seen + seen.add(page.cursor) + cursor = page.cursor + assert len(all_rows) == revision.row_count + payload = b"".join(canonical_json_utf8_v1(row) + b"\n" for row in all_rows) digest = await harness.destination.write( obligation.account_id, revision.reporting_revision_id, payload ) diff --git a/tests/conformance/reporting/test_reporting_materializer_contracts.py b/tests/conformance/reporting/test_reporting_materializer_contracts.py new file mode 100644 index 000000000..e164315c5 --- /dev/null +++ b/tests/conformance/reporting/test_reporting_materializer_contracts.py @@ -0,0 +1,248 @@ +"""Public closed contracts, URL consumers, import safety and real capability gates.""" + +import asyncio +import importlib +import inspect +from dataclasses import replace +from datetime import datetime, timedelta, timezone +from typing import get_args, get_type_hints + +import pytest +from pydantic import TypeAdapter + +from adcp.reporting.ledger import ( + InMemoryReportingLedgerStore, + ProducerOfferings, + ReportingDeliveryPrincipal, + ReportingProducer, +) +from adcp.reporting.ledger.delivery_models import MaterializationFailure +from adcp.reporting.materializer import ( + ReferenceReportingDestinationWriter, + ReportingDestinationPage, + ReportingIOContext, + ReportingWriterError, + ReportingWriterFailure, + ReportingWriterFailureCode, + reference_verifier, +) + +from ._generation_support import UncalledSource +from ._materializer_support import io_context, materializer_case +from ._reliable_support import NotificationHarness, reliable_factory + + +@pytest.mark.parametrize( + "key", + [ + "managed_delivery", + "reconciled_billing", + "reporting.delivery_ready", + "readiness_notification", + "status_notification", + "ledger_notification", + "supports_webhook_activity", + "consumer_status_task", + "receipt_task", + "reconciliation_task", + "status_task", + "delivery_task", + ], +) +@pytest.mark.parametrize("value", [False, True]) +def test_producer_extra_cannot_inject_sdk_owned_readiness(key, value): + producer = ReportingProducer( + source=UncalledSource(), offerings=ProducerOfferings(), store=InMemoryReportingLedgerStore() + ) + with pytest.raises(ValueError, match="SDK-owned"): + producer.advertised_reporting_delivery( + consumer_status_task=False, + offerings=(), + automated_recovery_window=timedelta(hours=1), + status_retention_days=30, + extra={key: value}, + ) + + +@pytest.mark.parametrize("backend", ["memory", "postgres"]) +async def test_real_outbox_capability_path_never_promotes_a_verified_reference_writer(backend): + async with reliable_factory(backend, notifications=True) as h: + case = await materializer_case(store=h.store) + locator = await case.io.write(case.prepared, context=io_context()) + await case.io.verify(case.prepared, locator, context=io_context()) + fields = ( + await NotificationHarness(h) + .worker() + .advertised_notifications(h.store, account_id="acct_a", ready_scope=case.delivery.scope) + ) + assert fields == { + "ledger_notification": "reporting.ledger_changed", + "supports_webhook_activity": False, + } + assert not set(fields).intersection( + { + "managed_delivery", + "reconciled_billing", + "reporting.delivery_ready", + "readiness_notification", + } + ) + assert case.writer.production_eligible is False + + +async def test_url_consumer_round_trips_all_binding_resolver_reconciliation_and_delivery_paths(): + case = await materializer_case() + principal = case.binding.principal + assert "://" in principal.consumer_id + assert ( + TypeAdapter(ReportingDeliveryPrincipal).validate_json( + TypeAdapter(ReportingDeliveryPrincipal).dump_json(principal) + ) + == principal + ) + frozen = await case.store.get_destination_binding( + caller=principal, generation_key=case.binding.generation_key + ) + assert frozen == case.binding + assert case.prepared.request.principal == principal + assert (await case.store.get_obligation_delivery(case.delivery.scope)) == case.delivery + page = await case.store.read_reconciliation_changes(caller=principal) + assert page.caller == principal and page.changes + snapshot = await case.store.read_status_snapshot(account_id=principal.account_id) + assert principal.consumer_id in snapshot.consumer_ids + locator = await case.io.write(case.prepared, context=io_context()) + verified = await case.io.verify(case.prepared, locator, context=io_context()) + assert verified.request.principal == principal + + +def test_new_failures_do_not_change_reviewed_persisted_failure_enum(): + assert get_args(MaterializationFailure) == ( + "WRITE_FAILED", + "VERIFICATION_FAILED", + "CONTENT_CORRUPT", + "RESOURCE_UNAVAILABLE", + ) + assert "CURRENT_REVISION_CHANGED" in get_args(ReportingWriterFailureCode) + assert "CURRENT_REVISION_CHANGED" not in get_args(MaterializationFailure) + for code in get_args(ReportingWriterFailureCode): + assert str(ReportingWriterError(ReportingWriterFailure(code))) == code + for changes in ( + {"code": "provider prose"}, + {"retry": "eventually"}, + {"effect": "probably_written"}, + {"retry_after_seconds": True}, + {"retry_after_seconds": -1}, + ): + with pytest.raises(ValueError): + ReportingWriterFailure(**{"code": "WRITE_FAILED", **changes}) + + +def test_reference_writer_production_flag_is_not_configurable_or_subclass_promotable(): + with pytest.raises(TypeError): + ReferenceReportingDestinationWriter((), production_eligible=True) + writer = ReferenceReportingDestinationWriter(()) + with pytest.raises(AttributeError): + writer.production_eligible = True + with pytest.raises(TypeError): + type( + "PromotedReference", + (ReferenceReportingDestinationWriter,), + {"production_eligible": True}, + ) + + +def test_readback_page_repr_never_includes_unverified_provider_body(): + body = b"https://provider.example.test/private?token=credential-sentinel" + page = ReportingDestinationPage("revision", (body,), 1, False, None, "jsonl", "producer") + assert "credential-sentinel" not in str(page) + repr(page) + assert "https://provider" not in str(page) + repr(page) + + +@pytest.mark.parametrize( + "field", ["report_definition_uri", "schema_uri", "schema_dialect", "schema_version"] +) +def test_public_verification_key_rejects_credentials_in_every_definition_coordinate(field): + verifier = reference_verifier() + with pytest.raises(ValueError) as caught: + replace( + verifier.key, + definition=replace( + verifier.key.definition, + **{field: "https://provider.example.test/data?token=do-not-expose"}, + ), + ) + assert "do-not-expose" not in str(caught.value) + repr(caught.value) + + +@pytest.mark.parametrize( + "field", ["report_definition_uri", "schema_uri", "schema_dialect", "schema_version"] +) +async def test_a_retained_definition_the_key_cannot_express_is_a_closed_binding_mismatch(field): + # ``ReportingDefinitionBinding`` is Core's loosely constrained retained + # record: it applies none of the verification key's HTTPS/digest/identifier + # screens, so real obligations carry values this key cannot express. Every + # public entry point must answer that with the closed BINDING_MISMATCH its + # callers already catch, never a raw ValueError through the failure contract. + case = await materializer_case() + loose = replace( + case.obligation.definition, + **{field: "https://provider.example.test/data?token=do-not-expose"}, + ) + obligation = replace(case.obligation, definition=loose) + with pytest.raises(ReportingWriterError) as caught: + await case.prepare(obligation=obligation) + assert caught.value.failure.code == "BINDING_MISMATCH" + assert case.writer.open_count == 0 + + locator = await case.io.write(case.prepared, context=io_context()) + prepared = replace(case.prepared, obligation=obligation) + for operation in ( + case.io.write(prepared, context=io_context()), + case.io.verify(prepared, locator, context=io_context()), + ): + with pytest.raises(ReportingWriterError) as caught: + await operation + assert caught.value.failure.code == "BINDING_MISMATCH" + assert "do-not-expose" not in str(caught.value) + repr(caught.value) + assert case.writer.open_count == case.writer.close_count == 1 + + +async def test_service_heartbeat_is_a_checkpoint_only_and_fences_before_source_read(): + case = await materializer_case() + calls = 0 + + class Heartbeat: + async def checkpoint(self): + nonlocal calls + calls += 1 + raise ReportingWriterError( + ReportingWriterFailure("LEASE_LOST", "same_identity", "not_started") + ) + + context = ReportingIOContext( + datetime.now(timezone.utc) + timedelta(seconds=10), asyncio.Event(), Heartbeat() + ) + with pytest.raises(ReportingWriterError, match="LEASE_LOST"): + await case.prepare(context=context) + assert calls == 1 and case.writer.open_count == 0 + + +def test_curated_all_exports_resolve_and_do_not_add_materializer_sql(): + for name in ( + "adcp.reporting.materializer", + "adcp.reporting.revision_selection", + "adcp.reporting.ledger", + "adcp.reporting.outbox", + ): + module = importlib.import_module(name) + assert len(module.__all__) == len(set(module.__all__)) + for public in module.__all__: + assert getattr(module, public) is not None + module = importlib.import_module("adcp.reporting.materializer") + assert not any( + name.endswith(("Coordinator", "Service", "Lease", "WorkQueue")) for name in module.__all__ + ) + assert inspect.isclass(module.ReportingDestinationSession) + assert get_type_hints(module.ReportingDestinationSession.write)["content"] is ( + module.ReportingPreparedRevision + ) diff --git a/tests/conformance/reporting/test_reporting_materializer_lifecycle.py b/tests/conformance/reporting/test_reporting_materializer_lifecycle.py new file mode 100644 index 000000000..636b19b41 --- /dev/null +++ b/tests/conformance/reporting/test_reporting_materializer_lifecycle.py @@ -0,0 +1,373 @@ +"""Adversarial asynchronous ownership: safe errors, partial opens and cancellation.""" + +import asyncio +import pickle +import traceback +from dataclasses import replace + +import pytest + +from adcp.reporting.materializer import ( + ReportingDestinationIO, + ReportingDestinationSession, + ReportingWriterCapability, + ReportingWriterError, + ReportingWriterFailure, +) +from adcp.reporting.materializer.reference import _Session + +from ._materializer_support import io_context, materializer_case + +SECRET = "https://provider.example.test/private?token=credential-sentinel&signature=provider-body" + + +class Phases: + def __init__(self, case, tmp_path, stage, mode): + self.case, self.tmp_path, self.stage, self.mode = case, tmp_path, stage, mode + self.entered = asyncio.Event() + self.release = asyncio.Event() + self.sessions = [] + self.spools = [] + self.stream_closes = 0 + + async def hit(self, stage): + if stage != self.stage: + return + self.entered.set() + if self.mode == "error": + try: + raise RuntimeError(SECRET) + except RuntimeError as exc: + raise ValueError(SECRET) from exc + await self.release.wait() + + def resolve(self, request, *, phase, context): + owner = self + + class Session(_Session): + async def _open(self): + await super()._open() + self._credential = SECRET + self.spool = owner.tmp_path / f"{len(owner.spools)}.spool" + self.spool.write_text(SECRET) + owner.spools.append(self.spool) + await owner.hit("resolve" if self.phase == "write" else "readback-resolve") + + async def _close(self): + try: + await owner.hit("close") + finally: + if hasattr(self, "spool"): + self.spool.unlink(missing_ok=True) + await super()._close() + + async def write(self, content): + await owner.hit("write-before") + locator = await super().write(content) + await owner.hit("write-after") + return locator + + async def read_rows(self, locator, *, cursor, limit): + await owner.hit("rows") + return await super().read_rows(locator, cursor=cursor, limit=limit) + + async def read_manifest(self, locator): + await owner.hit("manifest") + return await super().read_manifest(locator) + + async def observe_native_version(self, locator): + self.native_reads = getattr(self, "native_reads", 0) + 1 + await owner.hit("native-before" if self.native_reads == 1 else "native-after") + return await super().observe_native_version(locator) + + async def list_objects(self, locator): + await owner.hit("inventory") + return await super().list_objects(locator) + + async def read_object(self, locator, *, object_ref): + try: + await owner.hit("object") + async for chunk in super().read_object(locator, object_ref=object_ref): + yield chunk + finally: + owner.stream_closes += 1 + await owner.hit("stream-close") + + session = Session(self.case.resolver, request, phase, context) + self.sessions.append(session) + return session + + +def safe_exception(error): + rendered = ( + str(error) + + repr(error) + + "".join(traceback.format_exception(type(error), error, error.__traceback__)) + ) + assert ( + SECRET not in rendered + and "credential-sentinel" not in rendered + and "provider-body" not in rendered + ) + assert error.__cause__ is None + if isinstance(error, asyncio.CancelledError): + # Python 3.10 Task adds a clean CancelledError context even to an + # unconditionally canceled coroutine. Inspect the entire chain: no + # provider exception or cancellation message may survive that wrapper. + seen = set() + while error is not None: + assert id(error) not in seen + seen.add(id(error)) + assert type(error) is asyncio.CancelledError and error.args == () + assert error.__cause__ is None + error = error.__context__ + else: + assert error.__context__ is None + + +@pytest.mark.parametrize( + "stage", + [ + "resolve", + "readback-resolve", + "write-before", + "write-after", + "rows", + "manifest", + "inventory", + "object", + "close", + "stream-close", + "native-before", + "native-after", + ], +) +@pytest.mark.parametrize("mode", ["cancel", "signal", "timeout", "error"]) +async def test_each_io_phase_closes_once_redacts_errors_and_leaves_no_tasks_or_spools( + tmp_path, caplog, stage, mode +): + capability = ( + ReportingWriterCapability( + "dataset_share", + "reference-memory", + None, + "canonical_digest", + "representative_consumer", + "native_version", + "sha256", + "conditional_create", + ) + if stage.startswith("native") + else None + ) + case = await materializer_case(capability=capability) + locator = await case.io.write(case.prepared, context=io_context()) + owner = Phases(case, tmp_path, stage, mode) + io = ReportingDestinationIO(case.registry, owner) + context = io_context(0.5 if mode == "timeout" else 20) + if stage in {"close", "stream-close"}: + context = replace(context, close_timeout_seconds=0.08 if mode == "timeout" else 5) + before = set(asyncio.all_tasks()) + writing = stage in {"resolve", "write-before", "write-after", "close"} + call = ( + io.write(case.prepared, context=context) + if writing + else io.verify(case.prepared, locator, context=context) + ) + task = asyncio.create_task(call) + await asyncio.wait_for(owner.entered.wait(), 10) + if mode == "cancel": + task.cancel(SECRET) + await asyncio.sleep(0) + task.cancel(SECRET) # Repeated cancellation cannot abandon cleanup. + elif mode == "signal": + context.cancel.set() + if mode in {"cancel", "signal"} and stage in {"close", "stream-close"}: + await asyncio.sleep(0) + owner.release.set() + expected = asyncio.CancelledError if mode in {"cancel", "signal"} else ReportingWriterError + with pytest.raises(expected) as caught: + await asyncio.wait_for(task, 10) + safe_exception(caught.value) + assert all(session._closed and session._credential is None for session in owner.sessions) + assert case.writer.open_count == case.writer.close_count == 2 + assert all(not p.exists() for p in owner.spools) + assert list(tmp_path.iterdir()) == [] + assert not (set(asyncio.all_tasks()) - before) + assert SECRET not in caplog.text + if stage in {"object", "stream-close"}: + assert owner.stream_closes == 1 + for session in owner.sessions: + await session.aclose() # A repeated close is physically idle. + assert case.writer.close_count == 2 + + +@pytest.mark.parametrize("mode", ["cancel", "timeout", "error"]) +async def test_source_read_cancellation_removes_its_spool_before_any_resolver_io(tmp_path, mode): + case = await materializer_case() + owner = Phases(case, tmp_path, "source", mode) + path = tmp_path / "source.spool" + closed = 0 + + class Reader: + async def read_revision_rows(self, **kwargs): + nonlocal closed + path.write_text(SECRET) + try: + await owner.hit("source") + return await case.store.read_revision_rows(**kwargs) + finally: + path.unlink() + closed += 1 + + before = set(asyncio.all_tasks()) + task = asyncio.create_task( + case.prepare(reader=Reader(), context=io_context(0.05 if mode == "timeout" else 20)) + ) + await asyncio.wait_for(owner.entered.wait(), 5) + if mode == "cancel": + task.cancel(SECRET) + with pytest.raises( + asyncio.CancelledError if mode == "cancel" else ReportingWriterError + ) as caught: + await task + safe_exception(caught.value) + assert closed == 1 and not path.exists() and case.writer.open_count == 0 + assert not (set(asyncio.all_tasks()) - before) + + +@pytest.mark.parametrize("phase", ["open", "write"]) +@pytest.mark.parametrize("mode", ["cancel", "signal"]) +async def test_cancellation_during_failed_operation_cleanup_takes_precedence(tmp_path, phase, mode): + case = await materializer_case() + closing, release = asyncio.Event(), asyncio.Event() + spool = tmp_path / "failure.spool" + + class Session(_Session): + async def _open(self): + await super()._open() + spool.write_text(SECRET) + if phase == "open": + raise ValueError(SECRET) + + async def write(self, content): + raise ValueError(SECRET) + + async def _close(self): + closing.set() + try: + await release.wait() + finally: + spool.unlink(missing_ok=True) + await super()._close() + + class Resolver: + def resolve(self, request, *, phase, context): + return Session(case.resolver, request, phase, context) + + before = set(asyncio.all_tasks()) + context = io_context() + task = asyncio.create_task( + ReportingDestinationIO(case.registry, Resolver()).write(case.prepared, context=context) + ) + await asyncio.wait_for(closing.wait(), 5) + if mode == "cancel": + task.cancel(SECRET) + await asyncio.sleep(0) + task.cancel(SECRET) + else: + context.cancel.set() + release.set() + with pytest.raises(asyncio.CancelledError) as caught: + await task + safe_exception(caught.value) + assert not spool.exists() and case.writer.open_count == case.writer.close_count == 1 + assert not (set(asyncio.all_tasks()) - before) + + +async def test_invalid_locator_after_write_retains_unknown_effect_and_original_identity(): + case = await materializer_case() + + class Session(_Session): + async def write(self, content): + locator = await super().write(content) + return replace(locator, external_id="other-tenant-identity") + + class Resolver: + def resolve(self, request, *, phase, context): + return Session(case.resolver, request, phase, context) + + with pytest.raises(ReportingWriterError) as caught: + await ReportingDestinationIO(case.registry, Resolver()).write( + case.prepared, context=io_context() + ) + assert caught.value.failure == ReportingWriterFailure( + "BINDING_MISMATCH", "same_identity", "unknown" + ) + safe_exception(caught.value) + assert case.writer.write_effects == 1 and case.writer.open_count == case.writer.close_count == 1 + + +@pytest.mark.parametrize("cancel", [False, True]) +async def test_resolver_factory_never_exposes_provider_exception_or_cancel_message(cancel): + case = await materializer_case() + + class BrokenResolver: + def resolve(self, *args, **kwargs): + raise asyncio.CancelledError(SECRET) if cancel else ValueError(SECRET) + + with pytest.raises(asyncio.CancelledError if cancel else ReportingWriterError) as caught: + await ReportingDestinationIO(case.registry, BrokenResolver()).write( + case.prepared, context=io_context() + ) + safe_exception(caught.value) + assert case.writer.open_count == 0 + + +async def test_session_binding_is_checked_before_open_and_lifecycle_is_sdk_owned(tmp_path): + case = await materializer_case() + owner = Phases(case, tmp_path, "unused", "error") + + class WrongResolver: + def resolve(self, request, *, phase, context): + return owner.resolve( + replace(request, destination_ref="wrong-destination"), phase=phase, context=context + ) + + with pytest.raises(ReportingWriterError, match="BINDING_MISMATCH"): + await ReportingDestinationIO(case.registry, WrongResolver()).write( + case.prepared, context=io_context() + ) + assert case.writer.open_count == 0 and case.writer.close_count == 1 + session = owner.sessions[0] + assert str(session) == repr(session) == "" + with pytest.raises(TypeError, match="cannot be persisted"): + pickle.dumps(session) + for name in ("request", "phase", "context"): + with pytest.raises(AttributeError): + setattr(session, name, SECRET) + for name in ("__repr__", "__aenter__", "__aexit__", "aclose", "__reduce__"): + with pytest.raises(TypeError, match="belong to the SDK"): + type("UnsafeSession", (ReportingDestinationSession,), {name: lambda *args: SECRET}) + + +async def test_closed_failures_preserve_known_failure_and_unknown_identity_contract(): + async def known(): + raise ReportingWriterError( + ReportingWriterFailure("WRITE_FAILED", "new_attempt", "not_started") + ) + + async def connection_loss(): + raise ConnectionError(SECRET) + + for operation, expected in ( + (known, ("WRITE_FAILED", "new_attempt", "not_started")), + (connection_loss, ("RESOURCE_UNAVAILABLE", "same_identity", "unknown")), + ): + with pytest.raises(ReportingWriterError) as caught: + await io_context().run(operation, effect="unknown") + record = caught.value.failure + assert (record.code, record.retry, record.effect) == expected + safe_exception(caught.value) + with pytest.raises(ValueError): + ReportingWriterFailure("RESOURCE_UNAVAILABLE", "new_attempt", "unknown") diff --git a/tests/conformance/reporting/test_reporting_materializer_packaging.py b/tests/conformance/reporting/test_reporting_materializer_packaging.py new file mode 100644 index 000000000..007a279bf --- /dev/null +++ b/tests/conformance/reporting/test_reporting_materializer_packaging.py @@ -0,0 +1,129 @@ +"""Non-editable VCS/sdist wheels, actual Python 3.10, strict adopter and no PG.""" + +import hashlib +import json +import os +import shutil +import sys +import tarfile +import zipfile +from pathlib import Path + +import pytest + +from . import test_reporting_notification_packaging as distribution + +built_distribution = distribution.built_distribution +ROOT = distribution.ROOT +run_step = distribution.run_step +ASSETS = ROOT / "src/adcp/reporting/materializer/assets" + + +@pytest.fixture(scope="module") +def b1_wheels(built_distribution): + path, sdist_wheel, source = built_distribution + direct = path / "vcs-wheel" + # This build starts in the actual VCS checkout, including its build hook. + run_step( + [sys.executable, "-m", "build", "--wheel", "--outdir", str(direct), str(ROOT)], + label="b1-vcs-wheel", + cwd=ROOT, + timeout=180, + ) + vcs_wheel = next(direct.glob("*.whl")) + expected = {f.name: f.read_bytes() for f in ASSETS.glob("*.json")} + with ( + zipfile.ZipFile(vcs_wheel) as vcs, + zipfile.ZipFile(sdist_wheel) as wheel, + tarfile.open(source) as tar, + ): + prefix = tar.getnames()[0].split("/")[0] + for name, raw in expected.items(): + member = f"adcp/reporting/materializer/assets/{name}" + assert vcs.read(member) == wheel.read(member) == raw + assert tar.extractfile(f"{prefix}/src/{member}").read() == raw + for relative in ( + "ledger/reporting_status_selector_version.sql", + "outbox/required_status_selector_schema.json", + ): + assert ( + vcs.read(f"adcp/reporting/{relative}") + == wheel.read(f"adcp/reporting/{relative}") + == (ROOT / "src/adcp/reporting" / relative).read_bytes() + ) + assert not any( + "reporting_materializer_" in name and name.endswith(".sql") for name in vcs.namelist() + ) + return ( + path, + {"vcs": vcs_wheel, "sdist": sdist_wheel}, + {name: hashlib.sha256(raw).hexdigest() for name, raw in expected.items()}, + ) + + +@pytest.mark.parametrize("kind", ["vcs", "sdist"]) +def test_python310_installed_wheel_exports_verifier_reference_and_strict_adopter(b1_wheels, kind): + path, wheels, hashes = b1_wheels + interpreter = os.environ.get("ADCP_PYTHON310") or ( + sys.executable if sys.version_info[:2] == (3, 10) else None + ) + if interpreter is None: + pytest.skip( + "Python 3.10 matrix job runs this gate; ADCP_PYTHON310 enables it on other hosts" + ) + environment = path / f"b1-python310-{kind}" + run_step( + [interpreter, "-m", "venv", str(environment)], + label=f"b1-{kind}-python310-environment", + cwd=path, + ) + python = environment / "bin/python" + installer = ( + [shutil.which("uv"), "pip", "install", "--python", str(python)] + if shutil.which("uv") + else [str(python), "-m", "pip", "install"] + ) + run_step( + [*installer, str(wheels[kind]), "mypy==1.20.2"], + label=f"b1-{kind}-noneditable-install", + cwd=path, + timeout=180, + ) + smoke, example, adopter = ( + path / f"{name}-{kind}.py" for name in ("smoke", "example", "adopter") + ) + shutil.copy2(Path(__file__).with_name("_materializer_installed.py"), smoke) + shutil.copy2(ROOT / "examples/reporting_destination_writer.py", example) + shutil.copy2(ROOT / "tests/type_checks/reporting_destination_writer.py", adopter) + result = json.loads( + run_step( + [str(python), "-I", str(smoke)], + label=f"b1-{kind}-isolated-python310-smoke", + cwd=path, + value={"workspace": str(ROOT), "example": str(example), "assets": hashes}, + timeout=120, + ) + ) + assert result == {"python": "3.10", "rows": [0, 501], "installed": True, "assets": hashes} + config = path / "mypy.ini" + config.write_text( + "[mypy]\npython_version = 3.10\nstrict = True\n" + "plugins = adcp.types.mypy_plugin\nfollow_imports = silent\n" + ) + run_step( + [ + str(python), + "-I", + "-m", + "mypy", + "--config-file", + str(config), + "--strict", + "--no-incremental", + str(adopter), + str(example), + ], + label=f"b1-{kind}-installed-adopter-types", + cwd=path, + timeout=120, + ) diff --git a/tests/conformance/reporting/test_reporting_materializer_readers.py b/tests/conformance/reporting/test_reporting_materializer_readers.py new file mode 100644 index 000000000..ddd0c0ff4 --- /dev/null +++ b/tests/conformance/reporting/test_reporting_materializer_readers.py @@ -0,0 +1,426 @@ +"""Hostile source/destination pagination, bounded streams and exact contract assets.""" + +import asyncio +import base64 +import hashlib +import json +from dataclasses import replace + +import pytest + +from adcp.reporting.ledger import LedgerConflictError +from adcp.reporting.ledger.store import encode_cursor +from adcp.reporting.materializer import ( + ReportingDestinationIO, + ReportingDestinationPage, + ReportingRevisionVerifierRegistry, + ReportingVerificationLimits, + ReportingWriterCapability, + ReportingWriterError, + reference_verifier, + strict_reporting_json, +) +from adcp.reporting.materializer.reference import _Session + +from ._materializer_support import io_context, materializer_case +from ._reliable_support import reliable_factory + + +@pytest.mark.parametrize( + "field,value", + [ + ("row_count", True), + ("row_count", -1), + ("readable", 1), + ("readable", 0), + ("readable_at_commit", 1), + ("revision_content_sha256", "private-invalid-body"), + ], +) +async def test_foundation_count_flags_and_digest_are_strict_before_source_io(field, value): + case = await materializer_case(1) + + class Reader: + async def read_revision_rows(self, **kwargs): + pytest.fail("invalid frozen metadata must fail before source I/O") + + with pytest.raises(ReportingWriterError, match="SOURCE_INVALID") as caught: + await case.prepare(reader=Reader(), revisions=(replace(case.revision, **{field: value}),)) + assert "private-invalid-body" not in str(caught.value) + assert case.writer.open_count == 0 + + +@pytest.mark.parametrize( + "damage", + [ + "revision", + "cursor-revision", + "cursor-offset", + "cursor-cycle", + "cursor-duplicate-keys", + "cursor-type", + "total-change", + "bool-total", + "unpaired-cursor", + "empty-more", + "truncated", + "extra", + "second-page", + "wrong-page-type", + ], +) +async def test_source_walk_rejects_incomplete_or_substituted_revision_pages_before_destination_io( + damage, +): + case = await materializer_case(501) + calls = 0 + + class Reader: + async def read_revision_rows(self, **kwargs): + nonlocal calls + calls += 1 + page = await case.store.read_revision_rows(**kwargs) + if damage == "revision": + return replace(page, reporting_revision_id="other-revision") + if damage == "cursor-revision" and page.has_more: + return replace( + page, cursor=encode_cursor({"revision": "other-revision", "offset": 500}) + ) + if damage == "cursor-offset" and page.has_more: + return replace( + page, + cursor=encode_cursor( + {"revision": case.revision.reporting_revision_id, "offset": 0} + ), + ) + if damage == "cursor-cycle": + return replace( + page, + rows=(case.rows[0],), + has_more=True, + cursor=encode_cursor( + {"revision": case.revision.reporting_revision_id, "offset": 1} + ), + ) + if damage == "cursor-duplicate-keys": + raw = b'{"revision":"other","revision":"revision-first","offset":500}' + return replace(page, cursor=base64.urlsafe_b64encode(raw).decode()) + if damage == "cursor-type": + return replace(page, cursor=True) + if damage == "total-change" and calls == 2: + return replace(page, total_count=500) + if damage == "bool-total": + return replace(page, total_count=True) + if damage == "unpaired-cursor": + return replace(page, has_more=False) + if damage == "empty-more": + return replace(page, rows=()) + if damage == "truncated": + return replace(page, has_more=False, cursor=None) + if damage == "extra": + return replace(page, rows=(*page.rows, case.rows[0])) + if damage == "second-page" and calls == 2: + return replace(page, rows=({**case.rows[-1], "impressions": True},)) + if damage == "wrong-page-type": + return {"rows": page.rows, "provider": "private-response"} + return page + + with pytest.raises(ReportingWriterError, match="SOURCE_INVALID"): + await case.prepare(reader=Reader()) + assert case.writer.open_count == case.writer.write_effects == 0 + assert calls == (2 if damage in {"cursor-cycle", "total-change", "second-page"} else 1) + + +@pytest.mark.parametrize("backend", ["memory", "postgres"]) +async def test_real_store_cursor_binds_revision_and_all_501_rows(backend): + async with reliable_factory(backend, notifications=True) as h: + case = await materializer_case(501, store=h.store) + first = await case.store.read_revision_rows( + account_id="acct_a", reporting_revision_id=case.revision.reporting_revision_id + ) + second = await case.store.read_revision_rows( + account_id="acct_a", + reporting_revision_id=case.revision.reporting_revision_id, + cursor=first.cursor, + ) + assert ( + first.reporting_revision_id + == second.reporting_revision_id + == case.revision.reporting_revision_id + ) + assert len(first.rows) == 500 and len(second.rows) == 1 + assert first.total_count == second.total_count == 501 and not second.has_more + for cursor in ( + encode_cursor({"revision": "another", "offset": 500}), + encode_cursor({"revision": case.revision.reporting_revision_id, "offset": True}), + encode_cursor({"revision": case.revision.reporting_revision_id, "offset": -1}), + ): + with pytest.raises(LedgerConflictError, match="does not bind"): + await case.store.read_revision_rows( + account_id="acct_a", + reporting_revision_id=case.revision.reporting_revision_id, + cursor=cursor, + ) + for limit in (True, 0, 501): + with pytest.raises(LedgerConflictError, match="page size"): + await case.store.read_revision_rows( + account_id="acct_a", + reporting_revision_id=case.revision.reporting_revision_id, + limit=limit, + ) + + +@pytest.mark.parametrize( + "damage", + ["replay", "cycle", "revision", "path", "format", "totals", "wrong-type", "empty", "extra"], +) +async def test_destination_pages_are_verifier_controlled(damage): + case = await materializer_case(3) + locator = await case.io.write(case.prepared, context=io_context()) + requests = [] + + class Session(_Session): + async def read_rows(self, locator, *, cursor, limit): + requests.append(cursor) + index = len(requests) - 1 + next_cursor = f"page-{index}" if index < 2 else None + page = ReportingDestinationPage( + case.revision.reporting_revision_id, + (case.prepared.rows[index],), + 3, + next_cursor is not None, + next_cursor, + "jsonl", + "producer", + ) + if damage == "replay": + return replace(page, rows=(case.prepared.rows[0],)) + if damage == "cycle": + return replace(page, has_more=True, cursor="repeat") + if damage == "revision": + return replace(page, reporting_revision_id="other") + if damage == "path": + return replace(page, verification_path="destination") + if damage == "format": + return replace(page, format="csv") + if damage == "totals": + return replace(page, total_count=2) + if damage == "wrong-type": + return object() + if damage == "empty": + return replace(page, rows=()) + if damage == "extra": + return replace(page, rows=tuple(case.prepared.rows) + (case.prepared.rows[0],)) + return page + + class Resolver: + def resolve(self, request, *, phase, context): + return Session(case.resolver, request, phase, context) + + with pytest.raises(ReportingWriterError, match="DESTINATION_CORRUPT"): + await ReportingDestinationIO(case.registry, Resolver()).verify( + case.prepared, locator, context=io_context() + ) + assert len(requests) <= 2 and case.writer.open_count == case.writer.close_count == 2 + + +@pytest.mark.parametrize( + "bound", ["max_rows", "max_pages", "max_total_bytes", "max_items", "max_chunks", "max_objects"] +) +async def test_walks_and_repeating_streams_have_finite_budgets(bound): + case = await materializer_case(501 if bound in {"max_pages", "max_objects"} else 100) + locator = await case.io.write(case.prepared, context=io_context()) + limits = replace( + ReportingVerificationLimits(), + **{bound: 1 if bound in {"max_rows", "max_pages", "max_chunks", "max_objects"} else 1000}, + ) + # Registry assets have their own bounded construction. Item/byte budgets + # deliberately exceed the vectors but not the full source/destination. + if bound in {"max_rows", "max_total_bytes", "max_items"}: + limits = replace(limits, max_rows=2 if bound == "max_rows" else 100_000) + case = await materializer_case(100) + locator = await case.io.write(case.prepared, context=io_context()) + verifier = replace(case.verifier, limits=limits) + registry = ReportingRevisionVerifierRegistry((verifier,)) + if bound in {"max_rows", "max_pages", "max_total_bytes", "max_items"}: + with pytest.raises(ReportingWriterError, match="LIMIT_EXCEEDED"): + await registry.prepare( + key=verifier.key, + binding=case.binding, + delivery=case.delivery, + obligation=case.obligation, + revisions=(case.revision,), + attempt=case.attempt, + reader=case.store, + context=io_context(), + ) + else: + with pytest.raises(ReportingWriterError, match="LIMIT_EXCEEDED|DESTINATION_CORRUPT"): + await ReportingDestinationIO(registry, case.resolver).verify( + case.prepared, locator, context=io_context() + ) + + +async def test_infinite_object_stream_is_bounded_and_closed_without_background_tasks(): + case = await materializer_case() + locator = await case.io.write(case.prepared, context=io_context()) + verifier = replace(case.verifier, limits=ReportingVerificationLimits(max_chunks=3)) + registry = ReportingRevisionVerifierRegistry((verifier,)) + chunks, closed = 0, 0 + + class Session(_Session): + async def read_object(self, locator, *, object_ref): + nonlocal chunks, closed + try: + while True: + chunks += 1 + yield b" " + finally: + closed += 1 + + class Resolver: + def resolve(self, request, *, phase, context): + return Session(case.resolver, request, phase, context) + + before = set(asyncio.all_tasks()) + with pytest.raises(ReportingWriterError, match="LIMIT_EXCEEDED"): + await ReportingDestinationIO(registry, Resolver()).verify( + case.prepared, locator, context=io_context() + ) + assert chunks == 4 and closed == 1 + assert not (set(asyncio.all_tasks()) - before) + assert case.writer.open_count == case.writer.close_count == 2 + + +@pytest.mark.parametrize("damage", ["page-version", "final-version", "final-path"]) +async def test_native_identity_is_pinned_across_pages_and_after_readback(damage): + capability = ReportingWriterCapability( + "dataset_share", + "reference-memory", + None, + "canonical_digest", + "representative_consumer", + "native_version", + "sha256", + "conditional_create", + ) + case = await materializer_case(501, capability=capability) + locator = await case.io.write(case.prepared, context=io_context()) + observations = 0 + + class Session(_Session): + async def read_rows(self, locator, *, cursor, limit): + result = await super().read_rows(locator, cursor=cursor, limit=limit) + if cursor is not None and damage == "page-version": + return replace(result, native_version_ref="different-version") + return result + + async def observe_native_version(self, locator): + nonlocal observations + observations += 1 + result = await super().observe_native_version(locator) + if observations == 2: + return replace( + result, + **{ + "native_version_ref" if damage == "final-version" else "location": "changed" + }, + ) + return result + + class Resolver: + def resolve(self, request, *, phase, context): + return Session(case.resolver, request, phase, context) + + with pytest.raises(ReportingWriterError, match="DESTINATION_CORRUPT"): + await ReportingDestinationIO(case.registry, Resolver()).verify( + case.prepared, locator, context=io_context() + ) + assert observations == (1 if damage == "page-version" else 2) + assert case.writer.open_count == case.writer.close_count == 2 + + +@pytest.mark.parametrize( + "mutation", + [ + "remote-ref", + "nested-id", + "missing-total", + "extra-total", + "expression", + "golden-digest", + "golden-bytes", + "golden-member-order", + "golden-row-order", + "golden-duplicate-name", + "canonical-algorithm", + ], +) +def test_installed_contracts_are_executable_complete_closed_and_network_free(mutation, monkeypatch): + verifier = reference_verifier() + definition, schema, contract = ( + json.loads(raw) + for raw in ( + verifier.definition_bytes, + verifier.schema_bytes, + verifier.canonicalization_bytes, + ) + ) + if mutation == "remote-ref": + schema["properties"]["details"] = {"$ref": "https://never-fetch.example.test/private"} + elif mutation == "nested-id": + schema["properties"]["details"] = {"$id": "https://never-fetch.example.test/private"} + elif mutation == "missing-total": + del schema["properties"]["spend"]["x-adcp-control-total"] + elif mutation == "extra-total": + schema["properties"]["extra"] = { + "type": "integer", + "x-adcp-control-total": {"value_type": "integer"}, + } + elif mutation == "expression": + definition["metrics"][0]["source_expression"] = "impressions * 2" + elif mutation == "golden-digest": + contract["golden_vectors"]["ordering_encoding"]["sha256"] = "0" * 64 + elif mutation == "golden-bytes": + contract["golden_vectors"]["ordering_encoding"]["canonical_utf8_base64"] = "e30=" + elif mutation == "golden-member-order": + contract["golden_vectors"]["ordering_encoding"]["input_rows"] = json.loads( + strict_reporting_json(contract["golden_vectors"]["ordering_encoding"]["input_rows"]) + ) + elif mutation == "golden-row-order": + contract["golden_vectors"]["ordering_encoding"]["input_rows"].reverse() + elif mutation == "golden-duplicate-name": + contract["golden_vectors"]["ordering_encoding"]["name"] = contract["golden_vectors"][ + "empty_report" + ]["name"] + else: + contract["algorithm"] = "provider-assertions-v1" + raw_schema = json.dumps(schema).encode() + contract["schema_sha256"] = hashlib.sha256(raw_schema).hexdigest() + raw_definition, raw_contract = json.dumps(definition).encode(), json.dumps(contract).encode() + key = replace( + verifier.key, + definition=replace( + verifier.key.definition, + report_definition_sha256=hashlib.sha256(raw_definition).hexdigest(), + schema_sha256=hashlib.sha256(raw_schema).hexdigest(), + ), + canonicalization=replace( + verifier.key.canonicalization, + canonicalization_sha256=hashlib.sha256(raw_contract).hexdigest(), + ), + ) + import socket + + def network(*args, **kwargs): + pytest.fail("canonicalizer attempted network fallback") + + monkeypatch.setattr(socket, "create_connection", network) + with pytest.raises(ReportingWriterError, match="UNSUPPORTED_VERIFICATION"): + replace( + verifier, + key=key, + schema_bytes=raw_schema, + definition_bytes=raw_definition, + canonicalization_bytes=raw_contract, + ) diff --git a/tests/conformance/reporting/test_reporting_materializer_verification.py b/tests/conformance/reporting/test_reporting_materializer_verification.py new file mode 100644 index 000000000..f14abb6ce --- /dev/null +++ b/tests/conformance/reporting/test_reporting_materializer_verification.py @@ -0,0 +1,601 @@ +"""Actual bytes, whole walks, typed totals and exact immutable destination paths.""" + +import hashlib +import json +from dataclasses import FrozenInstanceError, replace +from datetime import datetime, timedelta +from decimal import Decimal +from itertools import product + +import pytest + +from adcp.reporting.ledger import ReportingDeliveryPrincipal, revision_content_sha256 +from adcp.reporting.materializer import ( + ReferenceReportingDestinationWriter, + ReferenceReportingResolver, + ReportingDestinationIO, + ReportingRevisionVerifierRegistry, + ReportingVerificationLimits, + ReportingWriterCapability, + ReportingWriterError, + parse_reporting_json, + reference_verifier, + strict_reporting_json, + validate_materialization_target, +) +from adcp.reporting.outbox import InMemoryReportingOutbox + +from ._materializer_support import io_context, materializer_case + +CAPABILITIES = [ + ReportingWriterCapability( + method, "reference-memory", fmt, profile, path, immutable, "sha256", "conditional_create" + ) + for method, fmt, path, immutable, profiles in ( + ( + "file_transfer", + "jsonl", + "producer", + "immutable_location", + ("canonical_digest", "manifest_checksums"), + ), + ( + "dataset_share", + None, + "representative_consumer", + "native_version", + ("canonical_digest", "native_commit"), + ), + ( + "warehouse_materialization", + None, + "destination", + "native_version", + ("canonical_digest", "native_commit"), + ), + ) + for profile in profiles +] + + +@pytest.mark.parametrize("count,capability", tuple(product((0, 1, 501), CAPABILITIES))) +async def test_entire_source_and_destination_walk_in_every_supported_profile(count, capability): + case = await materializer_case(count, capability=capability) + locator = await case.io.write(case.prepared, context=io_context()) + case.resolver.rotate() + result = await case.io.verify(case.prepared, locator, context=io_context()) + assert result.verification.row_count == count + assert result.verification.control_totals == case.revision.managed_control_totals + assert result.request.principal == ReportingDeliveryPrincipal( + "acct_a", "https://buyer.example.test/agents/reporting" + ) + assert result.verification.verification_path == capability.verification_path + destination_operation_1 = await case.io.write(case.prepared, context=io_context()) + assert destination_operation_1 == locator + assert case.writer.write_effects == 1 and case.writer.open_count == case.writer.close_count == 3 + assert case.writer.production_eligible is False + assert (await case.store.get_materialization(case.attempt.key)).outcome is None + assert len(await InMemoryReportingOutbox(case.store).list_events(account_id="acct_a")) >= 1 + assert all( + e.notification_type != "reporting.delivery_ready" + for e in await InMemoryReportingOutbox(case.store).list_events(account_id="acct_a") + ) + + +@pytest.mark.parametrize( + "damage", + [ + "rows", + "row-order", + "row-missing", + "row-extra", + "object", + "object-utf8", + "object-missing", + "object-extra", + "object-order", + "manifest", + "manifest-schema", + "manifest-count", + "manifest-total-type", + "manifest-total-unit", + "manifest-total-missing", + "manifest-period", + "manifest-object-checksum", + "manifest-object-count", + "native-version", + "native-path", + ], +) +async def test_writer_locators_cannot_prove_corrupt_or_changed_destination(damage): + cap = CAPABILITIES[2] if damage.startswith("native") else CAPABILITIES[0] + case = await materializer_case(501, capability=cap) + locator = await case.io.write(case.prepared, context=io_context()) + artifact = case.writer._artifacts[locator.external_id] + if damage == "rows": + row = json.loads(artifact.rows[0]) + row["details"]["active"] = 1 + artifact = replace(artifact, rows=(strict_reporting_json(row), *artifact.rows[1:])) + elif damage == "row-order": + artifact = replace(artifact, rows=artifact.rows[::-1]) + elif damage == "row-missing": + artifact = replace(artifact, rows=artifact.rows[:-1]) + elif damage == "row-extra": + artifact = replace(artifact, rows=(*artifact.rows, artifact.rows[0])) + elif damage in {"object", "object-utf8"}: + artifact = replace( + artifact, + objects=( + (artifact.objects[0][0], b"\xff\n" if damage == "object-utf8" else b"{}\n"), + *artifact.objects[1:], + ), + ) + elif damage == "object-missing": + artifact = replace(artifact, objects=artifact.objects[:-1]) + elif damage == "object-extra": + artifact = replace(artifact, objects=(*artifact.objects, ("extra.jsonl", b"{}\n"))) + elif damage == "object-order": + artifact = replace(artifact, objects=artifact.objects[::-1]) + elif damage.startswith("manifest"): + manifest = json.loads(artifact.manifest) + if damage == "manifest": + artifact = replace(artifact, manifest=artifact.manifest + b" ") + else: + if damage == "manifest-schema": + manifest["unexpected"] = "provider prose" + if damage == "manifest-count": + manifest["row_count"] = True + if damage == "manifest-total-type": + manifest["control_totals"][0]["value_type"] = "decimal" + if damage == "manifest-total-unit": + manifest["control_totals"][1]["unit"] = "EUR" + if damage == "manifest-total-missing": + manifest["control_totals"].pop() + if damage == "manifest-period": + manifest["period"]["end"] = manifest["period"]["start"] + if damage == "manifest-object-checksum": + manifest["files"][0]["sha256"] = "0" * 64 + if damage == "manifest-object-count": + manifest["files"][0]["row_count"] = 249 + raw = strict_reporting_json(manifest) + locator = replace( + locator, + resource=replace(locator.resource, manifest_sha256=hashlib.sha256(raw).hexdigest()), + ) + artifact = replace(artifact, manifest=raw) + elif damage == "native-version": + artifact = replace( + artifact, + locator=replace( + locator, resource=replace(locator.resource, native_version_ref="changed-version") + ), + ) + elif damage == "native-path": + artifact = replace( + artifact, + locator=replace( + locator, resource=replace(locator.resource, location="different/table") + ), + ) + case.writer._artifacts[locator.external_id] = artifact + with pytest.raises(ReportingWriterError): + await case.io.verify(case.prepared, locator, context=io_context()) + assert case.writer.open_count == case.writer.close_count == 2 + assert (await case.store.get_materialization(case.attempt.key)).outcome is None + + +@pytest.mark.parametrize( + "value", + [ + 1.0, + float("nan"), + float("inf"), + Decimal("1"), + datetime(2026, 1, 1), + b"abc", + {1}, + (1,), + {1: "value"}, + 2**53, + "\ud800", + ], +) +def test_exact_recursive_json_rejects_non_json_values(value): + with pytest.raises(ReportingWriterError): + strict_reporting_json({"nested": [value]}) + + +@pytest.mark.parametrize("base,value", [(int, 1), (str, "x"), (dict, {}), (list, [])]) +def test_json_subclasses_are_never_coerced(base, value): + subclass = type("NotExact", (base,), {}) + with pytest.raises(ReportingWriterError): + strict_reporting_json({"value": subclass(value)}) + + +@pytest.mark.parametrize( + "raw", + [ + b'{"a":1,"a":2}', + b'{"a":{"x":true,"x":1}}', + b"NaN", + b"Infinity", + b"-Infinity", + b"1.0", + b'"\xff"', + b'"\\udfff"', + ], +) +def test_parser_rejects_ambiguous_or_invalid_bytes(raw): + with pytest.raises(ReportingWriterError) as caught: + parse_reporting_json(raw) + assert caught.value.__context__ is None and caught.value.__cause__ is None + + +def test_type_distinction_unicode_and_bounded_json(): + assert strict_reporting_json(True) != strict_reporting_json(1) + assert strict_reporting_json("é") != strict_reporting_json("e\u0301") + assert strict_reporting_json({"\ue000": 1, "😀": 2}) == '{"😀":2,"\ue000":1}'.encode() + for value, limits in ( + ([[[[]]]], ReportingVerificationLimits(max_depth=2)), + ({"large": "x" * 30}, ReportingVerificationLimits(max_value_bytes=20)), + ([1, 2, 3], ReportingVerificationLimits(max_items=2)), + ): + with pytest.raises(ReportingWriterError): + strict_reporting_json(value, limits) + with pytest.raises(ReportingWriterError): + parse_reporting_json(b"[" * 1000 + b"]" * 1000) + + +@pytest.mark.parametrize("value", ['😀é\x00\n\\"', [True, False, None, -1], {"nested": [1, 2]}]) +def test_exact_utf8_and_json_escape_budget(value): + encoded = strict_reporting_json(value) + assert ( + strict_reporting_json(value, ReportingVerificationLimits(max_value_bytes=len(encoded))) + == encoded + ) + with pytest.raises(ReportingWriterError, match="LIMIT_EXCEEDED"): + strict_reporting_json(value, ReportingVerificationLimits(max_value_bytes=len(encoded) - 1)) + + +def test_aggregate_byte_budget_rejects_before_allocating_canonical_output(monkeypatch): + import adcp.reporting.materializer._json as boundary + + def forbidden(value): + pytest.fail("oversized values must fail before canonical output allocation") + + monkeypatch.setattr(boundary, "canonical_json_utf8_v1", forbidden) + with pytest.raises(ReportingWriterError, match="LIMIT_EXCEEDED"): + boundary.strict_reporting_json( + ["x" * 1000] * 1000, ReportingVerificationLimits(max_value_bytes=2000) + ) + + +@pytest.mark.parametrize( + "part", + [ + "definition_id", + "profile", + "definition_uri", + "definition_hash", + "schema_version", + "schema_uri", + "schema_hash", + "dialect", + "ref_policy", + "canonical_id", + "canonical_uri", + "canonical_hash", + "method", + "transport", + "format", + "verification_profile", + "path", + "immutability", + "write_semantics", + ], +) +async def test_registry_key_is_the_complete_frozen_tuple(part): + case = await materializer_case() + key = case.verifier.key + if part in {"definition_id", "profile"}: + key = replace( + key, + **{ + ( + "report_definition_id" if part == "definition_id" else "reporting_profile" + ): "different" + }, + ) + elif part.startswith("canonical"): + field = { + "canonical_id": "canonicalization_id", + "canonical_uri": "canonicalization_uri", + "canonical_hash": "canonicalization_sha256", + }[part] + key = replace( + key, + canonicalization=replace( + key.canonicalization, + **{ + field: ( + "0" * 64 + if part.endswith("hash") + else ( + "https://different.example.test/contract" + if part.endswith("uri") + else "different" + ) + ) + }, + ), + ) + elif part in { + "method", + "transport", + "format", + "verification_profile", + "path", + "immutability", + "write_semantics", + }: + changes = { + "method": "warehouse_materialization", + "transport": "other", + "format": "csv", + "verification_profile": "manifest_checksums", + "verification_path": "destination", + "immutability": "native_version", + "write_semantics": "idempotent", + } + field = "verification_path" if part == "path" else part + values = {field: changes[field]} + if part in {"method", "immutability"}: + values["verification_path"] = "destination" + key = replace(key, capability=replace(key.capability, **values)) + else: + field = { + "definition_uri": "report_definition_uri", + "definition_hash": "report_definition_sha256", + "schema_hash": "schema_sha256", + "dialect": "schema_dialect", + "ref_policy": "schema_ref_policy", + }.get(part, part) + value = ( + "0" * 64 + if part.endswith("hash") + else ( + "https://different.example.test/schema" + if part.endswith("uri") or part == "dialect" + else "other" + ) + ) + key = replace(key, definition=replace(key.definition, **{field: value})) + with pytest.raises(ReportingWriterError, match="UNSUPPORTED_VERIFICATION"): + await case.prepare(key=key) + assert case.writer.open_count == 0 + + +async def test_uppercase_digest_evidence_is_semantically_equal_and_output_is_lowercase(): + case = await materializer_case() + revision = replace( + case.revision, + canonical_content_digest=replace( + case.revision.canonical_content_digest, + value=case.revision.canonical_content_digest.value.upper(), + canonicalization_sha256=case.revision.canonical_content_digest.canonicalization_sha256.upper(), + ), + ) + prepared = await case.prepare(revisions=(revision,)) + locator = await case.io.write(prepared, context=io_context()) + verified = await case.io.verify(prepared, locator, context=io_context()) + assert ( + verified.verification.canonical_content_digest.value + == case.revision.canonical_content_digest.value + ) + + +@pytest.mark.parametrize("claim", ["canonical", "core", "total-value", "total-missing"]) +async def test_validly_shaped_ledger_claims_are_recomputed_before_resolver_io(claim): + case = await materializer_case(501) + if claim == "canonical": + revision = replace( + case.revision, + canonical_content_digest=replace( + case.revision.canonical_content_digest, value="a" * 64 + ), + ) + elif claim == "core": + revision = replace(case.revision, revision_content_sha256="b" * 64) + else: + totals = case.revision.managed_control_totals + totals = ( + (replace(totals[0], value="0"), *totals[1:]) if claim == "total-value" else totals[:-1] + ) + pairs = tuple((total.name, total.value) for total in totals) + revision = replace( + case.revision, + managed_control_totals=totals, + control_totals=pairs, + revision_content_sha256=revision_content_sha256( + reporting_revision_id=case.revision.reporting_revision_id, + row_count=case.revision.row_count, + control_totals=pairs, + reporting_rows=case.rows, + control_total_evidence=totals, + ), + ) + with pytest.raises(ReportingWriterError, match="SOURCE_INVALID"): + await case.prepare(revisions=(revision,)) + assert case.writer.open_count == case.writer.write_effects == 0 + + +async def test_rotation_revocation_is_checked_again_before_readback(): + case = await materializer_case() + locator = await case.io.write(case.prepared, context=io_context()) + case.resolver.revoke(case.binding.principal) + with pytest.raises(ReportingWriterError, match="AUTHORIZATION_DENIED"): + await case.io.verify(case.prepared, locator, context=io_context()) + assert case.writer.open_count == case.writer.close_count == 2 + + +async def test_binding_and_tenant_collisions_do_not_share_effect_identity_or_resolution(): + first, second = await materializer_case(account="acct_a"), await materializer_case( + account="acct_b" + ) + assert first.prepared.request.external_id != second.prepared.request.external_id + wrong = ReportingDestinationIO( + first.registry, ReferenceReportingResolver(first.writer, first.registry, (second.binding,)) + ) + with pytest.raises(ReportingWriterError, match="AUTHORIZATION_DENIED"): + await wrong.write(first.prepared, context=io_context()) + assert first.writer.write_effects == 0 + revised = replace(first.prepared.request, reporting_revision_id="different") + assert revised.external_id != first.prepared.request.external_id # Equal attempt 1 is isolated. + + +@pytest.mark.parametrize("coordinate", ["account", "consumer"]) +async def test_colliding_public_ids_in_one_destination_never_cross_principals(coordinate): + first = await materializer_case() + second = await materializer_case( + **{ + coordinate: ( + "acct_b" if coordinate == "account" else "https://another.example.test/agent" + ) + } + ) + resolver = ReferenceReportingResolver( + first.writer, first.registry, (first.binding, second.binding) + ) + io = ReportingDestinationIO(first.registry, resolver) + left = await io.write(first.prepared, context=io_context()) + right = await io.write(second.prepared, context=io_context()) + assert ( + left.external_id != right.external_id and left.resource.location != right.resource.location + ) + assert first.writer.write_effects == 2 + with pytest.raises(ReportingWriterError, match="BINDING_MISMATCH"): + await io.verify(first.prepared, right, context=io_context()) + assert first.writer.open_count == first.writer.close_count == 2 + for case, locator in ((first, left), (second, right)): + result = await io.verify(case.prepared, locator, context=io_context()) + assert result.request.principal == case.binding.principal + + +async def test_registry_and_prepared_bytes_are_immutable_and_target_reselection_fails_closed(): + case = await materializer_case() + with pytest.raises(FrozenInstanceError): + case.registry.verifiers = () + assert all(type(r) is bytes for r in case.prepared.rows) + official = replace( + case.revision, + reporting_revision_id="official", + finality="official", + finality_basis="source_final", + finality_policy_id="reference-final", + finalized_at=case.revision.created_at, + ) + with pytest.raises(ReportingWriterError, match="CURRENT_REVISION_CHANGED"): + validate_materialization_target( + case.prepared, binding=case.binding, revisions=(case.revision, official) + ) + with pytest.raises(ReportingWriterError, match="HISTORY_CORRUPT"): + validate_materialization_target( + case.prepared, binding=case.binding, revisions=(case.revision, case.revision) + ) + assert case.writer.write_effects == 0 + + +@pytest.mark.parametrize( + "path", + [ + "../private", + "/private", + "a//private", + "a/./private", + "a/%2e%2e/private", + "a\\private", + "C:/private", + "C:private", + ], +) +async def test_object_locator_traversal_fails_before_readback_authorization(path): + case = await materializer_case() + locator = await case.io.write(case.prepared, context=io_context()) + with pytest.raises((ReportingWriterError, ValueError)): + locator = replace(locator, resource=replace(locator.resource, object_refs=(path,))) + await case.io.verify(case.prepared, locator, context=io_context()) + assert case.writer.open_count == case.writer.close_count == 1 + + +@pytest.mark.parametrize("count", [0, 501]) +async def test_snapshot_to_official_retains_both_histories_and_separate_attempt_one_identity(count): + case = await materializer_case(count) + snapshot_locator = await case.io.write(case.prepared, context=io_context()) + official = replace( + case.revision, + reporting_revision_id="revision-official", + finality="official", + finality_basis="source_final", + finality_policy_id="reference-final", + finalized_at=case.revision.created_at, + revision_content_sha256=revision_content_sha256( + reporting_revision_id="revision-official", + row_count=count, + control_totals=case.revision.control_totals, + reporting_rows=case.rows, + control_total_evidence=case.revision.managed_control_totals, + ), + ) + await case.store.commit_revision(official, case.rows) + history = await case.store.list_revisions( + account_id=case.obligation.account_id, + reporting_obligation_id=case.obligation.reporting_obligation_id, + ) + assert len(history) == 2 and official.supersedes_reporting_revision_id is None + with pytest.raises(ReportingWriterError, match="CURRENT_REVISION_CHANGED"): + validate_materialization_target(case.prepared, binding=case.binding, revisions=history) + attempt = replace( + case.attempt, + reporting_revision_id=official.reporting_revision_id, + reporting_materialization_id="materialization-official", + created_at=case.attempt.created_at + timedelta(seconds=1), + ) + await case.store.commit_materialization_attempt(attempt) + prepared = await case.prepare(revisions=history, attempt=attempt) + locator = await case.io.write(prepared, context=io_context()) + verified = await case.io.verify(prepared, locator, context=io_context()) + assert verified.request.reporting_revision_id == official.reporting_revision_id + assert attempt.attempt == case.attempt.attempt == 1 + assert locator.external_id != snapshot_locator.external_id + assert locator.resource.location != snapshot_locator.resource.location + assert case.writer.write_effects == 2 + assert (await case.store.get_materialization(case.attempt.key)).outcome is None + assert (await case.store.get_materialization(attempt.key)).outcome is None + + +async def test_official_required_preparation_never_falls_back_to_a_snapshot(): + case = await materializer_case(finality="official") + snapshot = replace( + case.revision, + reporting_revision_id="snapshot-only", + finality="snapshot", + finality_basis=None, + finality_policy_id=None, + finalized_at=None, + ) + with pytest.raises(ReportingWriterError, match="REVISION_NOT_READY"): + await case.prepare(revisions=(snapshot,)) + assert case.writer.open_count == 0 + locator = await case.io.write(case.prepared, context=io_context()) + result = await case.io.verify(case.prepared, locator, context=io_context()) + assert result.request.reporting_revision_id == case.revision.reporting_revision_id + + +def test_reference_writer_has_no_production_config_override(): + writer = ReferenceReportingDestinationWriter(()) + with pytest.raises(AttributeError): + writer.production_eligible = True + with pytest.raises(TypeError): + ReferenceReportingDestinationWriter((), production_eligible=True) + with pytest.raises(ReportingWriterError): + ReportingRevisionVerifierRegistry((reference_verifier(), reference_verifier())) diff --git a/tests/conformance/reporting/test_reporting_notification_packaging.py b/tests/conformance/reporting/test_reporting_notification_packaging.py index 6fe17e7f0..a068aec69 100644 --- a/tests/conformance/reporting/test_reporting_notification_packaging.py +++ b/tests/conformance/reporting/test_reporting_notification_packaging.py @@ -231,6 +231,8 @@ def installed_distribution(built_distribution, request): assert files("adcp.reporting.outbox").joinpath("required_schema.json").is_file() assert files("adcp.reporting.outbox").joinpath("required_status_schema.json").is_file() assert files("adcp.reporting.ledger").joinpath("reporting_status_notifications.sql").is_file() +assert files("adcp.reporting.ledger").joinpath("reporting_status_selector_version.sql").is_file() +assert files("adcp.reporting.outbox").joinpath("required_status_selector_schema.json").is_file() version = files("adcp").joinpath("ADCP_VERSION").read_text().strip() assert get_named_validator("core/reporting-status-changed-webhook.json", version=version) is not None assert get_named_validator("core/webhook-activity-record.json", version=version) is not None @@ -255,6 +257,7 @@ def test_wheel_and_sdist_contain_exact_complete_sql_chain(built_distribution): *CHAIN, "reporting_webhook_activity.sql", "reporting_status_notifications.sql", + "reporting_status_selector_version.sql", ): expected = (ROOT / "src" / "adcp" / "reporting" / "ledger" / name).read_bytes() assert archive.read(f"adcp/reporting/ledger/{name}") == expected diff --git a/tests/conformance/reporting/test_reporting_notification_readiness.py b/tests/conformance/reporting/test_reporting_notification_readiness.py index ce88a85cd..e51bbe772 100644 --- a/tests/conformance/reporting/test_reporting_notification_readiness.py +++ b/tests/conformance/reporting/test_reporting_notification_readiness.py @@ -39,7 +39,7 @@ async def test_capability_fragment_exposes_only_complete_notifications(notificat ) -async def test_managed_capability_requires_retained_configuration_and_frozen_scope( +async def test_frozen_binding_alone_never_advertises_managed_readiness( notification_harness, ): h = notification_harness @@ -49,7 +49,6 @@ async def test_managed_capability_requires_retained_configuration_and_frozen_sco ) assert fields == { "ledger_notification": "reporting.ledger_changed", - "readiness_notification": "reporting.delivery_ready", "supports_webhook_activity": False, } with pytest.raises(ReportingNotificationError): diff --git a/tests/conformance/reporting/test_reporting_revision_selection.py b/tests/conformance/reporting/test_reporting_revision_selection.py new file mode 100644 index 000000000..028908981 --- /dev/null +++ b/tests/conformance/reporting/test_reporting_revision_selection.py @@ -0,0 +1,293 @@ +"""Whole-history selection is identical for Core, C, ingest and B1 preparation.""" + +import asyncio +from dataclasses import replace +from itertools import permutations + +import pytest + +from adcp.reporting.fixtures import SNAPSHOT_OFFERING_ID +from adcp.reporting.ledger import ( + InMemoryReportingLedgerStore, + LedgerConflictError, + ProducerOfferings, + ReportingProducer, + current_required_revision, + project_obligation_health, + select_reporting_revision, +) +from adcp.reporting.ledger.status import ReportingStatusCaller, ReportingStatusHandler +from adcp.reporting.ledger.status_projection import ( + ReportingStatusSnapshot, + StatusProjectionInput, + lifecycle_intents, + project_status_scope, +) +from adcp.reporting.ledger.status_snapshot import validate_status_evidence +from adcp.reporting.outbox import ReportingStatusScope +from adcp.reporting.revision_selection import RevisionHistoryEntry + +from ._generation_support import NOW, UncalledSource, configuration, obligation_for, revision_for +from ._reliable_support import complete_fetch, reliable_factory +from ._reliable_support import configuration as reliable_configuration +from .test_reporting_notification_outbox import statement + + +def entry(name, predecessor=None, finality="snapshot", account="acct_a", obligation="rpo_acct_a"): + return RevisionHistoryEntry(account, obligation, name, finality, predecessor) + + +VECTORS = ( + ((), "snapshot", "not_ready", "empty_history"), + ((), "official", "not_ready", "empty_history"), + ((entry("a"),), "official", "not_ready", "official_required"), + ((entry("a"),), "snapshot", "selected", "a"), + ((entry("a"), entry("b", "a")), "snapshot", "selected", "b"), + ( + (entry("a"), entry("b", "a"), entry("official", finality="official")), + "snapshot", + "selected", + "official", + ), + ((entry("official", finality="official"),), "official", "selected", "official"), + ((entry("a"), entry("a")), "snapshot", "corrupt", "duplicate_revision_id"), + ((entry("a", account="acct_b"),), "snapshot", "corrupt", "ownership_mismatch"), + ((entry("a", obligation="rpo_other"),), "snapshot", "corrupt", "ownership_mismatch"), + ((entry(""),), "snapshot", "corrupt", "invalid_revision_identity"), + ((entry("a", finality="draft"),), "snapshot", "corrupt", "invalid_finality"), + ((entry("a", "absent"),), "official", "corrupt", "missing_predecessor"), + ( + (entry("a", "o"), entry("o", finality="official")), + "official", + "corrupt", + "cross_finality_edge", + ), + ((entry("a"), entry("o", "a", "official")), "official", "corrupt", "cross_finality_edge"), + ( + (entry("o", finality="official"), entry("p", "o", "official")), + "official", + "corrupt", + "official_predecessor", + ), + ( + (entry("a"), entry("b", "a"), entry("c", "a")), + "official", + "corrupt", + "forked_snapshot_history", + ), + ((entry("a"), entry("b")), "official", "corrupt", "disconnected_snapshot_history"), + ((entry("a", "a"),), "snapshot", "corrupt", "revision_cycle"), + ((entry("a", "b"), entry("b", "a")), "snapshot", "corrupt", "revision_cycle"), + ( + (entry("unique-leaf"), entry("a", "b"), entry("b", "a")), + "snapshot", + "corrupt", + "revision_cycle", + ), + ( + (entry("a", finality="official"), entry("b", finality="official")), + "snapshot", + "corrupt", + "multiple_officials", + ), +) + + +@pytest.mark.parametrize("history,required,kind,detail", VECTORS) +def test_complete_history_in_any_input_order(history, required, kind, detail): + for rows in permutations(history): + result = select_reporting_revision( + rows, + account_id="acct_a", + reporting_obligation_id="rpo_acct_a", + required_finality=required, + ) + assert result.kind == kind + assert ( + result.revision.reporting_revision_id if kind == "selected" else result.reason + ) == detail + if kind == "corrupt" and not any(r.finality == "official" for r in history): + # A unique official never excuses a damaged retained snapshot history. + result = select_reporting_revision( + (*history, entry("official", finality="official")), + account_id="acct_a", + reporting_obligation_id="rpo_acct_a", + required_finality="official", + ) + assert result.kind == "corrupt" + + +@pytest.mark.parametrize( + "field,value", + [ + ("account_id", []), + ("reporting_obligation_id", {}), + ("reporting_revision_id", []), + ("supersedes_reporting_revision_id", []), + ("finality", {}), + ], +) +def test_damaged_identity_types_return_corrupt_without_hashing_them(field, value): + result = select_reporting_revision( + (replace(entry("a"), **{field: value}),), + account_id="acct_a", + reporting_obligation_id="rpo_acct_a", + required_finality="official", + ) + assert result.kind == "corrupt" + + +def records(history, obligation): + revision, _ = revision_for(obligation) + result = [] + for r in history: + record = replace( + revision, + account_id=r.account_id, + reporting_obligation_id=r.reporting_obligation_id, + reporting_revision_id=r.reporting_revision_id, + finality=r.finality, + supersedes_reporting_revision_id=None, + finality_basis="source_final" if r.finality == "official" else None, + finality_policy_id="closed" if r.finality == "official" else None, + finalized_at=NOW if r.finality == "official" else None, + ) + # Model a damaged custom-store image, including edges normal dataclass + # construction already rejects. The pure wire adapter covers these too. + object.__setattr__( + record, "supersedes_reporting_revision_id", r.supersedes_reporting_revision_id + ) + result.append(record) + return tuple(result) + + +@pytest.mark.parametrize("history,required,kind,detail", VECTORS) +def test_compatibility_health_handler_projection_and_ingest_agree(history, required, kind, detail): + if any(not r.reporting_revision_id or r.finality == "draft" for r in history): + return # Persisted records have their own closed field validation. + config = replace(configuration(), required_finality=required) + obligation = obligation_for(config) + revisions = records(history, obligation) + current = current_required_revision(obligation, revisions) + assert (current.reporting_revision_id if current else None) == ( + detail if kind == "selected" else None + ) + if any(r.reporting_obligation_id != obligation.reporting_obligation_id for r in revisions): + return # Snapshot readers partition the complete account set by obligation. + projection = project_obligation_health( + obligation, revisions, ledger_as_of=NOW, scope_closed=True + ) + snapshot = ReportingStatusSnapshot("acct_a", NOW, (config,), (obligation,), revisions) + scoped = project_status_scope( + StatusProjectionInput(snapshot, ReportingStatusScope.for_obligation(obligation)) + ) + response = ReportingStatusHandler(InMemoryReportingLedgerStore()).render_snapshot( + {"view": "summary"}, caller=ReportingStatusCaller("acct_a", "buyer"), snapshot=snapshot + ) + assert projection.health == scoped.health == response["health"] + if kind == "corrupt": + assert projection.current_revision is None + assert [i.code for i in scoped.issues] == ["HISTORY_UNAVAILABLE"] + disputed = replace(statement(obligation), consumer_status="missing") + with pytest.raises(LedgerConflictError, match="history requires repair"): + validate_status_evidence(disputed, snapshot) + with pytest.raises(LedgerConflictError, match="history requires repair"): + validate_status_evidence(replace(disputed, reporting_obligation_id=None), snapshot) + assert lifecycle_intents(replace(snapshot, statuses=(disputed,))) == () + + +@pytest.mark.parametrize( + "history", + [ + v[0] + for v in VECTORS + if v[2] == "corrupt" + and all(r.reporting_revision_id and r.finality != "draft" for r in v[0]) + ], +) +async def test_producer_rejects_corruption_before_source_or_adapter_io(history): + class DamagedStore(InMemoryReportingLedgerStore): + async def list_revisions(self, **kwargs): + return records(history, obligation) + + config = configuration() + store = DamagedStore() + await store.put_configuration(config) + obligation = await store.commit_obligation(obligation_for(config)) + producer = ReportingProducer( + source=UncalledSource(), offerings=ProducerOfferings(), store=store + ) + with pytest.raises(LedgerConflictError, match="history requires repair"): + await producer.acquire_obligation(config, obligation, restate=True) + + +def test_unreadable_official_never_falls_back_to_materialized_snapshot(): + obligation = obligation_for(configuration()) + snapshot, _ = revision_for(obligation) + official = replace( + snapshot, + reporting_revision_id="official", + finality="official", + readable=False, + finality_basis="source_final", + finality_policy_id="closed", + finalized_at=NOW, + ) + assert current_required_revision(obligation, (snapshot, official)) is official + result = project_obligation_health( + obligation, (snapshot, official), ledger_as_of=NOW, scope_closed=True + ) + assert result.health == "action_required" and result.current_revision is official + + +@pytest.mark.parametrize("backend", ["memory", "postgres"]) +async def test_snapshot_restatement_after_an_official_close_keeps_one_chain(backend): + """A retained official close must never be mistaken for the snapshot leaf. + + Whole-history selection answers "what does this obligation currently + publish", and a unique official wins that outright. Reusing that answer to + choose ``supersedes`` roots the next restatement at ``None``, so the + obligation ends up with two snapshot roots -- ``disconnected_snapshot_history`` + over immutable rows, which every selector caller then parks for a repair + nothing in the SDK can perform. + """ + async with reliable_factory(backend) as h: + config = reliable_configuration("eur") + await h.store.put_configuration(config) + producer = h.producer(h.source(complete_fetch), managed=False) + (obligation,) = await producer.close_elapsed_periods(config) + + async def commit(observation, finality): + request = producer._build_slice( # noqa: SLF001 - the suite's slice fixture + config, obligation, SNAPSHOT_OFFERING_ID, now=h.clock(), observation=observation + ) + result = await producer._source.execute(request, cancel=asyncio.Event()) # noqa: SLF001 + return await h.commit_slice(producer, obligation, request, result, finality=finality) + + first = await commit(0, "snapshot") + second = await commit(1, "snapshot") + official = await commit(2, "official") + assert first.supersedes_reporting_revision_id is None + assert second.supersedes_reporting_revision_id == first.reporting_revision_id + assert official.supersedes_reporting_revision_id is None + + restated = await commit(3, "snapshot") + assert restated.supersedes_reporting_revision_id == second.reporting_revision_id + + history = await h.store.list_revisions( + account_id=obligation.account_id, + reporting_obligation_id=obligation.reporting_obligation_id, + ) + selection = select_reporting_revision( + history, + account_id=obligation.account_id, + reporting_obligation_id=obligation.reporting_obligation_id, + required_finality=obligation.required_finality, + ) + assert selection.kind == "selected" + assert selection.revision.reporting_revision_id == official.reporting_revision_id + projection = project_obligation_health( + obligation, history, ledger_as_of=h.clock(), scope_closed=True + ) + assert [i.code for i in projection.issues] == [] + assert projection.current_revision is not None diff --git a/tests/conformance/reporting/test_reporting_status_migration.py b/tests/conformance/reporting/test_reporting_status_migration.py index bb819f01d..d4d96e89f 100644 --- a/tests/conformance/reporting/test_reporting_status_migration.py +++ b/tests/conformance/reporting/test_reporting_status_migration.py @@ -24,7 +24,11 @@ ReportingNotificationWorker, ) from adcp.reporting.outbox._schema import REQUIRED_OBJECTS, schema_objects, validate_schema -from adcp.reporting.outbox.status_schema import REQUIRED_STATUS_OBJECTS, validate_status_schema +from adcp.reporting.outbox.status_schema import ( + REQUIRED_STATUS_OBJECTS, + REQUIRED_STATUS_SELECTOR_OBJECTS, + validate_status_schema, +) from . import test_reporting_notification_process_matrix as _process from ._generation_support import ( @@ -205,9 +209,10 @@ async def test_populated_repeated_c_manifest_preserves_every_a_b_object_and_row( async with pool.connection() as conn: objects = await schema_objects(conn) assert {k: objects[k] for k in REQUIRED_OBJECTS} == REQUIRED_OBJECTS - assert { - k: v for k, v in objects.items() if k not in REQUIRED_OBJECTS - } == REQUIRED_STATUS_OBJECTS + assert {k: v for k, v in objects.items() if k not in REQUIRED_OBJECTS} == { + **REQUIRED_STATUS_OBJECTS, + **REQUIRED_STATUS_SELECTOR_OBJECTS, + } assert ( json.dumps(REQUIRED_STATUS_OBJECTS, sort_keys=True, indent=2) + "\n" == files("adcp.reporting.outbox") diff --git a/tests/conformance/reporting/test_reporting_status_process_matrix.py b/tests/conformance/reporting/test_reporting_status_process_matrix.py index 77a537a51..e3c039441 100644 --- a/tests/conformance/reporting/test_reporting_status_process_matrix.py +++ b/tests/conformance/reporting/test_reporting_status_process_matrix.py @@ -14,6 +14,7 @@ ) from adcp.reporting.ledger.status_snapshot import read_snapshot_on from adcp.reporting.outbox import PgStatusNotificationStore, ReportingStatusSweeper +from adcp.reporting.outbox.status import escalation_identity from adcp.reporting.outbox.status_pg import _replay_storage from ._generation_support import ( @@ -139,13 +140,13 @@ async def database_seed(pool, case="expected", *, baseline=True): await c.execute( "INSERT INTO reporting_status_accounts (account_id, policy)" " VALUES ('acct_a', %s::jsonb)", - (json.dumps(policy.to_wire() if policy else {}),), + (json.dumps(escalation_identity(policy)),), ) await status._apply_on(c, snapshot, through=through, baseline=True) await c.execute( "UPDATE reporting_status_accounts SET baseline_complete=TRUE," " baseline_highwater=%s, dirty_sequence=%s, baseline_at=%s," - " replay_lifecycles=%s::jsonb WHERE account_id='acct_a'", + " replay_lifecycles=%s::jsonb, selector_target_version=2, selector_transition='complete' WHERE account_id='acct_a'", (through, through, snapshot.as_of, _replay_storage(snapshot)), ) return ledger, status, at, escalation @@ -237,7 +238,7 @@ async def test_real_clock_checkpoint_event_crash_restart_converges_once(crash): await child.kill() await settle_status(pool) # Capture expiry into durable state; reclaim uses production DB time. - async with pool.connection() as c: + async with status._transaction("acct_a") as c: await c.execute( "UPDATE reporting_status_scope_checkpoints SET lease_expires_at=clock_timestamp()" " WHERE lease_token IS NOT NULL" diff --git a/tests/conformance/reporting/test_reporting_status_selector_epoch.py b/tests/conformance/reporting/test_reporting_status_selector_epoch.py new file mode 100644 index 000000000..2780ae268 --- /dev/null +++ b/tests/conformance/reporting/test_reporting_status_selector_epoch.py @@ -0,0 +1,429 @@ +"""Shared memory/PostgreSQL cutover, directional scope and replay contract.""" + +from dataclasses import replace +from datetime import timedelta + +import pytest + +from adcp.reporting.ledger import InMemoryReportingLedgerStore +from adcp.reporting.ledger.status_projection import StatusProjectionInput, project_status_scope +from adcp.reporting.outbox import ReportingStatusScope, ReportingStatusSweeper + +from . import test_reporting_status_projection_contract as contract +from ._generation_support import configuration, obligation_for, revision_for +from ._reliable_support import SimulatedCrash + +status_harness = contract.status_harness + + +def old_selected(scope, target): + """Exact v1 scope predicate, frozen here for shared-state vector setup.""" + return ( + scope.account_id == target.account_id + and (scope.consumer_id is None or scope.consumer_id == target.consumer_id) + and ( + scope.generation_key is None + or target.generation_key is None + or scope.generation_key == target.generation_key + ) + and ( + scope.reporting_obligation_id is None + or target.reporting_obligation_id is None + or scope.reporting_obligation_id == target.reporting_obligation_id + ) + ) + + +async def mark_v1(h): + """Restore old metadata only in a private test schema/state image. + + Actual unmodified C binaries establish their own baselines in the rolling + tests. This helper makes the same vectors usable by the memory reference. + """ + if isinstance(h.ledger, InMemoryReportingLedgerStore): + state = h.status._state + state.selector_accounts.clear() + state.accounts = { + a: (seq, {k: v for k, v in policy.items() if k != "selector_semantics_version"}) + for a, (seq, policy) in state.accounts.items() + } + state.checkpoints = { + key: replace(c, selector_semantics_version=1, selector_writer_floor=1) + for key, c in state.checkpoints.items() + } + else: + async with h.ledger._pool.connection() as c, c.transaction(): + await c.execute( + "ALTER TABLE reporting_status_scope_checkpoints" + " DISABLE TRIGGER reporting_status_selector_writer_v2" + ) + await c.execute( + "UPDATE reporting_status_scope_checkpoints" + " SET selector_semantics_version=1, selector_writer_floor=1" + ) + await c.execute( + "UPDATE reporting_status_accounts SET policy=policy-'selector_semantics_version'," + " selector_target_version=1, selector_transition='pending'" + ) + await c.execute( + "ALTER TABLE reporting_status_scope_checkpoints" + " ENABLE TRIGGER reporting_status_selector_writer_v2" + ) + + +async def restart(h): + cls = type(h.status) + await h.reliable.restart() + h.status = cls(h.ledger) + + +async def two_feeds(h, *, issue=True): + owners = {} + for feed in ("analytics", "billing"): + config = replace(configuration(), delivery_config_id=feed, feed_purpose=feed) + await h.ledger.put_configuration(config) + obligation = replace(obligation_for(config), reporting_obligation_id=f"rpo_{feed}") + await h.ledger.commit_obligation(obligation) + revision, rows = revision_for(obligation, suffix=feed) + await h.ledger.commit_revision(revision, rows) + owners[feed] = obligation + issues = {} + if issue: + for key, scope in ( + ( + "billing-partial", + ReportingStatusScope("acct_a", consumer_id="buyer", feed_purpose="billing"), + ), + ("analytics-public", ReportingStatusScope.for_obligation(owners["analytics"])), + ("billing-private", ReportingStatusScope.for_obligation(owners["billing"], "auditor")), + ): + issues[key] = await h.ledger.ensure_issue_opened( + issue_key=key, + account_id="acct_a", + consumer_id=scope.consumer_id, + observed_at=h.clock(), + status_scope=scope, + ) + return owners, issues + + +async def test_directional_feed_public_private_configuration_obligation_scopes(status_harness): + h = status_harness + owners, issues = await two_feeds(h) + snapshot = await h.ledger.read_status_snapshot(account_id="acct_a") + for consumer in (None, "buyer", "auditor"): + for feed in (None, "analytics", "billing"): + for level in ("account", "configuration", "obligation"): + if level != "account" and feed is None: + continue + owner = owners[feed] if feed else None + scope = ReportingStatusScope( + "acct_a", + owner.generation_key if level != "account" else None, + owner.reporting_obligation_id if level == "obligation" else None, + consumer, + feed, + ) + result = project_status_scope(StatusProjectionInput(snapshot, scope)) + expected = set() + if feed in (None, "analytics"): + expected.add(issues["analytics-public"].issue_id) + if feed in (None, "billing") and consumer == "buyer": + expected.add(issues["billing-partial"].issue_id) + if feed in (None, "billing") and consumer == "auditor": + expected.add(issues["billing-private"].issue_id) + assert {i.issue_id for i in result.issues} == expected, (consumer, feed, level) + assert all( + feed is None or o.obligation.feed_purpose == feed for o in result.obligations + ) + + +async def test_contaminated_epoch_corrects_changed_issue_set_once_and_preserves_every_identity( + status_harness, monkeypatch +): + h = status_harness + await two_feeds(h) + import adcp.reporting.ledger.status_projection as projection + + with monkeypatch.context() as patch: + patch.setattr(projection, "_selected", old_selected) + await h.status.baseline(account_id="acct_a") + await mark_v1(h) + before = {c.scope.checkpoint_key: c for c in await h.status.checkpoints(account_id="acct_a")} + assert not await h.status.baseline_ready(account_id="acct_a") + destination_operation_1 = await h.status.rebuild_one() + assert (destination_operation_1).events == 0 # Durable fence only. + fenced = await h.status.checkpoints(account_id="acct_a") + assert all(c.selector_writer_floor == 2 and c.selector_semantics_version == 1 for c in fenced) + destination_operation_2 = await h.status.claim_due(account_id="acct_a") + assert not destination_operation_2 + await restart(h) + await h.drain() + events = await h.status.outbox.list_events(account_id="acct_a") + after = {c.scope.checkpoint_key: c for c in await h.status.checkpoints(account_id="acct_a")} + changed = {k for k, c in after.items() if c.fingerprint != before[k].fingerprint} + assert changed and len(changed) == len(events) == 2 + assert all(e.cause.health == e.cause.previous_health == "action_required" for e in events) + for key, current in after.items(): + old = before[key] + assert current.scope == old.scope and current.baseline == old.baseline + assert current.source_sequence == old.source_sequence + assert ( + current.lease_token == old.lease_token + and current.lease_expires_at == old.lease_expires_at + ) + assert current.generation == old.generation + (key in changed) + assert current.selector_semantics_version == current.selector_writer_floor == 2 + assert await h.status.baseline_ready(account_id="acct_a") + destination_operation_3 = await h.status.rebuild_one() + assert not (destination_operation_3).did_work + destination_operation_4 = await h.status.project_one(account_id="acct_a") + assert not (destination_operation_4).did_work + assert await h.status.checkpoints(account_id="acct_a") == tuple(after.values()) + assert await h.status.outbox.list_events(account_id="acct_a") == events + + +async def test_captured_reversal_boundaries_precede_chronological_overdue_deadlines(status_harness): + h = status_harness + first, revision, _ = await h.seed(readable=True) + config = replace(configuration(), delivery_config_id="second") + await h.ledger.put_configuration(config) + waiting = replace(obligation_for(config), reporting_obligation_id="rpo_waiting") + await h.ledger.commit_obligation(waiting) + h.clock.now = waiting.period.expected_at - timedelta(seconds=5) + await h.status.baseline(account_id="acct_a") + await mark_v1(h) + for readable in (False, True, False, True): + h.clock.advance() + await h.ledger.set_revision_readable( + account_id="acct_a", + reporting_revision_id=revision.reporting_revision_id, + readable=readable, + ) + h.clock.now = waiting.automated_recovery_deadline_at + timedelta(hours=2) + for _ in range(30): + turn = await h.status.rebuild_one() + if not turn.did_work: + break + await restart(h) # Every committed fence/boundary/deadline/final mark survives restart. + else: + pytest.fail("selector epoch did not converge") + events = await h.events() + ready_events = sorted( + ( + e + for e in events + if e.cause.scope.reporting_obligation_id == first.reporting_obligation_id + ), + key=lambda e: e.cause.checkpoint_generation, + ) + assert [e.cause.health for e in ready_events] == [ + "action_required", + "complete", + "action_required", + "complete", + ] + assert [e.cause.previous_health for e in ready_events] == [ + "complete", + "action_required", + "complete", + "action_required", + ] + late = sorted( + ( + e + for e in events + if e.cause.scope.reporting_obligation_id == waiting.reporting_obligation_id + ), + key=lambda e: e.cause.checkpoint_generation, + ) + assert [e.cause.health for e in late] == ["delayed", "action_required"] + assert [e.cause.previous_health for e in late] == ["waiting", "delayed"] + assert len({e.notification_id for e in events}) == len(events) + destination_operation_5 = await ReportingStatusSweeper(h.status).run_once(account_id="acct_a") + assert not (destination_operation_5).did_work + + +@pytest.mark.parametrize("fail_at", [1, 2, 4]) +async def test_checkpoint_event_failure_rolls_back_epoch_turn_and_restart_is_once_only( + status_harness, monkeypatch, fail_at +): + h = status_harness + await two_feeds(h, issue=False) + await h.status.baseline(account_id="acct_a") + await mark_v1(h) + destination_operation_6 = await h.status.rebuild_one() + assert (destination_operation_6).did_work + await h.ledger.ensure_issue_opened( + issue_key="late-public", + account_id="acct_a", + consumer_id=None, + observed_at=h.clock(), + status_scope=ReportingStatusScope("acct_a"), + ) + before = await h.status.checkpoints(account_id="acct_a") + from adcp.reporting.outbox import status_memory, status_pg + + original = status_pg.advance_checkpoint + called = 0 + + def crash(*args, **kwargs): + nonlocal called + called += 1 + if called == fail_at: + raise SimulatedCrash("selector-checkpoint-event-pre-commit") + return original(*args, **kwargs) + + with monkeypatch.context() as patch: + patch.setattr(status_memory, "advance_checkpoint", crash) + patch.setattr(status_pg, "advance_checkpoint", crash) + with pytest.raises(SimulatedCrash): + await h.status.rebuild_one() + assert await h.status.checkpoints(account_id="acct_a") == before + assert not await h.status.outbox.list_events(account_id="acct_a") + await restart(h) + await h.drain() + events = await h.status.outbox.list_events(account_id="acct_a") + assert len(events) == 4 and {e.cause.checkpoint_generation for e in events} == {1} + destination_operation_7 = await h.status.rebuild_one() + assert not (destination_operation_7).did_work + + +@pytest.mark.parametrize("phase", ["fence", "final_mark"]) +async def test_crash_before_fence_or_final_mark_commit_is_restartable( + status_harness, monkeypatch, phase +): + h = status_harness + await h.seed(readable=True) + await h.status.baseline(account_id="acct_a") + await mark_v1(h) + if phase == "final_mark": + destination_operation_12 = await h.status.rebuild_one() + assert (destination_operation_12).did_work + before = await h.status.checkpoints(account_id="acct_a") + name = "_rebuild" if isinstance(h.ledger, InMemoryReportingLedgerStore) else "_rebuild_on" + original = getattr(h.status, name) + + def memory_crash(*args): + original(*args) + raise SimulatedCrash("selector-epoch-pre-commit") + + async def pg_crash(*args): + await original(*args) + raise SimulatedCrash("selector-epoch-pre-commit") + + with monkeypatch.context() as patch: + patch.setattr(h.status, name, memory_crash if name == "_rebuild" else pg_crash) + with pytest.raises(SimulatedCrash): + await h.status.rebuild_one() + assert await h.status.checkpoints(account_id="acct_a") == before + assert not await h.status.baseline_ready(account_id="acct_a") + await restart(h) + await h.drain() + assert await h.status.baseline_ready(account_id="acct_a") + assert not await h.status.outbox.list_events(account_id="acct_a") + + +async def test_retained_scopes_outside_current_discovery_are_reprojected_without_deletion( + status_harness, monkeypatch +): + h = status_harness + await h.seed(readable=True) + await h.status.baseline(account_id="acct_a") + await mark_v1(h) + before = await h.status.checkpoints(account_id="acct_a") + expired = h.clock() - timedelta(seconds=1) + if isinstance(h.ledger, InMemoryReportingLedgerStore): + h.status._state.checkpoints = { + key: replace(c, next_due_at=expired) for key, c in h.status._state.checkpoints.items() + } + else: + async with h.ledger._pool.connection() as c: + await c.execute( + "UPDATE reporting_status_scope_checkpoints SET next_due_at=%s", (expired,) + ) + from adcp.reporting.outbox import status_memory, status_pg + + with monkeypatch.context() as patch: + patch.setattr(status_memory, "projection_scopes", lambda snapshot: ()) + patch.setattr(status_pg, "projection_scopes", lambda snapshot: ()) + await h.drain() + after = await h.status.checkpoints(account_id="acct_a") + assert len(after) == len(before) + assert [(c.scope, c.fingerprint, c.generation, c.baseline) for c in after] == [ + (c.scope, c.fingerprint, c.generation, c.baseline) for c in before + ] + assert all(c.selector_semantics_version == 2 and c.next_due_at is None for c in after) + destination_operation_8 = await h.status.rebuild_one() + assert not (destination_operation_8).did_work + assert not await h.status.outbox.list_events(account_id="acct_a") + + +async def test_account_epoch_readiness_and_new_baseline_are_isolated(status_harness): + h = status_harness + for account in ("acct_a", "acct_b"): + await h.ledger.put_configuration(configuration(account)) + await h.status.baseline(account_id=account) + await mark_v1(h) + assert not await h.status.baseline_ready(account_id="acct_a") + assert not await h.status.baseline_ready(account_id="acct_b") + await h.drain() + assert await h.status.baseline_ready(account_id="acct_a") + assert not await h.status.baseline_ready(account_id="acct_b") + await restart(h) + assert await h.status.baseline_ready(account_id="acct_a") + assert not await h.status.baseline_ready(account_id="acct_b") + destination_operation_9 = await h.status.baseline(account_id="new-account") + assert destination_operation_9 + assert await h.status.baseline_ready(account_id="new-account") + assert not await h.status.outbox.list_events(account_id="new-account") + + +async def test_old_memory_state_image_without_any_epoch_fields_is_imported_as_v1(): + from types import SimpleNamespace + + from adcp.reporting.outbox import InMemoryStatusNotificationStore + + from ._reliable_support import reliable_factory + + async with reliable_factory("memory", notifications=True) as reliable: + h = contract.StatusHarness(reliable, InMemoryStatusNotificationStore(reliable.store)) + await h.seed(readable=True) + await h.status.baseline(account_id="acct_a") + before = await h.status.checkpoints(account_id="acct_a") + state = h.status._state + del state.selector_accounts + state.accounts = {a: (seq, {}) for a, (seq, _) in state.accounts.items()} + state.checkpoints = { + key: SimpleNamespace( + **{k: v for k, v in vars(c).items() if not k.startswith("selector_")} + ) + for key, c in state.checkpoints.items() + } + h.status = InMemoryStatusNotificationStore(h.ledger) + assert all( + c.selector_semantics_version == 1 + for c in await h.status.checkpoints(account_id="acct_a") + ) + assert not await h.status.baseline_ready(account_id="acct_a") + await h.drain() + after = await h.status.checkpoints(account_id="acct_a") + assert after == before + assert not await h.status.outbox.list_events(account_id="acct_a") + + +async def test_existing_c_service_discovers_old_accounts_without_an_account_list(status_harness): + from types import SimpleNamespace + + from adcp.reporting.outbox import ReportingStatusService + + h = status_harness + await h.seed(readable=True) + await h.status.baseline(account_id="acct_a") + await mark_v1(h) + service = ReportingStatusService(SimpleNamespace(store=h.status, account_ids=())) + destination_operation_10 = await service.drain() + assert destination_operation_10 >= 2 + assert await h.status.baseline_ready(account_id="acct_a") + destination_operation_11 = await service.drain() + assert destination_operation_11 == 0 diff --git a/tests/conformance/reporting/test_reporting_status_selector_pg.py b/tests/conformance/reporting/test_reporting_status_selector_pg.py new file mode 100644 index 000000000..7ac7fc318 --- /dev/null +++ b/tests/conformance/reporting/test_reporting_status_selector_pg.py @@ -0,0 +1,455 @@ +"""Populated real-C cutover, immutable retention and old row-only lease races.""" + +import asyncio +import json +import subprocess +import sys +from contextlib import asynccontextmanager +from dataclasses import replace +from datetime import datetime, timedelta, timezone +from importlib.resources import files +from pathlib import Path + +import pytest + +from adcp.reporting.ledger import PgReportingReconciliationStore +from adcp.reporting.outbox import ( + ActivityRequest, + PgStatusNotificationStore, + ReportingEnvelopeCipher, + ReportingNotificationWorker, + ReportingStatusSweeper, +) +from adcp.reporting.outbox.status_schema import validate_status_schema + +from ._generation_support import ( + NOW, + configuration, + isolated_reporting_pool, + obligation_for, + require_rolling_database, + revision_for, +) +from ._reliable_support import ( + FailurePlan, + ScriptedSigning, + ScriptedSubscriptions, + notification_subscription, +) +from .test_reporting_notification_migration import retained_physical_rows +from .test_reporting_status_migration import C_QUEUES, physical_rows + +ROOT = Path(__file__).resolve().parents[3] +C_SHA = "967b6e286301d7e5d089aea6fdbb90bea8ee5a16" + + +@pytest.fixture(scope="module") +def frozen_c(tmp_path_factory): + require_rolling_database() + root = tmp_path_factory.mktemp("frozen-c-selector") / "source" + subprocess.run( + ["git", "worktree", "add", "--detach", str(root), C_SHA], + cwd=ROOT, + check=True, + capture_output=True, + timeout=60, + ) + try: + yield root + finally: + subprocess.run( + ["git", "worktree", "remove", "--force", str(root)], + cwd=ROOT, + check=True, + capture_output=True, + timeout=60, + ) + + +class OldC: + def __init__(self, process): + self.process = process + + async def send(self, **command): + self.process.stdin.write( + (json.dumps(command, default=lambda v: v.isoformat()) + "\n").encode() + ) + await self.process.stdin.drain() + + async def receive(self): + line = await asyncio.wait_for(self.process.stdout.readline(), 30) + assert line, "frozen C process exited before replying" + return json.loads(line) + + async def call(self, **command): + await self.send(**command) + return await self.receive() + + +@asynccontextmanager +async def old_c(pool, source, *, now=NOW): + process = await asyncio.create_subprocess_exec( + sys.executable, + "-I", + str(Path(__file__).with_name("_frozen_status_c.py")), + stdin=asyncio.subprocess.PIPE, + stdout=asyncio.subprocess.PIPE, + stderr=asyncio.subprocess.PIPE, + ) + child = OldC(process) + try: + await child.send(source=str(source), conninfo=pool.conninfo, kwargs=pool.kwargs, now=now) + destination_operation_3 = await child.receive() + assert Path((destination_operation_3)["origin"]).is_relative_to(source) + yield child + destination_operation_4 = await child.call(action="stop") + assert destination_operation_4 == {"stopped": True} + await asyncio.wait_for(process.wait(), 10) + assert process.returncode == 0 + finally: + if process.returncode is None: + process.kill() + await process.wait() + error = await process.stderr.read() + assert not error, error.decode() + + +async def seed(ledger, *, account="acct_a", readable=False, name="daily"): + config = replace(configuration(account), delivery_config_id=name) + await ledger.put_configuration(config) + obligation = replace(obligation_for(config), reporting_obligation_id=f"rpo_{account}_{name}") + await ledger.commit_obligation(obligation) + revision, rows = revision_for(obligation, suffix=name) + if readable: + await ledger.commit_revision(revision, rows) + return obligation, revision + + +async def drain(status, account="acct_a"): + for _ in range(40): + if not (await status.project_one(account_id=account)).did_work: + return + pytest.fail("selector migration failed to converge") + + +async def populate_status_activity(status): + failures = FailurePlan() + subscriptions = ScriptedSubscriptions(failures) + subscriptions.put(notification_subscription(events=("reporting.status_changed",))) + worker = ReportingNotificationWorker( + outbox=status.outbox, + subscriptions=subscriptions, + signing=ScriptedSigning(failures), + cipher=ReportingEnvelopeCipher(b"e" * 32), + activity=status.outbox, + ) + while await worker.expand_one(account_id="acct_a"): + pass + now = datetime.now(timezone.utc) + lease = await status.outbox.claim_delivery(account_id="acct_a", now=now, lease_seconds=60) + assert lease is not None + destination_operation_1 = await status.outbox.reserve_attempt( + lease, request=ActivityRequest("https://receiver.example.test/reporting", 1), now=now + ) + assert destination_operation_1 + destination_operation_2 = await status.outbox.finish_delivery( + lease, state="pending", retry_at=now, now=now + ) + assert destination_operation_2 + + +async def test_actual_c_claim_fence_pool_marker_two_projectors_two_sweepers_and_source(frozen_c): + from psycopg_pool import AsyncConnectionPool + + async with isolated_reporting_pool(autocommit=True) as owner, old_c(owner, frozen_c) as old: + destination_operation_5 = await old.call(action="schema") + assert destination_operation_5 == {"result": True} + clock = [NOW] + ledger = PgReportingReconciliationStore( + pool=owner, notifications=True, clock=lambda: clock[0] + ) + waiting, _ = await seed(ledger) + _, ready = await seed(ledger, readable=True, name="ready") + await seed(ledger, account="acct_b") + baseline_at = waiting.period.expected_at - timedelta(seconds=1) + for account in ("acct_a", "acct_b"): + destination_operation_34 = await old.call( + action="baseline", account=account, now=baseline_at + ) + assert destination_operation_34 == {"result": True} + destination_operation_35 = await old.call(action="ready", account=account) + assert destination_operation_35 == {"result": True} + # A populated C event and every C activity table exist before cutover. + await old.call( + action="source", revision=ready.reporting_revision_id, readable=False, now=baseline_at + ) + destination_operation_6 = await old.call(action="project") + assert (destination_operation_6)["result"]["events"] == 2 + status = PgStatusNotificationStore(ledger) + await populate_status_activity(status) + immutable = await retained_physical_rows(owner) + queue_rows = await physical_rows(owner) + assert all(queue_rows.values()) + await status.create_schema() + assert await retained_physical_rows(owner) == immutable + assert await physical_rows(owner) == queue_rows + assert not await status.baseline_ready(account_id="acct_a") + destination_operation_7 = await old.call(action="ready") + assert destination_operation_7 == {"result": True} # Schema alone isn't cutover. + # Old claim owns only a checkpoint row. The new fence owns the account + # then waits for that row; the old trigger must never acquire account. + await old.send(action="claim", now=NOW, hold="claim") + destination_operation_8 = await old.receive() + assert destination_operation_8 == {"held": True} + fencing = asyncio.create_task(status.project_one(account_id="acct_a")) + await asyncio.sleep(0.02) + assert not fencing.done() + await old.send(action="release_hold") + old_lease = (await old.receive())["result"] + assert old_lease is not None + destination_operation_9 = await asyncio.wait_for(fencing, 10) + assert (destination_operation_9).events == 0 + # The fence preserves existing lease identity; old claims/completion + # fail before a write, while v2 migration doesn't wait for lease expiry. + assert any( + c.lease_token == old_lease["token"] + for c in await status.checkpoints(account_id="acct_a") + ) + fenced_rows = await physical_rows( + owner, + tables=("reporting_status_accounts", "reporting_status_scope_checkpoints", *C_QUEUES), + ) + destination_operation_10 = await old.call(action="claim") + assert (destination_operation_10)["error"] == "database_fence" + destination_operation_11 = await old.call(action="complete", lease=0) + assert (destination_operation_11)["error"] == "status_policy_conflict" + destination_operation_12 = await old.call(action="release", lease=0) + assert (destination_operation_12)["error"] == "database_fence" + destination_operation_13 = await old.call(action="ready") + assert (destination_operation_13)["error"] == "status_policy_conflict" + destination_operation_14 = await old.call(action="project") + assert (destination_operation_14)["error"] == "status_policy_conflict" + assert await physical_rows(owner, tables=tuple(fenced_rows)) == fenced_rows + destination_operation_15 = await old.call(action="schema") + assert destination_operation_15 == {"result": True} + assert await physical_rows(owner, tables=tuple(fenced_rows)) == fenced_rows + async with owner.connection() as c: + await validate_status_schema(c, activity=True) + assert ( + await ( + await c.execute( + "SELECT current_setting('adcp.reporting.selector_semantics_version',true)" + ) + ).fetchone() + )[0] in (None, "") + # Four distinct size-one pools exercise the account lock across every + # new projector/sweeper path; source dirties arrive concurrently. + pools = [ + AsyncConnectionPool( + owner.conninfo, kwargs=owner.kwargs, min_size=1, max_size=1, open=False + ) + for _ in range(4) + ] + try: + for pool in pools: + await pool.open(wait=True) + stores = [ + PgStatusNotificationStore( + PgReportingReconciliationStore( + pool=p, notifications=True, clock=lambda: clock[0] + ) + ) + for p in pools + ] + + async def sweep(s): + for _ in range(12): + await ReportingStatusSweeper(s).run_once(account_id="acct_a") + + await asyncio.wait_for( + asyncio.gather( + drain(stores[0]), + drain(stores[1]), + sweep(stores[2]), + sweep(stores[3]), + ledger.set_revision_readable( + account_id="acct_a", + reporting_revision_id=ready.reporting_revision_id, + readable=True, + ), + ), + 25, + ) + await drain(stores[0]) + for pool in pools: + async with pool.connection() as c: + assert ( + await ( + await c.execute( + "SELECT current_setting(" + "'adcp.reporting.selector_semantics_version',true)" + ) + ).fetchone() + )[0] in (None, "") + finally: + for pool in pools: + await pool.close() + assert await status.baseline_ready(account_id="acct_a") + assert not await status.baseline_ready(account_id="acct_b") + destination_operation_16 = await old.call(action="ready") + assert (destination_operation_16)["error"] == "status_policy_conflict" + destination_operation_17 = await old.call(action="ready", account="acct_b") + assert destination_operation_17 == {"result": True} + clock[0] = waiting.automated_recovery_deadline_at + destination_operation_18 = await ReportingStatusSweeper(status).run_once( + account_id="acct_a" + ) + assert (destination_operation_18).did_work + events = await status.outbox.list_events(account_id="acct_a") + late = sorted( + ( + e + for e in events + if e.cause.scope.reporting_obligation_id == waiting.reporting_obligation_id + ), + key=lambda e: e.cause.checkpoint_generation, + ) + assert [(e.cause.previous_health, e.cause.health) for e in late] == [ + ("waiting", "delayed"), + ("delayed", "action_required"), + ] + assert len( + {(e.cause.scope.checkpoint_key, e.cause.checkpoint_generation) for e in events} + ) == len(events) + after = await physical_rows(owner) + assert all(all(row in after[table] for row in rows) for table, rows in queue_rows.items()) + destination_operation_19 = await status.baseline(account_id="brand-new") + assert destination_operation_19 + assert await status.baseline_ready(account_id="brand-new") + assert not await status.outbox.list_events(account_id="brand-new") + + +async def test_actual_c_unique_looking_snapshot_corruption_corrects_without_baseline_reset( + frozen_c, +): + async with isolated_reporting_pool(autocommit=True) as pool, old_c(pool, frozen_c) as old: + destination_operation_20 = await old.call(action="schema") + assert destination_operation_20 == {"result": True} + ledger = PgReportingReconciliationStore(pool=pool, notifications=True, clock=lambda: NOW) + obligation, _ = await seed(ledger, readable=True) + second, rows = revision_for(obligation, suffix="disconnected") + await ledger.commit_revision(second, rows) # v1 persistence accepts another snapshot root. + destination_operation_21 = await old.call(action="baseline") + assert destination_operation_21 == {"result": True} + status = PgStatusNotificationStore(ledger) + immutable = await retained_physical_rows(pool) + await status.create_schema() + before = await status.checkpoints(account_id="acct_a") + assert all( + c.snapshot["health"] == "complete" and c.selector_semantics_version == 1 for c in before + ) + await drain(status) + after = await status.checkpoints(account_id="acct_a") + events = await status.outbox.list_events(account_id="acct_a") + assert len(events) == 2 + assert all( + e.cause.previous_health == "complete" and e.cause.health == "action_required" + for e in events + ) + assert all(c.snapshot["issues"][0]["code"] == "HISTORY_UNAVAILABLE" for c in after) + assert [(c.scope, c.baseline, c.source_sequence) for c in after] == [ + (c.scope, c.baseline, c.source_sequence) for c in before + ] + assert await retained_physical_rows(pool) == immutable + physical = await physical_rows( + pool, + tables=("reporting_status_accounts", "reporting_status_scope_checkpoints", *C_QUEUES), + ) + destination_operation_22 = await status.rebuild_one() + assert not (destination_operation_22).did_work + destination_operation_23 = await status.project_one(account_id="acct_a") + assert not (destination_operation_23).did_work + assert await physical_rows(pool, tables=tuple(physical)) == physical + + +async def test_actual_c_inflight_projector_commits_before_v2_fence_and_source_serializes(frozen_c): + async with isolated_reporting_pool(autocommit=True) as pool, old_c(pool, frozen_c) as old: + destination_operation_24 = await old.call(action="schema") + assert destination_operation_24 == {"result": True} + ledger = PgReportingReconciliationStore(pool=pool, notifications=True, clock=lambda: NOW) + _, revision = await seed(ledger, readable=True) + destination_operation_25 = await old.call(action="baseline") + assert destination_operation_25 == {"result": True} + await old.call(action="source", revision=revision.reporting_revision_id, readable=False) + status = PgStatusNotificationStore(ledger) + await status.create_schema() + await old.send(action="project", hold="project") + destination_operation_26 = await old.receive() + assert destination_operation_26 == {"held": True} + fence = asyncio.create_task(status.project_one(account_id="acct_a")) + publication = asyncio.create_task( + ledger.set_revision_readable( + account_id="acct_a", + reporting_revision_id=revision.reporting_revision_id, + readable=True, + ) + ) + await asyncio.sleep(0.02) + assert not fence.done() and not publication.done() + await old.send(action="release_hold") + destination_operation_27 = await old.receive() + assert (destination_operation_27)["result"]["events"] == 2 + destination_operation_28 = await asyncio.wait_for(fence, 10) + assert (destination_operation_28).events == 0 + await asyncio.wait_for(publication, 10) + old_events = await physical_rows(pool) + await drain(status) + assert await status.baseline_ready(account_id="acct_a") + events = await status.outbox.list_events(account_id="acct_a") + assert len(events) == 4 + assert all( + c.snapshot["health"] == "complete" + for c in await status.checkpoints(account_id="acct_a") + ) + after = await physical_rows(pool) + assert all(all(row in after[table] for row in rows) for table, rows in old_events.items()) + destination_operation_29 = await old.call(action="project") + assert (destination_operation_29)["error"] == "status_policy_conflict" + + +async def test_populated_selector_sql_rollback_and_repeated_concurrent_install(frozen_c): + async with isolated_reporting_pool(autocommit=True) as pool, old_c(pool, frozen_c) as old: + destination_operation_30 = await old.call(action="schema") + assert destination_operation_30 == {"result": True} + ledger = PgReportingReconciliationStore(pool=pool, notifications=True, clock=lambda: NOW) + await seed(ledger, readable=True) + destination_operation_31 = await old.call(action="baseline") + assert destination_operation_31 == {"result": True} + rows = await physical_rows( + pool, + tables=("reporting_status_accounts", "reporting_status_scope_checkpoints", *C_QUEUES), + ) + immutable = await retained_physical_rows(pool) + migration = ( + files("adcp.reporting.ledger") + .joinpath("reporting_status_selector_version.sql") + .read_text() + ) + with pytest.raises(RuntimeError, match="interrupt migration"): + async with pool.connection() as c, c.transaction(): + await c.execute(migration) + raise RuntimeError("interrupt migration") + assert await physical_rows(pool, tables=tuple(rows)) == rows + assert await retained_physical_rows(pool) == immutable + destination_operation_32 = await old.call(action="ready") + assert destination_operation_32 == {"result": True} + status = PgStatusNotificationStore(ledger) + await asyncio.wait_for(asyncio.gather(status.create_schema(), status.create_schema()), 30) + await status.create_schema() + assert await retained_physical_rows(pool) == immutable + assert not await status.baseline_ready(account_id="acct_a") + destination_operation_33 = await old.call(action="ready") + assert destination_operation_33 == {"result": True} + await drain(status) + assert await status.baseline_ready(account_id="acct_a") + assert not await status.outbox.list_events(account_id="acct_a") diff --git a/tests/test_reliable_reporting_service.py b/tests/test_reliable_reporting_service.py index 0f0c69097..e62500ff1 100644 --- a/tests/test_reliable_reporting_service.py +++ b/tests/test_reliable_reporting_service.py @@ -226,9 +226,12 @@ async def get_media_buy_delivery(self, params: Any, context: Any = None) -> Any: assert {"get_products", "get_reporting_status", "get_media_buy_delivery"} <= tools -async def test_capability_block_is_schema_valid_and_only_advertises_installed_tiers() -> None: +@pytest.mark.parametrize("consumer_status_enabled", [False, True]) +async def test_capability_block_is_schema_valid_and_only_advertises_installed_tiers( + consumer_status_enabled: bool, +) -> None: service = ReliableReportingService.memory( - account_context=_account_context, consumer_status_enabled=True + account_context=_account_context, consumer_status_enabled=consumer_status_enabled ) service.sources.register("gam", ScriptedReportingAdapter(redacted_capabilities(), [_rows(1)])) await service.configure(_configuration()) @@ -236,7 +239,11 @@ async def test_capability_block_is_schema_valid_and_only_advertises_installed_ti assert block["managed_delivery"] is False assert block["reconciled_billing"] is False - assert block["consumer_status_task"] == "sync_reporting_status" + assert "receipt_task" not in block + if consumer_status_enabled: + assert block["consumer_status_task"] == "sync_reporting_status" + else: + assert "consumer_status_task" not in block validator = get_named_validator("core/reporting-delivery-capabilities.json") assert validator is not None assert list(validator.iter_errors(block)) == [] diff --git a/tests/test_reporting_consumer_loop.py b/tests/test_reporting_consumer_loop.py index 9dd4f2d68..69b9a878c 100644 --- a/tests/test_reporting_consumer_loop.py +++ b/tests/test_reporting_consumer_loop.py @@ -774,9 +774,29 @@ def test_an_obligation_with_a_revision_and_no_reading_refuses_to_guess() -> None [_obligation(revision_count=1)], now=EXPECTED_AT + RECOVERY, automated_recovery_window=RECOVERY, + obligation_revisions={"rpo_1": [_revision()]}, ) +@pytest.mark.parametrize( + "history,count", + [(None, 1), ([], 1), ([_revision()], 2), ([_revision(), _revision()], 2)], +) +def test_an_incomplete_obligation_partition_names_obligation_revisions(history, count) -> None: + # The missing input is the obligation's retained history, not a reading: + # a message that says "reading" sends the adopter to fix the wrong + # argument and fail again on the next turn. + with pytest.raises(ConsumerStatusPlanError, match="obligation_revisions") as caught: + plan_consumer_statuses( + [_obligation(revision_count=count)], + now=EXPECTED_AT + RECOVERY, + automated_recovery_window=RECOVERY, + obligation_revisions=None if history is None else {"rpo_1": history}, + ) + assert "rpo_1" in str(caught.value) + assert "no reading was supplied" not in str(caught.value) + + def test_required_finality_decides_whether_a_revision_counts() -> None: # An obligation needing `official` is not satisfied by snapshots, so a # snapshot-only ledger really has no required revision. diff --git a/tests/type_checks/reporting_destination_writer.py b/tests/type_checks/reporting_destination_writer.py new file mode 100644 index 000000000..c9d42a325 --- /dev/null +++ b/tests/type_checks/reporting_destination_writer.py @@ -0,0 +1,134 @@ +"""Public B1 adoption, without private model imports, Any, or type suppressions.""" + +from __future__ import annotations + +import asyncio +from collections.abc import AsyncIterator, Sequence +from dataclasses import dataclass +from datetime import datetime +from typing import Literal + +from adcp.reporting.ledger import ( + ReportingObligationRecord, + ReportingRevisionRecord, + select_reporting_revision, +) +from adcp.reporting.materializer import ( + ReferenceReportingDestinationWriter, + ReferenceReportingResolver, + ReportingDestinationBinding, + ReportingDestinationIO, + ReportingDestinationLocator, + ReportingDestinationPage, + ReportingDestinationRequest, + ReportingDestinationResolver, + ReportingDestinationSession, + ReportingDestinationWriter, + ReportingIOContext, + ReportingIOPhase, + ReportingMaterializationAttempt, + ReportingNativeObservation, + ReportingObligationDeliveryRecord, + ReportingPreparedRevision, + ReportingRevisionRowReader, + ReportingRevisionVerifierRegistry, + ReportingVerifiedDestination, + reference_verifier, +) + + +@dataclass(frozen=True) +class TrustedResolver: + """Real adopters acquire credentials in session._open and release in _close.""" + + delegate: ReferenceReportingResolver + + def resolve( + self, + request: ReportingDestinationRequest, + *, + phase: ReportingIOPhase, + context: ReportingIOContext, + ) -> ReportingDestinationSession: + return self.delegate.resolve(request, phase=phase, context=context) + + +class AdopterSession(ReportingDestinationSession): + """Typed provider hooks; the SDK owns redaction and the async context lifecycle.""" + + async def _open(self) -> None: + pass + + async def _close(self) -> None: + pass + + async def write(self, content: ReportingPreparedRevision) -> ReportingDestinationLocator: + raise NotImplementedError + + async def read_rows( + self, locator: ReportingDestinationLocator, *, cursor: str | None, limit: int + ) -> ReportingDestinationPage: + raise NotImplementedError + + async def read_manifest(self, locator: ReportingDestinationLocator) -> bytes: + raise NotImplementedError + + async def list_objects(self, locator: ReportingDestinationLocator) -> tuple[str, ...]: + raise NotImplementedError + + async def read_object( + self, locator: ReportingDestinationLocator, *, object_ref: str + ) -> AsyncIterator[bytes]: + yield b"" + + async def observe_native_version( + self, locator: ReportingDestinationLocator + ) -> ReportingNativeObservation: + raise NotImplementedError + + +async def adopter( + reader: ReportingRevisionRowReader, + binding: ReportingDestinationBinding, + delivery: ReportingObligationDeliveryRecord, + obligation: ReportingObligationRecord, + history: Sequence[ReportingRevisionRecord], + attempt: ReportingMaterializationAttempt, + deadline: datetime, +) -> ReportingVerifiedDestination | None: + selection = select_reporting_revision( + history, + account_id=obligation.account_id, + reporting_obligation_id=obligation.reporting_obligation_id, + required_finality=obligation.required_finality, + ) + if selection.kind == "corrupt" or selection.kind == "not_ready": + reason: str = selection.reason + assert reason + return None + chosen: ReportingRevisionRecord = selection.revision + assert chosen.reporting_revision_id == attempt.reporting_revision_id + verifier = reference_verifier() + registry = ReportingRevisionVerifierRegistry((verifier,)) + reference = ReferenceReportingDestinationWriter((verifier.key.capability,)) + production: Literal[False] = reference.production_eligible + assert production is False + writer: ReportingDestinationWriter = reference + assert writer.capabilities + resolver: ReportingDestinationResolver = TrustedResolver( + ReferenceReportingResolver(reference, registry, (binding,)) + ) + cancel = asyncio.Event() + prepared = await registry.prepare( + key=verifier.key, + binding=binding, + delivery=delivery, + obligation=obligation, + revisions=history, + attempt=attempt, + reader=reader, + context=ReportingIOContext(deadline, cancel), + ) + io = ReportingDestinationIO(registry, resolver) + locator = await io.write(prepared, context=ReportingIOContext(deadline, cancel)) + return await io.verify(prepared, locator, context=ReportingIOContext(deadline, cancel))