diff --git a/docs/catalog-curation-backlog.md b/docs/catalog-curation-backlog.md index d729b6f1..64acfd0b 100644 --- a/docs/catalog-curation-backlog.md +++ b/docs/catalog-curation-backlog.md @@ -171,6 +171,35 @@ were closed on 2026-08-09 by chronicle#153: replay rejects the hand-forged plan before any write reaches disk, and the builder end-to-end refuses the stale catalog, leaving both artifacts untouched. +- 2026-10-16: #153 added the live-binding check but kept the first-owner + conjunct (`uuid_owner[row_uuid] == prior_own_key`) alongside it. Because + the first owner never moves, that conjunct refused every second + enrichment hop. A placeholder that had already been enriched once (a + geography-free placeholder enriched to geography-only when its docket + entry learned a country) could not take its first print. + `usaspending.dod.prime_award_obligations` (a2649fd9) hit this in Thesis + run 38962977031. The conjunct is gone (#360). The contract is now the + live binding plus the grammar's hand-over rules: the placeholder's key + is live and bound to the row's UUID, it is the same concept as the + identity it hands over to, and it has not handed over before. A key that + handed over and later returned as the successor of another hop is live + again but stays consumed. When the binding holds but either hand-over + rule fails, the builder refuses at plan time and names the cause. + Tests pin the chain, the stale-row refusals, both hand-over refusals + and the curation each refusal suggests. A property test checks that the + builder plans an enrichment exactly when the registry grammar admits + the retire + succeeds pair. +- 2026-10-18: when a catalog row's UUID differed from its key's binding, + `resolve_uuid` planned the curator remint (revive the key if retired, + then supersede it to the row's UUID) even when the key had retired after + handing its lineage over. The grammar refuses to revive a handed-over + key, so staging rejected the plan before any write. A docket entry that + gains a country reaches this: it re-keys a geography-free placeholder + row onto the country key (finding L2 of #360's second review). The + builder now reclaims the key on the row's UUID. That needs no ceremony, + because no live binding changes. A row whose UUID the registry already + knows is refused at plan time, and the message names the cause. The + enrichment differential now has a docket call-site arm as well. ## Country geography on Colorado-scoped concepts, and one raw series id (2026-09-04) diff --git a/scripts/build_series_catalog.py b/scripts/build_series_catalog.py index 8b6f47c9..1fc57ca5 100644 --- a/scripts/build_series_catalog.py +++ b/scripts/build_series_catalog.py @@ -1416,7 +1416,7 @@ class UuidRegistry: """The append-only UUID minting ledger. One JSON object per line, binding one identity (concept, geography - level/id/vintage, entity name/role) to a UUID. Five event kinds, chained + level/id/vintage, entity name/role) to a UUID. Six event kinds, chained per identity and never edited or removed (``--verify-registry-append- only`` and the git-HEAD prefix check enforce growth-only): @@ -1431,7 +1431,12 @@ class UuidRegistry: * retire — ``retired: true`` with the unchanged UUID; ``note`` required. The identity left the catalog; its binding stays reserved but dormant. * revive — ``revived: true`` with the unchanged UUID; written - automatically when a retired identity is observed again. + automatically when a retired identity is observed again. Never for + a key whose lineage was handed over (consumed by a succeeds event): + that key is terminal. + * reclaim — ``reclaimed: true`` on a retired, handed-over key, with a + UUID new to the registry; ``note`` required. The key starts a NEW + lineage and is ordinary again (later drops retire, returns revive). Invariants ``--check`` builds on: live bindings and catalog rows are in BIJECTION (same identity, same UUID, both directions), and live @@ -2129,14 +2134,21 @@ def _plan_enrichment( entity: dict | None, row_uuid: str, ) -> bool: - """Plan a retire + succeeds pair when ``prior`` is a genuine - placeholder whose live binding owns ``row_uuid``; True if planned. + """Plan a retire + succeeds pair when ``prior`` is a docket + placeholder that holds ``row_uuid`` live and may hand it over to + ``canon_key``; True if planned. + + A placeholder that holds ``row_uuid`` live but may not hand it over + (``canon_key`` is another concept, or the placeholder already + handed over once) raises a diagnostic naming that cause. Every + caller would refuse such a row anyway, since the UUID is + registered, but with a UUID-reuse or remint message that misstates + why. """ prior_own_key = UuidRegistry.entry_key(prior) - owner_key = registry.uuid_owner.get(uuid_module.UUID(row_uuid).int) prior_geo = prior.get("geography") or None geo = geography or {} - enrichment_shaped = ( + placeholder_shaped = ( prior.get("status") == "docket-only" and prior.get("entity") is None and ( @@ -2149,23 +2161,57 @@ def _plan_enrichment( ) ) ) - # uuid_owner records FIRST ownership and never moves, so alone it - # would accept a stale catalog row still carrying a UUID its key - # has since superseded away from. The contract is about the LIVE - # binding: demand it be row_uuid itself. Staging replay would - # reject such a plan anyway; this fails at plan time instead. + # The contract is the LIVE binding plus the grammar's hand-over + # rules. The row's own key must be live and bound to row_uuid right + # now: that refuses a stale row still carrying a UUID its key has + # since superseded away from, and a key that was retired, handed + # over or reclaimed. Staging replay would reject each of these + # plans anyway; this fails at plan time instead. Historical + # ownership (uuid_owner) is not part of the contract: it records + # FIRST ownership and never moves, so a placeholder that was itself + # the successor of an earlier enrichment never owns its UUID and + # could never be enriched again. if not ( - enrichment_shaped - and owner_key == prior_own_key + placeholder_shaped and registry.is_live(prior_own_key) and registry.binding(prior_own_key) == row_uuid ): return False + # Lineage never crosses concepts. Observed buckets always take the + # prior's concept; a docket entry matched through a curated alias + # can name another one, and the grammar refuses that succeeds. + if prior.get("concept") != canon_key[0]: + raise SystemExit( + f"identity {canon_key} matches docket placeholder " + f"{prior_own_key} (uuid {row_uuid}) only through the " + f"curated alias {canon_key[0]!r}; lineage never crosses " + "concepts, so the placeholder cannot be enriched into it — " + f"name the docket entry {prior['concept']!r}, the " + "placeholder's own concept" + ) + # A key hands over at most once. One that handed over and later + # returned as the successor of another hop is live again, but it + # stays consumed and the grammar refuses it as a predecessor. + if prior_own_key in registry.consumed: + raise SystemExit( + f"identity {canon_key} would take uuid {row_uuid} from docket " + f"placeholder {prior_own_key}, which already handed its " + "lineage over once and holds this uuid as the successor of a " + "later enrichment; a placeholder hands over at most once. " + "Curate it: for example, delete the placeholder's catalog row " + "and rerun with --allow-remint, so the observed identity " + "takes its own lineage (reviving its retired binding, or " + "minting if it has none) and the placeholder's binding " + "retires" + ) # Docket-placeholder enrichment: the binding MOVES to the observed # identity via an explicit retire + succeeds pair. UUID continuity # is preserved, so no ceremony flag is needed — this holds for the - # first enrichment and for a re-enrichment after a withdraw/reclaim - # cycle (the grammar accepts succeeds on a retired successor key). + # first enrichment, for a chain of them (a geography-free + # placeholder enriched to a geography-only one, then to the + # observed identity), and for a re-enrichment after a + # withdraw/reclaim cycle (the grammar accepts succeeds on a retired + # successor key). plan["enrich_retires"].append( dict( _registry_event( @@ -2193,6 +2239,33 @@ def _plan_enrichment( ) return True + def _plan_reclaim( + canon_key: tuple[str, str, str], + geography: dict | None, + entity: dict | None, + row_uuid: str, + ) -> str: + """Plan a reclaim of ``canon_key`` on ``row_uuid``, which must be + new to the registry; return ``row_uuid``. + + The key handed its lineage over through a succeeds event and is + retired, so it is terminal: the grammar refuses to revive it (and + so to supersede it, which needs a live binding). Anything that + reappears under it is a NEW series, and a reclaim is the one event + that starts one there. The old UUID stays with its lineage. + """ + plan["mints"].append( + dict( + _registry_event(canon_key[0], geography, entity, row_uuid), + reclaimed=True, + note=( + "identity re-established after its lineage was " + "handed over" + ), + ) + ) + return row_uuid + def resolve_uuid( canon_key: tuple[str, str, str], prior: dict | None, @@ -2201,6 +2274,12 @@ def resolve_uuid( ) -> str: binding = registry.binding(canon_key) prior_uuid = prior["uuid"] if prior else None + # Retired after handing its lineage over through a succeeds event. + # A key that returned as a later hop's successor is live, so it is + # not handed over here (the grammar still bars it as a predecessor). + handed_over = ( + not registry.is_live(canon_key) and canon_key in registry.consumed + ) if prior_uuid and binding and prior_uuid != binding: # A RETIRED successor returning on the CURRENT placeholder # lineage is a re-enrichment, not a remint: plan the retire + @@ -2213,12 +2292,23 @@ def resolve_uuid( ) ): return prior_uuid - # The catalog row disagrees with the registry: an explicit, - # gated remint (the curator edited the row's uuid on purpose). - # The replacement must be new to the registry outright. + # The catalog row disagrees with the registry: the curator + # edited the row's uuid on purpose. Either event that can carry + # it (supersede, or reclaim for a handed-over key) needs a UUID + # new to the registry outright. replacement_owner = registry.uuid_owner.get( uuid_module.UUID(prior_uuid).int ) + if replacement_owner is not None and handed_over: + raise SystemExit( + f"identity {canon_key} handed its lineage over, so only " + "a reclaim can re-establish it, and the catalog row's " + f"uuid {prior_uuid} is already known to the registry " + f"(first bound to {replacement_owner}) — a reclaimed " + "identity is a NEW series and mints fresh: give the " + "row an unregistered uuid, or delete it so the builder " + "reclaims with a fresh one" + ) if replacement_owner is not None: raise SystemExit( f"identity {canon_key} would supersede to uuid " @@ -2226,8 +2316,16 @@ def resolve_uuid( f"(first bound to {replacement_owner}) — superseding " "UUIDs must be new" ) + # A handed-over key is terminal: the grammar refuses the revive + # a supersede would need. The row's uuid starts the key's new + # lineage through a reclaim instead, as the handed-over branch + # below does with a fresh uuid. No live binding changes, so no + # remint ceremony applies. + if handed_over: + return _plan_reclaim(canon_key, geography, entity, prior_uuid) # A retired binding is revived first so the event chain stays - # valid (revives are staged before supersedes). + # valid (revives are staged before supersedes). The supersede + # is the gated remint. if not registry.is_live(canon_key): plan["revives"].append( dict( @@ -2243,9 +2341,7 @@ def resolve_uuid( ) ) return prior_uuid - if binding and not ( - not registry.is_live(canon_key) and canon_key in registry.consumed - ): + if binding and not handed_over: if not registry.is_live(canon_key): # A retired identity is being observed again: same UUID, # explicit revive event (no ceremony — resuming an identity @@ -2260,22 +2356,12 @@ def resolve_uuid( ) return binding if binding: - # The identity handed its lineage to a successor: terminal. - # Anything reappearing under the old key is a NEW series claim - # and mints fresh through an explicit reclaim event; the old - # UUID stays with its lineage. - row_uuid = str(uuid_module.uuid4()) - plan["mints"].append( - dict( - _registry_event(canon_key[0], geography, entity, row_uuid), - reclaimed=True, - note=( - "identity re-established after its lineage was " - "handed over" - ), - ) + # The identity handed its lineage to a successor and the row + # carries no UUID of its own (or carries exactly the one that + # was handed over): reclaim on a fresh UUID. + return _plan_reclaim( + canon_key, geography, entity, str(uuid_module.uuid4()) ) - return row_uuid row_uuid = prior_uuid if prior_uuid else str(uuid_module.uuid4()) owner_key = registry.uuid_owner.get(uuid_module.UUID(row_uuid).int) if owner_key is not None: @@ -2285,7 +2371,8 @@ def resolve_uuid( return row_uuid raise SystemExit( f"identity {canon_key} would mint uuid {row_uuid}, which " - f"the registry already binds to {owner_key} — UUID reuse " + f"the registry already knows (first bound to {owner_key}) " + "— UUID reuse " "requires an explicit ceremony (merge via curated alias + " "--allow-remint, or placeholder enrichment)" ) diff --git a/tests/test_build_series_catalog.py b/tests/test_build_series_catalog.py index 98510253..cfc02d6b 100644 --- a/tests/test_build_series_catalog.py +++ b/tests/test_build_series_catalog.py @@ -16,9 +16,11 @@ import pathlib import sys import tempfile +import uuid import pytest from hypothesis import HealthCheck, given, settings +from hypothesis import event as hypothesis_event from hypothesis import strategies as st ROOT = pathlib.Path(__file__).resolve().parents[1] @@ -2014,7 +2016,7 @@ def test_mint_never_reuses_a_bound_uuid(tmp_path: pathlib.Path) -> None: }, ] } - with pytest.raises(SystemExit, match="already binds"): + with pytest.raises(SystemExit, match=r"already knows \(first bound to"): _build( tmp_path, [_row("new.one"), _row("new.two")], @@ -4436,3 +4438,1143 @@ def observed_total(catalog: dict) -> int: undone = json.loads((tmp_path / "catalog.json").read_text()) assert _without_digests(undone) == _without_digests(before) assert bsc.main(argv + ["--check"]) == 0 + + +# --- chained placeholder enrichment (2026-10-16) ---------------------------- +# +# A docket placeholder can be enriched more than once. The geography-free +# placeholder becomes a geography-only one when its docket entry learns a +# country, and that one becomes the observed identity when the first print +# lands. Each hop is a retire + succeeds pair carrying one UUID. The guard is +# the LIVE binding of the row's own key. uuid_owner records first ownership +# and never moves, so it names the first hop's predecessor, never the +# second's: requiring it (the guard before this fix) refused every second +# hop. Thesis run 38962977031 hit exactly that on +# usaspending.dod.prime_award_obligations. + +DOD = "usaspending.dod.prime_award_obligations" +CHAIN_NOTE = "docket placeholder enriched by first observed identity" +US_UNDECLARED = {"level": "country", "id": "0100000US", "vintage": None} +U3 = "cccccccc-3333-4333-8333-333333333333" +U4 = "eeeeeeee-5555-4555-8555-555555555555" + + +def _key(fields: dict) -> tuple[str, str, str]: + return bsc.UuidRegistry.entry_key(fields) + + +def _owner(registry: bsc.UuidRegistry, row_uuid: str): + return registry.uuid_owner.get(uuid.UUID(row_uuid).int) + + +def _plan_events(plan: dict) -> list[dict]: + """Every registry event a plan stages, in main's order.""" + return ( + plan["enrich_retires"] + plan["mints"] + plan["revives"] + + plan["supersedes"] + plan["retire_pending"] + ) + + +def _event_kind(event: dict) -> str: + # Planned events carry absent markers as None; rendered lines omit them. + for marker in ("succeeds", "supersedes", "retired", "revived", "reclaimed"): + if event.get(marker) is not None: + return marker + return "mint" + + +def _registry_lines(tmp_path: pathlib.Path) -> list[dict]: + return [ + json.loads(line) + for line in (tmp_path / "registry.jsonl").read_text().splitlines() + ] + + +def _dod_observation() -> dict: + """The FY2026 first print as Thesis records it (registered snapshot).""" + return _row( + f"{DOD}.fy2026", + rid=f"{DOD}.fy2026.registered_query_snapshot", + unit="billions USD", + period={"type": "year", "value": "2026"}, + ) + + +def _dod_chain() -> list[dict]: + """Registry lines 196, 249 and 250 for a2649fd9, with U1 for the UUID.""" + return [ + _mint(DOD, U1), + dict(_mint(DOD, U1), retired=True, note=CHAIN_NOTE), + dict( + _mint(DOD, U1, geography=US_UNDECLARED), + succeeds={"concept": DOD, "geography": None, "entity": None}, + ), + ] + + +def _docket_row(concept: str, row_uuid: str, geography: dict | None) -> dict: + return { + "uuid": row_uuid, + "concept": concept, + "geography": geography, + "entity": None, + "aliases": [], + "status": "docket-only", + } + + +def test_a_placeholder_enriched_once_is_enriched_again( + tmp_path: pathlib.Path, +) -> None: + # Production repro (Thesis run 38962977031, 2026-10-16). The catalog + # still holds the geography-only placeholder docket-only; its first + # print must take the second hop instead of falling through to the + # UUID-reuse refusal. + registry = _registry(tmp_path, _dod_chain()) + geo_key = _key({"concept": DOD, "geography": US_UNDECLARED, "entity": None}) + # The trap: the live placeholder holds U1 without having first owned it. + assert registry.is_live(geo_key) and registry.binding(geo_key) == U1 + assert _owner(registry, U1) != geo_key + existing = { + "series": [ + _docket_row(DOD, U1, dict(bsc.COUNTRY_GEOGRAPHY["US"])) + ] + } + catalog, plan = _build( + tmp_path, [_dod_observation()], existing=existing, + registry_entries=_dod_chain(), + ) + (row,) = catalog["series"] + assert (row["concept"], row["uuid"], row["status"]) == (DOD, U1, "observed") + assert row["entity"] == ECONOMY + # Exactly the pair the diagnosis's patched generator wrote (and that + # thesis#362's append guard admits): retire the geography-only + # placeholder, then succeed it at the observed identity, one UUID. + assert [ + bsc.UuidRegistry.render_entry(e) for e in _plan_events(plan) + ] == [ + bsc.UuidRegistry.render_entry(e) + for e in ( + dict(_mint(DOD, U1, geography=US_UNDECLARED), retired=True, + note=CHAIN_NOTE), + dict( + _mint(DOD, U1, geography=US_IDENTITY, entity=ECONOMY), + succeeds={"concept": DOD, "geography": US_UNDECLARED, + "entity": None}, + ), + ) + ] + assert not plan["dropped"] + staged = registry.stage(_plan_events(plan)) + observed_key = _key(row) + assert staged.is_live(observed_key) and staged.binding(observed_key) == U1 + assert not staged.is_live(geo_key) and geo_key in staged.consumed + assert [k for k, e in staged.live_bindings() if e["uuid"] == U1] == [ + observed_key + ] + assert bsc.registry_agreement_problems(catalog, staged) == [] + + +def test_placeholder_chain_keeps_one_uuid_through_main( + tmp_path: pathlib.Path, +) -> None: + # The same chain end to end through main: the docket entry starts with + # no country, learns "US" (first hop, docket to docket), then its first + # print lands (second hop). Neither hop needs a ceremony flag, the UUID + # never changes, and --check passes after each. + entry = {"series": DOD, "cadence": "annual", + "extras": {"targetUnit": "billions USD"}} + bystander = _row("bls.cps.unemployment_rate") + argv = _repo(tmp_path, [bystander], seed={"series": [entry]}) + + def rows_by_concept() -> dict[str, dict]: + catalog = json.loads((tmp_path / "catalog.json").read_text()) + return {r["concept"]: r for r in catalog["series"]} + + assert bsc.main(argv) == 0 + placeholder = rows_by_concept()[DOD] + assert placeholder["status"] == "docket-only" + assert placeholder["geography"] is None + lineage = placeholder["uuid"] + bystander_uuid = rows_by_concept()["bls.cps.unemployment_rate"]["uuid"] + + entry["extras"]["country"] = "US" + (tmp_path / "seed.json").write_text( + json.dumps({"series": [entry]}), encoding="utf-8" + ) + assert bsc.main(argv) == 0 + country = rows_by_concept()[DOD] + assert (country["uuid"], country["status"]) == (lineage, "docket-only") + assert country["geography"] == bsc.COUNTRY_GEOGRAPHY["US"] + assert bsc.main(argv + ["--check"]) == 0 + + (tmp_path / "obs.jsonl").write_text( + "".join(json.dumps(r) + "\n" for r in [bystander, _dod_observation()]), + encoding="utf-8", + ) + assert bsc.main(argv) == 0 + observed = rows_by_concept()[DOD] + assert (observed["uuid"], observed["status"]) == (lineage, "observed") + assert observed["entity"] == ECONOMY + assert rows_by_concept()["bls.cps.unemployment_rate"]["uuid"] == ( + bystander_uuid + ) + assert bsc.main(argv + ["--check"]) == 0 + + events = _registry_lines(tmp_path) + assert [ + (e["geography"], e["entity"], _event_kind(e)) + for e in events + if e["concept"] == DOD + ] == [ + (None, None, "mint"), + (None, None, "retired"), + (US_UNDECLARED, None, "succeeds"), + (US_UNDECLARED, None, "retired"), + (US_IDENTITY, ECONOMY, "succeeds"), + ] + assert {e["uuid"] for e in events if e["concept"] == DOD} == {lineage} + assert not any( + _event_kind(e) in ("supersedes", "reclaimed") for e in events + ) + catalog_bytes = (tmp_path / "catalog.json").read_bytes() + registry_bytes = (tmp_path / "registry.jsonl").read_bytes() + assert bsc.main(argv) == 0 + assert (tmp_path / "catalog.json").read_bytes() == catalog_bytes + assert (tmp_path / "registry.jsonl").read_bytes() == registry_bytes + + +# Each stage is (docket entry declares a country, observation present). +_CHAINS = ( + ((False, False), (True, False)), + ((False, False), (False, True)), + ((True, False), (True, True)), + ((False, False), (True, False), (True, True)), +) + + +@settings( + max_examples=60, + deadline=None, + suppress_health_check=[HealthCheck.function_scoped_fixture], +) +@given( + chain=st.sampled_from(_CHAINS), + country=st.sampled_from(sorted(bsc.COUNTRY_GEOGRAPHY)), + vintage=st.sampled_from(["current", "2020", None]), + entity=st.sampled_from([ECONOMY, HAND_ENTITY]), +) +def test_any_placeholder_chain_keeps_one_uuid_and_passes_check( + tmp_path_factory: pytest.TempPathFactory, + chain: tuple, + country: str, + vintage: str | None, + entity: dict, +) -> None: + """For every chain of placeholder enrichments the builder can reach + (geography-free -> country, geography-free -> observed, country -> + observed, and all three hops), in any country, at any observed + vintage and entity: every hop writes exactly one retire + succeeds + pair handing over the binding it retired, needs no ceremony flag, + keeps the lineage's one UUID, leaves exactly one live binding for it + (at the row's identity), leaves unrelated series untouched, passes + --check, and is byte-idempotent.""" + tmp_path = tmp_path_factory.mktemp("chain") + concept = "agency.chain.series" + neighbour = {"series": "agency.chain.neighbour", "cadence": "monthly", + "extras": {"country": "US", "targetUnit": "percent"}} + bystander = _row("bls.cps.unemployment_rate") + observation = _row( + concept, + geography={**bsc.COUNTRY_GEOGRAPHY[country], "vintage": vintage}, + entity=entity, + ) + + def write_stage(stage: tuple[bool, bool]) -> None: + declares_country, observed = stage + extras = {"targetUnit": "percent"} + if declares_country: + extras["country"] = country + seed = {"series": [ + {"series": concept, "cadence": "monthly", "extras": extras}, + neighbour, + ]} + (tmp_path / "seed.json").write_text(json.dumps(seed), encoding="utf-8") + rows = [bystander] + ([observation] if observed else []) + (tmp_path / "obs.jsonl").write_text( + "".join(json.dumps(r) + "\n" for r in rows), encoding="utf-8" + ) + + def catalog_rows() -> list[dict]: + return json.loads((tmp_path / "catalog.json").read_text())["series"] + + argv = _repo(tmp_path, []) + write_stage(chain[0]) + assert bsc.main(argv) == 0 + (first,) = [r for r in catalog_rows() if r["concept"] == concept] + lineage = first["uuid"] + others = { + _key(r): r["uuid"] for r in catalog_rows() if r["concept"] != concept + } + for stage in chain[1:]: + before = len(_registry_lines(tmp_path)) + write_stage(stage) + assert bsc.main(argv) == 0 # no --allow-remint + (row,) = [r for r in catalog_rows() if r["concept"] == concept] + assert row["uuid"] == lineage + assert row["status"] == ("observed" if stage[1] else "docket-only") + hop = _registry_lines(tmp_path)[before:] + assert [_event_kind(e) for e in hop] == ["retired", "succeeds"] + assert {e["uuid"] for e in hop} == {lineage} + assert _key(hop[1]["succeeds"]) == _key(hop[0]) + assert _key(hop[1]) == _key(row) + assert { + _key(r): r["uuid"] for r in catalog_rows() if r["concept"] != concept + } == others + registry = bsc.UuidRegistry.load(tmp_path / "registry.jsonl") + assert [ + k for k, e in registry.live_bindings() if e["uuid"] == lineage + ] == [_key(row)] + assert bsc.main(argv + ["--check"]) == 0 + snapshot = ( + (tmp_path / "catalog.json").read_bytes(), + (tmp_path / "registry.jsonl").read_bytes(), + ) + assert bsc.main(argv) == 0 + assert ( + (tmp_path / "catalog.json").read_bytes(), + (tmp_path / "registry.jsonl").read_bytes(), + ) == snapshot + assert { + e["uuid"] for e in _registry_lines(tmp_path) if e["concept"] == concept + } == {lineage} + + +def _stale_after_chain(scenario: str): + """(registry events, stale docket-only row, observed identity): the + DOD chain (geography-free -> geography-only, U1) and then whatever made + the committed catalog's row stale.""" + chain = _dod_chain() + observed_economy = {"concept": DOD, "geography": US_IDENTITY, + "entity": ECONOMY} + observed_hand = dict(observed_economy, entity=HAND_ENTITY) + geo_row = dict(bsc.COUNTRY_GEOGRAPHY["US"]) + handed_on = chain + [ + dict(_mint(DOD, U1, geography=US_UNDECLARED), retired=True, + note=CHAIN_NOTE), + dict(observed_economy, uuid=U1, + succeeds={"concept": DOD, "geography": US_UNDECLARED, + "entity": None}), + ] + if scenario == "superseded": + # A curated remint moved the placeholder to U2; the row kept U1. + events = chain + [ + dict(_mint(DOD, U2, geography=US_UNDECLARED), supersedes=U1, + note="curated remint of the placeholder"), + ] + return events, _docket_row(DOD, U1, geo_row), observed_economy + if scenario == "retired": + # The docket entry was withdrawn (gated retire); the row lingers. + events = chain + [ + dict(_mint(DOD, U1, geography=US_UNDECLARED), retired=True, + note="docket entry withdrawn"), + ] + return events, _docket_row(DOD, U1, geo_row), observed_economy + if scenario == "handed over": + # The placeholder already took its second hop; the row predates it. + return handed_on, _docket_row(DOD, U1, geo_row), observed_hand + if scenario == "reclaimed": + # After the hand-over the observation was withdrawn and the + # placeholder re-established on a fresh UUID; the row kept U1. + events = handed_on + [ + dict(observed_economy, uuid=U1, retired=True, + note="observation withdrawn upstream"), + dict(_mint(DOD, U2, geography=US_UNDECLARED), reclaimed=True, + note="placeholder re-established after handover"), + ] + return events, _docket_row(DOD, U1, geo_row), observed_hand + if scenario == "chain root": + # The row is the geography-free placeholder the first hop retired. + return chain, _docket_row(DOD, U1, None), observed_economy + if scenario == "another series' uuid": + # The row carries a UUID that another live series holds. + events = chain + [_mint("agency.other", U3)] + return events, _docket_row(DOD, U3, geo_row), observed_economy + raise AssertionError(scenario) + + +@pytest.mark.parametrize( + "scenario", + ["superseded", "retired", "handed over", "reclaimed", "chain root", + "another series' uuid"], +) +def test_stale_rows_after_a_chain_are_still_refused( + tmp_path: pathlib.Path, scenario: str +) -> None: + # The case the dropped owner check was written for: a catalog row that + # no longer carries its key's LIVE binding. With the owner check gone, + # the live-binding conditions alone must refuse each one at plan time + # with the ceremony diagnostic, and main must leave both artifacts + # byte-untouched. + events, stale_row, observed = _stale_after_chain(scenario) + registry = _registry(tmp_path, events) + stale_key = _key(stale_row) + assert not ( + registry.is_live(stale_key) + and registry.binding(stale_key) == stale_row["uuid"] + ) + observation = _row( + DOD, geography=dict(observed["geography"], name="United States"), + entity=observed["entity"], + ) + with pytest.raises(SystemExit, match="explicit ceremony"): + _build( + tmp_path, [observation], existing={"series": [stale_row]}, + registry_entries=events, + ) + argv = _repo(tmp_path, [observation], seed={"series": []}) + _registry(tmp_path, events) + (tmp_path / "catalog.json").write_text( + json.dumps({"series": [stale_row]}) + "\n", encoding="utf-8" + ) + catalog_before = (tmp_path / "catalog.json").read_bytes() + registry_before = (tmp_path / "registry.jsonl").read_bytes() + with pytest.raises(SystemExit, match="explicit ceremony"): + bsc.main(argv) + assert (tmp_path / "catalog.json").read_bytes() == catalog_before + assert (tmp_path / "registry.jsonl").read_bytes() == registry_before + + +def test_a_returned_placeholder_never_hands_over_twice( + tmp_path: pathlib.Path, +) -> None: + # Independent-review counterexample (chronicle#360, 2026-10-18). A + # placeholder B hands U1 to its observed identity C. The observation is + # withdrawn and the docket loses its country (fresh placeholder A/U2), + # then the country returns and B takes U2 from A. B is live and bound + # to U2, but it is still consumed: it already handed over once, and the + # grammar never lets a predecessor hand over twice. When C's + # observation returns, the builder must refuse at plan time, naming the + # cause. Without the consumed check it planned retire(B/U2) + C + # succeeds B, which staging rejected. Base refused too (B never + # first-owned U2), but with a remint message that blamed a supersede + # nobody made. The curation the diagnostic suggests must then work. + concept = "agency.lineage" + bystander = _row("bls.cps.unemployment_rate") + observed = _row(concept) + + def stage(country: bool, rows: list[dict]) -> None: + extras = {"targetUnit": "percent"} + if country: + extras["country"] = "US" + seed = {"series": [ + {"series": concept, "cadence": "monthly", "extras": extras}, + ]} + (tmp_path / "seed.json").write_text(json.dumps(seed), encoding="utf-8") + (tmp_path / "obs.jsonl").write_text( + "".join(json.dumps(r) + "\n" for r in rows), encoding="utf-8" + ) + + argv = _repo(tmp_path, []) + remint = ["--allow-remint", "--remint-note", "observation withdrawn"] + stage(True, [bystander]) + assert bsc.main(argv) == 0 # mint B/U1 + stage(True, [bystander, observed]) + assert bsc.main(argv) == 0 # retire B, C succeeds B with U1 + stage(False, [bystander]) + assert bsc.main(argv + remint) == 0 # mint A/U2, retire C + stage(True, [bystander]) + assert bsc.main(argv) == 0 # retire A, B succeeds A with U2 + assert bsc.main(argv + ["--check"]) == 0 + a_key = (concept, bsc._geo_key(None), bsc._entity_key(None)) + b_key = (concept, bsc._geo_key(US_UNDECLARED), bsc._entity_key(None)) + c_key = (concept, bsc._geo_key(US_IDENTITY), bsc._entity_key(ECONOMY)) + assert [ + (_key(e), _event_kind(e)) + for e in _registry_lines(tmp_path) + if e["concept"] == concept + ] == [ + (b_key, "mint"), + (b_key, "retired"), + (c_key, "succeeds"), + (a_key, "mint"), + (c_key, "retired"), + (a_key, "retired"), + (b_key, "succeeds"), + ] + registry = bsc.UuidRegistry.load(tmp_path / "registry.jsonl") + u2 = registry.binding(a_key) + assert registry.is_live(b_key) and registry.binding(b_key) == u2 + assert b_key in registry.consumed # the state the live binding misses + + stage(True, [bystander, observed]) + catalog_before = (tmp_path / "catalog.json").read_bytes() + registry_before = (tmp_path / "registry.jsonl").read_bytes() + for flags in ([], remint): + with pytest.raises(SystemExit, match="hands over at most once"): + bsc.main(argv + flags) + assert (tmp_path / "catalog.json").read_bytes() == catalog_before + assert (tmp_path / "registry.jsonl").read_bytes() == registry_before + + # The suggested curation: delete the placeholder's catalog row and + # rerun with --allow-remint. C revives its own U1; B's U2 retires. + catalog = json.loads(catalog_before) + catalog["series"] = [ + r for r in catalog["series"] if r["concept"] != concept + ] + (tmp_path / "catalog.json").write_text( + json.dumps(catalog, indent=2) + "\n", encoding="utf-8" + ) + assert bsc.main(argv) == 1 # retiring B's live binding is gated + assert bsc.main(argv + remint) == 0 + assert bsc.main(argv + ["--check"]) == 0 + registry = bsc.UuidRegistry.load(tmp_path / "registry.jsonl") + u1 = registry.binding(c_key) + assert registry.is_live(c_key) and u1 != u2 + assert not registry.is_live(b_key) and registry.binding(b_key) == u2 + assert [ + (_key(e), _event_kind(e)) + for e in _registry_lines(tmp_path)[len(registry_before.splitlines()):] + ] == [(c_key, "revived"), (b_key, "retired")] + (row,) = [ + r for r in json.loads((tmp_path / "catalog.json").read_text())["series"] + if r["concept"] == concept + ] + assert (row["uuid"], row["status"], _key(row)) == (u1, "observed", c_key) + + +def test_a_docket_alias_never_plans_a_cross_concept_hop( + tmp_path: pathlib.Path, +) -> None: + # Independent-review finding (chronicle#360, LOW, pre-existing): a + # docket entry naming a placeholder by its curated alias keys the + # docket row under the alias, so the planned succeeds crossed concepts + # and died only in staging replay. The builder now refuses at plan time + # naming the cause, and the remedy it names (call the docket entry by + # the placeholder's own concept) enriches cleanly. + existing = {"series": [ + dict(_docket_row("agency.a", U1, None), aliases=["agency.b"]), + ]} + entry = {"series": "agency.b", "cadence": "monthly", + "extras": {"country": "US", "targetUnit": "percent"}} + with pytest.raises(SystemExit, match="never crosses concepts") as refused: + _build( + tmp_path, [], existing=existing, + registry_entries=[_mint("agency.a", U1)], + docket={"series": [entry]}, + ) + assert "name the docket entry 'agency.a'" in str(refused.value) + catalog, plan = _build( + tmp_path, [], existing=existing, + registry_entries=[_mint("agency.a", U1)], + docket={"series": [dict(entry, series="agency.a")]}, + ) + (row,) = catalog["series"] + assert (row["concept"], row["uuid"], row["status"]) == ( + "agency.a", U1, "docket-only" + ) + assert [_event_kind(e) for e in _plan_events(plan)] == [ + "retired", "succeeds" + ] + staged = _registry(tmp_path, [_mint("agency.a", U1)]).stage( + _plan_events(plan) + ) + assert bsc.registry_agreement_problems(catalog, staged) == [] + + +# A small identity universe for generated registry histories: two concepts +# (so a row can carry another series' UUID), geographies at every +# refinement level, three entities, and six UUIDs (so stale rows collide +# with known values). +_GEN_CONCEPT, _GEN_OTHER = "agency.lineage", "agency.elsewhere" +_PLACEHOLDER_GEOS = ( + None, + {"level": "country", "id": "0100000US", "vintage": None}, + {"level": "country", "id": "0100000US", "vintage": "current"}, + {"level": "country", "id": "0100000US", "vintage": "2020"}, + {"level": "country", "id": "GB", "vintage": None}, +) +_OBSERVED_GEOS = ( + {"level": "country", "id": "0100000US", "vintage": "current"}, + {"level": "country", "id": "0100000US", "vintage": "2020"}, + {"level": "country", "id": "GB", "vintage": "current"}, +) +_GEN_GEOS = _PLACEHOLDER_GEOS + _OBSERVED_GEOS[2:] +_GEN_ENTITIES = (None, ECONOMY, HAND_ENTITY) +_GEN_UUIDS = ( + U1, U2, U3, U4, + "ffffffff-6666-4666-8666-666666666666", + "abababab-7777-4777-8777-777777777777", +) +_GEN_IDENTS = tuple( + {"concept": concept, "geography": geography, "entity": entity} + for concept in (_GEN_CONCEPT, _GEN_OTHER) + for geography in _GEN_GEOS + for entity in _GEN_ENTITIES +) + + +def _ident(entry: dict) -> dict: + return {key: entry[key] for key in ("concept", "geography", "entity")} + + +def _refines(placeholder: dict | None, geography: dict | None) -> bool: + """Whether an enrichment hop from ``placeholder`` to ``geography`` is + plausible. Used only to steer generation toward chains; the registry + grammar alone decides what a history may contain.""" + if placeholder is None: + return True + geography = geography or {} + return (placeholder["level"], placeholder["id"]) == ( + geography.get("level"), geography.get("id") + ) and placeholder["vintage"] in (None, geography.get("vintage")) + + +def _generated_registry(data: st.DataObject) -> bsc.UuidRegistry: + """A registry grown from random events that the grammar accepts. + + Each step draws a move the current state allows: mint an unbound + identity on an unused UUID, retire a live binding, revive a retired + one, supersede or reclaim onto an unused UUID, take an enrichment hop + (retire an entity-less binding and succeed it at a refinement), or + return (mint a coarser placeholder and hop it onto a retired key that + already handed over). + The move is kept only if UuidRegistry accepts it, so every history is + one the grammar itself admits. Hops chain, so placeholders enriched + once, twice and three times occur alongside stale, superseded, + retired, handed-over and reclaimed keys, and keys that handed over and + then returned as a later hop's successor (live, but consumed).""" + registry = bsc.UuidRegistry(pathlib.Path("generated.jsonl"), b"") + for _ in range(data.draw(st.integers(1, 12), label="steps")): + held = list(registry.latest.values()) + live = [e for e in held if not e.get("retired")] + retired = [e for e in held if e.get("retired")] + consumed = [e for e in retired if _key(e) in registry.consumed] + unbound = [i for i in _GEN_IDENTS if _key(i) not in registry.latest] + unused = [u for u in _GEN_UUIDS if _owner(registry, u) is None] + moves = [] + if unbound and unused: + moves += ["mint", "mint"] + if live: + moves += ["retire"] + if live and unused: + moves += ["supersede"] + if retired: + moves += ["revive"] + if consumed and unused: + moves += ["reclaim"] + if any(e["entity"] is None for e in live): + moves += ["hop", "hop", "hop"] + # A consumed placeholder can take a lineage again as a later hop's + # successor (main writes this when a docket entry loses its + # country, which mints a fresh coarser placeholder, and regains + # it): mint a coarser placeholder and hop it straight onto the key. + returns = [ + (coarser, entry) + for entry in consumed + if entry["entity"] is None + for coarser in unbound + if coarser["concept"] == entry["concept"] + and coarser["entity"] is None + and _key(coarser) != _key(entry) + and _refines(coarser["geography"], entry["geography"]) + ] + if returns and unused: + moves += ["return", "return"] + if not moves: + break + move = data.draw(st.sampled_from(moves)) + if move == "mint": + # Placeholders (entity-less) are minted twice as often. + pool = [i for i in unbound if i["entity"] is None] * 2 + unbound + events = [dict(data.draw(st.sampled_from(pool)), + uuid=data.draw(st.sampled_from(unused)))] + elif move == "retire": + entry = data.draw(st.sampled_from(live)) + events = [dict(_ident(entry), uuid=entry["uuid"], retired=True, + note="generated retirement")] + elif move == "supersede": + entry = data.draw(st.sampled_from(live)) + events = [dict(_ident(entry), + uuid=data.draw(st.sampled_from(unused)), + supersedes=entry["uuid"], + note="generated curated remint")] + elif move == "revive": + entry = data.draw(st.sampled_from(retired)) + events = [dict(_ident(entry), uuid=entry["uuid"], revived=True)] + elif move == "reclaim": + entry = data.draw(st.sampled_from(consumed)) + events = [dict(_ident(entry), + uuid=data.draw(st.sampled_from(unused)), + reclaimed=True, + note="generated reclaim after handover")] + elif move == "return": + coarser, entry = data.draw(st.sampled_from(returns)) + fresh = data.draw(st.sampled_from(unused)) + events = [ + dict(coarser, uuid=fresh), + dict(coarser, uuid=fresh, retired=True, note=CHAIN_NOTE), + dict(_ident(entry), uuid=fresh, succeeds=coarser), + ] + else: + entry = data.draw(st.sampled_from( + [e for e in live if e["entity"] is None] + )) + targets = [ + ident for ident in _GEN_IDENTS + if ident["concept"] == entry["concept"] + and _key(ident) != _key(entry) + and _refines(entry["geography"], ident["geography"]) + ] + # Entity-less targets are weighted up so hops chain, and retired + # ones further still so lineages return onto keys that already + # handed over (live again, but consumed). + retired_keys = {_key(e) for e in retired} + target = data.draw(st.sampled_from( + [t for t in targets if _key(t) in retired_keys] * 6 + + [t for t in targets if t["entity"] is None] * 3 + + targets + )) + events = [ + dict(_ident(entry), uuid=entry["uuid"], retired=True, + note=CHAIN_NOTE), + dict(target, uuid=entry["uuid"], succeeds=_ident(entry)), + ] + try: + registry = registry.stage(events) + except SystemExit: + continue + return registry + + +@settings(max_examples=400, deadline=None) +@given(data=st.data()) +def test_a_live_binding_its_key_never_first_owned_came_by_succeeds( + data: st.DataObject, +) -> None: + """Why the owner check was redundant where it was right. In every + history the grammar admits, a live binding (K, U) either has K as U's + first owner, or K's binding was established by a succeeds event. So + the conjunct uuid_owner[U] == K, on top of the live-binding + conditions, refused nothing but enrichment chains.""" + registry = _generated_registry(data) + for key, entry in registry.live_bindings(): + if _owner(registry, entry["uuid"]) == key: + continue + establishing = [ + e for e in registry.entries + if _key(e) == key and _event_kind(e) not in ("retired", "revived") + ] + assert _event_kind(establishing[-1]) == "succeeds", (key, entry) + + +@settings( + max_examples=400, + deadline=None, + suppress_health_check=[HealthCheck.function_scoped_fixture], +) +@given(data=st.data()) +def test_enrichment_is_planned_exactly_when_the_grammar_admits_it( + tmp_path_factory: pytest.TempPathFactory, data: st.DataObject +) -> None: + """Differential: plan-time guard vs. the registry grammar. + + For any history the grammar admits, any docket-only row (live, live + but already handed over, stale, superseded, retired, handed over, + reclaimed, the root of a chain, or carrying another series' UUID) and + any observation of its concept: + + * completeness: the builder plans the retire + succeeds pair exactly + when UuidRegistry would accept that very pair (so no enrichment + chain is refused); + * soundness: everything the builder plans stages cleanly and agrees + with the catalog it built (so no stale row slips through); + * a refusal is a plan-time diagnostic, never a crash; + * when no enrichment is planned, the row takes its own key's binding + or a UUID new to the registry, never a UUID another key holds. + """ + tmp_path = tmp_path_factory.mktemp("differential") + registry = _generated_registry(data) + (tmp_path / "registry.jsonl").write_bytes(registry.raw) + placeholders = [ + e for e in registry.latest.values() + if e["concept"] == _GEN_CONCEPT and e["entity"] is None + ] + # Steer toward rows at keys the history holds, carrying their last + # binding: live ones (the chained case lives here), live ones that + # already handed over (consumed), and stale ones (retired, handed + # over). Any geography with any UUID stays possible. + live = [e for e in placeholders if not e.get("retired")] + candidates = ( + [(e["geography"], e["uuid"]) for e in live] * 8 + + [(e["geography"], e["uuid"]) for e in live + if _key(e) in registry.consumed] * 16 + + [(e["geography"], e["uuid"]) for e in placeholders] * 2 + + [(g, u) for g in _PLACEHOLDER_GEOS for u in _GEN_UUIDS] + ) + geography, row_uuid = data.draw(st.sampled_from(candidates)) + placeholder = {"concept": _GEN_CONCEPT, "geography": geography, + "entity": None} + # Observations that refine the row's geography are weighted up; any + # observed geography stays possible (cross-country, vintage clash). + observed = { + "concept": _GEN_CONCEPT, + "geography": data.draw(st.sampled_from( + [g for g in _OBSERVED_GEOS if _refines(geography, g)] * 3 + + list(_OBSERVED_GEOS) + )), + "entity": data.draw(st.sampled_from((ECONOMY, HAND_ENTITY))), + } + retire = dict(placeholder, uuid=row_uuid, retired=True, note=CHAIN_NOTE) + succeed = dict(observed, uuid=row_uuid, succeeds=placeholder) + try: + registry.stage([retire, succeed]) + admitted = True + except SystemExit: + admitted = False + observation = _row( + _GEN_CONCEPT, + geography=dict(observed["geography"], name=None), + entity=observed["entity"], + ) + try: + catalog, plan = _build( + tmp_path, [observation], + existing={"series": [_docket_row(_GEN_CONCEPT, row_uuid, + geography)]}, + ) + except SystemExit as exc: + assert not admitted, exc + assert any( + diagnostic in str(exc) + for diagnostic in ( + "explicit ceremony", + "superseding UUIDs must be new", + "hands over at most once", + ) + ), exc + return + hop = [e for e in plan["mints"] if "succeeds" in e] + assert bool(hop) == admitted + (row,) = catalog["series"] + if admitted: + assert row["uuid"] == row_uuid + assert [ + bsc.UuidRegistry.render_entry(e) + for e in plan["enrich_retires"] + hop + ] == [bsc.UuidRegistry.render_entry(e) for e in (retire, succeed)] + else: + assert row["uuid"] == registry.binding(_key(observed)) or ( + _owner(registry, row["uuid"]) is None + ) + # main attaches the ceremony note to gated events before staging. + for event in plan["supersedes"] + plan["retire_pending"]: + event["note"] = "generated ceremony" + staged = registry.stage(_plan_events(plan)) + assert bsc.registry_agreement_problems(catalog, staged) == [] + + +# --- handed-over keys at the docket call site (2026-10-18) ------------------- +# +# Finding L2 of the chronicle#360 round-2 review (pre-existing). When a +# catalog row carried a UUID other than its key's binding, resolve_uuid +# planned the gated remint (revive the key if retired, then supersede it to +# the row's UUID) even when the key had retired after handing its lineage +# over through a succeeds event. The grammar refuses to revive such a key, +# so staging rejected the plan before any write. A docket entry that gains +# a country reaches this: it re-keys a geography-free placeholder row onto +# the country key. The builder now reclaims the key on the row's UUID, the +# move the grammar provides for a handed-over key. + +LINEAGE = "agency.lineage" +RECLAIM_NOTE = "identity re-established after its lineage was handed over" +_B = {"concept": LINEAGE, "geography": US_UNDECLARED, "entity": None} +_E = {"concept": LINEAGE, "geography": US_UNDECLARED, "entity": ECONOMY} +_US_ENTRY = {"series": LINEAGE, "cadence": "monthly", + "extras": {"country": "US", "targetUnit": "percent"}} + + +def _b_handed_to_e() -> list[dict]: + """The reviewer's repro: B minted on U1, retired, succeeded by E.""" + return [ + dict(_B, uuid=U1), + dict(_B, uuid=U1, retired=True, note=CHAIN_NOTE), + dict(_E, uuid=U1, succeeds=_B), + ] + + +def test_a_curated_row_uuid_reclaims_a_handed_over_key( + tmp_path: pathlib.Path, +) -> None: + # The catalog holds a geography-free docket row carrying a hand-edited, + # unregistered U2, and the docket entry gains country US, so the row + # re-keys onto B. Base planned revive B + supersede B -> U2, and staging + # refused both ("revives a consumed lineage", then "share uuid"). B is + # terminal: U2 starts its new lineage through a reclaim, which needs no + # ceremony because no live binding changes. + events = _b_handed_to_e() + registry = _registry(tmp_path, events) + assert not registry.is_live(_key(_B)) and _key(_B) in registry.consumed + assert _owner(registry, U2) is None + catalog, plan = _build( + tmp_path, [], existing={"series": [_docket_row(LINEAGE, U2, None)]}, + registry_entries=events, docket={"series": [_US_ENTRY]}, + ) + (row,) = catalog["series"] + assert (_key(row), row["uuid"], row["status"]) == ( + _key(_B), U2, "docket-only" + ) + assert plan["enrich_retires"] == plan["revives"] == plan["supersedes"] == [] + assert [bsc.UuidRegistry.render_entry(e) for e in plan["mints"]] == [ + bsc.UuidRegistry.render_entry( + dict(_B, uuid=U2, reclaimed=True, note=RECLAIM_NOTE) + ) + ] + # E has no row in this synthetic catalog, so its live binding is the + # one gated event, as on base. + assert [(_key(e), e["uuid"]) for e in plan["retire_pending"]] == [ + (_key(_E), U1) + ] + for event in plan["retire_pending"]: + event["note"] = "synthetic gate approval" + staged = registry.stage(_plan_events(plan)) + assert staged.is_live(_key(_B)) and staged.binding(_key(_B)) == U2 + assert _key(_B) not in staged.consumed # an ordinary lineage again + assert staged.binding(_key(_E)) == U1 # U1 stays with its lineage + assert bsc.registry_agreement_problems(catalog, staged) == [] + + +def test_a_handed_over_key_reclaims_the_row_uuid_through_main( + tmp_path: pathlib.Path, +) -> None: + # The same move end to end, with E's observation withdrawn (E retired) + # so nothing else changes: main writes exactly the reclaim without + # --allow-remint, --check passes, and a rerun is byte-idempotent. + events = _b_handed_to_e() + [ + dict(_E, uuid=U1, retired=True, note="observation withdrawn upstream"), + ] + argv = _repo(tmp_path, [], seed={"series": [_US_ENTRY]}) + _registry(tmp_path, events) + (tmp_path / "catalog.json").write_text( + json.dumps({"series": [_docket_row(LINEAGE, U2, None)]}) + "\n", + encoding="utf-8", + ) + assert bsc.main(argv + ["--check"]) == 1 # the reclaim is unrecorded + assert bsc.main(argv) == 0 + assert bsc.main(argv + ["--check"]) == 0 + assert [ + (_key(e), e["uuid"], _event_kind(e), e.get("note")) + for e in _registry_lines(tmp_path)[len(events):] + ] == [(_key(_B), U2, "reclaimed", RECLAIM_NOTE)] + (row,) = json.loads((tmp_path / "catalog.json").read_text())["series"] + assert (_key(row), row["uuid"], row["status"]) == ( + _key(_B), U2, "docket-only" + ) + snapshot = ( + (tmp_path / "catalog.json").read_bytes(), + (tmp_path / "registry.jsonl").read_bytes(), + ) + assert bsc.main(argv) == 0 + assert ( + (tmp_path / "catalog.json").read_bytes(), + (tmp_path / "registry.jsonl").read_bytes(), + ) == snapshot + + +def test_a_registered_row_uuid_never_reclaims_a_handed_over_key( + tmp_path: pathlib.Path, +) -> None: + # A reclaim needs a UUID new to the registry. When the row carries one + # the registry already knows (here U3, B's own uuid before a curated + # remint), the builder refuses at plan time naming the cause, with or + # without --allow-remint, and leaves both artifacts untouched. Base + # refused too, but blamed a supersede. The remedy the refusal names + # works: delete the row, and the builder reclaims B on a fresh UUID. + bystander = _row("bls.cps.unemployment_rate") + argv = _repo(tmp_path, [bystander], seed={"series": []}) + assert bsc.main(argv) == 0 + events = _registry_lines(tmp_path) + [ + dict(_B, uuid=U3), + dict(_B, uuid=U1, supersedes=U3, + note="curated remint of the placeholder"), + dict(_B, uuid=U1, retired=True, note=CHAIN_NOTE), + dict(_E, uuid=U1, succeeds=_B), + dict(_E, uuid=U1, retired=True, note="observation withdrawn upstream"), + ] + _registry(tmp_path, events) + catalog = json.loads((tmp_path / "catalog.json").read_text()) + catalog["series"].append(_docket_row(LINEAGE, U3, None)) + (tmp_path / "catalog.json").write_text( + json.dumps(catalog, indent=2) + "\n", encoding="utf-8" + ) + (tmp_path / "seed.json").write_text( + json.dumps({"series": [_US_ENTRY]}), encoding="utf-8" + ) + catalog_before = (tmp_path / "catalog.json").read_bytes() + registry_before = (tmp_path / "registry.jsonl").read_bytes() + for flags in ([], ["--allow-remint", "--remint-note", "curated"]): + with pytest.raises( + SystemExit, match="a reclaimed identity is a NEW series" + ) as refused: + bsc.main(argv + flags) + assert "handed its lineage over" in str(refused.value) + assert (tmp_path / "catalog.json").read_bytes() == catalog_before + assert (tmp_path / "registry.jsonl").read_bytes() == registry_before + + catalog["series"] = [r for r in catalog["series"] if r["concept"] != LINEAGE] + (tmp_path / "catalog.json").write_text( + json.dumps(catalog, indent=2) + "\n", encoding="utf-8" + ) + assert bsc.main(argv) == 0 + assert bsc.main(argv + ["--check"]) == 0 + (reclaim,) = _registry_lines(tmp_path)[len(events):] + assert (_key(reclaim), _event_kind(reclaim)) == (_key(_B), "reclaimed") + assert reclaim["uuid"] not in {U1, U3} + (row,) = [ + r for r in json.loads((tmp_path / "catalog.json").read_text())["series"] + if r["concept"] == LINEAGE + ] + assert (_key(row), row["uuid"]) == (_key(_B), reclaim["uuid"]) + + +def _docket_identity(country: str | None) -> dict: + """The identity a docket entry for _GEN_CONCEPT keys its row under.""" + geography = ( + None if country is None + else bsc._identity_geography(bsc.COUNTRY_GEOGRAPHY[country]) + ) + return {"concept": _GEN_CONCEPT, "geography": geography, "entity": None} + + +@settings( + max_examples=400, + deadline=None, + suppress_health_check=[HealthCheck.function_scoped_fixture], +) +@given(data=st.data()) +def test_docket_enrichment_is_planned_exactly_when_the_grammar_admits_it( + tmp_path_factory: pytest.TempPathFactory, data: st.DataObject +) -> None: + """The differential above, at the docket call site. + + A docket entry keys its row by its own name and country, not by the + row it matches. So unlike an observed bucket it can re-key a + placeholder onto a key that handed its lineage over and retired (the + class of finding L2), and it can reach a placeholder of another + concept through a curated alias. For any history the grammar admits, + any existing docket-only row (as above, or carrying a UUID the + registry never met; of the entry's concept, or of another concept + that lists the entry's name as an alias) and a docket entry declaring + no country, US or GB, the same four properties hold: a hop is planned + exactly when the grammar admits it, every plan stages and agrees with + the catalog, a refusal is a plan-time diagnostic, and without a hop the + row takes its key's binding or a UUID new to the registry. + """ + tmp_path = tmp_path_factory.mktemp("docket-differential") + registry = _generated_registry(data) + # A docket hop starts from the live geography-free placeholder. When + # the history lacks one, sometimes mint it (a move the grammar checks). + root = _docket_identity(None) + unused = [u for u in _GEN_UUIDS if _owner(registry, u) is None] + if _key(root) not in registry.latest and unused and data.draw( + st.booleans(), label="mint the geography-free placeholder" + ): + registry = registry.stage( + [dict(root, uuid=data.draw(st.sampled_from(unused)))] + ) + (tmp_path / "registry.jsonl").write_bytes(registry.raw) + + def handed_over(key: tuple[str, str, str]) -> bool: + return key in registry.consumed and not registry.is_live(key) + + # Entries whose key handed over and retired are weighted up most, then + # entries whose key a hop could land on (unbound or retired). + countries = [] + for option in (None, "US", "GB"): + key = _key(_docket_identity(option)) + weight = 9 if handed_over(key) else 2 if registry.is_live(key) else 4 + countries += [option] * weight + country = data.draw(st.sampled_from(countries), label="country") + target = _docket_identity(country) + target_key = _key(target) + row_concept = data.draw( + st.sampled_from([_GEN_CONCEPT] * 4 + [_GEN_OTHER]), label="row concept" + ) + placeholders = [ + e for e in registry.latest.values() + if e["concept"] == row_concept and e["entity"] is None + ] + live = [e for e in placeholders if not e.get("retired")] + unused = [u for u in _GEN_UUIDS if _owner(registry, u) is None] + candidates = ( + [(e["geography"], e["uuid"]) for e in live] * 8 + + [(e["geography"], e["uuid"]) for e in live + if _key(e) in registry.consumed] * 16 + # The live geography-free placeholder, which a country hands over. + + [(None, e["uuid"]) for e in live if e["geography"] is None] * 16 + + [(e["geography"], e["uuid"]) for e in placeholders] * 2 + # A hand-edited UUID the registry never met, on a row the entry + # re-keys (geography-free) or keeps where it is. + + [(g, u) for g in (None, target["geography"]) for u in unused] * 4 + + [(g, u) for g in _PLACEHOLDER_GEOS for u in _GEN_UUIDS] + ) + geography, row_uuid = data.draw(st.sampled_from(candidates)) + placeholder = {"concept": row_concept, "geography": geography, + "entity": None} + retire = dict(placeholder, uuid=row_uuid, retired=True, note=CHAIN_NOTE) + succeed = dict(target, uuid=row_uuid, succeeds=placeholder) + try: + registry.stage([retire, succeed]) + admitted = True + except SystemExit: + admitted = False + if handed_over(target_key) and row_uuid != registry.binding(target_key): + hypothesis_event( + "entry key handed over; row uuid " + + ("unregistered" if _owner(registry, row_uuid) is None + else "registered") + ) + hypothesis_event(f"admitted={admitted}") + row = _docket_row(row_concept, row_uuid, geography) + if row_concept != _GEN_CONCEPT: + row["aliases"] = [_GEN_CONCEPT] + extras = {"targetUnit": "percent"} + if country is not None: + extras["country"] = country + entry = {"series": _GEN_CONCEPT, "cadence": "monthly", "extras": extras} + try: + catalog, plan = _build( + tmp_path, [], existing={"series": [row]}, + docket={"series": [entry]}, + ) + except SystemExit as exc: + assert not admitted, exc + assert any( + diagnostic in str(exc) + for diagnostic in ( + "explicit ceremony", + "superseding UUIDs must be new", + "a reclaimed identity is a NEW series", + "hands over at most once", + "never crosses concepts", + ) + ), exc + return + hop = [e for e in plan["mints"] if "succeeds" in e] + assert bool(hop) == admitted + (built,) = catalog["series"] + assert _key(built) == target_key + if admitted: + assert built["uuid"] == row_uuid + assert [ + bsc.UuidRegistry.render_entry(e) + for e in plan["enrich_retires"] + hop + ] == [bsc.UuidRegistry.render_entry(e) for e in (retire, succeed)] + else: + assert built["uuid"] == registry.binding(target_key) or ( + _owner(registry, built["uuid"]) is None + ) + for event in plan["supersedes"] + plan["retire_pending"]: + event["note"] = "generated ceremony" + staged = registry.stage(_plan_events(plan)) + assert bsc.registry_agreement_problems(catalog, staged) == []