From 66aaeafa0b245ef6021c93690989727c959e41ed Mon Sep 17 00:00:00 2001 From: Max Ghenis Date: Fri, 16 Oct 2026 18:58:48 -0400 Subject: [PATCH 1/5] Let an enriched docket placeholder be enriched again _plan_enrichment required uuid_owner[row_uuid] == the row's own key. uuid_owner records FIRST ownership and never moves, so a placeholder that was itself the successor of an earlier enrichment (geography-free -> geography-only) never owns its UUID, and its observation fell to the mint path and was refused as UUID reuse. This blocked usaspending.dod.prime_award_obligations (a2649fd9) from taking its FY2026 observation (Thesis run 38962977031). Drop that conjunct. The live-binding conditions that remain (enrichment-shaped, the row's key live, and bound to row_uuid now) are the contract and still refuse stale, superseded, retired, handed-over and reclaimed rows. The mint-path refusal now says "first bound to", since uuid_owner is not the current binding. Co-Authored-By: Claude Opus 5.5 --- scripts/build_series_catalog.py | 26 ++++++++++++++++---------- 1 file changed, 16 insertions(+), 10 deletions(-) diff --git a/scripts/build_series_catalog.py b/scripts/build_series_catalog.py index 8b6f47c9..c514b0f4 100644 --- a/scripts/build_series_catalog.py +++ b/scripts/build_series_catalog.py @@ -2133,7 +2133,6 @@ def _plan_enrichment( placeholder whose live binding owns ``row_uuid``; True if planned. """ prior_own_key = UuidRegistry.entry_key(prior) - owner_key = registry.uuid_owner.get(uuid_module.UUID(row_uuid).int) prior_geo = prior.get("geography") or None geo = geography or {} enrichment_shaped = ( @@ -2149,14 +2148,17 @@ def _plan_enrichment( ) ) ) - # uuid_owner records FIRST ownership and never moves, so alone it - # would accept a stale catalog row still carrying a UUID its key - # has since superseded away from. The contract is about the LIVE - # binding: demand it be row_uuid itself. Staging replay would - # reject such a plan anyway; this fails at plan time instead. + # The contract is the LIVE binding: the row's own key must be live + # and bound to row_uuid right now. That alone refuses a stale row + # still carrying a UUID its key has since superseded away from, or + # a key that was retired, handed over or reclaimed. Staging replay + # would reject such a plan anyway; this fails at plan time instead. + # Historical ownership (uuid_owner) is not part of the contract: it + # records FIRST ownership and never moves, so a placeholder that + # was itself the successor of an earlier enrichment never owns its + # UUID and could never be enriched again. if not ( enrichment_shaped - and owner_key == prior_own_key and registry.is_live(prior_own_key) and registry.binding(prior_own_key) == row_uuid ): @@ -2164,8 +2166,11 @@ def _plan_enrichment( # Docket-placeholder enrichment: the binding MOVES to the observed # identity via an explicit retire + succeeds pair. UUID continuity # is preserved, so no ceremony flag is needed — this holds for the - # first enrichment and for a re-enrichment after a withdraw/reclaim - # cycle (the grammar accepts succeeds on a retired successor key). + # first enrichment, for a chain of them (a geography-free + # placeholder enriched to a geography-only one, then to the + # observed identity), and for a re-enrichment after a + # withdraw/reclaim cycle (the grammar accepts succeeds on a retired + # successor key). plan["enrich_retires"].append( dict( _registry_event( @@ -2285,7 +2290,8 @@ def resolve_uuid( return row_uuid raise SystemExit( f"identity {canon_key} would mint uuid {row_uuid}, which " - f"the registry already binds to {owner_key} — UUID reuse " + f"the registry already knows (first bound to {owner_key}) " + "— UUID reuse " "requires an explicit ceremony (merge via curated alias + " "--allow-remint, or placeholder enrichment)" ) From adb190152a4f31e23af581f0f62d50a1c15e32f6 Mon Sep 17 00:00:00 2001 From: Max Ghenis Date: Fri, 16 Oct 2026 19:14:57 -0400 Subject: [PATCH 2/5] Pin chained placeholder enrichment and the stale-row refusals MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Regressions: - the production state (registry lines 196/249/250 for a2649fd9): the geography-only placeholder takes its first print as exactly the retire + succeeds pair Thesis's append guard admits; - the chain through main (docket entry learns a country, then the first print lands): no ceremony, one UUID, --check after each hop; - six stale rows after a chain (superseded, retired, handed over, reclaimed, chain root, another series' UUID) still refused at plan time by the live-binding conditions alone, artifacts byte-untouched. Properties (Hypothesis): - every builder-reachable chain, any country, vintage and entity, keeps one UUID, writes one retire + succeeds pair per hop, passes --check and is byte-idempotent; - in any grammar-admitted history, a live binding its key did not first own was established by succeeds (why the owner conjunct only ever refused chains); - differential: the builder plans an enrichment exactly when UuidRegistry accepts the pair, everything it plans stages and agrees with its catalog, and refusals are plan-time diagnostics. Counterfactually verified: with the owner conjunct restored, both regressions, the chain property and the differential property fail (the differential on 5/5 seeds, shrinking to the dod shape); the stale-row tests pass either way. The mint-path diagnostic now says "already knows (first bound to …)", so test_mint_never_reuses_a_bound_uuid matches that. Co-Authored-By: Claude Opus 5.5 --- docs/catalog-curation-backlog.md | 11 + tests/test_build_series_catalog.py | 658 ++++++++++++++++++++++++++++- 2 files changed, 668 insertions(+), 1 deletion(-) diff --git a/docs/catalog-curation-backlog.md b/docs/catalog-curation-backlog.md index d729b6f1..5c854429 100644 --- a/docs/catalog-curation-backlog.md +++ b/docs/catalog-curation-backlog.md @@ -171,6 +171,17 @@ were closed on 2026-08-09 by chronicle#153: replay rejects the hand-forged plan before any write reaches disk, and the builder end-to-end refuses the stale catalog, leaving both artifacts untouched. +- 2026-10-16: #153 added the live-binding check but kept the first-owner + conjunct (`uuid_owner[row_uuid] == prior_own_key`) alongside it. Because + the first owner never moves, that conjunct refused every second + enrichment hop. A placeholder that had already been enriched once (a + geography-free placeholder enriched to geography-only when its docket + entry learned a country) could not take its first print. The conjunct is + gone, and the live binding alone is the contract. + `usaspending.dod.prime_award_obligations` (a2649fd9) hit this in Thesis + run 38962977031. Tests now pin the chain and the stale-row refusals. A + property test also checks that the builder plans an enrichment exactly + when the registry grammar admits the retire + succeeds pair. ## Country geography on Colorado-scoped concepts, and one raw series id (2026-09-04) diff --git a/tests/test_build_series_catalog.py b/tests/test_build_series_catalog.py index 8bfd0fb7..860dde7a 100644 --- a/tests/test_build_series_catalog.py +++ b/tests/test_build_series_catalog.py @@ -16,6 +16,7 @@ import pathlib import sys import tempfile +import uuid import pytest from hypothesis import HealthCheck, given, settings @@ -2014,7 +2015,7 @@ def test_mint_never_reuses_a_bound_uuid(tmp_path: pathlib.Path) -> None: }, ] } - with pytest.raises(SystemExit, match="already binds"): + with pytest.raises(SystemExit, match=r"already knows \(first bound to"): _build( tmp_path, [_row("new.one"), _row("new.two")], @@ -4424,3 +4425,658 @@ def observed_total(catalog: dict) -> int: undone = json.loads((tmp_path / "catalog.json").read_text()) assert _without_digests(undone) == _without_digests(before) assert bsc.main(argv + ["--check"]) == 0 + + +# --- chained placeholder enrichment (2026-10-16) ---------------------------- +# +# A docket placeholder can be enriched more than once. The geography-free +# placeholder becomes a geography-only one when its docket entry learns a +# country, and that one becomes the observed identity when the first print +# lands. Each hop is a retire + succeeds pair carrying one UUID. The guard is +# the LIVE binding of the row's own key. uuid_owner records first ownership +# and never moves, so it names the first hop's predecessor, never the +# second's: requiring it (the guard before this fix) refused every second +# hop. Thesis run 38962977031 hit exactly that on +# usaspending.dod.prime_award_obligations. + +DOD = "usaspending.dod.prime_award_obligations" +CHAIN_NOTE = "docket placeholder enriched by first observed identity" +US_UNDECLARED = {"level": "country", "id": "0100000US", "vintage": None} +U3 = "cccccccc-3333-4333-8333-333333333333" +U4 = "eeeeeeee-5555-4555-8555-555555555555" + + +def _key(fields: dict) -> tuple[str, str, str]: + return bsc.UuidRegistry.entry_key(fields) + + +def _owner(registry: bsc.UuidRegistry, row_uuid: str): + return registry.uuid_owner.get(uuid.UUID(row_uuid).int) + + +def _plan_events(plan: dict) -> list[dict]: + """Every registry event a plan stages, in main's order.""" + return ( + plan["enrich_retires"] + plan["mints"] + plan["revives"] + + plan["supersedes"] + plan["retire_pending"] + ) + + +def _event_kind(event: dict) -> str: + for marker in ("succeeds", "supersedes", "retired", "revived", "reclaimed"): + if marker in event: + return marker + return "mint" + + +def _registry_lines(tmp_path: pathlib.Path) -> list[dict]: + return [ + json.loads(line) + for line in (tmp_path / "registry.jsonl").read_text().splitlines() + ] + + +def _dod_observation() -> dict: + """The FY2026 first print as Thesis records it (registered snapshot).""" + return _row( + f"{DOD}.fy2026", + rid=f"{DOD}.fy2026.registered_query_snapshot", + unit="billions USD", + period={"type": "year", "value": "2026"}, + ) + + +def _dod_chain() -> list[dict]: + """Registry lines 196, 249 and 250 for a2649fd9, with U1 for the UUID.""" + return [ + _mint(DOD, U1), + dict(_mint(DOD, U1), retired=True, note=CHAIN_NOTE), + dict( + _mint(DOD, U1, geography=US_UNDECLARED), + succeeds={"concept": DOD, "geography": None, "entity": None}, + ), + ] + + +def _docket_row(concept: str, row_uuid: str, geography: dict | None) -> dict: + return { + "uuid": row_uuid, + "concept": concept, + "geography": geography, + "entity": None, + "aliases": [], + "status": "docket-only", + } + + +def test_a_placeholder_enriched_once_is_enriched_again( + tmp_path: pathlib.Path, +) -> None: + # Production repro (Thesis run 38962977031, 2026-10-16). The catalog + # still holds the geography-only placeholder docket-only; its first + # print must take the second hop instead of falling through to the + # UUID-reuse refusal. + registry = _registry(tmp_path, _dod_chain()) + geo_key = _key({"concept": DOD, "geography": US_UNDECLARED, "entity": None}) + # The trap: the live placeholder holds U1 without having first owned it. + assert registry.is_live(geo_key) and registry.binding(geo_key) == U1 + assert _owner(registry, U1) != geo_key + existing = { + "series": [ + _docket_row(DOD, U1, dict(bsc.COUNTRY_GEOGRAPHY["US"])) + ] + } + catalog, plan = _build( + tmp_path, [_dod_observation()], existing=existing, + registry_entries=_dod_chain(), + ) + (row,) = catalog["series"] + assert (row["concept"], row["uuid"], row["status"]) == (DOD, U1, "observed") + assert row["entity"] == ECONOMY + # Exactly the pair the diagnosis's patched generator wrote (and that + # thesis#362's append guard admits): retire the geography-only + # placeholder, then succeed it at the observed identity, one UUID. + assert [ + bsc.UuidRegistry.render_entry(e) for e in _plan_events(plan) + ] == [ + bsc.UuidRegistry.render_entry(e) + for e in ( + dict(_mint(DOD, U1, geography=US_UNDECLARED), retired=True, + note=CHAIN_NOTE), + dict( + _mint(DOD, U1, geography=US_IDENTITY, entity=ECONOMY), + succeeds={"concept": DOD, "geography": US_UNDECLARED, + "entity": None}, + ), + ) + ] + assert not plan["dropped"] + staged = registry.stage(_plan_events(plan)) + observed_key = _key(row) + assert staged.is_live(observed_key) and staged.binding(observed_key) == U1 + assert not staged.is_live(geo_key) and geo_key in staged.consumed + assert [k for k, e in staged.live_bindings() if e["uuid"] == U1] == [ + observed_key + ] + assert bsc.registry_agreement_problems(catalog, staged) == [] + + +def test_placeholder_chain_keeps_one_uuid_through_main( + tmp_path: pathlib.Path, +) -> None: + # The same chain end to end through main: the docket entry starts with + # no country, learns "US" (first hop, docket to docket), then its first + # print lands (second hop). Neither hop needs a ceremony flag, the UUID + # never changes, and --check passes after each. + entry = {"series": DOD, "cadence": "annual", + "extras": {"targetUnit": "billions USD"}} + bystander = _row("bls.cps.unemployment_rate") + argv = _repo(tmp_path, [bystander], seed={"series": [entry]}) + + def rows_by_concept() -> dict[str, dict]: + catalog = json.loads((tmp_path / "catalog.json").read_text()) + return {r["concept"]: r for r in catalog["series"]} + + assert bsc.main(argv) == 0 + placeholder = rows_by_concept()[DOD] + assert placeholder["status"] == "docket-only" + assert placeholder["geography"] is None + lineage = placeholder["uuid"] + bystander_uuid = rows_by_concept()["bls.cps.unemployment_rate"]["uuid"] + + entry["extras"]["country"] = "US" + (tmp_path / "seed.json").write_text( + json.dumps({"series": [entry]}), encoding="utf-8" + ) + assert bsc.main(argv) == 0 + country = rows_by_concept()[DOD] + assert (country["uuid"], country["status"]) == (lineage, "docket-only") + assert country["geography"] == bsc.COUNTRY_GEOGRAPHY["US"] + assert bsc.main(argv + ["--check"]) == 0 + + (tmp_path / "obs.jsonl").write_text( + "".join(json.dumps(r) + "\n" for r in [bystander, _dod_observation()]), + encoding="utf-8", + ) + assert bsc.main(argv) == 0 + observed = rows_by_concept()[DOD] + assert (observed["uuid"], observed["status"]) == (lineage, "observed") + assert observed["entity"] == ECONOMY + assert rows_by_concept()["bls.cps.unemployment_rate"]["uuid"] == ( + bystander_uuid + ) + assert bsc.main(argv + ["--check"]) == 0 + + events = _registry_lines(tmp_path) + assert [ + (e["geography"], e["entity"], _event_kind(e)) + for e in events + if e["concept"] == DOD + ] == [ + (None, None, "mint"), + (None, None, "retired"), + (US_UNDECLARED, None, "succeeds"), + (US_UNDECLARED, None, "retired"), + (US_IDENTITY, ECONOMY, "succeeds"), + ] + assert {e["uuid"] for e in events if e["concept"] == DOD} == {lineage} + assert not any( + _event_kind(e) in ("supersedes", "reclaimed") for e in events + ) + catalog_bytes = (tmp_path / "catalog.json").read_bytes() + registry_bytes = (tmp_path / "registry.jsonl").read_bytes() + assert bsc.main(argv) == 0 + assert (tmp_path / "catalog.json").read_bytes() == catalog_bytes + assert (tmp_path / "registry.jsonl").read_bytes() == registry_bytes + + +# Each stage is (docket entry declares a country, observation present). +_CHAINS = ( + ((False, False), (True, False)), + ((False, False), (False, True)), + ((True, False), (True, True)), + ((False, False), (True, False), (True, True)), +) + + +@settings( + max_examples=60, + deadline=None, + suppress_health_check=[HealthCheck.function_scoped_fixture], +) +@given( + chain=st.sampled_from(_CHAINS), + country=st.sampled_from(sorted(bsc.COUNTRY_GEOGRAPHY)), + vintage=st.sampled_from(["current", "2020", None]), + entity=st.sampled_from([ECONOMY, HAND_ENTITY]), +) +def test_any_placeholder_chain_keeps_one_uuid_and_passes_check( + tmp_path_factory: pytest.TempPathFactory, + chain: tuple, + country: str, + vintage: str | None, + entity: dict, +) -> None: + """For every chain of placeholder enrichments the builder can reach + (geography-free -> country, geography-free -> observed, country -> + observed, and all three hops), in any country, at any observed + vintage and entity: every hop writes exactly one retire + succeeds + pair handing over the binding it retired, needs no ceremony flag, + keeps the lineage's one UUID, leaves exactly one live binding for it + (at the row's identity), leaves unrelated series untouched, passes + --check, and is byte-idempotent.""" + tmp_path = tmp_path_factory.mktemp("chain") + concept = "agency.chain.series" + neighbour = {"series": "agency.chain.neighbour", "cadence": "monthly", + "extras": {"country": "US", "targetUnit": "percent"}} + bystander = _row("bls.cps.unemployment_rate") + observation = _row( + concept, + geography={**bsc.COUNTRY_GEOGRAPHY[country], "vintage": vintage}, + entity=entity, + ) + + def write_stage(stage: tuple[bool, bool]) -> None: + declares_country, observed = stage + extras = {"targetUnit": "percent"} + if declares_country: + extras["country"] = country + seed = {"series": [ + {"series": concept, "cadence": "monthly", "extras": extras}, + neighbour, + ]} + (tmp_path / "seed.json").write_text(json.dumps(seed), encoding="utf-8") + rows = [bystander] + ([observation] if observed else []) + (tmp_path / "obs.jsonl").write_text( + "".join(json.dumps(r) + "\n" for r in rows), encoding="utf-8" + ) + + def catalog_rows() -> list[dict]: + return json.loads((tmp_path / "catalog.json").read_text())["series"] + + argv = _repo(tmp_path, []) + write_stage(chain[0]) + assert bsc.main(argv) == 0 + (first,) = [r for r in catalog_rows() if r["concept"] == concept] + lineage = first["uuid"] + others = { + _key(r): r["uuid"] for r in catalog_rows() if r["concept"] != concept + } + for stage in chain[1:]: + before = len(_registry_lines(tmp_path)) + write_stage(stage) + assert bsc.main(argv) == 0 # no --allow-remint + (row,) = [r for r in catalog_rows() if r["concept"] == concept] + assert row["uuid"] == lineage + assert row["status"] == ("observed" if stage[1] else "docket-only") + hop = _registry_lines(tmp_path)[before:] + assert [_event_kind(e) for e in hop] == ["retired", "succeeds"] + assert {e["uuid"] for e in hop} == {lineage} + assert _key(hop[1]["succeeds"]) == _key(hop[0]) + assert _key(hop[1]) == _key(row) + assert { + _key(r): r["uuid"] for r in catalog_rows() if r["concept"] != concept + } == others + registry = bsc.UuidRegistry.load(tmp_path / "registry.jsonl") + assert [ + k for k, e in registry.live_bindings() if e["uuid"] == lineage + ] == [_key(row)] + assert bsc.main(argv + ["--check"]) == 0 + snapshot = ( + (tmp_path / "catalog.json").read_bytes(), + (tmp_path / "registry.jsonl").read_bytes(), + ) + assert bsc.main(argv) == 0 + assert ( + (tmp_path / "catalog.json").read_bytes(), + (tmp_path / "registry.jsonl").read_bytes(), + ) == snapshot + assert { + e["uuid"] for e in _registry_lines(tmp_path) if e["concept"] == concept + } == {lineage} + + +def _stale_after_chain(scenario: str): + """(registry events, stale docket-only row, observed identity): the + DOD chain (geography-free -> geography-only, U1) and then whatever made + the committed catalog's row stale.""" + chain = _dod_chain() + observed_economy = {"concept": DOD, "geography": US_IDENTITY, + "entity": ECONOMY} + observed_hand = dict(observed_economy, entity=HAND_ENTITY) + geo_row = dict(bsc.COUNTRY_GEOGRAPHY["US"]) + handed_on = chain + [ + dict(_mint(DOD, U1, geography=US_UNDECLARED), retired=True, + note=CHAIN_NOTE), + dict(observed_economy, uuid=U1, + succeeds={"concept": DOD, "geography": US_UNDECLARED, + "entity": None}), + ] + if scenario == "superseded": + # A curated remint moved the placeholder to U2; the row kept U1. + events = chain + [ + dict(_mint(DOD, U2, geography=US_UNDECLARED), supersedes=U1, + note="curated remint of the placeholder"), + ] + return events, _docket_row(DOD, U1, geo_row), observed_economy + if scenario == "retired": + # The docket entry was withdrawn (gated retire); the row lingers. + events = chain + [ + dict(_mint(DOD, U1, geography=US_UNDECLARED), retired=True, + note="docket entry withdrawn"), + ] + return events, _docket_row(DOD, U1, geo_row), observed_economy + if scenario == "handed over": + # The placeholder already took its second hop; the row predates it. + return handed_on, _docket_row(DOD, U1, geo_row), observed_hand + if scenario == "reclaimed": + # After the hand-over the observation was withdrawn and the + # placeholder re-established on a fresh UUID; the row kept U1. + events = handed_on + [ + dict(observed_economy, uuid=U1, retired=True, + note="observation withdrawn upstream"), + dict(_mint(DOD, U2, geography=US_UNDECLARED), reclaimed=True, + note="placeholder re-established after handover"), + ] + return events, _docket_row(DOD, U1, geo_row), observed_hand + if scenario == "chain root": + # The row is the geography-free placeholder the first hop retired. + return chain, _docket_row(DOD, U1, None), observed_economy + if scenario == "another series' uuid": + # The row carries a UUID that another live series holds. + events = chain + [_mint("agency.other", U3)] + return events, _docket_row(DOD, U3, geo_row), observed_economy + raise AssertionError(scenario) + + +@pytest.mark.parametrize( + "scenario", + ["superseded", "retired", "handed over", "reclaimed", "chain root", + "another series' uuid"], +) +def test_stale_rows_after_a_chain_are_still_refused( + tmp_path: pathlib.Path, scenario: str +) -> None: + # The case the dropped owner check was written for: a catalog row that + # no longer carries its key's LIVE binding. With the owner check gone, + # the live-binding conditions alone must refuse each one at plan time + # with the ceremony diagnostic, and main must leave both artifacts + # byte-untouched. + events, stale_row, observed = _stale_after_chain(scenario) + registry = _registry(tmp_path, events) + stale_key = _key(stale_row) + assert not ( + registry.is_live(stale_key) + and registry.binding(stale_key) == stale_row["uuid"] + ) + observation = _row( + DOD, geography=dict(observed["geography"], name="United States"), + entity=observed["entity"], + ) + with pytest.raises(SystemExit, match="explicit ceremony"): + _build( + tmp_path, [observation], existing={"series": [stale_row]}, + registry_entries=events, + ) + argv = _repo(tmp_path, [observation], seed={"series": []}) + _registry(tmp_path, events) + (tmp_path / "catalog.json").write_text( + json.dumps({"series": [stale_row]}) + "\n", encoding="utf-8" + ) + catalog_before = (tmp_path / "catalog.json").read_bytes() + registry_before = (tmp_path / "registry.jsonl").read_bytes() + with pytest.raises(SystemExit, match="explicit ceremony"): + bsc.main(argv) + assert (tmp_path / "catalog.json").read_bytes() == catalog_before + assert (tmp_path / "registry.jsonl").read_bytes() == registry_before + + +# A small identity universe for generated registry histories: two concepts +# (so a row can carry another series' UUID), geographies at every +# refinement level, three entities, and six UUIDs (so stale rows collide +# with known values). +_GEN_CONCEPT, _GEN_OTHER = "agency.lineage", "agency.elsewhere" +_PLACEHOLDER_GEOS = ( + None, + {"level": "country", "id": "0100000US", "vintage": None}, + {"level": "country", "id": "0100000US", "vintage": "current"}, + {"level": "country", "id": "0100000US", "vintage": "2020"}, + {"level": "country", "id": "GB", "vintage": None}, +) +_OBSERVED_GEOS = ( + {"level": "country", "id": "0100000US", "vintage": "current"}, + {"level": "country", "id": "0100000US", "vintage": "2020"}, + {"level": "country", "id": "GB", "vintage": "current"}, +) +_GEN_GEOS = _PLACEHOLDER_GEOS + _OBSERVED_GEOS[2:] +_GEN_ENTITIES = (None, ECONOMY, HAND_ENTITY) +_GEN_UUIDS = ( + U1, U2, U3, U4, + "ffffffff-6666-4666-8666-666666666666", + "abababab-7777-4777-8777-777777777777", +) +_GEN_IDENTS = tuple( + {"concept": concept, "geography": geography, "entity": entity} + for concept in (_GEN_CONCEPT, _GEN_OTHER) + for geography in _GEN_GEOS + for entity in _GEN_ENTITIES +) + + +def _ident(entry: dict) -> dict: + return {key: entry[key] for key in ("concept", "geography", "entity")} + + +def _refines(placeholder: dict | None, geography: dict | None) -> bool: + """Whether an enrichment hop from ``placeholder`` to ``geography`` is + plausible. Used only to steer generation toward chains; the registry + grammar alone decides what a history may contain.""" + if placeholder is None: + return True + geography = geography or {} + return (placeholder["level"], placeholder["id"]) == ( + geography.get("level"), geography.get("id") + ) and placeholder["vintage"] in (None, geography.get("vintage")) + + +def _generated_registry(data: st.DataObject) -> bsc.UuidRegistry: + """A registry grown from random events that the grammar accepts. + + Each step draws a move the current state allows: mint an unbound + identity on an unused UUID, retire a live binding, revive a retired + one, supersede or reclaim onto an unused UUID, or take an enrichment + hop (retire an entity-less binding and succeed it at a refinement). + The move is kept only if UuidRegistry accepts it, so every history is + one the grammar itself admits. Hops chain, so placeholders enriched + once, twice and three times occur alongside stale, superseded, + retired, handed-over and reclaimed keys.""" + registry = bsc.UuidRegistry(pathlib.Path("generated.jsonl"), b"") + for _ in range(data.draw(st.integers(1, 12), label="steps")): + held = list(registry.latest.values()) + live = [e for e in held if not e.get("retired")] + retired = [e for e in held if e.get("retired")] + consumed = [e for e in retired if _key(e) in registry.consumed] + unbound = [i for i in _GEN_IDENTS if _key(i) not in registry.latest] + unused = [u for u in _GEN_UUIDS if _owner(registry, u) is None] + moves = [] + if unbound and unused: + moves += ["mint", "mint"] + if live: + moves += ["retire"] + if live and unused: + moves += ["supersede"] + if retired: + moves += ["revive"] + if consumed and unused: + moves += ["reclaim"] + if any(e["entity"] is None for e in live): + moves += ["hop", "hop", "hop"] + if not moves: + break + move = data.draw(st.sampled_from(moves)) + if move == "mint": + # Placeholders (entity-less) are minted twice as often. + pool = [i for i in unbound if i["entity"] is None] * 2 + unbound + events = [dict(data.draw(st.sampled_from(pool)), + uuid=data.draw(st.sampled_from(unused)))] + elif move == "retire": + entry = data.draw(st.sampled_from(live)) + events = [dict(_ident(entry), uuid=entry["uuid"], retired=True, + note="generated retirement")] + elif move == "supersede": + entry = data.draw(st.sampled_from(live)) + events = [dict(_ident(entry), + uuid=data.draw(st.sampled_from(unused)), + supersedes=entry["uuid"], + note="generated curated remint")] + elif move == "revive": + entry = data.draw(st.sampled_from(retired)) + events = [dict(_ident(entry), uuid=entry["uuid"], revived=True)] + elif move == "reclaim": + entry = data.draw(st.sampled_from(consumed)) + events = [dict(_ident(entry), + uuid=data.draw(st.sampled_from(unused)), + reclaimed=True, + note="generated reclaim after handover")] + else: + entry = data.draw(st.sampled_from( + [e for e in live if e["entity"] is None] + )) + targets = [ + ident for ident in _GEN_IDENTS + if ident["concept"] == entry["concept"] + and _key(ident) != _key(entry) + and _refines(entry["geography"], ident["geography"]) + ] + # Entity-less targets are weighted up so hops chain. + target = data.draw(st.sampled_from( + [t for t in targets if t["entity"] is None] * 3 + targets + )) + events = [ + dict(_ident(entry), uuid=entry["uuid"], retired=True, + note=CHAIN_NOTE), + dict(target, uuid=entry["uuid"], succeeds=_ident(entry)), + ] + try: + registry = registry.stage(events) + except SystemExit: + continue + return registry + + +@settings(max_examples=400, deadline=None) +@given(data=st.data()) +def test_a_live_binding_its_key_never_first_owned_came_by_succeeds( + data: st.DataObject, +) -> None: + """Why the owner check was redundant where it was right. In every + history the grammar admits, a live binding (K, U) either has K as U's + first owner, or K's binding was established by a succeeds event. So + the conjunct uuid_owner[U] == K, on top of the live-binding + conditions, refused nothing but enrichment chains.""" + registry = _generated_registry(data) + for key, entry in registry.live_bindings(): + if _owner(registry, entry["uuid"]) == key: + continue + establishing = [ + e for e in registry.entries + if _key(e) == key and _event_kind(e) not in ("retired", "revived") + ] + assert _event_kind(establishing[-1]) == "succeeds", (key, entry) + + +@settings( + max_examples=400, + deadline=None, + suppress_health_check=[HealthCheck.function_scoped_fixture], +) +@given(data=st.data()) +def test_enrichment_is_planned_exactly_when_the_grammar_admits_it( + tmp_path_factory: pytest.TempPathFactory, data: st.DataObject +) -> None: + """Differential: plan-time guard vs. the registry grammar. + + For any history the grammar admits, any docket-only row (live, stale, + superseded, retired, handed over, reclaimed, the root of a chain, or + carrying another series' UUID) and any observation of its concept: + + * completeness: the builder plans the retire + succeeds pair exactly + when UuidRegistry would accept that very pair (so no enrichment + chain is refused); + * soundness: everything the builder plans stages cleanly and agrees + with the catalog it built (so no stale row slips through); + * a refusal is a plan-time diagnostic, never a crash; + * when no enrichment is planned, the row takes its own key's binding + or a UUID new to the registry, never a UUID another key holds. + """ + tmp_path = tmp_path_factory.mktemp("differential") + registry = _generated_registry(data) + (tmp_path / "registry.jsonl").write_bytes(registry.raw) + placeholders = [ + e for e in registry.latest.values() + if e["concept"] == _GEN_CONCEPT and e["entity"] is None + ] + # Steer toward rows at keys the history holds, carrying their last + # binding: live ones (the chained case lives here) and stale ones + # (retired, handed over). Any geography with any UUID stays possible. + candidates = ( + [(e["geography"], e["uuid"]) for e in placeholders + if not e.get("retired")] * 8 + + [(e["geography"], e["uuid"]) for e in placeholders] * 2 + + [(g, u) for g in _PLACEHOLDER_GEOS for u in _GEN_UUIDS] + ) + geography, row_uuid = data.draw(st.sampled_from(candidates)) + placeholder = {"concept": _GEN_CONCEPT, "geography": geography, + "entity": None} + # Observations that refine the row's geography are weighted up; any + # observed geography stays possible (cross-country, vintage clash). + observed = { + "concept": _GEN_CONCEPT, + "geography": data.draw(st.sampled_from( + [g for g in _OBSERVED_GEOS if _refines(geography, g)] * 3 + + list(_OBSERVED_GEOS) + )), + "entity": data.draw(st.sampled_from((ECONOMY, HAND_ENTITY))), + } + retire = dict(placeholder, uuid=row_uuid, retired=True, note=CHAIN_NOTE) + succeed = dict(observed, uuid=row_uuid, succeeds=placeholder) + try: + registry.stage([retire, succeed]) + admitted = True + except SystemExit: + admitted = False + observation = _row( + _GEN_CONCEPT, + geography=dict(observed["geography"], name=None), + entity=observed["entity"], + ) + try: + catalog, plan = _build( + tmp_path, [observation], + existing={"series": [_docket_row(_GEN_CONCEPT, row_uuid, + geography)]}, + ) + except SystemExit as exc: + assert not admitted, exc + assert "explicit ceremony" in str(exc) or ( + "superseding UUIDs must be new" in str(exc) + ), exc + return + hop = [e for e in plan["mints"] if "succeeds" in e] + assert bool(hop) == admitted + (row,) = catalog["series"] + if admitted: + assert row["uuid"] == row_uuid + assert [ + bsc.UuidRegistry.render_entry(e) + for e in plan["enrich_retires"] + hop + ] == [bsc.UuidRegistry.render_entry(e) for e in (retire, succeed)] + else: + assert row["uuid"] == registry.binding(_key(observed)) or ( + _owner(registry, row["uuid"]) is None + ) + # main attaches the ceremony note to gated events before staging. + for event in plan["supersedes"] + plan["retire_pending"]: + event["note"] = "generated ceremony" + staged = registry.stage(_plan_events(plan)) + assert bsc.registry_agreement_problems(catalog, staged) == [] From ac8652da07a0713145cf034682b8c2202c30b791 Mon Sep 17 00:00:00 2001 From: Max Ghenis Date: Sun, 18 Oct 2026 12:10:21 -0400 Subject: [PATCH 3/5] Refuse a consumed or cross-concept predecessor at plan time Independent review of #360 found a state the live binding misses. A placeholder B hands its lineage to C, then (docket loses and regains its country) takes a fresh UUID back from a coarser placeholder A. B is live and bound to that UUID but still consumed, and the grammar never lets a predecessor hand over twice. Without the dropped owner conjunct the builder planned retire(B) + C succeeds B, which staging then rejected. _plan_enrichment now also requires prior_own_key not in registry.consumed, so it refuses at plan time again (as base did). The review's LOW finding, pre-existing: a docket entry matched to a placeholder through a curated alias keyed the docket row under the alias, so the planned succeeds crossed concepts and died in staging. Enrichment now also requires the prior's concept to equal the canonical one. Observed buckets always take the prior's concept, so only that docket path changes, from a staging failure to a plan-time refusal. Tests: - the review's lifecycle through main (seven events, B live and consumed), refused with and without --allow-remint, artifacts byte-untouched; - the cross-concept docket alias, refused at plan time; - the registry generator gains a "return" move (mint a coarser placeholder and hop it onto a retired key that already handed over), and retired hop targets and consumed rows are weighted up, so the differential property reaches live-but-consumed keys. Counterfactuals: without the consumed check, the lifecycle regression and the differential property fail on 5/5 seeds; without the concept check, the alias regression fails. Co-Authored-By: Claude Opus 5.5 --- scripts/build_series_catalog.py | 19 +++- tests/test_build_series_catalog.py | 151 ++++++++++++++++++++++++++--- 2 files changed, 153 insertions(+), 17 deletions(-) diff --git a/scripts/build_series_catalog.py b/scripts/build_series_catalog.py index c514b0f4..0af4c366 100644 --- a/scripts/build_series_catalog.py +++ b/scripts/build_series_catalog.py @@ -2135,9 +2135,13 @@ def _plan_enrichment( prior_own_key = UuidRegistry.entry_key(prior) prior_geo = prior.get("geography") or None geo = geography or {} + # Lineage never crosses concepts. Observed buckets always take the + # prior's concept; a docket entry matched through a curated alias + # can name another one, and the grammar would refuse that succeeds. enrichment_shaped = ( prior.get("status") == "docket-only" and prior.get("entity") is None + and prior.get("concept") == canon_key[0] and ( prior_geo is None or ( @@ -2148,11 +2152,15 @@ def _plan_enrichment( ) ) ) - # The contract is the LIVE binding: the row's own key must be live - # and bound to row_uuid right now. That alone refuses a stale row - # still carrying a UUID its key has since superseded away from, or - # a key that was retired, handed over or reclaimed. Staging replay - # would reject such a plan anyway; this fails at plan time instead. + # The contract is the LIVE binding: the row's own key must be live, + # bound to row_uuid right now, and still able to hand its lineage + # over. Liveness and binding refuse a stale row still carrying a + # UUID its key has since superseded away from, and a key that was + # retired, handed over or reclaimed. A key hands over at most once: + # one that handed over and later returned as the successor of + # another hop is live again but stays consumed, and the grammar + # refuses it as a predecessor. Staging replay would reject each of + # these plans anyway; this fails at plan time instead. # Historical ownership (uuid_owner) is not part of the contract: it # records FIRST ownership and never moves, so a placeholder that # was itself the successor of an earlier enrichment never owns its @@ -2161,6 +2169,7 @@ def _plan_enrichment( enrichment_shaped and registry.is_live(prior_own_key) and registry.binding(prior_own_key) == row_uuid + and prior_own_key not in registry.consumed ): return False # Docket-placeholder enrichment: the binding MOVES to the observed diff --git a/tests/test_build_series_catalog.py b/tests/test_build_series_catalog.py index 860dde7a..7aa0c75a 100644 --- a/tests/test_build_series_catalog.py +++ b/tests/test_build_series_catalog.py @@ -4831,6 +4831,97 @@ def test_stale_rows_after_a_chain_are_still_refused( assert (tmp_path / "registry.jsonl").read_bytes() == registry_before +def test_a_returned_placeholder_never_hands_over_twice( + tmp_path: pathlib.Path, +) -> None: + # Independent-review counterexample (chronicle#360, 2026-10-18). A + # placeholder B hands U1 to its observed identity C. The observation is + # withdrawn and the docket loses its country (fresh placeholder A/U2), + # then the country returns and B takes U2 from A. B is live and bound + # to U2, but it is still consumed: it already handed over once, and the + # grammar never lets a predecessor hand over twice. When C's + # observation returns, the builder must refuse at plan time. Without the + # consumed check it planned retire(B/U2) + C succeeds B, which staging + # rejected. Base refused too (B never first-owned U2), so this pins the + # status quo for this lineage, not a regression. + concept = "agency.lineage" + bystander = _row("bls.cps.unemployment_rate") + observed = _row(concept) + + def stage(country: bool, rows: list[dict]) -> None: + extras = {"targetUnit": "percent"} + if country: + extras["country"] = "US" + seed = {"series": [ + {"series": concept, "cadence": "monthly", "extras": extras}, + ]} + (tmp_path / "seed.json").write_text(json.dumps(seed), encoding="utf-8") + (tmp_path / "obs.jsonl").write_text( + "".join(json.dumps(r) + "\n" for r in rows), encoding="utf-8" + ) + + argv = _repo(tmp_path, []) + remint = ["--allow-remint", "--remint-note", "observation withdrawn"] + stage(True, [bystander]) + assert bsc.main(argv) == 0 # mint B/U1 + stage(True, [bystander, observed]) + assert bsc.main(argv) == 0 # retire B, C succeeds B with U1 + stage(False, [bystander]) + assert bsc.main(argv + remint) == 0 # mint A/U2, retire C + stage(True, [bystander]) + assert bsc.main(argv) == 0 # retire A, B succeeds A with U2 + assert bsc.main(argv + ["--check"]) == 0 + a_key = (concept, bsc._geo_key(None), bsc._entity_key(None)) + b_key = (concept, bsc._geo_key(US_UNDECLARED), bsc._entity_key(None)) + c_key = (concept, bsc._geo_key(US_IDENTITY), bsc._entity_key(ECONOMY)) + assert [ + (_key(e), _event_kind(e)) + for e in _registry_lines(tmp_path) + if e["concept"] == concept + ] == [ + (b_key, "mint"), + (b_key, "retired"), + (c_key, "succeeds"), + (a_key, "mint"), + (c_key, "retired"), + (a_key, "retired"), + (b_key, "succeeds"), + ] + registry = bsc.UuidRegistry.load(tmp_path / "registry.jsonl") + u2 = registry.binding(a_key) + assert registry.is_live(b_key) and registry.binding(b_key) == u2 + assert b_key in registry.consumed # the state the live binding misses + + stage(True, [bystander, observed]) + catalog_before = (tmp_path / "catalog.json").read_bytes() + registry_before = (tmp_path / "registry.jsonl").read_bytes() + for flags in ([], remint): + with pytest.raises(SystemExit, match="superseding UUIDs must be new"): + bsc.main(argv + flags) + assert (tmp_path / "catalog.json").read_bytes() == catalog_before + assert (tmp_path / "registry.jsonl").read_bytes() == registry_before + + +def test_a_docket_alias_never_plans_a_cross_concept_hop( + tmp_path: pathlib.Path, +) -> None: + # Independent-review finding (chronicle#360, LOW, pre-existing): a + # docket entry naming a placeholder by its curated alias keys the + # docket row under the alias, so the planned succeeds crossed concepts + # and died only in staging replay. The builder now refuses at plan time. + existing = {"series": [ + dict(_docket_row("agency.a", U1, None), aliases=["agency.b"]), + ]} + docket = {"series": [{"series": "agency.b", "cadence": "monthly", + "extras": {"country": "US", + "targetUnit": "percent"}}]} + with pytest.raises(SystemExit, match="explicit ceremony"): + _build( + tmp_path, [], existing=existing, + registry_entries=[_mint("agency.a", U1)], docket=docket, + ) + + # A small identity universe for generated registry histories: two concepts # (so a row can carry another series' UUID), geographies at every # refinement level, three entities, and six UUIDs (so stale rows collide @@ -4884,12 +4975,15 @@ def _generated_registry(data: st.DataObject) -> bsc.UuidRegistry: Each step draws a move the current state allows: mint an unbound identity on an unused UUID, retire a live binding, revive a retired - one, supersede or reclaim onto an unused UUID, or take an enrichment - hop (retire an entity-less binding and succeed it at a refinement). + one, supersede or reclaim onto an unused UUID, take an enrichment hop + (retire an entity-less binding and succeed it at a refinement), or + return (mint a coarser placeholder and hop it onto a retired key that + already handed over). The move is kept only if UuidRegistry accepts it, so every history is one the grammar itself admits. Hops chain, so placeholders enriched once, twice and three times occur alongside stale, superseded, - retired, handed-over and reclaimed keys.""" + retired, handed-over and reclaimed keys, and keys that handed over and + then returned as a later hop's successor (live, but consumed).""" registry = bsc.UuidRegistry(pathlib.Path("generated.jsonl"), b"") for _ in range(data.draw(st.integers(1, 12), label="steps")): held = list(registry.latest.values()) @@ -4911,6 +5005,22 @@ def _generated_registry(data: st.DataObject) -> bsc.UuidRegistry: moves += ["reclaim"] if any(e["entity"] is None for e in live): moves += ["hop", "hop", "hop"] + # A consumed placeholder can take a lineage again as a later hop's + # successor (main writes this when a docket entry loses its + # country, which mints a fresh coarser placeholder, and regains + # it): mint a coarser placeholder and hop it straight onto the key. + returns = [ + (coarser, entry) + for entry in consumed + if entry["entity"] is None + for coarser in unbound + if coarser["concept"] == entry["concept"] + and coarser["entity"] is None + and _key(coarser) != _key(entry) + and _refines(coarser["geography"], entry["geography"]) + ] + if returns and unused: + moves += ["return", "return"] if not moves: break move = data.draw(st.sampled_from(moves)) @@ -4938,6 +5048,14 @@ def _generated_registry(data: st.DataObject) -> bsc.UuidRegistry: uuid=data.draw(st.sampled_from(unused)), reclaimed=True, note="generated reclaim after handover")] + elif move == "return": + coarser, entry = data.draw(st.sampled_from(returns)) + fresh = data.draw(st.sampled_from(unused)) + events = [ + dict(coarser, uuid=fresh), + dict(coarser, uuid=fresh, retired=True, note=CHAIN_NOTE), + dict(_ident(entry), uuid=fresh, succeeds=coarser), + ] else: entry = data.draw(st.sampled_from( [e for e in live if e["entity"] is None] @@ -4948,9 +5066,14 @@ def _generated_registry(data: st.DataObject) -> bsc.UuidRegistry: and _key(ident) != _key(entry) and _refines(entry["geography"], ident["geography"]) ] - # Entity-less targets are weighted up so hops chain. + # Entity-less targets are weighted up so hops chain, and retired + # ones further still so lineages return onto keys that already + # handed over (live again, but consumed). + retired_keys = {_key(e) for e in retired} target = data.draw(st.sampled_from( - [t for t in targets if t["entity"] is None] * 3 + targets + [t for t in targets if _key(t) in retired_keys] * 6 + + [t for t in targets if t["entity"] is None] * 3 + + targets )) events = [ dict(_ident(entry), uuid=entry["uuid"], retired=True, @@ -4996,9 +5119,10 @@ def test_enrichment_is_planned_exactly_when_the_grammar_admits_it( ) -> None: """Differential: plan-time guard vs. the registry grammar. - For any history the grammar admits, any docket-only row (live, stale, - superseded, retired, handed over, reclaimed, the root of a chain, or - carrying another series' UUID) and any observation of its concept: + For any history the grammar admits, any docket-only row (live, live + but already handed over, stale, superseded, retired, handed over, + reclaimed, the root of a chain, or carrying another series' UUID) and + any observation of its concept: * completeness: the builder plans the retire + succeeds pair exactly when UuidRegistry would accept that very pair (so no enrichment @@ -5017,11 +5141,14 @@ def test_enrichment_is_planned_exactly_when_the_grammar_admits_it( if e["concept"] == _GEN_CONCEPT and e["entity"] is None ] # Steer toward rows at keys the history holds, carrying their last - # binding: live ones (the chained case lives here) and stale ones - # (retired, handed over). Any geography with any UUID stays possible. + # binding: live ones (the chained case lives here), live ones that + # already handed over (consumed), and stale ones (retired, handed + # over). Any geography with any UUID stays possible. + live = [e for e in placeholders if not e.get("retired")] candidates = ( - [(e["geography"], e["uuid"]) for e in placeholders - if not e.get("retired")] * 8 + [(e["geography"], e["uuid"]) for e in live] * 8 + + [(e["geography"], e["uuid"]) for e in live + if _key(e) in registry.consumed] * 16 + [(e["geography"], e["uuid"]) for e in placeholders] * 2 + [(g, u) for g in _PLACEHOLDER_GEOS for u in _GEN_UUIDS] ) From 5be9ff8ac41f1fb4a198f14f7b35504bab7fac3a Mon Sep 17 00:00:00 2001 From: Max Ghenis Date: Sun, 18 Oct 2026 14:09:29 -0400 Subject: [PATCH 4/5] Name the cause when a placeholder may not hand over Round-2 review of #360 (LOW L1, L3; NIT N1): - L1 (new in this PR): a docket entry matching a placeholder through a curated alias was refused with the UUID-reuse message, whose remedy (curated alias + --allow-remint) cannot help. - L3 (pre-existing): a placeholder that already handed over once was refused with "superseding UUIDs must be new", blaming a remint nobody made. When the placeholder holds the row's UUID live but either hand-over rule fails, _plan_enrichment now raises a diagnostic naming the cause: lineage never crosses concepts (name the docket entry by the placeholder's own concept), or a placeholder hands over at most once (delete its catalog row and rerun with --allow-remint). Both callers already refused every such row, since the UUID is registered, so only the message changes. Tests exercise each suggested curation: renaming the docket entry enriches cleanly, and deleting the consumed placeholder's row then --allow-remint revives the observed identity's own UUID and retires the placeholder, with --check passing. _event_kind now reads planned events too (absent markers there are None, not missing). N1: the backlog bullet and docstring said the live binding alone is the contract; they now state the hand-over rules too. Co-Authored-By: Claude Opus 5.5 --- docs/catalog-curation-backlog.md | 17 +++++-- scripts/build_series_catalog.py | 71 ++++++++++++++++++-------- tests/test_build_series_catalog.py | 80 ++++++++++++++++++++++++------ 3 files changed, 127 insertions(+), 41 deletions(-) diff --git a/docs/catalog-curation-backlog.md b/docs/catalog-curation-backlog.md index 5c854429..13f0a009 100644 --- a/docs/catalog-curation-backlog.md +++ b/docs/catalog-curation-backlog.md @@ -176,12 +176,19 @@ were closed on 2026-08-09 by chronicle#153: the first owner never moves, that conjunct refused every second enrichment hop. A placeholder that had already been enriched once (a geography-free placeholder enriched to geography-only when its docket - entry learned a country) could not take its first print. The conjunct is - gone, and the live binding alone is the contract. + entry learned a country) could not take its first print. `usaspending.dod.prime_award_obligations` (a2649fd9) hit this in Thesis - run 38962977031. Tests now pin the chain and the stale-row refusals. A - property test also checks that the builder plans an enrichment exactly - when the registry grammar admits the retire + succeeds pair. + run 38962977031. The conjunct is gone (#360). The contract is now the + live binding plus the grammar's hand-over rules: the placeholder's key + is live and bound to the row's UUID, it is the same concept as the + identity it hands over to, and it has not handed over before. A key that + handed over and later returned as the successor of another hop is live + again but stays consumed. When the binding holds but either hand-over + rule fails, the builder refuses at plan time and names the cause. + Tests pin the chain, the stale-row refusals, both hand-over refusals + and the curation each refusal suggests. A property test checks that the + builder plans an enrichment exactly when the registry grammar admits + the retire + succeeds pair. ## Country geography on Colorado-scoped concepts, and one raw series id (2026-09-04) diff --git a/scripts/build_series_catalog.py b/scripts/build_series_catalog.py index 0af4c366..685c5f59 100644 --- a/scripts/build_series_catalog.py +++ b/scripts/build_series_catalog.py @@ -2129,19 +2129,23 @@ def _plan_enrichment( entity: dict | None, row_uuid: str, ) -> bool: - """Plan a retire + succeeds pair when ``prior`` is a genuine - placeholder whose live binding owns ``row_uuid``; True if planned. + """Plan a retire + succeeds pair when ``prior`` is a docket + placeholder that holds ``row_uuid`` live and may hand it over to + ``canon_key``; True if planned. + + A placeholder that holds ``row_uuid`` live but may not hand it over + (``canon_key`` is another concept, or the placeholder already + handed over once) raises a diagnostic naming that cause. Every + caller would refuse such a row anyway, since the UUID is + registered, but with a UUID-reuse or remint message that misstates + why. """ prior_own_key = UuidRegistry.entry_key(prior) prior_geo = prior.get("geography") or None geo = geography or {} - # Lineage never crosses concepts. Observed buckets always take the - # prior's concept; a docket entry matched through a curated alias - # can name another one, and the grammar would refuse that succeeds. - enrichment_shaped = ( + placeholder_shaped = ( prior.get("status") == "docket-only" and prior.get("entity") is None - and prior.get("concept") == canon_key[0] and ( prior_geo is None or ( @@ -2152,26 +2156,49 @@ def _plan_enrichment( ) ) ) - # The contract is the LIVE binding: the row's own key must be live, - # bound to row_uuid right now, and still able to hand its lineage - # over. Liveness and binding refuse a stale row still carrying a - # UUID its key has since superseded away from, and a key that was - # retired, handed over or reclaimed. A key hands over at most once: - # one that handed over and later returned as the successor of - # another hop is live again but stays consumed, and the grammar - # refuses it as a predecessor. Staging replay would reject each of - # these plans anyway; this fails at plan time instead. - # Historical ownership (uuid_owner) is not part of the contract: it - # records FIRST ownership and never moves, so a placeholder that - # was itself the successor of an earlier enrichment never owns its - # UUID and could never be enriched again. + # The contract is the LIVE binding plus the grammar's hand-over + # rules. The row's own key must be live and bound to row_uuid right + # now: that refuses a stale row still carrying a UUID its key has + # since superseded away from, and a key that was retired, handed + # over or reclaimed. Staging replay would reject each of these + # plans anyway; this fails at plan time instead. Historical + # ownership (uuid_owner) is not part of the contract: it records + # FIRST ownership and never moves, so a placeholder that was itself + # the successor of an earlier enrichment never owns its UUID and + # could never be enriched again. if not ( - enrichment_shaped + placeholder_shaped and registry.is_live(prior_own_key) and registry.binding(prior_own_key) == row_uuid - and prior_own_key not in registry.consumed ): return False + # Lineage never crosses concepts. Observed buckets always take the + # prior's concept; a docket entry matched through a curated alias + # can name another one, and the grammar refuses that succeeds. + if prior.get("concept") != canon_key[0]: + raise SystemExit( + f"identity {canon_key} matches docket placeholder " + f"{prior_own_key} (uuid {row_uuid}) only through the " + f"curated alias {canon_key[0]!r}; lineage never crosses " + "concepts, so the placeholder cannot be enriched into it — " + f"name the docket entry {prior['concept']!r}, the " + "placeholder's own concept" + ) + # A key hands over at most once. One that handed over and later + # returned as the successor of another hop is live again, but it + # stays consumed and the grammar refuses it as a predecessor. + if prior_own_key in registry.consumed: + raise SystemExit( + f"identity {canon_key} would take uuid {row_uuid} from docket " + f"placeholder {prior_own_key}, which already handed its " + "lineage over once and holds this uuid as the successor of a " + "later enrichment; a placeholder hands over at most once. " + "Curate it: for example, delete the placeholder's catalog row " + "and rerun with --allow-remint, so the observed identity " + "takes its own lineage (reviving its retired binding, or " + "minting if it has none) and the placeholder's binding " + "retires" + ) # Docket-placeholder enrichment: the binding MOVES to the observed # identity via an explicit retire + succeeds pair. UUID continuity # is preserved, so no ceremony flag is needed — this holds for the diff --git a/tests/test_build_series_catalog.py b/tests/test_build_series_catalog.py index 7aa0c75a..ec326087 100644 --- a/tests/test_build_series_catalog.py +++ b/tests/test_build_series_catalog.py @@ -4463,8 +4463,9 @@ def _plan_events(plan: dict) -> list[dict]: def _event_kind(event: dict) -> str: + # Planned events carry absent markers as None; rendered lines omit them. for marker in ("succeeds", "supersedes", "retired", "revived", "reclaimed"): - if marker in event: + if event.get(marker) is not None: return marker return "mint" @@ -4840,10 +4841,11 @@ def test_a_returned_placeholder_never_hands_over_twice( # then the country returns and B takes U2 from A. B is live and bound # to U2, but it is still consumed: it already handed over once, and the # grammar never lets a predecessor hand over twice. When C's - # observation returns, the builder must refuse at plan time. Without the - # consumed check it planned retire(B/U2) + C succeeds B, which staging - # rejected. Base refused too (B never first-owned U2), so this pins the - # status quo for this lineage, not a regression. + # observation returns, the builder must refuse at plan time, naming the + # cause. Without the consumed check it planned retire(B/U2) + C + # succeeds B, which staging rejected. Base refused too (B never + # first-owned U2), but with a remint message that blamed a supersede + # nobody made. The curation the diagnostic suggests must then work. concept = "agency.lineage" bystander = _row("bls.cps.unemployment_rate") observed = _row(concept) @@ -4896,11 +4898,37 @@ def stage(country: bool, rows: list[dict]) -> None: catalog_before = (tmp_path / "catalog.json").read_bytes() registry_before = (tmp_path / "registry.jsonl").read_bytes() for flags in ([], remint): - with pytest.raises(SystemExit, match="superseding UUIDs must be new"): + with pytest.raises(SystemExit, match="hands over at most once"): bsc.main(argv + flags) assert (tmp_path / "catalog.json").read_bytes() == catalog_before assert (tmp_path / "registry.jsonl").read_bytes() == registry_before + # The suggested curation: delete the placeholder's catalog row and + # rerun with --allow-remint. C revives its own U1; B's U2 retires. + catalog = json.loads(catalog_before) + catalog["series"] = [ + r for r in catalog["series"] if r["concept"] != concept + ] + (tmp_path / "catalog.json").write_text( + json.dumps(catalog, indent=2) + "\n", encoding="utf-8" + ) + assert bsc.main(argv) == 1 # retiring B's live binding is gated + assert bsc.main(argv + remint) == 0 + assert bsc.main(argv + ["--check"]) == 0 + registry = bsc.UuidRegistry.load(tmp_path / "registry.jsonl") + u1 = registry.binding(c_key) + assert registry.is_live(c_key) and u1 != u2 + assert not registry.is_live(b_key) and registry.binding(b_key) == u2 + assert [ + (_key(e), _event_kind(e)) + for e in _registry_lines(tmp_path)[len(registry_before.splitlines()):] + ] == [(c_key, "revived"), (b_key, "retired")] + (row,) = [ + r for r in json.loads((tmp_path / "catalog.json").read_text())["series"] + if r["concept"] == concept + ] + assert (row["uuid"], row["status"], _key(row)) == (u1, "observed", c_key) + def test_a_docket_alias_never_plans_a_cross_concept_hop( tmp_path: pathlib.Path, @@ -4908,18 +4936,37 @@ def test_a_docket_alias_never_plans_a_cross_concept_hop( # Independent-review finding (chronicle#360, LOW, pre-existing): a # docket entry naming a placeholder by its curated alias keys the # docket row under the alias, so the planned succeeds crossed concepts - # and died only in staging replay. The builder now refuses at plan time. + # and died only in staging replay. The builder now refuses at plan time + # naming the cause, and the remedy it names (call the docket entry by + # the placeholder's own concept) enriches cleanly. existing = {"series": [ dict(_docket_row("agency.a", U1, None), aliases=["agency.b"]), ]} - docket = {"series": [{"series": "agency.b", "cadence": "monthly", - "extras": {"country": "US", - "targetUnit": "percent"}}]} - with pytest.raises(SystemExit, match="explicit ceremony"): + entry = {"series": "agency.b", "cadence": "monthly", + "extras": {"country": "US", "targetUnit": "percent"}} + with pytest.raises(SystemExit, match="never crosses concepts") as refused: _build( tmp_path, [], existing=existing, - registry_entries=[_mint("agency.a", U1)], docket=docket, + registry_entries=[_mint("agency.a", U1)], + docket={"series": [entry]}, ) + assert "name the docket entry 'agency.a'" in str(refused.value) + catalog, plan = _build( + tmp_path, [], existing=existing, + registry_entries=[_mint("agency.a", U1)], + docket={"series": [dict(entry, series="agency.a")]}, + ) + (row,) = catalog["series"] + assert (row["concept"], row["uuid"], row["status"]) == ( + "agency.a", U1, "docket-only" + ) + assert [_event_kind(e) for e in _plan_events(plan)] == [ + "retired", "succeeds" + ] + staged = _registry(tmp_path, [_mint("agency.a", U1)]).stage( + _plan_events(plan) + ) + assert bsc.registry_agreement_problems(catalog, staged) == [] # A small identity universe for generated registry histories: two concepts @@ -5185,8 +5232,13 @@ def test_enrichment_is_planned_exactly_when_the_grammar_admits_it( ) except SystemExit as exc: assert not admitted, exc - assert "explicit ceremony" in str(exc) or ( - "superseding UUIDs must be new" in str(exc) + assert any( + diagnostic in str(exc) + for diagnostic in ( + "explicit ceremony", + "superseding UUIDs must be new", + "hands over at most once", + ) ), exc return hop = [e for e in plan["mints"] if "succeeds" in e] From cd39f8c63f644c2277314679bf1813f6b7e96c07 Mon Sep 17 00:00:00 2001 From: Max Ghenis Date: Sun, 18 Oct 2026 14:45:44 -0400 Subject: [PATCH 5/5] Reclaim a handed-over key on a curated row uuid instead of reviving it When a catalog row carried a uuid other than its key's binding, resolve_uuid planned revive + supersede even when the key had retired after handing its lineage over through a succeeds event. The grammar refuses to revive a consumed key, so staging rejected the plan before any write. A docket entry gaining a country reached it by re-keying a geography-free placeholder row onto the handed-over country key (finding L2 of the chronicle#360 round-2 review). The builder now plans a reclaim on the row's uuid, which is new to the registry on that path, as the handed-over branch already did with a fresh uuid. A row whose uuid the registry already knows is refused at plan time with a message naming the cause. The enrichment differential gains a docket call-site arm (alias rows, no-country/US/GB entries, unregistered row uuids), steered so the handed-over class is reached on every seed. Co-Authored-By: Claude Opus 5.5 --- docs/catalog-curation-backlog.md | 11 ++ scripts/build_series_catalog.py | 93 ++++++--- tests/test_build_series_catalog.py | 307 +++++++++++++++++++++++++++++ 3 files changed, 387 insertions(+), 24 deletions(-) diff --git a/docs/catalog-curation-backlog.md b/docs/catalog-curation-backlog.md index 13f0a009..64acfd0b 100644 --- a/docs/catalog-curation-backlog.md +++ b/docs/catalog-curation-backlog.md @@ -189,6 +189,17 @@ were closed on 2026-08-09 by chronicle#153: and the curation each refusal suggests. A property test checks that the builder plans an enrichment exactly when the registry grammar admits the retire + succeeds pair. +- 2026-10-18: when a catalog row's UUID differed from its key's binding, + `resolve_uuid` planned the curator remint (revive the key if retired, + then supersede it to the row's UUID) even when the key had retired after + handing its lineage over. The grammar refuses to revive a handed-over + key, so staging rejected the plan before any write. A docket entry that + gains a country reaches this: it re-keys a geography-free placeholder + row onto the country key (finding L2 of #360's second review). The + builder now reclaims the key on the row's UUID. That needs no ceremony, + because no live binding changes. A row whose UUID the registry already + knows is refused at plan time, and the message names the cause. The + enrichment differential now has a docket call-site arm as well. ## Country geography on Colorado-scoped concepts, and one raw series id (2026-09-04) diff --git a/scripts/build_series_catalog.py b/scripts/build_series_catalog.py index 685c5f59..1fc57ca5 100644 --- a/scripts/build_series_catalog.py +++ b/scripts/build_series_catalog.py @@ -1416,7 +1416,7 @@ class UuidRegistry: """The append-only UUID minting ledger. One JSON object per line, binding one identity (concept, geography - level/id/vintage, entity name/role) to a UUID. Five event kinds, chained + level/id/vintage, entity name/role) to a UUID. Six event kinds, chained per identity and never edited or removed (``--verify-registry-append- only`` and the git-HEAD prefix check enforce growth-only): @@ -1431,7 +1431,12 @@ class UuidRegistry: * retire — ``retired: true`` with the unchanged UUID; ``note`` required. The identity left the catalog; its binding stays reserved but dormant. * revive — ``revived: true`` with the unchanged UUID; written - automatically when a retired identity is observed again. + automatically when a retired identity is observed again. Never for + a key whose lineage was handed over (consumed by a succeeds event): + that key is terminal. + * reclaim — ``reclaimed: true`` on a retired, handed-over key, with a + UUID new to the registry; ``note`` required. The key starts a NEW + lineage and is ordinary again (later drops retire, returns revive). Invariants ``--check`` builds on: live bindings and catalog rows are in BIJECTION (same identity, same UUID, both directions), and live @@ -2234,6 +2239,33 @@ def _plan_enrichment( ) return True + def _plan_reclaim( + canon_key: tuple[str, str, str], + geography: dict | None, + entity: dict | None, + row_uuid: str, + ) -> str: + """Plan a reclaim of ``canon_key`` on ``row_uuid``, which must be + new to the registry; return ``row_uuid``. + + The key handed its lineage over through a succeeds event and is + retired, so it is terminal: the grammar refuses to revive it (and + so to supersede it, which needs a live binding). Anything that + reappears under it is a NEW series, and a reclaim is the one event + that starts one there. The old UUID stays with its lineage. + """ + plan["mints"].append( + dict( + _registry_event(canon_key[0], geography, entity, row_uuid), + reclaimed=True, + note=( + "identity re-established after its lineage was " + "handed over" + ), + ) + ) + return row_uuid + def resolve_uuid( canon_key: tuple[str, str, str], prior: dict | None, @@ -2242,6 +2274,12 @@ def resolve_uuid( ) -> str: binding = registry.binding(canon_key) prior_uuid = prior["uuid"] if prior else None + # Retired after handing its lineage over through a succeeds event. + # A key that returned as a later hop's successor is live, so it is + # not handed over here (the grammar still bars it as a predecessor). + handed_over = ( + not registry.is_live(canon_key) and canon_key in registry.consumed + ) if prior_uuid and binding and prior_uuid != binding: # A RETIRED successor returning on the CURRENT placeholder # lineage is a re-enrichment, not a remint: plan the retire + @@ -2254,12 +2292,23 @@ def resolve_uuid( ) ): return prior_uuid - # The catalog row disagrees with the registry: an explicit, - # gated remint (the curator edited the row's uuid on purpose). - # The replacement must be new to the registry outright. + # The catalog row disagrees with the registry: the curator + # edited the row's uuid on purpose. Either event that can carry + # it (supersede, or reclaim for a handed-over key) needs a UUID + # new to the registry outright. replacement_owner = registry.uuid_owner.get( uuid_module.UUID(prior_uuid).int ) + if replacement_owner is not None and handed_over: + raise SystemExit( + f"identity {canon_key} handed its lineage over, so only " + "a reclaim can re-establish it, and the catalog row's " + f"uuid {prior_uuid} is already known to the registry " + f"(first bound to {replacement_owner}) — a reclaimed " + "identity is a NEW series and mints fresh: give the " + "row an unregistered uuid, or delete it so the builder " + "reclaims with a fresh one" + ) if replacement_owner is not None: raise SystemExit( f"identity {canon_key} would supersede to uuid " @@ -2267,8 +2316,16 @@ def resolve_uuid( f"(first bound to {replacement_owner}) — superseding " "UUIDs must be new" ) + # A handed-over key is terminal: the grammar refuses the revive + # a supersede would need. The row's uuid starts the key's new + # lineage through a reclaim instead, as the handed-over branch + # below does with a fresh uuid. No live binding changes, so no + # remint ceremony applies. + if handed_over: + return _plan_reclaim(canon_key, geography, entity, prior_uuid) # A retired binding is revived first so the event chain stays - # valid (revives are staged before supersedes). + # valid (revives are staged before supersedes). The supersede + # is the gated remint. if not registry.is_live(canon_key): plan["revives"].append( dict( @@ -2284,9 +2341,7 @@ def resolve_uuid( ) ) return prior_uuid - if binding and not ( - not registry.is_live(canon_key) and canon_key in registry.consumed - ): + if binding and not handed_over: if not registry.is_live(canon_key): # A retired identity is being observed again: same UUID, # explicit revive event (no ceremony — resuming an identity @@ -2301,22 +2356,12 @@ def resolve_uuid( ) return binding if binding: - # The identity handed its lineage to a successor: terminal. - # Anything reappearing under the old key is a NEW series claim - # and mints fresh through an explicit reclaim event; the old - # UUID stays with its lineage. - row_uuid = str(uuid_module.uuid4()) - plan["mints"].append( - dict( - _registry_event(canon_key[0], geography, entity, row_uuid), - reclaimed=True, - note=( - "identity re-established after its lineage was " - "handed over" - ), - ) + # The identity handed its lineage to a successor and the row + # carries no UUID of its own (or carries exactly the one that + # was handed over): reclaim on a fresh UUID. + return _plan_reclaim( + canon_key, geography, entity, str(uuid_module.uuid4()) ) - return row_uuid row_uuid = prior_uuid if prior_uuid else str(uuid_module.uuid4()) owner_key = registry.uuid_owner.get(uuid_module.UUID(row_uuid).int) if owner_key is not None: diff --git a/tests/test_build_series_catalog.py b/tests/test_build_series_catalog.py index 3636633c..cfc02d6b 100644 --- a/tests/test_build_series_catalog.py +++ b/tests/test_build_series_catalog.py @@ -20,6 +20,7 @@ import pytest from hypothesis import HealthCheck, given, settings +from hypothesis import event as hypothesis_event from hypothesis import strategies as st ROOT = pathlib.Path(__file__).resolve().parents[1] @@ -5271,3 +5272,309 @@ def test_enrichment_is_planned_exactly_when_the_grammar_admits_it( event["note"] = "generated ceremony" staged = registry.stage(_plan_events(plan)) assert bsc.registry_agreement_problems(catalog, staged) == [] + + +# --- handed-over keys at the docket call site (2026-10-18) ------------------- +# +# Finding L2 of the chronicle#360 round-2 review (pre-existing). When a +# catalog row carried a UUID other than its key's binding, resolve_uuid +# planned the gated remint (revive the key if retired, then supersede it to +# the row's UUID) even when the key had retired after handing its lineage +# over through a succeeds event. The grammar refuses to revive such a key, +# so staging rejected the plan before any write. A docket entry that gains +# a country reaches this: it re-keys a geography-free placeholder row onto +# the country key. The builder now reclaims the key on the row's UUID, the +# move the grammar provides for a handed-over key. + +LINEAGE = "agency.lineage" +RECLAIM_NOTE = "identity re-established after its lineage was handed over" +_B = {"concept": LINEAGE, "geography": US_UNDECLARED, "entity": None} +_E = {"concept": LINEAGE, "geography": US_UNDECLARED, "entity": ECONOMY} +_US_ENTRY = {"series": LINEAGE, "cadence": "monthly", + "extras": {"country": "US", "targetUnit": "percent"}} + + +def _b_handed_to_e() -> list[dict]: + """The reviewer's repro: B minted on U1, retired, succeeded by E.""" + return [ + dict(_B, uuid=U1), + dict(_B, uuid=U1, retired=True, note=CHAIN_NOTE), + dict(_E, uuid=U1, succeeds=_B), + ] + + +def test_a_curated_row_uuid_reclaims_a_handed_over_key( + tmp_path: pathlib.Path, +) -> None: + # The catalog holds a geography-free docket row carrying a hand-edited, + # unregistered U2, and the docket entry gains country US, so the row + # re-keys onto B. Base planned revive B + supersede B -> U2, and staging + # refused both ("revives a consumed lineage", then "share uuid"). B is + # terminal: U2 starts its new lineage through a reclaim, which needs no + # ceremony because no live binding changes. + events = _b_handed_to_e() + registry = _registry(tmp_path, events) + assert not registry.is_live(_key(_B)) and _key(_B) in registry.consumed + assert _owner(registry, U2) is None + catalog, plan = _build( + tmp_path, [], existing={"series": [_docket_row(LINEAGE, U2, None)]}, + registry_entries=events, docket={"series": [_US_ENTRY]}, + ) + (row,) = catalog["series"] + assert (_key(row), row["uuid"], row["status"]) == ( + _key(_B), U2, "docket-only" + ) + assert plan["enrich_retires"] == plan["revives"] == plan["supersedes"] == [] + assert [bsc.UuidRegistry.render_entry(e) for e in plan["mints"]] == [ + bsc.UuidRegistry.render_entry( + dict(_B, uuid=U2, reclaimed=True, note=RECLAIM_NOTE) + ) + ] + # E has no row in this synthetic catalog, so its live binding is the + # one gated event, as on base. + assert [(_key(e), e["uuid"]) for e in plan["retire_pending"]] == [ + (_key(_E), U1) + ] + for event in plan["retire_pending"]: + event["note"] = "synthetic gate approval" + staged = registry.stage(_plan_events(plan)) + assert staged.is_live(_key(_B)) and staged.binding(_key(_B)) == U2 + assert _key(_B) not in staged.consumed # an ordinary lineage again + assert staged.binding(_key(_E)) == U1 # U1 stays with its lineage + assert bsc.registry_agreement_problems(catalog, staged) == [] + + +def test_a_handed_over_key_reclaims_the_row_uuid_through_main( + tmp_path: pathlib.Path, +) -> None: + # The same move end to end, with E's observation withdrawn (E retired) + # so nothing else changes: main writes exactly the reclaim without + # --allow-remint, --check passes, and a rerun is byte-idempotent. + events = _b_handed_to_e() + [ + dict(_E, uuid=U1, retired=True, note="observation withdrawn upstream"), + ] + argv = _repo(tmp_path, [], seed={"series": [_US_ENTRY]}) + _registry(tmp_path, events) + (tmp_path / "catalog.json").write_text( + json.dumps({"series": [_docket_row(LINEAGE, U2, None)]}) + "\n", + encoding="utf-8", + ) + assert bsc.main(argv + ["--check"]) == 1 # the reclaim is unrecorded + assert bsc.main(argv) == 0 + assert bsc.main(argv + ["--check"]) == 0 + assert [ + (_key(e), e["uuid"], _event_kind(e), e.get("note")) + for e in _registry_lines(tmp_path)[len(events):] + ] == [(_key(_B), U2, "reclaimed", RECLAIM_NOTE)] + (row,) = json.loads((tmp_path / "catalog.json").read_text())["series"] + assert (_key(row), row["uuid"], row["status"]) == ( + _key(_B), U2, "docket-only" + ) + snapshot = ( + (tmp_path / "catalog.json").read_bytes(), + (tmp_path / "registry.jsonl").read_bytes(), + ) + assert bsc.main(argv) == 0 + assert ( + (tmp_path / "catalog.json").read_bytes(), + (tmp_path / "registry.jsonl").read_bytes(), + ) == snapshot + + +def test_a_registered_row_uuid_never_reclaims_a_handed_over_key( + tmp_path: pathlib.Path, +) -> None: + # A reclaim needs a UUID new to the registry. When the row carries one + # the registry already knows (here U3, B's own uuid before a curated + # remint), the builder refuses at plan time naming the cause, with or + # without --allow-remint, and leaves both artifacts untouched. Base + # refused too, but blamed a supersede. The remedy the refusal names + # works: delete the row, and the builder reclaims B on a fresh UUID. + bystander = _row("bls.cps.unemployment_rate") + argv = _repo(tmp_path, [bystander], seed={"series": []}) + assert bsc.main(argv) == 0 + events = _registry_lines(tmp_path) + [ + dict(_B, uuid=U3), + dict(_B, uuid=U1, supersedes=U3, + note="curated remint of the placeholder"), + dict(_B, uuid=U1, retired=True, note=CHAIN_NOTE), + dict(_E, uuid=U1, succeeds=_B), + dict(_E, uuid=U1, retired=True, note="observation withdrawn upstream"), + ] + _registry(tmp_path, events) + catalog = json.loads((tmp_path / "catalog.json").read_text()) + catalog["series"].append(_docket_row(LINEAGE, U3, None)) + (tmp_path / "catalog.json").write_text( + json.dumps(catalog, indent=2) + "\n", encoding="utf-8" + ) + (tmp_path / "seed.json").write_text( + json.dumps({"series": [_US_ENTRY]}), encoding="utf-8" + ) + catalog_before = (tmp_path / "catalog.json").read_bytes() + registry_before = (tmp_path / "registry.jsonl").read_bytes() + for flags in ([], ["--allow-remint", "--remint-note", "curated"]): + with pytest.raises( + SystemExit, match="a reclaimed identity is a NEW series" + ) as refused: + bsc.main(argv + flags) + assert "handed its lineage over" in str(refused.value) + assert (tmp_path / "catalog.json").read_bytes() == catalog_before + assert (tmp_path / "registry.jsonl").read_bytes() == registry_before + + catalog["series"] = [r for r in catalog["series"] if r["concept"] != LINEAGE] + (tmp_path / "catalog.json").write_text( + json.dumps(catalog, indent=2) + "\n", encoding="utf-8" + ) + assert bsc.main(argv) == 0 + assert bsc.main(argv + ["--check"]) == 0 + (reclaim,) = _registry_lines(tmp_path)[len(events):] + assert (_key(reclaim), _event_kind(reclaim)) == (_key(_B), "reclaimed") + assert reclaim["uuid"] not in {U1, U3} + (row,) = [ + r for r in json.loads((tmp_path / "catalog.json").read_text())["series"] + if r["concept"] == LINEAGE + ] + assert (_key(row), row["uuid"]) == (_key(_B), reclaim["uuid"]) + + +def _docket_identity(country: str | None) -> dict: + """The identity a docket entry for _GEN_CONCEPT keys its row under.""" + geography = ( + None if country is None + else bsc._identity_geography(bsc.COUNTRY_GEOGRAPHY[country]) + ) + return {"concept": _GEN_CONCEPT, "geography": geography, "entity": None} + + +@settings( + max_examples=400, + deadline=None, + suppress_health_check=[HealthCheck.function_scoped_fixture], +) +@given(data=st.data()) +def test_docket_enrichment_is_planned_exactly_when_the_grammar_admits_it( + tmp_path_factory: pytest.TempPathFactory, data: st.DataObject +) -> None: + """The differential above, at the docket call site. + + A docket entry keys its row by its own name and country, not by the + row it matches. So unlike an observed bucket it can re-key a + placeholder onto a key that handed its lineage over and retired (the + class of finding L2), and it can reach a placeholder of another + concept through a curated alias. For any history the grammar admits, + any existing docket-only row (as above, or carrying a UUID the + registry never met; of the entry's concept, or of another concept + that lists the entry's name as an alias) and a docket entry declaring + no country, US or GB, the same four properties hold: a hop is planned + exactly when the grammar admits it, every plan stages and agrees with + the catalog, a refusal is a plan-time diagnostic, and without a hop the + row takes its key's binding or a UUID new to the registry. + """ + tmp_path = tmp_path_factory.mktemp("docket-differential") + registry = _generated_registry(data) + # A docket hop starts from the live geography-free placeholder. When + # the history lacks one, sometimes mint it (a move the grammar checks). + root = _docket_identity(None) + unused = [u for u in _GEN_UUIDS if _owner(registry, u) is None] + if _key(root) not in registry.latest and unused and data.draw( + st.booleans(), label="mint the geography-free placeholder" + ): + registry = registry.stage( + [dict(root, uuid=data.draw(st.sampled_from(unused)))] + ) + (tmp_path / "registry.jsonl").write_bytes(registry.raw) + + def handed_over(key: tuple[str, str, str]) -> bool: + return key in registry.consumed and not registry.is_live(key) + + # Entries whose key handed over and retired are weighted up most, then + # entries whose key a hop could land on (unbound or retired). + countries = [] + for option in (None, "US", "GB"): + key = _key(_docket_identity(option)) + weight = 9 if handed_over(key) else 2 if registry.is_live(key) else 4 + countries += [option] * weight + country = data.draw(st.sampled_from(countries), label="country") + target = _docket_identity(country) + target_key = _key(target) + row_concept = data.draw( + st.sampled_from([_GEN_CONCEPT] * 4 + [_GEN_OTHER]), label="row concept" + ) + placeholders = [ + e for e in registry.latest.values() + if e["concept"] == row_concept and e["entity"] is None + ] + live = [e for e in placeholders if not e.get("retired")] + unused = [u for u in _GEN_UUIDS if _owner(registry, u) is None] + candidates = ( + [(e["geography"], e["uuid"]) for e in live] * 8 + + [(e["geography"], e["uuid"]) for e in live + if _key(e) in registry.consumed] * 16 + # The live geography-free placeholder, which a country hands over. + + [(None, e["uuid"]) for e in live if e["geography"] is None] * 16 + + [(e["geography"], e["uuid"]) for e in placeholders] * 2 + # A hand-edited UUID the registry never met, on a row the entry + # re-keys (geography-free) or keeps where it is. + + [(g, u) for g in (None, target["geography"]) for u in unused] * 4 + + [(g, u) for g in _PLACEHOLDER_GEOS for u in _GEN_UUIDS] + ) + geography, row_uuid = data.draw(st.sampled_from(candidates)) + placeholder = {"concept": row_concept, "geography": geography, + "entity": None} + retire = dict(placeholder, uuid=row_uuid, retired=True, note=CHAIN_NOTE) + succeed = dict(target, uuid=row_uuid, succeeds=placeholder) + try: + registry.stage([retire, succeed]) + admitted = True + except SystemExit: + admitted = False + if handed_over(target_key) and row_uuid != registry.binding(target_key): + hypothesis_event( + "entry key handed over; row uuid " + + ("unregistered" if _owner(registry, row_uuid) is None + else "registered") + ) + hypothesis_event(f"admitted={admitted}") + row = _docket_row(row_concept, row_uuid, geography) + if row_concept != _GEN_CONCEPT: + row["aliases"] = [_GEN_CONCEPT] + extras = {"targetUnit": "percent"} + if country is not None: + extras["country"] = country + entry = {"series": _GEN_CONCEPT, "cadence": "monthly", "extras": extras} + try: + catalog, plan = _build( + tmp_path, [], existing={"series": [row]}, + docket={"series": [entry]}, + ) + except SystemExit as exc: + assert not admitted, exc + assert any( + diagnostic in str(exc) + for diagnostic in ( + "explicit ceremony", + "superseding UUIDs must be new", + "a reclaimed identity is a NEW series", + "hands over at most once", + "never crosses concepts", + ) + ), exc + return + hop = [e for e in plan["mints"] if "succeeds" in e] + assert bool(hop) == admitted + (built,) = catalog["series"] + assert _key(built) == target_key + if admitted: + assert built["uuid"] == row_uuid + assert [ + bsc.UuidRegistry.render_entry(e) + for e in plan["enrich_retires"] + hop + ] == [bsc.UuidRegistry.render_entry(e) for e in (retire, succeed)] + else: + assert built["uuid"] == registry.binding(target_key) or ( + _owner(registry, built["uuid"]) is None + ) + for event in plan["supersedes"] + plan["retire_pending"]: + event["note"] = "generated ceremony" + staged = registry.stage(_plan_events(plan)) + assert bsc.registry_agreement_problems(catalog, staged) == []